diff options
131 files changed, 1203 insertions, 566 deletions
diff --git a/.rubocop.yml b/.rubocop.yml index cc3055ef66e..cdcfd8150e5 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -681,7 +681,7 @@ Style/UnlessElse: # Checks for %W when interpolation is not needed. Style/UnneededCapitalW: - Enabled: false + Enabled: true # TODO: Enable UnneededInterpolation Cop. # Checks for strings that are just an interpolated expression. diff --git a/CHANGELOG b/CHANGELOG index 7a5b778f63a..1d83894b424 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -3,9 +3,12 @@ Please view this file on the master branch, on stable branches it's out of date. v 8.10.0 (unreleased) - Fix commit builds API, return all builds for all pipelines for given commit. !4849 - Replace Haml with Hamlit to make view rendering faster. !3666 + - Refactor repository paths handling to allow multiple git mount points + - Add Application Setting to configure default Repository Path for new projects - Wrap code blocks on Activies and Todos page. !4783 (winniehell) - Align flash messages with left side of page content !4959 (winniehell) - Display last commit of deleted branch in push events !4699 (winniehell) + - Apply the trusted_proxies config to the rack request object for use with rack_attack - Add Sidekiq queue duration to transaction metrics. - Let Workhorse serve format-patch diffs - Make images fit to the size of the viewport !4810 @@ -15,10 +18,12 @@ v 8.10.0 (unreleased) - Exclude email check from the standard health check - Fix changing issue state columns in milestone view - Add notification settings dropdown for groups + - Allow importing from Github using Personal Access Tokens. (Eric K Idema) - Fix user creation with stronger minimum password requirements !4054 (nathan-pmt) - PipelinesFinder uses git cache data - Check for conflicts with existing Project's wiki path when creating a new project. - Remove unused front-end variable -> default_issues_tracker + - Better caching of git calls on ProjectsController#show. - Add API endpoint for a group issues !4520 (mahcsig) - Add Bugzilla integration !4930 (iamtjg) - Allow [ci skip] to be in any case and allow [skip ci]. !4785 (simon_w) @@ -45,6 +50,7 @@ v 8.9.3 - Reduce overhead and optimize ProjectTeam#max_member_access performance. !4973 - Use update_columns to by_pass all the dirty code on active_record. !4985 - Decreased min width of screen to 1280px for pinned sidebar + - Fix restore Rake task warning message output !4980 - Fix encrypted data backwards compatibility after upgrading attr_encrypted gem - Update mobile button icons to be more inline with typical UI paradigms @@ -97,7 +103,6 @@ v 8.9.1 - Remove duplicate 'New Page' button on edit wiki page v 8.9.0 -v 8.9.0 (unreleased) - Fix group visibility form layout in application settings - Fix builds API response not including commit data - Fix error when CI job variables key specified but not defined @@ -244,6 +249,7 @@ v 8.9.0 (unreleased) - Filter parameters for request_uri value on instrumented transactions. - Remove duplicated keys add UNIQUE index to keys fingerprint column - ExtractsPath get ref_names from repository cache, if not there access git. + - Show a flash warning about the error detail of XHR requests which failed with status code 404 and 500 - Cache user todo counts from TodoService - Ensure Todos counters doesn't count Todos for projects pending delete - Add left/right arrows horizontal navigation diff --git a/GITLAB_SHELL_VERSION b/GITLAB_SHELL_VERSION index 4a36342fcab..fd2a01863fd 100644 --- a/GITLAB_SHELL_VERSION +++ b/GITLAB_SHELL_VERSION @@ -1 +1 @@ -3.0.0 +3.1.0 @@ -303,7 +303,6 @@ group :development, :test do gem 'rubocop', '~> 0.40.0', require: false gem 'rubocop-rspec', '~> 1.5.0', require: false gem 'scss_lint', '~> 0.47.0', require: false - gem 'coveralls', '~> 0.8.2', require: false gem 'simplecov', '~> 0.11.0', require: false gem 'flog', require: false gem 'flay', require: false @@ -350,3 +349,4 @@ gem 'health_check', '~> 1.5.1' # System information gem 'vmstat', '~> 2.1.0' +gem 'sys-filesystem', '~> 1.1.6' diff --git a/Gemfile.lock b/Gemfile.lock index 4c5350ba639..f99b373dbbd 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -141,12 +141,6 @@ GEM colorize (0.7.7) concurrent-ruby (1.0.2) connection_pool (2.2.0) - coveralls (0.8.13) - json (~> 1.8) - simplecov (~> 0.11.0) - term-ansicolor (~> 1.3) - thor (~> 0.19.1) - tins (~> 1.6.0) crack (0.4.3) safe_yaml (~> 1.0.0) creole (0.5.0) @@ -721,6 +715,8 @@ GEM activerecord (>= 4.1, < 5.1) state_machines-activemodel (>= 0.3.0) stringex (2.5.2) + sys-filesystem (1.1.6) + ffi systemu (2.6.5) task_list (1.0.2) html-pipeline @@ -729,8 +725,6 @@ GEM teaspoon-jasmine (2.2.0) teaspoon (>= 1.0.0) temple (0.7.7) - term-ansicolor (1.3.2) - tins (~> 1.0) test_after_commit (0.4.2) activerecord (>= 3.2) thin (1.6.4) @@ -751,7 +745,6 @@ GEM mime-types multi_json (~> 1.7) twitter-stream (~> 0.1) - tins (1.6.0) turbolinks (2.5.3) coffee-rails twitter-stream (0.1.16) @@ -841,7 +834,6 @@ DEPENDENCIES chronic_duration (~> 0.10.6) coffee-rails (~> 4.1.0) connection_pool (~> 2.0) - coveralls (~> 0.8.2) creole (~> 0.5.0) d3_rails (~> 3.5.0) database_cleaner (~> 1.4.0) @@ -976,6 +968,7 @@ DEPENDENCIES spring-commands-teaspoon (~> 0.0.2) sprockets (~> 3.6.0) state_machines-activerecord (~> 0.4.0) + sys-filesystem (~> 1.1.6) task_list (~> 1.0.2) teaspoon (~> 1.1.0) teaspoon-jasmine (~> 2.2.0) diff --git a/app/assets/javascripts/application.js.coffee b/app/assets/javascripts/application.js.coffee index b6dbf2d0cc1..20fe5a5cc27 100644 --- a/app/assets/javascripts/application.js.coffee +++ b/app/assets/javascripts/application.js.coffee @@ -185,6 +185,15 @@ $ -> else buttons.enable() + $(document).ajaxError (e, xhrObj, xhrSetting, xhrErrorText) -> + + if xhrObj.status is 401 + new Flash 'You need to be logged in.', 'alert' + + else if xhrObj.status in [ 404, 500 ] + new Flash 'Something went wrong on our end.', 'alert' + + # Show/Hide the profile menu when hovering the account box $('.account-box').hover -> $(@).toggleClass('hover') @@ -260,8 +269,8 @@ $ -> new Aside() # Sidenav pinning - if $window.width() < 1280 and $.cookie('pin_nav') is 'true' - $.cookie('pin_nav', 'false', { path: '/' }) + if $window.width() < 1024 and $.cookie('pin_nav') is 'true' + $.cookie('pin_nav', 'false', { path: '/', expires: 365 * 10 }) $('.page-with-sidebar') .toggleClass('page-sidebar-collapsed page-sidebar-expanded') .removeClass('page-sidebar-pinned') @@ -292,7 +301,7 @@ $ -> .toggleClass('header-collapsed header-expanded') # Save settings - $.cookie 'pin_nav', doPinNav, { path: '/' } + $.cookie 'pin_nav', doPinNav, { path: '/', expires: 365 * 10 } if $.cookie('pin_nav') is 'true' or doPinNav tooltipText = 'Unpin navigation' diff --git a/app/assets/stylesheets/framework/variables.scss b/app/assets/stylesheets/framework/variables.scss index 87f8a17659f..211a9af2348 100644 --- a/app/assets/stylesheets/framework/variables.scss +++ b/app/assets/stylesheets/framework/variables.scss @@ -7,7 +7,7 @@ $gutter_collapsed_width: 62px; $gutter_width: 290px; $gutter_inner_width: 258px; $sidebar-transition-duration: .15s; -$sidebar-breakpoint: 1280px; +$sidebar-breakpoint: 1024px; /* * UI elements diff --git a/app/controllers/admin/application_settings_controller.rb b/app/controllers/admin/application_settings_controller.rb index f4eda864aac..5f65dd3aff0 100644 --- a/app/controllers/admin/application_settings_controller.rb +++ b/app/controllers/admin/application_settings_controller.rb @@ -109,6 +109,7 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController :metrics_packet_size, :send_user_confirmation_email, :container_registry_token_expire_delay, + :repository_storage, restricted_visibility_levels: [], import_sources: [], disabled_oauth_sign_in_sources: [] diff --git a/app/controllers/admin/system_info_controller.rb b/app/controllers/admin/system_info_controller.rb index 3c67370b667..cc63009cdc0 100644 --- a/app/controllers/admin/system_info_controller.rb +++ b/app/controllers/admin/system_info_controller.rb @@ -1,13 +1,32 @@ class Admin::SystemInfoController < Admin::ApplicationController def show + excluded_mounts = [ + "nobrowse", + "read-only", + "ro" + ] + system_info = Vmstat.snapshot + mounts = Sys::Filesystem.mounts + + @disks = [] + mounts.each do |mount| + options = mount.options.split(', ') + + next unless excluded_mounts.each { |em| break if options.include?(em) } + + disk = Sys::Filesystem.stat(mount.mount_point) + @disks.push({ + bytes_total: disk.bytes_total, + bytes_used: disk.bytes_used, + disk_name: mount.name, + mount_path: disk.path + }) + end @cpus = system_info.cpus.length @mem_used = system_info.memory.active_bytes @mem_total = system_info.memory.total_bytes - - @disk_used = system_info.disks[0].used_bytes - @disk_total = system_info.disks[0].total_bytes end end diff --git a/app/controllers/import/github_controller.rb b/app/controllers/import/github_controller.rb index 67bf4190e7e..9c1b0eb20f4 100644 --- a/app/controllers/import/github_controller.rb +++ b/app/controllers/import/github_controller.rb @@ -1,14 +1,29 @@ class Import::GithubController < Import::BaseController before_action :verify_github_import_enabled - before_action :github_auth, except: :callback + before_action :github_auth, only: [:status, :jobs, :create] rescue_from Octokit::Unauthorized, with: :github_unauthorized + helper_method :logged_in_with_github? + + def new + if logged_in_with_github? + go_to_github_for_permissions + elsif session[:github_access_token] + redirect_to status_import_github_url + end + end + def callback session[:github_access_token] = client.get_token(params[:code]) redirect_to status_import_github_url end + def personal_access_token + session[:github_access_token] = params[:personal_access_token] + redirect_to status_import_github_url + end + def status @repos = client.repos @already_added_projects = current_user.created_projects.where(import_type: "github") @@ -57,10 +72,14 @@ class Import::GithubController < Import::BaseController end def github_unauthorized - go_to_github_for_permissions + session[:github_access_token] = nil + redirect_to new_import_github_url, + alert: 'Access denied to your GitHub account.' end - private + def logged_in_with_github? + current_user.identities.exists?(provider: 'github') + end def access_params { github_access_token: session[:github_access_token] } diff --git a/app/helpers/application_settings_helper.rb b/app/helpers/application_settings_helper.rb index 55313fd8357..6e580c62ccd 100644 --- a/app/helpers/application_settings_helper.rb +++ b/app/helpers/application_settings_helper.rb @@ -78,4 +78,12 @@ module ApplicationSettingsHelper end end end + + def repository_storage_options_for_select + options = Gitlab.config.repositories.storages.map do |name, path| + ["#{name} - #{path}", name] + end + + options_for_select(options, @application_setting.repository_storage) + end end diff --git a/app/helpers/projects_helper.rb b/app/helpers/projects_helper.rb index 26796b08fac..f312a7ccca3 100644 --- a/app/helpers/projects_helper.rb +++ b/app/helpers/projects_helper.rb @@ -327,9 +327,9 @@ module ProjectsHelper end end - def sanitize_repo_path(message) + def sanitize_repo_path(project, message) return '' unless message.present? - message.strip.gsub(Gitlab.config.gitlab_shell.repos_path.chomp('/'), "[REPOS PATH]") + message.strip.gsub(project.repository_storage_path.chomp('/'), "[REPOS PATH]") end end diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index d914b0b26eb..5fa6eacd234 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -55,6 +55,10 @@ class ApplicationSetting < ActiveRecord::Base presence: true, numericality: { only_integer: true, greater_than: 0 } + validates :repository_storage, + presence: true, + inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } } + validates_each :restricted_visibility_levels do |record, attr, value| unless value.nil? value.each do |level| @@ -134,6 +138,7 @@ class ApplicationSetting < ActiveRecord::Base disabled_oauth_sign_in_sources: [], send_user_confirmation_email: false, container_registry_token_expire_delay: 5, + repository_storage: 'default', ) end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index da19462f265..8b52cc824cd 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -21,8 +21,10 @@ class Namespace < ActiveRecord::Base delegate :name, to: :owner, allow_nil: true, prefix: true - after_create :ensure_dir_exist after_update :move_dir, if: :path_changed? + + # Save the storage paths before the projects are destroyed to use them on after destroy + before_destroy(prepend: true) { @old_repository_storage_paths = repository_storage_paths } after_destroy :rm_dir scope :root, -> { where('type IS NULL') } @@ -87,51 +89,35 @@ class Namespace < ActiveRecord::Base owner_name end - def ensure_dir_exist - gitlab_shell.add_namespace(path) - end - - def rm_dir - # Move namespace directory into trash. - # We will remove it later async - new_path = "#{path}+#{id}+deleted" - - if gitlab_shell.mv_namespace(path, new_path) - message = "Namespace directory \"#{path}\" moved to \"#{new_path}\"" - Gitlab::AppLogger.info message - - # Remove namespace directroy async with delay so - # GitLab has time to remove all projects first - GitlabShellWorker.perform_in(5.minutes, :rm_namespace, new_path) - end - end - def move_dir - # Ensure old directory exists before moving it - gitlab_shell.add_namespace(path_was) - if any_project_has_container_registry_tags? raise Exception.new('Namespace cannot be moved, because at least one project has tags in container registry') end - if gitlab_shell.mv_namespace(path_was, path) - Gitlab::UploadsTransfer.new.rename_namespace(path_was, path) - - # If repositories moved successfully we need to - # send update instructions to users. - # However we cannot allow rollback since we moved namespace dir - # So we basically we mute exceptions in next actions - begin - send_update_instructions - rescue - # Returning false does not rollback after_* transaction but gives - # us information about failing some of tasks - false + # Move the namespace directory in all storages paths used by member projects + repository_storage_paths.each do |repository_storage_path| + # Ensure old directory exists before moving it + gitlab_shell.add_namespace(repository_storage_path, path_was) + + unless gitlab_shell.mv_namespace(repository_storage_path, path_was, path) + # if we cannot move namespace directory we should rollback + # db changes in order to prevent out of sync between db and fs + raise Exception.new('namespace directory cannot be moved') end - else - # if we cannot move namespace directory we should rollback - # db changes in order to prevent out of sync between db and fs - raise Exception.new('namespace directory cannot be moved') + end + + Gitlab::UploadsTransfer.new.rename_namespace(path_was, path) + + # If repositories moved successfully we need to + # send update instructions to users. + # However we cannot allow rollback since we moved namespace dir + # So we basically we mute exceptions in next actions + begin + send_update_instructions + rescue + # Returning false does not rollback after_* transaction but gives + # us information about failing some of tasks + false end end @@ -152,4 +138,33 @@ class Namespace < ActiveRecord::Base def find_fork_of(project) projects.joins(:forked_project_link).find_by('forked_project_links.forked_from_project_id = ?', project.id) end + + private + + def repository_storage_paths + # We need to get the storage paths for all the projects, even the ones that are + # pending delete. Unscoping also get rids of the default order, which causes + # problems with SELECT DISTINCT. + Project.unscoped do + projects.select('distinct(repository_storage)').to_a.map(&:repository_storage_path) + end + end + + def rm_dir + # Remove the namespace directory in all storages paths used by member projects + @old_repository_storage_paths.each do |repository_storage_path| + # Move namespace directory into trash. + # We will remove it later async + new_path = "#{path}+#{id}+deleted" + + if gitlab_shell.mv_namespace(repository_storage_path, path, new_path) + message = "Namespace directory \"#{path}\" moved to \"#{new_path}\"" + Gitlab::AppLogger.info message + + # Remove namespace directroy async with delay so + # GitLab has time to remove all projects first + GitlabShellWorker.perform_in(5.minutes, :rm_namespace, repository_storage_path, new_path) + end + end + end end diff --git a/app/models/project.rb b/app/models/project.rb index 73ded09c162..6a950ee830d 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -24,8 +24,12 @@ class Project < ActiveRecord::Base default_value_for :wiki_enabled, gitlab_config_features.wiki default_value_for :snippets_enabled, gitlab_config_features.snippets default_value_for :container_registry_enabled, gitlab_config_features.container_registry + default_value_for(:repository_storage) { current_application_settings.repository_storage } default_value_for(:shared_runners_enabled) { current_application_settings.shared_runners_enabled } + after_create :ensure_dir_exist + after_save :ensure_dir_exist, if: :namespace_id_changed? + # set last_activity_at to the same as created_at after_create :set_last_activity_at def set_last_activity_at @@ -165,6 +169,9 @@ class Project < ActiveRecord::Base validate :visibility_level_allowed_by_group validate :visibility_level_allowed_as_fork validate :check_wiki_path_conflict + validates :repository_storage, + presence: true, + inclusion: { in: ->(_object) { Gitlab.config.repositories.storages.keys } } add_authentication_token_field :runners_token before_save :ensure_runners_token @@ -376,6 +383,10 @@ class Project < ActiveRecord::Base end end + def repository_storage_path + Gitlab.config.repositories.storages[repository_storage] + end + def team @team ||= ProjectTeam.new(self) end @@ -842,12 +853,12 @@ class Project < ActiveRecord::Base raise Exception.new('Project cannot be renamed, because tags are present in its container registry') end - if gitlab_shell.mv_repository(old_path_with_namespace, new_path_with_namespace) + if gitlab_shell.mv_repository(repository_storage_path, old_path_with_namespace, new_path_with_namespace) # If repository moved successfully we need to send update instructions to users. # However we cannot allow rollback since we moved repository # So we basically we mute exceptions in next actions begin - gitlab_shell.mv_repository("#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki") + gitlab_shell.mv_repository(repository_storage_path, "#{old_path_with_namespace}.wiki", "#{new_path_with_namespace}.wiki") send_move_instructions(old_path_with_namespace) reset_events_cache @@ -988,7 +999,7 @@ class Project < ActiveRecord::Base def create_repository # Forked import is handled asynchronously unless forked? - if gitlab_shell.add_repository(path_with_namespace) + if gitlab_shell.add_repository(repository_storage_path, path_with_namespace) repository.after_create true else @@ -1140,4 +1151,8 @@ class Project < ActiveRecord::Base _, status = Gitlab::Popen.popen(%W(find #{export_path} -not -path #{export_path} -delete)) status.zero? end + + def ensure_dir_exist + gitlab_shell.add_namespace(repository_storage_path, namespace.path) + end end diff --git a/app/models/project_wiki.rb b/app/models/project_wiki.rb index 25d82929c0b..a255710f577 100644 --- a/app/models/project_wiki.rb +++ b/app/models/project_wiki.rb @@ -159,7 +159,7 @@ class ProjectWiki private def init_repo(path_with_namespace) - gitlab_shell.add_repository(path_with_namespace) + gitlab_shell.add_repository(project.repository_storage_path, path_with_namespace) end def commit_details(action, message = nil, title = nil) @@ -173,7 +173,7 @@ class ProjectWiki end def path_to_repo - @path_to_repo ||= File.join(Gitlab.config.gitlab_shell.repos_path, "#{path_with_namespace}.git") + @path_to_repo ||= File.join(project.repository_storage_path, "#{path_with_namespace}.git") end def update_project_activity diff --git a/app/models/repository.rb b/app/models/repository.rb index 2a6a3b086c2..eb232ea681b 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -39,7 +39,7 @@ class Repository # Return absolute path to repository def path_to_repo @path_to_repo ||= File.expand_path( - File.join(Gitlab.config.gitlab_shell.repos_path, path_with_namespace + ".git") + File.join(@project.repository_storage_path, path_with_namespace + ".git") ) end @@ -246,24 +246,26 @@ class Repository end end + # Keys for data that can be affected for any commit push. def cache_keys - %i(size branch_names tag_names branch_count tag_count commit_count + %i(size commit_count readme version contribution_guide changelog license_blob license_key gitignore) end + # Keys for data on branch/tag operations. + def cache_keys_for_branches_and_tags + %i(branch_names tag_names branch_count tag_count) + end + def build_cache - cache_keys.each do |key| + (cache_keys + cache_keys_for_branches_and_tags).each do |key| unless cache.exist?(key) send(key) end end end - def expire_gitignore - cache.expire(:gitignore) - end - def expire_tags_cache cache.expire(:tag_names) @tags = nil @@ -286,8 +288,6 @@ class Repository # This ensures this particular cache is flushed after the first commit to a # new repository. expire_emptiness_caches if empty? - expire_branch_count_cache - expire_tag_count_cache end def expire_branch_cache(branch_name = nil) diff --git a/app/services/projects/destroy_service.rb b/app/services/projects/destroy_service.rb index f09072975c3..882606e38d0 100644 --- a/app/services/projects/destroy_service.rb +++ b/app/services/projects/destroy_service.rb @@ -51,13 +51,13 @@ module Projects return true if params[:skip_repo] == true # There is a possibility project does not have repository or wiki - return true unless gitlab_shell.exists?(path + '.git') + return true unless gitlab_shell.exists?(project.repository_storage_path, path + '.git') new_path = removal_path(path) - if gitlab_shell.mv_repository(path, new_path) + if gitlab_shell.mv_repository(project.repository_storage_path, path, new_path) log_info("Repository \"#{path}\" moved to \"#{new_path}\"") - GitlabShellWorker.perform_in(5.minutes, :remove_repository, new_path) + GitlabShellWorker.perform_in(5.minutes, :remove_repository, project.repository_storage_path, new_path) else false end diff --git a/app/services/projects/housekeeping_service.rb b/app/services/projects/housekeeping_service.rb index 43db29315a1..a47df22f1ba 100644 --- a/app/services/projects/housekeeping_service.rb +++ b/app/services/projects/housekeeping_service.rb @@ -24,7 +24,7 @@ module Projects def execute raise LeaseTaken unless try_obtain_lease - GitlabShellOneShotWorker.perform_async(:gc, @project.path_with_namespace) + GitlabShellOneShotWorker.perform_async(:gc, @project.repository_storage_path, @project.path_with_namespace) ensure Gitlab::Metrics.measure(:reset_pushes_since_gc) do @project.update_column(:pushes_since_gc, 0) diff --git a/app/services/projects/import_service.rb b/app/services/projects/import_service.rb index 9159ec08959..163ebf26c84 100644 --- a/app/services/projects/import_service.rb +++ b/app/services/projects/import_service.rb @@ -42,7 +42,7 @@ module Projects def import_repository begin - gitlab_shell.import_repository(project.path_with_namespace, project.import_url) + gitlab_shell.import_repository(project.repository_storage_path, project.path_with_namespace, project.import_url) rescue Gitlab::Shell::Error => e raise Error, "Error importing repository #{project.import_url} into #{project.path_with_namespace} - #{e.message}" end diff --git a/app/services/projects/transfer_service.rb b/app/services/projects/transfer_service.rb index 03b57dea51e..bc7f8bf433b 100644 --- a/app/services/projects/transfer_service.rb +++ b/app/services/projects/transfer_service.rb @@ -50,12 +50,12 @@ module Projects project.send_move_instructions(old_path) # Move main repository - unless gitlab_shell.mv_repository(old_path, new_path) + unless gitlab_shell.mv_repository(project.repository_storage_path, old_path, new_path) raise TransferError.new('Cannot move project') end # Move wiki repo also if present - gitlab_shell.mv_repository("#{old_path}.wiki", "#{new_path}.wiki") + gitlab_shell.mv_repository(project.repository_storage_path, "#{old_path}.wiki", "#{new_path}.wiki") # clear project cached events project.reset_events_cache diff --git a/app/views/admin/application_settings/_form.html.haml b/app/views/admin/application_settings/_form.html.haml index 30ab0717164..c1f70bc1866 100644 --- a/app/views/admin/application_settings/_form.html.haml +++ b/app/views/admin/application_settings/_form.html.haml @@ -311,6 +311,15 @@ = f.text_field :sentry_dsn, class: 'form-control' %fieldset + %legend Repository Storage + .form-group + = f.label :repository_storage, 'Storage path for new projects', class: 'control-label col-sm-2' + .col-sm-10 + = f.select :repository_storage, repository_storage_options_for_select, {}, class: 'form-control' + .help-block + You can manage the repository storage paths in your gitlab.yml configuration file + + %fieldset %legend Repository Checks .form-group .col-sm-offset-2.col-sm-10 diff --git a/app/views/admin/background_jobs/show.html.haml b/app/views/admin/background_jobs/show.html.haml index 654d261aa99..4f680b507c4 100644 --- a/app/views/admin/background_jobs/show.html.haml +++ b/app/views/admin/background_jobs/show.html.haml @@ -2,7 +2,7 @@ - page_title "Background Jobs" = render 'admin/background_jobs/head' -%div{ class: (container_class) } +%div{ class: container_class } %h3.page-title Background Jobs %p.light GitLab uses #{link_to "sidekiq", "http://sidekiq.org/"} library for async job processing diff --git a/app/views/admin/builds/index.html.haml b/app/views/admin/builds/index.html.haml index efd5b12cfeb..1e60205f91a 100644 --- a/app/views/admin/builds/index.html.haml +++ b/app/views/admin/builds/index.html.haml @@ -1,7 +1,7 @@ - @no_container = true = render "admin/dashboard/head" -%div{ class: (container_class) } +%div{ class: container_class } .top-area %ul.nav-links diff --git a/app/views/admin/dashboard/index.html.haml b/app/views/admin/dashboard/index.html.haml index 4682016a886..a2ac407c159 100644 --- a/app/views/admin/dashboard/index.html.haml +++ b/app/views/admin/dashboard/index.html.haml @@ -1,7 +1,7 @@ - @no_container = true = render "admin/dashboard/head" -%div{ class: (container_class) } +%div{ class: container_class } .admin-dashboard.prepend-top-default .row .col-md-4 diff --git a/app/views/admin/groups/index.html.haml b/app/views/admin/groups/index.html.haml index 4f1996ef7ab..94aa5f5a942 100644 --- a/app/views/admin/groups/index.html.haml +++ b/app/views/admin/groups/index.html.haml @@ -2,7 +2,7 @@ - page_title "Groups" = render "admin/dashboard/head" -%div{ class: (container_class) } +%div{ class: container_class } %h3.page-title Groups (#{number_with_delimiter(@groups.total_count)}) @@ -39,7 +39,6 @@ = link_to 'New Group', new_admin_group_path, class: "btn btn-new" %ul.content-list - - @groups.each do |group| - = render 'group', group: group + = render @groups = paginate @groups, theme: "gitlab" diff --git a/app/views/admin/health_check/show.html.haml b/app/views/admin/health_check/show.html.haml index 7b8407f9152..e79303240f0 100644 --- a/app/views/admin/health_check/show.html.haml +++ b/app/views/admin/health_check/show.html.haml @@ -2,7 +2,7 @@ - page_title "Health Check" = render 'admin/background_jobs/head' -%div{ class: (container_class) } +%div{ class: container_class } %h3.page-title Health Check .bs-callout.clearfix diff --git a/app/views/admin/logs/show.html.haml b/app/views/admin/logs/show.html.haml index 5ddc3b9ea85..676812121d7 100644 --- a/app/views/admin/logs/show.html.haml +++ b/app/views/admin/logs/show.html.haml @@ -5,7 +5,7 @@ Gitlab::RepositoryCheckLogger] = render 'admin/background_jobs/head' -%div{ class: (container_class) } +%div{ class: container_class } %ul.nav-links.log-tabs - loggers.each do |klass| %li{ class: (klass == Gitlab::GitLogger ? 'active' : '') } diff --git a/app/views/admin/projects/index.html.haml b/app/views/admin/projects/index.html.haml index 4822cb693c2..7d2eb423223 100644 --- a/app/views/admin/projects/index.html.haml +++ b/app/views/admin/projects/index.html.haml @@ -3,7 +3,7 @@ = render 'shared/show_aside' = render "admin/dashboard/head" -%div{ class: (container_class) } +%div{ class: container_class } .row.prepend-top-default %aside.col-md-3 .panel.admin-filter diff --git a/app/views/admin/runners/index.html.haml b/app/views/admin/runners/index.html.haml index 5eff77aff2d..114bea92fc2 100644 --- a/app/views/admin/runners/index.html.haml +++ b/app/views/admin/runners/index.html.haml @@ -1,7 +1,7 @@ - @no_container = true = render "admin/dashboard/head" -%div{ class: (container_class) } +%div{ class: container_class } %p.prepend-top-default %span diff --git a/app/views/admin/system_info/show.html.haml b/app/views/admin/system_info/show.html.haml index 3ef2f20b589..6956e5ab795 100644 --- a/app/views/admin/system_info/show.html.haml +++ b/app/views/admin/system_info/show.html.haml @@ -2,7 +2,7 @@ - page_title "System Info" = render 'admin/background_jobs/head' -%div{ class: (container_class) } +%div{ class: container_class } .prepend-top-default .row .col-sm-4 @@ -17,6 +17,9 @@ %h1= "#{number_to_human_size(@mem_used)} / #{number_to_human_size(@mem_total)}" .col-sm-4 .light-well - %h4 Disk + %h4 Disks .data - %h1= "#{number_to_human_size(@disk_used)} / #{number_to_human_size(@disk_total)}" + - @disks.each do |disk| + %h1= "#{number_to_human_size(disk[:bytes_used])} / #{number_to_human_size(disk[:bytes_total])}" + %p= "#{disk[:disk_name]}" + %p= "#{disk[:mount_path]}" diff --git a/app/views/admin/users/index.html.haml b/app/views/admin/users/index.html.haml index d0a696da64b..21bb99a792c 100644 --- a/app/views/admin/users/index.html.haml +++ b/app/views/admin/users/index.html.haml @@ -3,7 +3,7 @@ = render 'shared/show_aside' = render "admin/dashboard/head" -%div{ class: (container_class) } +%div{ class: container_class } .admin-filter %ul.nav-links %li{class: "#{'active' unless params[:filter]}"} diff --git a/app/views/groups/show.html.haml b/app/views/groups/show.html.haml index a0a6762edcf..69f634e75b9 100644 --- a/app/views/groups/show.html.haml +++ b/app/views/groups/show.html.haml @@ -5,7 +5,7 @@ = auto_discovery_link_tag(:atom, group_url(@group, format: :atom, private_token: current_user.private_token), title: "#{@group.name} activity") .cover-block.groups-cover-block - %div{ class: (container_class) } + %div{ class: container_class } = link_to group_icon(@group), target: '_blank' do = image_tag group_icon(@group), class: "avatar group-avatar s70" .group-info diff --git a/app/views/import/github/new.html.haml b/app/views/import/github/new.html.haml new file mode 100644 index 00000000000..435ed7bd4cb --- /dev/null +++ b/app/views/import/github/new.html.haml @@ -0,0 +1,43 @@ +- page_title "GitHub Import" +- header_title "Projects", root_path + +%h3.page-title + = icon 'github', text: 'Import Projects from GitHub' + +- if github_import_configured? + %p + To import a GitHub project, you first need to authorize GitLab to access + the list of your GitHub repositories: + + = link_to 'List Your GitHub Repositories', status_import_github_path, class: 'btn btn-success' + + %hr + +%p + - if github_import_configured? + Alternatively, + - else + To import a GitHub project, + you can use a + = succeed '.' do + = link_to 'Personal Access Token', 'https://github.com/settings/tokens' + When you create your Personal Access Token, + you will need to select the <code>repo</code> scope, so we can display a + list of your public and private repositories which are available for import. + += form_tag personal_access_token_import_github_path, method: :post, class: 'form-inline' do + .form-group + = text_field_tag :personal_access_token, '', class: 'form-control', placeholder: "Personal Access Token", size: 40 + = submit_tag 'List Your GitHub Repositories', class: 'btn btn-success' + +- unless github_import_configured? + %hr + %p + Note: + - if current_user.admin? + As an administrator you may like to configure + - else + Consider asking your GitLab administrator to configure + = link_to 'GitHub integration', help_page_path("integration", "github") + which will allow login via GitHub and allow importing projects without + generating a Personal Access Token. diff --git a/app/views/projects/_github_import_modal.html.haml b/app/views/projects/_github_import_modal.html.haml deleted file mode 100644 index 46ad1559356..00000000000 --- a/app/views/projects/_github_import_modal.html.haml +++ /dev/null @@ -1,13 +0,0 @@ -%div#github_import_modal.modal - .modal-dialog - .modal-content - .modal-header - %a.close{href: "#", "data-dismiss" => "modal"} × - %h3 Import projects from GitHub - .modal-body - To enable importing projects from GitHub, - - if current_user.admin? - as administrator you need to configure - - else - ask your Gitlab administrator to configure - == #{link_to 'OAuth integration', help_page_path("integration", "github")}. diff --git a/app/views/projects/_home_panel.html.haml b/app/views/projects/_home_panel.html.haml index 86ea08dd229..8ce23379fb4 100644 --- a/app/views/projects/_home_panel.html.haml +++ b/app/views/projects/_home_panel.html.haml @@ -1,6 +1,6 @@ - empty_repo = @project.empty_repo? .project-home-panel.cover-block.clearfix{:class => ("empty-project" if empty_repo)} - %div{ class: (container_class) } + %div{ class: container_class } .row .project-image-container = project_icon(@project, alt: '', class: 'project-avatar avatar s70') diff --git a/app/views/projects/_last_push.html.haml b/app/views/projects/_last_push.html.haml index e0ca2a3109c..434d8644b83 100644 --- a/app/views/projects/_last_push.html.haml +++ b/app/views/projects/_last_push.html.haml @@ -1,7 +1,7 @@ - if event = last_push_event - if show_last_push_widget?(event) .row-content-block.top-block.clear-block.hidden-xs - %div{ class: (container_class) } + %div{ class: container_class } .event-last-push .event-last-push-text %span You pushed to diff --git a/app/views/projects/blob/show.html.haml b/app/views/projects/blob/show.html.haml index 8d77bdbe382..0ab78a39cf9 100644 --- a/app/views/projects/blob/show.html.haml +++ b/app/views/projects/blob/show.html.haml @@ -2,7 +2,7 @@ - page_title @blob.path, @ref = render "projects/commits/head" -%div{ class: (container_class) } +%div{ class: container_class } = render 'projects/last_push' %div#tree-holder.tree-holder diff --git a/app/views/projects/branches/index.html.haml b/app/views/projects/branches/index.html.haml index e0367c40272..77b405f1f39 100644 --- a/app/views/projects/branches/index.html.haml +++ b/app/views/projects/branches/index.html.haml @@ -2,7 +2,7 @@ - page_title "Branches" = render "projects/commits/head" -%div{ class: (container_class) } +%div{ class: container_class } .top-area .nav-text Protected branches can be managed in project settings @@ -27,7 +27,7 @@ = sort_title_recently_updated = link_to namespace_project_branches_path(sort: 'last_updated') do = sort_title_oldest_updated - - unless @branches.empty? + - if @branches.any? %ul.content-list.all-branches - @branches.each do |branch| = render "projects/branches/branch", branch: branch diff --git a/app/views/projects/builds/index.html.haml b/app/views/projects/builds/index.html.haml index 181547316aa..a131289ee97 100644 --- a/app/views/projects/builds/index.html.haml +++ b/app/views/projects/builds/index.html.haml @@ -2,7 +2,7 @@ - page_title "Builds" = render "projects/pipelines/head" -%div{ class: (container_class) } +%div{ class: container_class } .top-area %ul.nav-links %li{class: ('active' if @scope.nil?)} diff --git a/app/views/projects/commits/show.html.haml b/app/views/projects/commits/show.html.haml index 51ca4eb903e..9a44ba94970 100644 --- a/app/views/projects/commits/show.html.haml +++ b/app/views/projects/commits/show.html.haml @@ -7,7 +7,7 @@ = render "head" -%div{ class: (container_class) } +%div{ class: container_class } .row-content-block.second-block.content-component-block .tree-ref-holder = render 'shared/ref_switcher', destination: 'commits' diff --git a/app/views/projects/compare/index.html.haml b/app/views/projects/compare/index.html.haml index b22285c11e0..e9ff8e90dd5 100644 --- a/app/views/projects/compare/index.html.haml +++ b/app/views/projects/compare/index.html.haml @@ -2,7 +2,7 @@ - page_title "Compare" = render "projects/commits/head" -%div{ class: (container_class) } +%div{ class: container_class } .sub-header-block Compare branches, tags or commit ranges. %br diff --git a/app/views/projects/compare/show.html.haml b/app/views/projects/compare/show.html.haml index f4ec7b767f6..28a50e7031a 100644 --- a/app/views/projects/compare/show.html.haml +++ b/app/views/projects/compare/show.html.haml @@ -2,7 +2,7 @@ - page_title "#{params[:from]}...#{params[:to]}" = render "projects/commits/head" -%div{ class: (container_class) } +%div{ class: container_class } .sub-header-block.no-bottom-space = render "form" diff --git a/app/views/projects/environments/index.html.haml b/app/views/projects/environments/index.html.haml index a03f117291f..5242021243e 100644 --- a/app/views/projects/environments/index.html.haml +++ b/app/views/projects/environments/index.html.haml @@ -2,7 +2,7 @@ - page_title "Environments" = render "projects/pipelines/head" -%div{ class: (container_class) } +%div{ class: container_class } - if can?(current_user, :create_environment, @project) && !@environments.blank? .top-area .nav-controls diff --git a/app/views/projects/environments/show.html.haml b/app/views/projects/environments/show.html.haml index 4c15e2759d6..53c62ef234d 100644 --- a/app/views/projects/environments/show.html.haml +++ b/app/views/projects/environments/show.html.haml @@ -2,7 +2,7 @@ - page_title "Environments" = render "projects/pipelines/head" -%div{ class: (container_class) } +%div{ class: container_class } .top-area .col-md-9 %h3.page-title= @environment.name.titleize diff --git a/app/views/projects/graphs/ci.html.haml b/app/views/projects/graphs/ci.html.haml index e695d3ae369..6be4273b6ab 100644 --- a/app/views/projects/graphs/ci.html.haml +++ b/app/views/projects/graphs/ci.html.haml @@ -2,7 +2,7 @@ - page_title "Continuous Integration", "Graphs" = render 'head' -%div{ class: (container_class) } +%div{ class: container_class } .sub-header-block .oneline A collection of graphs for Continuous Integration diff --git a/app/views/projects/graphs/commits.html.haml b/app/views/projects/graphs/commits.html.haml index 0daffe68f6f..65db8af494d 100644 --- a/app/views/projects/graphs/commits.html.haml +++ b/app/views/projects/graphs/commits.html.haml @@ -2,7 +2,7 @@ - page_title "Commits", "Graphs" = render 'head' -%div{ class: (container_class) } +%div{ class: container_class } .sub-header-block .tree-ref-holder = render 'shared/ref_switcher', destination: 'graphs_commits' diff --git a/app/views/projects/graphs/languages.html.haml b/app/views/projects/graphs/languages.html.haml index 6d97f552a8e..fcfcae0be20 100644 --- a/app/views/projects/graphs/languages.html.haml +++ b/app/views/projects/graphs/languages.html.haml @@ -2,7 +2,7 @@ - page_title "Languages", "Graphs" = render 'head' -%div{ class: (container_class) } +%div{ class: container_class } .sub-header-block .oneline Programming languages used in this repository diff --git a/app/views/projects/graphs/show.html.haml b/app/views/projects/graphs/show.html.haml index 9f7e2a361ff..a985b442b2d 100644 --- a/app/views/projects/graphs/show.html.haml +++ b/app/views/projects/graphs/show.html.haml @@ -2,7 +2,7 @@ - page_title "Contributors", "Graphs" = render 'head' -%div{ class: (container_class) } +%div{ class: container_class } .sub-header-block .tree-ref-holder = render 'shared/ref_switcher', destination: 'graphs' diff --git a/app/views/projects/imports/new.html.haml b/app/views/projects/imports/new.html.haml index a8a8caf7280..2cd8d03e30e 100644 --- a/app/views/projects/imports/new.html.haml +++ b/app/views/projects/imports/new.html.haml @@ -10,7 +10,7 @@ .panel-body %pre :preserve - #{sanitize_repo_path(@project.import_error)} + #{sanitize_repo_path(@project, @project.import_error)} = form_for @project, url: namespace_project_import_path(@project.namespace, @project), method: :post, html: { class: 'form-horizontal' } do |f| = render "shared/import_form", f: f diff --git a/app/views/projects/issues/index.html.haml b/app/views/projects/issues/index.html.haml index cd876b5ea62..7ce4c1e5555 100644 --- a/app/views/projects/issues/index.html.haml +++ b/app/views/projects/issues/index.html.haml @@ -6,7 +6,7 @@ - if current_user = auto_discovery_link_tag(:atom, namespace_project_issues_url(@project.namespace, @project, :atom, private_token: current_user.private_token), title: "#{@project.name} issues") -%div{ class: (container_class) } +%div{ class: container_class } .top-area = render 'shared/issuable/nav', type: :issues .nav-controls diff --git a/app/views/projects/labels/index.html.haml b/app/views/projects/labels/index.html.haml index aa4d69550ec..db66a0edbd8 100644 --- a/app/views/projects/labels/index.html.haml +++ b/app/views/projects/labels/index.html.haml @@ -3,7 +3,7 @@ - hide_class = '' = render "projects/issues/head" -%div{ class: (container_class) } +%div{ class: container_class } .top-area.adjust .nav-text Labels can be applied to issues and merge requests. Star a label to make it a priority label. Order the prioritized labels to change their relative priority, by dragging. diff --git a/app/views/projects/merge_requests/index.html.haml b/app/views/projects/merge_requests/index.html.haml index 9f948d41dda..ace275c689b 100644 --- a/app/views/projects/merge_requests/index.html.haml +++ b/app/views/projects/merge_requests/index.html.haml @@ -3,7 +3,7 @@ = render "projects/issues/head" = render 'projects/last_push' -%div{ class: (container_class) } +%div{ class: container_class } .top-area = render 'shared/issuable/nav', type: :merge_requests .nav-controls diff --git a/app/views/projects/milestones/index.html.haml b/app/views/projects/milestones/index.html.haml index b0e0bdfff5a..ad2bfbec915 100644 --- a/app/views/projects/milestones/index.html.haml +++ b/app/views/projects/milestones/index.html.haml @@ -2,7 +2,7 @@ - page_title "Milestones" = render "projects/issues/head" -%div{ class: (container_class) } +%div{ class: container_class } .top-area = render 'shared/milestones_filter' diff --git a/app/views/projects/network/_head.html.haml b/app/views/projects/network/_head.html.haml index 86295a3d011..8f6805268d5 100644 --- a/app/views/projects/network/_head.html.haml +++ b/app/views/projects/network/_head.html.haml @@ -1,6 +1,6 @@ - @no_container = true -%div{ class: (container_class) } +%div{ class: container_class } .row-content-block.second-block.content-component-block .tree-ref-holder = render partial: 'shared/ref_switcher', locals: {destination: 'graph'} diff --git a/app/views/projects/network/show.html.haml b/app/views/projects/network/show.html.haml index 3ca30b4ba6b..091af4df4a1 100644 --- a/app/views/projects/network/show.html.haml +++ b/app/views/projects/network/show.html.haml @@ -4,7 +4,7 @@ = page_specific_javascript_tag('network/application.js') = render "projects/commits/head" = render "head" -%div{ class: (container_class) } +%div{ class: container_class } .project-network .controls = form_tag namespace_project_network_path(@project.namespace, @project, @id), method: :get, class: 'form-inline network-form' do |f| diff --git a/app/views/projects/new.html.haml b/app/views/projects/new.html.haml index 8a73b077357..05f33b78a47 100644 --- a/app/views/projects/new.html.haml +++ b/app/views/projects/new.html.haml @@ -23,6 +23,7 @@ .input-group-addon = root_url = f.select :namespace_id, namespaces_options(params[:namespace_id] || :current_user, display_path: true), {}, {class: 'select2 js-select-namespace', tabindex: 1} + - else .input-group-addon.static-namespace #{root_url}#{current_user.username}/ @@ -44,15 +45,8 @@ .col-sm-12.import-buttons %div - if github_import_enabled? - - if github_import_configured? - = link_to status_import_github_path, class: 'btn import_github' do - %i.fa.fa-github - GitHub - - else - = link_to '#', class: 'how_to_import_link btn import_github' do - %i.fa.fa-github - GitHub - = render 'github_import_modal' + = link_to new_import_github_path, class: 'btn import_github' do + = icon 'github', text: 'GitHub' %div - if bitbucket_import_enabled? - if bitbucket_import_configured? diff --git a/app/views/projects/pipelines/index.html.haml b/app/views/projects/pipelines/index.html.haml index b70693eeb62..28b475d5c2f 100644 --- a/app/views/projects/pipelines/index.html.haml +++ b/app/views/projects/pipelines/index.html.haml @@ -2,7 +2,7 @@ - page_title "Pipelines" = render "projects/pipelines/head" -%div{ class: (container_class) } +%div{ class: container_class } .top-area %ul.nav-links %li{class: ('active' if @scope.nil?)} diff --git a/app/views/projects/show.html.haml b/app/views/projects/show.html.haml index 15f0d85194b..f6e81af2638 100644 --- a/app/views/projects/show.html.haml +++ b/app/views/projects/show.html.haml @@ -13,7 +13,7 @@ = render "home_panel" .project-stats.row-content-block.second-block - %div{ class: (container_class) } + %div{ class: container_class } %ul.nav %li = link_to project_files_path(@project) do diff --git a/app/views/projects/tags/index.html.haml b/app/views/projects/tags/index.html.haml index 4ca1f58ac5c..c375bb6dd1b 100644 --- a/app/views/projects/tags/index.html.haml +++ b/app/views/projects/tags/index.html.haml @@ -2,7 +2,7 @@ - page_title "Tags" = render "projects/commits/head" -%div{ class: (container_class) } +%div{ class: container_class } .top-area .nav-text Tags give the ability to mark specific points in history as being important @@ -25,7 +25,7 @@ = sort_title_oldest_updated .tags - - unless @tags.empty? + - if @tags.any? %ul.content-list = render partial: 'tag', collection: @tags diff --git a/app/views/projects/tree/show.html.haml b/app/views/projects/tree/show.html.haml index 2abcfcdd7b2..bf5360b4dee 100644 --- a/app/views/projects/tree/show.html.haml +++ b/app/views/projects/tree/show.html.haml @@ -7,7 +7,7 @@ = render 'projects/last_push' = render "projects/commits/head" -%div{ class: (container_class) } +%div{ class: container_class } .tree-controls = render 'projects/find_file_link' - if can? current_user, :download_code, @project diff --git a/app/views/projects/wikis/_new.html.haml b/app/views/projects/wikis/_new.html.haml index 4f8abcdc8e1..c32cb122c26 100644 --- a/app/views/projects/wikis/_new.html.haml +++ b/app/views/projects/wikis/_new.html.haml @@ -1,6 +1,6 @@ - @no_container = true -%div{ class: (container_class) } +%div{ class: container_class } %div#modal-new-wiki.modal .modal-dialog .modal-content diff --git a/app/views/projects/wikis/edit.html.haml b/app/views/projects/wikis/edit.html.haml index 817bf9b3f69..233538bb488 100644 --- a/app/views/projects/wikis/edit.html.haml +++ b/app/views/projects/wikis/edit.html.haml @@ -2,7 +2,7 @@ - page_title "Edit", @page.title.capitalize, "Wiki" = render 'nav' -%div{ class: (container_class) } +%div{ class: container_class } .top-area .nav-text %strong diff --git a/app/views/projects/wikis/git_access.html.haml b/app/views/projects/wikis/git_access.html.haml index 6caf7230f35..b8811a28dd6 100644 --- a/app/views/projects/wikis/git_access.html.haml +++ b/app/views/projects/wikis/git_access.html.haml @@ -2,7 +2,7 @@ - page_title "Git Access", "Wiki" = render 'nav' -%div{ class: (container_class) } +%div{ class: container_class } .sub-header-block %span.oneline Git access for diff --git a/app/views/projects/wikis/history.html.haml b/app/views/projects/wikis/history.html.haml index 630ee35b70b..4c0b14e2c42 100644 --- a/app/views/projects/wikis/history.html.haml +++ b/app/views/projects/wikis/history.html.haml @@ -1,6 +1,6 @@ - page_title "History", @page.title.capitalize, "Wiki" = render 'nav' -%div{ class: (container_class) } +%div{ class: container_class } .top-area .nav-text %strong diff --git a/app/views/projects/wikis/pages.html.haml b/app/views/projects/wikis/pages.html.haml index 81d9f391c1c..9c10acd4cb6 100644 --- a/app/views/projects/wikis/pages.html.haml +++ b/app/views/projects/wikis/pages.html.haml @@ -3,7 +3,7 @@ = render 'nav' -%div{ class: (container_class) } +%div{ class: container_class } %ul.content-list - @wiki_pages.each do |wiki_page| %li diff --git a/app/views/projects/wikis/show.html.haml b/app/views/projects/wikis/show.html.haml index 76f9b1ecd76..5cebb538cf5 100644 --- a/app/views/projects/wikis/show.html.haml +++ b/app/views/projects/wikis/show.html.haml @@ -2,7 +2,7 @@ - page_title @page.title.capitalize, "Wiki" = render 'nav' -%div{ class: (container_class) } +%div{ class: container_class } .top-area .nav-text %strong= @page.title.capitalize diff --git a/app/workers/post_receive.rb b/app/workers/post_receive.rb index f3327ca9e61..09035a7cf2d 100644 --- a/app/workers/post_receive.rb +++ b/app/workers/post_receive.rb @@ -4,10 +4,10 @@ class PostReceive sidekiq_options queue: :post_receive def perform(repo_path, identifier, changes) - if repo_path.start_with?(Gitlab.config.gitlab_shell.repos_path.to_s) - repo_path.gsub!(Gitlab.config.gitlab_shell.repos_path.to_s, "") + if path = Gitlab.config.repositories.storages.find { |p| repo_path.start_with?(p[1].to_s) } + repo_path.gsub!(path[1].to_s, "") else - log("Check gitlab.yml config for correct gitlab_shell.repos_path variable. \"#{Gitlab.config.gitlab_shell.repos_path}\" does not match \"#{repo_path}\"") + log("Check gitlab.yml config for correct repositories.storages values. No repository storage path matches \"#{repo_path}\"") end post_received = Gitlab::GitPostReceive.new(repo_path, identifier, changes) diff --git a/app/workers/repository_fork_worker.rb b/app/workers/repository_fork_worker.rb index d947f105516..f7604e48f83 100644 --- a/app/workers/repository_fork_worker.rb +++ b/app/workers/repository_fork_worker.rb @@ -12,7 +12,7 @@ class RepositoryForkWorker return end - result = gitlab_shell.fork_repository(source_path, target_path) + result = gitlab_shell.fork_repository(project.repository_storage_path, source_path, target_path) unless result logger.error("Unable to fork project #{project_id} for repository #{source_path} -> #{target_path}") project.mark_import_as_failed('The project could not be forked.') diff --git a/config/dependency_decisions.yml b/config/dependency_decisions.yml index 436a2c5e17a..293f2b71d65 100644 --- a/config/dependency_decisions.yml +++ b/config/dependency_decisions.yml @@ -181,3 +181,9 @@ :why: Equivalent to LGPLv2 :versions: [] :when: 2016-06-07 17:14:10.907682000 Z +- - :whitelist + - Artistic 2.0 + - :who: Josh Frye + :why: Disk/mount information display on Admin pages + :versions: [] + :when: 2016-06-29 16:32:45.432113000 Z diff --git a/config/gitlab.teatro.yml b/config/gitlab.teatro.yml index 01c8dc5ff98..75b79b837e0 100644 --- a/config/gitlab.teatro.yml +++ b/config/gitlab.teatro.yml @@ -47,11 +47,13 @@ production: &base backup: path: "tmp/backups" # Relative paths are relative to Rails.root (default: tmp/backups/) + repositories: + storages: # REPO PATHS MUST NOT BE A SYMLINK!!! + default: /apps/repositories/ + gitlab_shell: path: /apps/gitlab-shell/ - # REPOS_PATH MUST NOT BE A SYMLINK!!! - repos_path: /apps/repositories/ hooks_path: /apps/gitlab-shell/hooks/ upload_pack: true diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index 75e1a3c1093..325eca72862 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -428,6 +428,13 @@ production: &base satellites: path: /home/git/gitlab-satellites/ + ## Repositories settings + repositories: + # Paths where repositories can be stored. Give the canonicalized absolute pathname. + # NOTE: REPOS PATHS MUST NOT CONTAIN ANY SYMLINK!!! + storages: # You must have at least a `default` storage path. + default: /home/git/repositories/ + ## Backup settings backup: path: "tmp/backups" # Relative paths are relative to Rails.root (default: tmp/backups/) @@ -452,9 +459,6 @@ production: &base ## GitLab Shell settings gitlab_shell: path: /home/git/gitlab-shell/ - - # REPOS_PATH MUST NOT BE A SYMLINK!!! - repos_path: /home/git/repositories/ hooks_path: /home/git/gitlab-shell/hooks/ # File that contains the secret key for verifying access for gitlab-shell. @@ -528,11 +532,13 @@ test: # user: YOUR_USERNAME satellites: path: tmp/tests/gitlab-satellites/ + repositories: + storages: + default: tmp/tests/repositories/ backup: path: tmp/tests/backups gitlab_shell: path: tmp/tests/gitlab-shell/ - repos_path: tmp/tests/repositories/ hooks_path: tmp/tests/gitlab-shell/hooks/ issues_tracker: redmine: diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index c6dc1e4ab38..a93996cec72 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -304,7 +304,6 @@ Settings.gitlab_shell['hooks_path'] ||= Settings.gitlab['user_home'] + '/gitla Settings.gitlab_shell['secret_file'] ||= Rails.root.join('.gitlab_shell_secret') Settings.gitlab_shell['receive_pack'] = true if Settings.gitlab_shell['receive_pack'].nil? Settings.gitlab_shell['upload_pack'] = true if Settings.gitlab_shell['upload_pack'].nil? -Settings.gitlab_shell['repos_path'] ||= Settings.gitlab['user_home'] + '/repositories/' Settings.gitlab_shell['ssh_host'] ||= Settings.gitlab.ssh_host Settings.gitlab_shell['ssh_port'] ||= 22 Settings.gitlab_shell['ssh_user'] ||= Settings.gitlab.user @@ -312,6 +311,14 @@ Settings.gitlab_shell['owner_group'] ||= Settings.gitlab.user Settings.gitlab_shell['ssh_path_prefix'] ||= Settings.send(:build_gitlab_shell_ssh_path_prefix) # +# Repositories +# +Settings['repositories'] ||= Settingslogic.new({}) +Settings.repositories['storages'] ||= {} +# Setting gitlab_shell.repos_path is DEPRECATED and WILL BE REMOVED in version 9.0 +Settings.repositories.storages['default'] ||= Settings.gitlab_shell['repos_path'] || Settings.gitlab['user_home'] + '/repositories/' + +# # Backup # Settings['backup'] ||= Settingslogic.new({}) diff --git a/config/initializers/6_validations.rb b/config/initializers/6_validations.rb new file mode 100644 index 00000000000..3ba9e36c567 --- /dev/null +++ b/config/initializers/6_validations.rb @@ -0,0 +1,24 @@ +def storage_name_valid?(name) + !!(name =~ /\A[a-zA-Z0-9\-_]+\z/) +end + +def find_parent_path(name, path) + Gitlab.config.repositories.storages.detect do |n, p| + name != n && path.chomp('/').start_with?(p.chomp('/')) + end +end + +def error(message) + raise "#{message}. Please fix this in your gitlab.yml before starting GitLab." +end + +error('No repository storage path defined') if Gitlab.config.repositories.storages.empty? + +Gitlab.config.repositories.storages.each do |name, path| + error("\"#{name}\" is not a valid storage name") unless storage_name_valid?(name) + + parent_name, _parent_path = find_parent_path(name, path) + if parent_name + error("#{name} is a nested path of #{parent_name}. Nested paths are not supported for repository storages") + end +end diff --git a/config/initializers/gitlab_shell_secret_token.rb b/config/initializers/gitlab_shell_secret_token.rb index 751fccead07..7454c33c9dd 100644 --- a/config/initializers/gitlab_shell_secret_token.rb +++ b/config/initializers/gitlab_shell_secret_token.rb @@ -1,19 +1 @@ -# Be sure to restart your server when you modify this file. - -require 'securerandom' - -# Your secret key for verifying the gitlab_shell. - - -secret_file = Gitlab.config.gitlab_shell.secret_file - -unless File.exist? secret_file - # Generate a new token of 16 random hexadecimal characters and store it in secret_file. - token = SecureRandom.hex(16) - File.write(secret_file, token) -end - -link_path = File.join(Gitlab.config.gitlab_shell.path, '.gitlab_shell_secret') -if File.exist?(Gitlab.config.gitlab_shell.path) && !File.exist?(link_path) - FileUtils.symlink(secret_file, link_path) -end +Gitlab::Shell.new.generate_and_link_secret_token diff --git a/config/initializers/rack_attack.rb.example b/config/initializers/rack_attack.rb.example index 30d05f16153..69052c029f2 100644 --- a/config/initializers/rack_attack.rb.example +++ b/config/initializers/rack_attack.rb.example @@ -10,7 +10,8 @@ paths_to_be_protected = [ "#{Rails.application.config.relative_url_root}/api/#{API::API.version}/session", "#{Rails.application.config.relative_url_root}/users", "#{Rails.application.config.relative_url_root}/users/confirmation", - "#{Rails.application.config.relative_url_root}/unsubscribes/" + "#{Rails.application.config.relative_url_root}/unsubscribes/", + "#{Rails.application.config.relative_url_root}/import/github/personal_access_token" ] diff --git a/config/initializers/trusted_proxies.rb b/config/initializers/trusted_proxies.rb index d256a16d42b..df4a933e22f 100644 --- a/config/initializers/trusted_proxies.rb +++ b/config/initializers/trusted_proxies.rb @@ -1,3 +1,16 @@ +# Override Rack::Request to make use of the same list of trusted_proxies +# as the ActionDispatch::Request object. This is necessary for libraries +# like rack_attack where they don't use ActionDispatch, and we want them +# to block/throttle requests on private networks. +# Rack Attack specific issue: https://github.com/kickstarter/rack-attack/issues/145 +module Rack + class Request + def trusted_proxy?(ip) + Rails.application.config.action_dispatch.trusted_proxies.any? { |proxy| proxy === ip } + end + end +end + Rails.application.config.action_dispatch.trusted_proxies = ( [ '127.0.0.1', '::1' ] + Array(Gitlab.config.gitlab.trusted_proxies) ).map { |proxy| IPAddr.new(proxy) } diff --git a/config/routes.rb b/config/routes.rb index 2aab73720f2..c04780fec88 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -139,6 +139,7 @@ Rails.application.routes.draw do # namespace :import do resource :github, only: [:create, :new], controller: :github do + post :personal_access_token get :status get :callback get :jobs diff --git a/db/migrate/20160608195742_add_repository_storage_to_projects.rb b/db/migrate/20160608195742_add_repository_storage_to_projects.rb new file mode 100644 index 00000000000..c700d2b569d --- /dev/null +++ b/db/migrate/20160608195742_add_repository_storage_to_projects.rb @@ -0,0 +1,12 @@ +class AddRepositoryStorageToProjects < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + disable_ddl_transaction! + + def up + add_column_with_default(:projects, :repository_storage, :string, default: 'default') + end + + def down + remove_column(:projects, :repository_storage) + end +end diff --git a/db/migrate/20160614182521_add_repository_storage_to_application_settings.rb b/db/migrate/20160614182521_add_repository_storage_to_application_settings.rb new file mode 100644 index 00000000000..6dae91b700b --- /dev/null +++ b/db/migrate/20160614182521_add_repository_storage_to_application_settings.rb @@ -0,0 +1,5 @@ +class AddRepositoryStorageToApplicationSettings < ActiveRecord::Migration + def change + add_column :application_settings, :repository_storage, :string, default: 'default' + end +end diff --git a/db/schema.rb b/db/schema.rb index 7a8377f687c..beb723c3bc5 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -85,6 +85,7 @@ ActiveRecord::Schema.define(version: 20160620115026) do t.boolean "send_user_confirmation_email", default: false t.integer "container_registry_token_expire_delay", default: 5 t.text "after_sign_up_text" + t.string "repository_storage", default: "default" end create_table "audit_events", force: :cascade do |t| @@ -796,38 +797,39 @@ ActiveRecord::Schema.define(version: 20160620115026) do t.datetime "created_at" t.datetime "updated_at" t.integer "creator_id" - t.boolean "issues_enabled", default: true, null: false - t.boolean "merge_requests_enabled", default: true, null: false - t.boolean "wiki_enabled", default: true, null: false + t.boolean "issues_enabled", default: true, null: false + t.boolean "merge_requests_enabled", default: true, null: false + t.boolean "wiki_enabled", default: true, null: false t.integer "namespace_id" - t.boolean "snippets_enabled", default: true, null: false + t.boolean "snippets_enabled", default: true, null: false t.datetime "last_activity_at" t.string "import_url" - t.integer "visibility_level", default: 0, null: false - t.boolean "archived", default: false, null: false + t.integer "visibility_level", default: 0, null: false + t.boolean "archived", default: false, null: false t.string "avatar" t.string "import_status" t.float "repository_size", default: 0.0 - t.integer "star_count", default: 0, null: false + t.integer "star_count", default: 0, null: false t.string "import_type" t.string "import_source" t.integer "commit_count", default: 0 t.text "import_error" t.integer "ci_id" - t.boolean "builds_enabled", default: true, null: false - t.boolean "shared_runners_enabled", default: true, null: false + t.boolean "builds_enabled", default: true, null: false + t.boolean "shared_runners_enabled", default: true, null: false t.string "runners_token" t.string "build_coverage_regex" - t.boolean "build_allow_git_fetch", default: true, null: false - t.integer "build_timeout", default: 3600, null: false + t.boolean "build_allow_git_fetch", default: true, null: false + t.integer "build_timeout", default: 3600, null: false t.boolean "pending_delete", default: false - t.boolean "public_builds", default: true, null: false + t.boolean "public_builds", default: true, null: false t.integer "pushes_since_gc", default: 0 t.boolean "last_repository_check_failed" t.datetime "last_repository_check_at" t.boolean "container_registry_enabled" - t.boolean "only_allow_merge_if_build_succeeds", default: false, null: false + t.boolean "only_allow_merge_if_build_succeeds", default: false, null: false t.boolean "has_external_issue_tracker" + t.string "repository_storage", default: "default", null: false end add_index "projects", ["builds_enabled", "shared_runners_enabled"], name: "index_projects_on_builds_enabled_and_shared_runners_enabled", using: :btree diff --git a/doc/README.md b/doc/README.md index be0d17084c7..b98d6812a81 100644 --- a/doc/README.md +++ b/doc/README.md @@ -34,6 +34,7 @@ - [Operations](operations/README.md) Keeping GitLab up and running. - [Raketasks](raketasks/README.md) Backups, maintenance, automatic webhook setup and the importing of projects. - [Repository checks](administration/repository_checks.md) Periodic Git repository checks. +- [Repository storages](administration/repository_storages.md) Manage the paths used to store repositories. - [Security](security/README.md) Learn what you can do to further secure your GitLab instance. - [System hooks](system_hooks/system_hooks.md) Notifications when users, projects and keys are changed. - [Update](update/README.md) Update guides to upgrade your installation. diff --git a/doc/administration/repository_storages.md b/doc/administration/repository_storages.md new file mode 100644 index 00000000000..81bfe173151 --- /dev/null +++ b/doc/administration/repository_storages.md @@ -0,0 +1,18 @@ +# Repository storages + +GitLab allows you to define repository storage paths to enable distribution of +storage load between several mount points. + +## For installations from source + +Add your repository storage paths in your `gitlab.yml` under repositories -> storages, using key -> value pairs. + +>**Notes:** +- You must have at least one storage path called `default`. +- In order for backups to work correctly the storage path must **not** be a +mount point and the GitLab user should have correct permissions for the parent +directory of the path. + +## For omnibus installations + +Follow the instructions at https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/doc/settings/configuration.md#storing-git-data-in-an-alternative-directory diff --git a/doc/api/settings.md b/doc/api/settings.md index b5152311f28..741c5a29581 100644 --- a/doc/api/settings.md +++ b/doc/api/settings.md @@ -38,7 +38,8 @@ Example response: "default_project_visibility" : 0, "gravatar_enabled" : true, "sign_in_text" : null, - "container_registry_token_expire_delay": 5 + "container_registry_token_expire_delay": 5, + "repository_storage": "default" } ``` @@ -66,6 +67,7 @@ PUT /application/settings | `user_oauth_applications` | boolean | no | Allow users to register any application to use GitLab as an OAuth provider | | `after_sign_out_path` | string | no | Where to redirect users after logout | | `container_registry_token_expire_delay` | integer | no | Container Registry token duration in minutes | +| `repository_storage` | string | no | Storage path for new projects. The value should be the name of one of the repository storage paths defined in your gitlab.yml | ```bash curl -X PUT -H "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v3/application/settings?signup_enabled=false&default_project_visibility=1 @@ -93,6 +95,7 @@ Example response: "restricted_signup_domains": [], "user_oauth_applications": true, "after_sign_out_path": "", - "container_registry_token_expire_delay": 5 + "container_registry_token_expire_delay": 5, + "repository_storage": "default" } ``` diff --git a/doc/raketasks/import.md b/doc/raketasks/import.md index 8a38937062e..2b305cb5c99 100644 --- a/doc/raketasks/import.md +++ b/doc/raketasks/import.md @@ -14,7 +14,8 @@ - For omnibus-gitlab, it is located at: `/var/opt/gitlab/git-data/repositories` by default, unless you changed it in the `/etc/gitlab/gitlab.rb` file. - For installations from source, it is usually located at: `/home/git/repositories` or you can see where -your repositories are located by looking at `config/gitlab.yml` under the `gitlab_shell => repos_path` entry. +your repositories are located by looking at `config/gitlab.yml` under the `repositories => storages` entries +(you'll usually use the `default` storage path to start). New folder needs to have git user ownership and read/write/execute access for git user and its group: diff --git a/doc/workflow/importing/import_projects_from_github.md b/doc/workflow/importing/import_projects_from_github.md index a7dfac2c120..a2b2a4b88f9 100644 --- a/doc/workflow/importing/import_projects_from_github.md +++ b/doc/workflow/importing/import_projects_from_github.md @@ -1,8 +1,10 @@ # Import your project from GitHub to GitLab
>**Note:**
-In order to enable the GitHub import setting, you should first
-enable the [GitHub integration][gh-import] in your GitLab instance.
+In order to enable the GitHub import setting, you may also want to
+enable the [GitHub integration][gh-import] in your GitLab instance. This
+configuration is optional, you will be able import your GitHub repositories
+with a Personal Access Token.
At its current state, GitHub importer can import:
@@ -20,9 +22,15 @@ It is not yet possible to import your cross-repository pull requests (those from forks). We are working on improving this in the near future.
The importer page is visible when you [create a new project][new-project].
-Click on the **GitHub** link and you will be redirected to GitHub for
-permission to access your projects. After accepting, you'll be automatically
-redirected to the importer.
+Click on the **GitHub** link and, if you are logged in via the GitHub
+integration, you will be redirected to GitHub for permission to access your
+projects. After accepting, you'll be automatically redirected to the importer.
+
+If you are not using the GitHub integration, you can still perform a one-off
+authorization with GitHub to access your projects.
+
+Alternatively, you can also enter a GitHub Personal Access Token. Once you enter
+your token, you'll be taken to the importer.
diff --git a/features/dashboard/new_project.feature b/features/dashboard/new_project.feature index 56b4a639c01..8ddafb6a7ac 100644 --- a/features/dashboard/new_project.feature +++ b/features/dashboard/new_project.feature @@ -21,7 +21,7 @@ Background: Scenario: I should see instructions on how to import from GitHub Given I see "New Project" page When I click on "Import project from GitHub" - Then I see instructions on how to import from GitHub + Then I am redirected to the GitHub import page @javascript Scenario: I should see Google Code import page diff --git a/features/steps/dashboard/new_project.rb b/features/steps/dashboard/new_project.rb index 31f8924c38c..09373168dad 100644 --- a/features/steps/dashboard/new_project.rb +++ b/features/steps/dashboard/new_project.rb @@ -28,14 +28,8 @@ class Spinach::Features::NewProject < Spinach::FeatureSteps first('.import_github').click end - step 'I see instructions on how to import from GitHub' do - github_modal = first('.modal-body') - expect(github_modal).to be_visible - expect(github_modal).to have_content "To enable importing projects from GitHub" - - page.all('.modal-body').each do |element| - expect(element).not_to be_visible unless element == github_modal - end + step 'I am redirected to the GitHub import page' do + expect(current_path).to eq new_import_github_path end step 'I click on "Repo by URL"' do diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 5a23a18fe9c..4e2a43e45e2 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -376,6 +376,7 @@ module API expose :user_oauth_applications expose :after_sign_out_path expose :container_registry_token_expire_delay + expose :repository_storage end class Release < Grape::Entity diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 1d361569d59..b32503e8516 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -20,6 +20,20 @@ module API @wiki ||= params[:project].end_with?('.wiki') && !Project.find_with_namespace(params[:project]) end + + def project + @project ||= begin + project_path = params[:project] + + # Check for *.wiki repositories. + # Strip out the .wiki from the pathname before finding the + # project. This applies the correct project permissions to + # the wiki repository as well. + project_path.chomp!('.wiki') if wiki? + + Project.find_with_namespace(project_path) + end + end end post "/allowed" do @@ -32,16 +46,6 @@ module API User.find_by(id: params[:user_id]) end - project_path = params[:project] - - # Check for *.wiki repositories. - # Strip out the .wiki from the pathname before finding the - # project. This applies the correct project permissions to - # the wiki repository as well. - project_path.chomp!('.wiki') if wiki? - - project = Project.find_with_namespace(project_path) - access = if wiki? Gitlab::GitAccessWiki.new(actor, project) @@ -49,7 +53,17 @@ module API Gitlab::GitAccess.new(actor, project) end - access.check(params[:action], params[:changes]) + access_status = access.check(params[:action], params[:changes]) + + response = { status: access_status.status, message: access_status.message } + + if access_status.status + # Return the repository full path so that gitlab-shell has it when + # handling ssh commands + response[:repository_path] = project.repository.path_to_repo + end + + response end # diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 7b91215d50b..b9773f98d75 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -2,8 +2,6 @@ require 'yaml' module Backup class Repository - attr_reader :repos_path - def dump prepare @@ -50,10 +48,12 @@ module Backup end def restore - if File.exists?(repos_path) + Gitlab.config.repositories.storages.each do |name, path| + next unless File.exists?(path) + # Move repos dir to 'repositories.old' dir - bk_repos_path = File.join(repos_path, '..', 'repositories.old.' + Time.now.to_i.to_s) - FileUtils.mv(repos_path, bk_repos_path) + bk_repos_path = File.join(path, '..', 'repositories.old.' + Time.now.to_i.to_s) + FileUtils.mv(path, bk_repos_path) end FileUtils.mkdir_p(repos_path) @@ -61,7 +61,7 @@ module Backup Project.find_each(batch_size: 1000) do |project| $progress.print " * #{project.path_with_namespace} ... " - project.namespace.ensure_dir_exist if project.namespace + project.ensure_dir_exist if File.exists?(path_to_bundle(project)) FileUtils.mkdir_p(path_to_repo(project)) @@ -100,8 +100,8 @@ module Backup end $progress.print 'Put GitLab hooks in repositories dirs'.color(:yellow) - cmd = "#{Gitlab.config.gitlab_shell.path}/bin/create-hooks" - if system(cmd) + cmd = %W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args + if system(*cmd) $progress.puts " [DONE]".color(:green) else puts " [FAILED]".color(:red) @@ -120,10 +120,6 @@ module Backup File.join(backup_repos_path, project.path_with_namespace + ".bundle") end - def repos_path - Gitlab.config.gitlab_shell.repos_path - end - def backup_repos_path File.join(Gitlab.config.backup.path, "repositories") end @@ -139,5 +135,11 @@ module Backup def silent {err: '/dev/null', out: '/dev/null'} end + + private + + def repository_storage_paths_args + Gitlab.config.repositories.storages.values + end end end diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 81d66271136..d77a5e3ff09 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -160,11 +160,7 @@ module Banzai title = object_link_title(object) klass = reference_class(object_sym) - data = data_attribute( - original: link_text || match, - project: project.id, - object_sym => object.id - ) + data = data_attributes_for(link_text || match, project, object) if matches.names.include?("url") && matches[:url] url = matches[:url] @@ -183,6 +179,14 @@ module Banzai end end + def data_attributes_for(text, project, object) + data_attribute( + original: text, + project: project.id, + object_sym => object.id + ) + end + def object_link_text_extras(object, matches) extras = [] diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb index 5351272f42d..4042e9a4c25 100644 --- a/lib/banzai/filter/issue_reference_filter.rb +++ b/lib/banzai/filter/issue_reference_filter.rb @@ -46,6 +46,26 @@ module Banzai end end + def object_link_title(object) + if object.is_a?(ExternalIssue) + "Issue in #{object.project.external_issue_tracker.title}" + else + super + end + end + + def data_attributes_for(text, project, object) + if object.is_a?(ExternalIssue) + data_attribute( + project: project.id, + external_issue: object.id, + reference_type: ExternalIssueReferenceFilter.reference_type + ) + else + super + end + end + def find_projects_for_paths(paths) super(paths).includes(:gitlab_issue_tracker_service) end diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 2d6f34c9cd8..bf058241cda 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -29,7 +29,7 @@ module Banzai def data_attribute(attributes = {}) attributes = attributes.reject { |_, v| v.nil? } - attributes[:reference_type] = self.class.reference_type + attributes[:reference_type] ||= self.class.reference_type attributes.delete(:original) if context[:no_original_data] attributes.map { |key, value| %Q(data-#{key.to_s.dasherize}="#{escape_once(value)}") }.join(" ") end diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index 3e3986d6382..34e0143a82e 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -1,3 +1,5 @@ +require 'securerandom' + module Gitlab class Shell class Error < StandardError; end @@ -18,77 +20,82 @@ module Gitlab # Init new repository # + # storage - project's storage path # name - project path with namespace # # Ex. - # add_repository("gitlab/gitlab-ci") + # add_repository("/path/to/storage", "gitlab/gitlab-ci") # - def add_repository(name) + def add_repository(storage, name) Gitlab::Utils.system_silent([gitlab_shell_projects_path, - 'add-project', "#{name}.git"]) + 'add-project', storage, "#{name}.git"]) end # Import repository # + # storage - project's storage path # name - project path with namespace # # Ex. - # import_repository("gitlab/gitlab-ci", "https://github.com/randx/six.git") + # import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://github.com/randx/six.git") # - def import_repository(name, url) - output, status = Popen::popen([gitlab_shell_projects_path, 'import-project', "#{name}.git", url, '900']) + def import_repository(storage, name, url) + output, status = Popen::popen([gitlab_shell_projects_path, 'import-project', + storage, "#{name}.git", url, '900']) raise Error, output unless status.zero? true end # Move repository - # + # storage - project's storage path # path - project path with namespace # new_path - new project path with namespace # # Ex. - # mv_repository("gitlab/gitlab-ci", "randx/gitlab-ci-new") + # mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new") # - def mv_repository(path, new_path) + def mv_repository(storage, path, new_path) Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'mv-project', - "#{path}.git", "#{new_path}.git"]) + storage, "#{path}.git", "#{new_path}.git"]) end # Fork repository to new namespace - # + # storage - project's storage path # path - project path with namespace # fork_namespace - namespace for forked project # # Ex. - # fork_repository("gitlab/gitlab-ci", "randx") + # fork_repository("/path/to/storage", "gitlab/gitlab-ci", "randx") # - def fork_repository(path, fork_namespace) + def fork_repository(storage, path, fork_namespace) Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'fork-project', - "#{path}.git", fork_namespace]) + storage, "#{path}.git", fork_namespace]) end # Remove repository from file system # + # storage - project's storage path # name - project path with namespace # # Ex. - # remove_repository("gitlab/gitlab-ci") + # remove_repository("/path/to/storage", "gitlab/gitlab-ci") # - def remove_repository(name) + def remove_repository(storage, name) Gitlab::Utils.system_silent([gitlab_shell_projects_path, - 'rm-project', "#{name}.git"]) + 'rm-project', storage, "#{name}.git"]) end # Gc repository # + # storage - project storage path # path - project path with namespace # # Ex. - # gc("gitlab/gitlab-ci") + # gc("/path/to/storage", "gitlab/gitlab-ci") # - def gc(path) + def gc(storage, path) Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'gc', - "#{path}.git"]) + storage, "#{path}.git"]) end # Add new key to gitlab-shell @@ -133,31 +140,31 @@ module Gitlab # Add empty directory for storing repositories # # Ex. - # add_namespace("gitlab") + # add_namespace("/path/to/storage", "gitlab") # - def add_namespace(name) - FileUtils.mkdir(full_path(name), mode: 0770) unless exists?(name) + def add_namespace(storage, name) + FileUtils.mkdir(full_path(storage, name), mode: 0770) unless exists?(storage, name) end # Remove directory from repositories storage # Every repository inside this directory will be removed too # # Ex. - # rm_namespace("gitlab") + # rm_namespace("/path/to/storage", "gitlab") # - def rm_namespace(name) - FileUtils.rm_r(full_path(name), force: true) + def rm_namespace(storage, name) + FileUtils.rm_r(full_path(storage, name), force: true) end # Move namespace directory inside repositories storage # # Ex. - # mv_namespace("gitlab", "gitlabhq") + # mv_namespace("/path/to/storage", "gitlab", "gitlabhq") # - def mv_namespace(old_name, new_name) - return false if exists?(new_name) || !exists?(old_name) + def mv_namespace(storage, old_name, new_name) + return false if exists?(storage, new_name) || !exists?(storage, old_name) - FileUtils.mv(full_path(old_name), full_path(new_name)) + FileUtils.mv(full_path(storage, old_name), full_path(storage, new_name)) end def url_to_repo(path) @@ -176,11 +183,26 @@ module Gitlab # Check if such directory exists in repositories. # # Usage: - # exists?('gitlab') - # exists?('gitlab/cookies.git') + # exists?(storage, 'gitlab') + # exists?(storage, 'gitlab/cookies.git') # - def exists?(dir_name) - File.exist?(full_path(dir_name)) + def exists?(storage, dir_name) + File.exist?(full_path(storage, dir_name)) + end + + # Create (if necessary) and link the secret token file + def generate_and_link_secret_token + secret_file = Gitlab.config.gitlab_shell.secret_file + unless File.exist? secret_file + # Generate a new token of 16 random hexadecimal characters and store it in secret_file. + token = SecureRandom.hex(16) + File.write(secret_file, token) + end + + link_path = File.join(gitlab_shell_path, '.gitlab_shell_secret') + if File.exist?(gitlab_shell_path) && !File.exist?(link_path) + FileUtils.symlink(secret_file, link_path) + end end protected @@ -193,14 +215,10 @@ module Gitlab File.expand_path("~#{Gitlab.config.gitlab_shell.ssh_user}") end - def repos_path - Gitlab.config.gitlab_shell.repos_path - end - - def full_path(dir_name) + def full_path(storage, dir_name) raise ArgumentError.new("Directory name can't be blank") if dir_name.blank? - File.join(repos_path, dir_name) + File.join(storage, dir_name) end def gitlab_shell_projects_path diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index d325eca6d99..043f10d96a9 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -4,26 +4,39 @@ module Gitlab GITHUB_SAFE_REMAINING_REQUESTS = 100 GITHUB_SAFE_SLEEP_TIME = 500 - attr_reader :client, :api + attr_reader :access_token def initialize(access_token) - @client = ::OAuth2::Client.new( - config.app_id, - config.app_secret, - github_options.merge(ssl: { verify: config['verify_ssl'] }) - ) + @access_token = access_token if access_token ::Octokit.auto_paginate = false + end + end + + def api + @api ||= ::Octokit::Client.new( + access_token: access_token, + api_endpoint: github_options[:site], + # If there is no config, we're connecting to github.com and we + # should verify ssl. + connection_options: { + ssl: { verify: config ? config['verify_ssl'] : true } + } + ) + end - @api = ::Octokit::Client.new( - access_token: access_token, - api_endpoint: github_options[:site], - connection_options: { - ssl: { verify: config['verify_ssl'] } - } - ) + def client + unless config + raise Projects::ImportService::Error, + 'OAuth configuration for GitHub missing.' end + + @client ||= ::OAuth2::Client.new( + config.app_id, + config.app_secret, + github_options.merge(ssl: { verify: config['verify_ssl'] }) + ) end def authorize_url(redirect_uri) @@ -56,7 +69,11 @@ module Gitlab end def github_options - config["args"]["client_options"].deep_symbolize_keys + if config + config["args"]["client_options"].deep_symbolize_keys + else + OmniAuth::Strategies::GitHub.default_options[:client_options].symbolize_keys + end end def rate_limit diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 2286ac8829c..730978d502b 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -167,7 +167,7 @@ module Gitlab def import_wiki unless project.wiki_enabled? wiki = WikiFormatter.new(project) - gitlab_shell.import_repository(wiki.path_with_namespace, wiki.import_url) + gitlab_shell.import_repository(project.repository_storage_path, wiki.path_with_namespace, wiki.import_url) project.update_attribute(:wiki_enabled, true) end diff --git a/lib/gitlab/key_fingerprint.rb b/lib/gitlab/key_fingerprint.rb index 8684b4636ea..b75ae512d92 100644 --- a/lib/gitlab/key_fingerprint.rb +++ b/lib/gitlab/key_fingerprint.rb @@ -39,7 +39,7 @@ module Gitlab # OpenSSH 6.8 introduces a new default output format for fingerprints. # Check the version and decide which command to use. - version_output, version_status = popen(%W(ssh -V)) + version_output, version_status = popen(%w(ssh -V)) return false unless version_status.zero? version_matches = version_output.match(/OpenSSH_(?<major>\d+)\.(?<minor>\d+)/) diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 9ee72fde92f..b43ee5b3383 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -33,12 +33,13 @@ namespace :gitlab do unless backup.skipped?('db') unless ENV['force'] == 'yes' - warning = warning = <<-MSG.strip_heredoc + warning = <<-MSG.strip_heredoc Before restoring the database we recommend removing all existing tables to avoid future upgrade problems. Be aware that if you have custom tables in the GitLab database these tables and all data will be removed. MSG + puts warning.color(:red) ask_to_continue puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.color(:yellow) sleep(5) diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 12d6ac45fb6..e9a4e37ec48 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -356,97 +356,108 @@ namespace :gitlab do ######################## def check_repo_base_exists - print "Repo base directory exists? ... " + puts "Repo base directory exists?" - repo_base_path = Gitlab.config.gitlab_shell.repos_path + Gitlab.config.repositories.storages.each do |name, repo_base_path| + print "#{name}... " - if File.exists?(repo_base_path) - puts "yes".color(:green) - else - puts "no".color(:red) - puts "#{repo_base_path} is missing".color(:red) - try_fixing_it( - "This should have been created when setting up GitLab Shell.", - "Make sure it's set correctly in config/gitlab.yml", - "Make sure GitLab Shell is installed correctly." - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun + if File.exists?(repo_base_path) + puts "yes".color(:green) + else + puts "no".color(:red) + puts "#{repo_base_path} is missing".color(:red) + try_fixing_it( + "This should have been created when setting up GitLab Shell.", + "Make sure it's set correctly in config/gitlab.yml", + "Make sure GitLab Shell is installed correctly." + ) + for_more_information( + see_installation_guide_section "GitLab Shell" + ) + fix_and_rerun + end end end def check_repo_base_is_not_symlink - print "Repo base directory is a symlink? ... " + puts "Repo storage directories are symlinks?" - repo_base_path = Gitlab.config.gitlab_shell.repos_path - unless File.exists?(repo_base_path) - puts "can't check because of previous errors".color(:magenta) - return - end + Gitlab.config.repositories.storages.each do |name, repo_base_path| + print "#{name}... " - unless File.symlink?(repo_base_path) - puts "no".color(:green) - else - puts "yes".color(:red) - try_fixing_it( - "Make sure it's set to the real directory in config/gitlab.yml" - ) - fix_and_rerun + unless File.exists?(repo_base_path) + puts "can't check because of previous errors".color(:magenta) + return + end + + unless File.symlink?(repo_base_path) + puts "no".color(:green) + else + puts "yes".color(:red) + try_fixing_it( + "Make sure it's set to the real directory in config/gitlab.yml" + ) + fix_and_rerun + end end end def check_repo_base_permissions - print "Repo base access is drwxrws---? ... " + puts "Repo paths access is drwxrws---?" - repo_base_path = Gitlab.config.gitlab_shell.repos_path - unless File.exists?(repo_base_path) - puts "can't check because of previous errors".color(:magenta) - return - end + Gitlab.config.repositories.storages.each do |name, repo_base_path| + print "#{name}... " - if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770") - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}", - "sudo chmod -R ug-s #{repo_base_path}", - "sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s" - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun + unless File.exists?(repo_base_path) + puts "can't check because of previous errors".color(:magenta) + return + end + + if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770") + puts "yes".color(:green) + else + puts "no".color(:red) + try_fixing_it( + "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}", + "sudo chmod -R ug-s #{repo_base_path}", + "sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s" + ) + for_more_information( + see_installation_guide_section "GitLab Shell" + ) + fix_and_rerun + end end end def check_repo_base_user_and_group gitlab_shell_ssh_user = Gitlab.config.gitlab_shell.ssh_user gitlab_shell_owner_group = Gitlab.config.gitlab_shell.owner_group - print "Repo base owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}? ... " + puts "Repo paths owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}?" - repo_base_path = Gitlab.config.gitlab_shell.repos_path - unless File.exists?(repo_base_path) - puts "can't check because of previous errors".color(:magenta) - return - end + Gitlab.config.repositories.storages.each do |name, repo_base_path| + print "#{name}... " - uid = uid_for(gitlab_shell_ssh_user) - gid = gid_for(gitlab_shell_owner_group) - if File.stat(repo_base_path).uid == uid && File.stat(repo_base_path).gid == gid - puts "yes".color(:green) - else - puts "no".color(:red) - puts " User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".color(:blue) - try_fixing_it( - "sudo chown -R #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group} #{repo_base_path}" - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun + unless File.exists?(repo_base_path) + puts "can't check because of previous errors".color(:magenta) + return + end + + uid = uid_for(gitlab_shell_ssh_user) + gid = gid_for(gitlab_shell_owner_group) + if File.stat(repo_base_path).uid == uid && File.stat(repo_base_path).gid == gid + puts "yes".color(:green) + else + puts "no".color(:red) + puts " User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".color(:blue) + try_fixing_it( + "sudo chown -R #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group} #{repo_base_path}" + ) + for_more_information( + see_installation_guide_section "GitLab Shell" + ) + fix_and_rerun + end end end @@ -473,7 +484,7 @@ namespace :gitlab do else puts "wrong or missing hooks".color(:red) try_fixing_it( - sudo_gitlab("#{File.join(gitlab_shell_path, 'bin/create-hooks')}"), + sudo_gitlab("#{File.join(gitlab_shell_path, 'bin/create-hooks')} #{repository_storage_paths_args.join(' ')}"), 'Check the hooks_path in config/gitlab.yml', 'Check your gitlab-shell installation' ) @@ -785,13 +796,13 @@ namespace :gitlab do namespace :repo do desc "GitLab | Check the integrity of the repositories managed by GitLab" task check: :environment do - namespace_dirs = Dir.glob( - File.join(Gitlab.config.gitlab_shell.repos_path, '*') - ) + Gitlab.config.repositories.storages.each do |name, path| + namespace_dirs = Dir.glob(File.join(path, '*')) - namespace_dirs.each do |namespace_dir| - repo_dirs = Dir.glob(File.join(namespace_dir, '*')) - repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) } + namespace_dirs.each do |namespace_dir| + repo_dirs = Dir.glob(File.join(namespace_dir, '*')) + repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) } + end end end end @@ -799,12 +810,12 @@ namespace :gitlab do namespace :user do desc "GitLab | Check the integrity of a specific user's repositories" task :check_repos, [:username] => :environment do |t, args| - username = args[:username] || prompt("Check repository integrity for which username? ".color(:blue)) + username = args[:username] || prompt("Check repository integrity for fsername? ".color(:blue)) user = User.find_by(username: username) if user repo_dirs = user.authorized_projects.map do |p| File.join( - Gitlab.config.gitlab_shell.repos_path, + p.repository_storage_path, "#{p.path_with_namespace}.git" ) end diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index ab0028d6603..b7cbdc6cd78 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -5,36 +5,36 @@ namespace :gitlab do warn_user_is_not_gitlab remove_flag = ENV['REMOVE'] - namespaces = Namespace.pluck(:path) - git_base_path = Gitlab.config.gitlab_shell.repos_path - all_dirs = Dir.glob(git_base_path + '/*') + Gitlab.config.repositories.storages.each do |name, git_base_path| + all_dirs = Dir.glob(git_base_path + '/*') - puts git_base_path.color(:yellow) - puts "Looking for directories to remove... " + puts git_base_path.color(:yellow) + puts "Looking for directories to remove... " - all_dirs.reject! do |dir| - # skip if git repo - dir =~ /.git$/ - end + all_dirs.reject! do |dir| + # skip if git repo + dir =~ /.git$/ + end - all_dirs.reject! do |dir| - dir_name = File.basename dir + all_dirs.reject! do |dir| + dir_name = File.basename dir - # skip if namespace present - namespaces.include?(dir_name) - end + # skip if namespace present + namespaces.include?(dir_name) + end - all_dirs.each do |dir_path| + all_dirs.each do |dir_path| - if remove_flag - if FileUtils.rm_rf dir_path - puts "Removed...#{dir_path}".color(:red) + if remove_flag + if FileUtils.rm_rf dir_path + puts "Removed...#{dir_path}".color(:red) + else + puts "Cannot remove #{dir_path}".color(:red) + end else - puts "Cannot remove #{dir_path}".color(:red) + puts "Can be removed: #{dir_path}".color(:red) end - else - puts "Can be removed: #{dir_path}".color(:red) end end @@ -48,20 +48,21 @@ namespace :gitlab do warn_user_is_not_gitlab move_suffix = "+orphaned+#{Time.now.to_i}" - repo_root = Gitlab.config.gitlab_shell.repos_path - # Look for global repos (legacy, depth 1) and normal repos (depth 2) - IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find| - find.each_line do |path| - path.chomp! - repo_with_namespace = path. - sub(repo_root, ''). - sub(%r{^/*}, ''). - chomp('.git'). - chomp('.wiki') - next if Project.find_with_namespace(repo_with_namespace) - new_path = path + move_suffix - puts path.inspect + ' -> ' + new_path.inspect - File.rename(path, new_path) + Gitlab.config.repositories.storages.each do |name, repo_root| + # Look for global repos (legacy, depth 1) and normal repos (depth 2) + IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find| + find.each_line do |path| + path.chomp! + repo_with_namespace = path. + sub(repo_root, ''). + sub(%r{^/*}, ''). + chomp('.git'). + chomp('.wiki') + next if Project.find_with_namespace(repo_with_namespace) + new_path = path + move_suffix + puts path.inspect + ' -> ' + new_path.inspect + File.rename(path, new_path) + end end end end diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake index 4753f00c26a..dbdd4e977e8 100644 --- a/lib/tasks/gitlab/import.rake +++ b/lib/tasks/gitlab/import.rake @@ -2,73 +2,73 @@ namespace :gitlab do namespace :import do # How to use: # - # 1. copy the bare repos under the repos_path (commonly /home/git/repositories) + # 1. copy the bare repos under the repository storage paths (commonly the default path is /home/git/repositories) # 2. run: bundle exec rake gitlab:import:repos RAILS_ENV=production # # Notes: # * The project owner will set to the first administator of the system # * Existing projects will be skipped # - desc "GitLab | Import bare repositories from gitlab_shell -> repos_path into GitLab project instance" + desc "GitLab | Import bare repositories from repositories -> storages into GitLab project instance" task repos: :environment do + Gitlab.config.repositories.storages.each do |name, git_base_path| + repos_to_import = Dir.glob(git_base_path + '/**/*.git') - git_base_path = Gitlab.config.gitlab_shell.repos_path - repos_to_import = Dir.glob(git_base_path + '/**/*.git') + repos_to_import.each do |repo_path| + # strip repo base path + repo_path[0..git_base_path.length] = '' - repos_to_import.each do |repo_path| - # strip repo base path - repo_path[0..git_base_path.length] = '' + path = repo_path.sub(/\.git$/, '') + group_name, name = File.split(path) + group_name = nil if group_name == '.' - path = repo_path.sub(/\.git$/, '') - group_name, name = File.split(path) - group_name = nil if group_name == '.' + puts "Processing #{repo_path}".color(:yellow) - puts "Processing #{repo_path}".color(:yellow) - - if path.end_with?('.wiki') - puts " * Skipping wiki repo" - next - end + if path.end_with?('.wiki') + puts " * Skipping wiki repo" + next + end - project = Project.find_with_namespace(path) + project = Project.find_with_namespace(path) - if project - puts " * #{project.name} (#{repo_path}) exists" - else - user = User.admins.reorder("id").first + if project + puts " * #{project.name} (#{repo_path}) exists" + else + user = User.admins.reorder("id").first - project_params = { - name: name, - path: name - } + project_params = { + name: name, + path: name + } - # find group namespace - if group_name - group = Namespace.find_by(path: group_name) - # create group namespace - unless group - group = Group.new(:name => group_name) - group.path = group_name - group.owner = user - if group.save - puts " * Created Group #{group.name} (#{group.id})".color(:green) - else - puts " * Failed trying to create group #{group.name}".color(:red) + # find group namespace + if group_name + group = Namespace.find_by(path: group_name) + # create group namespace + unless group + group = Group.new(:name => group_name) + group.path = group_name + group.owner = user + if group.save + puts " * Created Group #{group.name} (#{group.id})".color(:green) + else + puts " * Failed trying to create group #{group.name}".color(:red) + end end + # set project group + project_params[:namespace_id] = group.id end - # set project group - project_params[:namespace_id] = group.id - end - project = Projects::CreateService.new(user, project_params).execute + project = Projects::CreateService.new(user, project_params).execute - if project.persisted? - puts " * Created #{project.name} (#{repo_path})".color(:green) - project.update_repository_size - project.update_commit_count - else - puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red) - puts " Errors: #{project.errors.messages}".color(:red) + if project.persisted? + puts " * Created #{project.name} (#{repo_path})".color(:green) + project.update_repository_size + project.update_commit_count + else + puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red) + puts " Errors: #{project.errors.messages}".color(:red) + end end end end diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index 352b566df24..fe43d40e6d2 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -62,7 +62,10 @@ namespace :gitlab do puts "" puts "GitLab Shell".color(:yellow) puts "Version:\t#{gitlab_shell_version || "unknown".color(:red)}" - puts "Repositories:\t#{Gitlab.config.gitlab_shell.repos_path}" + puts "Repository storage paths:" + Gitlab.config.repositories.storages.each do |name, path| + puts "- #{name}: \t#{path}" + end puts "Hooks:\t\t#{Gitlab.config.gitlab_shell.hooks_path}" puts "Git:\t\t#{Gitlab.config.git.bin_path}" diff --git a/lib/tasks/gitlab/list_repos.rake b/lib/tasks/gitlab/list_repos.rake index c7596e7abcb..ffcc76e5498 100644 --- a/lib/tasks/gitlab/list_repos.rake +++ b/lib/tasks/gitlab/list_repos.rake @@ -9,7 +9,7 @@ namespace :gitlab do scope = scope.where('id IN (?) OR namespace_id in (?)', project_ids, namespace_ids) end scope.find_each do |project| - base = File.join(Gitlab.config.gitlab_shell.repos_path, project.path_with_namespace) + base = File.join(project.repository_storage_path, project.path_with_namespace) puts base + '.git' puts base + '.wiki.git' end diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index b1648a4602a..c85ebdf8619 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -12,7 +12,6 @@ namespace :gitlab do gitlab_url = Gitlab.config.gitlab.url # gitlab-shell requires a / at the end of the url gitlab_url += '/' unless gitlab_url.end_with?('/') - repos_path = Gitlab.config.gitlab_shell.repos_path target_dir = Gitlab.config.gitlab_shell.path # Clone if needed @@ -35,7 +34,6 @@ namespace :gitlab do user: user, gitlab_url: gitlab_url, http_settings: {self_signed_cert: false}.stringify_keys, - repos_path: repos_path, auth_file: File.join(home_dir, ".ssh", "authorized_keys"), redis: { bin: %x{which redis-cli}.chomp, @@ -58,10 +56,10 @@ namespace :gitlab do File.open("config.yml", "w+") {|f| f.puts config.to_yaml} # Launch installation process - system(*%W(bin/install)) + system(*%W(bin/install) + repository_storage_paths_args) # (Re)create hooks - system(*%W(bin/create-hooks)) + system(*%W(bin/create-hooks) + repository_storage_paths_args) end # Required for debian packaging with PKGR: Setup .ssh/environment with @@ -73,6 +71,8 @@ namespace :gitlab do File.open(File.join(home_dir, ".ssh", "environment"), "w+") do |f| f.puts "PATH=#{ENV['PATH']}" end + + Gitlab::Shell.new.generate_and_link_secret_token end desc "GitLab | Setup gitlab-shell" @@ -87,7 +87,8 @@ namespace :gitlab do if File.exists?(path_to_repo) print '-' else - if Gitlab::Shell.new.add_repository(project.path_with_namespace) + if Gitlab::Shell.new.add_repository(project.repository_storage_path, + project.path_with_namespace) print '.' else print 'F' @@ -138,4 +139,3 @@ namespace :gitlab do system(*%W(#{Gitlab.config.git.bin_path} reset --hard #{tag})) end end - diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake index d0c019044b7..ab96b1d3593 100644 --- a/lib/tasks/gitlab/task_helpers.rake +++ b/lib/tasks/gitlab/task_helpers.rake @@ -125,10 +125,16 @@ namespace :gitlab do end def all_repos - IO.popen(%W(find #{Gitlab.config.gitlab_shell.repos_path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| - find.each_line do |path| - yield path.chomp + Gitlab.config.repositories.storages.each do |name, path| + IO.popen(%W(find #{path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| + find.each_line do |path| + yield path.chomp + end end end end + + def repository_storage_paths_args + Gitlab.config.repositories.storages.values + end end diff --git a/lib/tasks/test.rake b/lib/tasks/test.rake index c5666d49e61..21c0e5f1d41 100644 --- a/lib/tasks/test.rake +++ b/lib/tasks/test.rake @@ -6,8 +6,6 @@ task :test do end unless Rails.env.production? - require 'coveralls/rake/task' - Coveralls::RakeTask.new desc "GitLab | Run all tests on CI with simplecov" - task :test_ci => [:rubocop, :brakeman, 'teaspoon', :spinach, :spec, 'coveralls:push'] + task test_ci: [:rubocop, :brakeman, 'teaspoon', :spinach, :spec] end diff --git a/spec/controllers/import/github_controller_spec.rb b/spec/controllers/import/github_controller_spec.rb index c55a3c28208..51d59526854 100644 --- a/spec/controllers/import/github_controller_spec.rb +++ b/spec/controllers/import/github_controller_spec.rb @@ -16,6 +16,24 @@ describe Import::GithubController do allow(controller).to receive(:github_import_enabled?).and_return(true) end + describe "GET new" do + it "redirects to GitHub for an access token if logged in with GitHub" do + allow(controller).to receive(:logged_in_with_github?).and_return(true) + expect(controller).to receive(:go_to_github_for_permissions) + + get :new + end + + it "redirects to status if we already have a token" do + assign_session_token + allow(controller).to receive(:logged_in_with_github?).and_return(false) + + get :new + + expect(controller).to redirect_to(status_import_github_url) + end + end + describe "GET callback" do it "updates access token" do token = "asdasd12345" @@ -32,6 +50,20 @@ describe Import::GithubController do end end + describe "POST personal_access_token" do + it "updates access token" do + token = "asdfasdf9876" + + allow_any_instance_of(Gitlab::GithubImport::Client). + to receive(:user).and_return(true) + + post :personal_access_token, personal_access_token: token + + expect(session[:github_access_token]).to eq(token) + expect(controller).to redirect_to(status_import_github_url) + end + end + describe "GET status" do before do @repo = OpenStruct.new(login: 'vim', full_name: 'asd/vim') @@ -59,6 +91,17 @@ describe Import::GithubController do expect(assigns(:already_added_projects)).to eq([@project]) expect(assigns(:repos)).to eq([]) end + + it "handles an invalid access token" do + allow_any_instance_of(Gitlab::GithubImport::Client). + to receive(:repos).and_raise(Octokit::Unauthorized) + + get :status + + expect(session[:github_access_token]).to eq(nil) + expect(controller).to redirect_to(new_import_github_url) + expect(flash[:alert]).to eq('Access denied to your GitHub account.') + end end describe "POST create" do diff --git a/spec/features/admin/admin_system_info_spec.rb b/spec/features/admin/admin_system_info_spec.rb index dbc1d829b67..f4e5c26b519 100644 --- a/spec/features/admin/admin_system_info_spec.rb +++ b/spec/features/admin/admin_system_info_spec.rb @@ -11,7 +11,7 @@ describe 'Admin System Info' do expect(page).to have_content 'CPU' expect(page).to have_content 'Memory' - expect(page).to have_content 'Disk' + expect(page).to have_content 'Disks' end end end diff --git a/spec/helpers/projects_helper_spec.rb b/spec/helpers/projects_helper_spec.rb index 09e0bbfd00b..604204cca0a 100644 --- a/spec/helpers/projects_helper_spec.rb +++ b/spec/helpers/projects_helper_spec.rb @@ -123,11 +123,17 @@ describe ProjectsHelper do end describe '#sanitized_import_error' do + let(:project) { create(:project) } + + before do + allow(project).to receive(:repository_storage_path).and_return('/base/repo/path') + end + it 'removes the repo path' do - repo = File.join(Gitlab.config.gitlab_shell.repos_path, '/namespace/test.git') + repo = '/base/repo/path/namespace/test.git' import_error = "Could not clone #{repo}\n" - expect(sanitize_repo_path(import_error)).to eq('Could not clone [REPOS PATH]/namespace/test.git') + expect(sanitize_repo_path(project, import_error)).to eq('Could not clone [REPOS PATH]/namespace/test.git') end end end diff --git a/spec/initializers/6_validations_spec.rb b/spec/initializers/6_validations_spec.rb new file mode 100644 index 00000000000..5178bd130f4 --- /dev/null +++ b/spec/initializers/6_validations_spec.rb @@ -0,0 +1,41 @@ +require 'spec_helper' + +describe '6_validations', lib: true do + context 'with correct settings' do + before do + mock_storages('foo' => '/a/b/c', 'bar' => 'a/b/d') + end + + it 'passes through' do + expect { load_validations }.not_to raise_error + end + end + + context 'with invalid storage names' do + before do + mock_storages('name with spaces' => '/a/b/c') + end + + it 'throws an error' do + expect { load_validations }.to raise_error('"name with spaces" is not a valid storage name. Please fix this in your gitlab.yml before starting GitLab.') + end + end + + context 'with nested storage paths' do + before do + mock_storages('foo' => '/a/b/c', 'bar' => '/a/b/c/d') + end + + it 'throws an error' do + expect { load_validations }.to raise_error('bar is a nested path of foo. Nested paths are not supported for repository storages. Please fix this in your gitlab.yml before starting GitLab.') + end + end + + def mock_storages(storages) + allow(Gitlab.config.repositories).to receive(:storages).and_return(storages) + end + + def load_validations + load File.join(__dir__, '../../config/initializers/6_validations.rb') + end +end diff --git a/spec/initializers/trusted_proxies_spec.rb b/spec/initializers/trusted_proxies_spec.rb index 4bb149f25ff..14c8df954a6 100644 --- a/spec/initializers/trusted_proxies_spec.rb +++ b/spec/initializers/trusted_proxies_spec.rb @@ -6,14 +6,16 @@ describe 'trusted_proxies', lib: true do set_trusted_proxies([]) end - it 'preserves private IPs as remote_ip' do + it 'preserves private IPs' do request = stub_request('HTTP_X_FORWARDED_FOR' => '10.1.5.89') expect(request.remote_ip).to eq('10.1.5.89') + expect(request.ip).to eq('10.1.5.89') end - it 'filters out localhost from remote_ip' do + it 'filters out localhost' do request = stub_request('HTTP_X_FORWARDED_FOR' => '1.1.1.1, 10.1.5.89, 127.0.0.1') expect(request.remote_ip).to eq('10.1.5.89') + expect(request.ip).to eq('10.1.5.89') end end @@ -22,9 +24,10 @@ describe 'trusted_proxies', lib: true do set_trusted_proxies([ "10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16" ]) end - it 'filters out private and local IPs from remote_ip' do + it 'filters out private and local IPs' do request = stub_request('HTTP_X_FORWARDED_FOR' => '1.2.3.6, 1.1.1.1, 10.1.5.89, 127.0.0.1') expect(request.remote_ip).to eq('1.1.1.1') + expect(request.ip).to eq('1.1.1.1') end end @@ -33,9 +36,10 @@ describe 'trusted_proxies', lib: true do set_trusted_proxies([ "60.98.25.47" ]) end - it 'filters out proxy IP from remote_ip' do + it 'filters out proxy IP' do request = stub_request('HTTP_X_FORWARDED_FOR' => '1.2.3.6, 1.1.1.1, 60.98.25.47, 127.0.0.1') expect(request.remote_ip).to eq('1.1.1.1') + expect(request.ip).to eq('1.1.1.1') end end diff --git a/spec/lib/banzai/filter/issue_reference_filter_spec.rb b/spec/lib/banzai/filter/issue_reference_filter_spec.rb index 8d6ce114aa9..a005b4990e7 100644 --- a/spec/lib/banzai/filter/issue_reference_filter_spec.rb +++ b/spec/lib/banzai/filter/issue_reference_filter_spec.rb @@ -199,6 +199,19 @@ describe Banzai::Filter::IssueReferenceFilter, lib: true do end end + context 'referencing external issues' do + let(:project) { create(:redmine_project) } + + it 'renders internal issue IDs as external issue links' do + doc = reference_filter('#1') + link = doc.css('a').first + + expect(link.attr('data-reference-type')).to eq('external_issue') + expect(link.attr('title')).to eq('Issue in Redmine') + expect(link.attr('data-external-issue')).to eq('1') + end + end + describe '#issues_per_Project' do context 'using an internal issue tracker' do it 'returns a Hash containing the issues per project' do diff --git a/spec/lib/gitlab/backend/shell_spec.rb b/spec/lib/gitlab/backend/shell_spec.rb index fd869f48b5c..6e5ba211382 100644 --- a/spec/lib/gitlab/backend/shell_spec.rb +++ b/spec/lib/gitlab/backend/shell_spec.rb @@ -13,9 +13,37 @@ describe Gitlab::Shell, lib: true do it { is_expected.to respond_to :add_repository } it { is_expected.to respond_to :remove_repository } it { is_expected.to respond_to :fork_repository } + it { is_expected.to respond_to :gc } + it { is_expected.to respond_to :add_namespace } + it { is_expected.to respond_to :rm_namespace } + it { is_expected.to respond_to :mv_namespace } + it { is_expected.to respond_to :exists? } it { expect(gitlab_shell.url_to_repo('diaspora')).to eq(Gitlab.config.gitlab_shell.ssh_path_prefix + "diaspora.git") } + describe 'generate_and_link_secret_token' do + let(:secret_file) { 'tmp/tests/.secret_shell_test' } + let(:link_file) { 'tmp/tests/shell-secret-test/.gitlab_shell_secret' } + + before do + allow(Gitlab.config.gitlab_shell).to receive(:path).and_return('tmp/tests/shell-secret-test') + allow(Gitlab.config.gitlab_shell).to receive(:secret_file).and_return(secret_file) + FileUtils.mkdir('tmp/tests/shell-secret-test') + gitlab_shell.generate_and_link_secret_token + end + + after do + FileUtils.rm_rf('tmp/tests/shell-secret-test') + FileUtils.rm_rf(secret_file) + end + + it 'creates and links the secret token file' do + expect(File.exist?(secret_file)).to be(true) + expect(File.symlink?(link_file)).to be(true) + expect(File.readlink(link_file)).to eq(secret_file) + end + end + describe Gitlab::Shell::KeyAdder, lib: true do describe '#add_key' do it 'normalizes space characters in the key' do diff --git a/spec/lib/gitlab/github_import/client_spec.rb b/spec/lib/gitlab/github_import/client_spec.rb index 7c21cbe96d9..3b023a35446 100644 --- a/spec/lib/gitlab/github_import/client_spec.rb +++ b/spec/lib/gitlab/github_import/client_spec.rb @@ -20,6 +20,20 @@ describe Gitlab::GithubImport::Client, lib: true do expect { client.api }.not_to raise_error end + context 'when config is missing' do + before do + allow(Gitlab.config.omniauth).to receive(:providers).and_return([]) + end + + it 'is still possible to get an Octokit client' do + expect { client.api }.not_to raise_error + end + + it 'is not be possible to get an OAuth2 client' do + expect { client.client }.to raise_error(Projects::ImportService::Error) + end + end + context 'allow SSL verification to be configurable on API' do before do github_provider['verify_ssl'] = false diff --git a/spec/lib/gitlab/popen_spec.rb b/spec/lib/gitlab/popen_spec.rb index 795cf241278..e8b236426e9 100644 --- a/spec/lib/gitlab/popen_spec.rb +++ b/spec/lib/gitlab/popen_spec.rb @@ -10,7 +10,7 @@ describe 'Gitlab::Popen', lib: true, no_db: true do context 'zero status' do before do - @output, @status = @klass.new.popen(%W(ls), path) + @output, @status = @klass.new.popen(%w(ls), path) end it { expect(@status).to be_zero } @@ -19,7 +19,7 @@ describe 'Gitlab::Popen', lib: true, no_db: true do context 'non-zero status' do before do - @output, @status = @klass.new.popen(%W(cat NOTHING), path) + @output, @status = @klass.new.popen(%w(cat NOTHING), path) end it { expect(@status).to eq(1) } @@ -34,7 +34,7 @@ describe 'Gitlab::Popen', lib: true, no_db: true do context 'without a directory argument' do before do - @output, @status = @klass.new.popen(%W(ls)) + @output, @status = @klass.new.popen(%w(ls)) end it { expect(@status).to be_zero } diff --git a/spec/models/application_setting_spec.rb b/spec/models/application_setting_spec.rb index d84f3e998f5..2ea1320267c 100644 --- a/spec/models/application_setting_spec.rb +++ b/spec/models/application_setting_spec.rb @@ -40,6 +40,16 @@ describe ApplicationSetting, models: true do it_behaves_like 'an object with email-formated attributes', :admin_notification_email do subject { setting } end + + context 'repository storages inclussion' do + before do + storages = { 'custom' => 'tmp/tests/custom_repositories' } + allow(Gitlab.config.repositories).to receive(:storages).and_return(storages) + end + + it { is_expected.to allow_value('custom').for(:repository_storage) } + it { is_expected.not_to allow_value('alternative').for(:repository_storage) } + end end context 'restricted signup domains' do diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb index 4e68ac5e63a..cbea407f9bf 100644 --- a/spec/models/namespace_spec.rb +++ b/spec/models/namespace_spec.rb @@ -57,6 +57,7 @@ describe Namespace, models: true do describe :move_dir do before do @namespace = create :namespace + @project = create :project, namespace: @namespace allow(@namespace).to receive(:path_changed?).and_return(true) end @@ -87,8 +88,13 @@ describe Namespace, models: true do end describe :rm_dir do - it "should remove dir" do - expect(namespace.rm_dir).to be_truthy + let!(:project) { create(:project, namespace: namespace) } + let!(:path) { File.join(Gitlab.config.repositories.storages.default, namespace.path) } + + before { namespace.destroy } + + it "should remove its dirs when deleted" do + expect(File.exist?(path)).to be(false) end end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index d305cd9ff1e..42308035d8c 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -56,6 +56,7 @@ describe Project, models: true do it { is_expected.to validate_length_of(:description).is_within(0..2000) } it { is_expected.to validate_presence_of(:creator) } it { is_expected.to validate_presence_of(:namespace) } + it { is_expected.to validate_presence_of(:repository_storage) } it 'should not allow new projects beyond user limits' do project2 = build(:project) @@ -84,6 +85,20 @@ describe Project, models: true do end end end + + context 'repository storages inclussion' do + let(:project2) { build(:project, repository_storage: 'missing') } + + before do + storages = { 'custom' => 'tmp/tests/custom_repositories' } + allow(Gitlab.config.repositories).to receive(:storages).and_return(storages) + end + + it "should not allow repository storages that don't match a label in the configuration" do + expect(project2).not_to be_valid + expect(project2.errors[:repository_storage].first).to match(/is not included in the list/) + end + end end describe 'default_scope' do @@ -131,6 +146,24 @@ describe Project, models: true do end end + describe '#repository_storage_path' do + let(:project) { create(:project, repository_storage: 'custom') } + + before do + FileUtils.mkdir('tmp/tests/custom_repositories') + storages = { 'custom' => 'tmp/tests/custom_repositories' } + allow(Gitlab.config.repositories).to receive(:storages).and_return(storages) + end + + after do + FileUtils.rm_rf('tmp/tests/custom_repositories') + end + + it 'returns the repository storage path' do + expect(project.repository_storage_path).to eq('tmp/tests/custom_repositories') + end + end + it 'should return valid url to repo' do project = Project.new(path: 'somewhere') expect(project.url_to_repo).to eq(Gitlab.config.gitlab_shell.ssh_path_prefix + 'somewhere.git') @@ -574,6 +607,21 @@ describe Project, models: true do end end + context 'repository storage by default' do + let(:project) { create(:empty_project) } + + subject { project.repository_storage } + + before do + storages = { 'alternative_storage' => '/some/path' } + allow(Gitlab.config.repositories).to receive(:storages).and_return(storages) + stub_application_setting(repository_storage: 'alternative_storage') + allow_any_instance_of(Project).to receive(:ensure_dir_exist).and_return(true) + end + + it { is_expected.to eq('alternative_storage') } + end + context 'shared runners by default' do let(:project) { create(:empty_project) } @@ -729,12 +777,12 @@ describe Project, models: true do expect(gitlab_shell).to receive(:mv_repository). ordered. - with("#{ns}/foo", "#{ns}/#{project.path}"). + with(project.repository_storage_path, "#{ns}/foo", "#{ns}/#{project.path}"). and_return(true) expect(gitlab_shell).to receive(:mv_repository). ordered. - with("#{ns}/foo.wiki", "#{ns}/#{project.path}.wiki"). + with(project.repository_storage_path, "#{ns}/foo.wiki", "#{ns}/#{project.path}.wiki"). and_return(true) expect_any_instance_of(SystemHooksService). @@ -826,7 +874,7 @@ describe Project, models: true do context 'using a regular repository' do it 'creates the repository' do expect(shell).to receive(:add_repository). - with(project.path_with_namespace). + with(project.repository_storage_path, project.path_with_namespace). and_return(true) expect(project.repository).to receive(:after_create) @@ -836,7 +884,7 @@ describe Project, models: true do it 'adds an error if the repository could not be created' do expect(shell).to receive(:add_repository). - with(project.path_with_namespace). + with(project.repository_storage_path, project.path_with_namespace). and_return(false) expect(project.repository).not_to receive(:after_create) diff --git a/spec/models/repository_spec.rb b/spec/models/repository_spec.rb index d8350000bf6..8ae083d7132 100644 --- a/spec/models/repository_spec.rb +++ b/spec/models/repository_spec.rb @@ -531,8 +531,6 @@ describe Repository, models: true do describe '#expire_cache' do it 'expires all caches' do expect(repository).to receive(:expire_branch_cache) - expect(repository).to receive(:expire_branch_count_cache) - expect(repository).to receive(:expire_tag_count_cache) repository.expire_cache end @@ -1055,12 +1053,14 @@ describe Repository, models: true do let(:cache) { repository.send(:cache) } it 'builds the caches if they do not already exist' do + cache_keys = repository.cache_keys + repository.cache_keys_for_branches_and_tags + expect(cache).to receive(:exist?). - exactly(repository.cache_keys.length). + exactly(cache_keys.length). times. and_return(false) - repository.cache_keys.each do |key| + cache_keys.each do |key| expect(repository).to receive(key) end @@ -1068,12 +1068,14 @@ describe Repository, models: true do end it 'does not build any caches that already exist' do + cache_keys = repository.cache_keys + repository.cache_keys_for_branches_and_tags + expect(cache).to receive(:exist?). - exactly(repository.cache_keys.length). + exactly(cache_keys.length). times. and_return(true) - repository.cache_keys.each do |key| + cache_keys.each do |key| expect(repository).not_to receive(key) end diff --git a/spec/requests/api/internal_spec.rb b/spec/requests/api/internal_spec.rb index 437c89c3577..fcea45f19ba 100644 --- a/spec/requests/api/internal_spec.rb +++ b/spec/requests/api/internal_spec.rb @@ -72,6 +72,7 @@ describe API::API, api: true do expect(response).to have_http_status(200) expect(json_response["status"]).to be_truthy + expect(json_response["repository_path"]).to eq(project.repository.path_to_repo) end end @@ -81,6 +82,7 @@ describe API::API, api: true do expect(response).to have_http_status(200) expect(json_response["status"]).to be_truthy + expect(json_response["repository_path"]).to eq(project.repository.path_to_repo) end end end diff --git a/spec/requests/api/settings_spec.rb b/spec/requests/api/settings_spec.rb index f756101c514..6629a5a65e2 100644 --- a/spec/requests/api/settings_spec.rb +++ b/spec/requests/api/settings_spec.rb @@ -14,16 +14,23 @@ describe API::API, 'Settings', api: true do expect(json_response).to be_an Hash expect(json_response['default_projects_limit']).to eq(42) expect(json_response['signin_enabled']).to be_truthy + expect(json_response['repository_storage']).to eq('default') end end describe "PUT /application/settings" do + before do + storages = { 'custom' => 'tmp/tests/custom_repositories' } + allow(Gitlab.config.repositories).to receive(:storages).and_return(storages) + end + it "should update application settings" do put api("/application/settings", admin), - default_projects_limit: 3, signin_enabled: false + default_projects_limit: 3, signin_enabled: false, repository_storage: 'custom' expect(response).to have_http_status(200) expect(json_response['default_projects_limit']).to eq(3) expect(json_response['signin_enabled']).to be_falsey + expect(json_response['repository_storage']).to eq('custom') end end end diff --git a/spec/services/destroy_group_service_spec.rb b/spec/services/destroy_group_service_spec.rb index afa89b84175..eca8ddd8ea4 100644 --- a/spec/services/destroy_group_service_spec.rb +++ b/spec/services/destroy_group_service_spec.rb @@ -23,8 +23,8 @@ describe DestroyGroupService, services: true do Sidekiq::Testing.inline! { destroy_group(group, user) } end - it { expect(gitlab_shell.exists?(group.path)).to be_falsey } - it { expect(gitlab_shell.exists?(remove_path)).to be_falsey } + it { expect(gitlab_shell.exists?(project.repository_storage_path, group.path)).to be_falsey } + it { expect(gitlab_shell.exists?(project.repository_storage_path, remove_path)).to be_falsey } end context 'Sidekiq fake' do @@ -33,8 +33,8 @@ describe DestroyGroupService, services: true do Sidekiq::Testing.fake! { destroy_group(group, user) } end - it { expect(gitlab_shell.exists?(group.path)).to be_falsey } - it { expect(gitlab_shell.exists?(remove_path)).to be_truthy } + it { expect(gitlab_shell.exists?(project.repository_storage_path, group.path)).to be_falsey } + it { expect(gitlab_shell.exists?(project.repository_storage_path, remove_path)).to be_truthy } end end diff --git a/spec/services/git_push_service_spec.rb b/spec/services/git_push_service_spec.rb index f99ad046f0d..48d374883d7 100644 --- a/spec/services/git_push_service_spec.rb +++ b/spec/services/git_push_service_spec.rb @@ -40,6 +40,18 @@ describe GitPushService, services: true do subject end + + it 'flushes the branches cache' do + expect(project.repository).to receive(:expire_branches_cache) + + subject + end + + it 'flushes the branch count cache' do + expect(project.repository).to receive(:expire_branch_count_cache) + + subject + end end context 'existing branch' do @@ -52,6 +64,18 @@ describe GitPushService, services: true do subject end + + it 'does not flush the branches cache' do + expect(project.repository).not_to receive(:expire_branches_cache) + + subject + end + + it 'does not flush the branch count cache' do + expect(project.repository).not_to receive(:expire_branch_count_cache) + + subject + end end context 'rm branch' do @@ -66,6 +90,18 @@ describe GitPushService, services: true do subject end + it 'flushes the branches cache' do + expect(project.repository).to receive(:expire_branches_cache) + + subject + end + + it 'flushes the branch count cache' do + expect(project.repository).to receive(:expire_branch_count_cache) + + subject + end + it 'flushes general cached data' do expect(project.repository).to receive(:expire_cache). with('master', newrev) @@ -314,6 +350,8 @@ describe GitPushService, services: true do it "doesn't close issues when external issue tracker is in use" do allow_any_instance_of(Project).to receive(:default_issues_tracker?). and_return(false) + external_issue_tracker = double(title: 'My Tracker', issue_path: issue.iid) + allow_any_instance_of(Project).to receive(:external_issue_tracker).and_return(external_issue_tracker) # The push still shouldn't create cross-reference notes. expect do diff --git a/spec/services/git_tag_push_service_spec.rb b/spec/services/git_tag_push_service_spec.rb index a63656e6268..a4fcd44882d 100644 --- a/spec/services/git_tag_push_service_spec.rb +++ b/spec/services/git_tag_push_service_spec.rb @@ -11,6 +11,31 @@ describe GitTagPushService, services: true do let(:newrev) { "8a2a6eb295bb170b34c24c76c49ed0e9b2eaf34b" } # gitlab-test: git rev-parse refs/tags/v1.1.0 let(:ref) { 'refs/tags/v1.1.0' } + describe "Push tags" do + subject do + service.execute + service + end + + it 'flushes general cached data' do + expect(project.repository).to receive(:expire_cache) + + subject + end + + it 'flushes the tags cache' do + expect(project.repository).to receive(:expire_tags_cache) + + subject + end + + it 'flushes the tag count cache' do + expect(project.repository).to receive(:expire_tag_count_cache) + + subject + end + end + describe "Git Tag Push Data" do before do service.execute diff --git a/spec/services/projects/housekeeping_service_spec.rb b/spec/services/projects/housekeeping_service_spec.rb index 4c5ced7e746..bd4dc6a0f79 100644 --- a/spec/services/projects/housekeeping_service_spec.rb +++ b/spec/services/projects/housekeeping_service_spec.rb @@ -12,7 +12,7 @@ describe Projects::HousekeepingService do it 'enqueues a sidekiq job' do expect(subject).to receive(:try_obtain_lease).and_return(true) - expect(GitlabShellOneShotWorker).to receive(:perform_async).with(:gc, project.path_with_namespace) + expect(GitlabShellOneShotWorker).to receive(:perform_async).with(:gc, project.repository_storage_path, project.path_with_namespace) subject.execute expect(project.pushes_since_gc).to eq(0) diff --git a/spec/services/projects/import_service_spec.rb b/spec/services/projects/import_service_spec.rb index 068c9a1219c..d5d4d7c56ef 100644 --- a/spec/services/projects/import_service_spec.rb +++ b/spec/services/projects/import_service_spec.rb @@ -36,7 +36,7 @@ describe Projects::ImportService, services: true do end it 'succeeds if repository import is successfully' do - expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_return(true) + expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_return(true) result = subject.execute @@ -44,7 +44,7 @@ describe Projects::ImportService, services: true do end it 'fails if repository import fails' do - expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_raise(Gitlab::Shell::Error.new('Failed to import the repository')) + expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_raise(Gitlab::Shell::Error.new('Failed to import the repository')) result = subject.execute @@ -64,7 +64,7 @@ describe Projects::ImportService, services: true do end it 'succeeds if importer succeeds' do - expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_return(true) + expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_return(true) expect_any_instance_of(Gitlab::GithubImport::Importer).to receive(:execute).and_return(true) result = subject.execute @@ -74,7 +74,7 @@ describe Projects::ImportService, services: true do it 'flushes various caches' do expect_any_instance_of(Gitlab::Shell).to receive(:import_repository). - with(project.path_with_namespace, project.import_url). + with(project.repository_storage_path, project.path_with_namespace, project.import_url). and_return(true) expect_any_instance_of(Gitlab::GithubImport::Importer).to receive(:execute). @@ -90,7 +90,7 @@ describe Projects::ImportService, services: true do end it 'fails if importer fails' do - expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_return(true) + expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_return(true) expect_any_instance_of(Gitlab::GithubImport::Importer).to receive(:execute).and_return(false) result = subject.execute @@ -100,7 +100,7 @@ describe Projects::ImportService, services: true do end it 'fails if importer raise an error' do - expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.path_with_namespace, project.import_url).and_return(true) + expect_any_instance_of(Gitlab::Shell).to receive(:import_repository).with(project.repository_storage_path, project.path_with_namespace, project.import_url).and_return(true) expect_any_instance_of(Gitlab::GithubImport::Importer).to receive(:execute).and_raise(Projects::ImportService::Error.new('Github: failed to connect API')) result = subject.execute diff --git a/spec/support/test_env.rb b/spec/support/test_env.rb index 426bf53f198..9f9ef20f99b 100644 --- a/spec/support/test_env.rb +++ b/spec/support/test_env.rb @@ -80,7 +80,7 @@ module TestEnv end def setup_gitlab_shell - unless File.directory?(Rails.root.join(*%w(tmp tests gitlab-shell))) + unless File.directory?(Gitlab.config.gitlab_shell.path) `rake gitlab:shell:install` end end @@ -127,14 +127,14 @@ module TestEnv def copy_repo(project) base_repo_path = File.expand_path(factory_repo_path_bare) - target_repo_path = File.expand_path(repos_path + "/#{project.namespace.path}/#{project.path}.git") + target_repo_path = File.expand_path(project.repository_storage_path + "/#{project.namespace.path}/#{project.path}.git") FileUtils.mkdir_p(target_repo_path) FileUtils.cp_r("#{base_repo_path}/.", target_repo_path) FileUtils.chmod_R 0755, target_repo_path end def repos_path - Gitlab.config.gitlab_shell.repos_path + Gitlab.config.repositories.storages.default end def backup_path @@ -143,7 +143,7 @@ module TestEnv def copy_forked_repo_with_submodules(project) base_repo_path = File.expand_path(forked_repo_path_bare) - target_repo_path = File.expand_path(repos_path + "/#{project.namespace.path}/#{project.path}.git") + target_repo_path = File.expand_path(project.repository_storage_path + "/#{project.namespace.path}/#{project.path}.git") FileUtils.mkdir_p(target_repo_path) FileUtils.cp_r("#{base_repo_path}/.", target_repo_path) FileUtils.chmod_R 0755, target_repo_path diff --git a/spec/tasks/gitlab/backup_rake_spec.rb b/spec/tasks/gitlab/backup_rake_spec.rb index 25da0917134..02308530d13 100644 --- a/spec/tasks/gitlab/backup_rake_spec.rb +++ b/spec/tasks/gitlab/backup_rake_spec.rb @@ -98,67 +98,107 @@ describe 'gitlab:app namespace rake task' do @backup_tar = tars_glob.first end - before do - create_backup - end - - after do - FileUtils.rm(@backup_tar) - end + context 'tar creation' do + before do + create_backup + end - context 'archive file permissions' do - it 'should set correct permissions on the tar file' do - expect(File.exist?(@backup_tar)).to be_truthy - expect(File::Stat.new(@backup_tar).mode.to_s(8)).to eq('100600') + after do + FileUtils.rm(@backup_tar) end - context 'with custom archive_permissions' do - before do - allow(Gitlab.config.backup).to receive(:archive_permissions).and_return(0651) - # We created a backup in a before(:all) so it got the default permissions. - # We now need to do some work to create a _new_ backup file using our stub. - FileUtils.rm(@backup_tar) - create_backup + context 'archive file permissions' do + it 'should set correct permissions on the tar file' do + expect(File.exist?(@backup_tar)).to be_truthy + expect(File::Stat.new(@backup_tar).mode.to_s(8)).to eq('100600') end - it 'uses the custom permissions' do - expect(File::Stat.new(@backup_tar).mode.to_s(8)).to eq('100651') + context 'with custom archive_permissions' do + before do + allow(Gitlab.config.backup).to receive(:archive_permissions).and_return(0651) + # We created a backup in a before(:all) so it got the default permissions. + # We now need to do some work to create a _new_ backup file using our stub. + FileUtils.rm(@backup_tar) + create_backup + end + + it 'uses the custom permissions' do + expect(File::Stat.new(@backup_tar).mode.to_s(8)).to eq('100651') + end end end - end - it 'should set correct permissions on the tar contents' do - tar_contents, exit_status = Gitlab::Popen.popen( - %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz lfs.tar.gz registry.tar.gz} - ) - expect(exit_status).to eq(0) - expect(tar_contents).to match('db/') - expect(tar_contents).to match('uploads.tar.gz') - expect(tar_contents).to match('repositories/') - expect(tar_contents).to match('builds.tar.gz') - expect(tar_contents).to match('artifacts.tar.gz') - expect(tar_contents).to match('lfs.tar.gz') - expect(tar_contents).to match('registry.tar.gz') - expect(tar_contents).not_to match(/^.{4,9}[rwx].* (database.sql.gz|uploads.tar.gz|repositories|builds.tar.gz|artifacts.tar.gz|registry.tar.gz)\/$/) - end + it 'should set correct permissions on the tar contents' do + tar_contents, exit_status = Gitlab::Popen.popen( + %W{tar -tvf #{@backup_tar} db uploads.tar.gz repositories builds.tar.gz artifacts.tar.gz lfs.tar.gz registry.tar.gz} + ) + expect(exit_status).to eq(0) + expect(tar_contents).to match('db/') + expect(tar_contents).to match('uploads.tar.gz') + expect(tar_contents).to match('repositories/') + expect(tar_contents).to match('builds.tar.gz') + expect(tar_contents).to match('artifacts.tar.gz') + expect(tar_contents).to match('lfs.tar.gz') + expect(tar_contents).to match('registry.tar.gz') + expect(tar_contents).not_to match(/^.{4,9}[rwx].* (database.sql.gz|uploads.tar.gz|repositories|builds.tar.gz|artifacts.tar.gz|registry.tar.gz)\/$/) + end - it 'should delete temp directories' do - temp_dirs = Dir.glob( - File.join(Gitlab.config.backup.path, '{db,repositories,uploads,builds,artifacts,lfs,registry}') - ) + it 'should delete temp directories' do + temp_dirs = Dir.glob( + File.join(Gitlab.config.backup.path, '{db,repositories,uploads,builds,artifacts,lfs,registry}') + ) + + expect(temp_dirs).to be_empty + end - expect(temp_dirs).to be_empty + context 'registry disabled' do + let(:enable_registry) { false } + + it 'should not create registry.tar.gz' do + tar_contents, exit_status = Gitlab::Popen.popen( + %W{tar -tvf #{@backup_tar}} + ) + expect(exit_status).to eq(0) + expect(tar_contents).not_to match('registry.tar.gz') + end + end end - context 'registry disabled' do - let(:enable_registry) { false } + context 'multiple repository storages' do + let(:project_a) { create(:project, repository_storage: 'default') } + let(:project_b) { create(:project, repository_storage: 'custom') } + + before do + FileUtils.mkdir('tmp/tests/default_storage') + FileUtils.mkdir('tmp/tests/custom_storage') + storages = { + 'default' => 'tmp/tests/default_storage', + 'custom' => 'tmp/tests/custom_storage' + } + allow(Gitlab.config.repositories).to receive(:storages).and_return(storages) + + # Create the projects now, after mocking the settings but before doing the backup + project_a + project_b + + # We only need a backup of the repositories for this test + ENV["SKIP"] = "db,uploads,builds,artifacts,lfs,registry" + create_backup + end + + after do + FileUtils.rm_rf('tmp/tests/default_storage') + FileUtils.rm_rf('tmp/tests/custom_storage') + FileUtils.rm(@backup_tar) + end - it 'should not create registry.tar.gz' do + it 'should include repositories in all repository storages' do tar_contents, exit_status = Gitlab::Popen.popen( - %W{tar -tvf #{@backup_tar}} + %W{tar -tvf #{@backup_tar} repositories} ) expect(exit_status).to eq(0) - expect(tar_contents).not_to match('registry.tar.gz') + expect(tar_contents).to match("repositories/#{project_a.path_with_namespace}.bundle") + expect(tar_contents).to match("repositories/#{project_b.path_with_namespace}.bundle") end end end # backup_create task diff --git a/spec/workers/post_receive_spec.rb b/spec/workers/post_receive_spec.rb index b8e73682c91..20b1a343c27 100644 --- a/spec/workers/post_receive_spec.rb +++ b/spec/workers/post_receive_spec.rb @@ -91,6 +91,6 @@ describe PostReceive do end def pwd(project) - File.join(Gitlab.config.gitlab_shell.repos_path, project.path_with_namespace) + File.join(Gitlab.config.repositories.storages.default, project.path_with_namespace) end end diff --git a/spec/workers/repository_fork_worker_spec.rb b/spec/workers/repository_fork_worker_spec.rb index 4ef05eb29d2..5f762282b5e 100644 --- a/spec/workers/repository_fork_worker_spec.rb +++ b/spec/workers/repository_fork_worker_spec.rb @@ -14,6 +14,7 @@ describe RepositoryForkWorker do describe "#perform" do it "creates a new repository from a fork" do expect(shell).to receive(:fork_repository).with( + project.repository_storage_path, project.path_with_namespace, fork_project.namespace.path ).and_return(true) @@ -25,9 +26,11 @@ describe RepositoryForkWorker do end it 'flushes various caches' do - expect(shell).to receive(:fork_repository). - with(project.path_with_namespace, fork_project.namespace.path). - and_return(true) + expect(shell).to receive(:fork_repository).with( + project.repository_storage_path, + project.path_with_namespace, + fork_project.namespace.path + ).and_return(true) expect_any_instance_of(Repository).to receive(:expire_emptiness_caches). and_call_original |