diff options
author | Filipa Lacerda <filipa@gitlab.com> | 2018-01-05 11:37:06 +0000 |
---|---|---|
committer | Filipa Lacerda <filipa@gitlab.com> | 2018-01-05 11:37:06 +0000 |
commit | f0c4fbea1c0621a6c5c12e9c9dd5eb534ee58157 (patch) | |
tree | c6dcad3a1f49194f741d24bc3de3581b59d34b2c | |
parent | 2cf2df8ea07703852247fdfbde73113d585aeb3d (diff) | |
parent | 96ef3e70383082f881c437754031e7c219eb50e1 (diff) | |
download | gitlab-ce-f0c4fbea1c0621a6c5c12e9c9dd5eb534ee58157.tar.gz |
Merge branch 'master' into 22643-manual-job-page
* master: (24 commits)
Allow local tests to use a modified Gitaly
Update check.md
add deprecation and removal issue to docs
Add status attribute to runner api entity
Fix typos in a code comment
Just try to detect and assign once
Modify `LDAP::Person` to return username value based on attributes
Create Kubernetes based on Application Templates
Fix a bug where charlock_holmes was used needlessly to encode strings
Ignore the Migration/Datetime cop in a migration that fix a column type to datetime_with_timezone
Use `__` instead of `s__` when context is not required
Enclose props in quotes
Update some Gitaly annotations in Gitlab::Shell
Prepare Gitlab::Git::Repository#rebase for Gitaly migration
add missing changelog
refactor spec, add docs
deprecate check integrity task
add lock specs
add locks chek
remove max-depth flag so it works with subgroups
...
38 files changed, 588 insertions, 191 deletions
diff --git a/app/assets/javascripts/groups/components/item_stats.vue b/app/assets/javascripts/groups/components/item_stats.vue index 803dc63d39c..2e42fb6c9a6 100644 --- a/app/assets/javascripts/groups/components/item_stats.vue +++ b/app/assets/javascripts/groups/components/item_stats.vue @@ -42,28 +42,28 @@ export default { v-if="isGroup" css-class="number-subgroups" icon-name="folder" - :title="s__('Subgroups')" - :value=item.subgroupCount + :title="__('Subgroups')" + :value="item.subgroupCount" /> <item-stats-value v-if="isGroup" css-class="number-projects" icon-name="bookmark" - :title="s__('Projects')" - :value=item.projectCount + :title="__('Projects')" + :value="item.projectCount" /> <item-stats-value v-if="isGroup" css-class="number-users" icon-name="users" - :title="s__('Members')" - :value=item.memberCount + :title="__('Members')" + :value="item.memberCount" /> <item-stats-value v-if="isProject" css-class="project-stars" icon-name="star" - :value=item.starCount + :value="item.starCount" /> <item-stats-value css-class="item-visibility" diff --git a/app/models/concerns/deployment_platform.rb b/app/models/concerns/deployment_platform.rb new file mode 100644 index 00000000000..89d0474a596 --- /dev/null +++ b/app/models/concerns/deployment_platform.rb @@ -0,0 +1,48 @@ +module DeploymentPlatform + def deployment_platform + @deployment_platform ||= + find_cluster_platform_kubernetes || + find_kubernetes_service_integration || + build_cluster_and_deployment_platform + end + + private + + def find_cluster_platform_kubernetes + clusters.find_by(enabled: true)&.platform_kubernetes + end + + def find_kubernetes_service_integration + services.deployment.reorder(nil).find_by(active: true) + end + + def build_cluster_and_deployment_platform + return unless kubernetes_service_template + + cluster = ::Clusters::Cluster.create(cluster_attributes_from_service_template) + cluster.platform_kubernetes if cluster.persisted? + end + + def kubernetes_service_template + @kubernetes_service_template ||= KubernetesService.active.find_by_template + end + + def cluster_attributes_from_service_template + { + name: 'kubernetes-template', + projects: [self], + provider_type: :user, + platform_type: :kubernetes, + platform_kubernetes_attributes: platform_kubernetes_attributes_from_service_template + } + end + + def platform_kubernetes_attributes_from_service_template + { + api_url: kubernetes_service_template.api_url, + ca_pem: kubernetes_service_template.ca_pem, + token: kubernetes_service_template.token, + namespace: kubernetes_service_template.namespace + } + end +end diff --git a/app/models/project.rb b/app/models/project.rb index 9c0bbf697e2..5d6c1b30587 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -19,6 +19,7 @@ class Project < ActiveRecord::Base include Routable include GroupDescendant include Gitlab::SQL::Pattern + include DeploymentPlatform extend Gitlab::ConfigHelper extend Gitlab::CurrentSettings @@ -904,12 +905,6 @@ class Project < ActiveRecord::Base @ci_service ||= ci_services.reorder(nil).find_by(active: true) end - # TODO: This will be extended for multiple enviroment clusters - def deployment_platform - @deployment_platform ||= clusters.find_by(enabled: true)&.platform_kubernetes - @deployment_platform ||= services.where(category: :deployment).reorder(nil).find_by(active: true) - end - def monitoring_services services.where(category: :monitoring) end diff --git a/app/models/service.rb b/app/models/service.rb index 176b472e724..24ba3039707 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -44,6 +44,7 @@ class Service < ActiveRecord::Base scope :pipeline_hooks, -> { where(pipeline_events: true, active: true) } scope :wiki_page_hooks, -> { where(wiki_page_events: true, active: true) } scope :external_issue_trackers, -> { issue_trackers.active.without_defaults } + scope :deployment, -> { where(category: 'deployment') } default_value_for :category, 'common' @@ -271,6 +272,10 @@ class Service < ActiveRecord::Base nil end + def self.find_by_template + find_by(template: true) + end + private def cache_project_has_external_issue_tracker diff --git a/changelogs/unreleased/40228-verify-integrity-of-repositories.yml b/changelogs/unreleased/40228-verify-integrity-of-repositories.yml new file mode 100644 index 00000000000..261d48652db --- /dev/null +++ b/changelogs/unreleased/40228-verify-integrity-of-repositories.yml @@ -0,0 +1,5 @@ +--- +title: Fix gitlab-rake gitlab:import:repos import schedule +merge_request: 15931 +author: +type: fixed diff --git a/changelogs/unreleased/41056-create-cluster-from-kubernetes-integration-application-template.yml b/changelogs/unreleased/41056-create-cluster-from-kubernetes-integration-application-template.yml new file mode 100644 index 00000000000..2dd6fc5f1b5 --- /dev/null +++ b/changelogs/unreleased/41056-create-cluster-from-kubernetes-integration-application-template.yml @@ -0,0 +1,5 @@ +--- +title: Allow automatic creation of Kubernetes Integration from template +merge_request: 16104 +author: +type: added diff --git a/changelogs/unreleased/feature-api_runners_online.yml b/changelogs/unreleased/feature-api_runners_online.yml new file mode 100644 index 00000000000..08f4dd16f28 --- /dev/null +++ b/changelogs/unreleased/feature-api_runners_online.yml @@ -0,0 +1,5 @@ +--- +title: Add online and status attribute to runner api entity +merge_request: 11750 +author: +type: added diff --git a/changelogs/unreleased/ldap_username_attributes.yml b/changelogs/unreleased/ldap_username_attributes.yml new file mode 100644 index 00000000000..89bbca58fc9 --- /dev/null +++ b/changelogs/unreleased/ldap_username_attributes.yml @@ -0,0 +1,5 @@ +--- +title: Modify `LDAP::Person` to return username value based on attributes +merge_request: +author: +type: fixed diff --git a/db/post_migrate/20171221140220_schedule_issues_closed_at_type_change.rb b/db/post_migrate/20171221140220_schedule_issues_closed_at_type_change.rb index be18c5866ae..eeecc7b1de0 100644 --- a/db/post_migrate/20171221140220_schedule_issues_closed_at_type_change.rb +++ b/db/post_migrate/20171221140220_schedule_issues_closed_at_type_change.rb @@ -1,6 +1,6 @@ # See http://doc.gitlab.com/ce/development/migration_style_guide.html # for more information on how to write migrations for GitLab. - +# rubocop:disable Migration/Datetime class ScheduleIssuesClosedAtTypeChange < ActiveRecord::Migration include Gitlab::Database::MigrationHelpers diff --git a/doc/administration/raketasks/check.md b/doc/administration/raketasks/check.md index c8b5434c068..c39cb49b1c6 100644 --- a/doc/administration/raketasks/check.md +++ b/doc/administration/raketasks/check.md @@ -28,19 +28,25 @@ exactly which repositories are causing the trouble. ### Check all GitLab repositories +>**Note:** +> +> - `gitlab:repo:check` has been deprecated in favor of `gitlab:git:fsck` +> - [Deprecated][ce-15931] in GitLab 10.4. +> - `gitlab:repo:check` will be removed in the future. [Removal issue][ce-41699] + This task loops through all repositories on the GitLab server and runs the 3 integrity checks described previously. **Omnibus Installation** ``` -sudo gitlab-rake gitlab:repo:check +sudo gitlab-rake gitlab:git:fsck ``` **Source Installation** ```bash -sudo -u git -H bundle exec rake gitlab:repo:check RAILS_ENV=production +sudo -u git -H bundle exec rake gitlab:git:fsck RAILS_ENV=production ``` ### Check repositories for a specific user @@ -76,3 +82,6 @@ The LDAP check Rake task will test the bind_dn and password credentials (if configured) and will list a sample of LDAP users. This task is also executed as part of the `gitlab:check` task, but can run independently. See [LDAP Rake Tasks - LDAP Check](ldap.md#check) for details. + +[ce-15931]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/15931 +[ce-41699]: https://gitlab.com/gitlab-org/gitlab-ce/issues/41699 diff --git a/doc/api/runners.md b/doc/api/runners.md index 015b09a745e..7495c6cdedb 100644 --- a/doc/api/runners.md +++ b/doc/api/runners.md @@ -30,14 +30,18 @@ Example response: "description": "test-1-20150125", "id": 6, "is_shared": false, - "name": null + "name": null, + "online": true, + "status": "online" }, { "active": true, "description": "test-2-20150125", "id": 8, "is_shared": false, - "name": null + "name": null, + "online": false, + "status": "offline" } ] ``` @@ -69,28 +73,36 @@ Example response: "description": "shared-runner-1", "id": 1, "is_shared": true, - "name": null + "name": null, + "online": true, + "status": "online" }, { "active": true, "description": "shared-runner-2", "id": 3, "is_shared": true, - "name": null + "name": null, + "online": false + "status": "offline" }, { "active": true, "description": "test-1-20150125", "id": 6, "is_shared": false, - "name": null + "name": null, + "online": true + "status": "paused" }, { "active": true, "description": "test-2-20150125", "id": 8, "is_shared": false, - "name": null + "name": null, + "online": false, + "status": "offline" } ] ``` @@ -122,6 +134,8 @@ Example response: "is_shared": false, "contacted_at": "2016-01-25T16:39:48.066Z", "name": null, + "online": true, + "status": "online", "platform": null, "projects": [ { @@ -176,6 +190,8 @@ Example response: "is_shared": false, "contacted_at": "2016-01-25T16:39:48.066Z", "name": null, + "online": true, + "status": "online", "platform": null, "projects": [ { @@ -327,14 +343,18 @@ Example response: "description": "test-2-20150125", "id": 8, "is_shared": false, - "name": null + "name": null, + "online": false, + "status": "offline" }, { "active": true, "description": "development_runner", "id": 5, "is_shared": true, - "name": null + "name": null, + "online": true + "status": "paused" } ] ``` @@ -364,7 +384,9 @@ Example response: "description": "test-2016-02-01", "id": 9, "is_shared": false, - "name": null + "name": null, + "online": true, + "status": "online" } ``` diff --git a/doc/development/gitaly.md b/doc/development/gitaly.md index ca2048c7019..26abf967dcf 100644 --- a/doc/development/gitaly.md +++ b/doc/development/gitaly.md @@ -97,6 +97,29 @@ describe 'Gitaly Request count tests' do end ``` +## Running tests with a locally modified version of Gitaly + +Normally, gitlab-ce/ee tests use a local clone of Gitaly in `tmp/tests/gitaly` +pinned at the version specified in GITALY_SERVER_VERSION. If you want +to run tests locally against a modified version of Gitaly you can +replace `tmp/tests/gitaly` with a symlink. + +```shell +rm -rf tmp/tests/gitaly +ln -s /path/to/gitaly tmp/tests/gitaly +``` + +Make sure you run `make` in your local Gitaly directory before running +tests. Otherwise, Gitaly will fail to boot. + +If you make changes to your local Gitaly in between test runs you need +to manually run `make` again. + +Note that CI tests will not use your locally modified version of +Gitaly. To use a custom Gitaly version in CI you need to update +GITALY_SERVER_VERSION. You can use the format `=revision` to use a +non-tagged commit from https://gitlab.com/gitlab-org/gitaly in CI. + --- [Return to Development documentation](README.md) diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 4ad4a1f7867..f5fa5fef389 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -862,6 +862,8 @@ module API expose :active expose :is_shared expose :name + expose :online?, as: :online + expose :status end class RunnerDetails < Runner diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 6b53eb4533d..c0edcabc6fd 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -14,14 +14,7 @@ module Gitlab ENCODING_CONFIDENCE_THRESHOLD = 50 def encode!(message) - return nil unless message.respond_to?(:force_encoding) - return message if message.encoding == Encoding::UTF_8 && message.valid_encoding? - - if message.respond_to?(:frozen?) && message.frozen? - message = message.dup - end - - message.force_encoding("UTF-8") + message = force_encode_utf8(message) return message if message.valid_encoding? # return message if message type is binary @@ -35,6 +28,8 @@ module Gitlab # encode and clean the bad chars message.replace clean(message) + rescue ArgumentError + return nil rescue encoding = detect ? detect[:encoding] : "unknown" "--broken encoding: #{encoding}" @@ -54,8 +49,8 @@ module Gitlab end def encode_utf8(message) - return nil unless message.is_a?(String) - return message if message.encoding == Encoding::UTF_8 && message.valid_encoding? + message = force_encode_utf8(message) + return message if message.valid_encoding? detect = CharlockHolmes::EncodingDetector.detect(message) if detect && detect[:encoding] @@ -69,6 +64,8 @@ module Gitlab else clean(message) end + rescue ArgumentError + return nil end def encode_binary(s) @@ -83,6 +80,15 @@ module Gitlab private + def force_encode_utf8(message) + raise ArgumentError unless message.respond_to?(:force_encoding) + return message if message.encoding == Encoding::UTF_8 && message.valid_encoding? + + message = message.dup if message.respond_to?(:frozen?) && message.frozen? + + message.force_encoding("UTF-8") + end + def clean(message) message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "") .encode("UTF-8") diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 1f7c35cafaa..71647099f83 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -11,7 +11,7 @@ module Gitlab include Gitlab::EncodingHelper def ref_name(ref) - encode_utf8(ref).sub(/\Arefs\/(tags|heads|remotes)\//, '') + encode!(ref).sub(/\Arefs\/(tags|heads|remotes)\//, '') end def branch_name(ref) diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 228d97a87ab..a1755143abe 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -50,10 +50,19 @@ module Gitlab # to the caller to limit the number of blobs and blob_size_limit. # # Gitaly migration issue: https://gitlab.com/gitlab-org/gitaly/issues/798 - def batch(repository, blob_references, blob_size_limit: nil) - blob_size_limit ||= MAX_DATA_DISPLAY_SIZE - blob_references.map do |sha, path| - find_by_rugged(repository, sha, path, limit: blob_size_limit) + def batch(repository, blob_references, blob_size_limit: MAX_DATA_DISPLAY_SIZE) + Gitlab::GitalyClient.migrate(:list_blobs_by_sha_path) do |is_enabled| + if is_enabled + Gitlab::GitalyClient.allow_n_plus_1_calls do + blob_references.map do |sha, path| + find_by_gitaly(repository, sha, path, limit: blob_size_limit) + end + end + else + blob_references.map do |sha, path| + find_by_rugged(repository, sha, path, limit: blob_size_limit) + end + end end end @@ -122,13 +131,23 @@ module Gitlab ) end - def find_by_gitaly(repository, sha, path) + def find_by_gitaly(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE) path = path.sub(/\A\/*/, '') path = '/' if path.empty? name = File.basename(path) - entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, MAX_DATA_DISPLAY_SIZE) + + # Gitaly will think that setting the limit to 0 means unlimited, while + # the client might only need the metadata and thus set the limit to 0. + # In this method we'll then set the limit to 1, but clear the byte of data + # that we got back so for the outside world it looks like the limit was + # actually 0. + req_limit = limit == 0 ? 1 : limit + + entry = Gitlab::GitalyClient::CommitService.new(repository).tree_entry(sha, path, req_limit) return unless entry + entry.data = "" if limit == 0 + case entry.type when :COMMIT new( diff --git a/lib/gitlab/git/operation_service.rb b/lib/gitlab/git/operation_service.rb index ef5bdbaf819..3fb0e2eed93 100644 --- a/lib/gitlab/git/operation_service.rb +++ b/lib/gitlab/git/operation_service.rb @@ -97,6 +97,11 @@ module Gitlab end end + def update_branch(branch_name, newrev, oldrev) + ref = Gitlab::Git::BRANCH_REF_PREFIX + branch_name + update_ref_in_hooks(ref, newrev, oldrev) + end + private # Returns [newrev, should_run_after_create, should_run_after_create_branch] diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index aec85f971ca..17c05c44d7e 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -1219,9 +1219,16 @@ module Gitlab rebase_path = worktree_path(REBASE_WORKTREE_PREFIX, rebase_id) env = git_env_for_user(user) + if remote_repository.is_a?(RemoteRepository) + env.merge!(remote_repository.fetch_env) + remote_repo_path = GITALY_INTERNAL_URL + else + remote_repo_path = remote_repository.path + end + with_worktree(rebase_path, branch, env: env) do run_git!( - %W(pull --rebase #{remote_repository.path} #{remote_branch}), + %W(pull --rebase #{remote_repo_path} #{remote_branch}), chdir: rebase_path, env: env ) diff --git a/lib/gitlab/ldap/adapter.rb b/lib/gitlab/ldap/adapter.rb index 0afaa2306b5..76863e77dc3 100644 --- a/lib/gitlab/ldap/adapter.rb +++ b/lib/gitlab/ldap/adapter.rb @@ -74,7 +74,7 @@ module Gitlab def user_options(fields, value, limit) options = { - attributes: Gitlab::LDAP::Person.ldap_attributes(config).compact.uniq, + attributes: Gitlab::LDAP::Person.ldap_attributes(config), base: config.base } diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index c8f19cd52d5..0d9a554fc18 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -148,7 +148,7 @@ module Gitlab def default_attributes { - 'username' => %w(uid userid sAMAccountName), + 'username' => %w(uid sAMAccountName userid), 'email' => %w(mail email userPrincipalName), 'name' => 'cn', 'first_name' => 'givenName', diff --git a/lib/gitlab/ldap/person.rb b/lib/gitlab/ldap/person.rb index 38d7a9ba2f5..e81cec6ba1a 100644 --- a/lib/gitlab/ldap/person.rb +++ b/lib/gitlab/ldap/person.rb @@ -6,6 +6,8 @@ module Gitlab # Source: http://ctogonewild.com/2009/09/03/bitmask-searches-in-ldap/ AD_USER_DISABLED = Net::LDAP::Filter.ex("userAccountControl:1.2.840.113556.1.4.803", "2") + InvalidEntryError = Class.new(StandardError) + attr_accessor :entry, :provider def self.find_by_uid(uid, adapter) @@ -29,11 +31,12 @@ module Gitlab def self.ldap_attributes(config) [ - 'dn', # Used in `dn` - config.uid, # Used in `uid` - *config.attributes['name'], # Used in `name` - *config.attributes['email'] # Used in `email` - ] + 'dn', + config.uid, + *config.attributes['name'], + *config.attributes['email'], + *config.attributes['username'] + ].compact.uniq end def self.normalize_dn(dn) @@ -60,6 +63,8 @@ module Gitlab Rails.logger.debug { "Instantiating #{self.class.name} with LDIF:\n#{entry.to_ldif}" } @entry = entry @provider = provider + + validate_entry end def name @@ -71,7 +76,13 @@ module Gitlab end def username - uid + username = attribute_value(:username) + + # Depending on the attribute, multiple values may + # be returned. We need only one for username. + # Ex. `uid` returns only one value but `mail` may + # return an array of multiple email addresses. + [username].flatten.first end def email @@ -104,6 +115,19 @@ module Gitlab entry.public_send(selected_attr) # rubocop:disable GitlabSecurity/PublicSend end + + def validate_entry + allowed_attrs = self.class.ldap_attributes(config).map(&:downcase) + + # Net::LDAP::Entry transforms keys to symbols. Change to strings to compare. + entry_attrs = entry.attribute_names.map { |n| n.to_s.downcase } + invalid_attrs = entry_attrs - allowed_attrs + + if invalid_attrs.any? + raise InvalidEntryError, + "#{self.class.name} initialized with Net::LDAP::Entry containing invalid attributes(s): #{invalid_attrs}" + end + end end end end diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb new file mode 100644 index 00000000000..d01213bb6e0 --- /dev/null +++ b/lib/gitlab/setup_helper.rb @@ -0,0 +1,61 @@ +module Gitlab + module SetupHelper + class << self + # We cannot create config.toml files for all possible Gitaly configuations. + # For instance, if Gitaly is running on another machine then it makes no + # sense to write a config.toml file on the current machine. This method will + # only generate a configuration for the most common and simplest case: when + # we have exactly one Gitaly process and we are sure it is running locally + # because it uses a Unix socket. + # For development and testing purposes, an extra storage is added to gitaly, + # which is not known to Rails, but must be explicitly stubbed. + def gitaly_configuration_toml(gitaly_dir, gitaly_ruby: true) + storages = [] + address = nil + + Gitlab.config.repositories.storages.each do |key, val| + if address + if address != val['gitaly_address'] + raise ArgumentError, "Your gitlab.yml contains more than one gitaly_address." + end + elsif URI(val['gitaly_address']).scheme != 'unix' + raise ArgumentError, "Automatic config.toml generation only supports 'unix:' addresses." + else + address = val['gitaly_address'] + end + + storages << { name: key, path: val['path'] } + end + + if Rails.env.test? + storages << { name: 'test_second_storage', path: Rails.root.join('tmp', 'tests', 'second_storage').to_s } + end + + config = { socket_path: address.sub(%r{\Aunix:}, ''), storage: storages } + config[:auth] = { token: 'secret' } if Rails.env.test? + config[:'gitaly-ruby'] = { dir: File.join(gitaly_dir, 'ruby') } if gitaly_ruby + config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path } + config[:bin_dir] = Gitlab.config.gitaly.client_path + + TOML.dump(config) + end + + # rubocop:disable Rails/Output + def create_gitaly_configuration(dir, force: false) + config_path = File.join(dir, 'config.toml') + FileUtils.rm_f(config_path) if force + + File.open(config_path, File::WRONLY | File::CREAT | File::EXCL) do |f| + f.puts gitaly_configuration_toml(dir) + end + rescue Errno::EEXIST + puts "Skipping config.toml generation:" + puts "A configuration file already exists." + rescue ArgumentError => e + puts "Skipping config.toml generation:" + puts e.message + end + # rubocop:enable Rails/Output + end + end +end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index 40650fc5ee7..b9cc97c9244 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -71,7 +71,6 @@ module Gitlab # Ex. # add_repository("/path/to/storage", "gitlab/gitlab-ci") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def add_repository(storage, name) relative_path = name.dup relative_path << '.git' unless relative_path.end_with?('.git') @@ -100,7 +99,7 @@ module Gitlab # Ex. # import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://gitlab.com/gitlab-org/gitlab-test.git") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/874 def import_repository(storage, name, url) # The timeout ensures the subprocess won't hang forever cmd = gitlab_projects(storage, "#{name}.git") @@ -122,7 +121,6 @@ module Gitlab # Ex. # fetch_remote(my_repo, "upstream") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 def fetch_remote(repository, remote, ssh_auth: nil, forced: false, no_tags: false) gitaly_migrate(:fetch_remote) do |is_enabled| if is_enabled @@ -142,7 +140,7 @@ module Gitlab # Ex. # mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/873 def mv_repository(storage, path, new_path) gitlab_projects(storage, "#{path}.git").mv_project("#{new_path}.git") end @@ -156,7 +154,7 @@ module Gitlab # Ex. # fork_repository("/path/to/forked_from/storage", "gitlab/gitlab-ci", "/path/to/forked_to/storage", "new-namespace/gitlab-ci") # - # Gitaly note: JV: not easy to migrate because this involves two Gitaly servers, not one. + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/817 def fork_repository(forked_from_storage, forked_from_disk_path, forked_to_storage, forked_to_disk_path) gitlab_projects(forked_from_storage, "#{forked_from_disk_path}.git") .fork_repository(forked_to_storage, "#{forked_to_disk_path}.git") @@ -170,7 +168,7 @@ module Gitlab # Ex. # remove_repository("/path/to/storage", "gitlab/gitlab-ci") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/387 + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/873 def remove_repository(storage, name) gitlab_projects(storage, "#{name}.git").rm_project end @@ -221,7 +219,6 @@ module Gitlab # Ex. # add_namespace("/path/to/storage", "gitlab") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def add_namespace(storage, name) Gitlab::GitalyClient.migrate(:add_namespace) do |enabled| if enabled @@ -243,7 +240,6 @@ module Gitlab # Ex. # rm_namespace("/path/to/storage", "gitlab") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def rm_namespace(storage, name) Gitlab::GitalyClient.migrate(:remove_namespace) do |enabled| if enabled @@ -261,7 +257,6 @@ module Gitlab # Ex. # mv_namespace("/path/to/storage", "gitlab", "gitlabhq") # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/385 def mv_namespace(storage, old_name, new_name) Gitlab::GitalyClient.migrate(:rename_namespace) do |enabled| if enabled diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index dfade1f3885..903e84359cd 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -387,14 +387,8 @@ namespace :gitlab do namespace :repo do desc "GitLab | Check the integrity of the repositories managed by GitLab" task check: :environment do - Gitlab.config.repositories.storages.each do |name, repository_storage| - namespace_dirs = Dir.glob(File.join(repository_storage['path'], '*')) - - namespace_dirs.each do |namespace_dir| - repo_dirs = Dir.glob(File.join(namespace_dir, '*')) - repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) } - end - end + puts "This task is deprecated. Please use gitlab:git:fsck instead".color(:red) + Rake::Task["gitlab:git:fsck"].execute end end @@ -461,35 +455,4 @@ namespace :gitlab do puts "FAIL. Please update gitlab-shell to #{required_version} from #{current_version}".color(:red) end end - - def check_repo_integrity(repo_dir) - puts "\nChecking repo at #{repo_dir.color(:yellow)}" - - git_fsck(repo_dir) - check_config_lock(repo_dir) - check_ref_locks(repo_dir) - end - - def git_fsck(repo_dir) - puts "Running `git fsck`".color(:yellow) - system(*%W(#{Gitlab.config.git.bin_path} fsck), chdir: repo_dir) - end - - def check_config_lock(repo_dir) - config_exists = File.exist?(File.join(repo_dir, 'config.lock')) - config_output = config_exists ? 'yes'.color(:red) : 'no'.color(:green) - puts "'config.lock' file exists?".color(:yellow) + " ... #{config_output}" - end - - def check_ref_locks(repo_dir) - lock_files = Dir.glob(File.join(repo_dir, 'refs/heads/*.lock')) - if lock_files.present? - puts "Ref lock files exist:".color(:red) - lock_files.each do |lock_file| - puts " #{lock_file}" - end - else - puts "No ref lock files exist".color(:green) - end - end end diff --git a/lib/tasks/gitlab/git.rake b/lib/tasks/gitlab/git.rake index cf82134d97e..3f5dd2ae3b3 100644 --- a/lib/tasks/gitlab/git.rake +++ b/lib/tasks/gitlab/git.rake @@ -30,6 +30,20 @@ namespace :gitlab do end end + desc 'GitLab | Git | Check all repos integrity' + task fsck: :environment do + failures = perform_git_cmd(%W(#{Gitlab.config.git.bin_path} fsck --name-objects --no-progress), "Checking integrity") do |repo| + check_config_lock(repo) + check_ref_locks(repo) + end + + if failures.empty? + puts "Done".color(:green) + else + output_failures(failures) + end + end + def perform_git_cmd(cmd, message) puts "Starting #{message} on all repositories" @@ -40,6 +54,8 @@ namespace :gitlab do else failures << repo end + + yield(repo) if block_given? end failures @@ -49,5 +65,24 @@ namespace :gitlab do puts "The following repositories reported errors:".color(:red) failures.each { |f| puts "- #{f}" } end + + def check_config_lock(repo_dir) + config_exists = File.exist?(File.join(repo_dir, 'config.lock')) + config_output = config_exists ? 'yes'.color(:red) : 'no'.color(:green) + + puts "'config.lock' file exists?".color(:yellow) + " ... #{config_output}" + end + + def check_ref_locks(repo_dir) + lock_files = Dir.glob(File.join(repo_dir, 'refs/heads/*.lock')) + + if lock_files.present? + puts "Ref lock files exist:".color(:red) + + lock_files.each { |lock_file| puts " #{lock_file}" } + else + puts "No ref lock files exist".color(:green) + end + end end end diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index 4d880c05f99..4507b841964 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -21,8 +21,8 @@ namespace :gitlab do command << 'BUNDLE_FLAGS=--no-deployment' if Rails.env.test? + Gitlab::SetupHelper.create_gitaly_configuration(args.dir) Dir.chdir(args.dir) do - create_gitaly_configuration # In CI we run scripts/gitaly-test-build instead of this command unless ENV['CI'].present? Bundler.with_original_env { run_command!(command) } @@ -39,60 +39,7 @@ namespace :gitlab do # Exclude gitaly-ruby configuration because that depends on the gitaly # installation directory. - puts gitaly_configuration_toml(gitaly_ruby: false) - end - - private - - # We cannot create config.toml files for all possible Gitaly configuations. - # For instance, if Gitaly is running on another machine then it makes no - # sense to write a config.toml file on the current machine. This method will - # only generate a configuration for the most common and simplest case: when - # we have exactly one Gitaly process and we are sure it is running locally - # because it uses a Unix socket. - # For development and testing purposes, an extra storage is added to gitaly, - # which is not known to Rails, but must be explicitly stubbed. - def gitaly_configuration_toml(gitaly_ruby: true) - storages = [] - address = nil - - Gitlab.config.repositories.storages.each do |key, val| - if address - if address != val['gitaly_address'] - raise ArgumentError, "Your gitlab.yml contains more than one gitaly_address." - end - elsif URI(val['gitaly_address']).scheme != 'unix' - raise ArgumentError, "Automatic config.toml generation only supports 'unix:' addresses." - else - address = val['gitaly_address'] - end - - storages << { name: key, path: val['path'] } - end - - if Rails.env.test? - storages << { name: 'test_second_storage', path: Rails.root.join('tmp', 'tests', 'second_storage').to_s } - end - - config = { socket_path: address.sub(%r{\Aunix:}, ''), storage: storages } - config[:auth] = { token: 'secret' } if Rails.env.test? - config[:'gitaly-ruby'] = { dir: File.join(Dir.pwd, 'ruby') } if gitaly_ruby - config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path } - config[:bin_dir] = Gitlab.config.gitaly.client_path - - TOML.dump(config) - end - - def create_gitaly_configuration - File.open("config.toml", File::WRONLY | File::CREAT | File::EXCL) do |f| - f.puts gitaly_configuration_toml - end - rescue Errno::EEXIST - puts "Skipping config.toml generation:" - puts "A configuration file already exists." - rescue ArgumentError => e - puts "Skipping config.toml generation:" - puts e.message + puts Gitlab::SetupHelper.gitaly_configuration_toml('', gitaly_ruby: false) end end end diff --git a/lib/tasks/gitlab/task_helpers.rb b/lib/tasks/gitlab/task_helpers.rb index 6723662703c..c1182af1014 100644 --- a/lib/tasks/gitlab/task_helpers.rb +++ b/lib/tasks/gitlab/task_helpers.rb @@ -130,7 +130,7 @@ module Gitlab def all_repos Gitlab.config.repositories.storages.each_value do |repository_storage| - IO.popen(%W(find #{repository_storage['path']} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| + IO.popen(%W(find #{repository_storage['path']} -mindepth 2 -type d -name *.git)) do |find| find.each_line do |path| yield path.chomp end diff --git a/spec/features/projects/import_export/export_file_spec.rb b/spec/features/projects/import_export/export_file_spec.rb index 461aa39d0ad..6732cf61767 100644 --- a/spec/features/projects/import_export/export_file_spec.rb +++ b/spec/features/projects/import_export/export_file_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' # Integration test that exports a file using the Import/Export feature # It looks up for any sensitive word inside the JSON, so if a sensitive word is found -# we''l have to either include it adding the model that includes it to the +safe_list+ +# we'll have to either include it adding the model that includes it to the +safe_list+ # or make sure the attribute is blacklisted in the +import_export.yml+ configuration feature 'Import/Export - project export integration test', :js do include Select2Helper diff --git a/spec/lib/gitlab/encoding_helper_spec.rb b/spec/lib/gitlab/encoding_helper_spec.rb index 87ec2698fc1..4e9367323cb 100644 --- a/spec/lib/gitlab/encoding_helper_spec.rb +++ b/spec/lib/gitlab/encoding_helper_spec.rb @@ -120,6 +120,24 @@ describe Gitlab::EncodingHelper do it 'returns empty string on conversion errors' do expect { ext_class.encode_utf8('') }.not_to raise_error(ArgumentError) end + + context 'with strings that can be forcefully encoded into utf8' do + let(:test_string) do + "refs/heads/FixSymbolsTitleDropdown".encode("ASCII-8BIT") + end + let(:expected_string) do + "refs/heads/FixSymbolsTitleDropdown".encode("UTF-8") + end + + subject { ext_class.encode_utf8(test_string) } + + it "doesn't use CharlockHolmes if the encoding can be forced into utf_8" do + expect(CharlockHolmes::EncodingDetector).not_to receive(:detect) + + expect(subject).to eq(expected_string) + expect(subject.encoding.name).to eq('UTF-8') + end + end end describe '#clean' do diff --git a/spec/lib/gitlab/git/blob_spec.rb b/spec/lib/gitlab/git/blob_spec.rb index c04a9688503..7f5946b1658 100644 --- a/spec/lib/gitlab/git/blob_spec.rb +++ b/spec/lib/gitlab/git/blob_spec.rb @@ -202,16 +202,6 @@ describe Gitlab::Git::Blob, seed_helper: true do context 'limiting' do subject { described_class.batch(repository, blob_references, blob_size_limit: blob_size_limit) } - context 'default' do - let(:blob_size_limit) { nil } - - it 'limits to MAX_DATA_DISPLAY_SIZE' do - stub_const('Gitlab::Git::Blob::MAX_DATA_DISPLAY_SIZE', 100) - - expect(subject.first.data.size).to eq(100) - end - end - context 'positive' do let(:blob_size_limit) { 10 } @@ -221,7 +211,10 @@ describe Gitlab::Git::Blob, seed_helper: true do context 'zero' do let(:blob_size_limit) { 0 } - it { expect(subject.first.data).to eq('') } + it 'only loads the metadata' do + expect(subject.first.size).not_to be(0) + expect(subject.first.data).to eq('') + end end context 'negative' do diff --git a/spec/lib/gitlab/ldap/adapter_spec.rb b/spec/lib/gitlab/ldap/adapter_spec.rb index d9ddb4326be..6132abd9b35 100644 --- a/spec/lib/gitlab/ldap/adapter_spec.rb +++ b/spec/lib/gitlab/ldap/adapter_spec.rb @@ -16,7 +16,7 @@ describe Gitlab::LDAP::Adapter do expect(adapter).to receive(:ldap_search) do |arg| expect(arg[:filter].to_s).to eq('(uid=johndoe)') expect(arg[:base]).to eq('dc=example,dc=com') - expect(arg[:attributes]).to match(%w{dn uid cn mail email userPrincipalName}) + expect(arg[:attributes]).to match(ldap_attributes) end.and_return({}) adapter.users('uid', 'johndoe') @@ -26,7 +26,7 @@ describe Gitlab::LDAP::Adapter do expect(adapter).to receive(:ldap_search).with( base: 'uid=johndoe,ou=users,dc=example,dc=com', scope: Net::LDAP::SearchScope_BaseObject, - attributes: %w{dn uid cn mail email userPrincipalName}, + attributes: ldap_attributes, filter: nil ).and_return({}) @@ -63,7 +63,7 @@ describe Gitlab::LDAP::Adapter do it 'uses the right uid attribute when non-default' do stub_ldap_config(uid: 'sAMAccountName') expect(adapter).to receive(:ldap_search).with( - hash_including(attributes: %w{dn sAMAccountName cn mail email userPrincipalName}) + hash_including(attributes: ldap_attributes) ).and_return({}) adapter.users('sAMAccountName', 'johndoe') @@ -137,4 +137,8 @@ describe Gitlab::LDAP::Adapter do end end end + + def ldap_attributes + Gitlab::LDAP::Person.ldap_attributes(Gitlab::LDAP::Config.new('ldapmain')) + end end diff --git a/spec/lib/gitlab/ldap/person_spec.rb b/spec/lib/gitlab/ldap/person_spec.rb index d204050ef66..ff29d9aa5be 100644 --- a/spec/lib/gitlab/ldap/person_spec.rb +++ b/spec/lib/gitlab/ldap/person_spec.rb @@ -8,13 +8,16 @@ describe Gitlab::LDAP::Person do before do stub_ldap_config( options: { + 'uid' => 'uid', 'attributes' => { - 'name' => 'cn', - 'email' => %w(mail email userPrincipalName) + 'name' => 'cn', + 'email' => %w(mail email userPrincipalName), + 'username' => username_attribute } } ) end + let(:username_attribute) { %w(uid sAMAccountName userid) } describe '.normalize_dn' do subject { described_class.normalize_dn(given) } @@ -44,6 +47,34 @@ describe Gitlab::LDAP::Person do end end + describe '.ldap_attributes' do + it 'returns a compact and unique array' do + stub_ldap_config( + options: { + 'uid' => nil, + 'attributes' => { + 'name' => 'cn', + 'email' => 'mail', + 'username' => %w(uid mail memberof) + } + } + ) + config = Gitlab::LDAP::Config.new('ldapmain') + ldap_attributes = described_class.ldap_attributes(config) + + expect(ldap_attributes).to match_array(%w(dn uid cn mail memberof)) + end + end + + describe '.validate_entry' do + it 'raises InvalidEntryError' do + entry['foo'] = 'bar' + + expect { described_class.new(entry, 'ldapmain') } + .to raise_error(Gitlab::LDAP::Person::InvalidEntryError) + end + end + describe '#name' do it 'uses the configured name attribute and handles values as an array' do name = 'John Doe' @@ -72,6 +103,44 @@ describe Gitlab::LDAP::Person do end end + describe '#username' do + context 'with default uid username attribute' do + let(:username_attribute) { 'uid' } + + it 'returns the proper username value' do + attr_value = 'johndoe' + entry[username_attribute] = attr_value + person = described_class.new(entry, 'ldapmain') + + expect(person.username).to eq(attr_value) + end + end + + context 'with a different username attribute' do + let(:username_attribute) { 'sAMAccountName' } + + it 'returns the proper username value' do + attr_value = 'johndoe' + entry[username_attribute] = attr_value + person = described_class.new(entry, 'ldapmain') + + expect(person.username).to eq(attr_value) + end + end + + context 'with a non-standard username attribute' do + let(:username_attribute) { 'mail' } + + it 'returns the proper username value' do + attr_value = 'john.doe@example.com' + entry[username_attribute] = attr_value + person = described_class.new(entry, 'ldapmain') + + expect(person.username).to eq(attr_value) + end + end + end + def assert_generic_test(test_description, got, expected) test_failure_message = "Failed test description: '#{test_description}'\n\n expected: #{expected}\n got: #{got}" expect(got).to eq(expected), test_failure_message diff --git a/spec/lib/gitlab/o_auth/user_spec.rb b/spec/lib/gitlab/o_auth/user_spec.rb index 6334bcd0156..45fff4c5787 100644 --- a/spec/lib/gitlab/o_auth/user_spec.rb +++ b/spec/lib/gitlab/o_auth/user_spec.rb @@ -275,6 +275,26 @@ describe Gitlab::OAuth::User do end end + context 'and a corresponding LDAP person with a non-default username' do + before do + allow(ldap_user).to receive(:uid) { uid } + allow(ldap_user).to receive(:username) { 'johndoe@example.com' } + allow(ldap_user).to receive(:email) { %w(johndoe@example.com john2@example.com) } + allow(ldap_user).to receive(:dn) { dn } + end + + context 'and no account for the LDAP user' do + it 'creates a user favoring the LDAP username and strips email domain' do + allow(Gitlab::LDAP::Person).to receive(:find_by_uid).and_return(ldap_user) + + oauth_user.save + + expect(gl_user).to be_valid + expect(gl_user.username).to eql 'johndoe' + end + end + end + context "and no corresponding LDAP person" do before do allow(Gitlab::LDAP::Person).to receive(:find_by_uid).and_return(nil) diff --git a/spec/models/concerns/deployment_platform_spec.rb b/spec/models/concerns/deployment_platform_spec.rb new file mode 100644 index 00000000000..7bb89fe41dc --- /dev/null +++ b/spec/models/concerns/deployment_platform_spec.rb @@ -0,0 +1,73 @@ +require 'rails_helper' + +describe DeploymentPlatform do + let(:project) { create(:project) } + + describe '#deployment_platform' do + subject { project.deployment_platform } + + context 'with no Kubernetes configuration on CI/CD, no Kubernetes Service and a Kubernetes template configured' do + let!(:kubernetes_service) { create(:kubernetes_service, template: true) } + + it 'returns a platform kubernetes' do + expect(subject).to be_a_kind_of(Clusters::Platforms::Kubernetes) + end + + it 'creates a cluster and a platform kubernetes' do + expect { subject } + .to change { Clusters::Cluster.count }.by(1) + .and change { Clusters::Platforms::Kubernetes.count }.by(1) + end + + it 'includes appropriate attributes for Cluster' do + cluster = subject.cluster + expect(cluster.name).to eq('kubernetes-template') + expect(cluster.project).to eq(project) + expect(cluster.provider_type).to eq('user') + expect(cluster.platform_type).to eq('kubernetes') + end + + it 'creates a platform kubernetes' do + expect { subject }.to change { Clusters::Platforms::Kubernetes.count }.by(1) + end + + it 'copies attributes from Clusters::Platform::Kubernetes template into the new Cluster::Platforms::Kubernetes' do + expect(subject.api_url).to eq(kubernetes_service.api_url) + expect(subject.ca_pem).to eq(kubernetes_service.ca_pem) + expect(subject.token).to eq(kubernetes_service.token) + expect(subject.namespace).to eq(kubernetes_service.namespace) + end + end + + context 'with no Kubernetes configuration on CI/CD, no Kubernetes Service and no Kubernetes template configured' do + it { is_expected.to be_nil } + end + + context 'when user configured kubernetes from CI/CD > Clusters' do + let!(:cluster) { create(:cluster, :provided_by_gcp, projects: [project]) } + let(:platform_kubernetes) { cluster.platform_kubernetes } + + it 'returns the Kubernetes platform' do + expect(subject).to eq(platform_kubernetes) + end + end + + context 'when user configured kubernetes integration from project services' do + let!(:kubernetes_service) { create(:kubernetes_service, project: project) } + + it 'returns the Kubernetes service' do + expect(subject).to eq(kubernetes_service) + end + end + + context 'when the cluster creation fails' do + let!(:kubernetes_service) { create(:kubernetes_service, template: true) } + + before do + allow_any_instance_of(Clusters::Cluster).to receive(:persisted?).and_return(false) + end + + it { is_expected.to be_nil } + end + end +end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index cea22bbd184..3c2ed043b82 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -3137,25 +3137,6 @@ describe Project do end end - describe '#deployment_platform' do - subject { project.deployment_platform } - - let(:project) { create(:project) } - - context 'when user configured kubernetes from Integration > Kubernetes' do - let!(:kubernetes_service) { create(:kubernetes_service, project: project) } - - it { is_expected.to eq(kubernetes_service) } - end - - context 'when user configured kubernetes from CI/CD > Clusters' do - let!(:cluster) { create(:cluster, :provided_by_gcp, projects: [project]) } - let(:platform_kubernetes) { cluster.platform_kubernetes } - - it { is_expected.to eq(platform_kubernetes) } - end - end - describe '#write_repository_config' do set(:project) { create(:project, :repository) } diff --git a/spec/models/service_spec.rb b/spec/models/service_spec.rb index 540615de117..ab6678cab38 100644 --- a/spec/models/service_spec.rb +++ b/spec/models/service_spec.rb @@ -272,4 +272,12 @@ describe Service do expect(service.deprecation_message).to be_nil end end + + describe '.find_by_template' do + let!(:kubernetes_service) { create(:kubernetes_service, template: true) } + + it 'returns service template' do + expect(KubernetesService.find_by_template).to eq(kubernetes_service) + end + end end diff --git a/spec/support/test_env.rb b/spec/support/test_env.rb index 1d99746b09f..664698fcbaf 100644 --- a/spec/support/test_env.rb +++ b/spec/support/test_env.rb @@ -1,4 +1,5 @@ require 'rspec/mocks' +require 'toml' module TestEnv extend self @@ -147,6 +148,9 @@ module TestEnv version: Gitlab::GitalyClient.expected_server_version, task: "gitlab:gitaly:install[#{gitaly_dir}]") do + # Always re-create config, in case it's outdated. This is fast anyway. + Gitlab::SetupHelper.create_gitaly_configuration(gitaly_dir, force: true) + start_gitaly(gitaly_dir) end end @@ -347,6 +351,9 @@ module TestEnv end def component_needs_update?(component_folder, expected_version) + # Allow local overrides of the component for tests during development + return false if Rails.env.test? && File.symlink?(component_folder) + version = File.read(File.join(component_folder, 'VERSION')).strip # Notice that this will always yield true when using branch versions diff --git a/spec/tasks/gitlab/git_rake_spec.rb b/spec/tasks/gitlab/git_rake_spec.rb new file mode 100644 index 00000000000..dacc5dc5ae7 --- /dev/null +++ b/spec/tasks/gitlab/git_rake_spec.rb @@ -0,0 +1,38 @@ +require 'rake_helper' + +describe 'gitlab:git rake tasks' do + before do + Rake.application.rake_require 'tasks/gitlab/git' + + storages = { 'default' => { 'path' => Settings.absolute('tmp/tests/default_storage') } } + + FileUtils.mkdir_p(Settings.absolute('tmp/tests/default_storage/@hashed/1/2/test.git')) + allow(Gitlab.config.repositories).to receive(:storages).and_return(storages) + allow_any_instance_of(String).to receive(:color) { |string, _color| string } + + stub_warn_user_is_not_gitlab + end + + after do + FileUtils.rm_rf(Settings.absolute('tmp/tests/default_storage')) + end + + describe 'fsck' do + it 'outputs the integrity check for a repo' do + expect { run_rake_task('gitlab:git:fsck') }.to output(/Performed Checking integrity at .*@hashed\/1\/2\/test.git/).to_stdout + end + + it 'errors out about config.lock issues' do + FileUtils.touch(Settings.absolute('tmp/tests/default_storage/@hashed/1/2/test.git/config.lock')) + + expect { run_rake_task('gitlab:git:fsck') }.to output(/file exists\? ... yes/).to_stdout + end + + it 'errors out about ref lock issues' do + FileUtils.mkdir_p(Settings.absolute('tmp/tests/default_storage/@hashed/1/2/test.git/refs/heads')) + FileUtils.touch(Settings.absolute('tmp/tests/default_storage/@hashed/1/2/test.git/refs/heads/blah.lock')) + + expect { run_rake_task('gitlab:git:fsck') }.to output(/Ref lock files exist:/).to_stdout + end + end +end |