diff options
42 files changed, 440 insertions, 200 deletions
diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index c64d9d48a48..a9a7f3fec01 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -0.105.1 +0.107.0 @@ -419,7 +419,7 @@ group :ed25519 do end # Gitaly GRPC client -gem 'gitaly-proto', '~> 0.101.0', require: 'gitaly' +gem 'gitaly-proto', '~> 0.102.0', require: 'gitaly' gem 'grpc', '~> 1.11.0' # Locked until https://github.com/google/protobuf/issues/4210 is closed diff --git a/Gemfile.lock b/Gemfile.lock index 883e580b86b..8d508cc0f39 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -283,7 +283,7 @@ GEM gettext_i18n_rails (>= 0.7.1) po_to_json (>= 1.0.0) rails (>= 3.2.0) - gitaly-proto (0.101.0) + gitaly-proto (0.102.0) google-protobuf (~> 3.1) grpc (~> 1.10) github-linguist (5.3.3) @@ -1039,7 +1039,7 @@ DEPENDENCIES gettext (~> 3.2.2) gettext_i18n_rails (~> 1.8.0) gettext_i18n_rails_js (~> 1.3) - gitaly-proto (~> 0.101.0) + gitaly-proto (~> 0.102.0) github-linguist (~> 5.3.3) gitlab-flowdock-git-hook (~> 1.0.1) gitlab-gollum-lib (~> 4.2) diff --git a/Gemfile.rails5.lock b/Gemfile.rails5.lock index 223717f1818..fd560290d6c 100644 --- a/Gemfile.rails5.lock +++ b/Gemfile.rails5.lock @@ -286,7 +286,7 @@ GEM gettext_i18n_rails (>= 0.7.1) po_to_json (>= 1.0.0) rails (>= 3.2.0) - gitaly-proto (0.101.0) + gitaly-proto (0.102.0) google-protobuf (~> 3.1) grpc (~> 1.10) github-linguist (5.3.3) @@ -1049,7 +1049,7 @@ DEPENDENCIES gettext (~> 3.2.2) gettext_i18n_rails (~> 1.8.0) gettext_i18n_rails_js (~> 1.3) - gitaly-proto (~> 0.101.0) + gitaly-proto (~> 0.102.0) github-linguist (~> 5.3.3) gitlab-flowdock-git-hook (~> 1.0.1) gitlab-gollum-lib (~> 4.2) diff --git a/app/assets/javascripts/vue_shared/components/expand_button.vue b/app/assets/javascripts/vue_shared/components/expand_button.vue index 0fdea651130..e6e92594b65 100644 --- a/app/assets/javascripts/vue_shared/components/expand_button.vue +++ b/app/assets/javascripts/vue_shared/components/expand_button.vue @@ -1,5 +1,7 @@ <script> import { __ } from '~/locale'; +import Icon from '~/vue_shared/components/icon.vue'; + /** * Port of detail_behavior expand button. * @@ -12,6 +14,9 @@ import { __ } from '~/locale'; */ export default { name: 'ExpandButton', + components: { + Icon, + }, data() { return { isCollapsed: true, @@ -22,6 +27,9 @@ export default { return __('Click to expand text'); }, }, + destroyed() { + this.isCollapsed = true; + }, methods: { onClick() { this.isCollapsed = !this.isCollapsed; @@ -37,7 +45,10 @@ export default { type="button" class="text-expander btn-blank" @click="onClick"> - ... + <icon + :size="12" + name="ellipsis_h" + /> </button> <span v-if="!isCollapsed"> <slot name="expanded"></slot> diff --git a/app/assets/stylesheets/pages/commits.scss b/app/assets/stylesheets/pages/commits.scss index dc8842212e0..49226ae8eac 100644 --- a/app/assets/stylesheets/pages/commits.scss +++ b/app/assets/stylesheets/pages/commits.scss @@ -135,10 +135,10 @@ } .text-expander { - display: inline-block; + display: inline-flex; background: $white-light; color: $gl-text-color-secondary; - padding: 0 4px; + padding: 1px $gl-padding-4; cursor: pointer; border: 1px solid $border-gray-dark; border-radius: $border-radius-default; @@ -180,6 +180,11 @@ .commit-content { padding-right: 10px; white-space: normal; + + .commit-title { + display: flex; + align-items: center; + } } .commit-actions { @@ -253,7 +258,6 @@ .generic_commit_status { a, button { - color: $gl-text-color; vertical-align: baseline; } diff --git a/app/controllers/dashboard_controller.rb b/app/controllers/dashboard_controller.rb index 68d328fa797..ff133001b84 100644 --- a/app/controllers/dashboard_controller.rb +++ b/app/controllers/dashboard_controller.rb @@ -54,7 +54,7 @@ class DashboardController < Dashboard::ApplicationController return unless @no_filters_set respond_to do |format| - format.html + format.html { render } format.atom { head :bad_request } end end diff --git a/app/helpers/projects_helper.rb b/app/helpers/projects_helper.rb index daad829faa2..c405e6d117f 100644 --- a/app/helpers/projects_helper.rb +++ b/app/helpers/projects_helper.rb @@ -407,6 +407,7 @@ module ProjectsHelper @ref || @repository.try(:root_ref) end + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/1235 def sanitize_repo_path(project, message) return '' unless message.present? diff --git a/app/models/concerns/redis_cacheable.rb b/app/models/concerns/redis_cacheable.rb index b5425295130..3bdc1330d23 100644 --- a/app/models/concerns/redis_cacheable.rb +++ b/app/models/concerns/redis_cacheable.rb @@ -48,7 +48,7 @@ module RedisCacheable def cast_value_from_cache(attribute, value) if Gitlab.rails5? - self.class.type_for_attribute(attribute).cast(value) + self.class.type_for_attribute(attribute.to_s).cast(value) else self.class.column_for_attribute(attribute).type_cast_from_database(value) end diff --git a/app/models/repository.rb b/app/models/repository.rb index e4202505634..c2f62badbcb 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -154,7 +154,10 @@ class Repository # Returns a list of commits that are not present in any reference def new_commits(newrev) - refs = ::Gitlab::Git::RevList.new(raw, newrev: newrev).new_refs + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/1233 + refs = Gitlab::GitalyClient::StorageSettings.allow_disk_access do + ::Gitlab::Git::RevList.new(raw, newrev: newrev).new_refs + end refs.map { |sha| commit(sha.strip) } end diff --git a/app/views/projects/commits/_commit.html.haml b/app/views/projects/commits/_commit.html.haml index 12b27eb9b66..90e55fd0fb0 100644 --- a/app/views/projects/commits/_commit.html.haml +++ b/app/views/projects/commits/_commit.html.haml @@ -34,7 +34,8 @@ .d-block.d-sm-none = render_commit_status(commit, ref: ref) - if commit.description? - %button.text-expander.d-none.d-sm-inline-block.js-toggle-button{ type: "button" } ... + %button.text-expander.js-toggle-button + = sprite_icon('ellipsis_h', size: 12) .commiter - commit_author_link = commit_author_link(commit, avatar: false, size: 24) diff --git a/app/views/projects/pipelines/_info.html.haml b/app/views/projects/pipelines/_info.html.haml index aa53fc3ea28..04131a90a57 100644 --- a/app/views/projects/pipelines/_info.html.haml +++ b/app/views/projects/pipelines/_info.html.haml @@ -29,7 +29,7 @@ = link_to @commit.short_id, project_commit_path(@project, @pipeline.sha), class: "commit-sha js-details-short" = link_to("#", class: "js-details-expand d-none d-sm-none d-md-inline") do %span.text-expander - \... + = sprite_icon('ellipsis_h', size: 12) %span.js-details-content.hide = link_to @pipeline.sha, project_commit_path(@project, @pipeline.sha), class: "commit-sha commit-hash-full" = clipboard_button(text: @pipeline.sha, title: "Copy commit SHA to clipboard") diff --git a/changelogs/unreleased/ce-5024-filename-search.yml b/changelogs/unreleased/ce-5024-filename-search.yml new file mode 100644 index 00000000000..a8bf9b1f802 --- /dev/null +++ b/changelogs/unreleased/ce-5024-filename-search.yml @@ -0,0 +1,5 @@ +--- +title: Add filename filtering to code search +merge_request: 19509 +author: +type: added diff --git a/changelogs/unreleased/rails5-fix-48104.yml b/changelogs/unreleased/rails5-fix-48104.yml new file mode 100644 index 00000000000..6cf519ad791 --- /dev/null +++ b/changelogs/unreleased/rails5-fix-48104.yml @@ -0,0 +1,6 @@ +--- +title: 'Rails5 fix expected: 1 time with arguments: (97, anything, {"squash"=>false}) + received: 0 times' +merge_request: 20004 +author: Jasper Maes +type: fixed diff --git a/changelogs/unreleased/rails5-fix-48141.yml b/changelogs/unreleased/rails5-fix-48141.yml new file mode 100644 index 00000000000..5e2aa23b8fb --- /dev/null +++ b/changelogs/unreleased/rails5-fix-48141.yml @@ -0,0 +1,6 @@ +--- +title: 'Rails5 fix expected: 0 times with any arguments received: 1 time with arguments: + DashboardController' +merge_request: 20018 +author: Jasper Maes +type: fixed diff --git a/changelogs/unreleased/rails5-fix-pages-controller.yml b/changelogs/unreleased/rails5-fix-pages-controller.yml new file mode 100644 index 00000000000..eeb3747c4eb --- /dev/null +++ b/changelogs/unreleased/rails5-fix-pages-controller.yml @@ -0,0 +1,5 @@ +--- +title: Rails5 fix Projects::PagesController spec +merge_request: 20007 +author: Jasper Maes +type: fixed diff --git a/changelogs/unreleased/text-expander-icon-update.yml b/changelogs/unreleased/text-expander-icon-update.yml new file mode 100644 index 00000000000..be9dc98728f --- /dev/null +++ b/changelogs/unreleased/text-expander-icon-update.yml @@ -0,0 +1,5 @@ +--- +title: Updated the icon for expand buttons to ellipsis +merge_request: 18793 +author: Constance Okoghenun +type: changed
\ No newline at end of file diff --git a/changelogs/unreleased/use-backup-custom-hooks-gitaly.yml b/changelogs/unreleased/use-backup-custom-hooks-gitaly.yml new file mode 100644 index 00000000000..4b9766332c3 --- /dev/null +++ b/changelogs/unreleased/use-backup-custom-hooks-gitaly.yml @@ -0,0 +1,5 @@ +--- +title: migrate backup rake task to gitaly +merge_request: +author: +type: added diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 12d09150127..3d3448cb4d6 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -394,6 +394,7 @@ repositories_storages = Settings.repositories.storages.values repository_downloads_path = Settings.gitlab['repository_downloads_path'].to_s.gsub(%r{/$}, '') repository_downloads_full_path = File.expand_path(repository_downloads_path, Settings.gitlab['user_home']) +# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/1237 Gitlab::GitalyClient::StorageSettings.allow_disk_access do if repository_downloads_path.blank? || repositories_storages.any? { |rs| [repository_downloads_path, repository_downloads_full_path].include?(rs.legacy_disk_path.gsub(%r{/$}, '')) } Settings.gitlab['repository_downloads_path'] = File.join(Settings.shared['path'], 'cache/archive') diff --git a/config/initializers/6_validations.rb b/config/initializers/6_validations.rb index 362a23164ab..ff6865608f0 100644 --- a/config/initializers/6_validations.rb +++ b/config/initializers/6_validations.rb @@ -37,6 +37,7 @@ def validate_storages_config end end +# Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/1237 def validate_storages_paths Gitlab::GitalyClient::StorageSettings.allow_disk_access do Gitlab.config.repositories.storages.each do |name, repository_storage| diff --git a/doc/api/pages_domains.md b/doc/api/pages_domains.md index 20275b902c6..da2ffcfe40a 100644 --- a/doc/api/pages_domains.md +++ b/doc/api/pages_domains.md @@ -122,11 +122,11 @@ POST /projects/:id/pages/domains | `key` | file/string | no | The certificate key in PEM format. | ```bash -curl --request POST --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" --form="domain=ssl.domain.example" --form="certificate=@/path/to/cert.pem" --form="key=@/path/to/key.pem" https://gitlab.example.com/api/v4/projects/5/pages/domains +curl --request POST --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" --form "domain=ssl.domain.example" --form "certificate=@/path/to/cert.pem" --form "key=@/path/to/key.pem" https://gitlab.example.com/api/v4/projects/5/pages/domains ``` ```bash -curl --request POST --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" --form="domain=ssl.domain.example" --form="certificate=$CERT_PEM" --form="key=$KEY_PEM" https://gitlab.example.com/api/v4/projects/5/pages/domains +curl --request POST --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" --form "domain=ssl.domain.example" --form "certificate=$CERT_PEM" --form "key=$KEY_PEM" https://gitlab.example.com/api/v4/projects/5/pages/domains ``` ```json @@ -158,11 +158,11 @@ PUT /projects/:id/pages/domains/:domain | `key` | file/string | no | The certificate key in PEM format. | ```bash -curl --request PUT --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" --form="certificate=@/path/to/cert.pem" --form="key=@/path/to/key.pem" https://gitlab.example.com/api/v4/projects/5/pages/domains/ssl.domain.example +curl --request PUT --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" --form "certificate=@/path/to/cert.pem" --form "key=@/path/to/key.pem" https://gitlab.example.com/api/v4/projects/5/pages/domains/ssl.domain.example ``` ```bash -curl --request PUT --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" --form="certificate=$CERT_PEM" --form="key=$KEY_PEM" https://gitlab.example.com/api/v4/projects/5/pages/domains/ssl.domain.example +curl --request PUT --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" --form "certificate=$CERT_PEM" --form "key=$KEY_PEM" https://gitlab.example.com/api/v4/projects/5/pages/domains/ssl.domain.example ``` ```json diff --git a/doc/api/search.md b/doc/api/search.md index 107ddaffa6a..9716f682ace 100644 --- a/doc/api/search.md +++ b/doc/api/search.md @@ -776,6 +776,15 @@ Example response: ### Scope: blobs +Filters are available for this scope: +- filename +- path +- extension + +to use a filter simply include it in your query like so: `a query filename:some_name*`. + +You may use wildcards (`*`) to use glob matching. + ```bash curl --request GET --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/projects/6/search?scope=blobs&search=installation ``` diff --git a/doc/development/what_requires_downtime.md b/doc/development/what_requires_downtime.md index b8be8daa157..f502866333e 100644 --- a/doc/development/what_requires_downtime.md +++ b/doc/development/what_requires_downtime.md @@ -252,6 +252,53 @@ Keep in mind that the relation passed to `change_column_type_using_background_migration` _must_ include `EachBatch`, otherwise it will raise a `TypeError`. +This migration then needs to be followed in a separate release (_not_ a patch +release) by a cleanup migration, which should steal from the queue and handle +any remaining rows. For example: + +```ruby +class MigrateRemainingIssuesClosedAt < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + class Issue < ActiveRecord::Base + self.table_name = 'issues' + include EachBatch + end + + def up + Gitlab::BackgroundMigration.steal('CopyColumn') + Gitlab::BackgroundMigration.steal('CleanupConcurrentTypeChange') + + migrate_remaining_rows if migrate_column_type? + end + + def down + # Previous migrations already revert the changes made here. + end + + def migrate_remaining_rows + Issue.where('closed_at_for_type_change IS NULL AND closed_at IS NOT NULL').each_batch do |batch| + batch.update_all('closed_at_for_type_change = closed_at') + end + + cleanup_concurrent_column_type_change(:issues, :closed_at) + end + + def migrate_column_type? + # Some environments may have already executed the previous version of this + # migration, thus we don't need to migrate those environments again. + column_for('issues', 'closed_at').type == :datetime # rubocop:disable Migration/Datetime + end +end +``` + +For more information, see [the documentation on cleaning up background +migrations](background_migrations.md#cleaning-up). + ## Adding Indexes Adding indexes is an expensive process that blocks INSERT and UPDATE queries for diff --git a/doc/install/installation.md b/doc/install/installation.md index 6cd1fb4c2d7..ef415246583 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -154,12 +154,12 @@ page](https://golang.org/dl). # Remove former Go installation folder sudo rm -rf /usr/local/go - - curl --remote-name --progress https://storage.googleapis.com/golang/go1.8.3.linux-amd64.tar.gz - echo '1862f4c3d3907e59b04a757cfda0ea7aa9ef39274af99a784f5be843c80c6772 go1.8.3.linux-amd64.tar.gz' | shasum -a256 -c - && \ - sudo tar -C /usr/local -xzf go1.8.3.linux-amd64.tar.gz + + curl --remote-name --progress https://dl.google.com/go/go1.10.3.linux-amd64.tar.gz + echo 'fa1b0e45d3b647c252f51f5e1204aba049cde4af177ef9f2181f43004f901035 go1.10.3.linux-amd64.tar.gz' | shasum -a256 -c - && \ + sudo tar -C /usr/local -xzf go1.10.3.linux-amd64.tar.gz sudo ln -sf /usr/local/go/bin/{go,godoc,gofmt} /usr/local/bin/ - rm go1.8.3.linux-amd64.tar.gz + rm go1.10.3.linux-amd64.tar.gz ## 4. Node diff --git a/doc/update/10.8-to-11.0.md b/doc/update/10.8-to-11.0.md index 78a47ab939f..f9b6044bd2f 100644 --- a/doc/update/10.8-to-11.0.md +++ b/doc/update/10.8-to-11.0.md @@ -81,8 +81,8 @@ More information can be found on the [yarn website](https://yarnpkg.com/en/docs/ ### 5. Update Go -NOTE: GitLab 9.2 and higher only supports Go 1.8.3 and dropped support for Go -1.5.x through 1.7.x. Be sure to upgrade your installation if necessary. +NOTE: GitLab 11.0 and higher only supports Go 1.9.x and newer, and dropped support for Go +1.5.x through 1.8.x. Be sure to upgrade your installation if necessary. You can check which version you are running with `go version`. @@ -92,11 +92,11 @@ Download and install Go: # Remove former Go installation folder sudo rm -rf /usr/local/go -curl --remote-name --progress https://storage.googleapis.com/golang/go1.8.3.linux-amd64.tar.gz -echo '1862f4c3d3907e59b04a757cfda0ea7aa9ef39274af99a784f5be843c80c6772 go1.8.3.linux-amd64.tar.gz' | shasum -a256 -c - && \ - sudo tar -C /usr/local -xzf go1.8.3.linux-amd64.tar.gz +curl --remote-name --progress https://dl.google.com/go/go1.10.3.linux-amd64.tar.gz +echo 'fa1b0e45d3b647c252f51f5e1204aba049cde4af177ef9f2181f43004f901035 go1.10.3.linux-amd64.tar.gz' | shasum -a256 -c - && \ + sudo tar -C /usr/local -xzf go1.10.3.linux-amd64.tar.gz sudo ln -sf /usr/local/go/bin/{go,godoc,gofmt} /usr/local/bin/ -rm go1.8.3.linux-amd64.tar.gz +rm go1.10.3.linux-amd64.tar.gz ``` ### 6. Get latest code diff --git a/doc/user/project/integrations/microsoft_teams.md b/doc/user/project/integrations/microsoft_teams.md index eaad2d5138a..5cf80a298ad 100644 --- a/doc/user/project/integrations/microsoft_teams.md +++ b/doc/user/project/integrations/microsoft_teams.md @@ -2,7 +2,7 @@ ## On Microsoft Teams -To enable Microsoft Teams integration you must create an incoming webhook integration on Microsoft Teams by following the steps described in this [document](https://msdn.microsoft.com/en-us/microsoft-teams/connectors). +To enable Microsoft Teams integration you must create an incoming webhook integration on Microsoft Teams by following the steps described in this [document](https://docs.microsoft.com/en-us/microsoftteams/platform/concepts/connectors#setting-up-a-custom-incoming-webhook). ## On GitLab diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 0119c5d6851..221ba52b490 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -4,7 +4,6 @@ require_relative 'helper' module Backup class Repository include Backup::Helper - # rubocop:disable Metrics/AbcSize attr_reader :progress @@ -18,61 +17,26 @@ module Backup Project.find_each(batch_size: 1000) do |project| progress.print " * #{display_repo_path(project)} ... " - path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do - path_to_repo(project) - end - path_to_project_bundle = path_to_bundle(project) - - # Create namespace dir or hashed path if missing if project.hashed_storage?(:repository) FileUtils.mkdir_p(File.dirname(File.join(backup_repos_path, project.disk_path))) else FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.full_path)) if project.namespace end - if empty_repo?(project) - progress.puts "[SKIPPED]".color(:cyan) + if !empty_repo?(project) + backup_project(project) + progress.puts "[DONE]".color(:green) else - in_path(path_to_project_repo) do |dir| - FileUtils.mkdir_p(path_to_tars(project)) - cmd = %W(tar -cf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir}) - output, status = Gitlab::Popen.popen(cmd) - - unless status.zero? - progress_warn(project, cmd.join(' '), output) - end - end - - cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all) - output, status = Gitlab::Popen.popen(cmd) - - if status.zero? - progress.puts "[DONE]".color(:green) - else - progress_warn(project, cmd.join(' '), output) - end + progress.puts "[SKIPPED]".color(:cyan) end wiki = ProjectWiki.new(project) - path_to_wiki_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do - path_to_repo(wiki) - end - path_to_wiki_bundle = path_to_bundle(wiki) - if File.exist?(path_to_wiki_repo) - progress.print " * #{display_repo_path(wiki)} ... " - - if empty_repo?(wiki) - progress.puts " [SKIPPED]".color(:cyan) - else - cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_wiki_repo} bundle create #{path_to_wiki_bundle} --all) - output, status = Gitlab::Popen.popen(cmd) - if status.zero? - progress.puts " [DONE]".color(:green) - else - progress_warn(wiki, cmd.join(' '), output) - end - end + if !empty_repo?(wiki) + backup_project(wiki) + progress.puts "[DONE] Wiki".color(:green) + else + progress.puts "[SKIPPED] Wiki".color(:cyan) end end end @@ -83,6 +47,38 @@ module Backup end end + def backup_project(project) + gitaly_migrate(:repository_backup) do |is_enabled| + if is_enabled + backup_project_gitaly(project) + else + backup_project_local(project) + end + end + + backup_custom_hooks(project) + rescue => e + progress_warn(project, e, 'Failed to backup repo') + end + + def backup_project_gitaly(project) + path_to_project_bundle = path_to_bundle(project) + Gitlab::GitalyClient::RepositoryService.new(project.repository) + .create_bundle(path_to_project_bundle) + end + + def backup_project_local(project) + path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do + path_to_repo(project) + end + + path_to_project_bundle = path_to_bundle(project) + + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all) + output, status = Gitlab::Popen.popen(cmd) + progress_warn(project, cmd.join(' '), output) unless status.zero? + end + def delete_all_repositories(name, repository_storage) gitaly_migrate(:delete_all_repositories) do |is_enabled| if is_enabled @@ -97,8 +93,6 @@ module Backup path = repository_storage.legacy_disk_path return unless File.exist?(path) - # Move all files in the existing repos directory except . and .. to - # repositories.old.<timestamp> directory bk_repos_path = File.join(Gitlab.config.backup.path, "tmp", "#{name}-repositories.old." + Time.now.to_i.to_s) FileUtils.mkdir_p(bk_repos_path, mode: 0700) files = Dir.glob(File.join(path, "*"), File::FNM_DOTMATCH) - [File.join(path, "."), File.join(path, "..")] @@ -129,13 +123,47 @@ module Backup .restore_custom_hooks(custom_hooks_path) end + def local_backup_custom_hooks(project) + in_path(path_to_tars(project)) do |dir| + path_to_project_repo = Gitlab::GitalyClient::StorageSettings.allow_disk_access do + path_to_repo(project) + end + break unless File.exist?(File.join(path_to_project_repo, dir)) + + FileUtils.mkdir_p(path_to_tars(project)) + cmd = %W(tar -cf #{path_to_tars(project, dir)} -c #{path_to_project_repo} #{dir}) + output, status = Gitlab::Popen.popen(cmd) + + unless status.zero? + progress_warn(project, cmd.join(' '), output) + end + end + end + + def gitaly_backup_custom_hooks(project) + FileUtils.mkdir_p(path_to_tars(project)) + custom_hooks_path = path_to_tars(project, 'custom_hooks') + Gitlab::GitalyClient::RepositoryService.new(project.repository) + .backup_custom_hooks(custom_hooks_path) + end + + def backup_custom_hooks(project) + gitaly_migrate(:backup_custom_hooks) do |is_enabled| + if is_enabled + gitaly_backup_custom_hooks(project) + else + local_backup_custom_hooks(project) + end + end + end + def restore_custom_hooks(project) in_path(path_to_tars(project)) do |dir| gitaly_migrate(:restore_custom_hooks) do |is_enabled| if is_enabled - local_restore_custom_hooks(project, dir) - else gitaly_restore_custom_hooks(project, dir) + else + local_restore_custom_hooks(project, dir) end end end @@ -186,7 +214,6 @@ module Backup end end end - # rubocop:enable Metrics/AbcSize protected @@ -224,9 +251,7 @@ module Backup def prepare FileUtils.rm_rf(backup_repos_path) - # Ensure the parent dir of backup_repos_path exists FileUtils.mkdir_p(Gitlab.config.backup.path) - # Fail if somebody raced to create backup_repos_path before us FileUtils.mkdir(backup_repos_path, mode: 0700) end @@ -242,7 +267,6 @@ module Backup end def empty_repo?(project_or_wiki) - # Protect against stale caches project_or_wiki.repository.expire_emptiness_caches project_or_wiki.repository.empty? end diff --git a/lib/gitlab/file_finder.rb b/lib/gitlab/file_finder.rb index f42088f980e..af8270c8db8 100644 --- a/lib/gitlab/file_finder.rb +++ b/lib/gitlab/file_finder.rb @@ -14,14 +14,21 @@ module Gitlab end def find(query) - by_content = find_by_content(query) + query = Gitlab::Search::Query.new(query) do + filter :filename, matcher: ->(filter, blob) { blob.filename =~ /#{filter[:regex_value]}$/i } + filter :path, matcher: ->(filter, blob) { blob.filename =~ /#{filter[:regex_value]}/i } + filter :extension, matcher: ->(filter, blob) { blob.filename =~ /\.#{filter[:regex_value]}$/i } + end + + by_content = find_by_content(query.term) already_found = Set.new(by_content.map(&:filename)) - by_filename = find_by_filename(query, except: already_found) + by_filename = find_by_filename(query.term, except: already_found) + + files = (by_content + by_filename) + .sort_by(&:filename) - (by_content + by_filename) - .sort_by(&:filename) - .map { |blob| [blob.filename, blob] } + query.filter_results(files).map { |blob| [blob.filename, blob] } end private diff --git a/lib/gitlab/git/lfs_changes.rb b/lib/gitlab/git/lfs_changes.rb index f3cc388ea41..f0fab1e76a3 100644 --- a/lib/gitlab/git/lfs_changes.rb +++ b/lib/gitlab/git/lfs_changes.rb @@ -7,67 +7,11 @@ module Gitlab end def new_pointers(object_limit: nil, not_in: nil) - @repository.gitaly_migrate(:blob_get_new_lfs_pointers) do |is_enabled| - if is_enabled - @repository.gitaly_blob_client.get_new_lfs_pointers(@newrev, object_limit, not_in) - else - git_new_pointers(object_limit, not_in) - end - end + @repository.gitaly_blob_client.get_new_lfs_pointers(@newrev, object_limit, not_in) end def all_pointers - @repository.gitaly_migrate(:blob_get_all_lfs_pointers) do |is_enabled| - if is_enabled - @repository.gitaly_blob_client.get_all_lfs_pointers(@newrev) - else - git_all_pointers - end - end - end - - private - - def git_new_pointers(object_limit, not_in) - @new_pointers ||= begin - rev_list.new_objects(rev_list_params(not_in: not_in)) do |object_ids| - object_ids = object_ids.take(object_limit) if object_limit - - Gitlab::Git::Blob.batch_lfs_pointers(@repository, object_ids) - end - end - end - - def git_all_pointers - params = {} - if rev_list_supports_new_options? - params[:options] = ["--filter=blob:limit=#{Gitlab::Git::Blob::LFS_POINTER_MAX_SIZE}"] - end - - rev_list.all_objects(rev_list_params(params)) do |object_ids| - Gitlab::Git::Blob.batch_lfs_pointers(@repository, object_ids) - end - end - - def rev_list - Gitlab::Git::RevList.new(@repository, newrev: @newrev) - end - - # We're passing the `--in-commit-order` arg to ensure we don't wait - # for git to traverse all commits before returning pointers. - # This is required in order to improve the performance of LFS integrity check - def rev_list_params(params = {}) - params[:options] ||= [] - params[:options] << "--in-commit-order" if rev_list_supports_new_options? - params[:require_path] = true - - params - end - - def rev_list_supports_new_options? - return @option_supported if defined?(@option_supported) - - @option_supported = Gitlab::Git.version >= Gitlab::VersionInfo.parse('2.16.0') + @repository.gitaly_blob_client.get_all_lfs_pointers(@newrev) end end end diff --git a/lib/gitlab/git/version.rb b/lib/gitlab/git/version.rb index 11184ca3457..1e14e8b652a 100644 --- a/lib/gitlab/git/version.rb +++ b/lib/gitlab/git/version.rb @@ -4,7 +4,7 @@ module Gitlab extend Gitlab::Git::Popen def self.git_version - Gitlab::VersionInfo.parse(popen(%W(#{Gitlab.config.git.bin_path} --version), nil).first) + Gitlab::VersionInfo.parse(Gitaly::Server.all.first.git_binary_version) end end end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 4340f779e53..ca986434221 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -196,20 +196,21 @@ module Gitlab end def create_bundle(save_path) - request = Gitaly::CreateBundleRequest.new(repository: @gitaly_repo) - response = GitalyClient.call( - @storage, - :repository_service, + gitaly_fetch_stream_to_file( + save_path, :create_bundle, - request, - timeout: GitalyClient.default_timeout + Gitaly::CreateBundleRequest, + GitalyClient.default_timeout ) + end - File.open(save_path, 'wb') do |f| - response.each do |message| - f.write(message.data) - end - end + def backup_custom_hooks(save_path) + gitaly_fetch_stream_to_file( + save_path, + :backup_custom_hooks, + Gitaly::BackupCustomHooksRequest, + GitalyClient.default_timeout + ) end def create_from_bundle(bundle_path) @@ -309,6 +310,25 @@ module Gitlab private + def gitaly_fetch_stream_to_file(save_path, rpc_name, request_class, timeout) + request = request_class.new(repository: @gitaly_repo) + response = GitalyClient.call( + @storage, + :repository_service, + rpc_name, + request, + timeout: timeout + ) + + File.open(save_path, 'wb') do |f| + response.each do |message| + f.write(message.data) + end + end + # If the file is empty means that we recieved an empty stream, we delete the file + FileUtils.rm(save_path) if File.zero?(save_path) + end + def gitaly_repo_stream_request(file_path, rpc_name, request_class, timeout) request = request_class.new(repository: @gitaly_repo) enum = Enumerator.new do |y| diff --git a/lib/gitlab/health_checks/fs_shards_check.rb b/lib/gitlab/health_checks/fs_shards_check.rb index fcbf266b80b..050fe7a5173 100644 --- a/lib/gitlab/health_checks/fs_shards_check.rb +++ b/lib/gitlab/health_checks/fs_shards_check.rb @@ -1,5 +1,6 @@ module Gitlab module HealthChecks + # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/1218 class FsShardsCheck extend BaseAbstractCheck RANDOM_STRING = SecureRandom.hex(1000).freeze diff --git a/lib/gitlab/search/parsed_query.rb b/lib/gitlab/search/parsed_query.rb new file mode 100644 index 00000000000..23595f23f01 --- /dev/null +++ b/lib/gitlab/search/parsed_query.rb @@ -0,0 +1,23 @@ +module Gitlab + module Search + class ParsedQuery + attr_reader :term, :filters + + def initialize(term, filters) + @term = term + @filters = filters + end + + def filter_results(results) + filters = @filters.reject { |filter| filter[:matcher].nil? } + return unless filters + + results.select do |result| + filters.all? do |filter| + filter[:matcher].call(filter, result) + end + end + end + end + end +end diff --git a/lib/gitlab/search/query.rb b/lib/gitlab/search/query.rb new file mode 100644 index 00000000000..8583bce7792 --- /dev/null +++ b/lib/gitlab/search/query.rb @@ -0,0 +1,55 @@ +module Gitlab + module Search + class Query < SimpleDelegator + def initialize(query, filter_opts = {}, &block) + @raw_query = query.dup + @filters = [] + @filter_options = { default_parser: :downcase.to_proc }.merge(filter_opts) + + self.instance_eval(&block) if block_given? + + @query = Gitlab::Search::ParsedQuery.new(*extract_filters) + # set the ParsedQuery as our default delegator thanks to SimpleDelegator + super(@query) + end + + private + + def filter(name, **attributes) + filter = { parser: @filter_options[:default_parser], name: name }.merge(attributes) + + @filters << filter + end + + def filter_options(**options) + @filter_options.merge!(options) + end + + def extract_filters + fragments = [] + + filters = @filters.each_with_object([]) do |filter, parsed_filters| + match = @raw_query.split.find { |part| part =~ /\A#{filter[:name]}:/ } + next unless match + + input = match.split(':')[1..-1].join + next if input.empty? + + filter[:value] = parse_filter(filter, input) + filter[:regex_value] = Regexp.escape(filter[:value]).gsub('\*', '.*?') + fragments << match + + parsed_filters << filter + end + + query = (@raw_query.split - fragments).join(' ') + + [query, filters] + end + + def parse_filter(filter, input) + filter[:parser].call(input) + end + end + end +end diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index 4a87f43597e..b2d75aac1d0 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -24,6 +24,7 @@ module Gitlab address = val['gitaly_address'] end + # https://gitlab.com/gitlab-org/gitaly/issues/1238 Gitlab::GitalyClient::StorageSettings.allow_disk_access do storages << { name: key, path: val.legacy_disk_path } end diff --git a/spec/controllers/projects/merge_requests_controller_spec.rb b/spec/controllers/projects/merge_requests_controller_spec.rb index a412e74581d..7f5f0b76c51 100644 --- a/spec/controllers/projects/merge_requests_controller_spec.rb +++ b/spec/controllers/projects/merge_requests_controller_spec.rb @@ -337,7 +337,12 @@ describe Projects::MergeRequestsController do context 'when the sha parameter matches the source SHA' do def merge_with_sha(params = {}) - post :merge, base_params.merge(sha: merge_request.diff_head_sha).merge(params) + post_params = base_params.merge(sha: merge_request.diff_head_sha).merge(params) + if Gitlab.rails5? + post :merge, params: post_params, as: :json + else + post :merge, post_params + end end it 'returns :success' do diff --git a/spec/controllers/projects/pages_controller_spec.rb b/spec/controllers/projects/pages_controller_spec.rb index 11f54eef531..8d2fa6a1740 100644 --- a/spec/controllers/projects/pages_controller_spec.rb +++ b/spec/controllers/projects/pages_controller_spec.rb @@ -71,7 +71,7 @@ describe Projects::PagesController do { namespace_id: project.namespace, project_id: project, - project: { pages_https_only: false } + project: { pages_https_only: 'false' } } end @@ -96,7 +96,7 @@ describe Projects::PagesController do it 'calls the update service' do expect(Projects::UpdateService) .to receive(:new) - .with(project, user, request_params[:project]) + .with(project, user, ActionController::Parameters.new(request_params[:project]).permit!) .and_return(update_service) patch :update, request_params diff --git a/spec/javascripts/vue_shared/components/expand_button_spec.js b/spec/javascripts/vue_shared/components/expand_button_spec.js index af9693c48fd..98fee9a74a5 100644 --- a/spec/javascripts/vue_shared/components/expand_button_spec.js +++ b/spec/javascripts/vue_shared/components/expand_button_spec.js @@ -19,7 +19,7 @@ describe('expand button', () => { }); it('renders a collpased button', () => { - expect(vm.$el.textContent.trim()).toEqual('...'); + expect(vm.$children[0].iconTestClass).toEqual('ic-ellipsis_h'); }); it('hides expander on click', done => { diff --git a/spec/lib/gitlab/file_finder_spec.rb b/spec/lib/gitlab/file_finder_spec.rb index d6d9e4001a3..b49c5817131 100644 --- a/spec/lib/gitlab/file_finder_spec.rb +++ b/spec/lib/gitlab/file_finder_spec.rb @@ -3,11 +3,29 @@ require 'spec_helper' describe Gitlab::FileFinder do describe '#find' do let(:project) { create(:project, :public, :repository) } + subject { described_class.new(project, project.default_branch) } it_behaves_like 'file finder' do - subject { described_class.new(project, project.default_branch) } let(:expected_file_by_name) { 'files/images/wm.svg' } let(:expected_file_by_content) { 'CHANGELOG' } end + + it 'filters by name' do + results = subject.find('files filename:wm.svg') + + expect(results.count).to eq(1) + end + + it 'filters by path' do + results = subject.find('white path:images') + + expect(results.count).to eq(1) + end + + it 'filters by extension' do + results = subject.find('files extension:svg') + + expect(results.count).to eq(1) + end end end diff --git a/spec/lib/gitlab/git/lfs_changes_spec.rb b/spec/lib/gitlab/git/lfs_changes_spec.rb index d0dd8c6303f..c5e7ab959b2 100644 --- a/spec/lib/gitlab/git/lfs_changes_spec.rb +++ b/spec/lib/gitlab/git/lfs_changes_spec.rb @@ -1,50 +1,19 @@ require 'spec_helper' describe Gitlab::Git::LfsChanges do - let(:project) { create(:project, :repository) } + set(:project) { create(:project, :repository) } let(:newrev) { '54fcc214b94e78d7a41a9a8fe6d87a5e59500e51' } let(:blob_object_id) { '0c304a93cb8430108629bbbcaa27db3343299bc0' } subject { described_class.new(project.repository, newrev) } describe '#new_pointers' do - shared_examples 'new pointers' do - it 'filters new objects to find lfs pointers' do - expect(subject.new_pointers(not_in: []).first.id).to eq(blob_object_id) - end - - it 'limits new_objects using object_limit' do - expect(subject.new_pointers(object_limit: 1)).to eq([]) - end - end - - context 'with gitaly enabled' do - it_behaves_like 'new pointers' + it 'filters new objects to find lfs pointers' do + expect(subject.new_pointers(not_in: []).first.id).to eq(blob_object_id) end - context 'with gitaly disabled', :skip_gitaly_mock do - it_behaves_like 'new pointers' - - it 'uses rev-list to find new objects' do - rev_list = double - allow(Gitlab::Git::RevList).to receive(:new).and_return(rev_list) - - expect(rev_list).to receive(:new_objects).and_return([]) - - subject.new_pointers - end - end - end - - describe '#all_pointers', :skip_gitaly_mock do - it 'uses rev-list to find all objects' do - rev_list = double - allow(Gitlab::Git::RevList).to receive(:new).and_return(rev_list) - allow(rev_list).to receive(:all_objects).and_yield([blob_object_id]) - - expect(Gitlab::Git::Blob).to receive(:batch_lfs_pointers).with(project.repository, [blob_object_id]) - - subject.all_pointers + it 'limits new_objects using object_limit' do + expect(subject.new_pointers(object_limit: 1)).to eq([]) end end end diff --git a/spec/lib/gitlab/search/query_spec.rb b/spec/lib/gitlab/search/query_spec.rb new file mode 100644 index 00000000000..2d00428fffa --- /dev/null +++ b/spec/lib/gitlab/search/query_spec.rb @@ -0,0 +1,39 @@ +require 'spec_helper' + +describe Gitlab::Search::Query do + let(:query) { 'base filter:wow anotherfilter:noway name:maybe other:mmm leftover' } + let(:subject) do + described_class.new(query) do + filter :filter + filter :name, parser: :upcase.to_proc + filter :other + end + end + + it { expect(described_class).to be < SimpleDelegator } + + it 'leaves undefined filters in the main query' do + expect(subject.term).to eq('base anotherfilter:noway leftover') + end + + it 'parses filters' do + expect(subject.filters.count).to eq(3) + expect(subject.filters.map { |f| f[:value] }).to match_array(%w[wow MAYBE mmm]) + end + + context 'with an empty filter' do + let(:query) { 'some bar name: baz' } + + it 'ignores empty filters' do + expect(subject.term).to eq('some bar name: baz') + end + end + + context 'with a pipe' do + let(:query) { 'base | nofilter' } + + it 'does not escape the pipe' do + expect(subject.term).to eq(query) + end + end +end diff --git a/spec/requests/api/search_spec.rb b/spec/requests/api/search_spec.rb index aca4aa40027..f8e468be170 100644 --- a/spec/requests/api/search_spec.rb +++ b/spec/requests/api/search_spec.rb @@ -312,6 +312,30 @@ describe API::Search do end it_behaves_like 'response is correct', schema: 'public_api/v4/blobs', size: 2 + + context 'filters' do + it 'by filename' do + get api("/projects/#{repo_project.id}/search", user), scope: 'blobs', search: 'mon filename:PROCESS.md' + + expect(response).to have_gitlab_http_status(200) + expect(json_response.size).to eq(2) + expect(json_response.first['filename']).to eq('PROCESS.md') + end + + it 'by path' do + get api("/projects/#{repo_project.id}/search", user), scope: 'blobs', search: 'mon path:markdown' + + expect(response).to have_gitlab_http_status(200) + expect(json_response.size).to eq(8) + end + + it 'by extension' do + get api("/projects/#{repo_project.id}/search", user), scope: 'blobs', search: 'mon extension:md' + + expect(response).to have_gitlab_http_status(200) + expect(json_response.size).to eq(11) + end + end end end end |