diff options
author | Shah El-Rahman <selrahman@gitlab.com> | 2018-02-07 21:41:40 -0600 |
---|---|---|
committer | Shah El-Rahman <selrahman@gitlab.com> | 2018-02-07 21:41:40 -0600 |
commit | 4988fe7467f40c4b1da9fa775512bd6dec0c7884 (patch) | |
tree | 457f967de16bb9eb3046ee92d901cb0392e1b1f3 | |
parent | 0cb8409a162bf6beaa3d808446a7e9aa80263340 (diff) | |
parent | 15762319e94457cc82dba8d15d5d9507ad527503 (diff) | |
download | gitlab-ce-41301-new-design-for-user-deletion-confirmation-in-admin-area.tar.gz |
Merge branch 'master' into 41301-new-design-for-user-deletion-confirmation-in-admin-area41301-new-design-for-user-deletion-confirmation-in-admin-area
-rw-r--r-- | Gemfile | 4 | ||||
-rw-r--r-- | Gemfile.lock | 8 | ||||
-rw-r--r-- | app/assets/javascripts/dispatcher.js | 5 | ||||
-rw-r--r-- | app/assets/javascripts/pages/admin/projects/index/components/delete_project_modal.vue | 125 | ||||
-rw-r--r-- | app/assets/javascripts/pages/admin/projects/index/index.js | 37 | ||||
-rw-r--r-- | app/views/admin/projects/_projects.html.haml | 9 | ||||
-rw-r--r-- | doc/update/mysql_to_postgresql.md | 264 | ||||
-rw-r--r-- | lib/gitlab/git/blob.rb | 6 | ||||
-rw-r--r-- | lib/gitlab/gitaly_client/blob_service.rb | 26 | ||||
-rw-r--r-- | lib/gitlab/gitaly_client/blobs_stitcher.rb | 47 | ||||
-rw-r--r-- | spec/lib/gitlab/git/blob_spec.rb | 94 | ||||
-rw-r--r-- | spec/lib/gitlab/gitaly_client/blobs_stitcher_spec.rb | 36 | ||||
-rw-r--r-- | vendor/gitlab-ci-yml/Auto-DevOps.gitlab-ci.yml | 7 |
13 files changed, 487 insertions, 181 deletions
@@ -294,7 +294,7 @@ group :metrics do gem 'influxdb', '~> 0.2', require: false # Prometheus - gem 'prometheus-client-mmap', '~> 0.9.1.pre.rc.2' + gem 'prometheus-client-mmap', '~> 0.9.1' gem 'raindrops', '~> 0.18' end @@ -410,7 +410,7 @@ group :ed25519 do end # Gitaly GRPC client -gem 'gitaly-proto', '~> 0.83.0', require: 'gitaly' +gem 'gitaly-proto', '~> 0.84.0', require: 'gitaly' # Locked until https://github.com/google/protobuf/issues/4210 is closed gem 'google-protobuf', '= 3.5.1' diff --git a/Gemfile.lock b/Gemfile.lock index d9665e38f45..22c4fc0ef28 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -285,7 +285,7 @@ GEM po_to_json (>= 1.0.0) rails (>= 3.2.0) gherkin-ruby (0.3.2) - gitaly-proto (0.83.0) + gitaly-proto (0.84.0) google-protobuf (~> 3.1) grpc (~> 1.0) github-linguist (4.7.6) @@ -636,7 +636,7 @@ GEM parser unparser procto (0.0.3) - prometheus-client-mmap (0.9.1.pre.rc.2) + prometheus-client-mmap (0.9.1) pry (0.10.4) coderay (~> 1.1.0) method_source (~> 0.8.1) @@ -1056,7 +1056,7 @@ DEPENDENCIES gettext (~> 3.2.2) gettext_i18n_rails (~> 1.8.0) gettext_i18n_rails_js (~> 1.2.0) - gitaly-proto (~> 0.83.0) + gitaly-proto (~> 0.84.0) github-linguist (~> 4.7.0) gitlab-flowdock-git-hook (~> 1.0.1) gitlab-markup (~> 1.6.2) @@ -1132,7 +1132,7 @@ DEPENDENCIES peek-sidekiq (~> 1.0.3) pg (~> 0.18.2) premailer-rails (~> 1.9.7) - prometheus-client-mmap (~> 0.9.1.pre.rc.2) + prometheus-client-mmap (~> 0.9.1) pry-byebug (~> 3.4.1) pry-rails (~> 0.3.4) rack-attack (~> 4.4.1) diff --git a/app/assets/javascripts/dispatcher.js b/app/assets/javascripts/dispatcher.js index 94695674ba0..aceaffdfcb9 100644 --- a/app/assets/javascripts/dispatcher.js +++ b/app/assets/javascripts/dispatcher.js @@ -109,6 +109,11 @@ var Dispatcher; .then(callDefault) .catch(fail); break; + case 'admin:projects:index': + import('./pages/admin/projects/index/index') + .then(callDefault) + .catch(fail); + break; case 'admin:users:index': import('./pages/admin/users/shared') .then(callDefault) diff --git a/app/assets/javascripts/pages/admin/projects/index/components/delete_project_modal.vue b/app/assets/javascripts/pages/admin/projects/index/components/delete_project_modal.vue new file mode 100644 index 00000000000..14315d5492e --- /dev/null +++ b/app/assets/javascripts/pages/admin/projects/index/components/delete_project_modal.vue @@ -0,0 +1,125 @@ +<script> + import _ from 'underscore'; + import modal from '~/vue_shared/components/modal.vue'; + import { s__, sprintf } from '~/locale'; + + export default { + components: { + modal, + }, + props: { + deleteProjectUrl: { + type: String, + required: false, + default: '', + }, + projectName: { + type: String, + required: false, + default: '', + }, + csrfToken: { + type: String, + required: false, + default: '', + }, + }, + data() { + return { + enteredProjectName: '', + }; + }, + computed: { + title() { + return sprintf(s__('AdminProjects|Delete Project %{projectName}?'), + { + projectName: `'${_.escape(this.projectName)}'`, + }, + false, + ); + }, + text() { + return sprintf(s__(`AdminProjects| + You’re about to permanently delete the project %{projectName}, its repository, + and all related resources including issues, merge requests, etc.. Once you confirm and press + %{strong_start}Delete project%{strong_end}, it cannot be undone or recovered.`), + { + projectName: `<strong>${_.escape(this.projectName)}</strong>`, + strong_start: '<strong>', + strong_end: '</strong>', + }, + false, + ); + }, + confirmationTextLabel() { + return sprintf(s__('AdminUsers|To confirm, type %{projectName}'), + { + projectName: `<code>${_.escape(this.projectName)}</code>`, + }, + false, + ); + }, + primaryButtonLabel() { + return s__('AdminProjects|Delete project'); + }, + canSubmit() { + return this.enteredProjectName === this.projectName; + }, + }, + methods: { + onCancel() { + this.enteredProjectName = ''; + }, + onSubmit() { + this.$refs.form.submit(); + this.enteredProjectName = ''; + }, + }, + }; +</script> + +<template> + <modal + id="delete-project-modal" + :title="title" + :text="text" + kind="danger" + :primary-button-label="primaryButtonLabel" + :submit-disabled="!canSubmit" + @submit="onSubmit" + @cancel="onCancel" + > + <template + slot="body" + slot-scope="props" + > + <p v-html="props.text"></p> + <p v-html="confirmationTextLabel"></p> + <form + ref="form" + :action="deleteProjectUrl" + method="post" + > + <input + ref="method" + type="hidden" + name="_method" + value="delete" + /> + <input + type="hidden" + name="authenticity_token" + :value="csrfToken" + /> + <input + name="projectName" + class="form-control" + type="text" + v-model="enteredProjectName" + aria-labelledby="input-label" + autocomplete="off" + /> + </form> + </template> + </modal> +</template> diff --git a/app/assets/javascripts/pages/admin/projects/index/index.js b/app/assets/javascripts/pages/admin/projects/index/index.js new file mode 100644 index 00000000000..a87b27090a8 --- /dev/null +++ b/app/assets/javascripts/pages/admin/projects/index/index.js @@ -0,0 +1,37 @@ +import Vue from 'vue'; + +import Translate from '~/vue_shared/translate'; +import csrf from '~/lib/utils/csrf'; + +import deleteProjectModal from './components/delete_project_modal.vue'; + +export default () => { + Vue.use(Translate); + + const deleteProjectModalEl = document.getElementById('delete-project-modal'); + + const deleteModal = new Vue({ + el: deleteProjectModalEl, + data: { + deleteProjectUrl: '', + projectName: '', + }, + render(createElement) { + return createElement(deleteProjectModal, { + props: { + deleteProjectUrl: this.deleteProjectUrl, + projectName: this.projectName, + csrfToken: csrf.token, + }, + }); + }, + }); + + $(document).on('shown.bs.modal', (event) => { + if (event.relatedTarget.classList.contains('delete-project-button')) { + const buttonProps = event.relatedTarget.dataset; + deleteModal.deleteProjectUrl = buttonProps.deleteProjectUrl; + deleteModal.projectName = buttonProps.projectName; + } + }); +}; diff --git a/app/views/admin/projects/_projects.html.haml b/app/views/admin/projects/_projects.html.haml index c69c2761189..b5d7b889504 100644 --- a/app/views/admin/projects/_projects.html.haml +++ b/app/views/admin/projects/_projects.html.haml @@ -5,7 +5,12 @@ %li.project-row{ class: ('no-description' if project.description.blank?) } .controls = link_to 'Edit', edit_project_path(project), id: "edit_#{dom_id(project)}", class: "btn" - = link_to 'Delete', [project.namespace.becomes(Namespace), project], data: { confirm: remove_project_message(project) }, method: :delete, class: "btn btn-remove" + %button.delete-project-button.btn.btn-danger{ data: { toggle: 'modal', + target: '#delete-project-modal', + delete_project_url: project_path(project), + project_name: project.name }, type: 'button' } + = s_('AdminProjects|Delete') + .stats %span.badge = storage_counter(project.statistics.storage_size) @@ -31,3 +36,5 @@ = paginate @projects, theme: 'gitlab' - else .nothing-here-block No projects found + + #delete-project-modal diff --git a/doc/update/mysql_to_postgresql.md b/doc/update/mysql_to_postgresql.md index fff47180099..44e9f6c5516 100644 --- a/doc/update/mysql_to_postgresql.md +++ b/doc/update/mysql_to_postgresql.md @@ -1,13 +1,15 @@ --- -last_updated: 2017-10-05 +last_updated: 2018-02-07 --- # Migrating from MySQL to PostgreSQL -> **Note:** This guide assumes you have a working Omnibus GitLab instance with +> **Note:** This guide assumes you have a working GitLab instance with > MySQL and want to migrate to bundled PostgreSQL database. -## Prerequisites +## Omnibus installation + +### Prerequisites First, we'll need to enable the bundled PostgreSQL database with up-to-date schema. Next, we'll use [pgloader](http://pgloader.io) to migrate the data @@ -19,7 +21,7 @@ Here's what you'll need to have installed: - Omnibus GitLab - MySQL -## Enable bundled PostgreSQL database +### Enable bundled PostgreSQL database 1. Stop GitLab: @@ -65,7 +67,7 @@ Here's what you'll need to have installed: After these steps, you'll have a fresh PostgreSQL database with up-to-date schema. -## Migrate data from MySQL to PostgreSQL +### Migrate data from MySQL to PostgreSQL Now, you can use pgloader to migrate the data from MySQL to PostgreSQL: @@ -104,122 +106,9 @@ the following: ----------------------------------------------- --------- --------- --------- -------------- public.abuse_reports 0 0 0 0.490s public.appearances 0 0 0 0.488s - public.approvals 0 0 0 0.273s - public.application_settings 1 1 0 0.266s - public.approvers 0 0 0 0.339s - public.approver_groups 0 0 0 0.357s - public.audit_events 1 1 0 0.410s - public.award_emoji 0 0 0 0.441s - public.boards 0 0 0 0.505s - public.broadcast_messages 0 0 0 0.498s - public.chat_names 0 0 0 0.576s - public.chat_teams 0 0 0 0.617s - public.ci_builds 0 0 0 0.611s - public.ci_group_variables 0 0 0 0.620s - public.ci_pipelines 0 0 0 0.599s - public.ci_pipeline_schedules 0 0 0 0.622s - public.ci_pipeline_schedule_variables 0 0 0 0.573s - public.ci_pipeline_variables 0 0 0 0.594s - public.ci_runners 0 0 0 0.533s - public.ci_runner_projects 0 0 0 0.584s - public.ci_sources_pipelines 0 0 0 0.564s - public.ci_stages 0 0 0 0.595s - public.ci_triggers 0 0 0 0.569s - public.ci_trigger_requests 0 0 0 0.596s - public.ci_variables 0 0 0 0.565s - public.container_repositories 0 0 0 0.605s - public.conversational_development_index_metrics 0 0 0 0.571s - public.deployments 0 0 0 0.607s - public.emails 0 0 0 0.602s - public.deploy_keys_projects 0 0 0 0.557s - public.events 160 160 0 0.677s - public.environments 0 0 0 0.567s - public.features 0 0 0 0.639s - public.events_for_migration 160 160 0 0.582s - public.feature_gates 0 0 0 0.579s - public.forked_project_links 0 0 0 0.660s - public.geo_nodes 0 0 0 0.686s - public.geo_event_log 0 0 0 0.626s - public.geo_repositories_changed_events 0 0 0 0.677s - public.geo_node_namespace_links 0 0 0 0.618s - public.geo_repository_renamed_events 0 0 0 0.696s - public.gpg_keys 0 0 0 0.704s - public.geo_repository_deleted_events 0 0 0 0.638s - public.historical_data 0 0 0 0.729s - public.geo_repository_updated_events 0 0 0 0.634s - public.index_statuses 0 0 0 0.746s - public.gpg_signatures 0 0 0 0.667s - public.issue_assignees 80 80 0 0.769s - public.identities 0 0 0 0.655s - public.issue_metrics 80 80 0 0.781s - public.issues 80 80 0 0.720s - public.labels 0 0 0 0.795s - public.issue_links 0 0 0 0.707s - public.label_priorities 0 0 0 0.793s - public.keys 0 0 0 0.734s - public.lfs_objects 0 0 0 0.812s - public.label_links 0 0 0 0.725s - public.licenses 0 0 0 0.813s - public.ldap_group_links 0 0 0 0.751s - public.members 52 52 0 0.830s - public.lfs_objects_projects 0 0 0 0.738s - public.merge_requests_closing_issues 0 0 0 0.825s - public.lists 0 0 0 0.769s - public.merge_request_diff_commits 0 0 0 0.840s - public.merge_request_metrics 0 0 0 0.837s - public.merge_requests 0 0 0 0.753s - public.merge_request_diffs 0 0 0 0.771s - public.namespaces 30 30 0 0.874s - public.merge_request_diff_files 0 0 0 0.775s - public.notes 0 0 0 0.849s - public.milestones 40 40 0 0.799s - public.oauth_access_grants 0 0 0 0.979s - public.namespace_statistics 0 0 0 0.797s - public.oauth_applications 0 0 0 0.899s - public.notification_settings 72 72 0 0.818s - public.oauth_access_tokens 0 0 0 0.807s - public.pages_domains 0 0 0 0.958s - public.oauth_openid_requests 0 0 0 0.832s - public.personal_access_tokens 0 0 0 0.965s - public.projects 8 8 0 0.987s - public.path_locks 0 0 0 0.925s - public.plans 0 0 0 0.923s - public.project_features 8 8 0 0.985s - public.project_authorizations 66 66 0 0.969s - public.project_import_data 8 8 0 1.002s - public.project_statistics 8 8 0 1.001s - public.project_group_links 0 0 0 0.949s - public.project_mirror_data 0 0 0 0.972s - public.protected_branch_merge_access_levels 0 0 0 1.017s - public.protected_branches 0 0 0 0.969s - public.protected_branch_push_access_levels 0 0 0 0.991s - public.protected_tags 0 0 0 1.009s - public.protected_tag_create_access_levels 0 0 0 0.985s - public.push_event_payloads 0 0 0 1.041s - public.push_rules 0 0 0 0.999s - public.redirect_routes 0 0 0 1.020s - public.remote_mirrors 0 0 0 1.034s - public.releases 0 0 0 0.993s - public.schema_migrations 896 896 0 1.057s - public.routes 38 38 0 1.021s - public.services 0 0 0 1.055s - public.sent_notifications 0 0 0 1.003s - public.slack_integrations 0 0 0 1.022s - public.spam_logs 0 0 0 1.024s - public.snippets 0 0 0 1.058s - public.subscriptions 0 0 0 1.069s - public.taggings 0 0 0 1.099s - public.timelogs 0 0 0 1.104s - public.system_note_metadata 0 0 0 1.038s - public.tags 0 0 0 1.034s - public.trending_projects 0 0 0 1.140s - public.uploads 0 0 0 1.129s - public.todos 80 80 0 1.085s - public.users_star_projects 0 0 0 1.153s - public.u2f_registrations 0 0 0 1.061s - public.web_hooks 0 0 0 1.179s - public.users 26 26 0 1.163s - public.user_agent_details 0 0 0 1.068s + . + . + . public.web_hook_logs 0 0 0 1.080s ----------------------------------------------- --------- --------- --------- -------------- COPY Threads Completion 4 4 0 2.008s @@ -240,9 +129,9 @@ the following: Now, you can verify that everything worked by visiting GitLab. -## Troubleshooting +### Troubleshooting -### Permissions +#### Permissions Note that the PostgreSQL user that you use for the above MUST have **superuser** privileges. Otherwise, you may see a similar message to the following: @@ -256,7 +145,7 @@ debugger invoked on a CL-POSTGRES-ERROR:INSUFFICIENT-PRIVILEGE in thread QUERY: ALTER TABLE approver_groups DISABLE TRIGGER ALL; ``` -### Experiencing 500 errors after the migration +#### Experiencing 500 errors after the migration If you experience 500 errors after the migration, try to clear the cache: @@ -265,3 +154,130 @@ sudo gitlab-rake cache:clear ``` [reconfigure GitLab]: ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure + +## Source installation + +### Prerequisites + +#### Install PostgreSQL and create database + +See [installation guide](../install/installation.md#6-database). + +#### Install [pgloader](http://pgloader.io) 3.4.1+ + +Install directly from your distro: +``` bash +sudo apt-get install pgloader +``` + +If this version is too old, use PostgreSQL's repository: +``` bash +# add repository +sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' + +# add key +sudo apt-get install wget ca-certificates +wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - + +# install package +sudo apt-get update +sudo apt-get install pgloader +``` + +### Enable bundled PostgreSQL database + +1. Stop GitLab: + + ``` bash + sudo service gitlab stop + ``` + +1. Switch database from MySQL to PostgreSQL + + ``` bash + cd /home/git/gitlab + sudo -u git mv config/database.yml config/database.yml.bak + sudo -u git cp config/database.yml.postgresql config/database.yml + sudo -u git -H chmod o-rwx config/database.yml + ``` + +1. Run the following commands to prepare the schema: + + ``` bash + sudo -u git -H bundle exec rake db:create db:migrate RAILS_ENV=production + ``` + +After these steps, you'll have a fresh PostgreSQL database with up-to-date schema. + +### Migrate data from MySQL to PostgreSQL + +Now, you can use pgloader to migrate the data from MySQL to PostgreSQL: + +1. Save the following snippet in a `commands.load` file, and edit with your + MySQL `username`, `password` and `host`: + + ``` + LOAD DATABASE + FROM mysql://username:password@host/gitlabhq_production + INTO postgresql://postgres@unix://var/run/postgresql:/gitlabhq_production + + WITH include no drop, truncate, disable triggers, create no tables, + create no indexes, preserve index names, no foreign keys, + data only + + ALTER SCHEMA 'gitlabhq_production' RENAME TO 'public' + + ; + ``` + +1. Start the migration: + + ``` bash + sudo -u postgres pgloader commands.load + ``` + +1. Once the migration finishes, you should see a summary table that looks like +the following: + + + ``` + table name read imported errors total time + ----------------------------------------------- --------- --------- --------- -------------- + fetch meta data 119 119 0 0.388s + Truncate 119 119 0 1.134s + ----------------------------------------------- --------- --------- --------- -------------- + public.abuse_reports 0 0 0 0.490s + public.appearances 0 0 0 0.488s + . + . + . + public.web_hook_logs 0 0 0 1.080s + ----------------------------------------------- --------- --------- --------- -------------- + COPY Threads Completion 4 4 0 2.008s + Reset Sequences 113 113 0 0.304s + Install Comments 0 0 0 0.000s + ----------------------------------------------- --------- --------- --------- -------------- + Total import time 1894 1894 0 12.497s + ``` + + If there is no output for more than 30 minutes, it's possible pgloader encountered an error. See + the [troubleshooting guide](#Troubleshooting) for more details. + +1. Start GitLab: + + ``` bash + sudo service gitlab start + ``` + +Now, you can verify that everything worked by visiting GitLab. + +### Troubleshooting + +#### Experiencing 500 errors after the migration + +If you experience 500 errors after the migration, try to clear the cache: + +``` bash +sudo -u git -H bundle exec rake cache:clear RAILS_ENV=production +``` + diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 4828301dbb9..b2fca2c16de 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -53,11 +53,7 @@ module Gitlab def batch(repository, blob_references, blob_size_limit: MAX_DATA_DISPLAY_SIZE) Gitlab::GitalyClient.migrate(:list_blobs_by_sha_path) do |is_enabled| if is_enabled - Gitlab::GitalyClient.allow_n_plus_1_calls do - blob_references.map do |sha, path| - find_by_gitaly(repository, sha, path, limit: blob_size_limit) - end - end + repository.gitaly_blob_client.get_blobs(blob_references, blob_size_limit).to_a else blob_references.map do |sha, path| find_by_rugged(repository, sha, path, limit: blob_size_limit) diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb index d70a1a7665e..dfa0fa43b0f 100644 --- a/lib/gitlab/gitaly_client/blob_service.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -1,6 +1,8 @@ module Gitlab module GitalyClient class BlobService + include Gitlab::EncodingHelper + def initialize(repository) @gitaly_repo = repository.gitaly_repository end @@ -54,6 +56,30 @@ module Gitlab end end end + + def get_blobs(revision_paths, limit = -1) + return [] if revision_paths.empty? + + revision_paths.map! do |rev, path| + Gitaly::GetBlobsRequest::RevisionPath.new(revision: rev, path: encode_binary(path)) + end + + request = Gitaly::GetBlobsRequest.new( + repository: @gitaly_repo, + revision_paths: revision_paths, + limit: limit + ) + + response = GitalyClient.call( + @gitaly_repo.storage_name, + :blob_service, + :get_blobs, + request, + timeout: GitalyClient.default_timeout + ) + + GitalyClient::BlobsStitcher.new(response) + end end end end diff --git a/lib/gitlab/gitaly_client/blobs_stitcher.rb b/lib/gitlab/gitaly_client/blobs_stitcher.rb new file mode 100644 index 00000000000..5ca592ff812 --- /dev/null +++ b/lib/gitlab/gitaly_client/blobs_stitcher.rb @@ -0,0 +1,47 @@ +module Gitlab + module GitalyClient + class BlobsStitcher + include Enumerable + + def initialize(rpc_response) + @rpc_response = rpc_response + end + + def each + current_blob_data = nil + + @rpc_response.each do |msg| + begin + if msg.oid.blank? && msg.data.blank? + next + elsif msg.oid.present? + yield new_blob(current_blob_data) if current_blob_data + + current_blob_data = msg.to_h.slice(:oid, :path, :size, :revision, :mode) + current_blob_data[:data] = msg.data.dup + else + current_blob_data[:data] << msg.data + end + end + end + + yield new_blob(current_blob_data) if current_blob_data + end + + private + + def new_blob(blob_data) + Gitlab::Git::Blob.new( + id: blob_data[:oid], + mode: blob_data[:mode].to_s(8), + name: File.basename(blob_data[:path]), + path: blob_data[:path], + size: blob_data[:size], + commit_id: blob_data[:revision], + data: blob_data[:data], + binary: Gitlab::Git::Blob.binary?(blob_data[:data]) + ) + end + end + end +end diff --git a/spec/lib/gitlab/git/blob_spec.rb b/spec/lib/gitlab/git/blob_spec.rb index 8ac960133c5..59e9e1cc94c 100644 --- a/spec/lib/gitlab/git/blob_spec.rb +++ b/spec/lib/gitlab/git/blob_spec.rb @@ -178,67 +178,77 @@ describe Gitlab::Git::Blob, seed_helper: true do end describe '.batch' do - let(:blob_references) do - [ - [SeedRepo::Commit::ID, "files/ruby/popen.rb"], - [SeedRepo::Commit::ID, 'six'] - ] - end + shared_examples 'loading blobs in batch' do + let(:blob_references) do + [ + [SeedRepo::Commit::ID, "files/ruby/popen.rb"], + [SeedRepo::Commit::ID, 'six'] + ] + end - subject { described_class.batch(repository, blob_references) } + subject { described_class.batch(repository, blob_references) } - it { expect(subject.size).to eq(blob_references.size) } + it { expect(subject.size).to eq(blob_references.size) } - context 'first blob' do - let(:blob) { subject[0] } + context 'first blob' do + let(:blob) { subject[0] } - it { expect(blob.id).to eq(SeedRepo::RubyBlob::ID) } - it { expect(blob.name).to eq(SeedRepo::RubyBlob::NAME) } - it { expect(blob.path).to eq("files/ruby/popen.rb") } - it { expect(blob.commit_id).to eq(SeedRepo::Commit::ID) } - it { expect(blob.data[0..10]).to eq(SeedRepo::RubyBlob::CONTENT[0..10]) } - it { expect(blob.size).to eq(669) } - it { expect(blob.mode).to eq("100644") } - end + it { expect(blob.id).to eq(SeedRepo::RubyBlob::ID) } + it { expect(blob.name).to eq(SeedRepo::RubyBlob::NAME) } + it { expect(blob.path).to eq("files/ruby/popen.rb") } + it { expect(blob.commit_id).to eq(SeedRepo::Commit::ID) } + it { expect(blob.data[0..10]).to eq(SeedRepo::RubyBlob::CONTENT[0..10]) } + it { expect(blob.size).to eq(669) } + it { expect(blob.mode).to eq("100644") } + end - context 'second blob' do - let(:blob) { subject[1] } + context 'second blob' do + let(:blob) { subject[1] } - it { expect(blob.id).to eq('409f37c4f05865e4fb208c771485f211a22c4c2d') } - it { expect(blob.data).to eq('') } - it 'does not mark the blob as binary' do - expect(blob).not_to be_binary + it { expect(blob.id).to eq('409f37c4f05865e4fb208c771485f211a22c4c2d') } + it { expect(blob.data).to eq('') } + it 'does not mark the blob as binary' do + expect(blob).not_to be_binary + end end - end - context 'limiting' do - subject { described_class.batch(repository, blob_references, blob_size_limit: blob_size_limit) } + context 'limiting' do + subject { described_class.batch(repository, blob_references, blob_size_limit: blob_size_limit) } - context 'positive' do - let(:blob_size_limit) { 10 } + context 'positive' do + let(:blob_size_limit) { 10 } - it { expect(subject.first.data.size).to eq(10) } - end + it { expect(subject.first.data.size).to eq(10) } + end - context 'zero' do - let(:blob_size_limit) { 0 } + context 'zero' do + let(:blob_size_limit) { 0 } - it 'only loads the metadata' do - expect(subject.first.size).not_to be(0) - expect(subject.first.data).to eq('') + it 'only loads the metadata' do + expect(subject.first.size).not_to be(0) + expect(subject.first.data).to eq('') + end end - end - context 'negative' do - let(:blob_size_limit) { -1 } + context 'negative' do + let(:blob_size_limit) { -1 } - it 'ignores MAX_DATA_DISPLAY_SIZE' do - stub_const('Gitlab::Git::Blob::MAX_DATA_DISPLAY_SIZE', 100) + it 'ignores MAX_DATA_DISPLAY_SIZE' do + stub_const('Gitlab::Git::Blob::MAX_DATA_DISPLAY_SIZE', 100) - expect(subject.first.data.size).to eq(669) + expect(subject.first.data.size).to eq(669) + end end end end + + context 'when Gitaly list_blobs_by_sha_path feature is enabled' do + it_behaves_like 'loading blobs in batch' + end + + context 'when Gitaly list_blobs_by_sha_path feature is disabled', :disable_gitaly do + it_behaves_like 'loading blobs in batch' + end end describe '.batch_lfs_pointers' do diff --git a/spec/lib/gitlab/gitaly_client/blobs_stitcher_spec.rb b/spec/lib/gitlab/gitaly_client/blobs_stitcher_spec.rb new file mode 100644 index 00000000000..9db710e759e --- /dev/null +++ b/spec/lib/gitlab/gitaly_client/blobs_stitcher_spec.rb @@ -0,0 +1,36 @@ +require 'spec_helper' + +describe Gitlab::GitalyClient::BlobsStitcher do + describe 'enumeration' do + it 'combines segregated blob messages together' do + messages = [ + OpenStruct.new(oid: 'abcdef1', path: 'path/to/file', size: 1642, revision: 'f00ba7', mode: 0100644, data: "first-line\n"), + OpenStruct.new(oid: '', data: 'second-line'), + OpenStruct.new(oid: '', data: '', revision: 'f00ba7', path: 'path/to/non-existent/file'), + OpenStruct.new(oid: 'abcdef2', path: 'path/to/another-file', size: 2461, revision: 'f00ba8', mode: 0100644, data: "GIF87a\x90\x01".b) + ] + + blobs = described_class.new(messages).to_a + + expect(blobs.size).to be(2) + + expect(blobs[0].id).to eq('abcdef1') + expect(blobs[0].mode).to eq('100644') + expect(blobs[0].name).to eq('file') + expect(blobs[0].path).to eq('path/to/file') + expect(blobs[0].size).to eq(1642) + expect(blobs[0].commit_id).to eq('f00ba7') + expect(blobs[0].data).to eq("first-line\nsecond-line") + expect(blobs[0].binary?).to be false + + expect(blobs[1].id).to eq('abcdef2') + expect(blobs[1].mode).to eq('100644') + expect(blobs[1].name).to eq('another-file') + expect(blobs[1].path).to eq('path/to/another-file') + expect(blobs[1].size).to eq(2461) + expect(blobs[1].commit_id).to eq('f00ba8') + expect(blobs[1].data).to eq("GIF87a\x90\x01".b) + expect(blobs[1].binary?).to be true + end + end +end diff --git a/vendor/gitlab-ci-yml/Auto-DevOps.gitlab-ci.yml b/vendor/gitlab-ci-yml/Auto-DevOps.gitlab-ci.yml index a7cd2bc972c..b2439d96e00 100644 --- a/vendor/gitlab-ci-yml/Auto-DevOps.gitlab-ci.yml +++ b/vendor/gitlab-ci-yml/Auto-DevOps.gitlab-ci.yml @@ -103,6 +103,7 @@ performance: artifacts: paths: - performance.json + - sitespeed-results/ only: refs: - branches @@ -503,16 +504,16 @@ production: export CI_ENVIRONMENT_URL=$(cat environment_url.txt) mkdir gitlab-exporter - wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/10-3/index.js + wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/10-5/index.js mkdir sitespeed-results if [ -f .gitlab-urls.txt ] then sed -i -e 's@^@'"$CI_ENVIRONMENT_URL"'@' .gitlab-urls.txt - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.0.3 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results .gitlab-urls.txt + docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results .gitlab-urls.txt else - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.0.3 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL" + docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL" fi mv sitespeed-results/data/performance.json performance.json |