summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorShah El-Rahman <selrahman@gitlab.com>2018-02-07 21:41:40 -0600
committerShah El-Rahman <selrahman@gitlab.com>2018-02-07 21:41:40 -0600
commit4988fe7467f40c4b1da9fa775512bd6dec0c7884 (patch)
tree457f967de16bb9eb3046ee92d901cb0392e1b1f3
parent0cb8409a162bf6beaa3d808446a7e9aa80263340 (diff)
parent15762319e94457cc82dba8d15d5d9507ad527503 (diff)
downloadgitlab-ce-41301-new-design-for-user-deletion-confirmation-in-admin-area.tar.gz
Merge branch 'master' into 41301-new-design-for-user-deletion-confirmation-in-admin-area41301-new-design-for-user-deletion-confirmation-in-admin-area
-rw-r--r--Gemfile4
-rw-r--r--Gemfile.lock8
-rw-r--r--app/assets/javascripts/dispatcher.js5
-rw-r--r--app/assets/javascripts/pages/admin/projects/index/components/delete_project_modal.vue125
-rw-r--r--app/assets/javascripts/pages/admin/projects/index/index.js37
-rw-r--r--app/views/admin/projects/_projects.html.haml9
-rw-r--r--doc/update/mysql_to_postgresql.md264
-rw-r--r--lib/gitlab/git/blob.rb6
-rw-r--r--lib/gitlab/gitaly_client/blob_service.rb26
-rw-r--r--lib/gitlab/gitaly_client/blobs_stitcher.rb47
-rw-r--r--spec/lib/gitlab/git/blob_spec.rb94
-rw-r--r--spec/lib/gitlab/gitaly_client/blobs_stitcher_spec.rb36
-rw-r--r--vendor/gitlab-ci-yml/Auto-DevOps.gitlab-ci.yml7
13 files changed, 487 insertions, 181 deletions
diff --git a/Gemfile b/Gemfile
index cb381aa12c8..880ed483c34 100644
--- a/Gemfile
+++ b/Gemfile
@@ -294,7 +294,7 @@ group :metrics do
gem 'influxdb', '~> 0.2', require: false
# Prometheus
- gem 'prometheus-client-mmap', '~> 0.9.1.pre.rc.2'
+ gem 'prometheus-client-mmap', '~> 0.9.1'
gem 'raindrops', '~> 0.18'
end
@@ -410,7 +410,7 @@ group :ed25519 do
end
# Gitaly GRPC client
-gem 'gitaly-proto', '~> 0.83.0', require: 'gitaly'
+gem 'gitaly-proto', '~> 0.84.0', require: 'gitaly'
# Locked until https://github.com/google/protobuf/issues/4210 is closed
gem 'google-protobuf', '= 3.5.1'
diff --git a/Gemfile.lock b/Gemfile.lock
index d9665e38f45..22c4fc0ef28 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -285,7 +285,7 @@ GEM
po_to_json (>= 1.0.0)
rails (>= 3.2.0)
gherkin-ruby (0.3.2)
- gitaly-proto (0.83.0)
+ gitaly-proto (0.84.0)
google-protobuf (~> 3.1)
grpc (~> 1.0)
github-linguist (4.7.6)
@@ -636,7 +636,7 @@ GEM
parser
unparser
procto (0.0.3)
- prometheus-client-mmap (0.9.1.pre.rc.2)
+ prometheus-client-mmap (0.9.1)
pry (0.10.4)
coderay (~> 1.1.0)
method_source (~> 0.8.1)
@@ -1056,7 +1056,7 @@ DEPENDENCIES
gettext (~> 3.2.2)
gettext_i18n_rails (~> 1.8.0)
gettext_i18n_rails_js (~> 1.2.0)
- gitaly-proto (~> 0.83.0)
+ gitaly-proto (~> 0.84.0)
github-linguist (~> 4.7.0)
gitlab-flowdock-git-hook (~> 1.0.1)
gitlab-markup (~> 1.6.2)
@@ -1132,7 +1132,7 @@ DEPENDENCIES
peek-sidekiq (~> 1.0.3)
pg (~> 0.18.2)
premailer-rails (~> 1.9.7)
- prometheus-client-mmap (~> 0.9.1.pre.rc.2)
+ prometheus-client-mmap (~> 0.9.1)
pry-byebug (~> 3.4.1)
pry-rails (~> 0.3.4)
rack-attack (~> 4.4.1)
diff --git a/app/assets/javascripts/dispatcher.js b/app/assets/javascripts/dispatcher.js
index 94695674ba0..aceaffdfcb9 100644
--- a/app/assets/javascripts/dispatcher.js
+++ b/app/assets/javascripts/dispatcher.js
@@ -109,6 +109,11 @@ var Dispatcher;
.then(callDefault)
.catch(fail);
break;
+ case 'admin:projects:index':
+ import('./pages/admin/projects/index/index')
+ .then(callDefault)
+ .catch(fail);
+ break;
case 'admin:users:index':
import('./pages/admin/users/shared')
.then(callDefault)
diff --git a/app/assets/javascripts/pages/admin/projects/index/components/delete_project_modal.vue b/app/assets/javascripts/pages/admin/projects/index/components/delete_project_modal.vue
new file mode 100644
index 00000000000..14315d5492e
--- /dev/null
+++ b/app/assets/javascripts/pages/admin/projects/index/components/delete_project_modal.vue
@@ -0,0 +1,125 @@
+<script>
+ import _ from 'underscore';
+ import modal from '~/vue_shared/components/modal.vue';
+ import { s__, sprintf } from '~/locale';
+
+ export default {
+ components: {
+ modal,
+ },
+ props: {
+ deleteProjectUrl: {
+ type: String,
+ required: false,
+ default: '',
+ },
+ projectName: {
+ type: String,
+ required: false,
+ default: '',
+ },
+ csrfToken: {
+ type: String,
+ required: false,
+ default: '',
+ },
+ },
+ data() {
+ return {
+ enteredProjectName: '',
+ };
+ },
+ computed: {
+ title() {
+ return sprintf(s__('AdminProjects|Delete Project %{projectName}?'),
+ {
+ projectName: `'${_.escape(this.projectName)}'`,
+ },
+ false,
+ );
+ },
+ text() {
+ return sprintf(s__(`AdminProjects|
+ You’re about to permanently delete the project %{projectName}, its repository,
+ and all related resources including issues, merge requests, etc.. Once you confirm and press
+ %{strong_start}Delete project%{strong_end}, it cannot be undone or recovered.`),
+ {
+ projectName: `<strong>${_.escape(this.projectName)}</strong>`,
+ strong_start: '<strong>',
+ strong_end: '</strong>',
+ },
+ false,
+ );
+ },
+ confirmationTextLabel() {
+ return sprintf(s__('AdminUsers|To confirm, type %{projectName}'),
+ {
+ projectName: `<code>${_.escape(this.projectName)}</code>`,
+ },
+ false,
+ );
+ },
+ primaryButtonLabel() {
+ return s__('AdminProjects|Delete project');
+ },
+ canSubmit() {
+ return this.enteredProjectName === this.projectName;
+ },
+ },
+ methods: {
+ onCancel() {
+ this.enteredProjectName = '';
+ },
+ onSubmit() {
+ this.$refs.form.submit();
+ this.enteredProjectName = '';
+ },
+ },
+ };
+</script>
+
+<template>
+ <modal
+ id="delete-project-modal"
+ :title="title"
+ :text="text"
+ kind="danger"
+ :primary-button-label="primaryButtonLabel"
+ :submit-disabled="!canSubmit"
+ @submit="onSubmit"
+ @cancel="onCancel"
+ >
+ <template
+ slot="body"
+ slot-scope="props"
+ >
+ <p v-html="props.text"></p>
+ <p v-html="confirmationTextLabel"></p>
+ <form
+ ref="form"
+ :action="deleteProjectUrl"
+ method="post"
+ >
+ <input
+ ref="method"
+ type="hidden"
+ name="_method"
+ value="delete"
+ />
+ <input
+ type="hidden"
+ name="authenticity_token"
+ :value="csrfToken"
+ />
+ <input
+ name="projectName"
+ class="form-control"
+ type="text"
+ v-model="enteredProjectName"
+ aria-labelledby="input-label"
+ autocomplete="off"
+ />
+ </form>
+ </template>
+ </modal>
+</template>
diff --git a/app/assets/javascripts/pages/admin/projects/index/index.js b/app/assets/javascripts/pages/admin/projects/index/index.js
new file mode 100644
index 00000000000..a87b27090a8
--- /dev/null
+++ b/app/assets/javascripts/pages/admin/projects/index/index.js
@@ -0,0 +1,37 @@
+import Vue from 'vue';
+
+import Translate from '~/vue_shared/translate';
+import csrf from '~/lib/utils/csrf';
+
+import deleteProjectModal from './components/delete_project_modal.vue';
+
+export default () => {
+ Vue.use(Translate);
+
+ const deleteProjectModalEl = document.getElementById('delete-project-modal');
+
+ const deleteModal = new Vue({
+ el: deleteProjectModalEl,
+ data: {
+ deleteProjectUrl: '',
+ projectName: '',
+ },
+ render(createElement) {
+ return createElement(deleteProjectModal, {
+ props: {
+ deleteProjectUrl: this.deleteProjectUrl,
+ projectName: this.projectName,
+ csrfToken: csrf.token,
+ },
+ });
+ },
+ });
+
+ $(document).on('shown.bs.modal', (event) => {
+ if (event.relatedTarget.classList.contains('delete-project-button')) {
+ const buttonProps = event.relatedTarget.dataset;
+ deleteModal.deleteProjectUrl = buttonProps.deleteProjectUrl;
+ deleteModal.projectName = buttonProps.projectName;
+ }
+ });
+};
diff --git a/app/views/admin/projects/_projects.html.haml b/app/views/admin/projects/_projects.html.haml
index c69c2761189..b5d7b889504 100644
--- a/app/views/admin/projects/_projects.html.haml
+++ b/app/views/admin/projects/_projects.html.haml
@@ -5,7 +5,12 @@
%li.project-row{ class: ('no-description' if project.description.blank?) }
.controls
= link_to 'Edit', edit_project_path(project), id: "edit_#{dom_id(project)}", class: "btn"
- = link_to 'Delete', [project.namespace.becomes(Namespace), project], data: { confirm: remove_project_message(project) }, method: :delete, class: "btn btn-remove"
+ %button.delete-project-button.btn.btn-danger{ data: { toggle: 'modal',
+ target: '#delete-project-modal',
+ delete_project_url: project_path(project),
+ project_name: project.name }, type: 'button' }
+ = s_('AdminProjects|Delete')
+
.stats
%span.badge
= storage_counter(project.statistics.storage_size)
@@ -31,3 +36,5 @@
= paginate @projects, theme: 'gitlab'
- else
.nothing-here-block No projects found
+
+ #delete-project-modal
diff --git a/doc/update/mysql_to_postgresql.md b/doc/update/mysql_to_postgresql.md
index fff47180099..44e9f6c5516 100644
--- a/doc/update/mysql_to_postgresql.md
+++ b/doc/update/mysql_to_postgresql.md
@@ -1,13 +1,15 @@
---
-last_updated: 2017-10-05
+last_updated: 2018-02-07
---
# Migrating from MySQL to PostgreSQL
-> **Note:** This guide assumes you have a working Omnibus GitLab instance with
+> **Note:** This guide assumes you have a working GitLab instance with
> MySQL and want to migrate to bundled PostgreSQL database.
-## Prerequisites
+## Omnibus installation
+
+### Prerequisites
First, we'll need to enable the bundled PostgreSQL database with up-to-date
schema. Next, we'll use [pgloader](http://pgloader.io) to migrate the data
@@ -19,7 +21,7 @@ Here's what you'll need to have installed:
- Omnibus GitLab
- MySQL
-## Enable bundled PostgreSQL database
+### Enable bundled PostgreSQL database
1. Stop GitLab:
@@ -65,7 +67,7 @@ Here's what you'll need to have installed:
After these steps, you'll have a fresh PostgreSQL database with up-to-date schema.
-## Migrate data from MySQL to PostgreSQL
+### Migrate data from MySQL to PostgreSQL
Now, you can use pgloader to migrate the data from MySQL to PostgreSQL:
@@ -104,122 +106,9 @@ the following:
----------------------------------------------- --------- --------- --------- --------------
public.abuse_reports 0 0 0 0.490s
public.appearances 0 0 0 0.488s
- public.approvals 0 0 0 0.273s
- public.application_settings 1 1 0 0.266s
- public.approvers 0 0 0 0.339s
- public.approver_groups 0 0 0 0.357s
- public.audit_events 1 1 0 0.410s
- public.award_emoji 0 0 0 0.441s
- public.boards 0 0 0 0.505s
- public.broadcast_messages 0 0 0 0.498s
- public.chat_names 0 0 0 0.576s
- public.chat_teams 0 0 0 0.617s
- public.ci_builds 0 0 0 0.611s
- public.ci_group_variables 0 0 0 0.620s
- public.ci_pipelines 0 0 0 0.599s
- public.ci_pipeline_schedules 0 0 0 0.622s
- public.ci_pipeline_schedule_variables 0 0 0 0.573s
- public.ci_pipeline_variables 0 0 0 0.594s
- public.ci_runners 0 0 0 0.533s
- public.ci_runner_projects 0 0 0 0.584s
- public.ci_sources_pipelines 0 0 0 0.564s
- public.ci_stages 0 0 0 0.595s
- public.ci_triggers 0 0 0 0.569s
- public.ci_trigger_requests 0 0 0 0.596s
- public.ci_variables 0 0 0 0.565s
- public.container_repositories 0 0 0 0.605s
- public.conversational_development_index_metrics 0 0 0 0.571s
- public.deployments 0 0 0 0.607s
- public.emails 0 0 0 0.602s
- public.deploy_keys_projects 0 0 0 0.557s
- public.events 160 160 0 0.677s
- public.environments 0 0 0 0.567s
- public.features 0 0 0 0.639s
- public.events_for_migration 160 160 0 0.582s
- public.feature_gates 0 0 0 0.579s
- public.forked_project_links 0 0 0 0.660s
- public.geo_nodes 0 0 0 0.686s
- public.geo_event_log 0 0 0 0.626s
- public.geo_repositories_changed_events 0 0 0 0.677s
- public.geo_node_namespace_links 0 0 0 0.618s
- public.geo_repository_renamed_events 0 0 0 0.696s
- public.gpg_keys 0 0 0 0.704s
- public.geo_repository_deleted_events 0 0 0 0.638s
- public.historical_data 0 0 0 0.729s
- public.geo_repository_updated_events 0 0 0 0.634s
- public.index_statuses 0 0 0 0.746s
- public.gpg_signatures 0 0 0 0.667s
- public.issue_assignees 80 80 0 0.769s
- public.identities 0 0 0 0.655s
- public.issue_metrics 80 80 0 0.781s
- public.issues 80 80 0 0.720s
- public.labels 0 0 0 0.795s
- public.issue_links 0 0 0 0.707s
- public.label_priorities 0 0 0 0.793s
- public.keys 0 0 0 0.734s
- public.lfs_objects 0 0 0 0.812s
- public.label_links 0 0 0 0.725s
- public.licenses 0 0 0 0.813s
- public.ldap_group_links 0 0 0 0.751s
- public.members 52 52 0 0.830s
- public.lfs_objects_projects 0 0 0 0.738s
- public.merge_requests_closing_issues 0 0 0 0.825s
- public.lists 0 0 0 0.769s
- public.merge_request_diff_commits 0 0 0 0.840s
- public.merge_request_metrics 0 0 0 0.837s
- public.merge_requests 0 0 0 0.753s
- public.merge_request_diffs 0 0 0 0.771s
- public.namespaces 30 30 0 0.874s
- public.merge_request_diff_files 0 0 0 0.775s
- public.notes 0 0 0 0.849s
- public.milestones 40 40 0 0.799s
- public.oauth_access_grants 0 0 0 0.979s
- public.namespace_statistics 0 0 0 0.797s
- public.oauth_applications 0 0 0 0.899s
- public.notification_settings 72 72 0 0.818s
- public.oauth_access_tokens 0 0 0 0.807s
- public.pages_domains 0 0 0 0.958s
- public.oauth_openid_requests 0 0 0 0.832s
- public.personal_access_tokens 0 0 0 0.965s
- public.projects 8 8 0 0.987s
- public.path_locks 0 0 0 0.925s
- public.plans 0 0 0 0.923s
- public.project_features 8 8 0 0.985s
- public.project_authorizations 66 66 0 0.969s
- public.project_import_data 8 8 0 1.002s
- public.project_statistics 8 8 0 1.001s
- public.project_group_links 0 0 0 0.949s
- public.project_mirror_data 0 0 0 0.972s
- public.protected_branch_merge_access_levels 0 0 0 1.017s
- public.protected_branches 0 0 0 0.969s
- public.protected_branch_push_access_levels 0 0 0 0.991s
- public.protected_tags 0 0 0 1.009s
- public.protected_tag_create_access_levels 0 0 0 0.985s
- public.push_event_payloads 0 0 0 1.041s
- public.push_rules 0 0 0 0.999s
- public.redirect_routes 0 0 0 1.020s
- public.remote_mirrors 0 0 0 1.034s
- public.releases 0 0 0 0.993s
- public.schema_migrations 896 896 0 1.057s
- public.routes 38 38 0 1.021s
- public.services 0 0 0 1.055s
- public.sent_notifications 0 0 0 1.003s
- public.slack_integrations 0 0 0 1.022s
- public.spam_logs 0 0 0 1.024s
- public.snippets 0 0 0 1.058s
- public.subscriptions 0 0 0 1.069s
- public.taggings 0 0 0 1.099s
- public.timelogs 0 0 0 1.104s
- public.system_note_metadata 0 0 0 1.038s
- public.tags 0 0 0 1.034s
- public.trending_projects 0 0 0 1.140s
- public.uploads 0 0 0 1.129s
- public.todos 80 80 0 1.085s
- public.users_star_projects 0 0 0 1.153s
- public.u2f_registrations 0 0 0 1.061s
- public.web_hooks 0 0 0 1.179s
- public.users 26 26 0 1.163s
- public.user_agent_details 0 0 0 1.068s
+ .
+ .
+ .
public.web_hook_logs 0 0 0 1.080s
----------------------------------------------- --------- --------- --------- --------------
COPY Threads Completion 4 4 0 2.008s
@@ -240,9 +129,9 @@ the following:
Now, you can verify that everything worked by visiting GitLab.
-## Troubleshooting
+### Troubleshooting
-### Permissions
+#### Permissions
Note that the PostgreSQL user that you use for the above MUST have **superuser** privileges. Otherwise, you may see
a similar message to the following:
@@ -256,7 +145,7 @@ debugger invoked on a CL-POSTGRES-ERROR:INSUFFICIENT-PRIVILEGE in thread
QUERY: ALTER TABLE approver_groups DISABLE TRIGGER ALL;
```
-### Experiencing 500 errors after the migration
+#### Experiencing 500 errors after the migration
If you experience 500 errors after the migration, try to clear the cache:
@@ -265,3 +154,130 @@ sudo gitlab-rake cache:clear
```
[reconfigure GitLab]: ../administration/restart_gitlab.md#omnibus-gitlab-reconfigure
+
+## Source installation
+
+### Prerequisites
+
+#### Install PostgreSQL and create database
+
+See [installation guide](../install/installation.md#6-database).
+
+#### Install [pgloader](http://pgloader.io) 3.4.1+
+
+Install directly from your distro:
+``` bash
+sudo apt-get install pgloader
+```
+
+If this version is too old, use PostgreSQL's repository:
+``` bash
+# add repository
+sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+
+# add key
+sudo apt-get install wget ca-certificates
+wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
+
+# install package
+sudo apt-get update
+sudo apt-get install pgloader
+```
+
+### Enable bundled PostgreSQL database
+
+1. Stop GitLab:
+
+ ``` bash
+ sudo service gitlab stop
+ ```
+
+1. Switch database from MySQL to PostgreSQL
+
+ ``` bash
+ cd /home/git/gitlab
+ sudo -u git mv config/database.yml config/database.yml.bak
+ sudo -u git cp config/database.yml.postgresql config/database.yml
+ sudo -u git -H chmod o-rwx config/database.yml
+ ```
+
+1. Run the following commands to prepare the schema:
+
+ ``` bash
+ sudo -u git -H bundle exec rake db:create db:migrate RAILS_ENV=production
+ ```
+
+After these steps, you'll have a fresh PostgreSQL database with up-to-date schema.
+
+### Migrate data from MySQL to PostgreSQL
+
+Now, you can use pgloader to migrate the data from MySQL to PostgreSQL:
+
+1. Save the following snippet in a `commands.load` file, and edit with your
+ MySQL `username`, `password` and `host`:
+
+ ```
+ LOAD DATABASE
+ FROM mysql://username:password@host/gitlabhq_production
+ INTO postgresql://postgres@unix://var/run/postgresql:/gitlabhq_production
+
+ WITH include no drop, truncate, disable triggers, create no tables,
+ create no indexes, preserve index names, no foreign keys,
+ data only
+
+ ALTER SCHEMA 'gitlabhq_production' RENAME TO 'public'
+
+ ;
+ ```
+
+1. Start the migration:
+
+ ``` bash
+ sudo -u postgres pgloader commands.load
+ ```
+
+1. Once the migration finishes, you should see a summary table that looks like
+the following:
+
+
+ ```
+ table name read imported errors total time
+ ----------------------------------------------- --------- --------- --------- --------------
+ fetch meta data 119 119 0 0.388s
+ Truncate 119 119 0 1.134s
+ ----------------------------------------------- --------- --------- --------- --------------
+ public.abuse_reports 0 0 0 0.490s
+ public.appearances 0 0 0 0.488s
+ .
+ .
+ .
+ public.web_hook_logs 0 0 0 1.080s
+ ----------------------------------------------- --------- --------- --------- --------------
+ COPY Threads Completion 4 4 0 2.008s
+ Reset Sequences 113 113 0 0.304s
+ Install Comments 0 0 0 0.000s
+ ----------------------------------------------- --------- --------- --------- --------------
+ Total import time 1894 1894 0 12.497s
+ ```
+
+ If there is no output for more than 30 minutes, it's possible pgloader encountered an error. See
+ the [troubleshooting guide](#Troubleshooting) for more details.
+
+1. Start GitLab:
+
+ ``` bash
+ sudo service gitlab start
+ ```
+
+Now, you can verify that everything worked by visiting GitLab.
+
+### Troubleshooting
+
+#### Experiencing 500 errors after the migration
+
+If you experience 500 errors after the migration, try to clear the cache:
+
+``` bash
+sudo -u git -H bundle exec rake cache:clear RAILS_ENV=production
+```
+
diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb
index 4828301dbb9..b2fca2c16de 100644
--- a/lib/gitlab/git/blob.rb
+++ b/lib/gitlab/git/blob.rb
@@ -53,11 +53,7 @@ module Gitlab
def batch(repository, blob_references, blob_size_limit: MAX_DATA_DISPLAY_SIZE)
Gitlab::GitalyClient.migrate(:list_blobs_by_sha_path) do |is_enabled|
if is_enabled
- Gitlab::GitalyClient.allow_n_plus_1_calls do
- blob_references.map do |sha, path|
- find_by_gitaly(repository, sha, path, limit: blob_size_limit)
- end
- end
+ repository.gitaly_blob_client.get_blobs(blob_references, blob_size_limit).to_a
else
blob_references.map do |sha, path|
find_by_rugged(repository, sha, path, limit: blob_size_limit)
diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb
index d70a1a7665e..dfa0fa43b0f 100644
--- a/lib/gitlab/gitaly_client/blob_service.rb
+++ b/lib/gitlab/gitaly_client/blob_service.rb
@@ -1,6 +1,8 @@
module Gitlab
module GitalyClient
class BlobService
+ include Gitlab::EncodingHelper
+
def initialize(repository)
@gitaly_repo = repository.gitaly_repository
end
@@ -54,6 +56,30 @@ module Gitlab
end
end
end
+
+ def get_blobs(revision_paths, limit = -1)
+ return [] if revision_paths.empty?
+
+ revision_paths.map! do |rev, path|
+ Gitaly::GetBlobsRequest::RevisionPath.new(revision: rev, path: encode_binary(path))
+ end
+
+ request = Gitaly::GetBlobsRequest.new(
+ repository: @gitaly_repo,
+ revision_paths: revision_paths,
+ limit: limit
+ )
+
+ response = GitalyClient.call(
+ @gitaly_repo.storage_name,
+ :blob_service,
+ :get_blobs,
+ request,
+ timeout: GitalyClient.default_timeout
+ )
+
+ GitalyClient::BlobsStitcher.new(response)
+ end
end
end
end
diff --git a/lib/gitlab/gitaly_client/blobs_stitcher.rb b/lib/gitlab/gitaly_client/blobs_stitcher.rb
new file mode 100644
index 00000000000..5ca592ff812
--- /dev/null
+++ b/lib/gitlab/gitaly_client/blobs_stitcher.rb
@@ -0,0 +1,47 @@
+module Gitlab
+ module GitalyClient
+ class BlobsStitcher
+ include Enumerable
+
+ def initialize(rpc_response)
+ @rpc_response = rpc_response
+ end
+
+ def each
+ current_blob_data = nil
+
+ @rpc_response.each do |msg|
+ begin
+ if msg.oid.blank? && msg.data.blank?
+ next
+ elsif msg.oid.present?
+ yield new_blob(current_blob_data) if current_blob_data
+
+ current_blob_data = msg.to_h.slice(:oid, :path, :size, :revision, :mode)
+ current_blob_data[:data] = msg.data.dup
+ else
+ current_blob_data[:data] << msg.data
+ end
+ end
+ end
+
+ yield new_blob(current_blob_data) if current_blob_data
+ end
+
+ private
+
+ def new_blob(blob_data)
+ Gitlab::Git::Blob.new(
+ id: blob_data[:oid],
+ mode: blob_data[:mode].to_s(8),
+ name: File.basename(blob_data[:path]),
+ path: blob_data[:path],
+ size: blob_data[:size],
+ commit_id: blob_data[:revision],
+ data: blob_data[:data],
+ binary: Gitlab::Git::Blob.binary?(blob_data[:data])
+ )
+ end
+ end
+ end
+end
diff --git a/spec/lib/gitlab/git/blob_spec.rb b/spec/lib/gitlab/git/blob_spec.rb
index 8ac960133c5..59e9e1cc94c 100644
--- a/spec/lib/gitlab/git/blob_spec.rb
+++ b/spec/lib/gitlab/git/blob_spec.rb
@@ -178,67 +178,77 @@ describe Gitlab::Git::Blob, seed_helper: true do
end
describe '.batch' do
- let(:blob_references) do
- [
- [SeedRepo::Commit::ID, "files/ruby/popen.rb"],
- [SeedRepo::Commit::ID, 'six']
- ]
- end
+ shared_examples 'loading blobs in batch' do
+ let(:blob_references) do
+ [
+ [SeedRepo::Commit::ID, "files/ruby/popen.rb"],
+ [SeedRepo::Commit::ID, 'six']
+ ]
+ end
- subject { described_class.batch(repository, blob_references) }
+ subject { described_class.batch(repository, blob_references) }
- it { expect(subject.size).to eq(blob_references.size) }
+ it { expect(subject.size).to eq(blob_references.size) }
- context 'first blob' do
- let(:blob) { subject[0] }
+ context 'first blob' do
+ let(:blob) { subject[0] }
- it { expect(blob.id).to eq(SeedRepo::RubyBlob::ID) }
- it { expect(blob.name).to eq(SeedRepo::RubyBlob::NAME) }
- it { expect(blob.path).to eq("files/ruby/popen.rb") }
- it { expect(blob.commit_id).to eq(SeedRepo::Commit::ID) }
- it { expect(blob.data[0..10]).to eq(SeedRepo::RubyBlob::CONTENT[0..10]) }
- it { expect(blob.size).to eq(669) }
- it { expect(blob.mode).to eq("100644") }
- end
+ it { expect(blob.id).to eq(SeedRepo::RubyBlob::ID) }
+ it { expect(blob.name).to eq(SeedRepo::RubyBlob::NAME) }
+ it { expect(blob.path).to eq("files/ruby/popen.rb") }
+ it { expect(blob.commit_id).to eq(SeedRepo::Commit::ID) }
+ it { expect(blob.data[0..10]).to eq(SeedRepo::RubyBlob::CONTENT[0..10]) }
+ it { expect(blob.size).to eq(669) }
+ it { expect(blob.mode).to eq("100644") }
+ end
- context 'second blob' do
- let(:blob) { subject[1] }
+ context 'second blob' do
+ let(:blob) { subject[1] }
- it { expect(blob.id).to eq('409f37c4f05865e4fb208c771485f211a22c4c2d') }
- it { expect(blob.data).to eq('') }
- it 'does not mark the blob as binary' do
- expect(blob).not_to be_binary
+ it { expect(blob.id).to eq('409f37c4f05865e4fb208c771485f211a22c4c2d') }
+ it { expect(blob.data).to eq('') }
+ it 'does not mark the blob as binary' do
+ expect(blob).not_to be_binary
+ end
end
- end
- context 'limiting' do
- subject { described_class.batch(repository, blob_references, blob_size_limit: blob_size_limit) }
+ context 'limiting' do
+ subject { described_class.batch(repository, blob_references, blob_size_limit: blob_size_limit) }
- context 'positive' do
- let(:blob_size_limit) { 10 }
+ context 'positive' do
+ let(:blob_size_limit) { 10 }
- it { expect(subject.first.data.size).to eq(10) }
- end
+ it { expect(subject.first.data.size).to eq(10) }
+ end
- context 'zero' do
- let(:blob_size_limit) { 0 }
+ context 'zero' do
+ let(:blob_size_limit) { 0 }
- it 'only loads the metadata' do
- expect(subject.first.size).not_to be(0)
- expect(subject.first.data).to eq('')
+ it 'only loads the metadata' do
+ expect(subject.first.size).not_to be(0)
+ expect(subject.first.data).to eq('')
+ end
end
- end
- context 'negative' do
- let(:blob_size_limit) { -1 }
+ context 'negative' do
+ let(:blob_size_limit) { -1 }
- it 'ignores MAX_DATA_DISPLAY_SIZE' do
- stub_const('Gitlab::Git::Blob::MAX_DATA_DISPLAY_SIZE', 100)
+ it 'ignores MAX_DATA_DISPLAY_SIZE' do
+ stub_const('Gitlab::Git::Blob::MAX_DATA_DISPLAY_SIZE', 100)
- expect(subject.first.data.size).to eq(669)
+ expect(subject.first.data.size).to eq(669)
+ end
end
end
end
+
+ context 'when Gitaly list_blobs_by_sha_path feature is enabled' do
+ it_behaves_like 'loading blobs in batch'
+ end
+
+ context 'when Gitaly list_blobs_by_sha_path feature is disabled', :disable_gitaly do
+ it_behaves_like 'loading blobs in batch'
+ end
end
describe '.batch_lfs_pointers' do
diff --git a/spec/lib/gitlab/gitaly_client/blobs_stitcher_spec.rb b/spec/lib/gitlab/gitaly_client/blobs_stitcher_spec.rb
new file mode 100644
index 00000000000..9db710e759e
--- /dev/null
+++ b/spec/lib/gitlab/gitaly_client/blobs_stitcher_spec.rb
@@ -0,0 +1,36 @@
+require 'spec_helper'
+
+describe Gitlab::GitalyClient::BlobsStitcher do
+ describe 'enumeration' do
+ it 'combines segregated blob messages together' do
+ messages = [
+ OpenStruct.new(oid: 'abcdef1', path: 'path/to/file', size: 1642, revision: 'f00ba7', mode: 0100644, data: "first-line\n"),
+ OpenStruct.new(oid: '', data: 'second-line'),
+ OpenStruct.new(oid: '', data: '', revision: 'f00ba7', path: 'path/to/non-existent/file'),
+ OpenStruct.new(oid: 'abcdef2', path: 'path/to/another-file', size: 2461, revision: 'f00ba8', mode: 0100644, data: "GIF87a\x90\x01".b)
+ ]
+
+ blobs = described_class.new(messages).to_a
+
+ expect(blobs.size).to be(2)
+
+ expect(blobs[0].id).to eq('abcdef1')
+ expect(blobs[0].mode).to eq('100644')
+ expect(blobs[0].name).to eq('file')
+ expect(blobs[0].path).to eq('path/to/file')
+ expect(blobs[0].size).to eq(1642)
+ expect(blobs[0].commit_id).to eq('f00ba7')
+ expect(blobs[0].data).to eq("first-line\nsecond-line")
+ expect(blobs[0].binary?).to be false
+
+ expect(blobs[1].id).to eq('abcdef2')
+ expect(blobs[1].mode).to eq('100644')
+ expect(blobs[1].name).to eq('another-file')
+ expect(blobs[1].path).to eq('path/to/another-file')
+ expect(blobs[1].size).to eq(2461)
+ expect(blobs[1].commit_id).to eq('f00ba8')
+ expect(blobs[1].data).to eq("GIF87a\x90\x01".b)
+ expect(blobs[1].binary?).to be true
+ end
+ end
+end
diff --git a/vendor/gitlab-ci-yml/Auto-DevOps.gitlab-ci.yml b/vendor/gitlab-ci-yml/Auto-DevOps.gitlab-ci.yml
index a7cd2bc972c..b2439d96e00 100644
--- a/vendor/gitlab-ci-yml/Auto-DevOps.gitlab-ci.yml
+++ b/vendor/gitlab-ci-yml/Auto-DevOps.gitlab-ci.yml
@@ -103,6 +103,7 @@ performance:
artifacts:
paths:
- performance.json
+ - sitespeed-results/
only:
refs:
- branches
@@ -503,16 +504,16 @@ production:
export CI_ENVIRONMENT_URL=$(cat environment_url.txt)
mkdir gitlab-exporter
- wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/10-3/index.js
+ wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/10-5/index.js
mkdir sitespeed-results
if [ -f .gitlab-urls.txt ]
then
sed -i -e 's@^@'"$CI_ENVIRONMENT_URL"'@' .gitlab-urls.txt
- docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.0.3 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results .gitlab-urls.txt
+ docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results .gitlab-urls.txt
else
- docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.0.3 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL"
+ docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL"
fi
mv sitespeed-results/data/performance.json performance.json