diff options
| author | Clement Ho <ClemMakesApps@gmail.com> | 2018-01-11 12:41:48 -0600 |
|---|---|---|
| committer | Clement Ho <ClemMakesApps@gmail.com> | 2018-01-11 12:41:48 -0600 |
| commit | 8a49e97bebb57eb763533b32e968547f37fa659e (patch) | |
| tree | 0475f802be1875bcfb25f664d9448e26ccf05e5a | |
| parent | f716049998ec41d8f26caedd65ebd1a247179926 (diff) | |
| parent | 685780d5b4606dce440ba8121e774fd6e7884ea2 (diff) | |
| download | gitlab-ce-8a49e97bebb57eb763533b32e968547f37fa659e.tar.gz | |
Merge branch 'master' into projects-l
222 files changed, 2749 insertions, 1462 deletions
diff --git a/GITLAB_WORKHORSE_VERSION b/GITLAB_WORKHORSE_VERSION index 18091983f59..1545d966571 100644 --- a/GITLAB_WORKHORSE_VERSION +++ b/GITLAB_WORKHORSE_VERSION @@ -1 +1 @@ -3.4.0 +3.5.0 @@ -385,9 +385,6 @@ gem 'ruby-prof', '~> 0.16.2' # OAuth gem 'oauth2', '~> 1.4' -# Soft deletion -gem 'paranoia', '~> 2.3.1' - # Health check gem 'health_check', '~> 2.6.0' diff --git a/Gemfile.lock b/Gemfile.lock index 40c4f73b8a6..cb6b0ebb3bc 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -580,8 +580,6 @@ GEM orm_adapter (0.5.0) os (0.9.6) parallel (1.12.0) - paranoia (2.3.1) - activerecord (>= 4.0, < 5.2) parser (2.4.0.2) ast (~> 2.3) parslet (1.5.0) @@ -1117,7 +1115,6 @@ DEPENDENCIES omniauth-twitter (~> 1.2.0) omniauth_crowd (~> 2.2.0) org-ruby (~> 0.9.12) - paranoia (~> 2.3.1) peek (~> 1.0.1) peek-gc (~> 0.0.2) peek-host (~> 1.0.0) diff --git a/app/assets/javascripts/dispatcher.js b/app/assets/javascripts/dispatcher.js index e8ec6daf687..1961762acdc 100644 --- a/app/assets/javascripts/dispatcher.js +++ b/app/assets/javascripts/dispatcher.js @@ -12,7 +12,6 @@ import notificationsDropdown from './notifications_dropdown'; import groupAvatar from './group_avatar'; import GroupLabelSubscription from './group_label_subscription'; import LineHighlighter from './line_highlighter'; -import groupsSelect from './groups_select'; import NewCommitForm from './new_commit_form'; import Project from './project'; import projectAvatar from './project_avatar'; @@ -40,7 +39,6 @@ import BlobLinePermalinkUpdater from './blob/blob_line_permalink_updater'; import BlobForkSuggestion from './blob/blob_fork_suggestion'; import UserCallout from './user_callout'; import ShortcutsWiki from './shortcuts_wiki'; -import Pipelines from './pipelines'; import BlobViewer from './blob/viewer/index'; import AutoWidthDropdownSelect from './issuable/auto_width_dropdown_select'; import UsersSelect from './users_select'; @@ -49,11 +47,9 @@ import GfmAutoComplete from './gfm_auto_complete'; import ShortcutsBlob from './shortcuts_blob'; import Star from './star'; import TreeView from './tree'; -import VersionCheckImage from './version_check_image'; import Wikis from './wikis'; import ZenMode from './zen_mode'; import initSettingsPanels from './settings_panels'; -import initExperimentalFlags from './experimental_flags'; import PerformanceBar from './performance_bar'; import initNotes from './init_notes'; import initLegacyFilters from './init_legacy_filters'; @@ -145,9 +141,6 @@ import Activities from './activities'; const filteredSearchEnabled = gl.FilteredSearchManager && document.querySelector('.filtered-search'); switch (page) { - case 'profiles:preferences:show': - initExperimentalFlags(); - break; case 'sessions:new': import('./pages/sessions/new') .then(callDefault) @@ -390,23 +383,16 @@ import Activities from './activities'; break; case 'projects:pipelines:new': case 'projects:pipelines:create': - new NewBranchForm($('.js-new-pipeline-form')); + import('./pages/projects/pipelines/new') + .then(callDefault) + .catch(fail); break; case 'projects:pipelines:builds': case 'projects:pipelines:failures': case 'projects:pipelines:show': - const { controllerAction } = document.querySelector('.js-pipeline-container').dataset; - const pipelineStatusUrl = `${document.querySelector('.js-pipeline-tab-link a').getAttribute('href')}/status.json`; - - new Pipelines({ - initTabs: true, - pipelineStatusUrl, - tabsOptions: { - action: controllerAction, - defaultAction: 'pipelines', - parentEl: '.pipelines-tabs', - }, - }); + import('./pages/projects/pipelines/builds') + .then(callDefault) + .catch(fail); break; case 'groups:activity': new Activities(); @@ -428,11 +414,9 @@ import Activities from './activities'; new UsersSelect(); break; case 'projects:project_members:index': - memberExpirationDate('.js-access-expiration-date-groups'); - groupsSelect(); - memberExpirationDate(); - new Members(); - new UsersSelect(); + import('./pages/projects/project_members/') + .then(callDefault) + .catch(fail); break; case 'groups:new': case 'groups:create': @@ -536,7 +520,9 @@ import Activities from './activities'; shortcut_handler = true; break; case 'help:index': - VersionCheckImage.bindErrorEvent($('img.js-version-status-badge')); + import('./pages/help') + .then(callDefault) + .catch(fail); break; case 'search:show': import('./pages/search/show') @@ -579,14 +565,16 @@ import Activities from './activities'; case 'import:fogbugz:new_user_map': import('./pages/import/fogbugz/new_user_map').then(m => m.default()).catch(fail); break; + case 'profiles:personal_access_tokens:index': + import('./pages/profiles/personal_access_tokens') + .then(callDefault) + .catch(fail); + break; case 'admin:impersonation_tokens:index': import('./pages/admin/impersonation_tokens') .then(callDefault) .catch(fail); break; - case 'profiles:personal_access_tokens:index': - new DueDateSelectors(); - break; case 'projects:clusters:show': import(/* webpackChunkName: "clusters" */ './clusters/clusters_bundle') .then(cluster => new cluster.default()) // eslint-disable-line new-cap @@ -672,8 +660,9 @@ import Activities from './activities'; new UserCallout(); break; case 'profiles': - new NotificationsForm(); - notificationsDropdown(); + import('./pages/profiles/index/') + .then(callDefault) + .catch(fail); break; case 'projects': new Project(); diff --git a/app/assets/javascripts/pages/help/index.js b/app/assets/javascripts/pages/help/index.js new file mode 100644 index 00000000000..4cf8afc4b7e --- /dev/null +++ b/app/assets/javascripts/pages/help/index.js @@ -0,0 +1,3 @@ +import VersionCheckImage from '../../version_check_image'; + +export default () => VersionCheckImage.bindErrorEvent($('img.js-version-status-badge')); diff --git a/app/assets/javascripts/pages/profiles/index/index.js b/app/assets/javascripts/pages/profiles/index/index.js new file mode 100644 index 00000000000..90eed38777a --- /dev/null +++ b/app/assets/javascripts/pages/profiles/index/index.js @@ -0,0 +1,7 @@ +import NotificationsForm from '../../../notifications_form'; +import notificationsDropdown from '../../../notifications_dropdown'; + +export default () => { + new NotificationsForm(); // eslint-disable-line no-new + notificationsDropdown(); +}; diff --git a/app/assets/javascripts/pages/profiles/personal_access_tokens/index.js b/app/assets/javascripts/pages/profiles/personal_access_tokens/index.js new file mode 100644 index 00000000000..030328a1363 --- /dev/null +++ b/app/assets/javascripts/pages/profiles/personal_access_tokens/index.js @@ -0,0 +1,3 @@ +import DueDateSelectors from '../../../due_date_select'; + +export default () => new DueDateSelectors(); diff --git a/app/assets/javascripts/pages/projects/pipelines/builds/index.js b/app/assets/javascripts/pages/projects/pipelines/builds/index.js new file mode 100644 index 00000000000..060a78b427e --- /dev/null +++ b/app/assets/javascripts/pages/projects/pipelines/builds/index.js @@ -0,0 +1,16 @@ +import Pipelines from '../../../../pipelines'; + +export default () => { + const { controllerAction } = document.querySelector('.js-pipeline-container').dataset; + const pipelineStatusUrl = `${document.querySelector('.js-pipeline-tab-link a').getAttribute('href')}/status.json`; + + new Pipelines({ // eslint-disable-line no-new + initTabs: true, + pipelineStatusUrl, + tabsOptions: { + action: controllerAction, + defaultAction: 'pipelines', + parentEl: '.pipelines-tabs', + }, + }); +}; diff --git a/app/assets/javascripts/pages/projects/pipelines/new/index.js b/app/assets/javascripts/pages/projects/pipelines/new/index.js new file mode 100644 index 00000000000..c54cc62bf05 --- /dev/null +++ b/app/assets/javascripts/pages/projects/pipelines/new/index.js @@ -0,0 +1,5 @@ +import NewBranchForm from '../../../../new_branch_form'; + +export default () => { + new NewBranchForm($('.js-new-pipeline-form')); // eslint-disable-line no-new +}; diff --git a/app/assets/javascripts/pages/projects/project_members/index.js b/app/assets/javascripts/pages/projects/project_members/index.js new file mode 100644 index 00000000000..f4643e7dba0 --- /dev/null +++ b/app/assets/javascripts/pages/projects/project_members/index.js @@ -0,0 +1,12 @@ +import memberExpirationDate from '../../../member_expiration_date'; +import UsersSelect from '../../../users_select'; +import groupsSelect from '../../../groups_select'; +import Members from '../../../members'; + +export default () => { + memberExpirationDate('.js-access-expiration-date-groups'); + groupsSelect(); + memberExpirationDate(); + new Members(); // eslint-disable-line no-new + new UsersSelect(); // eslint-disable-line no-new +}; diff --git a/app/assets/javascripts/vue_shared/components/clipboard_button.vue b/app/assets/javascripts/vue_shared/components/clipboard_button.vue index e18852af6e9..31d9b9d9c48 100644 --- a/app/assets/javascripts/vue_shared/components/clipboard_button.vue +++ b/app/assets/javascripts/vue_shared/components/clipboard_button.vue @@ -1,10 +1,14 @@ <script> + import tooltip from '../directives/tooltip'; /** * Falls back to the code used in `copy_to_clipboard.js` */ export default { name: 'ClipboardButton', + directives: { + tooltip, + }, props: { text: { type: String, @@ -14,6 +18,16 @@ type: String, required: true, }, + tooltipPlacement: { + type: String, + required: false, + default: 'top', + }, + tooltipContainer: { + type: [String, Boolean], + required: false, + default: false, + }, }, }; </script> @@ -22,8 +36,11 @@ <button type="button" class="btn btn-transparent btn-clipboard" - :data-title="title" + :title="title" :data-clipboard-text="text" + v-tooltip + :data-container="tooltipContainer" + :data-placement="tooltipPlacement" > <i aria-hidden="true" diff --git a/app/controllers/admin/runners_controller.rb b/app/controllers/admin/runners_controller.rb index 38b808cdc31..4b01904f2a1 100644 --- a/app/controllers/admin/runners_controller.rb +++ b/app/controllers/admin/runners_controller.rb @@ -65,6 +65,7 @@ class Admin::RunnersController < Admin::ApplicationController else Project.all end + @projects = @projects.where.not(id: runner.projects.select(:id)) if runner.projects.any? @projects = @projects.page(params[:page]).per(30) end diff --git a/app/controllers/concerns/group_tree.rb b/app/controllers/concerns/group_tree.rb index b10147835f3..b569029283f 100644 --- a/app/controllers/concerns/group_tree.rb +++ b/app/controllers/concerns/group_tree.rb @@ -8,6 +8,7 @@ module GroupTree # Only show root groups if no parent-id is given groups.where(parent_id: params[:parent_id]) end + @groups = @groups.with_selects_for_list(archived: params[:archived]) .sort(@sort = params[:sort]) .page(params[:page]) diff --git a/app/controllers/concerns/routable_actions.rb b/app/controllers/concerns/routable_actions.rb index 4199da9cdf5..f745deb083c 100644 --- a/app/controllers/concerns/routable_actions.rb +++ b/app/controllers/concerns/routable_actions.rb @@ -32,6 +32,7 @@ module RoutableActions if canonical_path.casecmp(requested_full_path) != 0 flash[:notice] = "#{routable.class.to_s.titleize} '#{requested_full_path}' was moved to '#{canonical_path}'. Please update any links and bookmarks that may still have the old path." end + redirect_to build_canonical_path(routable) end end diff --git a/app/controllers/metrics_controller.rb b/app/controllers/metrics_controller.rb index d81ad135198..33b682d2859 100644 --- a/app/controllers/metrics_controller.rb +++ b/app/controllers/metrics_controller.rb @@ -12,6 +12,7 @@ class MetricsController < ActionController::Base ) "# Metrics are disabled, see: #{help_page}\n" end + render text: response, content_type: 'text/plain; version=0.0.4' end diff --git a/app/controllers/omniauth_callbacks_controller.rb b/app/controllers/omniauth_callbacks_controller.rb index e3c18cba1dd..689d2e3db22 100644 --- a/app/controllers/omniauth_callbacks_controller.rb +++ b/app/controllers/omniauth_callbacks_controller.rb @@ -83,6 +83,7 @@ class OmniauthCallbacksController < Devise::OmniauthCallbacksController if ticket handle_service_ticket oauth['provider'], ticket end + handle_omniauth end @@ -90,6 +91,7 @@ class OmniauthCallbacksController < Devise::OmniauthCallbacksController if params['sid'] handle_service_ticket oauth['provider'], params['sid'] end + handle_omniauth end @@ -124,6 +126,7 @@ class OmniauthCallbacksController < Devise::OmniauthCallbacksController # Only allow properly saved users to login. if @user.persisted? && @user.valid? log_audit_event(@user, with: oauth['provider']) + if @user.two_factor_enabled? params[:remember_me] = '1' if remember_me? prompt_for_two_factor(@user) diff --git a/app/controllers/projects/blob_controller.rb b/app/controllers/projects/blob_controller.rb index d838b8dc29e..35e67730a27 100644 --- a/app/controllers/projects/blob_controller.rb +++ b/app/controllers/projects/blob_controller.rb @@ -150,6 +150,7 @@ class Projects::BlobController < Projects::ApplicationController if params[:file].present? params[:file_name] = params[:file].original_filename end + File.join(@path, params[:file_name]) elsif params[:file_path].present? params[:file_path] diff --git a/app/controllers/projects/deploy_keys_controller.rb b/app/controllers/projects/deploy_keys_controller.rb index cf8829ba95b..e06dda1baa4 100644 --- a/app/controllers/projects/deploy_keys_controller.rb +++ b/app/controllers/projects/deploy_keys_controller.rb @@ -27,6 +27,7 @@ class Projects::DeployKeysController < Projects::ApplicationController unless @key.valid? && @project.deploy_keys << @key flash[:alert] = @key.errors.full_messages.join(', ').html_safe end + redirect_to_repository_settings(@project) end diff --git a/app/controllers/projects/hooks_controller.rb b/app/controllers/projects/hooks_controller.rb index 85d35900c71..6f51e7b9b40 100644 --- a/app/controllers/projects/hooks_controller.rb +++ b/app/controllers/projects/hooks_controller.rb @@ -21,6 +21,7 @@ class Projects::HooksController < Projects::ApplicationController @hooks = @project.hooks.select(&:persisted?) flash[:alert] = @hook.errors.full_messages.join.html_safe end + redirect_to project_settings_integrations_path(@project) end diff --git a/app/controllers/projects/merge_requests/creations_controller.rb b/app/controllers/projects/merge_requests/creations_controller.rb index dc524b790a0..3d2926d5d75 100644 --- a/app/controllers/projects/merge_requests/creations_controller.rb +++ b/app/controllers/projects/merge_requests/creations_controller.rb @@ -48,6 +48,7 @@ class Projects::MergeRequests::CreationsController < Projects::MergeRequests::Ap else [] end + @diff_notes_disabled = true @environment = @merge_request.environments_for(current_user).last diff --git a/app/controllers/projects_controller.rb b/app/controllers/projects_controller.rb index 6f229b08c0c..e6e2b219e6a 100644 --- a/app/controllers/projects_controller.rb +++ b/app/controllers/projects_controller.rb @@ -203,6 +203,7 @@ class ProjectsController < Projects::ApplicationController else flash[:alert] = _("Project export could not be deleted.") end + redirect_to(edit_project_path(@project)) end diff --git a/app/controllers/sessions_controller.rb b/app/controllers/sessions_controller.rb index d79108c88fb..c73306a6b66 100644 --- a/app/controllers/sessions_controller.rb +++ b/app/controllers/sessions_controller.rb @@ -28,6 +28,7 @@ class SessionsController < Devise::SessionsController resource.update_attributes(reset_password_token: nil, reset_password_sent_at: nil) end + # hide the signed-in notification flash[:notice] = nil log_audit_event(current_user, resource, with: authentication_method) diff --git a/app/finders/group_descendants_finder.rb b/app/finders/group_descendants_finder.rb index 1a5f6063437..58570a580f1 100644 --- a/app/finders/group_descendants_finder.rb +++ b/app/finders/group_descendants_finder.rb @@ -63,6 +63,7 @@ class GroupDescendantsFinder groups_table = Group.arel_table visible_to_user = groups_table[:visibility_level] .in(Gitlab::VisibilityLevel.levels_for_user(current_user)) + if current_user authorized_groups = GroupsFinder.new(current_user, all_available: false) @@ -115,6 +116,7 @@ class GroupDescendantsFinder else direct_child_groups end + groups.with_selects_for_list(archived: params[:archived]).order_by(sort) end @@ -140,6 +142,7 @@ class GroupDescendantsFinder else direct_child_projects end + projects.with_route.order_by(sort) end diff --git a/app/finders/group_projects_finder.rb b/app/finders/group_projects_finder.rb index 6e8733bb49c..f2d3b90b8e2 100644 --- a/app/finders/group_projects_finder.rb +++ b/app/finders/group_projects_finder.rb @@ -34,6 +34,7 @@ class GroupProjectsFinder < ProjectsFinder else collection_without_user end + union(projects) end diff --git a/app/helpers/markup_helper.rb b/app/helpers/markup_helper.rb index f78d41a0448..2fe1927a189 100644 --- a/app/helpers/markup_helper.rb +++ b/app/helpers/markup_helper.rb @@ -203,6 +203,7 @@ module MarkupHelper node.content = node.content.truncate(num_remaining) truncated = true end + content_length += node.content.length end diff --git a/app/helpers/nav_helper.rb b/app/helpers/nav_helper.rb index 8ada746b244..680ea96a556 100644 --- a/app/helpers/nav_helper.rb +++ b/app/helpers/nav_helper.rb @@ -12,6 +12,7 @@ module NavHelper current_path?('projects/merge_requests/conflicts#show') || current_path?('issues#show') || current_path?('milestones#show') + if cookies[:collapsed_gutter] == 'true' %w[page-gutter right-sidebar-collapsed] else diff --git a/app/helpers/snippets_helper.rb b/app/helpers/snippets_helper.rb index b447d4952e7..00e7e4230b9 100644 --- a/app/helpers/snippets_helper.rb +++ b/app/helpers/snippets_helper.rb @@ -89,6 +89,7 @@ module SnippetsHelper snippet_chunk = [lined_content[line_number]] snippet_start_line = line_number end + last_line = line_number end # Add final chunk to chunk array diff --git a/app/helpers/submodule_helper.rb b/app/helpers/submodule_helper.rb index 40d69e30188..1db9ae3839c 100644 --- a/app/helpers/submodule_helper.rb +++ b/app/helpers/submodule_helper.rb @@ -58,6 +58,7 @@ module SubmoduleHelper url_no_dotgit = url.chomp('.git') return true if url_no_dotgit == [Gitlab.config.gitlab.url, '/', namespace, '/', project].join('') + url_with_dotgit = url_no_dotgit + '.git' url_with_dotgit == Gitlab::Shell.new.url_to_repo([namespace, '/', project].join('')) end diff --git a/app/helpers/todos_helper.rb b/app/helpers/todos_helper.rb index 2a7aa299e83..e7c953e749e 100644 --- a/app/helpers/todos_helper.rb +++ b/app/helpers/todos_helper.rb @@ -30,6 +30,7 @@ module TodosHelper else todo.target_reference end + link_to text, todo_target_path(todo), class: 'has-tooltip', title: todo.target.title end diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 8ab338d873d..80bda7f22ff 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -418,6 +418,7 @@ class ApplicationSetting < ActiveRecord::Base super(group_full_path) Gitlab::PerformanceBar.expire_allowed_user_ids_cache end + return end diff --git a/app/models/ci/pipeline_schedule.rb b/app/models/ci/pipeline_schedule.rb index 10ead6b6d3b..b6abc3d7681 100644 --- a/app/models/ci/pipeline_schedule.rb +++ b/app/models/ci/pipeline_schedule.rb @@ -2,8 +2,9 @@ module Ci class PipelineSchedule < ActiveRecord::Base extend Gitlab::Ci::Model include Importable + include IgnorableColumn - acts_as_paranoid + ignore_column :deleted_at belongs_to :project belongs_to :owner, class_name: 'User' diff --git a/app/models/ci/trigger.rb b/app/models/ci/trigger.rb index b5290bcaf53..aa065e33739 100644 --- a/app/models/ci/trigger.rb +++ b/app/models/ci/trigger.rb @@ -1,8 +1,9 @@ module Ci class Trigger < ActiveRecord::Base extend Gitlab::Ci::Model + include IgnorableColumn - acts_as_paranoid + ignore_column :deleted_at belongs_to :project belongs_to :owner, class_name: "User" diff --git a/app/models/concerns/internal_id.rb b/app/models/concerns/internal_id.rb index a3d0ac8d862..01079fb8bd6 100644 --- a/app/models/concerns/internal_id.rb +++ b/app/models/concerns/internal_id.rb @@ -10,7 +10,6 @@ module InternalId if iid.blank? parent = project || group records = parent.public_send(self.class.name.tableize) # rubocop:disable GitlabSecurity/PublicSend - records = records.with_deleted if self.paranoid? max_iid = records.maximum(:iid) self.iid = max_iid.to_i + 1 diff --git a/app/models/concerns/issuable.rb b/app/models/concerns/issuable.rb index 4251561a0a0..7049f340c9d 100644 --- a/app/models/concerns/issuable.rb +++ b/app/models/concerns/issuable.rb @@ -314,6 +314,7 @@ module Issuable includes = [] includes << :author unless notes.authors_loaded? includes << :award_emoji unless notes.award_emojis_loaded? + if includes.any? notes.includes(includes) else diff --git a/app/models/concerns/loaded_in_group_list.rb b/app/models/concerns/loaded_in_group_list.rb index dcb3b2b5ff3..935e9d10133 100644 --- a/app/models/concerns/loaded_in_group_list.rb +++ b/app/models/concerns/loaded_in_group_list.rb @@ -25,6 +25,7 @@ module LoadedInGroupList base_count = projects.project(Arel.star.count.as('preloaded_project_count')) .where(projects[:namespace_id].eq(namespaces[:id])) + if archived == 'only' base_count.where(projects[:archived].eq(true)) elsif Gitlab::Utils.to_boolean(archived) diff --git a/app/models/issue.rb b/app/models/issue.rb index ad4a3c737ff..93628b456f2 100644 --- a/app/models/issue.rb +++ b/app/models/issue.rb @@ -12,7 +12,7 @@ class Issue < ActiveRecord::Base include ThrottledTouch include IgnorableColumn - ignore_column :assignee_id, :branch_name + ignore_column :assignee_id, :branch_name, :deleted_at DueDateStruct = Struct.new(:title, :name).freeze NoDueDate = DueDateStruct.new('No Due Date', '0').freeze @@ -78,8 +78,6 @@ class Issue < ActiveRecord::Base end end - acts_as_paranoid - class << self alias_method :in_parents, :in_projects end diff --git a/app/models/label.rb b/app/models/label.rb index b5bfa6ea2dd..7538f2d8718 100644 --- a/app/models/label.rb +++ b/app/models/label.rb @@ -132,6 +132,7 @@ class Label < ActiveRecord::Base else priorities.find_by(project: project) end + priority.try(:priority) end diff --git a/app/models/merge_request.rb b/app/models/merge_request.rb index ef58816937c..2669d2a6ff3 100644 --- a/app/models/merge_request.rb +++ b/app/models/merge_request.rb @@ -11,7 +11,8 @@ class MergeRequest < ActiveRecord::Base include Gitlab::Utils::StrongMemoize ignore_column :locked_at, - :ref_fetched + :ref_fetched, + :deleted_at belongs_to :target_project, class_name: "Project" belongs_to :source_project, class_name: "Project" @@ -150,8 +151,6 @@ class MergeRequest < ActiveRecord::Base after_save :keep_around_commit - acts_as_paranoid - def self.reference_prefix '!' end @@ -794,6 +793,7 @@ class MergeRequest < ActiveRecord::Base if !include_description && closes_issues_references.present? message << "Closes #{closes_issues_references.to_sentence}" end + message << "#{description}" if include_description && description.present? message << "See merge request #{to_reference(full: true)}" diff --git a/app/models/namespace.rb b/app/models/namespace.rb index bdcc9159d26..37a7417cafc 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -1,6 +1,4 @@ class Namespace < ActiveRecord::Base - acts_as_paranoid without_default_scope: true - include CacheMarkdownField include Sortable include Gitlab::ShellAdapter @@ -10,6 +8,9 @@ class Namespace < ActiveRecord::Base include AfterCommitQueue include Storage::LegacyNamespace include Gitlab::SQL::Pattern + include IgnorableColumn + + ignore_column :deleted_at # Prevent users from creating unreasonably deep level of nesting. # The number 20 was taken based on maximum nesting level of @@ -221,12 +222,6 @@ class Namespace < ActiveRecord::Base has_parent? end - def soft_delete_without_removing_associations - # We can't use paranoia's `#destroy` since this will hard-delete projects. - # Project uses `pending_delete` instead of the acts_as_paranoia gem. - self.deleted_at = Time.now - end - private def refresh_access_of_projects_invited_groups diff --git a/app/models/network/graph.rb b/app/models/network/graph.rb index aec7b01e23a..c351d2012c6 100644 --- a/app/models/network/graph.rb +++ b/app/models/network/graph.rb @@ -224,6 +224,7 @@ module Network space_base = parents.first.space end end + space_base end diff --git a/app/models/notification_recipient.rb b/app/models/notification_recipient.rb index 183e098d819..ab5a96209c7 100644 --- a/app/models/notification_recipient.rb +++ b/app/models/notification_recipient.rb @@ -9,6 +9,7 @@ class NotificationRecipient group: nil, skip_read_ability: false ) + unless NotificationSetting.levels.key?(type) || type == :subscription raise ArgumentError, "invalid type: #{type.inspect}" end diff --git a/app/models/project.rb b/app/models/project.rb index 7dc5e980c1b..029f2da2e4e 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -633,6 +633,7 @@ class Project < ActiveRecord::Base project_import_data.data ||= {} project_import_data.data = project_import_data.data.merge(data) end + if credentials project_import_data.credentials ||= {} project_import_data.credentials = project_import_data.credentials.merge(credentials) diff --git a/app/models/project_services/hipchat_service.rb b/app/models/project_services/hipchat_service.rb index 768f0a7472e..bfe7ac29c18 100644 --- a/app/models/project_services/hipchat_service.rb +++ b/app/models/project_services/hipchat_service.rb @@ -110,6 +110,7 @@ class HipchatService < Service message = "" message << "#{push[:user_name]} " + if Gitlab::Git.blank_ref?(before) message << "pushed new #{ref_type} <a href=\""\ "#{project_url}/commits/#{CGI.escape(ref)}\">#{ref}</a>"\ diff --git a/app/models/repository.rb b/app/models/repository.rb index a84d6a1426b..d27212b2058 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -1014,6 +1014,7 @@ class Repository else cache.fetch(key, &block) end + instance_variable_set(ivar, value) rescue Rugged::ReferenceError, Gitlab::Git::Repository::NoRepository # Even if the above `#exists?` check passes these errors might still diff --git a/app/models/service.rb b/app/models/service.rb index 24ba3039707..7f260f7a96b 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -250,6 +250,7 @@ class Service < ActiveRecord::Base teamcity microsoft_teams ] + if Rails.env.development? service_names += %w[mock_ci mock_deployment mock_monitoring] end diff --git a/app/serializers/issue_entity.rb b/app/serializers/issue_entity.rb index 0bdd4d7a272..b5e2334b6e3 100644 --- a/app/serializers/issue_entity.rb +++ b/app/serializers/issue_entity.rb @@ -6,7 +6,6 @@ class IssueEntity < IssuableEntity expose :updated_by_id expose :created_at expose :updated_at - expose :deleted_at expose :milestone, using: API::Entities::Milestone expose :labels, using: LabelEntity expose :lock_version diff --git a/app/services/create_deployment_service.rb b/app/services/create_deployment_service.rb index 63b85c3de7d..88dfb7a4a90 100644 --- a/app/services/create_deployment_service.rb +++ b/app/services/create_deployment_service.rb @@ -16,6 +16,7 @@ class CreateDeploymentService ActiveRecord::Base.transaction do environment.external_url = expanded_environment_url if expanded_environment_url + environment.fire_state_event(action) return unless environment.save diff --git a/app/services/groups/destroy_service.rb b/app/services/groups/destroy_service.rb index e3f9d9ee95d..58e88688dfa 100644 --- a/app/services/groups/destroy_service.rb +++ b/app/services/groups/destroy_service.rb @@ -1,7 +1,6 @@ module Groups class DestroyService < Groups::BaseService def async_execute - group.soft_delete_without_removing_associations job_id = GroupDestroyWorker.perform_async(group.id, current_user.id) Rails.logger.info("User #{current_user.id} scheduled a deletion of group ID #{group.id} with job ID #{job_id}") end @@ -23,7 +22,7 @@ module Groups group.chat_team&.remove_mattermost_team(current_user) - group.really_destroy! + group.destroy end end end diff --git a/app/services/users/destroy_service.rb b/app/services/users/destroy_service.rb index 00db8a2c434..b71002433d6 100644 --- a/app/services/users/destroy_service.rb +++ b/app/services/users/destroy_service.rb @@ -53,7 +53,7 @@ module Users # Destroy the namespace after destroying the user since certain methods may depend on the namespace existing user_data = user.destroy - namespace.really_destroy! + namespace.destroy user_data end diff --git a/app/views/projects/_home_panel.html.haml b/app/views/projects/_home_panel.html.haml index 1d644dda177..b565f14747a 100644 --- a/app/views/projects/_home_panel.html.haml +++ b/app/views/projects/_home_panel.html.haml @@ -4,7 +4,7 @@ .limit-container-width{ class: container_class } .avatar-container.s70.project-avatar = project_icon(@project, alt: @project.name, class: 'avatar s70 avatar-tile') - %h1.project-title + %h1.project-title.qa-project-name = @project.name %span.visibility-icon.has-tooltip{ data: { container: 'body' }, title: visibility_icon_description(@project) } = visibility_level_icon(@project.visibility_level, fw: false) diff --git a/app/views/projects/_new_project_fields.html.haml b/app/views/projects/_new_project_fields.html.haml index a78a8e5d628..bd99eb93cc8 100644 --- a/app/views/projects/_new_project_fields.html.haml +++ b/app/views/projects/_new_project_fields.html.haml @@ -9,7 +9,7 @@ - if current_user.can_select_namespace? .input-group-addon = root_url - = f.select :namespace_id, namespaces_options(namespace_id_from(params) || :current_user, display_path: true, extra_group: namespace_id_from(params)), {}, { class: 'select2 js-select-namespace', tabindex: 1} + = f.select :namespace_id, namespaces_options(namespace_id_from(params) || :current_user, display_path: true, extra_group: namespace_id_from(params)), {}, { class: 'select2 js-select-namespace qa-project-namespace-select', tabindex: 1} - else .input-group-addon.static-namespace diff --git a/app/views/projects/buttons/_dropdown.html.haml b/app/views/projects/buttons/_dropdown.html.haml index 2589c53beae..8e8c911185a 100644 --- a/app/views/projects/buttons/_dropdown.html.haml +++ b/app/views/projects/buttons/_dropdown.html.haml @@ -30,12 +30,13 @@ %li = link_to project_new_blob_path(@project, @project.default_branch || 'master') do #{ _('New file') } - %li - = link_to new_project_branch_path(@project) do - #{ _('New branch') } - %li - = link_to new_project_tag_path(@project) do - #{ _('New tag') } + - unless @project.empty_repo? + %li + = link_to new_project_branch_path(@project) do + #{ _('New branch') } + %li + = link_to new_project_tag_path(@project) do + #{ _('New tag') } - elsif current_user && current_user.already_forked?(@project) %li = link_to project_new_blob_path(@project, @project.default_branch || 'master') do diff --git a/app/views/projects/milestones/show.html.haml b/app/views/projects/milestones/show.html.haml index 9fc297ab7f6..5dd4d2c949c 100644 --- a/app/views/projects/milestones/show.html.haml +++ b/app/views/projects/milestones/show.html.haml @@ -27,7 +27,7 @@ Edit - if @project.group - = link_to promote_project_milestone_path(@milestone.project, @milestone), title: "Promote to Group Milestone", class: 'btn btn-grouped', data: { confirm: "Promoting this milestone will make it available for all projects inside the group. Existing project milestones with the same name will be merged. Are you sure?", toggle: "tooltip" }, method: :post do + = link_to promote_project_milestone_path(@milestone.project, @milestone), title: "Promote to Group Milestone", class: 'btn btn-grouped', data: { confirm: "You are about to promote #{@milestone.title} to a group level. This will make this milestone available to all projects inside #{@project.group.name}. The existing project milestone will be merged into the group level. This action cannot be reversed.", toggle: "tooltip" }, method: :post do Promote - if @milestone.active? diff --git a/app/views/shared/_clone_panel.html.haml b/app/views/shared/_clone_panel.html.haml index 1cba4fc6c41..687cd4d1532 100644 --- a/app/views/shared/_clone_panel.html.haml +++ b/app/views/shared/_clone_panel.html.haml @@ -7,7 +7,7 @@ %span = enabled_project_button(project, enabled_protocol) - else - %a#clone-dropdown.btn.clone-dropdown-btn{ href: '#', data: { toggle: 'dropdown' } } + %a#clone-dropdown.btn.clone-dropdown-btn.qa-clone-dropdown{ href: '#', data: { toggle: 'dropdown' } } %span = default_clone_protocol.upcase = icon('caret-down') diff --git a/app/views/shared/_label.html.haml b/app/views/shared/_label.html.haml index 81d07074325..8e88cecaf9e 100644 --- a/app/views/shared/_label.html.haml +++ b/app/views/shared/_label.html.haml @@ -77,7 +77,7 @@ = icon('spinner spin', class: 'label-subscribe-button-loading') - if label.is_a?(ProjectLabel) && label.project.group && can?(current_user, :admin_label, label.project.group) - = link_to promote_project_label_path(label.project, label), title: "Promote to Group Label", class: 'btn btn-transparent btn-action', data: {confirm: "Promoting this label will make this label available to all projects inside this group. Existing project labels with the same name will be merged. Are you sure?", toggle: "tooltip"}, method: :post do + = link_to promote_project_label_path(label.project, label), title: "Promote to Group Label", class: 'btn btn-transparent btn-action', data: {confirm: "You are about to promote #{label.title} to a group level. This will make this milestone available to all projects inside #{label.project.group.name}. The existing project label will be merged into the group level. This action cannot be reversed.", toggle: "tooltip"}, method: :post do %span.sr-only Promote to Group = icon('level-up') - if can?(current_user, :admin_label, label) diff --git a/app/views/shared/milestones/_milestone.html.haml b/app/views/shared/milestones/_milestone.html.haml index 7ba8f9d4313..50f4901a2dd 100644 --- a/app/views/shared/milestones/_milestone.html.haml +++ b/app/views/shared/milestones/_milestone.html.haml @@ -51,7 +51,7 @@ \ - if @project.group - = link_to promote_project_milestone_path(milestone.project, milestone), title: "Promote to Group Milestone", class: 'btn btn-xs btn-grouped', data: { confirm: "Promoting this milestone will make it available for all projects inside the group. Existing project milestones with the same name will be merged. Are you sure?", toggle: "tooltip" }, method: :post do + = link_to promote_project_milestone_path(milestone.project, milestone), title: "Promote to Group Milestone", class: 'btn btn-xs btn-grouped', data: { confirm: "You are about to promote #{milestone.title} to a group level. This will make this milestone available to all projects inside #{@project.group.name}. The existing project milestone will be merged into the group level. This action cannot be reversed.", toggle: "tooltip" }, method: :post do Promote = link_to 'Close Milestone', project_milestone_path(@project, milestone, milestone: {state_event: :close }), method: :put, remote: true, class: "btn btn-xs btn-close btn-grouped" diff --git a/app/workers/group_destroy_worker.rb b/app/workers/group_destroy_worker.rb index f577b310b20..509bd09dc2e 100644 --- a/app/workers/group_destroy_worker.rb +++ b/app/workers/group_destroy_worker.rb @@ -4,7 +4,7 @@ class GroupDestroyWorker def perform(group_id, user_id) begin - group = Group.with_deleted.find(group_id) + group = Group.find(group_id) rescue ActiveRecord::RecordNotFound return end diff --git a/app/workers/pages_worker.rb b/app/workers/pages_worker.rb index 3ec81d040b4..d3b95009364 100644 --- a/app/workers/pages_worker.rb +++ b/app/workers/pages_worker.rb @@ -13,6 +13,7 @@ class PagesWorker if result[:status] == :success result = Projects::UpdatePagesConfigurationService.new(build.project).execute end + result end diff --git a/changelogs/unreleased/18040-line-breaks-around-conditional-blocks.yml b/changelogs/unreleased/18040-line-breaks-around-conditional-blocks.yml new file mode 100644 index 00000000000..447c65a3764 --- /dev/null +++ b/changelogs/unreleased/18040-line-breaks-around-conditional-blocks.yml @@ -0,0 +1,5 @@ +--- +title: Adds Rubocop rule for line break around conditionals +merge_request: 15739 +author: Jacopo Beschi @jacopo-beschi +type: added diff --git a/changelogs/unreleased/39988-hide-new-branch-tag-empty-repo.yml b/changelogs/unreleased/39988-hide-new-branch-tag-empty-repo.yml new file mode 100644 index 00000000000..4f2c87c44b3 --- /dev/null +++ b/changelogs/unreleased/39988-hide-new-branch-tag-empty-repo.yml @@ -0,0 +1,5 @@ +--- +title: Hide new branch and tag links for projects with an empty repo +merge_request: +author: +type: fixed diff --git a/changelogs/unreleased/remove-soft-removals.yml b/changelogs/unreleased/remove-soft-removals.yml new file mode 100644 index 00000000000..aa53d33e502 --- /dev/null +++ b/changelogs/unreleased/remove-soft-removals.yml @@ -0,0 +1,5 @@ +--- +title: Remove soft removals related code +merge_request: 15789 +author: +type: changed diff --git a/config/application.rb b/config/application.rb index 1110199b888..ea9a07cbde9 100644 --- a/config/application.rb +++ b/config/application.rb @@ -61,6 +61,7 @@ module Gitlab # - Any parameter containing `secret` # - Two-factor tokens (:otp_attempt) # - Repo/Project Import URLs (:import_url) + # - Build traces (:trace) # - Build variables (:variables) # - GitLab Pages SSL cert/key info (:certificate, :encrypted_key) # - Webhook URLs (:hook) @@ -75,6 +76,7 @@ module Gitlab key otp_attempt sentry_dsn + trace variables ) @@ -149,6 +151,7 @@ module Gitlab caching_config_hash[:pool_size] = Sidekiq.options[:concurrency] + 5 caching_config_hash[:pool_timeout] = 1 end + config.cache_store = :redis_store, caching_config_hash config.active_record.raise_in_transactional_callbacks = true diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index f10f0cdf42c..abc992e49dc 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -68,6 +68,7 @@ class Settings < Settingslogic end values.delete_if { |value| value.nil? } end + values end @@ -78,6 +79,7 @@ class Settings < Settingslogic if current.is_a? String value = modul.const_get(current.upcase) rescue default end + value end diff --git a/config/initializers/devise.rb b/config/initializers/devise.rb index 051ef93b205..fa25f3778fa 100644 --- a/config/initializers/devise.rb +++ b/config/initializers/devise.rb @@ -241,6 +241,7 @@ Devise.setup do |config| true end end + if provider['name'] == 'authentiq' provider['args'][:remote_sign_out_handler] = lambda do |request| authentiq_session = request.params['sid'] diff --git a/config/initializers/peek.rb b/config/initializers/peek.rb index 581397b26f8..e74b95f1646 100644 --- a/config/initializers/peek.rb +++ b/config/initializers/peek.rb @@ -2,6 +2,7 @@ Rails.application.config.peek.adapter = :redis, { client: ::Redis.new(Gitlab::Re Peek.into Peek::Views::Host Peek.into Peek::Views::PerformanceBar + if Gitlab::Database.mysql? require 'peek-mysql2' PEEK_DB_CLIENT = ::Mysql2::Client @@ -11,6 +12,7 @@ else PEEK_DB_CLIENT = ::PG::Connection PEEK_DB_VIEW = Peek::Views::PG end + Peek.into PEEK_DB_VIEW Peek.into Peek::Views::Redis Peek.into Peek::Views::Sidekiq diff --git a/db/migrate/20170928124105_create_fork_networks.rb b/db/migrate/20170928124105_create_fork_networks.rb index ca906b953a3..89e5b871967 100644 --- a/db/migrate/20170928124105_create_fork_networks.rb +++ b/db/migrate/20170928124105_create_fork_networks.rb @@ -23,6 +23,7 @@ class CreateForkNetworks < ActiveRecord::Migration if foreign_keys_for(:fork_networks, :root_project_id).any? remove_foreign_key :fork_networks, column: :root_project_id end + drop_table :fork_networks end end diff --git a/db/migrate/20170928133643_create_fork_network_members.rb b/db/migrate/20170928133643_create_fork_network_members.rb index 836f023efdc..8c7d9ba859a 100644 --- a/db/migrate/20170928133643_create_fork_network_members.rb +++ b/db/migrate/20170928133643_create_fork_network_members.rb @@ -21,6 +21,7 @@ class CreateForkNetworkMembers < ActiveRecord::Migration if foreign_keys_for(:fork_network_members, :forked_from_project_id).any? remove_foreign_key :fork_network_members, column: :forked_from_project_id end + drop_table :fork_network_members end end diff --git a/db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb b/db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb index 7cf1d0cec68..d1a039ed551 100644 --- a/db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb +++ b/db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb @@ -9,6 +9,7 @@ class AddIndexOnNamespacesLowerName < ActiveRecord::Migration return unless Gitlab::Database.postgresql? disable_statement_timeout + if Gitlab::Database.version.to_f >= 9.5 # Allow us to hot-patch the index manually ahead of the migration execute "CREATE INDEX CONCURRENTLY IF NOT EXISTS #{INDEX_NAME} ON namespaces (lower(name));" @@ -21,6 +22,7 @@ class AddIndexOnNamespacesLowerName < ActiveRecord::Migration return unless Gitlab::Database.postgresql? disable_statement_timeout + if Gitlab::Database.version.to_f >= 9.2 execute "DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME};" else diff --git a/db/post_migrate/20170518200835_rename_users_with_renamed_namespace.rb b/db/post_migrate/20170518200835_rename_users_with_renamed_namespace.rb index da0fcda87a6..17ad7de065d 100644 --- a/db/post_migrate/20170518200835_rename_users_with_renamed_namespace.rb +++ b/db/post_migrate/20170518200835_rename_users_with_renamed_namespace.rb @@ -31,6 +31,7 @@ class RenameUsersWithRenamedNamespace < ActiveRecord::Migration predicate = namespaces[:owner_id].eq(users[:id]) .and(namespaces[:type].eq(nil)) .and(users[:username].matches(path)) + update_sql = if Gitlab::Database.postgresql? "UPDATE users SET username = namespaces.path "\ "FROM namespaces WHERE #{predicate.to_sql}" diff --git a/db/post_migrate/20171207150343_remove_soft_removed_objects.rb b/db/post_migrate/20171207150343_remove_soft_removed_objects.rb new file mode 100644 index 00000000000..3e2dedfdd6a --- /dev/null +++ b/db/post_migrate/20171207150343_remove_soft_removed_objects.rb @@ -0,0 +1,208 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class RemoveSoftRemovedObjects < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + disable_ddl_transaction! + + module SoftRemoved + extend ActiveSupport::Concern + + included do + scope :soft_removed, -> { where('deleted_at IS NOT NULL') } + end + end + + class User < ActiveRecord::Base + self.table_name = 'users' + + include EachBatch + end + + class Issue < ActiveRecord::Base + self.table_name = 'issues' + + include EachBatch + include SoftRemoved + end + + class MergeRequest < ActiveRecord::Base + self.table_name = 'merge_requests' + + include EachBatch + include SoftRemoved + end + + class Namespace < ActiveRecord::Base + self.table_name = 'namespaces' + + include EachBatch + include SoftRemoved + + scope :soft_removed_personal, -> { soft_removed.where(type: nil) } + scope :soft_removed_group, -> { soft_removed.where(type: 'Group') } + end + + class Route < ActiveRecord::Base + self.table_name = 'routes' + + include EachBatch + include SoftRemoved + end + + class Project < ActiveRecord::Base + self.table_name = 'projects' + + include EachBatch + include SoftRemoved + end + + class CiPipelineSchedule < ActiveRecord::Base + self.table_name = 'ci_pipeline_schedules' + + include EachBatch + include SoftRemoved + end + + class CiTrigger < ActiveRecord::Base + self.table_name = 'ci_triggers' + + include EachBatch + include SoftRemoved + end + + MODELS = [Issue, MergeRequest, CiPipelineSchedule, CiTrigger].freeze + + def up + disable_statement_timeout + + remove_personal_routes + remove_personal_namespaces + remove_group_namespaces + remove_simple_soft_removed_rows + end + + def down + # The data removed by this migration can't be restored in an automated way. + end + + def remove_simple_soft_removed_rows + create_temporary_indexes + + MODELS.each do |model| + say_with_time("Removing soft removed rows from #{model.table_name}") do + model.soft_removed.each_batch do |batch, index| + batch.delete_all + end + end + end + ensure + remove_temporary_indexes + end + + def create_temporary_indexes + MODELS.each do |model| + index_name = temporary_index_name_for(model) + + # Without this index the removal process can take a very long time. For + # example, getting the next ID of a batch for the `issues` table in + # staging would take between 15 and 20 seconds. + next if temporary_index_exists?(model) + + say_with_time("Creating temporary index #{index_name}") do + add_concurrent_index( + model.table_name, + [:deleted_at, :id], + name: index_name, + where: 'deleted_at IS NOT NULL' + ) + end + end + end + + def remove_temporary_indexes + MODELS.each do |model| + index_name = temporary_index_name_for(model) + + next unless temporary_index_exists?(model) + + say_with_time("Removing temporary index #{index_name}") do + remove_concurrent_index_by_name(model.table_name, index_name) + end + end + end + + def temporary_index_name_for(model) + "index_on_#{model.table_name}_tmp" + end + + def temporary_index_exists?(model) + index_name = temporary_index_name_for(model) + + index_exists?(model.table_name, [:deleted_at, :id], name: index_name) + end + + def remove_personal_namespaces + # Some personal namespaces are left behind in case of GitLab.com. In these + # cases the associated data such as the projects and users has already been + # removed. + Namespace.soft_removed_personal.each_batch do |batch| + batch.delete_all + end + end + + def remove_group_namespaces + admin_id = id_for_admin_user + + unless admin_id + say 'Not scheduling soft removed groups for removal as no admin user ' \ + 'could be found. You will need to remove any such groups manually.' + + return + end + + # Left over groups can't be easily removed because we may also need to + # remove memberships, repositories, and other associated data. As a result + # we'll just schedule a Sidekiq job to remove these. + # + # As of January 5th, 2018 there are 36 groups that will be removed using + # this code. + Namespace.select(:id).soft_removed_group.each_batch(of: 10) do |batch, index| + batch.each do |ns| + schedule_group_removal(index * 5.minutes, ns.id, admin_id) + end + end + end + + def schedule_group_removal(delay, group_id, user_id) + if migrate_inline? + GroupDestroyWorker.new.perform(group_id, user_id) + else + GroupDestroyWorker.perform_in(delay, group_id, user_id) + end + end + + def remove_personal_routes + namespaces = Namespace.select(1) + .soft_removed + .where('namespaces.type IS NULL') + .where('routes.source_type = ?', 'Namespace') + .where('routes.source_id = namespaces.id') + + Route.where('EXISTS (?)', namespaces).each_batch do |batch| + batch.delete_all + end + end + + def id_for_admin_user + User.where(admin: true).limit(1).pluck(:id).first + end + + def migrate_inline? + Rails.env.test? || Rails.env.development? + end +end diff --git a/db/post_migrate/20171207150344_remove_deleted_at_columns.rb b/db/post_migrate/20171207150344_remove_deleted_at_columns.rb new file mode 100644 index 00000000000..154d7a1b926 --- /dev/null +++ b/db/post_migrate/20171207150344_remove_deleted_at_columns.rb @@ -0,0 +1,31 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class RemoveDeletedAtColumns < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + TABLES = %i[issues merge_requests namespaces ci_pipeline_schedules ci_triggers].freeze + COLUMN = :deleted_at + + def up + TABLES.each do |table| + remove_column(table, COLUMN) if column_exists?(table, COLUMN) + end + end + + def down + TABLES.each do |table| + unless column_exists?(table, COLUMN) + add_column(table, COLUMN, :datetime_with_timezone) + end + + unless index_exists?(table, COLUMN) + add_concurrent_index(table, COLUMN) + end + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 07620349057..8a6db61250b 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -357,7 +357,6 @@ ActiveRecord::Schema.define(version: 20180105212544) do t.integer "project_id" t.integer "owner_id" t.boolean "active", default: true - t.datetime "deleted_at" t.datetime "created_at" t.datetime "updated_at" end @@ -467,7 +466,6 @@ ActiveRecord::Schema.define(version: 20180105212544) do create_table "ci_triggers", force: :cascade do |t| t.string "token" - t.datetime "deleted_at" t.datetime "created_at" t.datetime "updated_at" t.integer "project_id" @@ -861,7 +859,6 @@ ActiveRecord::Schema.define(version: 20180105212544) do t.integer "iid" t.integer "updated_by_id" t.boolean "confidential", default: false, null: false - t.datetime "deleted_at" t.date "due_date" t.integer "moved_to_id" t.integer "lock_version" @@ -878,7 +875,6 @@ ActiveRecord::Schema.define(version: 20180105212544) do add_index "issues", ["author_id"], name: "index_issues_on_author_id", using: :btree add_index "issues", ["confidential"], name: "index_issues_on_confidential", using: :btree - add_index "issues", ["deleted_at"], name: "index_issues_on_deleted_at", using: :btree add_index "issues", ["description"], name: "index_issues_on_description_trigram", using: :gin, opclasses: {"description"=>"gin_trgm_ops"} add_index "issues", ["milestone_id"], name: "index_issues_on_milestone_id", using: :btree add_index "issues", ["moved_to_id"], name: "index_issues_on_moved_to_id", where: "(moved_to_id IS NOT NULL)", using: :btree @@ -1088,7 +1084,6 @@ ActiveRecord::Schema.define(version: 20180105212544) do t.boolean "merge_when_pipeline_succeeds", default: false, null: false t.integer "merge_user_id" t.string "merge_commit_sha" - t.datetime "deleted_at" t.string "in_progress_merge_commit_sha" t.integer "lock_version" t.text "title_html" @@ -1107,7 +1102,6 @@ ActiveRecord::Schema.define(version: 20180105212544) do add_index "merge_requests", ["assignee_id"], name: "index_merge_requests_on_assignee_id", using: :btree add_index "merge_requests", ["author_id"], name: "index_merge_requests_on_author_id", using: :btree add_index "merge_requests", ["created_at"], name: "index_merge_requests_on_created_at", using: :btree - add_index "merge_requests", ["deleted_at"], name: "index_merge_requests_on_deleted_at", using: :btree add_index "merge_requests", ["description"], name: "index_merge_requests_on_description_trigram", using: :gin, opclasses: {"description"=>"gin_trgm_ops"} add_index "merge_requests", ["head_pipeline_id"], name: "index_merge_requests_on_head_pipeline_id", using: :btree add_index "merge_requests", ["latest_merge_request_diff_id"], name: "index_merge_requests_on_latest_merge_request_diff_id", using: :btree @@ -1167,7 +1161,6 @@ ActiveRecord::Schema.define(version: 20180105212544) do t.boolean "share_with_group_lock", default: false t.integer "visibility_level", default: 20, null: false t.boolean "request_access_enabled", default: false, null: false - t.datetime "deleted_at" t.text "description_html" t.boolean "lfs_enabled" t.integer "parent_id" @@ -1177,7 +1170,6 @@ ActiveRecord::Schema.define(version: 20180105212544) do end add_index "namespaces", ["created_at"], name: "index_namespaces_on_created_at", using: :btree - add_index "namespaces", ["deleted_at"], name: "index_namespaces_on_deleted_at", using: :btree add_index "namespaces", ["name", "parent_id"], name: "index_namespaces_on_name_and_parent_id", unique: true, using: :btree add_index "namespaces", ["name"], name: "index_namespaces_on_name_trigram", using: :gin, opclasses: {"name"=>"gin_trgm_ops"} add_index "namespaces", ["owner_id"], name: "index_namespaces_on_owner_id", using: :btree diff --git a/doc/administration/high_availability/nfs.md b/doc/administration/high_availability/nfs.md index e09ccaba08c..d8928a7fe4c 100644 --- a/doc/administration/high_availability/nfs.md +++ b/doc/administration/high_availability/nfs.md @@ -32,7 +32,9 @@ options: ## AWS Elastic File System -GitLab does not recommend using AWS Elastic File System (EFS). +GitLab strongly recommends against using AWS Elastic File System (EFS). +Our support team will not be able to assist on performance issues related to +file system access. Customers and users have reported that AWS EFS does not perform well for GitLab's use-case. There are several issues that can cause problems. For these reasons diff --git a/doc/api/pipeline_triggers.md b/doc/api/pipeline_triggers.md index 9030ae32d17..e881e61d4ef 100644 --- a/doc/api/pipeline_triggers.md +++ b/doc/api/pipeline_triggers.md @@ -24,7 +24,6 @@ curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" "https://gitlab.example.com/ "id": 10, "description": "my trigger", "created_at": "2016-01-07T09:53:58.235Z", - "deleted_at": null, "last_used": null, "token": "6d056f63e50fe6f8c5f8f4aa10edb7", "updated_at": "2016-01-07T09:53:58.235Z", @@ -55,7 +54,6 @@ curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" "https://gitlab.example.com/ "id": 10, "description": "my trigger", "created_at": "2016-01-07T09:53:58.235Z", - "deleted_at": null, "last_used": null, "token": "6d056f63e50fe6f8c5f8f4aa10edb7", "updated_at": "2016-01-07T09:53:58.235Z", @@ -85,7 +83,6 @@ curl --request POST --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" --form descri "id": 10, "description": "my trigger", "created_at": "2016-01-07T09:53:58.235Z", - "deleted_at": null, "last_used": null, "token": "6d056f63e50fe6f8c5f8f4aa10edb7", "updated_at": "2016-01-07T09:53:58.235Z", @@ -116,7 +113,6 @@ curl --request PUT --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" --form descrip "id": 10, "description": "my trigger", "created_at": "2016-01-07T09:53:58.235Z", - "deleted_at": null, "last_used": null, "token": "6d056f63e50fe6f8c5f8f4aa10edb7", "updated_at": "2016-01-07T09:53:58.235Z", @@ -146,7 +142,6 @@ curl --request POST --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" "https://gitl "id": 10, "description": "my trigger", "created_at": "2016-01-07T09:53:58.235Z", - "deleted_at": null, "last_used": null, "token": "6d056f63e50fe6f8c5f8f4aa10edb7", "updated_at": "2016-01-07T09:53:58.235Z", diff --git a/doc/api/snippets.md b/doc/api/snippets.md index e57143e4215..42b760c107d 100644 --- a/doc/api/snippets.md +++ b/doc/api/snippets.md @@ -2,7 +2,7 @@ > [Introduced][ce-6373] in GitLab 8.15. -### Snippet visibility level +## Snippet visibility level Snippets in GitLab can be either private, internal, or public. You can set it with the `visibility` field in the snippet. @@ -273,4 +273,5 @@ Example response: } ``` -[ce-[ce-29508]: https://gitlab.com/gitlab-org/gitlab-ce/issues/29508]: https://gitlab.com/gitlab-org/gitlab-ce/issues/29508 +[ce-6373]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/6373 +[ce-29508]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/12655 diff --git a/doc/articles/index.md b/doc/articles/index.md index d8a7874b96d..8385ef936c6 100644 --- a/doc/articles/index.md +++ b/doc/articles/index.md @@ -17,7 +17,6 @@ Build, test, and deploy the software you develop with [GitLab CI/CD](../ci/READM | Article title | Category | Publishing date | | :------------ | :------: | --------------: | | [Autoscaling GitLab Runners on AWS](runner_autoscale_aws/index.md) | Admin guide | 2017-11-24 | -| [How to test and deploy Laravel/PHP applications with GitLab CI/CD and Envoy](laravel_with_gitlab_and_envoy/index.md) | Tutorial | 2017-08-31 | | [Making CI Easier with GitLab](https://about.gitlab.com/2017/07/13/making-ci-easier-with-gitlab/) | Concepts | 2017-07-13 | | [Dockerizing GitLab Review Apps](https://about.gitlab.com/2017/07/11/dockerizing-review-apps/) | Concepts | 2017-07-11 | | [Continuous Integration: From Jenkins to GitLab Using Docker](https://about.gitlab.com/2017/07/27/docker-my-precious/) | Concepts | 2017-07-27 | @@ -62,7 +61,6 @@ upgrade, integrate, migrate to GitLab: | :------------ | :------: | --------------: | | [Video Tutorial: Idea to Production on Google Container Engine (GKE)](https://about.gitlab.com/2017/01/23/video-tutorial-idea-to-production-on-google-container-engine-gke/) | Tutorial | 2017-01-23 | | [How to Setup a GitLab Instance on Microsoft Azure](https://about.gitlab.com/2016/07/13/how-to-setup-a-gitlab-instance-on-microsoft-azure/) | Tutorial | 2016-07-13 | -| [Get started with OpenShift Origin 3 and GitLab](openshift_and_gitlab/index.md) | Tutorial | 2016-06-28 | | [Getting started with GitLab and DigitalOcean](https://about.gitlab.com/2016/04/27/getting-started-with-gitlab-and-digitalocean/) | Tutorial | 2016-04-27 | ## Software development diff --git a/doc/articles/laravel_with_gitlab_and_envoy/index.md b/doc/articles/laravel_with_gitlab_and_envoy/index.md index b20bd8c247a..b092cdb0f7a 100644 --- a/doc/articles/laravel_with_gitlab_and_envoy/index.md +++ b/doc/articles/laravel_with_gitlab_and_envoy/index.md @@ -1,680 +1 @@ -# Test and deploy Laravel applications with GitLab CI/CD and Envoy - -> **[Article Type](../../development/writing_documentation.md#types-of-technical-articles):** tutorial || -> **Level:** intermediary || -> **Author:** [Mehran Rasulian](https://gitlab.com/mehranrasulian) || -> **Publication date:** 2017-08-31 - -## Introduction - -GitLab features our applications with Continuous Integration, and it is possible to easily deploy the new code changes to the production server whenever we want. - -In this tutorial, we'll show you how to initialize a [Laravel](http://laravel.com/) application and setup our [Envoy](https://laravel.com/docs/envoy) tasks, then we'll jump into see how to test and deploy it with [GitLab CI/CD](../../ci/README.md) via [Continuous Delivery](https://about.gitlab.com/2016/08/05/continuous-integration-delivery-and-deployment-with-gitlab/). - -We assume you have a basic experience with Laravel, Linux servers, -and you know how to use GitLab. - -Laravel is a high quality web framework written in PHP. -It has a great community with a [fantastic documentation](https://laravel.com/docs). -Aside from the usual routing, controllers, requests, responses, views, and (blade) templates, out of the box Laravel provides plenty of additional services such as cache, events, localization, authentication and many others. - -We will use [Envoy](https://laravel.com/docs/master/envoy) as an SSH task runner based on PHP. -It uses a clean, minimal [Blade syntax](https://laravel.com/docs/blade) to setup tasks that can run on remote servers, such as, cloning your project from the repository, installing the Composer dependencies, and running [Artisan commands](https://laravel.com/docs/artisan). - -## Initialize our Laravel app on GitLab - -We assume [you have installed a new laravel project](https://laravel.com/docs/installation#installation), so let's start with a unit test, and initialize Git for the project. - -### Unit Test - -Every new installation of Laravel (currently 5.4) comes with two type of tests, 'Feature' and 'Unit', placed in the tests directory. -Here's a unit test from `test/Unit/ExampleTest.php`: - -```php -<?php - -namespace Tests\Unit; - -... - -class ExampleTest extends TestCase -{ - public function testBasicTest() - { - $this->assertTrue(true); - } -} -``` - -This test is as simple as asserting that the given value is true. - -Laravel uses `PHPUnit` for tests by default. -If we run `vendor/bin/phpunit` we should see the green output: - -```bash -vendor/bin/phpunit -OK (1 test, 1 assertions) -``` - -This test will be used later for continuously testing our app with GitLab CI/CD. - -### Push to GitLab - -Since we have our app up and running locally, it's time to push the codebase to our remote repository. -Let's create [a new project](../../gitlab-basics/create-project.md) in GitLab named `laravel-sample`. -After that, follow the command line instructions displayed on the project's homepage to initiate the repository on our machine and push the first commit. - - -```bash -cd laravel-sample -git init -git remote add origin git@gitlab.example.com:<USERNAME>/laravel-sample.git -git add . -git commit -m 'Initial Commit' -git push -u origin master -``` - -## Configure the production server - -Before we begin setting up Envoy and GitLab CI/CD, let's quickly make sure the production server is ready for deployment. -We have installed LEMP stack which stands for Linux, Nginx, MySQL and PHP on our Ubuntu 16.04. - -### Create a new user - -Let's now create a new user that will be used to deploy our website and give it -the needed permissions using [Linux ACL](https://serversforhackers.com/video/linux-acls): - -```bash -# Create user deployer -sudo adduser deployer -# Give the read-write-execute permissions to deployer user for directory /var/www -sudo setfacl -R -m u:deployer:rwx /var/www -``` - -If you don't have ACL installed on your Ubuntu server, use this command to install it: - -```bash -sudo apt install acl -``` - -### Add SSH key - -Let's suppose we want to deploy our app to the production server from a private repository on GitLab. First, we need to [generate a new SSH key pair **with no passphrase**](../../ssh/README.md) for the deployer user. - -After that, we need to copy the private key, which will be used to connect to our server as the deployer user with SSH, to be able to automate our deployment process: - -```bash -# As the deployer user on server -# -# Copy the content of public key to authorized_keys -cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys -# Copy the private key text block -cat ~/.ssh/id_rsa -``` - -Now, let's add it to your GitLab project as a [secret variable](../../ci/variables/README.md#secret-variables). -Secret variables are user-defined variables and are stored out of `.gitlab-ci.yml`, for security purposes. -They can be added per project by navigating to the project's **Settings** > **CI/CD**. - - - -To the field **KEY**, add the name `SSH_PRIVATE_KEY`, and to the **VALUE** field, paste the private key you've copied earlier. -We'll use this variable in the `.gitlab-ci.yml` later, to easily connect to our remote server as the deployer user without entering its password. - -We also need to add the public key to **Project** > **Settings** > **Repository** as [Deploy Keys](../../ssh/README.md/#deploy-keys), which gives us the ability to access our repository from the server through [SSH protocol](../../gitlab-basics/command-line-commands.md/#start-working-on-your-project). - - -```bash -# As the deployer user on the server -# -# Copy the public key -cat ~/.ssh/id_rsa.pub -``` - - - -To the field **Title**, add any name you want, and paste the public key into the **Key** field. - -Now, let's clone our repository on the server just to make sure the `deployer` user has access to the repository. - -```bash -# As the deployer user on server -# -git clone git@gitlab.example.com:<USERNAME>/laravel-sample.git -``` - ->**Note:** -Answer **yes** if asked `Are you sure you want to continue connecting (yes/no)?`. -It adds GitLab.com to the known hosts. - -### Configuring Nginx - -Now, let's make sure our web server configuration points to the `current/public` rather than `public`. - -Open the default Nginx server block configuration file by typing: - -```bash -sudo nano /etc/nginx/sites-available/default -``` - -The configuration should be like this. - -``` -server { - root /var/www/app/current/public; - server_name example.com; - # Rest of the configuration -} -``` - ->**Note:** -You may replace the app's name in `/var/www/app/current/public` with the folder name of your application. - -## Setting up Envoy - -So we have our Laravel app ready for production. -The next thing is to use Envoy to perform the deploy. - -To use Envoy, we should first install it on our local machine [using the given instructions by Laravel](https://laravel.com/docs/envoy/#introduction). - -### How Envoy works - -The pros of Envoy is that it doesn't require Blade engine, it just uses Blade syntax to define tasks. -To start, we create an `Envoy.blade.php` in the root of our app with a simple task to test Envoy. - - -```php -@servers(['web' => 'remote_username@remote_host']) - -@task('list', [on => 'web']) - ls -l -@endtask -``` - -As you may expect, we have an array within `@servers` directive at the top of the file, which contains a key named `web` with a value of the server's address (e.g. `deployer@192.168.1.1`). -Then within our `@task` directive we define the bash commands that should be run on the server when the task is executed. - -On the local machine use the `run` command to run Envoy tasks. - -```bash -envoy run list -``` - -It should execute the `list` task we defined earlier, which connects to the server and lists directory contents. - -Envoy is not a dependency of Laravel, therefore you can use it for any PHP application. - -### Zero downtime deployment - -Every time we deploy to the production server, Envoy downloads the latest release of our app from GitLab repository and replace it with preview's release. -Envoy does this without any [downtime](https://en.wikipedia.org/wiki/Downtime), -so we don't have to worry during the deployment while someone might be reviewing the site. -Our deployment plan is to clone the latest release from GitLab repository, install the Composer dependencies and finally, activate the new release. - -#### @setup directive - -The first step of our deployment process is to define a set of variables within [@setup](https://laravel.com/docs/envoy/#setup) directive. -You may change the `app` to your application's name: - - -```php -... - -@setup - $repository = 'git@gitlab.example.com:<USERNAME>/laravel-sample.git'; - $releases_dir = '/var/www/app/releases'; - $app_dir = '/var/www/app'; - $release = date('YmdHis'); - $new_release_dir = $releases_dir .'/'. $release; -@endsetup - -... -``` - -- `$repository` is the address of our repository -- `$releases_dir` directory is where we deploy the app -- `$app_dir` is the actual location of the app that is live on the server -- `$release` contains a date, so every time that we deploy a new release of our app, we get a new folder with the current date as name -- `$new_release_dir` is the full path of the new release which is used just to make the tasks cleaner - -#### @story directive - -The [@story](https://laravel.com/docs/envoy/#stories) directive allows us define a list of tasks that can be run as a single task. -Here we have three tasks called `clone_repository`, `run_composer`, `update_symlinks`. These variables are usable to making our task's codes more cleaner: - - -```php -... - -@story('deploy') - clone_repository - run_composer - update_symlinks -@endstory - -... -``` - -Let's create these three tasks one by one. - -#### Clone the repository - -The first task will create the `releases` directory (if it doesn't exist), and then clone the `master` branch of the repository (by default) into the new release directory, given by the `$new_release_dir` variable. -The `releases` directory will hold all our deployments: - -```php -... - -@task('clone_repository') - echo 'Cloning repository' - [ -d {{ $releases_dir }} ] || mkdir {{ $releases_dir }} - git clone --depth 1 {{ $repository }} {{ $new_release_dir }} -@endtask - -... -``` - -While our project grows, its Git history will be very very long over time. -Since we are creating a directory per release, it might not be necessary to have the history of the project downloaded for each release. -The `--depth 1` option is a great solution which saves systems time and disk space as well. - -#### Installing dependencies with Composer - -As you may know, this task just navigates to the new release directory and runs Composer to install the application dependencies: - -```php -... - -@task('run_composer') - echo "Starting deployment ({{ $release }})" - cd {{ $new_release_dir }} - composer install --prefer-dist --no-scripts -q -o -@endtask - -... -``` - -#### Activate new release - -Next thing to do after preparing the requirements of our new release, is to remove the storage directory from it and to create two symbolic links to point the application's `storage` directory and `.env` file to the new release. -Then, we need to create another symbolic link to the new release with the name of `current` placed in the app directory. -The `current` symbolic link always points to the latest release of our app: - -```php -... - -@task('update_symlinks') - echo "Linking storage directory" - rm -rf {{ $new_release_dir }}/storage - ln -nfs {{ $app_dir }}/storage {{ $new_release_dir }}/storage - - echo 'Linking .env file' - ln -nfs {{ $app_dir }}/.env {{ $new_release_dir }}/.env - - echo 'Linking current release' - ln -nfs {{ $new_release_dir }} {{ $app_dir }}/current -@endtask -``` - -As you see, we use `-nfs` as an option for `ln` command, which says that the `storage`, `.env` and `current` no longer points to the preview's release and will point them to the new release by force (`f` from `-nfs` means force), which is the case when we are doing multiple deployments. - -### Full script - -The script is ready, but make sure to change the `deployer@192.168.1.1` to your server and also change `/var/www/app` with the directory you want to deploy your app. - -At the end, our `Envoy.blade.php` file will look like this: - -```php -@servers(['web' => 'deployer@192.168.1.1']) - -@setup - $repository = 'git@gitlab.example.com:<USERNAME>/laravel-sample.git'; - $releases_dir = '/var/www/app/releases'; - $app_dir = '/var/www/app'; - $release = date('YmdHis'); - $new_release_dir = $releases_dir .'/'. $release; -@endsetup - -@story('deploy') - clone_repository - run_composer - update_symlinks -@endstory - -@task('clone_repository') - echo 'Cloning repository' - [ -d {{ $releases_dir }} ] || mkdir {{ $releases_dir }} - git clone --depth 1 {{ $repository }} {{ $new_release_dir }} -@endtask - -@task('run_composer') - echo "Starting deployment ({{ $release }})" - cd {{ $new_release_dir }} - composer install --prefer-dist --no-scripts -q -o -@endtask - -@task('update_symlinks') - echo "Linking storage directory" - rm -rf {{ $new_release_dir }}/storage - ln -nfs {{ $app_dir }}/storage {{ $new_release_dir }}/storage - - echo 'Linking .env file' - ln -nfs {{ $app_dir }}/.env {{ $new_release_dir }}/.env - - echo 'Linking current release' - ln -nfs {{ $new_release_dir }} {{ $app_dir }}/current -@endtask -``` - -One more thing we should do before any deployment is to manually copy our application `storage` folder to the `/var/www/app` directory on the server for the first time. -You might want to create another Envoy task to do that for you. -We also create the `.env` file in the same path to setup our production environment variables for Laravel. -These are persistent data and will be shared to every new release. - -Now, we would need to deploy our app by running `envoy run deploy`, but it won't be necessary since GitLab can handle that for us with CI's [environments](../../ci/environments.md), which will be described [later](#setting-up-gitlab-ci-cd) in this tutorial. - -Now it's time to commit [Envoy.blade.php](https://gitlab.com/mehranrasulian/laravel-sample/blob/master/Envoy.blade.php) and push it to the `master` branch. -To keep things simple, we commit directly to `master`, without using [feature-branches](../../workflow/gitlab_flow.md/#github-flow-as-a-simpler-alternative) since collaboration is beyond the scope of this tutorial. -In a real world project, teams may use [Issue Tracker](../../user/project/issues/index.md) and [Merge Requests](../../user/project/merge_requests/index.md) to move their code across branches: - -```bash -git add Envoy.blade.php -git commit -m 'Add Envoy' -git push origin master -``` - -## Continuous Integration with GitLab - -We have our app ready on GitLab, and we also can deploy it manually. -But let's take a step forward to do it automatically with [Continuous Delivery](https://about.gitlab.com/2016/08/05/continuous-integration-delivery-and-deployment-with-gitlab/#continuous-delivery) method. -We need to check every commit with a set of automated tests to become aware of issues at the earliest, and then, we can deploy to the target environment if we are happy with the result of the tests. - -[GitLab CI/CD](../../ci/README.md) allows us to use [Docker](https://docker.com/) engine to handle the process of testing and deploying our app. -In the case you're not familiar with Docker, refer to [How to Automate Docker Deployments](http://paislee.io/how-to-automate-docker-deployments/). - -To be able to build, test, and deploy our app with GitLab CI/CD, we need to prepare our work environment. -To do that, we'll use a Docker image which has the minimum requirements that a Laravel app needs to run. -[There are other ways](../../ci/examples/php.md/#test-php-projects-using-the-docker-executor) to do that as well, but they may lead our builds run slowly, which is not what we want when there are faster options to use. - -With Docker images our builds run incredibly faster! - -### Create a Container Image - -Let's create a [Dockerfile](https://gitlab.com/mehranrasulian/laravel-sample/blob/master/Dockerfile) in the root directory of our app with the following content: - -```bash -# Set the base image for subsequent instructions -FROM php:7.1 - -# Update packages -RUN apt-get update - -# Install PHP and composer dependencies -RUN apt-get install -qq git curl libmcrypt-dev libjpeg-dev libpng-dev libfreetype6-dev libbz2-dev - -# Clear out the local repository of retrieved package files -RUN apt-get clean - -# Install needed extensions -# Here you can install any other extension that you need during the test and deployment process -RUN docker-php-ext-install mcrypt pdo_mysql zip - -# Install Composer -RUN curl --silent --show-error https://getcomposer.org/installer | php -- --install-dir=/usr/local/bin --filename=composer - -# Install Laravel Envoy -RUN composer global require "laravel/envoy=~1.0" -``` - -We added the [official PHP 7.1 Docker image](https://hub.docker.com/r/_/php/), which consist of a minimum installation of Debian Jessie with PHP pre-installed, and works perfectly for our use case. - -We used `docker-php-ext-install` (provided by the official PHP Docker image) to install the PHP extensions we need. - -#### Setting Up GitLab Container Registry - -Now that we have our `Dockerfile` let's build and push it to our [GitLab Container Registry](../../user/project/container_registry.md). - -> The registry is the place to store and tag images for later use. Developers may want to maintain their own registry for private, company images, or for throw-away images used only in testing. Using GitLab Container Registry means you don't need to set up and administer yet another service or use a public registry. - -On your GitLab project repository navigate to the **Registry** tab. - - - -You may need to [enable Container Registry](../../user/project/container_registry.md#enable-the-container-registry-for-your-project) to your project to see this tab. You'll find it under your project's **Settings > General > Sharing and permissions**. - - - -To start using Container Registry on our machine, we first need to login to the GitLab registry using our GitLab username and password: - -```bash -docker login registry.gitlab.com -``` -Then we can build and push our image to GitLab: - -```bash -docker build -t registry.gitlab.com/<USERNAME>/laravel-sample . - -docker push registry.gitlab.com/<USERNAME>/laravel-sample -``` - ->**Note:** -To run the above commands, we first need to have [Docker](https://docs.docker.com/engine/installation/) installed on our machine. - -Congratulations! You just pushed the first Docker image to the GitLab Registry, and if you refresh the page you should be able to see it: - - - ->**Note:** -You can also [use GitLab CI/CD](https://about.gitlab.com/2016/05/23/gitlab-container-registry/#use-with-gitlab-ci) to build and push your Docker images, rather than doing that on your machine. - -We'll use this image further down in the `.gitlab-ci.yml` configuration file to handle the process of testing and deploying our app. - -Let's commit the `Dockerfile` file. - -```bash -git add Dockerfile -git commit -m 'Add Dockerfile' -git push origin master -``` - -### Setting up GitLab CI/CD - -In order to build and test our app with GitLab CI/CD, we need a file called `.gitlab-ci.yml` in our repository's root. It is similar to Circle CI and Travis CI, but built-in GitLab. - -Our `.gitlab-ci.yml` file will look like this: - -```yaml -image: registry.gitlab.com/<USERNAME>/laravel-sample:latest - -services: - - mysql:5.7 - -variables: - MYSQL_DATABASE: homestead - MYSQL_ROOT_PASSWORD: secret - DB_HOST: mysql - DB_USERNAME: root - -stages: - - test - - deploy - -unit_test: - stage: test - script: - - cp .env.example .env - - composer install - - php artisan key:generate - - php artisan migrate - - vendor/bin/phpunit - -deploy_production: - stage: deploy - script: - - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - - eval $(ssh-agent -s) - - ssh-add <(echo "$SSH_PRIVATE_KEY") - - mkdir -p ~/.ssh - - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' - - - ~/.composer/vendor/bin/envoy run deploy - environment: - name: production - url: http://192.168.1.1 - when: manual - only: - - master -``` - -That's a lot to take in, isn't it? Let's run through it step by step. - -#### Image and Services - -[GitLab Runners](../../ci/runners/README.md) run the script defined by `.gitlab-ci.yml`. -The `image` keyword tells the Runners which image to use. -The `services` keyword defines additional images [that are linked to the main image](../../ci/docker/using_docker_images.md/#what-is-a-service). -Here we use the container image we created before as our main image and also use MySQL 5.7 as a service. - -```yaml -image: registry.gitlab.com/<USERNAME>/laravel-sample:latest - -services: - - mysql:5.7 - -... -``` - ->**Note:** -If you wish to test your app with different PHP versions and [database management systems](../../ci/services/README.md), you can define different `image` and `services` keywords for each test job. - -#### Variables - -GitLab CI/CD allows us to use [environment variables](../../ci/yaml/README.md#variables) in our jobs. -We defined MySQL as our database management system, which comes with a superuser root created by default. - -So we should adjust the configuration of MySQL instance by defining `MYSQL_DATABASE` variable as our database name and `MYSQL_ROOT_PASSWORD` variable as the password of `root`. -Find out more about MySQL variables at the [official MySQL Docker Image](https://hub.docker.com/r/_/mysql/). - -Also set the variables `DB_HOST` to `mysql` and `DB_USERNAME` to `root`, which are Laravel specific variables. -We define `DB_HOST` as `mysql` instead of `127.0.0.1`, as we use MySQL Docker image as a service which [is linked to the main Docker image](../../ci/docker/using_docker_images.md/#how-services-are-linked-to-the-build). - -```yaml -... - -variables: - MYSQL_DATABASE: homestead - MYSQL_ROOT_PASSWORD: secret - DB_HOST: mysql - DB_USERNAME: root - -... -``` - -#### Unit Test as the first job - -We defined the required shell scripts as an array of the [script](../../ci/yaml/README.md#script) variable to be executed when running `unit_test` job. - -These scripts are some Artisan commands to prepare the Laravel, and, at the end of the script, we'll run the tests by `PHPUnit`. - -```yaml -... - -unit_test: - script: - # Install app dependencies - - composer install - # Setup .env - - cp .env.example .env - # Generate an environment key - - php artisan key:generate - # Run migrations - - php artisan migrate - # Run tests - - vendor/bin/phpunit - -... -``` - -#### Deploy to production - -The job `deploy_production` will deploy the app to the production server. -To deploy our app with Envoy, we had to set up the `$SSH_PRIVATE_KEY` variable as an [SSH private key](../../ci/ssh_keys/README.md/#ssh-keys-when-using-the-docker-executor). -If the SSH keys have added successfully, we can run Envoy. - -As mentioned before, GitLab supports [Continuous Delivery](https://about.gitlab.com/2016/08/05/continuous-integration-delivery-and-deployment-with-gitlab/#continuous-delivery) methods as well. -The [environment](../../ci/yaml/README.md#environment) keyword tells GitLab that this job deploys to the `production` environment. -The `url` keyword is used to generate a link to our application on the GitLab Environments page. -The `only` keyword tells GitLab CI that the job should be executed only when the pipeline is building the `master` branch. -Lastly, `when: manual` is used to turn the job from running automatically to a manual action. - -```yaml -... - -deploy_production: - script: - # Add the private SSH key to the build environment - - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' - - eval $(ssh-agent -s) - - ssh-add <(echo "$SSH_PRIVATE_KEY") - - mkdir -p ~/.ssh - - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' - - # Run Envoy - - ~/.composer/vendor/bin/envoy run deploy - - environment: - name: production - url: http://192.168.1.1 - when: manual - only: - - master -``` - -You may also want to add another job for [staging environment](https://about.gitlab.com/2016/08/26/ci-deployment-and-environments), to final test your application before deploying to production. - -### Turn on GitLab CI/CD - -We have prepared everything we need to test and deploy our app with GitLab CI/CD. -To do that, commit and push `.gitlab-ci.yml` to the `master` branch. It will trigger a pipeline, which you can watch live under your project's **Pipelines**. - - - -Here we see our **Test** and **Deploy** stages. -The **Test** stage has the `unit_test` build running. -click on it to see the Runner's output. - - - -After our code passed through the pipeline successfully, we can deploy to our production server by clicking the **play** button on the right side. - - - -Once the deploy pipeline passed successfully, navigate to **Pipelines > Environments**. - - - -If something doesn't work as expected, you can roll back to the latest working version of your app. - - - -By clicking on the external link icon specified on the right side, GitLab opens the production website. -Our deployment successfully was done and we can see the application is live. - - - -In the case that you're interested to know how is the application directory structure on the production server after deployment, here are three directories named `current`, `releases` and `storage`. -As you know, the `current` directory is a symbolic link that points to the latest release. -The `.env` file consists of our Laravel environment variables. - - - -If you navigate to the `current` directory, you should see the application's content. -As you see, the `.env` is pointing to the `/var/www/app/.env` file and also `storage` is pointing to the `/var/www/app/storage/` directory. - - - -## Conclusion - -We configured GitLab CI to perform automated tests and used the method of [Continuous Delivery](https://continuousdelivery.com/) to deploy to production a Laravel application with Envoy, directly from the codebase. - -Envoy also was a great match to help us deploy the application without writing our custom bash script and doing Linux magics. +This document was moved to [another location](../../ci/examples/laravel_with_gitlab_and_envoy/index.md). diff --git a/doc/articles/openshift_and_gitlab/index.md b/doc/articles/openshift_and_gitlab/index.md index c0bbcfe2a8a..b7594cfef7f 100644 --- a/doc/articles/openshift_and_gitlab/index.md +++ b/doc/articles/openshift_and_gitlab/index.md @@ -1,510 +1 @@ -# Getting started with OpenShift Origin 3 and GitLab - -> **Article [Type](../../development/writing_documentation.html#types-of-technical-articles):** tutorial || -> **Level:** intermediary || -> **Author:** [Achilleas Pipinellis](https://gitlab.com/axil) || -> **Publication date:** 2016-06-28 - -## Introduction - -[OpenShift Origin][openshift] is an open source container application -platform created by [RedHat], based on [kubernetes] and [Docker]. That means -you can host your own PaaS for free and almost with no hassle. - -In this tutorial, we will see how to deploy GitLab in OpenShift using GitLab's -official Docker image while getting familiar with the web interface and CLI -tools that will help us achieve our goal. - ---- - -## Prerequisites - -OpenShift 3 is not yet deployed on RedHat's offered Online platform ([openshift.com]), -so in order to test it, we will use an [all-in-one Virtualbox image][vm] that is -offered by the OpenShift developers and managed by Vagrant. If you haven't done -already, go ahead and install the following components as they are essential to -test OpenShift easily: - -- [VirtualBox] -- [Vagrant] -- [OpenShift Client][oc] (`oc` for short) - -It is also important to mention that for the purposes of this tutorial, the -latest Origin release is used: - -- **oc** `v1.3.0` (must be [installed][oc-gh] locally on your computer) -- **openshift** `v1.3.0` (is pre-installed in the [VM image][vm-new]) -- **kubernetes** `v1.3.0` (is pre-installed in the [VM image][vm-new]) - ->**Note:** -If you intend to deploy GitLab on a production OpenShift cluster, there are some -limitations to bare in mind. Read on the [limitations](#current-limitations) -section for more information and follow the linked links for the relevant -discussions. - -Now that you have all batteries, let's see how easy it is to test OpenShift -on your computer. - -## Getting familiar with OpenShift Origin - -The environment we are about to use is based on CentOS 7 which comes with all -the tools needed pre-installed: Docker, kubernetes, OpenShift, etcd. - -### Test OpenShift using Vagrant - -As of this writing, the all-in-one VM is at version 1.3, and that's -what we will use in this tutorial. - -In short: - -1. Open a terminal and in a new directory run: - ```sh - vagrant init openshift/origin-all-in-one - ``` -1. This will generate a Vagrantfile based on the all-in-one VM image -1. In the same directory where you generated the Vagrantfile - enter: - - ```sh - vagrant up - ``` - -This will download the VirtualBox image and fire up the VM with some preconfigured -values as you can see in the Vagrantfile. As you may have noticed, you need -plenty of RAM (5GB in our example), so make sure you have enough. - -Now that OpenShift is setup, let's see how the web console looks like. - -### Explore the OpenShift web console - -Once Vagrant finishes its thing with the VM, you will be presented with a -message which has some important information. One of them is the IP address -of the deployed OpenShift platform and in particular <https://10.2.2.2:8443/console/>. -Open this link with your browser and accept the self-signed certificate in -order to proceed. - -Let's login as admin with username/password `admin/admin`. This is what the -landing page looks like: - - - -You can see that a number of [projects] are already created for testing purposes. - -If you head over the `openshift-infra` project, a number of services with their -respective pods are there to explore. - - - -We are not going to explore the whole interface, but if you want to learn about -the key concepts of OpenShift, read the [core concepts reference][core] in the -official documentation. - -### Explore the OpenShift CLI - -OpenShift Client (`oc`), is a powerful CLI tool that talks to the OpenShift API -and performs pretty much everything you can do from the web UI and much more. - -Assuming you have [installed][oc] it, let's explore some of its main -functionalities. - -Let's first see the version of `oc`: - -```sh -$ oc version - -oc v1.3.0 -kubernetes v1.3.0+52492b4 -``` - -With `oc help` you can see the top level arguments you can run with `oc` and -interact with your cluster, kubernetes, run applications, create projects and -much more. - -Let's login to the all-in-one VM and see how to achieve the same results like -when we visited the web console earlier. The username/password for the -administrator user is `admin/admin`. There is also a test user with username/ -password `user/user`, with limited access. Let's login as admin for the moment: - -```sh -$ oc login https://10.2.2.2:8443 - -Authentication required for https://10.2.2.2:8443 (openshift) -Username: admin -Password: -Login successful. - -You have access to the following projects and can switch between them with 'oc project <projectname>': - - * cockpit - * default (current) - * delete - * openshift - * openshift-infra - * sample - -Using project "default". -``` - -Switch to the `openshift-infra` project with: - -```sh -oc project openshift-infra -``` - -And finally, see its status: - -```sh -oc status -``` - -The last command should spit a bunch of information about the statuses of the -pods and the services, which if you look closely is what we encountered in the -second image when we explored the web console. - -You can always read more about `oc` in the [OpenShift CLI documentation][oc]. - -### Troubleshooting the all-in-one VM - -Using the all-in-one VM gives you the ability to test OpenShift whenever you -want. That means you get to play with it, shutdown the VM, and pick up where -you left off. - -Sometimes though, you may encounter some issues, like OpenShift not running -when booting up the VM. The web UI may not responding or you may see issues -when trying to login with `oc`, like: - -``` -The connection to the server 10.2.2.2:8443 was refused - did you specify the right host or port? -``` - -In that case, the OpenShift service might not be running, so in order to fix it: - -1. SSH into the VM by going to the directory where the Vagrantfile is and then - run: - - ```sh - vagrant ssh - ``` - -1. Run `systemctl` and verify by the output that the `openshift` service is not - running (it will be in red color). If that's the case start the service with: - - ```sh - sudo systemctl start openshift - ``` - -1. Verify the service is up with: - - ```sh - systemctl status openshift -l - ``` - -Now you will be able to login using `oc` (like we did before) and visit the web -console. - -## Deploy GitLab - -Now that you got a taste of what OpenShift looks like, let's deploy GitLab! - -### Create a new project - -First, we will create a new project to host our application. You can do this -either by running the CLI client: - -```bash -$ oc new-project gitlab -``` - -or by using the web interface: - - - -If you used the command line, `oc` automatically uses the new project and you -can see its status with: - -```sh -$ oc status - -In project gitlab on server https://10.2.2.2:8443 - -You have no services, deployment configs, or build configs. -Run 'oc new-app' to create an application. -``` - -If you visit the web console, you can now see `gitlab` listed in the projects list. - -The next step is to import the OpenShift template for GitLab. - -### Import the template - -The [template][templates] is basically a JSON file which describes a set of -related object definitions to be created together, as well as a set of -parameters for those objects. - -The template for GitLab resides in the Omnibus GitLab repository under the -docker directory. Let's download it locally with `wget`: - -```bash -wget https://gitlab.com/gitlab-org/omnibus-gitlab/raw/master/docker/openshift-template.json -``` - -And then let's import it in OpenShift: - -```bash -oc create -f openshift-template.json -n openshift -``` - ->**Note:** -The `-n openshift` namespace flag is a trick to make the template available to all -projects. If you recall from when we created the `gitlab` project, `oc` switched -to it automatically, and that can be verified by the `oc status` command. If -you omit the namespace flag, the application will be available only to the -current project, in our case `gitlab`. The `openshift` namespace is a global -one that the administrators should use if they want the application to be -available to all users. - -We are now ready to finally deploy GitLab! - -### Create a new application - -The next step is to use the template we previously imported. Head over to the -`gitlab` project and hit the **Add to Project** button. - - - -This will bring you to the catalog where you can find all the pre-defined -applications ready to deploy with the click of a button. Search for `gitlab` -and you will see the previously imported template: - - - -Select it, and in the following screen you will be presented with the predefined -values used with the GitLab template: - - - -Notice at the top that there are three resources to be created with this -template: - -- `gitlab-ce` -- `gitlab-ce-redis` -- `gitlab-ce-postgresql` - -While PostgreSQL and Redis are bundled in Omnibus GitLab, the template is using -separate images as you can see from [this line][line] in the template. - -The predefined values have been calculated for the purposes of testing out -GitLab in the all-in-one VM. You don't need to change anything here, hit -**Create** to start the deployment. - -If you are deploying to production you will want to change the **GitLab instance -hostname** and use greater values for the volume sizes. If you don't provide a -password for PostgreSQL, it will be created automatically. - ->**Note:** -The `gitlab.apps.10.2.2.2.xip.io` hostname that is used by default will -resolve to the host with IP `10.2.2.2` which is the IP our VM uses. It is a -trick to have distinct FQDNs pointing to services that are on our local network. -Read more on how this works in <http://xip.io>. - -Now that we configured this, let's see how to manage and scale GitLab. - -## Manage and scale GitLab - -Setting up GitLab for the first time might take a while depending on your -internet connection and the resources you have attached to the all-in-one VM. -GitLab's docker image is quite big (~500MB), so you'll have to wait until -it's downloaded and configured before you use it. - -### Watch while GitLab gets deployed - -Navigate to the `gitlab` project at **Overview**. You can notice that the -deployment is in progress by the orange color. The Docker images are being -downloaded and soon they will be up and running. - - - -Switch to the **Browse > Pods** and you will eventually see all 3 pods in a -running status. Remember the 3 resources that were to be created when we first -created the GitLab app? This is where you can see them in action. - - - -You can see GitLab being reconfigured by taking look at the logs in realtime. -Click on `gitlab-ce-2-j7ioe` (your ID will be different) and go to the **Logs** -tab. - - - -At a point you should see a _**gitlab Reconfigured!**_ message in the logs. -Navigate back to the **Overview** and hopefully all pods will be up and running. - - - -Congratulations! You can now navigate to your new shinny GitLab instance by -visiting <http://gitlab.apps.10.2.2.2.xip.io> where you will be asked to -change the root user password. Login using `root` as username and providing the -password you just set, and start using GitLab! - -### Scale GitLab with the push of a button - -If you reach to a point where your GitLab instance could benefit from a boost -of resources, you'd be happy to know that you can scale up with the push of a -button. - -In the **Overview** page just click the up arrow button in the pod where -GitLab is. The change is instant and you can see the number of [replicas] now -running scaled to 2. - - - -Upping the GitLab pods is actually like adding new application servers to your -cluster. You can see how that would work if you didn't use GitLab with -OpenShift by following the [HA documentation][ha] for the application servers. - -Bare in mind that you may need more resources (CPU, RAM, disk space) when you -scale up. If a pod is in pending state for too long, you can navigate to -**Browse > Events** and see the reason and message of the state. - - - -### Scale GitLab using the `oc` CLI - -Using `oc` is super easy to scale up the replicas of a pod. You may want to -skim through the [basic CLI operations][basic-cli] to get a taste how the CLI -commands are used. Pay extra attention to the object types as we will use some -of them and their abbreviated versions below. - -In order to scale up, we need to find out the name of the replication controller. -Let's see how to do that using the following steps. - -1. Make sure you are in the `gitlab` project: - - ```sh - oc project gitlab - ``` - -1. See what services are used for this project: - - ```sh - oc get svc - ``` - - The output will be similar to: - - ``` - NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE - gitlab-ce 172.30.243.177 <none> 22/TCP,80/TCP 5d - gitlab-ce-postgresql 172.30.116.75 <none> 5432/TCP 5d - gitlab-ce-redis 172.30.105.88 <none> 6379/TCP 5d - ``` - -1. We need to see the replication controllers of the `gitlab-ce` service. - Get a detailed view of the current ones: - - ```sh - oc describe rc gitlab-ce - ``` - - This will return a large detailed list of the current replication controllers. - Search for the name of the GitLab controller, usually `gitlab-ce-1` or if - that failed at some point and you spawned another one, it will be named - `gitlab-ce-2`. - -1. Scale GitLab using the previous information: - - ```sh - oc scale --replicas=2 replicationcontrollers gitlab-ce-2 - ``` - -1. Get the new replicas number to make sure scaling worked: - - ```sh - oc get rc gitlab-ce-2 - ``` - - which will return something like: - - ``` - NAME DESIRED CURRENT AGE - gitlab-ce-2 2 2 5d - ``` - -And that's it! We successfully scaled the replicas to 2 using the CLI. - -As always, you can find the name of the controller using the web console. Just -click on the service you are interested in and you will see the details in the -right sidebar. - - - -### Autoscaling GitLab - -In case you were wondering whether there is an option to autoscale a pod based -on the resources of your server, the answer is yes, of course there is. - -We will not expand on this matter, but feel free to read the documentation on -OpenShift's website about [autoscaling]. - -## Current limitations - -As stated in the [all-in-one VM][vm] page: - -> By default, OpenShift will not allow a container to run as root or even a -non-random container assigned userid. Most Docker images in the Dockerhub do not -follow this best practice and instead run as root. - -The all-in-one VM we are using has this security turned off so it will not -bother us. In any case, it is something to keep in mind when deploying GitLab -on a production cluster. - -In order to deploy GitLab on a production cluster, you will need to assign the -GitLab service account to the `anyuid` Security Context. - -1. Edit the Security Context: - ```sh - oc edit scc anyuid - ``` - -1. Add `system:serviceaccount:<project>:gitlab-ce-user` to the `users` section. - If you changed the Application Name from the default the user will - will be `<app-name>-user` instead of `gitlab-ce-user` - -1. Save and exit the editor - -## Conclusion - -By now, you should have an understanding of the basic OpenShift Origin concepts -and a sense of how things work using the web console or the CLI. - -GitLab was hard to install in previous versions of OpenShift, -but now that belongs to the past. Upload a template, create a project, add an -application and you are done. You are ready to login to your new GitLab instance. - -And remember that in this tutorial we just scratched the surface of what Origin -is capable of. As always, you can refer to the detailed -[documentation][openshift-docs] to learn more about deploying your own OpenShift -PaaS and managing your applications with the ease of containers. - -[RedHat]: https://www.redhat.com/en "RedHat website" -[openshift]: https://www.openshift.org "OpenShift Origin website" -[vm]: https://www.openshift.org/vm/ "OpenShift All-in-one VM" -[vm-new]: https://atlas.hashicorp.com/openshift/boxes/origin-all-in-one "Official OpenShift Vagrant box on Atlas" -[template]: https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/docker/openshift-template.json "OpenShift template for GitLab" -[openshift.com]: https://openshift.com "OpenShift Online" -[kubernetes]: http://kubernetes.io/ "Kubernetes website" -[Docker]: https://www.docker.com "Docker website" -[oc]: https://docs.openshift.org/latest/cli_reference/get_started_cli.html "Documentation - oc CLI documentation" -[VirtualBox]: https://www.virtualbox.org/wiki/Downloads "VirtualBox downloads" -[Vagrant]: https://www.vagrantup.com/downloads.html "Vagrant downloads" -[projects]: https://docs.openshift.org/latest/dev_guide/projects.html "Documentation - Projects overview" -[core]: https://docs.openshift.org/latest/architecture/core_concepts/index.html "Documentation - Core concepts of OpenShift Origin" -[templates]: https://docs.openshift.org/latest/architecture/core_concepts/templates.html "Documentation - OpenShift templates" -[old-post]: https://blog.openshift.com/deploy-gitlab-openshift/ "Old post - Deploy GitLab on OpenShift" -[line]: https://gitlab.com/gitlab-org/omnibus-gitlab/blob/658c065c8d022ce858dd63eaeeadb0b2ddc8deea/docker/openshift-template.json#L239 "GitLab - OpenShift template" -[oc-gh]: https://github.com/openshift/origin/releases/tag/v1.3.0 "Openshift 1.3.0 release on GitHub" -[ha]: http://docs.gitlab.com/ce/administration/high_availability/gitlab.html "Documentation - GitLab High Availability" -[replicas]: https://docs.openshift.org/latest/architecture/core_concepts/deployments.html#replication-controllers "Documentation - Replication controller" -[autoscaling]: https://docs.openshift.org/latest/dev_guide/pod_autoscaling.html "Documentation - Autoscale" -[basic-cli]: https://docs.openshift.org/latest/cli_reference/basic_cli_operations.html "Documentation - Basic CLI operations" -[openshift-docs]: https://docs.openshift.org "OpenShift documentation" +This document was moved to [another location](../../install/openshift_and_gitlab/index.html). diff --git a/doc/ci/README.md b/doc/ci/README.md index 05d792dea0f..5829aaee9c9 100644 --- a/doc/ci/README.md +++ b/doc/ci/README.md @@ -120,7 +120,7 @@ Here is an collection of tutorials and guides on setting up your CI pipeline. - [Run PHP Composer & NPM scripts then deploy them to a staging server](examples/deployment/composer-npm-deploy.md) - [Analyze code quality with the Code Climate CLI](examples/code_climate.md) - **Articles** - - [How to test and deploy Laravel/PHP applications with GitLab CI/CD and Envoy](../articles/laravel_with_gitlab_and_envoy/index.md) + - [How to test and deploy Laravel/PHP applications with GitLab CI/CD and Envoy](examples/laravel_with_gitlab_and_envoy/index.md) - [How to deploy Maven projects to Artifactory with GitLab CI/CD](examples/artifactory_and_gitlab/index.md) - [Automated Debian packaging](https://about.gitlab.com/2016/10/12/automated-debian-package-build-with-gitlab-ci/) - [Spring boot application with GitLab CI and Kubernetes](https://about.gitlab.com/2016/12/14/continuous-delivery-of-a-spring-boot-application-with-gitlab-ci-and-kubernetes/) diff --git a/doc/ci/examples/README.md b/doc/ci/examples/README.md index 25a0c5dcff5..d4590d0f495 100644 --- a/doc/ci/examples/README.md +++ b/doc/ci/examples/README.md @@ -16,6 +16,7 @@ Apart from those, here is an collection of tutorials and guides on setting up yo - [Testing a PHP application](php.md) - [Run PHP Composer & NPM scripts then deploy them to a staging server](deployment/composer-npm-deploy.md) +- [How to test and deploy Laravel/PHP applications with GitLab CI/CD and Envoy](laravel_with_gitlab_and_envoy/index.md) ### Ruby diff --git a/doc/articles/laravel_with_gitlab_and_envoy/img/container_registry_checkbox.png b/doc/ci/examples/laravel_with_gitlab_and_envoy/img/container_registry_checkbox.png Binary files differindex a56c07a0da7..a56c07a0da7 100644 --- a/doc/articles/laravel_with_gitlab_and_envoy/img/container_registry_checkbox.png +++ b/doc/ci/examples/laravel_with_gitlab_and_envoy/img/container_registry_checkbox.png diff --git a/doc/articles/laravel_with_gitlab_and_envoy/img/container_registry_page_empty_image.png b/doc/ci/examples/laravel_with_gitlab_and_envoy/img/container_registry_page_empty_image.png Binary files differindex b1406fed6b8..b1406fed6b8 100644 --- a/doc/articles/laravel_with_gitlab_and_envoy/img/container_registry_page_empty_image.png +++ b/doc/ci/examples/laravel_with_gitlab_and_envoy/img/container_registry_page_empty_image.png diff --git a/doc/articles/laravel_with_gitlab_and_envoy/img/container_registry_page_with_image.jpg b/doc/ci/examples/laravel_with_gitlab_and_envoy/img/container_registry_page_with_image.jpg Binary files differindex d1f0cbc08ab..d1f0cbc08ab 100644 --- a/doc/articles/laravel_with_gitlab_and_envoy/img/container_registry_page_with_image.jpg +++ b/doc/ci/examples/laravel_with_gitlab_and_envoy/img/container_registry_page_with_image.jpg diff --git a/doc/articles/laravel_with_gitlab_and_envoy/img/deploy_keys_page.png b/doc/ci/examples/laravel_with_gitlab_and_envoy/img/deploy_keys_page.png Binary files differindex 9aae11b8679..9aae11b8679 100644 --- a/doc/articles/laravel_with_gitlab_and_envoy/img/deploy_keys_page.png +++ b/doc/ci/examples/laravel_with_gitlab_and_envoy/img/deploy_keys_page.png diff --git a/doc/articles/laravel_with_gitlab_and_envoy/img/environment_page.png b/doc/ci/examples/laravel_with_gitlab_and_envoy/img/environment_page.png Binary files differindex a06b6d417cd..a06b6d417cd 100644 --- a/doc/articles/laravel_with_gitlab_and_envoy/img/environment_page.png +++ b/doc/ci/examples/laravel_with_gitlab_and_envoy/img/environment_page.png diff --git a/doc/articles/laravel_with_gitlab_and_envoy/img/environments_page.png b/doc/ci/examples/laravel_with_gitlab_and_envoy/img/environments_page.png Binary files differindex d357ecda7d2..d357ecda7d2 100644 --- a/doc/articles/laravel_with_gitlab_and_envoy/img/environments_page.png +++ b/doc/ci/examples/laravel_with_gitlab_and_envoy/img/environments_page.png diff --git a/doc/articles/laravel_with_gitlab_and_envoy/img/laravel_welcome_page.png b/doc/ci/examples/laravel_with_gitlab_and_envoy/img/laravel_welcome_page.png Binary files differindex 3bb21fd12b4..3bb21fd12b4 100644 --- a/doc/articles/laravel_with_gitlab_and_envoy/img/laravel_welcome_page.png +++ b/doc/ci/examples/laravel_with_gitlab_and_envoy/img/laravel_welcome_page.png diff --git a/doc/articles/laravel_with_gitlab_and_envoy/img/laravel_with_gitlab_and_envoy.png b/doc/ci/examples/laravel_with_gitlab_and_envoy/img/laravel_with_gitlab_and_envoy.png Binary files differindex bc188f83fb1..bc188f83fb1 100644 --- a/doc/articles/laravel_with_gitlab_and_envoy/img/laravel_with_gitlab_and_envoy.png +++ b/doc/ci/examples/laravel_with_gitlab_and_envoy/img/laravel_with_gitlab_and_envoy.png diff --git a/doc/articles/laravel_with_gitlab_and_envoy/img/pipeline_page.png b/doc/ci/examples/laravel_with_gitlab_and_envoy/img/pipeline_page.png Binary files differindex baf8dec499c..baf8dec499c 100644 --- a/doc/articles/laravel_with_gitlab_and_envoy/img/pipeline_page.png +++ b/doc/ci/examples/laravel_with_gitlab_and_envoy/img/pipeline_page.png diff --git a/doc/articles/laravel_with_gitlab_and_envoy/img/pipelines_page.png b/doc/ci/examples/laravel_with_gitlab_and_envoy/img/pipelines_page.png Binary files differindex d96c43bcf16..d96c43bcf16 100644 --- a/doc/articles/laravel_with_gitlab_and_envoy/img/pipelines_page.png +++ b/doc/ci/examples/laravel_with_gitlab_and_envoy/img/pipelines_page.png diff --git a/doc/articles/laravel_with_gitlab_and_envoy/img/pipelines_page_deploy_button.png b/doc/ci/examples/laravel_with_gitlab_and_envoy/img/pipelines_page_deploy_button.png Binary files differindex 997db10189f..997db10189f 100644 --- a/doc/articles/laravel_with_gitlab_and_envoy/img/pipelines_page_deploy_button.png +++ b/doc/ci/examples/laravel_with_gitlab_and_envoy/img/pipelines_page_deploy_button.png diff --git a/doc/articles/laravel_with_gitlab_and_envoy/img/production_server_app_directory.png b/doc/ci/examples/laravel_with_gitlab_and_envoy/img/production_server_app_directory.png Binary files differindex 6dbc29fc25c..6dbc29fc25c 100644 --- a/doc/articles/laravel_with_gitlab_and_envoy/img/production_server_app_directory.png +++ b/doc/ci/examples/laravel_with_gitlab_and_envoy/img/production_server_app_directory.png diff --git a/doc/articles/laravel_with_gitlab_and_envoy/img/production_server_current_directory.png b/doc/ci/examples/laravel_with_gitlab_and_envoy/img/production_server_current_directory.png Binary files differindex 8a6dcccfa38..8a6dcccfa38 100644 --- a/doc/articles/laravel_with_gitlab_and_envoy/img/production_server_current_directory.png +++ b/doc/ci/examples/laravel_with_gitlab_and_envoy/img/production_server_current_directory.png diff --git a/doc/articles/laravel_with_gitlab_and_envoy/img/secret_variables_page.png b/doc/ci/examples/laravel_with_gitlab_and_envoy/img/secret_variables_page.png Binary files differindex 658c0b5bcac..658c0b5bcac 100644 --- a/doc/articles/laravel_with_gitlab_and_envoy/img/secret_variables_page.png +++ b/doc/ci/examples/laravel_with_gitlab_and_envoy/img/secret_variables_page.png diff --git a/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md b/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md new file mode 100644 index 00000000000..e1aff6fdf36 --- /dev/null +++ b/doc/ci/examples/laravel_with_gitlab_and_envoy/index.md @@ -0,0 +1,684 @@ +--- +redirect_from: 'https://docs.gitlab.com/ee/articles/laravel_with_gitlab_and_envoy/index.html' +--- + +# Test and deploy Laravel applications with GitLab CI/CD and Envoy + +> **[Article Type](../../../development/writing_documentation.md#types-of-technical-articles):** tutorial || +> **Level:** intermediary || +> **Author:** [Mehran Rasulian](https://gitlab.com/mehranrasulian) || +> **Publication date:** 2017-08-31 + +## Introduction + +GitLab features our applications with Continuous Integration, and it is possible to easily deploy the new code changes to the production server whenever we want. + +In this tutorial, we'll show you how to initialize a [Laravel](http://laravel.com/) application and setup our [Envoy](https://laravel.com/docs/envoy) tasks, then we'll jump into see how to test and deploy it with [GitLab CI/CD](../README.md) via [Continuous Delivery](https://about.gitlab.com/2016/08/05/continuous-integration-delivery-and-deployment-with-gitlab/). + +We assume you have a basic experience with Laravel, Linux servers, +and you know how to use GitLab. + +Laravel is a high quality web framework written in PHP. +It has a great community with a [fantastic documentation](https://laravel.com/docs). +Aside from the usual routing, controllers, requests, responses, views, and (blade) templates, out of the box Laravel provides plenty of additional services such as cache, events, localization, authentication and many others. + +We will use [Envoy](https://laravel.com/docs/master/envoy) as an SSH task runner based on PHP. +It uses a clean, minimal [Blade syntax](https://laravel.com/docs/blade) to setup tasks that can run on remote servers, such as, cloning your project from the repository, installing the Composer dependencies, and running [Artisan commands](https://laravel.com/docs/artisan). + +## Initialize our Laravel app on GitLab + +We assume [you have installed a new laravel project](https://laravel.com/docs/installation#installation), so let's start with a unit test, and initialize Git for the project. + +### Unit Test + +Every new installation of Laravel (currently 5.4) comes with two type of tests, 'Feature' and 'Unit', placed in the tests directory. +Here's a unit test from `test/Unit/ExampleTest.php`: + +```php +<?php + +namespace Tests\Unit; + +... + +class ExampleTest extends TestCase +{ + public function testBasicTest() + { + $this->assertTrue(true); + } +} +``` + +This test is as simple as asserting that the given value is true. + +Laravel uses `PHPUnit` for tests by default. +If we run `vendor/bin/phpunit` we should see the green output: + +```bash +vendor/bin/phpunit +OK (1 test, 1 assertions) +``` + +This test will be used later for continuously testing our app with GitLab CI/CD. + +### Push to GitLab + +Since we have our app up and running locally, it's time to push the codebase to our remote repository. +Let's create [a new project](../../../gitlab-basics/create-project.md) in GitLab named `laravel-sample`. +After that, follow the command line instructions displayed on the project's homepage to initiate the repository on our machine and push the first commit. + + +```bash +cd laravel-sample +git init +git remote add origin git@gitlab.example.com:<USERNAME>/laravel-sample.git +git add . +git commit -m 'Initial Commit' +git push -u origin master +``` + +## Configure the production server + +Before we begin setting up Envoy and GitLab CI/CD, let's quickly make sure the production server is ready for deployment. +We have installed LEMP stack which stands for Linux, Nginx, MySQL and PHP on our Ubuntu 16.04. + +### Create a new user + +Let's now create a new user that will be used to deploy our website and give it +the needed permissions using [Linux ACL](https://serversforhackers.com/video/linux-acls): + +```bash +# Create user deployer +sudo adduser deployer +# Give the read-write-execute permissions to deployer user for directory /var/www +sudo setfacl -R -m u:deployer:rwx /var/www +``` + +If you don't have ACL installed on your Ubuntu server, use this command to install it: + +```bash +sudo apt install acl +``` + +### Add SSH key + +Let's suppose we want to deploy our app to the production server from a private repository on GitLab. First, we need to [generate a new SSH key pair **with no passphrase**](../../../ssh/README.md) for the deployer user. + +After that, we need to copy the private key, which will be used to connect to our server as the deployer user with SSH, to be able to automate our deployment process: + +```bash +# As the deployer user on server +# +# Copy the content of public key to authorized_keys +cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys +# Copy the private key text block +cat ~/.ssh/id_rsa +``` + +Now, let's add it to your GitLab project as a [secret variable](../../variables/README.md#secret-variables). +Secret variables are user-defined variables and are stored out of `.gitlab-ci.yml`, for security purposes. +They can be added per project by navigating to the project's **Settings** > **CI/CD**. + + + +To the field **KEY**, add the name `SSH_PRIVATE_KEY`, and to the **VALUE** field, paste the private key you've copied earlier. +We'll use this variable in the `.gitlab-ci.yml` later, to easily connect to our remote server as the deployer user without entering its password. + +We also need to add the public key to **Project** > **Settings** > **Repository** as [Deploy Keys](../../../ssh/README.md/#deploy-keys), which gives us the ability to access our repository from the server through [SSH protocol](../../../gitlab-basics/command-line-commands.md/#start-working-on-your-project). + + +```bash +# As the deployer user on the server +# +# Copy the public key +cat ~/.ssh/id_rsa.pub +``` + + + +To the field **Title**, add any name you want, and paste the public key into the **Key** field. + +Now, let's clone our repository on the server just to make sure the `deployer` user has access to the repository. + +```bash +# As the deployer user on server +# +git clone git@gitlab.example.com:<USERNAME>/laravel-sample.git +``` + +>**Note:** +Answer **yes** if asked `Are you sure you want to continue connecting (yes/no)?`. +It adds GitLab.com to the known hosts. + +### Configuring Nginx + +Now, let's make sure our web server configuration points to the `current/public` rather than `public`. + +Open the default Nginx server block configuration file by typing: + +```bash +sudo nano /etc/nginx/sites-available/default +``` + +The configuration should be like this. + +``` +server { + root /var/www/app/current/public; + server_name example.com; + # Rest of the configuration +} +``` + +>**Note:** +You may replace the app's name in `/var/www/app/current/public` with the folder name of your application. + +## Setting up Envoy + +So we have our Laravel app ready for production. +The next thing is to use Envoy to perform the deploy. + +To use Envoy, we should first install it on our local machine [using the given instructions by Laravel](https://laravel.com/docs/envoy/#introduction). + +### How Envoy works + +The pros of Envoy is that it doesn't require Blade engine, it just uses Blade syntax to define tasks. +To start, we create an `Envoy.blade.php` in the root of our app with a simple task to test Envoy. + + +```php +@servers(['web' => 'remote_username@remote_host']) + +@task('list', [on => 'web']) + ls -l +@endtask +``` + +As you may expect, we have an array within `@servers` directive at the top of the file, which contains a key named `web` with a value of the server's address (e.g. `deployer@192.168.1.1`). +Then within our `@task` directive we define the bash commands that should be run on the server when the task is executed. + +On the local machine use the `run` command to run Envoy tasks. + +```bash +envoy run list +``` + +It should execute the `list` task we defined earlier, which connects to the server and lists directory contents. + +Envoy is not a dependency of Laravel, therefore you can use it for any PHP application. + +### Zero downtime deployment + +Every time we deploy to the production server, Envoy downloads the latest release of our app from GitLab repository and replace it with preview's release. +Envoy does this without any [downtime](https://en.wikipedia.org/wiki/Downtime), +so we don't have to worry during the deployment while someone might be reviewing the site. +Our deployment plan is to clone the latest release from GitLab repository, install the Composer dependencies and finally, activate the new release. + +#### @setup directive + +The first step of our deployment process is to define a set of variables within [@setup](https://laravel.com/docs/envoy/#setup) directive. +You may change the `app` to your application's name: + + +```php +... + +@setup + $repository = 'git@gitlab.example.com:<USERNAME>/laravel-sample.git'; + $releases_dir = '/var/www/app/releases'; + $app_dir = '/var/www/app'; + $release = date('YmdHis'); + $new_release_dir = $releases_dir .'/'. $release; +@endsetup + +... +``` + +- `$repository` is the address of our repository +- `$releases_dir` directory is where we deploy the app +- `$app_dir` is the actual location of the app that is live on the server +- `$release` contains a date, so every time that we deploy a new release of our app, we get a new folder with the current date as name +- `$new_release_dir` is the full path of the new release which is used just to make the tasks cleaner + +#### @story directive + +The [@story](https://laravel.com/docs/envoy/#stories) directive allows us define a list of tasks that can be run as a single task. +Here we have three tasks called `clone_repository`, `run_composer`, `update_symlinks`. These variables are usable to making our task's codes more cleaner: + + +```php +... + +@story('deploy') + clone_repository + run_composer + update_symlinks +@endstory + +... +``` + +Let's create these three tasks one by one. + +#### Clone the repository + +The first task will create the `releases` directory (if it doesn't exist), and then clone the `master` branch of the repository (by default) into the new release directory, given by the `$new_release_dir` variable. +The `releases` directory will hold all our deployments: + +```php +... + +@task('clone_repository') + echo 'Cloning repository' + [ -d {{ $releases_dir }} ] || mkdir {{ $releases_dir }} + git clone --depth 1 {{ $repository }} {{ $new_release_dir }} +@endtask + +... +``` + +While our project grows, its Git history will be very very long over time. +Since we are creating a directory per release, it might not be necessary to have the history of the project downloaded for each release. +The `--depth 1` option is a great solution which saves systems time and disk space as well. + +#### Installing dependencies with Composer + +As you may know, this task just navigates to the new release directory and runs Composer to install the application dependencies: + +```php +... + +@task('run_composer') + echo "Starting deployment ({{ $release }})" + cd {{ $new_release_dir }} + composer install --prefer-dist --no-scripts -q -o +@endtask + +... +``` + +#### Activate new release + +Next thing to do after preparing the requirements of our new release, is to remove the storage directory from it and to create two symbolic links to point the application's `storage` directory and `.env` file to the new release. +Then, we need to create another symbolic link to the new release with the name of `current` placed in the app directory. +The `current` symbolic link always points to the latest release of our app: + +```php +... + +@task('update_symlinks') + echo "Linking storage directory" + rm -rf {{ $new_release_dir }}/storage + ln -nfs {{ $app_dir }}/storage {{ $new_release_dir }}/storage + + echo 'Linking .env file' + ln -nfs {{ $app_dir }}/.env {{ $new_release_dir }}/.env + + echo 'Linking current release' + ln -nfs {{ $new_release_dir }} {{ $app_dir }}/current +@endtask +``` + +As you see, we use `-nfs` as an option for `ln` command, which says that the `storage`, `.env` and `current` no longer points to the preview's release and will point them to the new release by force (`f` from `-nfs` means force), which is the case when we are doing multiple deployments. + +### Full script + +The script is ready, but make sure to change the `deployer@192.168.1.1` to your server and also change `/var/www/app` with the directory you want to deploy your app. + +At the end, our `Envoy.blade.php` file will look like this: + +```php +@servers(['web' => 'deployer@192.168.1.1']) + +@setup + $repository = 'git@gitlab.example.com:<USERNAME>/laravel-sample.git'; + $releases_dir = '/var/www/app/releases'; + $app_dir = '/var/www/app'; + $release = date('YmdHis'); + $new_release_dir = $releases_dir .'/'. $release; +@endsetup + +@story('deploy') + clone_repository + run_composer + update_symlinks +@endstory + +@task('clone_repository') + echo 'Cloning repository' + [ -d {{ $releases_dir }} ] || mkdir {{ $releases_dir }} + git clone --depth 1 {{ $repository }} {{ $new_release_dir }} +@endtask + +@task('run_composer') + echo "Starting deployment ({{ $release }})" + cd {{ $new_release_dir }} + composer install --prefer-dist --no-scripts -q -o +@endtask + +@task('update_symlinks') + echo "Linking storage directory" + rm -rf {{ $new_release_dir }}/storage + ln -nfs {{ $app_dir }}/storage {{ $new_release_dir }}/storage + + echo 'Linking .env file' + ln -nfs {{ $app_dir }}/.env {{ $new_release_dir }}/.env + + echo 'Linking current release' + ln -nfs {{ $new_release_dir }} {{ $app_dir }}/current +@endtask +``` + +One more thing we should do before any deployment is to manually copy our application `storage` folder to the `/var/www/app` directory on the server for the first time. +You might want to create another Envoy task to do that for you. +We also create the `.env` file in the same path to setup our production environment variables for Laravel. +These are persistent data and will be shared to every new release. + +Now, we would need to deploy our app by running `envoy run deploy`, but it won't be necessary since GitLab can handle that for us with CI's [environments](../../environments.md), which will be described [later](#setting-up-gitlab-ci-cd) in this tutorial. + +Now it's time to commit [Envoy.blade.php](https://gitlab.com/mehranrasulian/laravel-sample/blob/master/Envoy.blade.php) and push it to the `master` branch. +To keep things simple, we commit directly to `master`, without using [feature-branches](../../../workflow/gitlab_flow.md/#github-flow-as-a-simpler-alternative) since collaboration is beyond the scope of this tutorial. +In a real world project, teams may use [Issue Tracker](../../../user/project/issues/index.md) and [Merge Requests](../../../user/project/merge_requests/index.md) to move their code across branches: + +```bash +git add Envoy.blade.php +git commit -m 'Add Envoy' +git push origin master +``` + +## Continuous Integration with GitLab + +We have our app ready on GitLab, and we also can deploy it manually. +But let's take a step forward to do it automatically with [Continuous Delivery](https://about.gitlab.com/2016/08/05/continuous-integration-delivery-and-deployment-with-gitlab/#continuous-delivery) method. +We need to check every commit with a set of automated tests to become aware of issues at the earliest, and then, we can deploy to the target environment if we are happy with the result of the tests. + +[GitLab CI/CD](../../README.md) allows us to use [Docker](https://docker.com/) engine to handle the process of testing and deploying our app. +In the case you're not familiar with Docker, refer to [How to Automate Docker Deployments](http://paislee.io/how-to-automate-docker-deployments/). + +To be able to build, test, and deploy our app with GitLab CI/CD, we need to prepare our work environment. +To do that, we'll use a Docker image which has the minimum requirements that a Laravel app needs to run. +[There are other ways](../php.md/#test-php-projects-using-the-docker-executor) to do that as well, but they may lead our builds run slowly, which is not what we want when there are faster options to use. + +With Docker images our builds run incredibly faster! + +### Create a Container Image + +Let's create a [Dockerfile](https://gitlab.com/mehranrasulian/laravel-sample/blob/master/Dockerfile) in the root directory of our app with the following content: + +```bash +# Set the base image for subsequent instructions +FROM php:7.1 + +# Update packages +RUN apt-get update + +# Install PHP and composer dependencies +RUN apt-get install -qq git curl libmcrypt-dev libjpeg-dev libpng-dev libfreetype6-dev libbz2-dev + +# Clear out the local repository of retrieved package files +RUN apt-get clean + +# Install needed extensions +# Here you can install any other extension that you need during the test and deployment process +RUN docker-php-ext-install mcrypt pdo_mysql zip + +# Install Composer +RUN curl --silent --show-error https://getcomposer.org/installer | php -- --install-dir=/usr/local/bin --filename=composer + +# Install Laravel Envoy +RUN composer global require "laravel/envoy=~1.0" +``` + +We added the [official PHP 7.1 Docker image](https://hub.docker.com/r/_/php/), which consist of a minimum installation of Debian Jessie with PHP pre-installed, and works perfectly for our use case. + +We used `docker-php-ext-install` (provided by the official PHP Docker image) to install the PHP extensions we need. + +#### Setting Up GitLab Container Registry + +Now that we have our `Dockerfile` let's build and push it to our [GitLab Container Registry](../../../user/project/container_registry.md). + +> The registry is the place to store and tag images for later use. Developers may want to maintain their own registry for private, company images, or for throw-away images used only in testing. Using GitLab Container Registry means you don't need to set up and administer yet another service or use a public registry. + +On your GitLab project repository navigate to the **Registry** tab. + + + +You may need to [enable Container Registry](../../../user/project/container_registry.md#enable-the-container-registry-for-your-project) to your project to see this tab. You'll find it under your project's **Settings > General > Sharing and permissions**. + + + +To start using Container Registry on our machine, we first need to login to the GitLab registry using our GitLab username and password: + +```bash +docker login registry.gitlab.com +``` +Then we can build and push our image to GitLab: + +```bash +docker build -t registry.gitlab.com/<USERNAME>/laravel-sample . + +docker push registry.gitlab.com/<USERNAME>/laravel-sample +``` + +>**Note:** +To run the above commands, we first need to have [Docker](https://docs.docker.com/engine/installation/) installed on our machine. + +Congratulations! You just pushed the first Docker image to the GitLab Registry, and if you refresh the page you should be able to see it: + + + +>**Note:** +You can also [use GitLab CI/CD](https://about.gitlab.com/2016/05/23/gitlab-container-registry/#use-with-gitlab-ci) to build and push your Docker images, rather than doing that on your machine. + +We'll use this image further down in the `.gitlab-ci.yml` configuration file to handle the process of testing and deploying our app. + +Let's commit the `Dockerfile` file. + +```bash +git add Dockerfile +git commit -m 'Add Dockerfile' +git push origin master +``` + +### Setting up GitLab CI/CD + +In order to build and test our app with GitLab CI/CD, we need a file called `.gitlab-ci.yml` in our repository's root. It is similar to Circle CI and Travis CI, but built-in GitLab. + +Our `.gitlab-ci.yml` file will look like this: + +```yaml +image: registry.gitlab.com/<USERNAME>/laravel-sample:latest + +services: + - mysql:5.7 + +variables: + MYSQL_DATABASE: homestead + MYSQL_ROOT_PASSWORD: secret + DB_HOST: mysql + DB_USERNAME: root + +stages: + - test + - deploy + +unit_test: + stage: test + script: + - cp .env.example .env + - composer install + - php artisan key:generate + - php artisan migrate + - vendor/bin/phpunit + +deploy_production: + stage: deploy + script: + - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' + - eval $(ssh-agent -s) + - ssh-add <(echo "$SSH_PRIVATE_KEY") + - mkdir -p ~/.ssh + - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' + + - ~/.composer/vendor/bin/envoy run deploy + environment: + name: production + url: http://192.168.1.1 + when: manual + only: + - master +``` + +That's a lot to take in, isn't it? Let's run through it step by step. + +#### Image and Services + +[GitLab Runners](../../runners/README.md) run the script defined by `.gitlab-ci.yml`. +The `image` keyword tells the Runners which image to use. +The `services` keyword defines additional images [that are linked to the main image](../../docker/using_docker_images.md/#what-is-a-service). +Here we use the container image we created before as our main image and also use MySQL 5.7 as a service. + +```yaml +image: registry.gitlab.com/<USERNAME>/laravel-sample:latest + +services: + - mysql:5.7 + +... +``` + +>**Note:** +If you wish to test your app with different PHP versions and [database management systems](../../services/README.md), you can define different `image` and `services` keywords for each test job. + +#### Variables + +GitLab CI/CD allows us to use [environment variables](../../yaml/README.md#variables) in our jobs. +We defined MySQL as our database management system, which comes with a superuser root created by default. + +So we should adjust the configuration of MySQL instance by defining `MYSQL_DATABASE` variable as our database name and `MYSQL_ROOT_PASSWORD` variable as the password of `root`. +Find out more about MySQL variables at the [official MySQL Docker Image](https://hub.docker.com/r/_/mysql/). + +Also set the variables `DB_HOST` to `mysql` and `DB_USERNAME` to `root`, which are Laravel specific variables. +We define `DB_HOST` as `mysql` instead of `127.0.0.1`, as we use MySQL Docker image as a service which [is linked to the main Docker image](../../docker/using_docker_images.md/#how-services-are-linked-to-the-build). + +```yaml +... + +variables: + MYSQL_DATABASE: homestead + MYSQL_ROOT_PASSWORD: secret + DB_HOST: mysql + DB_USERNAME: root + +... +``` + +#### Unit Test as the first job + +We defined the required shell scripts as an array of the [script](../../yaml/README.md#script) variable to be executed when running `unit_test` job. + +These scripts are some Artisan commands to prepare the Laravel, and, at the end of the script, we'll run the tests by `PHPUnit`. + +```yaml +... + +unit_test: + script: + # Install app dependencies + - composer install + # Setup .env + - cp .env.example .env + # Generate an environment key + - php artisan key:generate + # Run migrations + - php artisan migrate + # Run tests + - vendor/bin/phpunit + +... +``` + +#### Deploy to production + +The job `deploy_production` will deploy the app to the production server. +To deploy our app with Envoy, we had to set up the `$SSH_PRIVATE_KEY` variable as an [SSH private key](../../ssh_keys/README.md/#ssh-keys-when-using-the-docker-executor). +If the SSH keys have added successfully, we can run Envoy. + +As mentioned before, GitLab supports [Continuous Delivery](https://about.gitlab.com/2016/08/05/continuous-integration-delivery-and-deployment-with-gitlab/#continuous-delivery) methods as well. +The [environment](../../yaml/README.md#environment) keyword tells GitLab that this job deploys to the `production` environment. +The `url` keyword is used to generate a link to our application on the GitLab Environments page. +The `only` keyword tells GitLab CI that the job should be executed only when the pipeline is building the `master` branch. +Lastly, `when: manual` is used to turn the job from running automatically to a manual action. + +```yaml +... + +deploy_production: + script: + # Add the private SSH key to the build environment + - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )' + - eval $(ssh-agent -s) + - ssh-add <(echo "$SSH_PRIVATE_KEY") + - mkdir -p ~/.ssh + - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' + + # Run Envoy + - ~/.composer/vendor/bin/envoy run deploy + + environment: + name: production + url: http://192.168.1.1 + when: manual + only: + - master +``` + +You may also want to add another job for [staging environment](https://about.gitlab.com/2016/08/26/ci-deployment-and-environments), to final test your application before deploying to production. + +### Turn on GitLab CI/CD + +We have prepared everything we need to test and deploy our app with GitLab CI/CD. +To do that, commit and push `.gitlab-ci.yml` to the `master` branch. It will trigger a pipeline, which you can watch live under your project's **Pipelines**. + + + +Here we see our **Test** and **Deploy** stages. +The **Test** stage has the `unit_test` build running. +click on it to see the Runner's output. + + + +After our code passed through the pipeline successfully, we can deploy to our production server by clicking the **play** button on the right side. + + + +Once the deploy pipeline passed successfully, navigate to **Pipelines > Environments**. + + + +If something doesn't work as expected, you can roll back to the latest working version of your app. + + + +By clicking on the external link icon specified on the right side, GitLab opens the production website. +Our deployment successfully was done and we can see the application is live. + + + +In the case that you're interested to know how is the application directory structure on the production server after deployment, here are three directories named `current`, `releases` and `storage`. +As you know, the `current` directory is a symbolic link that points to the latest release. +The `.env` file consists of our Laravel environment variables. + + + +If you navigate to the `current` directory, you should see the application's content. +As you see, the `.env` is pointing to the `/var/www/app/.env` file and also `storage` is pointing to the `/var/www/app/storage/` directory. + + + +## Conclusion + +We configured GitLab CI to perform automated tests and used the method of [Continuous Delivery](https://continuousdelivery.com/) to deploy to production a Laravel application with Envoy, directly from the codebase. + +Envoy also was a great match to help us deploy the application without writing our custom bash script and doing Linux magics. diff --git a/doc/install/README.md b/doc/install/README.md index 540cb0d3f38..43197351db3 100644 --- a/doc/install/README.md +++ b/doc/install/README.md @@ -27,7 +27,7 @@ the hardware requirements. - [Installing in Kubernetes](kubernetes/index.md) - Install GitLab into a Kubernetes Cluster using our official Helm Chart Repository. -- [Install GitLab on OpenShift](../articles/openshift_and_gitlab/index.md) +- [Install GitLab on OpenShift](openshift_and_gitlab/index.md) - [Install GitLab on DC/OS](https://mesosphere.com/blog/gitlab-dcos/) via [GitLab-Mesosphere integration](https://about.gitlab.com/2016/09/16/announcing-gitlab-and-mesosphere/) - [Install GitLab on Azure](azure/index.md) - [Install GitLab on Google Cloud Platform](google_cloud_platform/index.md) diff --git a/doc/articles/openshift_and_gitlab/img/add-gitlab-to-project.png b/doc/install/openshift_and_gitlab/img/add-gitlab-to-project.png Binary files differindex fcad4e59ae3..fcad4e59ae3 100644 --- a/doc/articles/openshift_and_gitlab/img/add-gitlab-to-project.png +++ b/doc/install/openshift_and_gitlab/img/add-gitlab-to-project.png diff --git a/doc/articles/openshift_and_gitlab/img/add-to-project.png b/doc/install/openshift_and_gitlab/img/add-to-project.png Binary files differindex bd915a229f6..bd915a229f6 100644 --- a/doc/articles/openshift_and_gitlab/img/add-to-project.png +++ b/doc/install/openshift_and_gitlab/img/add-to-project.png diff --git a/doc/articles/openshift_and_gitlab/img/create-project-ui.png b/doc/install/openshift_and_gitlab/img/create-project-ui.png Binary files differindex e72866f252a..e72866f252a 100644 --- a/doc/articles/openshift_and_gitlab/img/create-project-ui.png +++ b/doc/install/openshift_and_gitlab/img/create-project-ui.png diff --git a/doc/articles/openshift_and_gitlab/img/gitlab-logs.png b/doc/install/openshift_and_gitlab/img/gitlab-logs.png Binary files differindex 1e24080c7df..1e24080c7df 100644 --- a/doc/articles/openshift_and_gitlab/img/gitlab-logs.png +++ b/doc/install/openshift_and_gitlab/img/gitlab-logs.png diff --git a/doc/articles/openshift_and_gitlab/img/gitlab-overview.png b/doc/install/openshift_and_gitlab/img/gitlab-overview.png Binary files differindex 3c5df0ea101..3c5df0ea101 100644 --- a/doc/articles/openshift_and_gitlab/img/gitlab-overview.png +++ b/doc/install/openshift_and_gitlab/img/gitlab-overview.png diff --git a/doc/articles/openshift_and_gitlab/img/gitlab-running.png b/doc/install/openshift_and_gitlab/img/gitlab-running.png Binary files differindex c7db691cb30..c7db691cb30 100644 --- a/doc/articles/openshift_and_gitlab/img/gitlab-running.png +++ b/doc/install/openshift_and_gitlab/img/gitlab-running.png diff --git a/doc/articles/openshift_and_gitlab/img/gitlab-scale.png b/doc/install/openshift_and_gitlab/img/gitlab-scale.png Binary files differindex 4903c7d7498..4903c7d7498 100644 --- a/doc/articles/openshift_and_gitlab/img/gitlab-scale.png +++ b/doc/install/openshift_and_gitlab/img/gitlab-scale.png diff --git a/doc/articles/openshift_and_gitlab/img/gitlab-settings.png b/doc/install/openshift_and_gitlab/img/gitlab-settings.png Binary files differindex db4360ffef0..db4360ffef0 100644 --- a/doc/articles/openshift_and_gitlab/img/gitlab-settings.png +++ b/doc/install/openshift_and_gitlab/img/gitlab-settings.png diff --git a/doc/articles/openshift_and_gitlab/img/no-resources.png b/doc/install/openshift_and_gitlab/img/no-resources.png Binary files differindex 480fb766468..480fb766468 100644 --- a/doc/articles/openshift_and_gitlab/img/no-resources.png +++ b/doc/install/openshift_and_gitlab/img/no-resources.png diff --git a/doc/articles/openshift_and_gitlab/img/openshift-infra-project.png b/doc/install/openshift_and_gitlab/img/openshift-infra-project.png Binary files differindex 8b9f85aa341..8b9f85aa341 100644 --- a/doc/articles/openshift_and_gitlab/img/openshift-infra-project.png +++ b/doc/install/openshift_and_gitlab/img/openshift-infra-project.png diff --git a/doc/articles/openshift_and_gitlab/img/pods-overview.png b/doc/install/openshift_and_gitlab/img/pods-overview.png Binary files differindex e1cf08bd217..e1cf08bd217 100644 --- a/doc/articles/openshift_and_gitlab/img/pods-overview.png +++ b/doc/install/openshift_and_gitlab/img/pods-overview.png diff --git a/doc/articles/openshift_and_gitlab/img/rc-name.png b/doc/install/openshift_and_gitlab/img/rc-name.png Binary files differindex 889e34adbec..889e34adbec 100644 --- a/doc/articles/openshift_and_gitlab/img/rc-name.png +++ b/doc/install/openshift_and_gitlab/img/rc-name.png diff --git a/doc/articles/openshift_and_gitlab/img/running-pods.png b/doc/install/openshift_and_gitlab/img/running-pods.png Binary files differindex 3fd4e56662f..3fd4e56662f 100644 --- a/doc/articles/openshift_and_gitlab/img/running-pods.png +++ b/doc/install/openshift_and_gitlab/img/running-pods.png diff --git a/doc/articles/openshift_and_gitlab/img/storage-volumes.png b/doc/install/openshift_and_gitlab/img/storage-volumes.png Binary files differindex ae1e5381faa..ae1e5381faa 100644 --- a/doc/articles/openshift_and_gitlab/img/storage-volumes.png +++ b/doc/install/openshift_and_gitlab/img/storage-volumes.png diff --git a/doc/articles/openshift_and_gitlab/img/web-console.png b/doc/install/openshift_and_gitlab/img/web-console.png Binary files differindex aa1425d4f94..aa1425d4f94 100644 --- a/doc/articles/openshift_and_gitlab/img/web-console.png +++ b/doc/install/openshift_and_gitlab/img/web-console.png diff --git a/doc/install/openshift_and_gitlab/index.md b/doc/install/openshift_and_gitlab/index.md new file mode 100644 index 00000000000..8fba44aea02 --- /dev/null +++ b/doc/install/openshift_and_gitlab/index.md @@ -0,0 +1,510 @@ +# Getting started with OpenShift Origin 3 and GitLab + +> **[Article Type](../../development/writing_documentation.html#types-of-technical-articles):** tutorial || +> **Level:** intermediary || +> **Author:** [Achilleas Pipinellis](https://gitlab.com/axil) || +> **Publication date:** 2016-06-28 + +## Introduction + +[OpenShift Origin][openshift] is an open source container application +platform created by [RedHat], based on [kubernetes] and [Docker]. That means +you can host your own PaaS for free and almost with no hassle. + +In this tutorial, we will see how to deploy GitLab in OpenShift using GitLab's +official Docker image while getting familiar with the web interface and CLI +tools that will help us achieve our goal. + +--- + +## Prerequisites + +OpenShift 3 is not yet deployed on RedHat's offered Online platform ([openshift.com]), +so in order to test it, we will use an [all-in-one Virtualbox image][vm] that is +offered by the OpenShift developers and managed by Vagrant. If you haven't done +already, go ahead and install the following components as they are essential to +test OpenShift easily: + +- [VirtualBox] +- [Vagrant] +- [OpenShift Client][oc] (`oc` for short) + +It is also important to mention that for the purposes of this tutorial, the +latest Origin release is used: + +- **oc** `v1.3.0` (must be [installed][oc-gh] locally on your computer) +- **openshift** `v1.3.0` (is pre-installed in the [VM image][vm-new]) +- **kubernetes** `v1.3.0` (is pre-installed in the [VM image][vm-new]) + +>**Note:** +If you intend to deploy GitLab on a production OpenShift cluster, there are some +limitations to bare in mind. Read on the [limitations](#current-limitations) +section for more information and follow the linked links for the relevant +discussions. + +Now that you have all batteries, let's see how easy it is to test OpenShift +on your computer. + +## Getting familiar with OpenShift Origin + +The environment we are about to use is based on CentOS 7 which comes with all +the tools needed pre-installed: Docker, kubernetes, OpenShift, etcd. + +### Test OpenShift using Vagrant + +As of this writing, the all-in-one VM is at version 1.3, and that's +what we will use in this tutorial. + +In short: + +1. Open a terminal and in a new directory run: + ```sh + vagrant init openshift/origin-all-in-one + ``` +1. This will generate a Vagrantfile based on the all-in-one VM image +1. In the same directory where you generated the Vagrantfile + enter: + + ```sh + vagrant up + ``` + +This will download the VirtualBox image and fire up the VM with some preconfigured +values as you can see in the Vagrantfile. As you may have noticed, you need +plenty of RAM (5GB in our example), so make sure you have enough. + +Now that OpenShift is setup, let's see how the web console looks like. + +### Explore the OpenShift web console + +Once Vagrant finishes its thing with the VM, you will be presented with a +message which has some important information. One of them is the IP address +of the deployed OpenShift platform and in particular <https://10.2.2.2:8443/console/>. +Open this link with your browser and accept the self-signed certificate in +order to proceed. + +Let's login as admin with username/password `admin/admin`. This is what the +landing page looks like: + + + +You can see that a number of [projects] are already created for testing purposes. + +If you head over the `openshift-infra` project, a number of services with their +respective pods are there to explore. + + + +We are not going to explore the whole interface, but if you want to learn about +the key concepts of OpenShift, read the [core concepts reference][core] in the +official documentation. + +### Explore the OpenShift CLI + +OpenShift Client (`oc`), is a powerful CLI tool that talks to the OpenShift API +and performs pretty much everything you can do from the web UI and much more. + +Assuming you have [installed][oc] it, let's explore some of its main +functionalities. + +Let's first see the version of `oc`: + +```sh +$ oc version + +oc v1.3.0 +kubernetes v1.3.0+52492b4 +``` + +With `oc help` you can see the top level arguments you can run with `oc` and +interact with your cluster, kubernetes, run applications, create projects and +much more. + +Let's login to the all-in-one VM and see how to achieve the same results like +when we visited the web console earlier. The username/password for the +administrator user is `admin/admin`. There is also a test user with username/ +password `user/user`, with limited access. Let's login as admin for the moment: + +```sh +$ oc login https://10.2.2.2:8443 + +Authentication required for https://10.2.2.2:8443 (openshift) +Username: admin +Password: +Login successful. + +You have access to the following projects and can switch between them with 'oc project <projectname>': + + * cockpit + * default (current) + * delete + * openshift + * openshift-infra + * sample + +Using project "default". +``` + +Switch to the `openshift-infra` project with: + +```sh +oc project openshift-infra +``` + +And finally, see its status: + +```sh +oc status +``` + +The last command should spit a bunch of information about the statuses of the +pods and the services, which if you look closely is what we encountered in the +second image when we explored the web console. + +You can always read more about `oc` in the [OpenShift CLI documentation][oc]. + +### Troubleshooting the all-in-one VM + +Using the all-in-one VM gives you the ability to test OpenShift whenever you +want. That means you get to play with it, shutdown the VM, and pick up where +you left off. + +Sometimes though, you may encounter some issues, like OpenShift not running +when booting up the VM. The web UI may not responding or you may see issues +when trying to login with `oc`, like: + +``` +The connection to the server 10.2.2.2:8443 was refused - did you specify the right host or port? +``` + +In that case, the OpenShift service might not be running, so in order to fix it: + +1. SSH into the VM by going to the directory where the Vagrantfile is and then + run: + + ```sh + vagrant ssh + ``` + +1. Run `systemctl` and verify by the output that the `openshift` service is not + running (it will be in red color). If that's the case start the service with: + + ```sh + sudo systemctl start openshift + ``` + +1. Verify the service is up with: + + ```sh + systemctl status openshift -l + ``` + +Now you will be able to login using `oc` (like we did before) and visit the web +console. + +## Deploy GitLab + +Now that you got a taste of what OpenShift looks like, let's deploy GitLab! + +### Create a new project + +First, we will create a new project to host our application. You can do this +either by running the CLI client: + +```bash +$ oc new-project gitlab +``` + +or by using the web interface: + + + +If you used the command line, `oc` automatically uses the new project and you +can see its status with: + +```sh +$ oc status + +In project gitlab on server https://10.2.2.2:8443 + +You have no services, deployment configs, or build configs. +Run 'oc new-app' to create an application. +``` + +If you visit the web console, you can now see `gitlab` listed in the projects list. + +The next step is to import the OpenShift template for GitLab. + +### Import the template + +The [template][templates] is basically a JSON file which describes a set of +related object definitions to be created together, as well as a set of +parameters for those objects. + +The template for GitLab resides in the Omnibus GitLab repository under the +docker directory. Let's download it locally with `wget`: + +```bash +wget https://gitlab.com/gitlab-org/omnibus-gitlab/raw/master/docker/openshift-template.json +``` + +And then let's import it in OpenShift: + +```bash +oc create -f openshift-template.json -n openshift +``` + +>**Note:** +The `-n openshift` namespace flag is a trick to make the template available to all +projects. If you recall from when we created the `gitlab` project, `oc` switched +to it automatically, and that can be verified by the `oc status` command. If +you omit the namespace flag, the application will be available only to the +current project, in our case `gitlab`. The `openshift` namespace is a global +one that the administrators should use if they want the application to be +available to all users. + +We are now ready to finally deploy GitLab! + +### Create a new application + +The next step is to use the template we previously imported. Head over to the +`gitlab` project and hit the **Add to Project** button. + + + +This will bring you to the catalog where you can find all the pre-defined +applications ready to deploy with the click of a button. Search for `gitlab` +and you will see the previously imported template: + + + +Select it, and in the following screen you will be presented with the predefined +values used with the GitLab template: + + + +Notice at the top that there are three resources to be created with this +template: + +- `gitlab-ce` +- `gitlab-ce-redis` +- `gitlab-ce-postgresql` + +While PostgreSQL and Redis are bundled in Omnibus GitLab, the template is using +separate images as you can see from [this line][line] in the template. + +The predefined values have been calculated for the purposes of testing out +GitLab in the all-in-one VM. You don't need to change anything here, hit +**Create** to start the deployment. + +If you are deploying to production you will want to change the **GitLab instance +hostname** and use greater values for the volume sizes. If you don't provide a +password for PostgreSQL, it will be created automatically. + +>**Note:** +The `gitlab.apps.10.2.2.2.xip.io` hostname that is used by default will +resolve to the host with IP `10.2.2.2` which is the IP our VM uses. It is a +trick to have distinct FQDNs pointing to services that are on our local network. +Read more on how this works in <http://xip.io>. + +Now that we configured this, let's see how to manage and scale GitLab. + +## Manage and scale GitLab + +Setting up GitLab for the first time might take a while depending on your +internet connection and the resources you have attached to the all-in-one VM. +GitLab's docker image is quite big (~500MB), so you'll have to wait until +it's downloaded and configured before you use it. + +### Watch while GitLab gets deployed + +Navigate to the `gitlab` project at **Overview**. You can notice that the +deployment is in progress by the orange color. The Docker images are being +downloaded and soon they will be up and running. + + + +Switch to the **Browse > Pods** and you will eventually see all 3 pods in a +running status. Remember the 3 resources that were to be created when we first +created the GitLab app? This is where you can see them in action. + + + +You can see GitLab being reconfigured by taking look at the logs in realtime. +Click on `gitlab-ce-2-j7ioe` (your ID will be different) and go to the **Logs** +tab. + + + +At a point you should see a _**gitlab Reconfigured!**_ message in the logs. +Navigate back to the **Overview** and hopefully all pods will be up and running. + + + +Congratulations! You can now navigate to your new shinny GitLab instance by +visiting <http://gitlab.apps.10.2.2.2.xip.io> where you will be asked to +change the root user password. Login using `root` as username and providing the +password you just set, and start using GitLab! + +### Scale GitLab with the push of a button + +If you reach to a point where your GitLab instance could benefit from a boost +of resources, you'd be happy to know that you can scale up with the push of a +button. + +In the **Overview** page just click the up arrow button in the pod where +GitLab is. The change is instant and you can see the number of [replicas] now +running scaled to 2. + + + +Upping the GitLab pods is actually like adding new application servers to your +cluster. You can see how that would work if you didn't use GitLab with +OpenShift by following the [HA documentation][ha] for the application servers. + +Bare in mind that you may need more resources (CPU, RAM, disk space) when you +scale up. If a pod is in pending state for too long, you can navigate to +**Browse > Events** and see the reason and message of the state. + + + +### Scale GitLab using the `oc` CLI + +Using `oc` is super easy to scale up the replicas of a pod. You may want to +skim through the [basic CLI operations][basic-cli] to get a taste how the CLI +commands are used. Pay extra attention to the object types as we will use some +of them and their abbreviated versions below. + +In order to scale up, we need to find out the name of the replication controller. +Let's see how to do that using the following steps. + +1. Make sure you are in the `gitlab` project: + + ```sh + oc project gitlab + ``` + +1. See what services are used for this project: + + ```sh + oc get svc + ``` + + The output will be similar to: + + ``` + NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE + gitlab-ce 172.30.243.177 <none> 22/TCP,80/TCP 5d + gitlab-ce-postgresql 172.30.116.75 <none> 5432/TCP 5d + gitlab-ce-redis 172.30.105.88 <none> 6379/TCP 5d + ``` + +1. We need to see the replication controllers of the `gitlab-ce` service. + Get a detailed view of the current ones: + + ```sh + oc describe rc gitlab-ce + ``` + + This will return a large detailed list of the current replication controllers. + Search for the name of the GitLab controller, usually `gitlab-ce-1` or if + that failed at some point and you spawned another one, it will be named + `gitlab-ce-2`. + +1. Scale GitLab using the previous information: + + ```sh + oc scale --replicas=2 replicationcontrollers gitlab-ce-2 + ``` + +1. Get the new replicas number to make sure scaling worked: + + ```sh + oc get rc gitlab-ce-2 + ``` + + which will return something like: + + ``` + NAME DESIRED CURRENT AGE + gitlab-ce-2 2 2 5d + ``` + +And that's it! We successfully scaled the replicas to 2 using the CLI. + +As always, you can find the name of the controller using the web console. Just +click on the service you are interested in and you will see the details in the +right sidebar. + + + +### Autoscaling GitLab + +In case you were wondering whether there is an option to autoscale a pod based +on the resources of your server, the answer is yes, of course there is. + +We will not expand on this matter, but feel free to read the documentation on +OpenShift's website about [autoscaling]. + +## Current limitations + +As stated in the [all-in-one VM][vm] page: + +> By default, OpenShift will not allow a container to run as root or even a +non-random container assigned userid. Most Docker images in the Dockerhub do not +follow this best practice and instead run as root. + +The all-in-one VM we are using has this security turned off so it will not +bother us. In any case, it is something to keep in mind when deploying GitLab +on a production cluster. + +In order to deploy GitLab on a production cluster, you will need to assign the +GitLab service account to the `anyuid` Security Context. + +1. Edit the Security Context: + ```sh + oc edit scc anyuid + ``` + +1. Add `system:serviceaccount:<project>:gitlab-ce-user` to the `users` section. + If you changed the Application Name from the default the user will + will be `<app-name>-user` instead of `gitlab-ce-user` + +1. Save and exit the editor + +## Conclusion + +By now, you should have an understanding of the basic OpenShift Origin concepts +and a sense of how things work using the web console or the CLI. + +GitLab was hard to install in previous versions of OpenShift, +but now that belongs to the past. Upload a template, create a project, add an +application and you are done. You are ready to login to your new GitLab instance. + +And remember that in this tutorial we just scratched the surface of what Origin +is capable of. As always, you can refer to the detailed +[documentation][openshift-docs] to learn more about deploying your own OpenShift +PaaS and managing your applications with the ease of containers. + +[RedHat]: https://www.redhat.com/en "RedHat website" +[openshift]: https://www.openshift.org "OpenShift Origin website" +[vm]: https://www.openshift.org/vm/ "OpenShift All-in-one VM" +[vm-new]: https://atlas.hashicorp.com/openshift/boxes/origin-all-in-one "Official OpenShift Vagrant box on Atlas" +[template]: https://gitlab.com/gitlab-org/omnibus-gitlab/blob/master/docker/openshift-template.json "OpenShift template for GitLab" +[openshift.com]: https://openshift.com "OpenShift Online" +[kubernetes]: http://kubernetes.io/ "Kubernetes website" +[Docker]: https://www.docker.com "Docker website" +[oc]: https://docs.openshift.org/latest/cli_reference/get_started_cli.html "Documentation - oc CLI documentation" +[VirtualBox]: https://www.virtualbox.org/wiki/Downloads "VirtualBox downloads" +[Vagrant]: https://www.vagrantup.com/downloads.html "Vagrant downloads" +[projects]: https://docs.openshift.org/latest/dev_guide/projects.html "Documentation - Projects overview" +[core]: https://docs.openshift.org/latest/architecture/core_concepts/index.html "Documentation - Core concepts of OpenShift Origin" +[templates]: https://docs.openshift.org/latest/architecture/core_concepts/templates.html "Documentation - OpenShift templates" +[old-post]: https://blog.openshift.com/deploy-gitlab-openshift/ "Old post - Deploy GitLab on OpenShift" +[line]: https://gitlab.com/gitlab-org/omnibus-gitlab/blob/658c065c8d022ce858dd63eaeeadb0b2ddc8deea/docker/openshift-template.json#L239 "GitLab - OpenShift template" +[oc-gh]: https://github.com/openshift/origin/releases/tag/v1.3.0 "Openshift 1.3.0 release on GitHub" +[ha]: ../../administration/high_availability/gitlab.html "Documentation - GitLab High Availability" +[replicas]: https://docs.openshift.org/latest/architecture/core_concepts/deployments.html#replication-controllers "Documentation - Replication controller" +[autoscaling]: https://docs.openshift.org/latest/dev_guide/pod_autoscaling.html "Documentation - Autoscale" +[basic-cli]: https://docs.openshift.org/latest/cli_reference/basic_cli_operations.html "Documentation - Basic CLI operations" +[openshift-docs]: https://docs.openshift.org "OpenShift documentation" diff --git a/doc/university/high-availability/aws/README.md b/doc/university/high-availability/aws/README.md index 54625996dff..ddc853afded 100644 --- a/doc/university/high-availability/aws/README.md +++ b/doc/university/high-availability/aws/README.md @@ -26,6 +26,10 @@ Login to your AWS account through the `My Account` dropdown on Amazon Web Services console from where we can choose all of the services we'll be using to configure our cloud infrastructure. +### Reference Architecture + + + *** ## Network diff --git a/doc/university/high-availability/aws/img/reference-arch.png b/doc/university/high-availability/aws/img/reference-arch.png Binary files differnew file mode 100644 index 00000000000..271ee5bc614 --- /dev/null +++ b/doc/university/high-availability/aws/img/reference-arch.png diff --git a/features/steps/project/commits/commits.rb b/features/steps/project/commits/commits.rb index c623a516c47..bd3011b1cd8 100644 --- a/features/steps/project/commits/commits.rb +++ b/features/steps/project/commits/commits.rb @@ -180,11 +180,13 @@ class Spinach::Features::ProjectCommits < Spinach::FeatureSteps dropdown.find(".compare-dropdown-toggle").click dropdown.find('.dropdown-menu', visible: true) dropdown.fill_in("Filter by Git revision", with: selection) + if is_commit dropdown.find('input[type="search"]').send_keys(:return) else find_link(selection, visible: true).click end + dropdown.find('.dropdown-menu', visible: false) end end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index f574858be02..c4ef2c74658 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -918,7 +918,7 @@ module API class Trigger < Grape::Entity expose :id expose :token, :description - expose :created_at, :updated_at, :deleted_at, :last_used + expose :created_at, :updated_at, :last_used expose :owner, using: Entities::UserBasic end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index d6ce368efd5..6134ad2bfc7 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -26,6 +26,7 @@ module API check_unmodified_since!(last_updated) status 204 + if block_given? yield resource else diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 8bf53939751..063f0d6599c 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -103,6 +103,7 @@ module API elsif params[:user_id] user = User.find_by(id: params[:user_id]) end + present user, with: Entities::UserSafe end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 7aa10631d53..c99fe3ab5b3 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -175,6 +175,7 @@ module API issue = ::Issues::CreateService.new(user_project, current_user, issue_params.merge(request: request, api: true)).execute + if issue.spam? render_api_error!({ error: 'Spam detected' }, 400) end diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb index 74b3376a1f3..675c963bae2 100644 --- a/lib/api/pipelines.rb +++ b/lib/api/pipelines.rb @@ -48,6 +48,7 @@ module API current_user, declared_params(include_missing: false)) .execute(:api, ignore_skip_ci: true, save_on_errors: false) + if new_pipeline.persisted? present new_pipeline, with: Entities::Pipeline else diff --git a/lib/api/project_snippets.rb b/lib/api/project_snippets.rb index 2ccda1c1aa1..5bed58c2d63 100644 --- a/lib/api/project_snippets.rb +++ b/lib/api/project_snippets.rb @@ -13,6 +13,7 @@ module API if errors[:project_access].any? error!(errors[:project_access], 422) end + not_found! end diff --git a/lib/api/projects.rb b/lib/api/projects.rb index fa222bf2b1c..653126e79ea 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -154,6 +154,7 @@ module API if project.errors[:limit_reached].present? error!(project.errors[:limit_reached], 403) end + render_validation_error!(project) end end diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 4f36bbd760f..9638c53a1df 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -15,6 +15,7 @@ module API if errors[:project_access].any? error!(errors[:project_access], 422) end + not_found! end diff --git a/lib/api/v3/entities.rb b/lib/api/v3/entities.rb index c17b6f45ed8..64758dae7d3 100644 --- a/lib/api/v3/entities.rb +++ b/lib/api/v3/entities.rb @@ -207,7 +207,7 @@ module API end class Trigger < Grape::Entity - expose :token, :created_at, :updated_at, :deleted_at, :last_used + expose :token, :created_at, :updated_at, :last_used expose :owner, using: ::API::Entities::UserBasic end diff --git a/lib/api/v3/members.rb b/lib/api/v3/members.rb index 684860b553e..de226e4e573 100644 --- a/lib/api/v3/members.rb +++ b/lib/api/v3/members.rb @@ -67,6 +67,7 @@ module API unless member member = source.add_user(params[:user_id], params[:access_level], current_user: current_user, expires_at: params[:expires_at]) end + if member.persisted? && member.valid? present member.user, with: ::API::Entities::Member, member: member else diff --git a/lib/api/v3/merge_requests.rb b/lib/api/v3/merge_requests.rb index 1d6d823f32b..0a24fea52a3 100644 --- a/lib/api/v3/merge_requests.rb +++ b/lib/api/v3/merge_requests.rb @@ -126,6 +126,7 @@ module API if status == :deprecated detail DEPRECATION_MESSAGE end + success ::API::V3::Entities::MergeRequest end get path do diff --git a/lib/api/v3/project_snippets.rb b/lib/api/v3/project_snippets.rb index c41fee32610..6ba425ba8c7 100644 --- a/lib/api/v3/project_snippets.rb +++ b/lib/api/v3/project_snippets.rb @@ -14,6 +14,7 @@ module API if errors[:project_access].any? error!(errors[:project_access], 422) end + not_found! end diff --git a/lib/api/v3/projects.rb b/lib/api/v3/projects.rb index 7c260b8d910..446f804124b 100644 --- a/lib/api/v3/projects.rb +++ b/lib/api/v3/projects.rb @@ -41,6 +41,7 @@ module API # private or internal, use the more conservative option, private. attrs[:visibility_level] = (publik == true) ? Gitlab::VisibilityLevel::PUBLIC : Gitlab::VisibilityLevel::PRIVATE end + attrs end @@ -201,6 +202,7 @@ module API if project.errors[:limit_reached].present? error!(project.errors[:limit_reached], 403) end + render_validation_error!(project) end end diff --git a/lib/api/v3/repositories.rb b/lib/api/v3/repositories.rb index f9a47101e27..5b54734bb45 100644 --- a/lib/api/v3/repositories.rb +++ b/lib/api/v3/repositories.rb @@ -14,6 +14,7 @@ module API if errors[:project_access].any? error!(errors[:project_access], 422) end + not_found! end end diff --git a/lib/api/v3/snippets.rb b/lib/api/v3/snippets.rb index 126ec72248e..85613c8ed84 100644 --- a/lib/api/v3/snippets.rb +++ b/lib/api/v3/snippets.rb @@ -97,6 +97,7 @@ module API attrs = declared_params(include_missing: false) UpdateSnippetService.new(nil, current_user, snippet, attrs).execute + if snippet.persisted? present snippet, with: ::API::Entities::PersonalSnippet else diff --git a/lib/backup/database.rb b/lib/backup/database.rb index d97e5d98229..5e6828de597 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -31,6 +31,7 @@ module Backup pgsql_args << "-n" pgsql_args << Gitlab.config.backup.pg_schema end + spawn('pg_dump', *pgsql_args, config['database'], out: compress_wr) end compress_wr.close diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 2a04c03919d..6715159a1aa 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -47,6 +47,7 @@ module Backup if File.exist?(path_to_wiki_repo) progress.print " * #{display_repo_path(wiki)} ... " + if empty_repo?(wiki) progress.puts " [SKIPPED]".color(:cyan) else diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index e25916528f4..35eadf6fa93 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -148,6 +148,7 @@ module Gitlab stream.seek(@offset) append = @offset > 0 end + start_offset = @offset open_new_tag @@ -155,6 +156,7 @@ module Gitlab stream.each_line do |line| s = StringScanner.new(line) until s.eos? + if s.scan(Gitlab::Regex.build_trace_section_regex) handle_section(s) elsif s.scan(/\e([@-_])(.*?)([@-~])/) @@ -168,6 +170,7 @@ module Gitlab else @out << s.scan(/./m) end + @offset += s.matched_size end end @@ -236,8 +239,10 @@ module Gitlab if @style_mask & STYLE_SWITCHES[:bold] != 0 fg_color.sub!(/fg-([a-z]{2,}+)/, 'fg-l-\1') end + css_classes << fg_color end + css_classes << @bg_color unless @bg_color.nil? STYLE_SWITCHES.each do |css_class, flag| diff --git a/lib/gitlab/cycle_analytics/base_query.rb b/lib/gitlab/cycle_analytics/base_query.rb index dcbdf9a64b0..8b3bc3e440d 100644 --- a/lib/gitlab/cycle_analytics/base_query.rb +++ b/lib/gitlab/cycle_analytics/base_query.rb @@ -15,7 +15,6 @@ module Gitlab query = mr_closing_issues_table.join(issue_table).on(issue_table[:id].eq(mr_closing_issues_table[:issue_id])) .join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id])) .where(issue_table[:project_id].eq(@project.id)) # rubocop:disable Gitlab/ModuleWithInstanceVariables - .where(issue_table[:deleted_at].eq(nil)) .where(issue_table[:created_at].gteq(@options[:from])) # rubocop:disable Gitlab/ModuleWithInstanceVariables # Load merge_requests diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 7b35c24d153..592a1956ceb 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -512,6 +512,7 @@ module Gitlab batch_size: 10_000, interval: 10.minutes ) + unless relation.model < EachBatch raise TypeError, 'The relation must include the EachBatch module' end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb index d32616862f0..979225dd216 100644 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb @@ -26,6 +26,7 @@ module Gitlab move_repository(project, old_full_path, new_full_path) move_repository(project, "#{old_full_path}.wiki", "#{new_full_path}.wiki") end + move_uploads(old_full_path, new_full_path) unless project.hashed_storage?(:attachments) move_pages(old_full_path, new_full_path) end diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index b669ee5b799..0f897e6316c 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -14,6 +14,7 @@ module Gitlab else @diff_lines = diff_lines end + @raw_lines = @diff_lines.map(&:text) end diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb index 37face8e7d0..d3b49b1ec75 100644 --- a/lib/gitlab/ee_compat_check.rb +++ b/lib/gitlab/ee_compat_check.rb @@ -156,12 +156,14 @@ module Gitlab %W[git apply --3way #{patch_path}] ) do |output, status| puts output + unless status.zero? @failed_files = output.lines.reduce([]) do |memo, line| if line.start_with?('error: patch failed:') file = line.sub(/\Aerror: patch failed: /, '') memo << file unless file =~ IGNORED_FILES_REGEX end + memo end diff --git a/lib/gitlab/email/handler/create_merge_request_handler.rb b/lib/gitlab/email/handler/create_merge_request_handler.rb index e2f7c1d0257..3436306e122 100644 --- a/lib/gitlab/email/handler/create_merge_request_handler.rb +++ b/lib/gitlab/email/handler/create_merge_request_handler.rb @@ -10,6 +10,7 @@ module Gitlab def initialize(mail, mail_key) super(mail, mail_key) + if m = /\A([^\+]*)\+merge-request\+(.*)/.match(mail_key.to_s) @project_path, @incoming_email_token = m.captures end diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 5e426b13ade..8953bc8c148 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -112,6 +112,7 @@ module Gitlab [bug['sCategory'], bug['sPriority']].each do |label| unless label.blank? labels << label + unless @known_labels.include?(label) create_label(label) @known_labels << label @@ -265,6 +266,7 @@ module Gitlab if content.blank? content = '*(No description has been entered for this issue)*' end + body << content body.join("\n\n") @@ -278,6 +280,7 @@ module Gitlab if content.blank? content = "*(No comment has been entered for this change)*" end + body << content if updates.any? diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index fa9bc57dd79..6ada9a145db 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -571,7 +571,21 @@ module Gitlab end def merged_branch_names(branch_names = []) - Set.new(git_merged_branch_names(branch_names)) + return [] unless root_ref + + root_sha = find_branch(root_ref)&.target + + return [] unless root_sha + + branches = gitaly_migrate(:merged_branch_names) do |is_enabled| + if is_enabled + gitaly_merged_branch_names(branch_names, root_sha) + else + git_merged_branch_names(branch_names, root_sha) + end + end + + Set.new(branches) end # Return an array of Diff objects that represent the diff @@ -654,6 +668,7 @@ module Gitlab end end end + @refs_hash end @@ -1488,14 +1503,7 @@ module Gitlab sort_branches(branches, sort_by) end - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/695 - def git_merged_branch_names(branch_names = []) - return [] unless root_ref - - root_sha = find_branch(root_ref)&.target - - return [] unless root_sha - + def git_merged_branch_names(branch_names, root_sha) git_arguments = %W[branch --merged #{root_sha} --format=%(refname:short)\ %(objectname)] + branch_names @@ -1509,6 +1517,14 @@ module Gitlab end end + def gitaly_merged_branch_names(branch_names, root_sha) + qualified_branch_names = branch_names.map { |b| "refs/heads/#{b}" } + + gitaly_ref_client.merged_branches(qualified_branch_names) + .reject { |b| b.target == root_sha } + .map(&:name) + end + def process_count_commits_options(options) if options[:from] || options[:to] ref = diff --git a/lib/gitlab/git/storage/forked_storage_check.rb b/lib/gitlab/git/storage/forked_storage_check.rb index 1307f400700..0a4e557b59b 100644 --- a/lib/gitlab/git/storage/forked_storage_check.rb +++ b/lib/gitlab/git/storage/forked_storage_check.rb @@ -27,6 +27,7 @@ module Gitlab status = nil while status.nil? + if deadline > Time.now.utc sleep(wait_time) _pid, status = Process.wait2(filesystem_check_pid, Process::WNOHANG) diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index ae1753ff0ae..37ba1af8d6f 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -52,6 +52,7 @@ module Gitlab ) response = GitalyClient.call(@repository.storage, :operation_service, :user_create_branch, request) + if response.pre_receive_error.present? raise Gitlab::Git::HooksService::PreReceiveError.new(response.pre_receive_error) end diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 5bce1009878..f8e2a27f3fe 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -14,12 +14,18 @@ module Gitlab request = Gitaly::FindAllBranchesRequest.new(repository: @gitaly_repo) response = GitalyClient.call(@storage, :ref_service, :find_all_branches, request) - response.flat_map do |message| - message.branches.map do |branch| - target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target) - Gitlab::Git::Branch.new(@repository, branch.name, branch.target.id, target_commit) - end - end + consume_find_all_branches_response(response) + end + + def merged_branches(branch_names = []) + request = Gitaly::FindAllBranchesRequest.new( + repository: @gitaly_repo, + merged_only: true, + merged_branches: branch_names.map { |s| encode_binary(s) } + ) + response = GitalyClient.call(@storage, :ref_service, :find_all_branches, request) + + consume_find_all_branches_response(response) end def default_branch_name @@ -62,7 +68,7 @@ module Gitlab request = Gitaly::FindLocalBranchesRequest.new(repository: @gitaly_repo) request.sort_by = sort_by_param(sort_by) if sort_by response = GitalyClient.call(@storage, :ref_service, :find_local_branches, request) - consume_branches_response(response) + consume_find_local_branches_response(response) end def tags @@ -151,7 +157,7 @@ module Gitlab enum_value end - def consume_branches_response(response) + def consume_find_local_branches_response(response) response.flat_map do |message| message.branches.map do |gitaly_branch| Gitlab::Git::Branch.new( @@ -164,6 +170,15 @@ module Gitlab end end + def consume_find_all_branches_response(response) + response.flat_map do |message| + message.branches.map do |branch| + target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target) + Gitlab::Git::Branch.new(@repository, branch.name, branch.target.id, target_commit) + end + end + end + def consume_tags_response(response) response.flat_map do |message| message.tags.map { |gitaly_tag| Util.gitlab_tag_from_gitaly_tag(@repository, gitaly_tag) } diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index ab38c0c3e34..46b49128140 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -302,6 +302,7 @@ module Gitlab else "#{project.namespace.full_path}/#{name}##{id}" end + text = "~~#{text}~~" if deleted text end @@ -329,6 +330,7 @@ module Gitlab if content.blank? content = "*(No comment has been entered for this change)*" end + body << content if updates.any? @@ -352,6 +354,7 @@ module Gitlab if content.blank? content = "*(No description has been entered for this issue)*" end + body << content if attachments.any? diff --git a/lib/gitlab/hook_data/issue_builder.rb b/lib/gitlab/hook_data/issue_builder.rb index e29dd0d5b0e..f9b1a3caf5e 100644 --- a/lib/gitlab/hook_data/issue_builder.rb +++ b/lib/gitlab/hook_data/issue_builder.rb @@ -7,7 +7,6 @@ module Gitlab closed_at confidential created_at - deleted_at description due_date id diff --git a/lib/gitlab/hook_data/merge_request_builder.rb b/lib/gitlab/hook_data/merge_request_builder.rb index ae9b68eb648..aff786864f2 100644 --- a/lib/gitlab/hook_data/merge_request_builder.rb +++ b/lib/gitlab/hook_data/merge_request_builder.rb @@ -5,7 +5,6 @@ module Gitlab assignee_id author_id created_at - deleted_at description head_pipeline_id id diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index c518943be59..4b5f9f3a926 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -148,6 +148,7 @@ module Gitlab else relation_hash = relation_item[sub_relation.to_s] end + [relation_hash, sub_relation] end diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 05dbaf6322c..cb711a83433 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -267,6 +267,7 @@ module Gitlab else %w[title group_id] end + finder_hash = parsed_relation_hash.slice(*finder_attributes) if label? diff --git a/lib/gitlab/kubernetes/helm/pod.rb b/lib/gitlab/kubernetes/helm/pod.rb index 233f6bf6227..97ad3c97e95 100644 --- a/lib/gitlab/kubernetes/helm/pod.rb +++ b/lib/gitlab/kubernetes/helm/pod.rb @@ -14,6 +14,7 @@ module Gitlab generate_config_map spec['volumes'] = volumes_specification end + ::Kubeclient::Resource.new(metadata: metadata, spec: spec) end diff --git a/lib/gitlab/ldap/config.rb b/lib/gitlab/ldap/config.rb index 0d9a554fc18..cde60addcf7 100644 --- a/lib/gitlab/ldap/config.rb +++ b/lib/gitlab/ldap/config.rb @@ -42,6 +42,7 @@ module Gitlab else self.class.invalid_provider(provider) end + @options = config_for(@provider) # Use @provider, not provider end diff --git a/lib/gitlab/metrics/influx_db.rb b/lib/gitlab/metrics/influx_db.rb index 877cebf6786..ef44a13df51 100644 --- a/lib/gitlab/metrics/influx_db.rb +++ b/lib/gitlab/metrics/influx_db.rb @@ -169,6 +169,7 @@ module Gitlab end end end + @pool end end diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb index fee741b47be..cc1e92480be 100644 --- a/lib/gitlab/middleware/multipart.rb +++ b/lib/gitlab/middleware/multipart.rb @@ -47,6 +47,7 @@ module Gitlab else value = decorate_params_value(value, @request.params[key], tmp_path) end + @request.update_param(key, value) end @@ -60,6 +61,7 @@ module Gitlab unless path_hash.is_a?(Hash) && path_hash.count == 1 raise "invalid path: #{path_hash.inspect}" end + path_key, path_value = path_hash.first unless value_hash.is_a?(Hash) && value_hash[path_key] diff --git a/lib/gitlab/multi_collection_paginator.rb b/lib/gitlab/multi_collection_paginator.rb index c22d0a84860..43921a8c1c0 100644 --- a/lib/gitlab/multi_collection_paginator.rb +++ b/lib/gitlab/multi_collection_paginator.rb @@ -37,6 +37,7 @@ module Gitlab else per_page - first_collection_last_page_size end + hash[page] = second_collection.page(second_collection_page) .per(per_page - paginated_first_collection(page).size) .padding(offset) diff --git a/lib/gitlab/quick_actions/extractor.rb b/lib/gitlab/quick_actions/extractor.rb index 3ebfa3bd4b8..c0878a34fb1 100644 --- a/lib/gitlab/quick_actions/extractor.rb +++ b/lib/gitlab/quick_actions/extractor.rb @@ -126,6 +126,7 @@ module Gitlab command << match_data[1] unless match_data[1].empty? commands << command end + content = substitution.perform_substitution(self, content) end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 8ad06480575..4178b436acf 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -24,6 +24,7 @@ module Gitlab # the pool will be used in a multi-threaded context size += Sidekiq.options[:concurrency] end + size end @@ -104,6 +105,7 @@ module Gitlab db_numbers = queries["db"] if queries.key?("db") config[:db] = db_numbers[0].to_i if db_numbers.any? end + config else redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url) diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index ca48c6df602..70b639501fd 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -115,6 +115,7 @@ module Gitlab else merge_requests.full_search(query) end + merge_requests.order('updated_at DESC') end diff --git a/lib/gitlab/storage_check/cli.rb b/lib/gitlab/storage_check/cli.rb index 04bf1bf1d26..9b64c8e033a 100644 --- a/lib/gitlab/storage_check/cli.rb +++ b/lib/gitlab/storage_check/cli.rb @@ -59,9 +59,11 @@ module Gitlab if response.skipped_shards.any? warnings << "Skipped shards: #{response.skipped_shards.join(', ')}" end + if response.failing_shards.any? warnings << "Failing shards: #{response.failing_shards.join(', ')}" end + logger.warn(warnings.join(' - ')) if warnings.any? end end diff --git a/lib/gitlab/testing/request_blocker_middleware.rb b/lib/gitlab/testing/request_blocker_middleware.rb index 4a8e3c2eee0..53333b9b06b 100644 --- a/lib/gitlab/testing/request_blocker_middleware.rb +++ b/lib/gitlab/testing/request_blocker_middleware.rb @@ -37,12 +37,14 @@ module Gitlab def call(env) increment_active_requests + if block_requests? block_request(env) else sleep 0.2 if slow_requests? @app.call(env) end + ensure decrement_active_requests end diff --git a/lib/gitlab/timeless.rb b/lib/gitlab/timeless.rb index b290c716f97..76a1808c8ac 100644 --- a/lib/gitlab/timeless.rb +++ b/lib/gitlab/timeless.rb @@ -9,6 +9,7 @@ module Gitlab else block.call end + ensure model.record_timestamps = original_record_timestamps end diff --git a/lib/gitlab/upgrader.rb b/lib/gitlab/upgrader.rb index 961df0468a4..3b64cb32afa 100644 --- a/lib/gitlab/upgrader.rb +++ b/lib/gitlab/upgrader.rb @@ -12,6 +12,7 @@ module Gitlab puts "You are using the latest GitLab version" else puts "Newer GitLab version is available" + answer = if ARGV.first == "-y" "yes" else @@ -77,6 +78,7 @@ module Gitlab update_commands.each do |title, cmd| puts title puts " -> #{cmd.join(' ')}" + if system(env, *cmd) puts " -> OK" else diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 5ab6cd5a4ef..0de183858aa 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -42,6 +42,7 @@ module Gitlab else raise "Unsupported action: #{action}" end + if feature_enabled params[:GitalyServer] = server end @@ -97,6 +98,9 @@ module Gitlab ) end + # If present DisableCache must be a Boolean. Otherwise workhorse ignores it. + params['DisableCache'] = true if git_archive_cache_disabled? + [ SEND_DATA_HEADER, "git-archive:#{encode(params)}" @@ -244,6 +248,10 @@ module Gitlab right_commit_id: diff_refs.head_sha } end + + def git_archive_cache_disabled? + ENV['WORKHORSE_ARCHIVE_CACHE_DISABLED'].present? || Feature.enabled?(:workhorse_archive_cache_disabled) + end end end end diff --git a/lib/system_check/simple_executor.rb b/lib/system_check/simple_executor.rb index 8b145fb4511..d268f501b4a 100644 --- a/lib/system_check/simple_executor.rb +++ b/lib/system_check/simple_executor.rb @@ -66,6 +66,7 @@ module SystemCheck if check.can_repair? $stdout.print 'Trying to fix error automatically. ...' + if check.repair! $stdout.puts 'Success'.color(:green) return diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 9dcf44fdc3e..2383bcf954b 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -46,6 +46,7 @@ namespace :gitlab do puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.color(:yellow) sleep(5) end + # Drop all tables Load the schema to ensure we don't have any newer tables # hanging out from a failed upgrade $progress.puts 'Cleaning the database ... '.color(:blue) @@ -222,6 +223,7 @@ namespace :gitlab do task restore: :environment do $progress.puts "Restoring container registry images ... ".color(:blue) + if Gitlab.config.registry.enabled Backup::Registry.new.restore $progress.puts "done".color(:green) diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 903e84359cd..31cd6bfe6e1 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -180,6 +180,7 @@ namespace :gitlab do puts "can't check, you have no projects".color(:magenta) return end + puts "" Project.find_each(batch_size: 100) do |project| @@ -210,6 +211,7 @@ namespace :gitlab do gitlab_shell_repo_base = gitlab_shell_path check_cmd = File.expand_path('bin/check', gitlab_shell_repo_base) puts "Running #{check_cmd}" + if system(check_cmd, chdir: gitlab_shell_repo_base) puts 'gitlab-shell self-check successful'.color(:green) else @@ -285,6 +287,7 @@ namespace :gitlab do return if process_count.zero? print 'Number of Sidekiq processes ... ' + if process_count == 1 puts '1'.color(:green) else diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index eb0f757aea7..04d56509ac6 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -84,6 +84,7 @@ namespace :gitlab do next unless user.ldap_user? print "#{user.name} (#{user.ldap_identity.extern_uid}) ..." + if Gitlab::LDAP::Access.allowed?(user) puts " [OK]".color(:green) else diff --git a/lib/tasks/gitlab/dev.rake b/lib/tasks/gitlab/dev.rake index ba221e44e5d..77c28615856 100644 --- a/lib/tasks/gitlab/dev.rake +++ b/lib/tasks/gitlab/dev.rake @@ -14,6 +14,7 @@ namespace :gitlab do puts "Must specify a branch as an argument".color(:red) exit 1 end + args end diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index 4507b841964..a2e68c0471b 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -5,9 +5,11 @@ namespace :gitlab do require 'toml' warn_user_is_not_gitlab + unless args.dir.present? abort %(Please specify the directory where you want to install gitaly:\n rake "gitlab:gitaly:install[/home/git/gitaly]") end + args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitaly.git') version = Gitlab::GitalyClient.expected_server_version diff --git a/lib/tasks/gitlab/list_repos.rake b/lib/tasks/gitlab/list_repos.rake index b732db9db6e..d7f28691098 100644 --- a/lib/tasks/gitlab/list_repos.rake +++ b/lib/tasks/gitlab/list_repos.rake @@ -8,6 +8,7 @@ namespace :gitlab do namespace_ids = Namespace.where(['updated_at > ?', date]).pluck(:id).sort scope = scope.where('id IN (?) OR namespace_id in (?)', project_ids, namespace_ids) end + scope.find_each do |project| base = File.join(project.repository_storage_path, project.disk_path) puts base + '.git' diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake index f44abc2b81b..a25f7ce59c7 100644 --- a/lib/tasks/gitlab/update_templates.rake +++ b/lib/tasks/gitlab/update_templates.rake @@ -10,6 +10,7 @@ namespace :gitlab do puts "This rake task is not meant fo production instances".red exit(1) end + admin = User.find_by(admin: true) unless admin diff --git a/lib/tasks/gitlab/workhorse.rake b/lib/tasks/gitlab/workhorse.rake index e7ac0b5859f..308ffb0e284 100644 --- a/lib/tasks/gitlab/workhorse.rake +++ b/lib/tasks/gitlab/workhorse.rake @@ -3,9 +3,11 @@ namespace :gitlab do desc "GitLab | Install or upgrade gitlab-workhorse" task :install, [:dir, :repo] => :environment do |t, args| warn_user_is_not_gitlab + unless args.dir.present? abort %(Please specify the directory where you want to install gitlab-workhorse:\n rake "gitlab:workhorse:install[/home/git/gitlab-workhorse]") end + args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitlab-workhorse.git') version = Gitlab::Workhorse.version diff --git a/lib/tasks/migrate/migrate_iids.rake b/lib/tasks/migrate/migrate_iids.rake index fc2cea8c016..aa2d01730d7 100644 --- a/lib/tasks/migrate/migrate_iids.rake +++ b/lib/tasks/migrate/migrate_iids.rake @@ -4,6 +4,7 @@ task migrate_iids: :environment do Issue.where(iid: nil).find_each(batch_size: 100) do |issue| begin issue.set_iid + if issue.update_attribute(:iid, issue.iid) print '.' else @@ -19,6 +20,7 @@ task migrate_iids: :environment do MergeRequest.where(iid: nil).find_each(batch_size: 100) do |mr| begin mr.set_iid + if mr.update_attribute(:iid, mr.iid) print '.' else @@ -34,6 +36,7 @@ task migrate_iids: :environment do Milestone.where(iid: nil).find_each(batch_size: 100) do |m| begin m.set_iid + if m.update_attribute(:iid, m.iid) print '.' else diff --git a/qa/README.md b/qa/README.md index 7f2dd39ff63..8fa04e80825 100644 --- a/qa/README.md +++ b/qa/README.md @@ -27,13 +27,17 @@ following call would login to a local [GDK] instance and run all specs in bin/qa Test::Instance http://localhost:3000 ``` +### Writing tests + +1. [Using page objects](qa/page/README.md) + ### Running specific tests You can also supply specific tests to run as another parameter. For example, to -test the EE license specs, you can run: +run the repository-related specs, you can execute: ``` -EE_LICENSE="<YOUR LICENSE KEY>" bin/qa Test::Instance http://localhost qa/specs/features/ee +bin/qa Test::Instance http://localhost qa/specs/features/repository/ ``` Since the arguments would be passed to `rspec`, you could use all `rspec` diff --git a/qa/qa/factory/base.rb b/qa/qa/factory/base.rb index 00851a7bece..bd66b74a164 100644 --- a/qa/qa/factory/base.rb +++ b/qa/qa/factory/base.rb @@ -1,12 +1,19 @@ +require 'forwardable' + module QA module Factory class Base + extend SingleForwardable + + def_delegators :evaluator, :dependency, :dependencies + def_delegators :evaluator, :product, :attributes + def fabricate!(*_args) raise NotImplementedError end def self.fabricate!(*args) - Factory::Product.populate!(new) do |factory| + new.tap do |factory| yield factory if block_given? dependencies.each do |name, signature| @@ -14,19 +21,37 @@ module QA end factory.fabricate!(*args) + + return Factory::Product.populate!(self) end end - def self.dependencies - @dependencies ||= {} + def self.evaluator + @evaluator ||= Factory::Base::DSL.new(self) end - def self.dependency(factory, as:, &block) - as.tap do |name| - class_eval { attr_accessor name } + class DSL + attr_reader :dependencies, :attributes + + def initialize(base) + @base = base + @dependencies = {} + @attributes = {} + end + + def dependency(factory, as:, &block) + as.tap do |name| + @base.class_eval { attr_accessor name } + + Dependency::Signature.new(factory, block).tap do |signature| + @dependencies.store(name, signature) + end + end + end - Dependency::Signature.new(factory, block).tap do |signature| - dependencies.store(name, signature) + def product(attribute, &block) + Product::Attribute.new(attribute, block).tap do |signature| + @attributes.store(attribute, signature) end end end diff --git a/qa/qa/factory/product.rb b/qa/qa/factory/product.rb index df35bbbb443..d004e642f9b 100644 --- a/qa/qa/factory/product.rb +++ b/qa/qa/factory/product.rb @@ -5,8 +5,9 @@ module QA class Product include Capybara::DSL - def initialize(factory) - @factory = factory + Attribute = Struct.new(:name, :block) + + def initialize @location = current_url end @@ -15,11 +16,13 @@ module QA end def self.populate!(factory) - raise ArgumentError unless block_given? - - yield factory - - new(factory) + new.tap do |product| + factory.attributes.each_value do |attribute| + product.instance_exec(&attribute.block).tap do |value| + product.define_singleton_method(attribute.name) { value } + end + end + end end end end diff --git a/qa/qa/factory/resource/project.rb b/qa/qa/factory/resource/project.rb index 07c2e3086d1..7df2dc6618c 100644 --- a/qa/qa/factory/resource/project.rb +++ b/qa/qa/factory/resource/project.rb @@ -13,6 +13,10 @@ module QA @description = 'My awesome project' end + product :name do + Page::Project::Show.act { project_name } + end + def fabricate! group.visit! diff --git a/qa/qa/page/README.md b/qa/qa/page/README.md new file mode 100644 index 00000000000..f72fbfeafca --- /dev/null +++ b/qa/qa/page/README.md @@ -0,0 +1,112 @@ +# Page objects in GitLab QA + +In GitLab QA we are using a known pattern, called _Page Objects_. + +This means that we have built an abstraction for all GitLab pages that we use +to drive GitLab QA scenarios. Whenever we do something on a page, like filling +in a form, or clicking a button, we do that only through a page object +associated with this area of GitLab. + +For example, when GitLab QA test harness signs in into GitLab, it needs to fill +in a user login and user password. In order to do that, we have a class, called +`Page::Main::Login` and `sign_in_using_credentials` methods, that is the only +piece of the code, that has knowledge about `user_login` and `user_password` +fields. + +## Why do we need that? + +We need page objects, because we need to reduce duplication and avoid problems +whenever someone changes some selectors in GitLab's source code. + +Imagine that we have a hundred specs in GitLab QA, and we need to sign into +GitLab each time, before we make assertions. Without a page object one would +need to rely on volatile helpers or invoke Capybara methods directly. Imagine +invoking `fill_in :user_login` in every `*_spec.rb` file / test example. + +When someone later changes `t.text_field :login` in the view associated with +this page to `t.text_field :username` it will generate a different field +identifier, what would effectively break all tests. + +Because we are using `Page::Main::Login.act { sign_in_using_credentials }` +everywhere, when we want to sign into GitLab, the page object is the single +source of truth, and we will need to update `fill_in :user_login` +to `fill_in :user_username` only in a one place. + +## What problems did we have in the past? + +We do not run QA tests for every commit, because of performance reasons, and +the time it would take to build packages and test everything. + +That is why when someone changes `t.text_field :login` to +`t.text_field :username` in the _new session_ view we won't know about this +change until our GitLab QA nightly pipeline fails, or until someone triggers +`package-qa` action in their merge request. + +Obviously such a change would break all tests. We call this problem a _fragile +tests problem_. + +In order to make GitLab QA more reliable and robust, we had to solve this +problem by introducing coupling between GitLab CE / EE views and GitLab QA. + +## How did we solve fragile tests problem? + +Currently, when you add a new `Page::Base` derived class, you will also need to +define all selectors that your page objects depends on. + +Whenever you push your code to CE / EE repository, `qa:selectors` sanity test +job is going to be run as a part of a CI pipeline. + +This test is going to validate all page objects that we have implemented in +`qa/page` directory. When it fails, you will be notified about missing +or invalid views / selectors definition. + +## How to properly implement a page object? + +We have built a DSL to define coupling between a page object and GitLab views +it is actually implemented by. See an example below. + +```ruby +module Page + module Main + class Login < Page::Base + view 'app/views/devise/passwords/edit.html.haml' do + element :password_field, 'password_field :password' + element :password_confirmation, 'password_field :password_confirmation' + element :change_password_button, 'submit "Change your password"' + end + + view 'app/views/devise/sessions/_new_base.html.haml' do + element :login_field, 'text_field :login' + element :passowrd_field, 'password_field :password' + element :sign_in_button, 'submit "Sign in"' + end + + # ... + end +end +``` + +It is possible to use `element` DSL method without value, with a String value +or with a Regexp. + +```ruby +view 'app/views/my/view.html.haml' do + # Require `f.submit "Sign in"` to be present in `my/view.html.haml + element :my_button, 'f.submit "Sign in"' + + # Match every line in `my/view.html.haml` against + # `/link_to .* "My Profile"/` regexp. + element :profile_link, /link_to .* "My Profile"/ + + # Implicitly require `.qa-logout-button` CSS class to be present in the view + element :logout_button +end +``` + +## Where to ask for help? + +If you need more information, ask for help on `#qa` channel on Slack (GitLab +Team only). + +If you are not a Team Member, and you still need help to contribute, please +open an issue in GitLab QA issue tracker. diff --git a/qa/qa/page/project/new.rb b/qa/qa/page/project/new.rb index a87313b2cb9..9b1438f76d5 100644 --- a/qa/qa/page/project/new.rb +++ b/qa/qa/page/project/new.rb @@ -2,16 +2,18 @@ module QA module Page module Project class New < Page::Base - ## - # TODO, define all selectors required by this page object - # - # See gitlab-org/gitlab-qa#154 - # - view 'app/views/projects/new.html.haml' + view 'app/views/projects/_new_project_fields.html.haml' do + element :project_namespace_select + element :project_namespace_field, 'select :namespace_id' + element :project_path, 'text_field :path' + element :project_description, 'text_area :description' + element :project_create_button, "submit 'Create project'" + end def choose_test_namespace - find('#s2id_project_namespace_id').click - find('.select2-result-label', text: Runtime::Namespace.name).click + click_element :project_namespace_select + + first('li', text: Runtime::Namespace.path).click end def choose_name(name) diff --git a/qa/qa/page/project/show.rb b/qa/qa/page/project/show.rb index 90967e6cc15..c8af5ba6280 100644 --- a/qa/qa/page/project/show.rb +++ b/qa/qa/page/project/show.rb @@ -2,15 +2,21 @@ module QA module Page module Project class Show < Page::Base - ## - # TODO, define all selectors required by this page object - # - # See gitlab-org/gitlab-qa#154 - # - view 'app/views/projects/show.html.haml' + view 'app/views/shared/_clone_panel.html.haml' do + element :clone_dropdown + element :clone_options_dropdown, '.clone-options-dropdown' + end + + view 'app/views/shared/_clone_panel.html.haml' do + element :project_repository_location, 'text_field_tag :project_clone' + end + + view 'app/views/projects/_home_panel.html.haml' do + element :project_name + end def choose_repository_clone_http - find('#clone-dropdown').click + click_element :clone_dropdown page.within('.clone-options-dropdown') do click_link('HTTP') @@ -22,7 +28,7 @@ module QA end def project_name - find('.project-title').text + find('.qa-project-name').text end def wait_for_push diff --git a/qa/qa/runtime/namespace.rb b/qa/qa/runtime/namespace.rb index b00e925986b..a72c2d21898 100644 --- a/qa/qa/runtime/namespace.rb +++ b/qa/qa/runtime/namespace.rb @@ -11,6 +11,10 @@ module QA 'qa-test-' + time.strftime('%d-%m-%Y-%H-%M-%S') end + def path + "#{sandbox_name}/#{name}" + end + def sandbox_name 'gitlab-qa-sandbox' end diff --git a/qa/qa/specs/features/project/create_spec.rb b/qa/qa/specs/features/project/create_spec.rb index 61c19378ae0..b1c07249892 100644 --- a/qa/qa/specs/features/project/create_spec.rb +++ b/qa/qa/specs/features/project/create_spec.rb @@ -4,11 +4,13 @@ module QA Runtime::Browser.visit(:gitlab, Page::Main::Login) Page::Main::Login.act { sign_in_using_credentials } - Factory::Resource::Project.fabricate! do |project| + created_project = Factory::Resource::Project.fabricate! do |project| project.name = 'awesome-project' project.description = 'create awesome project test' end + expect(created_project.name).to match /^awesome-project-\h{16}$/ + expect(page).to have_content( /Project \S?awesome-project\S+ was successfully created/ ) diff --git a/qa/spec/factory/base_spec.rb b/qa/spec/factory/base_spec.rb index a3ba0176819..90dd58e20fd 100644 --- a/qa/spec/factory/base_spec.rb +++ b/qa/spec/factory/base_spec.rb @@ -1,8 +1,9 @@ describe QA::Factory::Base do + let(:factory) { spy('factory') } + let(:product) { spy('product') } + describe '.fabricate!' do subject { Class.new(described_class) } - let(:factory) { spy('factory') } - let(:product) { spy('product') } before do allow(QA::Factory::Product).to receive(:new).and_return(product) @@ -59,30 +60,63 @@ describe QA::Factory::Base do it 'defines dependency accessors' do expect(subject.new).to respond_to :mydep, :mydep= end - end - describe 'building dependencies' do - let(:dependency) { double('dependency') } - let(:instance) { spy('instance') } + describe 'dependencies fabrication' do + let(:dependency) { double('dependency') } + let(:instance) { spy('instance') } + + subject do + Class.new(described_class) do + dependency Some::MyDependency, as: :mydep + end + end + + before do + stub_const('Some::MyDependency', dependency) + allow(subject).to receive(:new).and_return(instance) + allow(instance).to receive(:mydep).and_return(nil) + allow(QA::Factory::Product).to receive(:new) + end + + it 'builds all dependencies first' do + expect(dependency).to receive(:fabricate!).once + + subject.fabricate! + end + end + end + + describe '.product' do subject do Class.new(described_class) do - dependency Some::MyDependency, as: :mydep + product :token do + page.do_something_on_page! + 'resulting value' + end end end - before do - stub_const('Some::MyDependency', dependency) - - allow(subject).to receive(:new).and_return(instance) - allow(instance).to receive(:mydep).and_return(nil) - allow(QA::Factory::Product).to receive(:new) + it 'appends new product attribute' do + expect(subject.attributes).to be_one + expect(subject.attributes).to have_key(:token) end - it 'builds all dependencies first' do - expect(dependency).to receive(:fabricate!).once + describe 'populating fabrication product with data' do + let(:page) { spy('page') } + + before do + allow(subject).to receive(:new).and_return(factory) + allow(QA::Factory::Product).to receive(:new).and_return(product) + allow(product).to receive(:page).and_return(page) + end - subject.fabricate! + it 'populates product after fabrication' do + subject.fabricate! + + expect(page).to have_received(:do_something_on_page!) + expect(product.token).to eq 'resulting value' + end end end end diff --git a/qa/spec/factory/product_spec.rb b/qa/spec/factory/product_spec.rb index 3d9e86a641b..fdfb1ec90cc 100644 --- a/qa/spec/factory/product_spec.rb +++ b/qa/spec/factory/product_spec.rb @@ -3,19 +3,8 @@ describe QA::Factory::Product do let(:product) { spy('product') } describe '.populate!' do - it 'instantiates and yields factory' do - expect(described_class).to receive(:new).with(factory) - - described_class.populate!(factory) do |instance| - instance.something = 'string' - end - - expect(factory).to have_received(:something=).with('string') - end - it 'returns a fabrication product' do - expect(described_class).to receive(:new) - .with(factory).and_return(product) + expect(described_class).to receive(:new).and_return(product) result = described_class.populate!(factory) do |instance| instance.something = 'string' @@ -23,11 +12,6 @@ describe QA::Factory::Product do expect(result).to be product end - - it 'raises unless block given' do - expect { described_class.populate!(factory) } - .to raise_error ArgumentError - end end describe '.visit!' do @@ -37,8 +21,7 @@ describe QA::Factory::Product do allow_any_instance_of(described_class) .to receive(:visit).and_return('visited some url') - expect(described_class.new(factory).visit!) - .to eq 'visited some url' + expect(subject.visit!).to eq 'visited some url' end end end diff --git a/rubocop/cop/line_break_around_conditional_block.rb b/rubocop/cop/line_break_around_conditional_block.rb new file mode 100644 index 00000000000..3e7021e724e --- /dev/null +++ b/rubocop/cop/line_break_around_conditional_block.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +module RuboCop + module Cop + # Ensures a line break around conditional blocks. + # + # @example + # # bad + # do_something + # if condition + # do_extra_stuff + # end + # do_something_more + # + # # good + # do_something + # + # if condition + # do_extra_stuff + # end + # + # do_something_more + # + # # bad + # do_something + # unless condition + # do_extra_stuff + # end + # + # do_something_more + # + # # good + # def a_method + # if condition + # do_something + # end + # end + # + # # good + # on_block do + # if condition + # do_something + # end + # end + class LineBreakAroundConditionalBlock < RuboCop::Cop::Cop + MSG = 'Add a line break around conditional blocks' + + def on_if(node) + return if node.single_line? + return unless node.if? || node.unless? + + add_offense(node, location: :expression, message: MSG) unless previous_line_valid?(node) + add_offense(node, location: :expression, message: MSG) unless last_line_valid?(node) + end + + def autocorrect(node) + lambda do |corrector| + line = range_by_whole_lines(node.source_range) + unless previous_line_valid?(node) + corrector.insert_before(line, "\n") + end + + unless last_line_valid?(node) + corrector.insert_after(line, "\n") + end + end + end + + private + + def previous_line_valid?(node) + previous_line(node).empty? || + start_clause_line?(previous_line(node)) || + block_start?(previous_line(node)) || + begin_line?(previous_line(node)) || + assignment_line?(previous_line(node)) + end + + def last_line_valid?(node) + last_line(node).empty? || + end_line?(last_line(node)) || + end_clause_line?(last_line(node)) + end + + def previous_line(node) + processed_source[node.loc.line - 2] + end + + def last_line(node) + processed_source[node.loc.last_line] + end + + def start_clause_line?(line) + line =~ /^\s*(def|=end|#|module|class|if|unless|else|elsif|ensure|when)/ + end + + def end_clause_line?(line) + line =~ /^\s*(rescue|else|elsif|when)/ + end + + def begin_line?(line) + # an assignment followed by a begin or ust a begin + line =~ /^\s*(@?(\w|\|+|=|\[|\]|\s)+begin|begin)/ + end + + def assignment_line?(line) + line =~ /^\s*.*=/ + end + + def block_start?(line) + line.match(/ (do|{)( \|.*?\|)?\s?$/) + end + + def end_line?(line) + line =~ /^\s*(end|})/ + end + end + end +end diff --git a/rubocop/rubocop.rb b/rubocop/rubocop.rb index 2f63babc425..57af87f7fb9 100644 --- a/rubocop/rubocop.rb +++ b/rubocop/rubocop.rb @@ -1,5 +1,6 @@ require_relative 'cop/gitlab/module_with_instance_variables' require_relative 'cop/include_sidekiq_worker' +require_relative 'cop/line_break_around_conditional_block' require_relative 'cop/migration/add_column' require_relative 'cop/migration/add_concurrent_foreign_key' require_relative 'cop/migration/add_concurrent_index' diff --git a/spec/factories/protected_branches.rb b/spec/factories/protected_branches.rb index 39460834d06..60956511834 100644 --- a/spec/factories/protected_branches.rb +++ b/spec/factories/protected_branches.rb @@ -53,6 +53,7 @@ FactoryBot.define do if evaluator.default_access_level && evaluator.default_push_level protected_branch.push_access_levels.new(access_level: Gitlab::Access::MASTER) end + if evaluator.default_access_level && evaluator.default_merge_level protected_branch.merge_access_levels.new(access_level: Gitlab::Access::MASTER) end diff --git a/spec/features/issues/bulk_assignment_labels_spec.rb b/spec/features/issues/bulk_assignment_labels_spec.rb index 587ece22ec7..cf283119f36 100644 --- a/spec/features/issues/bulk_assignment_labels_spec.rb +++ b/spec/features/issues/bulk_assignment_labels_spec.rb @@ -377,6 +377,7 @@ feature 'Issues > Labels bulk assignment' do items.map do |item| click_link item end + if unmark items.map do |item| # Make sure we are unmarking the item no matter the state it has currently diff --git a/spec/fixtures/api/schemas/entities/issue.json b/spec/fixtures/api/schemas/entities/issue.json index 3d3329a3406..38467b4ca20 100644 --- a/spec/fixtures/api/schemas/entities/issue.json +++ b/spec/fixtures/api/schemas/entities/issue.json @@ -28,7 +28,6 @@ "confidential": { "type": "boolean" }, "discussion_locked": { "type": ["boolean", "null"] }, "updated_by_id": { "type": ["string", "null"] }, - "deleted_at": { "type": ["string", "null"] }, "time_estimate": { "type": "integer" }, "total_time_spent": { "type": "integer" }, "human_time_estimate": { "type": ["integer", "null"] }, diff --git a/spec/fixtures/api/schemas/entities/merge_request_widget.json b/spec/fixtures/api/schemas/entities/merge_request_widget.json index 7f662098216..05461787f06 100644 --- a/spec/fixtures/api/schemas/entities/merge_request_widget.json +++ b/spec/fixtures/api/schemas/entities/merge_request_widget.json @@ -13,7 +13,6 @@ "updated_by_id": { "type": ["string", "null"] }, "created_at": { "type": "string" }, "updated_at": { "type": "string" }, - "deleted_at": { "type": ["string", "null"] }, "time_estimate": { "type": "integer" }, "total_time_spent": { "type": "integer" }, "human_time_estimate": { "type": ["integer", "null"] }, diff --git a/spec/javascripts/notes/mock_data.js b/spec/javascripts/notes/mock_data.js index 6b608adff15..b020a1020df 100644 --- a/spec/javascripts/notes/mock_data.js +++ b/spec/javascripts/notes/mock_data.js @@ -29,7 +29,6 @@ export const noteableDataMock = { can_create_note: true, can_update: true, }, - deleted_at: null, description: '', due_date: null, human_time_estimate: null, @@ -283,7 +282,6 @@ export const loggedOutnoteableData = { "updated_by_id": 1, "created_at": "2017-02-07T10:11:18.395Z", "updated_at": "2017-08-08T10:22:51.564Z", - "deleted_at": null, "time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, diff --git a/spec/javascripts/sidebar/mock_data.js b/spec/javascripts/sidebar/mock_data.js index 3b094d20838..7bc591d2d47 100644 --- a/spec/javascripts/sidebar/mock_data.js +++ b/spec/javascripts/sidebar/mock_data.js @@ -15,7 +15,6 @@ const RESPONSE_MAP = { updated_by_id: 1, created_at: '2017-02-02T21: 49: 49.664Z', updated_at: '2017-05-03T22: 26: 03.760Z', - deleted_at: null, time_estimate: 0, total_time_spent: 0, human_time_estimate: null, @@ -153,7 +152,6 @@ const RESPONSE_MAP = { updated_by_id: 1, created_at: '2017-06-27T19:54:42.437Z', updated_at: '2017-08-18T03:39:49.222Z', - deleted_at: null, time_estimate: 0, total_time_spent: 0, human_time_estimate: null, diff --git a/spec/javascripts/vue_mr_widget/mock_data.js b/spec/javascripts/vue_mr_widget/mock_data.js index ca29c9fee32..ae494267659 100644 --- a/spec/javascripts/vue_mr_widget/mock_data.js +++ b/spec/javascripts/vue_mr_widget/mock_data.js @@ -14,7 +14,6 @@ export default { "updated_by_id": null, "created_at": "2017-04-07T12:27:26.718Z", "updated_at": "2017-04-07T15:39:25.852Z", - "deleted_at": null, "time_estimate": 0, "total_time_spent": 0, "human_time_estimate": null, diff --git a/spec/javascripts/vue_shared/components/clipboard_button_spec.js b/spec/javascripts/vue_shared/components/clipboard_button_spec.js new file mode 100644 index 00000000000..08e4e1f8337 --- /dev/null +++ b/spec/javascripts/vue_shared/components/clipboard_button_spec.js @@ -0,0 +1,31 @@ +import Vue from 'vue'; +import clipboardButton from '~/vue_shared/components/clipboard_button.vue'; +import mountComponent from '../../helpers/vue_mount_component_helper'; + +describe('clipboard button', () => { + let vm; + + beforeEach(() => { + const Component = Vue.extend(clipboardButton); + vm = mountComponent(Component, { + text: 'copy me', + title: 'Copy this value into Clipboard!', + }); + }); + + afterEach(() => { + vm.$destroy(); + }); + + it('renders a button for clipboard', () => { + expect(vm.$el.tagName).toEqual('BUTTON'); + expect(vm.$el.getAttribute('data-clipboard-text')).toEqual('copy me'); + expect(vm.$el.querySelector('i').className).toEqual('fa fa-clipboard'); + }); + + it('should have a tooltip with default values', () => { + expect(vm.$el.getAttribute('data-original-title')).toEqual('Copy this value into Clipboard!'); + expect(vm.$el.getAttribute('data-placement')).toEqual('top'); + expect(vm.$el.getAttribute('data-container')).toEqual(null); + }); +}); diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb index f346a345f00..f4e781c599e 100644 --- a/spec/lib/gitlab/git/repository_spec.rb +++ b/spec/lib/gitlab/git/repository_spec.rb @@ -1283,48 +1283,58 @@ describe Gitlab::Git::Repository, seed_helper: true do end describe '#merged_branch_names' do - context 'when branch names are passed' do - it 'only returns the names we are asking' do - names = repository.merged_branch_names(%w[merge-test]) + shared_examples 'finding merged branch names' do + context 'when branch names are passed' do + it 'only returns the names we are asking' do + names = repository.merged_branch_names(%w[merge-test]) - expect(names).to contain_exactly('merge-test') - end + expect(names).to contain_exactly('merge-test') + end - it 'does not return unmerged branch names' do - names = repository.merged_branch_names(%w[feature]) + it 'does not return unmerged branch names' do + names = repository.merged_branch_names(%w[feature]) - expect(names).to be_empty + expect(names).to be_empty + end end - end - context 'when no root ref is available' do - it 'returns empty list' do - project = create(:project, :empty_repo) + context 'when no root ref is available' do + it 'returns empty list' do + project = create(:project, :empty_repo) - names = project.repository.merged_branch_names(%w[feature]) + names = project.repository.merged_branch_names(%w[feature]) - expect(names).to be_empty + expect(names).to be_empty + end end - end - context 'when no branch names are specified' do - before do - repository.create_branch('identical', 'master') - end + context 'when no branch names are specified' do + before do + repository.create_branch('identical', 'master') + end - after do - ensure_seeds - end + after do + ensure_seeds + end - it 'returns all merged branch names except for identical one' do - names = repository.merged_branch_names + it 'returns all merged branch names except for identical one' do + names = repository.merged_branch_names - expect(names).to include('merge-test') - expect(names).to include('fix-mode') - expect(names).not_to include('feature') - expect(names).not_to include('identical') + expect(names).to include('merge-test') + expect(names).to include('fix-mode') + expect(names).not_to include('feature') + expect(names).not_to include('identical') + end end end + + context 'when Gitaly merged_branch_names feature is enabled' do + it_behaves_like 'finding merged branch names' + end + + context 'when Gitaly merged_branch_names feature is disabled', :disable_gitaly do + it_behaves_like 'finding merged branch names' + end end describe "#ls_files" do diff --git a/spec/lib/gitlab/hook_data/issue_builder_spec.rb b/spec/lib/gitlab/hook_data/issue_builder_spec.rb index aeacd577d18..506b2c0be20 100644 --- a/spec/lib/gitlab/hook_data/issue_builder_spec.rb +++ b/spec/lib/gitlab/hook_data/issue_builder_spec.rb @@ -14,7 +14,6 @@ describe Gitlab::HookData::IssueBuilder do closed_at confidential created_at - deleted_at description due_date id diff --git a/spec/lib/gitlab/hook_data/merge_request_builder_spec.rb b/spec/lib/gitlab/hook_data/merge_request_builder_spec.rb index 78475403f9e..b61614e4790 100644 --- a/spec/lib/gitlab/hook_data/merge_request_builder_spec.rb +++ b/spec/lib/gitlab/hook_data/merge_request_builder_spec.rb @@ -12,7 +12,6 @@ describe Gitlab::HookData::MergeRequestBuilder do assignee_id author_id created_at - deleted_at description head_pipeline_id id diff --git a/spec/lib/gitlab/import_export/project.group.json b/spec/lib/gitlab/import_export/project.group.json index 82a1fbd2fc5..1a561e81e4a 100644 --- a/spec/lib/gitlab/import_export/project.group.json +++ b/spec/lib/gitlab/import_export/project.group.json @@ -54,7 +54,6 @@ "iid": 1, "updated_by_id": 1, "confidential": false, - "deleted_at": null, "due_date": null, "moved_to_id": null, "lock_version": null, @@ -134,7 +133,6 @@ "iid": 2, "updated_by_id": 1, "confidential": false, - "deleted_at": null, "due_date": null, "moved_to_id": null, "lock_version": null, diff --git a/spec/lib/gitlab/import_export/project.json b/spec/lib/gitlab/import_export/project.json index 7580b62cfc0..4cf33778d15 100644 --- a/spec/lib/gitlab/import_export/project.json +++ b/spec/lib/gitlab/import_export/project.json @@ -56,7 +56,6 @@ "iid": 10, "updated_by_id": null, "confidential": false, - "deleted_at": null, "due_date": null, "moved_to_id": null, "test_ee_field": "test", @@ -350,7 +349,6 @@ "iid": 9, "updated_by_id": null, "confidential": false, - "deleted_at": null, "due_date": null, "moved_to_id": null, "milestone": { @@ -586,7 +584,6 @@ "iid": 8, "updated_by_id": null, "confidential": false, - "deleted_at": null, "due_date": null, "moved_to_id": null, "label_links": [ @@ -820,7 +817,6 @@ "iid": 7, "updated_by_id": null, "confidential": false, - "deleted_at": null, "due_date": null, "moved_to_id": null, "notes": [ @@ -1033,7 +1029,6 @@ "iid": 6, "updated_by_id": null, "confidential": false, - "deleted_at": null, "due_date": null, "moved_to_id": null, "notes": [ @@ -1246,7 +1241,6 @@ "iid": 5, "updated_by_id": null, "confidential": false, - "deleted_at": null, "due_date": null, "moved_to_id": null, "notes": [ @@ -1459,7 +1453,6 @@ "iid": 4, "updated_by_id": null, "confidential": false, - "deleted_at": null, "due_date": null, "moved_to_id": null, "notes": [ @@ -1672,7 +1665,6 @@ "iid": 3, "updated_by_id": null, "confidential": false, - "deleted_at": null, "due_date": null, "moved_to_id": null, "notes": [ @@ -1885,7 +1877,6 @@ "iid": 2, "updated_by_id": null, "confidential": false, - "deleted_at": null, "due_date": null, "moved_to_id": null, "notes": [ @@ -2098,7 +2089,6 @@ "iid": 1, "updated_by_id": null, "confidential": false, - "deleted_at": null, "due_date": null, "moved_to_id": null, "notes": [ @@ -2504,7 +2494,6 @@ "merge_when_pipeline_succeeds": true, "merge_user_id": null, "merge_commit_sha": null, - "deleted_at": null, "notes": [ { "id": 671, @@ -2948,7 +2937,6 @@ "merge_when_pipeline_succeeds": false, "merge_user_id": null, "merge_commit_sha": null, - "deleted_at": null, "notes": [ { "id": 679, @@ -3228,7 +3216,6 @@ "merge_when_pipeline_succeeds": false, "merge_user_id": null, "merge_commit_sha": null, - "deleted_at": null, "notes": [ { "id": 777, @@ -3508,7 +3495,6 @@ "merge_when_pipeline_succeeds": false, "merge_user_id": null, "merge_commit_sha": null, - "deleted_at": null, "notes": [ { "id": 785, @@ -4198,7 +4184,6 @@ "merge_when_pipeline_succeeds": false, "merge_user_id": null, "merge_commit_sha": null, - "deleted_at": null, "notes": [ { "id": 793, @@ -4734,7 +4719,6 @@ "merge_when_pipeline_succeeds": false, "merge_user_id": null, "merge_commit_sha": null, - "deleted_at": null, "notes": [ { "id": 801, @@ -5223,7 +5207,6 @@ "merge_when_pipeline_succeeds": false, "merge_user_id": null, "merge_commit_sha": null, - "deleted_at": null, "notes": [ { "id": 809, @@ -5478,7 +5461,6 @@ "merge_when_pipeline_succeeds": false, "merge_user_id": null, "merge_commit_sha": null, - "deleted_at": null, "notes": [ { "id": 817, @@ -6168,7 +6150,6 @@ "merge_when_pipeline_succeeds": false, "merge_user_id": null, "merge_commit_sha": null, - "deleted_at": null, "notes": [ { "id": 825, @@ -6968,7 +6949,6 @@ "id": 123, "token": "cdbfasdf44a5958c83654733449e585", "project_id": 5, - "deleted_at": null, "created_at": "2017-01-16T15:25:28.637Z", "updated_at": "2017-01-16T15:25:28.637Z" } diff --git a/spec/lib/gitlab/import_export/project.light.json b/spec/lib/gitlab/import_export/project.light.json index 02450478a77..5dbf0ed289b 100644 --- a/spec/lib/gitlab/import_export/project.light.json +++ b/spec/lib/gitlab/import_export/project.light.json @@ -54,7 +54,6 @@ "iid": 20, "updated_by_id": 1, "confidential": false, - "deleted_at": null, "due_date": null, "moved_to_id": null, "lock_version": null, diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml index c5bf23e65b3..5a33fa3fd53 100644 --- a/spec/lib/gitlab/import_export/safe_model_attributes.yml +++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml @@ -14,7 +14,6 @@ Issue: - iid - updated_by_id - confidential -- deleted_at - closed_at - due_date - moved_to_id @@ -159,7 +158,6 @@ MergeRequest: - merge_when_pipeline_succeeds - merge_user_id - merge_commit_sha -- deleted_at - in_progress_merge_commit_sha - lock_version - milestone_id @@ -294,7 +292,6 @@ Ci::Trigger: - id - token - project_id -- deleted_at - created_at - updated_at - owner_id @@ -310,7 +307,6 @@ Ci::PipelineSchedule: - project_id - owner_id - active -- deleted_at - created_at - updated_at Clusters::Cluster: diff --git a/spec/lib/gitlab/workhorse_spec.rb b/spec/lib/gitlab/workhorse_spec.rb index 249c77dc636..2e7a0265a0b 100644 --- a/spec/lib/gitlab/workhorse_spec.rb +++ b/spec/lib/gitlab/workhorse_spec.rb @@ -26,11 +26,16 @@ describe Gitlab::Workhorse do 'GitalyRepository' => repository.gitaly_repository.to_h.deep_stringify_keys ) end + let(:cache_disabled) { false } subject do described_class.send_git_archive(repository, ref: ref, format: format) end + before do + allow(described_class).to receive(:git_archive_cache_disabled?).and_return(cache_disabled) + end + context 'when Gitaly workhorse_archive feature is enabled' do it 'sets the header correctly' do key, command, params = decode_workhorse_header(subject) @@ -39,6 +44,15 @@ describe Gitlab::Workhorse do expect(command).to eq('git-archive') expect(params).to include(gitaly_params) end + + context 'when archive caching is disabled' do + let(:cache_disabled) { true } + + it 'tells workhorse not to use the cache' do + _, _, params = decode_workhorse_header(subject) + expect(params).to include({ 'DisableCache' => true }) + end + end end context 'when Gitaly workhorse_archive feature is disabled', :skip_gitaly_mock do diff --git a/spec/migrations/remove_soft_removed_objects_spec.rb b/spec/migrations/remove_soft_removed_objects_spec.rb new file mode 100644 index 00000000000..ec089f9106d --- /dev/null +++ b/spec/migrations/remove_soft_removed_objects_spec.rb @@ -0,0 +1,77 @@ +require 'spec_helper' +require Rails.root.join('db', 'post_migrate', '20171207150343_remove_soft_removed_objects.rb') + +describe RemoveSoftRemovedObjects, :migration do + describe '#up' do + it 'removes various soft removed objects' do + 5.times do + create_with_deleted_at(:issue) + end + + regular_issue = create(:issue) + + run_migration + + expect(Issue.count).to eq(1) + expect(Issue.first).to eq(regular_issue) + end + + it 'removes the temporary indexes once soft removed data has been removed' do + migration = described_class.new + + run_migration + + disable_migrations_output do + expect(migration.temporary_index_exists?(Issue)).to eq(false) + end + end + + it 'removes routes of soft removed personal namespaces' do + namespace = create_with_deleted_at(:namespace) + group = create(:group) + + expect(Route.where(source: namespace).exists?).to eq(true) + expect(Route.where(source: group).exists?).to eq(true) + + run_migration + + expect(Route.where(source: namespace).exists?).to eq(false) + expect(Route.where(source: group).exists?).to eq(true) + end + + it 'schedules the removal of soft removed groups' do + group = create_with_deleted_at(:group) + admin = create(:user, admin: true) + + expect_any_instance_of(GroupDestroyWorker) + .to receive(:perform) + .with(group.id, admin.id) + + run_migration + end + + it 'does not remove soft removed groups when no admin user could be found' do + create_with_deleted_at(:group) + + expect_any_instance_of(GroupDestroyWorker) + .not_to receive(:perform) + + run_migration + end + end + + def run_migration + disable_migrations_output do + migrate! + end + end + + def create_with_deleted_at(*args) + row = create(*args) + + # We set "deleted_at" this way so we don't run into any column cache issues. + row.class.where(id: row.id).update_all(deleted_at: 1.year.ago) + + row + end +end diff --git a/spec/models/ci/pipeline_schedule_spec.rb b/spec/models/ci/pipeline_schedule_spec.rb index 9a278212efc..8ee15f0e734 100644 --- a/spec/models/ci/pipeline_schedule_spec.rb +++ b/spec/models/ci/pipeline_schedule_spec.rb @@ -12,7 +12,6 @@ describe Ci::PipelineSchedule do it { is_expected.to respond_to(:cron_timezone) } it { is_expected.to respond_to(:description) } it { is_expected.to respond_to(:next_run_at) } - it { is_expected.to respond_to(:deleted_at) } describe 'validations' do it 'does not allow invalid cron patters' do diff --git a/spec/models/issue_spec.rb b/spec/models/issue_spec.rb index 5ced000cdb6..f5c9f551e65 100644 --- a/spec/models/issue_spec.rb +++ b/spec/models/issue_spec.rb @@ -18,11 +18,6 @@ describe Issue do subject { create(:issue) } - describe "act_as_paranoid" do - it { is_expected.to have_db_column(:deleted_at) } - it { is_expected.to have_db_index(:deleted_at) } - end - describe 'callbacks' do describe '#ensure_metrics' do it 'creates metrics after saving' do diff --git a/spec/models/merge_request_spec.rb b/spec/models/merge_request_spec.rb index 07b3e1c1758..27e9c477d61 100644 --- a/spec/models/merge_request_spec.rb +++ b/spec/models/merge_request_spec.rb @@ -24,11 +24,6 @@ describe MergeRequest do it { is_expected.to include_module(Taskable) } end - describe "act_as_paranoid" do - it { is_expected.to have_db_column(:deleted_at) } - it { is_expected.to have_db_index(:deleted_at) } - end - describe 'validation' do it { is_expected.to validate_presence_of(:target_branch) } it { is_expected.to validate_presence_of(:source_branch) } diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb index b3f160f3119..c3673a0e2a3 100644 --- a/spec/models/namespace_spec.rb +++ b/spec/models/namespace_spec.rb @@ -410,17 +410,6 @@ describe Namespace do end end - describe '#soft_delete_without_removing_associations' do - let(:project1) { create(:project_empty_repo, namespace: namespace) } - - it 'updates the deleted_at timestamp but preserves projects' do - namespace.soft_delete_without_removing_associations - - expect(Project.all).to include(project1) - expect(namespace.deleted_at).not_to be_nil - end - end - describe '#user_ids_for_project_authorizations' do it 'returns the user IDs for which to refresh authorizations' do expect(namespace.user_ids_for_project_authorizations) diff --git a/spec/requests/api/commit_statuses_spec.rb b/spec/requests/api/commit_statuses_spec.rb index ffa17d296e8..f246bb79ab7 100644 --- a/spec/requests/api/commit_statuses_spec.rb +++ b/spec/requests/api/commit_statuses_spec.rb @@ -142,6 +142,7 @@ describe API::CommitStatuses do expect(json_response['ref']).not_to be_empty expect(json_response['target_url']).to be_nil expect(json_response['description']).to be_nil + if status == 'failed' expect(CommitStatus.find(json_response['id'])).to be_api_failure end diff --git a/spec/requests/api/merge_requests_spec.rb b/spec/requests/api/merge_requests_spec.rb index 6c0ebf4fc82..4eae3e50602 100644 --- a/spec/requests/api/merge_requests_spec.rb +++ b/spec/requests/api/merge_requests_spec.rb @@ -585,7 +585,7 @@ describe API::MergeRequests do project = create(:project, public_builds: false) merge_request = create(:merge_request, :simple, source_project: project) guest = create(:user) - project.team << [guest, :guest] + project.add_guest(guest) get api("/projects/#{project.id}/merge_requests/#{merge_request.iid}/pipelines", guest) diff --git a/spec/requests/api/runner_spec.rb b/spec/requests/api/runner_spec.rb index 679d391caa5..cb66d23b77c 100644 --- a/spec/requests/api/runner_spec.rb +++ b/spec/requests/api/runner_spec.rb @@ -1142,6 +1142,7 @@ describe API::Runner do else { 'file' => file } end + post api("/jobs/#{job.id}/artifacts"), params, headers end end diff --git a/spec/rubocop/cop/line_break_around_conditional_block_spec.rb b/spec/rubocop/cop/line_break_around_conditional_block_spec.rb new file mode 100644 index 00000000000..7ddf9141fcd --- /dev/null +++ b/spec/rubocop/cop/line_break_around_conditional_block_spec.rb @@ -0,0 +1,411 @@ +require 'spec_helper' +require 'rubocop' +require 'rubocop/rspec/support' +require_relative '../../../rubocop/cop/line_break_around_conditional_block' + +describe RuboCop::Cop::LineBreakAroundConditionalBlock do + include CopHelper + + subject(:cop) { described_class.new } + + shared_examples 'examples with conditional' do |conditional| + it "flags violation for #{conditional} without line break before" do + source = <<~RUBY + do_something + #{conditional} condition + do_something_more + end + RUBY + inspect_source(source) + + expect(cop.offenses.size).to eq(1) + offense = cop.offenses.first + + expect(offense.line).to eq(2) + expect(cop.highlights).to eq(["#{conditional} condition\n do_something_more\nend"]) + expect(offense.message).to eq('Add a line break around conditional blocks') + end + + it "flags violation for #{conditional} without line break after" do + source = <<~RUBY + #{conditional} condition + do_something + end + do_something_more + RUBY + inspect_source(source) + + expect(cop.offenses.size).to eq(1) + offense = cop.offenses.first + + expect(offense.line).to eq(1) + expect(cop.highlights).to eq(["#{conditional} condition\n do_something\nend"]) + expect(offense.message).to eq('Add a line break around conditional blocks') + end + + it "doesn't flag violation for #{conditional} with line break before and after" do + source = <<~RUBY + #{conditional} condition + do_something + end + RUBY + inspect_source(source) + + expect(cop.offenses).to be_empty + end + + it "doesn't flag violation for #{conditional} preceded by a method definition" do + source = <<~RUBY + def a_method + #{conditional} condition + do_something + end + end + RUBY + inspect_source(source) + + expect(cop.offenses).to be_empty + end + + it "doesn't flag violation for #{conditional} preceded by a class definition" do + source = <<~RUBY + class Foo + #{conditional} condition + do_something + end + end + RUBY + inspect_source(source) + + expect(cop.offenses).to be_empty + end + + it "doesn't flag violation for #{conditional} preceded by a module definition" do + source = <<~RUBY + module Foo + #{conditional} condition + do_something + end + end + RUBY + inspect_source(source) + + expect(cop.offenses).to be_empty + end + + it "doesn't flag violation for #{conditional} preceded by a begin definition" do + source = <<~RUBY + begin + #{conditional} condition + do_something + end + end + RUBY + inspect_source(source) + + expect(cop.offenses).to be_empty + end + + it "doesn't flag violation for #{conditional} preceded by an assign/begin definition" do + source = <<~RUBY + @project ||= begin + #{conditional} condition + do_something + end + end + RUBY + inspect_source(source) + + expect(cop.offenses).to be_empty + end + + it "doesn't flag violation for #{conditional} preceded by a block definition" do + source = <<~RUBY + on_block(param_a) do |item| + #{conditional} condition + do_something + end + end + RUBY + inspect_source(source) + + expect(cop.offenses).to be_empty + end + + it "doesn't flag violation for #{conditional} preceded by a block definition using brackets" do + source = <<~RUBY + on_block(param_a) { |item| + #{conditional} condition + do_something + end + } + RUBY + inspect_source(source) + + expect(cop.offenses).to be_empty + end + + it "doesn't flag violation for #{conditional} preceded by a comment" do + source = <<~RUBY + # a short comment + #{conditional} condition + do_something + end + RUBY + inspect_source(source) + + expect(cop.offenses).to be_empty + end + + it "doesn't flag violation for #{conditional} preceded by an assignment" do + source = <<~RUBY + foo = + #{conditional} condition + do_something + else + do_something_more + end + RUBY + inspect_source(source) + + expect(cop.offenses).to be_empty + end + + it "doesn't flag violation for #{conditional} preceded by a multiline comment" do + source = <<~RUBY + =begin + a multiline comment + =end + #{conditional} condition + do_something + end + RUBY + inspect_source(source) + + expect(cop.offenses).to be_empty + end + + it "doesn't flag violation for #{conditional} preceded by another conditional" do + source = <<~RUBY + #{conditional} condition_a + #{conditional} condition_b + do_something + end + end + RUBY + inspect_source(source) + + expect(cop.offenses).to be_empty + end + + it "doesn't flag violation for #{conditional} preceded by an else" do + source = <<~RUBY + if condition_a + do_something + else + #{conditional} condition_b + do_something_extra + end + end + RUBY + inspect_source(source) + + expect(cop.offenses).to be_empty + end + + it "doesn't flag violation for #{conditional} preceded by an elsif" do + source = <<~RUBY + if condition_a + do_something + elsif condition_b + #{conditional} condition_c + do_something_extra + end + end + RUBY + inspect_source(source) + + expect(cop.offenses).to be_empty + end + + it "doesn't flag violation for #{conditional} preceded by an ensure" do + source = <<~RUBY + def a_method + ensure + #{conditional} condition_c + do_something_extra + end + end + RUBY + inspect_source(source) + + expect(cop.offenses).to be_empty + end + + it "doesn't flag violation for #{conditional} preceded by a when" do + source = <<~RUBY + case field + when value + #{conditional} condition_c + do_something_extra + end + end + RUBY + inspect_source(source) + + expect(cop.offenses).to be_empty + end + + it "doesn't flag violation for #{conditional} followed by an end" do + source = <<~RUBY + class Foo + + #{conditional} condition + do_something + end + end + RUBY + inspect_source(source) + + expect(cop.offenses).to be_empty + end + + it "doesn't flag violation for #{conditional} followed by an else" do + source = <<~RUBY + #{conditional} condition_a + #{conditional} condition_b + do_something + end + else + do_something_extra + end + RUBY + inspect_source(source) + + expect(cop.offenses).to be_empty + end + + it "doesn't flag violation for #{conditional} followed by a when" do + source = <<~RUBY + case + when condition_a + #{conditional} condition_b + do_something + end + when condition_c + do_something_extra + end + RUBY + inspect_source(source) + + expect(cop.offenses).to be_empty + end + + it "doesn't flag violation for #{conditional} followed by an elsif" do + source = <<~RUBY + if condition_a + #{conditional} condition_b + do_something + end + elsif condition_c + do_something_extra + end + RUBY + inspect_source(source) + + expect(cop.offenses).to be_empty + end + + it "doesn't flag violation for #{conditional} followed by a rescue" do + source = <<~RUBY + def a_method + #{conditional} condition + do_something + end + rescue + do_something_extra + end + RUBY + inspect_source(source) + + expect(cop.offenses).to be_empty + end + + it "autocorrects #{conditional} without line break before" do + source = <<~RUBY + do_something + #{conditional} condition + do_something_more + end + RUBY + autocorrected = autocorrect_source(source) + + expected_source = <<~RUBY + do_something + + #{conditional} condition + do_something_more + end + RUBY + expect(autocorrected).to eql(expected_source) + end + + it "autocorrects #{conditional} without line break after" do + source = <<~RUBY + #{conditional} condition + do_something + end + do_something_more + RUBY + autocorrected = autocorrect_source(source) + + expected_source = <<~RUBY + #{conditional} condition + do_something + end + + do_something_more + RUBY + expect(autocorrected).to eql(expected_source) + end + + it "autocorrects #{conditional} without line break before and after" do + source = <<~RUBY + do_something + #{conditional} condition + do_something_more + end + do_something_extra + RUBY + autocorrected = autocorrect_source(source) + + expected_source = <<~RUBY + do_something + + #{conditional} condition + do_something_more + end + + do_something_extra + RUBY + expect(autocorrected).to eql(expected_source) + end + end + + %w[if unless].each do |example| + it_behaves_like 'examples with conditional', example + end + + it "doesn't flag violation for if with elsif" do + source = <<~RUBY + if condition + do_something + elsif another_condition + do_something_more + end + RUBY + inspect_source(source) + + expect(cop.offenses).to be_empty + end +end diff --git a/spec/services/projects/transfer_service_spec.rb b/spec/services/projects/transfer_service_spec.rb index 39f6388c25e..ef68742a463 100644 --- a/spec/services/projects/transfer_service_spec.rb +++ b/spec/services/projects/transfer_service_spec.rb @@ -150,6 +150,7 @@ describe Projects::TransferService do before do group.add_owner(user) + unless gitlab_shell.add_repository(repository_storage, "#{group.full_path}/#{project.path}") raise 'failed to add repository' end diff --git a/spec/services/system_note_service_spec.rb b/spec/services/system_note_service_spec.rb index 4e640a82dfc..965fd39c967 100644 --- a/spec/services/system_note_service_spec.rb +++ b/spec/services/system_note_service_spec.rb @@ -727,6 +727,7 @@ describe SystemNoteService do else "#{Settings.gitlab.base_url}/#{project.namespace.path}/#{project.path}/merge_requests/#{merge_request.iid}" end + link = double(object: { 'url' => url }) links << link expect(link).to receive(:save!) diff --git a/spec/services/users/destroy_service_spec.rb b/spec/services/users/destroy_service_spec.rb index aeba9cd60bc..bb3d73edf8e 100644 --- a/spec/services/users/destroy_service_spec.rb +++ b/spec/services/users/destroy_service_spec.rb @@ -15,7 +15,7 @@ describe Users::DestroyService do expect { user_data['email'].to eq(user.email) } expect { User.find(user.id) }.to raise_error(ActiveRecord::RecordNotFound) - expect { Namespace.with_deleted.find(namespace.id) }.to raise_error(ActiveRecord::RecordNotFound) + expect { Namespace.find(namespace.id) }.to raise_error(ActiveRecord::RecordNotFound) end it 'will delete the project' do diff --git a/spec/support/features/discussion_comments_shared_example.rb b/spec/support/features/discussion_comments_shared_example.rb index c24940393f9..fa94aa2ae3d 100644 --- a/spec/support/features/discussion_comments_shared_example.rb +++ b/spec/support/features/discussion_comments_shared_example.rb @@ -113,6 +113,7 @@ shared_examples 'discussion comments' do |resource_name| else expect(find(submit_selector).value).to eq 'Start discussion' end + expect(page).not_to have_selector menu_selector end @@ -200,6 +201,7 @@ shared_examples 'discussion comments' do |resource_name| else expect(find(submit_selector).value).to eq 'Comment' end + expect(page).not_to have_selector menu_selector end diff --git a/spec/support/filtered_search_helpers.rb b/spec/support/filtered_search_helpers.rb index 05021ea9054..f3f96bd1f0a 100644 --- a/spec/support/filtered_search_helpers.rb +++ b/spec/support/filtered_search_helpers.rb @@ -61,9 +61,11 @@ module FilteredSearchHelpers token_emoji = tokens[index][:emoji_name] expect(el.find('.name')).to have_content(token_name) + if token_value expect(el.find('.value')).to have_content(token_value) end + # gl-emoji content is blank when the emoji unicode is not supported if token_emoji selector = %(gl-emoji[data-name="#{token_emoji}"]) diff --git a/spec/support/generate-seed-repo-rb b/spec/support/generate-seed-repo-rb index 4ee33f9725b..876b3b8242d 100755 --- a/spec/support/generate-seed-repo-rb +++ b/spec/support/generate-seed-repo-rb @@ -24,6 +24,7 @@ def main unless system(*%W[git clone --bare #{SOURCE} #{REPO_NAME}], chdir: dir) abort "git clone failed" end + repo = File.join(dir, REPO_NAME) erb = ERB.new(DATA.read) erb.run(binding) diff --git a/spec/support/matchers/access_matchers_for_controller.rb b/spec/support/matchers/access_matchers_for_controller.rb index cdb62a5deee..42a9ed9ff34 100644 --- a/spec/support/matchers/access_matchers_for_controller.rb +++ b/spec/support/matchers/access_matchers_for_controller.rb @@ -43,6 +43,7 @@ module AccessMatchersForController user = create(:user) membership.public_send(:"add_#{role}", user) end + user end diff --git a/spec/support/select2_helper.rb b/spec/support/select2_helper.rb index 55da961e173..90618ba5b19 100644 --- a/spec/support/select2_helper.rb +++ b/spec/support/select2_helper.rb @@ -17,6 +17,7 @@ module Select2Helper selector = options.fetch(:from) first(selector, visible: false) + if options[:multiple] execute_script("$('#{selector}').select2('val', ['#{value}']).trigger('change');") else diff --git a/spec/support/stub_env.rb b/spec/support/stub_env.rb index f621463e621..695152e2d4e 100644 --- a/spec/support/stub_env.rb +++ b/spec/support/stub_env.rb @@ -4,6 +4,7 @@ module StubENV def stub_env(key_or_hash, value = nil) init_stub unless env_stubbed? + if key_or_hash.is_a? Hash key_or_hash.each { |k, v| add_stubbed_value(k, v) } else diff --git a/spec/support/test_env.rb b/spec/support/test_env.rb index 664698fcbaf..25ff6094408 100644 --- a/spec/support/test_env.rb +++ b/spec/support/test_env.rb @@ -325,6 +325,7 @@ module TestEnv if component_needs_update?(install_dir, version) # Cleanup the component entirely to ensure we start fresh FileUtils.rm_rf(install_dir) + unless system('rake', task) raise ComponentFailedToInstallError end diff --git a/spec/support/wait_for_requests.rb b/spec/support/wait_for_requests.rb index f4130d68271..fda0e29f983 100644 --- a/spec/support/wait_for_requests.rb +++ b/spec/support/wait_for_requests.rb @@ -53,6 +53,7 @@ module WaitForRequests wait_until = Time.now + max_wait_time.seconds loop do break if yield + if Time.now > wait_until raise "Condition not met: #{condition_name}" else diff --git a/spec/views/projects/buttons/_dropdown.html.haml_spec.rb b/spec/views/projects/buttons/_dropdown.html.haml_spec.rb new file mode 100644 index 00000000000..d0e692635b9 --- /dev/null +++ b/spec/views/projects/buttons/_dropdown.html.haml_spec.rb @@ -0,0 +1,39 @@ +require 'spec_helper' + +describe 'projects/buttons/_dropdown' do + let(:user) { create(:user) } + + context 'user with all abilities' do + before do + assign(:project, project) + + allow(view).to receive(:current_user).and_return(user) + allow(view).to receive(:can?).and_return(true) + end + + context 'empty repository' do + let(:project) { create(:project, :empty_repo) } + + it 'has a link to create a new file' do + render + + expect(view).to render_template('projects/buttons/_dropdown') + expect(rendered).to have_link('New file') + end + + it 'does not have a link to create a new branch' do + render + + expect(view).to render_template('projects/buttons/_dropdown') + expect(rendered).not_to have_link('New branch') + end + + it 'does not have a link to create a new tag' do + render + + expect(view).to render_template('projects/buttons/_dropdown') + expect(rendered).not_to have_link('New tag') + end + end + end +end |
