diff options
| author | Stan Hu <stanhu@gmail.com> | 2018-02-23 22:37:10 +0000 |
|---|---|---|
| committer | Stan Hu <stanhu@gmail.com> | 2018-02-23 22:37:10 +0000 |
| commit | 042c499997f8b32c2ab54e3fb7703f8b63a0919f (patch) | |
| tree | a069f5c33759474958edcb87c936ee38571db353 | |
| parent | 21c2ffea5f50f9aa204d8ec6ba6b5cc47cba636e (diff) | |
| parent | aee1d09de2762735147195c1394c650ceab92746 (diff) | |
| download | gitlab-ce-042c499997f8b32c2ab54e3fb7703f8b63a0919f.tar.gz | |
Merge branch '10-5-stable-patch-2' into '10-5-stable'
Prepare 10.5.2 release
See merge request gitlab-org/gitlab-ce!17322
86 files changed, 648 insertions, 368 deletions
diff --git a/app/assets/javascripts/filtered_search/dropdown_user.js b/app/assets/javascripts/filtered_search/dropdown_user.js index a9e2b65def0..907028fba33 100644 --- a/app/assets/javascripts/filtered_search/dropdown_user.js +++ b/app/assets/javascripts/filtered_search/dropdown_user.js @@ -12,7 +12,6 @@ class DropdownUser extends gl.FilteredSearchDropdown { endpoint: `${gon.relative_url_root || ''}/autocomplete/users.json`, searchKey: 'search', params: { - per_page: 20, active: true, group_id: this.getGroupId(), project_id: this.getProjectId(), diff --git a/app/assets/javascripts/single_file_diff.js b/app/assets/javascripts/single_file_diff.js index 48dd91bdf06..0cd6e69811a 100644 --- a/app/assets/javascripts/single_file_diff.js +++ b/app/assets/javascripts/single_file_diff.js @@ -88,6 +88,8 @@ export default class SingleFileDiff { if (cb) cb(); }) - .catch(createFlash(__('An error occurred while retrieving diff'))); + .catch(() => { + createFlash(__('An error occurred while retrieving diff')); + }); } } diff --git a/app/assets/javascripts/users_select.js b/app/assets/javascripts/users_select.js index eaed81cf79e..5835138ff2b 100644 --- a/app/assets/javascripts/users_select.js +++ b/app/assets/javascripts/users_select.js @@ -34,14 +34,13 @@ function UsersSelect(currentUser, els, options = {}) { var options = {}; var $block, $collapsedSidebar, $dropdown, $loading, $selectbox, $value, abilityName, assignTo, assigneeTemplate, collapsedAssigneeTemplate, defaultLabel, defaultNullUser, firstUser, issueURL, selectedId, selectedIdDefault, showAnyUser, showNullUser, showMenuAbove; $dropdown = $(dropdown); - options.projectId = $dropdown.data('project-id'); - options.groupId = $dropdown.data('group-id'); - options.showCurrentUser = $dropdown.data('current-user'); - options.todoFilter = $dropdown.data('todo-filter'); - options.todoStateFilter = $dropdown.data('todo-state-filter'); - options.perPage = $dropdown.data('per-page'); - showNullUser = $dropdown.data('null-user'); - defaultNullUser = $dropdown.data('null-user-default'); + options.projectId = $dropdown.data('projectId'); + options.groupId = $dropdown.data('groupId'); + options.showCurrentUser = $dropdown.data('currentUser'); + options.todoFilter = $dropdown.data('todoFilter'); + options.todoStateFilter = $dropdown.data('todoStateFilter'); + showNullUser = $dropdown.data('nullUser'); + defaultNullUser = $dropdown.data('nullUserDefault'); showMenuAbove = $dropdown.data('showMenuAbove'); showAnyUser = $dropdown.data('any-user'); firstUser = $dropdown.data('first-user'); @@ -669,7 +668,6 @@ UsersSelect.prototype.users = function(query, options, callback) { const url = this.buildUrl(this.usersPath); const params = { search: query, - per_page: options.perPage || 20, active: true, project_id: options.projectId || null, group_id: options.groupId || null, diff --git a/app/assets/javascripts/vue_merge_request_widget/mr_widget_options.js b/app/assets/javascripts/vue_merge_request_widget/mr_widget_options.js index edf67fcd0a7..d8f0442ef9d 100644 --- a/app/assets/javascripts/vue_merge_request_widget/mr_widget_options.js +++ b/app/assets/javascripts/vue_merge_request_widget/mr_widget_options.js @@ -152,6 +152,7 @@ export default { }, handleNotification(data) { if (data.ci_status === this.mr.ciStatus) return; + if (!data.pipeline) return; const label = data.pipeline.details.status.label; const title = `Pipeline ${label}`; diff --git a/app/assets/stylesheets/framework/stacked-progress-bar.scss b/app/assets/stylesheets/framework/stacked-progress-bar.scss index 4869cda73e5..528ba53a48b 100644 --- a/app/assets/stylesheets/framework/stacked-progress-bar.scss +++ b/app/assets/stylesheets/framework/stacked-progress-bar.scss @@ -10,7 +10,7 @@ .status-neutral, .status-red, { height: 100%; - min-width: 25px; + min-width: 30px; padding: 0 5px; font-size: $tooltip-font-size; font-weight: normal; diff --git a/app/controllers/concerns/uploads_actions.rb b/app/controllers/concerns/uploads_actions.rb index 7ad79a1e56c..3dbfabcae8a 100644 --- a/app/controllers/concerns/uploads_actions.rb +++ b/app/controllers/concerns/uploads_actions.rb @@ -24,7 +24,7 @@ module UploadsActions # - or redirect to its URL # def show - return render_404 unless uploader.exists? + return render_404 unless uploader&.exists? if uploader.file_storage? disposition = uploader.image_or_video? ? 'inline' : 'attachment' @@ -71,6 +71,9 @@ module UploadsActions def build_uploader_from_params uploader = uploader_class.new(model, secret: params[:secret]) + + return nil unless uploader.model_valid? + uploader.retrieve_from_store!(params[:filename]) uploader end diff --git a/app/finders/autocomplete_users_finder.rb b/app/finders/autocomplete_users_finder.rb index c3f5358b577..e8a03947f59 100644 --- a/app/finders/autocomplete_users_finder.rb +++ b/app/finders/autocomplete_users_finder.rb @@ -1,6 +1,12 @@ class AutocompleteUsersFinder + # The number of users to display in the results is hardcoded to 20, and + # pagination is not supported. This ensures that performance remains + # consistent and removes the need for implementing keyset pagination to ensure + # good performance. + LIMIT = 20 + attr_reader :current_user, :project, :group, :search, :skip_users, - :page, :per_page, :author_id, :params + :author_id, :params def initialize(params:, current_user:, project:, group:) @current_user = current_user @@ -8,8 +14,6 @@ class AutocompleteUsersFinder @group = group @search = params[:search] @skip_users = params[:skip_users] - @page = params[:page] - @per_page = params[:per_page] @author_id = params[:author_id] @params = params end @@ -20,7 +24,7 @@ class AutocompleteUsersFinder items = items.reorder(:name) items = items.search(search) if search.present? items = items.where.not(id: skip_users) if skip_users.present? - items = items.page(page).per(per_page) + items = items.limit(LIMIT) if params[:todo_filter].present? && current_user items = items.todo_authors(current_user.id, params[:todo_state_filter]) @@ -52,9 +56,13 @@ class AutocompleteUsersFinder end def users_from_project - user_ids = project.team.users.pluck(:id) - user_ids << author_id if author_id.present? + if author_id.present? + union = Gitlab::SQL::Union + .new([project.authorized_users, User.where(id: author_id)]) - User.where(id: user_ids) + User.from("(#{union.to_sql}) #{User.table_name}") + else + project.authorized_users + end end end diff --git a/app/finders/members_finder.rb b/app/finders/members_finder.rb index af24045886e..4734d97b8c7 100644 --- a/app/finders/members_finder.rb +++ b/app/finders/members_finder.rb @@ -10,26 +10,59 @@ class MembersFinder def execute project_members = project.project_members project_members = project_members.non_invite unless can?(current_user, :admin_project, project) - wheres = ["members.id IN (#{project_members.select(:id).to_sql})"] if group - # We need `.where.not(user_id: nil)` here otherwise when a group has an - # invitee, it would make the following query return 0 rows since a NULL - # user_id would be present in the subquery - # See http://stackoverflow.com/questions/129077/not-in-clause-and-null-values - non_null_user_ids = project_members.where.not(user_id: nil).select(:user_id) - group_members = GroupMembersFinder.new(group).execute - group_members = group_members.where.not(user_id: non_null_user_ids) - group_members = group_members.non_invite unless can?(current_user, :admin_group, group) + group_members = group_members.non_invite - wheres << "members.id IN (#{group_members.select(:id).to_sql})" - end + union = Gitlab::SQL::Union.new([project_members, group_members], remove_duplicates: false) - Member.where(wheres.join(' OR ')) + sql = distinct_on(union) + + Member.includes(:user).from("(#{sql}) AS #{Member.table_name}") + else + project_members + end end def can?(*args) Ability.allowed?(*args) end + + private + + def distinct_on(union) + # We're interested in a list of members without duplicates by user_id. + # We prefer project members over group members, project members should go first. + if Gitlab::Database.postgresql? + <<~SQL + SELECT DISTINCT ON (user_id, invite_email) member_union.* + FROM (#{union.to_sql}) AS member_union + ORDER BY user_id, + invite_email, + CASE + WHEN type = 'ProjectMember' THEN 1 + WHEN type = 'GroupMember' THEN 2 + ELSE 3 + END + SQL + else + # Older versions of MySQL do not support window functions (and DISTINCT ON is postgres-specific). + <<~SQL + SELECT t1.* + FROM (#{union.to_sql}) AS t1 + JOIN ( + SELECT + COALESCE(user_id, -1) AS user_id, + COALESCE(invite_email, 'NULL') AS invite_email, + MIN(CASE WHEN type = 'ProjectMember' THEN 1 WHEN type = 'GroupMember' THEN 2 ELSE 3 END) AS type_number + FROM + (#{union.to_sql}) AS t3 + GROUP BY COALESCE(user_id, -1), COALESCE(invite_email, 'NULL') + ) AS t2 ON COALESCE(t1.user_id, -1) = t2.user_id + AND COALESCE(t1.invite_email, 'NULL') = t2.invite_email + AND CASE WHEN t1.type = 'ProjectMember' THEN 1 WHEN t1.type = 'GroupMember' THEN 2 ELSE 3 END = t2.type_number + SQL + end + end end diff --git a/app/finders/snippets_finder.rb b/app/finders/snippets_finder.rb index 33359fa1efb..ec61fe1892e 100644 --- a/app/finders/snippets_finder.rb +++ b/app/finders/snippets_finder.rb @@ -56,8 +56,10 @@ class SnippetsFinder < UnionFinder end def feature_available_projects - projects = Project.public_or_visible_to_user(current_user) - .with_feature_available_for_user(:snippets, current_user).select(:id) + projects = Project.public_or_visible_to_user(current_user, use_where_in: false) do |part| + part.with_feature_available_for_user(:snippets, current_user) + end.select(:id) + arel_query = Arel::Nodes::SqlLiteral.new(projects.to_sql) table[:project_id].in(arel_query) end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 20534b8eed0..c48c5176987 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -467,7 +467,7 @@ module Ci if cache && project.jobs_cache_index cache = cache.merge( - key: "#{cache[:key]}_#{project.jobs_cache_index}") + key: "#{cache[:key]}-#{project.jobs_cache_index}") end [cache] diff --git a/app/models/project.rb b/app/models/project.rb index 2ba6a863500..79058d51af8 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -316,18 +316,42 @@ class Project < ActiveRecord::Base # Returns a collection of projects that is either public or visible to the # logged in user. - def self.public_or_visible_to_user(user = nil) - if user - authorized = user - .project_authorizations - .select(1) - .where('project_authorizations.project_id = projects.id') - - levels = Gitlab::VisibilityLevel.levels_for_user(user) - - where('EXISTS (?) OR projects.visibility_level IN (?)', authorized, levels) + # + # A caller may pass in a block to modify individual parts of + # the query, e.g. to apply .with_feature_available_for_user on top of it. + # This is useful for performance as we can stick those additional filters + # at the bottom of e.g. the UNION. + # + # Optionally, turning `use_where_in` off leads to returning a + # relation using #from instead of #where. This can perform much better + # but leads to trouble when used in conjunction with AR's #merge method. + def self.public_or_visible_to_user(user = nil, use_where_in: true, &block) + # If we don't get a block passed, use identity to avoid if/else repetitions + block = ->(part) { part } unless block_given? + + return block.call(public_to_user) unless user + + # If the user is allowed to see all projects, + # we can shortcut and just return. + return block.call(all) if user.full_private_access? + + authorized = user + .project_authorizations + .select(1) + .where('project_authorizations.project_id = projects.id') + authorized_projects = block.call(where('EXISTS (?)', authorized)) + + levels = Gitlab::VisibilityLevel.levels_for_user(user) + visible_projects = block.call(where(visibility_level: levels)) + + # We use a UNION here instead of OR clauses since this results in better + # performance. + union = Gitlab::SQL::Union.new([authorized_projects.select('projects.id'), visible_projects.select('projects.id')]) + + if use_where_in + where("projects.id IN (#{union.to_sql})") # rubocop:disable GitlabSecurity/SqlInjection else - public_to_user + from("(#{union.to_sql}) AS #{table_name}") end end diff --git a/app/models/user.rb b/app/models/user.rb index 05c93d3cb17..e81e555c090 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -59,6 +59,8 @@ class User < ActiveRecord::Base # Override Devise::Models::Trackable#update_tracked_fields! # to limit database writes to at most once every hour def update_tracked_fields!(request) + return if Gitlab::Database.read_only? + update_tracked_fields(request) lease = Gitlab::ExclusiveLease.new("user_update_tracked_fields:#{id}", timeout: 1.hour.to_i) @@ -325,8 +327,8 @@ class User < ActiveRecord::Base SQL where( - fuzzy_arel_match(:name, query) - .or(fuzzy_arel_match(:username, query)) + fuzzy_arel_match(:name, query, lower_exact_match: true) + .or(fuzzy_arel_match(:username, query, lower_exact_match: true)) .or(arel_table[:email].eq(query)) ).reorder(order % { query: ActiveRecord::Base.connection.quote(query) }, :name) end diff --git a/app/services/clusters/gcp/finalize_creation_service.rb b/app/services/clusters/gcp/finalize_creation_service.rb index cea56f4e849..15ab2d54404 100644 --- a/app/services/clusters/gcp/finalize_creation_service.rb +++ b/app/services/clusters/gcp/finalize_creation_service.rb @@ -30,10 +30,10 @@ module Clusters ca_cert: Base64.decode64(gke_cluster.master_auth.cluster_ca_certificate), username: gke_cluster.master_auth.username, password: gke_cluster.master_auth.password, - token: request_kuberenetes_token) + token: request_kubernetes_token) end - def request_kuberenetes_token + def request_kubernetes_token Ci::FetchKubernetesTokenService.new( 'https://' + gke_cluster.endpoint, Base64.decode64(gke_cluster.master_auth.cluster_ca_certificate), diff --git a/app/uploaders/gitlab_uploader.rb b/app/uploaders/gitlab_uploader.rb index a9e5c028b03..010100f2da1 100644 --- a/app/uploaders/gitlab_uploader.rb +++ b/app/uploaders/gitlab_uploader.rb @@ -67,6 +67,10 @@ class GitlabUploader < CarrierWave::Uploader::Base super || file&.filename end + def model_valid? + !!model + end + private # Designed to be overridden by child uploaders that have a dynamic path diff --git a/app/uploaders/personal_file_uploader.rb b/app/uploaders/personal_file_uploader.rb index e7d9ecd3222..f2ad0badd53 100644 --- a/app/uploaders/personal_file_uploader.rb +++ b/app/uploaders/personal_file_uploader.rb @@ -14,6 +14,12 @@ class PersonalFileUploader < FileUploader File.join(model.class.to_s.underscore, model.id.to_s) end + # model_path_segment does not require a model to be passed, so we can always + # generate a path, even when there's no model. + def model_valid? + true + end + # Revert-Override def store_dir File.join(base_dir, dynamic_segment) diff --git a/changelogs/unreleased/34130-null-pipes.yml b/changelogs/unreleased/34130-null-pipes.yml new file mode 100644 index 00000000000..a56e5cf8db2 --- /dev/null +++ b/changelogs/unreleased/34130-null-pipes.yml @@ -0,0 +1,5 @@ +--- +title: Prevent MR Widget error when no CI configured +merge_request: +author: +type: fixed diff --git a/changelogs/unreleased/41461-project-members-slow-due-to-sql.yml b/changelogs/unreleased/41461-project-members-slow-due-to-sql.yml new file mode 100644 index 00000000000..27eee7d943b --- /dev/null +++ b/changelogs/unreleased/41461-project-members-slow-due-to-sql.yml @@ -0,0 +1,5 @@ +--- +title: Improve query performance of MembersFinder. +merge_request: 17190 +author: +type: performance diff --git a/changelogs/unreleased/41619-turn-on-legacy-authorization-for-new-clusters-on-gke.yml b/changelogs/unreleased/41619-turn-on-legacy-authorization-for-new-clusters-on-gke.yml new file mode 100644 index 00000000000..507367c98c4 --- /dev/null +++ b/changelogs/unreleased/41619-turn-on-legacy-authorization-for-new-clusters-on-gke.yml @@ -0,0 +1,5 @@ +--- +title: Enable Legacy Authorization by default on Cluster creations +merge_request: 17302 +author: +type: fixed diff --git a/changelogs/unreleased/42877-snippets-dashboard-slow.yml b/changelogs/unreleased/42877-snippets-dashboard-slow.yml new file mode 100644 index 00000000000..839b44ad272 --- /dev/null +++ b/changelogs/unreleased/42877-snippets-dashboard-slow.yml @@ -0,0 +1,5 @@ +--- +title: Improve query performance for snippets dashboard. +merge_request: 17088 +author: +type: performance diff --git a/changelogs/unreleased/43373-fix-cache-index-appending.yml b/changelogs/unreleased/43373-fix-cache-index-appending.yml new file mode 100644 index 00000000000..fdb293ea04d --- /dev/null +++ b/changelogs/unreleased/43373-fix-cache-index-appending.yml @@ -0,0 +1,5 @@ +--- +title: Fix issue with cache key being empty when variable used as the key +merge_request: 17260 +author: +type: fixed diff --git a/changelogs/unreleased/fix-500-for-invalid-upload-path.yml b/changelogs/unreleased/fix-500-for-invalid-upload-path.yml new file mode 100644 index 00000000000..a4ce00c64c4 --- /dev/null +++ b/changelogs/unreleased/fix-500-for-invalid-upload-path.yml @@ -0,0 +1,5 @@ +--- +title: Fix 500 error when loading an invalid upload URL +merge_request: +author: +type: fixed diff --git a/changelogs/unreleased/flipper-caching.yml b/changelogs/unreleased/flipper-caching.yml new file mode 100644 index 00000000000..6db27fd579e --- /dev/null +++ b/changelogs/unreleased/flipper-caching.yml @@ -0,0 +1,5 @@ +--- +title: Increase feature flag cache TTL to one hour +merge_request: +author: +type: performance diff --git a/changelogs/unreleased/jej-fix-slow-lfs-object-check.yml b/changelogs/unreleased/jej-fix-slow-lfs-object-check.yml new file mode 100644 index 00000000000..09112fba85e --- /dev/null +++ b/changelogs/unreleased/jej-fix-slow-lfs-object-check.yml @@ -0,0 +1,5 @@ +--- +title: Only check LFS integrity for first ref in a push to avoid timeout +merge_request: 17098 +author: +type: performance diff --git a/changelogs/unreleased/kp-fix-stacked-bar-progress-value-clipping.yml b/changelogs/unreleased/kp-fix-stacked-bar-progress-value-clipping.yml new file mode 100644 index 00000000000..690536a533b --- /dev/null +++ b/changelogs/unreleased/kp-fix-stacked-bar-progress-value-clipping.yml @@ -0,0 +1,5 @@ +--- +title: Fix single digit value clipping for stacked progress bar +merge_request: 17217 +author: +type: fixed diff --git a/changelogs/unreleased/sh-guard-read-only-user-updates.yml b/changelogs/unreleased/sh-guard-read-only-user-updates.yml new file mode 100644 index 00000000000..b8dbd840ed9 --- /dev/null +++ b/changelogs/unreleased/sh-guard-read-only-user-updates.yml @@ -0,0 +1,5 @@ +--- +title: Don't attempt to update user tracked fields if database is in read-only +merge_request: +author: +type: fixed diff --git a/changelogs/unreleased/users-autocomplete.yml b/changelogs/unreleased/users-autocomplete.yml new file mode 100644 index 00000000000..2cb078a3a7c --- /dev/null +++ b/changelogs/unreleased/users-autocomplete.yml @@ -0,0 +1,5 @@ +--- +title: Improve performance of searching for and autocompleting of users +merge_request: +author: +type: performance diff --git a/changelogs/unreleased/zj-branch-contains-git-message.yml b/changelogs/unreleased/zj-branch-contains-git-message.yml new file mode 100644 index 00000000000..ce034e7ec87 --- /dev/null +++ b/changelogs/unreleased/zj-branch-contains-git-message.yml @@ -0,0 +1,5 @@ +--- +title: Allow branch names to be named the same as the sha it points to +merge_request: +author: +type: fixed diff --git a/config/initializers/flipper.rb b/config/initializers/flipper.rb index cc9167d29b9..c60ad535fd5 100644 --- a/config/initializers/flipper.rb +++ b/config/initializers/flipper.rb @@ -8,7 +8,7 @@ Flipper.configure do |config| cached_adapter = Flipper::Adapters::ActiveSupportCacheStore.new( adapter, Rails.cache, - expires_in: 10.seconds) + expires_in: 1.hour) Flipper.new(cached_adapter) end diff --git a/db/migrate/20180213131630_add_partial_index_to_projects_for_index_only_scans.rb b/db/migrate/20180213131630_add_partial_index_to_projects_for_index_only_scans.rb new file mode 100644 index 00000000000..cedf2510dda --- /dev/null +++ b/db/migrate/20180213131630_add_partial_index_to_projects_for_index_only_scans.rb @@ -0,0 +1,21 @@ +class AddPartialIndexToProjectsForIndexOnlyScans < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + INDEX_NAME = 'index_projects_on_id_partial_for_visibility' + + disable_ddl_transaction! + + # Adds a partial index to leverage index-only scans when looking up project ids + def up + unless index_exists?(:projects, :id, name: INDEX_NAME) + add_concurrent_index :projects, :id, name: INDEX_NAME, unique: true, where: 'visibility_level IN (10,20)' + end + end + + def down + if index_exists?(:projects, :id, name: INDEX_NAME) + remove_concurrent_index_by_name :projects, INDEX_NAME + end + end +end diff --git a/db/migrate/20180215181245_users_name_lower_index.rb b/db/migrate/20180215181245_users_name_lower_index.rb new file mode 100644 index 00000000000..d3f68cb7d45 --- /dev/null +++ b/db/migrate/20180215181245_users_name_lower_index.rb @@ -0,0 +1,29 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class UsersNameLowerIndex < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + INDEX_NAME = 'index_on_users_name_lower' + + disable_ddl_transaction! + + def up + return unless Gitlab::Database.postgresql? + + # On GitLab.com this produces an index with a size of roughly 60 MB. + execute "CREATE INDEX CONCURRENTLY #{INDEX_NAME} ON users (LOWER(name))" + end + + def down + return unless Gitlab::Database.postgresql? + + if supports_drop_index_concurrently? + execute "DROP INDEX CONCURRENTLY IF EXISTS #{INDEX_NAME}" + else + execute "DROP INDEX IF EXISTS #{INDEX_NAME}" + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 8efe5512d91..5bb461169f1 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1475,6 +1475,7 @@ ActiveRecord::Schema.define(version: 20180216121030) do add_index "projects", ["created_at"], name: "index_projects_on_created_at", using: :btree add_index "projects", ["creator_id"], name: "index_projects_on_creator_id", using: :btree add_index "projects", ["description"], name: "index_projects_on_description_trigram", using: :gin, opclasses: {"description"=>"gin_trgm_ops"} + add_index "projects", ["id"], name: "index_projects_on_id_partial_for_visibility", unique: true, where: "(visibility_level = ANY (ARRAY[10, 20]))", using: :btree add_index "projects", ["last_activity_at"], name: "index_projects_on_last_activity_at", using: :btree add_index "projects", ["last_repository_check_failed"], name: "index_projects_on_last_repository_check_failed", using: :btree add_index "projects", ["last_repository_updated_at"], name: "index_projects_on_last_repository_updated_at", using: :btree diff --git a/doc/README.md b/doc/README.md index c8b6b4f32b8..46fcb7c6baf 100644 --- a/doc/README.md +++ b/doc/README.md @@ -12,6 +12,10 @@ GitLab offers the most scalable Git-based fully integrated platform for software With GitLab self-hosted, you deploy your own GitLab instance on-premises or on a private cloud of your choice. GitLab self-hosted is available for [free and with paid subscriptions](https://about.gitlab.com/products/): Libre, Starter, Premium, and Ultimate. +Every feature available in Libre is also available in Starter, Premium, and Ultimate. +Starter features are also available in Premium and Ultimate, and Premium features are also +available in Ultimate. + GitLab.com is our SaaS offering. It's hosted, managed, and administered by GitLab, with [free and paid plans](https://about.gitlab.com/gitlab-com/) for individuals and teams: Free, Bronze, Silver, and Gold. ## Shortcuts to GitLab's most visited docs @@ -124,8 +128,8 @@ Manage your [repositories](user/project/repository/index.md) from the UI (user i ## Administrator documentation -[Administration documentation](administration/index.md) applies to admin users of [GitLab -self-hosted instances](#self-hosted-gitlab): Libre, Starter, Premium, Ultimate. +[Administration documentation](administration/index.md) applies to admin users of GitLab +self-hosted instances. Learn how to install, configure, update, upgrade, integrate, and maintain your own instance. Regular users don't have access to GitLab administration tools and settings. @@ -133,7 +137,7 @@ Regular users don't have access to GitLab administration tools and settings. ## Contributor documentation GitLab Community Edition is [open source](https://gitlab.com/gitlab-org/gitlab-ce/) -and Enterprise Editions are [open-core](https://gitlab.com/gitlab-org/gitlab-ee/). +and GitLab Enterprise Edition is [open-core](https://gitlab.com/gitlab-org/gitlab-ee/). Learn how to contribute to GitLab: - [Development](development/README.md): All styleguides and explanations how to contribute. diff --git a/doc/administration/custom_hooks.md b/doc/administration/custom_hooks.md index 4d35b20d0c3..960970aea30 100644 --- a/doc/administration/custom_hooks.md +++ b/doc/administration/custom_hooks.md @@ -4,8 +4,9 @@ **Note:** Custom Git hooks must be configured on the filesystem of the GitLab server. Only GitLab server administrators will be able to complete these tasks. Please explore [webhooks] as an option if you do not -have filesystem access. For a user configurable Git hook interface, please see -[GitLab Enterprise Edition Git Hooks](http://docs.gitlab.com/ee/git_hooks/git_hooks.html). +have filesystem access. For a user configurable Git hook interface, see +[Push Rules](https://docs.gitlab.com/ee/push_rules/push_rules.html), +available in GitLab Enterprise Edition. Git natively supports hooks that are executed on different actions. Examples of server-side git hooks include pre-receive, post-receive, and update. diff --git a/doc/administration/index.md b/doc/administration/index.md index e53268e5f3e..51444651bdb 100644 --- a/doc/administration/index.md +++ b/doc/administration/index.md @@ -1,9 +1,19 @@ # Administrator documentation Learn how to administer your GitLab instance (Community Edition and -[Enterprise Editions](https://about.gitlab.com/products/)). +Enterprise Edition). Regular users don't have access to GitLab administration tools and settings. +GitLab has two product distributions: the open source +[GitLab Community Edition (CE)](https://gitlab.com/gitlab-org/gitlab-ce), +and the open core [GitLab Enterprise Edition (EE)](https://gitlab.com/gitlab-org/gitlab-ee), +available through [different subscriptions](https://about.gitlab.com/products/). + +You can [install GitLab CE or GitLab EE](https://about.gitlab.com/installation/ce-or-ee/), +but the features you'll have access to depend on the subscription you choose +(Libre, Starter, Premium, or Ultimate). GitLab Community Edition installations +only have access to Libre features. + GitLab.com is administered by GitLab, Inc., therefore, only GitLab team members have access to its admin configurations. If you're a GitLab.com user, please check the [user documentation](../user/index.html). diff --git a/doc/administration/operations/fast_ssh_key_lookup.md b/doc/administration/operations/fast_ssh_key_lookup.md index 9d1589d84aa..fa4b2acf142 100644 --- a/doc/administration/operations/fast_ssh_key_lookup.md +++ b/doc/administration/operations/fast_ssh_key_lookup.md @@ -1,7 +1,7 @@ # Fast lookup of authorized SSH keys in the database > [Introduced](https://gitlab.com/gitlab-org/gitlab-ee/issues/1631) in -> [GitLab Enterprise Edition Standard](https://about.gitlab.com/gitlab-ee) 9.3. +> [GitLab Starter](https://about.gitlab.com/gitlab-ee) 9.3. > > [Available in](https://gitlab.com/gitlab-org/gitlab-ee/issues/3953) GitLab > Community Edition 10.4. diff --git a/doc/ci/examples/browser_performance.md b/doc/ci/examples/browser_performance.md index 7bd0514d406..42dc6ef36ba 100644 --- a/doc/ci/examples/browser_performance.md +++ b/doc/ci/examples/browser_performance.md @@ -13,16 +13,20 @@ Once you set up the Runner, add a new job to `.gitlab-ci.yml`, called `performan services: - docker:dind script: + - mkdir gitlab-exporter + - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/10-5/index.js - mkdir sitespeed-results - - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io --outputFolder sitespeed-results https://my.website.com + - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results https://my.website.com + - mv sitespeed-results/data/performance.json performance.json artifacts: paths: + - performance.json - sitespeed-results/ ``` -This will create a `performance` job in your CI/CD pipeline and will run Sitespeed.io against the webpage you define. The full HTML Sitespeed.io report will be saved as an artifact, and if you have Pages enabled it can be viewed directly in your browser. For further customization options of Sitespeed.io, including the ability to provide a list of URLs to test, please consult their [documentation](https://www.sitespeed.io/documentation/sitespeed.io/configuration/). +This will create a `performance` job in your CI/CD pipeline and will run Sitespeed.io against the webpage you define. The GitLab plugin for Sitespeed.io is downloaded in order to export key metrics to JSON. The full HTML Sitespeed.io report will also be saved as an artifact, and if you have Pages enabled it can be viewed directly in your browser. For further customization options of Sitespeed.io, including the ability to provide a list of URLs to test, please consult their [documentation](https://www.sitespeed.io/documentation/sitespeed.io/configuration/). -For [GitLab Premium](https://about.gitlab.com/products/) users, a performance score can be automatically +For [GitLab Premium](https://about.gitlab.com/products/) users, key metrics are automatically extracted and shown right in the merge request widget. Learn more about [Browser Performance Testing](https://docs.gitlab.com/ee/user/project/merge_requests/browser_performance_testing.html). ## Performance testing on Review Apps @@ -42,9 +46,15 @@ A simple `performance` job would look like: - docker:dind script: - export CI_ENVIRONMENT_URL=$(cat environment_url.txt) + - mkdir gitlab-exporter + - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/10-5/index.js - mkdir sitespeed-results - - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io --outputFolder sitespeed-results $CI_ENVIRONMENT_URL + - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:6.3.1 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL" + - mv sitespeed-results/data/performance.json performance.json artifacts: paths: + - performance.json - sitespeed-results/ ``` + +A complete example can be found in our [Auto DevOps CI YML](https://gitlab.com/gitlab-org/gitlab-ci-yml/blob/master/Auto-DevOps.gitlab-ci.yml).
\ No newline at end of file diff --git a/doc/customization/branded_login_page.md b/doc/customization/branded_login_page.md index d4d9f5f7b5e..b892f59d777 100644 --- a/doc/customization/branded_login_page.md +++ b/doc/customization/branded_login_page.md @@ -1,6 +1,6 @@ # Changing the appearance of the login page -GitLab Community Edition offers a way to put your company's identity on the login page of your GitLab server and make it a branded login page. +GitLab offers a way to put your company's identity on the login page of your GitLab server and make it a branded login page. By default, the page shows the GitLab logo and description. diff --git a/doc/development/architecture.md b/doc/development/architecture.md index d1ba7d3dfc3..31117b5e723 100644 --- a/doc/development/architecture.md +++ b/doc/development/architecture.md @@ -2,9 +2,11 @@ ## Software delivery -There are two editions of GitLab: [Enterprise Edition](https://about.gitlab.com/gitlab-ee/) (EE) and [Community Edition](https://about.gitlab.com/gitlab-ce/) (CE). GitLab CE is delivered via git from the [gitlabhq repository](https://gitlab.com/gitlab-org/gitlab-ce/tree/master). New versions of GitLab are released in stable branches and the master branch is for bleeding edge development. +There are two software distributions of GitLab: the open source [Community Edition](https://gitlab.com/gitlab-org/gitlab-ce/) (CE), and the open core [Enterprise Edition](https://gitlab.com/gitlab-org/gitlab-ee/) (EE). GitLab is available under [different subscriptions](https://about.gitlab.com/products/). -EE releases are available not long after CE releases. To obtain the GitLab EE there is a [repository at gitlab.com](https://gitlab.com/gitlab-org/gitlab-ee). For more information about the release process see the section 'New versions and upgrading' in the readme. +New versions of GitLab are released in stable branches and the master branch is for bleeding edge development. + +For information, see the [GitLab Release Process](https://gitlab.com/gitlab-org/release/docs/tree/master#gitlab-release-process). Both EE and CE require some add-on components called gitlab-shell and Gitaly. These components are available from the [gitlab-shell](https://gitlab.com/gitlab-org/gitlab-shell/tree/master) and [gitaly](https://gitlab.com/gitlab-org/gitaly/tree/master) repositories respectively. New versions are usually tags but staying on the master branch will give you the latest stable version. New releases are generally around the same time as GitLab CE releases with exception for informal security updates deemed critical. diff --git a/doc/development/doc_styleguide.md b/doc/development/doc_styleguide.md index cfeeed2506d..6fe5f647d6c 100644 --- a/doc/development/doc_styleguide.md +++ b/doc/development/doc_styleguide.md @@ -236,6 +236,11 @@ Inside the document: ## New features +New features must be shipped with its accompanying documentation and the doc +reviewed by a technical writer. + +### Mentioning GitLab versions and tiers + - Every piece of documentation that comes with a new feature should declare the GitLab version that feature got introduced. Right below the heading add a note: @@ -244,7 +249,7 @@ Inside the document: > Introduced in GitLab 8.3. ``` -- If possible every feature should have a link to the MR that introduced it. +- If possible every feature should have a link to the MR, issue, or epic that introduced it. The above note would be then transformed to: ``` @@ -254,11 +259,12 @@ Inside the document: , where the [link identifier](#links) is named after the repository (CE) and the MR number. -- If the feature is only in GitLab Enterprise Edition, don't forget to mention - it, like: +- If the feature is only available in GitLab Enterprise Edition, don't forget to mention + the [paid tier](https://about.gitlab.com/handbook/marketing/product-marketing/#tiers) + the feature is available in: ``` - > Introduced in GitLab Enterprise Edition 8.3. + > [Introduced][ee-1234] in [GitLab Starter](https://about.gitlab.com/products/) 8.3. ``` Otherwise, leave this mention out. diff --git a/doc/development/licensing.md b/doc/development/licensing.md index 274923c2d43..c06bc0d4731 100644 --- a/doc/development/licensing.md +++ b/doc/development/licensing.md @@ -1,6 +1,6 @@ # GitLab Licensing and Compatibility -GitLab CE is licensed under the terms of the MIT License. GitLab EE is licensed under "The GitLab Enterprise Edition (EE) license" wherein there are more restrictions. See their respective LICENSE files ([CE][CE], [EE][EE]) for more information. +[GitLab Community Edition](https://gitlab.com/gitlab-org/gitlab-ce/) (CE) is licensed [under the terms of the MIT License][CE]. [GitLab Enterprise Edition](https://gitlab.com/gitlab-org/gitlab-ee/) (EE) is licensed under "[The GitLab Enterprise Edition (EE) license][EE]" wherein there are more restrictions. ## Automated Testing diff --git a/doc/install/azure/index.md b/doc/install/azure/index.md index 7afe338ae8b..b0c3ad960bb 100644 --- a/doc/install/azure/index.md +++ b/doc/install/azure/index.md @@ -38,9 +38,10 @@ create SQL Databases, author websites, and perform lots of other cloud tasks. ## Create New VM The [Azure Marketplace][Azure-Marketplace] is an online store for pre-configured applications and -services which have been optimized for the cloud by software vendors like GitLab, and both -the [Community Edition ("CE")][CE] and the [Enterprise Edition ("EE")][EE] versions of GitLab are -available on the Azure Marketplace as pre-configured solutions. +services which have been optimized for the cloud by software vendors like GitLab, +available on the Azure Marketplace as pre-configured solutions. In this tutorial +we will install GitLab Community Edition, but for GitLab Enterprise Edition you +can follow the same process. To begin creating a new GitLab VM, click on the **+ New** icon, type "GitLab" into the search box, and then click the **"GitLab Community Edition"** search result: diff --git a/doc/user/project/integrations/img/prometheus_dashboard.png b/doc/user/project/integrations/img/prometheus_dashboard.png Binary files differnew file mode 100644 index 00000000000..bd19f1b44cc --- /dev/null +++ b/doc/user/project/integrations/img/prometheus_dashboard.png diff --git a/doc/user/project/integrations/img/prometheus_deploy.png b/doc/user/project/integrations/img/prometheus_deploy.png Binary files differnew file mode 100644 index 00000000000..d39081bcc7b --- /dev/null +++ b/doc/user/project/integrations/img/prometheus_deploy.png diff --git a/doc/user/project/integrations/img/prometheus_gcp_firewall_rule.png b/doc/user/project/integrations/img/prometheus_gcp_firewall_rule.png Binary files differdeleted file mode 100644 index e30cba211e6..00000000000 --- a/doc/user/project/integrations/img/prometheus_gcp_firewall_rule.png +++ /dev/null diff --git a/doc/user/project/integrations/img/prometheus_gcp_node_name.png b/doc/user/project/integrations/img/prometheus_gcp_node_name.png Binary files differdeleted file mode 100644 index ea289431454..00000000000 --- a/doc/user/project/integrations/img/prometheus_gcp_node_name.png +++ /dev/null diff --git a/doc/user/project/integrations/prometheus.md b/doc/user/project/integrations/prometheus.md index 5fefb3b69c4..249463fb86e 100644 --- a/doc/user/project/integrations/prometheus.md +++ b/doc/user/project/integrations/prometheus.md @@ -2,119 +2,69 @@ > [Introduced][ce-8935] in GitLab 9.0. -GitLab offers powerful integration with [Prometheus] for monitoring your apps. -Metrics are retrieved from the configured Prometheus server, and then displayed +GitLab offers powerful integration with [Prometheus] for monitoring key metrics your apps, directly within GitLab. +Metrics for each environment are retrieved from Prometheus, and then displayed within the GitLab interface. -Each project can be configured with its own specific Prometheus server, see the -[configuration](#configuration) section for more details. If you have a single -Prometheus server which monitors all of your infrastructure, you can pre-fill -the settings page with a default template. To configure the template, see the -[Services templates](services_templates.md) document. + -## Requirements +There are two ways to setup Prometheus integration, depending on where your apps are running: +* For deployments on Kubernetes, GitLab can automatically [deploy and manage Prometheus](#managed-prometheus-on-kubernetes) +* For other deployment targets, simply [specify the Prometheus server](#manual-configuration-of-prometheus). -Integration with Prometheus requires the following: - -1. GitLab 9.0 or higher -1. Prometheus must be configured to collect one of the [supported metrics](prometheus_library/metrics.md) -1. Each metric must be have a label to indicate the environment -1. GitLab must have network connectivity to the Prometheus server - -## Getting started with Prometheus monitoring +## Managed Prometheus on Kubernetes +> **Note**: [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/issues/28916) in GitLab 10.5 -Depending on your deployment and where you have located your GitLab server, there are a few options to get started with Prometheus monitoring. +GitLab can seamlessly deploy and manage Prometheus on a [connected Kubernetes cluster](../clusters/index.md), making monitoring of your apps easy. -* If both GitLab and your applications are installed in the same Kubernetes cluster, you can leverage the [bundled Prometheus server within GitLab](#configuring-omnibus-gitlab-prometheus-to-monitor-kubernetes). -* If your applications are deployed on Kubernetes, but GitLab is not in the same cluster, then you can [configure a Prometheus server in your Kubernetes cluster](#configuring-your-own-prometheus-server-within-kubernetes). -* If your applications are not running in Kubernetes, [get started with Prometheus](#getting-started-with-prometheus-outside-of-kubernetes). - -### Getting started with Prometheus outside of Kubernetes - -Installing and configuring Prometheus to monitor applications is fairly straight forward. - -1. [Install Prometheus](https://prometheus.io/docs/introduction/install/) -1. Set up one of the [supported monitoring targets](prometheus_library/metrics.md) -1. Configure the Prometheus server to [collect their metrics](https://prometheus.io/docs/operating/configuration/#scrape_config) +### Requirements -### Configuring Omnibus GitLab Prometheus to monitor Kubernetes deployments +* A [connected Kubernetes cluster](../clusters/index.md) +* Helm Tiller [installed by GitLab](../clusters/index.md#installing-applications) -With Omnibus GitLab running inside of Kubernetes, you can leverage the bundled -version of Prometheus to collect the supported metrics. Once enabled, Prometheus will automatically begin monitoring Kubernetes Nodes and any [annotated Pods](https://prometheus.io/docs/operating/configuration/#<kubernetes_sd_config>). +### Getting started -1. Read how to configure the bundled Prometheus server in the - [Administration guide][gitlab-prometheus-k8s-monitor]. -1. Now that Prometheus is configured, proceed on - [configuring the Prometheus project service in GitLab](#configuration-in-gitlab). +Once you have a connected Kubernetes cluster with Helm installed, deploying a managed Prometheus is as easy as a single click. -### Configuring your own Prometheus server within Kubernetes +1. Go to the `CI/CD > Kubernetes` page, to view your connected clusters +1. Select the cluster you would like to deploy Prometheus to +1. Click the **Install** button to deploy Prometheus to the cluster -Setting up and configuring Prometheus within Kubernetes is quick and painless. -The Prometheus project provides an [official Docker image][prometheus-docker-image] -which we can use as a starting point. + -To get started quickly, we have provided a [sample YML file][prometheus-yml] -that can be used as a template. This file will create a `prometheus` **Namespace**, -**Service**, **Deployment**, and **ConfigMap** in Kubernetes. You can upload -this file to the Kubernetes dashboard using **+ Create** at the top right. +### About managed Prometheus deployments - +Prometheus is deployed into the `gitlab-managed-apps` namespace, using the [official Helm chart](https://github.com/kubernetes/charts/tree/master/stable/prometheus). Prometheus is only accessible within the cluster, with GitLab communicating through the [Kubernetes API](https://kubernetes.io/docs/concepts/overview/kubernetes-api/). -Or use `kubectl`: +The Prometheus server will [automatically detect and monitor](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#%3Ckubernetes_sd_config%3E) nodes, pods, and endpoints. To configure a resource to be monitored by Prometheus, simply set the following [Kubernetes annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/): +* `prometheus.io/scrape` to `true` to enable monitoring of the resource. +* `prometheus.io/port` to define the port of the metrics endpoint. +* `prometheus.io/path` to define the path of the metrics endpoint. Defaults to `/metrics`. -```bash -kubectl apply -f path/to/prometheus.yml -``` +CPU and Memory consumption is monitored, but requires [naming conventions](prometheus_library/kubernetes.html#specifying-the-environment) in order to determine the environment. If you are using [Auto DevOps](../../../topics/autodevops/), this is handled automatically. -Once deployed, you should see the Prometheus service, deployment, and -pod start within the `prometheus` namespace. The server will begin to collect -metrics from each Kubernetes Node in the cluster, based on the configuration -provided in the template. It will also attempt to collect metrics from any Kubernetes Pods that have been [annotated for Prometheus](https://prometheus.io/docs/operating/configuration/#pod). +The [NGINX Ingress](../clusters/index.md#installing-applications) that is deployed by GitLab to clusters, is automatically annotated for monitoring providing key response metrics: latency, throughput, and error rates. -Since GitLab is not running within Kubernetes, the template provides external -network access via a `NodePort` running on `30090`. This method allows access -to be controlled using provider firewall rules, like within Google Compute Engine. +## Manual configuration of Prometheus -Since a `NodePort` does not automatically have firewall rules created for it, -one will need to be created manually to allow access. In GCP/GKE, you will want -to confirm the Node that the Prometheus pod is running on. This can be done -either by looking at the Pod in the Kubernetes dashboard, or by running: +### Requirements -```bash -kubectl describe pods -n prometheus -``` - -Next on GKE, we need to get the `tag` of the Node or VM Instance, so we can -create an accurate firewall rule. The easiest way to do this is to go into the -Google Cloud Platform Compute console and select the VM instance that matches -the name of the Node gathered from the step above. In this case, the node tag -needed is `gke-prometheus-demo-5d5ada10-node`. Also make a note of the -**External IP**, which will be the IP address the Prometheus server is reachable -on. - - - -Armed with the proper Node tag, the firewall rule can now be created -specifically for this node. To create the firewall rule, open the Google Cloud -Platform Networking console, and select **Firewall Rules**. - -Create a new rule: +Integration with Prometheus requires the following: -- Specify the source IP range to match your desired access list, which should - include your GitLab server. A sample of GitLab.com's IP address range is - available [in this issue][gitlab.com-ip-range], but note that GitLab.com's IPs - are subject to change without prior notification. -- Allowed protocol and port should be `tcp:30090`. -- The target tags should match the Node tag identified earlier in this step. +1. GitLab 9.0 or higher +1. Prometheus must be configured to collect one of the [supported metrics](prometheus_library/metrics.md) +1. Each metric must be have a label to indicate the environment +1. GitLab must have network connectivity to the Prometheus server - +### Getting started ---- +Installing and configuring Prometheus to monitor applications is fairly straight forward. -Now that Prometheus is configured, proceed to -[configure the Prometheus project service in GitLab](##configuration-in-gitlab). +1. [Install Prometheus](https://prometheus.io/docs/introduction/install/) +1. Set up one of the [supported monitoring targets](prometheus_library/metrics.md) +1. Configure the Prometheus server to [collect their metrics](https://prometheus.io/docs/operating/configuration/#scrape_config) -## Configuration in GitLab +### Configuration in GitLab The actual configuration of Prometheus integration within GitLab is very simple. All you will need is the DNS or IP address of the Prometheus server you'd like diff --git a/doc/user/project/integrations/prometheus_library/kubernetes.md b/doc/user/project/integrations/prometheus_library/kubernetes.md index a6673fa2a00..02adc562028 100644 --- a/doc/user/project/integrations/prometheus_library/kubernetes.md +++ b/doc/user/project/integrations/prometheus_library/kubernetes.md @@ -24,9 +24,10 @@ Prometheus server up and running. You have two options here: - If you have an Omnibus based GitLab installation within your Kubernetes cluster, you can leverage the bundled Prometheus server to [monitor Kubernetes](../../../../administration/monitoring/prometheus/index.md#configuring-prometheus-to-monitor-kubernetes). - To configure your own Prometheus server, you can follow the [Prometheus documentation](https://prometheus.io/docs/introduction/overview/) or [our guide](../../../../administration/monitoring/prometheus/index.md#configuring-your-own-prometheus-server-within-kubernetes). -## Specifying the Environment label +## Specifying the Environment -In order to isolate and only display relevant metrics for a given environment -however, GitLab needs a method to detect which labels are associated. To do this, GitLab will [look for an `environment` label](metrics.md#identifying-environments). +In order to isolate and only display relevant CPU and Memory metrics for a given environment, GitLab needs a method to detect which containers it is running. Because these metrics are tracked at the container level, traditional Kubernetes labels are not available. + +Instead, the [Deployment](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/) or [DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) name should begin with [CI_ENVIRONMENT_SLUG](../../../../ci/variables/README.md#predefined-variables-environment-variables). It can be followed by a `-` and additional content if desired. For example, a deployment name of `review-homepage-5620p5` would match the `review/homepage` environment. If you are using [GitLab Auto-Deploy](../../../../ci/autodeploy/index.md) and one of the two [provided Kubernetes monitoring solutions](../prometheus.md#getting-started-with-prometheus-monitoring), the `environment` label will be automatically added. diff --git a/doc/user/project/integrations/prometheus_library/nginx_ingress.md b/doc/user/project/integrations/prometheus_library/nginx_ingress.md index e6f13d0630b..49b34c82ae6 100644 --- a/doc/user/project/integrations/prometheus_library/nginx_ingress.md +++ b/doc/user/project/integrations/prometheus_library/nginx_ingress.md @@ -2,11 +2,11 @@ > [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/13438) in GitLab 9.5 -GitLab has support for automatically detecting and monitoring the Kubernetes NGINX ingress controller. This is provided by leveraging the built in Prometheus metrics included in [version 0.9.0](https://github.com/kubernetes/ingress/blob/master/controllers/nginx/Changelog.md#09-beta1) of the ingress. +GitLab has support for automatically detecting and monitoring the Kubernetes NGINX ingress controller. This is provided by leveraging the built in Prometheus metrics included in [version 0.9.0](https://github.com/kubernetes/ingress/blob/master/controllers/nginx/Changelog.md#09-beta1) and above of the ingress. ## Requirements -The [Prometheus service](../prometheus/index.md) must be enabled. +[Prometheus integration](../prometheus/index.md) must be active. ## Metrics supported @@ -18,24 +18,34 @@ The [Prometheus service](../prometheus/index.md) must be enabled. ## Configuring NGINX ingress monitoring -If you have deployed with the [gitlab-omnibus](https://docs.gitlab.com/ee/install/kubernetes/gitlab_omnibus.md) Helm chart, and your application is running in the same cluster, no further action is required. The ingress metrics will be automatically enabled and annotated for Prometheus monitoring. Simply ensure Prometheus monitoring is [enabled for your project](../prometheus.md), which is on by default. +If you have deployed NGINX Ingress using GitLab's [Kubernetes cluster integration](../../clusters/index.md#installing-applications), it will [automatically be monitored](#about-managed-nginx-ingress-deployments) by Prometheus. -For other deployments, there is some configuration required depending on your installation: -* NGINX Ingress should be version 0.9.0 or above +For other deployments, there is [some configuration](#manually-setting-up-nginx-ingress-for-prometheus-monitoring) required depending on your installation: +* NGINX Ingress should be version 0.9.0 or above, with metrics enabled * NGINX Ingress should be annotated for Prometheus monitoring * Prometheus should be configured to monitor annotated pods -### Setting up NGINX Ingress for Prometheus monitoring +### About managed NGINX Ingress deployments + +NGINX Ingress is deployed into the `gitlab-managed-apps` namespace, using the [official Helm chart](https://github.com/kubernetes/charts/tree/master/stable/nginx-ingress). NGINX Ingress will be [externally reachable via the Load Balancer's IP](https://docs.gitlab.com/ce/user/project/clusters/index.html#getting-the-external-ip-address). + +NGINX is configured for Prometheus monitoring, by setting: +* `enable-vts-status: "true"`, to export Prometheus metrics +* `prometheus.io/scrape: "true"`, to enable automatic discovery +* `prometheus.io/port: "10254"`, to specify the metrics port + +When used in conjunction with the GitLab deployed Prometheus service, response metrics will be automatically collected. + +### Manually setting up NGINX Ingress for Prometheus monitoring Version 0.9.0 and above of [NGINX ingress](https://github.com/kubernetes/ingress/tree/master/controllers/nginx) have built-in support for exporting Prometheus metrics. To enable, a ConfigMap setting must be passed: `enable-vts-status: "true"`. Once enabled, a Prometheus metrics endpoint will start running on port 10254. -With metric data now available, Prometheus needs to be configured to collect it. The easiest way to do this is to leverage Prometheus' [built-in Kubernetes service discovery](https://prometheus.io/docs/operating/configuration/#kubernetes_sd_config), which automatically detects a variety of Kubernetes components and makes them available for monitoring. Since NGINX ingress metrics are exposed per pod, a scrape job for Kubernetes pods is required. A sample pod scraping configuration [is available](https://github.com/prometheus/prometheus/blob/master/documentation/examples/prometheus-kubernetes.yml#L248). This configuration will detect pods and enable collection of metrics **only if** they have been specifically annotated for monitoring. +Next, the ingress needs to be annotated for Prometheus monitoring. Two new annotations need to be added: -Depending on how NGINX ingress was deployed, typically a DaemonSet or Deployment, edit the corresponding YML spec. Two new annotations need to be added: * `prometheus.io/scrape: "true"` * `prometheus.io/port: "10254"` -Prometheus should now be collecting NGINX ingress metrics. To validate view the Prometheus Targets, available under `Status > Targets` on the Prometheus dashboard. New entries for NGINX should be listed in the kubernetes pod monitoring job, `kubernetes-pods`. +Managing these settings depends on how NGINX ingress has been deployed. If you have deployed via the [official Helm chart](https://github.com/kubernetes/charts/tree/master/stable/nginx-ingress), metrics can be enabled with `controller.stats.enabled` along with the required annotations. Alternatively it is possible edit the NGINX ingress YML directly in the [Kubernetes dashboard](https://github.com/kubernetes/dashboard). ## Specifying the Environment label diff --git a/doc/user/project/integrations/samples/prometheus.yml b/doc/user/project/integrations/samples/prometheus.yml deleted file mode 100644 index 3a4735d282f..00000000000 --- a/doc/user/project/integrations/samples/prometheus.yml +++ /dev/null @@ -1,107 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: prometheus ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: prometheus - namespace: prometheus -data: - prometheus.yml: |- - scrape_configs: - - job_name: 'kubernetes-nodes' - scheme: https - tls_config: - ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - insecure_skip_verify: true - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - kubernetes_sd_configs: - - role: node - metric_relabel_configs: - - source_labels: [pod_name] - target_label: environment - regex: (.+)-.+-.+ - replacement: $1 - - job_name: kubernetes-pods - tls_config: - ca_file: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" - insecure_skip_verify: true - bearer_token_file: "/var/run/secrets/kubernetes.io/serviceaccount/token" - kubernetes_sd_configs: - - role: pod - api_server: https://kubernetes.default.svc:443 - tls_config: - ca_file: "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" - bearer_token_file: "/var/run/secrets/kubernetes.io/serviceaccount/token" - relabel_configs: - - source_labels: - - __meta_kubernetes_pod_annotation_prometheus_io_scrape - action: keep - regex: 'true' - - source_labels: - - __meta_kubernetes_pod_annotation_prometheus_io_path - action: replace - target_label: __metrics_path__ - regex: "(.+)" - - source_labels: - - __address__ - - __meta_kubernetes_pod_annotation_prometheus_io_port - action: replace - regex: "([^:]+)(?::[0-9]+)?;([0-9]+)" - replacement: "$1:$2" - target_label: __address__ - - action: labelmap - regex: __meta_kubernetes_pod_label_(.+) - - source_labels: - - __meta_kubernetes_namespace - action: replace - target_label: kubernetes_namespace - - source_labels: - - __meta_kubernetes_pod_name - action: replace - target_label: kubernetes_pod_name ---- -apiVersion: v1 -kind: Service -metadata: - name: prometheus - namespace: prometheus -spec: - selector: - app: prometheus - ports: - - name: prometheus - protocol: TCP - port: 9090 - nodePort: 30090 - type: NodePort ---- -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: prometheus - namespace: prometheus -spec: - replicas: 1 - template: - metadata: - labels: - app: prometheus - spec: - containers: - - name: prometheus - image: prom/prometheus:latest - args: - - '--config.file=/prometheus-data/prometheus.yml' - ports: - - name: prometheus - containerPort: 9090 - volumeMounts: - - name: data-volume - mountPath: /prometheus-data - volumes: - - name: data-volume - configMap: - name: prometheus diff --git a/doc/user/project/issues/issues_functionalities.md b/doc/user/project/issues/issues_functionalities.md index 0bef83d18e8..f2ca6a6822e 100644 --- a/doc/user/project/issues/issues_functionalities.md +++ b/doc/user/project/issues/issues_functionalities.md @@ -50,8 +50,8 @@ Often multiple people likely work on the same issue together, which can especially be difficult to track in large teams where there is shared ownership of an issue. -In GitLab Enterprise Edition, you can also select multiple assignees -to an issue. +In [GitLab Starter](https://about.gitlab.com/products/), you can also +select multiple assignees to an issue. Learn more on the [Multiple Assignees documentation](https://docs.gitlab.com/ee/user/project/issues/multiple_assignees_for_issues.html). diff --git a/doc/user/project/milestones/img/milestone_create.png b/doc/user/project/milestones/img/milestone_create.png Binary files differdeleted file mode 100644 index beb2caa897f..00000000000 --- a/doc/user/project/milestones/img/milestone_create.png +++ /dev/null diff --git a/doc/user/project/milestones/img/milestone_group_create.png b/doc/user/project/milestones/img/milestone_group_create.png Binary files differdeleted file mode 100644 index 7aaa7c56c15..00000000000 --- a/doc/user/project/milestones/img/milestone_group_create.png +++ /dev/null diff --git a/doc/user/project/milestones/img/milestones_new_group_milestone.png b/doc/user/project/milestones/img/milestones_new_group_milestone.png Binary files differnew file mode 100644 index 00000000000..8780394d72e --- /dev/null +++ b/doc/user/project/milestones/img/milestones_new_group_milestone.png diff --git a/doc/user/project/milestones/img/milestones_new_project_milestone.png b/doc/user/project/milestones/img/milestones_new_project_milestone.png Binary files differnew file mode 100644 index 00000000000..ba058428dfa --- /dev/null +++ b/doc/user/project/milestones/img/milestones_new_project_milestone.png diff --git a/doc/user/project/milestones/img/milestones_project_milestone_page.png b/doc/user/project/milestones/img/milestones_project_milestone_page.png Binary files differnew file mode 100644 index 00000000000..9717075b8d0 --- /dev/null +++ b/doc/user/project/milestones/img/milestones_project_milestone_page.png diff --git a/doc/user/project/milestones/img/milestones_promote_milestone.png b/doc/user/project/milestones/img/milestones_promote_milestone.png Binary files differnew file mode 100644 index 00000000000..99bee1240d4 --- /dev/null +++ b/doc/user/project/milestones/img/milestones_promote_milestone.png diff --git a/doc/user/project/milestones/img/sidebar.png b/doc/user/project/milestones/img/sidebar.png Binary files differdeleted file mode 100644 index 274962a936c..00000000000 --- a/doc/user/project/milestones/img/sidebar.png +++ /dev/null diff --git a/doc/user/project/milestones/index.md b/doc/user/project/milestones/index.md index 27832b0fa2b..d3e9bf9e6a8 100644 --- a/doc/user/project/milestones/index.md +++ b/doc/user/project/milestones/index.md @@ -1,63 +1,111 @@ # Milestones -Milestones allow you to organize issues and merge requests into a cohesive group, -optionally setting a due date. A common use is keeping track of an upcoming -software version. Milestones can be created per-project or per-group. +## Overview -## Creating a project milestone +Milestones in GitLab are a way to track issues and merge requests created to achieve a broader goal in a certain period of time. + +Milestones allow you to organize issues and merge requests into a cohesive group, with an optional start date and an optional due date. + +## Project milestones and group milestones + +- **Project milestones** can be assigned to issues or merge requests in that project only. +- **Group milestones** can be assigned to any issue or merge request of any project in that group. +- In the [future](https://gitlab.com/gitlab-org/gitlab-ce/issues/36862), you will be able to assign group milestones to issues and merge reqeusts of projects in [subgroups](../../group/subgroups/index.md). + +## Creating milestones + +>**Note:** +A permission level of `Developer` or higher is required to create milestones. + +### New project milestone + +To create a **project milestone**, navigate to **Issues > Milestones** in the project. + +Click the **New milestone** button. Enter the title, an optional description, an optional start date, and an optional due date. Click **Create milestone** to create the milestone. + + + +### New group milestone + +To create a **group milestone**, follow similar steps from above to project milestones. Navigate to **Issues > Milestones** in the group and create it from there. + + + +## Editing milestones >**Note:** -You need [Master permissions](../../permissions.md) in order to create a milestone. +A permission level of `Developer` or higher is required to edit milestones. + +You can update a milestone by navigating to **Issues > Milestones** in the project or group and clicking the **Edit** button. -You can find the milestones page under your project's **Issues ➔ Milestones**. -To create a new milestone, simply click the **New milestone** button when in the -milestones page. A milestone can have a title, a description and start/due dates. -Once you fill in all the details, hit the **Create milestone** button. +You can delete a milestone by clicking the **Delete** button. - +### Promoting project milestones to group milestones -## Creating a group milestone +If you are expanding from a few projects to a larger number of projects within the same group, you may want to share the same milestone among multiple projects in the same group. If you previously created a project milestone and now want to make it available for other milestones, you can promote it to a group milestone. + +From the project milestone list page, you can promote a project milestone to a group milestone. This will merge all project milestones across all projects in this group with the same name into a single group milestones. All issues and merge requests that previously were assigned one of these project milestones will now be assigned the new group milestones. This action cannot be reversed and the changes are permanent. >**Note:** -You need [Master permissions](../../permissions.md) in order to create a milestone. +Not all features on the project milestone view are available on the group milestone view. If you promote a project milestone to a group milestone, you will lose these features. See [Milestone view](#milestone-view) to see which features are missing from the group milestone view. + + + +## Assigning milestones from the sidebar + +Every issue and merge request can be assigned a milestone. The milestones are visible on every issue and merge request page, in the sidebar. They are also visible in the issue board. From the sidebar, you can assign or unassign a milestones to the object. You can also perform this as a [quick action](../quick_actions.md) in a comment. [As mentioned](#project-milestones-and-group-milestones), for a given issue or merge request, both project milestones and group milestones can be selected and assigned to the object. + +## Filtering issues and merge requests by milestone + +### Filtering in list pages + +From the project issue/merge request list pages and the group issue/merge request list pages, you can [filter](../../search/index.md#issues-and-merge-requests) by both group milestones and project milestones. + +### Filtering in issue boards + +From [project issue boards](../issue_board.md), you can filter by both group milestones and project milestones in the [search and filter bar](../../search/index.md#issue-boards). + +### Special milestone filters + +When filtering by milestone, in addition to choosing a specific project milestone or group milestone, you can choose a special milestone filter. -You can create a milestone for a group that will be shared across group projects. -On the group's **Issues ➔ Milestones** page, you will be able to see the state -of that milestone and the issues/merge requests count that it shares across the group projects. To create a new milestone click the **New milestone** button. The form is the same as when creating a milestone for a specific project which you can find in the previous item. +- **No Milestone**: Show issues or merge requests with no assigned milestone. +- **Upcoming**: Show issues or merge requests that have been assigned the open milestone that has the next upcoming due date (i.e. nearest due date in the future). +- **Started**: Show issues or merge requests that have an assigned milestone with a start date that is before today. -In addition to that you will be able to filter issues or merge requests by group milestones in all projects that belongs to the milestone group. +## Milestone view -## Milestone promotion +Not all features in the project milestone view are available in the group milestone view. This table summarizes the differences: -Project milestones can be promoted to group milestones if its project belongs to a group. When a milestone is promoted all other milestones across the group projects with the same title will be merged into it, which means all milestone's children like issues, merge requests and boards will be moved into the new promoted milestone. -The promote button can be found in the milestone view or milestones list. +| Feature | Project milestone view | Group milestone view | +|---|:---:|:---:| +| Title an description | ✓ | ✓ | +| Issues assigned to milestone | ✓ | | +| Merge requests assigned to milestone | ✓ | | +| Participants and labels used | ✓ | | +| Percentage complete | ✓ | ✓ | +| Start date and due date | ✓ | ✓ | +| Total issue time spent | ✓ | ✓ | +| Total issue weight | ✓ | | -## Special milestone filters +The milestone view shows the title and description. -In addition to the milestones that exist in the project or group, there are some -special options available when filtering by milestone: +### Project milestone features -* **No Milestone** - only show issues or merge requests without a milestone. -* **Upcoming** - show issues or merge request that belong to the next open - milestone with a due date, by project. (For example: if project A has - milestone v1 due in three days, and project B has milestone v2 due in a week, - then this will show issues or merge requests from milestone v1 in project A - and milestone v2 in project B.) -* **Started** - show issues or merge requests from any milestone with a start - date less than today. Note that this can return results from several - milestones in the same project. +These features are only available for project milestones and not group milestones. -## Milestone sidebar +- Issues assigned to the milestone are displayed in three columns: Unstarted issues, ongoing issues, and completed issues. +- Merge requests assigned to the milestone are displayed in four columns: Work in progress merge requests, waiting for merge, rejected, and closed. +- Participants and labels that are used in issues and merge requests that have the milestone assigned are displayed. -The milestone sidebar shows percentage complete, start date and due date, -issues, total issue weight, total issue time spent, and merge requests. +### Milestone sidebar -The percentage complete is calculated as: Closed and merged merge requests plus all closed issues divided by -total merge requests and issues. +The milestone sidebar on the milestone view shows the following: - +- Percentage complete, which is calculated as number of closed issues plus number of closed/merged merge requests divided by total number issues and merge requests. +- The start date and due date. +- The total time spent on all issues that have the milestone assigned. -## Quick actions +For project milestones only, the milestone sidebar shows the total issue weight of all issues that have the milestone assigned. -[Quick actions](../quick_actions.md) are available for assigning and removing -project and group milestones. +
\ No newline at end of file diff --git a/doc/user/project/repository/index.md b/doc/user/project/repository/index.md index ce081cedd71..dd0374983cc 100644 --- a/doc/user/project/repository/index.md +++ b/doc/user/project/repository/index.md @@ -66,8 +66,7 @@ your implementation with your team. You can live preview changes submitted to a new branch with [Review Apps](../../../ci/review_apps/index.md). -With [GitLab Enterprise Edition](https://about.gitlab.com/products/) -subscriptions, you can also request +With [GitLab Starter](https://about.gitlab.com/products/), you can also request [approval](https://docs.gitlab.com/ee/user/project/merge_requests/merge_request_approvals.html) from your managers. To create, delete, and [branches](branches/index.md) via GitLab's UI: diff --git a/doc/workflow/lfs/manage_large_binaries_with_git_lfs.md b/doc/workflow/lfs/manage_large_binaries_with_git_lfs.md index 8fff3d591fe..377eee69c11 100644 --- a/doc/workflow/lfs/manage_large_binaries_with_git_lfs.md +++ b/doc/workflow/lfs/manage_large_binaries_with_git_lfs.md @@ -34,7 +34,7 @@ Documentation for GitLab instance administrators is under [LFS administration do credentials store is recommended * Git LFS always assumes HTTPS so if you have GitLab server on HTTP you will have to add the URL to Git config manually (see [troubleshooting](#troubleshooting)) - + >**Note**: With 8.12 GitLab added LFS support to SSH. The Git LFS communication still goes over HTTP, but now the SSH client passes the correct credentials to the Git LFS client, so no action is required by the user. @@ -85,6 +85,8 @@ git lfs fetch master ## File Locking +> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/issues/35856) in GitLab 10.5. + The first thing to do before using File Locking is to tell Git LFS which kind of files are lockable. The following command will store PNG files in LFS and flag them as lockable: diff --git a/lib/gitlab/checks/change_access.rb b/lib/gitlab/checks/change_access.rb index d75e73dac10..3ce5f807989 100644 --- a/lib/gitlab/checks/change_access.rb +++ b/lib/gitlab/checks/change_access.rb @@ -16,11 +16,11 @@ module Gitlab lfs_objects_missing: 'LFS objects are missing. Ensure LFS is properly set up or try a manual "git lfs push --all".' }.freeze - attr_reader :user_access, :project, :skip_authorization, :protocol, :oldrev, :newrev, :ref, :branch_name, :tag_name + attr_reader :user_access, :project, :skip_authorization, :skip_lfs_integrity_check, :protocol, :oldrev, :newrev, :ref, :branch_name, :tag_name def initialize( change, user_access:, project:, skip_authorization: false, - protocol: + skip_lfs_integrity_check: false, protocol: ) @oldrev, @newrev, @ref = change.values_at(:oldrev, :newrev, :ref) @branch_name = Gitlab::Git.branch_name(@ref) @@ -28,6 +28,7 @@ module Gitlab @user_access = user_access @project = project @skip_authorization = skip_authorization + @skip_lfs_integrity_check = skip_lfs_integrity_check @protocol = protocol end @@ -37,7 +38,7 @@ module Gitlab push_checks branch_checks tag_checks - lfs_objects_exist_check + lfs_objects_exist_check unless skip_lfs_integrity_check commits_check unless skip_commits_check true @@ -120,6 +121,7 @@ module Gitlab def commits_check return if deletion? || newrev.nil? + return unless should_run_commit_validations? # n+1: https://gitlab.com/gitlab-org/gitlab-ee/issues/3593 ::Gitlab::GitalyClient.allow_n_plus_1_calls do @@ -138,6 +140,10 @@ module Gitlab private + def should_run_commit_validations? + commit_check.validate_lfs_file_locks? + end + def updated_from_web? protocol == 'web' end @@ -175,7 +181,7 @@ module Gitlab end def commits - project.repository.new_commits(newrev) + @commits ||= project.repository.new_commits(newrev) end end end diff --git a/lib/gitlab/checks/commit_check.rb b/lib/gitlab/checks/commit_check.rb index ae0cd142378..43a52b493bb 100644 --- a/lib/gitlab/checks/commit_check.rb +++ b/lib/gitlab/checks/commit_check.rb @@ -35,14 +35,14 @@ module Gitlab end end - private - def validate_lfs_file_locks? strong_memoize(:validate_lfs_file_locks) do project.lfs_enabled? && project.lfs_file_locks.any? && newrev && oldrev end end + private + def lfs_file_locks_validation lambda do |paths| lfs_lock = project.lfs_file_locks.where(path: paths).where.not(user_id: user.id).first diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index d95561fe1b2..ae27a138b7c 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -508,7 +508,7 @@ module Gitlab @committed_date = Time.at(commit.committer.date.seconds).utc @committer_name = commit.committer.name.dup @committer_email = commit.committer.email.dup - @parent_ids = commit.parent_ids + @parent_ids = Array(commit.parent_ids) end def serialize_keys diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 706930a96b7..e2b95b22f2d 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -1349,7 +1349,7 @@ module Gitlab if is_enabled gitaly_ref_client.branch_names_contains_sha(sha) else - refs_contains_sha(:branch, sha) + refs_contains_sha('refs/heads/', sha) end end end @@ -1359,7 +1359,7 @@ module Gitlab if is_enabled gitaly_ref_client.tag_names_contains_sha(sha) else - refs_contains_sha(:tag, sha) + refs_contains_sha('refs/tags/', sha) end end end @@ -1458,19 +1458,25 @@ module Gitlab end end - def refs_contains_sha(ref_type, sha) - args = %W(#{ref_type} --contains #{sha}) - names = run_git(args).first + def refs_contains_sha(refs_prefix, sha) + refs_prefix << "/" unless refs_prefix.ends_with?('/') - return [] unless names.respond_to?(:split) + # By forcing the output to %(refname) each line wiht a ref will start with + # the ref prefix. All other lines can be discarded. + args = %W(for-each-ref --contains=#{sha} --format=%(refname) #{refs_prefix}) + names, code = run_git(args) - names = names.split("\n").map(&:strip) + return [] unless code.zero? - names.each do |name| - name.slice! '* ' + refs = [] + left_slice_count = refs_prefix.length + names.lines.each do |line| + next unless line.start_with?(refs_prefix) + + refs << line.rstrip[left_slice_count..-1] end - names + refs end def rugged_write_config(full_path:) diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 8ec3386184a..bbdb593d4e2 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -198,7 +198,7 @@ module Gitlab end def check_repository_existence! - unless project.repository.exists? + unless repository.exists? raise UnauthorizedError, ERROR_MESSAGES[:no_repo] end end @@ -238,19 +238,22 @@ module Gitlab changes_list = Gitlab::ChangesList.new(changes) # Iterate over all changes to find if user allowed all of them to be applied - changes_list.each do |change| + changes_list.each.with_index do |change, index| + first_change = index == 0 + # If user does not have access to make at least one change, cancel all # push by allowing the exception to bubble up - check_single_change_access(change) + check_single_change_access(change, skip_lfs_integrity_check: !first_change) end end - def check_single_change_access(change) + def check_single_change_access(change, skip_lfs_integrity_check: false) Checks::ChangeAccess.new( change, user_access: user_access, project: project, skip_authorization: deploy_key?, + skip_lfs_integrity_check: skip_lfs_integrity_check, protocol: protocol ).exec end @@ -324,5 +327,9 @@ module Gitlab def push_to_read_only_message ERROR_MESSAGES[:cannot_push_to_read_only] end + + def repository + project.repository + end end end diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index 1c9477e84b2..a5b3902ebf4 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -13,7 +13,7 @@ module Gitlab authentication_abilities.include?(:download_code) && user_access.can_do_action?(:download_wiki_code) end - def check_single_change_access(change) + def check_single_change_access(change, _options = {}) unless user_access.can_do_action?(:create_wiki) raise UnauthorizedError, ERROR_MESSAGES[:write_to_wiki] end @@ -28,5 +28,11 @@ module Gitlab def push_to_read_only_message ERROR_MESSAGES[:read_only] end + + private + + def repository + project.wiki.repository + end end end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 45b9e14ba55..f3e48083c19 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -73,7 +73,7 @@ module Gitlab # event_name - The name of the event (e.g. "git_push"). # tags - A set of tags to attach to the event. def add_event(event_name, tags = {}) - self.class.transaction_metric(event_name, :counter, prefix: 'event_', tags: tags).increment(tags.merge(labels)) + self.class.transaction_metric(event_name, :counter, prefix: 'event_', use_feature_flag: true, tags: tags).increment(tags.merge(labels)) @metrics << Metric.new(EVENT_SERIES, { count: 1 }, tags.merge(event: event_name), :event) end @@ -150,11 +150,12 @@ module Gitlab with_feature :prometheus_metrics_transaction_allocated_memory end - def self.transaction_metric(name, type, prefix: nil, tags: {}) + def self.transaction_metric(name, type, prefix: nil, use_feature_flag: false, tags: {}) metric_name = "gitlab_transaction_#{prefix}#{name}_total".to_sym fetch_metric(type, metric_name) do docstring "Transaction #{prefix}#{name} #{type}" base_labels tags.merge(BASE_LABELS) + with_feature "prometheus_transaction_#{prefix}#{name}_total".to_sym if use_feature_flag if type == :gauge multiprocess_mode :livesum diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb index 5f0c98cb5a4..53744bad1f4 100644 --- a/lib/gitlab/sql/pattern.rb +++ b/lib/gitlab/sql/pattern.rb @@ -25,7 +25,11 @@ module Gitlab query.length >= MIN_CHARS_FOR_PARTIAL_MATCHING end - def fuzzy_arel_match(column, query) + # column - The column name to search in. + # query - The text to search for. + # lower_exact_match - When set to `true` we'll fall back to using + # `LOWER(column) = query` instead of using `ILIKE`. + def fuzzy_arel_match(column, query, lower_exact_match: false) query = query.squish return nil unless query.present? @@ -36,7 +40,13 @@ module Gitlab else # No words of at least 3 chars, but we can search for an exact # case insensitive match with the query as a whole - arel_table[column].matches(sanitize_sql_like(query)) + if lower_exact_match + Arel::Nodes::NamedFunction + .new('LOWER', [arel_table[column]]) + .eq(query) + else + arel_table[column].matches(sanitize_sql_like(query)) + end end end diff --git a/lib/google_api/cloud_platform/client.rb b/lib/google_api/cloud_platform/client.rb index ff638c07755..f30dd995695 100644 --- a/lib/google_api/cloud_platform/client.rb +++ b/lib/google_api/cloud_platform/client.rb @@ -76,9 +76,13 @@ module GoogleApi "initial_node_count": cluster_size, "node_config": { "machine_type": machine_type + }, + "legacy_abac": { + "enabled": true } } - } ) + } + ) service.create_cluster(project_id, zone, request_body, options: user_agent_header) end diff --git a/lib/tasks/migrate/setup_postgresql.rake b/lib/tasks/migrate/setup_postgresql.rake index 31cbd651edb..1c7a8a90f5c 100644 --- a/lib/tasks/migrate/setup_postgresql.rake +++ b/lib/tasks/migrate/setup_postgresql.rake @@ -8,6 +8,7 @@ task setup_postgresql: :environment do require Rails.root.join('db/migrate/20170503185032_index_redirect_routes_path_for_like') require Rails.root.join('db/migrate/20171220191323_add_index_on_namespaces_lower_name.rb') require Rails.root.join('db/migrate/20180113220114_rework_redirect_routes_indexes.rb') + require Rails.root.join('db/migrate/20180215181245_users_name_lower_index.rb') NamespacesProjectsPathLowerIndexes.new.up AddUsersLowerUsernameEmailIndexes.new.up @@ -17,4 +18,5 @@ task setup_postgresql: :environment do IndexRedirectRoutesPathForLike.new.up AddIndexOnNamespacesLowerName.new.up ReworkRedirectRoutesIndexes.new.up + UsersNameLowerIndex.new.up end diff --git a/spec/controllers/autocomplete_controller_spec.rb b/spec/controllers/autocomplete_controller_spec.rb index 73fff6eb5ca..b7257fac608 100644 --- a/spec/controllers/autocomplete_controller_spec.rb +++ b/spec/controllers/autocomplete_controller_spec.rb @@ -109,15 +109,17 @@ describe AutocompleteController do end context 'limited users per page' do - let(:per_page) { 2 } - before do + 25.times do + create(:user) + end + sign_in(user) - get(:users, per_page: per_page) + get(:users) end it { expect(json_response).to be_kind_of(Array) } - it { expect(json_response.size).to eq(per_page) } + it { expect(json_response.size).to eq(20) } end context 'unauthenticated user' do diff --git a/spec/controllers/projects/uploads_controller_spec.rb b/spec/controllers/projects/uploads_controller_spec.rb index d572085661d..eca9baed9c9 100644 --- a/spec/controllers/projects/uploads_controller_spec.rb +++ b/spec/controllers/projects/uploads_controller_spec.rb @@ -7,4 +7,12 @@ describe Projects::UploadsController do end it_behaves_like 'handle uploads' + + context 'when the URL the old style, without /-/system' do + it 'responds with a redirect to the login page' do + get :show, namespace_id: 'project', project_id: 'avatar', filename: 'foo.png', secret: 'bar' + + expect(response).to redirect_to(new_user_session_path) + end + end end diff --git a/spec/javascripts/vue_mr_widget/mr_widget_options_spec.js b/spec/javascripts/vue_mr_widget/mr_widget_options_spec.js index cd00d0a39a3..45035effe81 100644 --- a/spec/javascripts/vue_mr_widget/mr_widget_options_spec.js +++ b/spec/javascripts/vue_mr_widget/mr_widget_options_spec.js @@ -295,6 +295,15 @@ describe('mrWidgetOptions', () => { expect(notify.notifyMe).not.toHaveBeenCalled(); }); + + it('should not notify if no pipeline provided', () => { + vm.handleNotification({ + ...data, + pipeline: undefined, + }); + + expect(notify.notifyMe).not.toHaveBeenCalled(); + }); }); describe('resumePolling', () => { diff --git a/spec/lib/gitlab/checks/change_access_spec.rb b/spec/lib/gitlab/checks/change_access_spec.rb index 475b5c5cfb2..b49ddbfc780 100644 --- a/spec/lib/gitlab/checks/change_access_spec.rb +++ b/spec/lib/gitlab/checks/change_access_spec.rb @@ -190,7 +190,7 @@ describe Gitlab::Checks::ChangeAccess do context 'with LFS not enabled' do it 'skips the validation' do - expect_any_instance_of(described_class).not_to receive(:lfs_file_locks_validation) + expect_any_instance_of(Gitlab::Checks::CommitCheck).not_to receive(:validate) subject.exec end @@ -207,7 +207,7 @@ describe Gitlab::Checks::ChangeAccess do end end - context 'when change is sent by the author od the lock' do + context 'when change is sent by the author of the lock' do let(:user) { owner } it "doesn't raise any error" do diff --git a/spec/lib/gitlab/git/blob_spec.rb b/spec/lib/gitlab/git/blob_spec.rb index 59e9e1cc94c..a6341cd509b 100644 --- a/spec/lib/gitlab/git/blob_spec.rb +++ b/spec/lib/gitlab/git/blob_spec.rb @@ -276,6 +276,7 @@ describe Gitlab::Git::Blob, seed_helper: true do expect(blobs.count).to eq(1) expect(blobs).to all( be_a(Gitlab::Git::Blob) ) + expect(blobs).to be_an(Array) end it 'accepts blob IDs as a lazy enumerator' do diff --git a/spec/lib/gitlab/git/commit_spec.rb b/spec/lib/gitlab/git/commit_spec.rb index 85e6efd7ca2..0b20a6349a2 100644 --- a/spec/lib/gitlab/git/commit_spec.rb +++ b/spec/lib/gitlab/git/commit_spec.rb @@ -102,6 +102,10 @@ describe Gitlab::Git::Commit, seed_helper: true do expect(described_class.find(repository, SeedRepo::Commit::ID)).to be_valid_commit end + it "returns an array of parent ids" do + expect(described_class.find(repository, SeedRepo::Commit::ID).parent_ids).to be_an(Array) + end + it "should return valid commit for tag" do expect(described_class.find(repository, 'v1.0.0').id).to eq('6f6d7e7ed97bb5f0054f2b1df789b39ca89b6ff9') end diff --git a/spec/lib/gitlab/git/repository_spec.rb b/spec/lib/gitlab/git/repository_spec.rb index 0e9150964fa..9feaaec3f3b 100644 --- a/spec/lib/gitlab/git/repository_spec.rb +++ b/spec/lib/gitlab/git/repository_spec.rb @@ -600,6 +600,33 @@ describe Gitlab::Git::Repository, seed_helper: true do end end + describe '#branch_names_contains_sha' do + shared_examples 'returning the right branches' do + let(:head_id) { repository.rugged.head.target.oid } + let(:new_branch) { head_id } + + before do + repository.create_branch(new_branch, 'master') + end + + after do + repository.delete_branch(new_branch) + end + + it 'displays that branch' do + expect(repository.branch_names_contains_sha(head_id)).to include('master', new_branch) + end + end + + context 'when Gitaly is enabled' do + it_behaves_like 'returning the right branches' + end + + context 'when Gitaly is disabled', :disable_gitaly do + it_behaves_like 'returning the right branches' + end + end + describe "#refs_hash" do subject { repository.refs_hash } diff --git a/spec/lib/gitlab/git_access_spec.rb b/spec/lib/gitlab/git_access_spec.rb index 3c3697e7aa9..19d3f55501e 100644 --- a/spec/lib/gitlab/git_access_spec.rb +++ b/spec/lib/gitlab/git_access_spec.rb @@ -18,8 +18,9 @@ describe Gitlab::GitAccess do redirected_path: redirected_path) end - let(:push_access_check) { access.check('git-receive-pack', '_any') } - let(:pull_access_check) { access.check('git-upload-pack', '_any') } + let(:changes) { '_any' } + let(:push_access_check) { access.check('git-receive-pack', changes) } + let(:pull_access_check) { access.check('git-upload-pack', changes) } describe '#check with single protocols allowed' do def disable_protocol(protocol) @@ -646,6 +647,20 @@ describe Gitlab::GitAccess do end end + describe 'check LFS integrity' do + let(:changes) { ['6f6d7e7ed 570e7b2ab refs/heads/master', '6f6d7e7ed 570e7b2ab refs/heads/feature'] } + + before do + project.add_developer(user) + end + + it 'checks LFS integrity only for first change' do + expect_any_instance_of(Gitlab::Checks::LfsIntegrity).to receive(:objects_missing?).exactly(1).times + + push_access_check + end + end + describe '#check_push_access!' do before do merge_into_protected_branch diff --git a/spec/lib/gitlab/git_access_wiki_spec.rb b/spec/lib/gitlab/git_access_wiki_spec.rb index 186b2d9279d..215f1ecc9c5 100644 --- a/spec/lib/gitlab/git_access_wiki_spec.rb +++ b/spec/lib/gitlab/git_access_wiki_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' describe Gitlab::GitAccessWiki do let(:access) { described_class.new(user, project, 'web', authentication_abilities: authentication_abilities, redirected_path: redirected_path) } - let(:project) { create(:project, :repository) } + let(:project) { create(:project, :wiki_repo) } let(:user) { create(:user) } let(:changes) { ['6f6d7e7ed 570e7b2ab refs/heads/master'] } let(:redirected_path) { nil } @@ -48,6 +48,18 @@ describe Gitlab::GitAccessWiki do it 'give access to download wiki code' do expect { subject }.not_to raise_error end + + context 'when the wiki repository does not exist' do + it 'returns not found' do + wiki_repo = project.wiki.repository + FileUtils.rm_rf(wiki_repo.path) + + # Sanity check for rm_rf + expect(wiki_repo.exists?).to eq(false) + + expect { subject }.to raise_error(Gitlab::GitAccess::UnauthorizedError, 'A repository for this project does not exist yet.') + end + end end context 'when wiki feature is disabled' do diff --git a/spec/lib/gitlab/sql/pattern_spec.rb b/spec/lib/gitlab/sql/pattern_spec.rb index ef51e3cc8df..5b5052de372 100644 --- a/spec/lib/gitlab/sql/pattern_spec.rb +++ b/spec/lib/gitlab/sql/pattern_spec.rb @@ -154,6 +154,12 @@ describe Gitlab::SQL::Pattern do it 'returns a single equality condition' do expect(fuzzy_arel_match.to_sql).to match(/title.*I?LIKE 'fo'/) end + + it 'uses LOWER instead of ILIKE when LOWER is enabled' do + rel = Issue.fuzzy_arel_match(:title, query, lower_exact_match: true) + + expect(rel.to_sql).to match(/LOWER\(.*title.*\).*=.*'fo'/) + end end context 'with two words both equal to 3 chars' do diff --git a/spec/lib/google_api/cloud_platform/client_spec.rb b/spec/lib/google_api/cloud_platform/client_spec.rb index f65e41dfea3..db9d9158b29 100644 --- a/spec/lib/google_api/cloud_platform/client_spec.rb +++ b/spec/lib/google_api/cloud_platform/client_spec.rb @@ -115,6 +115,9 @@ describe GoogleApi::CloudPlatform::Client do "initial_node_count": cluster_size, "node_config": { "machine_type": machine_type + }, + "legacy_abac": { + "enabled": true } } } ) diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index 3e8b9ea472a..922b4e19df0 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -277,7 +277,7 @@ describe Ci::Build do allow_any_instance_of(Project).to receive(:jobs_cache_index).and_return(1) end - it { is_expected.to be_an(Array).and all(include(key: "key_1")) } + it { is_expected.to be_an(Array).and all(include(key: "key-1")) } end context 'when project does not have jobs_cache_index' do diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index cb02d526a98..dd1bfba1421 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -496,6 +496,14 @@ describe User do user2.update_tracked_fields!(request) end.to change { user2.reload.current_sign_in_at } end + + it 'does not write if the DB is in read-only mode' do + expect(Gitlab::Database).to receive(:read_only?).and_return(true) + + expect do + user.update_tracked_fields!(request) + end.not_to change { user.reload.current_sign_in_at } + end end shared_context 'user keys' do diff --git a/spec/requests/api/internal_spec.rb b/spec/requests/api/internal_spec.rb index ea6b0a71849..71db01d2c28 100644 --- a/spec/requests/api/internal_spec.rb +++ b/spec/requests/api/internal_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' describe API::Internal do let(:user) { create(:user) } let(:key) { create(:key, user: user) } - let(:project) { create(:project, :repository) } + let(:project) { create(:project, :repository, :wiki_repo) } let(:secret_token) { Gitlab::Shell.secret_token } let(:gl_repository) { "project-#{project.id}" } let(:reference_counter) { double('ReferenceCounter') } diff --git a/spec/requests/git_http_spec.rb b/spec/requests/git_http_spec.rb index 2e2dccdafad..3deca0ac076 100644 --- a/spec/requests/git_http_spec.rb +++ b/spec/requests/git_http_spec.rb @@ -150,7 +150,7 @@ describe 'Git HTTP requests' do let(:path) { "/#{wiki.repository.full_path}.git" } context "when the project is public" do - let(:project) { create(:project, :repository, :public, :wiki_enabled) } + let(:project) { create(:project, :wiki_repo, :public, :wiki_enabled) } it_behaves_like 'pushes require Basic HTTP Authentication' @@ -177,7 +177,7 @@ describe 'Git HTTP requests' do end context 'but the repo is disabled' do - let(:project) { create(:project, :repository, :public, :repository_disabled, :wiki_enabled) } + let(:project) { create(:project, :wiki_repo, :public, :repository_disabled, :wiki_enabled) } it_behaves_like 'pulls are allowed' it_behaves_like 'pushes are allowed' @@ -198,7 +198,7 @@ describe 'Git HTTP requests' do end context "when the project is private" do - let(:project) { create(:project, :repository, :private, :wiki_enabled) } + let(:project) { create(:project, :wiki_repo, :private, :wiki_enabled) } it_behaves_like 'pulls require Basic HTTP Authentication' it_behaves_like 'pushes require Basic HTTP Authentication' @@ -210,7 +210,7 @@ describe 'Git HTTP requests' do end context 'but the repo is disabled' do - let(:project) { create(:project, :repository, :private, :repository_disabled, :wiki_enabled) } + let(:project) { create(:project, :wiki_repo, :private, :repository_disabled, :wiki_enabled) } it 'allows clones' do download(path, user: user.username, password: user.password) do |response| diff --git a/spec/support/shared_examples/controllers/uploads_actions_shared_examples.rb b/spec/support/shared_examples/controllers/uploads_actions_shared_examples.rb index 7ce80c82439..ea7dbade171 100644 --- a/spec/support/shared_examples/controllers/uploads_actions_shared_examples.rb +++ b/spec/support/shared_examples/controllers/uploads_actions_shared_examples.rb @@ -89,6 +89,19 @@ shared_examples 'handle uploads' do end end + context "when neither the uploader nor the model exists" do + before do + allow_any_instance_of(Upload).to receive(:build_uploader).and_return(nil) + allow(controller).to receive(:find_model).and_return(nil) + end + + it "responds with status 404" do + show_upload + + expect(response).to have_gitlab_http_status(404) + end + end + context "when the file doesn't exist" do before do allow_any_instance_of(FileUploader).to receive(:exists?).and_return(false) |
