From e43b2e81dab3cade773d479f2ae56478e3113207 Mon Sep 17 00:00:00 2001 From: Andre Guedes Date: Thu, 27 Oct 2016 17:39:32 -0200 Subject: Added MR Road map --- lib/container_registry/ROADMAP.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 lib/container_registry/ROADMAP.md diff --git a/lib/container_registry/ROADMAP.md b/lib/container_registry/ROADMAP.md new file mode 100644 index 00000000000..e0a20776404 --- /dev/null +++ b/lib/container_registry/ROADMAP.md @@ -0,0 +1,7 @@ +## Road map + +### Initial thoughts + +- Determine if image names will be persisted or fetched from API +- If persisted, how to update the stored names upon modification +- If fetched, how to fetch only images of a given project -- cgit v1.2.1 From dcd4beb8eb7bb7d0c2f720ef85c3da9f97a3dfe6 Mon Sep 17 00:00:00 2001 From: Andre Guedes Date: Wed, 2 Nov 2016 00:33:35 -0200 Subject: Multi-level container image names backend implementation - Adds Registry events API endpoint - Adds container_images_repository and container_images models - Changes JWT authentication to allow multi-level scopes - Adds services for container image maintenance --- app/models/container_image.rb | 58 ++++++++++++++++++++++ app/models/container_images_repository.rb | 26 ++++++++++ app/models/project.rb | 1 + .../container_registry_authentication_service.rb | 9 +++- .../container_images/create_service.rb | 16 ++++++ .../container_images/destroy_service.rb | 11 ++++ .../container_images/push_service.rb | 26 ++++++++++ .../create_service.rb | 7 +++ ...029153736_create_container_images_repository.rb | 31 ++++++++++++ .../20161031013926_create_container_image.rb | 32 ++++++++++++ lib/api/api.rb | 1 + lib/api/registry_events.rb | 52 +++++++++++++++++++ lib/container_registry/client.rb | 4 ++ lib/container_registry/tag.rb | 8 ++- 14 files changed, 275 insertions(+), 7 deletions(-) create mode 100644 app/models/container_image.rb create mode 100644 app/models/container_images_repository.rb create mode 100644 app/services/container_images_repositories/container_images/create_service.rb create mode 100644 app/services/container_images_repositories/container_images/destroy_service.rb create mode 100644 app/services/container_images_repositories/container_images/push_service.rb create mode 100644 app/services/container_images_repositories/create_service.rb create mode 100644 db/migrate/20161029153736_create_container_images_repository.rb create mode 100644 db/migrate/20161031013926_create_container_image.rb create mode 100644 lib/api/registry_events.rb diff --git a/app/models/container_image.rb b/app/models/container_image.rb new file mode 100644 index 00000000000..dcc4a7af629 --- /dev/null +++ b/app/models/container_image.rb @@ -0,0 +1,58 @@ +class ContainerImage < ActiveRecord::Base + belongs_to :container_images_repository + + delegate :registry, :registry_path_with_namespace, :client, to: :container_images_repository + + validates :manifest, presence: true + + before_validation :update_token, on: :create + def update_token + paths = container_images_repository.allowed_paths << name_with_namespace + token = Auth::ContainerRegistryAuthenticationService.full_access_token(paths) + client.update_token(token) + end + + def path + [registry.path, name_with_namespace].compact.join('/') + end + + def name_with_namespace + [registry_path_with_namespace, name].compact.join('/') + end + + def tag(tag) + ContainerRegistry::Tag.new(self, tag) + end + + def manifest + @manifest ||= client.repository_tags(name_with_namespace) + end + + def tags + return @tags if defined?(@tags) + return [] unless manifest && manifest['tags'] + + @tags = manifest['tags'].map do |tag| + ContainerRegistry::Tag.new(self, tag) + end + end + + def blob(config) + ContainerRegistry::Blob.new(self, config) + end + + def delete_tags + return unless tags + + tags.all?(&:delete) + end + + def self.split_namespace(full_path) + image_name = full_path.split('/').last + namespace = full_path.gsub(/(.*)(#{Regexp.escape('/' + image_name)})/, '\1') + if namespace.count('/') < 1 + namespace, image_name = full_path, "" + end + return namespace, image_name + end +end diff --git a/app/models/container_images_repository.rb b/app/models/container_images_repository.rb new file mode 100644 index 00000000000..99e94d2a6d0 --- /dev/null +++ b/app/models/container_images_repository.rb @@ -0,0 +1,26 @@ +class ContainerImagesRepository < ActiveRecord::Base + + belongs_to :project + + has_many :container_images, dependent: :destroy + + delegate :client, to: :registry + + def registry_path_with_namespace + project.path_with_namespace.downcase + end + + def allowed_paths + @allowed_paths ||= [registry_path_with_namespace] + + container_images.map { |i| i.name_with_namespace } + end + + def registry + @registry ||= begin + token = Auth::ContainerRegistryAuthenticationService.full_access_token(allowed_paths) + url = Gitlab.config.registry.api_url + host_port = Gitlab.config.registry.host_port + ContainerRegistry::Registry.new(url, token: token, path: host_port) + end + end +end diff --git a/app/models/project.rb b/app/models/project.rb index 411299eef63..703e24eb79a 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -157,6 +157,7 @@ class Project < ActiveRecord::Base has_one :import_data, dependent: :destroy, class_name: "ProjectImportData" has_one :project_feature, dependent: :destroy has_one :statistics, class_name: 'ProjectStatistics', dependent: :delete + has_one :container_images_repository, dependent: :destroy has_many :commit_statuses, dependent: :destroy, foreign_key: :gl_project_id has_many :pipelines, dependent: :destroy, class_name: 'Ci::Pipeline', foreign_key: :gl_project_id diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb index 5cb7a86a5ee..6b83b38fa4d 100644 --- a/app/services/auth/container_registry_authentication_service.rb +++ b/app/services/auth/container_registry_authentication_service.rb @@ -16,7 +16,7 @@ module Auth { token: authorized_token(scope).encoded } end - def self.full_access_token(*names) + def self.full_access_token(names) registry = Gitlab.config.registry token = JSONWebToken::RSAToken.new(registry.key) token.issuer = registry.issuer @@ -61,7 +61,12 @@ module Auth end def process_repository_access(type, name, actions) - requested_project = Project.find_by_full_path(name) + # Strips image name due to lack of + # per image authentication. + # Removes only last occurence in light + # of future nested groups + namespace, _ = ContainerImage::split_namespace(name) + requested_project = Project.find_by_full_path(namespace) return unless requested_project actions = actions.select do |action| diff --git a/app/services/container_images_repositories/container_images/create_service.rb b/app/services/container_images_repositories/container_images/create_service.rb new file mode 100644 index 00000000000..0c2c69d5183 --- /dev/null +++ b/app/services/container_images_repositories/container_images/create_service.rb @@ -0,0 +1,16 @@ +module ContainerImagesRepositories + module ContainerImages + class CreateService < BaseService + def execute + @container_image = container_images_repository.container_images.create(params) + @container_image if @container_image.valid? + end + + private + + def container_images_repository + @container_images_repository ||= project.container_images_repository + end + end + end +end diff --git a/app/services/container_images_repositories/container_images/destroy_service.rb b/app/services/container_images_repositories/container_images/destroy_service.rb new file mode 100644 index 00000000000..91b8cfeea47 --- /dev/null +++ b/app/services/container_images_repositories/container_images/destroy_service.rb @@ -0,0 +1,11 @@ +module ContainerImagesRepositories + module ContainerImages + class DestroyService < BaseService + def execute(container_image) + return false unless container_image + + container_image.destroy + end + end + end +end diff --git a/app/services/container_images_repositories/container_images/push_service.rb b/app/services/container_images_repositories/container_images/push_service.rb new file mode 100644 index 00000000000..2731cf1d52e --- /dev/null +++ b/app/services/container_images_repositories/container_images/push_service.rb @@ -0,0 +1,26 @@ +module ContainerImagesRepositories + module ContainerImages + class PushService < BaseService + def execute(container_image_name, event) + find_or_create_container_image(container_image_name).valid? + end + + private + + def find_or_create_container_image(container_image_name) + options = {name: container_image_name} + container_images.find_by(options) || + ::ContainerImagesRepositories::ContainerImages::CreateService.new(project, + current_user, options).execute + end + + def container_images_repository + @container_images_repository ||= project.container_images_repository + end + + def container_images + @container_images ||= container_images_repository.container_images + end + end + end +end diff --git a/app/services/container_images_repositories/create_service.rb b/app/services/container_images_repositories/create_service.rb new file mode 100644 index 00000000000..7e9dd3abe5f --- /dev/null +++ b/app/services/container_images_repositories/create_service.rb @@ -0,0 +1,7 @@ +module ContainerImagesRepositories + class CreateService < BaseService + def execute + project.container_images_repository || ::ContainerImagesRepository.create(project: project) + end + end +end diff --git a/db/migrate/20161029153736_create_container_images_repository.rb b/db/migrate/20161029153736_create_container_images_repository.rb new file mode 100644 index 00000000000..d93180b1674 --- /dev/null +++ b/db/migrate/20161029153736_create_container_images_repository.rb @@ -0,0 +1,31 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class CreateContainerImagesRepository < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + # When a migration requires downtime you **must** uncomment the following + # constant and define a short and easy to understand explanation as to why the + # migration requires downtime. + # DOWNTIME_REASON = '' + + # When using the methods "add_concurrent_index" or "add_column_with_default" + # you must disable the use of transactions as these methods can not run in an + # existing transaction. When using "add_concurrent_index" make sure that this + # method is the _only_ method called in the migration, any other changes + # should go in a separate migration. This ensures that upon failure _only_ the + # index creation fails and can be retried or reverted easily. + # + # To disable transactions uncomment the following line and remove these + # comments: + # disable_ddl_transaction! + + def change + create_table :container_images_repositories do |t| + t.integer :project_id, null: false + end + end +end diff --git a/db/migrate/20161031013926_create_container_image.rb b/db/migrate/20161031013926_create_container_image.rb new file mode 100644 index 00000000000..94feae280a6 --- /dev/null +++ b/db/migrate/20161031013926_create_container_image.rb @@ -0,0 +1,32 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class CreateContainerImage < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + # When a migration requires downtime you **must** uncomment the following + # constant and define a short and easy to understand explanation as to why the + # migration requires downtime. + # DOWNTIME_REASON = '' + + # When using the methods "add_concurrent_index" or "add_column_with_default" + # you must disable the use of transactions as these methods can not run in an + # existing transaction. When using "add_concurrent_index" make sure that this + # method is the _only_ method called in the migration, any other changes + # should go in a separate migration. This ensures that upon failure _only_ the + # index creation fails and can be retried or reverted easily. + # + # To disable transactions uncomment the following line and remove these + # comments: + # disable_ddl_transaction! + + def change + create_table :container_images do |t| + t.integer :container_images_repository_id + t.string :name + end + end +end diff --git a/lib/api/api.rb b/lib/api/api.rb index a0282ff8deb..ed775f898d2 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -84,6 +84,7 @@ module API mount ::API::Namespaces mount ::API::Notes mount ::API::NotificationSettings + mount ::API::RegistryEvents mount ::API::Pipelines mount ::API::ProjectHooks mount ::API::Projects diff --git a/lib/api/registry_events.rb b/lib/api/registry_events.rb new file mode 100644 index 00000000000..c0473051424 --- /dev/null +++ b/lib/api/registry_events.rb @@ -0,0 +1,52 @@ +module API + # RegistryEvents API + class RegistryEvents < Grape::API + # before { authenticate! } + + content_type :json, 'application/vnd.docker.distribution.events.v1+json' + + params do + requires :events, type: Array, desc: 'The ID of a project' do + requires :id, type: String, desc: 'The ID of the event' + requires :timestamp, type: String, desc: 'Timestamp of the event' + requires :action, type: String, desc: 'Action performed by event' + requires :target, type: Hash, desc: 'Target of the event' do + optional :mediaType, type: String, desc: 'Media type of the target' + optional :size, type: Integer, desc: 'Size in bytes of the target' + requires :digest, type: String, desc: 'Digest of the target' + requires :repository, type: String, desc: 'Repository of target' + optional :url, type: String, desc: 'Url of the target' + optional :tag, type: String, desc: 'Tag of the target' + end + requires :request, type: Hash, desc: 'Request of the event' do + requires :id, type: String, desc: 'The ID of the request' + optional :addr, type: String, desc: 'IP Address of the request client' + optional :host, type: String, desc: 'Hostname of the registry instance' + requires :method, type: String, desc: 'Request method' + requires :useragent, type: String, desc: 'UserAgent header of the request' + end + requires :actor, type: Hash, desc: 'Actor that initiated the event' do + optional :name, type: String, desc: 'Actor name' + end + requires :source, type: Hash, desc: 'Source of the event' do + optional :addr, type: String, desc: 'Hostname of source registry node' + optional :instanceID, type: String, desc: 'Source registry node instanceID' + end + end + end + resource :registry_events do + post do + params['events'].each do |event| + repository = event['target']['repository'] + + if event['action'] == 'push' and !!event['target']['tag'] + namespace, container_image_name = ContainerImage::split_namespace(repository) + ::ContainerImagesRepositories::ContainerImages::PushService.new( + Project::find_with_namespace(namespace), current_user + ).execute(container_image_name, event) + end + end + end + end + end +end diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index 2edddb84fc3..2cbb7bfb67d 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -15,6 +15,10 @@ module ContainerRegistry @options = options end + def update_token(token) + @options[:token] = token + end + def repository_tags(name) response_body faraday.get("/v2/#{name}/tags/list") end diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb index 59040199920..68dd87c979d 100644 --- a/lib/container_registry/tag.rb +++ b/lib/container_registry/tag.rb @@ -22,9 +22,7 @@ module ContainerRegistry end def manifest - return @manifest if defined?(@manifest) - - @manifest = client.repository_manifest(repository.name, name) + @manifest ||= client.repository_manifest(repository.name_with_namespace, name) end def path @@ -40,7 +38,7 @@ module ContainerRegistry def digest return @digest if defined?(@digest) - @digest = client.repository_tag_digest(repository.name, name) + @digest = client.repository_tag_digest(repository.name_with_namespace, name) end def config_blob @@ -82,7 +80,7 @@ module ContainerRegistry def delete return unless digest - client.delete_repository_tag(repository.name, digest) + client.delete_repository_tag(repository.name_with_namespace, digest) end end end -- cgit v1.2.1 From eed0b85ad084ad4d13cc26907102063d9372fe75 Mon Sep 17 00:00:00 2001 From: Andre Guedes Date: Wed, 23 Nov 2016 14:50:30 -0200 Subject: First iteration of container_image view - Fixes project, container_image and tag deletion - Removed container_images_repository [ci skip] --- .../stylesheets/pages/container_registry.scss | 16 ++++++++++ .../projects/container_registry_controller.rb | 28 ++++++++++++----- app/models/container_image.rb | 20 +++++++++---- app/models/container_images_repository.rb | 26 ---------------- app/models/project.rb | 20 ++++++++----- app/services/container_images/destroy_service.rb | 32 ++++++++++++++++++++ .../container_images/create_service.rb | 16 ---------- .../container_images/destroy_service.rb | 11 ------- .../container_images/push_service.rb | 26 ---------------- .../create_service.rb | 7 ----- app/services/projects/destroy_service.rb | 10 ------- .../projects/container_registry/_image.html.haml | 35 ++++++++++++++++++++++ .../projects/container_registry/_tag.html.haml | 2 +- .../projects/container_registry/index.html.haml | 20 +++---------- ...029153736_create_container_images_repository.rb | 31 ------------------- .../20161031013926_create_container_image.rb | 2 +- lib/api/registry_events.rb | 16 ++++++++-- 17 files changed, 149 insertions(+), 169 deletions(-) create mode 100644 app/assets/stylesheets/pages/container_registry.scss delete mode 100644 app/models/container_images_repository.rb create mode 100644 app/services/container_images/destroy_service.rb delete mode 100644 app/services/container_images_repositories/container_images/create_service.rb delete mode 100644 app/services/container_images_repositories/container_images/destroy_service.rb delete mode 100644 app/services/container_images_repositories/container_images/push_service.rb delete mode 100644 app/services/container_images_repositories/create_service.rb create mode 100644 app/views/projects/container_registry/_image.html.haml delete mode 100644 db/migrate/20161029153736_create_container_images_repository.rb diff --git a/app/assets/stylesheets/pages/container_registry.scss b/app/assets/stylesheets/pages/container_registry.scss new file mode 100644 index 00000000000..7d68eae3c97 --- /dev/null +++ b/app/assets/stylesheets/pages/container_registry.scss @@ -0,0 +1,16 @@ +/** + * Container Registry + */ + +.container-image { + border-bottom: 1px solid #f0f0f0; +} + +.container-image-head { + padding: 0px 16px; + line-height: 4; +} + +.table.tags { + margin-bottom: 0px; +} diff --git a/app/controllers/projects/container_registry_controller.rb b/app/controllers/projects/container_registry_controller.rb index d1f46497207..54bcb5f504a 100644 --- a/app/controllers/projects/container_registry_controller.rb +++ b/app/controllers/projects/container_registry_controller.rb @@ -5,17 +5,22 @@ class Projects::ContainerRegistryController < Projects::ApplicationController layout 'project' def index - @tags = container_registry_repository.tags + @images = project.container_images end def destroy url = namespace_project_container_registry_index_path(project.namespace, project) - if tag.delete - redirect_to url + if tag + delete_tag(url) else - redirect_to url, alert: 'Failed to remove tag' + if image.destroy + redirect_to url + else + redirect_to url, alert: 'Failed to remove image' + end end + end private @@ -24,11 +29,20 @@ class Projects::ContainerRegistryController < Projects::ApplicationController render_404 unless Gitlab.config.registry.enabled end - def container_registry_repository - @container_registry_repository ||= project.container_registry_repository + def delete_tag(url) + if tag.delete + image.destroy if image.tags.empty? + redirect_to url + else + redirect_to url, alert: 'Failed to remove tag' + end + end + + def image + @image ||= project.container_images.find_by(id: params[:id]) end def tag - @tag ||= container_registry_repository.tag(params[:id]) + @tag ||= image.tag(params[:tag]) if params[:tag].present? end end diff --git a/app/models/container_image.rb b/app/models/container_image.rb index dcc4a7af629..7721c53a6fc 100644 --- a/app/models/container_image.rb +++ b/app/models/container_image.rb @@ -1,23 +1,28 @@ class ContainerImage < ActiveRecord::Base - belongs_to :container_images_repository + belongs_to :project - delegate :registry, :registry_path_with_namespace, :client, to: :container_images_repository + delegate :container_registry, :container_registry_allowed_paths, + :container_registry_path_with_namespace, to: :project + + delegate :client, to: :container_registry validates :manifest, presence: true + before_destroy :delete_tags + before_validation :update_token, on: :create def update_token - paths = container_images_repository.allowed_paths << name_with_namespace + paths = container_registry_allowed_paths << name_with_namespace token = Auth::ContainerRegistryAuthenticationService.full_access_token(paths) client.update_token(token) end def path - [registry.path, name_with_namespace].compact.join('/') + [container_registry.path, name_with_namespace].compact.join('/') end def name_with_namespace - [registry_path_with_namespace, name].compact.join('/') + [container_registry_path_with_namespace, name].compact.join('/') end def tag(tag) @@ -44,7 +49,10 @@ class ContainerImage < ActiveRecord::Base def delete_tags return unless tags - tags.all?(&:delete) + digests = tags.map {|tag| tag.digest }.to_set + digests.all? do |digest| + client.delete_repository_tag(name_with_namespace, digest) + end end def self.split_namespace(full_path) diff --git a/app/models/container_images_repository.rb b/app/models/container_images_repository.rb deleted file mode 100644 index 99e94d2a6d0..00000000000 --- a/app/models/container_images_repository.rb +++ /dev/null @@ -1,26 +0,0 @@ -class ContainerImagesRepository < ActiveRecord::Base - - belongs_to :project - - has_many :container_images, dependent: :destroy - - delegate :client, to: :registry - - def registry_path_with_namespace - project.path_with_namespace.downcase - end - - def allowed_paths - @allowed_paths ||= [registry_path_with_namespace] + - container_images.map { |i| i.name_with_namespace } - end - - def registry - @registry ||= begin - token = Auth::ContainerRegistryAuthenticationService.full_access_token(allowed_paths) - url = Gitlab.config.registry.api_url - host_port = Gitlab.config.registry.host_port - ContainerRegistry::Registry.new(url, token: token, path: host_port) - end - end -end diff --git a/app/models/project.rb b/app/models/project.rb index 703e24eb79a..afaf2095a4c 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -157,7 +157,7 @@ class Project < ActiveRecord::Base has_one :import_data, dependent: :destroy, class_name: "ProjectImportData" has_one :project_feature, dependent: :destroy has_one :statistics, class_name: 'ProjectStatistics', dependent: :delete - has_one :container_images_repository, dependent: :destroy + has_many :container_images, dependent: :destroy has_many :commit_statuses, dependent: :destroy, foreign_key: :gl_project_id has_many :pipelines, dependent: :destroy, class_name: 'Ci::Pipeline', foreign_key: :gl_project_id @@ -405,15 +405,19 @@ class Project < ActiveRecord::Base path_with_namespace.downcase end - def container_registry_repository + def container_registry_allowed_paths + @container_registry_allowed_paths ||= [container_registry_path_with_namespace] + + container_images.map { |i| i.name_with_namespace } + end + + def container_registry return unless Gitlab.config.registry.enabled - @container_registry_repository ||= begin - token = Auth::ContainerRegistryAuthenticationService.full_access_token(container_registry_path_with_namespace) + @container_registry ||= begin + token = Auth::ContainerRegistryAuthenticationService.full_access_token(container_registry_allowed_paths) url = Gitlab.config.registry.api_url host_port = Gitlab.config.registry.host_port - registry = ContainerRegistry::Registry.new(url, token: token, path: host_port) - registry.repository(container_registry_path_with_namespace) + ContainerRegistry::Registry.new(url, token: token, path: host_port) end end @@ -424,9 +428,9 @@ class Project < ActiveRecord::Base end def has_container_registry_tags? - return unless container_registry_repository + return unless container_images - container_registry_repository.tags.any? + container_images.first.tags.any? end def commit(ref = 'HEAD') diff --git a/app/services/container_images/destroy_service.rb b/app/services/container_images/destroy_service.rb new file mode 100644 index 00000000000..bc5b53fd055 --- /dev/null +++ b/app/services/container_images/destroy_service.rb @@ -0,0 +1,32 @@ +module ContainerImages + class DestroyService < BaseService + + class DestroyError < StandardError; end + + def execute(container_image) + @container_image = container_image + + return false unless can?(current_user, :remove_project, project) + + ContainerImage.transaction do + container_image.destroy! + + unless remove_container_image_tags + raise_error('Failed to remove container image tags. Please try again or contact administrator') + end + end + + true + end + + private + + def raise_error(message) + raise DestroyError.new(message) + end + + def remove_container_image_tags + container_image.delete_tags + end + end +end diff --git a/app/services/container_images_repositories/container_images/create_service.rb b/app/services/container_images_repositories/container_images/create_service.rb deleted file mode 100644 index 0c2c69d5183..00000000000 --- a/app/services/container_images_repositories/container_images/create_service.rb +++ /dev/null @@ -1,16 +0,0 @@ -module ContainerImagesRepositories - module ContainerImages - class CreateService < BaseService - def execute - @container_image = container_images_repository.container_images.create(params) - @container_image if @container_image.valid? - end - - private - - def container_images_repository - @container_images_repository ||= project.container_images_repository - end - end - end -end diff --git a/app/services/container_images_repositories/container_images/destroy_service.rb b/app/services/container_images_repositories/container_images/destroy_service.rb deleted file mode 100644 index 91b8cfeea47..00000000000 --- a/app/services/container_images_repositories/container_images/destroy_service.rb +++ /dev/null @@ -1,11 +0,0 @@ -module ContainerImagesRepositories - module ContainerImages - class DestroyService < BaseService - def execute(container_image) - return false unless container_image - - container_image.destroy - end - end - end -end diff --git a/app/services/container_images_repositories/container_images/push_service.rb b/app/services/container_images_repositories/container_images/push_service.rb deleted file mode 100644 index 2731cf1d52e..00000000000 --- a/app/services/container_images_repositories/container_images/push_service.rb +++ /dev/null @@ -1,26 +0,0 @@ -module ContainerImagesRepositories - module ContainerImages - class PushService < BaseService - def execute(container_image_name, event) - find_or_create_container_image(container_image_name).valid? - end - - private - - def find_or_create_container_image(container_image_name) - options = {name: container_image_name} - container_images.find_by(options) || - ::ContainerImagesRepositories::ContainerImages::CreateService.new(project, - current_user, options).execute - end - - def container_images_repository - @container_images_repository ||= project.container_images_repository - end - - def container_images - @container_images ||= container_images_repository.container_images - end - end - end -end diff --git a/app/services/container_images_repositories/create_service.rb b/app/services/container_images_repositories/create_service.rb deleted file mode 100644 index 7e9dd3abe5f..00000000000 --- a/app/services/container_images_repositories/create_service.rb +++ /dev/null @@ -1,7 +0,0 @@ -module ContainerImagesRepositories - class CreateService < BaseService - def execute - project.container_images_repository || ::ContainerImagesRepository.create(project: project) - end - end -end diff --git a/app/services/projects/destroy_service.rb b/app/services/projects/destroy_service.rb index 9716a1780a9..ba410b79e8c 100644 --- a/app/services/projects/destroy_service.rb +++ b/app/services/projects/destroy_service.rb @@ -31,10 +31,6 @@ module Projects project.team.truncate project.destroy! - unless remove_registry_tags - raise_error('Failed to remove project container registry. Please try again or contact administrator') - end - unless remove_repository(repo_path) raise_error('Failed to remove project repository. Please try again or contact administrator') end @@ -68,12 +64,6 @@ module Projects end end - def remove_registry_tags - return true unless Gitlab.config.registry.enabled - - project.container_registry_repository.delete_tags - end - def raise_error(message) raise DestroyError.new(message) end diff --git a/app/views/projects/container_registry/_image.html.haml b/app/views/projects/container_registry/_image.html.haml new file mode 100644 index 00000000000..b1d62e34a97 --- /dev/null +++ b/app/views/projects/container_registry/_image.html.haml @@ -0,0 +1,35 @@ +- expanded = false +.container-image.js-toggle-container + .container-image-head + = link_to "#", class: "js-toggle-button" do + - if expanded + = icon("chevron-up") + - else + = icon("chevron-down") + + = escape_once(image.name) + = clipboard_button(clipboard_text: "docker pull #{image.path}") + .controls.hidden-xs.pull-right + = link_to namespace_project_container_registry_path(@project.namespace, @project, image.id), class: 'btn btn-remove has-tooltip', title: "Remove", data: { confirm: "Are you sure?" }, method: :delete do + = icon("trash cred") + + + .container-image-tags.js-toggle-content{ class: ("hide" unless expanded) } + - if image.tags.blank? + %li + .nothing-here-block No tags in Container Registry for this container image. + + - else + .table-holder + %table.table.tags + %thead + %tr + %th Name + %th Image ID + %th Size + %th Created + - if can?(current_user, :update_container_image, @project) + %th + + - image.tags.each do |tag| + = render 'tag', tag: tag diff --git a/app/views/projects/container_registry/_tag.html.haml b/app/views/projects/container_registry/_tag.html.haml index 10822b6184c..00345ec26de 100644 --- a/app/views/projects/container_registry/_tag.html.haml +++ b/app/views/projects/container_registry/_tag.html.haml @@ -25,5 +25,5 @@ - if can?(current_user, :update_container_image, @project) %td.content .controls.hidden-xs.pull-right - = link_to namespace_project_container_registry_path(@project.namespace, @project, tag.name), class: 'btn btn-remove has-tooltip', title: "Remove", data: { confirm: "Are you sure?" }, method: :delete do + = link_to namespace_project_container_registry_path(@project.namespace, @project, { id: tag.repository.id, tag: tag.name} ), class: 'btn btn-remove has-tooltip', title: "Remove", data: { confirm: "Are you sure?" }, method: :delete do = icon("trash cred") diff --git a/app/views/projects/container_registry/index.html.haml b/app/views/projects/container_registry/index.html.haml index 993da27310f..f074ce6be6d 100644 --- a/app/views/projects/container_registry/index.html.haml +++ b/app/views/projects/container_registry/index.html.haml @@ -19,21 +19,9 @@ %br docker push #{escape_once(@project.container_registry_repository_url)} - - if @tags.blank? - %li - .nothing-here-block No images in Container Registry for this project. + - if @images.blank? + .nothing-here-block No container images in Container Registry for this project. - else - .table-holder - %table.table.tags - %thead - %tr - %th Name - %th Image ID - %th Size - %th Created - - if can?(current_user, :update_container_image, @project) - %th - - - @tags.each do |tag| - = render 'tag', tag: tag + - @images.each do |image| + = render 'image', image: image diff --git a/db/migrate/20161029153736_create_container_images_repository.rb b/db/migrate/20161029153736_create_container_images_repository.rb deleted file mode 100644 index d93180b1674..00000000000 --- a/db/migrate/20161029153736_create_container_images_repository.rb +++ /dev/null @@ -1,31 +0,0 @@ -# See http://doc.gitlab.com/ce/development/migration_style_guide.html -# for more information on how to write migrations for GitLab. - -class CreateContainerImagesRepository < ActiveRecord::Migration - include Gitlab::Database::MigrationHelpers - - # Set this constant to true if this migration requires downtime. - DOWNTIME = false - - # When a migration requires downtime you **must** uncomment the following - # constant and define a short and easy to understand explanation as to why the - # migration requires downtime. - # DOWNTIME_REASON = '' - - # When using the methods "add_concurrent_index" or "add_column_with_default" - # you must disable the use of transactions as these methods can not run in an - # existing transaction. When using "add_concurrent_index" make sure that this - # method is the _only_ method called in the migration, any other changes - # should go in a separate migration. This ensures that upon failure _only_ the - # index creation fails and can be retried or reverted easily. - # - # To disable transactions uncomment the following line and remove these - # comments: - # disable_ddl_transaction! - - def change - create_table :container_images_repositories do |t| - t.integer :project_id, null: false - end - end -end diff --git a/db/migrate/20161031013926_create_container_image.rb b/db/migrate/20161031013926_create_container_image.rb index 94feae280a6..85c0913b8f3 100644 --- a/db/migrate/20161031013926_create_container_image.rb +++ b/db/migrate/20161031013926_create_container_image.rb @@ -25,7 +25,7 @@ class CreateContainerImage < ActiveRecord::Migration def change create_table :container_images do |t| - t.integer :container_images_repository_id + t.integer :project_id t.string :name end end diff --git a/lib/api/registry_events.rb b/lib/api/registry_events.rb index c0473051424..dc7279d2b75 100644 --- a/lib/api/registry_events.rb +++ b/lib/api/registry_events.rb @@ -41,9 +41,19 @@ module API if event['action'] == 'push' and !!event['target']['tag'] namespace, container_image_name = ContainerImage::split_namespace(repository) - ::ContainerImagesRepositories::ContainerImages::PushService.new( - Project::find_with_namespace(namespace), current_user - ).execute(container_image_name, event) + project = Project::find_with_namespace(namespace) + + if project + container_image = project.container_images.find_or_create_by(name: container_image_name) + + if container_image.valid? + puts('Valid!') + else + render_api_error!({ error: "Failed to create container image!" }, 400) + end + else + not_found!('Project') + end end end end -- cgit v1.2.1 From 246df2bd1151d39a04ef553064144eb75ee3e980 Mon Sep 17 00:00:00 2001 From: Andre Guedes Date: Tue, 13 Dec 2016 23:42:43 -0200 Subject: Adding registry endpoint authorization --- .../admin/application_settings_controller.rb | 6 +++++ .../admin/container_registry_controller.rb | 11 ++++++++ app/models/application_setting.rb | 6 +++++ app/views/admin/container_registry/show.html.haml | 31 ++++++++++++++++++++++ app/views/admin/dashboard/_head.html.haml | 4 +++ config/routes/admin.rb | 2 ++ ...egistry_access_token_to_application_settings.rb | 29 ++++++++++++++++++++ doc/administration/container_registry.md | 22 +++++++++++++-- doc/ci/docker/using_docker_build.md | 8 +++--- doc/user/project/container_registry.md | 19 +++++++------ lib/api/helpers.rb | 10 +++++++ lib/api/registry_events.rb | 2 +- 12 files changed, 133 insertions(+), 17 deletions(-) create mode 100644 app/controllers/admin/container_registry_controller.rb create mode 100644 app/views/admin/container_registry/show.html.haml create mode 100644 db/migrate/20161213212947_add_container_registry_access_token_to_application_settings.rb diff --git a/app/controllers/admin/application_settings_controller.rb b/app/controllers/admin/application_settings_controller.rb index b0f5d4a9933..fb6df1a06d2 100644 --- a/app/controllers/admin/application_settings_controller.rb +++ b/app/controllers/admin/application_settings_controller.rb @@ -29,6 +29,12 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController redirect_to :back end + def reset_container_registry_token + @application_setting.reset_container_registry_access_token! + flash[:notice] = 'New container registry access token has been generated!' + redirect_to :back + end + def clear_repository_check_states RepositoryCheck::ClearWorker.perform_async diff --git a/app/controllers/admin/container_registry_controller.rb b/app/controllers/admin/container_registry_controller.rb new file mode 100644 index 00000000000..265c032c67d --- /dev/null +++ b/app/controllers/admin/container_registry_controller.rb @@ -0,0 +1,11 @@ +class Admin::ContainerRegistryController < Admin::ApplicationController + def show + @access_token = container_registry_access_token + end + + private + + def container_registry_access_token + current_application_settings.container_registry_access_token + end +end diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 74b358d8c40..b94a71e1ea7 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -4,6 +4,7 @@ class ApplicationSetting < ActiveRecord::Base add_authentication_token_field :runners_registration_token add_authentication_token_field :health_check_access_token + add_authentication_token_field :container_registry_access_token CACHE_KEY = 'application_setting.last' DOMAIN_LIST_SEPARATOR = %r{\s*[,;]\s* # comma or semicolon, optionally surrounded by whitespace @@ -141,6 +142,7 @@ class ApplicationSetting < ActiveRecord::Base before_save :ensure_runners_registration_token before_save :ensure_health_check_access_token + before_save :ensure_container_registry_access_token after_commit do Rails.cache.write(CACHE_KEY, self) @@ -276,6 +278,10 @@ class ApplicationSetting < ActiveRecord::Base ensure_health_check_access_token! end + def container_registry_access_token + ensure_container_registry_access_token! + end + def sidekiq_throttling_enabled? return false unless sidekiq_throttling_column_exists? diff --git a/app/views/admin/container_registry/show.html.haml b/app/views/admin/container_registry/show.html.haml new file mode 100644 index 00000000000..8803eddda69 --- /dev/null +++ b/app/views/admin/container_registry/show.html.haml @@ -0,0 +1,31 @@ +- @no_container = true += render "admin/dashboard/head" + +%div{ class: container_class } + + %p.prepend-top-default + %span + To properly configure the Container Registry you should add the following + access token to the Docker Registry config.yml as follows: + %pre + %code + :plain + notifications: + endpoints: + - ... + headers: + X-Registry-Token: [#{@access_token}] + %br + Access token is + %code{ id: 'registry-token' } #{@access_token} + + .bs-callout.clearfix + .pull-left + %p + You can reset container registry access token by pressing the button below. + %p + = button_to reset_container_registry_token_admin_application_settings_path, + method: :put, class: 'btn btn-default', + data: { confirm: 'Are you sure you want to reset container registry token?' } do + = icon('refresh') + Reset container registry access token diff --git a/app/views/admin/dashboard/_head.html.haml b/app/views/admin/dashboard/_head.html.haml index 7893c1dee97..dbd039547fa 100644 --- a/app/views/admin/dashboard/_head.html.haml +++ b/app/views/admin/dashboard/_head.html.haml @@ -27,3 +27,7 @@ = link_to admin_runners_path, title: 'Runners' do %span Runners + = nav_link path: 'container_registry#show' do + = link_to admin_container_registry_path, title: 'Registry' do + %span + Registry diff --git a/config/routes/admin.rb b/config/routes/admin.rb index 8e99239f350..b09c05826a7 100644 --- a/config/routes/admin.rb +++ b/config/routes/admin.rb @@ -58,6 +58,7 @@ namespace :admin do resource :background_jobs, controller: 'background_jobs', only: [:show] resource :system_info, controller: 'system_info', only: [:show] resources :requests_profiles, only: [:index, :show], param: :name, constraints: { name: /.+\.html/ } + resource :container_registry, controller: 'container_registry', only: [:show] resources :projects, only: [:index] @@ -88,6 +89,7 @@ namespace :admin do resources :services, only: [:index, :edit, :update] put :reset_runners_token put :reset_health_check_token + put :reset_container_registry_token put :clear_repository_check_states end diff --git a/db/migrate/20161213212947_add_container_registry_access_token_to_application_settings.rb b/db/migrate/20161213212947_add_container_registry_access_token_to_application_settings.rb new file mode 100644 index 00000000000..f89f9b00a5f --- /dev/null +++ b/db/migrate/20161213212947_add_container_registry_access_token_to_application_settings.rb @@ -0,0 +1,29 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class AddContainerRegistryAccessTokenToApplicationSettings < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + # When a migration requires downtime you **must** uncomment the following + # constant and define a short and easy to understand explanation as to why the + # migration requires downtime. + # DOWNTIME_REASON = '' + + # When using the methods "add_concurrent_index" or "add_column_with_default" + # you must disable the use of transactions as these methods can not run in an + # existing transaction. When using "add_concurrent_index" make sure that this + # method is the _only_ method called in the migration, any other changes + # should go in a separate migration. This ensures that upon failure _only_ the + # index creation fails and can be retried or reverted easily. + # + # To disable transactions uncomment the following line and remove these + # comments: + # disable_ddl_transaction! + + def change + add_column :application_settings, :container_registry_access_token, :string + end +end diff --git a/doc/administration/container_registry.md b/doc/administration/container_registry.md index a6300e18dc0..14795601246 100644 --- a/doc/administration/container_registry.md +++ b/doc/administration/container_registry.md @@ -76,7 +76,7 @@ you modify its settings. Read the upstream documentation on how to achieve that. At the absolute minimum, make sure your [Registry configuration][registry-auth] has `container_registry` as the service and `https://gitlab.example.com/jwt/auth` -as the realm: +as the realm. ``` auth: @@ -87,6 +87,23 @@ auth: rootcertbundle: /root/certs/certbundle ``` +Also a notification endpoint must be configured with the token from +Admin Area -> Overview -> Registry (`/admin/container_registry`) like in the following sample: + +``` +notifications: + endpoints: + - name: listener + url: https://gitlab.example.com/api/v3/registry_events + headers: + X-Registry-Token: [57Cx95fc2zHFh93VTiGD] + timeout: 500ms + threshold: 5 + backoff: 1s +``` + +Check the [Registry endpoint configuration][registry-endpoint] for details. + ## Container Registry domain configuration There are two ways you can configure the Registry's external domain. @@ -477,7 +494,7 @@ configurable in future releases. **GitLab 8.8 ([source docs][8-8-docs])** - GitLab Container Registry feature was introduced. - +i [reconfigure gitlab]: restart_gitlab.md#omnibus-gitlab-reconfigure [restart gitlab]: restart_gitlab.md#installations-from-source [wildcard certificate]: https://en.wikipedia.org/wiki/Wildcard_certificate @@ -487,6 +504,7 @@ configurable in future releases. [storage-config]: https://docs.docker.com/registry/configuration/#storage [registry-http-config]: https://docs.docker.com/registry/configuration/#http [registry-auth]: https://docs.docker.com/registry/configuration/#auth +[registry-endpoint]: https://docs.docker.com/registry/notifications/#/configuration [token-config]: https://docs.docker.com/registry/configuration/#token [8-8-docs]: https://gitlab.com/gitlab-org/gitlab-ce/blob/8-8-stable/doc/administration/container_registry.md [registry-ssl]: https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/support/nginx/registry-ssl diff --git a/doc/ci/docker/using_docker_build.md b/doc/ci/docker/using_docker_build.md index 8620984d40d..6ae6269b28a 100644 --- a/doc/ci/docker/using_docker_build.md +++ b/doc/ci/docker/using_docker_build.md @@ -299,8 +299,8 @@ could look like: stage: build script: - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN registry.example.com - - docker build -t registry.example.com/group/project:latest . - - docker push registry.example.com/group/project:latest + - docker build -t registry.example.com/group/project/image:latest . + - docker push registry.example.com/group/project/image:latest ``` You have to use the special `gitlab-ci-token` user created for you in order to @@ -350,8 +350,8 @@ stages: - deploy variables: - CONTAINER_TEST_IMAGE: registry.example.com/my-group/my-project:$CI_BUILD_REF_NAME - CONTAINER_RELEASE_IMAGE: registry.example.com/my-group/my-project:latest + CONTAINER_TEST_IMAGE: registry.example.com/my-group/my-project/my-image:$CI_BUILD_REF_NAME + CONTAINER_RELEASE_IMAGE: registry.example.com/my-group/my-project/my-image:latest before_script: - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN registry.example.com diff --git a/doc/user/project/container_registry.md b/doc/user/project/container_registry.md index 91b35c73b34..eada8e04227 100644 --- a/doc/user/project/container_registry.md +++ b/doc/user/project/container_registry.md @@ -10,6 +10,7 @@ - Starting from GitLab 8.12, if you have 2FA enabled in your account, you need to pass a personal access token instead of your password in order to login to GitLab's Container Registry. +- Multiple level image names support was added in GitLab ?8.15? With the Docker Container Registry integrated into GitLab, every project can have its own space to store its Docker images. @@ -54,26 +55,23 @@ sure that you are using the Registry URL with the namespace and project name that is hosted on GitLab: ``` -docker build -t registry.example.com/group/project . -docker push registry.example.com/group/project +docker build -t registry.example.com/group/project/image . +docker push registry.example.com/group/project/image ``` Your image will be named after the following scheme: ``` -// +/// ``` -As such, the name of the image is unique, but you can differentiate the images -using tags. - ## Use images from GitLab Container Registry To download and run a container from images hosted in GitLab Container Registry, use `docker run`: ``` -docker run [options] registry.example.com/group/project [arguments] +docker run [options] registry.example.com/group/project/image [arguments] ``` For more information on running Docker containers, visit the @@ -87,7 +85,8 @@ and click **Registry** in the project menu. This view will show you all tags in your project and will easily allow you to delete them. -![Container Registry panel](img/container_registry_panel.png) +![Container Registry panel](image-needs-update) +[//]: # (img/container_registry_panel.png) ## Build and push images using GitLab CI @@ -136,7 +135,7 @@ A user attempted to enable an S3-backed Registry. The `docker login` step went fine. However, when pushing an image, the output showed: ``` -The push refers to a repository [s3-testing.myregistry.com:4567/root/docker-test] +The push refers to a repository [s3-testing.myregistry.com:4567/root/docker-test/docker-image] dc5e59c14160: Pushing [==================================================>] 14.85 kB 03c20c1a019a: Pushing [==================================================>] 2.048 kB a08f14ef632e: Pushing [==================================================>] 2.048 kB @@ -229,7 +228,7 @@ a container image. You may need to run as root to do this. For example: ```sh docker login s3-testing.myregistry.com:4567 -docker push s3-testing.myregistry.com:4567/root/docker-test +docker push s3-testing.myregistry.com:4567/root/docker-test/docker-image ``` In the example above, we see the following trace on the mitmproxy window: diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index a1db2099693..0fd2b1587e3 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -111,6 +111,16 @@ module API end end + def authenticate_container_registry_access_token! + token = request.headers['X-Registry-Token'] + unless token.present? && ActiveSupport::SecurityUtils.variable_size_secure_compare( + token, + current_application_settings.container_registry_access_token + ) + unauthorized! + end + end + def authenticated_as_admin! authenticate! forbidden! unless current_user.is_admin? diff --git a/lib/api/registry_events.rb b/lib/api/registry_events.rb index dc7279d2b75..e52433339eb 100644 --- a/lib/api/registry_events.rb +++ b/lib/api/registry_events.rb @@ -1,7 +1,7 @@ module API # RegistryEvents API class RegistryEvents < Grape::API - # before { authenticate! } + before { authenticate_container_registry_access_token! } content_type :json, 'application/vnd.docker.distribution.events.v1+json' -- cgit v1.2.1 From e4fa80f3b67f1ef30c262cd4df28516ccff6336a Mon Sep 17 00:00:00 2001 From: Andre Guedes Date: Fri, 16 Dec 2016 01:24:05 -0200 Subject: Fixes broken and missing tests --- .../stylesheets/pages/container_registry.scss | 6 +- .../projects/container_registry_controller.rb | 1 - app/models/container_image.rb | 4 +- app/models/namespace.rb | 8 +-- app/models/project.rb | 18 ++---- .../container_registry_authentication_service.rb | 3 +- app/services/container_images/destroy_service.rb | 1 - app/services/projects/transfer_service.rb | 6 +- .../projects/container_registry/_image.html.haml | 2 +- .../projects/container_registry/_tag.html.haml | 2 +- .../projects/container_registry/index.html.haml | 4 +- db/schema.rb | 6 ++ lib/api/registry_events.rb | 4 +- lib/container_registry/blob.rb | 4 +- lib/container_registry/registry.rb | 4 -- lib/container_registry/repository.rb | 48 -------------- spec/factories/container_images.rb | 21 +++++++ spec/features/container_registry_spec.rb | 32 +++++++--- .../security/project/internal_access_spec.rb | 3 + .../security/project/private_access_spec.rb | 3 + .../security/project/public_access_spec.rb | 3 + spec/lib/container_registry/blob_spec.rb | 15 +++-- spec/lib/container_registry/registry_spec.rb | 2 +- spec/lib/container_registry/repository_spec.rb | 65 ------------------- spec/lib/container_registry/tag_spec.rb | 11 +++- spec/lib/gitlab/import_export/all_models.yml | 3 + spec/models/ci/build_spec.rb | 2 +- spec/models/container_image_spec.rb | 73 ++++++++++++++++++++++ spec/models/namespace_spec.rb | 8 ++- spec/models/project_spec.rb | 41 +++--------- spec/services/projects/destroy_service_spec.rb | 15 +++-- spec/services/projects/transfer_service_spec.rb | 3 + 32 files changed, 211 insertions(+), 210 deletions(-) delete mode 100644 lib/container_registry/repository.rb create mode 100644 spec/factories/container_images.rb delete mode 100644 spec/lib/container_registry/repository_spec.rb create mode 100644 spec/models/container_image_spec.rb diff --git a/app/assets/stylesheets/pages/container_registry.scss b/app/assets/stylesheets/pages/container_registry.scss index 7d68eae3c97..92543d7d714 100644 --- a/app/assets/stylesheets/pages/container_registry.scss +++ b/app/assets/stylesheets/pages/container_registry.scss @@ -3,14 +3,14 @@ */ .container-image { - border-bottom: 1px solid #f0f0f0; + border-bottom: 1px solid $white-normal; } .container-image-head { - padding: 0px 16px; + padding: 0 16px; line-height: 4; } .table.tags { - margin-bottom: 0px; + margin-bottom: 0; } diff --git a/app/controllers/projects/container_registry_controller.rb b/app/controllers/projects/container_registry_controller.rb index 54bcb5f504a..f656f86fcdb 100644 --- a/app/controllers/projects/container_registry_controller.rb +++ b/app/controllers/projects/container_registry_controller.rb @@ -20,7 +20,6 @@ class Projects::ContainerRegistryController < Projects::ApplicationController redirect_to url, alert: 'Failed to remove image' end end - end private diff --git a/app/models/container_image.rb b/app/models/container_image.rb index 7721c53a6fc..583cb977910 100644 --- a/app/models/container_image.rb +++ b/app/models/container_image.rb @@ -22,7 +22,7 @@ class ContainerImage < ActiveRecord::Base end def name_with_namespace - [container_registry_path_with_namespace, name].compact.join('/') + [container_registry_path_with_namespace, name].reject(&:blank?).join('/') end def tag(tag) @@ -55,6 +55,8 @@ class ContainerImage < ActiveRecord::Base end end + # rubocop:disable RedundantReturn + def self.split_namespace(full_path) image_name = full_path.split('/').last namespace = full_path.gsub(/(.*)(#{Regexp.escape('/' + image_name)})/, '\1') diff --git a/app/models/namespace.rb b/app/models/namespace.rb index bd0336c984a..c8e329044e0 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -118,8 +118,8 @@ class Namespace < ActiveRecord::Base end def move_dir - if any_project_has_container_registry_tags? - raise Gitlab::UpdatePathError.new('Namespace cannot be moved, because at least one project has tags in container registry') + if any_project_has_container_registry_images? + raise Gitlab::UpdatePathError.new('Namespace cannot be moved, because at least one project has images in container registry') end # Move the namespace directory in all storages paths used by member projects @@ -154,8 +154,8 @@ class Namespace < ActiveRecord::Base end end - def any_project_has_container_registry_tags? - projects.any?(&:has_container_registry_tags?) + def any_project_has_container_registry_images? + projects.any? { |project| project.container_images.present? } end def send_update_instructions diff --git a/app/models/project.rb b/app/models/project.rb index afaf2095a4c..d4f5584f53d 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -421,18 +421,12 @@ class Project < ActiveRecord::Base end end - def container_registry_repository_url + def container_registry_url if Gitlab.config.registry.enabled "#{Gitlab.config.registry.host_port}/#{container_registry_path_with_namespace}" end end - def has_container_registry_tags? - return unless container_images - - container_images.first.tags.any? - end - def commit(ref = 'HEAD') repository.commit(ref) end @@ -913,11 +907,11 @@ class Project < ActiveRecord::Base expire_caches_before_rename(old_path_with_namespace) - if has_container_registry_tags? - Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry tags are present" + if container_images.present? + Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry images are present" - # we currently doesn't support renaming repository if it contains tags in container registry - raise StandardError.new('Project cannot be renamed, because tags are present in its container registry') + # we currently doesn't support renaming repository if it contains images in container registry + raise StandardError.new('Project cannot be renamed, because images are present in its container registry') end if gitlab_shell.mv_repository(repository_storage_path, old_path_with_namespace, new_path_with_namespace) @@ -1264,7 +1258,7 @@ class Project < ActiveRecord::Base ] if container_registry_enabled? - variables << { key: 'CI_REGISTRY_IMAGE', value: container_registry_repository_url, public: true } + variables << { key: 'CI_REGISTRY_IMAGE', value: container_registry_url, public: true } end variables diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb index 6b83b38fa4d..5b2fcdf3b16 100644 --- a/app/services/auth/container_registry_authentication_service.rb +++ b/app/services/auth/container_registry_authentication_service.rb @@ -16,7 +16,8 @@ module Auth { token: authorized_token(scope).encoded } end - def self.full_access_token(names) + def self.full_access_token(*names) + names = names.flatten registry = Gitlab.config.registry token = JSONWebToken::RSAToken.new(registry.key) token.issuer = registry.issuer diff --git a/app/services/container_images/destroy_service.rb b/app/services/container_images/destroy_service.rb index bc5b53fd055..c73b6cfefba 100644 --- a/app/services/container_images/destroy_service.rb +++ b/app/services/container_images/destroy_service.rb @@ -1,6 +1,5 @@ module ContainerImages class DestroyService < BaseService - class DestroyError < StandardError; end def execute(container_image) diff --git a/app/services/projects/transfer_service.rb b/app/services/projects/transfer_service.rb index 20dfbddc823..3e241b9e7c0 100644 --- a/app/services/projects/transfer_service.rb +++ b/app/services/projects/transfer_service.rb @@ -36,9 +36,9 @@ module Projects raise TransferError.new("Project with same path in target namespace already exists") end - if project.has_container_registry_tags? - # we currently doesn't support renaming repository if it contains tags in container registry - raise TransferError.new('Project cannot be transferred, because tags are present in its container registry') + unless project.container_images.empty? + # we currently doesn't support renaming repository if it contains images in container registry + raise TransferError.new('Project cannot be transferred, because images are present in its container registry') end project.expire_caches_before_rename(old_path) diff --git a/app/views/projects/container_registry/_image.html.haml b/app/views/projects/container_registry/_image.html.haml index b1d62e34a97..5845efd345a 100644 --- a/app/views/projects/container_registry/_image.html.haml +++ b/app/views/projects/container_registry/_image.html.haml @@ -10,7 +10,7 @@ = escape_once(image.name) = clipboard_button(clipboard_text: "docker pull #{image.path}") .controls.hidden-xs.pull-right - = link_to namespace_project_container_registry_path(@project.namespace, @project, image.id), class: 'btn btn-remove has-tooltip', title: "Remove", data: { confirm: "Are you sure?" }, method: :delete do + = link_to namespace_project_container_registry_path(@project.namespace, @project, image.id), class: 'btn btn-remove has-tooltip', title: "Remove image", data: { confirm: "Are you sure?" }, method: :delete do = icon("trash cred") diff --git a/app/views/projects/container_registry/_tag.html.haml b/app/views/projects/container_registry/_tag.html.haml index 00345ec26de..b35a9cb621f 100644 --- a/app/views/projects/container_registry/_tag.html.haml +++ b/app/views/projects/container_registry/_tag.html.haml @@ -25,5 +25,5 @@ - if can?(current_user, :update_container_image, @project) %td.content .controls.hidden-xs.pull-right - = link_to namespace_project_container_registry_path(@project.namespace, @project, { id: tag.repository.id, tag: tag.name} ), class: 'btn btn-remove has-tooltip', title: "Remove", data: { confirm: "Are you sure?" }, method: :delete do + = link_to namespace_project_container_registry_path(@project.namespace, @project, { id: tag.repository.id, tag: tag.name} ), class: 'btn btn-remove has-tooltip', title: "Remove tag", data: { confirm: "Are you sure?" }, method: :delete do = icon("trash cred") diff --git a/app/views/projects/container_registry/index.html.haml b/app/views/projects/container_registry/index.html.haml index f074ce6be6d..ab6213f03d8 100644 --- a/app/views/projects/container_registry/index.html.haml +++ b/app/views/projects/container_registry/index.html.haml @@ -15,9 +15,9 @@ %br Then you are free to create and upload a container image with build and push commands: %pre - docker build -t #{escape_once(@project.container_registry_repository_url)} . + docker build -t #{escape_once(@project.container_registry_url)} . %br - docker push #{escape_once(@project.container_registry_repository_url)} + docker push #{escape_once(@project.container_registry_url)} - if @images.blank? .nothing-here-block No container images in Container Registry for this project. diff --git a/db/schema.rb b/db/schema.rb index 88aaa6c3c55..36df20fc8f2 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -109,6 +109,7 @@ ActiveRecord::Schema.define(version: 20170215200045) do t.boolean "html_emails_enabled", default: true t.string "plantuml_url" t.boolean "plantuml_enabled" + t.string "container_registry_access_token" t.integer "max_pages_size", default: 100, null: false t.integer "terminal_max_session_time", default: 0, null: false end @@ -392,6 +393,11 @@ ActiveRecord::Schema.define(version: 20170215200045) do add_index "ci_variables", ["gl_project_id"], name: "index_ci_variables_on_gl_project_id", using: :btree + create_table "container_images", force: :cascade do |t| + t.integer "project_id" + t.string "name" + end + create_table "deploy_keys_projects", force: :cascade do |t| t.integer "deploy_key_id", null: false t.integer "project_id", null: false diff --git a/lib/api/registry_events.rb b/lib/api/registry_events.rb index e52433339eb..12305a49f0f 100644 --- a/lib/api/registry_events.rb +++ b/lib/api/registry_events.rb @@ -46,9 +46,7 @@ module API if project container_image = project.container_images.find_or_create_by(name: container_image_name) - if container_image.valid? - puts('Valid!') - else + unless container_image.valid? render_api_error!({ error: "Failed to create container image!" }, 400) end else diff --git a/lib/container_registry/blob.rb b/lib/container_registry/blob.rb index eb5a2596177..8db8e483b1d 100644 --- a/lib/container_registry/blob.rb +++ b/lib/container_registry/blob.rb @@ -38,11 +38,11 @@ module ContainerRegistry end def delete - client.delete_blob(repository.name, digest) + client.delete_blob(repository.name_with_namespace, digest) end def data - @data ||= client.blob(repository.name, digest, type) + @data ||= client.blob(repository.name_with_namespace, digest, type) end end end diff --git a/lib/container_registry/registry.rb b/lib/container_registry/registry.rb index 0e634f6b6ef..63bce655f57 100644 --- a/lib/container_registry/registry.rb +++ b/lib/container_registry/registry.rb @@ -8,10 +8,6 @@ module ContainerRegistry @client = ContainerRegistry::Client.new(uri, options) end - def repository(name) - ContainerRegistry::Repository.new(self, name) - end - private def default_path diff --git a/lib/container_registry/repository.rb b/lib/container_registry/repository.rb deleted file mode 100644 index 0e4a7cb3cc9..00000000000 --- a/lib/container_registry/repository.rb +++ /dev/null @@ -1,48 +0,0 @@ -module ContainerRegistry - class Repository - attr_reader :registry, :name - - delegate :client, to: :registry - - def initialize(registry, name) - @registry, @name = registry, name - end - - def path - [registry.path, name].compact.join('/') - end - - def tag(tag) - ContainerRegistry::Tag.new(self, tag) - end - - def manifest - return @manifest if defined?(@manifest) - - @manifest = client.repository_tags(name) - end - - def valid? - manifest.present? - end - - def tags - return @tags if defined?(@tags) - return [] unless manifest && manifest['tags'] - - @tags = manifest['tags'].map do |tag| - ContainerRegistry::Tag.new(self, tag) - end - end - - def blob(config) - ContainerRegistry::Blob.new(self, config) - end - - def delete_tags - return unless tags - - tags.all?(&:delete) - end - end -end diff --git a/spec/factories/container_images.rb b/spec/factories/container_images.rb new file mode 100644 index 00000000000..6141a519a75 --- /dev/null +++ b/spec/factories/container_images.rb @@ -0,0 +1,21 @@ +FactoryGirl.define do + factory :container_image do + name "test_container_image" + project + + transient do + tags ['tag'] + stubbed true + end + + after(:build) do |image, evaluator| + if evaluator.stubbed + allow(Gitlab.config.registry).to receive(:enabled).and_return(true) + allow(image.client).to receive(:repository_tags).and_return({ + name: image.name_with_namespace, + tags: evaluator.tags + }) + end + end + end +end diff --git a/spec/features/container_registry_spec.rb b/spec/features/container_registry_spec.rb index 203e55a36f2..862c9fbf6c0 100644 --- a/spec/features/container_registry_spec.rb +++ b/spec/features/container_registry_spec.rb @@ -2,15 +2,18 @@ require 'spec_helper' describe "Container Registry" do let(:project) { create(:empty_project) } - let(:repository) { project.container_registry_repository } + let(:registry) { project.container_registry } let(:tag_name) { 'latest' } let(:tags) { [tag_name] } + let(:container_image) { create(:container_image) } + let(:image_name) { container_image.name } before do login_as(:user) project.team << [@user, :developer] - stub_container_registry_tags(*tags) stub_container_registry_config(enabled: true) + stub_container_registry_tags(*tags) + project.container_images << container_image unless container_image.nil? allow(Auth::ContainerRegistryAuthenticationService).to receive(:full_access_token).and_return('token') end @@ -19,15 +22,26 @@ describe "Container Registry" do visit namespace_project_container_registry_index_path(project.namespace, project) end - context 'when no tags' do - let(:tags) { [] } + context 'when no images' do + let(:container_image) { } + + it { expect(page).to have_content('No container images in Container Registry for this project') } + end - it { expect(page).to have_content('No images in Container Registry for this project') } + context 'when there are images' do + it { expect(page).to have_content(image_name) } end + end + + describe 'DELETE /:project/container_registry/:image_id' do + before do + visit namespace_project_container_registry_index_path(project.namespace, project) + end + + it do + expect_any_instance_of(ContainerImage).to receive(:delete_tags).and_return(true) - context 'when there are tags' do - it { expect(page).to have_content(tag_name) } - it { expect(page).to have_content('d7a513a66') } + click_on 'Remove image' end end @@ -39,7 +53,7 @@ describe "Container Registry" do it do expect_any_instance_of(::ContainerRegistry::Tag).to receive(:delete).and_return(true) - click_on 'Remove' + click_on 'Remove tag' end end end diff --git a/spec/features/security/project/internal_access_spec.rb b/spec/features/security/project/internal_access_spec.rb index 24af062d763..4e7a2c0ecc0 100644 --- a/spec/features/security/project/internal_access_spec.rb +++ b/spec/features/security/project/internal_access_spec.rb @@ -429,9 +429,12 @@ describe "Internal Project Access", feature: true do end describe "GET /:project_path/container_registry" do + let(:container_image) { create(:container_image) } + before do stub_container_registry_tags('latest') stub_container_registry_config(enabled: true) + project.container_images << container_image end subject { namespace_project_container_registry_index_path(project.namespace, project) } diff --git a/spec/features/security/project/private_access_spec.rb b/spec/features/security/project/private_access_spec.rb index c511dcfa18e..c74cdc05593 100644 --- a/spec/features/security/project/private_access_spec.rb +++ b/spec/features/security/project/private_access_spec.rb @@ -418,9 +418,12 @@ describe "Private Project Access", feature: true do end describe "GET /:project_path/container_registry" do + let(:container_image) { create(:container_image) } + before do stub_container_registry_tags('latest') stub_container_registry_config(enabled: true) + project.container_images << container_image end subject { namespace_project_container_registry_index_path(project.namespace, project) } diff --git a/spec/features/security/project/public_access_spec.rb b/spec/features/security/project/public_access_spec.rb index d8cc012c27e..485ef335b78 100644 --- a/spec/features/security/project/public_access_spec.rb +++ b/spec/features/security/project/public_access_spec.rb @@ -429,9 +429,12 @@ describe "Public Project Access", feature: true do end describe "GET /:project_path/container_registry" do + let(:container_image) { create(:container_image) } + before do stub_container_registry_tags('latest') stub_container_registry_config(enabled: true) + project.container_images << container_image end subject { namespace_project_container_registry_index_path(project.namespace, project) } diff --git a/spec/lib/container_registry/blob_spec.rb b/spec/lib/container_registry/blob_spec.rb index bbacdc67ebd..f092449c4bd 100644 --- a/spec/lib/container_registry/blob_spec.rb +++ b/spec/lib/container_registry/blob_spec.rb @@ -9,12 +9,19 @@ describe ContainerRegistry::Blob do 'size' => 1000 } end - let(:token) { 'authorization-token' } - - let(:registry) { ContainerRegistry::Registry.new('http://example.com', token: token) } - let(:repository) { registry.repository('group/test') } + let(:token) { 'token' } + + let(:group) { create(:group, name: 'group') } + let(:project) { create(:project, path: 'test', group: group) } + let(:example_host) { 'example.com' } + let(:registry_url) { 'http://' + example_host } + let(:repository) { create(:container_image, name: '', project: project) } let(:blob) { repository.blob(config) } + before do + stub_container_registry_config(enabled: true, api_url: registry_url, host_port: example_host) + end + it { expect(blob).to respond_to(:repository) } it { expect(blob).to delegate_method(:registry).to(:repository) } it { expect(blob).to delegate_method(:client).to(:repository) } diff --git a/spec/lib/container_registry/registry_spec.rb b/spec/lib/container_registry/registry_spec.rb index 4f3f8b24fc4..4d6eea94bf0 100644 --- a/spec/lib/container_registry/registry_spec.rb +++ b/spec/lib/container_registry/registry_spec.rb @@ -10,7 +10,7 @@ describe ContainerRegistry::Registry do it { is_expected.to respond_to(:uri) } it { is_expected.to respond_to(:path) } - it { expect(subject.repository('test')).not_to be_nil } + it { expect(subject).not_to be_nil } context '#path' do subject { registry.path } diff --git a/spec/lib/container_registry/repository_spec.rb b/spec/lib/container_registry/repository_spec.rb deleted file mode 100644 index c364e759108..00000000000 --- a/spec/lib/container_registry/repository_spec.rb +++ /dev/null @@ -1,65 +0,0 @@ -require 'spec_helper' - -describe ContainerRegistry::Repository do - let(:registry) { ContainerRegistry::Registry.new('http://example.com') } - let(:repository) { registry.repository('group/test') } - - it { expect(repository).to respond_to(:registry) } - it { expect(repository).to delegate_method(:client).to(:registry) } - it { expect(repository.tag('test')).not_to be_nil } - - context '#path' do - subject { repository.path } - - it { is_expected.to eq('example.com/group/test') } - end - - context 'manifest processing' do - before do - stub_request(:get, 'http://example.com/v2/group/test/tags/list'). - with(headers: { 'Accept' => 'application/vnd.docker.distribution.manifest.v2+json' }). - to_return( - status: 200, - body: JSON.dump(tags: ['test']), - headers: { 'Content-Type' => 'application/json' }) - end - - context '#manifest' do - subject { repository.manifest } - - it { is_expected.not_to be_nil } - end - - context '#valid?' do - subject { repository.valid? } - - it { is_expected.to be_truthy } - end - - context '#tags' do - subject { repository.tags } - - it { is_expected.not_to be_empty } - end - end - - context '#delete_tags' do - let(:tag) { ContainerRegistry::Tag.new(repository, 'tag') } - - before { expect(repository).to receive(:tags).twice.and_return([tag]) } - - subject { repository.delete_tags } - - context 'succeeds' do - before { expect(tag).to receive(:delete).and_return(true) } - - it { is_expected.to be_truthy } - end - - context 'any fails' do - before { expect(tag).to receive(:delete).and_return(false) } - - it { is_expected.to be_falsey } - end - end -end diff --git a/spec/lib/container_registry/tag_spec.rb b/spec/lib/container_registry/tag_spec.rb index c5e31ae82b6..cdd0fe66bc3 100644 --- a/spec/lib/container_registry/tag_spec.rb +++ b/spec/lib/container_registry/tag_spec.rb @@ -1,11 +1,18 @@ require 'spec_helper' describe ContainerRegistry::Tag do - let(:registry) { ContainerRegistry::Registry.new('http://example.com') } - let(:repository) { registry.repository('group/test') } + let(:group) { create(:group, name: 'group') } + let(:project) { create(:project, path: 'test', group: group) } + let(:example_host) { 'example.com' } + let(:registry_url) { 'http://' + example_host } + let(:repository) { create(:container_image, name: '', project: project) } let(:tag) { repository.tag('tag') } let(:headers) { { 'Accept' => 'application/vnd.docker.distribution.manifest.v2+json' } } + before do + stub_container_registry_config(enabled: true, api_url: registry_url, host_port: example_host) + end + it { expect(tag).to respond_to(:repository) } it { expect(tag).to delegate_method(:registry).to(:repository) } it { expect(tag).to delegate_method(:client).to(:repository) } diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index 06617f3b007..9c08f41fe82 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -114,6 +114,8 @@ merge_access_levels: - protected_branch push_access_levels: - protected_branch +container_images: +- name project: - taggings - base_tags @@ -197,6 +199,7 @@ project: - project_authorizations - route - statistics +- container_images award_emoji: - awardable - user diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index 2dfca8bcfce..83a2efb55b9 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -1397,7 +1397,7 @@ describe Ci::Build, :models do { key: 'CI_REGISTRY', value: 'registry.example.com', public: true } end let(:ci_registry_image) do - { key: 'CI_REGISTRY_IMAGE', value: project.container_registry_repository_url, public: true } + { key: 'CI_REGISTRY_IMAGE', value: project.container_registry_url, public: true } end context 'and is disabled for project' do diff --git a/spec/models/container_image_spec.rb b/spec/models/container_image_spec.rb new file mode 100644 index 00000000000..e0bea737f59 --- /dev/null +++ b/spec/models/container_image_spec.rb @@ -0,0 +1,73 @@ +require 'spec_helper' + +describe ContainerImage do + let(:group) { create(:group, name: 'group') } + let(:project) { create(:project, path: 'test', group: group) } + let(:example_host) { 'example.com' } + let(:registry_url) { 'http://' + example_host } + let(:container_image) { create(:container_image, name: '', project: project, stubbed: false) } + + before do + stub_container_registry_config(enabled: true, api_url: registry_url, host_port: example_host) + stub_request(:get, 'http://example.com/v2/group/test/tags/list'). + with(headers: { 'Accept' => 'application/vnd.docker.distribution.manifest.v2+json' }). + to_return( + status: 200, + body: JSON.dump(tags: ['test']), + headers: { 'Content-Type' => 'application/json' }) + end + + it { expect(container_image).to respond_to(:project) } + it { expect(container_image).to delegate_method(:container_registry).to(:project) } + it { expect(container_image).to delegate_method(:client).to(:container_registry) } + it { expect(container_image.tag('test')).not_to be_nil } + + context '#path' do + subject { container_image.path } + + it { is_expected.to eq('example.com/group/test') } + end + + context 'manifest processing' do + context '#manifest' do + subject { container_image.manifest } + + it { is_expected.not_to be_nil } + end + + context '#valid?' do + subject { container_image.valid? } + + it { is_expected.to be_truthy } + end + + context '#tags' do + subject { container_image.tags } + + it { is_expected.not_to be_empty } + end + end + + context '#delete_tags' do + let(:tag) { ContainerRegistry::Tag.new(container_image, 'tag') } + + before do + expect(container_image).to receive(:tags).twice.and_return([tag]) + expect(tag).to receive(:digest).and_return('sha256:4c8e63ca4cb663ce6c688cb06f1c3672a172b088dac5b6d7ad7d49cd620d85cf') + end + + subject { container_image.delete_tags } + + context 'succeeds' do + before { expect(container_image.client).to receive(:delete_repository_tag).and_return(true) } + + it { is_expected.to be_truthy } + end + + context 'any fails' do + before { expect(container_image.client).to receive(:delete_repository_tag).and_return(false) } + + it { is_expected.to be_falsey } + end + end +end diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb index 35d932f1c64..aeb4eeb0b55 100644 --- a/spec/models/namespace_spec.rb +++ b/spec/models/namespace_spec.rb @@ -134,18 +134,20 @@ describe Namespace, models: true do expect(@namespace.move_dir).to be_truthy end - context "when any project has container tags" do + context "when any project has container images" do + let(:container_image) { create(:container_image) } + before do stub_container_registry_config(enabled: true) stub_container_registry_tags('tag') - create(:empty_project, namespace: @namespace) + create(:empty_project, namespace: @namespace, container_images: [container_image]) allow(@namespace).to receive(:path_was).and_return(@namespace.path) allow(@namespace).to receive(:path).and_return('new_path') end - it { expect { @namespace.move_dir }.to raise_error('Namespace cannot be moved, because at least one project has tags in container registry') } + it { expect { @namespace.move_dir }.to raise_error('Namespace cannot be moved, because at least one project has images in container registry') } end end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index b0087a9e15d..77f2ff3d17b 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -1173,10 +1173,13 @@ describe Project, models: true do project.rename_repo end - context 'container registry with tags' do + context 'container registry with images' do + let(:container_image) { create(:container_image) } + before do stub_container_registry_config(enabled: true) stub_container_registry_tags('tag') + project.container_images << container_image end subject { project.rename_repo } @@ -1383,20 +1386,20 @@ describe Project, models: true do it { is_expected.to eq(project.path_with_namespace.downcase) } end - describe '#container_registry_repository' do + describe '#container_registry' do let(:project) { create(:empty_project) } before { stub_container_registry_config(enabled: true) } - subject { project.container_registry_repository } + subject { project.container_registry } it { is_expected.not_to be_nil } end - describe '#container_registry_repository_url' do + describe '#container_registry_url' do let(:project) { create(:empty_project) } - subject { project.container_registry_repository_url } + subject { project.container_registry_url } before { stub_container_registry_config(**registry_settings) } @@ -1422,34 +1425,6 @@ describe Project, models: true do end end - describe '#has_container_registry_tags?' do - let(:project) { create(:empty_project) } - - subject { project.has_container_registry_tags? } - - context 'for enabled registry' do - before { stub_container_registry_config(enabled: true) } - - context 'with tags' do - before { stub_container_registry_tags('test', 'test2') } - - it { is_expected.to be_truthy } - end - - context 'when no tags' do - before { stub_container_registry_tags } - - it { is_expected.to be_falsey } - end - end - - context 'for disabled registry' do - before { stub_container_registry_config(enabled: false) } - - it { is_expected.to be_falsey } - end - end - describe '#latest_successful_builds_for' do def create_pipeline(status = 'success') create(:ci_pipeline, project: project, diff --git a/spec/services/projects/destroy_service_spec.rb b/spec/services/projects/destroy_service_spec.rb index 74bfba44dfd..270e630e70e 100644 --- a/spec/services/projects/destroy_service_spec.rb +++ b/spec/services/projects/destroy_service_spec.rb @@ -90,25 +90,30 @@ describe Projects::DestroyService, services: true do end context 'container registry' do + let(:container_image) { create(:container_image) } + before do stub_container_registry_config(enabled: true) stub_container_registry_tags('tag') + project.container_images << container_image end - context 'tags deletion succeeds' do + context 'images deletion succeeds' do it do - expect_any_instance_of(ContainerRegistry::Tag).to receive(:delete).and_return(true) + expect_any_instance_of(ContainerImage).to receive(:delete_tags).and_return(true) destroy_project(project, user, {}) end end - context 'tags deletion fails' do - before { expect_any_instance_of(ContainerRegistry::Tag).to receive(:delete).and_return(false) } + context 'images deletion fails' do + before do + expect_any_instance_of(ContainerImage).to receive(:delete_tags).and_return(false) + end subject { destroy_project(project, user, {}) } - it { expect{subject}.to raise_error(Projects::DestroyService::DestroyError) } + it { expect{subject}.to raise_error(ActiveRecord::RecordNotDestroyed) } end end diff --git a/spec/services/projects/transfer_service_spec.rb b/spec/services/projects/transfer_service_spec.rb index 5c6fbea8d0e..5e56226ff91 100644 --- a/spec/services/projects/transfer_service_spec.rb +++ b/spec/services/projects/transfer_service_spec.rb @@ -29,9 +29,12 @@ describe Projects::TransferService, services: true do end context 'disallow transfering of project with tags' do + let(:container_image) { create(:container_image) } + before do stub_container_registry_config(enabled: true) stub_container_registry_tags('tag') + project.container_images << container_image end subject { transfer_project(project, user, group) } -- cgit v1.2.1 From 164ef8a348cac86097313bc453493ccf739adffe Mon Sep 17 00:00:00 2001 From: Andre Guedes Date: Fri, 16 Dec 2016 11:12:37 -0200 Subject: Fixing typos in docs --- doc/administration/container_registry.md | 4 ++-- doc/user/project/container_registry.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/administration/container_registry.md b/doc/administration/container_registry.md index 14795601246..4d1cb391e69 100644 --- a/doc/administration/container_registry.md +++ b/doc/administration/container_registry.md @@ -76,7 +76,7 @@ you modify its settings. Read the upstream documentation on how to achieve that. At the absolute minimum, make sure your [Registry configuration][registry-auth] has `container_registry` as the service and `https://gitlab.example.com/jwt/auth` -as the realm. +as the realm: ``` auth: @@ -494,7 +494,7 @@ configurable in future releases. **GitLab 8.8 ([source docs][8-8-docs])** - GitLab Container Registry feature was introduced. -i + [reconfigure gitlab]: restart_gitlab.md#omnibus-gitlab-reconfigure [restart gitlab]: restart_gitlab.md#installations-from-source [wildcard certificate]: https://en.wikipedia.org/wiki/Wildcard_certificate diff --git a/doc/user/project/container_registry.md b/doc/user/project/container_registry.md index eada8e04227..c5b2266ff19 100644 --- a/doc/user/project/container_registry.md +++ b/doc/user/project/container_registry.md @@ -10,7 +10,7 @@ - Starting from GitLab 8.12, if you have 2FA enabled in your account, you need to pass a personal access token instead of your password in order to login to GitLab's Container Registry. -- Multiple level image names support was added in GitLab ?8.15? +- Multiple level image names support was added in GitLab 8.15 With the Docker Container Registry integrated into GitLab, every project can have its own space to store its Docker images. -- cgit v1.2.1 From b408a192e0fbf630d4f9a4112f6835be50a681d8 Mon Sep 17 00:00:00 2001 From: Andre Guedes Date: Fri, 16 Dec 2016 12:07:20 -0200 Subject: Adding mock for full_access_token --- spec/factories/container_images.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/factories/container_images.rb b/spec/factories/container_images.rb index 6141a519a75..3693865101d 100644 --- a/spec/factories/container_images.rb +++ b/spec/factories/container_images.rb @@ -11,6 +11,7 @@ FactoryGirl.define do after(:build) do |image, evaluator| if evaluator.stubbed allow(Gitlab.config.registry).to receive(:enabled).and_return(true) + allow(Auth::ContainerRegistryAuthenticationService).to receive(:full_access_token).and_return('token') allow(image.client).to receive(:repository_tags).and_return({ name: image.name_with_namespace, tags: evaluator.tags -- cgit v1.2.1 From ea17df5c4c23890c48cd51af17e2517f04f7c88b Mon Sep 17 00:00:00 2001 From: Andre Guedes Date: Wed, 25 Jan 2017 10:02:09 -0200 Subject: Fixing minor view issues --- app/views/projects/container_registry/_image.html.haml | 6 +++--- app/views/projects/container_registry/_tag.html.haml | 2 +- app/views/projects/container_registry/index.html.haml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/views/projects/container_registry/_image.html.haml b/app/views/projects/container_registry/_image.html.haml index 5845efd345a..72f2103b862 100644 --- a/app/views/projects/container_registry/_image.html.haml +++ b/app/views/projects/container_registry/_image.html.haml @@ -8,7 +8,7 @@ = icon("chevron-down") = escape_once(image.name) - = clipboard_button(clipboard_text: "docker pull #{image.path}") + = clipboard_button(clipboard_text: "docker pull #{image.path}") .controls.hidden-xs.pull-right = link_to namespace_project_container_registry_path(@project.namespace, @project, image.id), class: 'btn btn-remove has-tooltip', title: "Remove image", data: { confirm: "Are you sure?" }, method: :delete do = icon("trash cred") @@ -24,8 +24,8 @@ %table.table.tags %thead %tr - %th Name - %th Image ID + %th Tag + %th Tag ID %th Size %th Created - if can?(current_user, :update_container_image, @project) diff --git a/app/views/projects/container_registry/_tag.html.haml b/app/views/projects/container_registry/_tag.html.haml index b35a9cb621f..f7161e85428 100644 --- a/app/views/projects/container_registry/_tag.html.haml +++ b/app/views/projects/container_registry/_tag.html.haml @@ -25,5 +25,5 @@ - if can?(current_user, :update_container_image, @project) %td.content .controls.hidden-xs.pull-right - = link_to namespace_project_container_registry_path(@project.namespace, @project, { id: tag.repository.id, tag: tag.name} ), class: 'btn btn-remove has-tooltip', title: "Remove tag", data: { confirm: "Are you sure?" }, method: :delete do + = link_to namespace_project_container_registry_path(@project.namespace, @project, { id: tag.repository.id, tag: tag.name} ), class: 'btn btn-remove has-tooltip', title: "Remove tag", data: { confirm: "Due to a Docker limitation, all tags with the same ID will also be deleted. Are you sure?" }, method: :delete do = icon("trash cred") diff --git a/app/views/projects/container_registry/index.html.haml b/app/views/projects/container_registry/index.html.haml index ab6213f03d8..5508a3de396 100644 --- a/app/views/projects/container_registry/index.html.haml +++ b/app/views/projects/container_registry/index.html.haml @@ -15,9 +15,9 @@ %br Then you are free to create and upload a container image with build and push commands: %pre - docker build -t #{escape_once(@project.container_registry_url)} . + docker build -t #{escape_once(@project.container_registry_url)}/image . %br - docker push #{escape_once(@project.container_registry_url)} + docker push #{escape_once(@project.container_registry_url)}/image - if @images.blank? .nothing-here-block No container images in Container Registry for this project. -- cgit v1.2.1 From 8294756fc110fdb84036e4ae097940410a8ad6de Mon Sep 17 00:00:00 2001 From: Andre Guedes Date: Wed, 25 Jan 2017 10:24:50 -0200 Subject: Improved readability in tag/image delete condition --- .../projects/container_registry_controller.rb | 28 +++++++++++++--------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/app/controllers/projects/container_registry_controller.rb b/app/controllers/projects/container_registry_controller.rb index f656f86fcdb..4981e57ed22 100644 --- a/app/controllers/projects/container_registry_controller.rb +++ b/app/controllers/projects/container_registry_controller.rb @@ -9,31 +9,37 @@ class Projects::ContainerRegistryController < Projects::ApplicationController end def destroy - url = namespace_project_container_registry_index_path(project.namespace, project) - if tag - delete_tag(url) + delete_tag else - if image.destroy - redirect_to url - else - redirect_to url, alert: 'Failed to remove image' - end + delete_image end end private + def registry_url + @registry_url ||= namespace_project_container_registry_index_path(project.namespace, project) + end + def verify_registry_enabled render_404 unless Gitlab.config.registry.enabled end - def delete_tag(url) + def delete_image + if image.destroy + redirect_to registry_url + else + redirect_to registry_url, alert: 'Failed to remove image' + end + end + + def delete_tag if tag.delete image.destroy if image.tags.empty? - redirect_to url + redirect_to registry_url else - redirect_to url, alert: 'Failed to remove tag' + redirect_to registry_url, alert: 'Failed to remove tag' end end -- cgit v1.2.1 From db5b4b8b1a9b8aa07c8310dde53b7c3ed391bafd Mon Sep 17 00:00:00 2001 From: Andre Guedes Date: Wed, 22 Feb 2017 11:19:23 -0300 Subject: Creates specs for destroy service and improves namespace container image query performance --- app/models/namespace.rb | 2 +- app/services/container_images/destroy_service.rb | 26 ++--------------- app/views/admin/container_registry/show.html.haml | 2 +- .../20161031013926_create_container_image.rb | 16 ---------- ...egistry_access_token_to_application_settings.rb | 16 ---------- db/schema.rb | 10 +++---- lib/api/registry_events.rb | 4 +-- .../container_images/destroy_service_spec.rb | 34 ++++++++++++++++++++++ 8 files changed, 45 insertions(+), 65 deletions(-) create mode 100644 spec/services/container_images/destroy_service_spec.rb diff --git a/app/models/namespace.rb b/app/models/namespace.rb index c8e329044e0..a803be2e780 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -155,7 +155,7 @@ class Namespace < ActiveRecord::Base end def any_project_has_container_registry_images? - projects.any? { |project| project.container_images.present? } + projects.joins(:container_images).any? end def send_update_instructions diff --git a/app/services/container_images/destroy_service.rb b/app/services/container_images/destroy_service.rb index c73b6cfefba..15dca227291 100644 --- a/app/services/container_images/destroy_service.rb +++ b/app/services/container_images/destroy_service.rb @@ -1,31 +1,9 @@ module ContainerImages class DestroyService < BaseService - class DestroyError < StandardError; end - def execute(container_image) - @container_image = container_image - - return false unless can?(current_user, :remove_project, project) - - ContainerImage.transaction do - container_image.destroy! - - unless remove_container_image_tags - raise_error('Failed to remove container image tags. Please try again or contact administrator') - end - end - - true - end - - private - - def raise_error(message) - raise DestroyError.new(message) - end + return false unless can?(current_user, :update_container_image, project) - def remove_container_image_tags - container_image.delete_tags + container_image.destroy! end end end diff --git a/app/views/admin/container_registry/show.html.haml b/app/views/admin/container_registry/show.html.haml index 8803eddda69..ffaa7736d65 100644 --- a/app/views/admin/container_registry/show.html.haml +++ b/app/views/admin/container_registry/show.html.haml @@ -17,7 +17,7 @@ X-Registry-Token: [#{@access_token}] %br Access token is - %code{ id: 'registry-token' } #{@access_token} + %code{ id: 'registry-token' }= @access_token .bs-callout.clearfix .pull-left diff --git a/db/migrate/20161031013926_create_container_image.rb b/db/migrate/20161031013926_create_container_image.rb index 85c0913b8f3..884c78880eb 100644 --- a/db/migrate/20161031013926_create_container_image.rb +++ b/db/migrate/20161031013926_create_container_image.rb @@ -7,22 +7,6 @@ class CreateContainerImage < ActiveRecord::Migration # Set this constant to true if this migration requires downtime. DOWNTIME = false - # When a migration requires downtime you **must** uncomment the following - # constant and define a short and easy to understand explanation as to why the - # migration requires downtime. - # DOWNTIME_REASON = '' - - # When using the methods "add_concurrent_index" or "add_column_with_default" - # you must disable the use of transactions as these methods can not run in an - # existing transaction. When using "add_concurrent_index" make sure that this - # method is the _only_ method called in the migration, any other changes - # should go in a separate migration. This ensures that upon failure _only_ the - # index creation fails and can be retried or reverted easily. - # - # To disable transactions uncomment the following line and remove these - # comments: - # disable_ddl_transaction! - def change create_table :container_images do |t| t.integer :project_id diff --git a/db/migrate/20161213212947_add_container_registry_access_token_to_application_settings.rb b/db/migrate/20161213212947_add_container_registry_access_token_to_application_settings.rb index f89f9b00a5f..23d87cc6d0a 100644 --- a/db/migrate/20161213212947_add_container_registry_access_token_to_application_settings.rb +++ b/db/migrate/20161213212947_add_container_registry_access_token_to_application_settings.rb @@ -7,22 +7,6 @@ class AddContainerRegistryAccessTokenToApplicationSettings < ActiveRecord::Migra # Set this constant to true if this migration requires downtime. DOWNTIME = false - # When a migration requires downtime you **must** uncomment the following - # constant and define a short and easy to understand explanation as to why the - # migration requires downtime. - # DOWNTIME_REASON = '' - - # When using the methods "add_concurrent_index" or "add_column_with_default" - # you must disable the use of transactions as these methods can not run in an - # existing transaction. When using "add_concurrent_index" make sure that this - # method is the _only_ method called in the migration, any other changes - # should go in a separate migration. This ensures that upon failure _only_ the - # index creation fails and can be retried or reverted easily. - # - # To disable transactions uncomment the following line and remove these - # comments: - # disable_ddl_transaction! - def change add_column :application_settings, :container_registry_access_token, :string end diff --git a/db/schema.rb b/db/schema.rb index 36df20fc8f2..08d11546800 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -61,6 +61,7 @@ ActiveRecord::Schema.define(version: 20170215200045) do t.boolean "shared_runners_enabled", default: true, null: false t.integer "max_artifacts_size", default: 100, null: false t.string "runners_registration_token" + t.integer "max_pages_size", default: 100, null: false t.boolean "require_two_factor_authentication", default: false t.integer "two_factor_grace_period", default: 48 t.boolean "metrics_enabled", default: false @@ -107,10 +108,9 @@ ActiveRecord::Schema.define(version: 20170215200045) do t.string "sidekiq_throttling_queues" t.decimal "sidekiq_throttling_factor" t.boolean "html_emails_enabled", default: true + t.string "container_registry_access_token" t.string "plantuml_url" t.boolean "plantuml_enabled" - t.string "container_registry_access_token" - t.integer "max_pages_size", default: 100, null: false t.integer "terminal_max_session_time", default: 0, null: false end @@ -586,9 +586,9 @@ ActiveRecord::Schema.define(version: 20170215200045) do end add_index "labels", ["group_id", "project_id", "title"], name: "index_labels_on_group_id_and_project_id_and_title", unique: true, using: :btree - add_index "labels", ["type", "project_id"], name: "index_labels_on_type_and_project_id", using: :btree add_index "labels", ["project_id"], name: "index_labels_on_project_id", using: :btree add_index "labels", ["title"], name: "index_labels_on_title", using: :btree + add_index "labels", ["type", "project_id"], name: "index_labels_on_type_and_project_id", using: :btree create_table "lfs_objects", force: :cascade do |t| t.string "oid", null: false @@ -761,8 +761,8 @@ ActiveRecord::Schema.define(version: 20170215200045) do t.integer "visibility_level", default: 20, null: false t.boolean "request_access_enabled", default: false, null: false t.datetime "deleted_at" - t.boolean "lfs_enabled" t.text "description_html" + t.boolean "lfs_enabled" t.integer "parent_id" end @@ -1283,8 +1283,8 @@ ActiveRecord::Schema.define(version: 20170215200045) do t.datetime "otp_grace_period_started_at" t.boolean "ldap_email", default: false, null: false t.boolean "external", default: false - t.string "organization" t.string "incoming_email_token" + t.string "organization" t.boolean "authorized_projects_populated" t.boolean "notified_of_own_activity", default: false, null: false end diff --git a/lib/api/registry_events.rb b/lib/api/registry_events.rb index 12305a49f0f..fc6fc0b97e0 100644 --- a/lib/api/registry_events.rb +++ b/lib/api/registry_events.rb @@ -39,9 +39,9 @@ module API params['events'].each do |event| repository = event['target']['repository'] - if event['action'] == 'push' and !!event['target']['tag'] + if event['action'] == 'push' && !!event['target']['tag'] namespace, container_image_name = ContainerImage::split_namespace(repository) - project = Project::find_with_namespace(namespace) + project = Project::find_by_full_path(namespace) if project container_image = project.container_images.find_or_create_by(name: container_image_name) diff --git a/spec/services/container_images/destroy_service_spec.rb b/spec/services/container_images/destroy_service_spec.rb new file mode 100644 index 00000000000..5b4dbaa7934 --- /dev/null +++ b/spec/services/container_images/destroy_service_spec.rb @@ -0,0 +1,34 @@ +require 'spec_helper' + +describe ContainerImages::DestroyService, services: true do + describe '#execute' do + let(:user) { create(:user) } + let(:container_image) { create(:container_image, name: '') } + let(:project) { create(:project, path: 'test', namespace: user.namespace, container_images: [container_image]) } + let(:example_host) { 'example.com' } + let(:registry_url) { 'http://' + example_host } + + it { expect(container_image).to be_valid } + it { expect(project.container_images).not_to be_empty } + + context 'when container image has tags' do + before do + project.team << [user, :master] + end + + it 'removes all tags before destroy' do + service = described_class.new(project, user) + + expect(container_image).to receive(:delete_tags).and_return(true) + expect { service.execute(container_image) }.to change(project.container_images, :count).by(-1) + end + + it 'fails when tags are not removed' do + service = described_class.new(project, user) + + expect(container_image).to receive(:delete_tags).and_return(false) + expect { service.execute(container_image) }.to raise_error(ActiveRecord::RecordNotDestroyed) + end + end + end +end -- cgit v1.2.1 From 53d332d3c73f8a883fa54d8eaaf91f92da73c33f Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 21 Mar 2017 13:39:57 +0100 Subject: Add configured container registry key to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 0b602d613c7..5e9f19d8319 100644 --- a/.gitignore +++ b/.gitignore @@ -30,6 +30,7 @@ eslint-report.html /config/unicorn.rb /config/secrets.yml /config/sidekiq.yml +/config/registry.key /coverage/* /coverage-javascript/ /db/*.sqlite3 -- cgit v1.2.1 From c64d36306cafac463f20d49e750f397a9b32960b Mon Sep 17 00:00:00 2001 From: Andre Guedes Date: Tue, 21 Mar 2017 10:35:02 -0300 Subject: Makes ContainerImages Routable Conflicts: db/schema.rb --- app/models/container_image.rb | 14 ++++++++++++-- .../auth/container_registry_authentication_service.rb | 2 +- app/views/projects/container_registry/_image.html.haml | 3 +-- app/views/projects/container_registry/index.html.haml | 3 +-- db/migrate/20161031013926_create_container_image.rb | 1 + db/schema.rb | 3 ++- lib/api/registry_events.rb | 2 +- 7 files changed, 19 insertions(+), 9 deletions(-) diff --git a/app/models/container_image.rb b/app/models/container_image.rb index 583cb977910..a362ea3adbc 100644 --- a/app/models/container_image.rb +++ b/app/models/container_image.rb @@ -1,4 +1,6 @@ class ContainerImage < ActiveRecord::Base + include Routable + belongs_to :project delegate :container_registry, :container_registry_allowed_paths, @@ -17,10 +19,18 @@ class ContainerImage < ActiveRecord::Base client.update_token(token) end - def path - [container_registry.path, name_with_namespace].compact.join('/') + def parent + project + end + + def parent_changed? + project_id_changed? end + # def path + # [container_registry.path, name_with_namespace].compact.join('/') + # end + def name_with_namespace [container_registry_path_with_namespace, name].reject(&:blank?).join('/') end diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb index 08fe6e3293e..a3c8d77bf09 100644 --- a/app/services/auth/container_registry_authentication_service.rb +++ b/app/services/auth/container_registry_authentication_service.rb @@ -66,7 +66,7 @@ module Auth # per image authentication. # Removes only last occurence in light # of future nested groups - namespace, _ = ContainerImage::split_namespace(name) + namespace, a = ContainerImage::split_namespace(name) requested_project = Project.find_by_full_path(namespace) return unless requested_project diff --git a/app/views/projects/container_registry/_image.html.haml b/app/views/projects/container_registry/_image.html.haml index 72f2103b862..4fd642a56c9 100644 --- a/app/views/projects/container_registry/_image.html.haml +++ b/app/views/projects/container_registry/_image.html.haml @@ -31,5 +31,4 @@ - if can?(current_user, :update_container_image, @project) %th - - image.tags.each do |tag| - = render 'tag', tag: tag + = render partial: 'tag', collection: image.tags diff --git a/app/views/projects/container_registry/index.html.haml b/app/views/projects/container_registry/index.html.haml index 5508a3de396..1b5d000e801 100644 --- a/app/views/projects/container_registry/index.html.haml +++ b/app/views/projects/container_registry/index.html.haml @@ -23,5 +23,4 @@ .nothing-here-block No container images in Container Registry for this project. - else - - @images.each do |image| - = render 'image', image: image + = render partial: 'image', collection: @images diff --git a/db/migrate/20161031013926_create_container_image.rb b/db/migrate/20161031013926_create_container_image.rb index 884c78880eb..06c409857da 100644 --- a/db/migrate/20161031013926_create_container_image.rb +++ b/db/migrate/20161031013926_create_container_image.rb @@ -11,6 +11,7 @@ class CreateContainerImage < ActiveRecord::Migration create_table :container_images do |t| t.integer :project_id t.string :name + t.string :path end end end diff --git a/db/schema.rb b/db/schema.rb index db57fb0a548..a1fc5dc1f58 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -108,7 +108,6 @@ ActiveRecord::Schema.define(version: 20170315194013) do t.string "sidekiq_throttling_queues" t.decimal "sidekiq_throttling_factor" t.boolean "html_emails_enabled", default: true - t.string "container_registry_access_token" t.string "plantuml_url" t.boolean "plantuml_enabled" t.integer "terminal_max_session_time", default: 0, null: false @@ -117,6 +116,7 @@ ActiveRecord::Schema.define(version: 20170315194013) do t.integer "unique_ips_limit_per_user" t.integer "unique_ips_limit_time_window" t.boolean "unique_ips_limit_enabled", default: false, null: false + t.string "container_registry_access_token" end create_table "audit_events", force: :cascade do |t| @@ -327,6 +327,7 @@ ActiveRecord::Schema.define(version: 20170315194013) do create_table "container_images", force: :cascade do |t| t.integer "project_id" t.string "name" + t.string "path" end create_table "deploy_keys_projects", force: :cascade do |t| diff --git a/lib/api/registry_events.rb b/lib/api/registry_events.rb index fc6fc0b97e0..8c53e0fcfc0 100644 --- a/lib/api/registry_events.rb +++ b/lib/api/registry_events.rb @@ -44,7 +44,7 @@ module API project = Project::find_by_full_path(namespace) if project - container_image = project.container_images.find_or_create_by(name: container_image_name) + container_image = project.container_images.find_or_create_by(name: container_image_name, path: container_image_name) unless container_image.valid? render_api_error!({ error: "Failed to create container image!" }, 400) -- cgit v1.2.1 From 68a2fa54dedcdbe893ec811413d1703e5f6ac2dc Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 22 Mar 2017 11:08:23 +0100 Subject: Remove out-of-scope changes for multi-level images --- .../admin/application_settings_controller.rb | 6 --- .../admin/container_registry_controller.rb | 11 ---- app/models/application_setting.rb | 6 --- app/views/admin/container_registry/show.html.haml | 31 ----------- app/views/admin/dashboard/_head.html.haml | 4 -- config/routes/admin.rb | 2 - ...egistry_access_token_to_application_settings.rb | 13 ----- doc/administration/container_registry.md | 18 ------- lib/api/api.rb | 1 - lib/api/helpers.rb | 10 ---- lib/api/registry_events.rb | 60 ---------------------- lib/container_registry/ROADMAP.md | 7 --- 12 files changed, 169 deletions(-) delete mode 100644 app/controllers/admin/container_registry_controller.rb delete mode 100644 app/views/admin/container_registry/show.html.haml delete mode 100644 db/migrate/20161213212947_add_container_registry_access_token_to_application_settings.rb delete mode 100644 lib/api/registry_events.rb delete mode 100644 lib/container_registry/ROADMAP.md diff --git a/app/controllers/admin/application_settings_controller.rb b/app/controllers/admin/application_settings_controller.rb index 1d0bd6e0b81..8d831ffdd70 100644 --- a/app/controllers/admin/application_settings_controller.rb +++ b/app/controllers/admin/application_settings_controller.rb @@ -29,12 +29,6 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController redirect_to :back end - def reset_container_registry_token - @application_setting.reset_container_registry_access_token! - flash[:notice] = 'New container registry access token has been generated!' - redirect_to :back - end - def clear_repository_check_states RepositoryCheck::ClearWorker.perform_async diff --git a/app/controllers/admin/container_registry_controller.rb b/app/controllers/admin/container_registry_controller.rb deleted file mode 100644 index 265c032c67d..00000000000 --- a/app/controllers/admin/container_registry_controller.rb +++ /dev/null @@ -1,11 +0,0 @@ -class Admin::ContainerRegistryController < Admin::ApplicationController - def show - @access_token = container_registry_access_token - end - - private - - def container_registry_access_token - current_application_settings.container_registry_access_token - end -end diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 9d01a70c77d..671a0fe98cc 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -4,7 +4,6 @@ class ApplicationSetting < ActiveRecord::Base add_authentication_token_field :runners_registration_token add_authentication_token_field :health_check_access_token - add_authentication_token_field :container_registry_access_token CACHE_KEY = 'application_setting.last'.freeze DOMAIN_LIST_SEPARATOR = %r{\s*[,;]\s* # comma or semicolon, optionally surrounded by whitespace @@ -158,7 +157,6 @@ class ApplicationSetting < ActiveRecord::Base before_save :ensure_runners_registration_token before_save :ensure_health_check_access_token - before_save :ensure_container_registry_access_token after_commit do Rails.cache.write(CACHE_KEY, self) @@ -332,10 +330,6 @@ class ApplicationSetting < ActiveRecord::Base ensure_health_check_access_token! end - def container_registry_access_token - ensure_container_registry_access_token! - end - def sidekiq_throttling_enabled? return false unless sidekiq_throttling_column_exists? diff --git a/app/views/admin/container_registry/show.html.haml b/app/views/admin/container_registry/show.html.haml deleted file mode 100644 index ffaa7736d65..00000000000 --- a/app/views/admin/container_registry/show.html.haml +++ /dev/null @@ -1,31 +0,0 @@ -- @no_container = true -= render "admin/dashboard/head" - -%div{ class: container_class } - - %p.prepend-top-default - %span - To properly configure the Container Registry you should add the following - access token to the Docker Registry config.yml as follows: - %pre - %code - :plain - notifications: - endpoints: - - ... - headers: - X-Registry-Token: [#{@access_token}] - %br - Access token is - %code{ id: 'registry-token' }= @access_token - - .bs-callout.clearfix - .pull-left - %p - You can reset container registry access token by pressing the button below. - %p - = button_to reset_container_registry_token_admin_application_settings_path, - method: :put, class: 'btn btn-default', - data: { confirm: 'Are you sure you want to reset container registry token?' } do - = icon('refresh') - Reset container registry access token diff --git a/app/views/admin/dashboard/_head.html.haml b/app/views/admin/dashboard/_head.html.haml index dbd039547fa..7893c1dee97 100644 --- a/app/views/admin/dashboard/_head.html.haml +++ b/app/views/admin/dashboard/_head.html.haml @@ -27,7 +27,3 @@ = link_to admin_runners_path, title: 'Runners' do %span Runners - = nav_link path: 'container_registry#show' do - = link_to admin_container_registry_path, title: 'Registry' do - %span - Registry diff --git a/config/routes/admin.rb b/config/routes/admin.rb index fcbe2e2c435..486ce3c5c87 100644 --- a/config/routes/admin.rb +++ b/config/routes/admin.rb @@ -63,7 +63,6 @@ namespace :admin do resource :background_jobs, controller: 'background_jobs', only: [:show] resource :system_info, controller: 'system_info', only: [:show] resources :requests_profiles, only: [:index, :show], param: :name, constraints: { name: /.+\.html/ } - resource :container_registry, controller: 'container_registry', only: [:show] resources :projects, only: [:index] @@ -94,7 +93,6 @@ namespace :admin do resources :services, only: [:index, :edit, :update] put :reset_runners_token put :reset_health_check_token - put :reset_container_registry_token put :clear_repository_check_states end diff --git a/db/migrate/20161213212947_add_container_registry_access_token_to_application_settings.rb b/db/migrate/20161213212947_add_container_registry_access_token_to_application_settings.rb deleted file mode 100644 index 23d87cc6d0a..00000000000 --- a/db/migrate/20161213212947_add_container_registry_access_token_to_application_settings.rb +++ /dev/null @@ -1,13 +0,0 @@ -# See http://doc.gitlab.com/ce/development/migration_style_guide.html -# for more information on how to write migrations for GitLab. - -class AddContainerRegistryAccessTokenToApplicationSettings < ActiveRecord::Migration - include Gitlab::Database::MigrationHelpers - - # Set this constant to true if this migration requires downtime. - DOWNTIME = false - - def change - add_column :application_settings, :container_registry_access_token, :string - end -end diff --git a/doc/administration/container_registry.md b/doc/administration/container_registry.md index dc4e57f25fb..f707039827b 100644 --- a/doc/administration/container_registry.md +++ b/doc/administration/container_registry.md @@ -87,23 +87,6 @@ auth: rootcertbundle: /root/certs/certbundle ``` -Also a notification endpoint must be configured with the token from -Admin Area -> Overview -> Registry (`/admin/container_registry`) like in the following sample: - -``` -notifications: - endpoints: - - name: listener - url: https://gitlab.example.com/api/v3/registry_events - headers: - X-Registry-Token: [57Cx95fc2zHFh93VTiGD] - timeout: 500ms - threshold: 5 - backoff: 1s -``` - -Check the [Registry endpoint configuration][registry-endpoint] for details. - ## Container Registry domain configuration There are two ways you can configure the Registry's external domain. @@ -600,7 +583,6 @@ notifications: [storage-config]: https://docs.docker.com/registry/configuration/#storage [registry-http-config]: https://docs.docker.com/registry/configuration/#http [registry-auth]: https://docs.docker.com/registry/configuration/#auth -[registry-endpoint]: https://docs.docker.com/registry/notifications/#/configuration [token-config]: https://docs.docker.com/registry/configuration/#token [8-8-docs]: https://gitlab.com/gitlab-org/gitlab-ce/blob/8-8-stable/doc/administration/container_registry.md [registry-ssl]: https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/support/nginx/registry-ssl diff --git a/lib/api/api.rb b/lib/api/api.rb index 7c7bfada7d0..1bf20f76ad6 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -104,7 +104,6 @@ module API mount ::API::Namespaces mount ::API::Notes mount ::API::NotificationSettings - mount ::API::RegistryEvents mount ::API::Pipelines mount ::API::ProjectHooks mount ::API::Projects diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 3c173b544aa..bd22b82476b 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -111,16 +111,6 @@ module API end end - def authenticate_container_registry_access_token! - token = request.headers['X-Registry-Token'] - unless token.present? && ActiveSupport::SecurityUtils.variable_size_secure_compare( - token, - current_application_settings.container_registry_access_token - ) - unauthorized! - end - end - def authenticated_as_admin! authenticate! forbidden! unless current_user.is_admin? diff --git a/lib/api/registry_events.rb b/lib/api/registry_events.rb deleted file mode 100644 index 8c53e0fcfc0..00000000000 --- a/lib/api/registry_events.rb +++ /dev/null @@ -1,60 +0,0 @@ -module API - # RegistryEvents API - class RegistryEvents < Grape::API - before { authenticate_container_registry_access_token! } - - content_type :json, 'application/vnd.docker.distribution.events.v1+json' - - params do - requires :events, type: Array, desc: 'The ID of a project' do - requires :id, type: String, desc: 'The ID of the event' - requires :timestamp, type: String, desc: 'Timestamp of the event' - requires :action, type: String, desc: 'Action performed by event' - requires :target, type: Hash, desc: 'Target of the event' do - optional :mediaType, type: String, desc: 'Media type of the target' - optional :size, type: Integer, desc: 'Size in bytes of the target' - requires :digest, type: String, desc: 'Digest of the target' - requires :repository, type: String, desc: 'Repository of target' - optional :url, type: String, desc: 'Url of the target' - optional :tag, type: String, desc: 'Tag of the target' - end - requires :request, type: Hash, desc: 'Request of the event' do - requires :id, type: String, desc: 'The ID of the request' - optional :addr, type: String, desc: 'IP Address of the request client' - optional :host, type: String, desc: 'Hostname of the registry instance' - requires :method, type: String, desc: 'Request method' - requires :useragent, type: String, desc: 'UserAgent header of the request' - end - requires :actor, type: Hash, desc: 'Actor that initiated the event' do - optional :name, type: String, desc: 'Actor name' - end - requires :source, type: Hash, desc: 'Source of the event' do - optional :addr, type: String, desc: 'Hostname of source registry node' - optional :instanceID, type: String, desc: 'Source registry node instanceID' - end - end - end - resource :registry_events do - post do - params['events'].each do |event| - repository = event['target']['repository'] - - if event['action'] == 'push' && !!event['target']['tag'] - namespace, container_image_name = ContainerImage::split_namespace(repository) - project = Project::find_by_full_path(namespace) - - if project - container_image = project.container_images.find_or_create_by(name: container_image_name, path: container_image_name) - - unless container_image.valid? - render_api_error!({ error: "Failed to create container image!" }, 400) - end - else - not_found!('Project') - end - end - end - end - end - end -end diff --git a/lib/container_registry/ROADMAP.md b/lib/container_registry/ROADMAP.md deleted file mode 100644 index e0a20776404..00000000000 --- a/lib/container_registry/ROADMAP.md +++ /dev/null @@ -1,7 +0,0 @@ -## Road map - -### Initial thoughts - -- Determine if image names will be persisted or fetched from API -- If persisted, how to update the stored names upon modification -- If fetched, how to fetch only images of a given project -- cgit v1.2.1 From 29c34267556198ee3dbbe2f13bc81708f5e60f10 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 22 Mar 2017 11:41:05 +0100 Subject: Move container images migration to the present time --- db/migrate/20161031013926_create_container_image.rb | 17 ----------------- db/migrate/20170322013926_create_container_image.rb | 13 +++++++++++++ 2 files changed, 13 insertions(+), 17 deletions(-) delete mode 100644 db/migrate/20161031013926_create_container_image.rb create mode 100644 db/migrate/20170322013926_create_container_image.rb diff --git a/db/migrate/20161031013926_create_container_image.rb b/db/migrate/20161031013926_create_container_image.rb deleted file mode 100644 index 06c409857da..00000000000 --- a/db/migrate/20161031013926_create_container_image.rb +++ /dev/null @@ -1,17 +0,0 @@ -# See http://doc.gitlab.com/ce/development/migration_style_guide.html -# for more information on how to write migrations for GitLab. - -class CreateContainerImage < ActiveRecord::Migration - include Gitlab::Database::MigrationHelpers - - # Set this constant to true if this migration requires downtime. - DOWNTIME = false - - def change - create_table :container_images do |t| - t.integer :project_id - t.string :name - t.string :path - end - end -end diff --git a/db/migrate/20170322013926_create_container_image.rb b/db/migrate/20170322013926_create_container_image.rb new file mode 100644 index 00000000000..c494f2a56c7 --- /dev/null +++ b/db/migrate/20170322013926_create_container_image.rb @@ -0,0 +1,13 @@ +class CreateContainerImage < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + create_table :container_images do |t| + t.integer :project_id + t.string :name + t.string :path + end + end +end -- cgit v1.2.1 From 95e2c0196b7e492f8c03c6cfeb6b37e97f75813e Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 22 Mar 2017 12:28:23 +0100 Subject: Clean code related to accessing registry from project [ci skip] --- app/models/container_image.rb | 28 ++++------------------------ app/models/project.rb | 16 +++++----------- 2 files changed, 9 insertions(+), 35 deletions(-) diff --git a/app/models/container_image.rb b/app/models/container_image.rb index a362ea3adbc..411617ccd71 100644 --- a/app/models/container_image.rb +++ b/app/models/container_image.rb @@ -3,36 +3,16 @@ class ContainerImage < ActiveRecord::Base belongs_to :project - delegate :container_registry, :container_registry_allowed_paths, - :container_registry_path_with_namespace, to: :project - + delegate :container_registry, to: :project delegate :client, to: :container_registry validates :manifest, presence: true before_destroy :delete_tags - before_validation :update_token, on: :create - def update_token - paths = container_registry_allowed_paths << name_with_namespace - token = Auth::ContainerRegistryAuthenticationService.full_access_token(paths) - client.update_token(token) - end - - def parent - project - end - - def parent_changed? - project_id_changed? - end - - # def path - # [container_registry.path, name_with_namespace].compact.join('/') - # end - - def name_with_namespace - [container_registry_path_with_namespace, name].reject(&:blank?).join('/') + def registry + # TODO, container registry with image access level + token = Auth::ContainerRegistryAuthenticationService.image_token(self) end def tag(tag) diff --git a/app/models/project.rb b/app/models/project.rb index 928965643a0..4aa9c6bb2f2 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -405,29 +405,23 @@ class Project < ActiveRecord::Base @repository ||= Repository.new(path_with_namespace, self) end - def container_registry_path_with_namespace - path_with_namespace.downcase - end - - def container_registry_allowed_paths - @container_registry_allowed_paths ||= [container_registry_path_with_namespace] + - container_images.map { |i| i.name_with_namespace } - end - def container_registry return unless Gitlab.config.registry.enabled @container_registry ||= begin - token = Auth::ContainerRegistryAuthenticationService.full_access_token(container_registry_allowed_paths) + token = Auth::ContainerRegistryAuthenticationService.full_access_token(project) + url = Gitlab.config.registry.api_url host_port = Gitlab.config.registry.host_port + # TODO, move configuration vars into ContainerRegistry::Registry, clean + # this method up afterwards ContainerRegistry::Registry.new(url, token: token, path: host_port) end end def container_registry_url if Gitlab.config.registry.enabled - "#{Gitlab.config.registry.host_port}/#{container_registry_path_with_namespace}" + "#{Gitlab.config.registry.host_port}/#{path_with_namespace.downcase}" end end -- cgit v1.2.1 From 896b13b929369c02f72fa881eda24ca4a6a0d900 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 22 Mar 2017 16:07:27 +0100 Subject: Refactor splitting container image full path [ci skip] --- app/models/container_image.rb | 17 +++++++---------- .../auth/container_registry_authentication_service.rb | 7 +------ 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/app/models/container_image.rb b/app/models/container_image.rb index 411617ccd71..6e9a060d7a8 100644 --- a/app/models/container_image.rb +++ b/app/models/container_image.rb @@ -1,6 +1,4 @@ class ContainerImage < ActiveRecord::Base - include Routable - belongs_to :project delegate :container_registry, to: :project @@ -45,14 +43,13 @@ class ContainerImage < ActiveRecord::Base end end - # rubocop:disable RedundantReturn + def self.from_path(full_path) + return unless full_path.include?('/') - def self.split_namespace(full_path) - image_name = full_path.split('/').last - namespace = full_path.gsub(/(.*)(#{Regexp.escape('/' + image_name)})/, '\1') - if namespace.count('/') < 1 - namespace, image_name = full_path, "" - end - return namespace, image_name + path = full_path[0...full_path.rindex('/')] + name = full_path[full_path.rindex('/')+1..-1] + project = Project.find_by_full_path(path) + + self.new(name: name, path: path, project: project) end end diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb index a3c8d77bf09..7e412040c7c 100644 --- a/app/services/auth/container_registry_authentication_service.rb +++ b/app/services/auth/container_registry_authentication_service.rb @@ -62,12 +62,7 @@ module Auth end def process_repository_access(type, name, actions) - # Strips image name due to lack of - # per image authentication. - # Removes only last occurence in light - # of future nested groups - namespace, a = ContainerImage::split_namespace(name) - requested_project = Project.find_by_full_path(namespace) + requested_project = ContainerImage.from_path(name).project return unless requested_project actions = actions.select do |action| -- cgit v1.2.1 From 4005eb643657e5ee8b1f328e36a3204253e3acf4 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 23 Mar 2017 11:41:16 +0100 Subject: Fix communication between GitLab and Container Registry --- app/models/container_image.rb | 23 ++++++++++++++-------- .../container_registry_authentication_service.rb | 17 ++++++++-------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/app/models/container_image.rb b/app/models/container_image.rb index 6e9a060d7a8..434302159b0 100644 --- a/app/models/container_image.rb +++ b/app/models/container_image.rb @@ -43,13 +43,20 @@ class ContainerImage < ActiveRecord::Base end end - def self.from_path(full_path) - return unless full_path.include?('/') - - path = full_path[0...full_path.rindex('/')] - name = full_path[full_path.rindex('/')+1..-1] - project = Project.find_by_full_path(path) - - self.new(name: name, path: path, project: project) + def self.project_from_path(image_path) + return unless image_path.include?('/') + + ## + # Projects are always located inside a namespace, so we can remove + # the last node, and see if project with that path exists. + # + truncated_path = image_path.slice(0...image_path.rindex('/')) + + ## + # We still make it possible to search projects by a full image path + # in order to maintain backwards compatibility. + # + Project.find_by_full_path(truncated_path) || + Project.find_by_full_path(image_path) end end diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb index 7e412040c7c..2205b0897e2 100644 --- a/app/services/auth/container_registry_authentication_service.rb +++ b/app/services/auth/container_registry_authentication_service.rb @@ -38,13 +38,13 @@ module Auth private def authorized_token(*accesses) - token = JSONWebToken::RSAToken.new(registry.key) - token.issuer = registry.issuer - token.audience = params[:service] - token.subject = current_user.try(:username) - token.expire_time = self.class.token_expire_at - token[:access] = accesses.compact - token + JSONWebToken::RSAToken.new(registry.key).tap do |token| + token.issuer = registry.issuer + token.audience = params[:service] + token.subject = current_user.try(:username) + token.expire_time = self.class.token_expire_at + token[:access] = accesses.compact + end end def scope @@ -62,7 +62,8 @@ module Auth end def process_repository_access(type, name, actions) - requested_project = ContainerImage.from_path(name).project + requested_project = ContainerImage.project_from_path(name) + return unless requested_project actions = actions.select do |action| -- cgit v1.2.1 From bd8c8df6ed8dd7321608ce652e23d86ef3bd6899 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 23 Mar 2017 12:01:12 +0100 Subject: Fix database schema --- db/schema.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/db/schema.rb b/db/schema.rb index a1fc5dc1f58..b5bd6fc3121 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20170315194013) do +ActiveRecord::Schema.define(version: 20170322013926) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -111,12 +111,10 @@ ActiveRecord::Schema.define(version: 20170315194013) do t.string "plantuml_url" t.boolean "plantuml_enabled" t.integer "terminal_max_session_time", default: 0, null: false - t.integer "max_pages_size", default: 100, null: false t.string "default_artifacts_expire_in", default: "0", null: false t.integer "unique_ips_limit_per_user" t.integer "unique_ips_limit_time_window" t.boolean "unique_ips_limit_enabled", default: false, null: false - t.string "container_registry_access_token" end create_table "audit_events", force: :cascade do |t| @@ -993,6 +991,7 @@ ActiveRecord::Schema.define(version: 20170315194013) do end add_index "routes", ["path"], name: "index_routes_on_path", unique: true, using: :btree + add_index "routes", ["path"], name: "index_routes_on_path_text_pattern_ops", using: :btree, opclasses: {"path"=>"varchar_pattern_ops"} add_index "routes", ["source_type", "source_id"], name: "index_routes_on_source_type_and_source_id", unique: true, using: :btree create_table "sent_notifications", force: :cascade do |t| -- cgit v1.2.1 From 01d159b409d8b24d36204979a73de249843d71bf Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 23 Mar 2017 14:00:41 +0100 Subject: Rename container image model to container repository --- app/models/container_image.rb | 62 ---------------------- app/models/container_repository.rb | 62 ++++++++++++++++++++++ .../container_registry_authentication_service.rb | 2 +- .../20170322013926_create_container_image.rb | 13 ----- .../20170322013926_create_container_repository.rb | 12 +++++ db/schema.rb | 10 ++-- 6 files changed, 79 insertions(+), 82 deletions(-) delete mode 100644 app/models/container_image.rb create mode 100644 app/models/container_repository.rb delete mode 100644 db/migrate/20170322013926_create_container_image.rb create mode 100644 db/migrate/20170322013926_create_container_repository.rb diff --git a/app/models/container_image.rb b/app/models/container_image.rb deleted file mode 100644 index 434302159b0..00000000000 --- a/app/models/container_image.rb +++ /dev/null @@ -1,62 +0,0 @@ -class ContainerImage < ActiveRecord::Base - belongs_to :project - - delegate :container_registry, to: :project - delegate :client, to: :container_registry - - validates :manifest, presence: true - - before_destroy :delete_tags - - def registry - # TODO, container registry with image access level - token = Auth::ContainerRegistryAuthenticationService.image_token(self) - end - - def tag(tag) - ContainerRegistry::Tag.new(self, tag) - end - - def manifest - @manifest ||= client.repository_tags(name_with_namespace) - end - - def tags - return @tags if defined?(@tags) - return [] unless manifest && manifest['tags'] - - @tags = manifest['tags'].map do |tag| - ContainerRegistry::Tag.new(self, tag) - end - end - - def blob(config) - ContainerRegistry::Blob.new(self, config) - end - - def delete_tags - return unless tags - - digests = tags.map {|tag| tag.digest }.to_set - digests.all? do |digest| - client.delete_repository_tag(name_with_namespace, digest) - end - end - - def self.project_from_path(image_path) - return unless image_path.include?('/') - - ## - # Projects are always located inside a namespace, so we can remove - # the last node, and see if project with that path exists. - # - truncated_path = image_path.slice(0...image_path.rindex('/')) - - ## - # We still make it possible to search projects by a full image path - # in order to maintain backwards compatibility. - # - Project.find_by_full_path(truncated_path) || - Project.find_by_full_path(image_path) - end -end diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb new file mode 100644 index 00000000000..2e78fc148b4 --- /dev/null +++ b/app/models/container_repository.rb @@ -0,0 +1,62 @@ +class ContainerRepository < ActiveRecord::Base + belongs_to :project + + delegate :container_registry, to: :project + delegate :client, to: :container_registry + + validates :manifest, presence: true + + before_destroy :delete_tags + + def registry + # TODO, container registry with image access level + token = Auth::ContainerRegistryAuthenticationService.image_token(self) + end + + def tag(tag) + ContainerRegistry::Tag.new(self, tag) + end + + def manifest + @manifest ||= client.repository_tags(self.path) + end + + def tags + return @tags if defined?(@tags) + return [] unless manifest && manifest['tags'] + + @tags = manifest['tags'].map do |tag| + ContainerRegistry::Tag.new(self, tag) + end + end + + def blob(config) + ContainerRegistry::Blob.new(self, config) + end + + def delete_tags + return unless tags + + digests = tags.map {|tag| tag.digest }.to_set + digests.all? do |digest| + client.delete_repository_tag(self.path, digest) + end + end + + def self.project_from_path(repository_path) + return unless repository_path.include?('/') + + ## + # Projects are always located inside a namespace, so we can remove + # the last node, and see if project with that path exists. + # + truncated_path = repository_path.slice(0...repository_path.rindex('/')) + + ## + # We still make it possible to search projects by a full image path + # in order to maintain backwards compatibility. + # + Project.find_by_full_path(truncated_path) || + Project.find_by_full_path(repository_path) + end +end diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb index 2205b0897e2..3d151c6a357 100644 --- a/app/services/auth/container_registry_authentication_service.rb +++ b/app/services/auth/container_registry_authentication_service.rb @@ -62,7 +62,7 @@ module Auth end def process_repository_access(type, name, actions) - requested_project = ContainerImage.project_from_path(name) + requested_project = ContainerRepository.project_from_path(name) return unless requested_project diff --git a/db/migrate/20170322013926_create_container_image.rb b/db/migrate/20170322013926_create_container_image.rb deleted file mode 100644 index c494f2a56c7..00000000000 --- a/db/migrate/20170322013926_create_container_image.rb +++ /dev/null @@ -1,13 +0,0 @@ -class CreateContainerImage < ActiveRecord::Migration - include Gitlab::Database::MigrationHelpers - - DOWNTIME = false - - def change - create_table :container_images do |t| - t.integer :project_id - t.string :name - t.string :path - end - end -end diff --git a/db/migrate/20170322013926_create_container_repository.rb b/db/migrate/20170322013926_create_container_repository.rb new file mode 100644 index 00000000000..0235fd7e096 --- /dev/null +++ b/db/migrate/20170322013926_create_container_repository.rb @@ -0,0 +1,12 @@ +class CreateContainerRepository < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + create_table :container_repositories do |t| + t.integer :project_id + t.string :path + end + end +end diff --git a/db/schema.rb b/db/schema.rb index b5bd6fc3121..be7604a14d5 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -61,7 +61,6 @@ ActiveRecord::Schema.define(version: 20170322013926) do t.boolean "shared_runners_enabled", default: true, null: false t.integer "max_artifacts_size", default: 100, null: false t.string "runners_registration_token" - t.integer "max_pages_size", default: 100, null: false t.boolean "require_two_factor_authentication", default: false t.integer "two_factor_grace_period", default: 48 t.boolean "metrics_enabled", default: false @@ -111,6 +110,7 @@ ActiveRecord::Schema.define(version: 20170322013926) do t.string "plantuml_url" t.boolean "plantuml_enabled" t.integer "terminal_max_session_time", default: 0, null: false + t.integer "max_pages_size", default: 100, null: false t.string "default_artifacts_expire_in", default: "0", null: false t.integer "unique_ips_limit_per_user" t.integer "unique_ips_limit_time_window" @@ -322,9 +322,8 @@ ActiveRecord::Schema.define(version: 20170322013926) do add_index "ci_variables", ["project_id"], name: "index_ci_variables_on_project_id", using: :btree - create_table "container_images", force: :cascade do |t| + create_table "container_repositories", force: :cascade do |t| t.integer "project_id" - t.string "name" t.string "path" end @@ -694,8 +693,8 @@ ActiveRecord::Schema.define(version: 20170322013926) do t.integer "visibility_level", default: 20, null: false t.boolean "request_access_enabled", default: false, null: false t.datetime "deleted_at" - t.text "description_html" t.boolean "lfs_enabled" + t.text "description_html" t.integer "parent_id" end @@ -991,7 +990,6 @@ ActiveRecord::Schema.define(version: 20170322013926) do end add_index "routes", ["path"], name: "index_routes_on_path", unique: true, using: :btree - add_index "routes", ["path"], name: "index_routes_on_path_text_pattern_ops", using: :btree, opclasses: {"path"=>"varchar_pattern_ops"} add_index "routes", ["source_type", "source_id"], name: "index_routes_on_source_type_and_source_id", unique: true, using: :btree create_table "sent_notifications", force: :cascade do |t| @@ -1238,8 +1236,8 @@ ActiveRecord::Schema.define(version: 20170322013926) do t.datetime "otp_grace_period_started_at" t.boolean "ldap_email", default: false, null: false t.boolean "external", default: false - t.string "incoming_email_token" t.string "organization" + t.string "incoming_email_token" t.boolean "authorized_projects_populated" t.boolean "ghost" end -- cgit v1.2.1 From ea16ea5bfcb78f66c6bb37e470d387bf1ac26c9f Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 23 Mar 2017 14:09:01 +0100 Subject: Identify container repository by a single name --- db/migrate/20170322013926_create_container_repository.rb | 2 +- db/schema.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/db/migrate/20170322013926_create_container_repository.rb b/db/migrate/20170322013926_create_container_repository.rb index 0235fd7e096..87a1523724c 100644 --- a/db/migrate/20170322013926_create_container_repository.rb +++ b/db/migrate/20170322013926_create_container_repository.rb @@ -6,7 +6,7 @@ class CreateContainerRepository < ActiveRecord::Migration def change create_table :container_repositories do |t| t.integer :project_id - t.string :path + t.string :name end end end diff --git a/db/schema.rb b/db/schema.rb index be7604a14d5..5d9b219ebea 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -324,7 +324,7 @@ ActiveRecord::Schema.define(version: 20170322013926) do create_table "container_repositories", force: :cascade do |t| t.integer "project_id" - t.string "path" + t.string "name" end create_table "deploy_keys_projects", force: :cascade do |t| -- cgit v1.2.1 From f451173a191474be681d208eceb6a0148ba2c0d0 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 23 Mar 2017 14:37:17 +0100 Subject: Fix specs for container repository model class --- app/models/container_repository.rb | 21 ++++--- spec/factories/container_images.rb | 22 -------- spec/factories/container_repositories.rb | 22 ++++++++ spec/models/container_image_spec.rb | 73 ------------------------ spec/models/container_repository_spec.rb | 96 ++++++++++++++++++++++++++++++++ 5 files changed, 132 insertions(+), 102 deletions(-) delete mode 100644 spec/factories/container_images.rb create mode 100644 spec/factories/container_repositories.rb delete mode 100644 spec/models/container_image_spec.rb create mode 100644 spec/models/container_repository_spec.rb diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index 2e78fc148b4..b3a8ec691de 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -1,16 +1,23 @@ class ContainerRepository < ActiveRecord::Base belongs_to :project - - delegate :container_registry, to: :project - delegate :client, to: :container_registry - + delegate :client, to: :registry validates :manifest, presence: true - + validates :name, presence: true before_destroy :delete_tags def registry - # TODO, container registry with image access level - token = Auth::ContainerRegistryAuthenticationService.image_token(self) + @registry ||= begin + token = Auth::ContainerRegistryAuthenticationService.full_access_token(path) + + url = Gitlab.config.registry.api_url + host_port = Gitlab.config.registry.host_port + + ContainerRegistry::Registry.new(url, token: token, path: host_port) + end + end + + def path + @path ||= "#{project.full_path}/#{name}" end def tag(tag) diff --git a/spec/factories/container_images.rb b/spec/factories/container_images.rb deleted file mode 100644 index 3693865101d..00000000000 --- a/spec/factories/container_images.rb +++ /dev/null @@ -1,22 +0,0 @@ -FactoryGirl.define do - factory :container_image do - name "test_container_image" - project - - transient do - tags ['tag'] - stubbed true - end - - after(:build) do |image, evaluator| - if evaluator.stubbed - allow(Gitlab.config.registry).to receive(:enabled).and_return(true) - allow(Auth::ContainerRegistryAuthenticationService).to receive(:full_access_token).and_return('token') - allow(image.client).to receive(:repository_tags).and_return({ - name: image.name_with_namespace, - tags: evaluator.tags - }) - end - end - end -end diff --git a/spec/factories/container_repositories.rb b/spec/factories/container_repositories.rb new file mode 100644 index 00000000000..fbf6bf62dfd --- /dev/null +++ b/spec/factories/container_repositories.rb @@ -0,0 +1,22 @@ +FactoryGirl.define do + factory :container_repository do + name "test_container_image" + project + + transient do + tags ['tag'] + end + + after(:build) do |image, evaluator| + # if evaluator.tags.to_a.any? + # allow(Gitlab.config.registry).to receive(:enabled).and_return(true) + # allow(Auth::ContainerRegistryAuthenticationService) + # .to receive(:full_access_token).and_return('token') + # allow(image.client).to receive(:repository_tags).and_return({ + # name: image.name_with_namespace, + # tags: evaluator.tags + # }) + # end + end + end +end diff --git a/spec/models/container_image_spec.rb b/spec/models/container_image_spec.rb deleted file mode 100644 index e0bea737f59..00000000000 --- a/spec/models/container_image_spec.rb +++ /dev/null @@ -1,73 +0,0 @@ -require 'spec_helper' - -describe ContainerImage do - let(:group) { create(:group, name: 'group') } - let(:project) { create(:project, path: 'test', group: group) } - let(:example_host) { 'example.com' } - let(:registry_url) { 'http://' + example_host } - let(:container_image) { create(:container_image, name: '', project: project, stubbed: false) } - - before do - stub_container_registry_config(enabled: true, api_url: registry_url, host_port: example_host) - stub_request(:get, 'http://example.com/v2/group/test/tags/list'). - with(headers: { 'Accept' => 'application/vnd.docker.distribution.manifest.v2+json' }). - to_return( - status: 200, - body: JSON.dump(tags: ['test']), - headers: { 'Content-Type' => 'application/json' }) - end - - it { expect(container_image).to respond_to(:project) } - it { expect(container_image).to delegate_method(:container_registry).to(:project) } - it { expect(container_image).to delegate_method(:client).to(:container_registry) } - it { expect(container_image.tag('test')).not_to be_nil } - - context '#path' do - subject { container_image.path } - - it { is_expected.to eq('example.com/group/test') } - end - - context 'manifest processing' do - context '#manifest' do - subject { container_image.manifest } - - it { is_expected.not_to be_nil } - end - - context '#valid?' do - subject { container_image.valid? } - - it { is_expected.to be_truthy } - end - - context '#tags' do - subject { container_image.tags } - - it { is_expected.not_to be_empty } - end - end - - context '#delete_tags' do - let(:tag) { ContainerRegistry::Tag.new(container_image, 'tag') } - - before do - expect(container_image).to receive(:tags).twice.and_return([tag]) - expect(tag).to receive(:digest).and_return('sha256:4c8e63ca4cb663ce6c688cb06f1c3672a172b088dac5b6d7ad7d49cd620d85cf') - end - - subject { container_image.delete_tags } - - context 'succeeds' do - before { expect(container_image.client).to receive(:delete_repository_tag).and_return(true) } - - it { is_expected.to be_truthy } - end - - context 'any fails' do - before { expect(container_image.client).to receive(:delete_repository_tag).and_return(false) } - - it { is_expected.to be_falsey } - end - end -end diff --git a/spec/models/container_repository_spec.rb b/spec/models/container_repository_spec.rb new file mode 100644 index 00000000000..3997c4ca682 --- /dev/null +++ b/spec/models/container_repository_spec.rb @@ -0,0 +1,96 @@ +require 'spec_helper' + +describe ContainerRepository do + let(:group) { create(:group, name: 'group') } + let(:project) { create(:project, path: 'test', group: group) } + + let(:container_repository) do + create(:container_repository, name: 'my_image', project: project) + end + + before do + stub_container_registry_config(enabled: true, + api_url: 'http://registry.gitlab', + host_port: 'registry.gitlab') + + stub_request(:get, 'http://registry.gitlab/v2/group/test/my_image/tags/list') + .with(headers: { + 'Accept' => 'application/vnd.docker.distribution.manifest.v2+json' }) + .to_return( + status: 200, + body: JSON.dump(tags: ['test_tag']), + headers: { 'Content-Type' => 'application/json' }) + end + + describe 'associations' do + it 'belongs to the project' do + expect(container_repository).to belong_to(:project) + end + end + + describe '#tag' do + it 'has a test tag' do + expect(container_repository.tag('test')).not_to be_nil + end + end + + describe '#path' do + it 'returns a full path to the repository' do + expect(container_repository.path).to eq('group/test/my_image') + end + end + + describe '#manifest' do + subject { container_repository.manifest } + + it { is_expected.not_to be_nil } + end + + describe '#valid?' do + subject { container_repository.valid? } + + it { is_expected.to be_truthy } + end + + describe '#tags' do + subject { container_repository.tags } + + it { is_expected.not_to be_empty } + end + + # TODO, improve these specs + # + describe '#delete_tags' do + let(:tag) { ContainerRegistry::Tag.new(container_repository, 'tag') } + + before do + allow(container_repository).to receive(:tags).twice.and_return([tag]) + allow(tag).to receive(:digest) + .and_return('sha256:4c8e63ca4cb663ce6c688cb06f1c3672a172b088dac5b6d7ad7d49cd620d85cf') + end + + context 'when action succeeds' do + before do + allow(container_repository.client) + .to receive(:delete_repository_tag) + .and_return(true) + end + + it 'returns status that indicates success' do + expect(container_repository.delete_tags).to be_truthy + end + end + + context 'when action fails' do + before do + allow(container_repository.client) + .to receive(:delete_repository_tag) + .and_return(false) + end + + it 'returns status that indicates failure' do + expect(container_repository.delete_tags).to be_falsey + end + end + end +end -- cgit v1.2.1 From 9c9aac3d16cfbbcbe24b9b33eb7a6d4c14c24f26 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 23 Mar 2017 09:46:05 -0400 Subject: Prevent Trigger action buttons from wrapping --- app/assets/stylesheets/pages/settings_ci_cd.scss | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/assets/stylesheets/pages/settings_ci_cd.scss b/app/assets/stylesheets/pages/settings_ci_cd.scss index b97a29cd1a0..fe22d186af1 100644 --- a/app/assets/stylesheets/pages/settings_ci_cd.scss +++ b/app/assets/stylesheets/pages/settings_ci_cd.scss @@ -6,6 +6,8 @@ } .trigger-actions { + white-space: nowrap; + .btn { margin-left: 10px; } -- cgit v1.2.1 From 249084b48a86a99c02eefe45b507ebcdf811c20f Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 23 Mar 2017 14:48:24 +0100 Subject: Fix some specs using the old ContainerImage const --- spec/features/container_registry_spec.rb | 2 +- spec/services/projects/destroy_service_spec.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/features/container_registry_spec.rb b/spec/features/container_registry_spec.rb index 862c9fbf6c0..0199d08ab63 100644 --- a/spec/features/container_registry_spec.rb +++ b/spec/features/container_registry_spec.rb @@ -39,7 +39,7 @@ describe "Container Registry" do end it do - expect_any_instance_of(ContainerImage).to receive(:delete_tags).and_return(true) + expect_any_instance_of(ContainerRepository).to receive(:delete_tags).and_return(true) click_on 'Remove image' end diff --git a/spec/services/projects/destroy_service_spec.rb b/spec/services/projects/destroy_service_spec.rb index 270e630e70e..f91d62ebdaf 100644 --- a/spec/services/projects/destroy_service_spec.rb +++ b/spec/services/projects/destroy_service_spec.rb @@ -100,7 +100,7 @@ describe Projects::DestroyService, services: true do context 'images deletion succeeds' do it do - expect_any_instance_of(ContainerImage).to receive(:delete_tags).and_return(true) + expect_any_instance_of(ContainerRepository).to receive(:delete_tags).and_return(true) destroy_project(project, user, {}) end @@ -108,7 +108,7 @@ describe Projects::DestroyService, services: true do context 'images deletion fails' do before do - expect_any_instance_of(ContainerImage).to receive(:delete_tags).and_return(false) + expect_any_instance_of(ContainerRepository).to receive(:delete_tags).and_return(false) end subject { destroy_project(project, user, {}) } -- cgit v1.2.1 From 3e01fed5cb36962065f5d19ab6a0cef1dfc14b48 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 23 Mar 2017 14:57:33 +0100 Subject: Fix Rubocop offenses in container repository code --- app/models/container_repository.rb | 2 +- spec/models/container_repository_spec.rb | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index b3a8ec691de..2f0fd3014a8 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -64,6 +64,6 @@ class ContainerRepository < ActiveRecord::Base # in order to maintain backwards compatibility. # Project.find_by_full_path(truncated_path) || - Project.find_by_full_path(repository_path) + Project.find_by_full_path(repository_path) end end diff --git a/spec/models/container_repository_spec.rb b/spec/models/container_repository_spec.rb index 3997c4ca682..e3180e01758 100644 --- a/spec/models/container_repository_spec.rb +++ b/spec/models/container_repository_spec.rb @@ -14,12 +14,11 @@ describe ContainerRepository do host_port: 'registry.gitlab') stub_request(:get, 'http://registry.gitlab/v2/group/test/my_image/tags/list') - .with(headers: { - 'Accept' => 'application/vnd.docker.distribution.manifest.v2+json' }) + .with(headers: { 'Accept' => 'application/vnd.docker.distribution.manifest.v2+json' }) .to_return( - status: 200, - body: JSON.dump(tags: ['test_tag']), - headers: { 'Content-Type' => 'application/json' }) + status: 200, + body: JSON.dump(tags: ['test_tag']), + headers: { 'Content-Type' => 'application/json' }) end describe 'associations' do -- cgit v1.2.1 From dcd2eeb1cfb633f4a28ddd9bc79deac0e3171d3f Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 23 Mar 2017 15:54:59 +0100 Subject: Rename container image to repository in specs --- app/models/namespace.rb | 2 +- app/models/project.rb | 4 ++-- app/services/projects/transfer_service.rb | 2 +- spec/features/container_registry_spec.rb | 8 ++++---- spec/features/security/project/internal_access_spec.rb | 4 ++-- spec/features/security/project/private_access_spec.rb | 4 ++-- spec/features/security/project/public_access_spec.rb | 4 ++-- spec/lib/container_registry/blob_spec.rb | 2 +- spec/lib/container_registry/tag_spec.rb | 2 +- spec/lib/gitlab/import_export/all_models.yml | 2 +- spec/models/namespace_spec.rb | 4 ++-- spec/models/project_spec.rb | 4 ++-- spec/services/container_images/destroy_service_spec.rb | 16 ++++++++-------- spec/services/projects/destroy_service_spec.rb | 4 ++-- spec/services/projects/transfer_service_spec.rb | 4 ++-- 15 files changed, 33 insertions(+), 33 deletions(-) diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 4ae9d0122f2..ac03f098908 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -150,7 +150,7 @@ class Namespace < ActiveRecord::Base end def any_project_has_container_registry_images? - projects.joins(:container_images).any? + projects.joins(:container_repositories).any? end def send_update_instructions diff --git a/app/models/project.rb b/app/models/project.rb index 4aa9c6bb2f2..5c6672c95b3 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -157,7 +157,7 @@ class Project < ActiveRecord::Base has_one :import_data, dependent: :destroy, class_name: "ProjectImportData" has_one :project_feature, dependent: :destroy has_one :statistics, class_name: 'ProjectStatistics', dependent: :delete - has_many :container_images, dependent: :destroy + has_many :container_repositories, dependent: :destroy has_many :commit_statuses, dependent: :destroy has_many :pipelines, dependent: :destroy, class_name: 'Ci::Pipeline' @@ -908,7 +908,7 @@ class Project < ActiveRecord::Base expire_caches_before_rename(old_path_with_namespace) - if container_images.present? + if container_repositories.present? Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry images are present" # we currently doesn't support renaming repository if it contains images in container registry diff --git a/app/services/projects/transfer_service.rb b/app/services/projects/transfer_service.rb index 6d9e7de4f24..f46bf884e37 100644 --- a/app/services/projects/transfer_service.rb +++ b/app/services/projects/transfer_service.rb @@ -36,7 +36,7 @@ module Projects raise TransferError.new("Project with same path in target namespace already exists") end - unless project.container_images.empty? + unless project.container_repositories.empty? # we currently doesn't support renaming repository if it contains images in container registry raise TransferError.new('Project cannot be transferred, because images are present in its container registry') end diff --git a/spec/features/container_registry_spec.rb b/spec/features/container_registry_spec.rb index 0199d08ab63..88642f72772 100644 --- a/spec/features/container_registry_spec.rb +++ b/spec/features/container_registry_spec.rb @@ -5,15 +5,15 @@ describe "Container Registry" do let(:registry) { project.container_registry } let(:tag_name) { 'latest' } let(:tags) { [tag_name] } - let(:container_image) { create(:container_image) } - let(:image_name) { container_image.name } + let(:container_repository) { create(:container_repository) } + let(:image_name) { container_repository.name } before do login_as(:user) project.team << [@user, :developer] stub_container_registry_config(enabled: true) stub_container_registry_tags(*tags) - project.container_images << container_image unless container_image.nil? + project.container_repositories << container_repository unless container_repository.nil? allow(Auth::ContainerRegistryAuthenticationService).to receive(:full_access_token).and_return('token') end @@ -23,7 +23,7 @@ describe "Container Registry" do end context 'when no images' do - let(:container_image) { } + let(:container_repository) { } it { expect(page).to have_content('No container images in Container Registry for this project') } end diff --git a/spec/features/security/project/internal_access_spec.rb b/spec/features/security/project/internal_access_spec.rb index 350de2e5b6b..a961d8b4f69 100644 --- a/spec/features/security/project/internal_access_spec.rb +++ b/spec/features/security/project/internal_access_spec.rb @@ -443,12 +443,12 @@ describe "Internal Project Access", feature: true do end describe "GET /:project_path/container_registry" do - let(:container_image) { create(:container_image) } + let(:container_repository) { create(:container_repository) } before do stub_container_registry_tags('latest') stub_container_registry_config(enabled: true) - project.container_images << container_image + project.container_repositories << container_repository end subject { namespace_project_container_registry_index_path(project.namespace, project) } diff --git a/spec/features/security/project/private_access_spec.rb b/spec/features/security/project/private_access_spec.rb index 62364206440..b7e42e67d82 100644 --- a/spec/features/security/project/private_access_spec.rb +++ b/spec/features/security/project/private_access_spec.rb @@ -432,12 +432,12 @@ describe "Private Project Access", feature: true do end describe "GET /:project_path/container_registry" do - let(:container_image) { create(:container_image) } + let(:container_repository) { create(:container_repository) } before do stub_container_registry_tags('latest') stub_container_registry_config(enabled: true) - project.container_images << container_image + project.container_repositories << container_repository end subject { namespace_project_container_registry_index_path(project.namespace, project) } diff --git a/spec/features/security/project/public_access_spec.rb b/spec/features/security/project/public_access_spec.rb index 0e0c3140fd0..02660984b29 100644 --- a/spec/features/security/project/public_access_spec.rb +++ b/spec/features/security/project/public_access_spec.rb @@ -443,12 +443,12 @@ describe "Public Project Access", feature: true do end describe "GET /:project_path/container_registry" do - let(:container_image) { create(:container_image) } + let(:container_repository) { create(:container_repository) } before do stub_container_registry_tags('latest') stub_container_registry_config(enabled: true) - project.container_images << container_image + project.container_repositories << container_repository end subject { namespace_project_container_registry_index_path(project.namespace, project) } diff --git a/spec/lib/container_registry/blob_spec.rb b/spec/lib/container_registry/blob_spec.rb index f092449c4bd..718a61ba291 100644 --- a/spec/lib/container_registry/blob_spec.rb +++ b/spec/lib/container_registry/blob_spec.rb @@ -15,7 +15,7 @@ describe ContainerRegistry::Blob do let(:project) { create(:project, path: 'test', group: group) } let(:example_host) { 'example.com' } let(:registry_url) { 'http://' + example_host } - let(:repository) { create(:container_image, name: '', project: project) } + let(:repository) { create(:container_repository, name: '', project: project) } let(:blob) { repository.blob(config) } before do diff --git a/spec/lib/container_registry/tag_spec.rb b/spec/lib/container_registry/tag_spec.rb index cdd0fe66bc3..01153a6eca9 100644 --- a/spec/lib/container_registry/tag_spec.rb +++ b/spec/lib/container_registry/tag_spec.rb @@ -5,7 +5,7 @@ describe ContainerRegistry::Tag do let(:project) { create(:project, path: 'test', group: group) } let(:example_host) { 'example.com' } let(:registry_url) { 'http://' + example_host } - let(:repository) { create(:container_image, name: '', project: project) } + let(:repository) { create(:container_repository, name: '', project: project) } let(:tag) { repository.tag('tag') } let(:headers) { { 'Accept' => 'application/vnd.docker.distribution.manifest.v2+json' } } diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index c3ee743035a..0429636486c 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -115,7 +115,7 @@ merge_access_levels: - protected_branch push_access_levels: - protected_branch -container_images: +container_repositories: - name project: - taggings diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb index d22447c602f..bc70c6f4aa2 100644 --- a/spec/models/namespace_spec.rb +++ b/spec/models/namespace_spec.rb @@ -149,13 +149,13 @@ describe Namespace, models: true do end context "when any project has container images" do - let(:container_image) { create(:container_image) } + let(:container_repository) { create(:container_repository) } before do stub_container_registry_config(enabled: true) stub_container_registry_tags('tag') - create(:empty_project, namespace: @namespace, container_images: [container_image]) + create(:empty_project, namespace: @namespace, container_repositories: [container_repository]) allow(@namespace).to receive(:path_was).and_return(@namespace.path) allow(@namespace).to receive(:path).and_return('new_path') diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index aefbedf0b93..69b7906bb4e 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -1186,12 +1186,12 @@ describe Project, models: true do end context 'container registry with images' do - let(:container_image) { create(:container_image) } + let(:container_repository) { create(:container_repository) } before do stub_container_registry_config(enabled: true) stub_container_registry_tags('tag') - project.container_images << container_image + project.container_repositories << container_repository end subject { project.rename_repo } diff --git a/spec/services/container_images/destroy_service_spec.rb b/spec/services/container_images/destroy_service_spec.rb index 5b4dbaa7934..ebc15598cce 100644 --- a/spec/services/container_images/destroy_service_spec.rb +++ b/spec/services/container_images/destroy_service_spec.rb @@ -3,13 +3,13 @@ require 'spec_helper' describe ContainerImages::DestroyService, services: true do describe '#execute' do let(:user) { create(:user) } - let(:container_image) { create(:container_image, name: '') } - let(:project) { create(:project, path: 'test', namespace: user.namespace, container_images: [container_image]) } + let(:container_repository) { create(:container_repository, name: '') } + let(:project) { create(:project, path: 'test', namespace: user.namespace, container_repositorys: [container_repository]) } let(:example_host) { 'example.com' } let(:registry_url) { 'http://' + example_host } - it { expect(container_image).to be_valid } - it { expect(project.container_images).not_to be_empty } + it { expect(container_repository).to be_valid } + it { expect(project.container_repositorys).not_to be_empty } context 'when container image has tags' do before do @@ -19,15 +19,15 @@ describe ContainerImages::DestroyService, services: true do it 'removes all tags before destroy' do service = described_class.new(project, user) - expect(container_image).to receive(:delete_tags).and_return(true) - expect { service.execute(container_image) }.to change(project.container_images, :count).by(-1) + expect(container_repository).to receive(:delete_tags).and_return(true) + expect { service.execute(container_repository) }.to change(project.container_repositorys, :count).by(-1) end it 'fails when tags are not removed' do service = described_class.new(project, user) - expect(container_image).to receive(:delete_tags).and_return(false) - expect { service.execute(container_image) }.to raise_error(ActiveRecord::RecordNotDestroyed) + expect(container_repository).to receive(:delete_tags).and_return(false) + expect { service.execute(container_repository) }.to raise_error(ActiveRecord::RecordNotDestroyed) end end end diff --git a/spec/services/projects/destroy_service_spec.rb b/spec/services/projects/destroy_service_spec.rb index f91d62ebdaf..daad65d478a 100644 --- a/spec/services/projects/destroy_service_spec.rb +++ b/spec/services/projects/destroy_service_spec.rb @@ -90,12 +90,12 @@ describe Projects::DestroyService, services: true do end context 'container registry' do - let(:container_image) { create(:container_image) } + let(:container_repository) { create(:container_repository) } before do stub_container_registry_config(enabled: true) stub_container_registry_tags('tag') - project.container_images << container_image + project.container_repositorys << container_repository end context 'images deletion succeeds' do diff --git a/spec/services/projects/transfer_service_spec.rb b/spec/services/projects/transfer_service_spec.rb index 5e56226ff91..adf8ede5086 100644 --- a/spec/services/projects/transfer_service_spec.rb +++ b/spec/services/projects/transfer_service_spec.rb @@ -29,12 +29,12 @@ describe Projects::TransferService, services: true do end context 'disallow transfering of project with tags' do - let(:container_image) { create(:container_image) } + let(:container_repository) { create(:container_repository) } before do stub_container_registry_config(enabled: true) stub_container_registry_tags('tag') - project.container_images << container_image + project.container_repositorys << container_repository end subject { transfer_project(project, user, group) } -- cgit v1.2.1 From af42dd29a0e81d524731f4ce3ced2ed17bac9903 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 24 Mar 2017 12:31:34 +0100 Subject: Fix specs for container repository tags --- app/models/container_repository.rb | 8 +++-- lib/container_registry/blob.rb | 4 +-- lib/container_registry/tag.rb | 6 ++-- spec/factories/container_repositories.rb | 23 ++++++------- spec/lib/container_registry/tag_spec.rb | 58 +++++++++++++++++++++++--------- 5 files changed, 64 insertions(+), 35 deletions(-) diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index 2f0fd3014a8..e5076f30c8e 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -1,8 +1,10 @@ class ContainerRepository < ActiveRecord::Base belongs_to :project - delegate :client, to: :registry + validates :manifest, presence: true - validates :name, presence: true + validates :name, length: { minimum: 0, allow_nil: false } + + delegate :client, to: :registry before_destroy :delete_tags def registry @@ -17,7 +19,7 @@ class ContainerRepository < ActiveRecord::Base end def path - @path ||= "#{project.full_path}/#{name}" + @path ||= [project.full_path, name].select(&:present?).join('/') end def tag(tag) diff --git a/lib/container_registry/blob.rb b/lib/container_registry/blob.rb index 8db8e483b1d..d5f85f9fcad 100644 --- a/lib/container_registry/blob.rb +++ b/lib/container_registry/blob.rb @@ -38,11 +38,11 @@ module ContainerRegistry end def delete - client.delete_blob(repository.name_with_namespace, digest) + client.delete_blob(repository.path, digest) end def data - @data ||= client.blob(repository.name_with_namespace, digest, type) + @data ||= client.blob(repository.path, digest, type) end end end diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb index 68dd87c979d..d653deb3bf1 100644 --- a/lib/container_registry/tag.rb +++ b/lib/container_registry/tag.rb @@ -22,7 +22,7 @@ module ContainerRegistry end def manifest - @manifest ||= client.repository_manifest(repository.name_with_namespace, name) + @manifest ||= client.repository_manifest(repository.path, name) end def path @@ -38,7 +38,7 @@ module ContainerRegistry def digest return @digest if defined?(@digest) - @digest = client.repository_tag_digest(repository.name_with_namespace, name) + @digest = client.repository_tag_digest(repository.path, name) end def config_blob @@ -80,7 +80,7 @@ module ContainerRegistry def delete return unless digest - client.delete_repository_tag(repository.name_with_namespace, digest) + client.delete_repository_tag(repository.path, digest) end end end diff --git a/spec/factories/container_repositories.rb b/spec/factories/container_repositories.rb index fbf6bf62dfd..295b3596ee9 100644 --- a/spec/factories/container_repositories.rb +++ b/spec/factories/container_repositories.rb @@ -1,22 +1,21 @@ FactoryGirl.define do factory :container_repository do - name "test_container_image" + name 'test_container_image' project transient do - tags ['tag'] + tags [] end - after(:build) do |image, evaluator| - # if evaluator.tags.to_a.any? - # allow(Gitlab.config.registry).to receive(:enabled).and_return(true) - # allow(Auth::ContainerRegistryAuthenticationService) - # .to receive(:full_access_token).and_return('token') - # allow(image.client).to receive(:repository_tags).and_return({ - # name: image.name_with_namespace, - # tags: evaluator.tags - # }) - # end + after(:build) do |repository, evaluator| + if evaluator.tags.any? + allow(repository.client) + .to receive(:repository_tags) + .and_return({ + name: repository.path, + tags: evaluator.tags + }) + end end end end diff --git a/spec/lib/container_registry/tag_spec.rb b/spec/lib/container_registry/tag_spec.rb index 01153a6eca9..37eaa10f4a4 100644 --- a/spec/lib/container_registry/tag_spec.rb +++ b/spec/lib/container_registry/tag_spec.rb @@ -3,30 +3,58 @@ require 'spec_helper' describe ContainerRegistry::Tag do let(:group) { create(:group, name: 'group') } let(:project) { create(:project, path: 'test', group: group) } - let(:example_host) { 'example.com' } - let(:registry_url) { 'http://' + example_host } - let(:repository) { create(:container_repository, name: '', project: project) } - let(:tag) { repository.tag('tag') } - let(:headers) { { 'Accept' => 'application/vnd.docker.distribution.manifest.v2+json' } } + + let(:repository) do + create(:container_repository, name: '', tags: %w[latest], project: project) + end + + # TODO, move stubs to helper with this header + let(:headers) do + { 'Accept' => 'application/vnd.docker.distribution.manifest.v2+json' } + end + + let(:tag) { described_class.new(repository, 'tag') } before do - stub_container_registry_config(enabled: true, api_url: registry_url, host_port: example_host) + stub_container_registry_config(enabled: true, + api_url: 'http://registry.gitlab', + host_port: 'registry.gitlab') end it { expect(tag).to respond_to(:repository) } it { expect(tag).to delegate_method(:registry).to(:repository) } it { expect(tag).to delegate_method(:client).to(:repository) } - context '#path' do - subject { tag.path } + describe '#path' do + context 'when tag belongs to zero-level repository' do + let(:repository) do + create(:container_repository, name: '', + tags: %w[rc1], + project: project) + end - it { is_expected.to eq('example.com/group/test:tag') } + it 'returns path to the image' do + expect(tag.path).to eq('group/test:tag') + end + end + + context 'when tag belongs to first-level repository' do + let(:repository) do + create(:container_repository, name: 'my_image', + tags: %w[latest], + project: project) + end + + it 'returns path to the image' do + expect(tag.path).to eq('group/test/my_image:tag') + end + end end context 'manifest processing' do context 'schema v1' do before do - stub_request(:get, 'http://example.com/v2/group/test/manifests/tag'). + stub_request(:get, 'http://registry.gitlab/v2/group/test/manifests/tag'). with(headers: headers). to_return( status: 200, @@ -63,7 +91,7 @@ describe ContainerRegistry::Tag do context 'schema v2' do before do - stub_request(:get, 'http://example.com/v2/group/test/manifests/tag'). + stub_request(:get, 'http://registry.gitlab/v2/group/test/manifests/tag'). with(headers: headers). to_return( status: 200, @@ -100,7 +128,7 @@ describe ContainerRegistry::Tag do context 'when locally stored' do before do - stub_request(:get, 'http://example.com/v2/group/test/blobs/sha256:d7a513a663c1a6dcdba9ed832ca53c02ac2af0c333322cd6ca92936d1d9917ac'). + stub_request(:get, 'http://registry.gitlab/v2/group/test/blobs/sha256:d7a513a663c1a6dcdba9ed832ca53c02ac2af0c333322cd6ca92936d1d9917ac'). with(headers: { 'Accept' => 'application/octet-stream' }). to_return( status: 200, @@ -112,7 +140,7 @@ describe ContainerRegistry::Tag do context 'when externally stored' do before do - stub_request(:get, 'http://example.com/v2/group/test/blobs/sha256:d7a513a663c1a6dcdba9ed832ca53c02ac2af0c333322cd6ca92936d1d9917ac'). + stub_request(:get, 'http://registry.gitlab/v2/group/test/blobs/sha256:d7a513a663c1a6dcdba9ed832ca53c02ac2af0c333322cd6ca92936d1d9917ac'). with(headers: { 'Accept' => 'application/octet-stream' }). to_return( status: 307, @@ -132,7 +160,7 @@ describe ContainerRegistry::Tag do context 'manifest digest' do before do - stub_request(:head, 'http://example.com/v2/group/test/manifests/tag'). + stub_request(:head, 'http://registry.gitlab/v2/group/test/manifests/tag'). with(headers: headers). to_return(status: 200, headers: { 'Docker-Content-Digest' => 'sha256:digest' }) end @@ -145,7 +173,7 @@ describe ContainerRegistry::Tag do context '#delete' do before do - stub_request(:delete, 'http://example.com/v2/group/test/manifests/sha256:digest'). + stub_request(:delete, 'http://registry.gitlab/v2/group/test/manifests/sha256:digest'). with(headers: headers). to_return(status: 200) end -- cgit v1.2.1 From 7db1f22673f1b6890b6ce4b9db9b367eae3988f0 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 24 Mar 2017 12:41:42 +0100 Subject: Fix specs for services related to container registry --- .../container_images/destroy_service_spec.rb | 66 ++++++++++++---------- spec/services/projects/destroy_service_spec.rb | 2 +- spec/services/projects/transfer_service_spec.rb | 2 +- 3 files changed, 39 insertions(+), 31 deletions(-) diff --git a/spec/services/container_images/destroy_service_spec.rb b/spec/services/container_images/destroy_service_spec.rb index ebc15598cce..ab5ebe5eac8 100644 --- a/spec/services/container_images/destroy_service_spec.rb +++ b/spec/services/container_images/destroy_service_spec.rb @@ -1,34 +1,42 @@ require 'spec_helper' -describe ContainerImages::DestroyService, services: true do - describe '#execute' do - let(:user) { create(:user) } - let(:container_repository) { create(:container_repository, name: '') } - let(:project) { create(:project, path: 'test', namespace: user.namespace, container_repositorys: [container_repository]) } - let(:example_host) { 'example.com' } - let(:registry_url) { 'http://' + example_host } - - it { expect(container_repository).to be_valid } - it { expect(project.container_repositorys).not_to be_empty } - - context 'when container image has tags' do - before do - project.team << [user, :master] - end - - it 'removes all tags before destroy' do - service = described_class.new(project, user) - - expect(container_repository).to receive(:delete_tags).and_return(true) - expect { service.execute(container_repository) }.to change(project.container_repositorys, :count).by(-1) - end - - it 'fails when tags are not removed' do - service = described_class.new(project, user) - - expect(container_repository).to receive(:delete_tags).and_return(false) - expect { service.execute(container_repository) }.to raise_error(ActiveRecord::RecordNotDestroyed) - end +describe ContainerImages::DestroyService, '#execute', :services do + let(:user) { create(:user) } + + let(:container_repository) do + create(:container_repository, name: 'myimage', tags: %w[latest]) + end + + let(:project) do + create(:project, path: 'test', + namespace: user.namespace, + container_repositories: [container_repository]) + end + + it { expect(container_repository).to be_valid } + it { expect(project.container_repositories).not_to be_empty } + + context 'when container image has tags' do + before do + project.add_master(user) + end + + it 'removes all tags before destroy' do + service = described_class.new(project, user) + + expect(container_repository) + .to receive(:delete_tags).and_return(true) + expect { service.execute(container_repository) } + .to change(project.container_repositories, :count).by(-1) + end + + it 'fails when tags are not removed' do + service = described_class.new(project, user) + + expect(container_repository) + .to receive(:delete_tags).and_return(false) + expect { service.execute(container_repository) } + .to raise_error(ActiveRecord::RecordNotDestroyed) end end end diff --git a/spec/services/projects/destroy_service_spec.rb b/spec/services/projects/destroy_service_spec.rb index daad65d478a..44e0286350b 100644 --- a/spec/services/projects/destroy_service_spec.rb +++ b/spec/services/projects/destroy_service_spec.rb @@ -95,7 +95,7 @@ describe Projects::DestroyService, services: true do before do stub_container_registry_config(enabled: true) stub_container_registry_tags('tag') - project.container_repositorys << container_repository + project.container_repositories << container_repository end context 'images deletion succeeds' do diff --git a/spec/services/projects/transfer_service_spec.rb b/spec/services/projects/transfer_service_spec.rb index adf8ede5086..a3babaf1e0b 100644 --- a/spec/services/projects/transfer_service_spec.rb +++ b/spec/services/projects/transfer_service_spec.rb @@ -34,7 +34,7 @@ describe Projects::TransferService, services: true do before do stub_container_registry_config(enabled: true) stub_container_registry_tags('tag') - project.container_repositorys << container_repository + project.container_repositories << container_repository end subject { transfer_project(project, user, group) } -- cgit v1.2.1 From 4fdcaca6a148475a18aa5931b62c4173301ded8c Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 24 Mar 2017 13:08:21 +0100 Subject: Fix container registry blob specs --- spec/lib/container_registry/blob_spec.rb | 114 ++++++++++++++++--------------- 1 file changed, 59 insertions(+), 55 deletions(-) diff --git a/spec/lib/container_registry/blob_spec.rb b/spec/lib/container_registry/blob_spec.rb index 718a61ba291..76ea29666ea 100644 --- a/spec/lib/container_registry/blob_spec.rb +++ b/spec/lib/container_registry/blob_spec.rb @@ -1,117 +1,121 @@ require 'spec_helper' describe ContainerRegistry::Blob do - let(:digest) { 'sha256:0123456789012345' } + let(:group) { create(:group, name: 'group') } + let(:project) { create(:project, path: 'test', group: group) } + + let(:repository) do + create(:container_repository, name: 'image', + tags: %w[latest rc1], + project: project) + end + let(:config) do - { - 'digest' => digest, + { 'digest' => 'sha256:0123456789012345', 'mediaType' => 'binary', - 'size' => 1000 - } + 'size' => 1000 } end - let(:token) { 'token' } - let(:group) { create(:group, name: 'group') } - let(:project) { create(:project, path: 'test', group: group) } - let(:example_host) { 'example.com' } - let(:registry_url) { 'http://' + example_host } - let(:repository) { create(:container_repository, name: '', project: project) } - let(:blob) { repository.blob(config) } + let(:blob) { described_class.new(repository, config) } before do - stub_container_registry_config(enabled: true, api_url: registry_url, host_port: example_host) + stub_container_registry_config(enabled: true, + api_url: 'http://registry.gitlab', + host_port: 'registry.gitlab') end it { expect(blob).to respond_to(:repository) } it { expect(blob).to delegate_method(:registry).to(:repository) } it { expect(blob).to delegate_method(:client).to(:repository) } - context '#path' do - subject { blob.path } - - it { is_expected.to eq('example.com/group/test@sha256:0123456789012345') } + describe '#path' do + it 'returns a valid path to the blob' do + expect(blob.path).to eq('group/test/image@sha256:0123456789012345') + end end - context '#digest' do - subject { blob.digest } - - it { is_expected.to eq(digest) } + describe '#digest' do + it 'return correct digest value' do + expect(blob.digest).to eq 'sha256:0123456789012345' + end end - context '#type' do - subject { blob.type } - - it { is_expected.to eq('binary') } + describe '#type' do + it 'returns a correct type' do + expect(blob.type).to eq 'binary' + end end - context '#revision' do - subject { blob.revision } - - it { is_expected.to eq('0123456789012345') } + describe '#revision' do + it 'returns a correct blob SHA' do + expect(blob.revision).to eq '0123456789012345' + end end - context '#short_revision' do - subject { blob.short_revision } - - it { is_expected.to eq('012345678') } + describe '#short_revision' do + it 'return a short SHA' do + expect(blob.short_revision).to eq '012345678' + end end - context '#delete' do + describe '#delete' do before do - stub_request(:delete, 'http://example.com/v2/group/test/blobs/sha256:0123456789012345'). - to_return(status: 200) + stub_request(:delete, 'http://registry.gitlab/v2/group/test/image/blobs/sha256:0123456789012345') + .to_return(status: 200) end - subject { blob.delete } - - it { is_expected.to be_truthy } + it 'returns true when blob has been successfuly deleted' do + expect(blob.delete).to be_truthy + end end - context '#data' do - let(:data) { '{"key":"value"}' } - - subject { blob.data } - + describe '#data' do context 'when locally stored' do before do - stub_request(:get, 'http://example.com/v2/group/test/blobs/sha256:0123456789012345'). + stub_request(:get, 'http://registry.gitlab/v2/group/test/image/blobs/sha256:0123456789012345'). to_return( status: 200, headers: { 'Content-Type' => 'application/json' }, - body: data) + body: '{"key":"value"}') end - it { is_expected.to eq(data) } + it 'returns a correct blob data' do + expect(blob.data).to eq '{"key":"value"}' + end end context 'when externally stored' do + let(:location) { 'http://external.com/blob/file' } + before do - stub_request(:get, 'http://example.com/v2/group/test/blobs/sha256:0123456789012345'). - with(headers: { 'Authorization' => "bearer #{token}" }). - to_return( + stub_request(:get, 'http://registry.gitlab/v2/group/test/image/blobs/sha256:0123456789012345') + .with(headers: { 'Authorization' => 'bearer token' }) + .to_return( status: 307, headers: { 'Location' => location }) end context 'for a valid address' do - let(:location) { 'http://external.com/blob/file' } - before do stub_request(:get, location). with(headers: { 'Authorization' => nil }). to_return( status: 200, headers: { 'Content-Type' => 'application/json' }, - body: data) + body: '{"key":"value"}') end - it { is_expected.to eq(data) } + it 'returns correct data' do + expect(blob.data).to eq '{"key":"value"}' + end end context 'for invalid file' do let(:location) { 'file:///etc/passwd' } - it { expect{ subject }.to raise_error(ArgumentError, 'invalid address') } + it 'raises an error' do + expect { blob.data }.to raise_error(ArgumentError, 'invalid address') + end end end end -- cgit v1.2.1 From 16e645c6563b81ce03e481b094abc2d331d05f36 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 24 Mar 2017 13:27:05 +0100 Subject: Remove container_registry method from project class --- app/models/project.rb | 14 -------------- spec/models/project_spec.rb | 29 +++-------------------------- 2 files changed, 3 insertions(+), 40 deletions(-) diff --git a/app/models/project.rb b/app/models/project.rb index 5c6672c95b3..a579ac7dc64 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -405,20 +405,6 @@ class Project < ActiveRecord::Base @repository ||= Repository.new(path_with_namespace, self) end - def container_registry - return unless Gitlab.config.registry.enabled - - @container_registry ||= begin - token = Auth::ContainerRegistryAuthenticationService.full_access_token(project) - - url = Gitlab.config.registry.api_url - host_port = Gitlab.config.registry.host_port - # TODO, move configuration vars into ContainerRegistry::Registry, clean - # this method up afterwards - ContainerRegistry::Registry.new(url, token: token, path: host_port) - end - end - def container_registry_url if Gitlab.config.registry.enabled "#{Gitlab.config.registry.host_port}/#{path_with_namespace.downcase}" diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 69b7906bb4e..e4e75cc6a09 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -1389,25 +1389,6 @@ describe Project, models: true do end end - describe '#container_registry_path_with_namespace' do - let(:project) { create(:empty_project, path: 'PROJECT') } - - subject { project.container_registry_path_with_namespace } - - it { is_expected.not_to eq(project.path_with_namespace) } - it { is_expected.to eq(project.path_with_namespace.downcase) } - end - - describe '#container_registry' do - let(:project) { create(:empty_project) } - - before { stub_container_registry_config(enabled: true) } - - subject { project.container_registry } - - it { is_expected.not_to be_nil } - end - describe '#container_registry_url' do let(:project) { create(:empty_project) } @@ -1417,10 +1398,8 @@ describe Project, models: true do context 'for enabled registry' do let(:registry_settings) do - { - enabled: true, - host_port: 'example.com', - } + { enabled: true, + host_port: 'example.com' } end it { is_expected.not_to be_nil } @@ -1428,9 +1407,7 @@ describe Project, models: true do context 'for disabled registry' do let(:registry_settings) do - { - enabled: false - } + { enabled: false } end it { is_expected.to be_nil } -- cgit v1.2.1 From d6f37a34c1c262f49a92f26dd187819419d56c2f Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 24 Mar 2017 13:27:18 +0100 Subject: Fix feature specs related to container registry --- app/controllers/projects/container_registry_controller.rb | 4 ++-- spec/lib/gitlab/import_export/all_models.yml | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/controllers/projects/container_registry_controller.rb b/app/controllers/projects/container_registry_controller.rb index 4981e57ed22..8929bd0aa55 100644 --- a/app/controllers/projects/container_registry_controller.rb +++ b/app/controllers/projects/container_registry_controller.rb @@ -5,7 +5,7 @@ class Projects::ContainerRegistryController < Projects::ApplicationController layout 'project' def index - @images = project.container_images + @images = project.container_repositories end def destroy @@ -44,7 +44,7 @@ class Projects::ContainerRegistryController < Projects::ApplicationController end def image - @image ||= project.container_images.find_by(id: params[:id]) + @image ||= project.container_repositories.find_by(id: params[:id]) end def tag diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index 0429636486c..e5bba29d85b 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -116,6 +116,7 @@ merge_access_levels: push_access_levels: - protected_branch container_repositories: +- project - name project: - taggings @@ -201,7 +202,7 @@ project: - project_authorizations - route - statistics -- container_images +- container_repositories - uploads award_emoji: - awardable -- cgit v1.2.1 From dce706bfcbddf7952e2c42c0c42825044cbb43a2 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 24 Mar 2017 13:47:29 +0100 Subject: Do not require a manifest for container repository Container repository can be empty - no tags or blogs is OK. --- app/models/container_repository.rb | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index e5076f30c8e..c8c56e69269 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -1,7 +1,6 @@ class ContainerRepository < ActiveRecord::Base belongs_to :project - validates :manifest, presence: true validates :name, length: { minimum: 0, allow_nil: false } delegate :client, to: :registry @@ -43,6 +42,8 @@ class ContainerRepository < ActiveRecord::Base ContainerRegistry::Blob.new(self, config) end + # TODO, add bang to this method + # def delete_tags return unless tags @@ -52,6 +53,14 @@ class ContainerRepository < ActiveRecord::Base end end + # TODO, specs needed + # + def empty? + tags.none? + end + + # TODO, we will return a new ContainerRepository object here + # def self.project_from_path(repository_path) return unless repository_path.include?('/') -- cgit v1.2.1 From 7ada193e0fd28b4a6eca1fda7dda6f0ebe6b2d72 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 24 Mar 2017 14:58:25 +0100 Subject: Fix specs for container repository destroy service --- spec/features/container_registry_spec.rb | 1 - spec/services/container_images/destroy_service_spec.rb | 4 ++++ spec/support/stub_gitlab_calls.rb | 3 ++- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/spec/features/container_registry_spec.rb b/spec/features/container_registry_spec.rb index 88642f72772..73b22fa1b7d 100644 --- a/spec/features/container_registry_spec.rb +++ b/spec/features/container_registry_spec.rb @@ -14,7 +14,6 @@ describe "Container Registry" do stub_container_registry_config(enabled: true) stub_container_registry_tags(*tags) project.container_repositories << container_repository unless container_repository.nil? - allow(Auth::ContainerRegistryAuthenticationService).to receive(:full_access_token).and_return('token') end describe 'GET /:project/container_registry' do diff --git a/spec/services/container_images/destroy_service_spec.rb b/spec/services/container_images/destroy_service_spec.rb index ab5ebe5eac8..6931b503e6d 100644 --- a/spec/services/container_images/destroy_service_spec.rb +++ b/spec/services/container_images/destroy_service_spec.rb @@ -13,6 +13,10 @@ describe ContainerImages::DestroyService, '#execute', :services do container_repositories: [container_repository]) end + before do + stub_container_registry_config(enabled: true) + end + it { expect(container_repository).to be_valid } it { expect(project.container_repositories).not_to be_empty } diff --git a/spec/support/stub_gitlab_calls.rb b/spec/support/stub_gitlab_calls.rb index a01ef576234..3949784aabb 100644 --- a/spec/support/stub_gitlab_calls.rb +++ b/spec/support/stub_gitlab_calls.rb @@ -27,7 +27,8 @@ module StubGitlabCalls def stub_container_registry_config(registry_settings) allow(Gitlab.config.registry).to receive_messages(registry_settings) - allow(Auth::ContainerRegistryAuthenticationService).to receive(:full_access_token).and_return('token') + allow(Auth::ContainerRegistryAuthenticationService) + .to receive(:full_access_token).and_return('token') end def stub_container_registry_tags(*tags) -- cgit v1.2.1 From f09e3fe85116743c0cb537fb958a8b0ab1ad19fb Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 27 Mar 2017 16:04:38 +0200 Subject: Add a few pending specs for container repository --- spec/models/container_repository_spec.rb | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/spec/models/container_repository_spec.rb b/spec/models/container_repository_spec.rb index e3180e01758..296b9e713a8 100644 --- a/spec/models/container_repository_spec.rb +++ b/spec/models/container_repository_spec.rb @@ -92,4 +92,29 @@ describe ContainerRepository do end end end + + describe '#from_repository_path' do + context 'when received multi-level repository path' do + let(:repository) do + described_class.from_repository_path('group/test/some/image/name') + end + + pending 'fabricates object within a correct project' do + expect(repository.project).to eq project + end + + pending 'it fabricates project with a correct name' do + expect(repository.name).to eq 'some/image/name' + end + end + + context 'when path contains too many nodes' do + end + + context 'when received multi-level repository with nested groups' do + end + + context 'when received root repository path' do + end + end end -- cgit v1.2.1 From b15d9042e2688f29b002f90e0154e793ff1544ff Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 28 Mar 2017 14:34:56 +0200 Subject: Implement container repository path class --- lib/container_registry/path.rb | 29 ++++++++ spec/lib/container_registry/path_spec.rb | 119 +++++++++++++++++++++++++++++++ 2 files changed, 148 insertions(+) create mode 100644 lib/container_registry/path.rb create mode 100644 spec/lib/container_registry/path_spec.rb diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb new file mode 100644 index 00000000000..f32df1bc0d1 --- /dev/null +++ b/lib/container_registry/path.rb @@ -0,0 +1,29 @@ +module ContainerRegistry + class Path + InvalidRegistryPathError = Class.new(StandardError) + + def initialize(name) + @nodes = name.to_s.split('/') + end + + def valid? + @nodes.size > 1 && + @nodes.size < Namespace::NUMBER_OF_ANCESTORS_ALLOWED + end + + def components + raise InvalidRegistryPathError unless valid? + + @components ||= @nodes.size.downto(2).map do |length| + @nodes.take(length).join('/') + end + end + + def repository_project + @project ||= Project.where_full_path_in(components.first(3))&.first + end + + def repository_name + end + end +end diff --git a/spec/lib/container_registry/path_spec.rb b/spec/lib/container_registry/path_spec.rb new file mode 100644 index 00000000000..32f25f5e527 --- /dev/null +++ b/spec/lib/container_registry/path_spec.rb @@ -0,0 +1,119 @@ +require 'spec_helper' + +describe ContainerRegistry::Path do + let(:path) { described_class.new(name) } + + describe '#components' do + context 'when repository path is valid' do + let(:name) { 'path/to/some/project' } + + it 'return all project-like components in reverse order' do + expect(path.components).to eq %w[path/to/some/project + path/to/some + path/to] + end + end + + context 'when repository path is invalid' do + let(:name) { '' } + + it 'rasises en error' do + expect { path.components } + .to raise_error described_class::InvalidRegistryPathError + end + end + end + + describe '#valid?' do + context 'when path has less than two components' do + let(:name) { 'something/' } + + it 'is not valid' do + expect(path).not_to be_valid + end + end + + context 'when path has more than allowed number of components' do + let(:name) { 'a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/r/s/t/u/w/y/z' } + + it 'is not valid' do + expect(path).not_to be_valid + end + end + + context 'when path has two or more components' do + let(:name) { 'some/path' } + + it 'is valid' do + expect(path).to be_valid + end + end + end + + describe '#repository_project' do + let(:group) { create(:group, path: 'some_group') } + + context 'when project for given path exists' do + let(:name) { 'some_group/some_project' } + + before do + create(:empty_project, group: group, name: 'some_project') + create(:empty_project, name: 'some_project') + end + + it 'returns a correct project' do + expect(path.repository_project.group).to eq group + end + end + + context 'when project for given path does not exist' do + let(:name) { 'not/matching' } + + it 'returns nil' do + expect(path.repository_project).to be_nil + end + end + + context 'when matching multi-level path' do + let(:project) do + create(:empty_project, group: group, name: 'some_project') + end + + context 'when using the zero-level path' do + let(:name) { project.full_path } + + it 'supports zero-level path' do + expect(path.repository_project).to eq project + end + end + + context 'when using first-level path' do + let(:name) { "#{project.full_path}/repository" } + + it 'supports first-level path' do + expect(path.repository_project).to eq project + end + end + + context 'when using second-level path' do + let(:name) { "#{project.full_path}/repository/name" } + + it 'supports second-level path' do + expect(path.repository_project).to eq project + end + end + + context 'when using too deep nesting in the path' do + let(:name) { "#{project.full_path}/repository/name/invalid" } + + it 'does not support three-levels of nesting' do + expect(path.repository_project).to be_nil + end + end + end + end + + describe '#repository_name' do + pending 'returns a correct name' + end +end -- cgit v1.2.1 From bdc1e1b9e005eeaf564b79698d61a801c3c6360f Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 28 Mar 2017 14:57:22 +0200 Subject: Implement method matching container repository names --- lib/container_registry/path.rb | 8 ++++++ spec/lib/container_registry/path_spec.rb | 45 +++++++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb index f32df1bc0d1..89ef396f374 100644 --- a/lib/container_registry/path.rb +++ b/lib/container_registry/path.rb @@ -3,6 +3,7 @@ module ContainerRegistry InvalidRegistryPathError = Class.new(StandardError) def initialize(name) + @name = name @nodes = name.to_s.split('/') end @@ -19,11 +20,18 @@ module ContainerRegistry end end + def has_repository? + # ContainerRepository.find_by_full_path(@name).present? + end + def repository_project @project ||= Project.where_full_path_in(components.first(3))&.first end def repository_name + return unless repository_project + + @name.remove(%r(^?#{Regexp.escape(repository_project.full_path)}/?)) end end end diff --git a/spec/lib/container_registry/path_spec.rb b/spec/lib/container_registry/path_spec.rb index 32f25f5e527..278b1fc1b55 100644 --- a/spec/lib/container_registry/path_spec.rb +++ b/spec/lib/container_registry/path_spec.rb @@ -114,6 +114,49 @@ describe ContainerRegistry::Path do end describe '#repository_name' do - pending 'returns a correct name' + context 'when project does not exist' do + let(:name) { 'some/name' } + + it 'returns nil' do + expect(path.repository_name).to be_nil + end + end + + context 'when project exists' do + let(:group) { create(:group, path: 'some_group') } + + let(:project) do + create(:empty_project, group: group, name: 'some_project') + end + + before do + allow(path).to receive(:repository_project) + .and_return(project) + end + + context 'when project path equal repository path' do + let(:name) { 'some_group/some_project' } + + it 'returns an empty string' do + expect(path.repository_name).to eq '' + end + end + + context 'when repository path has one additional level' do + let(:name) { 'some_group/some_project/repository' } + + it 'returns a correct repository name' do + expect(path.repository_name).to eq 'repository' + end + end + + context 'when repository path has two additional levels' do + let(:name) { 'some_group/some_project/repository/image' } + + it 'returns a correct repository name' do + expect(path.repository_name).to eq 'repository/image' + end + end + end end end -- cgit v1.2.1 From 8584798886a8c7d0077157c29e0dc05087656aaf Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 28 Mar 2017 16:20:16 +0200 Subject: Fix rubocop offense in container registry path class --- lib/container_registry/path.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb index 89ef396f374..0ca51ab3766 100644 --- a/lib/container_registry/path.rb +++ b/lib/container_registry/path.rb @@ -8,8 +8,7 @@ module ContainerRegistry end def valid? - @nodes.size > 1 && - @nodes.size < Namespace::NUMBER_OF_ANCESTORS_ALLOWED + @nodes.size > 1 && @nodes.size < Namespace::NUMBER_OF_ANCESTORS_ALLOWED end def components -- cgit v1.2.1 From 95faf5f5b7268ea1750f3a764cd0537b3e0d1e25 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 29 Mar 2017 12:14:06 +0200 Subject: Use new registry path class to match repository project --- app/models/container_repository.rb | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index c8c56e69269..149d65ddbff 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -62,19 +62,7 @@ class ContainerRepository < ActiveRecord::Base # TODO, we will return a new ContainerRepository object here # def self.project_from_path(repository_path) - return unless repository_path.include?('/') - - ## - # Projects are always located inside a namespace, so we can remove - # the last node, and see if project with that path exists. - # - truncated_path = repository_path.slice(0...repository_path.rindex('/')) - - ## - # We still make it possible to search projects by a full image path - # in order to maintain backwards compatibility. - # - Project.find_by_full_path(truncated_path) || - Project.find_by_full_path(repository_path) + ContainerRegistry::Path.new(repository_path) + .repository_project end end -- cgit v1.2.1 From a222486c3e06d02e93704aa8440299cb4a677cef Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 29 Mar 2017 12:14:29 +0200 Subject: Rename method for checking tags in container repository This is important because method `empty?` is triggered when validation happens, and we don't want to make API request to registry when record is validated. --- app/models/container_repository.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index 149d65ddbff..b128069ca0e 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -42,6 +42,12 @@ class ContainerRepository < ActiveRecord::Base ContainerRegistry::Blob.new(self, config) end + # TODO, specs needed + # + def has_tags? + tags.any? + end + # TODO, add bang to this method # def delete_tags @@ -53,12 +59,6 @@ class ContainerRepository < ActiveRecord::Base end end - # TODO, specs needed - # - def empty? - tags.none? - end - # TODO, we will return a new ContainerRepository object here # def self.project_from_path(repository_path) -- cgit v1.2.1 From 5a7f8cb5d2f9ab7bc7172cedeb220b7d78530f78 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 29 Mar 2017 12:30:38 +0200 Subject: Add readability improvements to registry auth specs --- ...ntainer_registry_authentication_service_spec.rb | 33 ++++++++++------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/spec/services/auth/container_registry_authentication_service_spec.rb b/spec/services/auth/container_registry_authentication_service_spec.rb index b91234ddb1e..fac7f1b1235 100644 --- a/spec/services/auth/container_registry_authentication_service_spec.rb +++ b/spec/services/auth/container_registry_authentication_service_spec.rb @@ -6,14 +6,15 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do let(:current_params) { {} } let(:rsa_key) { OpenSSL::PKey::RSA.generate(512) } let(:payload) { JWT.decode(subject[:token], rsa_key).first } + let(:authentication_abilities) do - [ - :read_container_image, - :create_container_image - ] + [:read_container_image, :create_container_image] end - subject { described_class.new(current_project, current_user, current_params).execute(authentication_abilities: authentication_abilities) } + subject do + described_class.new(current_project, current_user, current_params) + .execute(authentication_abilities: authentication_abilities) + end before do allow(Gitlab.config.registry).to receive_messages(enabled: true, issuer: 'rspec', key: nil) @@ -40,13 +41,11 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end end - shared_examples 'a accessible' do + shared_examples 'an accessible' do let(:access) do - [{ - 'type' => 'repository', + [{ 'type' => 'repository', 'name' => project.path_with_namespace, - 'actions' => actions, - }] + 'actions' => actions }] end it_behaves_like 'a valid token' @@ -59,19 +58,19 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end shared_examples 'a pullable' do - it_behaves_like 'a accessible' do + it_behaves_like 'an accessible' do let(:actions) { ['pull'] } end end shared_examples 'a pushable' do - it_behaves_like 'a accessible' do + it_behaves_like 'an accessible' do let(:actions) { ['push'] } end end shared_examples 'a pullable and pushable' do - it_behaves_like 'a accessible' do + it_behaves_like 'an accessible' do let(:actions) { %w(pull push) } end end @@ -87,7 +86,7 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do subject { { token: token } } - it_behaves_like 'a accessible' do + it_behaves_like 'an accessible' do let(:actions) { ['*'] } end end @@ -198,11 +197,9 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do context 'build authorized as user' do let(:current_project) { create(:empty_project) } let(:current_user) { create(:user) } + let(:authentication_abilities) do - [ - :build_read_container_image, - :build_create_container_image - ] + [:build_read_container_image, :build_create_container_image] end before do -- cgit v1.2.1 From 06bae00365cda6930063b98fde1a9b804f2ae3bc Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 29 Mar 2017 12:53:02 +0200 Subject: Make container repository path code more readable --- lib/container_registry/path.rb | 24 ++++++++--- spec/lib/container_registry/path_spec.rb | 70 ++++++++++++++++++-------------- 2 files changed, 58 insertions(+), 36 deletions(-) diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb index 0ca51ab3766..d3f8bf74e14 100644 --- a/lib/container_registry/path.rb +++ b/lib/container_registry/path.rb @@ -1,10 +1,24 @@ module ContainerRegistry + ## + # Class reponsible for extracting project and repository name from + # image repository path provided by a containers registry API response. + # + # Example: + # + # some/group/my_project/my/image -> + # project: some/group/my_project + # repository: my/image + # class Path InvalidRegistryPathError = Class.new(StandardError) - def initialize(name) - @name = name - @nodes = name.to_s.split('/') + def initialize(path) + @path = path + @nodes = path.to_s.split('/') + end + + def to_s + @path end def valid? @@ -20,7 +34,7 @@ module ContainerRegistry end def has_repository? - # ContainerRepository.find_by_full_path(@name).present? + # ContainerRepository.find_by_full_path(@path).present? end def repository_project @@ -30,7 +44,7 @@ module ContainerRegistry def repository_name return unless repository_project - @name.remove(%r(^?#{Regexp.escape(repository_project.full_path)}/?)) + @path.remove(%r(^?#{Regexp.escape(repository_project.full_path)}/?)) end end end diff --git a/spec/lib/container_registry/path_spec.rb b/spec/lib/container_registry/path_spec.rb index 278b1fc1b55..a680a0adcb2 100644 --- a/spec/lib/container_registry/path_spec.rb +++ b/spec/lib/container_registry/path_spec.rb @@ -1,51 +1,59 @@ require 'spec_helper' describe ContainerRegistry::Path do - let(:path) { described_class.new(name) } + subject { described_class.new(path) } describe '#components' do context 'when repository path is valid' do - let(:name) { 'path/to/some/project' } + let(:path) { 'path/to/some/project' } it 'return all project-like components in reverse order' do - expect(path.components).to eq %w[path/to/some/project + expect(subject.components).to eq %w[path/to/some/project path/to/some path/to] end end context 'when repository path is invalid' do - let(:name) { '' } + let(:path) { '' } it 'rasises en error' do - expect { path.components } + expect { subject.components } .to raise_error described_class::InvalidRegistryPathError end end end + describe '#to_s' do + let(:path) { 'some/image' } + + it 'return a string with a repository path' do + expect(subject.to_s).to eq path + end + end + describe '#valid?' do context 'when path has less than two components' do - let(:name) { 'something/' } + let(:path) { 'something/' } it 'is not valid' do - expect(path).not_to be_valid + expect(subject).not_to be_valid end end context 'when path has more than allowed number of components' do - let(:name) { 'a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/r/s/t/u/w/y/z' } + let(:path) { 'a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/r/s/t/u/w/y/z' } it 'is not valid' do - expect(path).not_to be_valid + expect(subject).not_to be_valid end end context 'when path has two or more components' do - let(:name) { 'some/path' } + let(:path) { 'some/path' } it 'is valid' do - expect(path).to be_valid + expect(subject).to be_valid end end end @@ -54,7 +62,7 @@ describe ContainerRegistry::Path do let(:group) { create(:group, path: 'some_group') } context 'when project for given path exists' do - let(:name) { 'some_group/some_project' } + let(:path) { 'some_group/some_project' } before do create(:empty_project, group: group, name: 'some_project') @@ -62,15 +70,15 @@ describe ContainerRegistry::Path do end it 'returns a correct project' do - expect(path.repository_project.group).to eq group + expect(subject.repository_project.group).to eq group end end context 'when project for given path does not exist' do - let(:name) { 'not/matching' } + let(:path) { 'not/matching' } it 'returns nil' do - expect(path.repository_project).to be_nil + expect(subject.repository_project).to be_nil end end @@ -80,34 +88,34 @@ describe ContainerRegistry::Path do end context 'when using the zero-level path' do - let(:name) { project.full_path } + let(:path) { project.full_path } it 'supports zero-level path' do - expect(path.repository_project).to eq project + expect(subject.repository_project).to eq project end end context 'when using first-level path' do - let(:name) { "#{project.full_path}/repository" } + let(:path) { "#{project.full_path}/repository" } it 'supports first-level path' do - expect(path.repository_project).to eq project + expect(subject.repository_project).to eq project end end context 'when using second-level path' do - let(:name) { "#{project.full_path}/repository/name" } + let(:path) { "#{project.full_path}/repository/name" } it 'supports second-level path' do - expect(path.repository_project).to eq project + expect(subject.repository_project).to eq project end end context 'when using too deep nesting in the path' do - let(:name) { "#{project.full_path}/repository/name/invalid" } + let(:path) { "#{project.full_path}/repository/name/invalid" } it 'does not support three-levels of nesting' do - expect(path.repository_project).to be_nil + expect(subject.repository_project).to be_nil end end end @@ -115,10 +123,10 @@ describe ContainerRegistry::Path do describe '#repository_name' do context 'when project does not exist' do - let(:name) { 'some/name' } + let(:path) { 'some/name' } it 'returns nil' do - expect(path.repository_name).to be_nil + expect(subject.repository_name).to be_nil end end @@ -135,26 +143,26 @@ describe ContainerRegistry::Path do end context 'when project path equal repository path' do - let(:name) { 'some_group/some_project' } + let(:path) { 'some_group/some_project' } it 'returns an empty string' do - expect(path.repository_name).to eq '' + expect(subject.repository_name).to eq '' end end context 'when repository path has one additional level' do - let(:name) { 'some_group/some_project/repository' } + let(:path) { 'some_group/some_project/repository' } it 'returns a correct repository name' do - expect(path.repository_name).to eq 'repository' + expect(subject.repository_name).to eq 'repository' end end context 'when repository path has two additional levels' do - let(:name) { 'some_group/some_project/repository/image' } + let(:path) { 'some_group/some_project/repository/image' } it 'returns a correct repository name' do - expect(path.repository_name).to eq 'repository/image' + expect(subject.repository_name).to eq 'repository/image' end end end -- cgit v1.2.1 From 3bfc05be5ee5b0262857febf90fc7e1f17895d4e Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 29 Mar 2017 13:01:48 +0200 Subject: Use container repository path inside auth service --- app/models/container_repository.rb | 7 ------- .../auth/container_registry_authentication_service.rb | 12 ++++++++---- spec/lib/container_registry/path_spec.rb | 4 ++-- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index b128069ca0e..98acf49c939 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -58,11 +58,4 @@ class ContainerRepository < ActiveRecord::Base client.delete_repository_tag(self.path, digest) end end - - # TODO, we will return a new ContainerRepository object here - # - def self.project_from_path(repository_path) - ContainerRegistry::Path.new(repository_path) - .repository_project - end end diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb index 3d151c6a357..7a2ec9664c1 100644 --- a/app/services/auth/container_registry_authentication_service.rb +++ b/app/services/auth/container_registry_authentication_service.rb @@ -56,13 +56,15 @@ module Auth def process_scope(scope) type, name, actions = scope.split(':', 3) actions = actions.split(',') + path = ContainerRegistry::Path.new(name) + return unless type == 'repository' - process_repository_access(type, name, actions) + process_repository_access(type, path, actions) end - def process_repository_access(type, name, actions) - requested_project = ContainerRepository.project_from_path(name) + def process_repository_access(type, path, actions) + requested_project = path.repository_project return unless requested_project @@ -70,7 +72,9 @@ module Auth can_access?(requested_project, action) end - { type: type, name: name, actions: actions } if actions.present? + return unless actions.present? + + { type: type, name: path.to_s, actions: actions } end def can_access?(requested_project, requested_action) diff --git a/spec/lib/container_registry/path_spec.rb b/spec/lib/container_registry/path_spec.rb index a680a0adcb2..6384850eb19 100644 --- a/spec/lib/container_registry/path_spec.rb +++ b/spec/lib/container_registry/path_spec.rb @@ -9,8 +9,8 @@ describe ContainerRegistry::Path do it 'return all project-like components in reverse order' do expect(subject.components).to eq %w[path/to/some/project - path/to/some - path/to] + path/to/some + path/to] end end -- cgit v1.2.1 From 313e35e817445271b68d40ab06f192f1d3e0ccf0 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 29 Mar 2017 13:14:06 +0200 Subject: Remove container images destroy service class --- app/services/container_images/destroy_service.rb | 9 ----- .../container_images/destroy_service_spec.rb | 46 ---------------------- 2 files changed, 55 deletions(-) delete mode 100644 app/services/container_images/destroy_service.rb delete mode 100644 spec/services/container_images/destroy_service_spec.rb diff --git a/app/services/container_images/destroy_service.rb b/app/services/container_images/destroy_service.rb deleted file mode 100644 index 15dca227291..00000000000 --- a/app/services/container_images/destroy_service.rb +++ /dev/null @@ -1,9 +0,0 @@ -module ContainerImages - class DestroyService < BaseService - def execute(container_image) - return false unless can?(current_user, :update_container_image, project) - - container_image.destroy! - end - end -end diff --git a/spec/services/container_images/destroy_service_spec.rb b/spec/services/container_images/destroy_service_spec.rb deleted file mode 100644 index 6931b503e6d..00000000000 --- a/spec/services/container_images/destroy_service_spec.rb +++ /dev/null @@ -1,46 +0,0 @@ -require 'spec_helper' - -describe ContainerImages::DestroyService, '#execute', :services do - let(:user) { create(:user) } - - let(:container_repository) do - create(:container_repository, name: 'myimage', tags: %w[latest]) - end - - let(:project) do - create(:project, path: 'test', - namespace: user.namespace, - container_repositories: [container_repository]) - end - - before do - stub_container_registry_config(enabled: true) - end - - it { expect(container_repository).to be_valid } - it { expect(project.container_repositories).not_to be_empty } - - context 'when container image has tags' do - before do - project.add_master(user) - end - - it 'removes all tags before destroy' do - service = described_class.new(project, user) - - expect(container_repository) - .to receive(:delete_tags).and_return(true) - expect { service.execute(container_repository) } - .to change(project.container_repositories, :count).by(-1) - end - - it 'fails when tags are not removed' do - service = described_class.new(project, user) - - expect(container_repository) - .to receive(:delete_tags).and_return(false) - expect { service.execute(container_repository) } - .to raise_error(ActiveRecord::RecordNotDestroyed) - end - end -end -- cgit v1.2.1 From 4407d3cf19bc815142ea3a7908003e85efde76ed Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 29 Mar 2017 14:01:05 +0200 Subject: Add comment to container registry auth service Comment explains why we still have authentication without user object there. The legacy authentication mechanism should be removed in 10.0. --- .../auth/container_registry_authentication_service.rb | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb index 7a2ec9664c1..b050f1dd51b 100644 --- a/app/services/auth/container_registry_authentication_service.rb +++ b/app/services/auth/container_registry_authentication_service.rb @@ -107,6 +107,11 @@ module Auth can?(current_user, :read_container_image, requested_project) end + ## + # We still support legacy pipeline triggers which do not have associated + # actor. New permissions model and new triggers are always associated with + # an actor, so this should be improved in 10.0 version of GitLab. + # def build_can_push?(requested_project) # Build can push only to the project from which it originates has_authentication_ability?(:build_create_container_image) && @@ -119,14 +124,11 @@ module Auth end def error(code, status:, message: '') - { - errors: [{ code: code, message: message }], - http_status: status - } + { errors: [{ code: code, message: message }], http_status: status } end def has_authentication_ability?(capability) - (@authentication_abilities || []).include?(capability) + @authentication_abilities.to_a.include?(capability) end end end -- cgit v1.2.1 From 031122eb54390b4ed792289237ecfb156ec69002 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 30 Mar 2017 13:31:33 +0200 Subject: Add container repository create service with specs --- app/models/container_repository.rb | 4 ++ .../create_repository_service.rb | 33 ++++++++++ .../create_repository_service_spec.rb | 74 ++++++++++++++++++++++ 3 files changed, 111 insertions(+) create mode 100644 app/services/container_registry/create_repository_service.rb create mode 100644 spec/services/container_registry/create_repository_service_spec.rb diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index 98acf49c939..33e2574d389 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -58,4 +58,8 @@ class ContainerRepository < ActiveRecord::Base client.delete_repository_tag(self.path, digest) end end + + def self.create_from_path(path) + self.create(project: path.repository_project, name: path.repository_name) + end end diff --git a/app/services/container_registry/create_repository_service.rb b/app/services/container_registry/create_repository_service.rb new file mode 100644 index 00000000000..84218702a4f --- /dev/null +++ b/app/services/container_registry/create_repository_service.rb @@ -0,0 +1,33 @@ +module ContainerRegistry + ## + # Service for creating a container repository. + # + # It is usually executed before registry authenticator returns + # a token for given request. + # + class CreateRepositoryService < BaseService + def execute(path) + @path = path + + return if path.has_repository? + + unless user_can_create? || legacy_trigger_can_create? + raise Gitlab::Access::AccessDeniedError + end + + ContainerRepository.create_from_path(path) + end + + private + + def user_can_create? + can?(@current_user, :create_container_image, @path.repository_project) + end + + ## TODO, remove it after removing legacy triggers. + # + def legacy_trigger_can_create? + @current_user.nil? && @project == @path.repository_project + end + end +end diff --git a/spec/services/container_registry/create_repository_service_spec.rb b/spec/services/container_registry/create_repository_service_spec.rb new file mode 100644 index 00000000000..dfd07c8cc02 --- /dev/null +++ b/spec/services/container_registry/create_repository_service_spec.rb @@ -0,0 +1,74 @@ +require 'spec_helper' + +describe ContainerRegistry::CreateRepositoryService, '#execute' do + let(:project) { create(:empty_project) } + let(:user) { create(:user) } + + let(:path) do + ContainerRegistry::Path.new("#{project.full_path}/my/image") + end + + let(:service) { described_class.new(project, user) } + + before do + stub_container_registry_config(enabled: true) + end + + context 'when container repository already exists' do + before do + create(:container_repository, project: project, name: 'my/image') + end + + it 'does not create container repository again' do + expect { service.execute(path) } + .to raise_error(Gitlab::Access::AccessDeniedError) + .and change { ContainerRepository.count }.by(0) + end + end + + context 'when repository is created by an user' do + context 'when user has no ability to create a repository' do + it 'does not create a new container repository' do + expect { service.execute(path) } + .to raise_error(Gitlab::Access::AccessDeniedError) + .and change { ContainerRepository.count }.by(0) + end + end + + context 'when user has ability do create a repository' do + before do + project.add_developer(user) + end + + it 'creates a new container repository' do + expect { service.execute(path) } + .to change { project.container_repositories.count }.by(1) + end + end + end + + context 'when repository is created by a legacy pipeline trigger' do + let(:user) { nil } + + context 'when repository path matches authenticated project' do + it 'creates a new container repository' do + expect { service.execute(path) } + .to change { project.container_repositories.count }.by(1) + end + end + + context 'when repository path does not match authenticated project' do + let(:private_project) { create(:empty_project, :private) } + + let(:path) do + ContainerRegistry::Path.new("#{private_project.full_path}/my/image") + end + + it 'does not create a new container repository' do + expect { service.execute(path) } + .to raise_error(Gitlab::Access::AccessDeniedError) + .and change { ContainerRepository.count }.by(0) + end + end + end +end -- cgit v1.2.1 From 003a51d17aea6af92953f7207e2b39a5cb6db8de Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 30 Mar 2017 13:45:54 +0200 Subject: Check container repository exists for a given path --- lib/container_registry/path.rb | 11 ++++++-- spec/lib/container_registry/path_spec.rb | 30 ++++++++++++++++++++- .../create_repository_service_spec.rb | 31 +++++++++++----------- 3 files changed, 54 insertions(+), 18 deletions(-) diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb index d3f8bf74e14..3a6fde08e8f 100644 --- a/lib/container_registry/path.rb +++ b/lib/container_registry/path.rb @@ -33,8 +33,15 @@ module ContainerRegistry end end + def has_project? + repository_project.present? + end + def has_repository? - # ContainerRepository.find_by_full_path(@path).present? + return false unless has_project? + + repository_project.container_repositories + .where(name: repository_name).any? end def repository_project @@ -42,7 +49,7 @@ module ContainerRegistry end def repository_name - return unless repository_project + return unless has_project? @path.remove(%r(^?#{Regexp.escape(repository_project.full_path)}/?)) end diff --git a/spec/lib/container_registry/path_spec.rb b/spec/lib/container_registry/path_spec.rb index 6384850eb19..906dd920031 100644 --- a/spec/lib/container_registry/path_spec.rb +++ b/spec/lib/container_registry/path_spec.rb @@ -39,7 +39,7 @@ describe ContainerRegistry::Path do it 'is not valid' do expect(subject).not_to be_valid end - end + end context 'when path has more than allowed number of components' do let(:path) { 'a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/r/s/t/u/w/y/z' } @@ -58,6 +58,34 @@ describe ContainerRegistry::Path do end end + describe '#has_repository?' do + context 'when project exists' do + let(:project) { create(:empty_project) } + let(:path) { "#{project.full_path}/my/image" } + + context 'when path already has matching repository' do + before do + create(:container_repository, project: project, name: 'my/image') + end + + it { is_expected.to have_repository } + it { is_expected.to have_project } + end + + context 'when path does not have matching repository' do + it { is_expected.not_to have_repository } + it { is_expected.to have_project } + end + end + + context 'when project does not exist' do + let(:path) { 'some/project/my/image' } + + it { is_expected.not_to have_repository } + it { is_expected.not_to have_project } + end + end + describe '#repository_project' do let(:group) { create(:group, path: 'some_group') } diff --git a/spec/services/container_registry/create_repository_service_spec.rb b/spec/services/container_registry/create_repository_service_spec.rb index dfd07c8cc02..ac7b147f92e 100644 --- a/spec/services/container_registry/create_repository_service_spec.rb +++ b/spec/services/container_registry/create_repository_service_spec.rb @@ -14,18 +14,6 @@ describe ContainerRegistry::CreateRepositoryService, '#execute' do stub_container_registry_config(enabled: true) end - context 'when container repository already exists' do - before do - create(:container_repository, project: project, name: 'my/image') - end - - it 'does not create container repository again' do - expect { service.execute(path) } - .to raise_error(Gitlab::Access::AccessDeniedError) - .and change { ContainerRepository.count }.by(0) - end - end - context 'when repository is created by an user' do context 'when user has no ability to create a repository' do it 'does not create a new container repository' do @@ -40,9 +28,22 @@ describe ContainerRegistry::CreateRepositoryService, '#execute' do project.add_developer(user) end - it 'creates a new container repository' do - expect { service.execute(path) } - .to change { project.container_repositories.count }.by(1) + context 'when repository already exists' do + before do + create(:container_repository, project: project, name: 'my/image') + end + + it 'does not create container repository again' do + expect { service.execute(path) } + .to_not change { ContainerRepository.count } + end + end + + context 'when repository does not exist yet' do + it 'creates a new container repository' do + expect { service.execute(path) } + .to change { project.container_repositories.count }.by(1) + end end end end -- cgit v1.2.1 From 236c9c170374190bcb6dc54a1ca3dddda2d3dcb2 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 30 Mar 2017 15:24:08 +0200 Subject: Simplify how we create container registry resources --- .../create_repository_service.rb | 33 ---------- .../create_repository_service_spec.rb | 75 ---------------------- 2 files changed, 108 deletions(-) delete mode 100644 app/services/container_registry/create_repository_service.rb delete mode 100644 spec/services/container_registry/create_repository_service_spec.rb diff --git a/app/services/container_registry/create_repository_service.rb b/app/services/container_registry/create_repository_service.rb deleted file mode 100644 index 84218702a4f..00000000000 --- a/app/services/container_registry/create_repository_service.rb +++ /dev/null @@ -1,33 +0,0 @@ -module ContainerRegistry - ## - # Service for creating a container repository. - # - # It is usually executed before registry authenticator returns - # a token for given request. - # - class CreateRepositoryService < BaseService - def execute(path) - @path = path - - return if path.has_repository? - - unless user_can_create? || legacy_trigger_can_create? - raise Gitlab::Access::AccessDeniedError - end - - ContainerRepository.create_from_path(path) - end - - private - - def user_can_create? - can?(@current_user, :create_container_image, @path.repository_project) - end - - ## TODO, remove it after removing legacy triggers. - # - def legacy_trigger_can_create? - @current_user.nil? && @project == @path.repository_project - end - end -end diff --git a/spec/services/container_registry/create_repository_service_spec.rb b/spec/services/container_registry/create_repository_service_spec.rb deleted file mode 100644 index ac7b147f92e..00000000000 --- a/spec/services/container_registry/create_repository_service_spec.rb +++ /dev/null @@ -1,75 +0,0 @@ -require 'spec_helper' - -describe ContainerRegistry::CreateRepositoryService, '#execute' do - let(:project) { create(:empty_project) } - let(:user) { create(:user) } - - let(:path) do - ContainerRegistry::Path.new("#{project.full_path}/my/image") - end - - let(:service) { described_class.new(project, user) } - - before do - stub_container_registry_config(enabled: true) - end - - context 'when repository is created by an user' do - context 'when user has no ability to create a repository' do - it 'does not create a new container repository' do - expect { service.execute(path) } - .to raise_error(Gitlab::Access::AccessDeniedError) - .and change { ContainerRepository.count }.by(0) - end - end - - context 'when user has ability do create a repository' do - before do - project.add_developer(user) - end - - context 'when repository already exists' do - before do - create(:container_repository, project: project, name: 'my/image') - end - - it 'does not create container repository again' do - expect { service.execute(path) } - .to_not change { ContainerRepository.count } - end - end - - context 'when repository does not exist yet' do - it 'creates a new container repository' do - expect { service.execute(path) } - .to change { project.container_repositories.count }.by(1) - end - end - end - end - - context 'when repository is created by a legacy pipeline trigger' do - let(:user) { nil } - - context 'when repository path matches authenticated project' do - it 'creates a new container repository' do - expect { service.execute(path) } - .to change { project.container_repositories.count }.by(1) - end - end - - context 'when repository path does not match authenticated project' do - let(:private_project) { create(:empty_project, :private) } - - let(:path) do - ContainerRegistry::Path.new("#{private_project.full_path}/my/image") - end - - it 'does not create a new container repository' do - expect { service.execute(path) } - .to raise_error(Gitlab::Access::AccessDeniedError) - .and change { ContainerRepository.count }.by(0) - end - end - end -end -- cgit v1.2.1 From 7d3d1ec5a7b3ee13397af587c6ecb3a3fc297006 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 30 Mar 2017 15:24:46 +0200 Subject: Create container repository on successful push auth Because we do not have yet two way communication between container registry and GitLab, we need to eagerly create a new container repository objects in database. We now do that after user/build successfully authenticates a push action using auth service. --- app/models/container_repository.rb | 3 +- .../container_registry_authentication_service.rb | 16 +++++++ lib/container_registry/path.rb | 4 ++ ...ntainer_registry_authentication_service_spec.rb | 52 ++++++++++++++++++++-- 4 files changed, 70 insertions(+), 5 deletions(-) diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index 33e2574d389..5663b3db92f 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -60,6 +60,7 @@ class ContainerRepository < ActiveRecord::Base end def self.create_from_path(path) - self.create(project: path.repository_project, name: path.repository_name) + self.create(project: path.repository_project, + name: path.repository_name) end end diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb index b050f1dd51b..839f514ad58 100644 --- a/app/services/auth/container_registry_authentication_service.rb +++ b/app/services/auth/container_registry_authentication_service.rb @@ -74,9 +74,25 @@ module Auth return unless actions.present? + # At this point user/build is already authenticated. + # + ensure_container_repository!(path, actions) + { type: type, name: path.to_s, actions: actions } end + ## + # Because we do not have two way communication with registry yet, + # we create a container repository image resource when push to the + # registry is successfuly authorized. + # + def ensure_container_repository!(path, actions) + return if path.has_repository? + return unless actions.include?('push') + + ContainerRepository.create_from_path(path) + end + def can_access?(requested_project, requested_action) return false unless requested_project.container_registry_enabled? diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb index 3a6fde08e8f..27e0e7897ff 100644 --- a/lib/container_registry/path.rb +++ b/lib/container_registry/path.rb @@ -44,6 +44,10 @@ module ContainerRegistry .where(name: repository_name).any? end + def root_repository? + @path == repository_project.full_path + end + def repository_project @project ||= Project.where_full_path_in(components.first(3))&.first end diff --git a/spec/services/auth/container_registry_authentication_service_spec.rb b/spec/services/auth/container_registry_authentication_service_spec.rb index fac7f1b1235..a4a6430011e 100644 --- a/spec/services/auth/container_registry_authentication_service_spec.rb +++ b/spec/services/auth/container_registry_authentication_service_spec.rb @@ -80,6 +80,19 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do it { is_expected.not_to include(:token) } end + shared_examples 'container repository factory' do + it 'creates a new containe repository resource' do + expect { subject } + .to change { project.container_repositories.count }.by(1) + end + end + + shared_examples 'container repository factory' do + it 'does not create a new container repository resource' do + expect { subject }.not_to change { ContainerRepository.count } + end + end + describe '#full_access_token' do let(:project) { create(:empty_project) } let(:token) { described_class.full_access_token(project.path_with_namespace) } @@ -89,6 +102,8 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do it_behaves_like 'an accessible' do let(:actions) { ['*'] } end + + it_behaves_like 'not a container repository factory' end context 'user authorization' do @@ -109,16 +124,20 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end it_behaves_like 'a pushable' + it_behaves_like 'container repository factory' end context 'allow reporter to pull images' do before { project.team << [current_user, :reporter] } - let(:current_params) do - { scope: "repository:#{project.path_with_namespace}:pull" } - end + context 'when pulling from root level repository' do + let(:current_params) do + { scope: "repository:#{project.path_with_namespace}:pull" } + end - it_behaves_like 'a pullable' + it_behaves_like 'a pullable' + it_behaves_like 'not a container repository factory' + end end context 'return a least of privileges' do @@ -129,6 +148,7 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end it_behaves_like 'a pullable' + it_behaves_like 'not a container repository factory' end context 'disallow guest to pull or push images' do @@ -139,6 +159,7 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end it_behaves_like 'an inaccessible' + it_behaves_like 'not a container repository factory' end end @@ -151,6 +172,7 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end it_behaves_like 'a pullable' + it_behaves_like 'not a container repository factory' end context 'disallow anyone to push images' do @@ -159,6 +181,7 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end it_behaves_like 'an inaccessible' + it_behaves_like 'not a container repository factory' end end @@ -172,6 +195,7 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end it_behaves_like 'a pullable' + it_behaves_like 'not a container repository factory' end context 'disallow anyone to push images' do @@ -180,6 +204,7 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end it_behaves_like 'an inaccessible' + it_behaves_like 'not a container repository factory' end end @@ -190,6 +215,7 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end it_behaves_like 'an inaccessible' + it_behaves_like 'not a container repository factory' end end end @@ -216,6 +242,10 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do it_behaves_like 'a pullable and pushable' do let(:project) { current_project } end + + it_behaves_like 'container repository factory' do + let(:project) { current_project } + end end context 'for other projects' do @@ -228,11 +258,13 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do let(:project) { create(:empty_project, :public) } it_behaves_like 'a pullable' + it_behaves_like 'not a container repository factory' end shared_examples 'pullable for being team member' do context 'when you are not member' do it_behaves_like 'an inaccessible' + it_behaves_like 'not a container repository factory' end context 'when you are member' do @@ -241,12 +273,14 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end it_behaves_like 'a pullable' + it_behaves_like 'not a container repository factory' end context 'when you are owner' do let(:project) { create(:empty_project, namespace: current_user.namespace) } it_behaves_like 'a pullable' + it_behaves_like 'not a container repository factory' end end @@ -260,6 +294,7 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do context 'when you are not member' do it_behaves_like 'an inaccessible' + it_behaves_like 'not a container repository factory' end context 'when you are member' do @@ -268,12 +303,14 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end it_behaves_like 'a pullable' + it_behaves_like 'not a container repository factory' end context 'when you are owner' do let(:project) { create(:empty_project, namespace: current_user.namespace) } it_behaves_like 'a pullable' + it_behaves_like 'not a container repository factory' end end end @@ -293,12 +330,14 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end it_behaves_like 'an inaccessible' + it_behaves_like 'not a container repository factory' end context 'when you are owner' do let(:project) { create(:empty_project, :public, namespace: current_user.namespace) } it_behaves_like 'an inaccessible' + it_behaves_like 'not a container repository factory' end end end @@ -315,6 +354,7 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end it_behaves_like 'an inaccessible' + it_behaves_like 'not a container repository factory' end end end @@ -322,6 +362,7 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do context 'unauthorized' do context 'disallow to use scope-less authentication' do it_behaves_like 'a forbidden' + it_behaves_like 'not a container repository factory' end context 'for invalid scope' do @@ -330,6 +371,7 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end it_behaves_like 'a forbidden' + it_behaves_like 'not a container repository factory' end context 'for private project' do @@ -351,6 +393,7 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end it_behaves_like 'a pullable' + it_behaves_like 'not a container repository factory' end context 'when pushing' do @@ -359,6 +402,7 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end it_behaves_like 'a forbidden' + it_behaves_like 'not a container repository factory' end end end -- cgit v1.2.1 From fffc8a59d7f193e451daace7c69f33603c906e8f Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 30 Mar 2017 15:41:35 +0200 Subject: Show full container repository path in the UI --- app/views/projects/container_registry/_image.html.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/projects/container_registry/_image.html.haml b/app/views/projects/container_registry/_image.html.haml index 4fd642a56c9..64b11e375c6 100644 --- a/app/views/projects/container_registry/_image.html.haml +++ b/app/views/projects/container_registry/_image.html.haml @@ -7,7 +7,7 @@ - else = icon("chevron-down") - = escape_once(image.name) + = escape_once(image.path) = clipboard_button(clipboard_text: "docker pull #{image.path}") .controls.hidden-xs.pull-right = link_to namespace_project_container_registry_path(@project.namespace, @project, image.id), class: 'btn btn-remove has-tooltip', title: "Remove image", data: { confirm: "Are you sure?" }, method: :delete do -- cgit v1.2.1 From cf042068b5f69b416640f9a4fcb21fbec1082268 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 30 Mar 2017 15:41:51 +0200 Subject: Do not allow registry requests for invalid repositories --- app/services/auth/container_registry_authentication_service.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb index 839f514ad58..dcb728b6151 100644 --- a/app/services/auth/container_registry_authentication_service.rb +++ b/app/services/auth/container_registry_authentication_service.rb @@ -64,6 +64,10 @@ module Auth end def process_repository_access(type, path, actions) + # TODO, add specs for invalid paths + # + return unless path.valid? + requested_project = path.repository_project return unless requested_project -- cgit v1.2.1 From 600bbe15a103b63e14daa295abaffdf1aeafaef3 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 31 Mar 2017 11:12:38 +0200 Subject: Fix rubocop offense in registry path specs --- spec/lib/container_registry/path_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/lib/container_registry/path_spec.rb b/spec/lib/container_registry/path_spec.rb index 906dd920031..68732b12542 100644 --- a/spec/lib/container_registry/path_spec.rb +++ b/spec/lib/container_registry/path_spec.rb @@ -39,7 +39,7 @@ describe ContainerRegistry::Path do it 'is not valid' do expect(subject).not_to be_valid end - end + end context 'when path has more than allowed number of components' do let(:path) { 'a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/r/s/t/u/w/y/z' } -- cgit v1.2.1 From a7466af3a6f31311d64654631a2ea2740c42b88e Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 31 Mar 2017 11:54:09 +0200 Subject: Improve code related to removing container image tags --- app/models/container_repository.rb | 11 +++++----- spec/factories/container_repositories.rb | 20 +++++++++++------ spec/features/container_registry_spec.rb | 3 ++- spec/models/container_repository_spec.rb | 30 ++++++++++---------------- spec/services/projects/destroy_service_spec.rb | 6 ++++-- 5 files changed, 36 insertions(+), 34 deletions(-) diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index 5663b3db92f..052d93c3bdc 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -45,15 +45,14 @@ class ContainerRepository < ActiveRecord::Base # TODO, specs needed # def has_tags? - tags.any? + tags.to_a.any? end - # TODO, add bang to this method - # - def delete_tags - return unless tags + def delete_tags! + return unless has_tags? + + digests = tags.map { |tag| tag.digest }.to_set - digests = tags.map {|tag| tag.digest }.to_set digests.all? do |digest| client.delete_repository_tag(self.path, digest) end diff --git a/spec/factories/container_repositories.rb b/spec/factories/container_repositories.rb index 295b3596ee9..4919a03cdf2 100644 --- a/spec/factories/container_repositories.rb +++ b/spec/factories/container_repositories.rb @@ -8,13 +8,21 @@ FactoryGirl.define do end after(:build) do |repository, evaluator| - if evaluator.tags.any? + next if evaluator.tags.to_a.none? + + allow(repository.client) + .to receive(:repository_tags) + .and_return({ + 'name' => repository.path, + 'tags' => evaluator.tags + }) + + evaluator.tags.each do |tag| allow(repository.client) - .to receive(:repository_tags) - .and_return({ - name: repository.path, - tags: evaluator.tags - }) + .to receive(:repository_tag_digest) + .with(repository.path, tag) + .and_return('sha256:4c8e63ca4cb663ce6c688cb06f1c3' \ + '72b088dac5b6d7ad7d49cd620d85cf72a15') end end end diff --git a/spec/features/container_registry_spec.rb b/spec/features/container_registry_spec.rb index 73b22fa1b7d..530e6af92d3 100644 --- a/spec/features/container_registry_spec.rb +++ b/spec/features/container_registry_spec.rb @@ -38,7 +38,8 @@ describe "Container Registry" do end it do - expect_any_instance_of(ContainerRepository).to receive(:delete_tags).and_return(true) + expect_any_instance_of(ContainerRepository) + .to receive(:delete_tags!).and_return(true) click_on 'Remove image' end diff --git a/spec/models/container_repository_spec.rb b/spec/models/container_repository_spec.rb index 296b9e713a8..92dccf76d71 100644 --- a/spec/models/container_repository_spec.rb +++ b/spec/models/container_repository_spec.rb @@ -57,38 +57,30 @@ describe ContainerRepository do it { is_expected.not_to be_empty } end - # TODO, improve these specs - # - describe '#delete_tags' do - let(:tag) { ContainerRegistry::Tag.new(container_repository, 'tag') } - - before do - allow(container_repository).to receive(:tags).twice.and_return([tag]) - allow(tag).to receive(:digest) - .and_return('sha256:4c8e63ca4cb663ce6c688cb06f1c3672a172b088dac5b6d7ad7d49cd620d85cf') + describe '#delete_tags!' do + let(:container_repository) do + create(:container_repository, name: 'my_image', + tags: %w[latest rc1], + project: project) end context 'when action succeeds' do - before do - allow(container_repository.client) + it 'returns status that indicates success' do + expect(container_repository.client) .to receive(:delete_repository_tag) .and_return(true) - end - it 'returns status that indicates success' do - expect(container_repository.delete_tags).to be_truthy + expect(container_repository.delete_tags!).to be_truthy end end context 'when action fails' do - before do - allow(container_repository.client) + it 'returns status that indicates failure' do + expect(container_repository.client) .to receive(:delete_repository_tag) .and_return(false) - end - it 'returns status that indicates failure' do - expect(container_repository.delete_tags).to be_falsey + expect(container_repository.delete_tags!).to be_falsey end end end diff --git a/spec/services/projects/destroy_service_spec.rb b/spec/services/projects/destroy_service_spec.rb index 44e0286350b..193ccd17282 100644 --- a/spec/services/projects/destroy_service_spec.rb +++ b/spec/services/projects/destroy_service_spec.rb @@ -100,7 +100,8 @@ describe Projects::DestroyService, services: true do context 'images deletion succeeds' do it do - expect_any_instance_of(ContainerRepository).to receive(:delete_tags).and_return(true) + expect_any_instance_of(ContainerRepository) + .to receive(:delete_tags!).and_return(true) destroy_project(project, user, {}) end @@ -108,7 +109,8 @@ describe Projects::DestroyService, services: true do context 'images deletion fails' do before do - expect_any_instance_of(ContainerRepository).to receive(:delete_tags).and_return(false) + expect_any_instance_of(ContainerRepository) + .to receive(:delete_tags!).and_return(false) end subject { destroy_project(project, user, {}) } -- cgit v1.2.1 From 60cdd2bcc894cf9cce4892570bf6a146dc45e536 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 31 Mar 2017 12:27:05 +0200 Subject: Add specs for container repository factory method --- app/models/container_repository.rb | 2 +- .../container_registry_authentication_service.rb | 2 +- spec/models/container_repository_spec.rb | 51 ++++++++++++++++++---- 3 files changed, 45 insertions(+), 10 deletions(-) diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index 052d93c3bdc..e27369c10d6 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -58,7 +58,7 @@ class ContainerRepository < ActiveRecord::Base end end - def self.create_from_path(path) + def self.create_from_path!(path) self.create(project: path.repository_project, name: path.repository_name) end diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb index dcb728b6151..d58ff589be1 100644 --- a/app/services/auth/container_registry_authentication_service.rb +++ b/app/services/auth/container_registry_authentication_service.rb @@ -94,7 +94,7 @@ module Auth return if path.has_repository? return unless actions.include?('push') - ContainerRepository.create_from_path(path) + ContainerRepository.create_from_path!(path) end def can_access?(requested_project, requested_action) diff --git a/spec/models/container_repository_spec.rb b/spec/models/container_repository_spec.rb index 92dccf76d71..884eac43719 100644 --- a/spec/models/container_repository_spec.rb +++ b/spec/models/container_repository_spec.rb @@ -85,28 +85,63 @@ describe ContainerRepository do end end - describe '#from_repository_path' do + describe '.create_from_path!' do + let(:repository) do + described_class.create_from_path!(ContainerRegistry::Path.new(path)) + end + + let(:repository_path) { ContainerRegistry::Path.new(path) } + context 'when received multi-level repository path' do - let(:repository) do - described_class.from_repository_path('group/test/some/image/name') - end + let(:path) { project.full_path + '/some/image' } - pending 'fabricates object within a correct project' do + it 'fabricates repository assigned to a correct project' do expect(repository.project).to eq project end - pending 'it fabricates project with a correct name' do - expect(repository.name).to eq 'some/image/name' + it 'fabricates repository with a correct name' do + expect(repository.name).to eq 'some/image' end end - context 'when path contains too many nodes' do + context 'when path is too long' do + let(:path) do + project.full_path + '/a/b/c/d/e/f/g/h/i/j/k/l/n/o/p/s/t/u/x/y/z' + end + + it 'does not create repository and raises error' do + expect { repository }.to raise_error( + ContainerRegistry::Path::InvalidRegistryPathError) + end end context 'when received multi-level repository with nested groups' do + let(:group) { create(:group, :nested, name: 'nested') } + let(:path) { project.full_path + '/some/image' } + + it 'fabricates repository assigned to a correct project' do + expect(repository.project).to eq project + end + + it 'fabricates repository with a correct name' do + expect(repository.name).to eq 'some/image' + end + + it 'has path including a nested group' do + expect(repository.path).to include 'nested/test/some/image' + end end context 'when received root repository path' do + let(:path) { project.full_path } + + it 'fabricates repository assigned to a correct project' do + expect(repository.project).to eq project + end + + it 'fabricates repository with an empty name' do + expect(repository.name).to be_empty + end end end end -- cgit v1.2.1 From 4726ff9dbee74d00544c7eb1ea188ecdfe16d7e8 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 31 Mar 2017 12:37:44 +0200 Subject: Add test example for invalid registry access request --- .../auth/container_registry_authentication_service.rb | 2 -- .../auth/container_registry_authentication_service_spec.rb | 13 +++++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/app/services/auth/container_registry_authentication_service.rb b/app/services/auth/container_registry_authentication_service.rb index d58ff589be1..5e151b0f044 100644 --- a/app/services/auth/container_registry_authentication_service.rb +++ b/app/services/auth/container_registry_authentication_service.rb @@ -64,8 +64,6 @@ module Auth end def process_repository_access(type, path, actions) - # TODO, add specs for invalid paths - # return unless path.valid? requested_project = path.repository_project diff --git a/spec/services/auth/container_registry_authentication_service_spec.rb b/spec/services/auth/container_registry_authentication_service_spec.rb index a4a6430011e..e273dfe1552 100644 --- a/spec/services/auth/container_registry_authentication_service_spec.rb +++ b/spec/services/auth/container_registry_authentication_service_spec.rb @@ -81,13 +81,13 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do end shared_examples 'container repository factory' do - it 'creates a new containe repository resource' do + it 'creates a new container repository resource' do expect { subject } .to change { project.container_repositories.count }.by(1) end end - shared_examples 'container repository factory' do + shared_examples 'not a container repository factory' do it 'does not create a new container repository resource' do expect { subject }.not_to change { ContainerRepository.count } end @@ -183,6 +183,15 @@ describe Auth::ContainerRegistryAuthenticationService, services: true do it_behaves_like 'an inaccessible' it_behaves_like 'not a container repository factory' end + + context 'when repository name is invalid' do + let(:current_params) do + { scope: 'repository:invalid:push' } + end + + it_behaves_like 'an inaccessible' + it_behaves_like 'not a container repository factory' + end end context 'for internal project' do -- cgit v1.2.1 From 41956773839ba010bb316e6bbe8d48c1ad7177de Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 31 Mar 2017 13:08:09 +0200 Subject: Reorganize container repository controllers and views --- .../projects/container_registry_controller.rb | 53 --------------------- .../projects/registry/application_controller.rb | 10 ++++ .../projects/registry/repositories_controller.rb | 54 ++++++++++++++++++++++ .../projects/registry/tags_controller.rb | 7 +++ .../projects/container_registry/_image.html.haml | 34 -------------- .../projects/container_registry/_tag.html.haml | 29 ------------ .../projects/container_registry/index.html.haml | 26 ----------- .../registry/repositories/_image.html.haml | 34 ++++++++++++++ .../projects/registry/repositories/_tag.html.haml | 29 ++++++++++++ .../projects/registry/repositories/index.html.haml | 26 +++++++++++ config/routes/project.rb | 5 +- 11 files changed, 164 insertions(+), 143 deletions(-) delete mode 100644 app/controllers/projects/container_registry_controller.rb create mode 100644 app/controllers/projects/registry/application_controller.rb create mode 100644 app/controllers/projects/registry/repositories_controller.rb create mode 100644 app/controllers/projects/registry/tags_controller.rb delete mode 100644 app/views/projects/container_registry/_image.html.haml delete mode 100644 app/views/projects/container_registry/_tag.html.haml delete mode 100644 app/views/projects/container_registry/index.html.haml create mode 100644 app/views/projects/registry/repositories/_image.html.haml create mode 100644 app/views/projects/registry/repositories/_tag.html.haml create mode 100644 app/views/projects/registry/repositories/index.html.haml diff --git a/app/controllers/projects/container_registry_controller.rb b/app/controllers/projects/container_registry_controller.rb deleted file mode 100644 index 8929bd0aa55..00000000000 --- a/app/controllers/projects/container_registry_controller.rb +++ /dev/null @@ -1,53 +0,0 @@ -class Projects::ContainerRegistryController < Projects::ApplicationController - before_action :verify_registry_enabled - before_action :authorize_read_container_image! - before_action :authorize_update_container_image!, only: [:destroy] - layout 'project' - - def index - @images = project.container_repositories - end - - def destroy - if tag - delete_tag - else - delete_image - end - end - - private - - def registry_url - @registry_url ||= namespace_project_container_registry_index_path(project.namespace, project) - end - - def verify_registry_enabled - render_404 unless Gitlab.config.registry.enabled - end - - def delete_image - if image.destroy - redirect_to registry_url - else - redirect_to registry_url, alert: 'Failed to remove image' - end - end - - def delete_tag - if tag.delete - image.destroy if image.tags.empty? - redirect_to registry_url - else - redirect_to registry_url, alert: 'Failed to remove tag' - end - end - - def image - @image ||= project.container_repositories.find_by(id: params[:id]) - end - - def tag - @tag ||= image.tag(params[:tag]) if params[:tag].present? - end -end diff --git a/app/controllers/projects/registry/application_controller.rb b/app/controllers/projects/registry/application_controller.rb new file mode 100644 index 00000000000..8710c219553 --- /dev/null +++ b/app/controllers/projects/registry/application_controller.rb @@ -0,0 +1,10 @@ +module Projects + module Registry + class ApplicationController < Projects::ApplicationController + layout 'project' + + before_action :verify_registry_enabled + before_action :authorize_read_container_image! + end + end +end diff --git a/app/controllers/projects/registry/repositories_controller.rb b/app/controllers/projects/registry/repositories_controller.rb new file mode 100644 index 00000000000..b953d5b3378 --- /dev/null +++ b/app/controllers/projects/registry/repositories_controller.rb @@ -0,0 +1,54 @@ +module Projects + module Registry + class RepositoriesController < ::Projects::Registry::ApplicationController + before_action :authorize_update_container_image!, only: [:destroy] + + def index + @images = project.container_repositories + end + + def destroy + if tag + delete_tag + else + delete_image + end + end + + private + + def registry_url + @registry_url ||= namespace_project_container_registry_index_path(project.namespace, project) + end + + def verify_registry_enabled + render_404 unless Gitlab.config.registry.enabled + end + + def delete_image + if image.destroy + redirect_to registry_url + else + redirect_to registry_url, alert: 'Failed to remove image' + end + end + + def delete_tag + if tag.delete + image.destroy if image.tags.empty? + redirect_to registry_url + else + redirect_to registry_url, alert: 'Failed to remove tag' + end + end + + def image + @image ||= project.container_repositories.find_by(id: params[:id]) + end + + def tag + @tag ||= image.tag(params[:tag]) if params[:tag].present? + end + end + end +end diff --git a/app/controllers/projects/registry/tags_controller.rb b/app/controllers/projects/registry/tags_controller.rb new file mode 100644 index 00000000000..e40489f67ad --- /dev/null +++ b/app/controllers/projects/registry/tags_controller.rb @@ -0,0 +1,7 @@ +module Projects + module Registry + class TagsController < ::Projects::Registry::ApplicationController + before_action :authorize_update_container_image!, only: [:destroy] + end + end +end diff --git a/app/views/projects/container_registry/_image.html.haml b/app/views/projects/container_registry/_image.html.haml deleted file mode 100644 index 64b11e375c6..00000000000 --- a/app/views/projects/container_registry/_image.html.haml +++ /dev/null @@ -1,34 +0,0 @@ -- expanded = false -.container-image.js-toggle-container - .container-image-head - = link_to "#", class: "js-toggle-button" do - - if expanded - = icon("chevron-up") - - else - = icon("chevron-down") - - = escape_once(image.path) - = clipboard_button(clipboard_text: "docker pull #{image.path}") - .controls.hidden-xs.pull-right - = link_to namespace_project_container_registry_path(@project.namespace, @project, image.id), class: 'btn btn-remove has-tooltip', title: "Remove image", data: { confirm: "Are you sure?" }, method: :delete do - = icon("trash cred") - - - .container-image-tags.js-toggle-content{ class: ("hide" unless expanded) } - - if image.tags.blank? - %li - .nothing-here-block No tags in Container Registry for this container image. - - - else - .table-holder - %table.table.tags - %thead - %tr - %th Tag - %th Tag ID - %th Size - %th Created - - if can?(current_user, :update_container_image, @project) - %th - - = render partial: 'tag', collection: image.tags diff --git a/app/views/projects/container_registry/_tag.html.haml b/app/views/projects/container_registry/_tag.html.haml deleted file mode 100644 index f7161e85428..00000000000 --- a/app/views/projects/container_registry/_tag.html.haml +++ /dev/null @@ -1,29 +0,0 @@ -%tr.tag - %td - = escape_once(tag.name) - = clipboard_button(clipboard_text: "docker pull #{tag.path}") - %td - - if tag.revision - %span.has-tooltip{ title: "#{tag.revision}" } - = tag.short_revision - - else - \- - %td - - if tag.total_size - = number_to_human_size(tag.total_size) - · - = pluralize(tag.layers.size, "layer") - - else - .light - \- - %td - - if tag.created_at - = time_ago_in_words(tag.created_at) - - else - .light - \- - - if can?(current_user, :update_container_image, @project) - %td.content - .controls.hidden-xs.pull-right - = link_to namespace_project_container_registry_path(@project.namespace, @project, { id: tag.repository.id, tag: tag.name} ), class: 'btn btn-remove has-tooltip', title: "Remove tag", data: { confirm: "Due to a Docker limitation, all tags with the same ID will also be deleted. Are you sure?" }, method: :delete do - = icon("trash cred") diff --git a/app/views/projects/container_registry/index.html.haml b/app/views/projects/container_registry/index.html.haml deleted file mode 100644 index 1b5d000e801..00000000000 --- a/app/views/projects/container_registry/index.html.haml +++ /dev/null @@ -1,26 +0,0 @@ -- page_title "Container Registry" - -%hr - -%ul.content-list - %li.light.prepend-top-default - %p - A 'container image' is a snapshot of a container. - You can host your container images with GitLab. - %br - To start using container images hosted on GitLab you first need to login: - %pre - %code - docker login #{Gitlab.config.registry.host_port} - %br - Then you are free to create and upload a container image with build and push commands: - %pre - docker build -t #{escape_once(@project.container_registry_url)}/image . - %br - docker push #{escape_once(@project.container_registry_url)}/image - - - if @images.blank? - .nothing-here-block No container images in Container Registry for this project. - - - else - = render partial: 'image', collection: @images diff --git a/app/views/projects/registry/repositories/_image.html.haml b/app/views/projects/registry/repositories/_image.html.haml new file mode 100644 index 00000000000..64b11e375c6 --- /dev/null +++ b/app/views/projects/registry/repositories/_image.html.haml @@ -0,0 +1,34 @@ +- expanded = false +.container-image.js-toggle-container + .container-image-head + = link_to "#", class: "js-toggle-button" do + - if expanded + = icon("chevron-up") + - else + = icon("chevron-down") + + = escape_once(image.path) + = clipboard_button(clipboard_text: "docker pull #{image.path}") + .controls.hidden-xs.pull-right + = link_to namespace_project_container_registry_path(@project.namespace, @project, image.id), class: 'btn btn-remove has-tooltip', title: "Remove image", data: { confirm: "Are you sure?" }, method: :delete do + = icon("trash cred") + + + .container-image-tags.js-toggle-content{ class: ("hide" unless expanded) } + - if image.tags.blank? + %li + .nothing-here-block No tags in Container Registry for this container image. + + - else + .table-holder + %table.table.tags + %thead + %tr + %th Tag + %th Tag ID + %th Size + %th Created + - if can?(current_user, :update_container_image, @project) + %th + + = render partial: 'tag', collection: image.tags diff --git a/app/views/projects/registry/repositories/_tag.html.haml b/app/views/projects/registry/repositories/_tag.html.haml new file mode 100644 index 00000000000..f7161e85428 --- /dev/null +++ b/app/views/projects/registry/repositories/_tag.html.haml @@ -0,0 +1,29 @@ +%tr.tag + %td + = escape_once(tag.name) + = clipboard_button(clipboard_text: "docker pull #{tag.path}") + %td + - if tag.revision + %span.has-tooltip{ title: "#{tag.revision}" } + = tag.short_revision + - else + \- + %td + - if tag.total_size + = number_to_human_size(tag.total_size) + · + = pluralize(tag.layers.size, "layer") + - else + .light + \- + %td + - if tag.created_at + = time_ago_in_words(tag.created_at) + - else + .light + \- + - if can?(current_user, :update_container_image, @project) + %td.content + .controls.hidden-xs.pull-right + = link_to namespace_project_container_registry_path(@project.namespace, @project, { id: tag.repository.id, tag: tag.name} ), class: 'btn btn-remove has-tooltip', title: "Remove tag", data: { confirm: "Due to a Docker limitation, all tags with the same ID will also be deleted. Are you sure?" }, method: :delete do + = icon("trash cred") diff --git a/app/views/projects/registry/repositories/index.html.haml b/app/views/projects/registry/repositories/index.html.haml new file mode 100644 index 00000000000..1b5d000e801 --- /dev/null +++ b/app/views/projects/registry/repositories/index.html.haml @@ -0,0 +1,26 @@ +- page_title "Container Registry" + +%hr + +%ul.content-list + %li.light.prepend-top-default + %p + A 'container image' is a snapshot of a container. + You can host your container images with GitLab. + %br + To start using container images hosted on GitLab you first need to login: + %pre + %code + docker login #{Gitlab.config.registry.host_port} + %br + Then you are free to create and upload a container image with build and push commands: + %pre + docker build -t #{escape_once(@project.container_registry_url)}/image . + %br + docker push #{escape_once(@project.container_registry_url)}/image + + - if @images.blank? + .nothing-here-block No container images in Container Registry for this project. + + - else + = render partial: 'image', collection: @images diff --git a/config/routes/project.rb b/config/routes/project.rb index 44b8ae7aedd..34f4bd917f7 100644 --- a/config/routes/project.rb +++ b/config/routes/project.rb @@ -219,7 +219,10 @@ constraints(ProjectUrlConstrainer.new) do end end - resources :container_registry, only: [:index, :destroy], constraints: { id: Gitlab::Regex.container_registry_reference_regex } + resources :container_registry, + controller: 'registry/repositories', + only: [:index, :destroy], + constraints: { id: Gitlab::Regex.container_registry_reference_regex } resources :milestones, constraints: { id: /\d+/ } do member do -- cgit v1.2.1 From f32d269cfa08f280c0d0dd86c4b50c4dcfb7409f Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 31 Mar 2017 13:16:16 +0200 Subject: Remove unused method from container registry client --- lib/container_registry/client.rb | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index 196cdd36a88..7f5f6d9ddb6 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -15,10 +15,6 @@ module ContainerRegistry @options = options end - def update_token(token) - @options[:token] = token - end - def repository_tags(name) response_body faraday.get("/v2/#{name}/tags/list") end -- cgit v1.2.1 From 00319e595ab52906d12ef027a10e08ac92ea1337 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 31 Mar 2017 13:56:07 +0200 Subject: Move code related to registry to multiple controllers --- .../projects/registry/application_controller.rb | 8 ++++- .../projects/registry/repositories_controller.rb | 36 +++------------------- .../projects/registry/tags_controller.rb | 20 ++++++++++++ .../projects/registry/repositories/_tag.html.haml | 9 ++++-- config/routes/project.rb | 6 ++++ 5 files changed, 44 insertions(+), 35 deletions(-) diff --git a/app/controllers/projects/registry/application_controller.rb b/app/controllers/projects/registry/application_controller.rb index 8710c219553..a56f9c58726 100644 --- a/app/controllers/projects/registry/application_controller.rb +++ b/app/controllers/projects/registry/application_controller.rb @@ -3,8 +3,14 @@ module Projects class ApplicationController < Projects::ApplicationController layout 'project' - before_action :verify_registry_enabled + before_action :verify_registry_enabled! before_action :authorize_read_container_image! + + private + + def verify_registry_enabled! + render_404 unless Gitlab.config.registry.enabled + end end end end diff --git a/app/controllers/projects/registry/repositories_controller.rb b/app/controllers/projects/registry/repositories_controller.rb index b953d5b3378..e3e30176b30 100644 --- a/app/controllers/projects/registry/repositories_controller.rb +++ b/app/controllers/projects/registry/repositories_controller.rb @@ -8,47 +8,19 @@ module Projects end def destroy - if tag - delete_tag - else - delete_image - end - end - - private - - def registry_url - @registry_url ||= namespace_project_container_registry_index_path(project.namespace, project) - end - - def verify_registry_enabled - render_404 unless Gitlab.config.registry.enabled - end - - def delete_image if image.destroy - redirect_to registry_url + redirect_to project_container_registry_path(@project) else - redirect_to registry_url, alert: 'Failed to remove image' + redirect_to project_container_registry_path(@project), + alert: 'Failed to remove images repository!' end end - def delete_tag - if tag.delete - image.destroy if image.tags.empty? - redirect_to registry_url - else - redirect_to registry_url, alert: 'Failed to remove tag' - end - end + private def image @image ||= project.container_repositories.find_by(id: params[:id]) end - - def tag - @tag ||= image.tag(params[:tag]) if params[:tag].present? - end end end end diff --git a/app/controllers/projects/registry/tags_controller.rb b/app/controllers/projects/registry/tags_controller.rb index e40489f67ad..8f0a1aff394 100644 --- a/app/controllers/projects/registry/tags_controller.rb +++ b/app/controllers/projects/registry/tags_controller.rb @@ -2,6 +2,26 @@ module Projects module Registry class TagsController < ::Projects::Registry::ApplicationController before_action :authorize_update_container_image!, only: [:destroy] + + def destroy + if tag.delete + redirect_to project_container_registry_path(@project) + else + redirect_to project_container_registry_path(@project), + alert: 'Failed to remove repository tag!' + end + end + + private + + def repository + @image ||= project.container_repositories + .find_by(id: params[:repository_id]) + end + + def tag + @tag ||= repository.tag(params[:id]) if params[:id].present? + end end end end diff --git a/app/views/projects/registry/repositories/_tag.html.haml b/app/views/projects/registry/repositories/_tag.html.haml index f7161e85428..c9b3644ff93 100644 --- a/app/views/projects/registry/repositories/_tag.html.haml +++ b/app/views/projects/registry/repositories/_tag.html.haml @@ -25,5 +25,10 @@ - if can?(current_user, :update_container_image, @project) %td.content .controls.hidden-xs.pull-right - = link_to namespace_project_container_registry_path(@project.namespace, @project, { id: tag.repository.id, tag: tag.name} ), class: 'btn btn-remove has-tooltip', title: "Remove tag", data: { confirm: "Due to a Docker limitation, all tags with the same ID will also be deleted. Are you sure?" }, method: :delete do - = icon("trash cred") + - notice = 'Due to a Docker limitation, all tags with the same ID will also be deleted. Are you sure?' + = link_to namespace_project_registry_repository_tag_path(@project.namespace, @project, tag.repository, tag.name), + method: :delete, + class: 'btn btn-remove has-tooltip', + title: 'Remove image tag', + data: { confirm: notice } do + = icon('trash cred') diff --git a/config/routes/project.rb b/config/routes/project.rb index 34f4bd917f7..0269857f9fb 100644 --- a/config/routes/project.rb +++ b/config/routes/project.rb @@ -224,6 +224,12 @@ constraints(ProjectUrlConstrainer.new) do only: [:index, :destroy], constraints: { id: Gitlab::Regex.container_registry_reference_regex } + namespace :registry do + resources :repository, only: [] do + resources :tags, only: [:destroy] + end + end + resources :milestones, constraints: { id: /\d+/ } do member do put :sort_issues -- cgit v1.2.1 From 83d1fe9b5aeb947c1387666205ecaca81f2bf3a2 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 31 Mar 2017 15:10:15 +0200 Subject: Add serveral minor improvements to container registry --- app/controllers/projects/registry/repositories_controller.rb | 3 ++- app/controllers/projects/registry/tags_controller.rb | 3 ++- app/models/container_repository.rb | 3 ++- app/views/projects/registry/repositories/_image.html.haml | 6 +++++- lib/container_registry/tag.rb | 4 +--- 5 files changed, 12 insertions(+), 7 deletions(-) diff --git a/app/controllers/projects/registry/repositories_controller.rb b/app/controllers/projects/registry/repositories_controller.rb index e3e30176b30..2901d83fcef 100644 --- a/app/controllers/projects/registry/repositories_controller.rb +++ b/app/controllers/projects/registry/repositories_controller.rb @@ -9,7 +9,8 @@ module Projects def destroy if image.destroy - redirect_to project_container_registry_path(@project) + redirect_to project_container_registry_path(@project), + notice: 'Images repository has been removed successfully!' else redirect_to project_container_registry_path(@project), alert: 'Failed to remove images repository!' diff --git a/app/controllers/projects/registry/tags_controller.rb b/app/controllers/projects/registry/tags_controller.rb index 8f0a1aff394..aab130787e9 100644 --- a/app/controllers/projects/registry/tags_controller.rb +++ b/app/controllers/projects/registry/tags_controller.rb @@ -5,7 +5,8 @@ module Projects def destroy if tag.delete - redirect_to project_container_registry_path(@project) + redirect_to project_container_registry_path(@project), + notice: 'Tag removed successfull!' else redirect_to project_container_registry_path(@project), alert: 'Failed to remove repository tag!' diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index e27369c10d6..ceb82af2e95 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -4,7 +4,8 @@ class ContainerRepository < ActiveRecord::Base validates :name, length: { minimum: 0, allow_nil: false } delegate :client, to: :registry - before_destroy :delete_tags + + before_destroy :delete_tags! def registry @registry ||= begin diff --git a/app/views/projects/registry/repositories/_image.html.haml b/app/views/projects/registry/repositories/_image.html.haml index 64b11e375c6..c3f8580d25b 100644 --- a/app/views/projects/registry/repositories/_image.html.haml +++ b/app/views/projects/registry/repositories/_image.html.haml @@ -10,7 +10,11 @@ = escape_once(image.path) = clipboard_button(clipboard_text: "docker pull #{image.path}") .controls.hidden-xs.pull-right - = link_to namespace_project_container_registry_path(@project.namespace, @project, image.id), class: 'btn btn-remove has-tooltip', title: "Remove image", data: { confirm: "Are you sure?" }, method: :delete do + = link_to namespace_project_container_registry_path(@project.namespace, @project, image), + class: 'btn btn-remove has-tooltip', + title: 'Remove repository', + data: { confirm: 'Are you sure?' }, + method: :delete do = icon("trash cred") diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb index d653deb3bf1..d00e6191e7e 100644 --- a/lib/container_registry/tag.rb +++ b/lib/container_registry/tag.rb @@ -36,9 +36,7 @@ module ContainerRegistry end def digest - return @digest if defined?(@digest) - - @digest = client.repository_tag_digest(repository.path, name) + @digest ||= client.repository_tag_digest(repository.path, name) end def config_blob -- cgit v1.2.1 From 058dfbb0fd5d058e31ee627dff0527b3f4d41664 Mon Sep 17 00:00:00 2001 From: Phil Hughes Date: Mon, 3 Apr 2017 09:05:21 +0100 Subject: Fixed user profile tabs causing the page to scroll Closes #30338 --- app/assets/stylesheets/pages/profile.scss | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/assets/stylesheets/pages/profile.scss b/app/assets/stylesheets/pages/profile.scss index 703c5fc8869..8c6dd392865 100644 --- a/app/assets/stylesheets/pages/profile.scss +++ b/app/assets/stylesheets/pages/profile.scss @@ -230,6 +230,14 @@ font-size: 0; } + .fade-right { + right: 0; + } + + .fade-left { + left: 0; + } + @media (max-width: $screen-xs-max) { .cover-block { padding-top: 20px; -- cgit v1.2.1 From 662d2e68173ecc8ee1bfe9dddf8cc38766bd310b Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Apr 2017 10:38:37 +0200 Subject: Refactor feature specs for container registry --- .../projects/registry/repositories/_tag.html.haml | 2 +- spec/features/container_registry_spec.rb | 65 +++++++++++----------- spec/support/stub_gitlab_calls.rb | 16 +++--- 3 files changed, 42 insertions(+), 41 deletions(-) diff --git a/app/views/projects/registry/repositories/_tag.html.haml b/app/views/projects/registry/repositories/_tag.html.haml index c9b3644ff93..73566689d57 100644 --- a/app/views/projects/registry/repositories/_tag.html.haml +++ b/app/views/projects/registry/repositories/_tag.html.haml @@ -29,6 +29,6 @@ = link_to namespace_project_registry_repository_tag_path(@project.namespace, @project, tag.repository, tag.name), method: :delete, class: 'btn btn-remove has-tooltip', - title: 'Remove image tag', + title: 'Remove tag', data: { confirm: notice } do = icon('trash cred') diff --git a/spec/features/container_registry_spec.rb b/spec/features/container_registry_spec.rb index 530e6af92d3..42431cbe731 100644 --- a/spec/features/container_registry_spec.rb +++ b/spec/features/container_registry_spec.rb @@ -1,59 +1,60 @@ require 'spec_helper' describe "Container Registry" do + let(:user) { create(:user) } let(:project) { create(:empty_project) } - let(:registry) { project.container_registry } - let(:tag_name) { 'latest' } - let(:tags) { [tag_name] } - let(:container_repository) { create(:container_repository) } - let(:image_name) { container_repository.name } + + let(:container_repository) do + create(:container_repository, name: 'my/image') + end before do - login_as(:user) - project.team << [@user, :developer] + login_as(user) + project.add_developer(user) stub_container_registry_config(enabled: true) - stub_container_registry_tags(*tags) - project.container_repositories << container_repository unless container_repository.nil? + stub_container_registry_tags(%w[latest]) end - describe 'GET /:project/container_registry' do - before do - visit namespace_project_container_registry_index_path(project.namespace, project) - end - - context 'when no images' do - let(:container_repository) { } - - it { expect(page).to have_content('No container images in Container Registry for this project') } - end + context 'when there are no image repositories' do + scenario 'user visits container registry main page' do + visit_container_registry - context 'when there are images' do - it { expect(page).to have_content(image_name) } + expect(page).to have_content 'No container images' end end - describe 'DELETE /:project/container_registry/:image_id' do + context 'when there are image repositories' do before do - visit namespace_project_container_registry_index_path(project.namespace, project) + project.container_repositories << container_repository + end + + scenario 'user wants to see multi-level container repository' do + visit_container_registry + + expect(page).to have_content('my/image') end - it do + scenario 'user removes entire container repository' do + visit_container_registry + expect_any_instance_of(ContainerRepository) .to receive(:delete_tags!).and_return(true) - click_on 'Remove image' + click_on 'Remove repository' end - end - describe 'DELETE /:project/container_registry/tag' do - before do - visit namespace_project_container_registry_index_path(project.namespace, project) - end + scenario 'user removes a specific tag from container repository' do + visit_container_registry - it do - expect_any_instance_of(::ContainerRegistry::Tag).to receive(:delete).and_return(true) + expect_any_instance_of(ContainerRegistry::Tag) + .to receive(:delete).and_return(true) click_on 'Remove tag' end end + + def visit_container_registry + visit namespace_project_container_registry_index_path( + project.namespace, project) + end end diff --git a/spec/support/stub_gitlab_calls.rb b/spec/support/stub_gitlab_calls.rb index 3949784aabb..dbf3ace37c3 100644 --- a/spec/support/stub_gitlab_calls.rb +++ b/spec/support/stub_gitlab_calls.rb @@ -32,15 +32,15 @@ module StubGitlabCalls end def stub_container_registry_tags(*tags) - allow_any_instance_of(ContainerRegistry::Client).to receive(:repository_tags).and_return( - { "tags" => tags } - ) - allow_any_instance_of(ContainerRegistry::Client).to receive(:repository_manifest).and_return( - JSON.parse(File.read(Rails.root + 'spec/fixtures/container_registry/tag_manifest.json')) - ) + allow_any_instance_of(ContainerRegistry::Client) + .to receive(:repository_tags).and_return({ 'tags' => tags }) + + allow_any_instance_of(ContainerRegistry::Client) + .to receive(:repository_manifest).and_return( + JSON.parse(File.read(Rails.root + 'spec/fixtures/container_registry/tag_manifest.json'))) + allow_any_instance_of(ContainerRegistry::Client).to receive(:blob).and_return( - File.read(Rails.root + 'spec/fixtures/container_registry/config_blob.json') - ) + File.read(Rails.root + 'spec/fixtures/container_registry/config_blob.json')) end private -- cgit v1.2.1 From 10b0fb1b4241e6c93f62a07fd1eac58835a8c7e2 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Apr 2017 10:53:16 +0200 Subject: Remove redundant stubs from container image tag specs --- spec/lib/container_registry/tag_spec.rb | 35 ++++++++++++++++----------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/spec/lib/container_registry/tag_spec.rb b/spec/lib/container_registry/tag_spec.rb index 37eaa10f4a4..bc1912d8e6c 100644 --- a/spec/lib/container_registry/tag_spec.rb +++ b/spec/lib/container_registry/tag_spec.rb @@ -5,10 +5,9 @@ describe ContainerRegistry::Tag do let(:project) { create(:project, path: 'test', group: group) } let(:repository) do - create(:container_repository, name: '', tags: %w[latest], project: project) + create(:container_repository, name: '', project: project) end - # TODO, move stubs to helper with this header let(:headers) do { 'Accept' => 'application/vnd.docker.distribution.manifest.v2+json' } end @@ -41,7 +40,7 @@ describe ContainerRegistry::Tag do context 'when tag belongs to first-level repository' do let(:repository) do create(:container_repository, name: 'my_image', - tags: %w[latest], + tags: %w[tag], project: project) end @@ -158,29 +157,29 @@ describe ContainerRegistry::Tag do end end - context 'manifest digest' do + context 'with stubbed digest' do before do - stub_request(:head, 'http://registry.gitlab/v2/group/test/manifests/tag'). - with(headers: headers). - to_return(status: 200, headers: { 'Docker-Content-Digest' => 'sha256:digest' }) + stub_request(:head, 'http://registry.gitlab/v2/group/test/manifests/tag') + .with(headers: headers) + .to_return(status: 200, headers: { 'Docker-Content-Digest' => 'sha256:digest' }) end - context '#digest' do - subject { tag.digest } - - it { is_expected.to eq('sha256:digest') } + describe '#digest' do + it 'returns a correct tag digest' do + expect(tag.digest).to eq 'sha256:digest' + end end - context '#delete' do + describe '#delete' do before do - stub_request(:delete, 'http://registry.gitlab/v2/group/test/manifests/sha256:digest'). - with(headers: headers). - to_return(status: 200) + stub_request(:delete, 'http://registry.gitlab/v2/group/test/manifests/sha256:digest') + .with(headers: headers) + .to_return(status: 200) end - subject { tag.delete } - - it { is_expected.to be_truthy } + it 'correctly deletes the tag' do + expect(tag.delete).to be_truthy + end end end end -- cgit v1.2.1 From 01280a5ad56f67ae653dade815faa5649bcee81f Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Apr 2017 10:57:12 +0200 Subject: Add missing test example for container repository tags --- app/models/container_repository.rb | 2 -- spec/models/container_repository_spec.rb | 6 ++++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index ceb82af2e95..ab04258ab8d 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -43,8 +43,6 @@ class ContainerRepository < ActiveRecord::Base ContainerRegistry::Blob.new(self, config) end - # TODO, specs needed - # def has_tags? tags.to_a.any? end diff --git a/spec/models/container_repository_spec.rb b/spec/models/container_repository_spec.rb index 884eac43719..f794085fc45 100644 --- a/spec/models/container_repository_spec.rb +++ b/spec/models/container_repository_spec.rb @@ -57,6 +57,12 @@ describe ContainerRepository do it { is_expected.not_to be_empty } end + describe '#has_tags?' do + it 'has tags' do + expect(container_repository).to have_tags + end + end + describe '#delete_tags!' do let(:container_repository) do create(:container_repository, name: 'my_image', -- cgit v1.2.1 From 1a47986b3d7cd8e6d5bdbfbc20a841cf5586f773 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Apr 2017 11:38:39 +0200 Subject: Check registry repository name against regexp This regexp is extracted from Docker Distribution 2.4.1 docs, contains additional `/` element that can be a separator of components. --- lib/container_registry/path.rb | 4 +++- lib/gitlab/regex.rb | 7 +++++++ spec/lib/container_registry/path_spec.rb | 24 +++++++++++++++--------- 3 files changed, 25 insertions(+), 10 deletions(-) diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb index 27e0e7897ff..6e8d62b77c7 100644 --- a/lib/container_registry/path.rb +++ b/lib/container_registry/path.rb @@ -22,7 +22,9 @@ module ContainerRegistry end def valid? - @nodes.size > 1 && @nodes.size < Namespace::NUMBER_OF_ANCESTORS_ALLOWED + @path =~ Gitlab::Regex.container_repository_name_regex && + @nodes.size > 1 && + @nodes.size < Namespace::NUMBER_OF_ANCESTORS_ALLOWED end def components diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 5e5f5ff1589..e599dd4a656 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -121,6 +121,13 @@ module Gitlab git_reference_regex end + ## + # Docker Distribution Registry 2.4.1 repository name rules + # + def container_repository_name_regex + @container_repository_regex ||= %r{\A[a-z0-9]+(?:[-._/][a-z0-9]+)*\Z} + end + def environment_name_regex @environment_name_regex ||= /\A[a-zA-Z0-9_\\\/\${}. -]+\z/.freeze end diff --git a/spec/lib/container_registry/path_spec.rb b/spec/lib/container_registry/path_spec.rb index 68732b12542..1973da65f0a 100644 --- a/spec/lib/container_registry/path_spec.rb +++ b/spec/lib/container_registry/path_spec.rb @@ -36,25 +36,31 @@ describe ContainerRegistry::Path do context 'when path has less than two components' do let(:path) { 'something/' } - it 'is not valid' do - expect(subject).not_to be_valid - end + it { is_expected.not_to be_valid } end context 'when path has more than allowed number of components' do let(:path) { 'a/b/c/d/e/f/g/h/i/j/k/l/m/n/o/p/r/s/t/u/w/y/z' } - it 'is not valid' do - expect(subject).not_to be_valid - end + it { is_expected.not_to be_valid } + end + + context 'when path has invalid characters' do + let(:path) { 'some\path' } + + it { is_expected.not_to be_valid } end context 'when path has two or more components' do let(:path) { 'some/path' } - it 'is valid' do - expect(subject).to be_valid - end + it { is_expected.to be_valid } + end + + context 'when path is related to multi-level image' do + let(:path) { 'some/path/my/image' } + + it { is_expected.to be_valid } end end -- cgit v1.2.1 From e10dae3e3c2a5fb6077244c72e44c912b7281349 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Apr 2017 11:42:37 +0200 Subject: Improve code in container repository path class --- lib/container_registry/path.rb | 23 +++++++++++++---------- spec/lib/container_registry/path_spec.rb | 8 ++++++++ 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb index 6e8d62b77c7..f76c6489381 100644 --- a/lib/container_registry/path.rb +++ b/lib/container_registry/path.rb @@ -14,24 +14,23 @@ module ContainerRegistry def initialize(path) @path = path - @nodes = path.to_s.split('/') - end - - def to_s - @path end def valid? @path =~ Gitlab::Regex.container_repository_name_regex && - @nodes.size > 1 && - @nodes.size < Namespace::NUMBER_OF_ANCESTORS_ALLOWED + nodes.size > 1 && + nodes.size < Namespace::NUMBER_OF_ANCESTORS_ALLOWED + end + + def nodes + @nodes ||= @path.to_s.split('/') end def components raise InvalidRegistryPathError unless valid? - @components ||= @nodes.size.downto(2).map do |length| - @nodes.take(length).join('/') + @components ||= nodes.size.downto(2).map do |length| + nodes.take(length).join('/') end end @@ -51,7 +50,7 @@ module ContainerRegistry end def repository_project - @project ||= Project.where_full_path_in(components.first(3))&.first + @project ||= Project.where_full_path_in(components.first(3)).first end def repository_name @@ -59,5 +58,9 @@ module ContainerRegistry @path.remove(%r(^?#{Regexp.escape(repository_project.full_path)}/?)) end + + def to_s + @path + end end end diff --git a/spec/lib/container_registry/path_spec.rb b/spec/lib/container_registry/path_spec.rb index 1973da65f0a..825c61beeb3 100644 --- a/spec/lib/container_registry/path_spec.rb +++ b/spec/lib/container_registry/path_spec.rb @@ -3,6 +3,14 @@ require 'spec_helper' describe ContainerRegistry::Path do subject { described_class.new(path) } + describe '#nodes' do + let(:path) { 'path/to/some/project' } + + it 'splits elements by a forward slash' do + expect(subject.nodes).to eq %w[path to some project] + end + end + describe '#components' do context 'when repository path is valid' do let(:path) { 'path/to/some/project' } -- cgit v1.2.1 From 6fefa794304a6233368b592422ea0fb71a2700f0 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Apr 2017 11:51:13 +0200 Subject: Validate uniqueness of container repository name --- app/models/container_repository.rb | 1 + spec/models/container_repository_spec.rb | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index ab04258ab8d..0cf781799e9 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -2,6 +2,7 @@ class ContainerRepository < ActiveRecord::Base belongs_to :project validates :name, length: { minimum: 0, allow_nil: false } + validates :name, uniqueness: { scope: :project_id } delegate :client, to: :registry diff --git a/spec/models/container_repository_spec.rb b/spec/models/container_repository_spec.rb index f794085fc45..503da6556c0 100644 --- a/spec/models/container_repository_spec.rb +++ b/spec/models/container_repository_spec.rb @@ -21,6 +21,13 @@ describe ContainerRepository do headers: { 'Content-Type' => 'application/json' }) end + describe 'validations' do + it 'validates uniqueness of name scoped to project' do + expect(subject).to validate_uniqueness_of(:name) + .scoped_to(:project_id) + end + end + describe 'associations' do it 'belongs to the project' do expect(container_repository).to belong_to(:project) -- cgit v1.2.1 From fd30b3d4972180b6e91d0180b140f32622c48b1d Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Apr 2017 12:49:54 +0200 Subject: Ensure root container repository when visiting registry Root container repository is a images repository that had been created before 9.1, before we introduced multi-level images support. --- .../projects/registry/repositories_controller.rb | 16 +++++ app/models/container_repository.rb | 12 +++- .../registry/repositories_controller_spec.rb | 82 ++++++++++++++++++++++ spec/factories/container_repositories.rb | 4 ++ spec/models/container_repository_spec.rb | 37 ++++++++++ 5 files changed, 149 insertions(+), 2 deletions(-) create mode 100644 spec/controllers/projects/registry/repositories_controller_spec.rb diff --git a/app/controllers/projects/registry/repositories_controller.rb b/app/controllers/projects/registry/repositories_controller.rb index 2901d83fcef..ab4344bcf3c 100644 --- a/app/controllers/projects/registry/repositories_controller.rb +++ b/app/controllers/projects/registry/repositories_controller.rb @@ -2,6 +2,7 @@ module Projects module Registry class RepositoriesController < ::Projects::Registry::ApplicationController before_action :authorize_update_container_image!, only: [:destroy] + before_action :ensure_root_container_repository!, only: [:index] def index @images = project.container_repositories @@ -22,6 +23,21 @@ module Projects def image @image ||= project.container_repositories.find_by(id: params[:id]) end + + ## + # Container repository object for root project path. + # + # Needed to maintain a backwards compatibility. + # + def ensure_root_container_repository! + ContainerRegistry::Path.new(@project.full_path).tap do |path| + return if path.has_repository? + + ContainerRepository.build_from_path(path).tap do |repository| + repository.save if repository.has_tags? + end + end + end end end end diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index 0cf781799e9..36158d75ae8 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -48,6 +48,10 @@ class ContainerRepository < ActiveRecord::Base tags.to_a.any? end + def root_repository? + name.empty? + end + def delete_tags! return unless has_tags? @@ -58,8 +62,12 @@ class ContainerRepository < ActiveRecord::Base end end + def self.build_from_path(path) + self.new(project: path.repository_project, + name: path.repository_name) + end + def self.create_from_path!(path) - self.create(project: path.repository_project, - name: path.repository_name) + build_from_path(path).tap(&:save!) end end diff --git a/spec/controllers/projects/registry/repositories_controller_spec.rb b/spec/controllers/projects/registry/repositories_controller_spec.rb new file mode 100644 index 00000000000..e514f535f1d --- /dev/null +++ b/spec/controllers/projects/registry/repositories_controller_spec.rb @@ -0,0 +1,82 @@ +require 'spec_helper' + +describe Projects::Registry::RepositoriesController do + let(:user) { create(:user) } + let(:project) { create(:empty_project, :private) } + + before do + sign_in(user) + end + + context 'when user has access to registry' do + before do + project.add_developer(user) + end + + describe 'GET index' do + context 'when root container repository exists' do + before do + create(:container_repository, :root, project: project) + end + + it 'does not create root container repository' do + expect { go_to_index }.not_to change { ContainerRepository.all.count } + end + end + + context 'when root container repository does not exist' do + context 'when there are tags for this repository' do + before do + stub_container_registry_tags(%w[rc1 latest]) + end + + it 'successfully renders container repositories' do + go_to_index + + expect(response).to have_http_status(:ok) + end + + it 'creates a root container repository' do + expect { go_to_index }.to change { ContainerRepository.all.count }.by(1) + expect(ContainerRepository.first).to be_root_repository + end + end + + context 'when there are no tags for this repository' do + before do + stub_container_registry_tags(*[]) + end + + it 'successfully renders container repositories' do + go_to_index + + expect(response).to have_http_status(:ok) + end + + it 'does not ensure root container repository' do + expect { go_to_index }.not_to change { ContainerRepository.all.count } + end + end + end + end + end + + context 'when user does not have access to registry' do + describe 'GET index' do + it 'responds with 404' do + go_to_index + + expect(response).to have_http_status(:not_found) + end + + it 'does not ensure root container repository' do + expect { go_to_index }.not_to change { ContainerRepository.all.count } + end + end + end + + def go_to_index + get :index, namespace_id: project.namespace, + project_id: project + end +end diff --git a/spec/factories/container_repositories.rb b/spec/factories/container_repositories.rb index 4919a03cdf2..3fcad9fd4b3 100644 --- a/spec/factories/container_repositories.rb +++ b/spec/factories/container_repositories.rb @@ -7,6 +7,10 @@ FactoryGirl.define do tags [] end + trait :root do + name '' + end + after(:build) do |repository, evaluator| next if evaluator.tags.to_a.none? diff --git a/spec/models/container_repository_spec.rb b/spec/models/container_repository_spec.rb index 503da6556c0..b23ed8e30b6 100644 --- a/spec/models/container_repository_spec.rb +++ b/spec/models/container_repository_spec.rb @@ -98,6 +98,43 @@ describe ContainerRepository do end end + describe '#root_repository?' do + context 'when repository is a root repository' do + let(:repository) { create(:container_repository, :root) } + + it 'returns true' do + expect(repository).to be_root_repository + end + end + + context 'when repository is not a root repository' do + it 'returns false' do + expect(container_repository).not_to be_root_repository + end + end + end + + describe '.build_from_path' do + let(:path) { project.full_path + '/some/image' } + let(:repository_path) { ContainerRegistry::Path.new(path) } + + let(:repository) do + described_class.build_from_path(ContainerRegistry::Path.new(path)) + end + + it 'fabricates repository assigned to a correct project' do + expect(repository.project).to eq project + end + + it 'fabricates repository with a correct name' do + expect(repository.name).to eq 'some/image' + end + + it 'is not persisted' do + expect(repository).not_to be_persisted + end + end + describe '.create_from_path!' do let(:repository) do described_class.create_from_path!(ContainerRegistry::Path.new(path)) -- cgit v1.2.1 From 0af4cbc57266cd7d2c433442c537da5c8970a3da Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Apr 2017 13:05:46 +0200 Subject: Simplify container repository build method specs --- spec/models/container_repository_spec.rb | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/spec/models/container_repository_spec.rb b/spec/models/container_repository_spec.rb index b23ed8e30b6..1a29cc9a096 100644 --- a/spec/models/container_repository_spec.rb +++ b/spec/models/container_repository_spec.rb @@ -115,11 +115,12 @@ describe ContainerRepository do end describe '.build_from_path' do - let(:path) { project.full_path + '/some/image' } - let(:repository_path) { ContainerRegistry::Path.new(path) } + let(:registry_path) do + ContainerRegistry::Path.new(project.full_path + '/some/image') + end let(:repository) do - described_class.build_from_path(ContainerRegistry::Path.new(path)) + described_class.build_from_path(registry_path) end it 'fabricates repository assigned to a correct project' do -- cgit v1.2.1 From b90f83e93319d54d558b29838d7453444f3fcf11 Mon Sep 17 00:00:00 2001 From: Annabel Dunstone Gray Date: Fri, 31 Mar 2017 15:13:32 -0500 Subject: Start adding profile icons --- app/assets/stylesheets/pages/events.scss | 16 ++++++---------- app/views/events/_event.html.haml | 5 +++-- app/views/events/event/_common.html.haml | 10 +++++++++- app/views/shared/icons/_code_fork.svg | 1 + app/views/shared/icons/_comment_o.svg | 1 + app/views/shared/icons/_icon_status_closed.svg | 1 + app/views/shared/icons/_icon_status_open.svg | 1 + app/views/shared/icons/_trash_o.svg | 1 + 8 files changed, 23 insertions(+), 13 deletions(-) create mode 100644 app/views/shared/icons/_code_fork.svg create mode 100644 app/views/shared/icons/_comment_o.svg create mode 100644 app/views/shared/icons/_icon_status_closed.svg create mode 100644 app/views/shared/icons/_icon_status_open.svg create mode 100644 app/views/shared/icons/_trash_o.svg diff --git a/app/assets/stylesheets/pages/events.scss b/app/assets/stylesheets/pages/events.scss index 08398bb43a2..14ff6ac9a9a 100644 --- a/app/assets/stylesheets/pages/events.scss +++ b/app/assets/stylesheets/pages/events.scss @@ -4,16 +4,11 @@ */ .event-item { font-size: $gl-font-size; - padding: $gl-padding-top 0 $gl-padding-top ($gl-avatar-size + $gl-padding-top); + padding: $gl-padding; border-bottom: 1px solid $white-normal; color: $list-text-color; &.event-inline { - .avatar { - position: relative; - top: -2px; - } - .event-title, .event-item-timestamp { line-height: 40px; @@ -24,16 +19,17 @@ color: $gl-text-color; } - .avatar { - margin-left: -($gl-avatar-size + $gl-padding-top); - } - .event-title { @include str-truncated(calc(100% - 174px)); font-weight: 600; color: $list-text-color; } + svg { + height: 16px; + width: 16px; + } + .event-body { margin-right: 174px; diff --git a/app/views/events/_event.html.haml b/app/views/events/_event.html.haml index a0bd14df209..bf7a9af99cd 100644 --- a/app/views/events/_event.html.haml +++ b/app/views/events/_event.html.haml @@ -3,13 +3,14 @@ .event-item-timestamp #{time_ago_with_tooltip(event.created_at)} - = author_avatar(event, size: 40) - - if event.created_project? + = custom_icon("icon_status_open") = render "events/event/created_project", event: event - elsif event.push? + = custom_icon("icon_commit") = render "events/event/push", event: event - elsif event.commented? + = custom_icon("comment_o") = render "events/event/note", event: event - else = render "events/event/common", event: event diff --git a/app/views/events/event/_common.html.haml b/app/views/events/event/_common.html.haml index 2fb6b5647da..0d230e3f3f2 100644 --- a/app/views/events/event/_common.html.haml +++ b/app/views/events/event/_common.html.haml @@ -1,5 +1,13 @@ +- if event.target + - if event.target_type == "MergeRequest" + - if event.action_name == "opened" + = custom_icon("icon_status_open") + - elsif event.action_name == "closed" + = custom_icon("icon_status_closed") + - else + = custom_icon("code_fork") + .event-title - %span.author_name= link_to_author event %span{ class: event.action_name } - if event.target = event.action_name diff --git a/app/views/shared/icons/_code_fork.svg b/app/views/shared/icons/_code_fork.svg new file mode 100644 index 00000000000..8347dce2d5b --- /dev/null +++ b/app/views/shared/icons/_code_fork.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/app/views/shared/icons/_comment_o.svg b/app/views/shared/icons/_comment_o.svg new file mode 100644 index 00000000000..55807f0840a --- /dev/null +++ b/app/views/shared/icons/_comment_o.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/app/views/shared/icons/_icon_status_closed.svg b/app/views/shared/icons/_icon_status_closed.svg new file mode 100644 index 00000000000..de448ee1194 --- /dev/null +++ b/app/views/shared/icons/_icon_status_closed.svg @@ -0,0 +1 @@ + diff --git a/app/views/shared/icons/_icon_status_open.svg b/app/views/shared/icons/_icon_status_open.svg new file mode 100644 index 00000000000..ed58d23c626 --- /dev/null +++ b/app/views/shared/icons/_icon_status_open.svg @@ -0,0 +1 @@ + diff --git a/app/views/shared/icons/_trash_o.svg b/app/views/shared/icons/_trash_o.svg new file mode 100644 index 00000000000..ea073d7fe67 --- /dev/null +++ b/app/views/shared/icons/_trash_o.svg @@ -0,0 +1 @@ + \ No newline at end of file -- cgit v1.2.1 From 633457563a9e9ef9473a00019dd243430aa08e80 Mon Sep 17 00:00:00 2001 From: Annabel Dunstone Gray Date: Fri, 31 Mar 2017 15:39:31 -0500 Subject: Differentiate between event types --- app/assets/stylesheets/pages/events.scss | 7 ++++++- app/views/events/_event.html.haml | 3 --- app/views/events/event/_common.html.haml | 14 ++++++++------ app/views/events/event/_created_project.html.haml | 4 +++- app/views/events/event/_note.html.haml | 4 +++- app/views/events/event/_push.html.haml | 5 +++-- 6 files changed, 23 insertions(+), 14 deletions(-) diff --git a/app/assets/stylesheets/pages/events.scss b/app/assets/stylesheets/pages/events.scss index 14ff6ac9a9a..35571748118 100644 --- a/app/assets/stylesheets/pages/events.scss +++ b/app/assets/stylesheets/pages/events.scss @@ -4,7 +4,7 @@ */ .event-item { font-size: $gl-font-size; - padding: $gl-padding; + padding: $gl-padding 0; border-bottom: 1px solid $white-normal; color: $list-text-color; @@ -25,6 +25,11 @@ color: $list-text-color; } + .event-icon { + display: inline-block; + margin: 0 10px; + } + svg { height: 16px; width: 16px; diff --git a/app/views/events/_event.html.haml b/app/views/events/_event.html.haml index bf7a9af99cd..53a33adc14d 100644 --- a/app/views/events/_event.html.haml +++ b/app/views/events/_event.html.haml @@ -4,13 +4,10 @@ #{time_ago_with_tooltip(event.created_at)} - if event.created_project? - = custom_icon("icon_status_open") = render "events/event/created_project", event: event - elsif event.push? - = custom_icon("icon_commit") = render "events/event/push", event: event - elsif event.commented? - = custom_icon("comment_o") = render "events/event/note", event: event - else = render "events/event/common", event: event diff --git a/app/views/events/event/_common.html.haml b/app/views/events/event/_common.html.haml index 0d230e3f3f2..98a4dda5b4d 100644 --- a/app/views/events/event/_common.html.haml +++ b/app/views/events/event/_common.html.haml @@ -1,11 +1,13 @@ - if event.target - - if event.target_type == "MergeRequest" - - if event.action_name == "opened" + - if event.action_name == "opened" + .event-icon.open-icon = custom_icon("icon_status_open") - - elsif event.action_name == "closed" - = custom_icon("icon_status_closed") - - else - = custom_icon("code_fork") + - elsif event.action_name == "closed" + .event-icon.closed-icon + = custom_icon("icon_status_closed") + - else + .event-icon.fork-icon + = custom_icon("code_fork") .event-title %span{ class: event.action_name } diff --git a/app/views/events/event/_created_project.html.haml b/app/views/events/event/_created_project.html.haml index 80cf2344fe1..6fb084df374 100644 --- a/app/views/events/event/_created_project.html.haml +++ b/app/views/events/event/_created_project.html.haml @@ -1,5 +1,7 @@ +.event-icon.open-icon += custom_icon("icon_status_open") + .event-title - %span.author_name= link_to_author event %span{ class: event.action_name } = event_action_name(event) diff --git a/app/views/events/event/_note.html.haml b/app/views/events/event/_note.html.haml index 64b5a733b77..755750b88d1 100644 --- a/app/views/events/event/_note.html.haml +++ b/app/views/events/event/_note.html.haml @@ -1,5 +1,7 @@ +.event-icon + = custom_icon("comment_o") + .event-title - %span.author_name= link_to_author event = event.action_name = event_note_title_html(event) diff --git a/app/views/events/event/_push.html.haml b/app/views/events/event/_push.html.haml index efd13aabf20..be1730541ef 100644 --- a/app/views/events/event/_push.html.haml +++ b/app/views/events/event/_push.html.haml @@ -1,7 +1,9 @@ - project = event.project +.event-icon + = custom_icon("icon_commit") + .event-title - %span.author_name= link_to_author event %span.pushed #{event.action_name} #{event.ref_type} %strong - commits_link = namespace_project_commits_path(project.namespace, project, event.ref_name) @@ -48,4 +50,3 @@ .event-body %ul.well-list.event_commits = render "events/commit", commit: last_commit, project: project, event: event - -- cgit v1.2.1 From 32bb33f717ccae968949d30d058e828e31d72cf2 Mon Sep 17 00:00:00 2001 From: Annabel Dunstone Gray Date: Fri, 31 Mar 2017 16:09:58 -0500 Subject: Change color of icons --- app/assets/stylesheets/pages/events.scss | 43 ++++++++++++++++------- app/views/events/event/_common.html.haml | 10 +++--- app/views/events/event/_created_project.html.haml | 4 +-- app/views/events/event/_note.html.haml | 2 +- app/views/events/event/_push.html.haml | 2 +- 5 files changed, 40 insertions(+), 21 deletions(-) diff --git a/app/assets/stylesheets/pages/events.scss b/app/assets/stylesheets/pages/events.scss index 35571748118..14ad26bcd6b 100644 --- a/app/assets/stylesheets/pages/events.scss +++ b/app/assets/stylesheets/pages/events.scss @@ -4,11 +4,16 @@ */ .event-item { font-size: $gl-font-size; - padding: $gl-padding 0; + padding: $gl-padding-top 0 $gl-padding-top ($gl-avatar-size + $gl-padding-top); border-bottom: 1px solid $white-normal; color: $list-text-color; + position: relative; &.event-inline { + .profile-icon { + top: 20px; + } + .event-title, .event-item-timestamp { line-height: 40px; @@ -19,22 +24,36 @@ color: $gl-text-color; } + .profile-icon { + position: absolute; + left: 0; + top: 14px; + + svg { + width: 20px; + height: auto; + fill: $gl-text-color-secondary; + } + + &.open-icon svg { + fill: $green-300; + } + + &.closed-icon svg { + fill: $red-300; + } + + &.fork-icon svg { + fill: $blue-300; + } + } + .event-title { @include str-truncated(calc(100% - 174px)); font-weight: 600; color: $list-text-color; } - .event-icon { - display: inline-block; - margin: 0 10px; - } - - svg { - height: 16px; - width: 16px; - } - .event-body { margin-right: 174px; @@ -164,7 +183,7 @@ max-width: 100%; } - .avatar { + .profile-icon { display: none; } diff --git a/app/views/events/event/_common.html.haml b/app/views/events/event/_common.html.haml index 98a4dda5b4d..2a98e58a03a 100644 --- a/app/views/events/event/_common.html.haml +++ b/app/views/events/event/_common.html.haml @@ -1,13 +1,13 @@ - if event.target - if event.action_name == "opened" - .event-icon.open-icon + .profile-icon.open-icon = custom_icon("icon_status_open") - elsif event.action_name == "closed" - .event-icon.closed-icon - = custom_icon("icon_status_closed") + .profile-icon.closed-icon + = custom_icon("icon_status_closed") - else - .event-icon.fork-icon - = custom_icon("code_fork") + .profile-icon.fork-icon + = custom_icon("code_fork") .event-title %span{ class: event.action_name } diff --git a/app/views/events/event/_created_project.html.haml b/app/views/events/event/_created_project.html.haml index 6fb084df374..340d8c61026 100644 --- a/app/views/events/event/_created_project.html.haml +++ b/app/views/events/event/_created_project.html.haml @@ -1,5 +1,5 @@ -.event-icon.open-icon -= custom_icon("icon_status_open") +.profile-icon.open-icon + = custom_icon("icon_status_open") .event-title %span{ class: event.action_name } diff --git a/app/views/events/event/_note.html.haml b/app/views/events/event/_note.html.haml index 755750b88d1..603bed6d705 100644 --- a/app/views/events/event/_note.html.haml +++ b/app/views/events/event/_note.html.haml @@ -1,4 +1,4 @@ -.event-icon +.profile-icon = custom_icon("comment_o") .event-title diff --git a/app/views/events/event/_push.html.haml b/app/views/events/event/_push.html.haml index be1730541ef..4abda4d9db4 100644 --- a/app/views/events/event/_push.html.haml +++ b/app/views/events/event/_push.html.haml @@ -1,6 +1,6 @@ - project = event.project -.event-icon +.profile-icon = custom_icon("icon_commit") .event-title -- cgit v1.2.1 From 99ff4d30b45bcef8d01ef24801842f7b9ffc66f5 Mon Sep 17 00:00:00 2001 From: Annabel Dunstone Gray Date: Fri, 31 Mar 2017 16:17:34 -0500 Subject: Add deleted branch icon --- app/views/events/event/_push.html.haml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/views/events/event/_push.html.haml b/app/views/events/event/_push.html.haml index 4abda4d9db4..1583f380737 100644 --- a/app/views/events/event/_push.html.haml +++ b/app/views/events/event/_push.html.haml @@ -1,7 +1,10 @@ - project = event.project .profile-icon - = custom_icon("icon_commit") + - if event.action_name == "deleted" + = custom_icon("trash_o") + - else + = custom_icon("icon_commit") .event-title %span.pushed #{event.action_name} #{event.ref_type} -- cgit v1.2.1 From c998ce66ada6b5281c7b1a2a8227866b0159c9ce Mon Sep 17 00:00:00 2001 From: Annabel Dunstone Gray Date: Fri, 31 Mar 2017 16:24:15 -0500 Subject: Add changelog --- app/assets/stylesheets/pages/events.scss | 2 +- changelogs/unreleased/29128-profile-page-icons.yml | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 changelogs/unreleased/29128-profile-page-icons.yml diff --git a/app/assets/stylesheets/pages/events.scss b/app/assets/stylesheets/pages/events.scss index 14ad26bcd6b..e7f9bbbc62f 100644 --- a/app/assets/stylesheets/pages/events.scss +++ b/app/assets/stylesheets/pages/events.scss @@ -4,7 +4,7 @@ */ .event-item { font-size: $gl-font-size; - padding: $gl-padding-top 0 $gl-padding-top ($gl-avatar-size + $gl-padding-top); + padding: $gl-padding-top 0 $gl-padding-top 40px; border-bottom: 1px solid $white-normal; color: $list-text-color; position: relative; diff --git a/changelogs/unreleased/29128-profile-page-icons.yml b/changelogs/unreleased/29128-profile-page-icons.yml new file mode 100644 index 00000000000..0215f5c0e8f --- /dev/null +++ b/changelogs/unreleased/29128-profile-page-icons.yml @@ -0,0 +1,4 @@ +--- +title: Add helpful icons to profile events +merge_request: +author: -- cgit v1.2.1 From d4377b8c12e106892098d66321059ca6bc1affdd Mon Sep 17 00:00:00 2001 From: Annabel Dunstone Gray Date: Mon, 3 Apr 2017 08:33:02 -0500 Subject: Update commit icon; fix specs --- app/views/shared/icons/_icon_commit.svg | 4 +--- features/steps/shared/project.rb | 2 +- spec/features/dashboard/project_member_activity_index_spec.rb | 6 +++--- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/app/views/shared/icons/_icon_commit.svg b/app/views/shared/icons/_icon_commit.svg index 0e96035b7b7..15e83dfdb53 100644 --- a/app/views/shared/icons/_icon_commit.svg +++ b/app/views/shared/icons/_icon_commit.svg @@ -1,3 +1 @@ - - - + \ No newline at end of file diff --git a/features/steps/shared/project.rb b/features/steps/shared/project.rb index 345a28f27dc..47bdc708e09 100644 --- a/features/steps/shared/project.rb +++ b/features/steps/shared/project.rb @@ -97,7 +97,7 @@ module SharedProject step 'I should see project "Shop" activity feed' do project = Project.find_by(name: "Shop") - expect(page).to have_content "#{@user.name} pushed new branch fix at #{project.name_with_namespace}" + expect(page).to have_content "pushed new branch fix at #{project.name_with_namespace}" end step 'I should see project settings' do diff --git a/spec/features/dashboard/project_member_activity_index_spec.rb b/spec/features/dashboard/project_member_activity_index_spec.rb index 49d93db58a9..d62839a09ef 100644 --- a/spec/features/dashboard/project_member_activity_index_spec.rb +++ b/spec/features/dashboard/project_member_activity_index_spec.rb @@ -21,20 +21,20 @@ feature 'Project member activity', feature: true, js: true do context 'when a user joins the project' do before { visit_activities_and_wait_with_event(Event::JOINED) } - it { is_expected.to eq("#{user.name} joined project") } + it { is_expected.to eq("joined project") } end context 'when a user leaves the project' do before { visit_activities_and_wait_with_event(Event::LEFT) } - it { is_expected.to eq("#{user.name} left project") } + it { is_expected.to eq("left project") } end context 'when a users membership expires for the project' do before { visit_activities_and_wait_with_event(Event::EXPIRED) } it "presents the correct message" do - message = "#{user.name} removed due to membership expiration from project" + message = "removed due to membership expiration from project" is_expected.to eq(message) end end -- cgit v1.2.1 From baa00d542478759be225a45dc805d0314e1921d2 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Apr 2017 15:52:24 +0200 Subject: Refactor container registry repository tag stubs --- .../registry/repositories_controller_spec.rb | 7 +++--- spec/features/container_registry_spec.rb | 3 ++- .../security/project/internal_access_spec.rb | 2 +- .../security/project/private_access_spec.rb | 2 +- .../security/project/public_access_spec.rb | 2 +- spec/models/namespace_spec.rb | 2 +- spec/models/project_spec.rb | 2 +- spec/services/projects/destroy_service_spec.rb | 2 +- spec/services/projects/transfer_service_spec.rb | 2 +- spec/support/stub_gitlab_calls.rb | 28 +++++++++++++++++----- 10 files changed, 35 insertions(+), 17 deletions(-) diff --git a/spec/controllers/projects/registry/repositories_controller_spec.rb b/spec/controllers/projects/registry/repositories_controller_spec.rb index e514f535f1d..29f0a65483f 100644 --- a/spec/controllers/projects/registry/repositories_controller_spec.rb +++ b/spec/controllers/projects/registry/repositories_controller_spec.rb @@ -24,10 +24,11 @@ describe Projects::Registry::RepositoriesController do end end - context 'when root container repository does not exist' do + context 'when root container repository is not created' do context 'when there are tags for this repository' do before do - stub_container_registry_tags(%w[rc1 latest]) + stub_container_registry_tags(repository: project.full_path, + tags: %w[rc1 latest]) end it 'successfully renders container repositories' do @@ -44,7 +45,7 @@ describe Projects::Registry::RepositoriesController do context 'when there are no tags for this repository' do before do - stub_container_registry_tags(*[]) + stub_container_registry_tags(repository: :any, tags: []) end it 'successfully renders container repositories' do diff --git a/spec/features/container_registry_spec.rb b/spec/features/container_registry_spec.rb index 42431cbe731..fa7adbe71ea 100644 --- a/spec/features/container_registry_spec.rb +++ b/spec/features/container_registry_spec.rb @@ -12,7 +12,7 @@ describe "Container Registry" do login_as(user) project.add_developer(user) stub_container_registry_config(enabled: true) - stub_container_registry_tags(%w[latest]) + stub_container_registry_tags(repository: :any, tags: []) end context 'when there are no image repositories' do @@ -25,6 +25,7 @@ describe "Container Registry" do context 'when there are image repositories' do before do + stub_container_registry_tags(repository: %r{my/image}, tags: %w[latest]) project.container_repositories << container_repository end diff --git a/spec/features/security/project/internal_access_spec.rb b/spec/features/security/project/internal_access_spec.rb index a961d8b4f69..6ecdc8cbb71 100644 --- a/spec/features/security/project/internal_access_spec.rb +++ b/spec/features/security/project/internal_access_spec.rb @@ -446,7 +446,7 @@ describe "Internal Project Access", feature: true do let(:container_repository) { create(:container_repository) } before do - stub_container_registry_tags('latest') + stub_container_registry_tags(repository: :any, tags: ['latest']) stub_container_registry_config(enabled: true) project.container_repositories << container_repository end diff --git a/spec/features/security/project/private_access_spec.rb b/spec/features/security/project/private_access_spec.rb index b7e42e67d82..a8fc0624588 100644 --- a/spec/features/security/project/private_access_spec.rb +++ b/spec/features/security/project/private_access_spec.rb @@ -435,7 +435,7 @@ describe "Private Project Access", feature: true do let(:container_repository) { create(:container_repository) } before do - stub_container_registry_tags('latest') + stub_container_registry_tags(repository: :any, tags: ['latest']) stub_container_registry_config(enabled: true) project.container_repositories << container_repository end diff --git a/spec/features/security/project/public_access_spec.rb b/spec/features/security/project/public_access_spec.rb index 02660984b29..08cf2bf5291 100644 --- a/spec/features/security/project/public_access_spec.rb +++ b/spec/features/security/project/public_access_spec.rb @@ -446,7 +446,7 @@ describe "Public Project Access", feature: true do let(:container_repository) { create(:container_repository) } before do - stub_container_registry_tags('latest') + stub_container_registry_tags(repository: :any, tags:['latest']) stub_container_registry_config(enabled: true) project.container_repositories << container_repository end diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb index e1bfd20a6aa..197fd558dd7 100644 --- a/spec/models/namespace_spec.rb +++ b/spec/models/namespace_spec.rb @@ -153,7 +153,7 @@ describe Namespace, models: true do before do stub_container_registry_config(enabled: true) - stub_container_registry_tags('tag') + stub_container_registry_tags(repository: :any, tags: ['tag']) create(:empty_project, namespace: @namespace, container_repositories: [container_repository]) diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 841c7d4cb5b..062d7fdd4ae 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -1190,7 +1190,7 @@ describe Project, models: true do before do stub_container_registry_config(enabled: true) - stub_container_registry_tags('tag') + stub_container_registry_tags(repository: :any, tags: ['tag']) project.container_repositories << container_repository end diff --git a/spec/services/projects/destroy_service_spec.rb b/spec/services/projects/destroy_service_spec.rb index cf1f90becfd..5ef07c8275e 100644 --- a/spec/services/projects/destroy_service_spec.rb +++ b/spec/services/projects/destroy_service_spec.rb @@ -94,7 +94,7 @@ describe Projects::DestroyService, services: true do before do stub_container_registry_config(enabled: true) - stub_container_registry_tags('tag') + stub_container_registry_tags(repository: :any, tags: ['tag']) project.container_repositories << container_repository end diff --git a/spec/services/projects/transfer_service_spec.rb b/spec/services/projects/transfer_service_spec.rb index 81e15f9dba6..29ccce59c53 100644 --- a/spec/services/projects/transfer_service_spec.rb +++ b/spec/services/projects/transfer_service_spec.rb @@ -33,7 +33,7 @@ describe Projects::TransferService, services: true do before do stub_container_registry_config(enabled: true) - stub_container_registry_tags('tag') + stub_container_registry_tags(repository: :any, tags: ['tag']) project.container_repositories << container_repository end diff --git a/spec/support/stub_gitlab_calls.rb b/spec/support/stub_gitlab_calls.rb index dbf3ace37c3..ded2d593059 100644 --- a/spec/support/stub_gitlab_calls.rb +++ b/spec/support/stub_gitlab_calls.rb @@ -31,20 +31,36 @@ module StubGitlabCalls .to receive(:full_access_token).and_return('token') end - def stub_container_registry_tags(*tags) + def stub_container_registry_tags(repository: :any, tags:) + repository = any_args if repository == :any + allow_any_instance_of(ContainerRegistry::Client) - .to receive(:repository_tags).and_return({ 'tags' => tags }) + .to receive(:repository_tags).with(repository) + .and_return({ 'tags' => tags }) allow_any_instance_of(ContainerRegistry::Client) - .to receive(:repository_manifest).and_return( - JSON.parse(File.read(Rails.root + 'spec/fixtures/container_registry/tag_manifest.json'))) + .to receive(:repository_manifest).with(repository) + .and_return(stub_container_registry_tag_manifest) - allow_any_instance_of(ContainerRegistry::Client).to receive(:blob).and_return( - File.read(Rails.root + 'spec/fixtures/container_registry/config_blob.json')) + allow_any_instance_of(ContainerRegistry::Client) + .to receive(:blob).with(repository) + .and_return(stub_container_registry_blob) end private + def stub_container_registry_tag_manifest + fixture_path = 'spec/fixtures/container_registry/tag_manifest.json' + + JSON.parse(File.read(Rails.root + fixture_path)) + end + + def stub_container_registry_blob + fixture_path = 'spec/fixtures/container_registry/config_blob.json' + + File.read(Rails.root + fixture_path) + end + def gitlab_url Gitlab.config.gitlab.url end -- cgit v1.2.1 From fc5eb3157082d923b80596485b302504ce0671ea Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Apr 2017 15:53:51 +0200 Subject: Fix Rubocop offenses in code related to the registry --- app/controllers/projects/registry/repositories_controller.rb | 2 +- spec/features/security/project/public_access_spec.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/projects/registry/repositories_controller.rb b/app/controllers/projects/registry/repositories_controller.rb index ab4344bcf3c..cb1ea45dd04 100644 --- a/app/controllers/projects/registry/repositories_controller.rb +++ b/app/controllers/projects/registry/repositories_controller.rb @@ -31,7 +31,7 @@ module Projects # def ensure_root_container_repository! ContainerRegistry::Path.new(@project.full_path).tap do |path| - return if path.has_repository? + break if path.has_repository? ContainerRepository.build_from_path(path).tap do |repository| repository.save if repository.has_tags? diff --git a/spec/features/security/project/public_access_spec.rb b/spec/features/security/project/public_access_spec.rb index 08cf2bf5291..c4d2f50ca14 100644 --- a/spec/features/security/project/public_access_spec.rb +++ b/spec/features/security/project/public_access_spec.rb @@ -446,7 +446,7 @@ describe "Public Project Access", feature: true do let(:container_repository) { create(:container_repository) } before do - stub_container_registry_tags(repository: :any, tags:['latest']) + stub_container_registry_tags(repository: :any, tags: ['latest']) stub_container_registry_config(enabled: true) project.container_repositories << container_repository end -- cgit v1.2.1 From a9b221d91a5529c514615b640cdbbaf6b99bf790 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Apr 2017 16:29:11 +0200 Subject: Refine method for checking project registry tags --- app/models/project.rb | 20 +++++++++++++++ spec/models/project_spec.rb | 60 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+) diff --git a/app/models/project.rb b/app/models/project.rb index 649d0e100c3..0adec807f34 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -406,6 +406,11 @@ class Project < ActiveRecord::Base end end + def has_container_registry_tags? + container_repositories.to_a.any?(&:has_tags?) || + has_root_container_repository_tags? + end + def commit(ref = 'HEAD') repository.commit(ref) end @@ -1373,4 +1378,19 @@ class Project < ActiveRecord::Base Project.unscoped.where(pending_delete: true).find_by_full_path(path_with_namespace) end + + ## + # This method is here because of support for legacy container repository + # which has exactly the same path like project does, but which might not be + # persisted in `container_repositories` table. + # + def has_root_container_repository_tags? + return false unless Gitlab.config.registry.enabled + + ContainerRegistry::Path.new(self.full_path).tap do |path| + ContainerRepository.build_from_path(path).tap do |repository| + return repository.has_tags? + end + end + end end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 062d7fdd4ae..4c13e53d831 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -1414,6 +1414,66 @@ describe Project, models: true do end end + describe '#has_container_registry_tags?' do + let(:project) { create(:empty_project) } + + context 'when container registry is enabled' do + before { stub_container_registry_config(enabled: true) } + + context 'when tags are present for multi-level registries' do + before do + create(:container_repository, project: project, name: 'image') + + stub_container_registry_tags(repository: /image/, + tags: %w[latest rc1]) + end + + it 'should have image tags' do + expect(project).to have_container_registry_tags + end + end + + context 'when tags are present for root repository' do + before do + stub_container_registry_tags(repository: project.full_path, + tags: %w[latest rc1 pre1]) + end + + it 'should have image tags' do + expect(project).to have_container_registry_tags + end + end + + context 'when there are no tags at all' do + before do + stub_container_registry_tags(repository: :any, tags: []) + end + + it 'should not have image tags' do + expect(project).not_to have_container_registry_tags + end + end + end + + context 'when container registry is disabled' do + before { stub_container_registry_config(enabled: false) } + + it 'should not have image tags' do + expect(project).not_to have_container_registry_tags + end + + it 'should not check root repository tags' do + expect(project).not_to receive(:full_path) + expect(project).not_to have_container_registry_tags + end + + it 'should iterate through container repositories' do + expect(project).to receive(:container_repositories) + expect(project).not_to have_container_registry_tags + end + end + end + describe '#latest_successful_builds_for' do def create_pipeline(status = 'success') create(:ci_pipeline, project: project, -- cgit v1.2.1 From e9d5e95c4403d39072b6d29555569b2d09b02fe6 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Apr 2017 16:34:18 +0200 Subject: Revert changes in services related to moving projects --- app/models/namespace.rb | 2 +- app/models/project.rb | 4 ++-- app/services/projects/transfer_service.rb | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 2c3473f9186..b57ed258af7 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -150,7 +150,7 @@ class Namespace < ActiveRecord::Base end def any_project_has_container_registry_images? - projects.joins(:container_repositories).any? + projects.any?(&:has_container_registry_tags?) end def send_update_instructions diff --git a/app/models/project.rb b/app/models/project.rb index 0adec807f34..fa64ccbf7e4 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -894,8 +894,8 @@ class Project < ActiveRecord::Base expire_caches_before_rename(old_path_with_namespace) - if container_repositories.present? - Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry images are present" + if has_container_registry_tags? + Rails.logger.error "Project #{old_path_with_namespace} cannot be renamed because container registry tags are present!" # we currently doesn't support renaming repository if it contains images in container registry raise StandardError.new('Project cannot be renamed, because images are present in its container registry') diff --git a/app/services/projects/transfer_service.rb b/app/services/projects/transfer_service.rb index f46bf884e37..c2c2addeebe 100644 --- a/app/services/projects/transfer_service.rb +++ b/app/services/projects/transfer_service.rb @@ -36,9 +36,9 @@ module Projects raise TransferError.new("Project with same path in target namespace already exists") end - unless project.container_repositories.empty? + if project.has_container_registry_tags? # we currently doesn't support renaming repository if it contains images in container registry - raise TransferError.new('Project cannot be transferred, because images are present in its container registry') + raise TransferError.new('Project cannot be transferred, because tags are present in its container registry') end project.expire_caches_before_rename(old_path) -- cgit v1.2.1 From 325908ad3931f12dcada654d164ae09ff5f00c6e Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Apr 2017 16:38:51 +0200 Subject: Remove changes unnecessary changes from namespace model --- app/models/namespace.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/models/namespace.rb b/app/models/namespace.rb index b57ed258af7..1d4b1f7d590 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -113,8 +113,8 @@ class Namespace < ActiveRecord::Base end def move_dir - if any_project_has_container_registry_images? - raise Gitlab::UpdatePathError.new('Namespace cannot be moved, because at least one project has images in container registry') + if any_project_has_container_registry_tags? + raise Gitlab::UpdatePathError.new('Namespace cannot be moved, because at least one project has tags in container registry') end # Move the namespace directory in all storages paths used by member projects @@ -149,7 +149,7 @@ class Namespace < ActiveRecord::Base end end - def any_project_has_container_registry_images? + def any_project_has_container_registry_tags? projects.any?(&:has_container_registry_tags?) end -- cgit v1.2.1 From d2a9c5d81fce78540cf2c2a70f020fc0b234ce5a Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Apr 2017 16:41:56 +0200 Subject: Remove unecessary changes from project transfer service --- app/services/projects/transfer_service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/projects/transfer_service.rb b/app/services/projects/transfer_service.rb index c2c2addeebe..da6e6acd4a7 100644 --- a/app/services/projects/transfer_service.rb +++ b/app/services/projects/transfer_service.rb @@ -37,7 +37,7 @@ module Projects end if project.has_container_registry_tags? - # we currently doesn't support renaming repository if it contains images in container registry + # we currently doesn't support renaming repository if it contains tags in container registry raise TransferError.new('Project cannot be transferred, because tags are present in its container registry') end -- cgit v1.2.1 From 1d6ddb05bb12dce9e90c76b73a22a21082506091 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Apr 2017 21:25:52 +0200 Subject: Fix namespace specs related to container registry --- spec/models/namespace_spec.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb index 197fd558dd7..a7e565ec645 100644 --- a/spec/models/namespace_spec.rb +++ b/spec/models/namespace_spec.rb @@ -161,7 +161,9 @@ describe Namespace, models: true do allow(@namespace).to receive(:path).and_return('new_path') end - it { expect { @namespace.move_dir }.to raise_error('Namespace cannot be moved, because at least one project has images in container registry') } + it 'raises an error about not movable project' do + expect { @namespace.move_dir }.to raise_error(/Namespace cannot be moved/) + end end context 'renaming a sub-group' do -- cgit v1.2.1 From 4fab9f24c07af110441a6db6d4f4d26197c155b7 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 3 Apr 2017 21:27:08 +0200 Subject: Fix documentation related to container registry --- doc/user/project/container_registry.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/user/project/container_registry.md b/doc/user/project/container_registry.md index 7524e70957f..6de75e43ed3 100644 --- a/doc/user/project/container_registry.md +++ b/doc/user/project/container_registry.md @@ -85,8 +85,7 @@ and click **Registry** in the project menu. This view will show you all tags in your project and will easily allow you to delete them. -![Container Registry panel](image-needs-update) -[//]: # (img/container_registry_panel.png) +![Container Registry panel](img/container_registry_panel.png) ## Build and push images using GitLab CI -- cgit v1.2.1 From 7e46d0a95fc4ce8d0d66e75e6b1c4ef473606e79 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Apr 2017 11:13:12 +0200 Subject: Fix container registry controller specs --- spec/controllers/projects/registry/repositories_controller_spec.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/controllers/projects/registry/repositories_controller_spec.rb b/spec/controllers/projects/registry/repositories_controller_spec.rb index 29f0a65483f..464302824a8 100644 --- a/spec/controllers/projects/registry/repositories_controller_spec.rb +++ b/spec/controllers/projects/registry/repositories_controller_spec.rb @@ -6,6 +6,7 @@ describe Projects::Registry::RepositoriesController do before do sign_in(user) + stub_container_registry_config(enabled: true) end context 'when user has access to registry' do -- cgit v1.2.1 From 008441c49e493b885586ba38993f54bdcb03799c Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Apr 2017 11:46:14 +0200 Subject: Improve container registry repository view partials --- .../registry/repositories/_image.html.haml | 25 +++++++++------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/app/views/projects/registry/repositories/_image.html.haml b/app/views/projects/registry/repositories/_image.html.haml index c3f8580d25b..e0d74789207 100644 --- a/app/views/projects/registry/repositories/_image.html.haml +++ b/app/views/projects/registry/repositories/_image.html.haml @@ -1,29 +1,21 @@ -- expanded = false .container-image.js-toggle-container .container-image-head = link_to "#", class: "js-toggle-button" do - - if expanded - = icon("chevron-up") - - else - = icon("chevron-down") - + = icon("chevron-down") = escape_once(image.path) + = clipboard_button(clipboard_text: "docker pull #{image.path}") + .controls.hidden-xs.pull-right = link_to namespace_project_container_registry_path(@project.namespace, @project, image), class: 'btn btn-remove has-tooltip', title: 'Remove repository', data: { confirm: 'Are you sure?' }, method: :delete do - = icon("trash cred") - - - .container-image-tags.js-toggle-content{ class: ("hide" unless expanded) } - - if image.tags.blank? - %li - .nothing-here-block No tags in Container Registry for this container image. + = icon('trash cred') - - else + .container-image-tags.js-toggle-content{ class: 'hide' } + - if image.has_tags? .table-holder %table.table.tags %thead @@ -34,5 +26,8 @@ %th Created - if can?(current_user, :update_container_image, @project) %th - = render partial: 'tag', collection: image.tags + - else + %li + .nothing-here-block No tags in Container Registry for this container image. + -- cgit v1.2.1 From 59b5843eb0d329f314a58c074c42ecaa4bd24f25 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Apr 2017 11:50:26 +0200 Subject: Improve wording in registry notifications in the UI --- app/controllers/projects/registry/repositories_controller.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/projects/registry/repositories_controller.rb b/app/controllers/projects/registry/repositories_controller.rb index cb1ea45dd04..2e18075d40f 100644 --- a/app/controllers/projects/registry/repositories_controller.rb +++ b/app/controllers/projects/registry/repositories_controller.rb @@ -11,10 +11,10 @@ module Projects def destroy if image.destroy redirect_to project_container_registry_path(@project), - notice: 'Images repository has been removed successfully!' + notice: 'Image repository has been removed successfully!' else redirect_to project_container_registry_path(@project), - alert: 'Failed to remove images repository!' + alert: 'Failed to remove image repository!' end end -- cgit v1.2.1 From 7cf91f7b78225957d8b79e96f7d0080cdd15c0cd Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Apr 2017 11:52:56 +0200 Subject: Improve wording in partials related to the registry --- app/views/projects/registry/repositories/_tag.html.haml | 3 +-- app/views/projects/registry/repositories/index.html.haml | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/app/views/projects/registry/repositories/_tag.html.haml b/app/views/projects/registry/repositories/_tag.html.haml index 73566689d57..ee1ec0e8f9a 100644 --- a/app/views/projects/registry/repositories/_tag.html.haml +++ b/app/views/projects/registry/repositories/_tag.html.haml @@ -25,10 +25,9 @@ - if can?(current_user, :update_container_image, @project) %td.content .controls.hidden-xs.pull-right - - notice = 'Due to a Docker limitation, all tags with the same ID will also be deleted. Are you sure?' = link_to namespace_project_registry_repository_tag_path(@project.namespace, @project, tag.repository, tag.name), method: :delete, class: 'btn btn-remove has-tooltip', title: 'Remove tag', - data: { confirm: notice } do + data: { confirm: 'Are you sure you want to delete this tag?' } do = icon('trash cred') diff --git a/app/views/projects/registry/repositories/index.html.haml b/app/views/projects/registry/repositories/index.html.haml index 1b5d000e801..be128e92fa7 100644 --- a/app/views/projects/registry/repositories/index.html.haml +++ b/app/views/projects/registry/repositories/index.html.haml @@ -20,7 +20,7 @@ docker push #{escape_once(@project.container_registry_url)}/image - if @images.blank? - .nothing-here-block No container images in Container Registry for this project. + .nothing-here-block No container image repositories in Container Registry for this project. - else = render partial: 'image', collection: @images -- cgit v1.2.1 From 97c6cf59b361a0fe1b22cc999a2b2dd64b0a30f3 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Apr 2017 11:56:32 +0200 Subject: Swap method names in containe registry path class --- lib/container_registry/path.rb | 16 ++++++++-------- spec/lib/container_registry/path_spec.rb | 18 +++++++++--------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb index f76c6489381..2291291d1fc 100644 --- a/lib/container_registry/path.rb +++ b/lib/container_registry/path.rb @@ -18,19 +18,19 @@ module ContainerRegistry def valid? @path =~ Gitlab::Regex.container_repository_name_regex && - nodes.size > 1 && - nodes.size < Namespace::NUMBER_OF_ANCESTORS_ALLOWED + components.size > 1 && + components.size < Namespace::NUMBER_OF_ANCESTORS_ALLOWED end - def nodes - @nodes ||= @path.to_s.split('/') + def components + @components ||= @path.to_s.split('/') end - def components + def nodes raise InvalidRegistryPathError unless valid? - @components ||= nodes.size.downto(2).map do |length| - nodes.take(length).join('/') + @nodes ||= components.size.downto(2).map do |length| + components.take(length).join('/') end end @@ -50,7 +50,7 @@ module ContainerRegistry end def repository_project - @project ||= Project.where_full_path_in(components.first(3)).first + @project ||= Project.where_full_path_in(nodes.first(3)).first end def repository_name diff --git a/spec/lib/container_registry/path_spec.rb b/spec/lib/container_registry/path_spec.rb index 825c61beeb3..b9c4572c269 100644 --- a/spec/lib/container_registry/path_spec.rb +++ b/spec/lib/container_registry/path_spec.rb @@ -3,22 +3,22 @@ require 'spec_helper' describe ContainerRegistry::Path do subject { described_class.new(path) } - describe '#nodes' do + describe '#components' do let(:path) { 'path/to/some/project' } - it 'splits elements by a forward slash' do - expect(subject.nodes).to eq %w[path to some project] + it 'splits components by a forward slash' do + expect(subject.components).to eq %w[path to some project] end end - describe '#components' do + describe '#nodes' do context 'when repository path is valid' do let(:path) { 'path/to/some/project' } - it 'return all project-like components in reverse order' do - expect(subject.components).to eq %w[path/to/some/project - path/to/some - path/to] + it 'return all project path like node in reverse order' do + expect(subject.nodes).to eq %w[path/to/some/project + path/to/some + path/to] end end @@ -26,7 +26,7 @@ describe ContainerRegistry::Path do let(:path) { '' } it 'rasises en error' do - expect { subject.components } + expect { subject.nodes } .to raise_error described_class::InvalidRegistryPathError end end -- cgit v1.2.1 From 1c91d52a70407e15f9b106bafc6505895214f3b8 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Apr 2017 11:58:15 +0200 Subject: Remove unneeded char in registry repository path --- lib/container_registry/path.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb index 2291291d1fc..89973b2e7b8 100644 --- a/lib/container_registry/path.rb +++ b/lib/container_registry/path.rb @@ -56,7 +56,7 @@ module ContainerRegistry def repository_name return unless has_project? - @path.remove(%r(^?#{Regexp.escape(repository_project.full_path)}/?)) + @path.remove(%r(^#{Regexp.escape(repository_project.full_path)}/?)) end def to_s -- cgit v1.2.1 From cb2ce8452fe2e9e156add5ccfe8fd2ec5cda9ace Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Apr 2017 12:57:38 +0200 Subject: Remove legacy registry tags when deleting a project --- app/models/container_repository.rb | 4 ++ app/services/projects/destroy_service.rb | 23 +++++++-- spec/models/container_repository_spec.rb | 18 +++++++ spec/services/projects/destroy_service_spec.rb | 68 +++++++++++++++++++------- 4 files changed, 92 insertions(+), 21 deletions(-) diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index 36158d75ae8..463eb5b7d69 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -70,4 +70,8 @@ class ContainerRepository < ActiveRecord::Base def self.create_from_path!(path) build_from_path(path).tap(&:save!) end + + def self.build_root_repository(project) + self.new(project: project, name: '') + end end diff --git a/app/services/projects/destroy_service.rb b/app/services/projects/destroy_service.rb index 4e1964f79dd..a47e74ba9b0 100644 --- a/app/services/projects/destroy_service.rb +++ b/app/services/projects/destroy_service.rb @@ -29,15 +29,20 @@ module Projects Project.transaction do project.team.truncate - project.destroy! + + unless remove_legacy_registry_tags + raise_error('Failed to remove some tags in project container registry. Please try again or contact administrator.') + end unless remove_repository(repo_path) - raise_error('Failed to remove project repository. Please try again or contact administrator') + raise_error('Failed to remove project repository. Please try again or contact administrator.') end unless remove_repository(wiki_path) - raise_error('Failed to remove wiki repository. Please try again or contact administrator') + raise_error('Failed to remove wiki repository. Please try again or contact administrator.') end + + project.destroy! end log_info("Project \"#{project.path_with_namespace}\" was removed") @@ -64,6 +69,18 @@ module Projects end end + ## + # This method makes sure that we correctly remove registry tags + # for legacy image repository (when repository path equals project path). + # + def remove_legacy_registry_tags + return true unless Gitlab.config.registry.enabled + + ContainerRepository.build_root_repository(project).tap do |repository| + return repository.delete_tags! if repository.has_tags? + end + end + def raise_error(message) raise DestroyError.new(message) end diff --git a/spec/models/container_repository_spec.rb b/spec/models/container_repository_spec.rb index 1a29cc9a096..3e6082ec326 100644 --- a/spec/models/container_repository_spec.rb +++ b/spec/models/container_repository_spec.rb @@ -195,4 +195,22 @@ describe ContainerRepository do end end end + + describe '.build_root_repository' do + let(:repository) do + described_class.build_root_repository(project) + end + + it 'fabricates a root repository object' do + expect(repository).to be_root_repository + end + + it 'assignes it to the correct project' do + expect(repository.project).to eq project + end + + it 'does not persist it' do + expect(repository).not_to be_persisted + end + end end diff --git a/spec/services/projects/destroy_service_spec.rb b/spec/services/projects/destroy_service_spec.rb index 5ef07c8275e..4b8589b2736 100644 --- a/spec/services/projects/destroy_service_spec.rb +++ b/spec/services/projects/destroy_service_spec.rb @@ -7,6 +7,11 @@ describe Projects::DestroyService, services: true do let!(:remove_path) { path.sub(/\.git\Z/, "+#{project.id}+deleted.git") } let!(:async) { false } # execute or async_execute + before do + stub_container_registry_config(enabled: true) + stub_container_registry_tags(repository: :any, tags: []) + end + shared_examples 'deleting the project' do it 'deletes the project' do expect(Project.unscoped.all).not_to include(project) @@ -89,37 +94,64 @@ describe Projects::DestroyService, services: true do it_behaves_like 'deleting the project with pipeline and build' end - context 'container registry' do - let(:container_repository) { create(:container_repository) } + describe 'container registry' do + context 'when there are regular container repositories' do + let(:container_repository) { create(:container_repository) } - before do - stub_container_registry_config(enabled: true) - stub_container_registry_tags(repository: :any, tags: ['tag']) - project.container_repositories << container_repository - end + before do + stub_container_registry_tags(repository: project.full_path + '/image', + tags: ['tag']) + project.container_repositories << container_repository + end - context 'images deletion succeeds' do - it do - expect_any_instance_of(ContainerRepository) - .to receive(:delete_tags!).and_return(true) + context 'when image repository deletion succeeds' do + it 'removes tags' do + expect_any_instance_of(ContainerRepository) + .to receive(:delete_tags!).and_return(true) - destroy_project(project, user, {}) + destroy_project(project, user) + end + end + + context 'when image repository deletion fails' do + it 'raises an exception' do + expect_any_instance_of(ContainerRepository) + .to receive(:delete_tags!).and_return(false) + + expect{ destroy_project(project, user) } + .to raise_error(ActiveRecord::RecordNotDestroyed) + end end end - context 'images deletion fails' do + context 'when there are tags for legacy root repository' do before do - expect_any_instance_of(ContainerRepository) - .to receive(:delete_tags!).and_return(false) + stub_container_registry_tags(repository: project.full_path, + tags: ['tag']) + end + + context 'when image repository tags deletion succeeds' do + it 'removes tags' do + expect_any_instance_of(ContainerRepository) + .to receive(:delete_tags!).and_return(true) + + destroy_project(project, user) + end end - subject { destroy_project(project, user, {}) } + context 'when image repository tags deletion fails' do + it 'raises an exception' do + expect_any_instance_of(ContainerRepository) + .to receive(:delete_tags!).and_return(false) - it { expect{subject}.to raise_error(ActiveRecord::RecordNotDestroyed) } + expect { destroy_project(project, user) } + .to raise_error(Projects::DestroyService::DestroyError) + end + end end end - def destroy_project(project, user, params) + def destroy_project(project, user, params = {}) if async Projects::DestroyService.new(project, user, params).async_execute else -- cgit v1.2.1 From f60820ed0345513d43a1fd9a8b93caf4d39756f4 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Apr 2017 13:02:52 +0200 Subject: Fix wording in registry tags controller notifications --- app/controllers/projects/registry/tags_controller.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/controllers/projects/registry/tags_controller.rb b/app/controllers/projects/registry/tags_controller.rb index aab130787e9..138f9d14ee1 100644 --- a/app/controllers/projects/registry/tags_controller.rb +++ b/app/controllers/projects/registry/tags_controller.rb @@ -6,10 +6,10 @@ module Projects def destroy if tag.delete redirect_to project_container_registry_path(@project), - notice: 'Tag removed successfull!' + notice: 'Registry tag has been removed successfully!' else redirect_to project_container_registry_path(@project), - alert: 'Failed to remove repository tag!' + alert: 'Failed to remove registry tag!' end end -- cgit v1.2.1 From 666ba382e900b0972d457af4f8784ef61324be5c Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Apr 2017 13:06:11 +0200 Subject: Simplify registry-related code in project model --- app/models/project.rb | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/app/models/project.rb b/app/models/project.rb index fa64ccbf7e4..2bbfba90b6b 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1387,10 +1387,6 @@ class Project < ActiveRecord::Base def has_root_container_repository_tags? return false unless Gitlab.config.registry.enabled - ContainerRegistry::Path.new(self.full_path).tap do |path| - ContainerRepository.build_from_path(path).tap do |repository| - return repository.has_tags? - end - end + ContainerRepository.build_root_repository(self).has_tags? end end -- cgit v1.2.1 From 33329d40df85c81b0a932c53a6152e03fc1f7363 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Apr 2017 13:13:51 +0200 Subject: Fix project specs related to container registry --- spec/models/project_spec.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 4c13e53d831..da1372fe761 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -1157,11 +1157,12 @@ describe Project, models: true do # Project#gitlab_shell returns a new instance of Gitlab::Shell on every # call. This makes testing a bit easier. allow(project).to receive(:gitlab_shell).and_return(gitlab_shell) - allow(project).to receive(:previous_changes).and_return('path' => ['foo']) end it 'renames a repository' do + stub_container_registry_config(enabled: false) + expect(gitlab_shell).to receive(:mv_repository). ordered. with(project.repository_storage_path, "#{project.namespace.full_path}/foo", "#{project.full_path}"). -- cgit v1.2.1 From b03f1699c47ce8a08f67ef458107d22cbafbc0bd Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Apr 2017 13:24:13 +0200 Subject: Extend registry docs regarding multi-level repositories --- doc/user/project/container_registry.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/doc/user/project/container_registry.md b/doc/user/project/container_registry.md index 6de75e43ed3..34762c1bf46 100644 --- a/doc/user/project/container_registry.md +++ b/doc/user/project/container_registry.md @@ -10,7 +10,7 @@ - Starting from GitLab 8.12, if you have 2FA enabled in your account, you need to pass a personal access token instead of your password in order to login to GitLab's Container Registry. -- Multiple level image names support was added in GitLab 8.15 +- Multiple level image names support was added in GitLab 9.1 With the Docker Container Registry integrated into GitLab, every project can have its own space to store its Docker images. @@ -65,6 +65,16 @@ Your image will be named after the following scheme: /// ``` +GitLab supports up to three levels of image repository names. + +Following image repository names are valid: + +``` +registry.example.com// +registry.example.com///image +registry.example.com///image/type +``` + ## Use images from GitLab Container Registry To download and run a container from images hosted in GitLab Container Registry, -- cgit v1.2.1 From ec28f0b59b54a00ddd4ac33dfa74fa6adfb7f09e Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Apr 2017 13:39:42 +0200 Subject: Add changelog entry for multi-level image repositories --- .../unreleased/feature-multi-level-container-registry-images.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changelogs/unreleased/feature-multi-level-container-registry-images.yml diff --git a/changelogs/unreleased/feature-multi-level-container-registry-images.yml b/changelogs/unreleased/feature-multi-level-container-registry-images.yml new file mode 100644 index 00000000000..6d39a6c17c0 --- /dev/null +++ b/changelogs/unreleased/feature-multi-level-container-registry-images.yml @@ -0,0 +1,4 @@ +--- +title: Add support for multi-level container image repository names +merge_request: 10109 +author: André Guede -- cgit v1.2.1 From 2f939b4a81be2b3f09b33313b0d13db28d0b1a31 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Apr 2017 14:58:51 +0200 Subject: Fix container registry specs after changing texts --- spec/features/container_registry_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/features/container_registry_spec.rb b/spec/features/container_registry_spec.rb index fa7adbe71ea..b86609e07c5 100644 --- a/spec/features/container_registry_spec.rb +++ b/spec/features/container_registry_spec.rb @@ -19,7 +19,7 @@ describe "Container Registry" do scenario 'user visits container registry main page' do visit_container_registry - expect(page).to have_content 'No container images' + expect(page).to have_content 'No container image repositories' end end -- cgit v1.2.1 From 22d68f2bc38b0c55a33b41410ed3587b1d7d5b74 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 4 Apr 2017 15:41:38 +0200 Subject: Fix HAML lint offense in repository image partial --- app/views/projects/registry/repositories/_image.html.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/projects/registry/repositories/_image.html.haml b/app/views/projects/registry/repositories/_image.html.haml index e0d74789207..b0b09354e02 100644 --- a/app/views/projects/registry/repositories/_image.html.haml +++ b/app/views/projects/registry/repositories/_image.html.haml @@ -14,7 +14,7 @@ method: :delete do = icon('trash cred') - .container-image-tags.js-toggle-content{ class: 'hide' } + .container-image-tags.js-toggle-content.hide - if image.has_tags? .table-holder %table.table.tags -- cgit v1.2.1 From ee6f50027c3b8366d97a0170f99c24dfc23cd101 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 5 Apr 2017 14:13:46 +0200 Subject: Improve container repository tags controller route --- config/routes/project.rb | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/config/routes/project.rb b/config/routes/project.rb index f85521fe6d3..909a12a0204 100644 --- a/config/routes/project.rb +++ b/config/routes/project.rb @@ -221,14 +221,13 @@ constraints(ProjectUrlConstrainer.new) do end end - resources :container_registry, - controller: 'registry/repositories', - only: [:index, :destroy], - constraints: { id: Gitlab::Regex.container_registry_reference_regex } + resources :container_registry, only: [:index, :destroy], + controller: 'registry/repositories' namespace :registry do resources :repository, only: [] do - resources :tags, only: [:destroy] + resources :tags, only: [:destroy], + constraints: { id: Gitlab::Regex.container_registry_reference_regex } end end -- cgit v1.2.1 From ed0547b7c069eb1df002f6441cfa313ba6c6381e Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 5 Apr 2017 14:18:42 +0200 Subject: Require container registry entities in controllers --- app/controllers/projects/registry/repositories_controller.rb | 2 +- app/controllers/projects/registry/tags_controller.rb | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/app/controllers/projects/registry/repositories_controller.rb b/app/controllers/projects/registry/repositories_controller.rb index 2e18075d40f..737f8424ebf 100644 --- a/app/controllers/projects/registry/repositories_controller.rb +++ b/app/controllers/projects/registry/repositories_controller.rb @@ -21,7 +21,7 @@ module Projects private def image - @image ||= project.container_repositories.find_by(id: params[:id]) + @image ||= project.container_repositories.find(params[:id]) end ## diff --git a/app/controllers/projects/registry/tags_controller.rb b/app/controllers/projects/registry/tags_controller.rb index 138f9d14ee1..7a9f290e946 100644 --- a/app/controllers/projects/registry/tags_controller.rb +++ b/app/controllers/projects/registry/tags_controller.rb @@ -17,11 +17,13 @@ module Projects def repository @image ||= project.container_repositories - .find_by(id: params[:repository_id]) + .find(params[:repository_id]) end def tag - @tag ||= repository.tag(params[:id]) if params[:id].present? + return render_404 unless params[:id].present? + + @tag ||= repository.tag(params[:id]) end end end -- cgit v1.2.1 From b31c53320ea930f85dabbea674103d2faa59471f Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 5 Apr 2017 14:20:33 +0200 Subject: Revert unneeded change in project destroy service --- app/services/projects/destroy_service.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/services/projects/destroy_service.rb b/app/services/projects/destroy_service.rb index a47e74ba9b0..f2d12402239 100644 --- a/app/services/projects/destroy_service.rb +++ b/app/services/projects/destroy_service.rb @@ -29,6 +29,7 @@ module Projects Project.transaction do project.team.truncate + project.destroy! unless remove_legacy_registry_tags raise_error('Failed to remove some tags in project container registry. Please try again or contact administrator.') @@ -41,8 +42,6 @@ module Projects unless remove_repository(wiki_path) raise_error('Failed to remove wiki repository. Please try again or contact administrator.') end - - project.destroy! end log_info("Project \"#{project.path_with_namespace}\" was removed") -- cgit v1.2.1 From b611da70afeffbffd4a8fc8d5ddf61aa2362617d Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 5 Apr 2017 14:32:45 +0200 Subject: Fix status when removing legacy tags from registry --- app/services/projects/destroy_service.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/services/projects/destroy_service.rb b/app/services/projects/destroy_service.rb index f2d12402239..06d8d143231 100644 --- a/app/services/projects/destroy_service.rb +++ b/app/services/projects/destroy_service.rb @@ -76,7 +76,7 @@ module Projects return true unless Gitlab.config.registry.enabled ContainerRepository.build_root_repository(project).tap do |repository| - return repository.delete_tags! if repository.has_tags? + return repository.has_tags? ? repository.delete_tags! : true end end -- cgit v1.2.1 From c753975b087cf71d82712624fa5fcd9fa6d79844 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 5 Apr 2017 14:37:50 +0200 Subject: Add some minor improvements into registry partials --- app/assets/stylesheets/pages/container_registry.scss | 2 +- app/views/projects/registry/repositories/_image.html.haml | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/app/assets/stylesheets/pages/container_registry.scss b/app/assets/stylesheets/pages/container_registry.scss index 92543d7d714..3266714396e 100644 --- a/app/assets/stylesheets/pages/container_registry.scss +++ b/app/assets/stylesheets/pages/container_registry.scss @@ -8,7 +8,7 @@ .container-image-head { padding: 0 16px; - line-height: 4; + line-height: 4em; } .table.tags { diff --git a/app/views/projects/registry/repositories/_image.html.haml b/app/views/projects/registry/repositories/_image.html.haml index b0b09354e02..d183ce34a3a 100644 --- a/app/views/projects/registry/repositories/_image.html.haml +++ b/app/views/projects/registry/repositories/_image.html.haml @@ -1,7 +1,7 @@ .container-image.js-toggle-container .container-image-head = link_to "#", class: "js-toggle-button" do - = icon("chevron-down") + = icon('chevron-down', 'aria-hidden': 'true') = escape_once(image.path) = clipboard_button(clipboard_text: "docker pull #{image.path}") @@ -12,7 +12,7 @@ title: 'Remove repository', data: { confirm: 'Are you sure?' }, method: :delete do - = icon('trash cred') + = icon('trash cred', 'aria-hidden': 'true') .container-image-tags.js-toggle-content.hide - if image.has_tags? @@ -28,6 +28,5 @@ %th = render partial: 'tag', collection: image.tags - else - %li - .nothing-here-block No tags in Container Registry for this container image. + .nothing-here-block No tags in Container Registry for this container image. -- cgit v1.2.1 From 000af87190bee6c99ad4e734818b3ba37c21a292 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 5 Apr 2017 14:44:35 +0200 Subject: Remove redundant code from container registry classes --- app/controllers/projects/registry/repositories_controller.rb | 2 +- app/models/container_repository.rb | 4 ++-- spec/lib/container_registry/blob_spec.rb | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/controllers/projects/registry/repositories_controller.rb b/app/controllers/projects/registry/repositories_controller.rb index 737f8424ebf..17f391ba07f 100644 --- a/app/controllers/projects/registry/repositories_controller.rb +++ b/app/controllers/projects/registry/repositories_controller.rb @@ -34,7 +34,7 @@ module Projects break if path.has_repository? ContainerRepository.build_from_path(path).tap do |repository| - repository.save if repository.has_tags? + repository.save! if repository.has_tags? end end end diff --git a/app/models/container_repository.rb b/app/models/container_repository.rb index 463eb5b7d69..9682df3a586 100644 --- a/app/models/container_repository.rb +++ b/app/models/container_repository.rb @@ -28,7 +28,7 @@ class ContainerRepository < ActiveRecord::Base end def manifest - @manifest ||= client.repository_tags(self.path) + @manifest ||= client.repository_tags(path) end def tags @@ -45,7 +45,7 @@ class ContainerRepository < ActiveRecord::Base end def has_tags? - tags.to_a.any? + tags.any? end def root_repository? diff --git a/spec/lib/container_registry/blob_spec.rb b/spec/lib/container_registry/blob_spec.rb index 76ea29666ea..f06e5fd54a2 100644 --- a/spec/lib/container_registry/blob_spec.rb +++ b/spec/lib/container_registry/blob_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' describe ContainerRegistry::Blob do let(:group) { create(:group, name: 'group') } - let(:project) { create(:project, path: 'test', group: group) } + let(:project) { create(:empty_project, path: 'test', group: group) } let(:repository) do create(:container_repository, name: 'image', -- cgit v1.2.1 From 6b565f534a17ce893ce288e8afd1e3e4d5d3dd3a Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 5 Apr 2017 14:47:28 +0200 Subject: Improve docs for multi-level container registry images --- doc/user/project/container_registry.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/user/project/container_registry.md b/doc/user/project/container_registry.md index 34762c1bf46..6a2ca7fb428 100644 --- a/doc/user/project/container_registry.md +++ b/doc/user/project/container_registry.md @@ -67,12 +67,12 @@ Your image will be named after the following scheme: GitLab supports up to three levels of image repository names. -Following image repository names are valid: +Following examples of image tags are valid: ``` -registry.example.com// -registry.example.com///image -registry.example.com///image/type +registry.example.com/group/project:some-tag +registry.example.com/group/project/image:latest +registry.example.com/group/project/my/image:rc1 ``` ## Use images from GitLab Container Registry -- cgit v1.2.1 From efe2d96a90cb5e2cc0c368294a021423aeeaeabe Mon Sep 17 00:00:00 2001 From: Tiago Botelho Date: Mon, 3 Apr 2017 16:37:23 +0100 Subject: adds initial microsoft teams integration --- app/models/project.rb | 1 + .../project_services/chat_message/note_message.rb | 2 +- .../project_services/chat_message/push_message.rb | 31 +++++++++- .../project_services/chat_notification_service.rb | 3 +- .../project_services/microsoft_teams_service.rb | 66 ++++++++++++++++++++++ app/models/service.rb | 1 + lib/microsoft_teams/notifier.rb | 37 ++++++++++++ spec/lib/gitlab/import_export/all_models.yml | 1 + 8 files changed, 137 insertions(+), 5 deletions(-) create mode 100644 app/models/project_services/microsoft_teams_service.rb create mode 100644 lib/microsoft_teams/notifier.rb diff --git a/app/models/project.rb b/app/models/project.rb index 12fd0668ff8..fb0e4a6ed5f 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -116,6 +116,7 @@ class Project < ActiveRecord::Base has_one :mock_ci_service, dependent: :destroy has_one :mock_deployment_service, dependent: :destroy has_one :mock_monitoring_service, dependent: :destroy + has_one :microsoft_teams_service, dependent: :destroy has_one :forked_project_link, dependent: :destroy, foreign_key: "forked_to_project_id" has_one :forked_from_project, through: :forked_project_link diff --git a/app/models/project_services/chat_message/note_message.rb b/app/models/project_services/chat_message/note_message.rb index 552113bac29..e06c37df69e 100644 --- a/app/models/project_services/chat_message/note_message.rb +++ b/app/models/project_services/chat_message/note_message.rb @@ -10,6 +10,7 @@ module ChatMessage def initialize(params) params = HashWithIndifferentAccess.new(params) @user_name = params[:user][:username] + @user_avatar = params[:user][:avatar] @project_name = params[:project_name] @project_url = params[:project_url] @@ -18,7 +19,6 @@ module ChatMessage @note = obj_attr[:note] @note_url = obj_attr[:url] noteable_type = obj_attr[:noteable_type] - case noteable_type when "Commit" create_commit_note(HashWithIndifferentAccess.new(params[:commit])) diff --git a/app/models/project_services/chat_message/push_message.rb b/app/models/project_services/chat_message/push_message.rb index 2d73b71ec37..9fb77d806ed 100644 --- a/app/models/project_services/chat_message/push_message.rb +++ b/app/models/project_services/chat_message/push_message.rb @@ -8,6 +8,7 @@ module ChatMessage attr_reader :ref attr_reader :ref_type attr_reader :user_name + attr_reader :user_avatar def initialize(params) @after = params[:after] @@ -18,16 +19,27 @@ module ChatMessage @ref_type = Gitlab::Git.tag_ref?(params[:ref]) ? 'tag' : 'branch' @ref = Gitlab::Git.ref_name(params[:ref]) @user_name = params[:user_name] + @user_avatar = params[:user_avatar] + @format = params[:format] + end + + def activity + { + title: activity_title, + subtitle: "to: #{project_link}", + text: compare_link, + image: params[:user_avatar] + } end def pretext - format(message) + @format ? format(message) : message end def attachments return [] if new_branch? || removed_branch? - commit_message_attachments + @format ? commit_message_attachments : commit_messages end private @@ -59,7 +71,7 @@ module ChatMessage end def commit_messages - commits.map { |commit| compose_commit_message(commit) }.join("\n") + commits.map { |commit| compose_commit_message(commit) }.join("\n\n") end def commit_message_attachments @@ -103,6 +115,19 @@ module ChatMessage "[Compare changes](#{compare_url})" end + def activity_title + action = + if new_branch? + "created" + elsif removed_branch? + "removed" + else + "pushed to" + end + + "#{user_name} #{action} #{ref_type}" + end + def attachment_color '#345' end diff --git a/app/models/project_services/chat_notification_service.rb b/app/models/project_services/chat_notification_service.rb index 75834103db5..2957237faac 100644 --- a/app/models/project_services/chat_notification_service.rb +++ b/app/models/project_services/chat_notification_service.rb @@ -51,7 +51,8 @@ class ChatNotificationService < Service data = data.merge( project_url: project_url, - project_name: project_name + project_name: project_name, + format: true ) # WebHook events often have an 'update' event that follows a 'open' or diff --git a/app/models/project_services/microsoft_teams_service.rb b/app/models/project_services/microsoft_teams_service.rb new file mode 100644 index 00000000000..c0a3d2c0025 --- /dev/null +++ b/app/models/project_services/microsoft_teams_service.rb @@ -0,0 +1,66 @@ +class MicrosoftTeamsService < ChatNotificationService + def title + 'Microsoft Teams Notification' + end + + def description + 'Receive event notifications in Microsoft Team' + end + + def self.to_param + 'microsoft_teams' + end + + #TODO: Setup the description accordingly + def help + 'This service sends notifications about projects events to Microsoft Teams channels.
+ To set up this service: +
    +
  1. Getting started with 365 Office Connectors For Microsoft Teams.
    2. +
  2. Paste the Webhook URL into the field below.
    3. +
  3. Select events below to enable notifications.
    4. +
' + end + + def default_channel_placeholder + "Channel name (e.g. general)" + end + + def webhook_placeholder + 'https://outlook.office.com/webhook/…' + end + + def event_field(event) + end + + def default_fields + [ + { type: 'text', name: 'webhook', placeholder: "e.g. #{webhook_placeholder}" }, + { type: 'checkbox', name: 'notify_only_broken_pipelines' }, + { type: 'checkbox', name: 'notify_only_default_branch' }, + ] + end + + def execute(data) + return unless supported_events.include?(data[:object_kind]) + return unless webhook.present? + + object_kind = data[:object_kind] + + data = data.merge( + project_url: project_url, + project_name: project_name, + format: false + ) + + message = get_message(object_kind, data) + + return false unless message + + MicrosoftTeams::Notifier.new(webhook).ping({ + title: message.project_name, + activity: message.activity, + attachments: message.attachments, + }) + end +end diff --git a/app/models/service.rb b/app/models/service.rb index 5a0ec58d193..dc76bf925d3 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -237,6 +237,7 @@ class Service < ActiveRecord::Base slack_slash_commands slack teamcity + microsoft_teams ] if Rails.env.development? service_names += %w[mock_ci mock_deployment mock_monitoring] diff --git a/lib/microsoft_teams/notifier.rb b/lib/microsoft_teams/notifier.rb new file mode 100644 index 00000000000..deff5fd26ee --- /dev/null +++ b/lib/microsoft_teams/notifier.rb @@ -0,0 +1,37 @@ +module MicrosoftTeams + class Notifier + def initialize(webhook) + @webhook = webhook + end + + def ping(options = {}) + HTTParty.post( + @webhook.to_str, + headers: { 'Content-type' => 'application/json' }, + body: body(options) + ) + end + + private + + def body(options = {}) + result = { 'sections' => [] } + + result['title'] = options[:title] if options[:title] + result['summary'] = options[:activity][:title] + result['sections'] << { + 'activityTitle' => options[:activity][:title], + 'activitySubtitle' => options[:activity][:subtitle], + 'activityText' => options[:activity][:text], + 'activityImage' => options[:activity][:image] + } + result['sections'] << { 'title' => 'Details', 'facts' => attachments(options[:attachments]) } if options[:attachments] + + result.to_json + end + + def attachments(content) + [{ 'name' => 'Attachments', 'value' => content }] + end + end +end diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index 24654bf6afd..f4b542f7bbb 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -143,6 +143,7 @@ project: - asana_service - gemnasium_service - slack_service +- microsoft_teams_service - mattermost_service - buildkite_service - bamboo_service -- cgit v1.2.1 From 7dd645e6c97dcc1dfeea4c0ad46ec1feabc87173 Mon Sep 17 00:00:00 2001 From: Tiago Botelho Date: Mon, 3 Apr 2017 17:39:17 +0100 Subject: add note message integration with microsoft teams --- .../project_services/chat_message/base_message.rb | 2 +- .../project_services/chat_message/note_message.rb | 64 +++++++++++++--------- .../project_services/chat_message/push_message.rb | 11 ++-- .../project_services/chat_notification_service.rb | 2 +- .../project_services/microsoft_teams_service.rb | 2 +- 5 files changed, 45 insertions(+), 36 deletions(-) diff --git a/app/models/project_services/chat_message/base_message.rb b/app/models/project_services/chat_message/base_message.rb index 86d271a3f69..a35c32ad43b 100644 --- a/app/models/project_services/chat_message/base_message.rb +++ b/app/models/project_services/chat_message/base_message.rb @@ -7,7 +7,7 @@ module ChatMessage end def pretext - format(message) + markdown_format ? message : format(message) end def fallback diff --git a/app/models/project_services/chat_message/note_message.rb b/app/models/project_services/chat_message/note_message.rb index e06c37df69e..da03e4d946e 100644 --- a/app/models/project_services/chat_message/note_message.rb +++ b/app/models/project_services/chat_message/note_message.rb @@ -1,16 +1,18 @@ module ChatMessage class NoteMessage < BaseMessage - attr_reader :message attr_reader :user_name + attr_reader :user_avatar attr_reader :project_name attr_reader :project_url attr_reader :note attr_reader :note_url + attr_reader :comment_attrs + attr_reader :markdown_format def initialize(params) params = HashWithIndifferentAccess.new(params) - @user_name = params[:user][:username] - @user_avatar = params[:user][:avatar] + @user_name = params[:user][:name] + @user_avatar = params[:user][:avatar_url] @project_name = params[:project_name] @project_url = params[:project_url] @@ -18,7 +20,30 @@ module ChatMessage obj_attr = HashWithIndifferentAccess.new(obj_attr) @note = obj_attr[:note] @note_url = obj_attr[:url] - noteable_type = obj_attr[:noteable_type] + @comment_attrs = comment_params(obj_attr[:noteable_type], params) + @markdown_format = params[:format] + end + + def activity + { + title: "#{user_name} #{link('commented on ' + comment_attrs[:target], note_url)}", + subtitle: "to: #{project_link}", + text: "*#{comment_attrs[:title]}*", + image: user_avatar + } + end + + def attachments + markdown_format ? note : description_message + end + + private + + def message + commented_on_message(comment_attrs) + end + + def comment_params(noteable_type, params) case noteable_type when "Commit" create_commit_note(HashWithIndifferentAccess.new(params[:commit])) @@ -31,40 +56,27 @@ module ChatMessage end end - def attachments - description_message - end - - private - def format_title(title) title.lines.first.chomp end + def create_issue_note(issue) + { target: "issue ##{issue[:iid]}", title: format_title(issue[:title]) } + end + def create_commit_note(commit) commit_sha = commit[:id] commit_sha = Commit.truncate_sha(commit_sha) - commented_on_message( - "commit #{commit_sha}", - format_title(commit[:message])) - end - def create_issue_note(issue) - commented_on_message( - "issue ##{issue[:iid]}", - format_title(issue[:title])) + { target: "commit #{commit_sha}", title: format_title(commit[:message]) } end def create_merge_note(merge_request) - commented_on_message( - "merge request !#{merge_request[:iid]}", - format_title(merge_request[:title])) + { target: "merge request !#{merge_request[:iid]}", title: format_title(merge_request[:title]) } end def create_snippet_note(snippet) - commented_on_message( - "snippet ##{snippet[:id]}", - format_title(snippet[:title])) + { target: "snippet ##{snippet[:id]}", title: format_title(snippet[:title]) } end def description_message @@ -75,8 +87,8 @@ module ChatMessage link(project_name, project_url) end - def commented_on_message(target, title) - @message = "#{user_name} #{link('commented on ' + target, note_url)} in #{project_link}: *#{title}*" + def commented_on_message(target:, title:) + "#{user_name} #{link('commented on ' + target, note_url)} in #{project_link}: *#{title}*" end end end diff --git a/app/models/project_services/chat_message/push_message.rb b/app/models/project_services/chat_message/push_message.rb index 9fb77d806ed..8e11ddba594 100644 --- a/app/models/project_services/chat_message/push_message.rb +++ b/app/models/project_services/chat_message/push_message.rb @@ -9,6 +9,7 @@ module ChatMessage attr_reader :ref_type attr_reader :user_name attr_reader :user_avatar + attr_reader :markdown_format def initialize(params) @after = params[:after] @@ -20,7 +21,7 @@ module ChatMessage @ref = Gitlab::Git.ref_name(params[:ref]) @user_name = params[:user_name] @user_avatar = params[:user_avatar] - @format = params[:format] + @markdown_format = params[:format] end def activity @@ -28,18 +29,14 @@ module ChatMessage title: activity_title, subtitle: "to: #{project_link}", text: compare_link, - image: params[:user_avatar] + image: user_avatar } end - def pretext - @format ? format(message) : message - end - def attachments return [] if new_branch? || removed_branch? - @format ? commit_message_attachments : commit_messages + markdown_format ? commit_messages : commit_message_attachments end private diff --git a/app/models/project_services/chat_notification_service.rb b/app/models/project_services/chat_notification_service.rb index 2957237faac..9a23038b7f2 100644 --- a/app/models/project_services/chat_notification_service.rb +++ b/app/models/project_services/chat_notification_service.rb @@ -52,7 +52,7 @@ class ChatNotificationService < Service data = data.merge( project_url: project_url, project_name: project_name, - format: true + format: false ) # WebHook events often have an 'update' event that follows a 'open' or diff --git a/app/models/project_services/microsoft_teams_service.rb b/app/models/project_services/microsoft_teams_service.rb index c0a3d2c0025..08eaeb06228 100644 --- a/app/models/project_services/microsoft_teams_service.rb +++ b/app/models/project_services/microsoft_teams_service.rb @@ -50,7 +50,7 @@ class MicrosoftTeamsService < ChatNotificationService data = data.merge( project_url: project_url, project_name: project_name, - format: false + format: true ) message = get_message(object_kind, data) -- cgit v1.2.1 From 9bfa0a5ffab279f25c52f2e1e9644616168bf994 Mon Sep 17 00:00:00 2001 From: Tiago Botelho Date: Mon, 3 Apr 2017 18:16:24 +0100 Subject: adds remaining messages --- .../project_services/chat_message/base_message.rb | 4 +++ .../project_services/chat_message/issue_message.rb | 24 +++++++++++------ .../project_services/chat_message/merge_message.rb | 13 ++++++++-- .../chat_message/pipeline_message.rb | 30 +++++++++++++++++----- .../chat_message/wiki_page_message.rb | 15 ++++++++++- lib/microsoft_teams/notifier.rb | 5 +++- 6 files changed, 72 insertions(+), 19 deletions(-) diff --git a/app/models/project_services/chat_message/base_message.rb b/app/models/project_services/chat_message/base_message.rb index a35c32ad43b..e17d0181691 100644 --- a/app/models/project_services/chat_message/base_message.rb +++ b/app/models/project_services/chat_message/base_message.rb @@ -17,6 +17,10 @@ module ChatMessage raise NotImplementedError end + def activity + raise NotImplementedError + end + private def message diff --git a/app/models/project_services/chat_message/issue_message.rb b/app/models/project_services/chat_message/issue_message.rb index 791e5b0cec7..398ea1dee3d 100644 --- a/app/models/project_services/chat_message/issue_message.rb +++ b/app/models/project_services/chat_message/issue_message.rb @@ -1,6 +1,7 @@ module ChatMessage class IssueMessage < BaseMessage attr_reader :user_name + attr_reader :user_avatar attr_reader :title attr_reader :project_name attr_reader :project_url @@ -9,9 +10,11 @@ module ChatMessage attr_reader :action attr_reader :state attr_reader :description + attr_reader :markdown_format def initialize(params) @user_name = params[:user][:username] + @user_avatar = params[:user][:avatar_url] @project_name = params[:project_name] @project_url = params[:project_url] @@ -23,23 +26,28 @@ module ChatMessage @action = obj_attr[:action] @state = obj_attr[:state] @description = obj_attr[:description] || '' + @markdown_format = params[:format] end def attachments return [] unless opened_issue? - description_message + markdown_format ? description : description_message + end + + def activity + { + title: "Issue #{state} by #{user_name}", + subtitle: "to: #{project_link}", + text: issue_link, + image: user_avatar + } end private def message - case state - when "opened" - "[#{project_link}] Issue #{state} by #{user_name}" - else - "[#{project_link}] Issue #{issue_link} #{state} by #{user_name}" - end + "[#{project_link}] Issue #{issue_link} #{state} by #{user_name}" end def opened_issue? @@ -52,7 +60,7 @@ module ChatMessage title_link: issue_url, text: format(description), color: "#C95823" - }] + }] end def project_link diff --git a/app/models/project_services/chat_message/merge_message.rb b/app/models/project_services/chat_message/merge_message.rb index 5e5efca7bec..13a894df980 100644 --- a/app/models/project_services/chat_message/merge_message.rb +++ b/app/models/project_services/chat_message/merge_message.rb @@ -1,6 +1,7 @@ module ChatMessage class MergeMessage < BaseMessage attr_reader :user_name + attr_reader :user_avatar attr_reader :project_name attr_reader :project_url attr_reader :merge_request_id @@ -8,9 +9,11 @@ module ChatMessage attr_reader :target_branch attr_reader :state attr_reader :title + attr_reader :markdown_format def initialize(params) @user_name = params[:user][:username] + @user_avatar = params[:user][:avatar_url] @project_name = params[:project_name] @project_url = params[:project_url] @@ -21,10 +24,16 @@ module ChatMessage @target_branch = obj_attr[:target_branch] @state = obj_attr[:state] @title = format_title(obj_attr[:title]) + @markdown_format = params[:format] end - def pretext - format(message) + def activity + { + title: "Merge Request #{state} by #{user_name}", + subtitle: "to: #{project_link}", + text: merge_request_link, + image: user_avatar + } end def attachments diff --git a/app/models/project_services/chat_message/pipeline_message.rb b/app/models/project_services/chat_message/pipeline_message.rb index 210027565a8..4c1cf120614 100644 --- a/app/models/project_services/chat_message/pipeline_message.rb +++ b/app/models/project_services/chat_message/pipeline_message.rb @@ -1,7 +1,15 @@ module ChatMessage class PipelineMessage < BaseMessage - attr_reader :ref_type, :ref, :status, :project_name, :project_url, - :user_name, :duration, :pipeline_id + attr_reader :ref_type + attr_reader :ref + attr_reader :status + attr_reader :project_name + attr_reader :project_url + attr_reader :user_name + attr_reader :duration + attr_reader :pipeline_id + attr_reader :user_avatar + attr_reader :markdown_format def initialize(data) pipeline_attributes = data[:object_attributes] @@ -14,6 +22,9 @@ module ChatMessage @project_name = data[:project][:path_with_namespace] @project_url = data[:project][:web_url] @user_name = (data[:user] && data[:user][:name]) || 'API' + @user_avatar = data[:user][:avatar_url] || '' + + @markdown_format = params[:format] end def pretext @@ -24,8 +35,17 @@ module ChatMessage format(message) end + def activity + { + title: "Pipeline #{pipeline_link} of #{branch_link} #{ref_type} by #{user_name} #{humanized_status}", + subtitle: "to: #{project_link}", + text: "in #{duration} #{'second'.pluralize(duration)}", + image: user_avatar + } + end + def attachments - [{ text: format(message), color: attachment_color }] + markdown_format ? message : [{ text: format(message), color: attachment_color }] end private @@ -34,10 +54,6 @@ module ChatMessage "#{project_link}: Pipeline #{pipeline_link} of #{branch_link} #{ref_type} by #{user_name} #{humanized_status} in #{duration} #{'second'.pluralize(duration)}" end - def format(string) - Slack::Notifier::LinkFormatter.format(string) - end - def humanized_status case status when 'success' diff --git a/app/models/project_services/chat_message/wiki_page_message.rb b/app/models/project_services/chat_message/wiki_page_message.rb index 134083e4504..4f5c2e719c9 100644 --- a/app/models/project_services/chat_message/wiki_page_message.rb +++ b/app/models/project_services/chat_message/wiki_page_message.rb @@ -7,9 +7,11 @@ module ChatMessage attr_reader :wiki_page_url attr_reader :action attr_reader :description + attr_reader :markdown_format def initialize(params) @user_name = params[:user][:username] + @user_avatar = params[:user][:avatar_url] @project_name = params[:project_name] @project_url = params[:project_url] @@ -26,10 +28,21 @@ module ChatMessage when "update" "edited" end + + @markdown_format = params[:format] + end + + def activity + { + title: "#{user_name} #{action} #{wiki_page_link}", + subtitle: "in: #{project_link}", + text: title, + image: user_avatar + } end def attachments - description_message + markdown_format ? @description : description_message end private diff --git a/lib/microsoft_teams/notifier.rb b/lib/microsoft_teams/notifier.rb index deff5fd26ee..8e422575636 100644 --- a/lib/microsoft_teams/notifier.rb +++ b/lib/microsoft_teams/notifier.rb @@ -25,7 +25,10 @@ module MicrosoftTeams 'activityText' => options[:activity][:text], 'activityImage' => options[:activity][:image] } - result['sections'] << { 'title' => 'Details', 'facts' => attachments(options[:attachments]) } if options[:attachments] + + if options[:attachments].present? && !options[:attachments].empty? + result['sections'] << { 'title' => 'Details', 'facts' => attachments(options[:attachments]) } + end result.to_json end -- cgit v1.2.1 From 05f69ef21246f1a2c0b83540d3e205c9e4d77819 Mon Sep 17 00:00:00 2001 From: Tiago Botelho Date: Tue, 4 Apr 2017 10:16:24 +0100 Subject: adds microsoft team activity object and refactors code accordingly --- .../project_services/chat_message/base_message.rb | 4 ++- .../project_services/chat_message/issue_message.rb | 18 +++++----- .../project_services/chat_message/merge_message.rb | 16 ++++----- .../project_services/chat_message/note_message.rb | 39 ++++++++++------------ .../chat_message/pipeline_message.rb | 23 +++++++------ .../project_services/chat_message/push_message.rb | 38 ++++++++++----------- .../chat_message/wiki_page_message.rb | 17 +++++----- .../project_services/chat_notification_service.rb | 3 +- .../project_services/microsoft_teams_service.rb | 5 +-- lib/microsoft_teams/activity.rb | 19 +++++++++++ lib/microsoft_teams/notifier.rb | 23 +++++-------- 11 files changed, 107 insertions(+), 98 deletions(-) create mode 100644 lib/microsoft_teams/activity.rb diff --git a/app/models/project_services/chat_message/base_message.rb b/app/models/project_services/chat_message/base_message.rb index e17d0181691..0f0657998c7 100644 --- a/app/models/project_services/chat_message/base_message.rb +++ b/app/models/project_services/chat_message/base_message.rb @@ -2,8 +2,10 @@ require 'slack-notifier' module ChatMessage class BaseMessage + attr_reader :markdown_format + def initialize(params) - raise NotImplementedError + @markdown_format = params[:markdown_format] || false end def pretext diff --git a/app/models/project_services/chat_message/issue_message.rb b/app/models/project_services/chat_message/issue_message.rb index 398ea1dee3d..aebd9aa93e5 100644 --- a/app/models/project_services/chat_message/issue_message.rb +++ b/app/models/project_services/chat_message/issue_message.rb @@ -10,9 +10,10 @@ module ChatMessage attr_reader :action attr_reader :state attr_reader :description - attr_reader :markdown_format def initialize(params) + super(params) + @user_name = params[:user][:username] @user_avatar = params[:user][:avatar_url] @project_name = params[:project_name] @@ -26,7 +27,6 @@ module ChatMessage @action = obj_attr[:action] @state = obj_attr[:state] @description = obj_attr[:description] || '' - @markdown_format = params[:format] end def attachments @@ -36,12 +36,12 @@ module ChatMessage end def activity - { - title: "Issue #{state} by #{user_name}", - subtitle: "to: #{project_link}", - text: issue_link, - image: user_avatar - } + MicrosoftTeams::Activity.new( + "Issue #{state} by #{user_name}", + "to: #{project_link}", + issue_link, + user_avatar + ).to_json end private @@ -60,7 +60,7 @@ module ChatMessage title_link: issue_url, text: format(description), color: "#C95823" - }] + }] end def project_link diff --git a/app/models/project_services/chat_message/merge_message.rb b/app/models/project_services/chat_message/merge_message.rb index 13a894df980..e9ef211438b 100644 --- a/app/models/project_services/chat_message/merge_message.rb +++ b/app/models/project_services/chat_message/merge_message.rb @@ -9,9 +9,10 @@ module ChatMessage attr_reader :target_branch attr_reader :state attr_reader :title - attr_reader :markdown_format def initialize(params) + super(params) + @user_name = params[:user][:username] @user_avatar = params[:user][:avatar_url] @project_name = params[:project_name] @@ -24,16 +25,15 @@ module ChatMessage @target_branch = obj_attr[:target_branch] @state = obj_attr[:state] @title = format_title(obj_attr[:title]) - @markdown_format = params[:format] end def activity - { - title: "Merge Request #{state} by #{user_name}", - subtitle: "to: #{project_link}", - text: merge_request_link, - image: user_avatar - } + MicrosoftTeams::Activity.new( + "Merge Request #{state} by #{user_name}", + "to: #{project_link}", + merge_request_link, + user_avatar + ).to_json end def attachments diff --git a/app/models/project_services/chat_message/note_message.rb b/app/models/project_services/chat_message/note_message.rb index da03e4d946e..26d944007a6 100644 --- a/app/models/project_services/chat_message/note_message.rb +++ b/app/models/project_services/chat_message/note_message.rb @@ -7,30 +7,30 @@ module ChatMessage attr_reader :note attr_reader :note_url attr_reader :comment_attrs - attr_reader :markdown_format def initialize(params) params = HashWithIndifferentAccess.new(params) + + super(params) + @user_name = params[:user][:name] @user_avatar = params[:user][:avatar_url] @project_name = params[:project_name] @project_url = params[:project_url] - obj_attr = params[:object_attributes] - obj_attr = HashWithIndifferentAccess.new(obj_attr) + obj_attr = HashWithIndifferentAccess.new(params[:object_attributes]) @note = obj_attr[:note] @note_url = obj_attr[:url] @comment_attrs = comment_params(obj_attr[:noteable_type], params) - @markdown_format = params[:format] end def activity - { - title: "#{user_name} #{link('commented on ' + comment_attrs[:target], note_url)}", - subtitle: "to: #{project_link}", - text: "*#{comment_attrs[:title]}*", - image: user_avatar - } + MicrosoftTeams::Activity.new( + "#{user_name} #{link('commented on ' + comment_attrs[:target], note_url)}", + "to: #{project_link}", + comment_attrs[:title], + user_avatar + ).to_json end def attachments @@ -40,19 +40,21 @@ module ChatMessage private def message - commented_on_message(comment_attrs) + "#{user_name} #{link('commented on ' + comment_attrs[:target], note_url)} in #{project_link}: *#{comment_attrs[:title]}*" end def comment_params(noteable_type, params) + params = HashWithIndifferentAccess.new(params) + case noteable_type when "Commit" - create_commit_note(HashWithIndifferentAccess.new(params[:commit])) + create_commit_note(params[:commit]) when "Issue" - create_issue_note(HashWithIndifferentAccess.new(params[:issue])) + create_issue_note(params[:issue]) when "MergeRequest" - create_merge_note(HashWithIndifferentAccess.new(params[:merge_request])) + create_merge_note(params[:merge_request]) when "Snippet" - create_snippet_note(HashWithIndifferentAccess.new(params[:snippet])) + create_snippet_note(params[:snippet]) end end @@ -65,8 +67,7 @@ module ChatMessage end def create_commit_note(commit) - commit_sha = commit[:id] - commit_sha = Commit.truncate_sha(commit_sha) + commit_sha = Commit.truncate_sha(commit[:id]) { target: "commit #{commit_sha}", title: format_title(commit[:message]) } end @@ -86,9 +87,5 @@ module ChatMessage def project_link link(project_name, project_url) end - - def commented_on_message(target:, title:) - "#{user_name} #{link('commented on ' + target, note_url)} in #{project_link}: *#{title}*" - end end end diff --git a/app/models/project_services/chat_message/pipeline_message.rb b/app/models/project_services/chat_message/pipeline_message.rb index 4c1cf120614..8f4fcf09f64 100644 --- a/app/models/project_services/chat_message/pipeline_message.rb +++ b/app/models/project_services/chat_message/pipeline_message.rb @@ -9,7 +9,6 @@ module ChatMessage attr_reader :duration attr_reader :pipeline_id attr_reader :user_avatar - attr_reader :markdown_format def initialize(data) pipeline_attributes = data[:object_attributes] @@ -19,12 +18,12 @@ module ChatMessage @duration = pipeline_attributes[:duration] @pipeline_id = pipeline_attributes[:id] + super(data) + @project_name = data[:project][:path_with_namespace] @project_url = data[:project][:web_url] @user_name = (data[:user] && data[:user][:name]) || 'API' @user_avatar = data[:user][:avatar_url] || '' - - @markdown_format = params[:format] end def pretext @@ -36,12 +35,12 @@ module ChatMessage end def activity - { - title: "Pipeline #{pipeline_link} of #{branch_link} #{ref_type} by #{user_name} #{humanized_status}", - subtitle: "to: #{project_link}", - text: "in #{duration} #{'second'.pluralize(duration)}", - image: user_avatar - } + MicrosoftTeams::Activity.new( + "Pipeline #{pipeline_link} of #{branch_link} #{ref_type} by #{user_name} #{humanized_status}", + "to: #{project_link}", + "in #{duration} #{time_measure}", + user_avatar + ).to_json end def attachments @@ -51,7 +50,7 @@ module ChatMessage private def message - "#{project_link}: Pipeline #{pipeline_link} of #{branch_link} #{ref_type} by #{user_name} #{humanized_status} in #{duration} #{'second'.pluralize(duration)}" + "#{project_link}: Pipeline #{pipeline_link} of #{branch_link} #{ref_type} by #{user_name} #{humanized_status} in #{duration} #{time_measure}" end def humanized_status @@ -90,5 +89,9 @@ module ChatMessage def pipeline_link "[##{pipeline_id}](#{pipeline_url})" end + + def time_measure + 'second'.pluralize(duration) + end end end diff --git a/app/models/project_services/chat_message/push_message.rb b/app/models/project_services/chat_message/push_message.rb index 8e11ddba594..71fa77712c0 100644 --- a/app/models/project_services/chat_message/push_message.rb +++ b/app/models/project_services/chat_message/push_message.rb @@ -9,9 +9,10 @@ module ChatMessage attr_reader :ref_type attr_reader :user_name attr_reader :user_avatar - attr_reader :markdown_format def initialize(params) + super(params) + @after = params[:after] @before = params[:before] @commits = params.fetch(:commits, []) @@ -21,16 +22,24 @@ module ChatMessage @ref = Gitlab::Git.ref_name(params[:ref]) @user_name = params[:user_name] @user_avatar = params[:user_avatar] - @markdown_format = params[:format] end def activity - { - title: activity_title, - subtitle: "to: #{project_link}", - text: compare_link, - image: user_avatar - } + action = + if new_branch? + "created" + elsif removed_branch? + "removed" + else + "pushed to" + end + + MicrosoftTeams::Activity.new( + "#{user_name} #{action} #{ref_type}", + "to: #{project_link}", + compare_link, + user_avatar + ).to_json end def attachments @@ -112,19 +121,6 @@ module ChatMessage "[Compare changes](#{compare_url})" end - def activity_title - action = - if new_branch? - "created" - elsif removed_branch? - "removed" - else - "pushed to" - end - - "#{user_name} #{action} #{ref_type}" - end - def attachment_color '#345' end diff --git a/app/models/project_services/chat_message/wiki_page_message.rb b/app/models/project_services/chat_message/wiki_page_message.rb index 4f5c2e719c9..85c2efd75bc 100644 --- a/app/models/project_services/chat_message/wiki_page_message.rb +++ b/app/models/project_services/chat_message/wiki_page_message.rb @@ -7,9 +7,10 @@ module ChatMessage attr_reader :wiki_page_url attr_reader :action attr_reader :description - attr_reader :markdown_format def initialize(params) + super(params) + @user_name = params[:user][:username] @user_avatar = params[:user][:avatar_url] @project_name = params[:project_name] @@ -28,17 +29,15 @@ module ChatMessage when "update" "edited" end - - @markdown_format = params[:format] end def activity - { - title: "#{user_name} #{action} #{wiki_page_link}", - subtitle: "in: #{project_link}", - text: title, - image: user_avatar - } + MicrosoftTeams::Activity.new( + "#{user_name} #{action} #{wiki_page_link}", + "in: #{project_link}", + title, + user_avatar + ).to_json end def attachments diff --git a/app/models/project_services/chat_notification_service.rb b/app/models/project_services/chat_notification_service.rb index 9a23038b7f2..75834103db5 100644 --- a/app/models/project_services/chat_notification_service.rb +++ b/app/models/project_services/chat_notification_service.rb @@ -51,8 +51,7 @@ class ChatNotificationService < Service data = data.merge( project_url: project_url, - project_name: project_name, - format: false + project_name: project_name ) # WebHook events often have an 'update' event that follows a 'open' or diff --git a/app/models/project_services/microsoft_teams_service.rb b/app/models/project_services/microsoft_teams_service.rb index 08eaeb06228..2a8ea3f7e21 100644 --- a/app/models/project_services/microsoft_teams_service.rb +++ b/app/models/project_services/microsoft_teams_service.rb @@ -50,7 +50,7 @@ class MicrosoftTeamsService < ChatNotificationService data = data.merge( project_url: project_url, project_name: project_name, - format: true + markdown_format: true ) message = get_message(object_kind, data) @@ -59,8 +59,9 @@ class MicrosoftTeamsService < ChatNotificationService MicrosoftTeams::Notifier.new(webhook).ping({ title: message.project_name, + pretext: message.pretext, activity: message.activity, - attachments: message.attachments, + attachments: message.attachments }) end end diff --git a/lib/microsoft_teams/activity.rb b/lib/microsoft_teams/activity.rb new file mode 100644 index 00000000000..e25ce262cbf --- /dev/null +++ b/lib/microsoft_teams/activity.rb @@ -0,0 +1,19 @@ +module MicrosoftTeams + class Activity + def initialize(title, subtitle, text, image) + @title = title + @subtitle = subtitle + @text = text + @image = image + end + + def to_json + { + 'activityTitle' => @title, + 'activitySubtitle' => @subtitle, + 'activityText' => @text, + 'activityImage' => @image + }.to_json + end + end +end diff --git a/lib/microsoft_teams/notifier.rb b/lib/microsoft_teams/notifier.rb index 8e422575636..a5d2fe15ab6 100644 --- a/lib/microsoft_teams/notifier.rb +++ b/lib/microsoft_teams/notifier.rb @@ -15,26 +15,19 @@ module MicrosoftTeams private def body(options = {}) + attachments = options[:attachments] result = { 'sections' => [] } - result['title'] = options[:title] if options[:title] - result['summary'] = options[:activity][:title] - result['sections'] << { - 'activityTitle' => options[:activity][:title], - 'activitySubtitle' => options[:activity][:subtitle], - 'activityText' => options[:activity][:text], - 'activityImage' => options[:activity][:image] - } + result['title'] = options[:title] + result['summary'] = options[:pretext] + result['sections'] << options[:activity] - if options[:attachments].present? && !options[:attachments].empty? - result['sections'] << { 'title' => 'Details', 'facts' => attachments(options[:attachments]) } - end + result['sections'] << { + 'title' => 'Details', + 'facts' => [{ 'name' => 'Attachments', 'value' => attachments }] + } if attachments.present? && attachments.empty? result.to_json end - - def attachments(content) - [{ 'name' => 'Attachments', 'value' => content }] - end end end -- cgit v1.2.1 From 1e54181523f42c2734056b6ea548ebc2134f57a6 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 5 Apr 2017 15:12:29 +0200 Subject: Improve migration for container repositories table --- db/migrate/20170322013926_create_container_repository.rb | 9 +++++++-- db/schema.rb | 10 ++++++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/db/migrate/20170322013926_create_container_repository.rb b/db/migrate/20170322013926_create_container_repository.rb index 87a1523724c..32eba3903b1 100644 --- a/db/migrate/20170322013926_create_container_repository.rb +++ b/db/migrate/20170322013926_create_container_repository.rb @@ -5,8 +5,13 @@ class CreateContainerRepository < ActiveRecord::Migration def change create_table :container_repositories do |t| - t.integer :project_id - t.string :name + t.references :project, foreign_key: true, null: false + t.string :name, null: false + + t.timestamps null: false end + + add_index :container_repositories, :project_id + add_index :container_repositories, [:project_id, :name], unique: true end end diff --git a/db/schema.rb b/db/schema.rb index 3c11e68da2c..9ee1d0e6aef 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -324,10 +324,15 @@ ActiveRecord::Schema.define(version: 20170329124448) do add_index "ci_variables", ["project_id"], name: "index_ci_variables_on_project_id", using: :btree create_table "container_repositories", force: :cascade do |t| - t.integer "project_id" - t.string "name" + t.integer "project_id", null: false + t.string "name", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false end + add_index "container_repositories", ["project_id", "name"], name: "index_container_repositories_on_project_id_and_name", unique: true, using: :btree + add_index "container_repositories", ["project_id"], name: "index_container_repositories_on_project_id", using: :btree + create_table "deploy_keys_projects", force: :cascade do |t| t.integer "deploy_key_id", null: false t.integer "project_id", null: false @@ -1304,6 +1309,7 @@ ActiveRecord::Schema.define(version: 20170329124448) do add_foreign_key "boards", "projects" add_foreign_key "chat_teams", "namespaces", on_delete: :cascade add_foreign_key "ci_triggers", "users", column: "owner_id", name: "fk_e8e10d1964", on_delete: :cascade + add_foreign_key "container_repositories", "projects" add_foreign_key "issue_metrics", "issues", on_delete: :cascade add_foreign_key "label_priorities", "labels", on_delete: :cascade add_foreign_key "label_priorities", "projects", on_delete: :cascade -- cgit v1.2.1 From 82dea6cffe339456c5ba0fd090464926e970dccf Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 5 Apr 2017 15:19:48 +0200 Subject: Fix Rubocop offenses related to registry routes --- config/routes/project.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/routes/project.rb b/config/routes/project.rb index 909a12a0204..757e1852da4 100644 --- a/config/routes/project.rb +++ b/config/routes/project.rb @@ -222,12 +222,12 @@ constraints(ProjectUrlConstrainer.new) do end resources :container_registry, only: [:index, :destroy], - controller: 'registry/repositories' + controller: 'registry/repositories' namespace :registry do resources :repository, only: [] do resources :tags, only: [:destroy], - constraints: { id: Gitlab::Regex.container_registry_reference_regex } + constraints: { id: Gitlab::Regex.container_registry_reference_regex } end end -- cgit v1.2.1 From 7348e0a19bda2410d0cd9e37b248b271992d4827 Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Fri, 31 Mar 2017 10:48:16 +0100 Subject: Ask people to create EE MRs on the 7th --- PROCESS.md | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/PROCESS.md b/PROCESS.md index fead93bd4cf..eaf89e61207 100644 --- a/PROCESS.md +++ b/PROCESS.md @@ -33,7 +33,7 @@ core team members will mention this person. ### Merge request coaching Several people from the [GitLab team][team] are helping community members to get -their contributions accepted by meeting our [Definition of done](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/CONTRIBUTING.md#definition-of-done). +their contributions accepted by meeting our [Definition of done][done]. What you can expect from them is described at https://about.gitlab.com/jobs/merge-request-coach/. @@ -64,6 +64,26 @@ Merge requests may still be merged into master during this period, but they will go into the _next_ release, unless they are manually cherry-picked into the stable branch. By freezing the stable branches 2 weeks prior to a release, we reduce the risk of a last minute merge request potentially breaking things. +### On the 7th + +Merge requests should still be complete, following the +[definition of done][done]. The single exception is documentation, and this can +only be left until after the freeze if: + +* There is a follow-up issue to add documentation. +* It is assigned to the person writing documentation for this feature, and they + are aware of it. +* It is in the correct milestone, with the ~Deliverable label. + +All Community Edition merge requests from GitLab team members merged on the +freeze date (the 7th) should have a corresponding Enterprise Edition merge +request, even if there are no conflicts. This is to reduce the size of the +subsequent EE merge, as we often merge a lot to CE on the release date. For more +information, see +[limit conflicts with EE when developing on CE][limit_ee_conflicts]. + +### Between the 7th and the 22nd + Once the stable branch is frozen, only fixes for regressions (bugs introduced in that same release) and security issues will be cherry-picked into the stable branch. Any merge requests cherry-picked into the stable branch for a previous release will also be picked into the latest stable branch. @@ -158,3 +178,5 @@ still an issue I encourage you to open it on the [GitLab.com issue tracker](http [contribution acceptance criteria]: https://gitlab.com/gitlab-org/gitlab-ce/blob/master/CONTRIBUTING.md#contribution-acceptance-criteria ["Implement design & UI elements" guidelines]: https://gitlab.com/gitlab-org/gitlab-ce/blob/master/CONTRIBUTING.md#implement-design-ui-elements [Thoughtbot code review guide]: https://github.com/thoughtbot/guides/tree/master/code-review +[done]: https://gitlab.com/gitlab-org/gitlab-ce/blob/master/CONTRIBUTING.md#definition-of-done +[limit_ee_conflicts]: https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/development/limit_ee_conflicts.md -- cgit v1.2.1 From d59ddf51f9dfd2f6a823f5441a7ae85c304910ca Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Wed, 5 Apr 2017 17:42:21 +0100 Subject: Large features by the 1st, small ones by the 3rd --- PROCESS.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/PROCESS.md b/PROCESS.md index eaf89e61207..2a2aafbd9ff 100644 --- a/PROCESS.md +++ b/PROCESS.md @@ -64,6 +64,29 @@ Merge requests may still be merged into master during this period, but they will go into the _next_ release, unless they are manually cherry-picked into the stable branch. By freezing the stable branches 2 weeks prior to a release, we reduce the risk of a last minute merge request potentially breaking things. +### Between the 1st and the 7th + +These types of merge requests need special consideration: + +* **Large features**: a large feature is one that is highlighted in the kick-off + and the release blogpost; typically this will have its own channel in Slack + and a dedicated team with front-end, back-end, and UX. +* **Small features**: any other feature request. + +**Large features** must be with a maintainer **by the 1st**. It's OK if they +aren't completely done, but this allows the maintainer enough time to make the +decision about whether this can make it in before the freeze. If the maintainer +doesn't think it will make it, they should inform the developers working on it +and the Product Manager responsible for the feature. + +**Small features** must be with a reviewer (not necessarily maintainer) **by the +3rd**. + +Most merge requests from the community do not have a specific release +target. However, if one does and falls into either of the above categories, it's +the reviewer's responsibility to manage the above communication and assignment +on behalf of the community member. + ### On the 7th Merge requests should still be complete, following the -- cgit v1.2.1 From f51adf8c6ef403858068554022ff34aaca97a7ba Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Wed, 5 Apr 2017 20:20:11 +0300 Subject: Add more tests for subgroups feature * subgroup can be created by owner of the group * project can be created inside subgroup by owner of the group Signed-off-by: Dmitriy Zaporozhets --- spec/features/groups_spec.rb | 39 ++++++++++++++++++++++++------ spec/features/projects/new_project_spec.rb | 16 ++++++++++++ 2 files changed, 47 insertions(+), 8 deletions(-) diff --git a/spec/features/groups_spec.rb b/spec/features/groups_spec.rb index c90cc06a8f5..8bfe6f4d54b 100644 --- a/spec/features/groups_spec.rb +++ b/spec/features/groups_spec.rb @@ -86,17 +86,40 @@ feature 'Group', feature: true do describe 'create a nested group' do let(:group) { create(:group, path: 'foo') } - before do - visit subgroups_group_path(group) - click_link 'New Subgroup' + context 'as admin' do + before do + visit subgroups_group_path(group) + click_link 'New Subgroup' + end + + it 'creates a nested group' do + fill_in 'Group path', with: 'bar' + click_button 'Create group' + + expect(current_path).to eq(group_path('foo/bar')) + expect(page).to have_content("Group 'bar' was successfully created.") + end end - it 'creates a nested group' do - fill_in 'Group path', with: 'bar' - click_button 'Create group' + context 'as group owner' do + let(:user) { create(:user) } - expect(current_path).to eq(group_path('foo/bar')) - expect(page).to have_content("Group 'bar' was successfully created.") + before do + group.add_owner(user) + logout + login_as(user) + + visit subgroups_group_path(group) + click_link 'New Subgroup' + end + + it 'creates a nested group' do + fill_in 'Group path', with: 'bar' + click_button 'Create group' + + expect(current_path).to eq(group_path('foo/bar')) + expect(page).to have_content("Group 'bar' was successfully created.") + end end end diff --git a/spec/features/projects/new_project_spec.rb b/spec/features/projects/new_project_spec.rb index 52196ce49bd..c66b9a34b86 100644 --- a/spec/features/projects/new_project_spec.rb +++ b/spec/features/projects/new_project_spec.rb @@ -71,6 +71,22 @@ feature "New project", feature: true do end end end + + context "with subgroup namespace" do + let(:group) { create(:group, :private, owner: user) } + let(:subgroup) { create(:group, parent: group) } + + before do + group.add_master(user) + visit new_project_path(namespace_id: subgroup.id) + end + + it "selects the group namespace" do + namespace = find("#project_namespace_id option[selected]") + + expect(namespace.text).to eq subgroup.full_path + end + end end context 'Import project options' do -- cgit v1.2.1 From 57374feabe1428b2ea06a6a3cac244612128095d Mon Sep 17 00:00:00 2001 From: Markus Koller Date: Tue, 24 Jan 2017 21:42:15 +0100 Subject: Move AuthHelper#two_factor_skippable? into ApplicationController --- app/controllers/application_controller.rb | 7 +++++++ app/helpers/auth_helper.rb | 12 ------------ 2 files changed, 7 insertions(+), 12 deletions(-) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 6a6e335d314..b197fd2157e 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -25,6 +25,7 @@ class ApplicationController < ActionController::Base helper_method :can?, :current_application_settings helper_method :import_sources_enabled?, :github_import_enabled?, :gitea_import_enabled?, :github_import_configured?, :gitlab_import_enabled?, :gitlab_import_configured?, :bitbucket_import_enabled?, :bitbucket_import_configured?, :google_code_import_enabled?, :fogbugz_import_enabled?, :git_import_enabled?, :gitlab_project_import_enabled? + helper_method :two_factor_grace_period_expired?, :two_factor_skippable? rescue_from Encoding::CompatibilityError do |exception| log_exception(exception) @@ -278,6 +279,12 @@ class ApplicationController < ActionController::Base date && (date + two_factor_grace_period.hours) < Time.current end + def two_factor_skippable? + two_factor_authentication_required? && + !current_user.two_factor_enabled? && + !two_factor_grace_period_expired? + end + def skip_two_factor? session[:skip_tfa] && session[:skip_tfa] > Time.current end diff --git a/app/helpers/auth_helper.rb b/app/helpers/auth_helper.rb index 101fe579da2..9c71d6c7f4c 100644 --- a/app/helpers/auth_helper.rb +++ b/app/helpers/auth_helper.rb @@ -64,18 +64,6 @@ module AuthHelper current_user.identities.exists?(provider: provider.to_s) end - def two_factor_skippable? - current_application_settings.require_two_factor_authentication && - !current_user.two_factor_enabled? && - current_application_settings.two_factor_grace_period && - !two_factor_grace_period_expired? - end - - def two_factor_grace_period_expired? - current_user.otp_grace_period_started_at && - (current_user.otp_grace_period_started_at + current_application_settings.two_factor_grace_period.hours) < Time.current - end - def unlink_allowed?(provider) %w(saml cas3).exclude?(provider.to_s) end -- cgit v1.2.1 From a3430f011f1adceaef8484f38a57018712a18ad2 Mon Sep 17 00:00:00 2001 From: Markus Koller Date: Tue, 24 Jan 2017 22:09:58 +0100 Subject: Support 2FA requirement per-group --- app/controllers/admin/groups_controller.rb | 4 +- app/controllers/application_controller.rb | 12 +- app/controllers/groups_controller.rb | 4 +- app/models/group.rb | 11 ++ app/models/members/group_member.rb | 5 + app/models/user.rb | 9 + app/views/admin/groups/_form.html.haml | 2 +- app/views/groups/_group_admin_settings.html.haml | 28 +++ app/views/groups/_group_lfs_settings.html.haml | 11 -- app/views/groups/edit.html.haml | 2 +- .../unreleased/feature-enforce-2fa-per-group.yml | 4 + ...4193147_add_two_factor_columns_to_namespaces.rb | 21 +++ ...170124193205_add_two_factor_columns_to_users.rb | 17 ++ db/schema.rb | 5 + .../two_factor_authentication_group_settings.png | Bin 0 -> 44874 bytes doc/security/two_factor_authentication.md | 17 +- spec/controllers/application_controller_spec.rb | 201 ++++++++++++++++++++- spec/models/group_spec.rb | 42 +++++ spec/models/members/group_member_spec.rb | 17 +- spec/models/user_spec.rb | 42 +++++ 20 files changed, 433 insertions(+), 21 deletions(-) create mode 100644 app/views/groups/_group_admin_settings.html.haml delete mode 100644 app/views/groups/_group_lfs_settings.html.haml create mode 100644 changelogs/unreleased/feature-enforce-2fa-per-group.yml create mode 100644 db/migrate/20170124193147_add_two_factor_columns_to_namespaces.rb create mode 100644 db/migrate/20170124193205_add_two_factor_columns_to_users.rb create mode 100644 doc/security/img/two_factor_authentication_group_settings.png diff --git a/app/controllers/admin/groups_controller.rb b/app/controllers/admin/groups_controller.rb index cea3d088e94..f28bbdeff5a 100644 --- a/app/controllers/admin/groups_controller.rb +++ b/app/controllers/admin/groups_controller.rb @@ -72,7 +72,9 @@ class Admin::GroupsController < Admin::ApplicationController :name, :path, :request_access_enabled, - :visibility_level + :visibility_level, + :require_two_factor_authentication, + :two_factor_grace_period ] end end diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index b197fd2157e..28c4380ca84 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -267,11 +267,19 @@ class ApplicationController < ActionController::Base end def two_factor_authentication_required? - current_application_settings.require_two_factor_authentication + current_application_settings.require_two_factor_authentication || + current_user.try(:require_two_factor_authentication) end def two_factor_grace_period - current_application_settings.two_factor_grace_period + if current_user.try(:require_two_factor_authentication) + [ + current_application_settings.two_factor_grace_period, + current_user.two_factor_grace_period + ].min + else + current_application_settings.two_factor_grace_period + end end def two_factor_grace_period_expired? diff --git a/app/controllers/groups_controller.rb b/app/controllers/groups_controller.rb index 05f9ee1ee90..5f90df579a8 100644 --- a/app/controllers/groups_controller.rb +++ b/app/controllers/groups_controller.rb @@ -150,7 +150,9 @@ class GroupsController < Groups::ApplicationController :visibility_level, :parent_id, :create_chat_team, - :chat_team_name + :chat_team_name, + :require_two_factor_authentication, + :two_factor_grace_period ] end diff --git a/app/models/group.rb b/app/models/group.rb index 60274386103..106084175ff 100644 --- a/app/models/group.rb +++ b/app/models/group.rb @@ -27,11 +27,14 @@ class Group < Namespace validates :avatar, file_size: { maximum: 200.kilobytes.to_i } + validates :two_factor_grace_period, presence: true, numericality: { greater_than_or_equal_to: 0 } + mount_uploader :avatar, AvatarUploader has_many :uploads, as: :model, dependent: :destroy after_create :post_create_hook after_destroy :post_destroy_hook + after_save :update_two_factor_requirement class << self # Searches for groups matching the given query. @@ -223,4 +226,12 @@ class Group < Namespace type: public? ? 'O' : 'I' # Open vs Invite-only } end + + protected + + def update_two_factor_requirement + return unless require_two_factor_authentication_changed? || two_factor_grace_period_changed? + + users.find_each(&:update_two_factor_requirement) + end end diff --git a/app/models/members/group_member.rb b/app/models/members/group_member.rb index 446f9f8f8a7..483425cd30f 100644 --- a/app/models/members/group_member.rb +++ b/app/models/members/group_member.rb @@ -3,11 +3,16 @@ class GroupMember < Member belongs_to :group, foreign_key: 'source_id' + delegate :update_two_factor_requirement, to: :user + # Make sure group member points only to group as it source default_value_for :source_type, SOURCE_TYPE validates :source_type, format: { with: /\ANamespace\z/ } default_scope { where(source_type: SOURCE_TYPE) } + after_create :update_two_factor_requirement, unless: :invite? + after_destroy :update_two_factor_requirement, unless: :invite? + def self.access_level_roles Gitlab::Access.options_with_owner end diff --git a/app/models/user.rb b/app/models/user.rb index 95a766f2ede..564e99df77b 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -963,6 +963,15 @@ class User < ActiveRecord::Base super end + def update_two_factor_requirement + periods = groups.where(require_two_factor_authentication: true).pluck(:two_factor_grace_period) + + self.require_two_factor_authentication = periods.any? + self.two_factor_grace_period = periods.min || User.column_defaults['two_factor_grace_period'] + + save + end + private def ci_projects_union diff --git a/app/views/admin/groups/_form.html.haml b/app/views/admin/groups/_form.html.haml index 589f4557b52..d9f05003904 100644 --- a/app/views/admin/groups/_form.html.haml +++ b/app/views/admin/groups/_form.html.haml @@ -13,7 +13,7 @@ .col-sm-offset-2.col-sm-10 = render 'shared/allow_request_access', form: f - = render 'groups/group_lfs_settings', f: f + = render 'groups/group_admin_settings', f: f - if @group.new_record? .form-group diff --git a/app/views/groups/_group_admin_settings.html.haml b/app/views/groups/_group_admin_settings.html.haml new file mode 100644 index 00000000000..2ace1e2dd1e --- /dev/null +++ b/app/views/groups/_group_admin_settings.html.haml @@ -0,0 +1,28 @@ +- if current_user.admin? + .form-group + = f.label :lfs_enabled, 'Large File Storage', class: 'control-label' + .col-sm-10 + .checkbox + = f.label :lfs_enabled do + = f.check_box :lfs_enabled, checked: @group.lfs_enabled? + %strong + Allow projects within this group to use Git LFS + = link_to icon('question-circle'), help_page_path('workflow/lfs/manage_large_binaries_with_git_lfs') + %br/ + %span.descr This setting can be overridden in each project. + +- if can? current_user, :admin_group, @group + .form-group + = f.label :require_two_factor_authentication, 'Two-factor authentication', class: 'control-label col-sm-2' + .col-sm-10 + .checkbox + = f.label :require_two_factor_authentication do + = f.check_box :require_two_factor_authentication + %strong + Require all users in this group to setup Two-factor authentication + = link_to icon('question-circle'), help_page_path('security/two_factor_authentication', anchor: 'enforcing-2fa-for-all-users-in-a-group') + .form-group + .col-sm-offset-2.col-sm-10 + .checkbox + = f.text_field :two_factor_grace_period, class: 'form-control' + .help-block Amount of time (in hours) that users are allowed to skip forced configuration of two-factor authentication diff --git a/app/views/groups/_group_lfs_settings.html.haml b/app/views/groups/_group_lfs_settings.html.haml deleted file mode 100644 index 3c622ca5c3c..00000000000 --- a/app/views/groups/_group_lfs_settings.html.haml +++ /dev/null @@ -1,11 +0,0 @@ -- if current_user.admin? - .form-group - .col-sm-offset-2.col-sm-10 - .checkbox - = f.label :lfs_enabled do - = f.check_box :lfs_enabled, checked: @group.lfs_enabled? - %strong - Allow projects within this group to use Git LFS - = link_to icon('question-circle'), help_page_path('workflow/lfs/manage_large_binaries_with_git_lfs') - %br/ - %span.descr This setting can be overridden in each project. diff --git a/app/views/groups/edit.html.haml b/app/views/groups/edit.html.haml index 80a77dab97f..00ff40224ba 100644 --- a/app/views/groups/edit.html.haml +++ b/app/views/groups/edit.html.haml @@ -27,7 +27,7 @@ .col-sm-offset-2.col-sm-10 = render 'shared/allow_request_access', form: f - = render 'group_lfs_settings', f: f + = render 'group_admin_settings', f: f .form-group %hr diff --git a/changelogs/unreleased/feature-enforce-2fa-per-group.yml b/changelogs/unreleased/feature-enforce-2fa-per-group.yml new file mode 100644 index 00000000000..6dd99e4245f --- /dev/null +++ b/changelogs/unreleased/feature-enforce-2fa-per-group.yml @@ -0,0 +1,4 @@ +--- +title: Support 2FA requirement per-group +merge_request: 8763 +author: Markus Koller diff --git a/db/migrate/20170124193147_add_two_factor_columns_to_namespaces.rb b/db/migrate/20170124193147_add_two_factor_columns_to_namespaces.rb new file mode 100644 index 00000000000..ca4429c676c --- /dev/null +++ b/db/migrate/20170124193147_add_two_factor_columns_to_namespaces.rb @@ -0,0 +1,21 @@ +class AddTwoFactorColumnsToNamespaces < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_column_with_default(:namespaces, :require_two_factor_authentication, :boolean, default: false) + add_column_with_default(:namespaces, :two_factor_grace_period, :integer, default: 48) + + add_concurrent_index(:namespaces, :require_two_factor_authentication) + end + + def down + remove_column(:namespaces, :require_two_factor_authentication) + remove_column(:namespaces, :two_factor_grace_period) + + remove_index(:namespaces, :require_two_factor_authentication) if index_exists?(:namespaces, :require_two_factor_authentication) + end +end diff --git a/db/migrate/20170124193205_add_two_factor_columns_to_users.rb b/db/migrate/20170124193205_add_two_factor_columns_to_users.rb new file mode 100644 index 00000000000..bef1b2062c8 --- /dev/null +++ b/db/migrate/20170124193205_add_two_factor_columns_to_users.rb @@ -0,0 +1,17 @@ +class AddTwoFactorColumnsToUsers < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_column_with_default(:users, :require_two_factor_authentication, :boolean, default: false) + add_column_with_default(:users, :two_factor_grace_period, :integer, default: 48) + end + + def down + remove_column(:users, :require_two_factor_authentication) + remove_column(:users, :two_factor_grace_period) + end +end diff --git a/db/schema.rb b/db/schema.rb index ccf18d07179..2d2507828d4 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -692,6 +692,8 @@ ActiveRecord::Schema.define(version: 20170402231018) do t.text "description_html" t.boolean "lfs_enabled" t.integer "parent_id" + t.boolean "require_two_factor_authentication", default: false, null: false + t.integer "two_factor_grace_period", default: 48, null: false end add_index "namespaces", ["created_at"], name: "index_namespaces_on_created_at", using: :btree @@ -702,6 +704,7 @@ ActiveRecord::Schema.define(version: 20170402231018) do add_index "namespaces", ["parent_id", "id"], name: "index_namespaces_on_parent_id_and_id", unique: true, using: :btree add_index "namespaces", ["path"], name: "index_namespaces_on_path", using: :btree add_index "namespaces", ["path"], name: "index_namespaces_on_path_trigram", using: :gin, opclasses: {"path"=>"gin_trgm_ops"} + add_index "namespaces", ["require_two_factor_authentication"], name: "index_namespaces_on_require_two_factor_authentication", using: :btree add_index "namespaces", ["type"], name: "index_namespaces_on_type", using: :btree create_table "notes", force: :cascade do |t| @@ -1246,6 +1249,8 @@ ActiveRecord::Schema.define(version: 20170402231018) do t.boolean "authorized_projects_populated" t.boolean "ghost" t.boolean "notified_of_own_activity" + t.boolean "require_two_factor_authentication", default: false, null: false + t.integer "two_factor_grace_period", default: 48, null: false end add_index "users", ["admin"], name: "index_users_on_admin", using: :btree diff --git a/doc/security/img/two_factor_authentication_group_settings.png b/doc/security/img/two_factor_authentication_group_settings.png new file mode 100644 index 00000000000..a1b3c58bfdc Binary files /dev/null and b/doc/security/img/two_factor_authentication_group_settings.png differ diff --git a/doc/security/two_factor_authentication.md b/doc/security/two_factor_authentication.md index c8499380c18..f02f7b807cf 100644 --- a/doc/security/two_factor_authentication.md +++ b/doc/security/two_factor_authentication.md @@ -8,7 +8,7 @@ their phone. You can read more about it here: [Two-factor Authentication (2FA)](../profile/two_factor_authentication.md) -## Enabling 2FA +## Enforcing 2FA for all users Users on GitLab, can enable it without any admin's intervention. If you want to enforce everyone to setup 2FA, you can choose from two different ways: @@ -28,6 +28,21 @@ period to `0`. --- +## Enforcing 2FA for all users in a group + +If you want to enforce 2FA only for certain groups, you can enable it in the +group settings and specify a grace period as above. To change this setting you +need to be administrator or owner of the group. + +If there are multiple 2FA requirements (i.e. group + all users, or multiple +groups) the shortest grace period will be used. + +--- + +![Two factor authentication group settings](img/two_factor_authentication_group_settings.png) + +--- + ## Disabling 2FA for everyone There may be some special situations where you want to disable 2FA for everyone diff --git a/spec/controllers/application_controller_spec.rb b/spec/controllers/application_controller_spec.rb index 81cbccd5436..64cfb87da5d 100644 --- a/spec/controllers/application_controller_spec.rb +++ b/spec/controllers/application_controller_spec.rb @@ -100,8 +100,6 @@ describe ApplicationController do end describe '#route_not_found' do - let(:controller) { ApplicationController.new } - it 'renders 404 if authenticated' do allow(controller).to receive(:current_user).and_return(user) expect(controller).to receive(:not_found) @@ -115,4 +113,203 @@ describe ApplicationController do controller.send(:route_not_found) end end + + context 'two-factor authentication' do + let(:controller) { ApplicationController.new } + + describe '#check_2fa_requirement' do + subject { controller.send :check_2fa_requirement } + + it 'does not redirect if 2FA is not required' do + allow(controller).to receive(:two_factor_authentication_required?).and_return(false) + expect(controller).not_to receive(:redirect_to) + + subject + end + + it 'does not redirect if user is not logged in' do + allow(controller).to receive(:two_factor_authentication_required?).and_return(true) + allow(controller).to receive(:current_user).and_return(nil) + expect(controller).not_to receive(:redirect_to) + + subject + end + + it 'does not redirect if user has 2FA enabled' do + allow(controller).to receive(:two_factor_authentication_required?).and_return(true) + allow(controller).to receive(:current_user).twice.and_return(user) + allow(user).to receive(:two_factor_enabled?).and_return(true) + expect(controller).not_to receive(:redirect_to) + + subject + end + + it 'does not redirect if 2FA setup can be skipped' do + allow(controller).to receive(:two_factor_authentication_required?).and_return(true) + allow(controller).to receive(:current_user).twice.and_return(user) + allow(user).to receive(:two_factor_enabled?).and_return(false) + allow(controller).to receive(:skip_two_factor?).and_return(true) + expect(controller).not_to receive(:redirect_to) + + subject + end + + it 'redirects to 2FA setup otherwise' do + allow(controller).to receive(:two_factor_authentication_required?).and_return(true) + allow(controller).to receive(:current_user).twice.and_return(user) + allow(user).to receive(:two_factor_enabled?).and_return(false) + allow(controller).to receive(:skip_two_factor?).and_return(false) + allow(controller).to receive(:profile_two_factor_auth_path) + expect(controller).to receive(:redirect_to) + + subject + end + end + + describe '#two_factor_authentication_required?' do + subject { controller.send :two_factor_authentication_required? } + + it 'returns false if no 2FA requirement is present' do + allow(controller).to receive(:current_user).and_return(nil) + + expect(subject).to be_falsey + end + + it 'returns true if a 2FA requirement is set in the application settings' do + stub_application_setting require_two_factor_authentication: true + allow(controller).to receive(:current_user).and_return(nil) + + expect(subject).to be_truthy + end + + it 'returns true if a 2FA requirement is set on the user' do + user.require_two_factor_authentication = true + allow(controller).to receive(:current_user).and_return(user) + + expect(subject).to be_truthy + end + end + + describe '#two_factor_grace_period' do + subject { controller.send :two_factor_grace_period } + + it 'returns the grace period from the application settings' do + stub_application_setting two_factor_grace_period: 23 + allow(controller).to receive(:current_user).and_return(nil) + + expect(subject).to eq 23 + end + + context 'with a 2FA requirement set on the user' do + let(:user) { create :user, require_two_factor_authentication: true, two_factor_grace_period: 23 } + + it 'returns the user grace period if lower than the application grace period' do + stub_application_setting two_factor_grace_period: 24 + allow(controller).to receive(:current_user).and_return(user) + + expect(subject).to eq 23 + end + + it 'returns the application grace period if lower than the user grace period' do + stub_application_setting two_factor_grace_period: 22 + allow(controller).to receive(:current_user).and_return(user) + + expect(subject).to eq 22 + end + end + end + + describe '#two_factor_grace_period_expired?' do + subject { controller.send :two_factor_grace_period_expired? } + + before do + allow(controller).to receive(:current_user).and_return(user) + end + + it 'returns false if the user has not started their grace period yet' do + expect(subject).to be_falsey + end + + context 'with grace period started' do + let(:user) { create :user, otp_grace_period_started_at: 2.hours.ago } + + it 'returns true if the grace period has expired' do + allow(controller).to receive(:two_factor_grace_period).and_return(1) + + expect(subject).to be_truthy + end + + it 'returns false if the grace period is still active' do + allow(controller).to receive(:two_factor_grace_period).and_return(3) + + expect(subject).to be_falsey + end + end + end + + describe '#two_factor_skippable' do + subject { controller.send :two_factor_skippable? } + + before do + allow(controller).to receive(:current_user).and_return(user) + end + + it 'returns false if 2FA is not required' do + allow(controller).to receive(:two_factor_authentication_required?).and_return(false) + + expect(subject).to be_falsey + end + + it 'returns false if the user has already enabled 2FA' do + allow(controller).to receive(:two_factor_authentication_required?).and_return(true) + allow(user).to receive(:two_factor_enabled?).and_return(true) + + expect(subject).to be_falsey + end + + it 'returns false if the 2FA grace period has expired' do + allow(controller).to receive(:two_factor_authentication_required?).and_return(true) + allow(user).to receive(:two_factor_enabled?).and_return(false) + allow(controller).to receive(:two_factor_grace_period_expired?).and_return(true) + + expect(subject).to be_falsey + end + + it 'returns true otherwise' do + allow(controller).to receive(:two_factor_authentication_required?).and_return(true) + allow(user).to receive(:two_factor_enabled?).and_return(false) + allow(controller).to receive(:two_factor_grace_period_expired?).and_return(false) + + expect(subject).to be_truthy + end + end + + describe '#skip_two_factor?' do + subject { controller.send :skip_two_factor? } + + it 'returns false if 2FA setup was not skipped' do + allow(controller).to receive(:session).and_return({}) + + expect(subject).to be_falsey + end + + context 'with 2FA setup skipped' do + before do + allow(controller).to receive(:session).and_return({ skip_tfa: 2.hours.from_now }) + end + + it 'returns false if the grace period has expired' do + Timecop.freeze(3.hours.from_now) do + expect(subject).to be_falsey + end + end + + it 'returns true if the grace period is still active' do + Timecop.freeze(1.hour.from_now) do + expect(subject).to be_truthy + end + end + end + end + end end diff --git a/spec/models/group_spec.rb b/spec/models/group_spec.rb index 5d87938235a..8ffde6f7fbb 100644 --- a/spec/models/group_spec.rb +++ b/spec/models/group_spec.rb @@ -55,6 +55,8 @@ describe Group, models: true do it { is_expected.to validate_uniqueness_of(:name).scoped_to(:parent_id) } it { is_expected.to validate_presence_of :path } it { is_expected.not_to validate_presence_of :owner } + it { is_expected.to validate_presence_of :two_factor_grace_period } + it { is_expected.to validate_numericality_of(:two_factor_grace_period).is_greater_than_or_equal_to(0) } end describe '.visible_to_user' do @@ -315,4 +317,44 @@ describe Group, models: true do to include(master.id, developer.id) end end + + describe '#update_two_factor_requirement' do + let(:user) { create(:user) } + + before do + group.add_user(user, GroupMember::OWNER) + end + + it 'is called when require_two_factor_authentication is changed' do + expect_any_instance_of(User).to receive(:update_two_factor_requirement) + + group.update!(require_two_factor_authentication: true) + end + + it 'is called when two_factor_grace_period is changed' do + expect_any_instance_of(User).to receive(:update_two_factor_requirement) + + group.update!(two_factor_grace_period: 23) + end + + it 'is not called when other attributes are changed' do + expect_any_instance_of(User).not_to receive(:update_two_factor_requirement) + + group.update!(description: 'foobar') + end + + it 'calls #update_two_factor_requirement on each group member' do + other_user = create(:user) + group.add_user(other_user, GroupMember::OWNER) + + calls = 0 + allow_any_instance_of(User).to receive(:update_two_factor_requirement) do + calls += 1 + end + + group.update!(require_two_factor_authentication: true, two_factor_grace_period: 23) + + expect(calls).to eq 2 + end + end end diff --git a/spec/models/members/group_member_spec.rb b/spec/models/members/group_member_spec.rb index 370aeb9e0a9..024380b7ebb 100644 --- a/spec/models/members/group_member_spec.rb +++ b/spec/models/members/group_member_spec.rb @@ -61,7 +61,7 @@ describe GroupMember, models: true do describe '#after_accept_request' do it 'calls NotificationService.accept_group_access_request' do - member = create(:group_member, user: build_stubbed(:user), requested_at: Time.now) + member = create(:group_member, user: build(:user), requested_at: Time.now) expect_any_instance_of(NotificationService).to receive(:new_group_member) @@ -75,4 +75,19 @@ describe GroupMember, models: true do it { is_expected.to eq 'Group' } end end + + describe '#update_two_factor_requirement' do + let(:user) { build :user } + let(:group_member) { build :group_member, user: user } + + it 'is called after creation and deletion' do + expect(user).to receive(:update_two_factor_requirement) + + group_member.save + + expect(user).to receive(:update_two_factor_requirement) + + group_member.destroy + end + end end diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index a9e37be1157..b2f686a1819 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -1521,4 +1521,46 @@ describe User, models: true do end end end + + describe '#update_two_factor_requirement' do + let(:user) { create :user } + + context 'with 2FA requirement on groups' do + let(:group1) { create :group, require_two_factor_authentication: true, two_factor_grace_period: 23 } + let(:group2) { create :group, require_two_factor_authentication: true, two_factor_grace_period: 32 } + + before do + group1.add_user(user, GroupMember::OWNER) + group2.add_user(user, GroupMember::OWNER) + + user.update_two_factor_requirement + end + + it 'requires 2FA' do + expect(user.require_two_factor_authentication).to be true + end + + it 'uses the shortest grace period' do + expect(user.two_factor_grace_period).to be 23 + end + end + + context 'without 2FA requirement on groups' do + let(:group) { create :group } + + before do + group.add_user(user, GroupMember::OWNER) + + user.update_two_factor_requirement + end + + it 'does not require 2FA' do + expect(user.require_two_factor_authentication).to be false + end + + it 'falls back to the default grace period' do + expect(user.two_factor_grace_period).to be 48 + end + end + end end -- cgit v1.2.1 From 7140e09e39895d747bca7238a5c9f5a4d4637a85 Mon Sep 17 00:00:00 2001 From: Markus Koller Date: Mon, 6 Feb 2017 17:07:37 +0100 Subject: Extract 2FA-related code from ApplicationController --- app/controllers/application_controller.rb | 40 +----------------- .../concerns/enforces_two_factor_authentication.rb | 47 ++++++++++++++++++++++ 2 files changed, 48 insertions(+), 39 deletions(-) create mode 100644 app/controllers/concerns/enforces_two_factor_authentication.rb diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 28c4380ca84..e77094fe2a8 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -8,12 +8,12 @@ class ApplicationController < ActionController::Base include PageLayoutHelper include SentryHelper include WorkhorseHelper + include EnforcesTwoFactorAuthentication before_action :authenticate_user_from_private_token! before_action :authenticate_user! before_action :validate_user_service_ticket! before_action :check_password_expiration - before_action :check_2fa_requirement before_action :ldap_security_check before_action :sentry_context before_action :default_headers @@ -25,7 +25,6 @@ class ApplicationController < ActionController::Base helper_method :can?, :current_application_settings helper_method :import_sources_enabled?, :github_import_enabled?, :gitea_import_enabled?, :github_import_configured?, :gitlab_import_enabled?, :gitlab_import_configured?, :bitbucket_import_enabled?, :bitbucket_import_configured?, :google_code_import_enabled?, :fogbugz_import_enabled?, :git_import_enabled?, :gitlab_project_import_enabled? - helper_method :two_factor_grace_period_expired?, :two_factor_skippable? rescue_from Encoding::CompatibilityError do |exception| log_exception(exception) @@ -152,12 +151,6 @@ class ApplicationController < ActionController::Base end end - def check_2fa_requirement - if two_factor_authentication_required? && current_user && !current_user.two_factor_enabled? && !skip_two_factor? - redirect_to profile_two_factor_auth_path - end - end - def ldap_security_check if current_user && current_user.requires_ldap_check? return unless current_user.try_obtain_ldap_lease @@ -266,37 +259,6 @@ class ApplicationController < ActionController::Base current_application_settings.import_sources.include?('gitlab_project') end - def two_factor_authentication_required? - current_application_settings.require_two_factor_authentication || - current_user.try(:require_two_factor_authentication) - end - - def two_factor_grace_period - if current_user.try(:require_two_factor_authentication) - [ - current_application_settings.two_factor_grace_period, - current_user.two_factor_grace_period - ].min - else - current_application_settings.two_factor_grace_period - end - end - - def two_factor_grace_period_expired? - date = current_user.otp_grace_period_started_at - date && (date + two_factor_grace_period.hours) < Time.current - end - - def two_factor_skippable? - two_factor_authentication_required? && - !current_user.two_factor_enabled? && - !two_factor_grace_period_expired? - end - - def skip_two_factor? - session[:skip_tfa] && session[:skip_tfa] > Time.current - end - # U2F (universal 2nd factor) devices need a unique identifier for the application # to perform authentication. # https://developers.yubico.com/U2F/App_ID.html diff --git a/app/controllers/concerns/enforces_two_factor_authentication.rb b/app/controllers/concerns/enforces_two_factor_authentication.rb new file mode 100644 index 00000000000..b490f0058c7 --- /dev/null +++ b/app/controllers/concerns/enforces_two_factor_authentication.rb @@ -0,0 +1,47 @@ +# == EnforcesTwoFactorAuthentication +# +# Controller concern to enforce two-factor authentication requirements +# +# Upon inclusion, adds `check_2fa_requirement` as a before_action, and +# makes `two_factor_grace_period_expired?` and `two_factor_skippable?` +# available as view helpers. +module EnforcesTwoFactorAuthentication + extend ActiveSupport::Concern + + included do + before_action :check_2fa_requirement + helper_method :two_factor_grace_period_expired?, :two_factor_skippable? + end + + def check_2fa_requirement + if two_factor_authentication_required? && current_user && !current_user.two_factor_enabled? && !skip_two_factor? + redirect_to profile_two_factor_auth_path + end + end + + def two_factor_authentication_required? + current_application_settings.require_two_factor_authentication? || + current_user.try(:require_two_factor_authentication?) + end + + def two_factor_grace_period + periods = [current_application_settings.two_factor_grace_period] + periods << current_user.two_factor_grace_period if current_user.try(:require_two_factor_authentication?) + periods.min + end + + def two_factor_grace_period_expired? + date = current_user.otp_grace_period_started_at + date && (date + two_factor_grace_period.hours) < Time.current + end + + def two_factor_skippable? + two_factor_authentication_required? && + !current_user.two_factor_enabled? && + !two_factor_grace_period_expired? + end + + def skip_two_factor? + session[:skip_tfa] && session[:skip_tfa] > Time.current + end +end -- cgit v1.2.1 From 8e665140565a8c022bf1cad1589e244a543419fa Mon Sep 17 00:00:00 2001 From: Markus Koller Date: Tue, 7 Mar 2017 19:48:57 +0100 Subject: Rename check_2fa_requirement to check_two_factor_requirement --- app/controllers/concerns/enforces_two_factor_authentication.rb | 8 ++++---- app/controllers/profiles/two_factor_auths_controller.rb | 2 +- app/controllers/sessions_controller.rb | 2 +- spec/controllers/application_controller_spec.rb | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/app/controllers/concerns/enforces_two_factor_authentication.rb b/app/controllers/concerns/enforces_two_factor_authentication.rb index b490f0058c7..05d427f83a0 100644 --- a/app/controllers/concerns/enforces_two_factor_authentication.rb +++ b/app/controllers/concerns/enforces_two_factor_authentication.rb @@ -2,18 +2,18 @@ # # Controller concern to enforce two-factor authentication requirements # -# Upon inclusion, adds `check_2fa_requirement` as a before_action, and -# makes `two_factor_grace_period_expired?` and `two_factor_skippable?` +# Upon inclusion, adds `check_two_factor_requirement` as a before_action, +# and makes `two_factor_grace_period_expired?` and `two_factor_skippable?` # available as view helpers. module EnforcesTwoFactorAuthentication extend ActiveSupport::Concern included do - before_action :check_2fa_requirement + before_action :check_two_factor_requirement helper_method :two_factor_grace_period_expired?, :two_factor_skippable? end - def check_2fa_requirement + def check_two_factor_requirement if two_factor_authentication_required? && current_user && !current_user.two_factor_enabled? && !skip_two_factor? redirect_to profile_two_factor_auth_path end diff --git a/app/controllers/profiles/two_factor_auths_controller.rb b/app/controllers/profiles/two_factor_auths_controller.rb index 26e7e93533e..5e54fdfcf44 100644 --- a/app/controllers/profiles/two_factor_auths_controller.rb +++ b/app/controllers/profiles/two_factor_auths_controller.rb @@ -1,5 +1,5 @@ class Profiles::TwoFactorAuthsController < Profiles::ApplicationController - skip_before_action :check_2fa_requirement + skip_before_action :check_two_factor_requirement def show unless current_user.otp_secret diff --git a/app/controllers/sessions_controller.rb b/app/controllers/sessions_controller.rb index d8561871098..d3091a4f8e9 100644 --- a/app/controllers/sessions_controller.rb +++ b/app/controllers/sessions_controller.rb @@ -3,7 +3,7 @@ class SessionsController < Devise::SessionsController include Devise::Controllers::Rememberable include Recaptcha::ClientHelper - skip_before_action :check_2fa_requirement, only: [:destroy] + skip_before_action :check_two_factor_requirement, only: [:destroy] prepend_before_action :check_initial_setup, only: [:new] prepend_before_action :authenticate_with_two_factor, diff --git a/spec/controllers/application_controller_spec.rb b/spec/controllers/application_controller_spec.rb index 64cfb87da5d..004cbba3398 100644 --- a/spec/controllers/application_controller_spec.rb +++ b/spec/controllers/application_controller_spec.rb @@ -117,8 +117,8 @@ describe ApplicationController do context 'two-factor authentication' do let(:controller) { ApplicationController.new } - describe '#check_2fa_requirement' do - subject { controller.send :check_2fa_requirement } + describe '#check_two_factor_requirement' do + subject { controller.send :check_two_factor_requirement } it 'does not redirect if 2FA is not required' do allow(controller).to receive(:two_factor_authentication_required?).and_return(false) -- cgit v1.2.1 From a49c5d18364fc3f4b475d639e5de55fd1558351c Mon Sep 17 00:00:00 2001 From: Markus Koller Date: Tue, 7 Mar 2017 19:51:22 +0100 Subject: Rename skip_tfa session variable to skip_two_factor --- app/controllers/concerns/enforces_two_factor_authentication.rb | 2 +- app/controllers/profiles/two_factor_auths_controller.rb | 2 +- spec/controllers/application_controller_spec.rb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/controllers/concerns/enforces_two_factor_authentication.rb b/app/controllers/concerns/enforces_two_factor_authentication.rb index 05d427f83a0..a3696df47e7 100644 --- a/app/controllers/concerns/enforces_two_factor_authentication.rb +++ b/app/controllers/concerns/enforces_two_factor_authentication.rb @@ -42,6 +42,6 @@ module EnforcesTwoFactorAuthentication end def skip_two_factor? - session[:skip_tfa] && session[:skip_tfa] > Time.current + session[:skip_two_factor] && session[:skip_two_factor] > Time.current end end diff --git a/app/controllers/profiles/two_factor_auths_controller.rb b/app/controllers/profiles/two_factor_auths_controller.rb index 5e54fdfcf44..b52134d89a4 100644 --- a/app/controllers/profiles/two_factor_auths_controller.rb +++ b/app/controllers/profiles/two_factor_auths_controller.rb @@ -71,7 +71,7 @@ class Profiles::TwoFactorAuthsController < Profiles::ApplicationController if two_factor_grace_period_expired? redirect_to new_profile_two_factor_auth_path, alert: 'Cannot skip two factor authentication setup' else - session[:skip_tfa] = current_user.otp_grace_period_started_at + two_factor_grace_period.hours + session[:skip_two_factor] = current_user.otp_grace_period_started_at + two_factor_grace_period.hours redirect_to root_path end end diff --git a/spec/controllers/application_controller_spec.rb b/spec/controllers/application_controller_spec.rb index 004cbba3398..7427c93b593 100644 --- a/spec/controllers/application_controller_spec.rb +++ b/spec/controllers/application_controller_spec.rb @@ -295,7 +295,7 @@ describe ApplicationController do context 'with 2FA setup skipped' do before do - allow(controller).to receive(:session).and_return({ skip_tfa: 2.hours.from_now }) + allow(controller).to receive(:session).and_return({ skip_two_factor: 2.hours.from_now }) end it 'returns false if the grace period has expired' do -- cgit v1.2.1 From b7ca7330ec9119c6a5eea00df20ddc690d4dafe1 Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Wed, 8 Mar 2017 12:09:15 +0100 Subject: state the reason to the user for the required 2fa --- .../concerns/enforces_two_factor_authentication.rb | 11 ++ .../profiles/two_factor_auths_controller.rb | 21 +++- spec/features/login_spec.rb | 131 ++++++++++++++++----- 3 files changed, 130 insertions(+), 33 deletions(-) diff --git a/app/controllers/concerns/enforces_two_factor_authentication.rb b/app/controllers/concerns/enforces_two_factor_authentication.rb index a3696df47e7..3e0c62172de 100644 --- a/app/controllers/concerns/enforces_two_factor_authentication.rb +++ b/app/controllers/concerns/enforces_two_factor_authentication.rb @@ -24,6 +24,17 @@ module EnforcesTwoFactorAuthentication current_user.try(:require_two_factor_authentication?) end + def two_factor_authentication_reason(global: -> {}, group: -> {}) + if two_factor_authentication_required? + if current_application_settings.require_two_factor_authentication? + global.call + else + groups = current_user.groups.where(require_two_factor_authentication: true).reorder(name: :asc) + group.call(groups) + end + end + end + def two_factor_grace_period periods = [current_application_settings.two_factor_grace_period] periods << current_user.two_factor_grace_period if current_user.try(:require_two_factor_authentication?) diff --git a/app/controllers/profiles/two_factor_auths_controller.rb b/app/controllers/profiles/two_factor_auths_controller.rb index b52134d89a4..d3fa81cd623 100644 --- a/app/controllers/profiles/two_factor_auths_controller.rb +++ b/app/controllers/profiles/two_factor_auths_controller.rb @@ -13,11 +13,24 @@ class Profiles::TwoFactorAuthsController < Profiles::ApplicationController current_user.save! if current_user.changed? if two_factor_authentication_required? && !current_user.two_factor_enabled? - if two_factor_grace_period_expired? - flash.now[:alert] = 'You must enable Two-Factor Authentication for your account.' - else + two_factor_authentication_reason( + global: lambda do + flash.now[:alert] = + 'The global settings require you to enable Two-Factor Authentication for your account.' + end, + group: lambda do |groups| + group_links = groups.map { |group| view_context.link_to group.full_name, group_path(group) }.to_sentence + + flash.now[:alert] = %{ + The group settings for #{group_links} require you to enable + Two-Factor Authentication for your account. + }.html_safe + end + ) + + unless two_factor_grace_period_expired? grace_period_deadline = current_user.otp_grace_period_started_at + two_factor_grace_period.hours - flash.now[:alert] = "You must enable Two-Factor Authentication for your account before #{l(grace_period_deadline)}." + flash.now[:alert] << " You need to do this before #{l(grace_period_deadline)}." end end diff --git a/spec/features/login_spec.rb b/spec/features/login_spec.rb index f32d1f78b40..11d417c253d 100644 --- a/spec/features/login_spec.rb +++ b/spec/features/login_spec.rb @@ -199,52 +199,125 @@ feature 'Login', feature: true do describe 'with required two-factor authentication enabled' do let(:user) { create(:user) } - before(:each) { stub_application_setting(require_two_factor_authentication: true) } + # TODO: otp_grace_period_started_at - context 'with grace period defined' do - before(:each) do - stub_application_setting(two_factor_grace_period: 48) - login_with(user) - end + context 'global setting' do + before(:each) { stub_application_setting(require_two_factor_authentication: true) } - context 'within the grace period' do - it 'redirects to two-factor configuration page' do - expect(current_path).to eq profile_two_factor_auth_path - expect(page).to have_content('You must enable Two-Factor Authentication for your account before') + context 'with grace period defined' do + before(:each) do + stub_application_setting(two_factor_grace_period: 48) + login_with(user) end - it 'allows skipping two-factor configuration', js: true do - expect(current_path).to eq profile_two_factor_auth_path + context 'within the grace period' do + it 'redirects to two-factor configuration page' do + expect(current_path).to eq profile_two_factor_auth_path + expect(page).to have_content('The global settings require you to enable Two-Factor Authentication for your account. You need to do this before ') + end - click_link 'Configure it later' - expect(current_path).to eq root_path + it 'allows skipping two-factor configuration', js: true do + expect(current_path).to eq profile_two_factor_auth_path + + click_link 'Configure it later' + expect(current_path).to eq root_path + end end - end - context 'after the grace period' do - let(:user) { create(:user, otp_grace_period_started_at: 9999.hours.ago) } + context 'after the grace period' do + let(:user) { create(:user, otp_grace_period_started_at: 9999.hours.ago) } - it 'redirects to two-factor configuration page' do - expect(current_path).to eq profile_two_factor_auth_path - expect(page).to have_content('You must enable Two-Factor Authentication for your account.') + it 'redirects to two-factor configuration page' do + expect(current_path).to eq profile_two_factor_auth_path + expect(page).to have_content( + 'The global settings require you to enable Two-Factor Authentication for your account.' + ) + end + + it 'disallows skipping two-factor configuration', js: true do + expect(current_path).to eq profile_two_factor_auth_path + expect(page).not_to have_link('Configure it later') + end + end + end + + context 'without grace period defined' do + before(:each) do + stub_application_setting(two_factor_grace_period: 0) + login_with(user) end - it 'disallows skipping two-factor configuration', js: true do + it 'redirects to two-factor configuration page' do expect(current_path).to eq profile_two_factor_auth_path - expect(page).not_to have_link('Configure it later') + expect(page).to have_content( + 'The global settings require you to enable Two-Factor Authentication for your account.' + ) end end end - context 'without grace period defined' do - before(:each) do - stub_application_setting(two_factor_grace_period: 0) - login_with(user) + context 'group setting' do + before do + group1 = create :group, name: 'Group 1', require_two_factor_authentication: true + group1.add_user(user, GroupMember::DEVELOPER) + group2 = create :group, name: 'Group 2', require_two_factor_authentication: true + group2.add_user(user, GroupMember::DEVELOPER) end - it 'redirects to two-factor configuration page' do - expect(current_path).to eq profile_two_factor_auth_path - expect(page).to have_content('You must enable Two-Factor Authentication for your account.') + context 'with grace period defined' do + before(:each) do + stub_application_setting(two_factor_grace_period: 48) + login_with(user) + end + + context 'within the grace period' do + it 'redirects to two-factor configuration page' do + expect(current_path).to eq profile_two_factor_auth_path + expect(page).to have_content( + 'The group settings for Group 1 and Group 2 require you to enable ' \ + 'Two-Factor Authentication for your account. You need to do this ' \ + 'before ') + end + + it 'allows skipping two-factor configuration', js: true do + expect(current_path).to eq profile_two_factor_auth_path + + click_link 'Configure it later' + expect(current_path).to eq root_path + end + end + + context 'after the grace period' do + let(:user) { create(:user, otp_grace_period_started_at: 9999.hours.ago) } + + it 'redirects to two-factor configuration page' do + expect(current_path).to eq profile_two_factor_auth_path + expect(page).to have_content( + 'The group settings for Group 1 and Group 2 require you to enable ' \ + 'Two-Factor Authentication for your account.' + ) + end + + it 'disallows skipping two-factor configuration', js: true do + expect(current_path).to eq profile_two_factor_auth_path + expect(page).not_to have_link('Configure it later') + end + end + end + + context 'without grace period defined' do + before(:each) do + stub_application_setting(two_factor_grace_period: 0) + login_with(user) + end + + it 'redirects to two-factor configuration page' do + expect(current_path).to eq profile_two_factor_auth_path + expect(page).to have_content( + 'The group settings for Group 1 and Group 2 require you to enable ' \ + 'Two-Factor Authentication for your account.' + ) + end end end end -- cgit v1.2.1 From 5ea4e34f4755e9a15503a6f16fd1574dc7864b23 Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Thu, 9 Mar 2017 20:27:14 +0100 Subject: add method to get a full routable hierarchy --- app/models/concerns/routable.rb | 68 +++++++++++++++++++++++++ spec/models/concerns/routable_spec.rb | 93 ++++++++++++++++++++++++++++++++++- 2 files changed, 160 insertions(+), 1 deletion(-) diff --git a/app/models/concerns/routable.rb b/app/models/concerns/routable.rb index 529fb5ce988..9b8970a1f38 100644 --- a/app/models/concerns/routable.rb +++ b/app/models/concerns/routable.rb @@ -83,6 +83,74 @@ module Routable AND members.source_type = r2.source_type"). where('members.user_id = ?', user_id) end + + # Builds a relation to find multiple objects that are nested under user + # membership. Includes the parent, as opposed to `#member_descendants` + # which only includes the descendants. + # + # Usage: + # + # Klass.member_self_and_descendants(1) + # + # Returns an ActiveRecord::Relation. + def member_self_and_descendants(user_id) + joins(:route). + joins("INNER JOIN routes r2 ON routes.path LIKE CONCAT(r2.path, '/%') + OR routes.path = r2.path + INNER JOIN members ON members.source_id = r2.source_id + AND members.source_type = r2.source_type"). + where('members.user_id = ?', user_id) + end + + # Returns all objects in a hierarchy, where any node in the hierarchy is + # under the user membership. + # + # Usage: + # + # Klass.member_hierarchy(1) + # + # Examples: + # + # Given the following group tree... + # + # _______group_1_______ + # | | + # | | + # nested_group_1 nested_group_2 + # | | + # | | + # nested_group_1_1 nested_group_2_1 + # + # + # ... the following results are returned: + # + # * the user is a member of group 1 + # => 'group_1', + # 'nested_group_1', nested_group_1_1', + # 'nested_group_2', 'nested_group_2_1' + # + # * the user is a member of nested_group_2 + # => 'group1', + # 'nested_group_2', 'nested_group_2_1' + # + # * the user is a member of nested_group_2_1 + # => 'group1', + # 'nested_group_2', 'nested_group_2_1' + # + # Returns an ActiveRecord::Relation. + def member_hierarchy(user_id) + paths = member_self_and_descendants(user_id).pluck('routes.path') + + return none if paths.empty? + + leaf_paths = paths.group_by(&:length).flat_map(&:last) + + wheres = leaf_paths.map do |leaf_path| + "#{connection.quote(leaf_path)} LIKE CONCAT(routes.path, '%')" + end + + joins(:route).where(wheres.join(' OR ')) + end end def full_name diff --git a/spec/models/concerns/routable_spec.rb b/spec/models/concerns/routable_spec.rb index 677e60e1282..1bd0cb075f7 100644 --- a/spec/models/concerns/routable_spec.rb +++ b/spec/models/concerns/routable_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Group, 'Routable' do - let!(:group) { create(:group) } + let!(:group) { create(:group, name: 'group 1') } describe 'Validations' do it { is_expected.to validate_presence_of(:route) } @@ -81,6 +81,97 @@ describe Group, 'Routable' do it { is_expected.to eq([nested_group]) } end + describe '.member_self_and_descendants' do + let!(:user) { create(:user) } + let!(:nested_group) { create(:group, parent: group) } + + before { group.add_owner(user) } + subject { described_class.member_self_and_descendants(user.id) } + + it { is_expected.to match_array [group, nested_group] } + end + + describe '.member_hierarchy' do + let!(:user) { create(:user) } + + # _______ group _______ + # | | + # | | + # nested_group_1 nested_group_2 + # | | + # | | + # nested_group_1_1 nested_group_2_1 + # + let!(:nested_group_1) { create :group, parent: group, name: 'group 1-1' } + let!(:nested_group_1_1) { create :group, parent: nested_group_1, name: 'group 1-1-1' } + let!(:nested_group_2) { create :group, parent: group, name: 'group 1-2' } + let!(:nested_group_2_1) { create :group, parent: nested_group_2, name: 'group 1-2-1' } + + context 'user is not a member of any group' do + subject { described_class.member_hierarchy(user.id) } + + it 'returns an empty array' do + is_expected.to eq [] + end + end + + context 'user is member of all groups' do + before do + group.add_owner(user) + nested_group_1.add_owner(user) + nested_group_1_1.add_owner(user) + nested_group_2.add_owner(user) + nested_group_2_1.add_owner(user) + end + subject { described_class.member_hierarchy(user.id) } + + it 'returns all groups' do + is_expected.to match_array [ + group, + nested_group_1, nested_group_1_1, + nested_group_2, nested_group_2_1 + ] + end + end + + context 'user is member of the top group' do + before { group.add_owner(user) } + subject { described_class.member_hierarchy(user.id) } + + it 'returns all groups' do + is_expected.to match_array [ + group, + nested_group_1, nested_group_1_1, + nested_group_2, nested_group_2_1 + ] + end + end + + context 'user is member of the first child (internal node)' do + before { nested_group_1.add_owner(user) } + subject { described_class.member_hierarchy(user.id) } + + it 'returns the groups in the hierarchy' do + is_expected.to match_array [ + group, + nested_group_1, nested_group_1_1 + ] + end + end + + context 'user is member of the last child (leaf node)' do + before { nested_group_1_1.add_owner(user) } + subject { described_class.member_hierarchy(user.id) } + + it 'returns the groups in the hierarchy' do + is_expected.to match_array [ + group, + nested_group_1, nested_group_1_1 + ] + end + end + end + describe '#full_path' do let(:group) { create(:group) } let(:nested_group) { create(:group, parent: group) } -- cgit v1.2.1 From 20575859b1bf431421427d52c4ac5a33cf662df6 Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Thu, 9 Mar 2017 20:38:13 +0100 Subject: check all groups for 2fa requirement --- .../concerns/enforces_two_factor_authentication.rb | 2 +- app/models/concerns/routable.rb | 6 ++-- app/models/user.rb | 10 +++++- spec/models/user_spec.rb | 41 ++++++++++++++++++++++ 4 files changed, 53 insertions(+), 6 deletions(-) diff --git a/app/controllers/concerns/enforces_two_factor_authentication.rb b/app/controllers/concerns/enforces_two_factor_authentication.rb index 3e0c62172de..28ffe0ba4b1 100644 --- a/app/controllers/concerns/enforces_two_factor_authentication.rb +++ b/app/controllers/concerns/enforces_two_factor_authentication.rb @@ -29,7 +29,7 @@ module EnforcesTwoFactorAuthentication if current_application_settings.require_two_factor_authentication? global.call else - groups = current_user.groups.where(require_two_factor_authentication: true).reorder(name: :asc) + groups = current_user.expanded_groups_requiring_two_factor_authentication.reorder(name: :asc) group.call(groups) end end diff --git a/app/models/concerns/routable.rb b/app/models/concerns/routable.rb index 9b8970a1f38..b907e421743 100644 --- a/app/models/concerns/routable.rb +++ b/app/models/concerns/routable.rb @@ -143,10 +143,8 @@ module Routable return none if paths.empty? - leaf_paths = paths.group_by(&:length).flat_map(&:last) - - wheres = leaf_paths.map do |leaf_path| - "#{connection.quote(leaf_path)} LIKE CONCAT(routes.path, '%')" + wheres = paths.map do |path| + "#{connection.quote(path)} LIKE CONCAT(routes.path, '%')" end joins(:route).where(wheres.join(' OR ')) diff --git a/app/models/user.rb b/app/models/user.rb index 564e99df77b..fb30cebfda4 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -484,6 +484,14 @@ class User < ActiveRecord::Base Group.member_descendants(id) end + def all_expanded_groups + Group.member_hierarchy(id) + end + + def expanded_groups_requiring_two_factor_authentication + all_expanded_groups.where(require_two_factor_authentication: true) + end + def nested_groups_projects Project.joins(:namespace).where('namespaces.parent_id IS NOT NULL'). member_descendants(id) @@ -964,7 +972,7 @@ class User < ActiveRecord::Base end def update_two_factor_requirement - periods = groups.where(require_two_factor_authentication: true).pluck(:two_factor_grace_period) + periods = expanded_groups_requiring_two_factor_authentication.pluck(:two_factor_grace_period) self.require_two_factor_authentication = periods.any? self.two_factor_grace_period = periods.min || User.column_defaults['two_factor_grace_period'] diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index b2f686a1819..ddd438bfc25 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -1407,6 +1407,17 @@ describe User, models: true do it { expect(user.nested_groups).to eq([nested_group]) } end + describe '#all_expanded_groups' do + let!(:user) { create(:user) } + let!(:group) { create(:group) } + let!(:nested_group_1) { create(:group, parent: group) } + let!(:nested_group_2) { create(:group, parent: group) } + + before { nested_group_1.add_owner(user) } + + it { expect(user.all_expanded_groups).to match_array [group, nested_group_1] } + end + describe '#nested_groups_projects' do let!(:user) { create(:user) } let!(:group) { create(:group) } @@ -1545,6 +1556,36 @@ describe User, models: true do end end + context 'with 2FA requirement on nested parent group' do + let!(:group1) { create :group, require_two_factor_authentication: true } + let!(:group1a) { create :group, require_two_factor_authentication: false, parent: group1 } + + before do + group1a.add_user(user, GroupMember::OWNER) + + user.update_two_factor_requirement + end + + it 'requires 2FA' do + expect(user.require_two_factor_authentication).to be true + end + end + + context 'with 2FA requirement on nested child group' do + let!(:group1) { create :group, require_two_factor_authentication: false } + let!(:group1a) { create :group, require_two_factor_authentication: true, parent: group1 } + + before do + group1.add_user(user, GroupMember::OWNER) + + user.update_two_factor_requirement + end + + it 'requires 2FA' do + expect(user.require_two_factor_authentication).to be true + end + end + context 'without 2FA requirement on groups' do let(:group) { create :group } -- cgit v1.2.1 From 1735ed613910b38c4c069da9c4637bbc4856db36 Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Tue, 14 Mar 2017 14:34:21 +0100 Subject: rename cache db column with `_cached` suffix --- app/controllers/concerns/enforces_two_factor_authentication.rb | 4 ++-- app/models/user.rb | 2 +- db/migrate/20170124193205_add_two_factor_columns_to_users.rb | 4 ++-- db/schema.rb | 2 +- spec/controllers/application_controller_spec.rb | 4 ++-- spec/models/user_spec.rb | 8 ++++---- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/app/controllers/concerns/enforces_two_factor_authentication.rb b/app/controllers/concerns/enforces_two_factor_authentication.rb index 28ffe0ba4b1..688e8bd4a37 100644 --- a/app/controllers/concerns/enforces_two_factor_authentication.rb +++ b/app/controllers/concerns/enforces_two_factor_authentication.rb @@ -21,7 +21,7 @@ module EnforcesTwoFactorAuthentication def two_factor_authentication_required? current_application_settings.require_two_factor_authentication? || - current_user.try(:require_two_factor_authentication?) + current_user.try(:require_two_factor_authentication_from_group?) end def two_factor_authentication_reason(global: -> {}, group: -> {}) @@ -37,7 +37,7 @@ module EnforcesTwoFactorAuthentication def two_factor_grace_period periods = [current_application_settings.two_factor_grace_period] - periods << current_user.two_factor_grace_period if current_user.try(:require_two_factor_authentication?) + periods << current_user.two_factor_grace_period if current_user.try(:require_two_factor_authentication_from_group?) periods.min end diff --git a/app/models/user.rb b/app/models/user.rb index fb30cebfda4..9c9c1f9c5da 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -974,7 +974,7 @@ class User < ActiveRecord::Base def update_two_factor_requirement periods = expanded_groups_requiring_two_factor_authentication.pluck(:two_factor_grace_period) - self.require_two_factor_authentication = periods.any? + self.require_two_factor_authentication_from_group = periods.any? self.two_factor_grace_period = periods.min || User.column_defaults['two_factor_grace_period'] save diff --git a/db/migrate/20170124193205_add_two_factor_columns_to_users.rb b/db/migrate/20170124193205_add_two_factor_columns_to_users.rb index bef1b2062c8..1d1021fcbb3 100644 --- a/db/migrate/20170124193205_add_two_factor_columns_to_users.rb +++ b/db/migrate/20170124193205_add_two_factor_columns_to_users.rb @@ -6,12 +6,12 @@ class AddTwoFactorColumnsToUsers < ActiveRecord::Migration disable_ddl_transaction! def up - add_column_with_default(:users, :require_two_factor_authentication, :boolean, default: false) + add_column_with_default(:users, :require_two_factor_authentication_from_group, :boolean, default: false) add_column_with_default(:users, :two_factor_grace_period, :integer, default: 48) end def down - remove_column(:users, :require_two_factor_authentication) + remove_column(:users, :require_two_factor_authentication_from_group) remove_column(:users, :two_factor_grace_period) end end diff --git a/db/schema.rb b/db/schema.rb index 2d2507828d4..62b9a766ff3 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1249,7 +1249,7 @@ ActiveRecord::Schema.define(version: 20170402231018) do t.boolean "authorized_projects_populated" t.boolean "ghost" t.boolean "notified_of_own_activity" - t.boolean "require_two_factor_authentication", default: false, null: false + t.boolean "require_two_factor_authentication_from_group", default: false, null: false t.integer "two_factor_grace_period", default: 48, null: false end diff --git a/spec/controllers/application_controller_spec.rb b/spec/controllers/application_controller_spec.rb index 7427c93b593..760f33b09c1 100644 --- a/spec/controllers/application_controller_spec.rb +++ b/spec/controllers/application_controller_spec.rb @@ -183,7 +183,7 @@ describe ApplicationController do end it 'returns true if a 2FA requirement is set on the user' do - user.require_two_factor_authentication = true + user.require_two_factor_authentication_from_group = true allow(controller).to receive(:current_user).and_return(user) expect(subject).to be_truthy @@ -201,7 +201,7 @@ describe ApplicationController do end context 'with a 2FA requirement set on the user' do - let(:user) { create :user, require_two_factor_authentication: true, two_factor_grace_period: 23 } + let(:user) { create :user, require_two_factor_authentication_from_group: true, two_factor_grace_period: 23 } it 'returns the user grace period if lower than the application grace period' do stub_application_setting two_factor_grace_period: 24 diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index ddd438bfc25..3977af55176 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -1548,7 +1548,7 @@ describe User, models: true do end it 'requires 2FA' do - expect(user.require_two_factor_authentication).to be true + expect(user.require_two_factor_authentication_from_group).to be true end it 'uses the shortest grace period' do @@ -1567,7 +1567,7 @@ describe User, models: true do end it 'requires 2FA' do - expect(user.require_two_factor_authentication).to be true + expect(user.require_two_factor_authentication_from_group).to be true end end @@ -1582,7 +1582,7 @@ describe User, models: true do end it 'requires 2FA' do - expect(user.require_two_factor_authentication).to be true + expect(user.require_two_factor_authentication_from_group).to be true end end @@ -1596,7 +1596,7 @@ describe User, models: true do end it 'does not require 2FA' do - expect(user.require_two_factor_authentication).to be false + expect(user.require_two_factor_authentication_from_group).to be false end it 'falls back to the default grace period' do -- cgit v1.2.1 From 63e61cfd83bdc03d5c0657b5f93c3236d6a2d987 Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Wed, 15 Mar 2017 13:45:28 +0100 Subject: use more explicit and explanatory sql statement --- app/models/concerns/routable.rb | 4 +++- spec/models/concerns/routable_spec.rb | 30 +++++++++++++++++++++++------- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/app/models/concerns/routable.rb b/app/models/concerns/routable.rb index b907e421743..aca99feee53 100644 --- a/app/models/concerns/routable.rb +++ b/app/models/concerns/routable.rb @@ -144,7 +144,9 @@ module Routable return none if paths.empty? wheres = paths.map do |path| - "#{connection.quote(path)} LIKE CONCAT(routes.path, '%')" + "#{connection.quote(path)} = routes.path + OR + #{connection.quote(path)} LIKE CONCAT(routes.path, '/%')" end joins(:route).where(wheres.join(' OR ')) diff --git a/spec/models/concerns/routable_spec.rb b/spec/models/concerns/routable_spec.rb index 1bd0cb075f7..f191605dbdb 100644 --- a/spec/models/concerns/routable_spec.rb +++ b/spec/models/concerns/routable_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Group, 'Routable' do - let!(:group) { create(:group, name: 'group 1') } + let!(:group) { create(:group, name: 'foo') } describe 'Validations' do it { is_expected.to validate_presence_of(:route) } @@ -92,20 +92,24 @@ describe Group, 'Routable' do end describe '.member_hierarchy' do + # foo/bar would also match foo/barbaz instead of just foo/bar and foo/bar/baz let!(:user) { create(:user) } - # _______ group _______ + # group + # _______ (foo) _______ # | | # | | # nested_group_1 nested_group_2 + # (bar) (barbaz) # | | # | | # nested_group_1_1 nested_group_2_1 + # (baz) (baz) # - let!(:nested_group_1) { create :group, parent: group, name: 'group 1-1' } - let!(:nested_group_1_1) { create :group, parent: nested_group_1, name: 'group 1-1-1' } - let!(:nested_group_2) { create :group, parent: group, name: 'group 1-2' } - let!(:nested_group_2_1) { create :group, parent: nested_group_2, name: 'group 1-2-1' } + let!(:nested_group_1) { create :group, parent: group, name: 'bar' } + let!(:nested_group_1_1) { create :group, parent: nested_group_1, name: 'baz' } + let!(:nested_group_2) { create :group, parent: group, name: 'barbaz' } + let!(:nested_group_2_1) { create :group, parent: nested_group_2, name: 'baz' } context 'user is not a member of any group' do subject { described_class.member_hierarchy(user.id) } @@ -147,7 +151,7 @@ describe Group, 'Routable' do end end - context 'user is member of the first child (internal node)' do + context 'user is member of the first child (internal node), branch 1' do before { nested_group_1.add_owner(user) } subject { described_class.member_hierarchy(user.id) } @@ -159,6 +163,18 @@ describe Group, 'Routable' do end end + context 'user is member of the first child (internal node), branch 2' do + before { nested_group_2.add_owner(user) } + subject { described_class.member_hierarchy(user.id) } + + it 'returns the groups in the hierarchy' do + is_expected.to match_array [ + group, + nested_group_2, nested_group_2_1 + ] + end + end + context 'user is member of the last child (leaf node)' do before { nested_group_1_1.add_owner(user) } subject { described_class.member_hierarchy(user.id) } -- cgit v1.2.1 From 01be21d42705d8d9857a0d4e5f3146a30b40352e Mon Sep 17 00:00:00 2001 From: Alexis Reigel Date: Thu, 6 Apr 2017 08:30:26 +0200 Subject: user#update_two_factor_requirement is public --- app/models/user.rb | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/app/models/user.rb b/app/models/user.rb index 9c9c1f9c5da..c358b1b8d2b 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -963,14 +963,6 @@ class User < ActiveRecord::Base self.admin = (new_level == 'admin') end - protected - - # override, from Devise::Validatable - def password_required? - return false if internal? - super - end - def update_two_factor_requirement periods = expanded_groups_requiring_two_factor_authentication.pluck(:two_factor_grace_period) @@ -980,6 +972,14 @@ class User < ActiveRecord::Base save end + protected + + # override, from Devise::Validatable + def password_required? + return false if internal? + super + end + private def ci_projects_union -- cgit v1.2.1 From 714c408f222cc3bfef577b477f7bab0556f50599 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 6 Apr 2017 10:23:51 +0200 Subject: Add minor improvements to container registry code --- app/controllers/projects/registry/tags_controller.rb | 6 ++---- lib/container_registry/path.rb | 8 ++++++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/app/controllers/projects/registry/tags_controller.rb b/app/controllers/projects/registry/tags_controller.rb index 7a9f290e946..d689cade3ab 100644 --- a/app/controllers/projects/registry/tags_controller.rb +++ b/app/controllers/projects/registry/tags_controller.rb @@ -15,15 +15,13 @@ module Projects private - def repository + def image @image ||= project.container_repositories .find(params[:repository_id]) end def tag - return render_404 unless params[:id].present? - - @tag ||= repository.tag(params[:id]) + @tag ||= image.tag(params[:id]) end end end diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb index 89973b2e7b8..a4b5f2aba6c 100644 --- a/lib/container_registry/path.rb +++ b/lib/container_registry/path.rb @@ -1,6 +1,6 @@ module ContainerRegistry ## - # Class reponsible for extracting project and repository name from + # Class responsible for extracting project and repository name from # image repository path provided by a containers registry API response. # # Example: @@ -12,6 +12,8 @@ module ContainerRegistry class Path InvalidRegistryPathError = Class.new(StandardError) + LEVELS_SUPPORTED = 3 + def initialize(path) @path = path end @@ -50,7 +52,9 @@ module ContainerRegistry end def repository_project - @project ||= Project.where_full_path_in(nodes.first(3)).first + @project ||= Project + .where_full_path_in(nodes.first(LEVELS_SUPPORTED)) + .first end def repository_name -- cgit v1.2.1 From 0ec3a031f4a8f392b9d79c4b638d832b1e077718 Mon Sep 17 00:00:00 2001 From: Toon Claes Date: Fri, 31 Mar 2017 16:30:08 +0200 Subject: Show the test coverage if it is available Do not check if coverage is enabled, just show it when it is available. --- app/controllers/projects/merge_requests_controller.rb | 2 +- app/models/project.rb | 4 ---- app/views/projects/builds/_table.html.haml | 2 +- app/views/projects/ci/builds/_build.html.haml | 3 +-- app/views/projects/commit/_pipeline.html.haml | 3 +-- .../projects/generic_commit_statuses/_generic_commit_status.html.haml | 3 +-- app/views/projects/pipelines/_with_tabs.html.haml | 3 +-- app/views/projects/stage/_stage.html.haml | 4 ++-- changelogs/unreleased/tc-show-pipeline-coverage-if-avail.yml | 4 ++++ 9 files changed, 12 insertions(+), 16 deletions(-) create mode 100644 changelogs/unreleased/tc-show-pipeline-coverage-if-avail.yml diff --git a/app/controllers/projects/merge_requests_controller.rb b/app/controllers/projects/merge_requests_controller.rb index a79d801991a..c337534b297 100755 --- a/app/controllers/projects/merge_requests_controller.rb +++ b/app/controllers/projects/merge_requests_controller.rb @@ -452,7 +452,7 @@ class Projects::MergeRequestsController < Projects::ApplicationController if pipeline status = pipeline.status - coverage = pipeline.try(:coverage) + coverage = pipeline.coverage status = "success_with_warnings" if pipeline.success? && pipeline.has_warnings? diff --git a/app/models/project.rb b/app/models/project.rb index 12fd0668ff8..2695cb1fb93 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1110,10 +1110,6 @@ class Project < ActiveRecord::Base self.runners_token && ActiveSupport::SecurityUtils.variable_size_secure_compare(token, self.runners_token) end - def build_coverage_enabled? - build_coverage_regex.present? - end - def build_timeout_in_minutes build_timeout / 60 end diff --git a/app/views/projects/builds/_table.html.haml b/app/views/projects/builds/_table.html.haml index acfdb250aff..82806f022ee 100644 --- a/app/views/projects/builds/_table.html.haml +++ b/app/views/projects/builds/_table.html.haml @@ -20,6 +20,6 @@ %th Coverage %th - = render partial: "projects/ci/builds/build", collection: builds, as: :build, locals: { commit_sha: true, ref: true, pipeline_link: true, stage: true, allow_retry: true, coverage: admin || project.build_coverage_enabled?, admin: admin } + = render partial: "projects/ci/builds/build", collection: builds, as: :build, locals: { commit_sha: true, ref: true, pipeline_link: true, stage: true, allow_retry: true, admin: admin } = paginate builds, theme: 'gitlab' diff --git a/app/views/projects/ci/builds/_build.html.haml b/app/views/projects/ci/builds/_build.html.haml index aeed293a724..508465cbfb7 100644 --- a/app/views/projects/ci/builds/_build.html.haml +++ b/app/views/projects/ci/builds/_build.html.haml @@ -4,7 +4,6 @@ - retried = local_assigns.fetch(:retried, false) - pipeline_link = local_assigns.fetch(:pipeline_link, false) - stage = local_assigns.fetch(:stage, false) -- coverage = local_assigns.fetch(:coverage, false) - allow_retry = local_assigns.fetch(:allow_retry, false) %tr.build.commit{ class: ('retried' if retried) } @@ -88,7 +87,7 @@ %span= time_ago_with_tooltip(build.finished_at) %td.coverage - - if coverage && build.try(:coverage) + - if build.try(:coverage) #{build.coverage}% %td diff --git a/app/views/projects/commit/_pipeline.html.haml b/app/views/projects/commit/_pipeline.html.haml index c2b32a22170..3ee85723ebe 100644 --- a/app/views/projects/commit/_pipeline.html.haml +++ b/app/views/projects/commit/_pipeline.html.haml @@ -47,7 +47,6 @@ %th Job ID %th Name %th - - if pipeline.project.build_coverage_enabled? - %th Coverage + %th Coverage %th = render partial: "projects/stage/stage", collection: pipeline.stages, as: :stage diff --git a/app/views/projects/generic_commit_statuses/_generic_commit_status.html.haml b/app/views/projects/generic_commit_statuses/_generic_commit_status.html.haml index 07fb80750d6..f458646522c 100644 --- a/app/views/projects/generic_commit_statuses/_generic_commit_status.html.haml +++ b/app/views/projects/generic_commit_statuses/_generic_commit_status.html.haml @@ -4,7 +4,6 @@ - retried = local_assigns.fetch(:retried, false) - pipeline_link = local_assigns.fetch(:pipeline_link, false) - stage = local_assigns.fetch(:stage, false) -- coverage = local_assigns.fetch(:coverage, false) %tr.generic_commit_status{ class: ('retried' if retried) } %td.status @@ -80,7 +79,7 @@ %span= time_ago_with_tooltip(generic_commit_status.finished_at) %td.coverage - - if coverage && generic_commit_status.try(:coverage) + - if generic_commit_status.try(:coverage) #{generic_commit_status.coverage}% %td diff --git a/app/views/projects/pipelines/_with_tabs.html.haml b/app/views/projects/pipelines/_with_tabs.html.haml index 53067cdcba4..d7cefb8613e 100644 --- a/app/views/projects/pipelines/_with_tabs.html.haml +++ b/app/views/projects/pipelines/_with_tabs.html.haml @@ -36,7 +36,6 @@ %th Job ID %th Name %th - - if pipeline.project.build_coverage_enabled? - %th Coverage + %th Coverage %th = render partial: "projects/stage/stage", collection: pipeline.stages, as: :stage diff --git a/app/views/projects/stage/_stage.html.haml b/app/views/projects/stage/_stage.html.haml index 28e1c060875..f93994bebe3 100644 --- a/app/views/projects/stage/_stage.html.haml +++ b/app/views/projects/stage/_stage.html.haml @@ -6,8 +6,8 @@ = ci_icon_for_status(stage.status)   = stage.name.titleize -= render stage.statuses.latest_ordered, coverage: @project.build_coverage_enabled?, stage: false, ref: false, pipeline_link: false, allow_retry: true -= render stage.statuses.retried_ordered, coverage: @project.build_coverage_enabled?, stage: false, ref: false, pipeline_link: false, retried: true += render stage.statuses.latest_ordered, stage: false, ref: false, pipeline_link: false, allow_retry: true += render stage.statuses.retried_ordered, stage: false, ref: false, pipeline_link: false, retried: true %tr %td{ colspan: 10 }   diff --git a/changelogs/unreleased/tc-show-pipeline-coverage-if-avail.yml b/changelogs/unreleased/tc-show-pipeline-coverage-if-avail.yml new file mode 100644 index 00000000000..c0cc4fb18c8 --- /dev/null +++ b/changelogs/unreleased/tc-show-pipeline-coverage-if-avail.yml @@ -0,0 +1,4 @@ +--- +title: Show the build/pipeline coverage if it is available +merge_request: +author: -- cgit v1.2.1 From 0fc361c4dcf9b714fef8dc8a59e6856fd58f2425 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kim=20=22BKC=22=20Carlb=C3=A4cker?= Date: Tue, 28 Mar 2017 19:14:48 +0200 Subject: Use Gitaly for Environment#first_deployment_for --- app/models/repository.rb | 10 ++-------- lib/gitlab/git/repository.rb | 15 +++++++++++++++ lib/gitlab/gitaly_client/ref.rb | 10 ++++++++++ spec/models/environment_spec.rb | 23 +++++++++++++++++++---- 4 files changed, 46 insertions(+), 12 deletions(-) diff --git a/app/models/repository.rb b/app/models/repository.rb index dc1c1fab880..1293cb1d486 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -6,6 +6,8 @@ class Repository attr_accessor :path_with_namespace, :project + delegate :ref_name_for_sha, to: :raw_repository + CommitError = Class.new(StandardError) CreateTreeError = Class.new(StandardError) @@ -700,14 +702,6 @@ class Repository end end - def ref_name_for_sha(ref_path, sha) - args = %W(#{Gitlab.config.git.bin_path} for-each-ref --count=1 #{ref_path} --contains #{sha}) - - # Not found -> ["", 0] - # Found -> ["b8d95eb4969eefacb0a58f6a28f6803f8070e7b9 commit\trefs/environments/production/77\n", 0] - Gitlab::Popen.popen(args, path_to_repo).first.split.last - end - def refs_contains_sha(ref_type, sha) args = %W(#{Gitlab.config.git.bin_path} #{ref_type} --contains #{sha}) names = Gitlab::Popen.popen(args, path_to_repo).first diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 2e4314932c8..4fe6f340ee9 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -452,6 +452,21 @@ module Gitlab Gitlab::Git::DiffCollection.new(diff_patches(from, to, options, *paths), options) end + # Returns a RefName for a given SHA + def ref_name_for_sha(ref_path, sha) + Gitlab::GitalyClient.migrate(:find_ref_name) do |is_enabled| + if is_enabled + Gitlab::GitalyClient::Ref.find_ref_name(self, sha, ref_path) + else + args = %W(#{Gitlab.config.git.bin_path} for-each-ref --count=1 #{ref_path} --contains #{sha}) + + # Not found -> ["", 0] + # Found -> ["b8d95eb4969eefacb0a58f6a28f6803f8070e7b9 commit\trefs/environments/production/77\n", 0] + Gitlab::Popen.popen(args, @path).first.split.last + end + end + end + # Returns commits collection # # Ex. diff --git a/lib/gitlab/gitaly_client/ref.rb b/lib/gitlab/gitaly_client/ref.rb index bfc5fa573c7..4958d00c542 100644 --- a/lib/gitlab/gitaly_client/ref.rb +++ b/lib/gitlab/gitaly_client/ref.rb @@ -23,6 +23,16 @@ module Gitlab consume_refs_response(stub.find_all_tag_names(request), prefix: 'refs/tags/') end + def find_ref_name(commit_id, ref_prefix) + request = Gitaly::FindRefNameRequest.new( + repository: @repository, + commit_id: commit_id, + prefix: ref_prefix + ) + + stub.find_ref_name(request).name + end + private def consume_refs_response(response, prefix:) diff --git a/spec/models/environment_spec.rb b/spec/models/environment_spec.rb index 9f0e7fbbe26..3ad3a25d873 100644 --- a/spec/models/environment_spec.rb +++ b/spec/models/environment_spec.rb @@ -100,12 +100,27 @@ describe Environment, models: true do let(:head_commit) { project.commit } let(:commit) { project.commit.parent } - it 'returns deployment id for the environment' do - expect(environment.first_deployment_for(commit)).to eq deployment1 + context 'Gitaly find_ref_name feature disables' do + it 'returns deployment id for the environment' do + expect(environment.first_deployment_for(commit)).to eq deployment1 + end + + it 'return nil when no deployment is found' do + expect(environment.first_deployment_for(head_commit)).to eq nil + end end - it 'return nil when no deployment is found' do - expect(environment.first_deployment_for(head_commit)).to eq nil + context 'Gitaly find_ref_name feature enabled' do + before do + allow(Gitlab::GitalyClient).to receive(:feature_enabled?).with(:find_ref_name).and_return(true) + end + + it 'checks that GitalyClient is called' do + expect(Gitlab::GitalyClient::Ref).to receive(:find_ref_name).with(project.repository.raw_repository, commit.id, environment.ref_path) + + environment.first_deployment_for(commit) + end + end end -- cgit v1.2.1 From ae7ec00492775c75b56725169bcc754c015b51b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kim=20=22BKC=22=20Carlb=C3=A4cker?= Date: Tue, 4 Apr 2017 21:43:55 +0200 Subject: rubocop --- spec/models/environment_spec.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/spec/models/environment_spec.rb b/spec/models/environment_spec.rb index 3ad3a25d873..6d99ebcd68f 100644 --- a/spec/models/environment_spec.rb +++ b/spec/models/environment_spec.rb @@ -120,7 +120,6 @@ describe Environment, models: true do environment.first_deployment_for(commit) end - end end -- cgit v1.2.1 From 47907a417214e3d5c1ce350307795241f0cada38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kim=20=22BKC=22=20Carlb=C3=A4cker?= Date: Wed, 5 Apr 2017 22:56:40 +0200 Subject: Cleanup --- spec/models/environment_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/models/environment_spec.rb b/spec/models/environment_spec.rb index 6d99ebcd68f..c8f4e9ff50e 100644 --- a/spec/models/environment_spec.rb +++ b/spec/models/environment_spec.rb @@ -100,7 +100,7 @@ describe Environment, models: true do let(:head_commit) { project.commit } let(:commit) { project.commit.parent } - context 'Gitaly find_ref_name feature disables' do + context 'Gitaly find_ref_name feature disabled' do it 'returns deployment id for the environment' do expect(environment.first_deployment_for(commit)).to eq deployment1 end @@ -115,7 +115,7 @@ describe Environment, models: true do allow(Gitlab::GitalyClient).to receive(:feature_enabled?).with(:find_ref_name).and_return(true) end - it 'checks that GitalyClient is called' do + it 'calls GitalyClient' do expect(Gitlab::GitalyClient::Ref).to receive(:find_ref_name).with(project.repository.raw_repository, commit.id, environment.ref_path) environment.first_deployment_for(commit) -- cgit v1.2.1 From 70982bb420ad69e354db9b6999feed4e4fab9dc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kim=20=22BKC=22=20Carlb=C3=A4cker?= Date: Thu, 6 Apr 2017 12:10:03 +0200 Subject: Hopefully this works --- lib/gitlab/git/repository.rb | 2 +- spec/models/environment_spec.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 4fe6f340ee9..9e338282e96 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -456,7 +456,7 @@ module Gitlab def ref_name_for_sha(ref_path, sha) Gitlab::GitalyClient.migrate(:find_ref_name) do |is_enabled| if is_enabled - Gitlab::GitalyClient::Ref.find_ref_name(self, sha, ref_path) + gitaly_ref_client.find_ref_name(sha, ref_path) else args = %W(#{Gitlab.config.git.bin_path} for-each-ref --count=1 #{ref_path} --contains #{sha}) diff --git a/spec/models/environment_spec.rb b/spec/models/environment_spec.rb index c8f4e9ff50e..af7753caba6 100644 --- a/spec/models/environment_spec.rb +++ b/spec/models/environment_spec.rb @@ -116,7 +116,7 @@ describe Environment, models: true do end it 'calls GitalyClient' do - expect(Gitlab::GitalyClient::Ref).to receive(:find_ref_name).with(project.repository.raw_repository, commit.id, environment.ref_path) + expect_any_instance_of(Gitlab::GitalyClient::Ref).to receive(:find_ref_name) environment.first_deployment_for(commit) end -- cgit v1.2.1 From 8af788e08587905bfa3455a06832ade4b3ef15c1 Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Thu, 30 Mar 2017 15:38:05 +0100 Subject: Disable invalid service templates (again) --- ...-t-initialize-git-repo-for-new-project-in-group.yml | 4 ++++ ...0170330141723_disable_invalid_service_templates2.rb | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 changelogs/unreleased/30024-owner-can-t-initialize-git-repo-for-new-project-in-group.yml create mode 100644 db/migrate/20170330141723_disable_invalid_service_templates2.rb diff --git a/changelogs/unreleased/30024-owner-can-t-initialize-git-repo-for-new-project-in-group.yml b/changelogs/unreleased/30024-owner-can-t-initialize-git-repo-for-new-project-in-group.yml new file mode 100644 index 00000000000..c43d2732b9a --- /dev/null +++ b/changelogs/unreleased/30024-owner-can-t-initialize-git-repo-for-new-project-in-group.yml @@ -0,0 +1,4 @@ +--- +title: Disable invalid service templates +merge_request: +author: diff --git a/db/migrate/20170330141723_disable_invalid_service_templates2.rb b/db/migrate/20170330141723_disable_invalid_service_templates2.rb new file mode 100644 index 00000000000..8424e56d8a1 --- /dev/null +++ b/db/migrate/20170330141723_disable_invalid_service_templates2.rb @@ -0,0 +1,18 @@ +# This is the same as DisableInvalidServiceTemplates. Later migrations may have +# inadventently enabled some invalid templates again. +# +class DisableInvalidServiceTemplates2 < ActiveRecord::Migration + DOWNTIME = false + + unless defined?(Service) + class Service < ActiveRecord::Base + self.inheritance_column = nil + end + end + + def up + Service.where(template: true, active: true).each do |template| + template.update(active: false) unless template.valid? + end + end +end -- cgit v1.2.1 From 12566f3fd4c9e9072b07df7afa5eea4c2f60a123 Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Thu, 6 Apr 2017 12:25:24 +0100 Subject: Link to docs site for file in doc/ --- PROCESS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PROCESS.md b/PROCESS.md index 2a2aafbd9ff..2f331ee9169 100644 --- a/PROCESS.md +++ b/PROCESS.md @@ -202,4 +202,4 @@ still an issue I encourage you to open it on the [GitLab.com issue tracker](http ["Implement design & UI elements" guidelines]: https://gitlab.com/gitlab-org/gitlab-ce/blob/master/CONTRIBUTING.md#implement-design-ui-elements [Thoughtbot code review guide]: https://github.com/thoughtbot/guides/tree/master/code-review [done]: https://gitlab.com/gitlab-org/gitlab-ce/blob/master/CONTRIBUTING.md#definition-of-done -[limit_ee_conflicts]: https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/development/limit_ee_conflicts.md +[limit_ee_conflicts]: https://docs.gitlab.com/ce/development/limit_ee_conflicts.html -- cgit v1.2.1 From 7f2a2008389e09a0abf163e0ff67d672ef74df27 Mon Sep 17 00:00:00 2001 From: Sean McGivern Date: Thu, 6 Apr 2017 12:47:40 +0100 Subject: Fix RuboCop for removing index --- db/migrate/20170402231018_remove_index_for_users_current_sign_in_at.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/db/migrate/20170402231018_remove_index_for_users_current_sign_in_at.rb b/db/migrate/20170402231018_remove_index_for_users_current_sign_in_at.rb index 8316ee9eb9f..0237c3189a5 100644 --- a/db/migrate/20170402231018_remove_index_for_users_current_sign_in_at.rb +++ b/db/migrate/20170402231018_remove_index_for_users_current_sign_in_at.rb @@ -1,6 +1,7 @@ # See http://doc.gitlab.com/ce/development/migration_style_guide.html # for more information on how to write migrations for GitLab. +# rubocop:disable RemoveIndex class RemoveIndexForUsersCurrentSignInAt < ActiveRecord::Migration include Gitlab::Database::MigrationHelpers -- cgit v1.2.1 From 24507da42df4d48539570617c99825914f0a34dc Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 22 Mar 2017 14:22:05 -0400 Subject: Remove individual modal width styles --- app/assets/stylesheets/framework/modal.scss | 6 ++++-- app/assets/stylesheets/pages/merge_requests.scss | 2 -- app/assets/stylesheets/pages/tree.scss | 2 -- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/app/assets/stylesheets/framework/modal.scss b/app/assets/stylesheets/framework/modal.scss index 8cd49280e1c..7098203321d 100644 --- a/app/assets/stylesheets/framework/modal.scss +++ b/app/assets/stylesheets/framework/modal.scss @@ -16,6 +16,8 @@ body.modal-open { overflow: hidden; } -.modal .modal-dialog { - width: 860px; +@media (min-width: $screen-md-min) { + .modal-dialog { + width: 860px; + } } diff --git a/app/assets/stylesheets/pages/merge_requests.scss b/app/assets/stylesheets/pages/merge_requests.scss index 7c3172421c1..b54d2a5dabb 100644 --- a/app/assets/stylesheets/pages/merge_requests.scss +++ b/app/assets/stylesheets/pages/merge_requests.scss @@ -319,8 +319,6 @@ } #modal_merge_info .modal-dialog { - width: 600px; - .dark { margin-right: 40px; } diff --git a/app/assets/stylesheets/pages/tree.scss b/app/assets/stylesheets/pages/tree.scss index fc4da4c495f..f3916622b6f 100644 --- a/app/assets/stylesheets/pages/tree.scss +++ b/app/assets/stylesheets/pages/tree.scss @@ -145,8 +145,6 @@ margin: 0; } -#modal-remove-blob > .modal-dialog { width: 850px; } - .blob-upload-dropzone-previews { text-align: center; border: 2px; -- cgit v1.2.1 From 894f01cd05f523dc48956d945d011df10a729a65 Mon Sep 17 00:00:00 2001 From: Adam Niedzielski Date: Thu, 6 Apr 2017 14:46:55 +0200 Subject: Include endpoint in metrics for ETag caching middleware --- .../add-dimension-etag-caching-metrics.yml | 4 ++ lib/gitlab/etag_caching/middleware.rb | 44 ++++++++++++++-------- spec/lib/gitlab/etag_caching/middleware_spec.rb | 16 ++++---- 3 files changed, 40 insertions(+), 24 deletions(-) create mode 100644 changelogs/unreleased/add-dimension-etag-caching-metrics.yml diff --git a/changelogs/unreleased/add-dimension-etag-caching-metrics.yml b/changelogs/unreleased/add-dimension-etag-caching-metrics.yml new file mode 100644 index 00000000000..f2a13eb7c61 --- /dev/null +++ b/changelogs/unreleased/add-dimension-etag-caching-metrics.yml @@ -0,0 +1,4 @@ +--- +title: Include endpoint in metrics for ETag caching middleware +merge_request: 10495 +author: diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb index cd4e318033d..630fe4fa849 100644 --- a/lib/gitlab/etag_caching/middleware.rb +++ b/lib/gitlab/etag_caching/middleware.rb @@ -2,26 +2,34 @@ module Gitlab module EtagCaching class Middleware RESERVED_WORDS = NamespaceValidator::WILDCARD_ROUTES.map { |word| "/#{word}/" }.join('|') - ROUTE_REGEXP = Regexp.union( - %r(^(?!.*(#{RESERVED_WORDS})).*/noteable/issue/\d+/notes\z), - %r(^(?!.*(#{RESERVED_WORDS})).*/issues/\d+/rendered_title\z) - ) + ROUTES = [ + { + regexp: %r(^(?!.*(#{RESERVED_WORDS})).*/noteable/issue/\d+/notes\z), + name: 'issue_notes' + }, + { + regexp: %r(^(?!.*(#{RESERVED_WORDS})).*/issues/\d+/rendered_title\z), + name: 'issue_title' + } + ].freeze def initialize(app) @app = app end def call(env) - return @app.call(env) unless enabled_for_current_route?(env) - Gitlab::Metrics.add_event(:etag_caching_middleware_used) + route = match_current_route(env) + return @app.call(env) unless route + + track_event(:etag_caching_middleware_used, route) etag, cached_value_present = get_etag(env) if_none_match = env['HTTP_IF_NONE_MATCH'] if if_none_match == etag - handle_cache_hit(etag) + handle_cache_hit(etag, route) else - track_cache_miss(if_none_match, cached_value_present) + track_cache_miss(if_none_match, cached_value_present, route) status, headers, body = @app.call(env) headers['ETag'] = etag @@ -31,8 +39,8 @@ module Gitlab private - def enabled_for_current_route?(env) - ROUTE_REGEXP.match(env['PATH_INFO']) + def match_current_route(env) + ROUTES.find { |route| route[:regexp].match(env['PATH_INFO']) } end def get_etag(env) @@ -52,23 +60,27 @@ module Gitlab %Q{W/"#{value}"} end - def handle_cache_hit(etag) - Gitlab::Metrics.add_event(:etag_caching_cache_hit) + def handle_cache_hit(etag, route) + track_event(:etag_caching_cache_hit, route) status_code = Gitlab::PollingInterval.polling_enabled? ? 304 : 429 [status_code, { 'ETag' => etag }, ['']] end - def track_cache_miss(if_none_match, cached_value_present) + def track_cache_miss(if_none_match, cached_value_present, route) if if_none_match.blank? - Gitlab::Metrics.add_event(:etag_caching_header_missing) + track_event(:etag_caching_header_missing, route) elsif !cached_value_present - Gitlab::Metrics.add_event(:etag_caching_key_not_found) + track_event(:etag_caching_key_not_found, route) else - Gitlab::Metrics.add_event(:etag_caching_resource_changed) + track_event(:etag_caching_resource_changed, route) end end + + def track_event(name, route) + Gitlab::Metrics.add_event(name, endpoint: route[:name]) + end end end end diff --git a/spec/lib/gitlab/etag_caching/middleware_spec.rb b/spec/lib/gitlab/etag_caching/middleware_spec.rb index 6ec4360adc2..c872d8232b0 100644 --- a/spec/lib/gitlab/etag_caching/middleware_spec.rb +++ b/spec/lib/gitlab/etag_caching/middleware_spec.rb @@ -47,9 +47,9 @@ describe Gitlab::EtagCaching::Middleware do it 'tracks "etag_caching_key_not_found" event' do expect(Gitlab::Metrics).to receive(:add_event) - .with(:etag_caching_middleware_used) + .with(:etag_caching_middleware_used, endpoint: 'issue_notes') expect(Gitlab::Metrics).to receive(:add_event) - .with(:etag_caching_key_not_found) + .with(:etag_caching_key_not_found, endpoint: 'issue_notes') middleware.call(build_env(path, if_none_match)) end @@ -93,9 +93,9 @@ describe Gitlab::EtagCaching::Middleware do it 'tracks "etag_caching_cache_hit" event' do expect(Gitlab::Metrics).to receive(:add_event) - .with(:etag_caching_middleware_used) + .with(:etag_caching_middleware_used, endpoint: 'issue_notes') expect(Gitlab::Metrics).to receive(:add_event) - .with(:etag_caching_cache_hit) + .with(:etag_caching_cache_hit, endpoint: 'issue_notes') middleware.call(build_env(path, if_none_match)) end @@ -132,9 +132,9 @@ describe Gitlab::EtagCaching::Middleware do mock_app_response expect(Gitlab::Metrics).to receive(:add_event) - .with(:etag_caching_middleware_used) + .with(:etag_caching_middleware_used, endpoint: 'issue_notes') expect(Gitlab::Metrics).to receive(:add_event) - .with(:etag_caching_resource_changed) + .with(:etag_caching_resource_changed, endpoint: 'issue_notes') middleware.call(build_env(path, if_none_match)) end @@ -150,9 +150,9 @@ describe Gitlab::EtagCaching::Middleware do it 'tracks "etag_caching_header_missing" event' do expect(Gitlab::Metrics).to receive(:add_event) - .with(:etag_caching_middleware_used) + .with(:etag_caching_middleware_used, endpoint: 'issue_notes') expect(Gitlab::Metrics).to receive(:add_event) - .with(:etag_caching_header_missing) + .with(:etag_caching_header_missing, endpoint: 'issue_notes') middleware.call(build_env(path, if_none_match)) end -- cgit v1.2.1 From fca0097c0f457c6d4d8e326e679ff7848264f021 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Thu, 6 Apr 2017 12:47:40 +0000 Subject: Github import rake task --- changelogs/unreleased/feature-gh-rake-task.yml | 4 + doc/administration/raketasks/github_import.md | 36 +++++ lib/tasks/import.rake | 204 +++++++++++++++++++++++++ 3 files changed, 244 insertions(+) create mode 100644 changelogs/unreleased/feature-gh-rake-task.yml create mode 100644 doc/administration/raketasks/github_import.md create mode 100644 lib/tasks/import.rake diff --git a/changelogs/unreleased/feature-gh-rake-task.yml b/changelogs/unreleased/feature-gh-rake-task.yml new file mode 100644 index 00000000000..5b1d380690c --- /dev/null +++ b/changelogs/unreleased/feature-gh-rake-task.yml @@ -0,0 +1,4 @@ +--- +title: Add rake task to import GitHub projects from the command line +merge_request: +author: diff --git a/doc/administration/raketasks/github_import.md b/doc/administration/raketasks/github_import.md new file mode 100644 index 00000000000..affb4d17861 --- /dev/null +++ b/doc/administration/raketasks/github_import.md @@ -0,0 +1,36 @@ +# GitHub import + +>**Note:** +> +> - [Introduced][ce-10308] in GitLab 9.1. +> - You need a personal access token in order to retrieve and import GitHub +> projects. You can get it from: https://github.com/settings/tokens +> - You also need to pass an username as the second argument to the rake task +> which will become the owner of the project. + +To import a project from the list of your GitHub projects available: + +```bash +# Omnibus installations +sudo gitlab-rake import:github[access_token,root,foo/bar] + +# Installations from source +bundle exec rake import:github[access_token,root,foo/bar] RAILS_ENV=production +``` + +In this case, `access_token` is your GitHub personal access token, `root` +is your GitLab username, and `foo/bar` is the new GitLab namespace/project that +will get created from your GitHub project. Subgroups are also possible: `foo/foo/bar`. + + +To import a specific GitHub project (named `foo/github_repo` here): + +```bash +# Omnibus installations +sudo gitlab-rake import:github[access_token,root,foo/bar,foo/github_repo] + +# Installations from source +bundle exec rake import:github[access_token,root,foo/bar,foo/github_repo] RAILS_ENV=production +``` + +[ce-10308]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/10308 diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake new file mode 100644 index 00000000000..350afeb5c0b --- /dev/null +++ b/lib/tasks/import.rake @@ -0,0 +1,204 @@ +require 'benchmark' +require 'rainbow/ext/string' +require_relative '../gitlab/shell_adapter' +require_relative '../gitlab/github_import/importer' + +class NewImporter < ::Gitlab::GithubImport::Importer + def execute + # Same as ::Gitlab::GithubImport::Importer#execute, but showing some progress. + puts 'Importing repository...'.color(:aqua) + import_repository unless project.repository_exists? + + puts 'Importing labels...'.color(:aqua) + import_labels + + puts 'Importing milestones...'.color(:aqua) + import_milestones + + puts 'Importing pull requests...'.color(:aqua) + import_pull_requests + + puts 'Importing issues...'.color(:aqua) + import_issues + + puts 'Importing issue comments...'.color(:aqua) + import_comments(:issues) + + puts 'Importing pull request comments...'.color(:aqua) + import_comments(:pull_requests) + + puts 'Importing wiki...'.color(:aqua) + import_wiki + + # Gitea doesn't have a Release API yet + # See https://github.com/go-gitea/gitea/issues/330 + unless project.gitea_import? + import_releases + end + + handle_errors + + project.repository.after_import + project.import_finish + + true + end + + def import_repository + begin + raise 'Blocked import URL.' if Gitlab::UrlBlocker.blocked_url?(project.import_url) + + gitlab_shell.import_repository(project.repository_storage_path, project.path_with_namespace, project.import_url) + rescue => e + project.repository.before_import if project.repository_exists? + + raise "Error importing repository #{project.import_url} into #{project.path_with_namespace} - #{e.message}" + end + end +end + +class GithubImport + def self.run!(*args) + new(*args).run! + end + + def initialize(token, gitlab_username, project_path, extras) + @token = token + @project_path = project_path + @current_user = User.find_by_username(gitlab_username) + @github_repo = extras.empty? ? nil : extras.first + end + + def run! + @repo = GithubRepos.new(@token, @current_user, @github_repo).choose_one! + + raise 'No repo found!' unless @repo + + show_warning! + + @project = Project.find_by_full_path(@project_path) || new_project + + import! + end + + private + + def show_warning! + puts "This will import GH #{@repo.full_name.bright} into GL #{@project_path.bright} as #{@current_user.name}" + puts "Permission checks are ignored. Press any key to continue.".color(:red) + + STDIN.getch + + puts 'Starting the import...'.color(:green) + end + + def import! + import_url = @project.import_url.gsub(/\:\/\/(.*@)?/, "://#{@token}@") + @project.update(import_url: import_url) + + @project.import_start + + timings = Benchmark.measure do + NewImporter.new(@project).execute + end + + puts "Import finished. Timings: #{timings}".color(:green) + end + + def new_project + Project.transaction do + namespace_path, _sep, name = @project_path.rpartition('/') + namespace = find_or_create_namespace(namespace_path) + + Project.create!( + import_url: "https://#{@token}@github.com/#{@repo.full_name}.git", + name: name, + path: name, + description: @repo.description, + namespace: namespace, + visibility_level: visibility_level, + import_type: 'github', + import_source: @repo.full_name, + creator: @current_user + ) + end + end + + def find_or_create_namespace(names) + return @current_user.namespace if names == @current_user.namespace_path + return @current_user.namespace unless @current_user.can_create_group? + + names = params[:target_namespace].presence || names + full_path_namespace = Namespace.find_by_full_path(names) + + return full_path_namespace if full_path_namespace + + names.split('/').inject(nil) do |parent, name| + begin + namespace = Group.create!(name: name, + path: name, + owner: @current_user, + parent: parent) + namespace.add_owner(@current_user) + + namespace + rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid + Namespace.where(parent: parent).find_by_path_or_name(name) + end + end + end + + def full_path_namespace(names) + @full_path_namespace ||= Namespace.find_by_full_path(names) + end + + def visibility_level + @repo.private ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility + end +end + +class GithubRepos + def initialize(token, current_user, github_repo) + @token = token + @current_user = current_user + @github_repo = github_repo + end + + def choose_one! + return found_github_repo if @github_repo + + repos.each do |repo| + print "ID: #{repo[:id].to_s.bright} ".color(:green) + puts "- Name: #{repo[:full_name]}".color(:green) + end + + print 'ID? '.bright + + repos.find { |repo| repo[:id] == repo_id } + end + + def found_github_repo + repos.find { |repo| repo[:full_name] == @github_repo } + end + + def repo_id + @repo_id ||= STDIN.gets.chomp.to_i + end + + def repos + @repos ||= client.repos + end + + def client + @client ||= Gitlab::GithubImport::Client.new(@token, {}) + end +end + +namespace :import do + desc 'Import a GitHub project - Example: import:github[ToKeN,root,root/blah,my/github_repo] (optional my/github_repo)' + task :github, [:token, :gitlab_username, :project_path] => :environment do |_t, args| + abort 'Project path must be: namespace(s)/project_name'.color(:red) unless args.project_path.include?('/') + + GithubImport.run!(args.token, args.gitlab_username, args.project_path, args.extras) + end +end -- cgit v1.2.1 From 79d299f9af445bf47f068c1ebf4b8678d5cec95e Mon Sep 17 00:00:00 2001 From: Jose Ivan Vargas Lopez Date: Thu, 6 Apr 2017 13:24:33 +0000 Subject: Introduced empty/error UX states to environments monitoring. --- .../javascripts/monitoring/prometheus_graph.js | 67 ++++++++++++++----- app/assets/stylesheets/pages/environments.scss | 9 +++ .../environments/_metrics_button.html.haml | 2 +- app/views/projects/environments/metrics.html.haml | 77 ++++++++++++++++++---- .../empty_states/monitoring/_getting_started.svg | 1 + .../shared/empty_states/monitoring/_loading.svg | 1 + .../empty_states/monitoring/_unable_to_connect.svg | 1 + changelogs/unreleased/add-error-empty-states.yml | 4 ++ .../fixtures/environments/metrics.html.haml | 64 ++++++++++++++++-- .../monitoring/prometheus_graph_spec.js | 23 ++++++- 10 files changed, 213 insertions(+), 36 deletions(-) create mode 100644 app/views/shared/empty_states/monitoring/_getting_started.svg create mode 100644 app/views/shared/empty_states/monitoring/_loading.svg create mode 100644 app/views/shared/empty_states/monitoring/_unable_to_connect.svg create mode 100644 changelogs/unreleased/add-error-empty-states.yml diff --git a/app/assets/javascripts/monitoring/prometheus_graph.js b/app/assets/javascripts/monitoring/prometheus_graph.js index a6ffa0f59de..d82a4eb9642 100644 --- a/app/assets/javascripts/monitoring/prometheus_graph.js +++ b/app/assets/javascripts/monitoring/prometheus_graph.js @@ -6,7 +6,10 @@ import statusCodes from '~/lib/utils/http_status'; import { formatRelevantDigits } from '~/lib/utils/number_utils'; import '../flash'; +const prometheusContainer = '.prometheus-container'; +const prometheusParentGraphContainer = '.prometheus-graphs'; const prometheusGraphsContainer = '.prometheus-graph'; +const prometheusStatesContainer = '.prometheus-state'; const metricsEndpoint = 'metrics.json'; const timeFormat = d3.time.format('%H:%M'); const dayFormat = d3.time.format('%b %e, %a'); @@ -14,19 +17,30 @@ const bisectDate = d3.bisector(d => d.time).left; const extraAddedWidthParent = 100; class PrometheusGraph { - constructor() { - this.margin = { top: 80, right: 180, bottom: 80, left: 100 }; - this.marginLabelContainer = { top: 40, right: 0, bottom: 40, left: 0 }; - const parentContainerWidth = $(prometheusGraphsContainer).parent().width() + - extraAddedWidthParent; - this.originalWidth = parentContainerWidth; - this.originalHeight = 330; - this.width = parentContainerWidth - this.margin.left - this.margin.right; - this.height = this.originalHeight - this.margin.top - this.margin.bottom; - this.backOffRequestCounter = 0; - this.configureGraph(); - this.init(); + const $prometheusContainer = $(prometheusContainer); + const hasMetrics = $prometheusContainer.data('has-metrics'); + this.docLink = $prometheusContainer.data('doc-link'); + this.integrationLink = $prometheusContainer.data('prometheus-integration'); + + $(document).ajaxError(() => {}); + + if (hasMetrics) { + this.margin = { top: 80, right: 180, bottom: 80, left: 100 }; + this.marginLabelContainer = { top: 40, right: 0, bottom: 40, left: 0 }; + const parentContainerWidth = $(prometheusGraphsContainer).parent().width() + + extraAddedWidthParent; + this.originalWidth = parentContainerWidth; + this.originalHeight = 330; + this.width = parentContainerWidth - this.margin.left - this.margin.right; + this.height = this.originalHeight - this.margin.top - this.margin.bottom; + this.backOffRequestCounter = 0; + this.configureGraph(); + this.init(); + } else { + this.state = '.js-getting-started'; + this.updateState(); + } } createGraph() { @@ -40,8 +54,19 @@ class PrometheusGraph { init() { this.getData().then((metricsResponse) => { - if (Object.keys(metricsResponse).length === 0) { - new Flash('Empty metrics', 'alert'); + let enoughData = true; + Object.keys(metricsResponse.metrics).forEach((key) => { + let currentKey; + if (key === 'cpu_values' || key === 'memory_values') { + currentKey = metricsResponse.metrics[key]; + if (Object.keys(currentKey).length === 0) { + enoughData = false; + } + } + }); + if (!enoughData) { + this.state = '.js-loading'; + this.updateState(); } else { this.transformData(metricsResponse); this.createGraph(); @@ -345,14 +370,17 @@ class PrometheusGraph { } return resp.metrics; }) - .catch(() => new Flash('An error occurred while fetching metrics.', 'alert')); + .catch(() => { + this.state = '.js-unable-to-connect'; + this.updateState(); + }); } transformData(metricsResponse) { Object.keys(metricsResponse.metrics).forEach((key) => { if (key === 'cpu_values' || key === 'memory_values') { const metricValues = (metricsResponse.metrics[key])[0]; - if (typeof metricValues !== 'undefined') { + if (metricValues !== undefined) { this.graphSpecificProperties[key].data = metricValues.values.map(metric => ({ time: new Date(metric[0] * 1000), value: metric[1], @@ -361,6 +389,13 @@ class PrometheusGraph { } }); } + + updateState() { + const $statesContainer = $(prometheusStatesContainer); + $(prometheusParentGraphContainer).hide(); + $(`${this.state}`, $statesContainer).removeClass('hidden'); + $(prometheusStatesContainer).show(); + } } export default PrometheusGraph; diff --git a/app/assets/stylesheets/pages/environments.scss b/app/assets/stylesheets/pages/environments.scss index 6faa3794c83..72e7d42858d 100644 --- a/app/assets/stylesheets/pages/environments.scss +++ b/app/assets/stylesheets/pages/environments.scss @@ -233,6 +233,15 @@ stroke-width: 1; } +.prometheus-state { + margin-top: 10px; + display: none; + + .state-button-section { + margin-top: 10px; + } +} + .environments-actions { .external-url, .monitoring-url, diff --git a/app/views/projects/environments/_metrics_button.html.haml b/app/views/projects/environments/_metrics_button.html.haml index e27281d6917..b4102fcf103 100644 --- a/app/views/projects/environments/_metrics_button.html.haml +++ b/app/views/projects/environments/_metrics_button.html.haml @@ -1,6 +1,6 @@ - environment = local_assigns.fetch(:environment) -- return unless environment.has_metrics? && can?(current_user, :read_environment, environment) +- return unless can?(current_user, :read_environment, environment) = link_to environment_metrics_path(environment), title: 'See metrics', class: 'btn metrics-button' do = icon('area-chart') diff --git a/app/views/projects/environments/metrics.html.haml b/app/views/projects/environments/metrics.html.haml index 92dc58cd38d..2e54af698aa 100644 --- a/app/views/projects/environments/metrics.html.haml +++ b/app/views/projects/environments/metrics.html.haml @@ -5,7 +5,7 @@ = page_specific_javascript_bundle_tag('monitoring') = render "projects/pipelines/head" -%div{ class: container_class } +.prometheus-container{ class: container_class, 'data-has-metrics': "#{@environment.has_metrics?}" } .top-area .row .col-sm-6 @@ -16,13 +16,68 @@ .col-sm-6 .nav-controls = render 'projects/deployments/actions', deployment: @environment.last_deployment - .row - .col-sm-12 - %h4 - CPU utilization - %svg.prometheus-graph{ 'graph-type' => 'cpu_values' } - .row - .col-sm-12 - %h4 - Memory usage - %svg.prometheus-graph{ 'graph-type' => 'memory_values' } + .prometheus-state + .js-getting-started.hidden + .row + .col-md-4.col-md-offset-4.state-svg + = render "shared/empty_states/monitoring/getting_started.svg" + .row + .col-md-6.col-md-offset-3 + %h4.text-center.state-title + Get started with performance monitoring + .row + .col-md-6.col-md-offset-3 + .description-text.text-center.state-description + Stay updated about the performance and health of your environment by configuring Prometheus to monitor your deployments. + = link_to help_page_path('administration/monitoring/prometheus/index.md') do + Learn more about performance monitoring + .row.state-button-section + .col-md-4.col-md-offset-4.text-center.state-button + = link_to edit_namespace_project_service_path(@project.namespace, @project, 'prometheus'), class: 'btn btn-success' do + Configure Prometheus + .js-loading.hidden + .row + .col-md-4.col-md-offset-4.state-svg + = render "shared/empty_states/monitoring/loading.svg" + .row + .col-md-6.col-md-offset-3 + %h4.text-center.state-title + Waiting for performance data + .row + .col-md-6.col-md-offset-3 + .description-text.text-center.state-description + Creating graphs uses the data from the Prometheus server. If this takes a long time, ensure that data is available. + .row.state-button-section + .col-md-4.col-md-offset-4.text-center.state-button + = link_to help_page_path('administration/monitoring/prometheus/index.md'), class: 'btn btn-success' do + View documentation + .js-unable-to-connect.hidden + .row + .col-md-4.col-md-offset-4.state-svg + = render "shared/empty_states/monitoring/unable_to_connect.svg" + .row + .col-md-6.col-md-offset-3 + %h4.text-center.state-title + Unable to connect to Prometheus server + .row + .col-md-6.col-md-offset-3 + .description-text.text-center.state-description + Ensure connectivity is available from the GitLab server to the + = link_to edit_namespace_project_service_path(@project.namespace, @project, 'prometheus') do + Prometheus server + .row.state-button-section + .col-md-4.col-md-offset-4.text-center.state-button + = link_to help_page_path('administration/monitoring/prometheus/index.md'), class:'btn btn-success' do + View documentation + + .prometheus-graphs + .row + .col-sm-12 + %h4 + CPU utilization + %svg.prometheus-graph{ 'graph-type' => 'cpu_values' } + .row + .col-sm-12 + %h4 + Memory usage + %svg.prometheus-graph{ 'graph-type' => 'memory_values' } diff --git a/app/views/shared/empty_states/monitoring/_getting_started.svg b/app/views/shared/empty_states/monitoring/_getting_started.svg new file mode 100644 index 00000000000..db7a1c2e708 --- /dev/null +++ b/app/views/shared/empty_states/monitoring/_getting_started.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/app/views/shared/empty_states/monitoring/_loading.svg b/app/views/shared/empty_states/monitoring/_loading.svg new file mode 100644 index 00000000000..6bbd7a6c5b9 --- /dev/null +++ b/app/views/shared/empty_states/monitoring/_loading.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/app/views/shared/empty_states/monitoring/_unable_to_connect.svg b/app/views/shared/empty_states/monitoring/_unable_to_connect.svg new file mode 100644 index 00000000000..62537d87d5d --- /dev/null +++ b/app/views/shared/empty_states/monitoring/_unable_to_connect.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/changelogs/unreleased/add-error-empty-states.yml b/changelogs/unreleased/add-error-empty-states.yml new file mode 100644 index 00000000000..ec6c7b6dce9 --- /dev/null +++ b/changelogs/unreleased/add-error-empty-states.yml @@ -0,0 +1,4 @@ +--- +title: Introduced error/empty states for the environments performance metrics +merge_request: 10271 +author: diff --git a/spec/javascripts/fixtures/environments/metrics.html.haml b/spec/javascripts/fixtures/environments/metrics.html.haml index 483063fb889..e2dd9519898 100644 --- a/spec/javascripts/fixtures/environments/metrics.html.haml +++ b/spec/javascripts/fixtures/environments/metrics.html.haml @@ -1,12 +1,62 @@ -%div +.prometheus-container{ 'data-has-metrics': "false", 'data-doc-link': '/help/administration/monitoring/prometheus/index.md', 'data-prometheus-integration': '/root/hello-prometheus/services/prometheus/edit' } .top-area .row .col-sm-6 %h3.page-title Metrics for environment - .row - .col-sm-12 - %svg.prometheus-graph{ 'graph-type' => 'cpu_values' } - .row - .col-sm-12 - %svg.prometheus-graph{ 'graph-type' => 'memory_values' } \ No newline at end of file + .prometheus-state + .js-getting-started.hidden + .row + .col-md-4.col-md-offset-4.state-svg + %svg + .row + .col-md-6.col-md-offset-3 + %h4.text-center.state-title + Get started with performance monitoring + .row + .col-md-6.col-md-offset-3 + .description-text.text-center.state-description + Stay updated about the performance and health of your environment by configuring Prometheus to monitor your deployments. Learn more about performance monitoring + .row.state-button-section + .col-md-4.col-md-offset-4.text-center.state-button + %a.btn.btn-success + Configure Prometheus + .js-loading.hidden + .row + .col-md-4.col-md-offset-4.state-svg + %svg + .row + .col-md-6.col-md-offset-3 + %h4.text-center.state-title + Waiting for performance data + .row + .col-md-6.col-md-offset-3 + .description-text.text-center.state-description + Creating graphs uses the data from the Prometheus server. If this takes a long time, ensure that data is available. + .row.state-button-section + .col-md-4.col-md-offset-4.text-center.state-button + %a.btn.btn-success + View documentation + .js-unable-to-connect.hidden + .row + .col-md-4.col-md-offset-4.state-svg + %svg + .row + .col-md-6.col-md-offset-3 + %h4.text-center.state-title + Unable to connect to Prometheus server + .row + .col-md-6.col-md-offset-3 + .description-text.text-center.state-description + Ensure connectivity is available from the GitLab server to the Prometheus server + .row.state-button-section + .col-md-4.col-md-offset-4.text-center.state-button + %a.btn.btn-success + View documentation + .prometheus-graphs + .row + .col-sm-12 + %svg.prometheus-graph{ 'graph-type' => 'cpu_values' } + .row + .col-sm-12 + %svg.prometheus-graph{ 'graph-type' => 'memory_values' } diff --git a/spec/javascripts/monitoring/prometheus_graph_spec.js b/spec/javascripts/monitoring/prometheus_graph_spec.js index c2bcd9c0f7c..4b904fc2960 100644 --- a/spec/javascripts/monitoring/prometheus_graph_spec.js +++ b/spec/javascripts/monitoring/prometheus_graph_spec.js @@ -1,5 +1,4 @@ import 'jquery'; -import '~/lib/utils/common_utils'; import PrometheusGraph from '~/monitoring/prometheus_graph'; import { prometheusMockData } from './prometheus_mock_data'; @@ -12,6 +11,7 @@ describe('PrometheusGraph', () => { beforeEach(() => { loadFixtures(fixtureName); + $('.prometheus-container').data('has-metrics', 'true'); this.prometheusGraph = new PrometheusGraph(); const self = this; const fakeInit = (metricsResponse) => { @@ -75,3 +75,24 @@ describe('PrometheusGraph', () => { }); }); }); + +describe('PrometheusGraphs UX states', () => { + const fixtureName = 'static/environments/metrics.html.raw'; + preloadFixtures(fixtureName); + + beforeEach(() => { + loadFixtures(fixtureName); + this.prometheusGraph = new PrometheusGraph(); + }); + + it('shows a specified state', () => { + this.prometheusGraph.state = '.js-getting-started'; + this.prometheusGraph.updateState(); + const $state = $('.js-getting-started'); + expect($state).toBeDefined(); + expect($('.state-title', $state)).toBeDefined(); + expect($('.state-svg', $state)).toBeDefined(); + expect($('.state-description', $state)).toBeDefined(); + expect($('.state-button', $state)).toBeDefined(); + }); +}); -- cgit v1.2.1 From 72580f07af5a2c1e4df6bbc339ad804b5f5bb9ed Mon Sep 17 00:00:00 2001 From: Timothy Andrew Date: Wed, 5 Apr 2017 12:50:53 +0530 Subject: Move a user's merge requests to the ghost user. 1. When the user is deleted. 2. Refactor out code relating to "migrating records to the ghost user" into a `MigrateToGhostUser` concern, which is tested using a shared example. --- .../concerns/users/migrate_to_ghost_user.rb | 38 +++++ app/services/users/destroy_service.rb | 21 +-- spec/services/users/destroy_service_spec.rb | 161 +++++++++++++++++++++ spec/services/users/destroy_spec.rb | 145 ------------------- ...ervice_migrate_to_ghost_user_shared_examples.rb | 53 +++++++ 5 files changed, 255 insertions(+), 163 deletions(-) create mode 100644 app/services/concerns/users/migrate_to_ghost_user.rb create mode 100644 spec/services/users/destroy_service_spec.rb delete mode 100644 spec/services/users/destroy_spec.rb create mode 100644 spec/support/services/user_destroy_service_migrate_to_ghost_user_shared_examples.rb diff --git a/app/services/concerns/users/migrate_to_ghost_user.rb b/app/services/concerns/users/migrate_to_ghost_user.rb new file mode 100644 index 00000000000..ecbbf3026a0 --- /dev/null +++ b/app/services/concerns/users/migrate_to_ghost_user.rb @@ -0,0 +1,38 @@ +# When a user is destroyed, some of their associated records are +# moved to a "Ghost User", to prevent these associated records from +# being destroyed. +# +# For example, all the issues/MRs a user has created are _not_ destroyed +# when the user is destroyed. +module Users::MigrateToGhostUser + extend ActiveSupport::Concern + + attr_reader :ghost_user + + def move_associated_records_to_ghost_user(user) + # Block the user before moving records to prevent a data race. + # For example, if the user creates an issue after `move_issues_to_ghost_user` + # runs and before the user is destroyed, the destroy will fail with + # an exception. + user.block + + user.transaction do + @ghost_user = User.ghost + + move_issues_to_ghost_user(user) + move_merge_requests_to_ghost_user(user) + end + + user.reload + end + + private + + def move_issues_to_ghost_user(user) + user.issues.update_all(author_id: ghost_user.id) + end + + def move_merge_requests_to_ghost_user(user) + user.merge_requests.update_all(author_id: ghost_user.id) + end +end diff --git a/app/services/users/destroy_service.rb b/app/services/users/destroy_service.rb index a3b32a71a64..e6608e316dc 100644 --- a/app/services/users/destroy_service.rb +++ b/app/services/users/destroy_service.rb @@ -1,5 +1,7 @@ module Users class DestroyService + include MigrateToGhostUser + attr_accessor :current_user def initialize(current_user) @@ -26,7 +28,7 @@ module Users ::Projects::DestroyService.new(project, current_user, skip_repo: true).execute end - move_issues_to_ghost_user(user) + move_associated_records_to_ghost_user(user) # Destroy the namespace after destroying the user since certain methods may depend on the namespace existing namespace = user.namespace @@ -35,22 +37,5 @@ module Users user_data end - - private - - def move_issues_to_ghost_user(user) - # Block the user before moving issues to prevent a data race. - # If the user creates an issue after `move_issues_to_ghost_user` - # runs and before the user is destroyed, the destroy will fail with - # an exception. We block the user so that issues can't be created - # after `move_issues_to_ghost_user` runs and before the destroy happens. - user.block - - ghost_user = User.ghost - - user.issues.update_all(author_id: ghost_user.id) - - user.reload - end end end diff --git a/spec/services/users/destroy_service_spec.rb b/spec/services/users/destroy_service_spec.rb new file mode 100644 index 00000000000..a5b69e6ddf4 --- /dev/null +++ b/spec/services/users/destroy_service_spec.rb @@ -0,0 +1,161 @@ +require 'spec_helper' + +describe Users::DestroyService, services: true do + describe "Deletes a user and all their personal projects" do + let!(:user) { create(:user) } + let!(:admin) { create(:admin) } + let!(:namespace) { create(:namespace, owner: user) } + let!(:project) { create(:empty_project, namespace: namespace) } + let(:service) { described_class.new(admin) } + + context 'no options are given' do + it 'deletes the user' do + user_data = service.execute(user) + + expect { user_data['email'].to eq(user.email) } + expect { User.find(user.id) }.to raise_error(ActiveRecord::RecordNotFound) + expect { Namespace.with_deleted.find(user.namespace.id) }.to raise_error(ActiveRecord::RecordNotFound) + end + + it 'will delete the project' do + expect_any_instance_of(Projects::DestroyService).to receive(:execute).once + + service.execute(user) + end + end + + context 'projects in pending_delete' do + before do + project.pending_delete = true + project.save + end + + it 'destroys a project in pending_delete' do + expect_any_instance_of(Projects::DestroyService).to receive(:execute).once + + service.execute(user) + + expect { Project.find(project.id) }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "a deleted user's issues" do + let(:project) { create(:project) } + + before do + project.add_developer(user) + end + + context "for an issue the user has created" do + let!(:issue) { create(:issue, project: project, author: user) } + + before do + service.execute(user) + end + + it 'does not delete the issue' do + expect(Issue.find_by_id(issue.id)).to be_present + end + + it 'migrates the issue so that the "Ghost User" is the issue owner' do + migrated_issue = Issue.find_by_id(issue.id) + + expect(migrated_issue.author).to eq(User.ghost) + end + + it 'blocks the user before migrating issues to the "Ghost User' do + expect(user).to be_blocked + end + end + + context "for an issue the user was assigned to" do + let!(:issue) { create(:issue, project: project, assignee: user) } + + before do + service.execute(user) + end + + it 'does not delete issues the user is assigned to' do + expect(Issue.find_by_id(issue.id)).to be_present + end + + it 'migrates the issue so that it is "Unassigned"' do + migrated_issue = Issue.find_by_id(issue.id) + + expect(migrated_issue.assignee).to be_nil + end + end + end + + context "solo owned groups present" do + let(:solo_owned) { create(:group) } + let(:member) { create(:group_member) } + let(:user) { member.user } + + before do + solo_owned.group_members = [member] + service.execute(user) + end + + it 'does not delete the user' do + expect(User.find(user.id)).to eq user + end + end + + context "deletions with solo owned groups" do + let(:solo_owned) { create(:group) } + let(:member) { create(:group_member) } + let(:user) { member.user } + + before do + solo_owned.group_members = [member] + service.execute(user, delete_solo_owned_groups: true) + end + + it 'deletes solo owned groups' do + expect { Project.find(solo_owned.id) }.to raise_error(ActiveRecord::RecordNotFound) + end + + it 'deletes the user' do + expect { User.find(user.id) }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context "deletion permission checks" do + it 'does not delete the user when user is not an admin' do + other_user = create(:user) + + expect { described_class.new(other_user).execute(user) }.to raise_error(Gitlab::Access::AccessDeniedError) + expect(User.exists?(user.id)).to be(true) + end + + it 'allows admins to delete anyone' do + described_class.new(admin).execute(user) + + expect(User.exists?(user.id)).to be(false) + end + + it 'allows users to delete their own account' do + described_class.new(user).execute(user) + + expect(User.exists?(user.id)).to be(false) + end + end + + context 'migrating associated records to the ghost user' do + context 'issues' do + include_examples "migrating a deleted user's associated records to the ghost user", Issue do + let(:created_record) { create(:issue, project: project, author: user) } + let(:assigned_record) { create(:issue, project: project, assignee: user) } + end + end + + context 'merge requests' do + include_examples "migrating a deleted user's associated records to the ghost user", MergeRequest do + let(:created_record) { create(:merge_request, source_project: project, author: user, target_branch: "first") } + let(:assigned_record) { create(:merge_request, source_project: project, assignee: user, target_branch: 'second') } + end + end + end + end +end diff --git a/spec/services/users/destroy_spec.rb b/spec/services/users/destroy_spec.rb deleted file mode 100644 index 66c61b7f8ff..00000000000 --- a/spec/services/users/destroy_spec.rb +++ /dev/null @@ -1,145 +0,0 @@ -require 'spec_helper' - -describe Users::DestroyService, services: true do - describe "Deletes a user and all their personal projects" do - let!(:user) { create(:user) } - let!(:admin) { create(:admin) } - let!(:namespace) { create(:namespace, owner: user) } - let!(:project) { create(:empty_project, namespace: namespace) } - let(:service) { described_class.new(admin) } - - context 'no options are given' do - it 'deletes the user' do - user_data = service.execute(user) - - expect { user_data['email'].to eq(user.email) } - expect { User.find(user.id) }.to raise_error(ActiveRecord::RecordNotFound) - expect { Namespace.with_deleted.find(user.namespace.id) }.to raise_error(ActiveRecord::RecordNotFound) - end - - it 'will delete the project' do - expect_any_instance_of(Projects::DestroyService).to receive(:execute).once - - service.execute(user) - end - end - - context 'projects in pending_delete' do - before do - project.pending_delete = true - project.save - end - - it 'destroys a project in pending_delete' do - expect_any_instance_of(Projects::DestroyService).to receive(:execute).once - - service.execute(user) - - expect { Project.find(project.id) }.to raise_error(ActiveRecord::RecordNotFound) - end - end - - context "a deleted user's issues" do - let(:project) { create(:project) } - - before do - project.add_developer(user) - end - - context "for an issue the user has created" do - let!(:issue) { create(:issue, project: project, author: user) } - - before do - service.execute(user) - end - - it 'does not delete the issue' do - expect(Issue.find_by_id(issue.id)).to be_present - end - - it 'migrates the issue so that the "Ghost User" is the issue owner' do - migrated_issue = Issue.find_by_id(issue.id) - - expect(migrated_issue.author).to eq(User.ghost) - end - - it 'blocks the user before migrating issues to the "Ghost User' do - expect(user).to be_blocked - end - end - - context "for an issue the user was assigned to" do - let!(:issue) { create(:issue, project: project, assignee: user) } - - before do - service.execute(user) - end - - it 'does not delete issues the user is assigned to' do - expect(Issue.find_by_id(issue.id)).to be_present - end - - it 'migrates the issue so that it is "Unassigned"' do - migrated_issue = Issue.find_by_id(issue.id) - - expect(migrated_issue.assignee).to be_nil - end - end - end - - context "solo owned groups present" do - let(:solo_owned) { create(:group) } - let(:member) { create(:group_member) } - let(:user) { member.user } - - before do - solo_owned.group_members = [member] - service.execute(user) - end - - it 'does not delete the user' do - expect(User.find(user.id)).to eq user - end - end - - context "deletions with solo owned groups" do - let(:solo_owned) { create(:group) } - let(:member) { create(:group_member) } - let(:user) { member.user } - - before do - solo_owned.group_members = [member] - service.execute(user, delete_solo_owned_groups: true) - end - - it 'deletes solo owned groups' do - expect { Project.find(solo_owned.id) }.to raise_error(ActiveRecord::RecordNotFound) - end - - it 'deletes the user' do - expect { User.find(user.id) }.to raise_error(ActiveRecord::RecordNotFound) - end - end - - context "deletion permission checks" do - it 'does not delete the user when user is not an admin' do - other_user = create(:user) - - expect { described_class.new(other_user).execute(user) }.to raise_error(Gitlab::Access::AccessDeniedError) - expect(User.exists?(user.id)).to be(true) - end - - it 'allows admins to delete anyone' do - described_class.new(admin).execute(user) - - expect(User.exists?(user.id)).to be(false) - end - - it 'allows users to delete their own account' do - described_class.new(user).execute(user) - - expect(User.exists?(user.id)).to be(false) - end - end - end -end diff --git a/spec/support/services/user_destroy_service_migrate_to_ghost_user_shared_examples.rb b/spec/support/services/user_destroy_service_migrate_to_ghost_user_shared_examples.rb new file mode 100644 index 00000000000..8996e3420e6 --- /dev/null +++ b/spec/support/services/user_destroy_service_migrate_to_ghost_user_shared_examples.rb @@ -0,0 +1,53 @@ +require "spec_helper" + +shared_examples "migrating a deleted user's associated records to the ghost user" do |record_class| + record_class_name = record_class.to_s.titleize.downcase + + let(:project) { create(:project) } + + before do + project.add_developer(user) + end + + context "for a #{record_class_name} the user has created" do + let!(:record) { created_record } + + it "does not delete the #{record_class_name}" do + service.execute(user) + + expect(record_class.find_by_id(record.id)).to be_present + end + + it "migrates the #{record_class_name} so that the 'Ghost User' is the #{record_class_name} owner" do + service.execute(user) + + migrated_record = record_class.find_by_id(record.id) + + expect(migrated_record.author).to eq(User.ghost) + end + + it "blocks the user before migrating #{record_class_name}s to the 'Ghost User'" do + service.execute(user) + + expect(user).to be_blocked + end + end + + context "for a #{record_class_name} the user was assigned to" do + let!(:record) { assigned_record } + + before do + service.execute(user) + end + + it "does not delete #{record_class_name}s the user is assigned to" do + expect(record_class.find_by_id(record.id)).to be_present + end + + it "migrates the #{record_class_name} so that it is 'Unassigned'" do + migrated_record = record_class.find_by_id(record.id) + + expect(migrated_record.assignee).to be_nil + end + end +end -- cgit v1.2.1 From 97cbf7c223ec772e4747bab5083904d4053e2e63 Mon Sep 17 00:00:00 2001 From: Timothy Andrew Date: Wed, 5 Apr 2017 13:27:46 +0530 Subject: Move a user's notes to the ghost user ... when the user is destroyed. --- .../concerns/users/migrate_to_ghost_user.rb | 5 +++++ spec/services/users/destroy_service_spec.rb | 10 +++++++-- ...ervice_migrate_to_ghost_user_shared_examples.rb | 26 ++++++++++++---------- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/app/services/concerns/users/migrate_to_ghost_user.rb b/app/services/concerns/users/migrate_to_ghost_user.rb index ecbbf3026a0..0779d12cd3a 100644 --- a/app/services/concerns/users/migrate_to_ghost_user.rb +++ b/app/services/concerns/users/migrate_to_ghost_user.rb @@ -21,6 +21,7 @@ module Users::MigrateToGhostUser move_issues_to_ghost_user(user) move_merge_requests_to_ghost_user(user) + move_notes_to_ghost_user(user) end user.reload @@ -35,4 +36,8 @@ module Users::MigrateToGhostUser def move_merge_requests_to_ghost_user(user) user.merge_requests.update_all(author_id: ghost_user.id) end + + def move_notes_to_ghost_user(user) + user.notes.update_all(author_id: ghost_user.id) + end end diff --git a/spec/services/users/destroy_service_spec.rb b/spec/services/users/destroy_service_spec.rb index a5b69e6ddf4..3efa7c196dc 100644 --- a/spec/services/users/destroy_service_spec.rb +++ b/spec/services/users/destroy_service_spec.rb @@ -144,18 +144,24 @@ describe Users::DestroyService, services: true do context 'migrating associated records to the ghost user' do context 'issues' do - include_examples "migrating a deleted user's associated records to the ghost user", Issue do + include_examples "migrating a deleted user's associated records to the ghost user", Issue, {} do let(:created_record) { create(:issue, project: project, author: user) } let(:assigned_record) { create(:issue, project: project, assignee: user) } end end context 'merge requests' do - include_examples "migrating a deleted user's associated records to the ghost user", MergeRequest do + include_examples "migrating a deleted user's associated records to the ghost user", MergeRequest, {} do let(:created_record) { create(:merge_request, source_project: project, author: user, target_branch: "first") } let(:assigned_record) { create(:merge_request, source_project: project, assignee: user, target_branch: 'second') } end end + + context 'notes' do + include_examples "migrating a deleted user's associated records to the ghost user", Note, { skip_assignee_specs: true } do + let(:created_record) { create(:note, project: project, author: user) } + end + end end end end diff --git a/spec/support/services/user_destroy_service_migrate_to_ghost_user_shared_examples.rb b/spec/support/services/user_destroy_service_migrate_to_ghost_user_shared_examples.rb index 8996e3420e6..add3dd3d5bc 100644 --- a/spec/support/services/user_destroy_service_migrate_to_ghost_user_shared_examples.rb +++ b/spec/support/services/user_destroy_service_migrate_to_ghost_user_shared_examples.rb @@ -1,6 +1,6 @@ require "spec_helper" -shared_examples "migrating a deleted user's associated records to the ghost user" do |record_class| +shared_examples "migrating a deleted user's associated records to the ghost user" do |record_class, options| record_class_name = record_class.to_s.titleize.downcase let(:project) { create(:project) } @@ -33,21 +33,23 @@ shared_examples "migrating a deleted user's associated records to the ghost user end end - context "for a #{record_class_name} the user was assigned to" do - let!(:record) { assigned_record } + unless options[:skip_assignee_specs] + context "for a #{record_class_name} the user was assigned to" do + let!(:record) { assigned_record } - before do - service.execute(user) - end + before do + service.execute(user) + end - it "does not delete #{record_class_name}s the user is assigned to" do - expect(record_class.find_by_id(record.id)).to be_present - end + it "does not delete #{record_class_name}s the user is assigned to" do + expect(record_class.find_by_id(record.id)).to be_present + end - it "migrates the #{record_class_name} so that it is 'Unassigned'" do - migrated_record = record_class.find_by_id(record.id) + it "migrates the #{record_class_name} so that it is 'Unassigned'" do + migrated_record = record_class.find_by_id(record.id) - expect(migrated_record.assignee).to be_nil + expect(migrated_record.assignee).to be_nil + end end end end -- cgit v1.2.1 From 6a065074a3add27825631451dd478d1164c1a1cd Mon Sep 17 00:00:00 2001 From: Timothy Andrew Date: Wed, 5 Apr 2017 22:13:39 +0530 Subject: Fix a bug with the User#abuse_report association. Introduction ------------ 1. The foreign key was not explicitly specified on the association. 2. The `AbuseReport` model contains two references to user - `reporter_id` and `user_id` 3. `user.abuse_report` is supposed to return the single abuse report where `user_id` refers to the given user. Bug Description --------------- 1. `user.abuse_report` would return an abuse report where `reporter_id` referred to the current user, if such an abuse report was present. 2. This implies a slightly more serious bug as well: - Assume User A filed an abuse report against User B - We have an abuse report where `reporter_id` is User A and `user_id` is User B - If User A is updated (`user_a.block`, for example), the abuse report would also be updated, such that both `reporter_id` _and_ `user_id` point to User A. Fix --- Explicitly declare the foreign key `user_id` in the `has_one` declaration --- app/models/user.rb | 2 +- spec/models/user_spec.rb | 28 +++++++++++++++++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/app/models/user.rb b/app/models/user.rb index 95a766f2ede..8c2f0011be8 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -89,7 +89,7 @@ class User < ActiveRecord::Base has_many :subscriptions, dependent: :destroy has_many :recent_events, -> { order "id DESC" }, foreign_key: :author_id, class_name: "Event" has_many :oauth_applications, class_name: 'Doorkeeper::Application', as: :owner, dependent: :destroy - has_one :abuse_report, dependent: :destroy + has_one :abuse_report, dependent: :destroy, foreign_key: :user_id has_many :spam_logs, dependent: :destroy has_many :builds, dependent: :nullify, class_name: 'Ci::Build' has_many :pipelines, dependent: :nullify, class_name: 'Ci::Pipeline' diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index a9e37be1157..6a787f6b0df 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -28,7 +28,6 @@ describe User, models: true do it { is_expected.to have_many(:merge_requests).dependent(:destroy) } it { is_expected.to have_many(:assigned_merge_requests).dependent(:nullify) } it { is_expected.to have_many(:identities).dependent(:destroy) } - it { is_expected.to have_one(:abuse_report) } it { is_expected.to have_many(:spam_logs).dependent(:destroy) } it { is_expected.to have_many(:todos).dependent(:destroy) } it { is_expected.to have_many(:award_emoji).dependent(:destroy) } @@ -38,6 +37,33 @@ describe User, models: true do it { is_expected.to have_many(:chat_names).dependent(:destroy) } it { is_expected.to have_many(:uploads).dependent(:destroy) } + describe "#abuse_report" do + let(:current_user) { create(:user) } + let(:other_user) { create(:user) } + + it { is_expected.to have_one(:abuse_report) } + + it "refers to the abuse report whose user_id is the current user" do + abuse_report = create(:abuse_report, reporter: other_user, user: current_user) + + expect(current_user.abuse_report).to eq(abuse_report) + end + + it "does not refer to the abuse report whose reporter_id is the current user" do + create(:abuse_report, reporter: current_user, user: other_user) + + expect(current_user.abuse_report).to be_nil + end + + it "does not update the user_id of an abuse report when the user is updated" do + abuse_report = create(:abuse_report, reporter: current_user, user: other_user) + + current_user.block + + expect(abuse_report.reload.user).to eq(other_user) + end + end + describe '#group_members' do it 'does not include group memberships for which user is a requester' do user = create(:user) -- cgit v1.2.1 From 682987547a932c011f84c6455f0fd32bb500b308 Mon Sep 17 00:00:00 2001 From: Timothy Andrew Date: Wed, 5 Apr 2017 22:25:52 +0530 Subject: Move a user's abuse reports to the ghost user ... when the user is destroyed. To clarify, this regards abuse reports that the to-be-deleted user has _reported_. --- app/services/concerns/users/migrate_to_ghost_user.rb | 19 ++++++++++++------- spec/services/users/destroy_service_spec.rb | 7 +++++++ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/app/services/concerns/users/migrate_to_ghost_user.rb b/app/services/concerns/users/migrate_to_ghost_user.rb index 0779d12cd3a..5d1f0ff57d1 100644 --- a/app/services/concerns/users/migrate_to_ghost_user.rb +++ b/app/services/concerns/users/migrate_to_ghost_user.rb @@ -11,7 +11,7 @@ module Users::MigrateToGhostUser def move_associated_records_to_ghost_user(user) # Block the user before moving records to prevent a data race. - # For example, if the user creates an issue after `move_issues_to_ghost_user` + # For example, if the user creates an issue after `migrate_issues` # runs and before the user is destroyed, the destroy will fail with # an exception. user.block @@ -19,9 +19,10 @@ module Users::MigrateToGhostUser user.transaction do @ghost_user = User.ghost - move_issues_to_ghost_user(user) - move_merge_requests_to_ghost_user(user) - move_notes_to_ghost_user(user) + migrate_issues(user) + migrate_merge_requests(user) + migrate_notes(user) + migrate_abuse_reports(user) end user.reload @@ -29,15 +30,19 @@ module Users::MigrateToGhostUser private - def move_issues_to_ghost_user(user) + def migrate_issues(user) user.issues.update_all(author_id: ghost_user.id) end - def move_merge_requests_to_ghost_user(user) + def migrate_merge_requests(user) user.merge_requests.update_all(author_id: ghost_user.id) end - def move_notes_to_ghost_user(user) + def migrate_notes(user) user.notes.update_all(author_id: ghost_user.id) end + + def migrate_abuse_reports(user) + AbuseReport.where(reporter_id: user.id).update_all(reporter_id: ghost_user.id) + end end diff --git a/spec/services/users/destroy_service_spec.rb b/spec/services/users/destroy_service_spec.rb index 3efa7c196dc..028de62e4ca 100644 --- a/spec/services/users/destroy_service_spec.rb +++ b/spec/services/users/destroy_service_spec.rb @@ -162,6 +162,13 @@ describe Users::DestroyService, services: true do let(:created_record) { create(:note, project: project, author: user) } end end + + context 'abuse reports' do + include_examples "migrating a deleted user's associated records to the ghost user", AbuseReport, { skip_assignee_specs: true } do + let(:created_record) { create(:abuse_report, reporter: user, user: create(:user)) } + let(:author_method) { :reporter } + end + end end end end -- cgit v1.2.1 From 3e1a1242c67781fb52940433c5ad1bbefd346216 Mon Sep 17 00:00:00 2001 From: Timothy Andrew Date: Thu, 6 Apr 2017 15:36:36 +0530 Subject: Move a user's award emoji to the ghost user ... when the user is destroyed. 1. Normally, for a given awardable and award emoji name, a user is only allowed to create a single award emoji. 2. This validation needs to be removed for ghost users, since: - User A and User B have created award emoji - with the same name and against the same awardable - User A is deleted. Their award emoji is moved to the ghost user - User B is deleted. Their award emoji needs to be moved to the ghost user. However, this breaks the uniqueness validation, since the ghost user is only allowed to have one award emoji of a given name for a given awardable --- app/models/award_emoji.rb | 3 ++- app/models/concerns/ghost_user.rb | 7 +++++ .../concerns/users/migrate_to_ghost_user.rb | 5 ++++ spec/models/award_emoji_spec.rb | 14 ++++++++++ spec/services/users/destroy_service_spec.rb | 30 +++++++++++++++++++++- ...ervice_migrate_to_ghost_user_shared_examples.rb | 6 ++++- 6 files changed, 62 insertions(+), 3 deletions(-) create mode 100644 app/models/concerns/ghost_user.rb diff --git a/app/models/award_emoji.rb b/app/models/award_emoji.rb index 6937ad3bdd9..6ada6fae4eb 100644 --- a/app/models/award_emoji.rb +++ b/app/models/award_emoji.rb @@ -3,13 +3,14 @@ class AwardEmoji < ActiveRecord::Base UPVOTE_NAME = "thumbsup".freeze include Participable + include GhostUser belongs_to :awardable, polymorphic: true belongs_to :user validates :awardable, :user, presence: true validates :name, presence: true, inclusion: { in: Gitlab::Emoji.emojis_names } - validates :name, uniqueness: { scope: [:user, :awardable_type, :awardable_id] } + validates :name, uniqueness: { scope: [:user, :awardable_type, :awardable_id] }, unless: :ghost_user? participant :user diff --git a/app/models/concerns/ghost_user.rb b/app/models/concerns/ghost_user.rb new file mode 100644 index 00000000000..da696127a80 --- /dev/null +++ b/app/models/concerns/ghost_user.rb @@ -0,0 +1,7 @@ +module GhostUser + extend ActiveSupport::Concern + + def ghost_user? + user && user.ghost? + end +end diff --git a/app/services/concerns/users/migrate_to_ghost_user.rb b/app/services/concerns/users/migrate_to_ghost_user.rb index 5d1f0ff57d1..76335e45e54 100644 --- a/app/services/concerns/users/migrate_to_ghost_user.rb +++ b/app/services/concerns/users/migrate_to_ghost_user.rb @@ -23,6 +23,7 @@ module Users::MigrateToGhostUser migrate_merge_requests(user) migrate_notes(user) migrate_abuse_reports(user) + migrate_award_emoji(user) end user.reload @@ -45,4 +46,8 @@ module Users::MigrateToGhostUser def migrate_abuse_reports(user) AbuseReport.where(reporter_id: user.id).update_all(reporter_id: ghost_user.id) end + + def migrate_award_emoji(user) + user.award_emoji.update_all(user_id: ghost_user.id) + end end diff --git a/spec/models/award_emoji_spec.rb b/spec/models/award_emoji_spec.rb index cb3c592f8cd..2a9a27752c1 100644 --- a/spec/models/award_emoji_spec.rb +++ b/spec/models/award_emoji_spec.rb @@ -25,6 +25,20 @@ describe AwardEmoji, models: true do expect(new_award).not_to be_valid end + + # Assume User A and User B both created award emoji of the same name + # on the same awardable. When User A is deleted, User A's award emoji + # is moved to the ghost user. When User B is deleted, User B's award emoji + # also needs to be moved to the ghost user - this cannot happen unless + # the uniqueness validation is disabled for ghost users. + it "allows duplicate award emoji for ghost users" do + user = create(:user, :ghost) + issue = create(:issue) + create(:award_emoji, user: user, awardable: issue) + new_award = build(:award_emoji, user: user, awardable: issue) + + expect(new_award).to be_valid + end end end end diff --git a/spec/services/users/destroy_service_spec.rb b/spec/services/users/destroy_service_spec.rb index 028de62e4ca..d3463185ebe 100644 --- a/spec/services/users/destroy_service_spec.rb +++ b/spec/services/users/destroy_service_spec.rb @@ -166,7 +166,35 @@ describe Users::DestroyService, services: true do context 'abuse reports' do include_examples "migrating a deleted user's associated records to the ghost user", AbuseReport, { skip_assignee_specs: true } do let(:created_record) { create(:abuse_report, reporter: user, user: create(:user)) } - let(:author_method) { :reporter } + end + end + + context 'award emoji' do + include_examples "migrating a deleted user's associated records to the ghost user", AwardEmoji, { skip_assignee_specs: true } do + let(:created_record) { create(:award_emoji, user: user) } + let(:author_alias) { :user } + + context "when the awardable already has an award emoji of the same name assigned to the ghost user" do + let(:awardable) { create(:issue) } + let!(:existing_award_emoji) { create(:award_emoji, user: User.ghost, name: "thumbsup", awardable: awardable) } + let!(:award_emoji) { create(:award_emoji, user: user, name: "thumbsup", awardable: awardable) } + + it "migrates the award emoji regardless" do + service.execute(user) + + migrated_record = AwardEmoji.find_by_id(award_emoji.id) + + expect(migrated_record.user).to eq(User.ghost) + end + + it "does not leave the migrated award emoji in an invalid state" do + service.execute(user) + + migrated_record = AwardEmoji.find_by_id(award_emoji.id) + + expect(migrated_record).to be_valid + end + end end end end diff --git a/spec/support/services/user_destroy_service_migrate_to_ghost_user_shared_examples.rb b/spec/support/services/user_destroy_service_migrate_to_ghost_user_shared_examples.rb index add3dd3d5bc..3820ffc283f 100644 --- a/spec/support/services/user_destroy_service_migrate_to_ghost_user_shared_examples.rb +++ b/spec/support/services/user_destroy_service_migrate_to_ghost_user_shared_examples.rb @@ -23,7 +23,11 @@ shared_examples "migrating a deleted user's associated records to the ghost user migrated_record = record_class.find_by_id(record.id) - expect(migrated_record.author).to eq(User.ghost) + if migrated_record.respond_to?(:author) + expect(migrated_record.author).to eq(User.ghost) + else + expect(migrated_record.send(author_alias)).to eq(User.ghost) + end end it "blocks the user before migrating #{record_class_name}s to the 'Ghost User'" do -- cgit v1.2.1 From 07f365e369f56394dd4c927a7592d4e49d3f4bbd Mon Sep 17 00:00:00 2001 From: Timothy Andrew Date: Thu, 6 Apr 2017 16:30:12 +0530 Subject: Add CHANGELOG entry for !10467 --- .../unreleased/28695-move-all-associated-records-to-ghost-user.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changelogs/unreleased/28695-move-all-associated-records-to-ghost-user.yml diff --git a/changelogs/unreleased/28695-move-all-associated-records-to-ghost-user.yml b/changelogs/unreleased/28695-move-all-associated-records-to-ghost-user.yml new file mode 100644 index 00000000000..c5dcde48028 --- /dev/null +++ b/changelogs/unreleased/28695-move-all-associated-records-to-ghost-user.yml @@ -0,0 +1,4 @@ +--- +title: Deleting a user should not delete associated records +merge_request: 10467 +author: -- cgit v1.2.1 From e152f3f3daf09b25e5a5952a0e62580b3ef96c50 Mon Sep 17 00:00:00 2001 From: Timothy Andrew Date: Thu, 6 Apr 2017 16:46:17 +0530 Subject: Add documentation around migrating records to the ghost user after user deletion. --- doc/profile/README.md | 1 + doc/user/profile/account/delete_account.md | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 doc/user/profile/account/delete_account.md diff --git a/doc/profile/README.md b/doc/profile/README.md index 54e44d65959..aed64ac1228 100644 --- a/doc/profile/README.md +++ b/doc/profile/README.md @@ -2,3 +2,4 @@ - [Preferences](../user/profile/preferences.md) - [Two-factor Authentication (2FA)](../user/profile/account/two_factor_authentication.md) +- [Deleting your account](../user/profile/account/delete_account.md) diff --git a/doc/user/profile/account/delete_account.md b/doc/user/profile/account/delete_account.md new file mode 100644 index 00000000000..505248536c8 --- /dev/null +++ b/doc/user/profile/account/delete_account.md @@ -0,0 +1,25 @@ +# Deleting a User Account + +- As a user, you can delete your own account by navigating to **Settings** > **Account** and selecting **Delete account** +- As an admin, you can delete a user account by navigating to the **Admin Area**, selecting the **Users** tab, selecting a user, and clicking on **Remvoe user** + +## Associated Records + +> Introduced for issues in [GitLab 9.0][ce-7393], and for merge requests, award emoji, notes, and abuse reports in [GitLab 9.1][ce-10467]. + +When a user account is deleted, not all associated records are deleted with it. Here's a list of things that will not be deleted: + +- Issues that the user created +- Merge requests that the user created +- Notes that the user created +- Abuse reports that the user reported +- Award emoji that the user craeted + + +Instead of being deleted, these records will be moved to a system-wide "Ghost User", whose sole purpose is to act as a container for such records. + + +[ce-7393]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/7393 +[ce-10467]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/10467 + + -- cgit v1.2.1 From 80c9264967f07f1a42aa01c391d827cfec052480 Mon Sep 17 00:00:00 2001 From: Dimitrie Hoekstra Date: Thu, 6 Apr 2017 13:38:09 +0000 Subject: Award emoji button smiley animation --- app/assets/javascripts/awards_handler.js | 3 +- app/assets/stylesheets/framework/awards.scss | 58 +++++++++++++++++++++- app/assets/stylesheets/framework/variables.scss | 2 + app/assets/stylesheets/pages/notes.scss | 41 +++++++++++++-- app/views/award_emoji/_awards_block.html.haml | 4 +- app/views/projects/notes/_note.html.haml | 4 +- .../shared/icons/_emoji_slightly_smiling_face.svg | 1 + app/views/shared/icons/_emoji_smile.svg | 1 + app/views/shared/icons/_emoji_smiley.svg | 1 + .../award-emoji-button-smiley-animation.yml | 4 ++ 10 files changed, 111 insertions(+), 8 deletions(-) create mode 100644 app/views/shared/icons/_emoji_slightly_smiling_face.svg create mode 100644 app/views/shared/icons/_emoji_smile.svg create mode 100644 app/views/shared/icons/_emoji_smiley.svg create mode 100644 changelogs/unreleased/award-emoji-button-smiley-animation.yml diff --git a/app/assets/javascripts/awards_handler.js b/app/assets/javascripts/awards_handler.js index 4f63c7988f5..67106e85a37 100644 --- a/app/assets/javascripts/awards_handler.js +++ b/app/assets/javascripts/awards_handler.js @@ -263,7 +263,8 @@ AwardsHandler.prototype.addAward = function addAward( this.addAwardToEmojiBar(votesBlock, normalizedEmoji, checkMutuality); return typeof callback === 'function' ? callback() : undefined; }); - return $('.emoji-menu').removeClass('is-visible'); + $('.emoji-menu').removeClass('is-visible'); + $('.js-add-award.is-active').removeClass('is-active'); }; AwardsHandler.prototype.addAwardToEmojiBar = function addAwardToEmojiBar( diff --git a/app/assets/stylesheets/framework/awards.scss b/app/assets/stylesheets/framework/awards.scss index 1ae144fb471..b849cc2d853 100644 --- a/app/assets/stylesheets/framework/awards.scss +++ b/app/assets/stylesheets/framework/awards.scss @@ -91,7 +91,7 @@ .award-menu-holder { display: inline-block; - position: relative; + position: absolute; .tooltip { white-space: nowrap; @@ -117,11 +117,41 @@ &.active, &:hover, - &:active { + &:active, + &.is-active { background-color: $row-hover; border-color: $row-hover-border; box-shadow: none; outline: 0; + + .award-control-icon svg { + background: $award-emoji-positive-add-bg; + + path { + fill: $award-emoji-positive-add-lines; + } + } + + .award-control-icon-neutral { + opacity: 0; + } + + .award-control-icon-positive { + opacity: 1; + transform: scale(1.15); + } + } + + &.is-active { + .award-control-icon-positive { + opacity: 0; + transform: scale(1); + } + + .award-control-icon-super-positive { + opacity: 1; + transform: scale(1); + } } &.btn { @@ -162,9 +192,33 @@ color: $border-gray-normal; margin-top: 1px; padding: 0 2px; + + svg { + margin-bottom: 1px; + height: 18px; + width: 18px; + border-radius: 50%; + + path { + fill: $border-gray-normal; + } + } + } + + .award-control-icon-positive, + .award-control-icon-super-positive { + position: absolute; + left: 7px; + bottom: 9px; + opacity: 0; + @include transition(opacity, transform); } .award-control-text { vertical-align: middle; } } + +.note-awards .award-control-icon-positive { + left: 6px; +} diff --git a/app/assets/stylesheets/framework/variables.scss b/app/assets/stylesheets/framework/variables.scss index 97794a47df8..712eb7caf33 100644 --- a/app/assets/stylesheets/framework/variables.scss +++ b/app/assets/stylesheets/framework/variables.scss @@ -293,6 +293,8 @@ $badge-color: $gl-text-color-secondary; * Award emoji */ $award-emoji-menu-shadow: rgba(0,0,0,.175); +$award-emoji-positive-add-bg: #fed159; +$award-emoji-positive-add-lines: #bb9c13; /* * Search Box diff --git a/app/assets/stylesheets/pages/notes.scss b/app/assets/stylesheets/pages/notes.scss index 57cf8e136e2..603ef461ffe 100644 --- a/app/assets/stylesheets/pages/notes.scss +++ b/app/assets/stylesheets/pages/notes.scss @@ -398,13 +398,50 @@ ul.notes { font-size: 17px; } - &:hover { + svg { + height: 16px; + width: 16px; + fill: $gray-darkest; + vertical-align: text-top; + } + + .award-control-icon-positive, + .award-control-icon-super-positive { + position: absolute; + margin-left: -20px; + opacity: 0; + } + + &:hover, + &.is-active { .danger-highlight { color: $gl-text-red; } .link-highlight { color: $gl-link-color; + + svg { + fill: $gl-link-color; + } + } + + .award-control-icon-neutral { + opacity: 0; + } + + .award-control-icon-positive { + opacity: 1; + } + } + + &.is-active { + .award-control-icon-positive { + opacity: 0; + } + + .award-control-icon-super-positive { + opacity: 1; } } } @@ -508,7 +545,6 @@ ul.notes { } .line-resolve-all-container { - .btn-group { margin-left: -4px; } @@ -537,7 +573,6 @@ ul.notes { fill: $gray-darkest; } } - } .line-resolve-all { diff --git a/app/views/award_emoji/_awards_block.html.haml b/app/views/award_emoji/_awards_block.html.haml index 5aae410a63f..3ca45fbf751 100644 --- a/app/views/award_emoji/_awards_block.html.haml +++ b/app/views/award_emoji/_awards_block.html.haml @@ -13,5 +13,7 @@ %button.btn.award-control.has-tooltip.js-add-award{ type: 'button', 'aria-label': 'Add emoji', data: { title: 'Add emoji', placement: "bottom" } } - = icon('smile-o', class: "award-control-icon award-control-icon-normal") + %span{ class: "award-control-icon award-control-icon-neutral" }= custom_icon('emoji_slightly_smiling_face') + %span{ class: "award-control-icon award-control-icon-positive" }= custom_icon('emoji_smiley') + %span{ class: "award-control-icon award-control-icon-super-positive" }= custom_icon('emoji_smile') = icon('spinner spin', class: "award-control-icon award-control-icon-loading") diff --git a/app/views/projects/notes/_note.html.haml b/app/views/projects/notes/_note.html.haml index 6c0e6d48d6c..18afa811bad 100644 --- a/app/views/projects/notes/_note.html.haml +++ b/app/views/projects/notes/_note.html.haml @@ -59,7 +59,9 @@ - if note.emoji_awardable? = link_to '#', title: 'Award Emoji', class: 'note-action-button note-emoji-button js-add-award js-note-emoji', data: { position: 'right' } do = icon('spinner spin') - = icon('smile-o', class: 'link-highlight') + %span{ class: "link-highlight award-control-icon-neutral" }= custom_icon('emoji_slightly_smiling_face') + %span{ class: "link-highlight award-control-icon-positive" }= custom_icon('emoji_smiley') + %span{ class: "link-highlight award-control-icon-super-positive" }= custom_icon('emoji_smile') - if note_editable = link_to '#', title: 'Edit comment', class: 'note-action-button js-note-edit' do diff --git a/app/views/shared/icons/_emoji_slightly_smiling_face.svg b/app/views/shared/icons/_emoji_slightly_smiling_face.svg new file mode 100644 index 00000000000..56dbad91554 --- /dev/null +++ b/app/views/shared/icons/_emoji_slightly_smiling_face.svg @@ -0,0 +1 @@ + diff --git a/app/views/shared/icons/_emoji_smile.svg b/app/views/shared/icons/_emoji_smile.svg new file mode 100644 index 00000000000..ce645fee46f --- /dev/null +++ b/app/views/shared/icons/_emoji_smile.svg @@ -0,0 +1 @@ + diff --git a/app/views/shared/icons/_emoji_smiley.svg b/app/views/shared/icons/_emoji_smiley.svg new file mode 100644 index 00000000000..ddfae50e566 --- /dev/null +++ b/app/views/shared/icons/_emoji_smiley.svg @@ -0,0 +1 @@ + diff --git a/changelogs/unreleased/award-emoji-button-smiley-animation.yml b/changelogs/unreleased/award-emoji-button-smiley-animation.yml new file mode 100644 index 00000000000..31903aeb040 --- /dev/null +++ b/changelogs/unreleased/award-emoji-button-smiley-animation.yml @@ -0,0 +1,4 @@ +--- +title: Added award emoji animation and improved active state +merge_request: +author: -- cgit v1.2.1 From 5f715f1d32c6f5ce25b3721bde8f476173afadc8 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 23 Mar 2017 03:54:49 +0900 Subject: Add scheduled_trigger model. Add cron parser. Plus, specs. --- app/models/ci/scheduled_trigger.rb | 23 ++++ app/workers/scheduled_trigger_worker.rb | 18 +++ .../20170322070910_create_ci_scheduled_triggers.rb | 45 ++++++++ db/schema.rb | 24 +++- lib/ci/cron_parser.rb | 30 +++++ spec/factories/ci/scheduled_triggers.rb | 42 +++++++ spec/lib/ci/cron_parser_spec.rb | 128 +++++++++++++++++++++ spec/models/ci/scheduled_trigger_spec.rb | 38 ++++++ spec/workers/scheduled_trigger_worker_spec.rb | 11 ++ 9 files changed, 356 insertions(+), 3 deletions(-) create mode 100644 app/models/ci/scheduled_trigger.rb create mode 100644 app/workers/scheduled_trigger_worker.rb create mode 100644 db/migrate/20170322070910_create_ci_scheduled_triggers.rb create mode 100644 lib/ci/cron_parser.rb create mode 100644 spec/factories/ci/scheduled_triggers.rb create mode 100644 spec/lib/ci/cron_parser_spec.rb create mode 100644 spec/models/ci/scheduled_trigger_spec.rb create mode 100644 spec/workers/scheduled_trigger_worker_spec.rb diff --git a/app/models/ci/scheduled_trigger.rb b/app/models/ci/scheduled_trigger.rb new file mode 100644 index 00000000000..5b1ff7bd7a4 --- /dev/null +++ b/app/models/ci/scheduled_trigger.rb @@ -0,0 +1,23 @@ +module Ci + class ScheduledTrigger < ActiveRecord::Base + extend Ci::Model + + acts_as_paranoid + + belongs_to :project + belongs_to :owner, class_name: "User" + + def schedule_next_run! + next_time = Ci::CronParser.new(cron, cron_time_zone).next_time_from_now + update(:next_run_at => next_time) if next_time.present? + end + + def valid_ref? + true #TODO: + end + + def update_last_run! + update(:last_run_at => Time.now) + end + end +end diff --git a/app/workers/scheduled_trigger_worker.rb b/app/workers/scheduled_trigger_worker.rb new file mode 100644 index 00000000000..7dc17aa4332 --- /dev/null +++ b/app/workers/scheduled_trigger_worker.rb @@ -0,0 +1,18 @@ +class ScheduledTriggerWorker + include Sidekiq::Worker + include CronjobQueue + + def perform + # TODO: Update next_run_at + + Ci::ScheduledTriggers.where("next_run_at < ?", Time.now).find_each do |trigger| + begin + Ci::CreateTriggerRequestService.new.execute(trigger.project, trigger, trigger.ref) + rescue => e + Rails.logger.error "#{trigger.id}: Failed to trigger job: #{e.message}" + ensure + trigger.schedule_next_run! + end + end + end +end diff --git a/db/migrate/20170322070910_create_ci_scheduled_triggers.rb b/db/migrate/20170322070910_create_ci_scheduled_triggers.rb new file mode 100644 index 00000000000..91e4b42d2af --- /dev/null +++ b/db/migrate/20170322070910_create_ci_scheduled_triggers.rb @@ -0,0 +1,45 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class CreateCiScheduledTriggers < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + # When a migration requires downtime you **must** uncomment the following + # constant and define a short and easy to understand explanation as to why the + # migration requires downtime. + # DOWNTIME_REASON = '' + + # When using the methods "add_concurrent_index" or "add_column_with_default" + # you must disable the use of transactions as these methods can not run in an + # existing transaction. When using "add_concurrent_index" make sure that this + # method is the _only_ method called in the migration, any other changes + # should go in a separate migration. This ensures that upon failure _only_ the + # index creation fails and can be retried or reverted easily. + # + # To disable transactions uncomment the following line and remove these + # comments: + # disable_ddl_transaction! + + def change + create_table :ci_scheduled_triggers do |t| + t.integer "project_id" + t.datetime "deleted_at" + t.datetime "created_at" + t.datetime "updated_at" + t.integer "owner_id" + t.string "description" + t.string "cron" + t.string "cron_time_zone" + t.datetime "next_run_at" + t.datetime "last_run_at" + t.string "ref" + end + + add_index :ci_scheduled_triggers, ["next_run_at"], name: "index_ci_scheduled_triggers_on_next_run_at", using: :btree + add_index :ci_scheduled_triggers, ["project_id"], name: "index_ci_scheduled_triggers_on_project_id", using: :btree + add_foreign_key :ci_scheduled_triggers, :users, column: :owner_id, on_delete: :cascade + end +end diff --git a/db/schema.rb b/db/schema.rb index 582f68cbee7..a101ce280fe 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -61,7 +61,6 @@ ActiveRecord::Schema.define(version: 20170405080720) do t.boolean "shared_runners_enabled", default: true, null: false t.integer "max_artifacts_size", default: 100, null: false t.string "runners_registration_token" - t.integer "max_pages_size", default: 100, null: false t.boolean "require_two_factor_authentication", default: false t.integer "two_factor_grace_period", default: 48 t.boolean "metrics_enabled", default: false @@ -111,6 +110,7 @@ ActiveRecord::Schema.define(version: 20170405080720) do t.string "plantuml_url" t.boolean "plantuml_enabled" t.integer "terminal_max_session_time", default: 0, null: false + t.integer "max_pages_size", default: 100, null: false t.string "default_artifacts_expire_in", default: "0", null: false t.integer "unique_ips_limit_per_user" t.integer "unique_ips_limit_time_window" @@ -290,6 +290,23 @@ ActiveRecord::Schema.define(version: 20170405080720) do add_index "ci_runners", ["locked"], name: "index_ci_runners_on_locked", using: :btree add_index "ci_runners", ["token"], name: "index_ci_runners_on_token", using: :btree + create_table "ci_scheduled_triggers", force: :cascade do |t| + t.integer "project_id" + t.datetime "deleted_at" + t.datetime "created_at" + t.datetime "updated_at" + t.integer "owner_id" + t.string "description" + t.string "cron" + t.string "cron_time_zone" + t.datetime "next_run_at" + t.datetime "last_run_at" + t.string "ref" + end + + add_index "ci_scheduled_triggers", ["next_run_at"], name: "index_ci_scheduled_triggers_on_next_run_at", using: :btree + add_index "ci_scheduled_triggers", ["project_id"], name: "index_ci_scheduled_triggers_on_project_id", using: :btree + create_table "ci_trigger_requests", force: :cascade do |t| t.integer "trigger_id", null: false t.text "variables" @@ -689,8 +706,8 @@ ActiveRecord::Schema.define(version: 20170405080720) do t.integer "visibility_level", default: 20, null: false t.boolean "request_access_enabled", default: false, null: false t.datetime "deleted_at" - t.text "description_html" t.boolean "lfs_enabled" + t.text "description_html" t.integer "parent_id" end @@ -1242,8 +1259,8 @@ ActiveRecord::Schema.define(version: 20170405080720) do t.datetime "otp_grace_period_started_at" t.boolean "ldap_email", default: false, null: false t.boolean "external", default: false - t.string "incoming_email_token" t.string "organization" + t.string "incoming_email_token" t.boolean "authorized_projects_populated" t.boolean "ghost" t.boolean "notified_of_own_activity" @@ -1298,6 +1315,7 @@ ActiveRecord::Schema.define(version: 20170405080720) do add_foreign_key "boards", "projects" add_foreign_key "chat_teams", "namespaces", on_delete: :cascade + add_foreign_key "ci_scheduled_triggers", "users", column: "owner_id", on_delete: :cascade add_foreign_key "ci_triggers", "users", column: "owner_id", name: "fk_e8e10d1964", on_delete: :cascade add_foreign_key "issue_metrics", "issues", on_delete: :cascade add_foreign_key "label_priorities", "labels", on_delete: :cascade diff --git a/lib/ci/cron_parser.rb b/lib/ci/cron_parser.rb new file mode 100644 index 00000000000..163cfc86aa7 --- /dev/null +++ b/lib/ci/cron_parser.rb @@ -0,0 +1,30 @@ +require 'rufus-scheduler' # Included in sidekiq-cron + +module Ci + class CronParser + def initialize(cron, cron_time_zone = 'UTC') + @cron = cron + @cron_time_zone = cron_time_zone + end + + def next_time_from_now + cronLine = try_parse_cron + return nil unless cronLine.present? + cronLine.next_time + end + + def valid_syntax? + try_parse_cron.present? ? true : false + end + + private + + def try_parse_cron + begin + Rufus::Scheduler.parse("#{@cron} #{@cron_time_zone}") + rescue + nil + end + end + end +end diff --git a/spec/factories/ci/scheduled_triggers.rb b/spec/factories/ci/scheduled_triggers.rb new file mode 100644 index 00000000000..9d45f4b4962 --- /dev/null +++ b/spec/factories/ci/scheduled_triggers.rb @@ -0,0 +1,42 @@ +FactoryGirl.define do + factory :ci_scheduled_trigger, class: Ci::ScheduledTrigger do + project factory: :empty_project + owner factory: :user + ref 'master' + + trait :cron_nightly_build do + cron '0 1 * * *' + cron_time_zone 'Europe/Istanbul' + end + + trait :cron_weekly_build do + cron '0 1 * * 5' + cron_time_zone 'Europe/Istanbul' + end + + trait :cron_monthly_build do + cron '0 1 22 * *' + cron_time_zone 'Europe/Istanbul' + end + + trait :cron_every_5_minutes do + cron '*/5 * * * *' + cron_time_zone 'Europe/Istanbul' + end + + trait :cron_every_5_hours do + cron '* */5 * * *' + cron_time_zone 'Europe/Istanbul' + end + + trait :cron_every_5_days do + cron '* * */5 * *' + cron_time_zone 'Europe/Istanbul' + end + + trait :cron_every_5_months do + cron '* * * */5 *' + cron_time_zone 'Europe/Istanbul' + end + end +end diff --git a/spec/lib/ci/cron_parser_spec.rb b/spec/lib/ci/cron_parser_spec.rb new file mode 100644 index 00000000000..58eb26c9421 --- /dev/null +++ b/spec/lib/ci/cron_parser_spec.rb @@ -0,0 +1,128 @@ +require 'spec_helper' + +module Ci + describe CronParser, lib: true do + describe '#next_time_from_now' do + subject { described_class.new(cron, cron_time_zone).next_time_from_now } + + context 'when cron and cron_time_zone are valid' do + context 'at 00:00, 00:10, 00:20, 00:30, 00:40, 00:50' do + let(:cron) { '*/10 * * * *' } + let(:cron_time_zone) { 'US/Pacific' } + + it 'returns next time from now' do + time = Time.now.in_time_zone(cron_time_zone) + time = time + 10.minutes + time = time.change(sec: 0, min: time.min-time.min%10) + is_expected.to eq(time) + end + end + + context 'at 10:00, 20:00' do + let(:cron) { '0 */10 * * *' } + let(:cron_time_zone) { 'US/Pacific' } + + it 'returns next time from now' do + time = Time.now.in_time_zone(cron_time_zone) + time = time + 10.hours + time = time.change(sec: 0, min: 0, hour: time.hour-time.hour%10) + is_expected.to eq(time) + end + end + + context 'when cron is every 10 days' do + let(:cron) { '0 0 */10 * *' } + let(:cron_time_zone) { 'US/Pacific' } + + it 'returns next time from now' do + time = Time.now.in_time_zone(cron_time_zone) + time = time + 10.days + time = time.change(sec: 0, min: 0, hour: 0, day: time.day-time.day%10) + is_expected.to eq(time) + end + end + + context 'when cron is every week 2:00 AM' do + let(:cron) { '0 2 * * *' } + let(:cron_time_zone) { 'US/Pacific' } + + it 'returns next time from now' do + time = Time.now.in_time_zone(cron_time_zone) + is_expected.to eq(time.change(sec: 0, min: 0, hour: 2, day: time.day+1)) + end + end + + context 'when cron_time_zone is US/Pacific' do + let(:cron) { '0 1 * * *' } + let(:cron_time_zone) { 'US/Pacific' } + + it 'returns next time from now' do + time = Time.now.in_time_zone(cron_time_zone) + is_expected.to eq(time.change(sec: 0, min: 0, hour: 1, day: time.day+1)) + end + end + + context 'when cron_time_zone is Europe/London' do + let(:cron) { '0 1 * * *' } + let(:cron_time_zone) { 'Europe/London' } + + it 'returns next time from now' do + time = Time.now.in_time_zone(cron_time_zone) + is_expected.to eq(time.change(sec: 0, min: 0, hour: 1, day: time.day+1)) + end + end + + context 'when cron_time_zone is Asia/Tokyo' do + let(:cron) { '0 1 * * *' } + let(:cron_time_zone) { 'Asia/Tokyo' } + + it 'returns next time from now' do + time = Time.now.in_time_zone(cron_time_zone) + is_expected.to eq(time.change(sec: 0, min: 0, hour: 1, day: time.day+1)) + end + end + end + + context 'when cron is given and cron_time_zone is not given' do + let(:cron) { '0 1 * * *' } + + it 'returns next time from now in utc' do + obj = described_class.new(cron).next_time_from_now + time = Time.now.in_time_zone('UTC') + expect(obj).to eq(time.change(sec: 0, min: 0, hour: 1, day: time.day+1)) + end + end + + context 'when cron and cron_time_zone are invalid' do + let(:cron) { 'hack' } + let(:cron_time_zone) { 'hack' } + + it 'returns nil' do + is_expected.to be_nil + end + end + end + + describe '#valid_syntax?' do + subject { described_class.new(cron, cron_time_zone).valid_syntax? } + + context 'when cron and cron_time_zone are valid' do + let(:cron) { '* * * * *' } + let(:cron_time_zone) { 'Europe/Istanbul' } + + it 'returns true' do + is_expected.to eq(true) + end + end + + context 'when cron and cron_time_zone are invalid' do + let(:cron) { 'hack' } + let(:cron_time_zone) { 'hack' } + + it 'returns false' do + is_expected.to eq(false) + end + end + end + end +end diff --git a/spec/models/ci/scheduled_trigger_spec.rb b/spec/models/ci/scheduled_trigger_spec.rb new file mode 100644 index 00000000000..68ba9c379b8 --- /dev/null +++ b/spec/models/ci/scheduled_trigger_spec.rb @@ -0,0 +1,38 @@ +require 'spec_helper' +require 'rufus-scheduler' # Included in sidekiq-cron + +describe Ci::ScheduledTrigger, models: true do + + describe 'associations' do + it { is_expected.to belong_to(:project) } + it { is_expected.to belong_to(:owner) } + end + + describe '#schedule_next_run!' do + context 'when cron and cron_time_zone are vaild' do + context 'when nightly build' do + it 'schedules next run' do + scheduled_trigger = create(:ci_scheduled_trigger, :cron_nightly_build) + scheduled_trigger.schedule_next_run! + puts "scheduled_trigger: #{scheduled_trigger.inspect}" + + expect(scheduled_trigger.cron).to be_nil + end + end + + context 'when weekly build' do + + end + + context 'when monthly build' do + + end + end + + context 'when cron and cron_time_zone are invaild' do + it 'schedules nothing' do + + end + end + end +end diff --git a/spec/workers/scheduled_trigger_worker_spec.rb b/spec/workers/scheduled_trigger_worker_spec.rb new file mode 100644 index 00000000000..c17536720a4 --- /dev/null +++ b/spec/workers/scheduled_trigger_worker_spec.rb @@ -0,0 +1,11 @@ +require 'spec_helper' + +describe ScheduledTriggerWorker do + subject { described_class.new.perform } + + context '#perform' do # TODO: + it 'does' do + is_expected.to be_nil + end + end +end -- cgit v1.2.1 From 37d6d1e46130f44f2fe05171b814b5682696839c Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 24 Mar 2017 00:18:13 +0900 Subject: basic components --- app/models/ci/scheduled_trigger.rb | 10 ++- app/services/ci/create_pipeline_service.rb | 10 +-- app/workers/scheduled_trigger_worker.rb | 8 +-- spec/factories/ci/scheduled_triggers.rb | 18 ++++- spec/lib/ci/cron_parser_spec.rb | 91 ++++++++---------------- spec/models/ci/scheduled_trigger_spec.rb | 31 +++----- spec/services/ci/create_pipeline_service_spec.rb | 4 ++ spec/workers/scheduled_trigger_worker_spec.rb | 54 ++++++++++++-- 8 files changed, 127 insertions(+), 99 deletions(-) diff --git a/app/models/ci/scheduled_trigger.rb b/app/models/ci/scheduled_trigger.rb index 5b1ff7bd7a4..9af274243a5 100644 --- a/app/models/ci/scheduled_trigger.rb +++ b/app/models/ci/scheduled_trigger.rb @@ -9,15 +9,13 @@ module Ci def schedule_next_run! next_time = Ci::CronParser.new(cron, cron_time_zone).next_time_from_now - update(:next_run_at => next_time) if next_time.present? - end - - def valid_ref? - true #TODO: + if next_time.present? + update_attributes(next_run_at: next_time) + end end def update_last_run! - update(:last_run_at => Time.now) + update_attributes(last_run_at: Time.now) end end end diff --git a/app/services/ci/create_pipeline_service.rb b/app/services/ci/create_pipeline_service.rb index 38a85e9fc42..6e3880e1e63 100644 --- a/app/services/ci/create_pipeline_service.rb +++ b/app/services/ci/create_pipeline_service.rb @@ -2,14 +2,14 @@ module Ci class CreatePipelineService < BaseService attr_reader :pipeline - def execute(ignore_skip_ci: false, save_on_errors: true, trigger_request: nil) + def execute(ignore_skip_ci: false, save_on_errors: true, trigger_request: nil, scheduled_trigger: false) @pipeline = Ci::Pipeline.new( project: project, ref: ref, sha: sha, before_sha: before_sha, tag: tag?, - trigger_requests: Array(trigger_request), + trigger_requests: (scheduled_trigger) ? [] : Array(trigger_request), user: current_user ) @@ -17,8 +17,10 @@ module Ci return error('Pipeline is disabled') end - unless trigger_request || can?(current_user, :create_pipeline, project) - return error('Insufficient permissions to create a new pipeline') + unless scheduled_trigger + unless trigger_request || can?(current_user, :create_pipeline, project) + return error('Insufficient permissions to create a new pipeline') + end end unless branch? || tag? diff --git a/app/workers/scheduled_trigger_worker.rb b/app/workers/scheduled_trigger_worker.rb index 7dc17aa4332..5c2f03dee79 100644 --- a/app/workers/scheduled_trigger_worker.rb +++ b/app/workers/scheduled_trigger_worker.rb @@ -3,15 +3,15 @@ class ScheduledTriggerWorker include CronjobQueue def perform - # TODO: Update next_run_at - - Ci::ScheduledTriggers.where("next_run_at < ?", Time.now).find_each do |trigger| + Ci::ScheduledTrigger.where("next_run_at < ?", Time.now).find_each do |trigger| begin - Ci::CreateTriggerRequestService.new.execute(trigger.project, trigger, trigger.ref) + Ci::CreatePipelineService.new(trigger.project, trigger.owner, ref: trigger.ref). + execute(ignore_skip_ci: true, scheduled_trigger: true) rescue => e Rails.logger.error "#{trigger.id}: Failed to trigger job: #{e.message}" ensure trigger.schedule_next_run! + trigger.update_last_run! end end end diff --git a/spec/factories/ci/scheduled_triggers.rb b/spec/factories/ci/scheduled_triggers.rb index 9d45f4b4962..c97b2d14bd1 100644 --- a/spec/factories/ci/scheduled_triggers.rb +++ b/spec/factories/ci/scheduled_triggers.rb @@ -1,42 +1,58 @@ FactoryGirl.define do factory :ci_scheduled_trigger, class: Ci::ScheduledTrigger do - project factory: :empty_project + project factory: :project owner factory: :user ref 'master' + trait :force_triggable do + next_run_at Time.now - 1.month + end + trait :cron_nightly_build do cron '0 1 * * *' cron_time_zone 'Europe/Istanbul' + next_run_at do # TODO: Use CronParser + time = Time.now.in_time_zone(cron_time_zone) + time = time + 1.day if time.hour > 1 + time = time.change(sec: 0, min: 0, hour: 1) + time + end end trait :cron_weekly_build do cron '0 1 * * 5' cron_time_zone 'Europe/Istanbul' + # TODO: next_run_at end trait :cron_monthly_build do cron '0 1 22 * *' cron_time_zone 'Europe/Istanbul' + # TODO: next_run_at end trait :cron_every_5_minutes do cron '*/5 * * * *' cron_time_zone 'Europe/Istanbul' + # TODO: next_run_at end trait :cron_every_5_hours do cron '* */5 * * *' cron_time_zone 'Europe/Istanbul' + # TODO: next_run_at end trait :cron_every_5_days do cron '* * */5 * *' cron_time_zone 'Europe/Istanbul' + # TODO: next_run_at end trait :cron_every_5_months do cron '* * * */5 *' cron_time_zone 'Europe/Istanbul' + # TODO: next_run_at end end end diff --git a/spec/lib/ci/cron_parser_spec.rb b/spec/lib/ci/cron_parser_spec.rb index 58eb26c9421..f8c7e88edb3 100644 --- a/spec/lib/ci/cron_parser_spec.rb +++ b/spec/lib/ci/cron_parser_spec.rb @@ -6,91 +6,62 @@ module Ci subject { described_class.new(cron, cron_time_zone).next_time_from_now } context 'when cron and cron_time_zone are valid' do - context 'at 00:00, 00:10, 00:20, 00:30, 00:40, 00:50' do - let(:cron) { '*/10 * * * *' } - let(:cron_time_zone) { 'US/Pacific' } + context 'when specific time' do + let(:cron) { '3 4 5 6 *' } + let(:cron_time_zone) { 'Europe/London' } - it 'returns next time from now' do - time = Time.now.in_time_zone(cron_time_zone) - time = time + 10.minutes - time = time.change(sec: 0, min: time.min-time.min%10) - is_expected.to eq(time) + it 'returns exact time in the future' do + expect(subject).to be > Time.now.in_time_zone(cron_time_zone) + expect(subject.min).to eq(3) + expect(subject.hour).to eq(4) + expect(subject.day).to eq(5) + expect(subject.month).to eq(6) end end - context 'at 10:00, 20:00' do - let(:cron) { '0 */10 * * *' } - let(:cron_time_zone) { 'US/Pacific' } + context 'when specific day of week' do + let(:cron) { '* * * * 0' } + let(:cron_time_zone) { 'Europe/London' } - it 'returns next time from now' do - time = Time.now.in_time_zone(cron_time_zone) - time = time + 10.hours - time = time.change(sec: 0, min: 0, hour: time.hour-time.hour%10) - is_expected.to eq(time) + it 'returns exact day of week in the future' do + expect(subject).to be > Time.now.in_time_zone(cron_time_zone) + expect(subject.wday).to eq(0) end end - context 'when cron is every 10 days' do - let(:cron) { '0 0 */10 * *' } + context 'when slash used' do + let(:cron) { '*/10 */6 */10 */10 *' } let(:cron_time_zone) { 'US/Pacific' } - it 'returns next time from now' do - time = Time.now.in_time_zone(cron_time_zone) - time = time + 10.days - time = time.change(sec: 0, min: 0, hour: 0, day: time.day-time.day%10) - is_expected.to eq(time) + it 'returns exact minute' do + expect(subject).to be > Time.now.in_time_zone(cron_time_zone) + expect(subject.min).to be_in([0, 10, 20, 30, 40, 50]) + expect(subject.hour).to be_in([0, 6, 12, 18]) + expect(subject.day).to be_in([1, 11, 21, 31]) + expect(subject.month).to be_in([1, 11]) end end - context 'when cron is every week 2:00 AM' do - let(:cron) { '0 2 * * *' } + context 'when range used' do + let(:cron) { '0,20,40 * 1-5 * *' } let(:cron_time_zone) { 'US/Pacific' } it 'returns next time from now' do - time = Time.now.in_time_zone(cron_time_zone) - is_expected.to eq(time.change(sec: 0, min: 0, hour: 2, day: time.day+1)) + expect(subject).to be > Time.now.in_time_zone(cron_time_zone) + expect(subject.min).to be_in([0, 20, 40]) + expect(subject.day).to be_in((1..5).to_a) end end context 'when cron_time_zone is US/Pacific' do - let(:cron) { '0 1 * * *' } + let(:cron) { '* * * * *' } let(:cron_time_zone) { 'US/Pacific' } it 'returns next time from now' do - time = Time.now.in_time_zone(cron_time_zone) - is_expected.to eq(time.change(sec: 0, min: 0, hour: 1, day: time.day+1)) - end - end - - context 'when cron_time_zone is Europe/London' do - let(:cron) { '0 1 * * *' } - let(:cron_time_zone) { 'Europe/London' } - - it 'returns next time from now' do - time = Time.now.in_time_zone(cron_time_zone) - is_expected.to eq(time.change(sec: 0, min: 0, hour: 1, day: time.day+1)) + expect(subject).to be > Time.now.in_time_zone(cron_time_zone) + expect(subject.utc_offset/60/60).to eq(-7) end end - - context 'when cron_time_zone is Asia/Tokyo' do - let(:cron) { '0 1 * * *' } - let(:cron_time_zone) { 'Asia/Tokyo' } - - it 'returns next time from now' do - time = Time.now.in_time_zone(cron_time_zone) - is_expected.to eq(time.change(sec: 0, min: 0, hour: 1, day: time.day+1)) - end - end - end - - context 'when cron is given and cron_time_zone is not given' do - let(:cron) { '0 1 * * *' } - - it 'returns next time from now in utc' do - obj = described_class.new(cron).next_time_from_now - time = Time.now.in_time_zone('UTC') - expect(obj).to eq(time.change(sec: 0, min: 0, hour: 1, day: time.day+1)) - end end context 'when cron and cron_time_zone are invalid' do diff --git a/spec/models/ci/scheduled_trigger_spec.rb b/spec/models/ci/scheduled_trigger_spec.rb index 68ba9c379b8..bb5e969fa44 100644 --- a/spec/models/ci/scheduled_trigger_spec.rb +++ b/spec/models/ci/scheduled_trigger_spec.rb @@ -1,5 +1,4 @@ require 'spec_helper' -require 'rufus-scheduler' # Included in sidekiq-cron describe Ci::ScheduledTrigger, models: true do @@ -9,30 +8,22 @@ describe Ci::ScheduledTrigger, models: true do end describe '#schedule_next_run!' do - context 'when cron and cron_time_zone are vaild' do - context 'when nightly build' do - it 'schedules next run' do - scheduled_trigger = create(:ci_scheduled_trigger, :cron_nightly_build) - scheduled_trigger.schedule_next_run! - puts "scheduled_trigger: #{scheduled_trigger.inspect}" + subject { scheduled_trigger.schedule_next_run! } - expect(scheduled_trigger.cron).to be_nil - end - end + let(:scheduled_trigger) { create(:ci_scheduled_trigger, :cron_nightly_build, next_run_at: nil) } - context 'when weekly build' do - - end - - context 'when monthly build' do - - end + it 'updates next_run_at' do + is_expected.not_to be_nil end + end + + describe '#update_last_run!' do + subject { scheduled_trigger.update_last_run! } - context 'when cron and cron_time_zone are invaild' do - it 'schedules nothing' do + let(:scheduled_trigger) { create(:ci_scheduled_trigger, :cron_nightly_build, last_run_at: nil) } - end + it 'updates last_run_at' do + is_expected.not_to be_nil end end end diff --git a/spec/services/ci/create_pipeline_service_spec.rb b/spec/services/ci/create_pipeline_service_spec.rb index d2f0337c260..4e34acc3585 100644 --- a/spec/services/ci/create_pipeline_service_spec.rb +++ b/spec/services/ci/create_pipeline_service_spec.rb @@ -214,5 +214,9 @@ describe Ci::CreatePipelineService, services: true do expect(Environment.find_by(name: "review/master")).not_to be_nil end end + + context 'when scheduled_trigger' do + # TODO: spec if approved + end end end diff --git a/spec/workers/scheduled_trigger_worker_spec.rb b/spec/workers/scheduled_trigger_worker_spec.rb index c17536720a4..ffcb27602a1 100644 --- a/spec/workers/scheduled_trigger_worker_spec.rb +++ b/spec/workers/scheduled_trigger_worker_spec.rb @@ -1,11 +1,57 @@ require 'spec_helper' describe ScheduledTriggerWorker do - subject { described_class.new.perform } + let(:worker) { described_class.new } - context '#perform' do # TODO: - it 'does' do - is_expected.to be_nil + before do + stub_ci_pipeline_to_return_yaml_file + end + + context 'when there is a scheduled trigger within next_run_at' do + before do + create(:ci_scheduled_trigger, :cron_nightly_build, :force_triggable) + worker.perform + end + + it 'creates a new pipeline' do + expect(Ci::Pipeline.last.status).to eq('pending') + end + + it 'schedules next_run_at' do + scheduled_trigger2 = create(:ci_scheduled_trigger, :cron_nightly_build) + expect(Ci::ScheduledTrigger.last.next_run_at).to eq(scheduled_trigger2.next_run_at) + end + end + + context 'when there are no scheduled triggers within next_run_at' do + let!(:scheduled_trigger) { create(:ci_scheduled_trigger, :cron_nightly_build) } + + before do + worker.perform + end + + it 'do not create a new pipeline' do + expect(Ci::Pipeline.all).to be_empty + end + + it 'do not reschedule next_run_at' do + expect(Ci::ScheduledTrigger.last.next_run_at).to eq(scheduled_trigger.next_run_at) + end + end + + context 'when next_run_at is nil' do + let!(:scheduled_trigger) { create(:ci_scheduled_trigger, :cron_nightly_build, next_run_at: nil) } + + before do + worker.perform + end + + it 'do not create a new pipeline' do + expect(Ci::Pipeline.all).to be_empty + end + + it 'do not reschedule next_run_at' do + expect(Ci::ScheduledTrigger.last.next_run_at).to eq(scheduled_trigger.next_run_at) end end end -- cgit v1.2.1 From 531af92dd3759a78959c28c5307d9e73beac1901 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 24 Mar 2017 00:35:54 +0900 Subject: Add config for worker --- config/gitlab.yml.example | 3 +++ config/initializers/1_settings.rb | 3 +++ 2 files changed, 6 insertions(+) diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index 4314e902564..892064949ce 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -180,6 +180,9 @@ production: &base # Flag stuck CI jobs as failed stuck_ci_jobs_worker: cron: "0 * * * *" + # Execute scheduled triggers + scheduled_trigger_worker: + cron: "0 * * * *" # Remove expired build artifacts expire_build_artifacts_worker: cron: "50 * * * *" diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index f7cae84088e..15e6b382eb7 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -315,6 +315,9 @@ Settings['cron_jobs'] ||= Settingslogic.new({}) Settings.cron_jobs['stuck_ci_jobs_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['stuck_ci_jobs_worker']['cron'] ||= '0 * * * *' Settings.cron_jobs['stuck_ci_jobs_worker']['job_class'] = 'StuckCiJobsWorker' +Settings.cron_jobs['scheduled_trigger_worker'] ||= Settingslogic.new({}) +Settings.cron_jobs['scheduled_trigger_worker']['cron'] ||= '0 * * * *' +Settings.cron_jobs['scheduled_trigger_worker']['job_class'] = 'ScheduledTriggerWorker' Settings.cron_jobs['expire_build_artifacts_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['expire_build_artifacts_worker']['cron'] ||= '50 * * * *' Settings.cron_jobs['expire_build_artifacts_worker']['job_class'] = 'ExpireBuildArtifactsWorker' -- cgit v1.2.1 From 8f798e81843c155db3b6661a8d30505b3fe1d098 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 29 Mar 2017 18:50:08 +0900 Subject: Rollback --- db/schema.rb | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/db/schema.rb b/db/schema.rb index a101ce280fe..7f51f4f2fdb 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -290,23 +290,6 @@ ActiveRecord::Schema.define(version: 20170405080720) do add_index "ci_runners", ["locked"], name: "index_ci_runners_on_locked", using: :btree add_index "ci_runners", ["token"], name: "index_ci_runners_on_token", using: :btree - create_table "ci_scheduled_triggers", force: :cascade do |t| - t.integer "project_id" - t.datetime "deleted_at" - t.datetime "created_at" - t.datetime "updated_at" - t.integer "owner_id" - t.string "description" - t.string "cron" - t.string "cron_time_zone" - t.datetime "next_run_at" - t.datetime "last_run_at" - t.string "ref" - end - - add_index "ci_scheduled_triggers", ["next_run_at"], name: "index_ci_scheduled_triggers_on_next_run_at", using: :btree - add_index "ci_scheduled_triggers", ["project_id"], name: "index_ci_scheduled_triggers_on_project_id", using: :btree - create_table "ci_trigger_requests", force: :cascade do |t| t.integer "trigger_id", null: false t.text "variables" @@ -1315,7 +1298,6 @@ ActiveRecord::Schema.define(version: 20170405080720) do add_foreign_key "boards", "projects" add_foreign_key "chat_teams", "namespaces", on_delete: :cascade - add_foreign_key "ci_scheduled_triggers", "users", column: "owner_id", on_delete: :cascade add_foreign_key "ci_triggers", "users", column: "owner_id", name: "fk_e8e10d1964", on_delete: :cascade add_foreign_key "issue_metrics", "issues", on_delete: :cascade add_foreign_key "label_priorities", "labels", on_delete: :cascade -- cgit v1.2.1 From b4da589ee902654225fe915c22e0859522511e66 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 29 Mar 2017 18:50:54 +0900 Subject: Remove old migration file --- .../20170322070910_create_ci_scheduled_triggers.rb | 45 ---------------------- 1 file changed, 45 deletions(-) delete mode 100644 db/migrate/20170322070910_create_ci_scheduled_triggers.rb diff --git a/db/migrate/20170322070910_create_ci_scheduled_triggers.rb b/db/migrate/20170322070910_create_ci_scheduled_triggers.rb deleted file mode 100644 index 91e4b42d2af..00000000000 --- a/db/migrate/20170322070910_create_ci_scheduled_triggers.rb +++ /dev/null @@ -1,45 +0,0 @@ -# See http://doc.gitlab.com/ce/development/migration_style_guide.html -# for more information on how to write migrations for GitLab. - -class CreateCiScheduledTriggers < ActiveRecord::Migration - include Gitlab::Database::MigrationHelpers - - # Set this constant to true if this migration requires downtime. - DOWNTIME = false - - # When a migration requires downtime you **must** uncomment the following - # constant and define a short and easy to understand explanation as to why the - # migration requires downtime. - # DOWNTIME_REASON = '' - - # When using the methods "add_concurrent_index" or "add_column_with_default" - # you must disable the use of transactions as these methods can not run in an - # existing transaction. When using "add_concurrent_index" make sure that this - # method is the _only_ method called in the migration, any other changes - # should go in a separate migration. This ensures that upon failure _only_ the - # index creation fails and can be retried or reverted easily. - # - # To disable transactions uncomment the following line and remove these - # comments: - # disable_ddl_transaction! - - def change - create_table :ci_scheduled_triggers do |t| - t.integer "project_id" - t.datetime "deleted_at" - t.datetime "created_at" - t.datetime "updated_at" - t.integer "owner_id" - t.string "description" - t.string "cron" - t.string "cron_time_zone" - t.datetime "next_run_at" - t.datetime "last_run_at" - t.string "ref" - end - - add_index :ci_scheduled_triggers, ["next_run_at"], name: "index_ci_scheduled_triggers_on_next_run_at", using: :btree - add_index :ci_scheduled_triggers, ["project_id"], name: "index_ci_scheduled_triggers_on_project_id", using: :btree - add_foreign_key :ci_scheduled_triggers, :users, column: :owner_id, on_delete: :cascade - end -end -- cgit v1.2.1 From 4c2435f58e8b46c25af64b85345eb49dc5341f5a Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 29 Mar 2017 18:54:59 +0900 Subject: Add ref to ci_triggers --- db/migrate/20170329095325_add_ref_to_triggers.rb | 29 ++++++++++++++++++++++++ db/schema.rb | 1 + 2 files changed, 30 insertions(+) create mode 100644 db/migrate/20170329095325_add_ref_to_triggers.rb diff --git a/db/migrate/20170329095325_add_ref_to_triggers.rb b/db/migrate/20170329095325_add_ref_to_triggers.rb new file mode 100644 index 00000000000..f8236b5a711 --- /dev/null +++ b/db/migrate/20170329095325_add_ref_to_triggers.rb @@ -0,0 +1,29 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class AddRefToTriggers < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + # When a migration requires downtime you **must** uncomment the following + # constant and define a short and easy to understand explanation as to why the + # migration requires downtime. + # DOWNTIME_REASON = '' + + # When using the methods "add_concurrent_index" or "add_column_with_default" + # you must disable the use of transactions as these methods can not run in an + # existing transaction. When using "add_concurrent_index" make sure that this + # method is the _only_ method called in the migration, any other changes + # should go in a separate migration. This ensures that upon failure _only_ the + # index creation fails and can be retried or reverted easily. + # + # To disable transactions uncomment the following line and remove these + # comments: + # disable_ddl_transaction! + + def change + add_column :ci_triggers, :ref, :string + end +end diff --git a/db/schema.rb b/db/schema.rb index 7f51f4f2fdb..f20f9bdfb17 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -308,6 +308,7 @@ ActiveRecord::Schema.define(version: 20170405080720) do t.integer "project_id" t.integer "owner_id" t.string "description" + t.string "ref" end add_index "ci_triggers", ["project_id"], name: "index_ci_triggers_on_project_id", using: :btree -- cgit v1.2.1 From e32c1a5c92a80c350bbf3b70552be5cf29501fe7 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 29 Mar 2017 19:07:25 +0900 Subject: Add ci_trigger_schedules table --- .../20170329095907_create_ci_trigger_schedules.rb | 42 ++++++++++++++++++++++ db/schema.rb | 15 ++++++++ 2 files changed, 57 insertions(+) create mode 100644 db/migrate/20170329095907_create_ci_trigger_schedules.rb diff --git a/db/migrate/20170329095907_create_ci_trigger_schedules.rb b/db/migrate/20170329095907_create_ci_trigger_schedules.rb new file mode 100644 index 00000000000..42f9497cac7 --- /dev/null +++ b/db/migrate/20170329095907_create_ci_trigger_schedules.rb @@ -0,0 +1,42 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class CreateCiTriggerSchedules < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + # When a migration requires downtime you **must** uncomment the following + # constant and define a short and easy to understand explanation as to why the + # migration requires downtime. + # DOWNTIME_REASON = '' + + # When using the methods "add_concurrent_index" or "add_column_with_default" + # you must disable the use of transactions as these methods can not run in an + # existing transaction. When using "add_concurrent_index" make sure that this + # method is the _only_ method called in the migration, any other changes + # should go in a separate migration. This ensures that upon failure _only_ the + # index creation fails and can be retried or reverted easily. + # + # To disable transactions uncomment the following line and remove these + # comments: + # disable_ddl_transaction! + + def change + create_table :ci_trigger_schedules do |t| + t.integer "project_id" + t.integer "trigger_id", null: false + t.datetime "deleted_at" + t.datetime "created_at" + t.datetime "updated_at" + t.string "cron" + t.string "cron_time_zone" + t.datetime "next_run_at" + end + + add_index :ci_trigger_schedules, ["next_run_at"], name: "index_ci_trigger_schedules_on_next_run_at", using: :btree + add_index :ci_trigger_schedules, ["project_id"], name: "index_ci_trigger_schedules_on_project_id", using: :btree + add_foreign_key :ci_trigger_schedules, :ci_triggers, column: :trigger_id, on_delete: :cascade + end +end diff --git a/db/schema.rb b/db/schema.rb index f20f9bdfb17..8f3b3110548 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -300,6 +300,20 @@ ActiveRecord::Schema.define(version: 20170405080720) do add_index "ci_trigger_requests", ["commit_id"], name: "index_ci_trigger_requests_on_commit_id", using: :btree + create_table "ci_trigger_schedules", force: :cascade do |t| + t.integer "project_id" + t.integer "trigger_id", null: false + t.datetime "deleted_at" + t.datetime "created_at" + t.datetime "updated_at" + t.string "cron" + t.string "cron_time_zone" + t.datetime "next_run_at" + end + + add_index "ci_trigger_schedules", ["next_run_at"], name: "index_ci_trigger_schedules_on_next_run_at", using: :btree + add_index "ci_trigger_schedules", ["project_id"], name: "index_ci_trigger_schedules_on_project_id", using: :btree + create_table "ci_triggers", force: :cascade do |t| t.string "token" t.datetime "deleted_at" @@ -1299,6 +1313,7 @@ ActiveRecord::Schema.define(version: 20170405080720) do add_foreign_key "boards", "projects" add_foreign_key "chat_teams", "namespaces", on_delete: :cascade + add_foreign_key "ci_trigger_schedules", "ci_triggers", column: "trigger_id", on_delete: :cascade add_foreign_key "ci_triggers", "users", column: "owner_id", name: "fk_e8e10d1964", on_delete: :cascade add_foreign_key "issue_metrics", "issues", on_delete: :cascade add_foreign_key "label_priorities", "labels", on_delete: :cascade -- cgit v1.2.1 From c426763c42d41c9c0c9a9cfe544f3185eeaa984f Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 29 Mar 2017 20:49:47 +0900 Subject: Rename ScheduledTrigger to TriggerSchedule. Because table structure changed. --- app/models/ci/scheduled_trigger.rb | 21 ---------- app/models/ci/trigger.rb | 1 + app/models/ci/trigger_schedule.rb | 21 ++++++++++ app/workers/scheduled_trigger_worker.rb | 18 --------- app/workers/trigger_schedule_worker.rb | 17 ++++++++ config/gitlab.yml.example | 2 +- config/initializers/1_settings.rb | 6 +-- spec/factories/ci/scheduled_triggers.rb | 5 +-- spec/models/ci/scheduled_trigger_spec.rb | 29 -------------- spec/models/ci/trigger_schedule_spec.rb | 29 ++++++++++++++ spec/models/ci/trigger_spec.rb | 1 + spec/workers/scheduled_trigger_worker_spec.rb | 57 --------------------------- spec/workers/trigger_schedule_worker_spec.rb | 57 +++++++++++++++++++++++++++ 13 files changed, 132 insertions(+), 132 deletions(-) delete mode 100644 app/models/ci/scheduled_trigger.rb create mode 100644 app/models/ci/trigger_schedule.rb delete mode 100644 app/workers/scheduled_trigger_worker.rb create mode 100644 app/workers/trigger_schedule_worker.rb delete mode 100644 spec/models/ci/scheduled_trigger_spec.rb create mode 100644 spec/models/ci/trigger_schedule_spec.rb delete mode 100644 spec/workers/scheduled_trigger_worker_spec.rb create mode 100644 spec/workers/trigger_schedule_worker_spec.rb diff --git a/app/models/ci/scheduled_trigger.rb b/app/models/ci/scheduled_trigger.rb deleted file mode 100644 index 9af274243a5..00000000000 --- a/app/models/ci/scheduled_trigger.rb +++ /dev/null @@ -1,21 +0,0 @@ -module Ci - class ScheduledTrigger < ActiveRecord::Base - extend Ci::Model - - acts_as_paranoid - - belongs_to :project - belongs_to :owner, class_name: "User" - - def schedule_next_run! - next_time = Ci::CronParser.new(cron, cron_time_zone).next_time_from_now - if next_time.present? - update_attributes(next_run_at: next_time) - end - end - - def update_last_run! - update_attributes(last_run_at: Time.now) - end - end -end diff --git a/app/models/ci/trigger.rb b/app/models/ci/trigger.rb index cba1d81a861..0a89f3e0640 100644 --- a/app/models/ci/trigger.rb +++ b/app/models/ci/trigger.rb @@ -8,6 +8,7 @@ module Ci belongs_to :owner, class_name: "User" has_many :trigger_requests, dependent: :destroy + has_one :trigger_schedule, dependent: :destroy validates :token, presence: true, uniqueness: true diff --git a/app/models/ci/trigger_schedule.rb b/app/models/ci/trigger_schedule.rb new file mode 100644 index 00000000000..7efaa228a93 --- /dev/null +++ b/app/models/ci/trigger_schedule.rb @@ -0,0 +1,21 @@ +module Ci + class TriggerSchedule < ActiveRecord::Base + extend Ci::Model + + acts_as_paranoid + + belongs_to :project + belongs_to :trigger + + def schedule_next_run! + next_time = Ci::CronParser.new(cron, cron_time_zone).next_time_from_now + if next_time.present? + update_attributes(next_run_at: next_time) + end + end + + # def update_last_run! + # update_attributes(last_run_at: Time.now) + # end + end +end diff --git a/app/workers/scheduled_trigger_worker.rb b/app/workers/scheduled_trigger_worker.rb deleted file mode 100644 index 5c2f03dee79..00000000000 --- a/app/workers/scheduled_trigger_worker.rb +++ /dev/null @@ -1,18 +0,0 @@ -class ScheduledTriggerWorker - include Sidekiq::Worker - include CronjobQueue - - def perform - Ci::ScheduledTrigger.where("next_run_at < ?", Time.now).find_each do |trigger| - begin - Ci::CreatePipelineService.new(trigger.project, trigger.owner, ref: trigger.ref). - execute(ignore_skip_ci: true, scheduled_trigger: true) - rescue => e - Rails.logger.error "#{trigger.id}: Failed to trigger job: #{e.message}" - ensure - trigger.schedule_next_run! - trigger.update_last_run! - end - end - end -end diff --git a/app/workers/trigger_schedule_worker.rb b/app/workers/trigger_schedule_worker.rb new file mode 100644 index 00000000000..d55e9378e02 --- /dev/null +++ b/app/workers/trigger_schedule_worker.rb @@ -0,0 +1,17 @@ +class TriggerScheduleWorker + include Sidekiq::Worker + include CronjobQueue + + def perform + Ci::TriggerSchedule.where("next_run_at < ?", Time.now).find_each do |trigger| + begin + Ci::CreatePipelineService.new(trigger.project, trigger.owner, ref: trigger.ref). + execute(ignore_skip_ci: true, scheduled_trigger: true) + rescue => e + Rails.logger.error "#{trigger.id}: Failed to trigger job: #{e.message}" + ensure + trigger.schedule_next_run! + end + end + end +end diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index 892064949ce..6fb67426c8b 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -181,7 +181,7 @@ production: &base stuck_ci_jobs_worker: cron: "0 * * * *" # Execute scheduled triggers - scheduled_trigger_worker: + trigger_schedule_worker: cron: "0 * * * *" # Remove expired build artifacts expire_build_artifacts_worker: diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 15e6b382eb7..71342f435f1 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -315,9 +315,9 @@ Settings['cron_jobs'] ||= Settingslogic.new({}) Settings.cron_jobs['stuck_ci_jobs_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['stuck_ci_jobs_worker']['cron'] ||= '0 * * * *' Settings.cron_jobs['stuck_ci_jobs_worker']['job_class'] = 'StuckCiJobsWorker' -Settings.cron_jobs['scheduled_trigger_worker'] ||= Settingslogic.new({}) -Settings.cron_jobs['scheduled_trigger_worker']['cron'] ||= '0 * * * *' -Settings.cron_jobs['scheduled_trigger_worker']['job_class'] = 'ScheduledTriggerWorker' +Settings.cron_jobs['trigger_schedule_worker'] ||= Settingslogic.new({}) +Settings.cron_jobs['trigger_schedule_worker']['cron'] ||= '0 * * * *' +Settings.cron_jobs['trigger_schedule_worker']['job_class'] = 'TriggerScheduleWorker' Settings.cron_jobs['expire_build_artifacts_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['expire_build_artifacts_worker']['cron'] ||= '50 * * * *' Settings.cron_jobs['expire_build_artifacts_worker']['job_class'] = 'ExpireBuildArtifactsWorker' diff --git a/spec/factories/ci/scheduled_triggers.rb b/spec/factories/ci/scheduled_triggers.rb index c97b2d14bd1..f909e343bf2 100644 --- a/spec/factories/ci/scheduled_triggers.rb +++ b/spec/factories/ci/scheduled_triggers.rb @@ -1,8 +1,7 @@ FactoryGirl.define do - factory :ci_scheduled_trigger, class: Ci::ScheduledTrigger do + factory :ci_trigger_schedule, class: Ci::TriggerSchedule do project factory: :project - owner factory: :user - ref 'master' + trigger factory: :ci_trigger trait :force_triggable do next_run_at Time.now - 1.month diff --git a/spec/models/ci/scheduled_trigger_spec.rb b/spec/models/ci/scheduled_trigger_spec.rb deleted file mode 100644 index bb5e969fa44..00000000000 --- a/spec/models/ci/scheduled_trigger_spec.rb +++ /dev/null @@ -1,29 +0,0 @@ -require 'spec_helper' - -describe Ci::ScheduledTrigger, models: true do - - describe 'associations' do - it { is_expected.to belong_to(:project) } - it { is_expected.to belong_to(:owner) } - end - - describe '#schedule_next_run!' do - subject { scheduled_trigger.schedule_next_run! } - - let(:scheduled_trigger) { create(:ci_scheduled_trigger, :cron_nightly_build, next_run_at: nil) } - - it 'updates next_run_at' do - is_expected.not_to be_nil - end - end - - describe '#update_last_run!' do - subject { scheduled_trigger.update_last_run! } - - let(:scheduled_trigger) { create(:ci_scheduled_trigger, :cron_nightly_build, last_run_at: nil) } - - it 'updates last_run_at' do - is_expected.not_to be_nil - end - end -end diff --git a/spec/models/ci/trigger_schedule_spec.rb b/spec/models/ci/trigger_schedule_spec.rb new file mode 100644 index 00000000000..14b8530a65b --- /dev/null +++ b/spec/models/ci/trigger_schedule_spec.rb @@ -0,0 +1,29 @@ +require 'spec_helper' + +describe Ci::TriggerSchedule, models: true do + + describe 'associations' do + it { is_expected.to belong_to(:project) } + it { is_expected.to belong_to(:trigger) } + end + + describe '#schedule_next_run!' do + subject { trigger_schedule.schedule_next_run! } + + let(:trigger_schedule) { create(:ci_trigger_schedule, :cron_nightly_build, next_run_at: nil) } + + it 'updates next_run_at' do + is_expected.not_to be_nil + end + end + + # describe '#update_last_run!' do + # subject { scheduled_trigger.update_last_run! } + + # let(:scheduled_trigger) { create(:ci_scheduled_trigger, :cron_nightly_build, last_run_at: nil) } + + # it 'updates last_run_at' do + # is_expected.not_to be_nil + # end + # end +end diff --git a/spec/models/ci/trigger_spec.rb b/spec/models/ci/trigger_spec.rb index 1bcb673cb16..42170de0180 100644 --- a/spec/models/ci/trigger_spec.rb +++ b/spec/models/ci/trigger_spec.rb @@ -7,6 +7,7 @@ describe Ci::Trigger, models: true do it { is_expected.to belong_to(:project) } it { is_expected.to belong_to(:owner) } it { is_expected.to have_many(:trigger_requests) } + it { is_expected.to have_one(:trigger_schedule) } end describe 'before_validation' do diff --git a/spec/workers/scheduled_trigger_worker_spec.rb b/spec/workers/scheduled_trigger_worker_spec.rb deleted file mode 100644 index ffcb27602a1..00000000000 --- a/spec/workers/scheduled_trigger_worker_spec.rb +++ /dev/null @@ -1,57 +0,0 @@ -require 'spec_helper' - -describe ScheduledTriggerWorker do - let(:worker) { described_class.new } - - before do - stub_ci_pipeline_to_return_yaml_file - end - - context 'when there is a scheduled trigger within next_run_at' do - before do - create(:ci_scheduled_trigger, :cron_nightly_build, :force_triggable) - worker.perform - end - - it 'creates a new pipeline' do - expect(Ci::Pipeline.last.status).to eq('pending') - end - - it 'schedules next_run_at' do - scheduled_trigger2 = create(:ci_scheduled_trigger, :cron_nightly_build) - expect(Ci::ScheduledTrigger.last.next_run_at).to eq(scheduled_trigger2.next_run_at) - end - end - - context 'when there are no scheduled triggers within next_run_at' do - let!(:scheduled_trigger) { create(:ci_scheduled_trigger, :cron_nightly_build) } - - before do - worker.perform - end - - it 'do not create a new pipeline' do - expect(Ci::Pipeline.all).to be_empty - end - - it 'do not reschedule next_run_at' do - expect(Ci::ScheduledTrigger.last.next_run_at).to eq(scheduled_trigger.next_run_at) - end - end - - context 'when next_run_at is nil' do - let!(:scheduled_trigger) { create(:ci_scheduled_trigger, :cron_nightly_build, next_run_at: nil) } - - before do - worker.perform - end - - it 'do not create a new pipeline' do - expect(Ci::Pipeline.all).to be_empty - end - - it 'do not reschedule next_run_at' do - expect(Ci::ScheduledTrigger.last.next_run_at).to eq(scheduled_trigger.next_run_at) - end - end -end diff --git a/spec/workers/trigger_schedule_worker_spec.rb b/spec/workers/trigger_schedule_worker_spec.rb new file mode 100644 index 00000000000..6c7521e8339 --- /dev/null +++ b/spec/workers/trigger_schedule_worker_spec.rb @@ -0,0 +1,57 @@ +require 'spec_helper' + +describe TriggerScheduleWorker do + let(:worker) { described_class.new } + + before do + stub_ci_pipeline_to_return_yaml_file + end + + context 'when there is a scheduled trigger within next_run_at' do + before do + create(:ci_trigger_schedule, :cron_nightly_build, :force_triggable) + worker.perform + end + + it 'creates a new pipeline' do + expect(Ci::Pipeline.last.status).to eq('pending') + end + + it 'schedules next_run_at' do + trigger_schedule2 = create(:ci_trigger_schedule, :cron_nightly_build) + expect(Ci::TriggerSchedule.last.next_run_at).to eq(trigger_schedule2.next_run_at) + end + end + + context 'when there are no scheduled triggers within next_run_at' do + let!(:trigger_schedule) { create(:ci_trigger_schedule, :cron_nightly_build) } + + before do + worker.perform + end + + it 'do not create a new pipeline' do + expect(Ci::Pipeline.all).to be_empty + end + + it 'do not reschedule next_run_at' do + expect(Ci::TriggerSchedule.last.next_run_at).to eq(trigger_schedule.next_run_at) + end + end + + context 'when next_run_at is nil' do + let!(:trigger_schedule) { create(:ci_trigger_schedule, :cron_nightly_build, next_run_at: nil) } + + before do + worker.perform + end + + it 'do not create a new pipeline' do + expect(Ci::Pipeline.all).to be_empty + end + + it 'do not reschedule next_run_at' do + expect(Ci::TriggerSchedule.last.next_run_at).to eq(trigger_schedule.next_run_at) + end + end +end -- cgit v1.2.1 From 3d1bc4a44cf7197d3148d829c4f527e9afbf1ea6 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 29 Mar 2017 22:14:58 +0900 Subject: Fixed strcture for db change --- app/services/ci/create_pipeline_service.rb | 10 ++--- app/workers/trigger_schedule_worker.rb | 11 ++--- spec/factories/ci/scheduled_triggers.rb | 57 ------------------------ spec/factories/ci/trigger_schedules.rb | 57 ++++++++++++++++++++++++ spec/factories/ci/triggers.rb | 2 +- spec/services/ci/create_pipeline_service_spec.rb | 4 -- spec/workers/trigger_schedule_worker_spec.rb | 14 ++++-- 7 files changed, 79 insertions(+), 76 deletions(-) delete mode 100644 spec/factories/ci/scheduled_triggers.rb create mode 100644 spec/factories/ci/trigger_schedules.rb diff --git a/app/services/ci/create_pipeline_service.rb b/app/services/ci/create_pipeline_service.rb index 6e3880e1e63..38a85e9fc42 100644 --- a/app/services/ci/create_pipeline_service.rb +++ b/app/services/ci/create_pipeline_service.rb @@ -2,14 +2,14 @@ module Ci class CreatePipelineService < BaseService attr_reader :pipeline - def execute(ignore_skip_ci: false, save_on_errors: true, trigger_request: nil, scheduled_trigger: false) + def execute(ignore_skip_ci: false, save_on_errors: true, trigger_request: nil) @pipeline = Ci::Pipeline.new( project: project, ref: ref, sha: sha, before_sha: before_sha, tag: tag?, - trigger_requests: (scheduled_trigger) ? [] : Array(trigger_request), + trigger_requests: Array(trigger_request), user: current_user ) @@ -17,10 +17,8 @@ module Ci return error('Pipeline is disabled') end - unless scheduled_trigger - unless trigger_request || can?(current_user, :create_pipeline, project) - return error('Insufficient permissions to create a new pipeline') - end + unless trigger_request || can?(current_user, :create_pipeline, project) + return error('Insufficient permissions to create a new pipeline') end unless branch? || tag? diff --git a/app/workers/trigger_schedule_worker.rb b/app/workers/trigger_schedule_worker.rb index d55e9378e02..04a53a38adb 100644 --- a/app/workers/trigger_schedule_worker.rb +++ b/app/workers/trigger_schedule_worker.rb @@ -3,14 +3,15 @@ class TriggerScheduleWorker include CronjobQueue def perform - Ci::TriggerSchedule.where("next_run_at < ?", Time.now).find_each do |trigger| + Ci::TriggerSchedule.where("next_run_at < ?", Time.now).find_each do |trigger_schedule| begin - Ci::CreatePipelineService.new(trigger.project, trigger.owner, ref: trigger.ref). - execute(ignore_skip_ci: true, scheduled_trigger: true) + Ci::CreateTriggerRequestService.new.execute(trigger_schedule.trigger.project, + trigger_schedule.trigger, + trigger_schedule.trigger.ref) rescue => e - Rails.logger.error "#{trigger.id}: Failed to trigger job: #{e.message}" + Rails.logger.error "#{trigger_schedule.id}: Failed to trigger_schedule job: #{e.message}" ensure - trigger.schedule_next_run! + trigger_schedule.schedule_next_run! end end end diff --git a/spec/factories/ci/scheduled_triggers.rb b/spec/factories/ci/scheduled_triggers.rb deleted file mode 100644 index f909e343bf2..00000000000 --- a/spec/factories/ci/scheduled_triggers.rb +++ /dev/null @@ -1,57 +0,0 @@ -FactoryGirl.define do - factory :ci_trigger_schedule, class: Ci::TriggerSchedule do - project factory: :project - trigger factory: :ci_trigger - - trait :force_triggable do - next_run_at Time.now - 1.month - end - - trait :cron_nightly_build do - cron '0 1 * * *' - cron_time_zone 'Europe/Istanbul' - next_run_at do # TODO: Use CronParser - time = Time.now.in_time_zone(cron_time_zone) - time = time + 1.day if time.hour > 1 - time = time.change(sec: 0, min: 0, hour: 1) - time - end - end - - trait :cron_weekly_build do - cron '0 1 * * 5' - cron_time_zone 'Europe/Istanbul' - # TODO: next_run_at - end - - trait :cron_monthly_build do - cron '0 1 22 * *' - cron_time_zone 'Europe/Istanbul' - # TODO: next_run_at - end - - trait :cron_every_5_minutes do - cron '*/5 * * * *' - cron_time_zone 'Europe/Istanbul' - # TODO: next_run_at - end - - trait :cron_every_5_hours do - cron '* */5 * * *' - cron_time_zone 'Europe/Istanbul' - # TODO: next_run_at - end - - trait :cron_every_5_days do - cron '* * */5 * *' - cron_time_zone 'Europe/Istanbul' - # TODO: next_run_at - end - - trait :cron_every_5_months do - cron '* * * */5 *' - cron_time_zone 'Europe/Istanbul' - # TODO: next_run_at - end - end -end diff --git a/spec/factories/ci/trigger_schedules.rb b/spec/factories/ci/trigger_schedules.rb new file mode 100644 index 00000000000..f909e343bf2 --- /dev/null +++ b/spec/factories/ci/trigger_schedules.rb @@ -0,0 +1,57 @@ +FactoryGirl.define do + factory :ci_trigger_schedule, class: Ci::TriggerSchedule do + project factory: :project + trigger factory: :ci_trigger + + trait :force_triggable do + next_run_at Time.now - 1.month + end + + trait :cron_nightly_build do + cron '0 1 * * *' + cron_time_zone 'Europe/Istanbul' + next_run_at do # TODO: Use CronParser + time = Time.now.in_time_zone(cron_time_zone) + time = time + 1.day if time.hour > 1 + time = time.change(sec: 0, min: 0, hour: 1) + time + end + end + + trait :cron_weekly_build do + cron '0 1 * * 5' + cron_time_zone 'Europe/Istanbul' + # TODO: next_run_at + end + + trait :cron_monthly_build do + cron '0 1 22 * *' + cron_time_zone 'Europe/Istanbul' + # TODO: next_run_at + end + + trait :cron_every_5_minutes do + cron '*/5 * * * *' + cron_time_zone 'Europe/Istanbul' + # TODO: next_run_at + end + + trait :cron_every_5_hours do + cron '* */5 * * *' + cron_time_zone 'Europe/Istanbul' + # TODO: next_run_at + end + + trait :cron_every_5_days do + cron '* * */5 * *' + cron_time_zone 'Europe/Istanbul' + # TODO: next_run_at + end + + trait :cron_every_5_months do + cron '* * * */5 *' + cron_time_zone 'Europe/Istanbul' + # TODO: next_run_at + end + end +end diff --git a/spec/factories/ci/triggers.rb b/spec/factories/ci/triggers.rb index a27b04424e5..1feaa9b9fa1 100644 --- a/spec/factories/ci/triggers.rb +++ b/spec/factories/ci/triggers.rb @@ -1,7 +1,7 @@ FactoryGirl.define do factory :ci_trigger_without_token, class: Ci::Trigger do factory :ci_trigger do - token 'token' + token { SecureRandom.hex(10) } end end end diff --git a/spec/services/ci/create_pipeline_service_spec.rb b/spec/services/ci/create_pipeline_service_spec.rb index 4e34acc3585..d2f0337c260 100644 --- a/spec/services/ci/create_pipeline_service_spec.rb +++ b/spec/services/ci/create_pipeline_service_spec.rb @@ -214,9 +214,5 @@ describe Ci::CreatePipelineService, services: true do expect(Environment.find_by(name: "review/master")).not_to be_nil end end - - context 'when scheduled_trigger' do - # TODO: spec if approved - end end end diff --git a/spec/workers/trigger_schedule_worker_spec.rb b/spec/workers/trigger_schedule_worker_spec.rb index 6c7521e8339..2cf51a31c71 100644 --- a/spec/workers/trigger_schedule_worker_spec.rb +++ b/spec/workers/trigger_schedule_worker_spec.rb @@ -8,18 +8,26 @@ describe TriggerScheduleWorker do end context 'when there is a scheduled trigger within next_run_at' do + let(:user) { create(:user) } + let(:project) { create(:project) } + let(:trigger) { create(:ci_trigger, owner: user, project: project, ref: 'master') } + let!(:trigger_schedule) { create(:ci_trigger_schedule, :cron_nightly_build, :force_triggable, trigger: trigger, project: project) } + before do - create(:ci_trigger_schedule, :cron_nightly_build, :force_triggable) worker.perform end + it 'creates a new trigger request' do + expect(Ci::TriggerRequest.first.trigger_id).to eq(trigger.id) + end + it 'creates a new pipeline' do expect(Ci::Pipeline.last.status).to eq('pending') end it 'schedules next_run_at' do - trigger_schedule2 = create(:ci_trigger_schedule, :cron_nightly_build) - expect(Ci::TriggerSchedule.last.next_run_at).to eq(trigger_schedule2.next_run_at) + next_time = Ci::CronParser.new('0 1 * * *', 'Europe/Istanbul').next_time_from_now + expect(Ci::TriggerSchedule.last.next_run_at).to eq(next_time) end end -- cgit v1.2.1 From 75f5e710434fbe6d709e6895c8c9328c9e92962e Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 29 Mar 2017 22:22:40 +0900 Subject: Add rufus-scheduler to Gemfile --- Gemfile | 3 +++ Gemfile.lock | 1 + lib/ci/cron_parser.rb | 2 -- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Gemfile b/Gemfile index 6a45b3d7339..b16505b3aa2 100644 --- a/Gemfile +++ b/Gemfile @@ -144,6 +144,9 @@ gem 'sidekiq-cron', '~> 0.4.4' gem 'redis-namespace', '~> 1.5.2' gem 'sidekiq-limit_fetch', '~> 3.4' +# Cron Parser +gem 'rufus-scheduler', '~> 3.1.10' + # HTTP requests gem 'httparty', '~> 0.13.3' diff --git a/Gemfile.lock b/Gemfile.lock index 50ca9af7a7a..e4603df5f7f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -987,6 +987,7 @@ DEPENDENCIES rubocop-rspec (~> 1.12.0) ruby-fogbugz (~> 0.2.1) ruby-prof (~> 0.16.2) + rufus-scheduler (~> 3.1.10) rugged (~> 0.25.1.1) sanitize (~> 2.0) sass-rails (~> 5.0.6) diff --git a/lib/ci/cron_parser.rb b/lib/ci/cron_parser.rb index 163cfc86aa7..24c4dd676dc 100644 --- a/lib/ci/cron_parser.rb +++ b/lib/ci/cron_parser.rb @@ -1,5 +1,3 @@ -require 'rufus-scheduler' # Included in sidekiq-cron - module Ci class CronParser def initialize(cron, cron_time_zone = 'UTC') -- cgit v1.2.1 From 2a1a7310d04f6d64a983d2438fdcc515e7118d91 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 30 Mar 2017 03:33:23 +0900 Subject: Add validation to Ci::TriggerSchedule (Halfway) --- app/models/ci/trigger_schedule.rb | 35 ++++++++++++++++++++++++++++---- lib/ci/cron_parser.rb | 12 ++++++----- spec/factories/ci/trigger_schedules.rb | 16 ++++++++------- spec/factories/ci/triggers.rb | 2 +- spec/models/ci/trigger_schedule_spec.rb | 36 ++++++++++++++++++++++----------- 5 files changed, 72 insertions(+), 29 deletions(-) diff --git a/app/models/ci/trigger_schedule.rb b/app/models/ci/trigger_schedule.rb index 7efaa228a93..1b3b73971bb 100644 --- a/app/models/ci/trigger_schedule.rb +++ b/app/models/ci/trigger_schedule.rb @@ -7,15 +7,42 @@ module Ci belongs_to :project belongs_to :trigger + validates :trigger, presence: true + validates :cron, presence: true + validates :cron_time_zone, presence: true + validate :check_cron + validate :check_ref + + after_create :schedule_next_run! + def schedule_next_run! next_time = Ci::CronParser.new(cron, cron_time_zone).next_time_from_now if next_time.present? - update_attributes(next_run_at: next_time) + update!(next_run_at: next_time) end end - # def update_last_run! - # update_attributes(last_run_at: Time.now) - # end + private + + def check_cron + cron_parser = Ci::CronParser.new(cron, cron_time_zone) + is_valid_cron, is_valid_cron_time_zone = cron_parser.validation + + if !is_valid_cron + self.errors.add(:cron, " is invalid syntax") + elsif !is_valid_cron_time_zone + self.errors.add(:cron_time_zone, " is invalid timezone") + elsif (cron_parser.next_time_from_now - Time.now).abs < 1.hour + self.errors.add(:cron, " can not be less than 1 hour") + end + end + + def check_ref + if !trigger.ref.present? + self.errors.add(:ref, " is empty") + elsif trigger.project.repository.ref_exists?(trigger.ref) + self.errors.add(:ref, " does not exist") + end + end end end diff --git a/lib/ci/cron_parser.rb b/lib/ci/cron_parser.rb index 24c4dd676dc..b0ffb48dabc 100644 --- a/lib/ci/cron_parser.rb +++ b/lib/ci/cron_parser.rb @@ -6,20 +6,22 @@ module Ci end def next_time_from_now - cronLine = try_parse_cron + cronLine = try_parse_cron(@cron, @cron_time_zone) return nil unless cronLine.present? cronLine.next_time end - def valid_syntax? - try_parse_cron.present? ? true : false + def validation + is_valid_cron = try_parse_cron(@cron, 'Europe/Istanbul').present? + is_valid_cron_time_zone = try_parse_cron('* * * * *', @cron_time_zone).present? + return is_valid_cron, is_valid_cron_time_zone end private - def try_parse_cron + def try_parse_cron(cron, cron_time_zone) begin - Rufus::Scheduler.parse("#{@cron} #{@cron_time_zone}") + Rufus::Scheduler.parse("#{cron} #{cron_time_zone}") rescue nil end diff --git a/spec/factories/ci/trigger_schedules.rb b/spec/factories/ci/trigger_schedules.rb index f909e343bf2..0dd397435c1 100644 --- a/spec/factories/ci/trigger_schedules.rb +++ b/spec/factories/ci/trigger_schedules.rb @@ -4,18 +4,20 @@ FactoryGirl.define do trigger factory: :ci_trigger trait :force_triggable do - next_run_at Time.now - 1.month + after(:create) do |trigger_schedule, evaluator| + trigger_schedule.next_run_at -= 1.month + end end trait :cron_nightly_build do cron '0 1 * * *' cron_time_zone 'Europe/Istanbul' - next_run_at do # TODO: Use CronParser - time = Time.now.in_time_zone(cron_time_zone) - time = time + 1.day if time.hour > 1 - time = time.change(sec: 0, min: 0, hour: 1) - time - end + # next_run_at do # TODO: Use CronParser + # time = Time.now.in_time_zone(cron_time_zone) + # time = time + 1.day if time.hour > 1 + # time = time.change(sec: 0, min: 0, hour: 1) + # time + # end end trait :cron_weekly_build do diff --git a/spec/factories/ci/triggers.rb b/spec/factories/ci/triggers.rb index 1feaa9b9fa1..d38800b58f7 100644 --- a/spec/factories/ci/triggers.rb +++ b/spec/factories/ci/triggers.rb @@ -1,7 +1,7 @@ FactoryGirl.define do factory :ci_trigger_without_token, class: Ci::Trigger do factory :ci_trigger do - token { SecureRandom.hex(10) } + token { SecureRandom.hex(15) } end end end diff --git a/spec/models/ci/trigger_schedule_spec.rb b/spec/models/ci/trigger_schedule_spec.rb index 14b8530a65b..d8b6bd93954 100644 --- a/spec/models/ci/trigger_schedule_spec.rb +++ b/spec/models/ci/trigger_schedule_spec.rb @@ -1,29 +1,41 @@ require 'spec_helper' describe Ci::TriggerSchedule, models: true do + let(:user) { create(:user) } + let(:project) { create(:project) } + let(:trigger) { create(:ci_trigger, owner: user, project: project, ref: 'master') } describe 'associations' do it { is_expected.to belong_to(:project) } it { is_expected.to belong_to(:trigger) } end - describe '#schedule_next_run!' do - subject { trigger_schedule.schedule_next_run! } + describe 'validation' do + let(:trigger_schedule) { create(:ci_trigger_schedule, :cron_nightly_build, trigger: trigger) } - let(:trigger_schedule) { create(:ci_trigger_schedule, :cron_nightly_build, next_run_at: nil) } + it { expect(trigger_schedule).to validate_presence_of(:trigger) } + it { is_expected.to validate_presence_of(:cron) } + it { is_expected.to validate_presence_of(:cron_time_zone) } - it 'updates next_run_at' do - is_expected.not_to be_nil + it '#check_cron' do + subject.cron = 'Hack' + subject.valid? + subject.errors[:screen_name].to include(' is invalid syntax') + end + + it '#check_ref' do end end - # describe '#update_last_run!' do - # subject { scheduled_trigger.update_last_run! } + describe '#schedule_next_run!' do + let(:trigger_schedule) { create(:ci_trigger_schedule, :cron_nightly_build, next_run_at: nil, trigger: trigger) } - # let(:scheduled_trigger) { create(:ci_scheduled_trigger, :cron_nightly_build, last_run_at: nil) } + before do + trigger_schedule.schedule_next_run! + end - # it 'updates last_run_at' do - # is_expected.not_to be_nil - # end - # end + it 'updates next_run_at' do + expect(Ci::TriggerSchedule.last.next_run_at).not_to be_nil + end + end end -- cgit v1.2.1 From 36ee4877788b8c78d67839b8d917d94a431b1db1 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 30 Mar 2017 20:51:05 +0900 Subject: Change configuration in gitlab.com as trigger_schedule_worker will perform twice a day --- config/gitlab.yml.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index 6fb67426c8b..3c70f35b9d0 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -182,7 +182,7 @@ production: &base cron: "0 * * * *" # Execute scheduled triggers trigger_schedule_worker: - cron: "0 * * * *" + cron: "0 */12 * * *" # Remove expired build artifacts expire_build_artifacts_worker: cron: "50 * * * *" -- cgit v1.2.1 From da8db28d17c91d5e751aaa9dbd9d756d2be9b5db Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 30 Mar 2017 20:56:10 +0900 Subject: Change configuration in gitlab.com as trigger_schedule_worker will perform twice a day 2 --- config/initializers/1_settings.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 71342f435f1..4c9d829aa9f 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -316,7 +316,7 @@ Settings.cron_jobs['stuck_ci_jobs_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['stuck_ci_jobs_worker']['cron'] ||= '0 * * * *' Settings.cron_jobs['stuck_ci_jobs_worker']['job_class'] = 'StuckCiJobsWorker' Settings.cron_jobs['trigger_schedule_worker'] ||= Settingslogic.new({}) -Settings.cron_jobs['trigger_schedule_worker']['cron'] ||= '0 * * * *' +Settings.cron_jobs['trigger_schedule_worker']['cron'] ||= '0 */12 * * *' Settings.cron_jobs['trigger_schedule_worker']['job_class'] = 'TriggerScheduleWorker' Settings.cron_jobs['expire_build_artifacts_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['expire_build_artifacts_worker']['cron'] ||= '50 * * * *' -- cgit v1.2.1 From 13bac4c252d2e84484bd0038a6c463e8b0a9cced Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 30 Mar 2017 20:59:20 +0900 Subject: Use delegate for ref on ci_trigger --- app/models/ci/trigger_schedule.rb | 2 ++ app/workers/trigger_schedule_worker.rb | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/app/models/ci/trigger_schedule.rb b/app/models/ci/trigger_schedule.rb index 1b3b73971bb..b9a99d86500 100644 --- a/app/models/ci/trigger_schedule.rb +++ b/app/models/ci/trigger_schedule.rb @@ -7,6 +7,8 @@ module Ci belongs_to :project belongs_to :trigger + delegate :ref, to: :trigger + validates :trigger, presence: true validates :cron, presence: true validates :cron_time_zone, presence: true diff --git a/app/workers/trigger_schedule_worker.rb b/app/workers/trigger_schedule_worker.rb index 04a53a38adb..440c579b99d 100644 --- a/app/workers/trigger_schedule_worker.rb +++ b/app/workers/trigger_schedule_worker.rb @@ -5,9 +5,9 @@ class TriggerScheduleWorker def perform Ci::TriggerSchedule.where("next_run_at < ?", Time.now).find_each do |trigger_schedule| begin - Ci::CreateTriggerRequestService.new.execute(trigger_schedule.trigger.project, + Ci::CreateTriggerRequestService.new.execute(trigger_schedule.project, trigger_schedule.trigger, - trigger_schedule.trigger.ref) + trigger_schedule.ref) rescue => e Rails.logger.error "#{trigger_schedule.id}: Failed to trigger_schedule job: #{e.message}" ensure -- cgit v1.2.1 From d48658e340c6d8d8b5e028afa6d5962ec7616e24 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 30 Mar 2017 21:03:25 +0900 Subject: Use constraint for #validation --- lib/ci/cron_parser.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/ci/cron_parser.rb b/lib/ci/cron_parser.rb index b0ffb48dabc..be41474198d 100644 --- a/lib/ci/cron_parser.rb +++ b/lib/ci/cron_parser.rb @@ -12,8 +12,10 @@ module Ci end def validation - is_valid_cron = try_parse_cron(@cron, 'Europe/Istanbul').present? - is_valid_cron_time_zone = try_parse_cron('* * * * *', @cron_time_zone).present? + VALID_SYNTAX_TIME_ZONE = 'Europe/Istanbul' + VALID_SYNTAX_CRON = '* * * * *' + is_valid_cron = try_parse_cron(@cron, VALID_SYNTAX_TIME_ZONE).present? + is_valid_cron_time_zone = try_parse_cron(VALID_SYNTAX_CRON, @cron_time_zone).present? return is_valid_cron, is_valid_cron_time_zone end -- cgit v1.2.1 From 9573bb44bc94261814dbdbb384b9ad7acf2907ff Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 31 Mar 2017 03:16:24 +0900 Subject: real_next_run (WIP) --- app/models/ci/trigger_schedule.rb | 19 ++++++- lib/ci/cron_parser.rb | 9 ++-- spec/factories/ci/trigger_schedules.rb | 42 ++------------- spec/factories/ci/triggers.rb | 4 ++ spec/models/ci/trigger_schedule_spec.rb | 94 +++++++++++++++++++++++++-------- 5 files changed, 102 insertions(+), 66 deletions(-) diff --git a/app/models/ci/trigger_schedule.rb b/app/models/ci/trigger_schedule.rb index b9a99d86500..0df0a03d63e 100644 --- a/app/models/ci/trigger_schedule.rb +++ b/app/models/ci/trigger_schedule.rb @@ -18,12 +18,27 @@ module Ci after_create :schedule_next_run! def schedule_next_run! + puts "cron: #{cron.inspect} | cron_time_zone: #{cron_time_zone.inspect}" next_time = Ci::CronParser.new(cron, cron_time_zone).next_time_from_now if next_time.present? update!(next_run_at: next_time) end end + def real_next_run(worker_cron: nil, worker_time_zone: nil) + puts "worker_cron: #{worker_cron.inspect} | worker_time_zone: #{worker_time_zone.inspect}" + worker_cron = Settings.cron_jobs['trigger_schedule_worker']['cron'] unless worker_cron.present? + worker_time_zone = Time.zone.name unless worker_time_zone.present? + worker_next_time = Ci::CronParser.new(worker_cron, worker_time_zone).next_time_from_now + + puts "next_run_at: #{next_run_at.inspect} | worker_next_time: #{worker_next_time.inspect}" + if next_run_at > worker_next_time + next_run_at + else + worker_next_time + end + end + private def check_cron @@ -40,9 +55,9 @@ module Ci end def check_ref - if !trigger.ref.present? + if !ref.present? self.errors.add(:ref, " is empty") - elsif trigger.project.repository.ref_exists?(trigger.ref) + elsif project.repository.ref_exists?(ref) self.errors.add(:ref, " does not exist") end end diff --git a/lib/ci/cron_parser.rb b/lib/ci/cron_parser.rb index be41474198d..a569de293b3 100644 --- a/lib/ci/cron_parser.rb +++ b/lib/ci/cron_parser.rb @@ -1,5 +1,8 @@ module Ci class CronParser + VALID_SYNTAX_SAMPLE_TIME_ZONE = 'UTC' + VALID_SYNTAX_SAMPLE_CRON = '* * * * *' + def initialize(cron, cron_time_zone = 'UTC') @cron = cron @cron_time_zone = cron_time_zone @@ -12,10 +15,8 @@ module Ci end def validation - VALID_SYNTAX_TIME_ZONE = 'Europe/Istanbul' - VALID_SYNTAX_CRON = '* * * * *' - is_valid_cron = try_parse_cron(@cron, VALID_SYNTAX_TIME_ZONE).present? - is_valid_cron_time_zone = try_parse_cron(VALID_SYNTAX_CRON, @cron_time_zone).present? + is_valid_cron = try_parse_cron(@cron, VALID_SYNTAX_SAMPLE_TIME_ZONE).present? + is_valid_cron_time_zone = try_parse_cron(VALID_SYNTAX_SAMPLE_CRON, @cron_time_zone).present? return is_valid_cron, is_valid_cron_time_zone end diff --git a/spec/factories/ci/trigger_schedules.rb b/spec/factories/ci/trigger_schedules.rb index 0dd397435c1..f572540d9e3 100644 --- a/spec/factories/ci/trigger_schedules.rb +++ b/spec/factories/ci/trigger_schedules.rb @@ -1,7 +1,7 @@ FactoryGirl.define do factory :ci_trigger_schedule, class: Ci::TriggerSchedule do project factory: :project - trigger factory: :ci_trigger + trigger factory: :ci_trigger_with_ref trait :force_triggable do after(:create) do |trigger_schedule, evaluator| @@ -11,49 +11,17 @@ FactoryGirl.define do trait :cron_nightly_build do cron '0 1 * * *' - cron_time_zone 'Europe/Istanbul' - # next_run_at do # TODO: Use CronParser - # time = Time.now.in_time_zone(cron_time_zone) - # time = time + 1.day if time.hour > 1 - # time = time.change(sec: 0, min: 0, hour: 1) - # time - # end + cron_time_zone Ci::CronParser::VALID_SYNTAX_SAMPLE_TIME_ZONE end trait :cron_weekly_build do - cron '0 1 * * 5' - cron_time_zone 'Europe/Istanbul' - # TODO: next_run_at + cron '0 1 * * 6' + cron_time_zone Ci::CronParser::VALID_SYNTAX_SAMPLE_TIME_ZONE end trait :cron_monthly_build do cron '0 1 22 * *' - cron_time_zone 'Europe/Istanbul' - # TODO: next_run_at - end - - trait :cron_every_5_minutes do - cron '*/5 * * * *' - cron_time_zone 'Europe/Istanbul' - # TODO: next_run_at - end - - trait :cron_every_5_hours do - cron '* */5 * * *' - cron_time_zone 'Europe/Istanbul' - # TODO: next_run_at - end - - trait :cron_every_5_days do - cron '* * */5 * *' - cron_time_zone 'Europe/Istanbul' - # TODO: next_run_at - end - - trait :cron_every_5_months do - cron '* * * */5 *' - cron_time_zone 'Europe/Istanbul' - # TODO: next_run_at + cron_time_zone Ci::CronParser::VALID_SYNTAX_SAMPLE_TIME_ZONE end end end diff --git a/spec/factories/ci/triggers.rb b/spec/factories/ci/triggers.rb index d38800b58f7..c9d2687b28e 100644 --- a/spec/factories/ci/triggers.rb +++ b/spec/factories/ci/triggers.rb @@ -2,6 +2,10 @@ FactoryGirl.define do factory :ci_trigger_without_token, class: Ci::Trigger do factory :ci_trigger do token { SecureRandom.hex(15) } + + factory :ci_trigger_with_ref do + ref 'master' + end end end end diff --git a/spec/models/ci/trigger_schedule_spec.rb b/spec/models/ci/trigger_schedule_spec.rb index d8b6bd93954..11e8083fc86 100644 --- a/spec/models/ci/trigger_schedule_spec.rb +++ b/spec/models/ci/trigger_schedule_spec.rb @@ -5,37 +5,85 @@ describe Ci::TriggerSchedule, models: true do let(:project) { create(:project) } let(:trigger) { create(:ci_trigger, owner: user, project: project, ref: 'master') } - describe 'associations' do - it { is_expected.to belong_to(:project) } - it { is_expected.to belong_to(:trigger) } - end + it { is_expected.to belong_to(:project) } + it { is_expected.to belong_to(:trigger) } + # it { is_expected.to validate_presence_of :cron } + # it { is_expected.to validate_presence_of :cron_time_zone } + it { is_expected.to respond_to :ref } - describe 'validation' do - let(:trigger_schedule) { create(:ci_trigger_schedule, :cron_nightly_build, trigger: trigger) } + # describe '#schedule_next_run!' do + # let(:trigger_schedule) { create(:ci_trigger_schedule, :cron_nightly_build, next_run_at: nil, trigger: trigger) } - it { expect(trigger_schedule).to validate_presence_of(:trigger) } - it { is_expected.to validate_presence_of(:cron) } - it { is_expected.to validate_presence_of(:cron_time_zone) } + # before do + # trigger_schedule.schedule_next_run! + # end - it '#check_cron' do - subject.cron = 'Hack' - subject.valid? - subject.errors[:screen_name].to include(' is invalid syntax') - end + # it 'updates next_run_at' do + # expect(Ci::TriggerSchedule.last.next_run_at).not_to be_nil + # end + # end - it '#check_ref' do - end - end + describe '#real_next_run' do + subject { trigger_schedule.real_next_run(worker_cron: worker_cron, worker_time_zone: worker_time_zone) } + + context 'when next_run_at > worker_next_time' do + let(:worker_cron) { '0 */12 * * *' } # each 00:00, 12:00 + let(:worker_time_zone) { 'UTC' } + let(:trigger_schedule) { create(:ci_trigger_schedule, :cron_weekly_build, cron_time_zone: user_time_zone, trigger: trigger) } + + context 'when user is in Europe/London(+00:00)' do + let(:user_time_zone) { 'Europe/London' } + + it 'returns next_run_at' do + is_expected.to eq(trigger_schedule.next_run_at) + end + end + + context 'when user is in Asia/Hong_Kong(+08:00)' do + let(:user_time_zone) { 'Asia/Hong_Kong' } - describe '#schedule_next_run!' do - let(:trigger_schedule) { create(:ci_trigger_schedule, :cron_nightly_build, next_run_at: nil, trigger: trigger) } + it 'returns next_run_at' do + is_expected.to eq(trigger_schedule.next_run_at) + end + end - before do - trigger_schedule.schedule_next_run! + context 'when user is in Canada/Pacific(-08:00)' do + let(:user_time_zone) { 'Canada/Pacific' } + + it 'returns next_run_at' do + is_expected.to eq(trigger_schedule.next_run_at) + end + end end - it 'updates next_run_at' do - expect(Ci::TriggerSchedule.last.next_run_at).not_to be_nil + context 'when worker_next_time > next_run_at' do + let(:worker_cron) { '0 0 */2 * *' } # every 2 days + let(:worker_time_zone) { 'UTC' } + let(:trigger_schedule) { create(:ci_trigger_schedule, :cron_nightly_build, cron_time_zone: user_time_zone, trigger: trigger) } + + context 'when user is in Europe/London(+00:00)' do + let(:user_time_zone) { 'Europe/London' } + + it 'returns worker_next_time' do + is_expected.to eq(Ci::CronParser.new(worker_cron, worker_time_zone).next_time_from_now) + end + end + + context 'when user is in Asia/Hong_Kong(+08:00)' do + let(:user_time_zone) { 'Asia/Hong_Kong' } + + it 'returns worker_next_time' do + is_expected.to eq(Ci::CronParser.new(worker_cron, worker_time_zone).next_time_from_now) + end + end + + context 'when user is in Canada/Pacific(-08:00)' do + let(:user_time_zone) { 'Canada/Pacific' } + + it 'returns worker_next_time' do + is_expected.to eq(Ci::CronParser.new(worker_cron, worker_time_zone).next_time_from_now) + end + end end end end -- cgit v1.2.1 From d65c816ed78910eabd7ecbc9282e85d6b6f21796 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 31 Mar 2017 19:08:39 +0900 Subject: Brush up --- app/models/ci/trigger_schedule.rb | 21 ++++++++++----- app/workers/trigger_schedule_worker.rb | 3 +++ lib/ci/cron_parser.rb | 9 ++++--- spec/factories/ci/trigger_schedules.rb | 9 ++++--- spec/factories/ci/triggers.rb | 4 ++- spec/lib/ci/cron_parser_spec.rb | 38 +++++++++++++--------------- spec/models/ci/trigger_schedule_spec.rb | 31 ++++++++++++++--------- spec/workers/trigger_schedule_worker_spec.rb | 24 ++++++++---------- 8 files changed, 80 insertions(+), 59 deletions(-) diff --git a/app/models/ci/trigger_schedule.rb b/app/models/ci/trigger_schedule.rb index 0df0a03d63e..fc144d3958d 100644 --- a/app/models/ci/trigger_schedule.rb +++ b/app/models/ci/trigger_schedule.rb @@ -18,20 +18,22 @@ module Ci after_create :schedule_next_run! def schedule_next_run! - puts "cron: #{cron.inspect} | cron_time_zone: #{cron_time_zone.inspect}" - next_time = Ci::CronParser.new(cron, cron_time_zone).next_time_from_now - if next_time.present? + # puts "cron: #{cron.inspect} | cron_time_zone: #{cron_time_zone.inspect}" + next_time = Ci::CronParser.new(cron, cron_time_zone).next_time_from(Time.now) + + if next_time.present? && !less_than_1_hour_from_now?(next_time) update!(next_run_at: next_time) end end def real_next_run(worker_cron: nil, worker_time_zone: nil) - puts "worker_cron: #{worker_cron.inspect} | worker_time_zone: #{worker_time_zone.inspect}" worker_cron = Settings.cron_jobs['trigger_schedule_worker']['cron'] unless worker_cron.present? worker_time_zone = Time.zone.name unless worker_time_zone.present? - worker_next_time = Ci::CronParser.new(worker_cron, worker_time_zone).next_time_from_now + # puts "real_next_run: worker_cron: #{worker_cron.inspect} | worker_time_zone: #{worker_time_zone.inspect}" + + worker_next_time = Ci::CronParser.new(worker_cron, worker_time_zone).next_time_from(Time.now) - puts "next_run_at: #{next_run_at.inspect} | worker_next_time: #{worker_next_time.inspect}" + # puts "real_next_run: next_run_at: #{next_run_at.inspect} | worker_next_time: #{worker_next_time.inspect}" if next_run_at > worker_next_time next_run_at else @@ -41,15 +43,20 @@ module Ci private + def less_than_1_hour_from_now?(time) + ((time - Time.now).abs < 1.hour) ? true : false + end + def check_cron cron_parser = Ci::CronParser.new(cron, cron_time_zone) is_valid_cron, is_valid_cron_time_zone = cron_parser.validation + next_time = cron_parser.next_time_from(Time.now) if !is_valid_cron self.errors.add(:cron, " is invalid syntax") elsif !is_valid_cron_time_zone self.errors.add(:cron_time_zone, " is invalid timezone") - elsif (cron_parser.next_time_from_now - Time.now).abs < 1.hour + elsif less_than_1_hour_from_now?(next_time) self.errors.add(:cron, " can not be less than 1 hour") end end diff --git a/app/workers/trigger_schedule_worker.rb b/app/workers/trigger_schedule_worker.rb index 440c579b99d..9216103b8da 100644 --- a/app/workers/trigger_schedule_worker.rb +++ b/app/workers/trigger_schedule_worker.rb @@ -4,11 +4,14 @@ class TriggerScheduleWorker def perform Ci::TriggerSchedule.where("next_run_at < ?", Time.now).find_each do |trigger_schedule| + next if Ci::Pipeline.where(project: trigger_schedule.project, ref: trigger_schedule.ref).running_or_pending.count > 0 + begin Ci::CreateTriggerRequestService.new.execute(trigger_schedule.project, trigger_schedule.trigger, trigger_schedule.ref) rescue => e + puts "#{trigger_schedule.id}: Failed to trigger_schedule job: #{e.message}" # TODO: Remove before merge Rails.logger.error "#{trigger_schedule.id}: Failed to trigger_schedule job: #{e.message}" ensure trigger_schedule.schedule_next_run! diff --git a/lib/ci/cron_parser.rb b/lib/ci/cron_parser.rb index a569de293b3..2919543bbef 100644 --- a/lib/ci/cron_parser.rb +++ b/lib/ci/cron_parser.rb @@ -8,10 +8,13 @@ module Ci @cron_time_zone = cron_time_zone end - def next_time_from_now + def next_time_from(time) cronLine = try_parse_cron(@cron, @cron_time_zone) - return nil unless cronLine.present? - cronLine.next_time + if cronLine.present? + cronLine.next_time(time) + else + nil + end end def validation diff --git a/spec/factories/ci/trigger_schedules.rb b/spec/factories/ci/trigger_schedules.rb index f572540d9e3..7143db6961c 100644 --- a/spec/factories/ci/trigger_schedules.rb +++ b/spec/factories/ci/trigger_schedules.rb @@ -1,11 +1,14 @@ FactoryGirl.define do factory :ci_trigger_schedule, class: Ci::TriggerSchedule do - project factory: :project - trigger factory: :ci_trigger_with_ref + trigger factory: :ci_trigger_for_trigger_schedule + + after(:build) do |trigger_schedule, evaluator| + trigger_schedule.update!(project: trigger_schedule.trigger.project) + end trait :force_triggable do after(:create) do |trigger_schedule, evaluator| - trigger_schedule.next_run_at -= 1.month + trigger_schedule.update!(next_run_at: trigger_schedule.next_run_at - 1.year) end end diff --git a/spec/factories/ci/triggers.rb b/spec/factories/ci/triggers.rb index c9d2687b28e..c9fedf8a857 100644 --- a/spec/factories/ci/triggers.rb +++ b/spec/factories/ci/triggers.rb @@ -3,7 +3,9 @@ FactoryGirl.define do factory :ci_trigger do token { SecureRandom.hex(15) } - factory :ci_trigger_with_ref do + factory :ci_trigger_for_trigger_schedule do + owner factory: :user + project factory: :project ref 'master' end end diff --git a/spec/lib/ci/cron_parser_spec.rb b/spec/lib/ci/cron_parser_spec.rb index f8c7e88edb3..dd1449b5b02 100644 --- a/spec/lib/ci/cron_parser_spec.rb +++ b/spec/lib/ci/cron_parser_spec.rb @@ -2,8 +2,8 @@ require 'spec_helper' module Ci describe CronParser, lib: true do - describe '#next_time_from_now' do - subject { described_class.new(cron, cron_time_zone).next_time_from_now } + describe '#next_time_from' do + subject { described_class.new(cron, cron_time_zone).next_time_from(Time.now) } context 'when cron and cron_time_zone are valid' do context 'when specific time' do @@ -65,8 +65,8 @@ module Ci end context 'when cron and cron_time_zone are invalid' do - let(:cron) { 'hack' } - let(:cron_time_zone) { 'hack' } + let(:cron) { 'invalid_cron' } + let(:cron_time_zone) { 'invalid_cron_time_zone' } it 'returns nil' do is_expected.to be_nil @@ -74,25 +74,23 @@ module Ci end end - describe '#valid_syntax?' do - subject { described_class.new(cron, cron_time_zone).valid_syntax? } - - context 'when cron and cron_time_zone are valid' do - let(:cron) { '* * * * *' } - let(:cron_time_zone) { 'Europe/Istanbul' } - - it 'returns true' do - is_expected.to eq(true) - end + describe '#validation' do + it 'returns results' do + is_valid_cron, is_valid_cron_time_zone = described_class.new('* * * * *', 'Europe/Istanbul').validation + expect(is_valid_cron).to eq(true) + expect(is_valid_cron_time_zone).to eq(true) end - context 'when cron and cron_time_zone are invalid' do - let(:cron) { 'hack' } - let(:cron_time_zone) { 'hack' } + it 'returns results' do + is_valid_cron, is_valid_cron_time_zone = described_class.new('*********', 'Europe/Istanbul').validation + expect(is_valid_cron).to eq(false) + expect(is_valid_cron_time_zone).to eq(true) + end - it 'returns false' do - is_expected.to eq(false) - end + it 'returns results' do + is_valid_cron, is_valid_cron_time_zone = described_class.new('* * * * *', 'Invalid-zone').validation + expect(is_valid_cron).to eq(true) + expect(is_valid_cron_time_zone).to eq(false) end end end diff --git a/spec/models/ci/trigger_schedule_spec.rb b/spec/models/ci/trigger_schedule_spec.rb index 11e8083fc86..d47ab529bc0 100644 --- a/spec/models/ci/trigger_schedule_spec.rb +++ b/spec/models/ci/trigger_schedule_spec.rb @@ -1,9 +1,6 @@ require 'spec_helper' describe Ci::TriggerSchedule, models: true do - let(:user) { create(:user) } - let(:project) { create(:project) } - let(:trigger) { create(:ci_trigger, owner: user, project: project, ref: 'master') } it { is_expected.to belong_to(:project) } it { is_expected.to belong_to(:trigger) } @@ -11,17 +8,27 @@ describe Ci::TriggerSchedule, models: true do # it { is_expected.to validate_presence_of :cron_time_zone } it { is_expected.to respond_to :ref } - # describe '#schedule_next_run!' do - # let(:trigger_schedule) { create(:ci_trigger_schedule, :cron_nightly_build, next_run_at: nil, trigger: trigger) } + it 'should validate less_than_1_hour_from_now' do + trigger_schedule = create(:ci_trigger_schedule, :cron_nightly_build) + trigger_schedule.cron = '* * * * *' + trigger_schedule.valid? + expect(trigger_schedule.errors[:cron].first).to include('can not be less than 1 hour') + end + + describe '#schedule_next_run!' do + context 'when more_than_1_hour_from_now' do + let(:trigger_schedule) { create(:ci_trigger_schedule, :cron_nightly_build) } - # before do - # trigger_schedule.schedule_next_run! - # end + before do + trigger_schedule.schedule_next_run! + end - # it 'updates next_run_at' do - # expect(Ci::TriggerSchedule.last.next_run_at).not_to be_nil - # end - # end + it 'updates next_run_at' do + next_time = Ci::CronParser.new(trigger_schedule.cron, trigger_schedule.cron_time_zone).next_time_from(Time.now) + expect(Ci::TriggerSchedule.last.next_run_at).to eq(next_time) + end + end + end describe '#real_next_run' do subject { trigger_schedule.real_next_run(worker_cron: worker_cron, worker_time_zone: worker_time_zone) } diff --git a/spec/workers/trigger_schedule_worker_spec.rb b/spec/workers/trigger_schedule_worker_spec.rb index 2cf51a31c71..f0c7eeaedae 100644 --- a/spec/workers/trigger_schedule_worker_spec.rb +++ b/spec/workers/trigger_schedule_worker_spec.rb @@ -8,38 +8,36 @@ describe TriggerScheduleWorker do end context 'when there is a scheduled trigger within next_run_at' do - let(:user) { create(:user) } - let(:project) { create(:project) } - let(:trigger) { create(:ci_trigger, owner: user, project: project, ref: 'master') } - let!(:trigger_schedule) { create(:ci_trigger_schedule, :cron_nightly_build, :force_triggable, trigger: trigger, project: project) } + let!(:trigger_schedule) { create(:ci_trigger_schedule, :cron_nightly_build, :force_triggable) } before do worker.perform end it 'creates a new trigger request' do - expect(Ci::TriggerRequest.first.trigger_id).to eq(trigger.id) + expect(trigger_schedule.trigger.id).to eq(Ci::TriggerRequest.first.trigger_id) end it 'creates a new pipeline' do expect(Ci::Pipeline.last.status).to eq('pending') end - it 'schedules next_run_at' do - next_time = Ci::CronParser.new('0 1 * * *', 'Europe/Istanbul').next_time_from_now + it 'updates next_run_at' do + next_time = Ci::CronParser.new(trigger_schedule.cron, trigger_schedule.cron_time_zone).next_time_from(Time.now) expect(Ci::TriggerSchedule.last.next_run_at).to eq(next_time) end end - context 'when there are no scheduled triggers within next_run_at' do - let!(:trigger_schedule) { create(:ci_trigger_schedule, :cron_nightly_build) } + context 'when there is a scheduled trigger within next_run_at and a runnign pipeline' do + let!(:trigger_schedule) { create(:ci_trigger_schedule, :cron_nightly_build, :force_triggable) } before do + create(:ci_pipeline, project: trigger_schedule.project, ref: trigger_schedule.ref, status: 'running') worker.perform end it 'do not create a new pipeline' do - expect(Ci::Pipeline.all).to be_empty + expect(Ci::Pipeline.count).to eq(1) end it 'do not reschedule next_run_at' do @@ -47,15 +45,15 @@ describe TriggerScheduleWorker do end end - context 'when next_run_at is nil' do - let!(:trigger_schedule) { create(:ci_trigger_schedule, :cron_nightly_build, next_run_at: nil) } + context 'when there are no scheduled triggers within next_run_at' do + let!(:trigger_schedule) { create(:ci_trigger_schedule, :cron_nightly_build) } before do worker.perform end it 'do not create a new pipeline' do - expect(Ci::Pipeline.all).to be_empty + expect(Ci::Pipeline.count).to eq(0) end it 'do not reschedule next_run_at' do -- cgit v1.2.1 From 5720919cd0cd457aa83fa3e3c36e34867b0eed60 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Fri, 31 Mar 2017 23:18:07 +0900 Subject: Brush up 2 --- app/models/ci/trigger_schedule.rb | 5 +- app/workers/trigger_schedule_worker.rb | 2 - lib/ci/cron_parser.rb | 2 +- spec/lib/ci/cron_parser_spec.rb | 25 +++++---- spec/models/ci/trigger_schedule_spec.rb | 93 ++++++++++++++------------------- 5 files changed, 55 insertions(+), 72 deletions(-) diff --git a/app/models/ci/trigger_schedule.rb b/app/models/ci/trigger_schedule.rb index fc144d3958d..b861f41fed1 100644 --- a/app/models/ci/trigger_schedule.rb +++ b/app/models/ci/trigger_schedule.rb @@ -18,7 +18,6 @@ module Ci after_create :schedule_next_run! def schedule_next_run! - # puts "cron: #{cron.inspect} | cron_time_zone: #{cron_time_zone.inspect}" next_time = Ci::CronParser.new(cron, cron_time_zone).next_time_from(Time.now) if next_time.present? && !less_than_1_hour_from_now?(next_time) @@ -29,11 +28,9 @@ module Ci def real_next_run(worker_cron: nil, worker_time_zone: nil) worker_cron = Settings.cron_jobs['trigger_schedule_worker']['cron'] unless worker_cron.present? worker_time_zone = Time.zone.name unless worker_time_zone.present? - # puts "real_next_run: worker_cron: #{worker_cron.inspect} | worker_time_zone: #{worker_time_zone.inspect}" worker_next_time = Ci::CronParser.new(worker_cron, worker_time_zone).next_time_from(Time.now) - # puts "real_next_run: next_run_at: #{next_run_at.inspect} | worker_next_time: #{worker_next_time.inspect}" if next_run_at > worker_next_time next_run_at else @@ -64,7 +61,7 @@ module Ci def check_ref if !ref.present? self.errors.add(:ref, " is empty") - elsif project.repository.ref_exists?(ref) + elsif !project.repository.branch_exists?(ref) self.errors.add(:ref, " does not exist") end end diff --git a/app/workers/trigger_schedule_worker.rb b/app/workers/trigger_schedule_worker.rb index 9216103b8da..36e640592a7 100644 --- a/app/workers/trigger_schedule_worker.rb +++ b/app/workers/trigger_schedule_worker.rb @@ -4,8 +4,6 @@ class TriggerScheduleWorker def perform Ci::TriggerSchedule.where("next_run_at < ?", Time.now).find_each do |trigger_schedule| - next if Ci::Pipeline.where(project: trigger_schedule.project, ref: trigger_schedule.ref).running_or_pending.count > 0 - begin Ci::CreateTriggerRequestService.new.execute(trigger_schedule.project, trigger_schedule.trigger, diff --git a/lib/ci/cron_parser.rb b/lib/ci/cron_parser.rb index 2919543bbef..eb348306436 100644 --- a/lib/ci/cron_parser.rb +++ b/lib/ci/cron_parser.rb @@ -11,7 +11,7 @@ module Ci def next_time_from(time) cronLine = try_parse_cron(@cron, @cron_time_zone) if cronLine.present? - cronLine.next_time(time) + cronLine.next_time(time).in_time_zone(Time.zone) else nil end diff --git a/spec/lib/ci/cron_parser_spec.rb b/spec/lib/ci/cron_parser_spec.rb index dd1449b5b02..11d1e1c8a78 100644 --- a/spec/lib/ci/cron_parser_spec.rb +++ b/spec/lib/ci/cron_parser_spec.rb @@ -8,10 +8,10 @@ module Ci context 'when cron and cron_time_zone are valid' do context 'when specific time' do let(:cron) { '3 4 5 6 *' } - let(:cron_time_zone) { 'Europe/London' } + let(:cron_time_zone) { 'UTC' } it 'returns exact time in the future' do - expect(subject).to be > Time.now.in_time_zone(cron_time_zone) + expect(subject).to be > Time.now expect(subject.min).to eq(3) expect(subject.hour).to eq(4) expect(subject.day).to eq(5) @@ -21,20 +21,20 @@ module Ci context 'when specific day of week' do let(:cron) { '* * * * 0' } - let(:cron_time_zone) { 'Europe/London' } + let(:cron_time_zone) { 'UTC' } it 'returns exact day of week in the future' do - expect(subject).to be > Time.now.in_time_zone(cron_time_zone) + expect(subject).to be > Time.now expect(subject.wday).to eq(0) end end context 'when slash used' do let(:cron) { '*/10 */6 */10 */10 *' } - let(:cron_time_zone) { 'US/Pacific' } + let(:cron_time_zone) { 'UTC' } it 'returns exact minute' do - expect(subject).to be > Time.now.in_time_zone(cron_time_zone) + expect(subject).to be > Time.now expect(subject.min).to be_in([0, 10, 20, 30, 40, 50]) expect(subject.hour).to be_in([0, 6, 12, 18]) expect(subject.day).to be_in([1, 11, 21, 31]) @@ -44,22 +44,25 @@ module Ci context 'when range used' do let(:cron) { '0,20,40 * 1-5 * *' } - let(:cron_time_zone) { 'US/Pacific' } + let(:cron_time_zone) { 'UTC' } it 'returns next time from now' do - expect(subject).to be > Time.now.in_time_zone(cron_time_zone) + expect(subject).to be > Time.now expect(subject.min).to be_in([0, 20, 40]) expect(subject.day).to be_in((1..5).to_a) end end context 'when cron_time_zone is US/Pacific' do - let(:cron) { '* * * * *' } + let(:cron) { '0 0 * * *' } let(:cron_time_zone) { 'US/Pacific' } it 'returns next time from now' do - expect(subject).to be > Time.now.in_time_zone(cron_time_zone) - expect(subject.utc_offset/60/60).to eq(-7) + expect(subject).to be > Time.now + end + + it 'converts time in server time zone' do + expect(subject.hour).to eq(7) end end end diff --git a/spec/models/ci/trigger_schedule_spec.rb b/spec/models/ci/trigger_schedule_spec.rb index d47ab529bc0..da23611f1a0 100644 --- a/spec/models/ci/trigger_schedule_spec.rb +++ b/spec/models/ci/trigger_schedule_spec.rb @@ -8,11 +8,36 @@ describe Ci::TriggerSchedule, models: true do # it { is_expected.to validate_presence_of :cron_time_zone } it { is_expected.to respond_to :ref } - it 'should validate less_than_1_hour_from_now' do + it 'should validate ref existence' do trigger_schedule = create(:ci_trigger_schedule, :cron_nightly_build) - trigger_schedule.cron = '* * * * *' + trigger_schedule.trigger.ref = 'invalid-ref' trigger_schedule.valid? - expect(trigger_schedule.errors[:cron].first).to include('can not be less than 1 hour') + expect(trigger_schedule.errors[:ref].first).to include('does not exist') + end + + describe 'cron limitation' do + let(:trigger_schedule) { create(:ci_trigger_schedule, :cron_nightly_build) } + + before do + trigger_schedule.cron = cron + trigger_schedule.valid? + end + + context 'when every hour' do + let(:cron) { '0 * * * *' } # 00:00, 01:00, 02:00, ..., 23:00 + + it 'fails' do + expect(trigger_schedule.errors[:cron].first).to include('can not be less than 1 hour') + end + end + + context 'when each six hours' do + let(:cron) { '0 */6 * * *' } # 00:00, 06:00, 12:00, 18:00 + + it 'succeeds' do + expect(trigger_schedule.errors[:cron]).to be_empty + end + end end describe '#schedule_next_run!' do @@ -31,65 +56,25 @@ describe Ci::TriggerSchedule, models: true do end describe '#real_next_run' do - subject { trigger_schedule.real_next_run(worker_cron: worker_cron, worker_time_zone: worker_time_zone) } - - context 'when next_run_at > worker_next_time' do - let(:worker_cron) { '0 */12 * * *' } # each 00:00, 12:00 - let(:worker_time_zone) { 'UTC' } - let(:trigger_schedule) { create(:ci_trigger_schedule, :cron_weekly_build, cron_time_zone: user_time_zone, trigger: trigger) } + let(:trigger_schedule) { create(:ci_trigger_schedule, cron: user_cron, cron_time_zone: 'UTC') } - context 'when user is in Europe/London(+00:00)' do - let(:user_time_zone) { 'Europe/London' } + subject { trigger_schedule.real_next_run(worker_cron: worker_cron, worker_time_zone: 'UTC') } - it 'returns next_run_at' do - is_expected.to eq(trigger_schedule.next_run_at) - end - end - - context 'when user is in Asia/Hong_Kong(+08:00)' do - let(:user_time_zone) { 'Asia/Hong_Kong' } - - it 'returns next_run_at' do - is_expected.to eq(trigger_schedule.next_run_at) - end - end - - context 'when user is in Canada/Pacific(-08:00)' do - let(:user_time_zone) { 'Canada/Pacific' } + context 'when next_run_at > worker_next_time' do + let(:worker_cron) { '* * * * *' } # every minutes + let(:user_cron) { '0 0 1 1 *' } # every 00:00, January 1st - it 'returns next_run_at' do - is_expected.to eq(trigger_schedule.next_run_at) - end + it 'returns next_run_at' do + is_expected.to eq(trigger_schedule.next_run_at) end end context 'when worker_next_time > next_run_at' do - let(:worker_cron) { '0 0 */2 * *' } # every 2 days - let(:worker_time_zone) { 'UTC' } - let(:trigger_schedule) { create(:ci_trigger_schedule, :cron_nightly_build, cron_time_zone: user_time_zone, trigger: trigger) } - - context 'when user is in Europe/London(+00:00)' do - let(:user_time_zone) { 'Europe/London' } - - it 'returns worker_next_time' do - is_expected.to eq(Ci::CronParser.new(worker_cron, worker_time_zone).next_time_from_now) - end - end - - context 'when user is in Asia/Hong_Kong(+08:00)' do - let(:user_time_zone) { 'Asia/Hong_Kong' } - - it 'returns worker_next_time' do - is_expected.to eq(Ci::CronParser.new(worker_cron, worker_time_zone).next_time_from_now) - end - end - - context 'when user is in Canada/Pacific(-08:00)' do - let(:user_time_zone) { 'Canada/Pacific' } + let(:worker_cron) { '0 0 1 1 *' } # every 00:00, January 1st + let(:user_cron) { '0 */6 * * *' } # each six hours - it 'returns worker_next_time' do - is_expected.to eq(Ci::CronParser.new(worker_cron, worker_time_zone).next_time_from_now) - end + it 'returns worker_next_time' do + is_expected.to eq(Ci::CronParser.new(worker_cron, 'UTC').next_time_from(Time.now)) end end end -- cgit v1.2.1 From 21cabf381b55ab2747d773ae1eeb70d2bb40e9a5 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sat, 1 Apr 2017 00:19:46 +0900 Subject: Move real_next_run to helper --- app/helpers/triggers_helper.rb | 13 +++++++++++++ app/models/ci/trigger_schedule.rb | 13 ------------- spec/helpers/triggers_helper_spec.rb | 27 +++++++++++++++++++++++++++ spec/models/ci/trigger_schedule_spec.rb | 24 ------------------------ 4 files changed, 40 insertions(+), 37 deletions(-) create mode 100644 spec/helpers/triggers_helper_spec.rb diff --git a/app/helpers/triggers_helper.rb b/app/helpers/triggers_helper.rb index a48d4475e97..be5cce9aea0 100644 --- a/app/helpers/triggers_helper.rb +++ b/app/helpers/triggers_helper.rb @@ -10,4 +10,17 @@ module TriggersHelper def service_trigger_url(service) "#{Settings.gitlab.url}/api/v3/projects/#{service.project_id}/services/#{service.to_param}/trigger" end + + def real_next_run(trigger_schedule, worker_cron: nil, worker_time_zone: nil) + worker_cron = Settings.cron_jobs['trigger_schedule_worker']['cron'] unless worker_cron.present? + worker_time_zone = Time.zone.name unless worker_time_zone.present? + + worker_next_time = Ci::CronParser.new(worker_cron, worker_time_zone).next_time_from(Time.now) + + if trigger_schedule.next_run_at > worker_next_time + trigger_schedule.next_run_at + else + worker_next_time + end + end end diff --git a/app/models/ci/trigger_schedule.rb b/app/models/ci/trigger_schedule.rb index b861f41fed1..aedba8bdf54 100644 --- a/app/models/ci/trigger_schedule.rb +++ b/app/models/ci/trigger_schedule.rb @@ -25,19 +25,6 @@ module Ci end end - def real_next_run(worker_cron: nil, worker_time_zone: nil) - worker_cron = Settings.cron_jobs['trigger_schedule_worker']['cron'] unless worker_cron.present? - worker_time_zone = Time.zone.name unless worker_time_zone.present? - - worker_next_time = Ci::CronParser.new(worker_cron, worker_time_zone).next_time_from(Time.now) - - if next_run_at > worker_next_time - next_run_at - else - worker_next_time - end - end - private def less_than_1_hour_from_now?(time) diff --git a/spec/helpers/triggers_helper_spec.rb b/spec/helpers/triggers_helper_spec.rb new file mode 100644 index 00000000000..ce17f4442ab --- /dev/null +++ b/spec/helpers/triggers_helper_spec.rb @@ -0,0 +1,27 @@ +require 'rails_helper' + +describe TriggersHelper do + describe '#real_next_run' do + let(:trigger_schedule) { create(:ci_trigger_schedule, cron: user_cron, cron_time_zone: 'UTC') } + + subject { helper.real_next_run(trigger_schedule, worker_cron: worker_cron, worker_time_zone: 'UTC') } + + context 'when next_run_at > worker_next_time' do + let(:worker_cron) { '* * * * *' } # every minutes + let(:user_cron) { '0 0 1 1 *' } # every 00:00, January 1st + + it 'returns next_run_at' do + is_expected.to eq(trigger_schedule.next_run_at) + end + end + + context 'when worker_next_time > next_run_at' do + let(:worker_cron) { '0 0 1 1 *' } # every 00:00, January 1st + let(:user_cron) { '0 */6 * * *' } # each six hours + + it 'returns worker_next_time' do + is_expected.to eq(Ci::CronParser.new(worker_cron, 'UTC').next_time_from(Time.now)) + end + end + end +end diff --git a/spec/models/ci/trigger_schedule_spec.rb b/spec/models/ci/trigger_schedule_spec.rb index da23611f1a0..6a586a4e9b1 100644 --- a/spec/models/ci/trigger_schedule_spec.rb +++ b/spec/models/ci/trigger_schedule_spec.rb @@ -54,28 +54,4 @@ describe Ci::TriggerSchedule, models: true do end end end - - describe '#real_next_run' do - let(:trigger_schedule) { create(:ci_trigger_schedule, cron: user_cron, cron_time_zone: 'UTC') } - - subject { trigger_schedule.real_next_run(worker_cron: worker_cron, worker_time_zone: 'UTC') } - - context 'when next_run_at > worker_next_time' do - let(:worker_cron) { '* * * * *' } # every minutes - let(:user_cron) { '0 0 1 1 *' } # every 00:00, January 1st - - it 'returns next_run_at' do - is_expected.to eq(trigger_schedule.next_run_at) - end - end - - context 'when worker_next_time > next_run_at' do - let(:worker_cron) { '0 0 1 1 *' } # every 00:00, January 1st - let(:user_cron) { '0 */6 * * *' } # each six hours - - it 'returns worker_next_time' do - is_expected.to eq(Ci::CronParser.new(worker_cron, 'UTC').next_time_from(Time.now)) - end - end - end end -- cgit v1.2.1 From 57d082f3589060c90c2841dd52dda77574f5d984 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sat, 1 Apr 2017 02:02:26 +0900 Subject: Add validator --- app/models/ci/trigger_schedule.rb | 26 ++++++-------------------- app/validators/cron_validator.rb | 16 ++++++++++++++++ app/validators/ref_validator.rb | 10 ++++++++++ spec/models/ci/trigger_schedule_spec.rb | 6 ++---- spec/workers/trigger_schedule_worker_spec.rb | 17 ----------------- 5 files changed, 34 insertions(+), 41 deletions(-) create mode 100644 app/validators/cron_validator.rb create mode 100644 app/validators/ref_validator.rb diff --git a/app/models/ci/trigger_schedule.rb b/app/models/ci/trigger_schedule.rb index aedba8bdf54..6529e364fe8 100644 --- a/app/models/ci/trigger_schedule.rb +++ b/app/models/ci/trigger_schedule.rb @@ -10,10 +10,10 @@ module Ci delegate :ref, to: :trigger validates :trigger, presence: true - validates :cron, presence: true + validates :cron, cron: true, presence: true validates :cron_time_zone, presence: true - validate :check_cron - validate :check_ref + validates :ref, ref: true, presence: true + validate :check_cron_frequency after_create :schedule_next_run! @@ -31,26 +31,12 @@ module Ci ((time - Time.now).abs < 1.hour) ? true : false end - def check_cron - cron_parser = Ci::CronParser.new(cron, cron_time_zone) - is_valid_cron, is_valid_cron_time_zone = cron_parser.validation - next_time = cron_parser.next_time_from(Time.now) + def check_cron_frequency + next_time = Ci::CronParser.new(cron, cron_time_zone).next_time_from(Time.now) - if !is_valid_cron - self.errors.add(:cron, " is invalid syntax") - elsif !is_valid_cron_time_zone - self.errors.add(:cron_time_zone, " is invalid timezone") - elsif less_than_1_hour_from_now?(next_time) + if less_than_1_hour_from_now?(next_time) self.errors.add(:cron, " can not be less than 1 hour") end end - - def check_ref - if !ref.present? - self.errors.add(:ref, " is empty") - elsif !project.repository.branch_exists?(ref) - self.errors.add(:ref, " does not exist") - end - end end end diff --git a/app/validators/cron_validator.rb b/app/validators/cron_validator.rb new file mode 100644 index 00000000000..ad70e0897ba --- /dev/null +++ b/app/validators/cron_validator.rb @@ -0,0 +1,16 @@ +# CronValidator +# +# Custom validator for Cron. +class CronValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + cron_parser = Ci::CronParser.new(record.cron, record.cron_time_zone) + is_valid_cron, is_valid_cron_time_zone = cron_parser.validation + next_time = cron_parser.next_time_from(Time.now) + + if !is_valid_cron + record.errors.add(:cron, " is invalid syntax") + elsif !is_valid_cron_time_zone + record.errors.add(:cron_time_zone, " is invalid timezone") + end + end +end diff --git a/app/validators/ref_validator.rb b/app/validators/ref_validator.rb new file mode 100644 index 00000000000..2024255a770 --- /dev/null +++ b/app/validators/ref_validator.rb @@ -0,0 +1,10 @@ +# RefValidator +# +# Custom validator for Ref. +class RefValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + unless record.project.repository.branch_exists?(value) + record.errors.add(attribute, " does not exist") + end + end +end diff --git a/spec/models/ci/trigger_schedule_spec.rb b/spec/models/ci/trigger_schedule_spec.rb index 6a586a4e9b1..30972f2295e 100644 --- a/spec/models/ci/trigger_schedule_spec.rb +++ b/spec/models/ci/trigger_schedule_spec.rb @@ -4,8 +4,6 @@ describe Ci::TriggerSchedule, models: true do it { is_expected.to belong_to(:project) } it { is_expected.to belong_to(:trigger) } - # it { is_expected.to validate_presence_of :cron } - # it { is_expected.to validate_presence_of :cron_time_zone } it { is_expected.to respond_to :ref } it 'should validate ref existence' do @@ -26,7 +24,7 @@ describe Ci::TriggerSchedule, models: true do context 'when every hour' do let(:cron) { '0 * * * *' } # 00:00, 01:00, 02:00, ..., 23:00 - it 'fails' do + it 'gets an error' do expect(trigger_schedule.errors[:cron].first).to include('can not be less than 1 hour') end end @@ -34,7 +32,7 @@ describe Ci::TriggerSchedule, models: true do context 'when each six hours' do let(:cron) { '0 */6 * * *' } # 00:00, 06:00, 12:00, 18:00 - it 'succeeds' do + it 'gets no errors' do expect(trigger_schedule.errors[:cron]).to be_empty end end diff --git a/spec/workers/trigger_schedule_worker_spec.rb b/spec/workers/trigger_schedule_worker_spec.rb index f0c7eeaedae..950f72a68d9 100644 --- a/spec/workers/trigger_schedule_worker_spec.rb +++ b/spec/workers/trigger_schedule_worker_spec.rb @@ -28,23 +28,6 @@ describe TriggerScheduleWorker do end end - context 'when there is a scheduled trigger within next_run_at and a runnign pipeline' do - let!(:trigger_schedule) { create(:ci_trigger_schedule, :cron_nightly_build, :force_triggable) } - - before do - create(:ci_pipeline, project: trigger_schedule.project, ref: trigger_schedule.ref, status: 'running') - worker.perform - end - - it 'do not create a new pipeline' do - expect(Ci::Pipeline.count).to eq(1) - end - - it 'do not reschedule next_run_at' do - expect(Ci::TriggerSchedule.last.next_run_at).to eq(trigger_schedule.next_run_at) - end - end - context 'when there are no scheduled triggers within next_run_at' do let!(:trigger_schedule) { create(:ci_trigger_schedule, :cron_nightly_build) } -- cgit v1.2.1 From d9574c0cce97d859ca605d70374633283c93f3fa Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sat, 1 Apr 2017 02:31:58 +0900 Subject: Maintain MR --- app/workers/trigger_schedule_worker.rb | 1 - ...-on-a-schedule-idea1-basic-backend-implementation.yml | 4 ++++ db/migrate/20170329095325_add_ref_to_triggers.rb | 16 ---------------- db/migrate/20170329095907_create_ci_trigger_schedules.rb | 16 ---------------- spec/helpers/triggers_helper_spec.rb | 8 ++++---- 5 files changed, 8 insertions(+), 37 deletions(-) create mode 100644 changelogs/unreleased/2989-run-cicd-pipelines-on-a-schedule-idea1-basic-backend-implementation.yml diff --git a/app/workers/trigger_schedule_worker.rb b/app/workers/trigger_schedule_worker.rb index 36e640592a7..440c579b99d 100644 --- a/app/workers/trigger_schedule_worker.rb +++ b/app/workers/trigger_schedule_worker.rb @@ -9,7 +9,6 @@ class TriggerScheduleWorker trigger_schedule.trigger, trigger_schedule.ref) rescue => e - puts "#{trigger_schedule.id}: Failed to trigger_schedule job: #{e.message}" # TODO: Remove before merge Rails.logger.error "#{trigger_schedule.id}: Failed to trigger_schedule job: #{e.message}" ensure trigger_schedule.schedule_next_run! diff --git a/changelogs/unreleased/2989-run-cicd-pipelines-on-a-schedule-idea1-basic-backend-implementation.yml b/changelogs/unreleased/2989-run-cicd-pipelines-on-a-schedule-idea1-basic-backend-implementation.yml new file mode 100644 index 00000000000..dd56409c35b --- /dev/null +++ b/changelogs/unreleased/2989-run-cicd-pipelines-on-a-schedule-idea1-basic-backend-implementation.yml @@ -0,0 +1,4 @@ +--- +title: Resolve "Run CI/CD pipelines on a schedule" - "Basic backend implementation" +merge_request: 10133 +author: dosuken123 diff --git a/db/migrate/20170329095325_add_ref_to_triggers.rb b/db/migrate/20170329095325_add_ref_to_triggers.rb index f8236b5a711..6900ded4277 100644 --- a/db/migrate/20170329095325_add_ref_to_triggers.rb +++ b/db/migrate/20170329095325_add_ref_to_triggers.rb @@ -7,22 +7,6 @@ class AddRefToTriggers < ActiveRecord::Migration # Set this constant to true if this migration requires downtime. DOWNTIME = false - # When a migration requires downtime you **must** uncomment the following - # constant and define a short and easy to understand explanation as to why the - # migration requires downtime. - # DOWNTIME_REASON = '' - - # When using the methods "add_concurrent_index" or "add_column_with_default" - # you must disable the use of transactions as these methods can not run in an - # existing transaction. When using "add_concurrent_index" make sure that this - # method is the _only_ method called in the migration, any other changes - # should go in a separate migration. This ensures that upon failure _only_ the - # index creation fails and can be retried or reverted easily. - # - # To disable transactions uncomment the following line and remove these - # comments: - # disable_ddl_transaction! - def change add_column :ci_triggers, :ref, :string end diff --git a/db/migrate/20170329095907_create_ci_trigger_schedules.rb b/db/migrate/20170329095907_create_ci_trigger_schedules.rb index 42f9497cac7..7b2e2e2098b 100644 --- a/db/migrate/20170329095907_create_ci_trigger_schedules.rb +++ b/db/migrate/20170329095907_create_ci_trigger_schedules.rb @@ -7,22 +7,6 @@ class CreateCiTriggerSchedules < ActiveRecord::Migration # Set this constant to true if this migration requires downtime. DOWNTIME = false - # When a migration requires downtime you **must** uncomment the following - # constant and define a short and easy to understand explanation as to why the - # migration requires downtime. - # DOWNTIME_REASON = '' - - # When using the methods "add_concurrent_index" or "add_column_with_default" - # you must disable the use of transactions as these methods can not run in an - # existing transaction. When using "add_concurrent_index" make sure that this - # method is the _only_ method called in the migration, any other changes - # should go in a separate migration. This ensures that upon failure _only_ the - # index creation fails and can be retried or reverted easily. - # - # To disable transactions uncomment the following line and remove these - # comments: - # disable_ddl_transaction! - def change create_table :ci_trigger_schedules do |t| t.integer "project_id" diff --git a/spec/helpers/triggers_helper_spec.rb b/spec/helpers/triggers_helper_spec.rb index ce17f4442ab..d801760335b 100644 --- a/spec/helpers/triggers_helper_spec.rb +++ b/spec/helpers/triggers_helper_spec.rb @@ -7,8 +7,8 @@ describe TriggersHelper do subject { helper.real_next_run(trigger_schedule, worker_cron: worker_cron, worker_time_zone: 'UTC') } context 'when next_run_at > worker_next_time' do - let(:worker_cron) { '* * * * *' } # every minutes - let(:user_cron) { '0 0 1 1 *' } # every 00:00, January 1st + let(:worker_cron) { '0 0 1 1 *' } # every 00:00, January 1st + let(:user_cron) { '1 0 1 1 *' } # every 00:01, January 1st it 'returns next_run_at' do is_expected.to eq(trigger_schedule.next_run_at) @@ -16,8 +16,8 @@ describe TriggersHelper do end context 'when worker_next_time > next_run_at' do - let(:worker_cron) { '0 0 1 1 *' } # every 00:00, January 1st - let(:user_cron) { '0 */6 * * *' } # each six hours + let(:worker_cron) { '1 0 1 1 *' } # every 00:01, January 1st + let(:user_cron) { '0 0 1 1 *' } # every 00:00, January 1st it 'returns worker_next_time' do is_expected.to eq(Ci::CronParser.new(worker_cron, 'UTC').next_time_from(Time.now)) -- cgit v1.2.1 From 62480461c943b4ca4c72830c04932cd5bba9f4e7 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sat, 1 Apr 2017 02:55:55 +0900 Subject: Fixed failed tests --- spec/models/ci/trigger_schedule_spec.rb | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/spec/models/ci/trigger_schedule_spec.rb b/spec/models/ci/trigger_schedule_spec.rb index 30972f2295e..1d6d602ebda 100644 --- a/spec/models/ci/trigger_schedule_spec.rb +++ b/spec/models/ci/trigger_schedule_spec.rb @@ -21,7 +21,7 @@ describe Ci::TriggerSchedule, models: true do trigger_schedule.valid? end - context 'when every hour' do + context 'when cron frequency is too short' do let(:cron) { '0 * * * *' } # 00:00, 01:00, 02:00, ..., 23:00 it 'gets an error' do @@ -29,8 +29,8 @@ describe Ci::TriggerSchedule, models: true do end end - context 'when each six hours' do - let(:cron) { '0 */6 * * *' } # 00:00, 06:00, 12:00, 18:00 + context 'when cron frequency is eligible' do + let(:cron) { '0 0 1 1 *' } # every 00:00, January 1st it 'gets no errors' do expect(trigger_schedule.errors[:cron]).to be_empty @@ -39,17 +39,15 @@ describe Ci::TriggerSchedule, models: true do end describe '#schedule_next_run!' do - context 'when more_than_1_hour_from_now' do - let(:trigger_schedule) { create(:ci_trigger_schedule, :cron_nightly_build) } + let(:trigger_schedule) { create(:ci_trigger_schedule, :cron_nightly_build) } - before do - trigger_schedule.schedule_next_run! - end + before do + trigger_schedule.schedule_next_run! + end - it 'updates next_run_at' do - next_time = Ci::CronParser.new(trigger_schedule.cron, trigger_schedule.cron_time_zone).next_time_from(Time.now) - expect(Ci::TriggerSchedule.last.next_run_at).to eq(next_time) - end + it 'updates next_run_at' do + next_time = Ci::CronParser.new(trigger_schedule.cron, trigger_schedule.cron_time_zone).next_time_from(Time.now) + expect(Ci::TriggerSchedule.last.next_run_at).to eq(next_time) end end end -- cgit v1.2.1 From 934e949726adf4428a03970d78e23555cc1d7a72 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sat, 1 Apr 2017 17:57:52 +0900 Subject: Fix rubocop issues. Use add_concurrent_foreign_key. --- app/validators/cron_validator.rb | 1 - db/migrate/20170329095325_add_ref_to_triggers.rb | 4 ---- db/migrate/20170329095907_create_ci_trigger_schedules.rb | 15 +++++++++------ db/schema.rb | 2 +- lib/ci/cron_parser.rb | 10 +++++----- spec/models/ci/trigger_schedule_spec.rb | 1 - 6 files changed, 15 insertions(+), 18 deletions(-) diff --git a/app/validators/cron_validator.rb b/app/validators/cron_validator.rb index ad70e0897ba..4d9a0d62a4c 100644 --- a/app/validators/cron_validator.rb +++ b/app/validators/cron_validator.rb @@ -5,7 +5,6 @@ class CronValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) cron_parser = Ci::CronParser.new(record.cron, record.cron_time_zone) is_valid_cron, is_valid_cron_time_zone = cron_parser.validation - next_time = cron_parser.next_time_from(Time.now) if !is_valid_cron record.errors.add(:cron, " is invalid syntax") diff --git a/db/migrate/20170329095325_add_ref_to_triggers.rb b/db/migrate/20170329095325_add_ref_to_triggers.rb index 6900ded4277..4aa52dd8f8f 100644 --- a/db/migrate/20170329095325_add_ref_to_triggers.rb +++ b/db/migrate/20170329095325_add_ref_to_triggers.rb @@ -1,10 +1,6 @@ -# See http://doc.gitlab.com/ce/development/migration_style_guide.html -# for more information on how to write migrations for GitLab. - class AddRefToTriggers < ActiveRecord::Migration include Gitlab::Database::MigrationHelpers - # Set this constant to true if this migration requires downtime. DOWNTIME = false def change diff --git a/db/migrate/20170329095907_create_ci_trigger_schedules.rb b/db/migrate/20170329095907_create_ci_trigger_schedules.rb index 7b2e2e2098b..3dcd05175c0 100644 --- a/db/migrate/20170329095907_create_ci_trigger_schedules.rb +++ b/db/migrate/20170329095907_create_ci_trigger_schedules.rb @@ -1,13 +1,11 @@ -# See http://doc.gitlab.com/ce/development/migration_style_guide.html -# for more information on how to write migrations for GitLab. - class CreateCiTriggerSchedules < ActiveRecord::Migration include Gitlab::Database::MigrationHelpers - # Set this constant to true if this migration requires downtime. DOWNTIME = false - def change + disable_ddl_transaction! + + def up create_table :ci_trigger_schedules do |t| t.integer "project_id" t.integer "trigger_id", null: false @@ -21,6 +19,11 @@ class CreateCiTriggerSchedules < ActiveRecord::Migration add_index :ci_trigger_schedules, ["next_run_at"], name: "index_ci_trigger_schedules_on_next_run_at", using: :btree add_index :ci_trigger_schedules, ["project_id"], name: "index_ci_trigger_schedules_on_project_id", using: :btree - add_foreign_key :ci_trigger_schedules, :ci_triggers, column: :trigger_id, on_delete: :cascade + add_concurrent_foreign_key :ci_trigger_schedules, :ci_triggers, column: :trigger_id, on_delete: :cascade + end + + def down + remove_foreign_key :ci_trigger_schedules, column: :trigger_id + drop_table :ci_trigger_schedules end end diff --git a/db/schema.rb b/db/schema.rb index 8f3b3110548..7d9f969c2e1 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1313,7 +1313,7 @@ ActiveRecord::Schema.define(version: 20170405080720) do add_foreign_key "boards", "projects" add_foreign_key "chat_teams", "namespaces", on_delete: :cascade - add_foreign_key "ci_trigger_schedules", "ci_triggers", column: "trigger_id", on_delete: :cascade + add_foreign_key "ci_trigger_schedules", "ci_triggers", column: "trigger_id", name: "fk_90a406cc94", on_delete: :cascade add_foreign_key "ci_triggers", "users", column: "owner_id", name: "fk_e8e10d1964", on_delete: :cascade add_foreign_key "issue_metrics", "issues", on_delete: :cascade add_foreign_key "label_priorities", "labels", on_delete: :cascade diff --git a/lib/ci/cron_parser.rb b/lib/ci/cron_parser.rb index eb348306436..e0d589956a8 100644 --- a/lib/ci/cron_parser.rb +++ b/lib/ci/cron_parser.rb @@ -1,7 +1,7 @@ module Ci class CronParser - VALID_SYNTAX_SAMPLE_TIME_ZONE = 'UTC' - VALID_SYNTAX_SAMPLE_CRON = '* * * * *' + VALID_SYNTAX_SAMPLE_TIME_ZONE = 'UTC'.freeze + VALID_SYNTAX_SAMPLE_CRON = '* * * * *'.freeze def initialize(cron, cron_time_zone = 'UTC') @cron = cron @@ -9,9 +9,9 @@ module Ci end def next_time_from(time) - cronLine = try_parse_cron(@cron, @cron_time_zone) - if cronLine.present? - cronLine.next_time(time).in_time_zone(Time.zone) + cron_line = try_parse_cron(@cron, @cron_time_zone) + if cron_line.present? + cron_line.next_time(time).in_time_zone(Time.zone) else nil end diff --git a/spec/models/ci/trigger_schedule_spec.rb b/spec/models/ci/trigger_schedule_spec.rb index 1d6d602ebda..57ebcdfb3f1 100644 --- a/spec/models/ci/trigger_schedule_spec.rb +++ b/spec/models/ci/trigger_schedule_spec.rb @@ -1,7 +1,6 @@ require 'spec_helper' describe Ci::TriggerSchedule, models: true do - it { is_expected.to belong_to(:project) } it { is_expected.to belong_to(:trigger) } it { is_expected.to respond_to :ref } -- cgit v1.2.1 From b5f252bdf53365b02bea4415b0b8581ad59d0587 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sat, 1 Apr 2017 18:02:46 +0900 Subject: Fix spec/factories_spec.rb --- spec/factories/ci/trigger_schedules.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spec/factories/ci/trigger_schedules.rb b/spec/factories/ci/trigger_schedules.rb index 7143db6961c..2e6a35c6416 100644 --- a/spec/factories/ci/trigger_schedules.rb +++ b/spec/factories/ci/trigger_schedules.rb @@ -1,6 +1,8 @@ FactoryGirl.define do factory :ci_trigger_schedule, class: Ci::TriggerSchedule do trigger factory: :ci_trigger_for_trigger_schedule + cron '0 1 * * *' + cron_time_zone Ci::CronParser::VALID_SYNTAX_SAMPLE_TIME_ZONE after(:build) do |trigger_schedule, evaluator| trigger_schedule.update!(project: trigger_schedule.trigger.project) -- cgit v1.2.1 From 01cea0d59dd52ab6db2c7fb19faa2b8c71cf6052 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sat, 1 Apr 2017 20:27:37 +0900 Subject: Suppress Import/Export function warnings. Add comment for confirmation. --- lib/gitlab/import_export/import_export.yml | 2 +- spec/lib/gitlab/import_export/all_models.yml | 2 ++ spec/lib/gitlab/import_export/safe_model_attributes.yml | 10 ++++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index f69288f7d67..8d74418ed14 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -39,7 +39,7 @@ project_tree: - :author - :events - :statuses - - :triggers + - :triggers # TODO: Need to confirm - :deploy_keys - :services - :hooks diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index 24654bf6afd..06e8cd5a1cd 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -99,6 +99,7 @@ triggers: - project - trigger_requests - owner +- trigger_schedule deploy_keys: - user - deploy_keys_projects @@ -194,6 +195,7 @@ project: - runners - variables - triggers +- trigger_schedules - environments - deployments - project_feature diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml index 1ad16a9b57d..ecd8f2990c6 100644 --- a/spec/lib/gitlab/import_export/safe_model_attributes.yml +++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml @@ -240,6 +240,16 @@ Ci::Trigger: - updated_at - owner_id - description +- ref +Ci::TriggerSchedule: +- id +- project_id +- trigger_id +- deleted_at +- created_at +- updated_at +- cron +- next_run_at DeployKey: - id - user_id -- cgit v1.2.1 From a2c4a2a26621539787712c62d6b58e79de7f4fb8 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sat, 1 Apr 2017 20:30:35 +0900 Subject: Resolve error on trigger.spec --- spec/factories/ci/triggers.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spec/factories/ci/triggers.rb b/spec/factories/ci/triggers.rb index c9fedf8a857..2295455575d 100644 --- a/spec/factories/ci/triggers.rb +++ b/spec/factories/ci/triggers.rb @@ -1,9 +1,10 @@ FactoryGirl.define do factory :ci_trigger_without_token, class: Ci::Trigger do factory :ci_trigger do - token { SecureRandom.hex(15) } + token 'token' factory :ci_trigger_for_trigger_schedule do + token { SecureRandom.hex(15) } owner factory: :user project factory: :project ref 'master' -- cgit v1.2.1 From 1bd5494942853decbcd67434ef592ca2523a777b Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sat, 1 Apr 2017 21:18:07 +0900 Subject: Improve cron_parser_spec --- spec/lib/ci/cron_parser_spec.rb | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/spec/lib/ci/cron_parser_spec.rb b/spec/lib/ci/cron_parser_spec.rb index 11d1e1c8a78..fb4a5ec4d13 100644 --- a/spec/lib/ci/cron_parser_spec.rb +++ b/spec/lib/ci/cron_parser_spec.rb @@ -2,6 +2,10 @@ require 'spec_helper' module Ci describe CronParser, lib: true do + shared_examples_for "returns time in the future" do + it { is_expected.to be > Time.now } + end + describe '#next_time_from' do subject { described_class.new(cron, cron_time_zone).next_time_from(Time.now) } @@ -10,8 +14,9 @@ module Ci let(:cron) { '3 4 5 6 *' } let(:cron_time_zone) { 'UTC' } - it 'returns exact time in the future' do - expect(subject).to be > Time.now + it_behaves_like "returns time in the future" + + it 'returns exact time' do expect(subject.min).to eq(3) expect(subject.hour).to eq(4) expect(subject.day).to eq(5) @@ -23,8 +28,9 @@ module Ci let(:cron) { '* * * * 0' } let(:cron_time_zone) { 'UTC' } - it 'returns exact day of week in the future' do - expect(subject).to be > Time.now + it_behaves_like "returns time in the future" + + it 'returns exact day of week' do expect(subject.wday).to eq(0) end end @@ -33,8 +39,9 @@ module Ci let(:cron) { '*/10 */6 */10 */10 *' } let(:cron_time_zone) { 'UTC' } - it 'returns exact minute' do - expect(subject).to be > Time.now + it_behaves_like "returns time in the future" + + it 'returns specific time' do expect(subject.min).to be_in([0, 10, 20, 30, 40, 50]) expect(subject.hour).to be_in([0, 6, 12, 18]) expect(subject.day).to be_in([1, 11, 21, 31]) @@ -46,8 +53,9 @@ module Ci let(:cron) { '0,20,40 * 1-5 * *' } let(:cron_time_zone) { 'UTC' } - it 'returns next time from now' do - expect(subject).to be > Time.now + it_behaves_like "returns time in the future" + + it 'returns specific time' do expect(subject.min).to be_in([0, 20, 40]) expect(subject.day).to be_in((1..5).to_a) end @@ -57,9 +65,7 @@ module Ci let(:cron) { '0 0 * * *' } let(:cron_time_zone) { 'US/Pacific' } - it 'returns next time from now' do - expect(subject).to be > Time.now - end + it_behaves_like "returns time in the future" it 'converts time in server time zone' do expect(subject.hour).to eq(7) -- cgit v1.2.1 From 97cc6777368bfe171198af383bf715629e9b076f Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Sat, 1 Apr 2017 23:27:24 +0900 Subject: Commentout check_cron_frequency --- app/models/ci/trigger_schedule.rb | 26 ++++++++++++++------------ spec/models/ci/trigger_schedule_spec.rb | 27 +-------------------------- 2 files changed, 15 insertions(+), 38 deletions(-) diff --git a/app/models/ci/trigger_schedule.rb b/app/models/ci/trigger_schedule.rb index 6529e364fe8..be547af2114 100644 --- a/app/models/ci/trigger_schedule.rb +++ b/app/models/ci/trigger_schedule.rb @@ -13,30 +13,32 @@ module Ci validates :cron, cron: true, presence: true validates :cron_time_zone, presence: true validates :ref, ref: true, presence: true - validate :check_cron_frequency + # validate :check_cron_frequency after_create :schedule_next_run! def schedule_next_run! next_time = Ci::CronParser.new(cron, cron_time_zone).next_time_from(Time.now) - if next_time.present? && !less_than_1_hour_from_now?(next_time) + # if next_time.present? && !less_than_1_hour_from_now?(next_time) + if next_time.present? update!(next_run_at: next_time) end end - private + # private - def less_than_1_hour_from_now?(time) - ((time - Time.now).abs < 1.hour) ? true : false - end + # def less_than_1_hour_from_now?(time) + # puts "diff: #{(time - Time.now).abs.inspect}" + # ((time - Time.now).abs < 1.hour) ? true : false + # end - def check_cron_frequency - next_time = Ci::CronParser.new(cron, cron_time_zone).next_time_from(Time.now) + # def check_cron_frequency + # next_time = Ci::CronParser.new(cron, cron_time_zone).next_time_from(Time.now) - if less_than_1_hour_from_now?(next_time) - self.errors.add(:cron, " can not be less than 1 hour") - end - end + # if less_than_1_hour_from_now?(next_time) + # self.errors.add(:cron, " can not be less than 1 hour") + # end + # end end end diff --git a/spec/models/ci/trigger_schedule_spec.rb b/spec/models/ci/trigger_schedule_spec.rb index 57ebcdfb3f1..8b27ca1c8b2 100644 --- a/spec/models/ci/trigger_schedule_spec.rb +++ b/spec/models/ci/trigger_schedule_spec.rb @@ -12,33 +12,8 @@ describe Ci::TriggerSchedule, models: true do expect(trigger_schedule.errors[:ref].first).to include('does not exist') end - describe 'cron limitation' do - let(:trigger_schedule) { create(:ci_trigger_schedule, :cron_nightly_build) } - - before do - trigger_schedule.cron = cron - trigger_schedule.valid? - end - - context 'when cron frequency is too short' do - let(:cron) { '0 * * * *' } # 00:00, 01:00, 02:00, ..., 23:00 - - it 'gets an error' do - expect(trigger_schedule.errors[:cron].first).to include('can not be less than 1 hour') - end - end - - context 'when cron frequency is eligible' do - let(:cron) { '0 0 1 1 *' } # every 00:00, January 1st - - it 'gets no errors' do - expect(trigger_schedule.errors[:cron]).to be_empty - end - end - end - describe '#schedule_next_run!' do - let(:trigger_schedule) { create(:ci_trigger_schedule, :cron_nightly_build) } + let(:trigger_schedule) { create(:ci_trigger_schedule, :cron_nightly_build, next_run_at: nil) } before do trigger_schedule.schedule_next_run! -- cgit v1.2.1 From a67aff6d398467099121e7a7b4a542ff531d3f45 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 4 Apr 2017 16:55:14 +0900 Subject: Add Import/Export Setting for trigger_schedule. Remove ref validation. --- app/models/ci/trigger_schedule.rb | 9 +++++---- app/validators/ref_validator.rb | 10 ---------- lib/gitlab/import_export/import_export.yml | 3 ++- lib/gitlab/import_export/relation_factory.rb | 1 + spec/lib/gitlab/import_export/all_models.yml | 2 ++ spec/lib/gitlab/import_export/safe_model_attributes.yml | 1 + spec/models/ci/trigger_schedule_spec.rb | 7 ------- 7 files changed, 11 insertions(+), 22 deletions(-) delete mode 100644 app/validators/ref_validator.rb diff --git a/app/models/ci/trigger_schedule.rb b/app/models/ci/trigger_schedule.rb index be547af2114..58337b34d80 100644 --- a/app/models/ci/trigger_schedule.rb +++ b/app/models/ci/trigger_schedule.rb @@ -1,6 +1,7 @@ module Ci class TriggerSchedule < ActiveRecord::Base extend Ci::Model + include Importable acts_as_paranoid @@ -9,10 +10,10 @@ module Ci delegate :ref, to: :trigger - validates :trigger, presence: true - validates :cron, cron: true, presence: true - validates :cron_time_zone, presence: true - validates :ref, ref: true, presence: true + validates :trigger, presence: { unless: :importing? } + validates :cron, cron: true, presence: { unless: :importing? } + validates :cron_time_zone, presence: { unless: :importing? } + validates :ref, presence: { unless: :importing? } # validate :check_cron_frequency after_create :schedule_next_run! diff --git a/app/validators/ref_validator.rb b/app/validators/ref_validator.rb deleted file mode 100644 index 2024255a770..00000000000 --- a/app/validators/ref_validator.rb +++ /dev/null @@ -1,10 +0,0 @@ -# RefValidator -# -# Custom validator for Ref. -class RefValidator < ActiveModel::EachValidator - def validate_each(record, attribute, value) - unless record.project.repository.branch_exists?(value) - record.errors.add(attribute, " does not exist") - end - end -end diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 8d74418ed14..f5e1e385ff9 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -39,7 +39,8 @@ project_tree: - :author - :events - :statuses - - :triggers # TODO: Need to confirm + - triggers: + - :trigger_schedule - :deploy_keys - :services - :hooks diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index fb43e7ccdbb..2ba12f5f924 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -5,6 +5,7 @@ module Gitlab pipelines: 'Ci::Pipeline', statuses: 'commit_status', triggers: 'Ci::Trigger', + trigger_schedule: 'Ci::TriggerSchedule', builds: 'Ci::Build', hooks: 'ProjectHook', merge_access_levels: 'ProtectedBranch::MergeAccessLevel', diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index 06e8cd5a1cd..488ae9655bb 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -100,6 +100,8 @@ triggers: - trigger_requests - owner - trigger_schedule +trigger_schedule: +- trigger deploy_keys: - user - deploy_keys_projects diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml index ecd8f2990c6..42082ff3dee 100644 --- a/spec/lib/gitlab/import_export/safe_model_attributes.yml +++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml @@ -249,6 +249,7 @@ Ci::TriggerSchedule: - created_at - updated_at - cron +- cron_time_zone - next_run_at DeployKey: - id diff --git a/spec/models/ci/trigger_schedule_spec.rb b/spec/models/ci/trigger_schedule_spec.rb index 8b27ca1c8b2..99668ff17b8 100644 --- a/spec/models/ci/trigger_schedule_spec.rb +++ b/spec/models/ci/trigger_schedule_spec.rb @@ -5,13 +5,6 @@ describe Ci::TriggerSchedule, models: true do it { is_expected.to belong_to(:trigger) } it { is_expected.to respond_to :ref } - it 'should validate ref existence' do - trigger_schedule = create(:ci_trigger_schedule, :cron_nightly_build) - trigger_schedule.trigger.ref = 'invalid-ref' - trigger_schedule.valid? - expect(trigger_schedule.errors[:ref].first).to include('does not exist') - end - describe '#schedule_next_run!' do let(:trigger_schedule) { create(:ci_trigger_schedule, :cron_nightly_build, next_run_at: nil) } -- cgit v1.2.1 From f6be8c048555f2d1086e7beed336b6187edb4d58 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 4 Apr 2017 16:58:02 +0900 Subject: Remove less_than_1_hour_from_now comments. Dry up def schedule_next_run! --- app/models/ci/trigger_schedule.rb | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/app/models/ci/trigger_schedule.rb b/app/models/ci/trigger_schedule.rb index 58337b34d80..6e7c0b4f6c2 100644 --- a/app/models/ci/trigger_schedule.rb +++ b/app/models/ci/trigger_schedule.rb @@ -14,32 +14,12 @@ module Ci validates :cron, cron: true, presence: { unless: :importing? } validates :cron_time_zone, presence: { unless: :importing? } validates :ref, presence: { unless: :importing? } - # validate :check_cron_frequency after_create :schedule_next_run! def schedule_next_run! next_time = Ci::CronParser.new(cron, cron_time_zone).next_time_from(Time.now) - - # if next_time.present? && !less_than_1_hour_from_now?(next_time) - if next_time.present? - update!(next_run_at: next_time) - end + update!(next_run_at: next_time) if next_time.present? end - - # private - - # def less_than_1_hour_from_now?(time) - # puts "diff: #{(time - Time.now).abs.inspect}" - # ((time - Time.now).abs < 1.hour) ? true : false - # end - - # def check_cron_frequency - # next_time = Ci::CronParser.new(cron, cron_time_zone).next_time_from(Time.now) - - # if less_than_1_hour_from_now?(next_time) - # self.errors.add(:cron, " can not be less than 1 hour") - # end - # end end end -- cgit v1.2.1 From 27f981b2901098f894e587bbd96b09e2a0f0c404 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 4 Apr 2017 18:31:19 +0900 Subject: Improve real_next_run. Improve triggers_helper_spec. --- app/helpers/triggers_helper.rb | 16 +++++----------- spec/helpers/triggers_helper_spec.rb | 27 ++++++++++++++++++++------- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/app/helpers/triggers_helper.rb b/app/helpers/triggers_helper.rb index be5cce9aea0..932ba595b73 100644 --- a/app/helpers/triggers_helper.rb +++ b/app/helpers/triggers_helper.rb @@ -11,16 +11,10 @@ module TriggersHelper "#{Settings.gitlab.url}/api/v3/projects/#{service.project_id}/services/#{service.to_param}/trigger" end - def real_next_run(trigger_schedule, worker_cron: nil, worker_time_zone: nil) - worker_cron = Settings.cron_jobs['trigger_schedule_worker']['cron'] unless worker_cron.present? - worker_time_zone = Time.zone.name unless worker_time_zone.present? - - worker_next_time = Ci::CronParser.new(worker_cron, worker_time_zone).next_time_from(Time.now) - - if trigger_schedule.next_run_at > worker_next_time - trigger_schedule.next_run_at - else - worker_next_time - end + def real_next_run(trigger_schedule, + worker_cron: Settings.cron_jobs['trigger_schedule_worker']['cron'], + worker_time_zone: Time.zone.name) + Ci::CronParser.new(worker_cron, worker_time_zone) + .next_time_from(trigger_schedule.next_run_at) end end diff --git a/spec/helpers/triggers_helper_spec.rb b/spec/helpers/triggers_helper_spec.rb index d801760335b..61d233421b2 100644 --- a/spec/helpers/triggers_helper_spec.rb +++ b/spec/helpers/triggers_helper_spec.rb @@ -4,23 +4,36 @@ describe TriggersHelper do describe '#real_next_run' do let(:trigger_schedule) { create(:ci_trigger_schedule, cron: user_cron, cron_time_zone: 'UTC') } - subject { helper.real_next_run(trigger_schedule, worker_cron: worker_cron, worker_time_zone: 'UTC') } + subject { helper.real_next_run(trigger_schedule, arguments) } context 'when next_run_at > worker_next_time' do - let(:worker_cron) { '0 0 1 1 *' } # every 00:00, January 1st + let(:arguments) { { worker_cron: '0 0 1 1 *', worker_time_zone: 'UTC' } } # every 00:00, January 1st let(:user_cron) { '1 0 1 1 *' } # every 00:01, January 1st - it 'returns next_run_at' do - is_expected.to eq(trigger_schedule.next_run_at) + it 'returns nearest worker_next_time from next_run_at' do + is_expected.to eq(Ci::CronParser.new(arguments[:worker_cron], arguments[:worker_time_zone]) + .next_time_from(trigger_schedule.next_run_at)) end end context 'when worker_next_time > next_run_at' do - let(:worker_cron) { '1 0 1 1 *' } # every 00:01, January 1st + let(:arguments) { { worker_cron: '1 0 1 1 *', worker_time_zone: 'UTC' } } # every 00:01, January 1st let(:user_cron) { '0 0 1 1 *' } # every 00:00, January 1st - it 'returns worker_next_time' do - is_expected.to eq(Ci::CronParser.new(worker_cron, 'UTC').next_time_from(Time.now)) + it 'returns nearest worker_next_time from next_run_at' do + is_expected.to eq(Ci::CronParser.new(arguments[:worker_cron], arguments[:worker_time_zone]) + .next_time_from(trigger_schedule.next_run_at)) + end + end + + context 'when worker_cron and worker_time_zone are ommited' do + let(:arguments) { {} } + let(:user_cron) { '* * * * *' } # every minutes + + it 'returns nearest worker_next_time from next_run_at by server configuration' do + is_expected.to eq(Ci::CronParser.new(Settings.cron_jobs['trigger_schedule_worker']['cron'], + Time.zone.name) + .next_time_from(trigger_schedule.next_run_at)) end end end -- cgit v1.2.1 From 914bef671f54c04a0d36d8e0f8c9830d6dea7b03 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 4 Apr 2017 18:44:25 +0900 Subject: Move Ci::CronParser to Gitlab::Ci::CronParser --- app/helpers/triggers_helper.rb | 4 +-- app/models/ci/trigger_schedule.rb | 2 +- app/validators/cron_validator.rb | 2 +- lib/ci/cron_parser.rb | 36 -------------------------- lib/gitlab/ci/cron_parser.rb | 38 ++++++++++++++++++++++++++++ spec/factories/ci/trigger_schedules.rb | 8 +++--- spec/helpers/triggers_helper_spec.rb | 14 +++++----- spec/models/ci/trigger_schedule_spec.rb | 2 +- spec/workers/trigger_schedule_worker_spec.rb | 2 +- 9 files changed, 55 insertions(+), 53 deletions(-) delete mode 100644 lib/ci/cron_parser.rb create mode 100644 lib/gitlab/ci/cron_parser.rb diff --git a/app/helpers/triggers_helper.rb b/app/helpers/triggers_helper.rb index 932ba595b73..a415ac11893 100644 --- a/app/helpers/triggers_helper.rb +++ b/app/helpers/triggers_helper.rb @@ -14,7 +14,7 @@ module TriggersHelper def real_next_run(trigger_schedule, worker_cron: Settings.cron_jobs['trigger_schedule_worker']['cron'], worker_time_zone: Time.zone.name) - Ci::CronParser.new(worker_cron, worker_time_zone) - .next_time_from(trigger_schedule.next_run_at) + Gitlab::Ci::CronParser.new(worker_cron, worker_time_zone) + .next_time_from(trigger_schedule.next_run_at) end end diff --git a/app/models/ci/trigger_schedule.rb b/app/models/ci/trigger_schedule.rb index 6e7c0b4f6c2..092338be9ce 100644 --- a/app/models/ci/trigger_schedule.rb +++ b/app/models/ci/trigger_schedule.rb @@ -18,7 +18,7 @@ module Ci after_create :schedule_next_run! def schedule_next_run! - next_time = Ci::CronParser.new(cron, cron_time_zone).next_time_from(Time.now) + next_time = Gitlab::Ci::CronParser.new(cron, cron_time_zone).next_time_from(Time.now) update!(next_run_at: next_time) if next_time.present? end end diff --git a/app/validators/cron_validator.rb b/app/validators/cron_validator.rb index 4d9a0d62a4c..31eaa4147a5 100644 --- a/app/validators/cron_validator.rb +++ b/app/validators/cron_validator.rb @@ -3,7 +3,7 @@ # Custom validator for Cron. class CronValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) - cron_parser = Ci::CronParser.new(record.cron, record.cron_time_zone) + cron_parser = Gitlab::Ci::CronParser.new(record.cron, record.cron_time_zone) is_valid_cron, is_valid_cron_time_zone = cron_parser.validation if !is_valid_cron diff --git a/lib/ci/cron_parser.rb b/lib/ci/cron_parser.rb deleted file mode 100644 index e0d589956a8..00000000000 --- a/lib/ci/cron_parser.rb +++ /dev/null @@ -1,36 +0,0 @@ -module Ci - class CronParser - VALID_SYNTAX_SAMPLE_TIME_ZONE = 'UTC'.freeze - VALID_SYNTAX_SAMPLE_CRON = '* * * * *'.freeze - - def initialize(cron, cron_time_zone = 'UTC') - @cron = cron - @cron_time_zone = cron_time_zone - end - - def next_time_from(time) - cron_line = try_parse_cron(@cron, @cron_time_zone) - if cron_line.present? - cron_line.next_time(time).in_time_zone(Time.zone) - else - nil - end - end - - def validation - is_valid_cron = try_parse_cron(@cron, VALID_SYNTAX_SAMPLE_TIME_ZONE).present? - is_valid_cron_time_zone = try_parse_cron(VALID_SYNTAX_SAMPLE_CRON, @cron_time_zone).present? - return is_valid_cron, is_valid_cron_time_zone - end - - private - - def try_parse_cron(cron, cron_time_zone) - begin - Rufus::Scheduler.parse("#{cron} #{cron_time_zone}") - rescue - nil - end - end - end -end diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb new file mode 100644 index 00000000000..01f37142510 --- /dev/null +++ b/lib/gitlab/ci/cron_parser.rb @@ -0,0 +1,38 @@ +module Gitlab + module Ci + class CronParser + VALID_SYNTAX_SAMPLE_TIME_ZONE = 'UTC'.freeze + VALID_SYNTAX_SAMPLE_CRON = '* * * * *'.freeze + + def initialize(cron, cron_time_zone = 'UTC') + @cron = cron + @cron_time_zone = cron_time_zone + end + + def next_time_from(time) + cron_line = try_parse_cron(@cron, @cron_time_zone) + if cron_line.present? + cron_line.next_time(time).in_time_zone(Time.zone) + else + nil + end + end + + def validation + is_valid_cron = try_parse_cron(@cron, VALID_SYNTAX_SAMPLE_TIME_ZONE).present? + is_valid_cron_time_zone = try_parse_cron(VALID_SYNTAX_SAMPLE_CRON, @cron_time_zone).present? + return is_valid_cron, is_valid_cron_time_zone + end + + private + + def try_parse_cron(cron, cron_time_zone) + begin + Rufus::Scheduler.parse("#{cron} #{cron_time_zone}") + rescue + nil + end + end + end + end +end \ No newline at end of file diff --git a/spec/factories/ci/trigger_schedules.rb b/spec/factories/ci/trigger_schedules.rb index 2e6a35c6416..8aafbc1f81b 100644 --- a/spec/factories/ci/trigger_schedules.rb +++ b/spec/factories/ci/trigger_schedules.rb @@ -2,7 +2,7 @@ FactoryGirl.define do factory :ci_trigger_schedule, class: Ci::TriggerSchedule do trigger factory: :ci_trigger_for_trigger_schedule cron '0 1 * * *' - cron_time_zone Ci::CronParser::VALID_SYNTAX_SAMPLE_TIME_ZONE + cron_time_zone Gitlab::Ci::CronParser::VALID_SYNTAX_SAMPLE_TIME_ZONE after(:build) do |trigger_schedule, evaluator| trigger_schedule.update!(project: trigger_schedule.trigger.project) @@ -16,17 +16,17 @@ FactoryGirl.define do trait :cron_nightly_build do cron '0 1 * * *' - cron_time_zone Ci::CronParser::VALID_SYNTAX_SAMPLE_TIME_ZONE + cron_time_zone Gitlab::Ci::CronParser::VALID_SYNTAX_SAMPLE_TIME_ZONE end trait :cron_weekly_build do cron '0 1 * * 6' - cron_time_zone Ci::CronParser::VALID_SYNTAX_SAMPLE_TIME_ZONE + cron_time_zone Gitlab::Ci::CronParser::VALID_SYNTAX_SAMPLE_TIME_ZONE end trait :cron_monthly_build do cron '0 1 22 * *' - cron_time_zone Ci::CronParser::VALID_SYNTAX_SAMPLE_TIME_ZONE + cron_time_zone Gitlab::Ci::CronParser::VALID_SYNTAX_SAMPLE_TIME_ZONE end end end diff --git a/spec/helpers/triggers_helper_spec.rb b/spec/helpers/triggers_helper_spec.rb index 61d233421b2..ee3fd3fea0f 100644 --- a/spec/helpers/triggers_helper_spec.rb +++ b/spec/helpers/triggers_helper_spec.rb @@ -11,8 +11,8 @@ describe TriggersHelper do let(:user_cron) { '1 0 1 1 *' } # every 00:01, January 1st it 'returns nearest worker_next_time from next_run_at' do - is_expected.to eq(Ci::CronParser.new(arguments[:worker_cron], arguments[:worker_time_zone]) - .next_time_from(trigger_schedule.next_run_at)) + is_expected.to eq(Gitlab::Ci::CronParser.new(arguments[:worker_cron], arguments[:worker_time_zone]) + .next_time_from(trigger_schedule.next_run_at)) end end @@ -21,8 +21,8 @@ describe TriggersHelper do let(:user_cron) { '0 0 1 1 *' } # every 00:00, January 1st it 'returns nearest worker_next_time from next_run_at' do - is_expected.to eq(Ci::CronParser.new(arguments[:worker_cron], arguments[:worker_time_zone]) - .next_time_from(trigger_schedule.next_run_at)) + is_expected.to eq(Gitlab::Ci::CronParser.new(arguments[:worker_cron], arguments[:worker_time_zone]) + .next_time_from(trigger_schedule.next_run_at)) end end @@ -31,9 +31,9 @@ describe TriggersHelper do let(:user_cron) { '* * * * *' } # every minutes it 'returns nearest worker_next_time from next_run_at by server configuration' do - is_expected.to eq(Ci::CronParser.new(Settings.cron_jobs['trigger_schedule_worker']['cron'], - Time.zone.name) - .next_time_from(trigger_schedule.next_run_at)) + is_expected.to eq(Gitlab::Ci::CronParser.new(Settings.cron_jobs['trigger_schedule_worker']['cron'], + Time.zone.name) + .next_time_from(trigger_schedule.next_run_at)) end end end diff --git a/spec/models/ci/trigger_schedule_spec.rb b/spec/models/ci/trigger_schedule_spec.rb index 99668ff17b8..81104cb26b6 100644 --- a/spec/models/ci/trigger_schedule_spec.rb +++ b/spec/models/ci/trigger_schedule_spec.rb @@ -13,7 +13,7 @@ describe Ci::TriggerSchedule, models: true do end it 'updates next_run_at' do - next_time = Ci::CronParser.new(trigger_schedule.cron, trigger_schedule.cron_time_zone).next_time_from(Time.now) + next_time = Gitlab::Ci::CronParser.new(trigger_schedule.cron, trigger_schedule.cron_time_zone).next_time_from(Time.now) expect(Ci::TriggerSchedule.last.next_run_at).to eq(next_time) end end diff --git a/spec/workers/trigger_schedule_worker_spec.rb b/spec/workers/trigger_schedule_worker_spec.rb index 950f72a68d9..4df5731709b 100644 --- a/spec/workers/trigger_schedule_worker_spec.rb +++ b/spec/workers/trigger_schedule_worker_spec.rb @@ -23,7 +23,7 @@ describe TriggerScheduleWorker do end it 'updates next_run_at' do - next_time = Ci::CronParser.new(trigger_schedule.cron, trigger_schedule.cron_time_zone).next_time_from(Time.now) + next_time = Gitlab::Ci::CronParser.new(trigger_schedule.cron, trigger_schedule.cron_time_zone).next_time_from(Time.now) expect(Ci::TriggerSchedule.last.next_run_at).to eq(next_time) end end -- cgit v1.2.1 From 3d3df09713dcb70baceaeba7603fa49b89fc8007 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 4 Apr 2017 18:49:52 +0900 Subject: Dry up next_time_from. Move cron_parser_spec to appropriate location. --- lib/gitlab/ci/cron_parser.rb | 6 +- spec/lib/ci/cron_parser_spec.rb | 106 --------------------------------- spec/lib/gitlab/ci/cron_parser_spec.rb | 104 ++++++++++++++++++++++++++++++++ 3 files changed, 105 insertions(+), 111 deletions(-) delete mode 100644 spec/lib/ci/cron_parser_spec.rb create mode 100644 spec/lib/gitlab/ci/cron_parser_spec.rb diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb index 01f37142510..2d9ea26faf0 100644 --- a/lib/gitlab/ci/cron_parser.rb +++ b/lib/gitlab/ci/cron_parser.rb @@ -11,11 +11,7 @@ module Gitlab def next_time_from(time) cron_line = try_parse_cron(@cron, @cron_time_zone) - if cron_line.present? - cron_line.next_time(time).in_time_zone(Time.zone) - else - nil - end + cron_line.next_time(time).in_time_zone(Time.zone) if cron_line.present? end def validation diff --git a/spec/lib/ci/cron_parser_spec.rb b/spec/lib/ci/cron_parser_spec.rb deleted file mode 100644 index fb4a5ec4d13..00000000000 --- a/spec/lib/ci/cron_parser_spec.rb +++ /dev/null @@ -1,106 +0,0 @@ -require 'spec_helper' - -module Ci - describe CronParser, lib: true do - shared_examples_for "returns time in the future" do - it { is_expected.to be > Time.now } - end - - describe '#next_time_from' do - subject { described_class.new(cron, cron_time_zone).next_time_from(Time.now) } - - context 'when cron and cron_time_zone are valid' do - context 'when specific time' do - let(:cron) { '3 4 5 6 *' } - let(:cron_time_zone) { 'UTC' } - - it_behaves_like "returns time in the future" - - it 'returns exact time' do - expect(subject.min).to eq(3) - expect(subject.hour).to eq(4) - expect(subject.day).to eq(5) - expect(subject.month).to eq(6) - end - end - - context 'when specific day of week' do - let(:cron) { '* * * * 0' } - let(:cron_time_zone) { 'UTC' } - - it_behaves_like "returns time in the future" - - it 'returns exact day of week' do - expect(subject.wday).to eq(0) - end - end - - context 'when slash used' do - let(:cron) { '*/10 */6 */10 */10 *' } - let(:cron_time_zone) { 'UTC' } - - it_behaves_like "returns time in the future" - - it 'returns specific time' do - expect(subject.min).to be_in([0, 10, 20, 30, 40, 50]) - expect(subject.hour).to be_in([0, 6, 12, 18]) - expect(subject.day).to be_in([1, 11, 21, 31]) - expect(subject.month).to be_in([1, 11]) - end - end - - context 'when range used' do - let(:cron) { '0,20,40 * 1-5 * *' } - let(:cron_time_zone) { 'UTC' } - - it_behaves_like "returns time in the future" - - it 'returns specific time' do - expect(subject.min).to be_in([0, 20, 40]) - expect(subject.day).to be_in((1..5).to_a) - end - end - - context 'when cron_time_zone is US/Pacific' do - let(:cron) { '0 0 * * *' } - let(:cron_time_zone) { 'US/Pacific' } - - it_behaves_like "returns time in the future" - - it 'converts time in server time zone' do - expect(subject.hour).to eq(7) - end - end - end - - context 'when cron and cron_time_zone are invalid' do - let(:cron) { 'invalid_cron' } - let(:cron_time_zone) { 'invalid_cron_time_zone' } - - it 'returns nil' do - is_expected.to be_nil - end - end - end - - describe '#validation' do - it 'returns results' do - is_valid_cron, is_valid_cron_time_zone = described_class.new('* * * * *', 'Europe/Istanbul').validation - expect(is_valid_cron).to eq(true) - expect(is_valid_cron_time_zone).to eq(true) - end - - it 'returns results' do - is_valid_cron, is_valid_cron_time_zone = described_class.new('*********', 'Europe/Istanbul').validation - expect(is_valid_cron).to eq(false) - expect(is_valid_cron_time_zone).to eq(true) - end - - it 'returns results' do - is_valid_cron, is_valid_cron_time_zone = described_class.new('* * * * *', 'Invalid-zone').validation - expect(is_valid_cron).to eq(true) - expect(is_valid_cron_time_zone).to eq(false) - end - end - end -end diff --git a/spec/lib/gitlab/ci/cron_parser_spec.rb b/spec/lib/gitlab/ci/cron_parser_spec.rb new file mode 100644 index 00000000000..62d1ea3f087 --- /dev/null +++ b/spec/lib/gitlab/ci/cron_parser_spec.rb @@ -0,0 +1,104 @@ +require 'spec_helper' + +describe Gitlab::Ci::CronParser do + shared_examples_for "returns time in the future" do + it { is_expected.to be > Time.now } + end + + describe '#next_time_from' do + subject { described_class.new(cron, cron_time_zone).next_time_from(Time.now) } + + context 'when cron and cron_time_zone are valid' do + context 'when specific time' do + let(:cron) { '3 4 5 6 *' } + let(:cron_time_zone) { 'UTC' } + + it_behaves_like "returns time in the future" + + it 'returns exact time' do + expect(subject.min).to eq(3) + expect(subject.hour).to eq(4) + expect(subject.day).to eq(5) + expect(subject.month).to eq(6) + end + end + + context 'when specific day of week' do + let(:cron) { '* * * * 0' } + let(:cron_time_zone) { 'UTC' } + + it_behaves_like "returns time in the future" + + it 'returns exact day of week' do + expect(subject.wday).to eq(0) + end + end + + context 'when slash used' do + let(:cron) { '*/10 */6 */10 */10 *' } + let(:cron_time_zone) { 'UTC' } + + it_behaves_like "returns time in the future" + + it 'returns specific time' do + expect(subject.min).to be_in([0, 10, 20, 30, 40, 50]) + expect(subject.hour).to be_in([0, 6, 12, 18]) + expect(subject.day).to be_in([1, 11, 21, 31]) + expect(subject.month).to be_in([1, 11]) + end + end + + context 'when range used' do + let(:cron) { '0,20,40 * 1-5 * *' } + let(:cron_time_zone) { 'UTC' } + + it_behaves_like "returns time in the future" + + it 'returns specific time' do + expect(subject.min).to be_in([0, 20, 40]) + expect(subject.day).to be_in((1..5).to_a) + end + end + + context 'when cron_time_zone is US/Pacific' do + let(:cron) { '0 0 * * *' } + let(:cron_time_zone) { 'US/Pacific' } + + it_behaves_like "returns time in the future" + + it 'converts time in server time zone' do + expect(subject.hour).to eq(7) + end + end + end + + context 'when cron and cron_time_zone are invalid' do + let(:cron) { 'invalid_cron' } + let(:cron_time_zone) { 'invalid_cron_time_zone' } + + it 'returns nil' do + is_expected.to be_nil + end + end + end + + describe '#validation' do + it 'returns results' do + is_valid_cron, is_valid_cron_time_zone = described_class.new('* * * * *', 'Europe/Istanbul').validation + expect(is_valid_cron).to eq(true) + expect(is_valid_cron_time_zone).to eq(true) + end + + it 'returns results' do + is_valid_cron, is_valid_cron_time_zone = described_class.new('*********', 'Europe/Istanbul').validation + expect(is_valid_cron).to eq(false) + expect(is_valid_cron_time_zone).to eq(true) + end + + it 'returns results' do + is_valid_cron, is_valid_cron_time_zone = described_class.new('* * * * *', 'Invalid-zone').validation + expect(is_valid_cron).to eq(true) + expect(is_valid_cron_time_zone).to eq(false) + end + end +end -- cgit v1.2.1 From 4949e2b291bc59ee3855882a29df3bff9edfd4e5 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 4 Apr 2017 19:14:49 +0900 Subject: Separate cron_valid? and cron_time_zone_valid? --- app/models/ci/trigger_schedule.rb | 2 +- app/validators/cron_time_zone_validator.rb | 9 +++++++ app/validators/cron_validator.rb | 8 +------ lib/gitlab/ci/cron_parser.rb | 10 ++++---- spec/lib/gitlab/ci/cron_parser_spec.rb | 38 ++++++++++++++++++++---------- 5 files changed, 42 insertions(+), 25 deletions(-) create mode 100644 app/validators/cron_time_zone_validator.rb diff --git a/app/models/ci/trigger_schedule.rb b/app/models/ci/trigger_schedule.rb index 092338be9ce..9b1dfce969a 100644 --- a/app/models/ci/trigger_schedule.rb +++ b/app/models/ci/trigger_schedule.rb @@ -12,7 +12,7 @@ module Ci validates :trigger, presence: { unless: :importing? } validates :cron, cron: true, presence: { unless: :importing? } - validates :cron_time_zone, presence: { unless: :importing? } + validates :cron_time_zone, cron_time_zone: true, presence: { unless: :importing? } validates :ref, presence: { unless: :importing? } after_create :schedule_next_run! diff --git a/app/validators/cron_time_zone_validator.rb b/app/validators/cron_time_zone_validator.rb new file mode 100644 index 00000000000..9d4bbe1d458 --- /dev/null +++ b/app/validators/cron_time_zone_validator.rb @@ -0,0 +1,9 @@ +# CronTimeZoneValidator +# +# Custom validator for CronTimeZone. +class CronTimeZoneValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + cron_parser = Gitlab::Ci::CronParser.new(record.cron, record.cron_time_zone) + record.errors.add(attribute, " is invalid syntax") unless cron_parser.cron_time_zone_valid? + end +end diff --git a/app/validators/cron_validator.rb b/app/validators/cron_validator.rb index 31eaa4147a5..cc07011d56b 100644 --- a/app/validators/cron_validator.rb +++ b/app/validators/cron_validator.rb @@ -4,12 +4,6 @@ class CronValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) cron_parser = Gitlab::Ci::CronParser.new(record.cron, record.cron_time_zone) - is_valid_cron, is_valid_cron_time_zone = cron_parser.validation - - if !is_valid_cron - record.errors.add(:cron, " is invalid syntax") - elsif !is_valid_cron_time_zone - record.errors.add(:cron_time_zone, " is invalid timezone") - end + record.errors.add(attribute, " is invalid syntax") unless cron_parser.cron_valid? end end diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb index 2d9ea26faf0..521e556f769 100644 --- a/lib/gitlab/ci/cron_parser.rb +++ b/lib/gitlab/ci/cron_parser.rb @@ -14,10 +14,12 @@ module Gitlab cron_line.next_time(time).in_time_zone(Time.zone) if cron_line.present? end - def validation - is_valid_cron = try_parse_cron(@cron, VALID_SYNTAX_SAMPLE_TIME_ZONE).present? - is_valid_cron_time_zone = try_parse_cron(VALID_SYNTAX_SAMPLE_CRON, @cron_time_zone).present? - return is_valid_cron, is_valid_cron_time_zone + def cron_valid? + try_parse_cron(@cron, VALID_SYNTAX_SAMPLE_TIME_ZONE).present? + end + + def cron_time_zone_valid? + try_parse_cron(VALID_SYNTAX_SAMPLE_CRON, @cron_time_zone).present? end private diff --git a/spec/lib/gitlab/ci/cron_parser_spec.rb b/spec/lib/gitlab/ci/cron_parser_spec.rb index 62d1ea3f087..1cdd8c1d2e7 100644 --- a/spec/lib/gitlab/ci/cron_parser_spec.rb +++ b/spec/lib/gitlab/ci/cron_parser_spec.rb @@ -82,23 +82,35 @@ describe Gitlab::Ci::CronParser do end end - describe '#validation' do - it 'returns results' do - is_valid_cron, is_valid_cron_time_zone = described_class.new('* * * * *', 'Europe/Istanbul').validation - expect(is_valid_cron).to eq(true) - expect(is_valid_cron_time_zone).to eq(true) + describe '#cron_valid?' do + subject { described_class.new(cron, Gitlab::Ci::CronParser::VALID_SYNTAX_SAMPLE_TIME_ZONE).cron_valid? } + + context 'when cron is valid' do + let(:cron) { '* * * * *' } + + it { is_expected.to eq(true) } end - it 'returns results' do - is_valid_cron, is_valid_cron_time_zone = described_class.new('*********', 'Europe/Istanbul').validation - expect(is_valid_cron).to eq(false) - expect(is_valid_cron_time_zone).to eq(true) + context 'when cron is invalid' do + let(:cron) { '*********' } + + it { is_expected.to eq(false) } end + end + + describe '#cron_time_zone_valid?' do + subject { described_class.new(Gitlab::Ci::CronParser::VALID_SYNTAX_SAMPLE_CRON, cron_time_zone).cron_time_zone_valid? } + + context 'when cron is valid' do + let(:cron_time_zone) { 'Europe/Istanbul' } + + it { is_expected.to eq(true) } + end + + context 'when cron is invalid' do + let(:cron_time_zone) { 'Invalid-zone' } - it 'returns results' do - is_valid_cron, is_valid_cron_time_zone = described_class.new('* * * * *', 'Invalid-zone').validation - expect(is_valid_cron).to eq(true) - expect(is_valid_cron_time_zone).to eq(false) + it { is_expected.to eq(false) } end end end -- cgit v1.2.1 From 0c153af73df3d5b10915cf9d32af728f9b6e8e98 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 4 Apr 2017 21:18:51 +0900 Subject: Ommit begin block in try_parse_cron --- lib/gitlab/ci/cron_parser.rb | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb index 521e556f769..69dd8ad0fce 100644 --- a/lib/gitlab/ci/cron_parser.rb +++ b/lib/gitlab/ci/cron_parser.rb @@ -25,11 +25,9 @@ module Gitlab private def try_parse_cron(cron, cron_time_zone) - begin - Rufus::Scheduler.parse("#{cron} #{cron_time_zone}") - rescue - nil - end + Rufus::Scheduler.parse("#{cron} #{cron_time_zone}") + rescue + # noop end end end -- cgit v1.2.1 From 4a5c6a8e2953baeceb33d281d23d2c305ff884fa Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 4 Apr 2017 21:22:12 +0900 Subject: Rename cron_nightly_build to nightly --- spec/factories/ci/trigger_schedules.rb | 6 +++--- spec/models/ci/trigger_schedule_spec.rb | 2 +- spec/workers/trigger_schedule_worker_spec.rb | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/spec/factories/ci/trigger_schedules.rb b/spec/factories/ci/trigger_schedules.rb index 8aafbc1f81b..9c16d45b49a 100644 --- a/spec/factories/ci/trigger_schedules.rb +++ b/spec/factories/ci/trigger_schedules.rb @@ -14,17 +14,17 @@ FactoryGirl.define do end end - trait :cron_nightly_build do + trait :nightly do cron '0 1 * * *' cron_time_zone Gitlab::Ci::CronParser::VALID_SYNTAX_SAMPLE_TIME_ZONE end - trait :cron_weekly_build do + trait :weekly do cron '0 1 * * 6' cron_time_zone Gitlab::Ci::CronParser::VALID_SYNTAX_SAMPLE_TIME_ZONE end - trait :cron_monthly_build do + trait :monthly do cron '0 1 22 * *' cron_time_zone Gitlab::Ci::CronParser::VALID_SYNTAX_SAMPLE_TIME_ZONE end diff --git a/spec/models/ci/trigger_schedule_spec.rb b/spec/models/ci/trigger_schedule_spec.rb index 81104cb26b6..9a4bf122bf0 100644 --- a/spec/models/ci/trigger_schedule_spec.rb +++ b/spec/models/ci/trigger_schedule_spec.rb @@ -6,7 +6,7 @@ describe Ci::TriggerSchedule, models: true do it { is_expected.to respond_to :ref } describe '#schedule_next_run!' do - let(:trigger_schedule) { create(:ci_trigger_schedule, :cron_nightly_build, next_run_at: nil) } + let(:trigger_schedule) { create(:ci_trigger_schedule, :nightly, next_run_at: nil) } before do trigger_schedule.schedule_next_run! diff --git a/spec/workers/trigger_schedule_worker_spec.rb b/spec/workers/trigger_schedule_worker_spec.rb index 4df5731709b..75a98e42ac5 100644 --- a/spec/workers/trigger_schedule_worker_spec.rb +++ b/spec/workers/trigger_schedule_worker_spec.rb @@ -8,7 +8,7 @@ describe TriggerScheduleWorker do end context 'when there is a scheduled trigger within next_run_at' do - let!(:trigger_schedule) { create(:ci_trigger_schedule, :cron_nightly_build, :force_triggable) } + let!(:trigger_schedule) { create(:ci_trigger_schedule, :nightly, :force_triggable) } before do worker.perform @@ -29,7 +29,7 @@ describe TriggerScheduleWorker do end context 'when there are no scheduled triggers within next_run_at' do - let!(:trigger_schedule) { create(:ci_trigger_schedule, :cron_nightly_build) } + let!(:trigger_schedule) { create(:ci_trigger_schedule, :nightly) } before do worker.perform -- cgit v1.2.1 From f229290ac828d1d5743f86c5a4977168a0406365 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Tue, 4 Apr 2017 23:43:09 +0900 Subject: Remove triggers_helper. Add trigger_schedule_presenter. --- app/helpers/triggers_helper.rb | 7 --- app/models/ci/trigger_schedule.rb | 1 + app/presenters/ci/trigger_schedule_presenter.rb | 11 ++++ spec/helpers/triggers_helper_spec.rb | 40 -------------- .../ci/trigger_schedule_presenter_spec.rb | 64 ++++++++++++++++++++++ 5 files changed, 76 insertions(+), 47 deletions(-) create mode 100644 app/presenters/ci/trigger_schedule_presenter.rb delete mode 100644 spec/helpers/triggers_helper_spec.rb create mode 100644 spec/presenters/ci/trigger_schedule_presenter_spec.rb diff --git a/app/helpers/triggers_helper.rb b/app/helpers/triggers_helper.rb index a415ac11893..a48d4475e97 100644 --- a/app/helpers/triggers_helper.rb +++ b/app/helpers/triggers_helper.rb @@ -10,11 +10,4 @@ module TriggersHelper def service_trigger_url(service) "#{Settings.gitlab.url}/api/v3/projects/#{service.project_id}/services/#{service.to_param}/trigger" end - - def real_next_run(trigger_schedule, - worker_cron: Settings.cron_jobs['trigger_schedule_worker']['cron'], - worker_time_zone: Time.zone.name) - Gitlab::Ci::CronParser.new(worker_cron, worker_time_zone) - .next_time_from(trigger_schedule.next_run_at) - end end diff --git a/app/models/ci/trigger_schedule.rb b/app/models/ci/trigger_schedule.rb index 9b1dfce969a..2cf041df8a7 100644 --- a/app/models/ci/trigger_schedule.rb +++ b/app/models/ci/trigger_schedule.rb @@ -2,6 +2,7 @@ module Ci class TriggerSchedule < ActiveRecord::Base extend Ci::Model include Importable + include Presentable acts_as_paranoid diff --git a/app/presenters/ci/trigger_schedule_presenter.rb b/app/presenters/ci/trigger_schedule_presenter.rb new file mode 100644 index 00000000000..faef6a3d66f --- /dev/null +++ b/app/presenters/ci/trigger_schedule_presenter.rb @@ -0,0 +1,11 @@ +module Ci + class TriggerSchedulePresenter < Gitlab::View::Presenter::Delegated + presents :trigger_schedule + + def real_next_run(worker_cron: Settings.cron_jobs['trigger_schedule_worker']['cron'], + worker_time_zone: Time.zone.name) + Gitlab::Ci::CronParser.new(worker_cron, worker_time_zone) + .next_time_from(next_run_at) + end + end +end diff --git a/spec/helpers/triggers_helper_spec.rb b/spec/helpers/triggers_helper_spec.rb deleted file mode 100644 index ee3fd3fea0f..00000000000 --- a/spec/helpers/triggers_helper_spec.rb +++ /dev/null @@ -1,40 +0,0 @@ -require 'rails_helper' - -describe TriggersHelper do - describe '#real_next_run' do - let(:trigger_schedule) { create(:ci_trigger_schedule, cron: user_cron, cron_time_zone: 'UTC') } - - subject { helper.real_next_run(trigger_schedule, arguments) } - - context 'when next_run_at > worker_next_time' do - let(:arguments) { { worker_cron: '0 0 1 1 *', worker_time_zone: 'UTC' } } # every 00:00, January 1st - let(:user_cron) { '1 0 1 1 *' } # every 00:01, January 1st - - it 'returns nearest worker_next_time from next_run_at' do - is_expected.to eq(Gitlab::Ci::CronParser.new(arguments[:worker_cron], arguments[:worker_time_zone]) - .next_time_from(trigger_schedule.next_run_at)) - end - end - - context 'when worker_next_time > next_run_at' do - let(:arguments) { { worker_cron: '1 0 1 1 *', worker_time_zone: 'UTC' } } # every 00:01, January 1st - let(:user_cron) { '0 0 1 1 *' } # every 00:00, January 1st - - it 'returns nearest worker_next_time from next_run_at' do - is_expected.to eq(Gitlab::Ci::CronParser.new(arguments[:worker_cron], arguments[:worker_time_zone]) - .next_time_from(trigger_schedule.next_run_at)) - end - end - - context 'when worker_cron and worker_time_zone are ommited' do - let(:arguments) { {} } - let(:user_cron) { '* * * * *' } # every minutes - - it 'returns nearest worker_next_time from next_run_at by server configuration' do - is_expected.to eq(Gitlab::Ci::CronParser.new(Settings.cron_jobs['trigger_schedule_worker']['cron'], - Time.zone.name) - .next_time_from(trigger_schedule.next_run_at)) - end - end - end -end diff --git a/spec/presenters/ci/trigger_schedule_presenter_spec.rb b/spec/presenters/ci/trigger_schedule_presenter_spec.rb new file mode 100644 index 00000000000..92e38441a8a --- /dev/null +++ b/spec/presenters/ci/trigger_schedule_presenter_spec.rb @@ -0,0 +1,64 @@ +require 'spec_helper' + +describe Ci::TriggerSchedulePresenter do + let(:trigger_schedule) { create(:ci_trigger_schedule) } + + subject(:presenter) do + described_class.new(trigger_schedule) + end + + it 'inherits from Gitlab::View::Presenter::Delegated' do + expect(described_class.superclass).to eq(Gitlab::View::Presenter::Delegated) + end + + describe '#initialize' do + it 'takes a trigger_schedule and optional params' do + expect { presenter }.not_to raise_error + end + + it 'exposes trigger_schedule' do + expect(presenter.trigger_schedule).to eq(trigger_schedule) + end + + it 'forwards missing methods to trigger_schedule' do + expect(presenter.ref).to eq('master') + end + end + + describe '#real_next_run' do + let(:trigger_schedule) { create(:ci_trigger_schedule, cron: user_cron, cron_time_zone: 'UTC') } + + subject { described_class.new(trigger_schedule).real_next_run(arguments) } + + context 'when next_run_at > worker_next_time' do + let(:arguments) { { worker_cron: '0 0 1 1 *', worker_time_zone: 'UTC' } } # every 00:00, January 1st + let(:user_cron) { '1 0 1 1 *' } # every 00:01, January 1st + + it 'returns nearest worker_next_time from next_run_at' do + is_expected.to eq(Gitlab::Ci::CronParser.new(arguments[:worker_cron], arguments[:worker_time_zone]) + .next_time_from(trigger_schedule.next_run_at)) + end + end + + context 'when worker_next_time > next_run_at' do + let(:arguments) { { worker_cron: '1 0 1 1 *', worker_time_zone: 'UTC' } } # every 00:01, January 1st + let(:user_cron) { '0 0 1 1 *' } # every 00:00, January 1st + + it 'returns nearest worker_next_time from next_run_at' do + is_expected.to eq(Gitlab::Ci::CronParser.new(arguments[:worker_cron], arguments[:worker_time_zone]) + .next_time_from(trigger_schedule.next_run_at)) + end + end + + context 'when worker_cron and worker_time_zone are ommited' do + let(:arguments) { {} } + let(:user_cron) { '* * * * *' } # every minutes + + it 'returns nearest worker_next_time from next_run_at by server configuration' do + is_expected.to eq(Gitlab::Ci::CronParser.new(Settings.cron_jobs['trigger_schedule_worker']['cron'], + Time.zone.name) + .next_time_from(trigger_schedule.next_run_at)) + end + end + end +end -- cgit v1.2.1 From 1dbc888e3306f30ca0882aece86ccd1a817e0ab8 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 5 Apr 2017 01:29:10 +0900 Subject: Remove TriggerSchedulePresenter. This will go in another MR. --- app/models/ci/trigger_schedule.rb | 1 - app/presenters/ci/trigger_schedule_presenter.rb | 11 ---- .../ci/trigger_schedule_presenter_spec.rb | 64 ---------------------- 3 files changed, 76 deletions(-) delete mode 100644 app/presenters/ci/trigger_schedule_presenter.rb delete mode 100644 spec/presenters/ci/trigger_schedule_presenter_spec.rb diff --git a/app/models/ci/trigger_schedule.rb b/app/models/ci/trigger_schedule.rb index 2cf041df8a7..9b1dfce969a 100644 --- a/app/models/ci/trigger_schedule.rb +++ b/app/models/ci/trigger_schedule.rb @@ -2,7 +2,6 @@ module Ci class TriggerSchedule < ActiveRecord::Base extend Ci::Model include Importable - include Presentable acts_as_paranoid diff --git a/app/presenters/ci/trigger_schedule_presenter.rb b/app/presenters/ci/trigger_schedule_presenter.rb deleted file mode 100644 index faef6a3d66f..00000000000 --- a/app/presenters/ci/trigger_schedule_presenter.rb +++ /dev/null @@ -1,11 +0,0 @@ -module Ci - class TriggerSchedulePresenter < Gitlab::View::Presenter::Delegated - presents :trigger_schedule - - def real_next_run(worker_cron: Settings.cron_jobs['trigger_schedule_worker']['cron'], - worker_time_zone: Time.zone.name) - Gitlab::Ci::CronParser.new(worker_cron, worker_time_zone) - .next_time_from(next_run_at) - end - end -end diff --git a/spec/presenters/ci/trigger_schedule_presenter_spec.rb b/spec/presenters/ci/trigger_schedule_presenter_spec.rb deleted file mode 100644 index 92e38441a8a..00000000000 --- a/spec/presenters/ci/trigger_schedule_presenter_spec.rb +++ /dev/null @@ -1,64 +0,0 @@ -require 'spec_helper' - -describe Ci::TriggerSchedulePresenter do - let(:trigger_schedule) { create(:ci_trigger_schedule) } - - subject(:presenter) do - described_class.new(trigger_schedule) - end - - it 'inherits from Gitlab::View::Presenter::Delegated' do - expect(described_class.superclass).to eq(Gitlab::View::Presenter::Delegated) - end - - describe '#initialize' do - it 'takes a trigger_schedule and optional params' do - expect { presenter }.not_to raise_error - end - - it 'exposes trigger_schedule' do - expect(presenter.trigger_schedule).to eq(trigger_schedule) - end - - it 'forwards missing methods to trigger_schedule' do - expect(presenter.ref).to eq('master') - end - end - - describe '#real_next_run' do - let(:trigger_schedule) { create(:ci_trigger_schedule, cron: user_cron, cron_time_zone: 'UTC') } - - subject { described_class.new(trigger_schedule).real_next_run(arguments) } - - context 'when next_run_at > worker_next_time' do - let(:arguments) { { worker_cron: '0 0 1 1 *', worker_time_zone: 'UTC' } } # every 00:00, January 1st - let(:user_cron) { '1 0 1 1 *' } # every 00:01, January 1st - - it 'returns nearest worker_next_time from next_run_at' do - is_expected.to eq(Gitlab::Ci::CronParser.new(arguments[:worker_cron], arguments[:worker_time_zone]) - .next_time_from(trigger_schedule.next_run_at)) - end - end - - context 'when worker_next_time > next_run_at' do - let(:arguments) { { worker_cron: '1 0 1 1 *', worker_time_zone: 'UTC' } } # every 00:01, January 1st - let(:user_cron) { '0 0 1 1 *' } # every 00:00, January 1st - - it 'returns nearest worker_next_time from next_run_at' do - is_expected.to eq(Gitlab::Ci::CronParser.new(arguments[:worker_cron], arguments[:worker_time_zone]) - .next_time_from(trigger_schedule.next_run_at)) - end - end - - context 'when worker_cron and worker_time_zone are ommited' do - let(:arguments) { {} } - let(:user_cron) { '* * * * *' } # every minutes - - it 'returns nearest worker_next_time from next_run_at by server configuration' do - is_expected.to eq(Gitlab::Ci::CronParser.new(Settings.cron_jobs['trigger_schedule_worker']['cron'], - Time.zone.name) - .next_time_from(trigger_schedule.next_run_at)) - end - end - end -end -- cgit v1.2.1 From 4688eb47c6fe135fb9baad5a5d56b1dfa685cc7f Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 5 Apr 2017 01:54:45 +0900 Subject: Rename cron_time_zone to cron_timezone. Separate add_concurrent_foreign_key. --- app/models/ci/trigger_schedule.rb | 4 ++-- app/validators/cron_time_zone_validator.rb | 9 ------- app/validators/cron_timezone_validator.rb | 9 +++++++ app/validators/cron_validator.rb | 2 +- .../20170329095907_create_ci_trigger_schedules.rb | 16 ++++--------- .../20170404163427_add_trigger_id_foreign_key.rb | 15 ++++++++++++ db/schema.rb | 2 +- lib/gitlab/ci/cron_parser.rb | 14 +++++------ spec/factories/ci/trigger_schedules.rb | 8 +++---- spec/lib/gitlab/ci/cron_parser_spec.rb | 28 +++++++++++----------- .../gitlab/import_export/safe_model_attributes.yml | 2 +- spec/models/ci/trigger_schedule_spec.rb | 2 +- spec/workers/trigger_schedule_worker_spec.rb | 2 +- 13 files changed, 60 insertions(+), 53 deletions(-) delete mode 100644 app/validators/cron_time_zone_validator.rb create mode 100644 app/validators/cron_timezone_validator.rb create mode 100644 db/migrate/20170404163427_add_trigger_id_foreign_key.rb diff --git a/app/models/ci/trigger_schedule.rb b/app/models/ci/trigger_schedule.rb index 9b1dfce969a..d18dbea284e 100644 --- a/app/models/ci/trigger_schedule.rb +++ b/app/models/ci/trigger_schedule.rb @@ -12,13 +12,13 @@ module Ci validates :trigger, presence: { unless: :importing? } validates :cron, cron: true, presence: { unless: :importing? } - validates :cron_time_zone, cron_time_zone: true, presence: { unless: :importing? } + validates :cron_timezone, cron_timezone: true, presence: { unless: :importing? } validates :ref, presence: { unless: :importing? } after_create :schedule_next_run! def schedule_next_run! - next_time = Gitlab::Ci::CronParser.new(cron, cron_time_zone).next_time_from(Time.now) + next_time = Gitlab::Ci::CronParser.new(cron, cron_timezone).next_time_from(Time.now) update!(next_run_at: next_time) if next_time.present? end end diff --git a/app/validators/cron_time_zone_validator.rb b/app/validators/cron_time_zone_validator.rb deleted file mode 100644 index 9d4bbe1d458..00000000000 --- a/app/validators/cron_time_zone_validator.rb +++ /dev/null @@ -1,9 +0,0 @@ -# CronTimeZoneValidator -# -# Custom validator for CronTimeZone. -class CronTimeZoneValidator < ActiveModel::EachValidator - def validate_each(record, attribute, value) - cron_parser = Gitlab::Ci::CronParser.new(record.cron, record.cron_time_zone) - record.errors.add(attribute, " is invalid syntax") unless cron_parser.cron_time_zone_valid? - end -end diff --git a/app/validators/cron_timezone_validator.rb b/app/validators/cron_timezone_validator.rb new file mode 100644 index 00000000000..542c7d006ad --- /dev/null +++ b/app/validators/cron_timezone_validator.rb @@ -0,0 +1,9 @@ +# CronTimezoneValidator +# +# Custom validator for CronTimezone. +class CronTimezoneValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + cron_parser = Gitlab::Ci::CronParser.new(record.cron, record.cron_timezone) + record.errors.add(attribute, " is invalid syntax") unless cron_parser.cron_timezone_valid? + end +end diff --git a/app/validators/cron_validator.rb b/app/validators/cron_validator.rb index cc07011d56b..981fade47a6 100644 --- a/app/validators/cron_validator.rb +++ b/app/validators/cron_validator.rb @@ -3,7 +3,7 @@ # Custom validator for Cron. class CronValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) - cron_parser = Gitlab::Ci::CronParser.new(record.cron, record.cron_time_zone) + cron_parser = Gitlab::Ci::CronParser.new(record.cron, record.cron_timezone) record.errors.add(attribute, " is invalid syntax") unless cron_parser.cron_valid? end end diff --git a/db/migrate/20170329095907_create_ci_trigger_schedules.rb b/db/migrate/20170329095907_create_ci_trigger_schedules.rb index 3dcd05175c0..cfcfa27ebb5 100644 --- a/db/migrate/20170329095907_create_ci_trigger_schedules.rb +++ b/db/migrate/20170329095907_create_ci_trigger_schedules.rb @@ -3,9 +3,7 @@ class CreateCiTriggerSchedules < ActiveRecord::Migration DOWNTIME = false - disable_ddl_transaction! - - def up + def change create_table :ci_trigger_schedules do |t| t.integer "project_id" t.integer "trigger_id", null: false @@ -13,17 +11,11 @@ class CreateCiTriggerSchedules < ActiveRecord::Migration t.datetime "created_at" t.datetime "updated_at" t.string "cron" - t.string "cron_time_zone" + t.string "cron_timezone" t.datetime "next_run_at" end - add_index :ci_trigger_schedules, ["next_run_at"], name: "index_ci_trigger_schedules_on_next_run_at", using: :btree - add_index :ci_trigger_schedules, ["project_id"], name: "index_ci_trigger_schedules_on_project_id", using: :btree - add_concurrent_foreign_key :ci_trigger_schedules, :ci_triggers, column: :trigger_id, on_delete: :cascade - end - - def down - remove_foreign_key :ci_trigger_schedules, column: :trigger_id - drop_table :ci_trigger_schedules + add_index :ci_trigger_schedules, :next_run_at + add_index :ci_trigger_schedules, :project_id end end diff --git a/db/migrate/20170404163427_add_trigger_id_foreign_key.rb b/db/migrate/20170404163427_add_trigger_id_foreign_key.rb new file mode 100644 index 00000000000..6679a95ca11 --- /dev/null +++ b/db/migrate/20170404163427_add_trigger_id_foreign_key.rb @@ -0,0 +1,15 @@ +class AddTriggerIdForeignKey < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_foreign_key :ci_trigger_schedules, :ci_triggers, column: :trigger_id, on_delete: :cascade + end + + def down + remove_foreign_key :ci_trigger_schedules, column: :trigger_id + end +end diff --git a/db/schema.rb b/db/schema.rb index 7d9f969c2e1..a564a4b6a12 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -307,7 +307,7 @@ ActiveRecord::Schema.define(version: 20170405080720) do t.datetime "created_at" t.datetime "updated_at" t.string "cron" - t.string "cron_time_zone" + t.string "cron_timezone" t.datetime "next_run_at" end diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb index 69dd8ad0fce..d1877b76598 100644 --- a/lib/gitlab/ci/cron_parser.rb +++ b/lib/gitlab/ci/cron_parser.rb @@ -4,13 +4,13 @@ module Gitlab VALID_SYNTAX_SAMPLE_TIME_ZONE = 'UTC'.freeze VALID_SYNTAX_SAMPLE_CRON = '* * * * *'.freeze - def initialize(cron, cron_time_zone = 'UTC') + def initialize(cron, cron_timezone = 'UTC') @cron = cron - @cron_time_zone = cron_time_zone + @cron_timezone = cron_timezone end def next_time_from(time) - cron_line = try_parse_cron(@cron, @cron_time_zone) + cron_line = try_parse_cron(@cron, @cron_timezone) cron_line.next_time(time).in_time_zone(Time.zone) if cron_line.present? end @@ -18,14 +18,14 @@ module Gitlab try_parse_cron(@cron, VALID_SYNTAX_SAMPLE_TIME_ZONE).present? end - def cron_time_zone_valid? - try_parse_cron(VALID_SYNTAX_SAMPLE_CRON, @cron_time_zone).present? + def cron_timezone_valid? + try_parse_cron(VALID_SYNTAX_SAMPLE_CRON, @cron_timezone).present? end private - def try_parse_cron(cron, cron_time_zone) - Rufus::Scheduler.parse("#{cron} #{cron_time_zone}") + def try_parse_cron(cron, cron_timezone) + Rufus::Scheduler.parse("#{cron} #{cron_timezone}") rescue # noop end diff --git a/spec/factories/ci/trigger_schedules.rb b/spec/factories/ci/trigger_schedules.rb index 9c16d45b49a..49d2b29727f 100644 --- a/spec/factories/ci/trigger_schedules.rb +++ b/spec/factories/ci/trigger_schedules.rb @@ -2,7 +2,7 @@ FactoryGirl.define do factory :ci_trigger_schedule, class: Ci::TriggerSchedule do trigger factory: :ci_trigger_for_trigger_schedule cron '0 1 * * *' - cron_time_zone Gitlab::Ci::CronParser::VALID_SYNTAX_SAMPLE_TIME_ZONE + cron_timezone Gitlab::Ci::CronParser::VALID_SYNTAX_SAMPLE_TIME_ZONE after(:build) do |trigger_schedule, evaluator| trigger_schedule.update!(project: trigger_schedule.trigger.project) @@ -16,17 +16,17 @@ FactoryGirl.define do trait :nightly do cron '0 1 * * *' - cron_time_zone Gitlab::Ci::CronParser::VALID_SYNTAX_SAMPLE_TIME_ZONE + cron_timezone Gitlab::Ci::CronParser::VALID_SYNTAX_SAMPLE_TIME_ZONE end trait :weekly do cron '0 1 * * 6' - cron_time_zone Gitlab::Ci::CronParser::VALID_SYNTAX_SAMPLE_TIME_ZONE + cron_timezone Gitlab::Ci::CronParser::VALID_SYNTAX_SAMPLE_TIME_ZONE end trait :monthly do cron '0 1 22 * *' - cron_time_zone Gitlab::Ci::CronParser::VALID_SYNTAX_SAMPLE_TIME_ZONE + cron_timezone Gitlab::Ci::CronParser::VALID_SYNTAX_SAMPLE_TIME_ZONE end end end diff --git a/spec/lib/gitlab/ci/cron_parser_spec.rb b/spec/lib/gitlab/ci/cron_parser_spec.rb index 1cdd8c1d2e7..b07b84027fc 100644 --- a/spec/lib/gitlab/ci/cron_parser_spec.rb +++ b/spec/lib/gitlab/ci/cron_parser_spec.rb @@ -6,12 +6,12 @@ describe Gitlab::Ci::CronParser do end describe '#next_time_from' do - subject { described_class.new(cron, cron_time_zone).next_time_from(Time.now) } + subject { described_class.new(cron, cron_timezone).next_time_from(Time.now) } - context 'when cron and cron_time_zone are valid' do + context 'when cron and cron_timezone are valid' do context 'when specific time' do let(:cron) { '3 4 5 6 *' } - let(:cron_time_zone) { 'UTC' } + let(:cron_timezone) { 'UTC' } it_behaves_like "returns time in the future" @@ -25,7 +25,7 @@ describe Gitlab::Ci::CronParser do context 'when specific day of week' do let(:cron) { '* * * * 0' } - let(:cron_time_zone) { 'UTC' } + let(:cron_timezone) { 'UTC' } it_behaves_like "returns time in the future" @@ -36,7 +36,7 @@ describe Gitlab::Ci::CronParser do context 'when slash used' do let(:cron) { '*/10 */6 */10 */10 *' } - let(:cron_time_zone) { 'UTC' } + let(:cron_timezone) { 'UTC' } it_behaves_like "returns time in the future" @@ -50,7 +50,7 @@ describe Gitlab::Ci::CronParser do context 'when range used' do let(:cron) { '0,20,40 * 1-5 * *' } - let(:cron_time_zone) { 'UTC' } + let(:cron_timezone) { 'UTC' } it_behaves_like "returns time in the future" @@ -60,9 +60,9 @@ describe Gitlab::Ci::CronParser do end end - context 'when cron_time_zone is US/Pacific' do + context 'when cron_timezone is US/Pacific' do let(:cron) { '0 0 * * *' } - let(:cron_time_zone) { 'US/Pacific' } + let(:cron_timezone) { 'US/Pacific' } it_behaves_like "returns time in the future" @@ -72,9 +72,9 @@ describe Gitlab::Ci::CronParser do end end - context 'when cron and cron_time_zone are invalid' do + context 'when cron and cron_timezone are invalid' do let(:cron) { 'invalid_cron' } - let(:cron_time_zone) { 'invalid_cron_time_zone' } + let(:cron_timezone) { 'invalid_cron_timezone' } it 'returns nil' do is_expected.to be_nil @@ -98,17 +98,17 @@ describe Gitlab::Ci::CronParser do end end - describe '#cron_time_zone_valid?' do - subject { described_class.new(Gitlab::Ci::CronParser::VALID_SYNTAX_SAMPLE_CRON, cron_time_zone).cron_time_zone_valid? } + describe '#cron_timezone_valid?' do + subject { described_class.new(Gitlab::Ci::CronParser::VALID_SYNTAX_SAMPLE_CRON, cron_timezone).cron_timezone_valid? } context 'when cron is valid' do - let(:cron_time_zone) { 'Europe/Istanbul' } + let(:cron_timezone) { 'Europe/Istanbul' } it { is_expected.to eq(true) } end context 'when cron is invalid' do - let(:cron_time_zone) { 'Invalid-zone' } + let(:cron_timezone) { 'Invalid-zone' } it { is_expected.to eq(false) } end diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml index 42082ff3dee..0c43c5662e8 100644 --- a/spec/lib/gitlab/import_export/safe_model_attributes.yml +++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml @@ -249,7 +249,7 @@ Ci::TriggerSchedule: - created_at - updated_at - cron -- cron_time_zone +- cron_timezone - next_run_at DeployKey: - id diff --git a/spec/models/ci/trigger_schedule_spec.rb b/spec/models/ci/trigger_schedule_spec.rb index 9a4bf122bf0..fc01d702f65 100644 --- a/spec/models/ci/trigger_schedule_spec.rb +++ b/spec/models/ci/trigger_schedule_spec.rb @@ -13,7 +13,7 @@ describe Ci::TriggerSchedule, models: true do end it 'updates next_run_at' do - next_time = Gitlab::Ci::CronParser.new(trigger_schedule.cron, trigger_schedule.cron_time_zone).next_time_from(Time.now) + next_time = Gitlab::Ci::CronParser.new(trigger_schedule.cron, trigger_schedule.cron_timezone).next_time_from(Time.now) expect(Ci::TriggerSchedule.last.next_run_at).to eq(next_time) end end diff --git a/spec/workers/trigger_schedule_worker_spec.rb b/spec/workers/trigger_schedule_worker_spec.rb index 75a98e42ac5..9d8fd9305ca 100644 --- a/spec/workers/trigger_schedule_worker_spec.rb +++ b/spec/workers/trigger_schedule_worker_spec.rb @@ -23,7 +23,7 @@ describe TriggerScheduleWorker do end it 'updates next_run_at' do - next_time = Gitlab::Ci::CronParser.new(trigger_schedule.cron, trigger_schedule.cron_time_zone).next_time_from(Time.now) + next_time = Gitlab::Ci::CronParser.new(trigger_schedule.cron, trigger_schedule.cron_timezone).next_time_from(Time.now) expect(Ci::TriggerSchedule.last.next_run_at).to eq(next_time) end end -- cgit v1.2.1 From 1a244c64ebe7fae8b13ffc8e663118265eb76f5c Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 5 Apr 2017 01:57:06 +0900 Subject: Remove next_run_at: nil from trigger_schedule_spec --- spec/models/ci/trigger_schedule_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/models/ci/trigger_schedule_spec.rb b/spec/models/ci/trigger_schedule_spec.rb index fc01d702f65..01db6b49fee 100644 --- a/spec/models/ci/trigger_schedule_spec.rb +++ b/spec/models/ci/trigger_schedule_spec.rb @@ -6,7 +6,7 @@ describe Ci::TriggerSchedule, models: true do it { is_expected.to respond_to :ref } describe '#schedule_next_run!' do - let(:trigger_schedule) { create(:ci_trigger_schedule, :nightly, next_run_at: nil) } + let(:trigger_schedule) { create(:ci_trigger_schedule, :nightly) } before do trigger_schedule.schedule_next_run! -- cgit v1.2.1 From cc6ac794adfd66de4aa3ed28db7bf89cb6b46cb2 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 5 Apr 2017 01:57:41 +0900 Subject: Define next_time as let in trigger_schedule_spec --- spec/models/ci/trigger_schedule_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/models/ci/trigger_schedule_spec.rb b/spec/models/ci/trigger_schedule_spec.rb index 01db6b49fee..53c1349d9f8 100644 --- a/spec/models/ci/trigger_schedule_spec.rb +++ b/spec/models/ci/trigger_schedule_spec.rb @@ -7,13 +7,13 @@ describe Ci::TriggerSchedule, models: true do describe '#schedule_next_run!' do let(:trigger_schedule) { create(:ci_trigger_schedule, :nightly) } + let(:next_time) { Gitlab::Ci::CronParser.new(trigger_schedule.cron, trigger_schedule.cron_timezone).next_time_from(Time.now) } before do trigger_schedule.schedule_next_run! end it 'updates next_run_at' do - next_time = Gitlab::Ci::CronParser.new(trigger_schedule.cron, trigger_schedule.cron_timezone).next_time_from(Time.now) expect(Ci::TriggerSchedule.last.next_run_at).to eq(next_time) end end -- cgit v1.2.1 From 3b58966b6c9f0cb183e8ec3652ec0cd7e20ce99e Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 5 Apr 2017 01:58:42 +0900 Subject: Use parenthesis for respond_to :ref --- spec/models/ci/trigger_schedule_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/models/ci/trigger_schedule_spec.rb b/spec/models/ci/trigger_schedule_spec.rb index 53c1349d9f8..22c0790688c 100644 --- a/spec/models/ci/trigger_schedule_spec.rb +++ b/spec/models/ci/trigger_schedule_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' describe Ci::TriggerSchedule, models: true do it { is_expected.to belong_to(:project) } it { is_expected.to belong_to(:trigger) } - it { is_expected.to respond_to :ref } + it { is_expected.to respond_to(:ref) } describe '#schedule_next_run!' do let(:trigger_schedule) { create(:ci_trigger_schedule, :nightly) } -- cgit v1.2.1 From 31bd3962d7e37f6143721ab0d09d94daf9dd0481 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 5 Apr 2017 01:59:24 +0900 Subject: Add empty line in cron_parser.rb --- lib/gitlab/ci/cron_parser.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb index d1877b76598..59272dbeca5 100644 --- a/lib/gitlab/ci/cron_parser.rb +++ b/lib/gitlab/ci/cron_parser.rb @@ -31,4 +31,4 @@ module Gitlab end end end -end \ No newline at end of file +end -- cgit v1.2.1 From 7b09a484f1ecc422ce14602d9ee15e5a59100264 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 5 Apr 2017 02:31:15 +0900 Subject: Fix unnecessary changes in schema.rb --- db/schema.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/db/schema.rb b/db/schema.rb index a564a4b6a12..58425d637c1 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -61,6 +61,7 @@ ActiveRecord::Schema.define(version: 20170405080720) do t.boolean "shared_runners_enabled", default: true, null: false t.integer "max_artifacts_size", default: 100, null: false t.string "runners_registration_token" + t.integer "max_pages_size", default: 100, null: false t.boolean "require_two_factor_authentication", default: false t.integer "two_factor_grace_period", default: 48 t.boolean "metrics_enabled", default: false @@ -110,7 +111,6 @@ ActiveRecord::Schema.define(version: 20170405080720) do t.string "plantuml_url" t.boolean "plantuml_enabled" t.integer "terminal_max_session_time", default: 0, null: false - t.integer "max_pages_size", default: 100, null: false t.string "default_artifacts_expire_in", default: "0", null: false t.integer "unique_ips_limit_per_user" t.integer "unique_ips_limit_time_window" @@ -704,8 +704,8 @@ ActiveRecord::Schema.define(version: 20170405080720) do t.integer "visibility_level", default: 20, null: false t.boolean "request_access_enabled", default: false, null: false t.datetime "deleted_at" - t.boolean "lfs_enabled" t.text "description_html" + t.boolean "lfs_enabled" t.integer "parent_id" end @@ -1257,8 +1257,8 @@ ActiveRecord::Schema.define(version: 20170405080720) do t.datetime "otp_grace_period_started_at" t.boolean "ldap_email", default: false, null: false t.boolean "external", default: false - t.string "organization" t.string "incoming_email_token" + t.string "organization" t.boolean "authorized_projects_populated" t.boolean "ghost" t.boolean "notified_of_own_activity" -- cgit v1.2.1 From cb082ae291ba13120bfffeac5cd5a7c4532142ff Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 5 Apr 2017 16:47:20 +0900 Subject: Improve instantiate recursion in cron_parser.rb --- lib/gitlab/ci/cron_parser.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb index 59272dbeca5..a3cc350ef22 100644 --- a/lib/gitlab/ci/cron_parser.rb +++ b/lib/gitlab/ci/cron_parser.rb @@ -10,8 +10,8 @@ module Gitlab end def next_time_from(time) - cron_line = try_parse_cron(@cron, @cron_timezone) - cron_line.next_time(time).in_time_zone(Time.zone) if cron_line.present? + @cron_line ||= try_parse_cron(@cron, @cron_timezone) + @cron_line.next_time(time).in_time_zone(Time.zone) if @cron_line.present? end def cron_valid? -- cgit v1.2.1 From c2c3ee1bf9101b68b59685fa046759729eeadda1 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 5 Apr 2017 17:48:28 +0900 Subject: Clean up trigger_schedule_worker_spec.rb --- spec/workers/trigger_schedule_worker_spec.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/spec/workers/trigger_schedule_worker_spec.rb b/spec/workers/trigger_schedule_worker_spec.rb index 9d8fd9305ca..3997369489f 100644 --- a/spec/workers/trigger_schedule_worker_spec.rb +++ b/spec/workers/trigger_schedule_worker_spec.rb @@ -9,6 +9,7 @@ describe TriggerScheduleWorker do context 'when there is a scheduled trigger within next_run_at' do let!(:trigger_schedule) { create(:ci_trigger_schedule, :nightly, :force_triggable) } + let(:next_time) { Gitlab::Ci::CronParser.new(trigger_schedule.cron, trigger_schedule.cron_timezone).next_time_from(Time.now) } before do worker.perform @@ -23,7 +24,6 @@ describe TriggerScheduleWorker do end it 'updates next_run_at' do - next_time = Gitlab::Ci::CronParser.new(trigger_schedule.cron, trigger_schedule.cron_timezone).next_time_from(Time.now) expect(Ci::TriggerSchedule.last.next_run_at).to eq(next_time) end end @@ -35,12 +35,12 @@ describe TriggerScheduleWorker do worker.perform end - it 'do not create a new pipeline' do + it 'does not create a new pipeline' do expect(Ci::Pipeline.count).to eq(0) end - it 'do not reschedule next_run_at' do - expect(Ci::TriggerSchedule.last.next_run_at).to eq(trigger_schedule.next_run_at) + it 'does not update next_run_at' do + expect(trigger_schedule.next_run_at).to eq(Ci::TriggerSchedule.last.next_run_at) end end end -- cgit v1.2.1 From 12a5380f5cb2ce8b652344f053b2333f6f5e80a9 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Wed, 5 Apr 2017 18:29:15 +0900 Subject: Implement a offset calculation on cron_parser_spec --- spec/lib/gitlab/ci/cron_parser_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/lib/gitlab/ci/cron_parser_spec.rb b/spec/lib/gitlab/ci/cron_parser_spec.rb index b07b84027fc..0864bc7258d 100644 --- a/spec/lib/gitlab/ci/cron_parser_spec.rb +++ b/spec/lib/gitlab/ci/cron_parser_spec.rb @@ -67,7 +67,7 @@ describe Gitlab::Ci::CronParser do it_behaves_like "returns time in the future" it 'converts time in server time zone' do - expect(subject.hour).to eq(7) + expect(subject.hour).to eq((Time.zone.now.in_time_zone(cron_timezone).utc_offset / 60 / 60).abs) end end end -- cgit v1.2.1 From 48e07eab5755770bff9d5ee1aca33526e4120637 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 6 Apr 2017 18:52:48 +0900 Subject: Improve trigger_schedule.rb --- app/models/ci/trigger_schedule.rb | 11 +++-- spec/factories/ci/trigger_schedules.rb | 6 --- spec/models/ci/trigger_schedule_spec.rb | 68 +++++++++++++++++++++++++--- spec/workers/trigger_schedule_worker_spec.rb | 23 +++++++++- 4 files changed, 91 insertions(+), 17 deletions(-) diff --git a/app/models/ci/trigger_schedule.rb b/app/models/ci/trigger_schedule.rb index d18dbea284e..256e609f0d1 100644 --- a/app/models/ci/trigger_schedule.rb +++ b/app/models/ci/trigger_schedule.rb @@ -15,11 +15,16 @@ module Ci validates :cron_timezone, cron_timezone: true, presence: { unless: :importing? } validates :ref, presence: { unless: :importing? } - after_create :schedule_next_run! + before_save :set_next_run_at + + def set_next_run_at + self.next_run_at = Gitlab::Ci::CronParser.new(cron, cron_timezone).next_time_from(Time.now) + end def schedule_next_run! - next_time = Gitlab::Ci::CronParser.new(cron, cron_timezone).next_time_from(Time.now) - update!(next_run_at: next_time) if next_time.present? + save! # with set_next_run_at + rescue ActiveRecord::RecordInvalid => invalid + update_attribute(:next_run_at, nil) # update without validation end end end diff --git a/spec/factories/ci/trigger_schedules.rb b/spec/factories/ci/trigger_schedules.rb index 49d2b29727f..315bce16995 100644 --- a/spec/factories/ci/trigger_schedules.rb +++ b/spec/factories/ci/trigger_schedules.rb @@ -8,12 +8,6 @@ FactoryGirl.define do trigger_schedule.update!(project: trigger_schedule.trigger.project) end - trait :force_triggable do - after(:create) do |trigger_schedule, evaluator| - trigger_schedule.update!(next_run_at: trigger_schedule.next_run_at - 1.year) - end - end - trait :nightly do cron '0 1 * * *' cron_timezone Gitlab::Ci::CronParser::VALID_SYNTAX_SAMPLE_TIME_ZONE diff --git a/spec/models/ci/trigger_schedule_spec.rb b/spec/models/ci/trigger_schedule_spec.rb index 22c0790688c..75d21541cee 100644 --- a/spec/models/ci/trigger_schedule_spec.rb +++ b/spec/models/ci/trigger_schedule_spec.rb @@ -5,16 +5,72 @@ describe Ci::TriggerSchedule, models: true do it { is_expected.to belong_to(:trigger) } it { is_expected.to respond_to(:ref) } + describe '#set_next_run_at' do + context 'when creates new TriggerSchedule' do + before do + trigger_schedule = create(:ci_trigger_schedule, :nightly) + @expected_next_run_at = Gitlab::Ci::CronParser.new(trigger_schedule.cron, trigger_schedule.cron_timezone) + .next_time_from(Time.now) + end + + it 'updates next_run_at automatically' do + expect(Ci::TriggerSchedule.last.next_run_at).to eq(@expected_next_run_at) + end + end + + context 'when updates cron of exsisted TriggerSchedule' do + before do + trigger_schedule = create(:ci_trigger_schedule, :nightly) + new_cron = '0 0 1 1 *' + trigger_schedule.update!(cron: new_cron) # Subject + @expected_next_run_at = Gitlab::Ci::CronParser.new(new_cron, trigger_schedule.cron_timezone) + .next_time_from(Time.now) + end + + it 'updates next_run_at automatically' do + expect(Ci::TriggerSchedule.last.next_run_at).to eq(@expected_next_run_at) + end + end + end + describe '#schedule_next_run!' do - let(:trigger_schedule) { create(:ci_trigger_schedule, :nightly) } - let(:next_time) { Gitlab::Ci::CronParser.new(trigger_schedule.cron, trigger_schedule.cron_timezone).next_time_from(Time.now) } + context 'when reschedules after 10 days from now' do + before do + trigger_schedule = create(:ci_trigger_schedule, :nightly) + time_future = Time.now + 10.days + allow(Time).to receive(:now).and_return(time_future) + trigger_schedule.schedule_next_run! # Subject + @expected_next_run_at = Gitlab::Ci::CronParser.new(trigger_schedule.cron, trigger_schedule.cron_timezone) + .next_time_from(time_future) + end - before do - trigger_schedule.schedule_next_run! + it 'points to proper next_run_at' do + expect(Ci::TriggerSchedule.last.next_run_at).to eq(@expected_next_run_at) + end end - it 'updates next_run_at' do - expect(Ci::TriggerSchedule.last.next_run_at).to eq(next_time) + context 'when cron is invalid' do + before do + trigger_schedule = create(:ci_trigger_schedule, :nightly) + trigger_schedule.cron = 'Invalid-cron' + trigger_schedule.schedule_next_run! # Subject + end + + it 'sets nil to next_run_at' do + expect(Ci::TriggerSchedule.last.next_run_at).to be_nil + end + end + + context 'when cron_timezone is invalid' do + before do + trigger_schedule = create(:ci_trigger_schedule, :nightly) + trigger_schedule.cron_timezone = 'Invalid-cron_timezone' + trigger_schedule.schedule_next_run! # Subject + end + + it 'sets nil to next_run_at' do + expect(Ci::TriggerSchedule.last.next_run_at).to be_nil + end end end end diff --git a/spec/workers/trigger_schedule_worker_spec.rb b/spec/workers/trigger_schedule_worker_spec.rb index 3997369489f..c48f1406c34 100644 --- a/spec/workers/trigger_schedule_worker_spec.rb +++ b/spec/workers/trigger_schedule_worker_spec.rb @@ -8,10 +8,12 @@ describe TriggerScheduleWorker do end context 'when there is a scheduled trigger within next_run_at' do - let!(:trigger_schedule) { create(:ci_trigger_schedule, :nightly, :force_triggable) } - let(:next_time) { Gitlab::Ci::CronParser.new(trigger_schedule.cron, trigger_schedule.cron_timezone).next_time_from(Time.now) } + let!(:trigger_schedule) { create(:ci_trigger_schedule, :nightly) } + let(:next_time) { Gitlab::Ci::CronParser.new(trigger_schedule.cron, trigger_schedule.cron_timezone).next_time_from(@time_future) } before do + @time_future = Time.now + 10.days + allow(Time).to receive(:now).and_return(@time_future) worker.perform end @@ -43,4 +45,21 @@ describe TriggerScheduleWorker do expect(trigger_schedule.next_run_at).to eq(Ci::TriggerSchedule.last.next_run_at) end end + + context 'when next_run_at is nil' do + let!(:trigger_schedule) { create(:ci_trigger_schedule, :nightly) } + + before do + trigger_schedule.update_attribute(:next_run_at, nil) + worker.perform + end + + it 'does not create a new pipeline' do + expect(Ci::Pipeline.count).to eq(0) + end + + it 'does not update next_run_at' do + expect(trigger_schedule.next_run_at).to eq(Ci::TriggerSchedule.last.next_run_at) + end + end end -- cgit v1.2.1 From 059ec792cbd670d2b716c111b6faa50e5782ae9f Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 6 Apr 2017 18:55:16 +0900 Subject: Use be_pending --- spec/workers/trigger_schedule_worker_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/workers/trigger_schedule_worker_spec.rb b/spec/workers/trigger_schedule_worker_spec.rb index c48f1406c34..0bb0bcc5c42 100644 --- a/spec/workers/trigger_schedule_worker_spec.rb +++ b/spec/workers/trigger_schedule_worker_spec.rb @@ -22,7 +22,7 @@ describe TriggerScheduleWorker do end it 'creates a new pipeline' do - expect(Ci::Pipeline.last.status).to eq('pending') + expect(Ci::Pipeline.last).to be_pending end it 'updates next_run_at' do -- cgit v1.2.1 From fff6afbad1180cb39fd0a8d9032de91397ba6471 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 6 Apr 2017 19:13:29 +0900 Subject: Use change direction in spec --- spec/workers/trigger_schedule_worker_spec.rb | 33 +++++++++++----------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/spec/workers/trigger_schedule_worker_spec.rb b/spec/workers/trigger_schedule_worker_spec.rb index 0bb0bcc5c42..151e1c2f7b9 100644 --- a/spec/workers/trigger_schedule_worker_spec.rb +++ b/spec/workers/trigger_schedule_worker_spec.rb @@ -8,58 +8,51 @@ describe TriggerScheduleWorker do end context 'when there is a scheduled trigger within next_run_at' do - let!(:trigger_schedule) { create(:ci_trigger_schedule, :nightly) } - let(:next_time) { Gitlab::Ci::CronParser.new(trigger_schedule.cron, trigger_schedule.cron_timezone).next_time_from(@time_future) } - before do - @time_future = Time.now + 10.days - allow(Time).to receive(:now).and_return(@time_future) - worker.perform + trigger_schedule = create(:ci_trigger_schedule, :nightly) + time_future = Time.now + 10.days + allow(Time).to receive(:now).and_return(time_future) + @next_time = Gitlab::Ci::CronParser.new(trigger_schedule.cron, trigger_schedule.cron_timezone).next_time_from(time_future) end it 'creates a new trigger request' do - expect(trigger_schedule.trigger.id).to eq(Ci::TriggerRequest.first.trigger_id) + expect { worker.perform }.to change { Ci::TriggerRequest.count }.by(1) end it 'creates a new pipeline' do + expect { worker.perform }.to change { Ci::Pipeline.count }.by(1) expect(Ci::Pipeline.last).to be_pending end it 'updates next_run_at' do - expect(Ci::TriggerSchedule.last.next_run_at).to eq(next_time) + expect { worker.perform }.to change { Ci::TriggerSchedule.last.next_run_at }.to(@next_time) end end context 'when there are no scheduled triggers within next_run_at' do - let!(:trigger_schedule) { create(:ci_trigger_schedule, :nightly) } - - before do - worker.perform - end + before { create(:ci_trigger_schedule, :nightly) } it 'does not create a new pipeline' do - expect(Ci::Pipeline.count).to eq(0) + expect { worker.perform }.not_to change { Ci::Pipeline.count } end it 'does not update next_run_at' do - expect(trigger_schedule.next_run_at).to eq(Ci::TriggerSchedule.last.next_run_at) + expect { worker.perform }.not_to change { Ci::TriggerSchedule.last.next_run_at } end end context 'when next_run_at is nil' do - let!(:trigger_schedule) { create(:ci_trigger_schedule, :nightly) } - before do + trigger_schedule = create(:ci_trigger_schedule, :nightly) trigger_schedule.update_attribute(:next_run_at, nil) - worker.perform end it 'does not create a new pipeline' do - expect(Ci::Pipeline.count).to eq(0) + expect { worker.perform }.not_to change { Ci::Pipeline.count } end it 'does not update next_run_at' do - expect(trigger_schedule.next_run_at).to eq(Ci::TriggerSchedule.last.next_run_at) + expect { worker.perform }.not_to change { Ci::TriggerSchedule.last.next_run_at } end end end -- cgit v1.2.1 From 9441c01484e668892d06f387fc0f85fe2d4ff4b4 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Thu, 6 Apr 2017 21:01:30 +0900 Subject: Fix rubocop --- app/models/ci/trigger_schedule.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/ci/trigger_schedule.rb b/app/models/ci/trigger_schedule.rb index 256e609f0d1..10ea381ee31 100644 --- a/app/models/ci/trigger_schedule.rb +++ b/app/models/ci/trigger_schedule.rb @@ -23,7 +23,7 @@ module Ci def schedule_next_run! save! # with set_next_run_at - rescue ActiveRecord::RecordInvalid => invalid + rescue ActiveRecord::RecordInvalid update_attribute(:next_run_at, nil) # update without validation end end -- cgit v1.2.1 From b7ce488df57ad2c0e2e509c906747cc31c5bef1f Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Wed, 29 Mar 2017 21:04:05 -0500 Subject: Recent search history for issues Fixes https://gitlab.com/gitlab-org/gitlab-ce/issues/27262 --- .../components/recent_searches_dropdown_content.js | 87 +++++++++++ .../javascripts/filtered_search/dropdown_hint.js | 2 +- .../javascripts/filtered_search/dropdown_utils.js | 4 +- .../javascripts/filtered_search/event_hub.js | 3 + .../filtered_search/filtered_search_manager.js | 89 +++++++++-- .../filtered_search/recent_searches_root.js | 59 ++++++++ .../services/recent_searches_service.js | 26 ++++ .../stores/recent_searches_store.js | 23 +++ app/assets/stylesheets/framework/dropdowns.scss | 9 +- app/assets/stylesheets/framework/filters.scss | 135 +++++++++++++++-- app/helpers/dropdowns_helper.rb | 4 +- app/views/shared/issuable/_filter.html.haml | 8 +- app/views/shared/issuable/_search_bar.html.haml | 162 ++++++++++---------- .../unreleased/27262-issue-recent-searches.yml | 4 + spec/features/boards/boards_spec.rb | 2 +- spec/features/boards/modal_filter_spec.rb | 2 +- .../filtered_search/dropdown_assignee_spec.rb | 2 +- .../issues/filtered_search/dropdown_author_spec.rb | 2 +- .../issues/filtered_search/dropdown_label_spec.rb | 2 +- .../filtered_search/dropdown_milestone_spec.rb | 2 +- .../issues/filtered_search/filter_issues_spec.rb | 4 +- .../issues/filtered_search/recent_searches_spec.rb | 92 ++++++++++++ .../issues/filtered_search/search_bar_spec.rb | 8 +- spec/features/merge_requests/reset_filters_spec.rb | 2 +- .../recent_searches_dropdown_content_spec.js | 166 +++++++++++++++++++++ .../filtered_search_manager_spec.js | 6 +- .../services/recent_searches_service_spec.js | 56 +++++++ .../stores/recent_searches_store_spec.js | 59 ++++++++ spec/support/filtered_search_helpers.rb | 18 ++- 29 files changed, 906 insertions(+), 132 deletions(-) create mode 100644 app/assets/javascripts/filtered_search/components/recent_searches_dropdown_content.js create mode 100644 app/assets/javascripts/filtered_search/event_hub.js create mode 100644 app/assets/javascripts/filtered_search/recent_searches_root.js create mode 100644 app/assets/javascripts/filtered_search/services/recent_searches_service.js create mode 100644 app/assets/javascripts/filtered_search/stores/recent_searches_store.js create mode 100644 changelogs/unreleased/27262-issue-recent-searches.yml create mode 100644 spec/features/issues/filtered_search/recent_searches_spec.rb create mode 100644 spec/javascripts/filtered_search/components/recent_searches_dropdown_content_spec.js create mode 100644 spec/javascripts/filtered_search/services/recent_searches_service_spec.js create mode 100644 spec/javascripts/filtered_search/stores/recent_searches_store_spec.js diff --git a/app/assets/javascripts/filtered_search/components/recent_searches_dropdown_content.js b/app/assets/javascripts/filtered_search/components/recent_searches_dropdown_content.js new file mode 100644 index 00000000000..9126422b335 --- /dev/null +++ b/app/assets/javascripts/filtered_search/components/recent_searches_dropdown_content.js @@ -0,0 +1,87 @@ +import eventHub from '../event_hub'; + +export default { + name: 'RecentSearchesDropdownContent', + + props: { + items: { + type: Array, + required: true, + }, + }, + + computed: { + processedItems() { + return this.items.map((item) => { + const { tokens, searchToken } + = gl.FilteredSearchTokenizer.processTokens(item); + + const resultantTokens = tokens.map(token => ({ + prefix: `${token.key}:`, + suffix: `${token.symbol}${token.value}`, + })); + + return { + text: item, + tokens: resultantTokens, + searchToken, + }; + }); + }, + hasItems() { + return this.items.length > 0; + }, + }, + + methods: { + onItemActivated(text) { + eventHub.$emit('recentSearchesItemSelected', text); + }, + onRequestClearRecentSearches(e) { + // Stop the dropdown from closing + e.stopPropagation(); + + eventHub.$emit('requestClearRecentSearches'); + }, + }, + + template: ` +
+
    +
  • + +
    • +
  • +
  • + +
    • +
+
+ You don't have any recent searches +
+
+ `, +}; diff --git a/app/assets/javascripts/filtered_search/dropdown_hint.js b/app/assets/javascripts/filtered_search/dropdown_hint.js index 98dcb697af9..64d7153e547 100644 --- a/app/assets/javascripts/filtered_search/dropdown_hint.js +++ b/app/assets/javascripts/filtered_search/dropdown_hint.js @@ -56,7 +56,7 @@ require('./filtered_search_dropdown'); renderContent() { const dropdownData = []; - [].forEach.call(this.input.closest('.filtered-search-input-container').querySelectorAll('.dropdown-menu'), (dropdownMenu) => { + [].forEach.call(this.input.closest('.filtered-search-box-input-container').querySelectorAll('.dropdown-menu'), (dropdownMenu) => { const { icon, hint, tag, type } = dropdownMenu.dataset; if (icon && hint && tag) { dropdownData.push( diff --git a/app/assets/javascripts/filtered_search/dropdown_utils.js b/app/assets/javascripts/filtered_search/dropdown_utils.js index 432b0c0dfd2..6c5c20447f7 100644 --- a/app/assets/javascripts/filtered_search/dropdown_utils.js +++ b/app/assets/javascripts/filtered_search/dropdown_utils.js @@ -129,7 +129,9 @@ import FilteredSearchContainer from './container'; } }); - return values.join(' '); + return values + .map(value => value.trim()) + .join(' '); } static getSearchInput(filteredSearchInput) { diff --git a/app/assets/javascripts/filtered_search/event_hub.js b/app/assets/javascripts/filtered_search/event_hub.js new file mode 100644 index 00000000000..0948c2e5352 --- /dev/null +++ b/app/assets/javascripts/filtered_search/event_hub.js @@ -0,0 +1,3 @@ +import Vue from 'vue'; + +export default new Vue(); diff --git a/app/assets/javascripts/filtered_search/filtered_search_manager.js b/app/assets/javascripts/filtered_search/filtered_search_manager.js index 22352950452..2a8a6b81b3f 100644 --- a/app/assets/javascripts/filtered_search/filtered_search_manager.js +++ b/app/assets/javascripts/filtered_search/filtered_search_manager.js @@ -1,18 +1,56 @@ +/* global Flash */ + import FilteredSearchContainer from './container'; +import RecentSearchesRoot from './recent_searches_root'; +import RecentSearchesStore from './stores/recent_searches_store'; +import RecentSearchesService from './services/recent_searches_service'; +import eventHub from './event_hub'; (() => { class FilteredSearchManager { constructor(page) { this.container = FilteredSearchContainer.container; this.filteredSearchInput = this.container.querySelector('.filtered-search'); + this.filteredSearchInputForm = this.filteredSearchInput.form; this.clearSearchButton = this.container.querySelector('.clear-search'); this.tokensContainer = this.container.querySelector('.tokens-container'); this.filteredSearchTokenKeys = gl.FilteredSearchTokenKeys; + this.recentSearchesStore = new RecentSearchesStore(); + let recentSearchesKey = 'issue-recent-searches'; + if (page === 'merge_requests') { + recentSearchesKey = 'merge-request-recent-searches'; + } + this.recentSearchesService = new RecentSearchesService(recentSearchesKey); + + // Fetch recent searches from localStorage + this.fetchingRecentSearchesPromise = this.recentSearchesService.fetch() + .catch(() => { + // eslint-disable-next-line no-new + new Flash('An error occured while parsing recent searches'); + // Gracefully fail to empty array + return []; + }) + .then((searches) => { + // Put any searches that may have come in before + // we fetched the saved searches ahead of the already saved ones + const resultantSearches = this.recentSearchesStore.setRecentSearches( + this.recentSearchesStore.state.recentSearches.concat(searches), + ); + this.recentSearchesService.save(resultantSearches); + }); + if (this.filteredSearchInput) { this.tokenizer = gl.FilteredSearchTokenizer; this.dropdownManager = new gl.FilteredSearchDropdownManager(this.filteredSearchInput.getAttribute('data-base-endpoint') || '', page); + this.recentSearchesRoot = new RecentSearchesRoot( + this.recentSearchesStore, + this.recentSearchesService, + document.querySelector('.js-filtered-search-history-dropdown'), + ); + this.recentSearchesRoot.init(); + this.bindEvents(); this.loadSearchParamsFromURL(); this.dropdownManager.setDropdown(); @@ -25,6 +63,10 @@ import FilteredSearchContainer from './container'; cleanup() { this.unbindEvents(); document.removeEventListener('beforeunload', this.cleanupWrapper); + + if (this.recentSearchesRoot) { + this.recentSearchesRoot.destroy(); + } } bindEvents() { @@ -34,7 +76,7 @@ import FilteredSearchContainer from './container'; this.handleInputPlaceholderWrapper = this.handleInputPlaceholder.bind(this); this.handleInputVisualTokenWrapper = this.handleInputVisualToken.bind(this); this.checkForEnterWrapper = this.checkForEnter.bind(this); - this.clearSearchWrapper = this.clearSearch.bind(this); + this.onClearSearchWrapper = this.onClearSearch.bind(this); this.checkForBackspaceWrapper = this.checkForBackspace.bind(this); this.removeSelectedTokenWrapper = this.removeSelectedToken.bind(this); this.unselectEditTokensWrapper = this.unselectEditTokens.bind(this); @@ -42,8 +84,8 @@ import FilteredSearchContainer from './container'; this.tokenChange = this.tokenChange.bind(this); this.addInputContainerFocusWrapper = this.addInputContainerFocus.bind(this); this.removeInputContainerFocusWrapper = this.removeInputContainerFocus.bind(this); + this.onrecentSearchesItemSelectedWrapper = this.onrecentSearchesItemSelected.bind(this); - this.filteredSearchInputForm = this.filteredSearchInput.form; this.filteredSearchInputForm.addEventListener('submit', this.handleFormSubmit); this.filteredSearchInput.addEventListener('input', this.setDropdownWrapper); this.filteredSearchInput.addEventListener('input', this.toggleClearSearchButtonWrapper); @@ -56,11 +98,12 @@ import FilteredSearchContainer from './container'; this.filteredSearchInput.addEventListener('focus', this.addInputContainerFocusWrapper); this.tokensContainer.addEventListener('click', FilteredSearchManager.selectToken); this.tokensContainer.addEventListener('dblclick', this.editTokenWrapper); - this.clearSearchButton.addEventListener('click', this.clearSearchWrapper); + this.clearSearchButton.addEventListener('click', this.onClearSearchWrapper); document.addEventListener('click', gl.FilteredSearchVisualTokens.unselectTokens); document.addEventListener('click', this.unselectEditTokensWrapper); document.addEventListener('click', this.removeInputContainerFocusWrapper); document.addEventListener('keydown', this.removeSelectedTokenWrapper); + eventHub.$on('recentSearchesItemSelected', this.onrecentSearchesItemSelectedWrapper); } unbindEvents() { @@ -76,11 +119,12 @@ import FilteredSearchContainer from './container'; this.filteredSearchInput.removeEventListener('focus', this.addInputContainerFocusWrapper); this.tokensContainer.removeEventListener('click', FilteredSearchManager.selectToken); this.tokensContainer.removeEventListener('dblclick', this.editTokenWrapper); - this.clearSearchButton.removeEventListener('click', this.clearSearchWrapper); + this.clearSearchButton.removeEventListener('click', this.onClearSearchWrapper); document.removeEventListener('click', gl.FilteredSearchVisualTokens.unselectTokens); document.removeEventListener('click', this.unselectEditTokensWrapper); document.removeEventListener('click', this.removeInputContainerFocusWrapper); document.removeEventListener('keydown', this.removeSelectedTokenWrapper); + eventHub.$off('recentSearchesItemSelected', this.onrecentSearchesItemSelectedWrapper); } checkForBackspace(e) { @@ -131,7 +175,7 @@ import FilteredSearchContainer from './container'; } addInputContainerFocus() { - const inputContainer = this.filteredSearchInput.closest('.filtered-search-input-container'); + const inputContainer = this.filteredSearchInput.closest('.filtered-search-box'); if (inputContainer) { inputContainer.classList.add('focus'); @@ -139,7 +183,7 @@ import FilteredSearchContainer from './container'; } removeInputContainerFocus(e) { - const inputContainer = this.filteredSearchInput.closest('.filtered-search-input-container'); + const inputContainer = this.filteredSearchInput.closest('.filtered-search-box'); const isElementInFilteredSearch = inputContainer && inputContainer.contains(e.target); const isElementInDynamicFilterDropdown = e.target.closest('.filter-dropdown') !== null; const isElementInStaticFilterDropdown = e.target.closest('ul[data-dropdown]') !== null; @@ -161,7 +205,7 @@ import FilteredSearchContainer from './container'; } unselectEditTokens(e) { - const inputContainer = this.container.querySelector('.filtered-search-input-container'); + const inputContainer = this.container.querySelector('.filtered-search-box'); const isElementInFilteredSearch = inputContainer && inputContainer.contains(e.target); const isElementInFilterDropdown = e.target.closest('.filter-dropdown') !== null; const isElementTokensContainer = e.target.classList.contains('tokens-container'); @@ -215,9 +259,12 @@ import FilteredSearchContainer from './container'; } } - clearSearch(e) { + onClearSearch(e) { e.preventDefault(); + this.clearSearch(); + } + clearSearch() { this.filteredSearchInput.value = ''; const removeElements = []; @@ -289,6 +336,17 @@ import FilteredSearchContainer from './container'; this.search(); } + saveCurrentSearchQuery() { + // Don't save before we have fetched the already saved searches + this.fetchingRecentSearchesPromise.then(() => { + const searchQuery = gl.DropdownUtils.getSearchQuery(); + if (searchQuery.length > 0) { + const resultantSearches = this.recentSearchesStore.addRecentSearch(searchQuery); + this.recentSearchesService.save(resultantSearches); + } + }); + } + loadSearchParamsFromURL() { const params = gl.utils.getUrlParamsArray(); const usernameParams = this.getUsernameParams(); @@ -343,6 +401,8 @@ import FilteredSearchContainer from './container'; } }); + this.saveCurrentSearchQuery(); + if (hasFilteredSearch) { this.clearSearchButton.classList.remove('hidden'); this.handleInputPlaceholder(); @@ -351,8 +411,12 @@ import FilteredSearchContainer from './container'; search() { const paths = []; + const searchQuery = gl.DropdownUtils.getSearchQuery(); + + this.saveCurrentSearchQuery(); + const { tokens, searchToken } - = this.tokenizer.processTokens(gl.DropdownUtils.getSearchQuery()); + = this.tokenizer.processTokens(searchQuery); const currentState = gl.utils.getParameterByName('state') || 'opened'; paths.push(`state=${currentState}`); @@ -416,6 +480,13 @@ import FilteredSearchContainer from './container'; currentDropdownRef.dispatchInputEvent(); } } + + onrecentSearchesItemSelected(text) { + this.clearSearch(); + this.filteredSearchInput.value = text; + this.filteredSearchInput.dispatchEvent(new CustomEvent('input')); + this.search(); + } } window.gl = window.gl || {}; diff --git a/app/assets/javascripts/filtered_search/recent_searches_root.js b/app/assets/javascripts/filtered_search/recent_searches_root.js new file mode 100644 index 00000000000..4e38409e12a --- /dev/null +++ b/app/assets/javascripts/filtered_search/recent_searches_root.js @@ -0,0 +1,59 @@ +import Vue from 'vue'; +import RecentSearchesDropdownContent from './components/recent_searches_dropdown_content'; +import eventHub from './event_hub'; + +class RecentSearchesRoot { + constructor( + recentSearchesStore, + recentSearchesService, + wrapperElement, + ) { + this.store = recentSearchesStore; + this.service = recentSearchesService; + this.wrapperElement = wrapperElement; + } + + init() { + this.bindEvents(); + this.render(); + } + + bindEvents() { + this.onRequestClearRecentSearchesWrapper = this.onRequestClearRecentSearches.bind(this); + + eventHub.$on('requestClearRecentSearches', this.onRequestClearRecentSearchesWrapper); + } + + unbindEvents() { + eventHub.$off('requestClearRecentSearches', this.onRequestClearRecentSearchesWrapper); + } + + render() { + this.vm = new Vue({ + el: this.wrapperElement, + data: this.store.state, + template: ` + + `, + components: { + 'recent-searches-dropdown-content': RecentSearchesDropdownContent, + }, + }); + } + + onRequestClearRecentSearches() { + const resultantSearches = this.store.setRecentSearches([]); + this.service.save(resultantSearches); + } + + destroy() { + this.unbindEvents(); + if (this.vm) { + this.vm.$destroy(); + } + } + +} + +export default RecentSearchesRoot; diff --git a/app/assets/javascripts/filtered_search/services/recent_searches_service.js b/app/assets/javascripts/filtered_search/services/recent_searches_service.js new file mode 100644 index 00000000000..3e402d5aed0 --- /dev/null +++ b/app/assets/javascripts/filtered_search/services/recent_searches_service.js @@ -0,0 +1,26 @@ +class RecentSearchesService { + constructor(localStorageKey = 'issuable-recent-searches') { + this.localStorageKey = localStorageKey; + } + + fetch() { + const input = window.localStorage.getItem(this.localStorageKey); + + let searches = []; + if (input && input.length > 0) { + try { + searches = JSON.parse(input); + } catch (err) { + return Promise.reject(err); + } + } + + return Promise.resolve(searches); + } + + save(searches = []) { + window.localStorage.setItem(this.localStorageKey, JSON.stringify(searches)); + } +} + +export default RecentSearchesService; diff --git a/app/assets/javascripts/filtered_search/stores/recent_searches_store.js b/app/assets/javascripts/filtered_search/stores/recent_searches_store.js new file mode 100644 index 00000000000..066be69766a --- /dev/null +++ b/app/assets/javascripts/filtered_search/stores/recent_searches_store.js @@ -0,0 +1,23 @@ +import _ from 'underscore'; + +class RecentSearchesStore { + constructor(initialState = {}) { + this.state = Object.assign({ + recentSearches: [], + }, initialState); + } + + addRecentSearch(newSearch) { + this.setRecentSearches([newSearch].concat(this.state.recentSearches)); + + return this.state.recentSearches; + } + + setRecentSearches(searches = []) { + const trimmedSearches = searches.map(search => search.trim()); + this.state.recentSearches = _.uniq(trimmedSearches).slice(0, 5); + return this.state.recentSearches; + } +} + +export default RecentSearchesStore; diff --git a/app/assets/stylesheets/framework/dropdowns.scss b/app/assets/stylesheets/framework/dropdowns.scss index 2ede47e9de6..23cba57f83a 100644 --- a/app/assets/stylesheets/framework/dropdowns.scss +++ b/app/assets/stylesheets/framework/dropdowns.scss @@ -177,10 +177,6 @@ border-radius: $border-radius-base; box-shadow: 0 2px 4px $dropdown-shadow-color; - .filtered-search-input-container & { - max-width: 280px; - } - &.is-loading { .dropdown-content { display: none; @@ -467,6 +463,11 @@ overflow-y: auto; } +.dropdown-info-note { + color: $gl-text-color-secondary; + text-align: center; +} + .dropdown-footer { padding-top: 10px; margin-top: 10px; diff --git a/app/assets/stylesheets/framework/filters.scss b/app/assets/stylesheets/framework/filters.scss index 51805c5d734..484df6214d3 100644 --- a/app/assets/stylesheets/framework/filters.scss +++ b/app/assets/stylesheets/framework/filters.scss @@ -22,7 +22,6 @@ } @media (min-width: $screen-sm-min) { - .issues-filters, .issues_bulk_update { .dropdown-menu-toggle { width: 132px; @@ -56,7 +55,7 @@ } } -.filtered-search-container { +.filtered-search-wrapper { display: -webkit-flex; display: flex; @@ -151,11 +150,13 @@ width: 100%; } -.filtered-search-input-container { +.filtered-search-box { + position: relative; + flex: 1; display: -webkit-flex; display: flex; - position: relative; width: 100%; + min-width: 0; border: 1px solid $border-color; background-color: $white-light; @@ -163,14 +164,6 @@ -webkit-flex: 1 1 auto; flex: 1 1 auto; margin-bottom: 10px; - - .dropdown-menu { - width: auto; - left: 0; - right: 0; - max-width: none; - min-width: 100%; - } } &:hover { @@ -229,6 +222,118 @@ } } +.filtered-search-box-input-container { + flex: 1; + position: relative; + // Fix PhantomJS not supporting `flex: 1;` properly. + // This is important because it can change the expected `e.target` when clicking things in tests. + // See https://gitlab.com/gitlab-org/gitlab-ce/blob/b54acba8b732688c59fe2f38510c469dc86ee499/spec/features/issues/filtered_search/visual_tokens_spec.rb#L61 + // - With `width: 100%`: `e.target` = `.tokens-container`, https://i.imgur.com/jGq7wbx.png + // - Without `width: 100%`: `e.target` = `.filtered-search`, https://i.imgur.com/cNI2CyT.png + width: 100%; + min-width: 0; +} + +.filtered-search-input-dropdown-menu { + max-width: 280px; + + @media (max-width: $screen-xs-min) { + width: auto; + left: 0; + right: 0; + max-width: none; + min-width: 100%; + } +} + +.filtered-search-history-dropdown-toggle-button { + display: flex; + align-items: center; + width: auto; + height: 100%; + padding-top: 0; + padding-left: 0.75em; + padding-bottom: 0; + padding-right: 0.5em; + + background-color: transparent; + border-radius: 0; + border-top: 0; + border-left: 0; + border-bottom: 0; + border-right: 1px solid $border-color; + + color: $gl-text-color-secondary; + + transition: color 0.1s linear; + + &:hover, + &:focus { + color: $gl-text-color; + border-color: $dropdown-input-focus-border; + outline: none; + } + + .dropdown-toggle-text { + color: inherit; + + .fa { + color: inherit; + } + } + + .fa { + position: initial; + } + +} + +.filtered-search-history-dropdown-wrapper { + position: initial; + flex-shrink: 0; +} + +.filtered-search-history-dropdown { + width: 40%; + + @media (max-width: $screen-xs-min) { + left: 0; + right: 0; + max-width: none; + } +} + +.filtered-search-history-dropdown-content { + max-height: none; +} + +.filtered-search-history-dropdown-item, +.filtered-search-history-clear-button { + @include dropdown-link; + + overflow: hidden; + width: 100%; + margin: 0.5em 0; + + background-color: transparent; + border: 0; + text-align: left; + white-space: nowrap; + text-overflow: ellipsis; +} + +.filtered-search-history-dropdown-token { + display: inline; + + &:not(:last-child) { + margin-right: 0.3em; + } + + & > .value { + font-weight: 600; + } +} + .filter-dropdown-container { display: -webkit-flex; display: flex; @@ -248,10 +353,8 @@ } @media (min-width: $screen-sm-min) and (max-width: $screen-sm-max) { - .issues-details-filters { - .dropdown-menu-toggle { - width: 100px; - } + .issue-bulk-update-dropdown-toggle { + width: 100px; } } diff --git a/app/helpers/dropdowns_helper.rb b/app/helpers/dropdowns_helper.rb index 81e0b6bb5ae..8ed99642c7a 100644 --- a/app/helpers/dropdowns_helper.rb +++ b/app/helpers/dropdowns_helper.rb @@ -1,6 +1,6 @@ module DropdownsHelper def dropdown_tag(toggle_text, options: {}, &block) - content_tag :div, class: "dropdown" do + content_tag :div, class: "dropdown #{options[:wrapper_class] if options.has_key?(:wrapper_class)}" do data_attr = { toggle: "dropdown" } if options.has_key?(:data) @@ -20,7 +20,7 @@ module DropdownsHelper output << dropdown_filter(options[:placeholder]) end - output << content_tag(:div, class: "dropdown-content") do + output << content_tag(:div, class: "dropdown-content #{options[:content_class] if options.has_key?(:content_class)}") do capture(&block) if block && !options.has_key?(:footer_content) end diff --git a/app/views/shared/issuable/_filter.html.haml b/app/views/shared/issuable/_filter.html.haml index 847a86e2e68..c72268473ca 100644 --- a/app/views/shared/issuable/_filter.html.haml +++ b/app/views/shared/issuable/_filter.html.haml @@ -40,21 +40,21 @@ .issues_bulk_update.hide = form_tag [:bulk_update, @project.namespace.becomes(Namespace), @project, type], method: :post, class: 'bulk-update' do .filter-item.inline - = dropdown_tag("Status", options: { toggle_class: "js-issue-status", title: "Change status", dropdown_class: "dropdown-menu-status dropdown-menu-selectable", data: { field_name: "update[state_event]", default_label: "Status" } } ) do + = dropdown_tag("Status", options: { toggle_class: "issue-bulk-update-dropdown-toggle js-issue-status", title: "Change status", dropdown_class: "dropdown-menu-status dropdown-menu-selectable", data: { field_name: "update[state_event]", default_label: "Status" } } ) do %ul %li %a{ href: "#", data: { id: "reopen" } } Open %li %a{ href: "#", data: {id: "close" } } Closed .filter-item.inline - = dropdown_tag("Assignee", options: { toggle_class: "js-user-search js-update-assignee js-filter-submit js-filter-bulk-update", title: "Assign to", filter: true, dropdown_class: "dropdown-menu-user dropdown-menu-selectable", + = dropdown_tag("Assignee", options: { toggle_class: "issue-bulk-update-dropdown-toggle js-user-search js-update-assignee js-filter-submit js-filter-bulk-update", title: "Assign to", filter: true, dropdown_class: "dropdown-menu-user dropdown-menu-selectable", placeholder: "Search authors", data: { first_user: (current_user.username if current_user), null_user: true, current_user: true, project_id: @project.id, field_name: "update[assignee_id]", default_label: "Assignee" } }) .filter-item.inline - = dropdown_tag("Milestone", options: { title: "Assign milestone", toggle_class: 'js-milestone-select js-extra-options js-filter-submit js-filter-bulk-update', filter: true, dropdown_class: "dropdown-menu-selectable dropdown-menu-milestone", placeholder: "Search milestones", data: { show_no: true, field_name: "update[milestone_id]", default_label: "Milestone", project_id: @project.id, milestones: namespace_project_milestones_path(@project.namespace, @project, :json), use_id: true } }) + = dropdown_tag("Milestone", options: { title: "Assign milestone", toggle_class: 'issue-bulk-update-dropdown-toggle js-milestone-select js-extra-options js-filter-submit js-filter-bulk-update', filter: true, dropdown_class: "dropdown-menu-selectable dropdown-menu-milestone", placeholder: "Search milestones", data: { show_no: true, field_name: "update[milestone_id]", default_label: "Milestone", project_id: @project.id, milestones: namespace_project_milestones_path(@project.namespace, @project, :json), use_id: true } }) .filter-item.inline.labels-filter = render "shared/issuable/label_dropdown", classes: ['js-filter-bulk-update', 'js-multiselect'], dropdown_title: 'Apply a label', show_create: false, show_footer: false, extra_options: false, filter_submit: false, data_options: { persist_when_hide: "true", field_name: "update[label_ids][]", show_no: false, show_any: false, use_id: true } .filter-item.inline - = dropdown_tag("Subscription", options: { toggle_class: "js-subscription-event", title: "Change subscription", dropdown_class: "dropdown-menu-selectable", data: { field_name: "update[subscription_event]", default_label: "Subscription" } } ) do + = dropdown_tag("Subscription", options: { toggle_class: "issue-bulk-update-dropdown-toggle js-subscription-event", title: "Change subscription", dropdown_class: "dropdown-menu-selectable", data: { field_name: "update[subscription_event]", default_label: "Subscription" } } ) do %ul %li %a{ href: "#", data: { id: "subscribe" } } Subscribe diff --git a/app/views/shared/issuable/_search_bar.html.haml b/app/views/shared/issuable/_search_bar.html.haml index 330fa8a5b10..9e241c3ea12 100644 --- a/app/views/shared/issuable/_search_bar.html.haml +++ b/app/views/shared/issuable/_search_bar.html.haml @@ -10,85 +10,93 @@ .check-all-holder = check_box_tag "check_all_issues", nil, false, class: "check_all_issues left" - .issues-other-filters.filtered-search-container - .filtered-search-input-container - .scroll-container - %ul.tokens-container.list-unstyled - %li.input-token - %input.form-control.filtered-search{ placeholder: 'Search or filter results...', data: { id: "filtered-search-#{type.to_s}", 'project-id' => @project.id, 'username-params' => @users.to_json(only: [:id, :username]), 'base-endpoint' => namespace_project_path(@project.namespace, @project) } } - = icon('filter') - %button.clear-search.hidden{ type: 'button' } - = icon('times') - #js-dropdown-hint.dropdown-menu.hint-dropdown - %ul{ data: { dropdown: true } } - %li.filter-dropdown-item{ data: { action: 'submit' } } - %button.btn.btn-link - = icon('search') - %span - Press Enter or click to search - %ul.filter-dropdown{ data: { dynamic: true, dropdown: true } } - %li.filter-dropdown-item - %button.btn.btn-link - -# Encapsulate static class name `{{icon}}` inside #{} to bypass - -# haml lint's ClassAttributeWithStaticValue - %i.fa{ class: "#{'{{icon}}'}" } - %span.js-filter-hint - {{hint}} - %span.js-filter-tag.dropdown-light-content - {{tag}} - #js-dropdown-author.dropdown-menu{ data: { icon: 'pencil', hint: 'author', tag: '@author' } } - %ul.filter-dropdown{ data: { dynamic: true, dropdown: true } } - %li.filter-dropdown-item - %button.btn.btn-link.dropdown-user - %img.avatar{ alt: '{{name}}\'s avatar', width: '30', data: { src: '{{avatar_url}}' } } - .dropdown-user-details + .issues-other-filters.filtered-search-wrapper + .filtered-search-box + = dropdown_tag(content_tag(:i, '', class: 'fa fa-history'), + options: { wrapper_class: "filtered-search-history-dropdown-wrapper", + toggle_class: "filtered-search-history-dropdown-toggle-button", + dropdown_class: "filtered-search-history-dropdown", + content_class: "filtered-search-history-dropdown-content", + title: "Recent searches" }) do + .js-filtered-search-history-dropdown + .filtered-search-box-input-container + .scroll-container + %ul.tokens-container.list-unstyled + %li.input-token + %input.form-control.filtered-search{ placeholder: 'Search or filter results...', data: { id: "filtered-search-#{type.to_s}", 'project-id' => @project.id, 'username-params' => @users.to_json(only: [:id, :username]), 'base-endpoint' => namespace_project_path(@project.namespace, @project) } } + = icon('filter') + %button.clear-search.hidden{ type: 'button' } + = icon('times') + #js-dropdown-hint.filtered-search-input-dropdown-menu.dropdown-menu.hint-dropdown + %ul{ data: { dropdown: true } } + %li.filter-dropdown-item{ data: { action: 'submit' } } + %button.btn.btn-link + = icon('search') %span - {{name}} - %span.dropdown-light-content - @{{username}} - #js-dropdown-assignee.dropdown-menu{ data: { icon: 'user', hint: 'assignee', tag: '@assignee' } } - %ul{ data: { dropdown: true } } - %li.filter-dropdown-item{ data: { value: 'none' } } - %button.btn.btn-link - No Assignee - %li.divider - %ul.filter-dropdown{ data: { dynamic: true, dropdown: true } } - %li.filter-dropdown-item - %button.btn.btn-link.dropdown-user - %img.avatar{ alt: '{{name}}\'s avatar', width: '30', data: { src: '{{avatar_url}}' } } - .dropdown-user-details - %span - {{name}} - %span.dropdown-light-content - @{{username}} - #js-dropdown-milestone.dropdown-menu{ data: { icon: 'clock-o', hint: 'milestone', tag: '%milestone' } } - %ul{ data: { dropdown: true } } - %li.filter-dropdown-item{ data: { value: 'none' } } - %button.btn.btn-link - No Milestone - %li.filter-dropdown-item{ data: { value: 'upcoming' } } - %button.btn.btn-link - Upcoming - %li.filter-dropdown-item{ 'data-value' => 'started' } - %button.btn.btn-link - Started - %li.divider - %ul.filter-dropdown{ data: { dynamic: true, dropdown: true } } - %li.filter-dropdown-item - %button.btn.btn-link.js-data-value - {{title}} - #js-dropdown-label.dropdown-menu{ data: { icon: 'tag', hint: 'label', tag: '~label', type: 'array' } } - %ul{ data: { dropdown: true } } - %li.filter-dropdown-item{ data: { value: 'none' } } - %button.btn.btn-link - No Label - %li.divider - %ul.filter-dropdown{ data: { dynamic: true, dropdown: true } } - %li.filter-dropdown-item - %button.btn.btn-link - %span.dropdown-label-box{ style: 'background: {{color}}' } - %span.label-title.js-data-value + Press Enter or click to search + %ul.filter-dropdown{ data: { dynamic: true, dropdown: true } } + %li.filter-dropdown-item + %button.btn.btn-link + -# Encapsulate static class name `{{icon}}` inside #{} to bypass + -# haml lint's ClassAttributeWithStaticValue + %i.fa{ class: "#{'{{icon}}'}" } + %span.js-filter-hint + {{hint}} + %span.js-filter-tag.dropdown-light-content + {{tag}} + #js-dropdown-author.filtered-search-input-dropdown-menu.dropdown-menu{ data: { icon: 'pencil', hint: 'author', tag: '@author' } } + %ul.filter-dropdown{ data: { dynamic: true, dropdown: true } } + %li.filter-dropdown-item + %button.btn.btn-link.dropdown-user + %img.avatar{ alt: '{{name}}\'s avatar', width: '30', data: { src: '{{avatar_url}}' } } + .dropdown-user-details + %span + {{name}} + %span.dropdown-light-content + @{{username}} + #js-dropdown-assignee.filtered-search-input-dropdown-menu.dropdown-menu{ data: { icon: 'user', hint: 'assignee', tag: '@assignee' } } + %ul{ data: { dropdown: true } } + %li.filter-dropdown-item{ data: { value: 'none' } } + %button.btn.btn-link + No Assignee + %li.divider + %ul.filter-dropdown{ data: { dynamic: true, dropdown: true } } + %li.filter-dropdown-item + %button.btn.btn-link.dropdown-user + %img.avatar{ alt: '{{name}}\'s avatar', width: '30', data: { src: '{{avatar_url}}' } } + .dropdown-user-details + %span + {{name}} + %span.dropdown-light-content + @{{username}} + #js-dropdown-milestone.filtered-search-input-dropdown-menu.dropdown-menu{ data: { icon: 'clock-o', hint: 'milestone', tag: '%milestone' } } + %ul{ data: { dropdown: true } } + %li.filter-dropdown-item{ data: { value: 'none' } } + %button.btn.btn-link + No Milestone + %li.filter-dropdown-item{ data: { value: 'upcoming' } } + %button.btn.btn-link + Upcoming + %li.filter-dropdown-item{ 'data-value' => 'started' } + %button.btn.btn-link + Started + %li.divider + %ul.filter-dropdown{ data: { dynamic: true, dropdown: true } } + %li.filter-dropdown-item + %button.btn.btn-link.js-data-value {{title}} + #js-dropdown-label.filtered-search-input-dropdown-menu.dropdown-menu{ data: { icon: 'tag', hint: 'label', tag: '~label', type: 'array' } } + %ul{ data: { dropdown: true } } + %li.filter-dropdown-item{ data: { value: 'none' } } + %button.btn.btn-link + No Label + %li.divider + %ul.filter-dropdown{ data: { dynamic: true, dropdown: true } } + %li.filter-dropdown-item + %button.btn.btn-link + %span.dropdown-label-box{ style: 'background: {{color}}' } + %span.label-title.js-data-value + {{title}} .filter-dropdown-container - if type == :boards - if can?(current_user, :admin_list, @project) diff --git a/changelogs/unreleased/27262-issue-recent-searches.yml b/changelogs/unreleased/27262-issue-recent-searches.yml new file mode 100644 index 00000000000..4bdec5af31d --- /dev/null +++ b/changelogs/unreleased/27262-issue-recent-searches.yml @@ -0,0 +1,4 @@ +--- +title: Recent search history for issues +merge_request: +author: diff --git a/spec/features/boards/boards_spec.rb b/spec/features/boards/boards_spec.rb index e168585534d..30ad169e30e 100644 --- a/spec/features/boards/boards_spec.rb +++ b/spec/features/boards/boards_spec.rb @@ -590,7 +590,7 @@ describe 'Issue Boards', feature: true, js: true do end def click_filter_link(link_text) - page.within('.filtered-search-input-container') do + page.within('.filtered-search-box') do expect(page).to have_button(link_text) click_button(link_text) diff --git a/spec/features/boards/modal_filter_spec.rb b/spec/features/boards/modal_filter_spec.rb index e2281a7da55..4a4c13e79c8 100644 --- a/spec/features/boards/modal_filter_spec.rb +++ b/spec/features/boards/modal_filter_spec.rb @@ -219,7 +219,7 @@ describe 'Issue Boards add issue modal filtering', :feature, :js do end def click_filter_link(link_text) - page.within('.add-issues-modal .filtered-search-input-container') do + page.within('.add-issues-modal .filtered-search-box') do expect(page).to have_button(link_text) click_button(link_text) diff --git a/spec/features/issues/filtered_search/dropdown_assignee_spec.rb b/spec/features/issues/filtered_search/dropdown_assignee_spec.rb index 4dcc56a97d1..3d1a9ed1722 100644 --- a/spec/features/issues/filtered_search/dropdown_assignee_spec.rb +++ b/spec/features/issues/filtered_search/dropdown_assignee_spec.rb @@ -194,7 +194,7 @@ describe 'Dropdown assignee', :feature, :js do new_user = create(:user) project.team << [new_user, :master] - find('.filtered-search-input-container .clear-search').click + find('.filtered-search-box .clear-search').click filtered_search.set('assignee') filtered_search.send_keys(':') diff --git a/spec/features/issues/filtered_search/dropdown_author_spec.rb b/spec/features/issues/filtered_search/dropdown_author_spec.rb index 1772a120045..990e3b3e60c 100644 --- a/spec/features/issues/filtered_search/dropdown_author_spec.rb +++ b/spec/features/issues/filtered_search/dropdown_author_spec.rb @@ -172,7 +172,7 @@ describe 'Dropdown author', js: true, feature: true do new_user = create(:user) project.team << [new_user, :master] - find('.filtered-search-input-container .clear-search').click + find('.filtered-search-box .clear-search').click filtered_search.set('author') send_keys_to_filtered_search(':') diff --git a/spec/features/issues/filtered_search/dropdown_label_spec.rb b/spec/features/issues/filtered_search/dropdown_label_spec.rb index b192064b693..c8645e08c4b 100644 --- a/spec/features/issues/filtered_search/dropdown_label_spec.rb +++ b/spec/features/issues/filtered_search/dropdown_label_spec.rb @@ -33,7 +33,7 @@ describe 'Dropdown label', js: true, feature: true do end def clear_search_field - find('.filtered-search-input-container .clear-search').click + find('.filtered-search-box .clear-search').click end before do diff --git a/spec/features/issues/filtered_search/dropdown_milestone_spec.rb b/spec/features/issues/filtered_search/dropdown_milestone_spec.rb index ce96a420699..0a525bc68c9 100644 --- a/spec/features/issues/filtered_search/dropdown_milestone_spec.rb +++ b/spec/features/issues/filtered_search/dropdown_milestone_spec.rb @@ -252,7 +252,7 @@ describe 'Dropdown milestone', :feature, :js do expect(initial_size).to be > 0 create(:milestone, project: project) - find('.filtered-search-input-container .clear-search').click + find('.filtered-search-box .clear-search').click filtered_search.set('milestone:') expect(dropdown_milestone_size).to eq(initial_size) diff --git a/spec/features/issues/filtered_search/filter_issues_spec.rb b/spec/features/issues/filtered_search/filter_issues_spec.rb index 2f880c926e7..6f00066de4d 100644 --- a/spec/features/issues/filtered_search/filter_issues_spec.rb +++ b/spec/features/issues/filtered_search/filter_issues_spec.rb @@ -758,10 +758,10 @@ describe 'Filter issues', js: true, feature: true do expect_issues_list_count(2) - sort_toggle = find('.filtered-search-container .dropdown-toggle') + sort_toggle = find('.filtered-search-wrapper .dropdown-toggle') sort_toggle.click - find('.filtered-search-container .dropdown-menu li a', text: 'Oldest updated').click + find('.filtered-search-wrapper .dropdown-menu li a', text: 'Oldest updated').click wait_for_ajax expect(find('.issues-list .issue:first-of-type .issue-title-text a')).to have_content(old_issue.title) diff --git a/spec/features/issues/filtered_search/recent_searches_spec.rb b/spec/features/issues/filtered_search/recent_searches_spec.rb new file mode 100644 index 00000000000..f506065a242 --- /dev/null +++ b/spec/features/issues/filtered_search/recent_searches_spec.rb @@ -0,0 +1,92 @@ +require 'spec_helper' + +describe 'Recent searches', js: true, feature: true do + include FilteredSearchHelpers + include WaitForAjax + + let!(:group) { create(:group) } + let!(:project) { create(:project, group: group) } + let!(:user) { create(:user) } + + before do + Capybara.ignore_hidden_elements = false + project.add_master(user) + group.add_developer(user) + create(:issue, project: project) + login_as(user) + + remove_recent_searches + end + + after do + Capybara.ignore_hidden_elements = true + end + + it 'searching adds to recent searches' do + visit namespace_project_issues_path(project.namespace, project) + + input_filtered_search('foo', submit: true) + input_filtered_search('bar', submit: true) + + items = all('.filtered-search-history-dropdown-item', visible: false) + + expect(items.count).to eq(2) + expect(items[0].text).to eq('bar') + expect(items[1].text).to eq('foo') + end + + it 'visiting URL with search params adds to recent searches' do + visit namespace_project_issues_path(project.namespace, project, label_name: 'foo', search: 'bar') + visit namespace_project_issues_path(project.namespace, project, label_name: 'qux', search: 'garply') + + items = all('.filtered-search-history-dropdown-item', visible: false) + + expect(items.count).to eq(2) + expect(items[0].text).to eq('label:~qux garply') + expect(items[1].text).to eq('label:~foo bar') + end + + it 'saved recent searches are restored last on the list' do + set_recent_searches('["saved1", "saved2"]') + + visit namespace_project_issues_path(project.namespace, project, search: 'foo') + + items = all('.filtered-search-history-dropdown-item', visible: false) + + expect(items.count).to eq(3) + expect(items[0].text).to eq('foo') + expect(items[1].text).to eq('saved1') + expect(items[2].text).to eq('saved2') + end + + it 'clicking item fills search input' do + set_recent_searches('["foo", "bar"]') + visit namespace_project_issues_path(project.namespace, project) + + all('.filtered-search-history-dropdown-item', visible: false)[0].trigger('click') + wait_for_filtered_search('foo') + + expect(find('.filtered-search').value.strip).to eq('foo') + end + + it 'clear recent searches button, clears recent searches' do + set_recent_searches('["foo"]') + visit namespace_project_issues_path(project.namespace, project) + + items_before = all('.filtered-search-history-dropdown-item', visible: false) + + expect(items_before.count).to eq(1) + + find('.filtered-search-history-clear-button', visible: false).trigger('click') + items_after = all('.filtered-search-history-dropdown-item', visible: false) + + expect(items_after.count).to eq(0) + end + + it 'shows flash error when failed to parse saved history' do + set_recent_searches('fail') + visit namespace_project_issues_path(project.namespace, project) + + expect(find('.flash-alert')).to have_text('An error occured while parsing recent searches') + end +end diff --git a/spec/features/issues/filtered_search/search_bar_spec.rb b/spec/features/issues/filtered_search/search_bar_spec.rb index 59244d65eec..48e7af67616 100644 --- a/spec/features/issues/filtered_search/search_bar_spec.rb +++ b/spec/features/issues/filtered_search/search_bar_spec.rb @@ -44,7 +44,7 @@ describe 'Search bar', js: true, feature: true do filtered_search.set(search_text) expect(filtered_search.value).to eq(search_text) - find('.filtered-search-input-container .clear-search').click + find('.filtered-search-box .clear-search').click expect(filtered_search.value).to eq('') end @@ -55,7 +55,7 @@ describe 'Search bar', js: true, feature: true do it 'hides after clicked' do filtered_search.set('a') - find('.filtered-search-input-container .clear-search').click + find('.filtered-search-box .clear-search').click expect(page).to have_css('.clear-search', visible: false) end @@ -81,7 +81,7 @@ describe 'Search bar', js: true, feature: true do expect(page.all('#js-dropdown-hint .filter-dropdown .filter-dropdown-item').size).to eq(1) - find('.filtered-search-input-container .clear-search').click + find('.filtered-search-box .clear-search').click filtered_search.click expect(page.all('#js-dropdown-hint .filter-dropdown .filter-dropdown-item').size).to eq(original_size) @@ -96,7 +96,7 @@ describe 'Search bar', js: true, feature: true do expect(page.all('#js-dropdown-hint .filter-dropdown .filter-dropdown-item').size).to eq(0) - find('.filtered-search-input-container .clear-search').click + find('.filtered-search-box .clear-search').click filtered_search.click expect(page.all('#js-dropdown-hint .filter-dropdown .filter-dropdown-item').size).to be > 0 diff --git a/spec/features/merge_requests/reset_filters_spec.rb b/spec/features/merge_requests/reset_filters_spec.rb index 14511707af4..df5943f9136 100644 --- a/spec/features/merge_requests/reset_filters_spec.rb +++ b/spec/features/merge_requests/reset_filters_spec.rb @@ -14,7 +14,7 @@ feature 'Merge requests filter clear button', feature: true, js: true do let!(:mr2) { create(:merge_request, title: "Bugfix1", source_project: project, target_project: project, source_branch: "Bugfix1") } let(:merge_request_css) { '.merge-request' } - let(:clear_search_css) { '.filtered-search-input-container .clear-search' } + let(:clear_search_css) { '.filtered-search-box .clear-search' } before do mr2.labels << bug diff --git a/spec/javascripts/filtered_search/components/recent_searches_dropdown_content_spec.js b/spec/javascripts/filtered_search/components/recent_searches_dropdown_content_spec.js new file mode 100644 index 00000000000..2722882375f --- /dev/null +++ b/spec/javascripts/filtered_search/components/recent_searches_dropdown_content_spec.js @@ -0,0 +1,166 @@ +import Vue from 'vue'; +import eventHub from '~/filtered_search/event_hub'; +import RecentSearchesDropdownContent from '~/filtered_search/components/recent_searches_dropdown_content'; + +const createComponent = (propsData) => { + const Component = Vue.extend(RecentSearchesDropdownContent); + + return new Component({ + el: document.createElement('div'), + propsData, + }); +}; + +// Remove all the newlines and whitespace from the formatted markup +const trimMarkupWhitespace = text => text.replace(/(\n|\s)+/gm, ' ').trim(); + +describe('RecentSearchesDropdownContent', () => { + const propsDataWithoutItems = { + items: [], + }; + const propsDataWithItems = { + items: [ + 'foo', + 'author:@root label:~foo bar', + ], + }; + + let vm; + afterEach(() => { + if (vm) { + vm.$destroy(); + } + }); + + describe('with no items', () => { + let el; + + beforeEach(() => { + vm = createComponent(propsDataWithoutItems); + el = vm.$el; + }); + + it('should render empty state', () => { + expect(el.querySelector('.dropdown-info-note')).toBeDefined(); + + const items = el.querySelectorAll('.filtered-search-history-dropdown-item'); + expect(items.length).toEqual(propsDataWithoutItems.items.length); + }); + }); + + describe('with items', () => { + let el; + + beforeEach(() => { + vm = createComponent(propsDataWithItems); + el = vm.$el; + }); + + it('should render clear recent searches button', () => { + expect(el.querySelector('.filtered-search-history-clear-button')).toBeDefined(); + }); + + it('should render recent search items', () => { + const items = el.querySelectorAll('.filtered-search-history-dropdown-item'); + expect(items.length).toEqual(propsDataWithItems.items.length); + + expect(trimMarkupWhitespace(items[0].querySelector('.filtered-search-history-dropdown-search-token').textContent)).toEqual('foo'); + + const item1Tokens = items[1].querySelectorAll('.filtered-search-history-dropdown-token'); + expect(item1Tokens.length).toEqual(2); + expect(item1Tokens[0].querySelector('.name').textContent).toEqual('author:'); + expect(item1Tokens[0].querySelector('.value').textContent).toEqual('@root'); + expect(item1Tokens[1].querySelector('.name').textContent).toEqual('label:'); + expect(item1Tokens[1].querySelector('.value').textContent).toEqual('~foo'); + expect(trimMarkupWhitespace(items[1].querySelector('.filtered-search-history-dropdown-search-token').textContent)).toEqual('bar'); + }); + }); + + describe('computed', () => { + describe('processedItems', () => { + it('with items', () => { + vm = createComponent(propsDataWithItems); + const processedItems = vm.processedItems; + + expect(processedItems.length).toEqual(2); + + expect(processedItems[0].text).toEqual(propsDataWithItems.items[0]); + expect(processedItems[0].tokens).toEqual([]); + expect(processedItems[0].searchToken).toEqual('foo'); + + expect(processedItems[1].text).toEqual(propsDataWithItems.items[1]); + expect(processedItems[1].tokens.length).toEqual(2); + expect(processedItems[1].tokens[0].prefix).toEqual('author:'); + expect(processedItems[1].tokens[0].suffix).toEqual('@root'); + expect(processedItems[1].tokens[1].prefix).toEqual('label:'); + expect(processedItems[1].tokens[1].suffix).toEqual('~foo'); + expect(processedItems[1].searchToken).toEqual('bar'); + }); + + it('with no items', () => { + vm = createComponent(propsDataWithoutItems); + const processedItems = vm.processedItems; + + expect(processedItems.length).toEqual(0); + }); + }); + + describe('hasItems', () => { + it('with items', () => { + vm = createComponent(propsDataWithItems); + const hasItems = vm.hasItems; + expect(hasItems).toEqual(true); + }); + + it('with no items', () => { + vm = createComponent(propsDataWithoutItems); + const hasItems = vm.hasItems; + expect(hasItems).toEqual(false); + }); + }); + }); + + describe('methods', () => { + describe('onItemActivated', () => { + let onRecentSearchesItemSelectedSpy; + + beforeEach(() => { + onRecentSearchesItemSelectedSpy = jasmine.createSpy('spy'); + eventHub.$on('recentSearchesItemSelected', onRecentSearchesItemSelectedSpy); + + vm = createComponent(propsDataWithItems); + }); + + afterEach(() => { + eventHub.$off('recentSearchesItemSelected', onRecentSearchesItemSelectedSpy); + }); + + it('emits event', () => { + expect(onRecentSearchesItemSelectedSpy).not.toHaveBeenCalled(); + vm.onItemActivated('something'); + expect(onRecentSearchesItemSelectedSpy).toHaveBeenCalledWith('something'); + }); + }); + + describe('onRequestClearRecentSearches', () => { + let onRequestClearRecentSearchesSpy; + + beforeEach(() => { + onRequestClearRecentSearchesSpy = jasmine.createSpy('spy'); + eventHub.$on('requestClearRecentSearches', onRequestClearRecentSearchesSpy); + + vm = createComponent(propsDataWithItems); + }); + + afterEach(() => { + eventHub.$off('requestClearRecentSearches', onRequestClearRecentSearchesSpy); + }); + + it('emits event', () => { + expect(onRequestClearRecentSearchesSpy).not.toHaveBeenCalled(); + vm.onRequestClearRecentSearches({ stopPropagation: () => {} }); + expect(onRequestClearRecentSearchesSpy).toHaveBeenCalled(); + }); + }); + }); +}); diff --git a/spec/javascripts/filtered_search/filtered_search_manager_spec.js b/spec/javascripts/filtered_search/filtered_search_manager_spec.js index 5f7c05e9014..97af681429b 100644 --- a/spec/javascripts/filtered_search/filtered_search_manager_spec.js +++ b/spec/javascripts/filtered_search/filtered_search_manager_spec.js @@ -29,7 +29,7 @@ const FilteredSearchSpecHelper = require('../helpers/filtered_search_spec_helper beforeEach(() => { setFixtures(` -
+
    ${FilteredSearchSpecHelper.createInputHTML(placeholder)} @@ -264,12 +264,12 @@ const FilteredSearchSpecHelper = require('../helpers/filtered_search_spec_helper describe('toggleInputContainerFocus', () => { it('toggles on focus', () => { input.focus(); - expect(document.querySelector('.filtered-search-input-container').classList.contains('focus')).toEqual(true); + expect(document.querySelector('.filtered-search-box').classList.contains('focus')).toEqual(true); }); it('toggles on blur', () => { input.blur(); - expect(document.querySelector('.filtered-search-input-container').classList.contains('focus')).toEqual(false); + expect(document.querySelector('.filtered-search-box').classList.contains('focus')).toEqual(false); }); }); }); diff --git a/spec/javascripts/filtered_search/services/recent_searches_service_spec.js b/spec/javascripts/filtered_search/services/recent_searches_service_spec.js new file mode 100644 index 00000000000..2a58fb3a7df --- /dev/null +++ b/spec/javascripts/filtered_search/services/recent_searches_service_spec.js @@ -0,0 +1,56 @@ +import RecentSearchesService from '~/filtered_search/services/recent_searches_service'; + +describe('RecentSearchesService', () => { + let service; + + beforeEach(() => { + service = new RecentSearchesService(); + window.localStorage.removeItem(service.localStorageKey); + }); + + describe('fetch', () => { + it('should default to empty array', (done) => { + const fetchItemsPromise = service.fetch(); + + fetchItemsPromise + .then((items) => { + expect(items).toEqual([]); + done(); + }) + .catch((err) => { + done.fail('Shouldn\'t reject with empty localStorage key', err); + }); + }); + + it('should reject when unable to parse', (done) => { + window.localStorage.setItem(service.localStorageKey, 'fail'); + const fetchItemsPromise = service.fetch(); + + fetchItemsPromise + .catch(() => { + done(); + }); + }); + + it('should return items from localStorage', (done) => { + window.localStorage.setItem(service.localStorageKey, '["foo", "bar"]'); + const fetchItemsPromise = service.fetch(); + + fetchItemsPromise + .then((items) => { + expect(items).toEqual(['foo', 'bar']); + done(); + }); + }); + }); + + describe('setRecentSearches', () => { + it('should save things in localStorage', () => { + const items = ['foo', 'bar']; + service.save(items); + const newLocalStorageValue = + window.localStorage.getItem(service.localStorageKey); + expect(JSON.parse(newLocalStorageValue)).toEqual(items); + }); + }); +}); diff --git a/spec/javascripts/filtered_search/stores/recent_searches_store_spec.js b/spec/javascripts/filtered_search/stores/recent_searches_store_spec.js new file mode 100644 index 00000000000..1eebc6f2367 --- /dev/null +++ b/spec/javascripts/filtered_search/stores/recent_searches_store_spec.js @@ -0,0 +1,59 @@ +import RecentSearchesStore from '~/filtered_search/stores/recent_searches_store'; + +describe('RecentSearchesStore', () => { + let store; + + beforeEach(() => { + store = new RecentSearchesStore(); + }); + + describe('addRecentSearch', () => { + it('should add to the front of the list', () => { + store.addRecentSearch('foo'); + store.addRecentSearch('bar'); + + expect(store.state.recentSearches).toEqual(['bar', 'foo']); + }); + + it('should deduplicate', () => { + store.addRecentSearch('foo'); + store.addRecentSearch('bar'); + store.addRecentSearch('foo'); + + expect(store.state.recentSearches).toEqual(['foo', 'bar']); + }); + + it('only keeps track of 5 items', () => { + store.addRecentSearch('1'); + store.addRecentSearch('2'); + store.addRecentSearch('3'); + store.addRecentSearch('4'); + store.addRecentSearch('5'); + store.addRecentSearch('6'); + store.addRecentSearch('7'); + + expect(store.state.recentSearches).toEqual(['7', '6', '5', '4', '3']); + }); + }); + + describe('setRecentSearches', () => { + it('should override list', () => { + store.setRecentSearches([ + 'foo', + 'bar', + ]); + store.setRecentSearches([ + 'baz', + 'qux', + ]); + + expect(store.state.recentSearches).toEqual(['baz', 'qux']); + }); + + it('only keeps track of 5 items', () => { + store.setRecentSearches(['1', '2', '3', '4', '5', '6', '7']); + + expect(store.state.recentSearches).toEqual(['1', '2', '3', '4', '5']); + }); + }); +}); diff --git a/spec/support/filtered_search_helpers.rb b/spec/support/filtered_search_helpers.rb index 6b009b132b6..36be0bb6bf8 100644 --- a/spec/support/filtered_search_helpers.rb +++ b/spec/support/filtered_search_helpers.rb @@ -30,7 +30,7 @@ module FilteredSearchHelpers end def clear_search_field - find('.filtered-search-input-container .clear-search').click + find('.filtered-search-box .clear-search').click end def reset_filters @@ -51,7 +51,7 @@ module FilteredSearchHelpers # Iterates through each visual token inside # .tokens-container to make sure the correct names and values are rendered def expect_tokens(tokens) - page.find '.filtered-search-input-container .tokens-container' do + page.find '.filtered-search-box .tokens-container' do page.all(:css, '.tokens-container li').each_with_index do |el, index| token_name = tokens[index][:name] token_value = tokens[index][:value] @@ -71,4 +71,18 @@ module FilteredSearchHelpers def get_filtered_search_placeholder find('.filtered-search')['placeholder'] end + + def remove_recent_searches + execute_script('window.localStorage.removeItem(\'issue-recent-searches\');') + end + + def set_recent_searches(input) + execute_script("window.localStorage.setItem('issue-recent-searches', '#{input}');") + end + + def wait_for_filtered_search(text) + Timeout.timeout(Capybara.default_max_wait_time) do + loop until find('.filtered-search').value.strip == text + end + end end -- cgit v1.2.1 From 163e9f99ab29524bca204fbd0d0aabc1a1813e4a Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 6 Apr 2017 12:25:09 +0200 Subject: Fix indexes in container repositories table --- db/migrate/20170322013926_create_container_repository.rb | 3 +-- spec/models/container_repository_spec.rb | 7 ------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/db/migrate/20170322013926_create_container_repository.rb b/db/migrate/20170322013926_create_container_repository.rb index 32eba3903b1..91540bc88bd 100644 --- a/db/migrate/20170322013926_create_container_repository.rb +++ b/db/migrate/20170322013926_create_container_repository.rb @@ -5,13 +5,12 @@ class CreateContainerRepository < ActiveRecord::Migration def change create_table :container_repositories do |t| - t.references :project, foreign_key: true, null: false + t.references :project, foreign_key: true, index: true, null: false t.string :name, null: false t.timestamps null: false end - add_index :container_repositories, :project_id add_index :container_repositories, [:project_id, :name], unique: true end end diff --git a/spec/models/container_repository_spec.rb b/spec/models/container_repository_spec.rb index 3e6082ec326..f7ee0b57072 100644 --- a/spec/models/container_repository_spec.rb +++ b/spec/models/container_repository_spec.rb @@ -21,13 +21,6 @@ describe ContainerRepository do headers: { 'Content-Type' => 'application/json' }) end - describe 'validations' do - it 'validates uniqueness of name scoped to project' do - expect(subject).to validate_uniqueness_of(:name) - .scoped_to(:project_id) - end - end - describe 'associations' do it 'belongs to the project' do expect(container_repository).to belong_to(:project) -- cgit v1.2.1 From 542b0d8b0194658710320ec163ec1ddbe2e8702b Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Thu, 6 Apr 2017 15:05:06 +0200 Subject: Use config.toml to configure Gitaly --- GITALY_SERVER_VERSION | 2 +- doc/administration/gitaly/index.md | 32 +++++++++++++++----------------- doc/install/installation.md | 14 ++++++++------ doc/update/9.0-to-9.1.md | 19 ++++++++++++++++++- lib/support/init.d/gitlab | 3 +-- 5 files changed, 43 insertions(+), 27 deletions(-) diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index 8f0916f768f..a918a2aa18d 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -0.5.0 +0.6.0 diff --git a/doc/administration/gitaly/index.md b/doc/administration/gitaly/index.md index 30a4c08508d..2e22212ddde 100644 --- a/doc/administration/gitaly/index.md +++ b/doc/administration/gitaly/index.md @@ -2,7 +2,7 @@ [Gitaly](https://gitlab.com/gitlab-org/gitlay) (introduced in GitLab 9.0) is a service that provides high-level RPC access to Git -repositories. As of GitLab 9.0 it is still an optional component with +repositories. As of GitLab 9.1 it is still an optional component with limited scope. GitLab components that access Git repositories (gitlab-rails, @@ -11,28 +11,26 @@ not have direct access to Gitaly. ## Configuring Gitaly -The Gitaly service itself is configured via environment variables. -These variables are documented [in the gitaly +The Gitaly service itself is configured via a TOML configuration file. +This file is documented [in the gitaly repository](https://gitlab.com/gitlab-org/gitaly/blob/master/doc/configuration/README.md). -To change a Gitaly environment variable in Omnibus you can use -`gitaly['env']` in `/etc/gitlab/gitlab.rb`. Changes will be applied +To change a Gitaly setting in Omnibus you can use +`gitaly['my_setting']` in `/etc/gitlab/gitlab.rb`. Changes will be applied when you run `gitlab-ctl reconfigure`. ```ruby -gitaly['env'] = { - 'GITALY_MY_VARIABLE' => 'value' -} +gitaly['prometheus_listen_addr'] = 'localhost:9236' ``` -To change a Gitaly environment variable in installations from source -you can edit `/home/git/gitaly/env`. +To change a Gitaly setting in installations from source you can edit +`/home/git/gitaly/config.toml`. -```shell -GITALY_MY_VARIABLE='value' +```toml +prometheus_listen_addr = "localhost:9236" ``` -Changes to `/home/git/gitaly/env` are applied when you run `service +Changes to `/home/git/gitaly/config.toml` are applied when you run `service gitlab restart`. ## Configuring GitLab to not use Gitaly @@ -49,15 +47,15 @@ gitaly['enable'] = false ``` In source installations, edit `/home/git/gitlab/config/gitlab.yml` and -make sure `socket_path` in the `gitaly` section is commented out. This -does not disable the Gitaly service; it only prevents it from being -used. +make sure `enabled` in the `gitaly` section is set to 'false'. This +does not disable the Gitaly service in your init script; it only +prevents it from being used. Apply the change with `service gitlab restart`. ```yaml gitaly: - # socket_path: tmp/sockets/private/gitlay.socket + enabled: false ``` ## Disabling or enabling the Gitaly service diff --git a/doc/install/installation.md b/doc/install/installation.md index a2248a38435..5b72c2cce07 100644 --- a/doc/install/installation.md +++ b/doc/install/installation.md @@ -459,9 +459,9 @@ Make GitLab start on boot: ### Install Gitaly -As of GitLab 9.0 Gitaly is an **optional** component. Its -configuration is expected to change in GitLab 9.1. It is OK to wait -with setting up Gitaly until you upgrade to GitLab 9.1 or later. +As of GitLab 9.1 Gitaly is an **optional** component. Its +configuration is still changing regularly. It is OK to wait +with setting up Gitaly until you upgrade to GitLab 9.2 or later. # Fetch Gitaly source with Git and compile with Go sudo -u git -H bundle exec rake "gitlab:gitaly:install[/home/git/gitaly]" RAILS_ENV=production @@ -471,9 +471,11 @@ with setting up Gitaly until you upgrade to GitLab 9.1 or later. sudo chown git /home/git/gitlab/tmp/sockets/private # Configure Gitaly - echo 'GITALY_SOCKET_PATH=/home/git/gitlab/tmp/sockets/private/gitaly.socket' | \ - sudo -u git tee -a /home/git/gitaly/env - + cd /home/git/gitaly + sudo -u git cp config.toml.example config.toml + # If you are using non-default settings you need to update config.toml + sudo -u git -H editor config.toml + # Enable Gitaly in the init script echo 'gitaly_enabled=true' | sudo tee -a /etc/default/gitlab diff --git a/doc/update/9.0-to-9.1.md b/doc/update/9.0-to-9.1.md index 53cddb3f290..ae983dea384 100644 --- a/doc/update/9.0-to-9.1.md +++ b/doc/update/9.0-to-9.1.md @@ -297,7 +297,10 @@ during your 9.1 upgrade **you can skip this step**. If you have not yet set up Gitaly then follow [Gitaly section of the installation guide](../install/installation.md#install-gitaly). -If you installed Gitaly in GitLab 9.0 you need to make some changes in gitlab.yml. +If you installed Gitaly in GitLab 9.0 you need to make some changes in +gitlab.yml, and create a new config.toml file. + +#### Gitaly gitlab.yml changes Look for `socket_path:` the `gitaly:` section. Its value is usually `/home/git/gitlab/tmp/sockets/private/gitaly.socket`. Note what socket @@ -318,6 +321,20 @@ the socket path, but with `unix:` in front. Each entry under `storages:` should use the same `gitaly_address`. +#### Gitaly config.toml + +In GitLab 9.1 we are replacing environment variables in Gitaly with a +TOML configuration file. + +```shell +cd /home/git/gitaly + +sudo mv env env.old +sudo -u git cp config.toml.example config.toml +# If you are using custom repository storage paths they need to be in config.toml +sudo -u git -H editor config.toml +``` + ### 11. Start application ```bash diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index 09e121e5120..6e351365de0 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -326,8 +326,7 @@ start_gitlab() { echo "Gitaly is already running with pid $gapid, not restarting" else $app_root/bin/daemon_with_pidfile $gitaly_pid_path \ - $app_root/bin/with_env $gitaly_dir/env \ - $gitaly_dir/gitaly >> $gitaly_log 2>&1 & + $gitaly_dir/gitaly $gitaly_dir/config.toml >> $gitaly_log 2>&1 & fi fi -- cgit v1.2.1 From 828d81ee1f6aaaf6ab9de1ed0c675ff961831c17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Thu, 6 Apr 2017 16:20:27 +0000 Subject: Optimise trace handling code to use streaming instead of full read --- app/assets/javascripts/build.js | 4 +- app/controllers/projects/builds_controller.rb | 42 ++-- app/models/ci/build.rb | 156 ++---------- app/views/notify/pipeline_failed_email.html.haml | 2 +- app/views/notify/pipeline_failed_email.text.erb | 2 +- app/views/projects/builds/_sidebar.html.haml | 2 +- db/fixtures/development/14_pipelines.rb | 2 +- features/steps/project/builds/summary.rb | 2 +- features/steps/shared/builds.rb | 2 +- lib/api/jobs.rb | 2 +- lib/api/runner.rb | 12 +- lib/api/v3/builds.rb | 2 +- lib/ci/ansi2html.rb | 62 +++-- lib/ci/api/builds.rb | 12 +- lib/gitlab/ci/trace.rb | 136 ++++++++++ lib/gitlab/ci/trace/stream.rb | 119 +++++++++ lib/gitlab/ci/trace_reader.rb | 50 ---- spec/factories/ci/builds.rb | 2 +- spec/features/projects/builds_spec.rb | 71 ++---- spec/javascripts/build_spec.js | 6 +- spec/lib/ci/ansi2html_spec.rb | 110 +++++---- spec/lib/gitlab/ci/trace/stream_spec.rb | 201 +++++++++++++++ spec/lib/gitlab/ci/trace_reader_spec.rb | 52 ---- spec/lib/gitlab/ci/trace_spec.rb | 216 ++++++++++++++++ spec/models/ci/build_spec.rb | 300 +++++++---------------- spec/requests/api/jobs_spec.rb | 4 +- spec/requests/api/runner_spec.rb | 24 +- spec/requests/api/v3/builds_spec.rb | 4 +- spec/requests/ci/api/builds_spec.rb | 22 +- 29 files changed, 988 insertions(+), 633 deletions(-) create mode 100644 lib/gitlab/ci/trace.rb create mode 100644 lib/gitlab/ci/trace/stream.rb delete mode 100644 lib/gitlab/ci/trace_reader.rb create mode 100644 spec/lib/gitlab/ci/trace/stream_spec.rb delete mode 100644 spec/lib/gitlab/ci/trace_reader_spec.rb create mode 100644 spec/lib/gitlab/ci/trace_spec.rb diff --git a/app/assets/javascripts/build.js b/app/assets/javascripts/build.js index fe54ecffdfe..7cb022c848a 100644 --- a/app/assets/javascripts/build.js +++ b/app/assets/javascripts/build.js @@ -84,10 +84,10 @@ window.Build = (function() { var removeRefreshStatuses = ['success', 'failed', 'canceled', 'skipped']; return $.ajax({ - url: this.buildUrl, + url: this.pageUrl + "/trace.json", dataType: 'json', success: function(buildData) { - $('.js-build-output').html(buildData.trace_html); + $('.js-build-output').html(buildData.html); gl.utils.setCiStatusFavicon(`${this.pageUrl}/status.json`); if (window.location.hash === DOWN_BUILD_TRACE) { $("html,body").scrollTop(this.$buildTrace.height()); diff --git a/app/controllers/projects/builds_controller.rb b/app/controllers/projects/builds_controller.rb index 3f3c90a49ab..add66ce9f84 100644 --- a/app/controllers/projects/builds_controller.rb +++ b/app/controllers/projects/builds_controller.rb @@ -31,25 +31,25 @@ class Projects::BuildsController < Projects::ApplicationController @builds = @project.pipelines.find_by_sha(@build.sha).builds.order('id DESC') @builds = @builds.where("id not in (?)", @build.id) @pipeline = @build.pipeline - - respond_to do |format| - format.html - format.json do - render json: { - id: @build.id, - status: @build.status, - trace_html: @build.trace_html - } - end - end end def trace - respond_to do |format| - format.json do - state = params[:state].presence - render json: @build.trace_with_state(state: state). - merge!(id: @build.id, status: @build.status) + build.trace.read do |stream| + respond_to do |format| + format.json do + result = { + id: @build.id, status: @build.status, complete: @build.complete? + } + + if stream.valid? + stream.limit + state = params[:state].presence + trace = stream.html_with_state(state) + result.merge!(trace.to_h) + end + + render json: result + end end end end @@ -86,10 +86,12 @@ class Projects::BuildsController < Projects::ApplicationController end def raw - if @build.has_trace_file? - send_file @build.trace_file_path, type: 'text/plain; charset=utf-8', disposition: 'inline' - else - render_404 + build.trace.read do |stream| + if stream.file? + send_file stream.path, type: 'text/plain; charset=utf-8', disposition: 'inline' + else + render_404 + end end end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 8431c5f228c..70470414bc4 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -171,19 +171,6 @@ module Ci latest_builds.where('stage_idx < ?', stage_idx) end - def trace_html(**args) - trace_with_state(**args)[:html] || '' - end - - def trace_with_state(state: nil, last_lines: nil) - trace_ansi = trace(last_lines: last_lines) - if trace_ansi.present? - Ci::Ansi2html.convert(trace_ansi, state) - else - {} - end - end - def timeout project.build_timeout end @@ -244,136 +231,35 @@ module Ci end def update_coverage - coverage = extract_coverage(trace, coverage_regex) + coverage = trace.extract_coverage(coverage_regex) update_attributes(coverage: coverage) if coverage.present? end - def extract_coverage(text, regex) - return unless regex - - matches = text.scan(Regexp.new(regex)).last - matches = matches.last if matches.is_a?(Array) - coverage = matches.gsub(/\d+(\.\d+)?/).first - - if coverage.present? - coverage.to_f - end - rescue - # if bad regex or something goes wrong we dont want to interrupt transition - # so we just silentrly ignore error for now - end - - def has_trace_file? - File.exist?(path_to_trace) || has_old_trace_file? + def trace + Gitlab::Ci::Trace.new(self) end def has_trace? - raw_trace.present? + trace.exist? end - def raw_trace(last_lines: nil) - if File.exist?(trace_file_path) - Gitlab::Ci::TraceReader.new(trace_file_path). - read(last_lines: last_lines) - else - # backward compatibility - read_attribute :trace - end + def trace=(data) + raise NotImplementedError end - ## - # Deprecated - # - # This is a hotfix for CI build data integrity, see #4246 - def has_old_trace_file? - project.ci_id && File.exist?(old_path_to_trace) - end - - def trace(last_lines: nil) - hide_secrets(raw_trace(last_lines: last_lines)) + def old_trace + read_attribute(:trace) end - def trace_length - if raw_trace - raw_trace.bytesize - else - 0 - end - end - - def trace=(trace) - recreate_trace_dir - trace = hide_secrets(trace) - File.write(path_to_trace, trace) - end - - def recreate_trace_dir - unless Dir.exist?(dir_to_trace) - FileUtils.mkdir_p(dir_to_trace) - end - end - private :recreate_trace_dir - - def append_trace(trace_part, offset) - recreate_trace_dir - touch if needs_touch? - - trace_part = hide_secrets(trace_part) - - File.truncate(path_to_trace, offset) if File.exist?(path_to_trace) - File.open(path_to_trace, 'ab') do |f| - f.write(trace_part) - end + def erase_old_trace! + write_attribute(:trace, nil) + save end def needs_touch? Time.now - updated_at > 15.minutes.to_i end - def trace_file_path - if has_old_trace_file? - old_path_to_trace - else - path_to_trace - end - end - - def dir_to_trace - File.join( - Settings.gitlab_ci.builds_path, - created_at.utc.strftime("%Y_%m"), - project.id.to_s - ) - end - - def path_to_trace - "#{dir_to_trace}/#{id}.log" - end - - ## - # Deprecated - # - # This is a hotfix for CI build data integrity, see #4246 - # Should be removed in 8.4, after CI files migration has been done. - # - def old_dir_to_trace - File.join( - Settings.gitlab_ci.builds_path, - created_at.utc.strftime("%Y_%m"), - project.ci_id.to_s - ) - end - - ## - # Deprecated - # - # This is a hotfix for CI build data integrity, see #4246 - # Should be removed in 8.4, after CI files migration has been done. - # - def old_path_to_trace - "#{old_dir_to_trace}/#{id}.log" - end - ## # Deprecated # @@ -555,6 +441,15 @@ module Ci options[:dependencies]&.empty? end + def hide_secrets(trace) + return unless trace + + trace = trace.dup + Ci::MaskSecret.mask!(trace, project.runners_token) if project + Ci::MaskSecret.mask!(trace, token) + trace + end + private def update_artifacts_size @@ -566,7 +461,7 @@ module Ci end def erase_trace! - self.trace = nil + trace.erase! end def update_erased!(user = nil) @@ -628,15 +523,6 @@ module Ci pipeline.config_processor.build_attributes(name) end - def hide_secrets(trace) - return unless trace - - trace = trace.dup - Ci::MaskSecret.mask!(trace, project.runners_token) if project - Ci::MaskSecret.mask!(trace, token) - trace - end - def update_project_statistics return unless project diff --git a/app/views/notify/pipeline_failed_email.html.haml b/app/views/notify/pipeline_failed_email.html.haml index 4beb6fcee5d..a83faa839df 100644 --- a/app/views/notify/pipeline_failed_email.html.haml +++ b/app/views/notify/pipeline_failed_email.html.haml @@ -137,6 +137,6 @@ - if build.has_trace? %td{ colspan: "2", style: "font-family:'Helvetica Neue',Helvetica,Arial,sans-serif;padding:0 0 15px;" } %pre{ style: "font-family:Monaco,'Lucida Console','Courier New',Courier,monospace;background-color:#fafafa;border-radius:3px;overflow:hidden;white-space:pre-wrap;word-break:break-all;font-size:13px;line-height:1.4;padding:12px;color:#333333;margin:0;" } - = build.trace_html(last_lines: 10).html_safe + = build.trace.html(last_lines: 10).html_safe - else %td{ colspan: "2" } diff --git a/app/views/notify/pipeline_failed_email.text.erb b/app/views/notify/pipeline_failed_email.text.erb index c1a4ea40cf5..294238eee51 100644 --- a/app/views/notify/pipeline_failed_email.text.erb +++ b/app/views/notify/pipeline_failed_email.text.erb @@ -35,7 +35,7 @@ had <%= failed.size %> failed <%= 'build'.pluralize(failed.size) %>. Stage: <%= build.stage %> Name: <%= build.name %> <% if build.has_trace? -%> -Trace: <%= build.trace_with_state(last_lines: 10)[:text] %> +Trace: <%= build.trace.raw(last_lines: 10) %> <% end -%> <% end -%> diff --git a/app/views/projects/builds/_sidebar.html.haml b/app/views/projects/builds/_sidebar.html.haml index 6f45d5b0689..f4a66398c85 100644 --- a/app/views/projects/builds/_sidebar.html.haml +++ b/app/views/projects/builds/_sidebar.html.haml @@ -68,7 +68,7 @@ - elsif @build.runner \##{@build.runner.id} .btn-group.btn-group-justified{ role: :group } - - if @build.has_trace_file? + - if @build.has_trace? = link_to 'Raw', raw_namespace_project_build_path(@project.namespace, @project, @build), class: 'btn btn-sm btn-default' - if @build.active? = link_to "Cancel", cancel_namespace_project_build_path(@project.namespace, @project, @build), class: 'btn btn-sm btn-default', method: :post diff --git a/db/fixtures/development/14_pipelines.rb b/db/fixtures/development/14_pipelines.rb index 534847a7107..3c42f7db6d5 100644 --- a/db/fixtures/development/14_pipelines.rb +++ b/db/fixtures/development/14_pipelines.rb @@ -130,7 +130,7 @@ class Gitlab::Seeder::Pipelines def setup_build_log(build) if %w(running success failed).include?(build.status) - build.trace = FFaker::Lorem.paragraphs(6).join("\n\n") + build.trace.set(FFaker::Lorem.paragraphs(6).join("\n\n")) end end diff --git a/features/steps/project/builds/summary.rb b/features/steps/project/builds/summary.rb index 19ff92f6dc6..124582de6b9 100644 --- a/features/steps/project/builds/summary.rb +++ b/features/steps/project/builds/summary.rb @@ -22,9 +22,9 @@ class Spinach::Features::ProjectBuildsSummary < Spinach::FeatureSteps end step 'recent build has been erased' do + expect(@build).not_to have_trace expect(@build.artifacts_file.exists?).to be_falsy expect(@build.artifacts_metadata.exists?).to be_falsy - expect(@build.trace).to be_empty end step 'recent build summary does not have artifacts widget' do diff --git a/features/steps/shared/builds.rb b/features/steps/shared/builds.rb index 5bc3a1f5ac4..5549fc25525 100644 --- a/features/steps/shared/builds.rb +++ b/features/steps/shared/builds.rb @@ -47,7 +47,7 @@ module SharedBuilds end step 'recent build has a build trace' do - @build.trace = 'job trace' + @build.trace.set('job trace') end step 'download of build artifacts archive starts' do diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index ffab0aafe59..288b03d940c 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -118,7 +118,7 @@ module API content_type 'text/plain' env['api.format'] = :binary - trace = build.trace + trace = build.trace.raw body trace end diff --git a/lib/api/runner.rb b/lib/api/runner.rb index d288369e362..6fbb02cb3aa 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -115,7 +115,7 @@ module API put '/:id' do job = authenticate_job! - job.update_attributes(trace: params[:trace]) if params[:trace] + job.trace.set(params[:trace]) if params[:trace] Gitlab::Metrics.add_event(:update_build, project: job.project.path_with_namespace) @@ -145,16 +145,14 @@ module API content_range = request.headers['Content-Range'] content_range = content_range.split('-') - current_length = job.trace_length - unless current_length == content_range[0].to_i - return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{current_length}" }) + stream_size = job.trace.append(request.body.read, content_range[0].to_i) + if stream_size < 0 + return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{-stream_size}" }) end - job.append_trace(request.body.read, content_range[0].to_i) - status 202 header 'Job-Status', job.status - header 'Range', "0-#{job.trace_length}" + header 'Range', "0-#{stream_size}" end desc 'Authorize artifacts uploading for job' do diff --git a/lib/api/v3/builds.rb b/lib/api/v3/builds.rb index 6f97102c6ef..4dd03cdf24b 100644 --- a/lib/api/v3/builds.rb +++ b/lib/api/v3/builds.rb @@ -120,7 +120,7 @@ module API content_type 'text/plain' env['api.format'] = :binary - trace = build.trace + trace = build.trace.raw body trace end diff --git a/lib/ci/ansi2html.rb b/lib/ci/ansi2html.rb index b3ccad7b28d..1020452480a 100644 --- a/lib/ci/ansi2html.rb +++ b/lib/ci/ansi2html.rb @@ -132,34 +132,54 @@ module Ci STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask].freeze - def convert(raw, new_state) + def convert(stream, new_state) reset_state - restore_state(raw, new_state) if new_state.present? - - start = @offset - ansi = raw[@offset..-1] + restore_state(new_state, stream) if new_state.present? + + append = false + truncated = false + + cur_offset = stream.tell + if cur_offset > @offset + @offset = cur_offset + truncated = true + else + stream.seek(@offset) + append = @offset > 0 + end + start_offset = @offset open_new_tag - s = StringScanner.new(ansi) - until s.eos? - if s.scan(/\e([@-_])(.*?)([@-~])/) - handle_sequence(s) - elsif s.scan(/\e(([@-_])(.*?)?)?$/) - break - elsif s.scan(/' - else - @out << s.scan(/./m) + stream.each_line do |line| + s = StringScanner.new(line) + until s.eos? + if s.scan(/\e([@-_])(.*?)([@-~])/) + handle_sequence(s) + elsif s.scan(/\e(([@-_])(.*?)?)?$/) + break + elsif s.scan(/' + else + @out << s.scan(/./m) + end + @offset += s.matched_size end - @offset += s.matched_size end close_open_tags() - { state: state, html: @out, text: ansi[0, @offset - start], append: start > 0 } + OpenStruct.new( + html: @out, + state: state, + append: append, + truncated: truncated, + offset: start_offset, + size: stream.tell - start_offset, + total: stream.size + ) end def handle_sequence(s) @@ -240,10 +260,10 @@ module Ci Base64.urlsafe_encode64(state.to_json) end - def restore_state(raw, new_state) + def restore_state(new_state, stream) state = Base64.urlsafe_decode64(new_state) state = JSON.parse(state, symbolize_names: true) - return if state[:offset].to_i > raw.length + return if state[:offset].to_i > stream.size STATE_PARAMS.each do |param| send("#{param}=".to_sym, state[param]) diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 95cc6308c3b..67b269b330c 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -61,7 +61,7 @@ module Ci update_runner_info - build.update_attributes(trace: params[:trace]) if params[:trace] + build.trace.set(params[:trace]) if params[:trace] Gitlab::Metrics.add_event(:update_build, project: build.project.path_with_namespace) @@ -92,16 +92,14 @@ module Ci content_range = request.headers['Content-Range'] content_range = content_range.split('-') - current_length = build.trace_length - unless current_length == content_range[0].to_i - return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{current_length}" }) + stream_size = build.trace.append(request.body.read, content_range[0].to_i) + if stream_size < 0 + return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{-stream_size}" }) end - build.append_trace(request.body.read, content_range[0].to_i) - status 202 header 'Build-Status', build.status - header 'Range', "0-#{build.trace_length}" + header 'Range', "0-#{stream_size}" end # Authorize artifacts uploading for build - Runners only diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb new file mode 100644 index 00000000000..5b835bb669a --- /dev/null +++ b/lib/gitlab/ci/trace.rb @@ -0,0 +1,136 @@ +module Gitlab + module Ci + class Trace + attr_reader :job + + delegate :old_trace, to: :job + + def initialize(job) + @job = job + end + + def html(last_lines: nil) + read do |stream| + stream.html(last_lines: last_lines) + end + end + + def raw(last_lines: nil) + read do |stream| + stream.raw(last_lines: last_lines) + end + end + + def extract_coverage(regex) + read do |stream| + stream.extract_coverage(regex) + end + end + + def set(data) + write do |stream| + data = job.hide_secrets(data) + stream.set(data) + end + end + + def append(data, offset) + write do |stream| + current_length = stream.size + return -current_length unless current_length == offset + + data = job.hide_secrets(data) + stream.append(data, offset) + stream.size + end + end + + def exist? + current_path.present? || old_trace.present? + end + + def read + stream = Gitlab::Ci::Trace::Stream.new do + if current_path + File.open(current_path, "rb") + elsif old_trace + StringIO.new(old_trace) + end + end + + yield stream + ensure + stream&.close + end + + def write + stream = Gitlab::Ci::Trace::Stream.new do + File.open(ensure_path, "a+b") + end + + yield(stream).tap do + job.touch if job.needs_touch? + end + ensure + stream&.close + end + + def erase! + paths.each do |trace_path| + FileUtils.rm(trace_path, force: true) + end + + job.erase_old_trace! + end + + private + + def ensure_path + return current_path if current_path + + ensure_directory + default_path + end + + def ensure_directory + unless Dir.exist?(default_directory) + FileUtils.mkdir_p(default_directory) + end + end + + def current_path + @current_path ||= paths.find do |trace_path| + File.exist?(trace_path) + end + end + + def paths + [ + default_path, + deprecated_path + ].compact + end + + def default_directory + File.join( + Settings.gitlab_ci.builds_path, + job.created_at.utc.strftime("%Y_%m"), + job.project_id.to_s + ) + end + + def default_path + File.join(default_directory, "#{job.id}.log") + end + + def deprecated_path + File.join( + Settings.gitlab_ci.builds_path, + job.created_at.utc.strftime("%Y_%m"), + job.project.ci_id.to_s, + "#{job.id}.log" + ) if job.project&.ci_id + end + end + end +end diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb new file mode 100644 index 00000000000..2af94e2c60e --- /dev/null +++ b/lib/gitlab/ci/trace/stream.rb @@ -0,0 +1,119 @@ +module Gitlab + module Ci + class Trace + # This was inspired from: http://stackoverflow.com/a/10219411/1520132 + class Stream + BUFFER_SIZE = 4096 + LIMIT_SIZE = 50.kilobytes + + attr_reader :stream + + delegate :close, :tell, :seek, :size, :path, :truncate, to: :stream, allow_nil: true + + delegate :valid?, to: :stream, as: :present?, allow_nil: true + + def initialize + @stream = yield + end + + def valid? + self.stream.present? + end + + def file? + self.path.present? + end + + def limit(last_bytes = LIMIT_SIZE) + stream_size = size + if stream_size < last_bytes + last_bytes = stream_size + end + stream.seek(-last_bytes, IO::SEEK_END) + end + + def append(data, offset) + stream.truncate(offset) + stream.seek(0, IO::SEEK_END) + stream.write(data) + stream.flush() + end + + def set(data) + truncate(0) + stream.write(data) + stream.flush() + end + + def raw(last_lines: nil) + return unless valid? + + if last_lines.to_i > 0 + read_last_lines(last_lines) + else + stream.read + end + end + + def html_with_state(state = nil) + ::Ci::Ansi2html.convert(stream, state) + end + + def html(last_lines: nil) + text = raw(last_lines: last_lines) + stream = StringIO.new(text) + ::Ci::Ansi2html.convert(stream).html + end + + def extract_coverage(regex) + return unless valid? + return unless regex + + regex = Regexp.new(regex) + + match = "" + + stream.each_line do |line| + matches = line.scan(regex) + next unless matches.is_a?(Array) + + match = matches.flatten.last + coverage = match.gsub(/\d+(\.\d+)?/).first + return coverage.to_f if coverage.present? + end + rescue + # if bad regex or something goes wrong we dont want to interrupt transition + # so we just silentrly ignore error for now + end + + private + + def read_last_lines(last_lines) + chunks = [] + pos = lines = 0 + max = stream.size + + # We want an extra line to make sure fist line has full contents + while lines <= last_lines && pos < max + pos += BUFFER_SIZE + + buf = + if pos <= max + stream.seek(-pos, IO::SEEK_END) + stream.read(BUFFER_SIZE) + else # Reached the head, read only left + stream.seek(0) + stream.read(BUFFER_SIZE - (pos - max)) + end + + lines += buf.count("\n") + chunks.unshift(buf) + end + + chunks.join.lines.last(last_lines).join + .force_encoding(Encoding.default_external) + end + end + end + end +end diff --git a/lib/gitlab/ci/trace_reader.rb b/lib/gitlab/ci/trace_reader.rb deleted file mode 100644 index 1d7ddeb3e0f..00000000000 --- a/lib/gitlab/ci/trace_reader.rb +++ /dev/null @@ -1,50 +0,0 @@ -module Gitlab - module Ci - # This was inspired from: http://stackoverflow.com/a/10219411/1520132 - class TraceReader - BUFFER_SIZE = 4096 - - attr_accessor :path, :buffer_size - - def initialize(new_path, buffer_size: BUFFER_SIZE) - self.path = new_path - self.buffer_size = Integer(buffer_size) - end - - def read(last_lines: nil) - if last_lines - read_last_lines(last_lines) - else - File.read(path) - end - end - - def read_last_lines(max_lines) - File.open(path) do |file| - chunks = [] - pos = lines = 0 - max = file.size - - # We want an extra line to make sure fist line has full contents - while lines <= max_lines && pos < max - pos += buffer_size - - buf = if pos <= max - file.seek(-pos, IO::SEEK_END) - file.read(buffer_size) - else # Reached the head, read only left - file.seek(0) - file.read(buffer_size - (pos - max)) - end - - lines += buf.count("\n") - chunks.unshift(buf) - end - - chunks.join.lines.last(max_lines).join - .force_encoding(Encoding.default_external) - end - end - end - end -end diff --git a/spec/factories/ci/builds.rb b/spec/factories/ci/builds.rb index 87a0c95c4dc..b62def83ee4 100644 --- a/spec/factories/ci/builds.rb +++ b/spec/factories/ci/builds.rb @@ -111,7 +111,7 @@ FactoryGirl.define do trait :trace do after(:create) do |build, evaluator| - build.trace = 'BUILD TRACE' + build.trace.set('BUILD TRACE') end end diff --git a/spec/features/projects/builds_spec.rb b/spec/features/projects/builds_spec.rb index 2116721b224..ab10434e10c 100644 --- a/spec/features/projects/builds_spec.rb +++ b/spec/features/projects/builds_spec.rb @@ -205,21 +205,13 @@ feature 'Builds', :feature do it 'loads job trace' do expect(page).to have_content 'BUILD TRACE' - build.append_trace(' and more trace', 11) + build.trace.write do |stream| + stream.append(' and more trace', 11) + end expect(page).to have_content 'BUILD TRACE and more trace' end end - - context 'when build does not have an initial trace' do - let(:build) { create(:ci_build, pipeline: pipeline) } - - it 'loads new trace' do - build.append_trace('build trace', 0) - - expect(page).to have_content 'build trace' - end - end end feature 'Variables' do @@ -390,7 +382,7 @@ feature 'Builds', :feature do it 'sends the right headers' do expect(page.status_code).to eq(200) expect(page.response_headers['Content-Type']).to eq('text/plain; charset=utf-8') - expect(page.response_headers['X-Sendfile']).to eq(build.path_to_trace) + expect(page.response_headers['X-Sendfile']).to eq(build.trace.send(:current_path)) end end @@ -409,43 +401,24 @@ feature 'Builds', :feature do context 'storage form' do let(:existing_file) { Tempfile.new('existing-trace-file').path } - let(:non_existing_file) do - file = Tempfile.new('non-existing-trace-file') - path = file.path - file.unlink - path - end - context 'when build has trace in file' do - before do - Capybara.current_session.driver.header('X-Sendfile-Type', 'X-Sendfile') - build.run! - visit namespace_project_build_path(project.namespace, project, build) + before do + Capybara.current_session.driver.header('X-Sendfile-Type', 'X-Sendfile') - allow_any_instance_of(Project).to receive(:ci_id).and_return(nil) - allow_any_instance_of(Ci::Build).to receive(:path_to_trace).and_return(existing_file) - allow_any_instance_of(Ci::Build).to receive(:old_path_to_trace).and_return(non_existing_file) + build.run! - page.within('.js-build-sidebar') { click_link 'Raw' } - end + allow_any_instance_of(Gitlab::Ci::Trace).to receive(:paths) + .and_return(paths) - it 'sends the right headers' do - expect(page.status_code).to eq(200) - expect(page.response_headers['Content-Type']).to eq('text/plain; charset=utf-8') - expect(page.response_headers['X-Sendfile']).to eq(existing_file) - end + visit namespace_project_build_path(project.namespace, project, build) end - context 'when build has trace in old file' do - before do - Capybara.current_session.driver.header('X-Sendfile-Type', 'X-Sendfile') - build.run! - visit namespace_project_build_path(project.namespace, project, build) - - allow_any_instance_of(Project).to receive(:ci_id).and_return(999) - allow_any_instance_of(Ci::Build).to receive(:path_to_trace).and_return(non_existing_file) - allow_any_instance_of(Ci::Build).to receive(:old_path_to_trace).and_return(existing_file) + context 'when build has trace in file' do + let(:paths) do + [existing_file] + end + before do page.within('.js-build-sidebar') { click_link 'Raw' } end @@ -457,20 +430,10 @@ feature 'Builds', :feature do end context 'when build has trace in DB' do - before do - Capybara.current_session.driver.header('X-Sendfile-Type', 'X-Sendfile') - build.run! - visit namespace_project_build_path(project.namespace, project, build) - - allow_any_instance_of(Project).to receive(:ci_id).and_return(nil) - allow_any_instance_of(Ci::Build).to receive(:path_to_trace).and_return(non_existing_file) - allow_any_instance_of(Ci::Build).to receive(:old_path_to_trace).and_return(non_existing_file) - - page.within('.js-build-sidebar') { click_link 'Raw' } - end + let(:paths) { [] } it 'sends the right headers' do - expect(page.status_code).to eq(404) + expect(page.status_code).not_to have_link('Raw') end end end diff --git a/spec/javascripts/build_spec.js b/spec/javascripts/build_spec.js index beee6cb2969..cb48f14fd9a 100644 --- a/spec/javascripts/build_spec.js +++ b/spec/javascripts/build_spec.js @@ -72,12 +72,14 @@ describe('Build', () => { it('displays the initial build trace', () => { expect($.ajax.calls.count()).toBe(1); const [{ url, dataType, success, context }] = $.ajax.calls.argsFor(0); - expect(url).toBe(`${BUILD_URL}.json`); + expect(url).toBe( + `${BUILD_URL}/trace.json`, + ); expect(dataType).toBe('json'); expect(success).toEqual(jasmine.any(Function)); spyOn(gl.utils, 'setCiStatusFavicon').and.callFake(() => {}); - success.call(context, { trace_html: 'Example', status: 'running' }); + success.call(context, { html: 'Example', status: 'running' }); expect($('#build-trace .js-build-output').text()).toMatch(/Example/); }); diff --git a/spec/lib/ci/ansi2html_spec.rb b/spec/lib/ci/ansi2html_spec.rb index 0762fd7e56a..a5dfb49478a 100644 --- a/spec/lib/ci/ansi2html_spec.rb +++ b/spec/lib/ci/ansi2html_spec.rb @@ -1,159 +1,160 @@ require 'spec_helper' describe Ci::Ansi2html, lib: true do - subject { Ci::Ansi2html } + subject { described_class } it "prints non-ansi as-is" do - expect(subject.convert("Hello")[:html]).to eq('Hello') + expect(convert_html("Hello")).to eq('Hello') end it "strips non-color-changing controll sequences" do - expect(subject.convert("Hello \e[2Kworld")[:html]).to eq('Hello world') + expect(convert_html("Hello \e[2Kworld")).to eq('Hello world') end it "prints simply red" do - expect(subject.convert("\e[31mHello\e[0m")[:html]).to eq('Hello') + expect(convert_html("\e[31mHello\e[0m")).to eq('Hello') end it "prints simply red without trailing reset" do - expect(subject.convert("\e[31mHello")[:html]).to eq('Hello') + expect(convert_html("\e[31mHello")).to eq('Hello') end it "prints simply yellow" do - expect(subject.convert("\e[33mHello\e[0m")[:html]).to eq('Hello') + expect(convert_html("\e[33mHello\e[0m")).to eq('Hello') end it "prints default on blue" do - expect(subject.convert("\e[39;44mHello")[:html]).to eq('Hello') + expect(convert_html("\e[39;44mHello")).to eq('Hello') end it "prints red on blue" do - expect(subject.convert("\e[31;44mHello")[:html]).to eq('Hello') + expect(convert_html("\e[31;44mHello")).to eq('Hello') end it "resets colors after red on blue" do - expect(subject.convert("\e[31;44mHello\e[0m world")[:html]).to eq('Hello world') + expect(convert_html("\e[31;44mHello\e[0m world")).to eq('Hello world') end it "performs color change from red/blue to yellow/blue" do - expect(subject.convert("\e[31;44mHello \e[33mworld")[:html]).to eq('Hello world') + expect(convert_html("\e[31;44mHello \e[33mworld")).to eq('Hello world') end it "performs color change from red/blue to yellow/green" do - expect(subject.convert("\e[31;44mHello \e[33;42mworld")[:html]).to eq('Hello world') + expect(convert_html("\e[31;44mHello \e[33;42mworld")).to eq('Hello world') end it "performs color change from red/blue to reset to yellow/green" do - expect(subject.convert("\e[31;44mHello\e[0m \e[33;42mworld")[:html]).to eq('Hello world') + expect(convert_html("\e[31;44mHello\e[0m \e[33;42mworld")).to eq('Hello world') end it "ignores unsupported codes" do - expect(subject.convert("\e[51mHello\e[0m")[:html]).to eq('Hello') + expect(convert_html("\e[51mHello\e[0m")).to eq('Hello') end it "prints light red" do - expect(subject.convert("\e[91mHello\e[0m")[:html]).to eq('Hello') + expect(convert_html("\e[91mHello\e[0m")).to eq('Hello') end it "prints default on light red" do - expect(subject.convert("\e[101mHello\e[0m")[:html]).to eq('Hello') + expect(convert_html("\e[101mHello\e[0m")).to eq('Hello') end it "performs color change from red/blue to default/blue" do - expect(subject.convert("\e[31;44mHello \e[39mworld")[:html]).to eq('Hello world') + expect(convert_html("\e[31;44mHello \e[39mworld")).to eq('Hello world') end it "performs color change from light red/blue to default/blue" do - expect(subject.convert("\e[91;44mHello \e[39mworld")[:html]).to eq('Hello world') + expect(convert_html("\e[91;44mHello \e[39mworld")).to eq('Hello world') end it "prints bold text" do - expect(subject.convert("\e[1mHello")[:html]).to eq('Hello') + expect(convert_html("\e[1mHello")).to eq('Hello') end it "resets bold text" do - expect(subject.convert("\e[1mHello\e[21m world")[:html]).to eq('Hello world') - expect(subject.convert("\e[1mHello\e[22m world")[:html]).to eq('Hello world') + expect(convert_html("\e[1mHello\e[21m world")).to eq('Hello world') + expect(convert_html("\e[1mHello\e[22m world")).to eq('Hello world') end it "prints italic text" do - expect(subject.convert("\e[3mHello")[:html]).to eq('Hello') + expect(convert_html("\e[3mHello")).to eq('Hello') end it "resets italic text" do - expect(subject.convert("\e[3mHello\e[23m world")[:html]).to eq('Hello world') + expect(convert_html("\e[3mHello\e[23m world")).to eq('Hello world') end it "prints underlined text" do - expect(subject.convert("\e[4mHello")[:html]).to eq('Hello') + expect(convert_html("\e[4mHello")).to eq('Hello') end it "resets underlined text" do - expect(subject.convert("\e[4mHello\e[24m world")[:html]).to eq('Hello world') + expect(convert_html("\e[4mHello\e[24m world")).to eq('Hello world') end it "prints concealed text" do - expect(subject.convert("\e[8mHello")[:html]).to eq('Hello') + expect(convert_html("\e[8mHello")).to eq('Hello') end it "resets concealed text" do - expect(subject.convert("\e[8mHello\e[28m world")[:html]).to eq('Hello world') + expect(convert_html("\e[8mHello\e[28m world")).to eq('Hello world') end it "prints crossed-out text" do - expect(subject.convert("\e[9mHello")[:html]).to eq('Hello') + expect(convert_html("\e[9mHello")).to eq('Hello') end it "resets crossed-out text" do - expect(subject.convert("\e[9mHello\e[29m world")[:html]).to eq('Hello world') + expect(convert_html("\e[9mHello\e[29m world")).to eq('Hello world') end it "can print 256 xterm fg colors" do - expect(subject.convert("\e[38;5;16mHello")[:html]).to eq('Hello') + expect(convert_html("\e[38;5;16mHello")).to eq('Hello') end it "can print 256 xterm fg colors on normal magenta background" do - expect(subject.convert("\e[38;5;16;45mHello")[:html]).to eq('Hello') + expect(convert_html("\e[38;5;16;45mHello")).to eq('Hello') end it "can print 256 xterm bg colors" do - expect(subject.convert("\e[48;5;240mHello")[:html]).to eq('Hello') + expect(convert_html("\e[48;5;240mHello")).to eq('Hello') end it "can print 256 xterm bg colors on normal magenta foreground" do - expect(subject.convert("\e[48;5;16;35mHello")[:html]).to eq('Hello') + expect(convert_html("\e[48;5;16;35mHello")).to eq('Hello') end it "prints bold colored text vividly" do - expect(subject.convert("\e[1;31mHello\e[0m")[:html]).to eq('Hello') + expect(convert_html("\e[1;31mHello\e[0m")).to eq('Hello') end it "prints bold light colored text correctly" do - expect(subject.convert("\e[1;91mHello\e[0m")[:html]).to eq('Hello') + expect(convert_html("\e[1;91mHello\e[0m")).to eq('Hello') end it "prints <" do - expect(subject.convert("<")[:html]).to eq('<') + expect(convert_html("<")).to eq('<') end it "replaces newlines with line break tags" do - expect(subject.convert("\n")[:html]).to eq('
    ') + expect(convert_html("\n")).to eq('
    ') end it "groups carriage returns with newlines" do - expect(subject.convert("\r\n")[:html]).to eq('
    ') + expect(convert_html("\r\n")).to eq('
    ') end describe "incremental update" do shared_examples 'stateable converter' do - let(:pass1) { subject.convert(pre_text) } - let(:pass2) { subject.convert(pre_text + text, pass1[:state]) } + let(:pass1_stream) { StringIO.new(pre_text) } + let(:pass2_stream) { StringIO.new(pre_text + text) } + let(:pass1) { subject.convert(pass1_stream) } + let(:pass2) { subject.convert(pass2_stream, pass1.state) } it "to returns html to append" do - expect(pass2[:append]).to be_truthy - expect(pass2[:html]).to eq(html) - expect(pass1[:text] + pass2[:text]).to eq(pre_text + text) - expect(pass1[:html] + pass2[:html]).to eq(pre_html + html) + expect(pass2.append).to be_truthy + expect(pass2.html).to eq(html) + expect(pass1.html + pass2.html).to eq(pre_html + html) end end @@ -193,4 +194,27 @@ describe Ci::Ansi2html, lib: true do it_behaves_like 'stateable converter' end end + + describe "truncates" do + let(:text) { "Hello World" } + let(:stream) { StringIO.new(text) } + let(:subject) { described_class.convert(stream) } + + before do + stream.seek(3, IO::SEEK_SET) + end + + it "returns truncated output" do + expect(subject.truncated).to be_truthy + end + + it "does not append output" do + expect(subject.append).to be_falsey + end + end + + def convert_html(data) + stream = StringIO.new(data) + subject.convert(stream).html + end end diff --git a/spec/lib/gitlab/ci/trace/stream_spec.rb b/spec/lib/gitlab/ci/trace/stream_spec.rb new file mode 100644 index 00000000000..f1a1a71c528 --- /dev/null +++ b/spec/lib/gitlab/ci/trace/stream_spec.rb @@ -0,0 +1,201 @@ +require 'spec_helper' + +describe Gitlab::Ci::Trace::Stream do + describe 'delegates' do + subject { described_class.new { nil } } + + it { is_expected.to delegate_method(:close).to(:stream) } + it { is_expected.to delegate_method(:tell).to(:stream) } + it { is_expected.to delegate_method(:seek).to(:stream) } + it { is_expected.to delegate_method(:size).to(:stream) } + it { is_expected.to delegate_method(:path).to(:stream) } + it { is_expected.to delegate_method(:truncate).to(:stream) } + it { is_expected.to delegate_method(:valid?).to(:stream).as(:present?) } + it { is_expected.to delegate_method(:file?).to(:path).as(:present?) } + end + + describe '#limit' do + let(:stream) do + described_class.new do + StringIO.new("12345678") + end + end + + it 'if size is larger we start from beggining' do + stream.limit(10) + + expect(stream.tell).to eq(0) + end + + it 'if size is smaller we start from the end' do + stream.limit(2) + + expect(stream.tell).to eq(6) + end + end + + describe '#append' do + let(:stream) do + described_class.new do + StringIO.new("12345678") + end + end + + it "truncates and append content" do + stream.append("89", 4) + stream.seek(0) + + expect(stream.size).to eq(6) + expect(stream.raw).to eq("123489") + end + end + + describe '#set' do + let(:stream) do + described_class.new do + StringIO.new("12345678") + end + end + + before do + stream.set("8901") + end + + it "overwrite content" do + stream.seek(0) + + expect(stream.size).to eq(4) + expect(stream.raw).to eq("8901") + end + end + + describe '#raw' do + let(:path) { __FILE__ } + let(:lines) { File.readlines(path) } + let(:stream) do + described_class.new do + File.open(path) + end + end + + it 'returns all contents if last_lines is not specified' do + result = stream.raw + + expect(result).to eq(lines.join) + expect(result.encoding).to eq(Encoding.default_external) + end + + context 'limit max lines' do + before do + # specifying BUFFER_SIZE forces to seek backwards + allow(described_class).to receive(:BUFFER_SIZE) + .and_return(2) + end + + it 'returns last few lines' do + result = stream.raw(last_lines: 2) + + expect(result).to eq(lines.last(2).join) + expect(result.encoding).to eq(Encoding.default_external) + end + + it 'returns everything if trying to get too many lines' do + result = stream.raw(last_lines: lines.size * 2) + + expect(result).to eq(lines.join) + expect(result.encoding).to eq(Encoding.default_external) + end + end + end + + describe '#html_with_state' do + let(:stream) do + described_class.new do + StringIO.new("1234") + end + end + + it 'returns html content with state' do + result = stream.html_with_state + + expect(result.html).to eq("1234") + end + + context 'follow-up state' do + let!(:last_result) { stream.html_with_state } + + before do + stream.append("5678", 4) + stream.seek(0) + end + + it "returns appended trace" do + result = stream.html_with_state(last_result.state) + + expect(result.append).to be_truthy + expect(result.html).to eq("5678") + end + end + end + + describe '#html' do + let(:stream) do + described_class.new do + StringIO.new("12\n34\n56") + end + end + + it "returns html" do + expect(stream.html).to eq("12
    34
    56") + end + + it "returns html for last line only" do + expect(stream.html(last_lines: 1)).to eq("56") + end + end + + describe '#extract_coverage' do + let(:stream) do + described_class.new do + StringIO.new(data) + end + end + + subject { stream.extract_coverage(regex) } + + context 'valid content & regex' do + let(:data) { 'Coverage 1033 / 1051 LOC (98.29%) covered' } + let(:regex) { '\(\d+.\d+\%\) covered' } + + it { is_expected.to eq(98.29) } + end + + context 'valid content & bad regex' do + let(:data) { 'Coverage 1033 / 1051 LOC (98.29%) covered\n' } + let(:regex) { 'very covered' } + + it { is_expected.to be_nil } + end + + context 'no coverage content & regex' do + let(:data) { 'No coverage for today :sad:' } + let(:regex) { '\(\d+.\d+\%\) covered' } + + it { is_expected.to be_nil } + end + + context 'multiple results in content & regex' do + let(:data) { ' (98.39%) covered. (98.29%) covered' } + let(:regex) { '\(\d+.\d+\%\) covered' } + + it { is_expected.to eq(98.29) } + end + + context 'using a regex capture' do + let(:data) { 'TOTAL 9926 3489 65%' } + let(:regex) { 'TOTAL\s+\d+\s+\d+\s+(\d{1,3}\%)' } + + it { is_expected.to eq(65) } + end + end +end diff --git a/spec/lib/gitlab/ci/trace_reader_spec.rb b/spec/lib/gitlab/ci/trace_reader_spec.rb deleted file mode 100644 index ff5551bf703..00000000000 --- a/spec/lib/gitlab/ci/trace_reader_spec.rb +++ /dev/null @@ -1,52 +0,0 @@ -require 'spec_helper' - -describe Gitlab::Ci::TraceReader do - let(:path) { __FILE__ } - let(:lines) { File.readlines(path) } - let(:bytesize) { lines.sum(&:bytesize) } - - it 'returns last few lines' do - 10.times do - subject = build_subject - last_lines = random_lines - - expected = lines.last(last_lines).join - result = subject.read(last_lines: last_lines) - - expect(result).to eq(expected) - expect(result.encoding).to eq(Encoding.default_external) - end - end - - it 'returns everything if trying to get too many lines' do - result = build_subject.read(last_lines: lines.size * 2) - - expect(result).to eq(lines.join) - expect(result.encoding).to eq(Encoding.default_external) - end - - it 'returns all contents if last_lines is not specified' do - result = build_subject.read - - expect(result).to eq(lines.join) - expect(result.encoding).to eq(Encoding.default_external) - end - - it 'raises an error if not passing an integer for last_lines' do - expect do - build_subject.read(last_lines: lines) - end.to raise_error(ArgumentError) - end - - def random_lines - Random.rand(lines.size) + 1 - end - - def random_buffer - Random.rand(bytesize) + 1 - end - - def build_subject - described_class.new(__FILE__, buffer_size: random_buffer) - end -end diff --git a/spec/lib/gitlab/ci/trace_spec.rb b/spec/lib/gitlab/ci/trace_spec.rb new file mode 100644 index 00000000000..69e8dc9220d --- /dev/null +++ b/spec/lib/gitlab/ci/trace_spec.rb @@ -0,0 +1,216 @@ +require 'spec_helper' + +describe Gitlab::Ci::Trace do + let(:build) { create(:ci_build) } + let(:trace) { described_class.new(build) } + + describe "associations" do + it { expect(trace).to respond_to(:job) } + it { expect(trace).to delegate_method(:old_trace).to(:job) } + end + + describe '#html' do + before do + trace.set("12\n34") + end + + it "returns formatted html" do + expect(trace.html).to eq("12
    34") + end + + it "returns last line of formatted html" do + expect(trace.html(last_lines: 1)).to eq("34") + end + end + + describe '#raw' do + before do + trace.set("12\n34") + end + + it "returns raw output" do + expect(trace.raw).to eq("12\n34") + end + + it "returns last line of raw output" do + expect(trace.raw(last_lines: 1)).to eq("34") + end + end + + describe '#extract_coverage' do + let(:regex) { '\(\d+.\d+\%\) covered' } + + before do + trace.set('Coverage 1033 / 1051 LOC (98.29%) covered') + end + + it "returns valid coverage" do + expect(trace.extract_coverage(regex)).to eq(98.29) + end + end + + describe '#set' do + before do + trace.set("12") + end + + it "returns trace" do + expect(trace.raw).to eq("12") + end + + context 'overwrite trace' do + before do + trace.set("34") + end + + it "returns new trace" do + expect(trace.raw).to eq("34") + end + end + + context 'runners token' do + let(:token) { 'my_secret_token' } + + before do + build.project.update(runners_token: token) + trace.set(token) + end + + it "hides token" do + expect(trace.raw).not_to include(token) + end + end + + context 'hides build token' do + let(:token) { 'my_secret_token' } + + before do + build.update(token: token) + trace.set(token) + end + + it "hides token" do + expect(trace.raw).not_to include(token) + end + end + end + + describe '#append' do + before do + trace.set("1234") + end + + it "returns correct trace" do + expect(trace.append("56", 4)).to eq(6) + expect(trace.raw).to eq("123456") + end + + context 'tries to append trace at different offset' do + it "fails with append" do + expect(trace.append("56", 2)).to eq(-4) + expect(trace.raw).to eq("1234") + end + end + + context 'runners token' do + let(:token) { 'my_secret_token' } + + before do + build.project.update(runners_token: token) + trace.append(token, 0) + end + + it "hides token" do + expect(trace.raw).not_to include(token) + end + end + + context 'build token' do + let(:token) { 'my_secret_token' } + + before do + build.update(token: token) + trace.append(token, 0) + end + + it "hides token" do + expect(trace.raw).not_to include(token) + end + end + end + + describe 'trace handling' do + context 'trace does not exist' do + it { expect(trace.exist?).to be(false) } + end + + context 'new trace path is used' do + before do + trace.send(:ensure_directory) + + File.open(trace.send(:default_path), "w") do |file| + file.write("data") + end + end + + it "trace exist" do + expect(trace.exist?).to be(true) + end + + it "can be erased" do + trace.erase! + expect(trace.exist?).to be(false) + end + end + + context 'deprecated path' do + let(:path) { trace.send(:deprecated_path) } + + context 'with valid ci_id' do + before do + build.project.update(ci_id: 1000) + + FileUtils.mkdir_p(File.dirname(path)) + + File.open(path, "w") do |file| + file.write("data") + end + end + + it "trace exist" do + expect(trace.exist?).to be(true) + end + + it "can be erased" do + trace.erase! + expect(trace.exist?).to be(false) + end + end + + context 'without valid ci_id' do + it "does not return deprecated path" do + expect(path).to be_nil + end + end + end + + context 'stored in database' do + before do + build.send(:write_attribute, :trace, "data") + end + + it "trace exist" do + expect(trace.exist?).to be(true) + end + + it "can be erased" do + trace.erase! + expect(trace.exist?).to be(false) + end + + it "returns database data" do + expect(trace.raw).to eq("data") + end + end + end +end diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index 8dbcf50ee0c..f876cb00fc7 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -17,8 +17,9 @@ describe Ci::Build, :models do it { is_expected.to belong_to(:trigger_request) } it { is_expected.to belong_to(:erased_by) } it { is_expected.to have_many(:deployments) } - it { is_expected.to validate_presence_of :ref } - it { is_expected.to respond_to :trace_html } + it { is_expected.to validate_presence_of(:ref) } + it { is_expected.to respond_to(:has_trace?) } + it { is_expected.to respond_to(:trace) } describe '#actionize' do context 'when build is a created' do @@ -78,32 +79,6 @@ describe Ci::Build, :models do end end - describe '#append_trace' do - subject { build.trace_html } - - context 'when build.trace hides runners token' do - let(:token) { 'my_secret_token' } - - before do - build.project.update(runners_token: token) - build.append_trace(token, 0) - end - - it { is_expected.not_to include(token) } - end - - context 'when build.trace hides build token' do - let(:token) { 'my_secret_token' } - - before do - build.update(token: token) - build.append_trace(token, 0) - end - - it { is_expected.not_to include(token) } - end - end - describe '#artifacts?' do subject { build.artifacts? } @@ -272,12 +247,98 @@ describe Ci::Build, :models do describe '#update_coverage' do context "regarding coverage_regex's value," do - it "saves the correct extracted coverage value" do + before do build.coverage_regex = '\(\d+.\d+\%\) covered' - allow(build).to receive(:trace) { 'Coverage 1033 / 1051 LOC (98.29%) covered' } - expect(build).to receive(:update_attributes).with(coverage: 98.29) { true } - expect(build.update_coverage).to be true + build.trace.set('Coverage 1033 / 1051 LOC (98.29%) covered') + end + + it "saves the correct extracted coverage value" do + expect(build.update_coverage).to be(true) + expect(build.coverage).to eq(98.29) + end + end + end + + describe '#trace' do + subject { build.trace } + + it { is_expected.to be_a(Gitlab::Ci::Trace) } + end + + describe '#has_trace?' do + subject { build.has_trace? } + + it "expect to call exist? method" do + expect_any_instance_of(Gitlab::Ci::Trace).to receive(:exist?) + .and_return(true) + + is_expected.to be(true) + end + end + + describe '#trace=' do + it "expect to fail trace=" do + expect { build.trace = "new" }.to raise_error(NotImplementedError) + end + end + + describe '#old_trace' do + subject { build.old_trace } + + before do + build.update_column(:trace, 'old trace') + end + + it "expect to receive data from database" do + is_expected.to eq('old trace') + end + end + + describe '#erase_old_trace!' do + subject { build.send(:read_attribute, :trace) } + + before do + build.send(:write_attribute, :trace, 'old trace') + end + + it "expect to receive data from database" do + build.erase_old_trace! + + is_expected.to be_nil + end + end + + describe '#hide_secrets' do + let(:subject) { build.hide_secrets(data) } + + context 'hide runners token' do + let(:data) { 'new token data'} + + before do + build.project.update(runners_token: 'token') end + + it { is_expected.to eq('new xxxxx data') } + end + + context 'hide build token' do + let(:data) { 'new token data'} + + before do + build.update(token: 'token') + end + + it { is_expected.to eq('new xxxxx data') } + end + + context 'hide build token' do + let(:data) { 'new token data'} + + before do + build.update(token: 'token') + end + + it { is_expected.to eq('new xxxxx data') } end end @@ -438,7 +499,7 @@ describe Ci::Build, :models do end it 'erases build trace in trace file' do - expect(build.trace).to be_empty + expect(build).not_to have_trace end it 'sets erased to true' do @@ -532,38 +593,6 @@ describe Ci::Build, :models do end end - describe '#extract_coverage' do - context 'valid content & regex' do - subject { build.extract_coverage('Coverage 1033 / 1051 LOC (98.29%) covered', '\(\d+.\d+\%\) covered') } - - it { is_expected.to eq(98.29) } - end - - context 'valid content & bad regex' do - subject { build.extract_coverage('Coverage 1033 / 1051 LOC (98.29%) covered', 'very covered') } - - it { is_expected.to be_nil } - end - - context 'no coverage content & regex' do - subject { build.extract_coverage('No coverage for today :sad:', '\(\d+.\d+\%\) covered') } - - it { is_expected.to be_nil } - end - - context 'multiple results in content & regex' do - subject { build.extract_coverage(' (98.39%) covered. (98.29%) covered', '\(\d+.\d+\%\) covered') } - - it { is_expected.to eq(98.29) } - end - - context 'using a regex capture' do - subject { build.extract_coverage('TOTAL 9926 3489 65%', 'TOTAL\s+\d+\s+\d+\s+(\d{1,3}\%)') } - - it { is_expected.to eq(65) } - end - end - describe '#first_pending' do let!(:first) { create(:ci_build, pipeline: pipeline, status: 'pending', created_at: Date.yesterday) } let!(:second) { create(:ci_build, pipeline: pipeline, status: 'pending') } @@ -983,32 +1012,6 @@ describe Ci::Build, :models do it { is_expected.to eq(project.name) } end - describe '#raw_trace' do - subject { build.raw_trace } - - context 'when build.trace hides runners token' do - let(:token) { 'my_secret_token' } - - before do - build.project.update(runners_token: token) - build.update(trace: token) - end - - it { is_expected.not_to include(token) } - end - - context 'when build.trace hides build token' do - let(:token) { 'my_secret_token' } - - before do - build.update(token: token) - build.update(trace: token) - end - - it { is_expected.not_to include(token) } - end - end - describe '#ref_slug' do { 'master' => 'master', @@ -1074,61 +1077,6 @@ describe Ci::Build, :models do end end - describe '#trace' do - it 'obfuscates project runners token' do - allow(build).to receive(:raw_trace).and_return("Test: #{build.project.runners_token}") - - expect(build.trace).to eq("Test: xxxxxxxxxxxxxxxxxxxx") - end - - it 'empty project runners token' do - allow(build).to receive(:raw_trace).and_return(test_trace) - # runners_token can't normally be set to nil - allow(build.project).to receive(:runners_token).and_return(nil) - - expect(build.trace).to eq(test_trace) - end - - context 'when build does not have trace' do - it 'is is empty' do - expect(build.trace).to be_nil - end - end - - context 'when trace contains text' do - let(:text) { 'example output' } - before do - build.trace = text - end - - it { expect(build.trace).to eq(text) } - end - - context 'when trace hides runners token' do - let(:token) { 'my_secret_token' } - - before do - build.update(trace: token) - build.project.update(runners_token: token) - end - - it { expect(build.trace).not_to include(token) } - it { expect(build.raw_trace).to include(token) } - end - - context 'when build.trace hides build token' do - let(:token) { 'my_secret_token' } - - before do - build.update(trace: token) - build.update(token: token) - end - - it { expect(build.trace).not_to include(token) } - it { expect(build.raw_trace).to include(token) } - end - end - describe '#has_expiring_artifacts?' do context 'when artifacts have expiration date set' do before { build.update(artifacts_expire_at: 1.day.from_now) } @@ -1147,66 +1095,6 @@ describe Ci::Build, :models do end end - describe '#has_trace_file?' do - context 'when there is no trace' do - it { expect(build.has_trace_file?).to be_falsey } - it { expect(build.trace).to be_nil } - end - - context 'when there is a trace' do - context 'when trace is stored in file' do - let(:build_with_trace) { create(:ci_build, :trace) } - - it { expect(build_with_trace.has_trace_file?).to be_truthy } - it { expect(build_with_trace.trace).to eq('BUILD TRACE') } - end - - context 'when trace is stored in old file' do - before do - allow(build.project).to receive(:ci_id).and_return(999) - allow(File).to receive(:exist?).with(build.path_to_trace).and_return(false) - allow(File).to receive(:exist?).with(build.old_path_to_trace).and_return(true) - allow(File).to receive(:read).with(build.old_path_to_trace).and_return(test_trace) - end - - it { expect(build.has_trace_file?).to be_truthy } - it { expect(build.trace).to eq(test_trace) } - end - - context 'when trace is stored in DB' do - before do - allow(build.project).to receive(:ci_id).and_return(nil) - allow(build).to receive(:read_attribute).with(:trace).and_return(test_trace) - allow(File).to receive(:exist?).with(build.path_to_trace).and_return(false) - allow(File).to receive(:exist?).with(build.old_path_to_trace).and_return(false) - end - - it { expect(build.has_trace_file?).to be_falsey } - it { expect(build.trace).to eq(test_trace) } - end - end - end - - describe '#trace_file_path' do - context 'when trace is stored in file' do - before do - allow(build).to receive(:has_trace_file?).and_return(true) - allow(build).to receive(:has_old_trace_file?).and_return(false) - end - - it { expect(build.trace_file_path).to eq(build.path_to_trace) } - end - - context 'when trace is stored in old file' do - before do - allow(build).to receive(:has_trace_file?).and_return(true) - allow(build).to receive(:has_old_trace_file?).and_return(true) - end - - it { expect(build.trace_file_path).to eq(build.old_path_to_trace) } - end - end - describe '#update_project_statistics' do let!(:build) { create(:ci_build, artifacts_size: 23) } diff --git a/spec/requests/api/jobs_spec.rb b/spec/requests/api/jobs_spec.rb index 9450701064b..d8a56c02a63 100644 --- a/spec/requests/api/jobs_spec.rb +++ b/spec/requests/api/jobs_spec.rb @@ -320,7 +320,7 @@ describe API::Jobs, api: true do context 'authorized user' do it 'returns specific job trace' do expect(response).to have_http_status(200) - expect(response.body).to eq(build.trace) + expect(response.body).to eq(build.trace.raw) end end @@ -408,7 +408,7 @@ describe API::Jobs, api: true do it 'erases job content' do expect(response).to have_http_status(201) - expect(build.trace).to be_empty + expect(build).not_to have_trace expect(build.artifacts_file.exists?).to be_falsy expect(build.artifacts_metadata.exists?).to be_falsy end diff --git a/spec/requests/api/runner_spec.rb b/spec/requests/api/runner_spec.rb index 1cfac7353d4..409a59d6c23 100644 --- a/spec/requests/api/runner_spec.rb +++ b/spec/requests/api/runner_spec.rb @@ -592,7 +592,7 @@ describe API::Runner do update_job(trace: 'BUILD TRACE UPDATED') expect(response).to have_http_status(200) - expect(job.reload.trace).to eq 'BUILD TRACE UPDATED' + expect(job.reload.trace.raw).to eq 'BUILD TRACE UPDATED' end end @@ -600,7 +600,7 @@ describe API::Runner do it 'does not override trace information' do update_job - expect(job.reload.trace).to eq 'BUILD TRACE' + expect(job.reload.trace.raw).to eq 'BUILD TRACE' end end @@ -631,7 +631,7 @@ describe API::Runner do context 'when request is valid' do it 'gets correct response' do expect(response.status).to eq 202 - expect(job.reload.trace).to eq 'BUILD TRACE appended' + expect(job.reload.trace.raw).to eq 'BUILD TRACE appended' expect(response.header).to have_key 'Range' expect(response.header).to have_key 'Job-Status' end @@ -642,7 +642,7 @@ describe API::Runner do it "changes the job's trace" do patch_the_trace - expect(job.reload.trace).to eq 'BUILD TRACE appended appended' + expect(job.reload.trace.raw).to eq 'BUILD TRACE appended appended' end context 'when Runner makes a force-patch' do @@ -651,7 +651,7 @@ describe API::Runner do it "doesn't change the build.trace" do force_patch_the_trace - expect(job.reload.trace).to eq 'BUILD TRACE appended' + expect(job.reload.trace.raw).to eq 'BUILD TRACE appended' end end end @@ -664,7 +664,7 @@ describe API::Runner do it 'changes the job.trace' do patch_the_trace - expect(job.reload.trace).to eq 'BUILD TRACE appended appended' + expect(job.reload.trace.raw).to eq 'BUILD TRACE appended appended' end context 'when Runner makes a force-patch' do @@ -673,7 +673,7 @@ describe API::Runner do it "doesn't change the job.trace" do force_patch_the_trace - expect(job.reload.trace).to eq 'BUILD TRACE appended' + expect(job.reload.trace.raw).to eq 'BUILD TRACE appended' end end end @@ -698,7 +698,7 @@ describe API::Runner do it 'gets correct response' do expect(response.status).to eq 202 - expect(job.reload.trace).to eq 'BUILD TRACE appended' + expect(job.reload.trace.raw).to eq 'BUILD TRACE appended' expect(response.header).to have_key 'Range' expect(response.header).to have_key 'Job-Status' end @@ -738,9 +738,11 @@ describe API::Runner do def patch_the_trace(content = ' appended', request_headers = nil) unless request_headers - offset = job.trace_length - limit = offset + content.length - 1 - request_headers = headers.merge({ 'Content-Range' => "#{offset}-#{limit}" }) + job.trace.read do |stream| + offset = stream.size + limit = offset + content.length - 1 + request_headers = headers.merge({ 'Content-Range' => "#{offset}-#{limit}" }) + end end Timecop.travel(job.updated_at + update_interval) do diff --git a/spec/requests/api/v3/builds_spec.rb b/spec/requests/api/v3/builds_spec.rb index a50c22a6dd1..e97d2b0cee0 100644 --- a/spec/requests/api/v3/builds_spec.rb +++ b/spec/requests/api/v3/builds_spec.rb @@ -330,7 +330,7 @@ describe API::V3::Builds, api: true do context 'authorized user' do it 'returns specific job trace' do expect(response).to have_http_status(200) - expect(response.body).to eq(build.trace) + expect(response.body).to eq(build.trace.raw) end end @@ -418,7 +418,7 @@ describe API::V3::Builds, api: true do it 'erases job content' do expect(response.status).to eq 201 - expect(build.trace).to be_empty + expect(build).not_to have_trace expect(build.artifacts_file.exists?).to be_falsy expect(build.artifacts_metadata.exists?).to be_falsy end diff --git a/spec/requests/ci/api/builds_spec.rb b/spec/requests/ci/api/builds_spec.rb index c879f37f50d..ef30d8638dd 100644 --- a/spec/requests/ci/api/builds_spec.rb +++ b/spec/requests/ci/api/builds_spec.rb @@ -285,7 +285,7 @@ describe Ci::API::Builds do end it 'does not override trace information when no trace is given' do - expect(build.reload.trace).to eq 'BUILD TRACE' + expect(build.reload.trace.raw).to eq 'BUILD TRACE' end context 'job has been erased' do @@ -309,9 +309,11 @@ describe Ci::API::Builds do def patch_the_trace(content = ' appended', request_headers = nil) unless request_headers - offset = build.trace_length - limit = offset + content.length - 1 - request_headers = headers.merge({ 'Content-Range' => "#{offset}-#{limit}" }) + build.trace.read do |stream| + offset = stream.size + limit = offset + content.length - 1 + request_headers = headers.merge({ 'Content-Range' => "#{offset}-#{limit}" }) + end end Timecop.travel(build.updated_at + update_interval) do @@ -335,7 +337,7 @@ describe Ci::API::Builds do context 'when request is valid' do it 'gets correct response' do expect(response.status).to eq 202 - expect(build.reload.trace).to eq 'BUILD TRACE appended' + expect(build.reload.trace.raw).to eq 'BUILD TRACE appended' expect(response.header).to have_key 'Range' expect(response.header).to have_key 'Build-Status' end @@ -346,7 +348,7 @@ describe Ci::API::Builds do it 'changes the build trace' do patch_the_trace - expect(build.reload.trace).to eq 'BUILD TRACE appended appended' + expect(build.reload.trace.raw).to eq 'BUILD TRACE appended appended' end context 'when Runner makes a force-patch' do @@ -355,7 +357,7 @@ describe Ci::API::Builds do it "doesn't change the build.trace" do force_patch_the_trace - expect(build.reload.trace).to eq 'BUILD TRACE appended' + expect(build.reload.trace.raw).to eq 'BUILD TRACE appended' end end end @@ -368,7 +370,7 @@ describe Ci::API::Builds do it 'changes the build.trace' do patch_the_trace - expect(build.reload.trace).to eq 'BUILD TRACE appended appended' + expect(build.reload.trace.raw).to eq 'BUILD TRACE appended appended' end context 'when Runner makes a force-patch' do @@ -377,7 +379,7 @@ describe Ci::API::Builds do it "doesn't change the build.trace" do force_patch_the_trace - expect(build.reload.trace).to eq 'BUILD TRACE appended' + expect(build.reload.trace.raw).to eq 'BUILD TRACE appended' end end end @@ -403,7 +405,7 @@ describe Ci::API::Builds do it 'gets correct response' do expect(response.status).to eq 202 - expect(build.reload.trace).to eq 'BUILD TRACE appended' + expect(build.reload.trace.raw).to eq 'BUILD TRACE appended' expect(response.header).to have_key 'Range' expect(response.header).to have_key 'Build-Status' end -- cgit v1.2.1 From b42dc1a52b5687972262e634cad589345fe49373 Mon Sep 17 00:00:00 2001 From: Eric Eastwood Date: Thu, 6 Apr 2017 16:36:38 +0000 Subject: Linking to edit file directly --- .../javascripts/blob/blob_fork_suggestion.js | 15 +++ app/assets/javascripts/dispatcher.js | 7 + app/assets/stylesheets/framework/files.scss | 13 ++ app/controllers/projects/blob_controller.rb | 10 +- app/helpers/blob_helper.rb | 39 +++--- app/views/projects/blob/_blob.html.haml | 7 + app/views/projects/blob/_header.html.haml | 6 +- .../unreleased/28899-linking-to-edit-file.yml | 5 + features/project/source/browse_files.feature | 4 + features/steps/project/source/browse_files.rb | 12 +- spec/controllers/projects/blob_controller_spec.rb | 83 +++++++++++- spec/features/projects/blobs/blob_show_spec.rb | 23 ++++ spec/features/projects/blobs/edit_spec.rb | 145 +++++++++++++++++---- spec/helpers/blob_helper_spec.rb | 2 +- 14 files changed, 313 insertions(+), 58 deletions(-) create mode 100644 app/assets/javascripts/blob/blob_fork_suggestion.js create mode 100644 changelogs/unreleased/28899-linking-to-edit-file.yml create mode 100644 spec/features/projects/blobs/blob_show_spec.rb diff --git a/app/assets/javascripts/blob/blob_fork_suggestion.js b/app/assets/javascripts/blob/blob_fork_suggestion.js new file mode 100644 index 00000000000..aa9a4e1c99a --- /dev/null +++ b/app/assets/javascripts/blob/blob_fork_suggestion.js @@ -0,0 +1,15 @@ +function BlobForkSuggestion(openButton, cancelButton, suggestionSection) { + if (openButton) { + openButton.addEventListener('click', () => { + suggestionSection.classList.remove('hidden'); + }); + } + + if (cancelButton) { + cancelButton.addEventListener('click', () => { + suggestionSection.classList.add('hidden'); + }); + } +} + +export default BlobForkSuggestion; diff --git a/app/assets/javascripts/dispatcher.js b/app/assets/javascripts/dispatcher.js index 9c7acc903d1..307c6a306d1 100644 --- a/app/assets/javascripts/dispatcher.js +++ b/app/assets/javascripts/dispatcher.js @@ -43,6 +43,7 @@ import GroupsList from './groups_list'; import ProjectsList from './projects_list'; import MiniPipelineGraph from './mini_pipeline_graph_dropdown'; import BlobLinePermalinkUpdater from './blob/blob_line_permalink_updater'; +import BlobForkSuggestion from './blob/blob_fork_suggestion'; import UserCallout from './user_callout'; const ShortcutsBlob = require('./shortcuts_blob'); @@ -86,6 +87,12 @@ const ShortcutsBlob = require('./shortcuts_blob'); skipResetBindings: true, fileBlobPermalinkUrl, }); + + new BlobForkSuggestion( + document.querySelector('.js-edit-blob-link-fork-toggler'), + document.querySelector('.js-cancel-fork-suggestion'), + document.querySelector('.js-file-fork-suggestion-section'), + ); } switch (page) { diff --git a/app/assets/stylesheets/framework/files.scss b/app/assets/stylesheets/framework/files.scss index ddea1cf540b..a5a8522739e 100644 --- a/app/assets/stylesheets/framework/files.scss +++ b/app/assets/stylesheets/framework/files.scss @@ -281,3 +281,16 @@ span.idiff { display: none; } } + +.file-fork-suggestion { + display: flex; + align-items: center; + justify-content: flex-end; + background-color: $gray-light; + border-bottom: 1px solid $border-color; + padding: 5px $gl-padding; +} + +.file-fork-suggestion-note { + margin-right: 1.5em; +} diff --git a/app/controllers/projects/blob_controller.rb b/app/controllers/projects/blob_controller.rb index 80a95c6158b..73706bf8dae 100644 --- a/app/controllers/projects/blob_controller.rb +++ b/app/controllers/projects/blob_controller.rb @@ -7,9 +7,11 @@ class Projects::BlobController < Projects::ApplicationController # Raised when given an invalid file path InvalidPathError = Class.new(StandardError) + prepend_before_action :authenticate_user!, only: [:edit] + before_action :require_non_empty_project, except: [:new, :create] before_action :authorize_download_code! - before_action :authorize_edit_tree!, only: [:new, :create, :edit, :update, :destroy] + before_action :authorize_edit_tree!, only: [:new, :create, :update, :destroy] before_action :assign_blob_vars before_action :commit, except: [:new, :create] before_action :blob, except: [:new, :create] @@ -37,7 +39,11 @@ class Projects::BlobController < Projects::ApplicationController end def edit - blob.load_all_data!(@repository) + if can_collaborate_with_project? + blob.load_all_data!(@repository) + else + redirect_to action: 'show' + end end def update diff --git a/app/helpers/blob_helper.rb b/app/helpers/blob_helper.rb index 8631bc54509..91d6d1852cf 100644 --- a/app/helpers/blob_helper.rb +++ b/app/helpers/blob_helper.rb @@ -8,31 +8,36 @@ module BlobHelper %w(credits changelog news copying copyright license authors) end - def edit_blob_link(project = @project, ref = @ref, path = @path, options = {}) - return unless current_user + def edit_path(project = @project, ref = @ref, path = @path, options = {}) + namespace_project_edit_blob_path(project.namespace, project, + tree_join(ref, path), + options[:link_opts]) + end + def fork_path(project = @project, ref = @ref, path = @path, options = {}) + continue_params = { + to: edit_path, + notice: edit_in_new_fork_notice, + notice_now: edit_in_new_fork_notice_now + } + namespace_project_forks_path(project.namespace, project, namespace_key: current_user.namespace.id, continue: continue_params) + end + + def edit_blob_link(project = @project, ref = @ref, path = @path, options = {}) blob = options.delete(:blob) blob ||= project.repository.blob_at(ref, path) rescue nil return unless blob - edit_path = namespace_project_edit_blob_path(project.namespace, project, - tree_join(ref, path), - options[:link_opts]) + common_classes = "btn js-edit-blob #{options[:extra_class]}" if !on_top_of_branch?(project, ref) - button_tag "Edit", class: "btn disabled has-tooltip", title: "You can only edit files when you are on a branch", data: { container: 'body' } - elsif can_edit_blob?(blob, project, ref) - link_to "Edit", edit_path, class: 'btn btn-sm' - elsif can?(current_user, :fork_project, project) - continue_params = { - to: edit_path, - notice: edit_in_new_fork_notice, - notice_now: edit_in_new_fork_notice_now - } - fork_path = namespace_project_forks_path(project.namespace, project, namespace_key: current_user.namespace.id, continue: continue_params) - - link_to "Edit", fork_path, class: 'btn', method: :post + button_tag 'Edit', class: "#{common_classes} disabled has-tooltip", title: "You can only edit files when you are on a branch", data: { container: 'body' } + # This condition applies to anonymous or users who can edit directly + elsif !current_user || (current_user && can_edit_blob?(blob, project, ref)) + link_to 'Edit', edit_path(project, ref, path, options), class: "#{common_classes} btn-sm" + elsif current_user && can?(current_user, :fork_project, project) + button_tag 'Edit', class: "#{common_classes} js-edit-blob-link-fork-toggler" end end diff --git a/app/views/projects/blob/_blob.html.haml b/app/views/projects/blob/_blob.html.haml index 2b2ee6ed987..aa9b852035e 100644 --- a/app/views/projects/blob/_blob.html.haml +++ b/app/views/projects/blob/_blob.html.haml @@ -25,4 +25,11 @@ #blob-content-holder.blob-content-holder %article.file-holder = render "projects/blob/header", blob: blob + - if current_user + .js-file-fork-suggestion-section.file-fork-suggestion.hidden + %span.file-fork-suggestion-note + You don't have permission to edit this file. Try forking this project to edit the file. + = link_to 'Fork', fork_path, method: :post, class: 'btn btn-grouped btn-inverted btn-new' + %button.js-cancel-fork-suggestion.btn.btn-grouped{ type: 'button' } + Cancel = render blob.to_partial_path(@project), blob: blob diff --git a/app/views/projects/blob/_header.html.haml b/app/views/projects/blob/_header.html.haml index deeeae3d64a..6000e057ec4 100644 --- a/app/views/projects/blob/_header.html.haml +++ b/app/views/projects/blob/_header.html.haml @@ -32,8 +32,8 @@ = link_to 'Permalink', namespace_project_blob_path(@project.namespace, @project, tree_join(@commit.sha, @path)), class: 'btn btn-sm js-data-file-blob-permalink-url' - - if current_user - .btn-group{ role: "group" }< - = edit_blob_link if blob_text_viewable?(blob) + .btn-group{ role: "group" }< + = edit_blob_link if blob_text_viewable?(blob) + - if current_user = replace_blob_link = delete_blob_link diff --git a/changelogs/unreleased/28899-linking-to-edit-file.yml b/changelogs/unreleased/28899-linking-to-edit-file.yml new file mode 100644 index 00000000000..a9f5410693b --- /dev/null +++ b/changelogs/unreleased/28899-linking-to-edit-file.yml @@ -0,0 +1,5 @@ +--- +title: Linking to blob edit page handles anonymous users and users without enough permissions + to edit directly +merge_request: +author: diff --git a/features/project/source/browse_files.feature b/features/project/source/browse_files.feature index d4b91fec6e8..5ede935c562 100644 --- a/features/project/source/browse_files.feature +++ b/features/project/source/browse_files.feature @@ -158,6 +158,8 @@ Feature: Project Source Browse Files Given I don't have write access And I click on ".gitignore" file in repo And I click button "Edit" + Then I should see a Fork/Cancel combo + And I click button "Fork" Then I should see a notice about a new fork having been created And I can edit code @@ -180,6 +182,8 @@ Feature: Project Source Browse Files Given I don't have write access And I click on ".gitignore" file in repo And I click button "Edit" + Then I should see a Fork/Cancel combo + And I click button "Fork" And I edit code And I fill the commit message And I click on "Commit Changes" diff --git a/features/steps/project/source/browse_files.rb b/features/steps/project/source/browse_files.rb index 5c47eaf0279..29f628763db 100644 --- a/features/steps/project/source/browse_files.rb +++ b/features/steps/project/source/browse_files.rb @@ -56,13 +56,17 @@ class Spinach::Features::ProjectSourceBrowseFiles < Spinach::FeatureSteps end step 'I click button "Edit"' do - click_link 'Edit' + find('.js-edit-blob').click end step 'I cannot see the edit button' do expect(page).not_to have_link 'edit' end + step 'I click button "Fork"' do + click_link 'Fork' + end + step 'I can edit code' do set_new_content expect(evaluate_script('ace.edit("editor").getValue()')).to eq new_gitignore_content @@ -366,6 +370,12 @@ class Spinach::Features::ProjectSourceBrowseFiles < Spinach::FeatureSteps end end + step 'I should see a Fork/Cancel combo' do + expect(page).to have_link 'Fork' + expect(page).to have_button 'Cancel' + expect(page).to have_content 'You don\'t have permission to edit this file. Try forking this project to edit the file.' + end + step 'I should see a notice about a new fork having been created' do expect(page).to have_content "You're not allowed to make changes to this project directly. A fork of this project has been created that you can make changes in, so you can submit a merge request." end diff --git a/spec/controllers/projects/blob_controller_spec.rb b/spec/controllers/projects/blob_controller_spec.rb index ec36a64b415..3e9f272a0d8 100644 --- a/spec/controllers/projects/blob_controller_spec.rb +++ b/spec/controllers/projects/blob_controller_spec.rb @@ -2,15 +2,10 @@ require 'rails_helper' describe Projects::BlobController do let(:project) { create(:project, :public, :repository) } - let(:user) { create(:user) } - - before do - project.team << [user, :master] - - sign_in(user) - end describe 'GET diff' do + let(:user) { create(:user) } + render_views def do_get(opts = {}) @@ -20,6 +15,12 @@ describe Projects::BlobController do get :diff, params.merge(opts) end + before do + project.team << [user, :master] + + sign_in(user) + end + context 'when essential params are missing' do it 'renders nothing' do do_get @@ -37,7 +38,69 @@ describe Projects::BlobController do end end + describe 'GET edit' do + let(:default_params) do + { + namespace_id: project.namespace, + project_id: project, + id: 'master/CHANGELOG' + } + end + + context 'anonymous' do + before do + get :edit, default_params + end + + it 'redirects to sign in and returns' do + expect(response).to redirect_to(new_user_session_path) + end + end + + context 'as guest' do + let(:guest) { create(:user) } + + before do + sign_in(guest) + get :edit, default_params + end + + it 'redirects to blob show' do + expect(response).to redirect_to(namespace_project_blob_path(project.namespace, project, 'master/CHANGELOG')) + end + end + + context 'as developer' do + let(:developer) { create(:user) } + + before do + project.team << [developer, :developer] + sign_in(developer) + get :edit, default_params + end + + it 'redirects to blob show' do + expect(response).to have_http_status(200) + end + end + + context 'as master' do + let(:master) { create(:user) } + + before do + project.team << [master, :master] + sign_in(master) + get :edit, default_params + end + + it 'redirects to blob show' do + expect(response).to have_http_status(200) + end + end + end + describe 'PUT update' do + let(:user) { create(:user) } let(:default_params) do { namespace_id: project.namespace, @@ -53,6 +116,12 @@ describe Projects::BlobController do namespace_project_blob_path(project.namespace, project, 'master/CHANGELOG') end + before do + project.team << [user, :master] + + sign_in(user) + end + it 'redirects to blob' do put :update, default_params diff --git a/spec/features/projects/blobs/blob_show_spec.rb b/spec/features/projects/blobs/blob_show_spec.rb new file mode 100644 index 00000000000..01cd268ffe8 --- /dev/null +++ b/spec/features/projects/blobs/blob_show_spec.rb @@ -0,0 +1,23 @@ +require 'spec_helper' + +feature 'File blob', feature: true do + include WaitForAjax + include TreeHelper + + let(:project) { create(:project, :public, :test_repo) } + let(:merge_request) { create(:merge_request, source_project: project, source_branch: 'feature', target_branch: 'master') } + let(:branch) { 'master' } + let(:file_path) { project.repository.ls_files(project.repository.root_ref)[1] } + + context 'anonymous' do + context 'from blob file path' do + before do + visit namespace_project_blob_path(project.namespace, project, tree_join(branch, file_path)) + end + + it 'updates content' do + expect(page).to have_link 'Edit' + end + end + end +end diff --git a/spec/features/projects/blobs/edit_spec.rb b/spec/features/projects/blobs/edit_spec.rb index a820d07ab3b..46f2f487e0c 100644 --- a/spec/features/projects/blobs/edit_spec.rb +++ b/spec/features/projects/blobs/edit_spec.rb @@ -2,44 +2,135 @@ require 'spec_helper' feature 'Editing file blob', feature: true, js: true do include WaitForAjax + include TreeHelper - given(:user) { create(:user) } - given(:role) { :developer } - given(:merge_request) { create(:merge_request, source_branch: 'feature', target_branch: 'master') } - given(:project) { merge_request.target_project } + let(:project) { create(:project, :public, :test_repo) } + let(:merge_request) { create(:merge_request, source_project: project, source_branch: 'feature', target_branch: 'master') } + let(:branch) { 'master' } + let(:file_path) { project.repository.ls_files(project.repository.root_ref)[1] } - background do - login_as(user) - project.team << [user, role] - end - - def edit_and_commit - wait_for_ajax - first('.file-actions').click_link 'Edit' - execute_script('ace.edit("editor").setValue("class NextFeature\nend\n")') - click_button 'Commit Changes' - end + context 'as a developer' do + let(:user) { create(:user) } + let(:role) { :developer } - context 'from MR diff' do before do - visit diffs_namespace_project_merge_request_path(project.namespace, project, merge_request) - edit_and_commit + project.team << [user, role] + login_as(user) + end + + def edit_and_commit + wait_for_ajax + find('.js-edit-blob').click + execute_script('ace.edit("editor").setValue("class NextFeature\nend\n")') + click_button 'Commit Changes' + end + + context 'from MR diff' do + before do + visit diffs_namespace_project_merge_request_path(project.namespace, project, merge_request) + edit_and_commit + end + + it 'returns me to the mr' do + expect(page).to have_content(merge_request.title) + end end - scenario 'returns me to the mr' do - expect(page).to have_content(merge_request.title) + context 'from blob file path' do + before do + visit namespace_project_blob_path(project.namespace, project, tree_join(branch, file_path)) + edit_and_commit + end + + it 'updates content' do + expect(page).to have_content 'successfully committed' + expect(page).to have_content 'NextFeature' + end end end - context 'from blob file path' do - before do - visit namespace_project_blob_path(project.namespace, project, '/feature/files/ruby/feature.rb') - edit_and_commit + context 'visit blob edit' do + context 'redirects to sign in and returns' do + context 'as developer' do + let(:user) { create(:user) } + + before do + project.team << [user, :developer] + visit namespace_project_edit_blob_path(project.namespace, project, tree_join(branch, file_path)) + end + + it 'redirects to sign in and returns' do + expect(page).to have_current_path(new_user_session_path) + + login_as(user) + + expect(page).to have_current_path(namespace_project_edit_blob_path(project.namespace, project, tree_join(branch, file_path))) + end + end + + context 'as guest' do + let(:user) { create(:user) } + + before do + visit namespace_project_edit_blob_path(project.namespace, project, tree_join(branch, file_path)) + end + + it 'redirects to sign in and returns' do + expect(page).to have_current_path(new_user_session_path) + + login_as(user) + + expect(page).to have_current_path(namespace_project_blob_path(project.namespace, project, tree_join(branch, file_path))) + end + end + end + + context 'as developer' do + let(:user) { create(:user) } + let(:protected_branch) { 'protected-branch' } + + before do + project.team << [user, :developer] + project.repository.add_branch(user, protected_branch, 'master') + create(:protected_branch, project: project, name: protected_branch) + login_as(user) + end + + context 'on some branch' do + before do + visit namespace_project_edit_blob_path(project.namespace, project, tree_join(branch, file_path)) + end + + it 'shows blob editor with same branch' do + expect(page).to have_current_path(namespace_project_edit_blob_path(project.namespace, project, tree_join(branch, file_path))) + expect(find('.js-target-branch .dropdown-toggle-text').text).to eq(branch) + end + end + + context 'with protected branch' do + before do + visit namespace_project_edit_blob_path(project.namespace, project, tree_join(protected_branch, file_path)) + end + + it 'shows blob editor with patch branch' do + expect(find('.js-target-branch .dropdown-toggle-text').text).to eq('patch-1') + end + end end - scenario 'updates content' do - expect(page).to have_content 'successfully committed' - expect(page).to have_content 'NextFeature' + context 'as master' do + let(:user) { create(:user) } + + before do + project.team << [user, :master] + login_as(user) + visit namespace_project_edit_blob_path(project.namespace, project, tree_join(branch, file_path)) + end + + it 'shows blob editor with same branch' do + expect(page).to have_current_path(namespace_project_edit_blob_path(project.namespace, project, tree_join(branch, file_path))) + expect(find('.js-target-branch .dropdown-toggle-text').text).to eq(branch) + end end end end diff --git a/spec/helpers/blob_helper_spec.rb b/spec/helpers/blob_helper_spec.rb index bead7948486..508aeb7cf67 100644 --- a/spec/helpers/blob_helper_spec.rb +++ b/spec/helpers/blob_helper_spec.rb @@ -73,7 +73,7 @@ describe BlobHelper do let(:project) { create(:project, :repository, namespace: namespace) } before do - allow(self).to receive(:current_user).and_return(double) + allow(self).to receive(:current_user).and_return(nil) allow(self).to receive(:can_collaborate_with_project?).and_return(true) end -- cgit v1.2.1 From 2f7ff6eac57fbc32ae88bf1898f2a58961e04e25 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer Date: Thu, 6 Apr 2017 18:47:35 +0200 Subject: Fix rubocop offence --- lib/gitlab/gitaly_client/ref.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gitlab/gitaly_client/ref.rb b/lib/gitlab/gitaly_client/ref.rb index 4958d00c542..fcdf452d567 100644 --- a/lib/gitlab/gitaly_client/ref.rb +++ b/lib/gitlab/gitaly_client/ref.rb @@ -24,7 +24,7 @@ module Gitlab end def find_ref_name(commit_id, ref_prefix) - request = Gitaly::FindRefNameRequest.new( + request = Gitaly::FindRefNameRequest.new( repository: @repository, commit_id: commit_id, prefix: ref_prefix -- cgit v1.2.1 From 1c42505b026d922df50c59d5f9e85073b5f5345f Mon Sep 17 00:00:00 2001 From: Timothy Andrew Date: Thu, 6 Apr 2017 22:33:07 +0530 Subject: Implement review comments from @DouweM for !10467. 1. Have `MigrateToGhostUser` be a service rather than a mixed-in module, to keep things explicit. Specs testing the behavior of this class are moved into a separate service spec file. 2. Add a `user.reported_abuse_reports` association to make the `migrate_abuse_reports` method more consistent with the other `migrate_` methods. --- app/models/user.rb | 1 + .../concerns/users/migrate_to_ghost_user.rb | 53 ------------- app/services/users/destroy_service.rb | 4 +- .../users/migrate_to_ghost_user_service.rb | 59 ++++++++++++++ spec/models/user_spec.rb | 1 + spec/services/users/destroy_service_spec.rb | 91 ++++++---------------- .../users/migrate_to_ghost_user_service_spec.rb | 64 +++++++++++++++ ...igrate_to_ghost_user_service_shared_examples.rb | 39 ++++++++++ ...ervice_migrate_to_ghost_user_shared_examples.rb | 59 -------------- 9 files changed, 188 insertions(+), 183 deletions(-) delete mode 100644 app/services/concerns/users/migrate_to_ghost_user.rb create mode 100644 app/services/users/migrate_to_ghost_user_service.rb create mode 100644 spec/services/users/migrate_to_ghost_user_service_spec.rb create mode 100644 spec/support/services/migrate_to_ghost_user_service_shared_examples.rb delete mode 100644 spec/support/services/user_destroy_service_migrate_to_ghost_user_shared_examples.rb diff --git a/app/models/user.rb b/app/models/user.rb index 8c2f0011be8..50d7687c255 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -90,6 +90,7 @@ class User < ActiveRecord::Base has_many :recent_events, -> { order "id DESC" }, foreign_key: :author_id, class_name: "Event" has_many :oauth_applications, class_name: 'Doorkeeper::Application', as: :owner, dependent: :destroy has_one :abuse_report, dependent: :destroy, foreign_key: :user_id + has_many :reported_abuse_reports, dependent: :destroy, foreign_key: :reporter_id, class_name: "AbuseReport" has_many :spam_logs, dependent: :destroy has_many :builds, dependent: :nullify, class_name: 'Ci::Build' has_many :pipelines, dependent: :nullify, class_name: 'Ci::Pipeline' diff --git a/app/services/concerns/users/migrate_to_ghost_user.rb b/app/services/concerns/users/migrate_to_ghost_user.rb deleted file mode 100644 index 76335e45e54..00000000000 --- a/app/services/concerns/users/migrate_to_ghost_user.rb +++ /dev/null @@ -1,53 +0,0 @@ -# When a user is destroyed, some of their associated records are -# moved to a "Ghost User", to prevent these associated records from -# being destroyed. -# -# For example, all the issues/MRs a user has created are _not_ destroyed -# when the user is destroyed. -module Users::MigrateToGhostUser - extend ActiveSupport::Concern - - attr_reader :ghost_user - - def move_associated_records_to_ghost_user(user) - # Block the user before moving records to prevent a data race. - # For example, if the user creates an issue after `migrate_issues` - # runs and before the user is destroyed, the destroy will fail with - # an exception. - user.block - - user.transaction do - @ghost_user = User.ghost - - migrate_issues(user) - migrate_merge_requests(user) - migrate_notes(user) - migrate_abuse_reports(user) - migrate_award_emoji(user) - end - - user.reload - end - - private - - def migrate_issues(user) - user.issues.update_all(author_id: ghost_user.id) - end - - def migrate_merge_requests(user) - user.merge_requests.update_all(author_id: ghost_user.id) - end - - def migrate_notes(user) - user.notes.update_all(author_id: ghost_user.id) - end - - def migrate_abuse_reports(user) - AbuseReport.where(reporter_id: user.id).update_all(reporter_id: ghost_user.id) - end - - def migrate_award_emoji(user) - user.award_emoji.update_all(user_id: ghost_user.id) - end -end diff --git a/app/services/users/destroy_service.rb b/app/services/users/destroy_service.rb index e6608e316dc..ba58b174cc0 100644 --- a/app/services/users/destroy_service.rb +++ b/app/services/users/destroy_service.rb @@ -1,7 +1,5 @@ module Users class DestroyService - include MigrateToGhostUser - attr_accessor :current_user def initialize(current_user) @@ -28,7 +26,7 @@ module Users ::Projects::DestroyService.new(project, current_user, skip_repo: true).execute end - move_associated_records_to_ghost_user(user) + MigrateToGhostUserService.new(user).execute # Destroy the namespace after destroying the user since certain methods may depend on the namespace existing namespace = user.namespace diff --git a/app/services/users/migrate_to_ghost_user_service.rb b/app/services/users/migrate_to_ghost_user_service.rb new file mode 100644 index 00000000000..1e1ed1791ec --- /dev/null +++ b/app/services/users/migrate_to_ghost_user_service.rb @@ -0,0 +1,59 @@ +# When a user is destroyed, some of their associated records are +# moved to a "Ghost User", to prevent these associated records from +# being destroyed. +# +# For example, all the issues/MRs a user has created are _not_ destroyed +# when the user is destroyed. +module Users + class MigrateToGhostUserService + extend ActiveSupport::Concern + + attr_reader :ghost_user, :user + + def initialize(user) + @user = user + end + + def execute + # Block the user before moving records to prevent a data race. + # For example, if the user creates an issue after `migrate_issues` + # runs and before the user is destroyed, the destroy will fail with + # an exception. + user.block + + user.transaction do + @ghost_user = User.ghost + + migrate_issues + migrate_merge_requests + migrate_notes + migrate_abuse_reports + migrate_award_emoji + end + + user.reload + end + + private + + def migrate_issues + user.issues.update_all(author_id: ghost_user.id) + end + + def migrate_merge_requests + user.merge_requests.update_all(author_id: ghost_user.id) + end + + def migrate_notes + user.notes.update_all(author_id: ghost_user.id) + end + + def migrate_abuse_reports + user.reported_abuse_reports.update_all(reporter_id: ghost_user.id) + end + + def migrate_award_emoji + user.award_emoji.update_all(user_id: ghost_user.id) + end + end +end diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index 6a787f6b0df..3f9c013ce37 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -36,6 +36,7 @@ describe User, models: true do it { is_expected.to have_many(:pipelines).dependent(:nullify) } it { is_expected.to have_many(:chat_names).dependent(:destroy) } it { is_expected.to have_many(:uploads).dependent(:destroy) } + it { is_expected.to have_many(:reported_abuse_reports).dependent(:destroy).class_name('AbuseReport') } describe "#abuse_report" do let(:current_user) { create(:user) } diff --git a/spec/services/users/destroy_service_spec.rb b/spec/services/users/destroy_service_spec.rb index d3463185ebe..43c18992d1a 100644 --- a/spec/services/users/destroy_service_spec.rb +++ b/spec/services/users/destroy_service_spec.rb @@ -46,43 +46,47 @@ describe Users::DestroyService, services: true do project.add_developer(user) end - context "for an issue the user has created" do - let!(:issue) { create(:issue, project: project, author: user) } + context "for an issue the user was assigned to" do + let!(:issue) { create(:issue, project: project, assignee: user) } before do service.execute(user) end - it 'does not delete the issue' do + it 'does not delete issues the user is assigned to' do expect(Issue.find_by_id(issue.id)).to be_present end - it 'migrates the issue so that the "Ghost User" is the issue owner' do + it 'migrates the issue so that it is "Unassigned"' do migrated_issue = Issue.find_by_id(issue.id) - expect(migrated_issue.author).to eq(User.ghost) + expect(migrated_issue.assignee).to be_nil end + end + end - it 'blocks the user before migrating issues to the "Ghost User' do - expect(user).to be_blocked - end + context "a deleted user's merge_requests" do + let(:project) { create(:project) } + + before do + project.add_developer(user) end - context "for an issue the user was assigned to" do - let!(:issue) { create(:issue, project: project, assignee: user) } + context "for an merge request the user was assigned to" do + let!(:merge_request) { create(:merge_request, source_project: project, assignee: user) } before do service.execute(user) end - it 'does not delete issues the user is assigned to' do - expect(Issue.find_by_id(issue.id)).to be_present + it 'does not delete merge requests the user is assigned to' do + expect(MergeRequest.find_by_id(merge_request.id)).to be_present end - it 'migrates the issue so that it is "Unassigned"' do - migrated_issue = Issue.find_by_id(issue.id) + it 'migrates the merge request so that it is "Unassigned"' do + migrated_merge_request = MergeRequest.find_by_id(merge_request.id) - expect(migrated_issue.assignee).to be_nil + expect(migrated_merge_request.assignee).to be_nil end end end @@ -142,60 +146,11 @@ describe Users::DestroyService, services: true do end end - context 'migrating associated records to the ghost user' do - context 'issues' do - include_examples "migrating a deleted user's associated records to the ghost user", Issue, {} do - let(:created_record) { create(:issue, project: project, author: user) } - let(:assigned_record) { create(:issue, project: project, assignee: user) } - end - end - - context 'merge requests' do - include_examples "migrating a deleted user's associated records to the ghost user", MergeRequest, {} do - let(:created_record) { create(:merge_request, source_project: project, author: user, target_branch: "first") } - let(:assigned_record) { create(:merge_request, source_project: project, assignee: user, target_branch: 'second') } - end - end - - context 'notes' do - include_examples "migrating a deleted user's associated records to the ghost user", Note, { skip_assignee_specs: true } do - let(:created_record) { create(:note, project: project, author: user) } - end - end - - context 'abuse reports' do - include_examples "migrating a deleted user's associated records to the ghost user", AbuseReport, { skip_assignee_specs: true } do - let(:created_record) { create(:abuse_report, reporter: user, user: create(:user)) } - end - end + context "migrating associated records" do + it 'delegates to the `MigrateToGhostUser` service to move associated records to the ghost user' do + expect_any_instance_of(Users::MigrateToGhostUserService).to receive(:execute).once - context 'award emoji' do - include_examples "migrating a deleted user's associated records to the ghost user", AwardEmoji, { skip_assignee_specs: true } do - let(:created_record) { create(:award_emoji, user: user) } - let(:author_alias) { :user } - - context "when the awardable already has an award emoji of the same name assigned to the ghost user" do - let(:awardable) { create(:issue) } - let!(:existing_award_emoji) { create(:award_emoji, user: User.ghost, name: "thumbsup", awardable: awardable) } - let!(:award_emoji) { create(:award_emoji, user: user, name: "thumbsup", awardable: awardable) } - - it "migrates the award emoji regardless" do - service.execute(user) - - migrated_record = AwardEmoji.find_by_id(award_emoji.id) - - expect(migrated_record.user).to eq(User.ghost) - end - - it "does not leave the migrated award emoji in an invalid state" do - service.execute(user) - - migrated_record = AwardEmoji.find_by_id(award_emoji.id) - - expect(migrated_record).to be_valid - end - end - end + service.execute(user) end end end diff --git a/spec/services/users/migrate_to_ghost_user_service_spec.rb b/spec/services/users/migrate_to_ghost_user_service_spec.rb new file mode 100644 index 00000000000..8c5b7e41c15 --- /dev/null +++ b/spec/services/users/migrate_to_ghost_user_service_spec.rb @@ -0,0 +1,64 @@ +require 'spec_helper' + +describe Users::MigrateToGhostUserService, services: true do + let!(:user) { create(:user) } + let!(:project) { create(:project) } + let(:service) { described_class.new(user) } + + context "migrating a user's associated records to the ghost user" do + context 'issues' do + include_examples "migrating a deleted user's associated records to the ghost user", Issue do + let(:created_record) { create(:issue, project: project, author: user) } + let(:assigned_record) { create(:issue, project: project, assignee: user) } + end + end + + context 'merge requests' do + include_examples "migrating a deleted user's associated records to the ghost user", MergeRequest do + let(:created_record) { create(:merge_request, source_project: project, author: user, target_branch: "first") } + let(:assigned_record) { create(:merge_request, source_project: project, assignee: user, target_branch: 'second') } + end + end + + context 'notes' do + include_examples "migrating a deleted user's associated records to the ghost user", Note do + let(:created_record) { create(:note, project: project, author: user) } + end + end + + context 'abuse reports' do + include_examples "migrating a deleted user's associated records to the ghost user", AbuseReport do + let(:created_record) { create(:abuse_report, reporter: user, user: create(:user)) } + end + end + + context 'award emoji' do + include_examples "migrating a deleted user's associated records to the ghost user", AwardEmoji do + let(:created_record) { create(:award_emoji, user: user) } + let(:author_alias) { :user } + + context "when the awardable already has an award emoji of the same name assigned to the ghost user" do + let(:awardable) { create(:issue) } + let!(:existing_award_emoji) { create(:award_emoji, user: User.ghost, name: "thumbsup", awardable: awardable) } + let!(:award_emoji) { create(:award_emoji, user: user, name: "thumbsup", awardable: awardable) } + + it "migrates the award emoji regardless" do + service.execute + + migrated_record = AwardEmoji.find_by_id(award_emoji.id) + + expect(migrated_record.user).to eq(User.ghost) + end + + it "does not leave the migrated award emoji in an invalid state" do + service.execute + + migrated_record = AwardEmoji.find_by_id(award_emoji.id) + + expect(migrated_record).to be_valid + end + end + end + end + end +end diff --git a/spec/support/services/migrate_to_ghost_user_service_shared_examples.rb b/spec/support/services/migrate_to_ghost_user_service_shared_examples.rb new file mode 100644 index 00000000000..0eac587e973 --- /dev/null +++ b/spec/support/services/migrate_to_ghost_user_service_shared_examples.rb @@ -0,0 +1,39 @@ +require "spec_helper" + +shared_examples "migrating a deleted user's associated records to the ghost user" do |record_class| + record_class_name = record_class.to_s.titleize.downcase + + let(:project) { create(:project) } + + before do + project.add_developer(user) + end + + context "for a #{record_class_name} the user has created" do + let!(:record) { created_record } + + it "does not delete the #{record_class_name}" do + service.execute + + expect(record_class.find_by_id(record.id)).to be_present + end + + it "migrates the #{record_class_name} so that the 'Ghost User' is the #{record_class_name} owner" do + service.execute + + migrated_record = record_class.find_by_id(record.id) + + if migrated_record.respond_to?(:author) + expect(migrated_record.author).to eq(User.ghost) + else + expect(migrated_record.send(author_alias)).to eq(User.ghost) + end + end + + it "blocks the user before migrating #{record_class_name}s to the 'Ghost User'" do + service.execute + + expect(user).to be_blocked + end + end +end diff --git a/spec/support/services/user_destroy_service_migrate_to_ghost_user_shared_examples.rb b/spec/support/services/user_destroy_service_migrate_to_ghost_user_shared_examples.rb deleted file mode 100644 index 3820ffc283f..00000000000 --- a/spec/support/services/user_destroy_service_migrate_to_ghost_user_shared_examples.rb +++ /dev/null @@ -1,59 +0,0 @@ -require "spec_helper" - -shared_examples "migrating a deleted user's associated records to the ghost user" do |record_class, options| - record_class_name = record_class.to_s.titleize.downcase - - let(:project) { create(:project) } - - before do - project.add_developer(user) - end - - context "for a #{record_class_name} the user has created" do - let!(:record) { created_record } - - it "does not delete the #{record_class_name}" do - service.execute(user) - - expect(record_class.find_by_id(record.id)).to be_present - end - - it "migrates the #{record_class_name} so that the 'Ghost User' is the #{record_class_name} owner" do - service.execute(user) - - migrated_record = record_class.find_by_id(record.id) - - if migrated_record.respond_to?(:author) - expect(migrated_record.author).to eq(User.ghost) - else - expect(migrated_record.send(author_alias)).to eq(User.ghost) - end - end - - it "blocks the user before migrating #{record_class_name}s to the 'Ghost User'" do - service.execute(user) - - expect(user).to be_blocked - end - end - - unless options[:skip_assignee_specs] - context "for a #{record_class_name} the user was assigned to" do - let!(:record) { assigned_record } - - before do - service.execute(user) - end - - it "does not delete #{record_class_name}s the user is assigned to" do - expect(record_class.find_by_id(record.id)).to be_present - end - - it "migrates the #{record_class_name} so that it is 'Unassigned'" do - migrated_record = record_class.find_by_id(record.id) - - expect(migrated_record.assignee).to be_nil - end - end - end -end -- cgit v1.2.1 From 29cd3b0b3f0f7956986a63421bb13e6bf654ee83 Mon Sep 17 00:00:00 2001 From: Jose Ivan Vargas Date: Thu, 6 Apr 2017 13:02:57 -0500 Subject: Fixed rubocop failure for the add_two_factor_columns_migration --- db/migrate/20170124193147_add_two_factor_columns_to_namespaces.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/migrate/20170124193147_add_two_factor_columns_to_namespaces.rb b/db/migrate/20170124193147_add_two_factor_columns_to_namespaces.rb index ca4429c676c..df5cddeb205 100644 --- a/db/migrate/20170124193147_add_two_factor_columns_to_namespaces.rb +++ b/db/migrate/20170124193147_add_two_factor_columns_to_namespaces.rb @@ -16,6 +16,6 @@ class AddTwoFactorColumnsToNamespaces < ActiveRecord::Migration remove_column(:namespaces, :require_two_factor_authentication) remove_column(:namespaces, :two_factor_grace_period) - remove_index(:namespaces, :require_two_factor_authentication) if index_exists?(:namespaces, :require_two_factor_authentication) + remove_concurrent_index(:namespaces, :require_two_factor_authentication) if index_exists?(:namespaces, :require_two_factor_authentication) end end -- cgit v1.2.1 From 99ff816593747a4dc46e8f52f0e5002d3337012c Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Thu, 6 Apr 2017 11:33:14 -0700 Subject: Add text to break up diagrams [ci skip] --- doc/development/polling.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/development/polling.md b/doc/development/polling.md index 4042b8aaa61..05e19f0c515 100644 --- a/doc/development/polling.md +++ b/doc/development/polling.md @@ -22,7 +22,12 @@ Instead you should use polling mechanism with ETag caching in Redis. ## How it works +Cache Miss: + ![Cache miss](img/cache-miss.svg) + +Cache Hit: + ![Cache hit](img/cache-hit.svg) 1. Whenever a resource changes we generate a random value and store it in -- cgit v1.2.1 From 1f404065ca602593dc18aa051515fd88132ddfdf Mon Sep 17 00:00:00 2001 From: Tiago Botelho Date: Tue, 4 Apr 2017 16:10:21 +0100 Subject: adds relevant tests --- .../project_services/chat_message/base_message.rb | 16 +- .../project_services/chat_message/issue_message.rb | 34 ++- .../project_services/chat_message/merge_message.rb | 41 ++- .../project_services/chat_message/note_message.rb | 79 +++--- .../chat_message/pipeline_message.rb | 35 ++- .../project_services/chat_message/push_message.rb | 48 ++-- .../chat_message/wiki_page_message.rb | 30 +-- .../project_services/chat_notification_service.rb | 20 +- .../project_services/microsoft_teams_service.rb | 35 +-- .../unreleased/microsoft-teams-integration.yml | 4 + .../img/microsoft_teams_configuration.png | Bin 0 -> 350592 bytes doc/user/project/integrations/microsoft_teams.md | 33 +++ lib/api/services.rb | 9 + lib/api/v3/services.rb | 6 + lib/microsoft_teams/activity.rb | 6 +- lib/microsoft_teams/notifier.rb | 35 ++- spec/lib/microsoft_teams/activity_spec.rb | 16 ++ spec/lib/microsoft_teams/notifier_spec.rb | 55 ++++ .../chat_message/issue_message_spec.rb | 96 +++++-- .../chat_message/merge_message_spec.rb | 73 ++++-- .../chat_message/note_message_spec.rb | 242 +++++++++++------- .../chat_message/pipeline_message_spec.rb | 136 +++++++--- .../chat_message/push_message_spec.rb | 132 ++++++++-- .../chat_message/wiki_page_message_spec.rb | 143 ++++++++--- .../microsoft_teams_service_spec.rb | 277 +++++++++++++++++++++ spec/models/project_spec.rb | 1 + 26 files changed, 1174 insertions(+), 428 deletions(-) create mode 100644 changelogs/unreleased/microsoft-teams-integration.yml create mode 100644 doc/user/project/integrations/img/microsoft_teams_configuration.png create mode 100644 doc/user/project/integrations/microsoft_teams.md create mode 100644 spec/lib/microsoft_teams/activity_spec.rb create mode 100644 spec/lib/microsoft_teams/notifier_spec.rb create mode 100644 spec/models/project_services/microsoft_teams_service_spec.rb diff --git a/app/models/project_services/chat_message/base_message.rb b/app/models/project_services/chat_message/base_message.rb index 0f0657998c7..7621a5fa2d8 100644 --- a/app/models/project_services/chat_message/base_message.rb +++ b/app/models/project_services/chat_message/base_message.rb @@ -2,14 +2,24 @@ require 'slack-notifier' module ChatMessage class BaseMessage - attr_reader :markdown_format + attr_reader :markdown + attr_reader :user_name + attr_reader :user_avatar + attr_reader :project_name + attr_reader :project_url def initialize(params) - @markdown_format = params[:markdown_format] || false + @markdown = params[:markdown] || false + @project_name = params.dig(:project, :path_with_namespace) || params[:project_name] + @project_url = params.dig(:project, :web_url) || params[:project_url] + @user_name = params.dig(:user, :username) || params[:user_name] + @user_avatar = params.dig(:user, :avatar_url) || params[:user_avatar] end def pretext - markdown_format ? message : format(message) + return message if markdown + + format(message) end def fallback diff --git a/app/models/project_services/chat_message/issue_message.rb b/app/models/project_services/chat_message/issue_message.rb index aebd9aa93e5..4b9a2b1e1f3 100644 --- a/app/models/project_services/chat_message/issue_message.rb +++ b/app/models/project_services/chat_message/issue_message.rb @@ -1,10 +1,6 @@ module ChatMessage class IssueMessage < BaseMessage - attr_reader :user_name - attr_reader :user_avatar attr_reader :title - attr_reader :project_name - attr_reader :project_url attr_reader :issue_iid attr_reader :issue_url attr_reader :action @@ -12,12 +8,7 @@ module ChatMessage attr_reader :description def initialize(params) - super(params) - - @user_name = params[:user][:username] - @user_avatar = params[:user][:avatar_url] - @project_name = params[:project_name] - @project_url = params[:project_url] + super obj_attr = params[:object_attributes] obj_attr = HashWithIndifferentAccess.new(obj_attr) @@ -31,23 +22,28 @@ module ChatMessage def attachments return [] unless opened_issue? + return description if markdown - markdown_format ? description : description_message + description_message end def activity - MicrosoftTeams::Activity.new( - "Issue #{state} by #{user_name}", - "to: #{project_link}", - issue_link, - user_avatar - ).to_json + { + title: "Issue #{state} by #{user_name}", + subtitle: "in #{project_link}", + text: issue_link, + image: user_avatar + } end private def message - "[#{project_link}] Issue #{issue_link} #{state} by #{user_name}" + if state == 'opened' + "[#{project_link}] Issue #{state} by #{user_name}" + else + "[#{project_link}] Issue #{issue_link} #{state} by #{user_name}" + end end def opened_issue? @@ -72,7 +68,7 @@ module ChatMessage end def issue_title - "##{issue_iid} #{title}" + "#{Issue.reference_prefix}#{issue_iid} #{title}" end end end diff --git a/app/models/project_services/chat_message/merge_message.rb b/app/models/project_services/chat_message/merge_message.rb index e9ef211438b..7d0de81cdf0 100644 --- a/app/models/project_services/chat_message/merge_message.rb +++ b/app/models/project_services/chat_message/merge_message.rb @@ -1,45 +1,36 @@ module ChatMessage class MergeMessage < BaseMessage - attr_reader :user_name - attr_reader :user_avatar - attr_reader :project_name - attr_reader :project_url - attr_reader :merge_request_id + attr_reader :merge_request_iid attr_reader :source_branch attr_reader :target_branch attr_reader :state attr_reader :title def initialize(params) - super(params) - - @user_name = params[:user][:username] - @user_avatar = params[:user][:avatar_url] - @project_name = params[:project_name] - @project_url = params[:project_url] + super obj_attr = params[:object_attributes] obj_attr = HashWithIndifferentAccess.new(obj_attr) - @merge_request_id = obj_attr[:iid] + @merge_request_iid = obj_attr[:iid] @source_branch = obj_attr[:source_branch] @target_branch = obj_attr[:target_branch] @state = obj_attr[:state] @title = format_title(obj_attr[:title]) end - def activity - MicrosoftTeams::Activity.new( - "Merge Request #{state} by #{user_name}", - "to: #{project_link}", - merge_request_link, - user_avatar - ).to_json - end - def attachments [] end + def activity + { + title: "Merge Request #{state} by #{user_name}", + subtitle: "in #{project_link}", + text: merge_request_link, + image: user_avatar + } + end + private def format_title(title) @@ -59,11 +50,15 @@ module ChatMessage end def merge_request_link - link("merge request !#{merge_request_id}", merge_request_url) + link(merge_request_title, merge_request_url) + end + + def merge_request_title + "#{MergeRequest.reference_prefix}#{merge_request_iid} #{title}" end def merge_request_url - "#{project_url}/merge_requests/#{merge_request_id}" + "#{project_url}/merge_requests/#{merge_request_iid}" end end end diff --git a/app/models/project_services/chat_message/note_message.rb b/app/models/project_services/chat_message/note_message.rb index 26d944007a6..2da4c244229 100644 --- a/app/models/project_services/chat_message/note_message.rb +++ b/app/models/project_services/chat_message/note_message.rb @@ -1,83 +1,74 @@ module ChatMessage class NoteMessage < BaseMessage - attr_reader :user_name - attr_reader :user_avatar - attr_reader :project_name - attr_reader :project_url attr_reader :note attr_reader :note_url - attr_reader :comment_attrs + attr_reader :title + attr_reader :target def initialize(params) - params = HashWithIndifferentAccess.new(params) - - super(params) + super - @user_name = params[:user][:name] - @user_avatar = params[:user][:avatar_url] - @project_name = params[:project_name] - @project_url = params[:project_url] - - obj_attr = HashWithIndifferentAccess.new(params[:object_attributes]) + params = HashWithIndifferentAccess.new(params) + obj_attr = params[:object_attributes] @note = obj_attr[:note] @note_url = obj_attr[:url] - @comment_attrs = comment_params(obj_attr[:noteable_type], params) + @target, @title = case obj_attr[:noteable_type] + when "Commit" + create_commit_note(params[:commit]) + when "Issue" + create_issue_note(params[:issue]) + when "MergeRequest" + create_merge_note(params[:merge_request]) + when "Snippet" + create_snippet_note(params[:snippet]) + end end - def activity - MicrosoftTeams::Activity.new( - "#{user_name} #{link('commented on ' + comment_attrs[:target], note_url)}", - "to: #{project_link}", - comment_attrs[:title], - user_avatar - ).to_json + def attachments + return note if markdown + + description_message end - def attachments - markdown_format ? note : description_message + def activity + { + title: "#{user_name} #{link('commented on ' + target, note_url)}", + subtitle: "in #{project_link}", + text: formatted_title, + image: user_avatar + } end private def message - "#{user_name} #{link('commented on ' + comment_attrs[:target], note_url)} in #{project_link}: *#{comment_attrs[:title]}*" - end - - def comment_params(noteable_type, params) - params = HashWithIndifferentAccess.new(params) - - case noteable_type - when "Commit" - create_commit_note(params[:commit]) - when "Issue" - create_issue_note(params[:issue]) - when "MergeRequest" - create_merge_note(params[:merge_request]) - when "Snippet" - create_snippet_note(params[:snippet]) - end + "#{user_name} #{link('commented on ' + target, note_url)} in #{project_link}: *#{formatted_title}*" end def format_title(title) title.lines.first.chomp end + def formatted_title + format_title(title) + end + def create_issue_note(issue) - { target: "issue ##{issue[:iid]}", title: format_title(issue[:title]) } + ["issue #{Issue.reference_prefix}#{issue[:iid]}", issue[:title]] end def create_commit_note(commit) commit_sha = Commit.truncate_sha(commit[:id]) - { target: "commit #{commit_sha}", title: format_title(commit[:message]) } + ["commit #{commit_sha}", commit[:message]] end def create_merge_note(merge_request) - { target: "merge request !#{merge_request[:iid]}", title: format_title(merge_request[:title]) } + ["merge request #{MergeRequest.reference_prefix}#{merge_request[:iid]}", merge_request[:title]] end def create_snippet_note(snippet) - { target: "snippet ##{snippet[:id]}", title: format_title(snippet[:title]) } + ["snippet #{Snippet.reference_prefix}#{snippet[:id]}", snippet[:title]] end def description_message diff --git a/app/models/project_services/chat_message/pipeline_message.rb b/app/models/project_services/chat_message/pipeline_message.rb index 8f4fcf09f64..4628d9b1a7b 100644 --- a/app/models/project_services/chat_message/pipeline_message.rb +++ b/app/models/project_services/chat_message/pipeline_message.rb @@ -3,27 +3,20 @@ module ChatMessage attr_reader :ref_type attr_reader :ref attr_reader :status - attr_reader :project_name - attr_reader :project_url - attr_reader :user_name attr_reader :duration attr_reader :pipeline_id - attr_reader :user_avatar def initialize(data) + super + + @user_name = data.dig(:user, :name) || 'API' + pipeline_attributes = data[:object_attributes] @ref_type = pipeline_attributes[:tag] ? 'tag' : 'branch' @ref = pipeline_attributes[:ref] @status = pipeline_attributes[:status] @duration = pipeline_attributes[:duration] @pipeline_id = pipeline_attributes[:id] - - super(data) - - @project_name = data[:project][:path_with_namespace] - @project_url = data[:project][:web_url] - @user_name = (data[:user] && data[:user][:name]) || 'API' - @user_avatar = data[:user][:avatar_url] || '' end def pretext @@ -34,17 +27,19 @@ module ChatMessage format(message) end - def activity - MicrosoftTeams::Activity.new( - "Pipeline #{pipeline_link} of #{branch_link} #{ref_type} by #{user_name} #{humanized_status}", - "to: #{project_link}", - "in #{duration} #{time_measure}", - user_avatar - ).to_json + def attachments + return message if markdown + + [{ text: format(message), color: attachment_color }] end - def attachments - markdown_format ? message : [{ text: format(message), color: attachment_color }] + def activity + { + title: "Pipeline #{pipeline_link} of #{branch_link} #{ref_type} by #{user_name} #{humanized_status}", + subtitle: "in #{project_link}", + text: "in #{duration} #{time_measure}", + image: user_avatar || '' + } end private diff --git a/app/models/project_services/chat_message/push_message.rb b/app/models/project_services/chat_message/push_message.rb index 71fa77712c0..c52dd6ef8ef 100644 --- a/app/models/project_services/chat_message/push_message.rb +++ b/app/models/project_services/chat_message/push_message.rb @@ -3,49 +3,41 @@ module ChatMessage attr_reader :after attr_reader :before attr_reader :commits - attr_reader :project_name - attr_reader :project_url attr_reader :ref attr_reader :ref_type - attr_reader :user_name - attr_reader :user_avatar def initialize(params) - super(params) + super @after = params[:after] @before = params[:before] @commits = params.fetch(:commits, []) - @project_name = params[:project_name] - @project_url = params[:project_url] @ref_type = Gitlab::Git.tag_ref?(params[:ref]) ? 'tag' : 'branch' @ref = Gitlab::Git.ref_name(params[:ref]) - @user_name = params[:user_name] - @user_avatar = params[:user_avatar] - end - - def activity - action = - if new_branch? - "created" - elsif removed_branch? - "removed" - else - "pushed to" - end - - MicrosoftTeams::Activity.new( - "#{user_name} #{action} #{ref_type}", - "to: #{project_link}", - compare_link, - user_avatar - ).to_json end def attachments return [] if new_branch? || removed_branch? + return commit_messages if markdown - markdown_format ? commit_messages : commit_message_attachments + commit_message_attachments + end + + def activity + action = if new_branch? + "created" + elsif removed_branch? + "removed" + else + "pushed to" + end + + { + title: "#{user_name} #{action} #{ref_type}", + subtitle: "in #{project_link}", + text: compare_link, + image: user_avatar + } end private diff --git a/app/models/project_services/chat_message/wiki_page_message.rb b/app/models/project_services/chat_message/wiki_page_message.rb index 85c2efd75bc..a139a8ee727 100644 --- a/app/models/project_services/chat_message/wiki_page_message.rb +++ b/app/models/project_services/chat_message/wiki_page_message.rb @@ -1,20 +1,12 @@ module ChatMessage class WikiPageMessage < BaseMessage - attr_reader :user_name attr_reader :title - attr_reader :project_name - attr_reader :project_url attr_reader :wiki_page_url attr_reader :action attr_reader :description def initialize(params) - super(params) - - @user_name = params[:user][:username] - @user_avatar = params[:user][:avatar_url] - @project_name = params[:project_name] - @project_url = params[:project_url] + super obj_attr = params[:object_attributes] obj_attr = HashWithIndifferentAccess.new(obj_attr) @@ -31,17 +23,19 @@ module ChatMessage end end - def activity - MicrosoftTeams::Activity.new( - "#{user_name} #{action} #{wiki_page_link}", - "in: #{project_link}", - title, - user_avatar - ).to_json + def attachments + return description if markdown + + description_message end - def attachments - markdown_format ? @description : description_message + def activity + { + title: "#{user_name} #{action} #{wiki_page_link}", + subtitle: "in #{project_link}", + text: title, + image: user_avatar + } end private diff --git a/app/models/project_services/chat_notification_service.rb b/app/models/project_services/chat_notification_service.rb index 75834103db5..fa782c6fbb7 100644 --- a/app/models/project_services/chat_notification_service.rb +++ b/app/models/project_services/chat_notification_service.rb @@ -49,10 +49,7 @@ class ChatNotificationService < Service object_kind = data[:object_kind] - data = data.merge( - project_url: project_url, - project_name: project_name - ) + data = custom_data(data) # WebHook events often have an 'update' event that follows a 'open' or # 'close' action. Ignore update events for now to prevent duplicate @@ -68,8 +65,7 @@ class ChatNotificationService < Service opts[:channel] = channel_name if channel_name opts[:username] = username if username - notifier = Slack::Notifier.new(webhook, opts) - notifier.ping(message.pretext, attachments: message.attachments, fallback: message.fallback) + return false unless notify(message, opts) true end @@ -92,6 +88,18 @@ class ChatNotificationService < Service private + def notify(message, opts) + Slack::Notifier.new(webhook, opts).ping( + message.pretext, + attachments: message.attachments, + fallback: message.fallback + ) + end + + def custom_data(data) + data.merge(project_url: project_url, project_name: project_name) + end + def get_message(object_kind, data) case object_kind when "push", "tag_push" diff --git a/app/models/project_services/microsoft_teams_service.rb b/app/models/project_services/microsoft_teams_service.rb index 2a8ea3f7e21..9b218fd81b4 100644 --- a/app/models/project_services/microsoft_teams_service.rb +++ b/app/models/project_services/microsoft_teams_service.rb @@ -4,14 +4,13 @@ class MicrosoftTeamsService < ChatNotificationService end def description - 'Receive event notifications in Microsoft Team' + 'Receive event notifications in Microsoft Teams' end def self.to_param 'microsoft_teams' end - #TODO: Setup the description accordingly def help 'This service sends notifications about projects events to Microsoft Teams channels.
    To set up this service: @@ -22,10 +21,6 @@ class MicrosoftTeamsService < ChatNotificationService ' end - def default_channel_placeholder - "Channel name (e.g. general)" - end - def webhook_placeholder 'https://outlook.office.com/webhook/…' end @@ -33,6 +28,9 @@ class MicrosoftTeamsService < ChatNotificationService def event_field(event) end + def default_channel_placeholder + end + def default_fields [ { type: 'text', name: 'webhook', placeholder: "e.g. #{webhook_placeholder}" }, @@ -41,27 +39,18 @@ class MicrosoftTeamsService < ChatNotificationService ] end - def execute(data) - return unless supported_events.include?(data[:object_kind]) - return unless webhook.present? - - object_kind = data[:object_kind] + private - data = data.merge( - project_url: project_url, - project_name: project_name, - markdown_format: true - ) - - message = get_message(object_kind, data) - - return false unless message - - MicrosoftTeams::Notifier.new(webhook).ping({ + def notify(message, opts) + MicrosoftTeams::Notifier.new(webhook).ping( title: message.project_name, pretext: message.pretext, activity: message.activity, attachments: message.attachments - }) + ) + end + + def custom_data(data) + super(data).merge(markdown: true) end end diff --git a/changelogs/unreleased/microsoft-teams-integration.yml b/changelogs/unreleased/microsoft-teams-integration.yml new file mode 100644 index 00000000000..c01902d3401 --- /dev/null +++ b/changelogs/unreleased/microsoft-teams-integration.yml @@ -0,0 +1,4 @@ +--- +title: Integrates Microsoft Teams webhooks with GitLab +merge_request: 10412 +author: diff --git a/doc/user/project/integrations/img/microsoft_teams_configuration.png b/doc/user/project/integrations/img/microsoft_teams_configuration.png new file mode 100644 index 00000000000..b5c9efc3dd9 Binary files /dev/null and b/doc/user/project/integrations/img/microsoft_teams_configuration.png differ diff --git a/doc/user/project/integrations/microsoft_teams.md b/doc/user/project/integrations/microsoft_teams.md new file mode 100644 index 00000000000..fbf9c1de443 --- /dev/null +++ b/doc/user/project/integrations/microsoft_teams.md @@ -0,0 +1,33 @@ +# Microsoft Teams Service + +## On Microsoft Teams + +To enable Microsoft Teams integration you must create an incoming webhook integration on Microsoft Teams by following the steps described in this [document](https://msdn.microsoft.com/en-us/microsoft-teams/connectors) + +## On GitLab + +After you set up Microsoft Teams, it's time to set up GitLab. + +Navigate to the [Integrations page](project_services.md#accessing-the-project-services) +and select the **Microsoft Teams Notification** service to configure it. +There, you will see a checkbox with the following events that can be triggered: + +- Push +- Issue +- Confidential issue +- Merge request +- Note +- Tag push +- Pipeline +- Wiki page + +At the end fill in your Microsoft Teams details: + +| Field | Description | +| ----- | ----------- | +| **Webhook** | The incoming webhook URL which you have to setup on Microsoft Teams. | +| **Notify only broken pipelines** | If you choose to enable the **Pipeline** event and you want to be only notified about failed pipelines. | + +After you are all done, click **Save changes** for the changes to take effect. + +![Microsoft Teams configuration](img/microsoft_teams_configuration.png) \ No newline at end of file diff --git a/lib/api/services.rb b/lib/api/services.rb index 6802a99311e..65f86caaa51 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -488,6 +488,14 @@ module API desc: 'The channel name' } ], + 'microsoft-teams' => [ + { + required: true, + name: :webhook, + type: String, + desc: 'The Microsoft Teams webhook. e.g. https://outlook.office.com/webhook/…' + } + ], 'mattermost' => [ { required: true, @@ -550,6 +558,7 @@ module API RedmineService, SlackService, MattermostService, + MicrosoftTeamsService, TeamcityService, ] diff --git a/lib/api/v3/services.rb b/lib/api/v3/services.rb index 3bacaeee032..bbe07ed4212 100644 --- a/lib/api/v3/services.rb +++ b/lib/api/v3/services.rb @@ -501,6 +501,12 @@ module API desc: 'The channel name' } ], + 'microsoft-teams' => [ + required: true, + name: :webhook, + type: String, + desc: 'The Microsoft Teams webhook. e.g. https://outlook.office.com/webhook/…' + ], 'mattermost' => [ { required: true, diff --git a/lib/microsoft_teams/activity.rb b/lib/microsoft_teams/activity.rb index e25ce262cbf..d2c420efdaf 100644 --- a/lib/microsoft_teams/activity.rb +++ b/lib/microsoft_teams/activity.rb @@ -1,19 +1,19 @@ module MicrosoftTeams class Activity - def initialize(title, subtitle, text, image) + def initialize(title:, subtitle:, text:, image:) @title = title @subtitle = subtitle @text = text @image = image end - def to_json + def prepare { 'activityTitle' => @title, 'activitySubtitle' => @subtitle, 'activityText' => @text, 'activityImage' => @image - }.to_json + } end end end diff --git a/lib/microsoft_teams/notifier.rb b/lib/microsoft_teams/notifier.rb index a5d2fe15ab6..3bef68a1bcb 100644 --- a/lib/microsoft_teams/notifier.rb +++ b/lib/microsoft_teams/notifier.rb @@ -2,30 +2,43 @@ module MicrosoftTeams class Notifier def initialize(webhook) @webhook = webhook + @header = { 'Content-type' => 'application/json' } end def ping(options = {}) - HTTParty.post( - @webhook.to_str, - headers: { 'Content-type' => 'application/json' }, - body: body(options) - ) + result = false + + begin + response = HTTParty.post( + @webhook.to_str, + headers: @header, + body: body(options) + ) + + result = true if response + rescue HTTParty::Error, StandardError => error + Rails.logger.info("#{self.class.name}: Error while connecting to #{@webhook}: #{error.message}") + end + + result end private def body(options = {}) - attachments = options[:attachments] result = { 'sections' => [] } result['title'] = options[:title] result['summary'] = options[:pretext] - result['sections'] << options[:activity] + result['sections'] << MicrosoftTeams::Activity.new(options[:activity]).prepare - result['sections'] << { - 'title' => 'Details', - 'facts' => [{ 'name' => 'Attachments', 'value' => attachments }] - } if attachments.present? && attachments.empty? + attachments = options[:attachments] + unless attachments.blank? + result['sections'] << { + 'title' => 'Details', + 'facts' => [{ 'name' => 'Attachments', 'value' => attachments }] + } + end result.to_json end diff --git a/spec/lib/microsoft_teams/activity_spec.rb b/spec/lib/microsoft_teams/activity_spec.rb new file mode 100644 index 00000000000..7890ae2e7b0 --- /dev/null +++ b/spec/lib/microsoft_teams/activity_spec.rb @@ -0,0 +1,16 @@ +require 'spec_helper' + +describe MicrosoftTeams::Activity do + subject { described_class.new(title: 'title', subtitle: 'subtitle', text: 'text', image: 'image') } + + describe '#prepare' do + it 'returns the correct JSON object' do + expect(subject.prepare).to eq({ + 'activityTitle' => 'title', + 'activitySubtitle' => 'subtitle', + 'activityText' => 'text', + 'activityImage' => 'image' + }) + end + end +end diff --git a/spec/lib/microsoft_teams/notifier_spec.rb b/spec/lib/microsoft_teams/notifier_spec.rb new file mode 100644 index 00000000000..3035693812f --- /dev/null +++ b/spec/lib/microsoft_teams/notifier_spec.rb @@ -0,0 +1,55 @@ +require 'spec_helper' + +describe MicrosoftTeams::Notifier do + subject { described_class.new(webhook_url) } + + let(:webhook_url) { 'https://example.gitlab.com/'} + let(:header) { { 'Content-Type' => 'application/json' } } + let(:options) do + { + title: 'JohnDoe4/project2', + pretext: '[[JohnDoe4/project2](http://localhost/namespace2/gitlabhq)] Issue [#1 Awesome issue](http://localhost/namespace2/gitlabhq/issues/1) opened by user6', + activity: { + title: 'Issue opened by user6', + subtitle: 'in [JohnDoe4/project2](http://localhost/namespace2/gitlabhq)', + text: '[#1 Awesome issue](http://localhost/namespace2/gitlabhq/issues/1)', + image: 'http://someimage.com' + }, + attachments: 'please fix' + } + end + + let(:body) do + { + 'sections' => [ + { + 'activityTitle' => 'Issue opened by user6', + 'activitySubtitle' => 'in [JohnDoe4/project2](http://localhost/namespace2/gitlabhq)', + 'activityText' => '[#1 Awesome issue](http://localhost/namespace2/gitlabhq/issues/1)', + 'activityImage' => 'http://someimage.com' + }, + { + 'title' => 'Details', + 'facts' => [ + { + 'name' => 'Attachments', + 'value' => 'please fix' + } + ] + } + ], + 'title' => 'JohnDoe4/project2', + 'summary' => '[[JohnDoe4/project2](http://localhost/namespace2/gitlabhq)] Issue [#1 Awesome issue](http://localhost/namespace2/gitlabhq/issues/1) opened by user6' + } + end + + describe '#ping' do + before do + stub_request(:post, webhook_url).with(body: JSON(body), headers: { 'Content-Type' => 'application/json' }).to_return(status: 200, body: "", headers: {}) + end + + it 'expects to receive successfull answer' do + expect(subject.ping(options)).to be true + end + end +end diff --git a/spec/models/project_services/chat_message/issue_message_spec.rb b/spec/models/project_services/chat_message/issue_message_spec.rb index 190ff4c535d..34e2d94b1ed 100644 --- a/spec/models/project_services/chat_message/issue_message_spec.rb +++ b/spec/models/project_services/chat_message/issue_message_spec.rb @@ -7,7 +7,8 @@ describe ChatMessage::IssueMessage, models: true do { user: { name: 'Test User', - username: 'test.user' + username: 'test.user', + avatar_url: 'http://someavatar.com' }, project_name: 'project_name', project_url: 'http://somewhere.com', @@ -25,43 +26,84 @@ describe ChatMessage::IssueMessage, models: true do } end - let(:color) { '#C95823' } + context 'without markdown' do + let(:color) { '#C95823' } - context '#initialize' do - before do - args[:object_attributes][:description] = nil + context '#initialize' do + before do + args[:object_attributes][:description] = nil + end + + it 'returns a non-null description' do + expect(subject.description).to eq('') + end end - it 'returns a non-null description' do - expect(subject.description).to eq('') + context 'open' do + it 'returns a message regarding opening of issues' do + expect(subject.pretext).to eq( + '[] Issue opened by test.user') + expect(subject.attachments).to eq([ + { + title: "#100 Issue title", + title_link: "http://url.com", + text: "issue description", + color: color, + } + ]) + end end - end - context 'open' do - it 'returns a message regarding opening of issues' do - expect(subject.pretext).to eq( - '[] Issue opened by test.user') - expect(subject.attachments).to eq([ - { - title: "#100 Issue title", - title_link: "http://url.com", - text: "issue description", - color: color, - } - ]) + context 'close' do + before do + args[:object_attributes][:action] = 'close' + args[:object_attributes][:state] = 'closed' + end + + it 'returns a message regarding closing of issues' do + expect(subject.pretext). to eq( + '[] Issue closed by test.user') + expect(subject.attachments).to be_empty + end end end - context 'close' do + context 'with markdown' do before do - args[:object_attributes][:action] = 'close' - args[:object_attributes][:state] = 'closed' + args[:markdown] = true end - it 'returns a message regarding closing of issues' do - expect(subject.pretext). to eq( - '[] Issue closed by test.user') - expect(subject.attachments).to be_empty + context 'open' do + it 'returns a message regarding opening of issues' do + expect(subject.pretext).to eq( + '[[project_name](http://somewhere.com)] Issue opened by test.user') + expect(subject.attachments).to eq('issue description') + expect(subject.activity).to eq({ + title: 'Issue opened by test.user', + subtitle: 'in [project_name](http://somewhere.com)', + text: '[#100 Issue title](http://url.com)', + image: 'http://someavatar.com' + }) + end + end + + context 'close' do + before do + args[:object_attributes][:action] = 'close' + args[:object_attributes][:state] = 'closed' + end + + it 'returns a message regarding closing of issues' do + expect(subject.pretext). to eq( + '[[project_name](http://somewhere.com)] Issue [#100 Issue title](http://url.com) closed by test.user') + expect(subject.attachments).to be_empty + expect(subject.activity).to eq({ + title: 'Issue closed by test.user', + subtitle: 'in [project_name](http://somewhere.com)', + text: '[#100 Issue title](http://url.com)', + image: 'http://someavatar.com' + }) + end end end end diff --git a/spec/models/project_services/chat_message/merge_message_spec.rb b/spec/models/project_services/chat_message/merge_message_spec.rb index cc154112e90..fa0a1f4a5b7 100644 --- a/spec/models/project_services/chat_message/merge_message_spec.rb +++ b/spec/models/project_services/chat_message/merge_message_spec.rb @@ -7,45 +7,84 @@ describe ChatMessage::MergeMessage, models: true do { user: { name: 'Test User', - username: 'test.user' + username: 'test.user', + avatar_url: 'http://someavatar.com' }, project_name: 'project_name', project_url: 'http://somewhere.com', object_attributes: { - title: "Issue title\nSecond line", + title: "Merge Request title\nSecond line", id: 10, iid: 100, assignee_id: 1, url: 'http://url.com', state: 'opened', - description: 'issue description', + description: 'merge request description', source_branch: 'source_branch', target_branch: 'target_branch', } } end - let(:color) { '#345' } + context 'without markdown' do + let(:color) { '#345' } - context 'open' do - it 'returns a message regarding opening of merge requests' do - expect(subject.pretext).to eq( - 'test.user opened '\ - 'in : *Issue title*') - expect(subject.attachments).to be_empty + context 'open' do + it 'returns a message regarding opening of merge requests' do + expect(subject.pretext).to eq( + 'test.user opened in : *Merge Request title*') + expect(subject.attachments).to be_empty + end + end + + context 'close' do + before do + args[:object_attributes][:state] = 'closed' + end + it 'returns a message regarding closing of merge requests' do + expect(subject.pretext).to eq( + 'test.user closed in : *Merge Request title*') + expect(subject.attachments).to be_empty + end end end - context 'close' do + context 'with markdown' do before do - args[:object_attributes][:state] = 'closed' + args[:markdown] = true + end + + context 'open' do + it 'returns a message regarding opening of merge requests' do + expect(subject.pretext).to eq( + 'test.user opened [!100 *Merge Request title*](http://somewhere.com/merge_requests/100) in [project_name](http://somewhere.com): *Merge Request title*') + expect(subject.attachments).to be_empty + expect(subject.activity).to eq({ + title: 'Merge Request opened by test.user', + subtitle: 'in [project_name](http://somewhere.com)', + text: '[!100 *Merge Request title*](http://somewhere.com/merge_requests/100)', + image: 'http://someavatar.com' + }) + end end - it 'returns a message regarding closing of merge requests' do - expect(subject.pretext).to eq( - 'test.user closed '\ - 'in : *Issue title*') - expect(subject.attachments).to be_empty + + context 'close' do + before do + args[:object_attributes][:state] = 'closed' + end + + it 'returns a message regarding closing of merge requests' do + expect(subject.pretext).to eq( + 'test.user closed [!100 *Merge Request title*](http://somewhere.com/merge_requests/100) in [project_name](http://somewhere.com): *Merge Request title*') + expect(subject.attachments).to be_empty + expect(subject.activity).to eq({ + title: 'Merge Request closed by test.user', + subtitle: 'in [project_name](http://somewhere.com)', + text: '[!100 *Merge Request title*](http://somewhere.com/merge_requests/100)', + image: 'http://someavatar.com' + }) + end end end end diff --git a/spec/models/project_services/chat_message/note_message_spec.rb b/spec/models/project_services/chat_message/note_message_spec.rb index da700a08e57..7cd9c61ee2b 100644 --- a/spec/models/project_services/chat_message/note_message_spec.rb +++ b/spec/models/project_services/chat_message/note_message_spec.rb @@ -1,130 +1,190 @@ require 'spec_helper' describe ChatMessage::NoteMessage, models: true do - let(:color) { '#345' } + subject { described_class.new(args) } - before do - @args = { - user: { - name: 'Test User', - username: 'test.user', - avatar_url: 'http://fakeavatar' - }, - project_name: 'project_name', - project_url: 'http://somewhere.com', - repository: { - name: 'project_name', - url: 'http://somewhere.com', - }, - object_attributes: { - id: 10, - note: 'comment on a commit', - url: 'http://url.com', - noteable_type: 'Commit' - } + let(:color) { '#345' } + let(:args) do + { + user: { + name: 'Test User', + username: 'test.user', + avatar_url: 'http://fakeavatar' + }, + project_name: 'project_name', + project_url: 'http://somewhere.com', + repository: { + name: 'project_name', + url: 'http://somewhere.com', + }, + object_attributes: { + id: 10, + note: 'comment on a commit', + url: 'http://url.com', + noteable_type: 'Commit' + } } end context 'commit notes' do before do - @args[:object_attributes][:note] = 'comment on a commit' - @args[:object_attributes][:noteable_type] = 'Commit' - @args[:commit] = { - id: '5f163b2b95e6f53cbd428f5f0b103702a52b9a23', - message: "Added a commit message\ndetails\n123\n" + args[:object_attributes][:note] = 'comment on a commit' + args[:object_attributes][:noteable_type] = 'Commit' + args[:commit] = { + id: '5f163b2b95e6f53cbd428f5f0b103702a52b9a23', + message: "Added a commit message\ndetails\n123\n" } end - it 'returns a message regarding notes on commits' do - message = described_class.new(@args) - expect(message.pretext).to eq("test.user in : " \ - "*Added a commit message*") - expected_attachments = [ - { - text: "comment on a commit", - color: color, - } - ] - expect(message.attachments).to eq(expected_attachments) + context 'without markdown' do + it 'returns a message regarding notes on commits' do + expect(subject.pretext).to eq("test.user in : " \ + "*Added a commit message*") + expect(subject.attachments).to eq([{ + text: 'comment on a commit', + color: color + }]) + end + end + + context 'with markdown' do + before do + args[:markdown] = true + end + + it 'returns a message regarding notes on commits' do + expect(subject.pretext).to eq( + 'test.user [commented on commit 5f163b2b](http://url.com) in [project_name](http://somewhere.com): *Added a commit message*' + ) + expect(subject.attachments).to eq('comment on a commit') + expect(subject.activity).to eq({ + title: 'test.user [commented on commit 5f163b2b](http://url.com)', + subtitle: 'in [project_name](http://somewhere.com)', + text: 'Added a commit message', + image: 'http://fakeavatar' + }) + end end end context 'merge request notes' do before do - @args[:object_attributes][:note] = 'comment on a merge request' - @args[:object_attributes][:noteable_type] = 'MergeRequest' - @args[:merge_request] = { - id: 1, - iid: 30, - title: "merge request title\ndetails\n" + args[:object_attributes][:note] = 'comment on a merge request' + args[:object_attributes][:noteable_type] = 'MergeRequest' + args[:merge_request] = { + id: 1, + iid: 30, + title: "merge request title\ndetails\n" } end - it 'returns a message regarding notes on a merge request' do - message = described_class.new(@args) - expect(message.pretext).to eq("test.user in : " \ - "*merge request title*") - expected_attachments = [ - { - text: "comment on a merge request", - color: color, - } - ] - expect(message.attachments).to eq(expected_attachments) + context 'without markdown' do + it 'returns a message regarding notes on a merge request' do + expect(subject.pretext).to eq("test.user in : " \ + "*merge request title*") + expect(subject.attachments).to eq([{ + text: 'comment on a merge request', + color: color + }]) + end + end + + context 'with markdown' do + before do + args[:markdown] = true + end + + it 'returns a message regarding notes on a merge request' do + expect(subject.pretext).to eq( + 'test.user [commented on merge request !30](http://url.com) in [project_name](http://somewhere.com): *merge request title*') + expect(subject.attachments).to eq('comment on a merge request') + expect(subject.activity).to eq({ + title: 'test.user [commented on merge request !30](http://url.com)', + subtitle: 'in [project_name](http://somewhere.com)', + text: 'merge request title', + image: 'http://fakeavatar' + }) + end end end context 'issue notes' do before do - @args[:object_attributes][:note] = 'comment on an issue' - @args[:object_attributes][:noteable_type] = 'Issue' - @args[:issue] = { - id: 1, - iid: 20, - title: "issue title\ndetails\n" + args[:object_attributes][:note] = 'comment on an issue' + args[:object_attributes][:noteable_type] = 'Issue' + args[:issue] = { + id: 1, + iid: 20, + title: "issue title\ndetails\n" } end - it 'returns a message regarding notes on an issue' do - message = described_class.new(@args) - expect(message.pretext).to eq( - "test.user in : " \ - "*issue title*") - expected_attachments = [ - { - text: "comment on an issue", - color: color, - } - ] - expect(message.attachments).to eq(expected_attachments) + context 'without markdown' do + it 'returns a message regarding notes on an issue' do + expect(subject.pretext).to eq( + "test.user in : " \ + "*issue title*") + expect(subject.attachments).to eq([{ + text: 'comment on an issue', + color: color + }]) + end + end + + context 'with markdown' do + before do + args[:markdown] = true + end + + it 'returns a message regarding notes on an issue' do + expect(subject.pretext).to eq( + 'test.user [commented on issue #20](http://url.com) in [project_name](http://somewhere.com): *issue title*') + expect(subject.attachments).to eq('comment on an issue') + expect(subject.activity).to eq({ + title: 'test.user [commented on issue #20](http://url.com)', + subtitle: 'in [project_name](http://somewhere.com)', + text: 'issue title', + image: 'http://fakeavatar' + }) + end end end context 'project snippet notes' do before do - @args[:object_attributes][:note] = 'comment on a snippet' - @args[:object_attributes][:noteable_type] = 'Snippet' - @args[:snippet] = { - id: 5, - title: "snippet title\ndetails\n" + args[:object_attributes][:note] = 'comment on a snippet' + args[:object_attributes][:noteable_type] = 'Snippet' + args[:snippet] = { + id: 5, + title: "snippet title\ndetails\n" } end - it 'returns a message regarding notes on a project snippet' do - message = described_class.new(@args) - expect(message.pretext).to eq("test.user in : " \ - "*snippet title*") - expected_attachments = [ - { - text: "comment on a snippet", - color: color, - } - ] - expect(message.attachments).to eq(expected_attachments) + context 'without markdown' do + it 'returns a message regarding notes on a project snippet' do + expect(subject.pretext).to eq("test.user in : " \ + "*snippet title*") + expect(subject.attachments).to eq([{ + text: 'comment on a snippet', + color: color + }]) + end + end + + context 'with markdown' do + before do + args[:markdown] = true + end + + it 'returns a message regarding notes on a project snippet' do + expect(subject.pretext).to eq( + 'test.user [commented on snippet $5](http://url.com) in [project_name](http://somewhere.com): *snippet title*') + expect(subject.attachments).to eq('comment on a snippet') + end end end end diff --git a/spec/models/project_services/chat_message/pipeline_message_spec.rb b/spec/models/project_services/chat_message/pipeline_message_spec.rb index bf2a9616455..ec5c6c5e0ed 100644 --- a/spec/models/project_services/chat_message/pipeline_message_spec.rb +++ b/spec/models/project_services/chat_message/pipeline_message_spec.rb @@ -2,8 +2,8 @@ require 'spec_helper' describe ChatMessage::PipelineMessage do subject { described_class.new(args) } - let(:user) { { name: 'hacker' } } + let(:user) { { name: 'hacker' } } let(:args) do { object_attributes: { @@ -14,54 +14,122 @@ describe ChatMessage::PipelineMessage do status: status, duration: duration }, - project: { path_with_namespace: 'project_name', - web_url: 'http://example.gitlab.com' }, + project: { + path_with_namespace: 'project_name', + web_url: 'http://example.gitlab.com' + }, user: user } end - let(:message) { build_message } + context 'without markdown' do + context 'pipeline succeeded' do + let(:status) { 'success' } + let(:color) { 'good' } + let(:duration) { 10 } + let(:message) { build_message('passed') } + + it 'returns a message with information about succeeded build' do + expect(subject.pretext).to be_empty + expect(subject.fallback).to eq(message) + expect(subject.attachments).to eq([text: message, color: color]) + end + end - context 'pipeline succeeded' do - let(:status) { 'success' } - let(:color) { 'good' } - let(:duration) { 10 } - let(:message) { build_message('passed') } + context 'pipeline failed' do + let(:status) { 'failed' } + let(:color) { 'danger' } + let(:duration) { 10 } + let(:message) { build_message } - it 'returns a message with information about succeeded build' do - verify_message + it 'returns a message with information about failed build' do + expect(subject.pretext).to be_empty + expect(subject.fallback).to eq(message) + expect(subject.attachments).to eq([text: message, color: color]) + end + + context 'when triggered by API therefore lacking user' do + let(:user) { nil } + let(:message) { build_message(status, 'API') } + + it 'returns a message stating it is by API' do + expect(subject.pretext).to be_empty + expect(subject.fallback).to eq(message) + expect(subject.attachments).to eq([text: message, color: color]) + end + end end - end - context 'pipeline failed' do - let(:status) { 'failed' } - let(:color) { 'danger' } - let(:duration) { 10 } + def build_message(status_text = status, name = user[:name]) + ":" \ + " Pipeline " \ + " of branch" \ + " by #{name} #{status_text} in #{duration} #{'second'.pluralize(duration)}" + end + end - it 'returns a message with information about failed build' do - verify_message + context 'with markdown' do + before do + args[:markdown] = true end - context 'when triggered by API therefore lacking user' do - let(:user) { nil } - let(:message) { build_message(status, 'API') } + context 'pipeline succeeded' do + let(:status) { 'success' } + let(:color) { 'good' } + let(:duration) { 10 } + let(:message) { build_markdown_message('passed') } - it 'returns a message stating it is by API' do - verify_message + it 'returns a message with information about succeeded build' do + expect(subject.pretext).to be_empty + expect(subject.attachments).to eq(message) + expect(subject.activity).to eq({ + title: 'Pipeline [#123](http://example.gitlab.com/pipelines/123) of [develop](http://example.gitlab.com/commits/develop) branch by hacker passed', + subtitle: 'in [project_name](http://example.gitlab.com)', + text: 'in 10 seconds', + image: '' + }) end end - end - def verify_message - expect(subject.pretext).to be_empty - expect(subject.fallback).to eq(message) - expect(subject.attachments).to eq([text: message, color: color]) - end + context 'pipeline failed' do + let(:status) { 'failed' } + let(:color) { 'danger' } + let(:duration) { 10 } + let(:message) { build_markdown_message } + + it 'returns a message with information about failed build' do + expect(subject.pretext).to be_empty + expect(subject.attachments).to eq(message) + expect(subject.activity).to eq({ + title: 'Pipeline [#123](http://example.gitlab.com/pipelines/123) of [develop](http://example.gitlab.com/commits/develop) branch by hacker failed', + subtitle: 'in [project_name](http://example.gitlab.com)', + text: 'in 10 seconds', + image: '' + }) + end - def build_message(status_text = status, name = user[:name]) - ":" \ - " Pipeline " \ - " of branch" \ - " by #{name} #{status_text} in #{duration} #{'second'.pluralize(duration)}" + context 'when triggered by API therefore lacking user' do + let(:user) { nil } + let(:message) { build_markdown_message(status, 'API') } + + it 'returns a message stating it is by API' do + expect(subject.pretext).to be_empty + expect(subject.attachments).to eq(message) + expect(subject.activity).to eq({ + title: 'Pipeline [#123](http://example.gitlab.com/pipelines/123) of [develop](http://example.gitlab.com/commits/develop) branch by API failed', + subtitle: 'in [project_name](http://example.gitlab.com)', + text: 'in 10 seconds', + image: '' + }) + end + end + end + + def build_markdown_message(status_text = status, name = user[:name]) + "[project_name](http://example.gitlab.com):" \ + " Pipeline [#123](http://example.gitlab.com/pipelines/123)" \ + " of [develop](http://example.gitlab.com/commits/develop)" \ + " branch by #{name} #{status_text} in #{duration} #{'second'.pluralize(duration)}" + end end end diff --git a/spec/models/project_services/chat_message/push_message_spec.rb b/spec/models/project_services/chat_message/push_message_spec.rb index 24928873bad..63eb078c44e 100644 --- a/spec/models/project_services/chat_message/push_message_spec.rb +++ b/spec/models/project_services/chat_message/push_message_spec.rb @@ -10,6 +10,7 @@ describe ChatMessage::PushMessage, models: true do project_name: 'project_name', ref: 'refs/heads/master', user_name: 'test.user', + user_avatar: 'http://someavatar.com', project_url: 'http://url.com' } end @@ -24,18 +25,36 @@ describe ChatMessage::PushMessage, models: true do ] end - it 'returns a message regarding pushes' do - expect(subject.pretext).to eq( - 'test.user pushed to branch of '\ - ' ()' - ) - expect(subject.attachments).to eq([ - { - text: ": message1 - author1\n"\ - ": message2 - author2", + context 'without markdown' do + it 'returns a message regarding pushes' do + expect(subject.pretext).to eq( + 'test.user pushed to branch of '\ + ' ()') + expect(subject.attachments).to eq([{ + text: ": message1 - author1\n\n"\ + ": message2 - author2", color: color, - } - ]) + }]) + end + end + + context 'with markdown' do + before do + args[:markdown] = true + end + + it 'returns a message regarding pushes' do + expect(subject.pretext).to eq( + 'test.user pushed to branch [master](http://url.com/commits/master) of [project_name](http://url.com) ([Compare changes](http://url.com/compare/before...after))') + expect(subject.attachments).to eq( + "[abcdefgh](http://url1.com): message1 - author1\n\n[12345678](http://url2.com): message2 - author2") + expect(subject.activity).to eq({ + title: 'test.user pushed to branch', + subtitle: 'in [project_name](http://url.com)', + text: '[Compare changes](http://url.com/compare/before...after)', + image: 'http://someavatar.com' + }) + end end end @@ -47,15 +66,36 @@ describe ChatMessage::PushMessage, models: true do project_name: 'project_name', ref: 'refs/tags/new_tag', user_name: 'test.user', + user_avatar: 'http://someavatar.com', project_url: 'http://url.com' } end - it 'returns a message regarding pushes' do - expect(subject.pretext).to eq('test.user pushed new tag ' \ - ' to ' \ - '') - expect(subject.attachments).to be_empty + context 'without markdown' do + it 'returns a message regarding pushes' do + expect(subject.pretext).to eq('test.user pushed new tag ' \ + ' to ' \ + '') + expect(subject.attachments).to be_empty + end + end + + context 'with markdown' do + before do + args[:markdown] = true + end + + it 'returns a message regarding pushes' do + expect(subject.pretext).to eq( + 'test.user pushed new tag [new_tag](http://url.com/commits/new_tag) to [project_name](http://url.com)') + expect(subject.attachments).to be_empty + expect(subject.activity).to eq({ + title: 'test.user created tag', + subtitle: 'in [project_name](http://url.com)', + text: '[Compare changes](http://url.com/compare/0000000000000000000000000000000000000000...after)', + image: 'http://someavatar.com' + }) + end end end @@ -64,12 +104,31 @@ describe ChatMessage::PushMessage, models: true do args[:before] = Gitlab::Git::BLANK_SHA end - it 'returns a message regarding a new branch' do - expect(subject.pretext).to eq( - 'test.user pushed new branch to '\ - '' - ) - expect(subject.attachments).to be_empty + context 'without markdown' do + it 'returns a message regarding a new branch' do + expect(subject.pretext).to eq( + 'test.user pushed new branch to '\ + '') + expect(subject.attachments).to be_empty + end + end + + context 'with markdown' do + before do + args[:markdown] = true + end + + it 'returns a message regarding a new branch' do + expect(subject.pretext).to eq( + 'test.user pushed new branch [master](http://url.com/commits/master) to [project_name](http://url.com)') + expect(subject.attachments).to be_empty + expect(subject.activity).to eq({ + title: 'test.user created branch', + subtitle: 'in [project_name](http://url.com)', + text: '[Compare changes](http://url.com/compare/0000000000000000000000000000000000000000...after)', + image: 'http://someavatar.com' + }) + end end end @@ -78,11 +137,30 @@ describe ChatMessage::PushMessage, models: true do args[:after] = Gitlab::Git::BLANK_SHA end - it 'returns a message regarding a removed branch' do - expect(subject.pretext).to eq( - 'test.user removed branch master from ' - ) - expect(subject.attachments).to be_empty + context 'without markdown' do + it 'returns a message regarding a removed branch' do + expect(subject.pretext).to eq( + 'test.user removed branch master from ') + expect(subject.attachments).to be_empty + end + end + + context 'with markdown' do + before do + args[:markdown] = true + end + + it 'returns a message regarding a removed branch' do + expect(subject.pretext).to eq( + 'test.user removed branch master from [project_name](http://url.com)') + expect(subject.attachments).to be_empty + expect(subject.activity).to eq({ + title: 'test.user removed branch', + subtitle: 'in [project_name](http://url.com)', + text: '[Compare changes](http://url.com/compare/before...0000000000000000000000000000000000000000)', + image: 'http://someavatar.com' + }) + end end end end diff --git a/spec/models/project_services/chat_message/wiki_page_message_spec.rb b/spec/models/project_services/chat_message/wiki_page_message_spec.rb index a2ad61e38e7..0df7db2abc2 100644 --- a/spec/models/project_services/chat_message/wiki_page_message_spec.rb +++ b/spec/models/project_services/chat_message/wiki_page_message_spec.rb @@ -7,7 +7,8 @@ describe ChatMessage::WikiPageMessage, models: true do { user: { name: 'Test User', - username: 'test.user' + username: 'test.user', + avatar_url: 'http://someavatar.com' }, project_name: 'project_name', project_url: 'http://somewhere.com', @@ -19,54 +20,128 @@ describe ChatMessage::WikiPageMessage, models: true do } end - describe '#pretext' do - context 'when :action == "create"' do - before { args[:object_attributes][:action] = 'create' } + context 'without markdown' do + describe '#pretext' do + context 'when :action == "create"' do + before { args[:object_attributes][:action] = 'create' } - it 'returns a message that a new wiki page was created' do - expect(subject.pretext).to eq( - 'test.user created in : '\ - '*Wiki page title*') + it 'returns a message that a new wiki page was created' do + expect(subject.pretext).to eq( + 'test.user created in : '\ + '*Wiki page title*') + end + end + + context 'when :action == "update"' do + before { args[:object_attributes][:action] = 'update' } + + it 'returns a message that a wiki page was updated' do + expect(subject.pretext).to eq( + 'test.user edited in : '\ + '*Wiki page title*') + end end end - context 'when :action == "update"' do - before { args[:object_attributes][:action] = 'update' } + describe '#attachments' do + let(:color) { '#345' } - it 'returns a message that a wiki page was updated' do - expect(subject.pretext).to eq( - 'test.user edited in : '\ - '*Wiki page title*') + context 'when :action == "create"' do + before { args[:object_attributes][:action] = 'create' } + + it 'returns the attachment for a new wiki page' do + expect(subject.attachments).to eq([ + { + text: "Wiki page description", + color: color, + } + ]) + end + end + + context 'when :action == "update"' do + before { args[:object_attributes][:action] = 'update' } + + it 'returns the attachment for an updated wiki page' do + expect(subject.attachments).to eq([ + { + text: "Wiki page description", + color: color, + } + ]) + end end end end - describe '#attachments' do - let(:color) { '#345' } + context 'with markdown' do + before do + args[:markdown] = true + end + + describe '#pretext' do + context 'when :action == "create"' do + before { args[:object_attributes][:action] = 'create' } + + it 'returns a message that a new wiki page was created' do + expect(subject.pretext).to eq( + 'test.user created [wiki page](http://url.com) in [project_name](http://somewhere.com): *Wiki page title*') + end + end - context 'when :action == "create"' do - before { args[:object_attributes][:action] = 'create' } + context 'when :action == "update"' do + before { args[:object_attributes][:action] = 'update' } - it 'returns the attachment for a new wiki page' do - expect(subject.attachments).to eq([ - { - text: "Wiki page description", - color: color, - } - ]) + it 'returns a message that a wiki page was updated' do + expect(subject.pretext).to eq( + 'test.user edited [wiki page](http://url.com) in [project_name](http://somewhere.com): *Wiki page title*') + end end end - context 'when :action == "update"' do - before { args[:object_attributes][:action] = 'update' } + describe '#attachments' do + context 'when :action == "create"' do + before { args[:object_attributes][:action] = 'create' } + + it 'returns the attachment for a new wiki page' do + expect(subject.attachments).to eq('Wiki page description') + end + end + + context 'when :action == "update"' do + before { args[:object_attributes][:action] = 'update' } + + it 'returns the attachment for an updated wiki page' do + expect(subject.attachments).to eq('Wiki page description') + end + end + end + + describe '#activity' do + context 'when :action == "create"' do + before { args[:object_attributes][:action] = 'create' } + + it 'returns the attachment for a new wiki page' do + expect(subject.activity).to eq({ + title: 'test.user created [wiki page](http://url.com)', + subtitle: 'in [project_name](http://somewhere.com)', + text: 'Wiki page title', + image: 'http://someavatar.com' + }) + end + end + + context 'when :action == "update"' do + before { args[:object_attributes][:action] = 'update' } - it 'returns the attachment for an updated wiki page' do - expect(subject.attachments).to eq([ - { - text: "Wiki page description", - color: color, - } - ]) + it 'returns the attachment for an updated wiki page' do + expect(subject.activity).to eq({ + title: 'test.user edited [wiki page](http://url.com)', + subtitle: 'in [project_name](http://somewhere.com)', + text: 'Wiki page title', + image: 'http://someavatar.com' + }) + end end end end diff --git a/spec/models/project_services/microsoft_teams_service_spec.rb b/spec/models/project_services/microsoft_teams_service_spec.rb new file mode 100644 index 00000000000..facc034f69c --- /dev/null +++ b/spec/models/project_services/microsoft_teams_service_spec.rb @@ -0,0 +1,277 @@ +require 'spec_helper' + +describe MicrosoftTeamsService, models: true do + let(:chat_service) { described_class.new } + let(:webhook_url) { 'https://example.gitlab.com/' } + + describe "Associations" do + it { is_expected.to belong_to :project } + it { is_expected.to have_one :service_hook } + end + + describe 'Validations' do + context 'when service is active' do + before { subject.active = true } + + it { is_expected.to validate_presence_of(:webhook) } + it_behaves_like 'issue tracker service URL attribute', :webhook + end + + context 'when service is inactive' do + before { subject.active = false } + + it { is_expected.not_to validate_presence_of(:webhook) } + end + end + + describe "#execute" do + let(:user) { create(:user) } + let(:project) { create(:project, :repository) } + + before do + allow(chat_service).to receive_messages( + project: project, + project_id: project.id, + service_hook: true, + webhook: webhook_url + ) + + WebMock.stub_request(:post, webhook_url) + end + + context 'with push events' do + let(:push_sample_data) do + Gitlab::DataBuilder::Push.build_sample(project, user) + end + + it "calls Microsoft Teams API for push events" do + chat_service.execute(push_sample_data) + + expect(WebMock).to have_requested(:post, webhook_url).once + end + + it 'specifies the webhook when it is configured' do + expect(MicrosoftTeams::Notifier).to receive(:new).with(webhook_url).and_return(double(:microsoft_teams_service).as_null_object) + + chat_service.execute(push_sample_data) + end + end + + context 'with issue events' do + let(:opts) { { title: 'Awesome issue', description: 'please fix' } } + let(:issues_sample_data) do + service = Issues::CreateService.new(project, user, opts) + issue = service.execute + service.hook_data(issue, 'open') + end + + it "calls Microsoft Teams API" do + chat_service.execute(issues_sample_data) + + expect(WebMock).to have_requested(:post, webhook_url).once + end + end + + context 'with merge events' do + let(:opts) do + { + title: 'Awesome merge_request', + description: 'please fix', + source_branch: 'feature', + target_branch: 'master' + } + end + + let(:merge_sample_data) do + service = MergeRequests::CreateService.new(project, user, opts) + merge_request = service.execute + service.hook_data(merge_request, 'open') + end + + it "calls Microsoft Teams API" do + chat_service.execute(merge_sample_data) + + expect(WebMock).to have_requested(:post, webhook_url).once + end + end + + context 'with wiki page events' do + let(:opts) do + { + title: "Awesome wiki_page", + content: "Some text describing some thing or another", + format: "md", + message: "user created page: Awesome wiki_page" + } + end + + let(:wiki_page_sample_data) do + service = WikiPages::CreateService.new(project, user, opts) + wiki_page = service.execute + service.hook_data(wiki_page, 'create') + end + + it "calls Microsoft Teams API" do + chat_service.execute(wiki_page_sample_data) + + expect(WebMock).to have_requested(:post, webhook_url).once + end + end + end + + describe "Note events" do + let(:user) { create(:user) } + let(:project) { create(:project, :repository, creator: user) } + + before do + allow(chat_service).to receive_messages( + project: project, + project_id: project.id, + service_hook: true, + webhook: webhook_url + ) + + WebMock.stub_request(:post, webhook_url) + end + + context 'when commit comment event executed' do + let(:commit_note) do + create(:note_on_commit, author: user, + project: project, + commit_id: project.repository.commit.id, + note: 'a comment on a commit') + end + + it "calls Microsoft Teams API for commit comment events" do + data = Gitlab::DataBuilder::Note.build(commit_note, user) + + chat_service.execute(data) + + expect(WebMock).to have_requested(:post, webhook_url).once + end + end + + context 'when merge request comment event executed' do + let(:merge_request_note) do + create(:note_on_merge_request, project: project, + note: "merge request note") + end + + it "calls Microsoft Teams API for merge request comment events" do + data = Gitlab::DataBuilder::Note.build(merge_request_note, user) + + chat_service.execute(data) + + expect(WebMock).to have_requested(:post, webhook_url).once + end + end + + context 'when issue comment event executed' do + let(:issue_note) do + create(:note_on_issue, project: project, note: "issue note") + end + + it "calls Microsoft Teams API for issue comment events" do + data = Gitlab::DataBuilder::Note.build(issue_note, user) + + chat_service.execute(data) + + expect(WebMock).to have_requested(:post, webhook_url).once + end + end + + context 'when snippet comment event executed' do + let(:snippet_note) do + create(:note_on_project_snippet, project: project, + note: "snippet note") + end + + it "calls Microsoft Teams API for snippet comment events" do + data = Gitlab::DataBuilder::Note.build(snippet_note, user) + + chat_service.execute(data) + + expect(WebMock).to have_requested(:post, webhook_url).once + end + end + end + + describe 'Pipeline events' do + let(:user) { create(:user) } + let(:project) { create(:project, :repository) } + + let(:pipeline) do + create(:ci_pipeline, + project: project, status: status, + sha: project.commit.sha, ref: project.default_branch) + end + + before do + allow(chat_service).to receive_messages( + project: project, + service_hook: true, + webhook: webhook_url + ) + end + + shared_examples 'call Microsoft Teams API' do + before do + WebMock.stub_request(:post, webhook_url) + end + + it 'calls Microsoft Teams API for pipeline events' do + data = Gitlab::DataBuilder::Pipeline.build(pipeline) + + chat_service.execute(data) + + expect(WebMock).to have_requested(:post, webhook_url).once + end + end + + context 'with failed pipeline' do + let(:status) { 'failed' } + + it_behaves_like 'call Microsoft Teams API' + end + + context 'with succeeded pipeline' do + let(:status) { 'success' } + + context 'with default to notify_only_broken_pipelines' do + it 'does not call Microsoft Teams API for pipeline events' do + data = Gitlab::DataBuilder::Pipeline.build(pipeline) + result = chat_service.execute(data) + + expect(result).to be_falsy + end + end + + context 'with setting notify_only_broken_pipelines to false' do + before do + chat_service.notify_only_broken_pipelines = false + end + + it_behaves_like 'call Microsoft Teams API' + end + end + + context 'only notify for the default branch' do + context 'when enabled' do + let(:pipeline) do + create(:ci_pipeline, project: project, status: 'failed', ref: 'not-the-default-branch') + end + + before do + chat_service.notify_only_default_branch = true + end + + it 'does not call the Microsoft Teams API for pipeline events' do + data = Gitlab::DataBuilder::Pipeline.build(pipeline) + result = chat_service.execute(data) + + expect(result).to be_falsy + end + end + end + end +end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 59a2560ca06..71cb72ab11b 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -22,6 +22,7 @@ describe Project, models: true do it { is_expected.to have_many(:protected_branches).dependent(:destroy) } it { is_expected.to have_one(:forked_project_link).dependent(:destroy) } it { is_expected.to have_one(:slack_service).dependent(:destroy) } + it { is_expected.to have_one(:microsoft_teams_service).dependent(:destroy) } it { is_expected.to have_one(:mattermost_service).dependent(:destroy) } it { is_expected.to have_one(:pushover_service).dependent(:destroy) } it { is_expected.to have_one(:asana_service).dependent(:destroy) } -- cgit v1.2.1