diff options
Diffstat (limited to 'lib')
26 files changed, 765 insertions, 167 deletions
diff --git a/lib/api/jobs.rb b/lib/api/jobs.rb index ffab0aafe59..288b03d940c 100644 --- a/lib/api/jobs.rb +++ b/lib/api/jobs.rb @@ -118,7 +118,7 @@ module API content_type 'text/plain' env['api.format'] = :binary - trace = build.trace + trace = build.trace.raw body trace end diff --git a/lib/api/runner.rb b/lib/api/runner.rb index d288369e362..6fbb02cb3aa 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -115,7 +115,7 @@ module API put '/:id' do job = authenticate_job! - job.update_attributes(trace: params[:trace]) if params[:trace] + job.trace.set(params[:trace]) if params[:trace] Gitlab::Metrics.add_event(:update_build, project: job.project.path_with_namespace) @@ -145,16 +145,14 @@ module API content_range = request.headers['Content-Range'] content_range = content_range.split('-') - current_length = job.trace_length - unless current_length == content_range[0].to_i - return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{current_length}" }) + stream_size = job.trace.append(request.body.read, content_range[0].to_i) + if stream_size < 0 + return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{-stream_size}" }) end - job.append_trace(request.body.read, content_range[0].to_i) - status 202 header 'Job-Status', job.status - header 'Range', "0-#{job.trace_length}" + header 'Range', "0-#{stream_size}" end desc 'Authorize artifacts uploading for job' do diff --git a/lib/api/services.rb b/lib/api/services.rb index 6802a99311e..65f86caaa51 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -488,6 +488,14 @@ module API desc: 'The channel name' } ], + 'microsoft-teams' => [ + { + required: true, + name: :webhook, + type: String, + desc: 'The Microsoft Teams webhook. e.g. https://outlook.office.com/webhook/…' + } + ], 'mattermost' => [ { required: true, @@ -550,6 +558,7 @@ module API RedmineService, SlackService, MattermostService, + MicrosoftTeamsService, TeamcityService, ] diff --git a/lib/api/v3/builds.rb b/lib/api/v3/builds.rb index 6f97102c6ef..4dd03cdf24b 100644 --- a/lib/api/v3/builds.rb +++ b/lib/api/v3/builds.rb @@ -120,7 +120,7 @@ module API content_type 'text/plain' env['api.format'] = :binary - trace = build.trace + trace = build.trace.raw body trace end diff --git a/lib/api/v3/services.rb b/lib/api/v3/services.rb index 3bacaeee032..bbe07ed4212 100644 --- a/lib/api/v3/services.rb +++ b/lib/api/v3/services.rb @@ -501,6 +501,12 @@ module API desc: 'The channel name' } ], + 'microsoft-teams' => [ + required: true, + name: :webhook, + type: String, + desc: 'The Microsoft Teams webhook. e.g. https://outlook.office.com/webhook/…' + ], 'mattermost' => [ { required: true, diff --git a/lib/ci/ansi2html.rb b/lib/ci/ansi2html.rb index b3ccad7b28d..1020452480a 100644 --- a/lib/ci/ansi2html.rb +++ b/lib/ci/ansi2html.rb @@ -132,34 +132,54 @@ module Ci STATE_PARAMS = [:offset, :n_open_tags, :fg_color, :bg_color, :style_mask].freeze - def convert(raw, new_state) + def convert(stream, new_state) reset_state - restore_state(raw, new_state) if new_state.present? - - start = @offset - ansi = raw[@offset..-1] + restore_state(new_state, stream) if new_state.present? + + append = false + truncated = false + + cur_offset = stream.tell + if cur_offset > @offset + @offset = cur_offset + truncated = true + else + stream.seek(@offset) + append = @offset > 0 + end + start_offset = @offset open_new_tag - s = StringScanner.new(ansi) - until s.eos? - if s.scan(/\e([@-_])(.*?)([@-~])/) - handle_sequence(s) - elsif s.scan(/\e(([@-_])(.*?)?)?$/) - break - elsif s.scan(/</) - @out << '<' - elsif s.scan(/\r?\n/) - @out << '<br>' - else - @out << s.scan(/./m) + stream.each_line do |line| + s = StringScanner.new(line) + until s.eos? + if s.scan(/\e([@-_])(.*?)([@-~])/) + handle_sequence(s) + elsif s.scan(/\e(([@-_])(.*?)?)?$/) + break + elsif s.scan(/</) + @out << '<' + elsif s.scan(/\r?\n/) + @out << '<br>' + else + @out << s.scan(/./m) + end + @offset += s.matched_size end - @offset += s.matched_size end close_open_tags() - { state: state, html: @out, text: ansi[0, @offset - start], append: start > 0 } + OpenStruct.new( + html: @out, + state: state, + append: append, + truncated: truncated, + offset: start_offset, + size: stream.tell - start_offset, + total: stream.size + ) end def handle_sequence(s) @@ -240,10 +260,10 @@ module Ci Base64.urlsafe_encode64(state.to_json) end - def restore_state(raw, new_state) + def restore_state(new_state, stream) state = Base64.urlsafe_decode64(new_state) state = JSON.parse(state, symbolize_names: true) - return if state[:offset].to_i > raw.length + return if state[:offset].to_i > stream.size STATE_PARAMS.each do |param| send("#{param}=".to_sym, state[param]) diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 95cc6308c3b..67b269b330c 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -61,7 +61,7 @@ module Ci update_runner_info - build.update_attributes(trace: params[:trace]) if params[:trace] + build.trace.set(params[:trace]) if params[:trace] Gitlab::Metrics.add_event(:update_build, project: build.project.path_with_namespace) @@ -92,16 +92,14 @@ module Ci content_range = request.headers['Content-Range'] content_range = content_range.split('-') - current_length = build.trace_length - unless current_length == content_range[0].to_i - return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{current_length}" }) + stream_size = build.trace.append(request.body.read, content_range[0].to_i) + if stream_size < 0 + return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{-stream_size}" }) end - build.append_trace(request.body.read, content_range[0].to_i) - status 202 header 'Build-Status', build.status - header 'Range', "0-#{build.trace_length}" + header 'Range', "0-#{stream_size}" end # Authorize artifacts uploading for build - Runners only diff --git a/lib/container_registry/blob.rb b/lib/container_registry/blob.rb index eb5a2596177..d5f85f9fcad 100644 --- a/lib/container_registry/blob.rb +++ b/lib/container_registry/blob.rb @@ -38,11 +38,11 @@ module ContainerRegistry end def delete - client.delete_blob(repository.name, digest) + client.delete_blob(repository.path, digest) end def data - @data ||= client.blob(repository.name, digest, type) + @data ||= client.blob(repository.path, digest, type) end end end diff --git a/lib/container_registry/path.rb b/lib/container_registry/path.rb new file mode 100644 index 00000000000..a4b5f2aba6c --- /dev/null +++ b/lib/container_registry/path.rb @@ -0,0 +1,70 @@ +module ContainerRegistry + ## + # Class responsible for extracting project and repository name from + # image repository path provided by a containers registry API response. + # + # Example: + # + # some/group/my_project/my/image -> + # project: some/group/my_project + # repository: my/image + # + class Path + InvalidRegistryPathError = Class.new(StandardError) + + LEVELS_SUPPORTED = 3 + + def initialize(path) + @path = path + end + + def valid? + @path =~ Gitlab::Regex.container_repository_name_regex && + components.size > 1 && + components.size < Namespace::NUMBER_OF_ANCESTORS_ALLOWED + end + + def components + @components ||= @path.to_s.split('/') + end + + def nodes + raise InvalidRegistryPathError unless valid? + + @nodes ||= components.size.downto(2).map do |length| + components.take(length).join('/') + end + end + + def has_project? + repository_project.present? + end + + def has_repository? + return false unless has_project? + + repository_project.container_repositories + .where(name: repository_name).any? + end + + def root_repository? + @path == repository_project.full_path + end + + def repository_project + @project ||= Project + .where_full_path_in(nodes.first(LEVELS_SUPPORTED)) + .first + end + + def repository_name + return unless has_project? + + @path.remove(%r(^#{Regexp.escape(repository_project.full_path)}/?)) + end + + def to_s + @path + end + end +end diff --git a/lib/container_registry/registry.rb b/lib/container_registry/registry.rb index 0e634f6b6ef..63bce655f57 100644 --- a/lib/container_registry/registry.rb +++ b/lib/container_registry/registry.rb @@ -8,10 +8,6 @@ module ContainerRegistry @client = ContainerRegistry::Client.new(uri, options) end - def repository(name) - ContainerRegistry::Repository.new(self, name) - end - private def default_path diff --git a/lib/container_registry/repository.rb b/lib/container_registry/repository.rb deleted file mode 100644 index 0e4a7cb3cc9..00000000000 --- a/lib/container_registry/repository.rb +++ /dev/null @@ -1,48 +0,0 @@ -module ContainerRegistry - class Repository - attr_reader :registry, :name - - delegate :client, to: :registry - - def initialize(registry, name) - @registry, @name = registry, name - end - - def path - [registry.path, name].compact.join('/') - end - - def tag(tag) - ContainerRegistry::Tag.new(self, tag) - end - - def manifest - return @manifest if defined?(@manifest) - - @manifest = client.repository_tags(name) - end - - def valid? - manifest.present? - end - - def tags - return @tags if defined?(@tags) - return [] unless manifest && manifest['tags'] - - @tags = manifest['tags'].map do |tag| - ContainerRegistry::Tag.new(self, tag) - end - end - - def blob(config) - ContainerRegistry::Blob.new(self, config) - end - - def delete_tags - return unless tags - - tags.all?(&:delete) - end - end -end diff --git a/lib/container_registry/tag.rb b/lib/container_registry/tag.rb index 59040199920..d00e6191e7e 100644 --- a/lib/container_registry/tag.rb +++ b/lib/container_registry/tag.rb @@ -22,9 +22,7 @@ module ContainerRegistry end def manifest - return @manifest if defined?(@manifest) - - @manifest = client.repository_manifest(repository.name, name) + @manifest ||= client.repository_manifest(repository.path, name) end def path @@ -38,9 +36,7 @@ module ContainerRegistry end def digest - return @digest if defined?(@digest) - - @digest = client.repository_tag_digest(repository.name, name) + @digest ||= client.repository_tag_digest(repository.path, name) end def config_blob @@ -82,7 +78,7 @@ module ContainerRegistry def delete return unless digest - client.delete_repository_tag(repository.name, digest) + client.delete_repository_tag(repository.path, digest) end end end diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb new file mode 100644 index 00000000000..a3cc350ef22 --- /dev/null +++ b/lib/gitlab/ci/cron_parser.rb @@ -0,0 +1,34 @@ +module Gitlab + module Ci + class CronParser + VALID_SYNTAX_SAMPLE_TIME_ZONE = 'UTC'.freeze + VALID_SYNTAX_SAMPLE_CRON = '* * * * *'.freeze + + def initialize(cron, cron_timezone = 'UTC') + @cron = cron + @cron_timezone = cron_timezone + end + + def next_time_from(time) + @cron_line ||= try_parse_cron(@cron, @cron_timezone) + @cron_line.next_time(time).in_time_zone(Time.zone) if @cron_line.present? + end + + def cron_valid? + try_parse_cron(@cron, VALID_SYNTAX_SAMPLE_TIME_ZONE).present? + end + + def cron_timezone_valid? + try_parse_cron(VALID_SYNTAX_SAMPLE_CRON, @cron_timezone).present? + end + + private + + def try_parse_cron(cron, cron_timezone) + Rufus::Scheduler.parse("#{cron} #{cron_timezone}") + rescue + # noop + end + end + end +end diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb new file mode 100644 index 00000000000..5b835bb669a --- /dev/null +++ b/lib/gitlab/ci/trace.rb @@ -0,0 +1,136 @@ +module Gitlab + module Ci + class Trace + attr_reader :job + + delegate :old_trace, to: :job + + def initialize(job) + @job = job + end + + def html(last_lines: nil) + read do |stream| + stream.html(last_lines: last_lines) + end + end + + def raw(last_lines: nil) + read do |stream| + stream.raw(last_lines: last_lines) + end + end + + def extract_coverage(regex) + read do |stream| + stream.extract_coverage(regex) + end + end + + def set(data) + write do |stream| + data = job.hide_secrets(data) + stream.set(data) + end + end + + def append(data, offset) + write do |stream| + current_length = stream.size + return -current_length unless current_length == offset + + data = job.hide_secrets(data) + stream.append(data, offset) + stream.size + end + end + + def exist? + current_path.present? || old_trace.present? + end + + def read + stream = Gitlab::Ci::Trace::Stream.new do + if current_path + File.open(current_path, "rb") + elsif old_trace + StringIO.new(old_trace) + end + end + + yield stream + ensure + stream&.close + end + + def write + stream = Gitlab::Ci::Trace::Stream.new do + File.open(ensure_path, "a+b") + end + + yield(stream).tap do + job.touch if job.needs_touch? + end + ensure + stream&.close + end + + def erase! + paths.each do |trace_path| + FileUtils.rm(trace_path, force: true) + end + + job.erase_old_trace! + end + + private + + def ensure_path + return current_path if current_path + + ensure_directory + default_path + end + + def ensure_directory + unless Dir.exist?(default_directory) + FileUtils.mkdir_p(default_directory) + end + end + + def current_path + @current_path ||= paths.find do |trace_path| + File.exist?(trace_path) + end + end + + def paths + [ + default_path, + deprecated_path + ].compact + end + + def default_directory + File.join( + Settings.gitlab_ci.builds_path, + job.created_at.utc.strftime("%Y_%m"), + job.project_id.to_s + ) + end + + def default_path + File.join(default_directory, "#{job.id}.log") + end + + def deprecated_path + File.join( + Settings.gitlab_ci.builds_path, + job.created_at.utc.strftime("%Y_%m"), + job.project.ci_id.to_s, + "#{job.id}.log" + ) if job.project&.ci_id + end + end + end +end diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb new file mode 100644 index 00000000000..2af94e2c60e --- /dev/null +++ b/lib/gitlab/ci/trace/stream.rb @@ -0,0 +1,119 @@ +module Gitlab + module Ci + class Trace + # This was inspired from: http://stackoverflow.com/a/10219411/1520132 + class Stream + BUFFER_SIZE = 4096 + LIMIT_SIZE = 50.kilobytes + + attr_reader :stream + + delegate :close, :tell, :seek, :size, :path, :truncate, to: :stream, allow_nil: true + + delegate :valid?, to: :stream, as: :present?, allow_nil: true + + def initialize + @stream = yield + end + + def valid? + self.stream.present? + end + + def file? + self.path.present? + end + + def limit(last_bytes = LIMIT_SIZE) + stream_size = size + if stream_size < last_bytes + last_bytes = stream_size + end + stream.seek(-last_bytes, IO::SEEK_END) + end + + def append(data, offset) + stream.truncate(offset) + stream.seek(0, IO::SEEK_END) + stream.write(data) + stream.flush() + end + + def set(data) + truncate(0) + stream.write(data) + stream.flush() + end + + def raw(last_lines: nil) + return unless valid? + + if last_lines.to_i > 0 + read_last_lines(last_lines) + else + stream.read + end + end + + def html_with_state(state = nil) + ::Ci::Ansi2html.convert(stream, state) + end + + def html(last_lines: nil) + text = raw(last_lines: last_lines) + stream = StringIO.new(text) + ::Ci::Ansi2html.convert(stream).html + end + + def extract_coverage(regex) + return unless valid? + return unless regex + + regex = Regexp.new(regex) + + match = "" + + stream.each_line do |line| + matches = line.scan(regex) + next unless matches.is_a?(Array) + + match = matches.flatten.last + coverage = match.gsub(/\d+(\.\d+)?/).first + return coverage.to_f if coverage.present? + end + rescue + # if bad regex or something goes wrong we dont want to interrupt transition + # so we just silentrly ignore error for now + end + + private + + def read_last_lines(last_lines) + chunks = [] + pos = lines = 0 + max = stream.size + + # We want an extra line to make sure fist line has full contents + while lines <= last_lines && pos < max + pos += BUFFER_SIZE + + buf = + if pos <= max + stream.seek(-pos, IO::SEEK_END) + stream.read(BUFFER_SIZE) + else # Reached the head, read only left + stream.seek(0) + stream.read(BUFFER_SIZE - (pos - max)) + end + + lines += buf.count("\n") + chunks.unshift(buf) + end + + chunks.join.lines.last(last_lines).join + .force_encoding(Encoding.default_external) + end + end + end + end +end diff --git a/lib/gitlab/ci/trace_reader.rb b/lib/gitlab/ci/trace_reader.rb deleted file mode 100644 index 1d7ddeb3e0f..00000000000 --- a/lib/gitlab/ci/trace_reader.rb +++ /dev/null @@ -1,50 +0,0 @@ -module Gitlab - module Ci - # This was inspired from: http://stackoverflow.com/a/10219411/1520132 - class TraceReader - BUFFER_SIZE = 4096 - - attr_accessor :path, :buffer_size - - def initialize(new_path, buffer_size: BUFFER_SIZE) - self.path = new_path - self.buffer_size = Integer(buffer_size) - end - - def read(last_lines: nil) - if last_lines - read_last_lines(last_lines) - else - File.read(path) - end - end - - def read_last_lines(max_lines) - File.open(path) do |file| - chunks = [] - pos = lines = 0 - max = file.size - - # We want an extra line to make sure fist line has full contents - while lines <= max_lines && pos < max - pos += buffer_size - - buf = if pos <= max - file.seek(-pos, IO::SEEK_END) - file.read(buffer_size) - else # Reached the head, read only left - file.seek(0) - file.read(buffer_size - (pos - max)) - end - - lines += buf.count("\n") - chunks.unshift(buf) - end - - chunks.join.lines.last(max_lines).join - .force_encoding(Encoding.default_external) - end - end - end - end -end diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb index cd4e318033d..630fe4fa849 100644 --- a/lib/gitlab/etag_caching/middleware.rb +++ b/lib/gitlab/etag_caching/middleware.rb @@ -2,26 +2,34 @@ module Gitlab module EtagCaching class Middleware RESERVED_WORDS = NamespaceValidator::WILDCARD_ROUTES.map { |word| "/#{word}/" }.join('|') - ROUTE_REGEXP = Regexp.union( - %r(^(?!.*(#{RESERVED_WORDS})).*/noteable/issue/\d+/notes\z), - %r(^(?!.*(#{RESERVED_WORDS})).*/issues/\d+/rendered_title\z) - ) + ROUTES = [ + { + regexp: %r(^(?!.*(#{RESERVED_WORDS})).*/noteable/issue/\d+/notes\z), + name: 'issue_notes' + }, + { + regexp: %r(^(?!.*(#{RESERVED_WORDS})).*/issues/\d+/rendered_title\z), + name: 'issue_title' + } + ].freeze def initialize(app) @app = app end def call(env) - return @app.call(env) unless enabled_for_current_route?(env) - Gitlab::Metrics.add_event(:etag_caching_middleware_used) + route = match_current_route(env) + return @app.call(env) unless route + + track_event(:etag_caching_middleware_used, route) etag, cached_value_present = get_etag(env) if_none_match = env['HTTP_IF_NONE_MATCH'] if if_none_match == etag - handle_cache_hit(etag) + handle_cache_hit(etag, route) else - track_cache_miss(if_none_match, cached_value_present) + track_cache_miss(if_none_match, cached_value_present, route) status, headers, body = @app.call(env) headers['ETag'] = etag @@ -31,8 +39,8 @@ module Gitlab private - def enabled_for_current_route?(env) - ROUTE_REGEXP.match(env['PATH_INFO']) + def match_current_route(env) + ROUTES.find { |route| route[:regexp].match(env['PATH_INFO']) } end def get_etag(env) @@ -52,23 +60,27 @@ module Gitlab %Q{W/"#{value}"} end - def handle_cache_hit(etag) - Gitlab::Metrics.add_event(:etag_caching_cache_hit) + def handle_cache_hit(etag, route) + track_event(:etag_caching_cache_hit, route) status_code = Gitlab::PollingInterval.polling_enabled? ? 304 : 429 [status_code, { 'ETag' => etag }, ['']] end - def track_cache_miss(if_none_match, cached_value_present) + def track_cache_miss(if_none_match, cached_value_present, route) if if_none_match.blank? - Gitlab::Metrics.add_event(:etag_caching_header_missing) + track_event(:etag_caching_header_missing, route) elsif !cached_value_present - Gitlab::Metrics.add_event(:etag_caching_key_not_found) + track_event(:etag_caching_key_not_found, route) else - Gitlab::Metrics.add_event(:etag_caching_resource_changed) + track_event(:etag_caching_resource_changed, route) end end + + def track_event(name, route) + Gitlab::Metrics.add_event(name, endpoint: route[:name]) + end end end end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 2e4314932c8..9e338282e96 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -452,6 +452,21 @@ module Gitlab Gitlab::Git::DiffCollection.new(diff_patches(from, to, options, *paths), options) end + # Returns a RefName for a given SHA + def ref_name_for_sha(ref_path, sha) + Gitlab::GitalyClient.migrate(:find_ref_name) do |is_enabled| + if is_enabled + gitaly_ref_client.find_ref_name(sha, ref_path) + else + args = %W(#{Gitlab.config.git.bin_path} for-each-ref --count=1 #{ref_path} --contains #{sha}) + + # Not found -> ["", 0] + # Found -> ["b8d95eb4969eefacb0a58f6a28f6803f8070e7b9 commit\trefs/environments/production/77\n", 0] + Gitlab::Popen.popen(args, @path).first.split.last + end + end + end + # Returns commits collection # # Ex. diff --git a/lib/gitlab/gitaly_client/ref.rb b/lib/gitlab/gitaly_client/ref.rb index bfc5fa573c7..fcdf452d567 100644 --- a/lib/gitlab/gitaly_client/ref.rb +++ b/lib/gitlab/gitaly_client/ref.rb @@ -23,6 +23,16 @@ module Gitlab consume_refs_response(stub.find_all_tag_names(request), prefix: 'refs/tags/') end + def find_ref_name(commit_id, ref_prefix) + request = Gitaly::FindRefNameRequest.new( + repository: @repository, + commit_id: commit_id, + prefix: ref_prefix + ) + + stub.find_ref_name(request).name + end + private def consume_refs_response(response, prefix:) diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index f69288f7d67..f5e1e385ff9 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -39,7 +39,8 @@ project_tree: - :author - :events - :statuses - - :triggers + - triggers: + - :trigger_schedule - :deploy_keys - :services - :hooks diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index fb43e7ccdbb..2ba12f5f924 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -5,6 +5,7 @@ module Gitlab pipelines: 'Ci::Pipeline', statuses: 'commit_status', triggers: 'Ci::Trigger', + trigger_schedule: 'Ci::TriggerSchedule', builds: 'Ci::Build', hooks: 'ProjectHook', merge_access_levels: 'ProtectedBranch::MergeAccessLevel', diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 5e5f5ff1589..e599dd4a656 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -121,6 +121,13 @@ module Gitlab git_reference_regex end + ## + # Docker Distribution Registry 2.4.1 repository name rules + # + def container_repository_name_regex + @container_repository_regex ||= %r{\A[a-z0-9]+(?:[-._/][a-z0-9]+)*\Z} + end + def environment_name_regex @environment_name_regex ||= /\A[a-zA-Z0-9_\\\/\${}. -]+\z/.freeze end diff --git a/lib/microsoft_teams/activity.rb b/lib/microsoft_teams/activity.rb new file mode 100644 index 00000000000..d2c420efdaf --- /dev/null +++ b/lib/microsoft_teams/activity.rb @@ -0,0 +1,19 @@ +module MicrosoftTeams + class Activity + def initialize(title:, subtitle:, text:, image:) + @title = title + @subtitle = subtitle + @text = text + @image = image + end + + def prepare + { + 'activityTitle' => @title, + 'activitySubtitle' => @subtitle, + 'activityText' => @text, + 'activityImage' => @image + } + end + end +end diff --git a/lib/microsoft_teams/notifier.rb b/lib/microsoft_teams/notifier.rb new file mode 100644 index 00000000000..3bef68a1bcb --- /dev/null +++ b/lib/microsoft_teams/notifier.rb @@ -0,0 +1,46 @@ +module MicrosoftTeams + class Notifier + def initialize(webhook) + @webhook = webhook + @header = { 'Content-type' => 'application/json' } + end + + def ping(options = {}) + result = false + + begin + response = HTTParty.post( + @webhook.to_str, + headers: @header, + body: body(options) + ) + + result = true if response + rescue HTTParty::Error, StandardError => error + Rails.logger.info("#{self.class.name}: Error while connecting to #{@webhook}: #{error.message}") + end + + result + end + + private + + def body(options = {}) + result = { 'sections' => [] } + + result['title'] = options[:title] + result['summary'] = options[:pretext] + result['sections'] << MicrosoftTeams::Activity.new(options[:activity]).prepare + + attachments = options[:attachments] + unless attachments.blank? + result['sections'] << { + 'title' => 'Details', + 'facts' => [{ 'name' => 'Attachments', 'value' => attachments }] + } + end + + result.to_json + end + end +end diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index 09e121e5120..6e351365de0 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -326,8 +326,7 @@ start_gitlab() { echo "Gitaly is already running with pid $gapid, not restarting" else $app_root/bin/daemon_with_pidfile $gitaly_pid_path \ - $app_root/bin/with_env $gitaly_dir/env \ - $gitaly_dir/gitaly >> $gitaly_log 2>&1 & + $gitaly_dir/gitaly $gitaly_dir/config.toml >> $gitaly_log 2>&1 & fi fi diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake new file mode 100644 index 00000000000..350afeb5c0b --- /dev/null +++ b/lib/tasks/import.rake @@ -0,0 +1,204 @@ +require 'benchmark' +require 'rainbow/ext/string' +require_relative '../gitlab/shell_adapter' +require_relative '../gitlab/github_import/importer' + +class NewImporter < ::Gitlab::GithubImport::Importer + def execute + # Same as ::Gitlab::GithubImport::Importer#execute, but showing some progress. + puts 'Importing repository...'.color(:aqua) + import_repository unless project.repository_exists? + + puts 'Importing labels...'.color(:aqua) + import_labels + + puts 'Importing milestones...'.color(:aqua) + import_milestones + + puts 'Importing pull requests...'.color(:aqua) + import_pull_requests + + puts 'Importing issues...'.color(:aqua) + import_issues + + puts 'Importing issue comments...'.color(:aqua) + import_comments(:issues) + + puts 'Importing pull request comments...'.color(:aqua) + import_comments(:pull_requests) + + puts 'Importing wiki...'.color(:aqua) + import_wiki + + # Gitea doesn't have a Release API yet + # See https://github.com/go-gitea/gitea/issues/330 + unless project.gitea_import? + import_releases + end + + handle_errors + + project.repository.after_import + project.import_finish + + true + end + + def import_repository + begin + raise 'Blocked import URL.' if Gitlab::UrlBlocker.blocked_url?(project.import_url) + + gitlab_shell.import_repository(project.repository_storage_path, project.path_with_namespace, project.import_url) + rescue => e + project.repository.before_import if project.repository_exists? + + raise "Error importing repository #{project.import_url} into #{project.path_with_namespace} - #{e.message}" + end + end +end + +class GithubImport + def self.run!(*args) + new(*args).run! + end + + def initialize(token, gitlab_username, project_path, extras) + @token = token + @project_path = project_path + @current_user = User.find_by_username(gitlab_username) + @github_repo = extras.empty? ? nil : extras.first + end + + def run! + @repo = GithubRepos.new(@token, @current_user, @github_repo).choose_one! + + raise 'No repo found!' unless @repo + + show_warning! + + @project = Project.find_by_full_path(@project_path) || new_project + + import! + end + + private + + def show_warning! + puts "This will import GH #{@repo.full_name.bright} into GL #{@project_path.bright} as #{@current_user.name}" + puts "Permission checks are ignored. Press any key to continue.".color(:red) + + STDIN.getch + + puts 'Starting the import...'.color(:green) + end + + def import! + import_url = @project.import_url.gsub(/\:\/\/(.*@)?/, "://#{@token}@") + @project.update(import_url: import_url) + + @project.import_start + + timings = Benchmark.measure do + NewImporter.new(@project).execute + end + + puts "Import finished. Timings: #{timings}".color(:green) + end + + def new_project + Project.transaction do + namespace_path, _sep, name = @project_path.rpartition('/') + namespace = find_or_create_namespace(namespace_path) + + Project.create!( + import_url: "https://#{@token}@github.com/#{@repo.full_name}.git", + name: name, + path: name, + description: @repo.description, + namespace: namespace, + visibility_level: visibility_level, + import_type: 'github', + import_source: @repo.full_name, + creator: @current_user + ) + end + end + + def find_or_create_namespace(names) + return @current_user.namespace if names == @current_user.namespace_path + return @current_user.namespace unless @current_user.can_create_group? + + names = params[:target_namespace].presence || names + full_path_namespace = Namespace.find_by_full_path(names) + + return full_path_namespace if full_path_namespace + + names.split('/').inject(nil) do |parent, name| + begin + namespace = Group.create!(name: name, + path: name, + owner: @current_user, + parent: parent) + namespace.add_owner(@current_user) + + namespace + rescue ActiveRecord::RecordNotUnique, ActiveRecord::RecordInvalid + Namespace.where(parent: parent).find_by_path_or_name(name) + end + end + end + + def full_path_namespace(names) + @full_path_namespace ||= Namespace.find_by_full_path(names) + end + + def visibility_level + @repo.private ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility + end +end + +class GithubRepos + def initialize(token, current_user, github_repo) + @token = token + @current_user = current_user + @github_repo = github_repo + end + + def choose_one! + return found_github_repo if @github_repo + + repos.each do |repo| + print "ID: #{repo[:id].to_s.bright} ".color(:green) + puts "- Name: #{repo[:full_name]}".color(:green) + end + + print 'ID? '.bright + + repos.find { |repo| repo[:id] == repo_id } + end + + def found_github_repo + repos.find { |repo| repo[:full_name] == @github_repo } + end + + def repo_id + @repo_id ||= STDIN.gets.chomp.to_i + end + + def repos + @repos ||= client.repos + end + + def client + @client ||= Gitlab::GithubImport::Client.new(@token, {}) + end +end + +namespace :import do + desc 'Import a GitHub project - Example: import:github[ToKeN,root,root/blah,my/github_repo] (optional my/github_repo)' + task :github, [:token, :gitlab_username, :project_path] => :environment do |_t, args| + abort 'Project path must be: namespace(s)/project_name'.color(:red) unless args.project_path.include?('/') + + GithubImport.run!(args.token, args.gitlab_username, args.project_path, args.extras) + end +end |