diff options
author | Robert Speicher <robert@gitlab.com> | 2015-08-22 00:00:08 +0000 |
---|---|---|
committer | Robert Speicher <robert@gitlab.com> | 2015-08-22 00:00:08 +0000 |
commit | f0bdf7f8102405f272a42c04c1fa70dae7365854 (patch) | |
tree | fa30c63d0f739f21b5c8816ee4b34338a55b4549 /lib | |
parent | 0daa21ed8cf3fe917645b66baa27917923bfdd8f (diff) | |
parent | 15fc7bd6139f0b429c05c055b4cfab561c926e08 (diff) | |
download | gitlab-ce-f0bdf7f8102405f272a42c04c1fa70dae7365854.tar.gz |
Merge branch 'reply-by-email' into 'master'
Reply by email
Fixes #1360.
It's far from done, but _it works_.
See merge request !1173
Diffstat (limited to 'lib')
-rw-r--r-- | lib/gitlab/email/attachment_uploader.rb | 35 | ||||
-rw-r--r-- | lib/gitlab/email/receiver.rb | 106 | ||||
-rw-r--r-- | lib/gitlab/email/reply_parser.rb | 79 | ||||
-rw-r--r-- | lib/gitlab/reply_by_email.rb | 49 | ||||
-rwxr-xr-x | lib/support/init.d/gitlab | 105 | ||||
-rwxr-xr-x | lib/support/init.d/gitlab.default.example | 9 | ||||
-rw-r--r-- | lib/tasks/gitlab/check.rake | 169 |
7 files changed, 528 insertions, 24 deletions
diff --git a/lib/gitlab/email/attachment_uploader.rb b/lib/gitlab/email/attachment_uploader.rb new file mode 100644 index 00000000000..32cece8316b --- /dev/null +++ b/lib/gitlab/email/attachment_uploader.rb @@ -0,0 +1,35 @@ +module Gitlab + module Email + class AttachmentUploader + attr_accessor :message + + def initialize(message) + @message = message + end + + def execute(project) + attachments = [] + + message.attachments.each do |attachment| + tmp = Tempfile.new("gitlab-email-attachment") + begin + File.open(tmp.path, "w+b") { |f| f.write attachment.body.decoded } + + file = { + tempfile: tmp, + filename: attachment.filename, + content_type: attachment.content_type + } + + link = ::Projects::UploadService.new(project, file).execute + attachments << link if link + ensure + tmp.close! + end + end + + attachments + end + end + end +end diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb new file mode 100644 index 00000000000..355fbd27898 --- /dev/null +++ b/lib/gitlab/email/receiver.rb @@ -0,0 +1,106 @@ +# Inspired in great part by Discourse's Email::Receiver +module Gitlab + module Email + class Receiver + class ProcessingError < StandardError; end + class EmailUnparsableError < ProcessingError; end + class SentNotificationNotFoundError < ProcessingError; end + class EmptyEmailError < ProcessingError; end + class AutoGeneratedEmailError < ProcessingError; end + class UserNotFoundError < ProcessingError; end + class UserBlockedError < ProcessingError; end + class UserNotAuthorizedError < ProcessingError; end + class NoteableNotFoundError < ProcessingError; end + class InvalidNoteError < ProcessingError; end + + def initialize(raw) + @raw = raw + end + + def execute + raise EmptyEmailError if @raw.blank? + + raise SentNotificationNotFoundError unless sent_notification + + raise AutoGeneratedEmailError if message.header.to_s =~ /auto-(generated|replied)/ + + author = sent_notification.recipient + + raise UserNotFoundError unless author + + raise UserBlockedError if author.blocked? + + project = sent_notification.project + + raise UserNotAuthorizedError unless project && author.can?(:create_note, project) + + raise NoteableNotFoundError unless sent_notification.noteable + + reply = ReplyParser.new(message).execute.strip + + raise EmptyEmailError if reply.blank? + + reply = add_attachments(reply) + + note = create_note(reply) + + unless note.persisted? + message = "The comment could not be created for the following reasons:" + note.errors.full_messages.each do |error| + message << "\n\n- #{error}" + end + + raise InvalidNoteError, message + end + end + + private + + def message + @message ||= Mail::Message.new(@raw) + rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError => e + raise EmailUnparsableError, e + end + + def reply_key + reply_key = nil + message.to.each do |address| + reply_key = Gitlab::ReplyByEmail.reply_key_from_address(address) + break if reply_key + end + + reply_key + end + + def sent_notification + return nil unless reply_key + + SentNotification.for(reply_key) + end + + def add_attachments(reply) + attachments = Email::AttachmentUploader.new(message).execute(sent_notification.project) + + attachments.each do |link| + text = "[#{link[:alt]}](#{link[:url]})" + text.prepend("!") if link[:is_image] + + reply << "\n\n#{text}" + end + + reply + end + + def create_note(reply) + Notes::CreateService.new( + sent_notification.project, + sent_notification.recipient, + note: reply, + noteable_type: sent_notification.noteable_type, + noteable_id: sent_notification.noteable_id, + commit_id: sent_notification.commit_id + ).execute + end + end + end +end diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb new file mode 100644 index 00000000000..6ed36b51f12 --- /dev/null +++ b/lib/gitlab/email/reply_parser.rb @@ -0,0 +1,79 @@ +# Inspired in great part by Discourse's Email::Receiver +module Gitlab + module Email + class ReplyParser + attr_accessor :message + + def initialize(message) + @message = message + end + + def execute + body = select_body(message) + + encoding = body.encoding + + body = discourse_email_trimmer(body) + + body = EmailReplyParser.parse_reply(body) + + body.force_encoding(encoding).encode("UTF-8") + end + + private + + def select_body(message) + text = message.text_part if message.multipart? + text ||= message if message.content_type !~ /text\/html/ + + return "" unless text + + text = fix_charset(text) + + # Certain trigger phrases that means we didn't parse correctly + if text =~ /(Content\-Type\:|multipart\/alternative|text\/plain)/ + return "" + end + + text + end + + # Force encoding to UTF-8 on a Mail::Message or Mail::Part + def fix_charset(object) + return nil if object.nil? + + if object.charset + object.body.decoded.force_encoding(object.charset.gsub(/utf8/i, "UTF-8")).encode("UTF-8").to_s + else + object.body.to_s + end + rescue + nil + end + + REPLYING_HEADER_LABELS = %w(From Sent To Subject Reply To Cc Bcc Date) + REPLYING_HEADER_REGEX = Regexp.union(REPLYING_HEADER_LABELS.map { |label| "#{label}:" }) + + def discourse_email_trimmer(body) + lines = body.scrub.lines.to_a + range_end = 0 + + lines.each_with_index do |l, idx| + # This one might be controversial but so many reply lines have years, times and end with a colon. + # Let's try it and see how well it works. + break if (l =~ /\d{4}/ && l =~ /\d:\d\d/ && l =~ /\:$/) || + (l =~ /On \w+ \d+,? \d+,?.*wrote:/) + + # Headers on subsequent lines + break if (0..2).all? { |off| lines[idx+off] =~ REPLYING_HEADER_REGEX } + # Headers on the same line + break if REPLYING_HEADER_LABELS.count { |label| l.include?(label) } >= 3 + + range_end = idx + end + + lines[0..range_end].join.strip + end + end + end +end diff --git a/lib/gitlab/reply_by_email.rb b/lib/gitlab/reply_by_email.rb new file mode 100644 index 00000000000..c3fe6778f06 --- /dev/null +++ b/lib/gitlab/reply_by_email.rb @@ -0,0 +1,49 @@ +module Gitlab + module ReplyByEmail + class << self + def enabled? + config.enabled && address_formatted_correctly? + end + + def address_formatted_correctly? + config.address && + config.address.include?("%{reply_key}") + end + + def reply_key + return nil unless enabled? + + SecureRandom.hex(16) + end + + def reply_address(reply_key) + config.address.gsub('%{reply_key}', reply_key) + end + + def reply_key_from_address(address) + regex = address_regex + return unless regex + + match = address.match(regex) + return unless match + + match[1] + end + + private + + def config + Gitlab.config.reply_by_email + end + + def address_regex + wildcard_address = config.address + return nil unless wildcard_address + + regex = Regexp.escape(wildcard_address) + regex = regex.gsub(Regexp.escape('%{reply_key}'), "(.+)") + Regexp.new(regex).freeze + end + end + end +end diff --git a/lib/support/init.d/gitlab b/lib/support/init.d/gitlab index a3455728a94..41a2f254db6 100755 --- a/lib/support/init.d/gitlab +++ b/lib/support/init.d/gitlab @@ -35,6 +35,8 @@ pid_path="$app_root/tmp/pids" socket_path="$app_root/tmp/sockets" web_server_pid_path="$pid_path/unicorn.pid" sidekiq_pid_path="$pid_path/sidekiq.pid" +mail_room_enabled=false +mail_room_pid_path="$pid_path/mail_room.pid" shell_path="/bin/bash" # Read configuration variable file if it is present @@ -70,13 +72,20 @@ check_pids(){ else spid=0 fi + if [ "$mail_room_enabled" = true ]; then + if [ -f "$mail_room_pid_path" ]; then + mpid=$(cat "$mail_room_pid_path") + else + mpid=0 + fi + fi } ## Called when we have started the two processes and are waiting for their pid files. wait_for_pids(){ # We are sleeping a bit here mostly because sidekiq is slow at writing it's pid i=0; - while [ ! -f $web_server_pid_path -o ! -f $sidekiq_pid_path ]; do + while [ ! -f $web_server_pid_path ] || [ ! -f $sidekiq_pid_path ] || { [ "$mail_room_enabled" = true ] && [ ! -f $mail_room_pid_path ]; }; do sleep 0.1; i=$((i+1)) if [ $((i%10)) = 0 ]; then @@ -111,7 +120,15 @@ check_status(){ else sidekiq_status="-1" fi - if [ $web_status = 0 -a $sidekiq_status = 0 ]; then + if [ "$mail_room_enabled" = true ]; then + if [ $mpid -ne 0 ]; then + kill -0 "$mpid" 2>/dev/null + mail_room_status="$?" + else + mail_room_status="-1" + fi + fi + if [ $web_status = 0 ] && [ $sidekiq_status = 0 ] && { [ "$mail_room_enabled" != true ] || [ $mail_room_status = 0 ]; }; then gitlab_status=0 else # http://refspecs.linuxbase.org/LSB_4.1.0/LSB-Core-generic/LSB-Core-generic/iniscrptact.html @@ -125,26 +142,33 @@ check_stale_pids(){ check_status # If there is a pid it is something else than 0, the service is running if # *_status is == 0. - if [ "$wpid" != "0" -a "$web_status" != "0" ]; then + if [ "$wpid" != "0" ] && [ "$web_status" != "0" ]; then echo "Removing stale Unicorn web server pid. This is most likely caused by the web server crashing the last time it ran." if ! rm "$web_server_pid_path"; then echo "Unable to remove stale pid, exiting." exit 1 fi fi - if [ "$spid" != "0" -a "$sidekiq_status" != "0" ]; then + if [ "$spid" != "0" ] && [ "$sidekiq_status" != "0" ]; then echo "Removing stale Sidekiq job dispatcher pid. This is most likely caused by Sidekiq crashing the last time it ran." if ! rm "$sidekiq_pid_path"; then echo "Unable to remove stale pid, exiting" exit 1 fi fi + if [ "$mail_room_enabled" = true ] && [ "$mpid" != "0" ] && [ "$mail_room_status" != "0" ]; then + echo "Removing stale MailRoom job dispatcher pid. This is most likely caused by MailRoom crashing the last time it ran." + if ! rm "$mail_room_pid_path"; then + echo "Unable to remove stale pid, exiting" + exit 1 + fi + fi } ## If no parts of the service is running, bail out. exit_if_not_running(){ check_stale_pids - if [ "$web_status" != "0" -a "$sidekiq_status" != "0" ]; then + if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then echo "GitLab is not running." exit fi @@ -154,12 +178,14 @@ exit_if_not_running(){ start_gitlab() { check_stale_pids - if [ "$web_status" != "0" -a "$sidekiq_status" != "0" ]; then - echo -n "Starting both the GitLab Unicorn and Sidekiq" - elif [ "$web_status" != "0" ]; then - echo -n "Starting GitLab Unicorn" - elif [ "$sidekiq_status" != "0" ]; then - echo -n "Starting GitLab Sidekiq" + if [ "$web_status" != "0" ]; then + echo "Starting GitLab Unicorn" + fi + if [ "$sidekiq_status" != "0" ]; then + echo "Starting GitLab Sidekiq" + fi + if [ "$mail_room_enabled" = true ] && [ "$mail_room_status" != "0" ]; then + echo "Starting GitLab MailRoom" fi # Then check if the service is running. If it is: don't start again. @@ -179,22 +205,33 @@ start_gitlab() { RAILS_ENV=$RAILS_ENV bin/background_jobs start & fi + if [ "$mail_room_enabled" = true ]; then + # If MailRoom is already running, don't start it again. + if [ "$mail_room_status" = "0" ]; then + echo "The MailRoom email processor is already running with pid $mpid, not restarting" + else + RAILS_ENV=$RAILS_ENV bin/mail_room start & + fi + fi + # Wait for the pids to be planted wait_for_pids # Finally check the status to tell wether or not GitLab is running print_status } -## Asks the Unicorn and the Sidekiq if they would be so kind as to stop, if not kills them. +## Asks Unicorn, Sidekiq and MailRoom if they would be so kind as to stop, if not kills them. stop_gitlab() { exit_if_not_running - if [ "$web_status" = "0" -a "$sidekiq_status" = "0" ]; then - echo -n "Shutting down both Unicorn and Sidekiq" - elif [ "$web_status" = "0" ]; then - echo -n "Shutting down Unicorn" - elif [ "$sidekiq_status" = "0" ]; then - echo -n "Shutting down Sidekiq" + if [ "$web_status" = "0" ]; then + echo "Shutting down GitLab Unicorn" + fi + if [ "$sidekiq_status" = "0" ]; then + echo "Shutting down GitLab Sidekiq" + fi + if [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; then + echo "Shutting down GitLab MailRoom" fi # If the Unicorn web server is running, tell it to stop; @@ -205,13 +242,17 @@ stop_gitlab() { if [ "$sidekiq_status" = "0" ]; then RAILS_ENV=$RAILS_ENV bin/background_jobs stop fi + # And do the same thing for the MailRoom. + if [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; then + RAILS_ENV=$RAILS_ENV bin/mail_room stop + fi # If something needs to be stopped, lets wait for it to stop. Never use SIGKILL in a script. - while [ "$web_status" = "0" -o "$sidekiq_status" = "0" ]; do + while [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; }; do sleep 1 check_status printf "." - if [ "$web_status" != "0" -a "$sidekiq_status" != "0" ]; then + if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then printf "\n" break fi @@ -220,7 +261,10 @@ stop_gitlab() { sleep 1 # Cleaning up unused pids rm "$web_server_pid_path" 2>/dev/null - # rm "$sidekiq_pid_path" # Sidekiq seems to be cleaning up it's own pid. + # rm "$sidekiq_pid_path" 2>/dev/null # Sidekiq seems to be cleaning up it's own pid. + if [ "$mail_room_enabled" = true ]; then + rm "$mail_room_pid_path" 2>/dev/null + fi print_status } @@ -228,7 +272,7 @@ stop_gitlab() { ## Prints the status of GitLab and it's components. print_status() { check_status - if [ "$web_status" != "0" -a "$sidekiq_status" != "0" ]; then + if [ "$web_status" != "0" ] && [ "$sidekiq_status" != "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" != "0" ]; }; then echo "GitLab is not running." return fi @@ -242,7 +286,14 @@ print_status() { else printf "The GitLab Sidekiq job dispatcher is \033[31mnot running\033[0m.\n" fi - if [ "$web_status" = "0" -a "$sidekiq_status" = "0" ]; then + if [ "$mail_room_enabled" = true ]; then + if [ "$mail_room_status" = "0" ]; then + echo "The GitLab MailRoom email processor with pid $spid is running." + else + printf "The GitLab MailRoom email processor is \033[31mnot running\033[0m.\n" + fi + fi + if [ "$web_status" = "0" ] && [ "$sidekiq_status" = "0" ] && { [ "$mail_room_enabled" != true ] || [ "$mail_room_status" = "0" ]; }; then printf "GitLab and all its components are \033[32mup and running\033[0m.\n" fi } @@ -257,9 +308,15 @@ reload_gitlab(){ printf "Reloading GitLab Unicorn configuration... " RAILS_ENV=$RAILS_ENV bin/web reload echo "Done." + echo "Restarting GitLab Sidekiq since it isn't capable of reloading its config..." RAILS_ENV=$RAILS_ENV bin/background_jobs restart + if [ "$mail_room_enabled" != true ]; then + echo "Restarting GitLab MailRoom since it isn't capable of reloading its config..." + RAILS_ENV=$RAILS_ENV bin/mail_room restart + fi + wait_for_pids print_status } @@ -267,7 +324,7 @@ reload_gitlab(){ ## Restarts Sidekiq and Unicorn. restart_gitlab(){ check_status - if [ "$web_status" = "0" -o "$sidekiq_status" = "0" ]; then + if [ "$web_status" = "0" ] || [ "$sidekiq_status" = "0" ] || { [ "$mail_room_enabled" = true ] && [ "$mail_room_status" = "0" ]; }; then stop_gitlab fi start_gitlab diff --git a/lib/support/init.d/gitlab.default.example b/lib/support/init.d/gitlab.default.example index cf7f4198cbf..fd70cb7cc74 100755 --- a/lib/support/init.d/gitlab.default.example +++ b/lib/support/init.d/gitlab.default.example @@ -30,6 +30,15 @@ web_server_pid_path="$pid_path/unicorn.pid" # The default is "$pid_path/sidekiq.pid" sidekiq_pid_path="$pid_path/sidekiq.pid" +# mail_room_enabled specifies whether mail_room, which is used to process incoming email, is enabled. +# This is required for the Reply by email feature. +# The default is "false" +mail_room_enabled=false + +# mail_room_pid_path defines the path in which to create the pid file for mail_room +# The default is "$pid_path/mail_room.pid" +mail_room_pid_path="$pid_path/mail_room.pid" + # shell_path defines the path of shell for "$app_user" in case you are using # shell other than "bash" # The default is "/bin/bash" diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 60aa50e8751..2b9688c1b40 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -2,6 +2,7 @@ namespace :gitlab do desc "GitLab | Check the configuration of GitLab and its environment" task check: %w{gitlab:gitlab_shell:check gitlab:sidekiq:check + gitlab:reply_by_email:check gitlab:ldap:check gitlab:app:check} @@ -629,6 +630,174 @@ namespace :gitlab do end end + + namespace :reply_by_email do + desc "GitLab | Check the configuration of Reply by email" + task check: :environment do + warn_user_is_not_gitlab + start_checking "Reply by email" + + if Gitlab.config.reply_by_email.enabled + check_address_formatted_correctly + check_mail_room_config_exists + check_imap_authentication + + if Rails.env.production? + check_initd_configured_correctly + check_mail_room_running + else + check_foreman_configured_correctly + end + else + puts 'Reply by email is disabled in config/gitlab.yml' + end + + finished_checking "Reply by email" + end + + + # Checks + ######################## + + def check_address_formatted_correctly + print "Address formatted correctly? ... " + + if Gitlab::ReplyByEmail.address_formatted_correctly? + puts "yes".green + else + puts "no".red + try_fixing_it( + "Make sure that the address in config/gitlab.yml includes the '%{reply_key}' placeholder." + ) + fix_and_rerun + end + end + + def check_initd_configured_correctly + print "Init.d configured correctly? ... " + + path = "/etc/default/gitlab" + + if File.exist?(path) && File.read(path).include?("mail_room_enabled=true") + puts "yes".green + else + puts "no".red + try_fixing_it( + "Enable mail_room in the init.d configuration." + ) + for_more_information( + "doc/reply_by_email/README.md" + ) + fix_and_rerun + end + end + + def check_foreman_configured_correctly + print "Foreman configured correctly? ... " + + path = Rails.root.join("Procfile") + + if File.exist?(path) && File.read(path) =~ /^mail_room:/ + puts "yes".green + else + puts "no".red + try_fixing_it( + "Enable mail_room in your Procfile." + ) + for_more_information( + "doc/reply_by_email/README.md" + ) + fix_and_rerun + end + end + + def check_mail_room_running + print "MailRoom running? ... " + + path = "/etc/default/gitlab" + + unless File.exist?(path) && File.read(path).include?("mail_room_enabled=true") + puts "can't check because of previous errors".magenta + return + end + + if mail_room_running? + puts "yes".green + else + puts "no".red + try_fixing_it( + sudo_gitlab("RAILS_ENV=production bin/mail_room start") + ) + for_more_information( + see_installation_guide_section("Install Init Script"), + "see log/mail_room.log for possible errors" + ) + fix_and_rerun + end + end + + def check_mail_room_config_exists + print "MailRoom config exists? ... " + + mail_room_config_file = Rails.root.join("config", "mail_room.yml") + + if File.exists?(mail_room_config_file) + puts "yes".green + else + puts "no".red + try_fixing_it( + "Copy config/mail_room.yml.example to config/mail_room.yml", + "Check that the information in config/mail_room.yml is correct" + ) + for_more_information( + "doc/reply_by_email/README.md" + ) + fix_and_rerun + end + end + + def check_imap_authentication + print "IMAP server credentials are correct? ... " + + mail_room_config_file = Rails.root.join("config", "mail_room.yml") + + unless File.exists?(mail_room_config_file) + puts "can't check because of previous errors".magenta + return + end + + config = YAML.load_file(mail_room_config_file)[:mailboxes].first rescue nil + + if config + begin + imap = Net::IMAP.new(config[:host], port: config[:port], ssl: config[:ssl]) + imap.login(config[:email], config[:password]) + connected = true + rescue + connected = false + end + end + + if connected + puts "yes".green + else + puts "no".red + try_fixing_it( + "Check that the information in config/mail_room.yml is correct" + ) + for_more_information( + "doc/reply_by_email/README.md" + ) + fix_and_rerun + end + end + + def mail_room_running? + ps_ux, _ = Gitlab::Popen.popen(%W(ps ux)) + ps_ux.include?("mail_room") + end + end + namespace :ldap do task :check, [:limit] => :environment do |t, args| # Only show up to 100 results because LDAP directories can be very big. |