diff options
| -rw-r--r-- | .codeclimate.yml | 1 | ||||
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | .rubocop.yml | 1 | ||||
| -rw-r--r-- | app/services/system_hooks_service.rb | 2 | ||||
| -rw-r--r-- | app/workers/all_queues.yml | 1 | ||||
| -rw-r--r-- | app/workers/plugin_worker.rb | 15 | ||||
| -rw-r--r-- | changelogs/unreleased/dz-system-hooks-plugins.yml | 5 | ||||
| -rw-r--r-- | config/sidekiq_queues.yml | 1 | ||||
| -rw-r--r-- | doc/administration/plugins.md | 66 | ||||
| -rw-r--r-- | lib/gitlab/plugin.rb | 26 | ||||
| -rw-r--r-- | lib/gitlab/plugin_logger.rb | 7 | ||||
| -rw-r--r-- | lib/tasks/plugins.rake | 16 | ||||
| -rwxr-xr-x | plugins/examples/save_to_file.clj | 3 | ||||
| -rwxr-xr-x | plugins/examples/save_to_file.rb | 3 | ||||
| -rw-r--r-- | spec/lib/gitlab/plugin_spec.rb | 68 | ||||
| -rw-r--r-- | spec/workers/plugin_worker_spec.rb | 25 |
16 files changed, 241 insertions, 0 deletions
diff --git a/.codeclimate.yml b/.codeclimate.yml index 333a70f71d3..216ecf43beb 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -45,3 +45,4 @@ exclude_paths: - log/ - backups/ - coverage-javascript/ +- plugins/ diff --git a/.gitignore b/.gitignore index 2004c2a09b4..fa39ae01ff0 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,4 @@ eslint-report.html /locale/**/LC_MESSAGES /locale/**/*.time_stamp /.rspec +/plugins/* diff --git a/.rubocop.yml b/.rubocop.yml index 24edb641657..293f61fb725 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -17,6 +17,7 @@ AllCops: - 'bin/**/*' - 'generator_templates/**/*' - 'builds/**/*' + - 'plugins/**/*' CacheRootDirectory: tmp # This cop checks whether some constant value isn't a diff --git a/app/services/system_hooks_service.rb b/app/services/system_hooks_service.rb index a6b7a6e1416..af8c02a10b7 100644 --- a/app/services/system_hooks_service.rb +++ b/app/services/system_hooks_service.rb @@ -11,6 +11,8 @@ class SystemHooksService SystemHook.hooks_for(hooks_scope).find_each do |hook| hook.async_execute(data, 'system_hooks') end + + Gitlab::Plugin.execute_all_async(data) end private diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 28a5e5da037..a9415410f8a 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -84,6 +84,7 @@ - new_note - pages - pages_domain_verification +- plugin - post_receive - process_commit - project_cache diff --git a/app/workers/plugin_worker.rb b/app/workers/plugin_worker.rb new file mode 100644 index 00000000000..bfcc683d99a --- /dev/null +++ b/app/workers/plugin_worker.rb @@ -0,0 +1,15 @@ +class PluginWorker + include ApplicationWorker + + sidekiq_options retry: false + + def perform(file_name, data) + success, message = Gitlab::Plugin.execute(file_name, data) + + unless success + Gitlab::PluginLogger.error("Plugin Error => #{file_name}: #{message}") + end + + true + end +end diff --git a/changelogs/unreleased/dz-system-hooks-plugins.yml b/changelogs/unreleased/dz-system-hooks-plugins.yml new file mode 100644 index 00000000000..e6eb1dfb03b --- /dev/null +++ b/changelogs/unreleased/dz-system-hooks-plugins.yml @@ -0,0 +1,5 @@ +--- +title: Add ability to use external plugins as an alternative to system hooks +merge_request: 17003 +author: +type: added diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index f037e3d1221..4845dc28a4a 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -68,3 +68,4 @@ - [project_migrate_hashed_storage, 1] - [storage_migrator, 1] - [pages_domain_verification, 1] + - [plugin, 1] diff --git a/doc/administration/plugins.md b/doc/administration/plugins.md new file mode 100644 index 00000000000..c91ac3012b9 --- /dev/null +++ b/doc/administration/plugins.md @@ -0,0 +1,66 @@ +# Plugins + +**Note:** Plugins must be configured on the filesystem of the GitLab +server. Only GitLab server administrators will be able to complete these tasks. +Please explore [system hooks] or [webhooks] as an option if you do not +have filesystem access. + +Introduced in GitLab 10.6. + +A plugin will run on each event so it's up to you to filter events or projects within a plugin code. You can have as many plugins as you want. Each plugin will be triggered by GitLab asynchronously in case of an event. For a list of events please see [system hooks] documentation. + +## Setup + +Plugins must be placed directly into `plugins` directory, subdirectories will be ignored. +There is an `example` directory inside `plugins` where you can find some basic examples. + +Follow the steps below to set up a custom hook: + +1. On the GitLab server, navigate to the project's plugin directory. + For an installation from source the path is usually + `/home/git/gitlab/plugins/`. For Omnibus installs the path is + usually `/opt/gitlab/embedded/service/gitlab-rails/plugins`. +1. Inside the `plugins` directory, create a file with a name of your choice, but without spaces or special characters. +1. Make the hook file executable and make sure it's owned by the git user. +1. Write the code to make the plugin function as expected. Plugin can be + in any language. Ensure the 'shebang' at the top properly reflects the language + type. For example, if the script is in Ruby the shebang will probably be + `#!/usr/bin/env ruby`. +1. The data to the plugin will be provided as JSON on STDIN. It will be exactly same as one for [system hooks] + +That's it! Assuming the plugin code is properly implemented the hook will fire +as appropriate. Plugins file list is updated for each event. There is no need to restart GitLab to apply a new plugin. + +If a plugin executes with non-zero exit code or GitLab fails to execute it, a +message will be logged to `plugin.log`. + +## Validation + +Writing own plugin can be tricky and its easier if you can check it without altering the system. +We provided a rake task you can use with staging environment to test your plugin before using it in production. +The rake task will use a sample data and execute each of plugins. By output you should be able to determine if +system sees your plugin and if it was executed without errors. + +```bash +# Omnibus installations +sudo gitlab-rake plugins:validate + +# Installations from source +bundle exec rake plugins:validate RAILS_ENV=production +``` + +Example of output can be next: + +``` +-> bundle exec rake plugins:validate RAILS_ENV=production +Validating plugins from /plugins directory +* /home/git/gitlab/plugins/save_to_file.clj succeed (zero exit code) +* /home/git/gitlab/plugins/save_to_file.rb failure (non-zero exit code) +``` + +[hooks]: https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks#Server-Side-Hooks +[system hooks]: ../system_hooks/system_hooks.md +[webhooks]: ../user/project/integrations/webhooks.md +[5073]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/5073 +[93]: https://gitlab.com/gitlab-org/gitlab-shell/merge_requests/93 + diff --git a/lib/gitlab/plugin.rb b/lib/gitlab/plugin.rb new file mode 100644 index 00000000000..0d1cb16b378 --- /dev/null +++ b/lib/gitlab/plugin.rb @@ -0,0 +1,26 @@ +module Gitlab + module Plugin + def self.files + Dir.glob(Rails.root.join('plugins/*')).select do |entry| + File.file?(entry) + end + end + + def self.execute_all_async(data) + args = files.map { |file| [file, data] } + + PluginWorker.bulk_perform_async(args) + end + + def self.execute(file, data) + result = Gitlab::Popen.popen_with_detail([file]) do |stdin| + stdin.write(data.to_json) + end + + exit_status = result.status&.exitstatus + [exit_status.zero?, result.stderr] + rescue => e + [false, e.message] + end + end +end diff --git a/lib/gitlab/plugin_logger.rb b/lib/gitlab/plugin_logger.rb new file mode 100644 index 00000000000..c4f6ec3e21d --- /dev/null +++ b/lib/gitlab/plugin_logger.rb @@ -0,0 +1,7 @@ +module Gitlab + class PluginLogger < Gitlab::Logger + def self.file_name_noext + 'plugin' + end + end +end diff --git a/lib/tasks/plugins.rake b/lib/tasks/plugins.rake new file mode 100644 index 00000000000..e73dd7e68df --- /dev/null +++ b/lib/tasks/plugins.rake @@ -0,0 +1,16 @@ +namespace :plugins do + desc 'Validate existing plugins' + task validate: :environment do + puts 'Validating plugins from /plugins directory' + + Gitlab::Plugin.files.each do |file| + success, message = Gitlab::Plugin.execute(file, Gitlab::DataBuilder::Push::SAMPLE_DATA) + + if success + puts "* #{file} succeed (zero exit code)." + else + puts "* #{file} failure (non-zero exit code). #{message}" + end + end + end +end diff --git a/plugins/examples/save_to_file.clj b/plugins/examples/save_to_file.clj new file mode 100755 index 00000000000..a59d83749d3 --- /dev/null +++ b/plugins/examples/save_to_file.clj @@ -0,0 +1,3 @@ +#!/usr/bin/env clojure +(let [in (slurp *in*)] + (spit "/tmp/clj-data.txt" in)) diff --git a/plugins/examples/save_to_file.rb b/plugins/examples/save_to_file.rb new file mode 100755 index 00000000000..61b0df9bfd6 --- /dev/null +++ b/plugins/examples/save_to_file.rb @@ -0,0 +1,3 @@ +#!/usr/bin/env ruby +x = STDIN.read +File.write('/tmp/rb-data.txt', x) diff --git a/spec/lib/gitlab/plugin_spec.rb b/spec/lib/gitlab/plugin_spec.rb new file mode 100644 index 00000000000..33dd4f79130 --- /dev/null +++ b/spec/lib/gitlab/plugin_spec.rb @@ -0,0 +1,68 @@ +require 'spec_helper' + +describe Gitlab::Plugin do + describe '.execute' do + let(:data) { Gitlab::DataBuilder::Push::SAMPLE_DATA } + let(:plugin) { Rails.root.join('plugins', 'test.rb') } + let(:tmp_file) { Tempfile.new('plugin-dump') } + let(:result) { described_class.execute(plugin.to_s, data) } + let(:success) { result.first } + let(:message) { result.last } + + let(:plugin_source) do + <<~EOS + #!/usr/bin/env ruby + x = STDIN.read + File.write('#{tmp_file.path}', x) + EOS + end + + before do + File.write(plugin, plugin_source) + end + + after do + FileUtils.rm(plugin) + end + + context 'successful execution' do + before do + File.chmod(0o777, plugin) + end + + after do + tmp_file.close! + end + + it { expect(success).to be true } + it { expect(message).to be_empty } + + it 'ensures plugin received data via stdin' do + result + + expect(File.read(tmp_file.path)).to eq(data.to_json) + end + end + + context 'non-executable' do + it { expect(success).to be false } + it { expect(message).to include('Permission denied') } + end + + context 'non-zero exit' do + let(:plugin_source) do + <<~EOS + #!/usr/bin/env ruby + exit 1 + EOS + end + + before do + File.chmod(0o777, plugin) + end + + it { expect(success).to be false } + it { expect(message).to be_empty } + end + end +end diff --git a/spec/workers/plugin_worker_spec.rb b/spec/workers/plugin_worker_spec.rb new file mode 100644 index 00000000000..9238a8199bc --- /dev/null +++ b/spec/workers/plugin_worker_spec.rb @@ -0,0 +1,25 @@ +require 'spec_helper' + +describe PluginWorker do + include RepoHelpers + + let(:filename) { 'my_plugin.rb' } + let(:data) { { 'event_name' => 'project_create' } } + + subject { described_class.new } + + describe '#perform' do + it 'executes Gitlab::Plugin with expected values' do + allow(Gitlab::Plugin).to receive(:execute).with(filename, data).and_return([true, '']) + + expect(subject.perform(filename, data)).to be_truthy + end + + it 'logs message in case of plugin execution failure' do + allow(Gitlab::Plugin).to receive(:execute).with(filename, data).and_return([false, 'permission denied']) + + expect(Gitlab::PluginLogger).to receive(:error) + expect(subject.perform(filename, data)).to be_truthy + end + end +end |