Merge branch 'master' into issue-discussions-refactor

* master: (481 commits)
  Make sure that we have author and committer
  disable file upload button while uploading
  Fix bar chart does not display label at hour 0
  Fixed activity not loading on project homepage
  Expose noteable_iid in Note
  Fix fly-out width when it has long items
  Add a test to show that threshold 40 would corrupt
  Add changelog entry
  Raise encoding confidence threshold to 50
  Fix the /projects/:id/repository/commits endpoint to handle dots in the ref name when the project full path contains a `/`
  Fix the /projects/:id/repository/tags endpoint to handle dots in the tag name when the project full path contains a `/`
  Add Italian translations of Pipeline Schedules
  Restrict InlineJavaScript for haml_lint to dev and test environment
  Incorporate Gitaly's CommitService.FindCommit RPC
  Move `deltas` and `diff_from_parents` logic to Gitlab::Git::Commit
  fix repo_edit_button_spec.js
  fix test failures in repo_preview_spec.js
  fix repo_loading_file_spec tests
  Refactor Gitlab::Git::Commit to include a repository
  use 100vh instead of flip flopping between the two - works on all suported browsers
  ...

9 files changed, 119 insertions, 9 deletions

diff --git a/config/application.rb b/config/application.rb
index f7145566262..47887bf8596 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -181,7 +181,11 @@ module Gitlab
         end
       end
 
+      # We add the MilestonesRoutingHelper because we know that this does not
+      # conflict with the methods defined in `project_url_helpers`, and we want
+      # these methods available in the same places.
       Gitlab::Routing.add_helpers(project_url_helpers)
+      Gitlab::Routing.add_helpers(MilestonesRoutingHelper)
     end
   end
 end
diff --git a/config/dependency_decisions.yml b/config/dependency_decisions.yml
index 59c7050a14d..ca5b941aebf 100644
--- a/config/dependency_decisions.yml
+++ b/config/dependency_decisions.yml
@@ -398,3 +398,9 @@
     :why: https://github.com/remy/undefsafe/blob/master/LICENSE
     :versions: []
     :when: 2017-04-10 06:30:00.002555000 Z
+- - :approve
+  - thunky
+  - :who: Mike Greiling
+    :why: https://github.com/mafintosh/thunky/blob/master/README.md#license
+    :versions: []
+    :when: 2017-08-07 05:56:09.907045000 Z
diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example
index 73a68c6da1b..e73db08fcac 100644
--- a/config/gitlab.yml.example
+++ b/config/gitlab.yml.example
@@ -506,6 +506,11 @@ production: &base
         path: /home/git/repositories/
         gitaly_address: unix:/home/git/gitlab/tmp/sockets/private/gitaly.socket # TCP connections are supported too (e.g. tcp://host:port)
         # gitaly_token: 'special token' # Optional: override global gitaly.token for this storage.
+        failure_count_threshold: 10 # number of failures before stopping attempts
+        failure_wait_time: 30 # Seconds after an access failure before allowing access again
+        failure_reset_time: 1800 # Time in seconds to expire failures
+        storage_timeout: 5 # Time in seconds to wait before aborting a storage access attempt
+
 
   ## Backup settings
   backup:
@@ -585,6 +590,12 @@ production: &base
     ip_whitelist:
       - 127.0.0.0/8
 
+    # Sidekiq exporter is webserver built in to Sidekiq to expose Prometheus metrics
+    sidekiq_exporter:
+    #  enabled: true
+    #  address: localhost
+    #  port: 3807
+
   #
   # 5. Extra customization
   # ==========================
@@ -638,6 +649,10 @@ test:
       default:
         path: tmp/tests/repositories/
         gitaly_address: unix:tmp/tests/gitaly/gitaly.socket
+      broken:
+        path: tmp/tests/non-existent-repositories
+        gitaly_address: unix:tmp/tests/gitaly/gitaly.socket
+
   gitaly:
     enabled: true
     token: secret
diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index 63f4c8c9e0a..2699173fc61 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -395,6 +395,10 @@ Settings.cron_jobs['remove_old_web_hook_logs_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['remove_old_web_hook_logs_worker']['cron'] ||= '40 0 * * *'
 Settings.cron_jobs['remove_old_web_hook_logs_worker']['job_class'] = 'RemoveOldWebHookLogsWorker'
 
+Settings.cron_jobs['stuck_merge_jobs_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['stuck_merge_jobs_worker']['cron'] ||= '0 */2 * * *'
+Settings.cron_jobs['stuck_merge_jobs_worker']['job_class'] = 'StuckMergeJobsWorker'
+
 #
 # GitLab Shell
 #
@@ -433,6 +437,17 @@ end
 Settings.repositories.storages.values.each do |storage|
   # Expand relative paths
   storage['path'] = Settings.absolute(storage['path'])
+  # Set failure defaults
+  storage['failure_count_threshold'] ||= 10
+  storage['failure_wait_time'] ||= 30
+  storage['failure_reset_time'] ||= 1800
+  storage['storage_timeout'] ||= 5
+  # Set turn strings into numbers
+  storage['failure_count_threshold'] = storage['failure_count_threshold'].to_i
+  storage['failure_wait_time'] = storage['failure_wait_time'].to_i
+  storage['failure_reset_time'] = storage['failure_reset_time'].to_i
+  # We might want to have a timeout shorter than 1 second.
+  storage['storage_timeout'] = storage['storage_timeout'].to_f
 end
 
 #
@@ -513,6 +528,10 @@ Settings.webpack.dev_server['port']    ||= 3808
 Settings['monitoring'] ||= Settingslogic.new({})
 Settings.monitoring['ip_whitelist'] ||= ['127.0.0.1/8']
 Settings.monitoring['unicorn_sampler_interval'] ||= 10
+Settings.monitoring['sidekiq_exporter'] ||= Settingslogic.new({})
+Settings.monitoring.sidekiq_exporter['enabled'] ||= false
+Settings.monitoring.sidekiq_exporter['address'] ||= 'localhost'
+Settings.monitoring.sidekiq_exporter['port'] ||= 3807
 
 #
 # Testing settings
diff --git a/config/initializers/6_validations.rb b/config/initializers/6_validations.rb
index 9e24f42d284..92ce4dd03cd 100644
--- a/config/initializers/6_validations.rb
+++ b/config/initializers/6_validations.rb
@@ -7,6 +7,13 @@ def find_parent_path(name, path)
   Gitlab.config.repositories.storages.detect do |n, rs|
     name != n && Pathname.new(rs['path']).realpath == parent
   end
+rescue Errno::EIO, Errno::ENOENT => e
+  warning = "WARNING: couldn't verify #{path} (#{name}). "\
+            "If this is an external storage, it might be offline."
+  message = "#{warning}\n#{e.message}"
+  Rails.logger.error("#{message}\n\t" + e.backtrace.join("\n\t"))
+
+  nil
 end
 
 def storage_validation_error(message)
@@ -29,6 +36,15 @@ def validate_storages_config
     if !repository_storage.is_a?(Hash) || repository_storage['path'].nil?
       storage_validation_error("#{name} is not a valid storage, because it has no `path` key. Refer to gitlab.yml.example for an updated example")
     end
+
+    %w(failure_count_threshold failure_wait_time failure_reset_time storage_timeout).each do |setting|
+      # Falling back to the defaults is fine!
+      next if repository_storage[setting].nil?
+
+      unless repository_storage[setting].to_f > 0
+        storage_validation_error("#{setting}, for storage `#{name}` needs to be greater than 0")
+      end
+    end
   end
 end
 
diff --git a/config/initializers/7_prometheus_metrics.rb b/config/initializers/7_prometheus_metrics.rb
index a2f8421f5d7..54c797e0714 100644
--- a/config/initializers/7_prometheus_metrics.rb
+++ b/config/initializers/7_prometheus_metrics.rb
@@ -10,3 +10,9 @@ Prometheus::Client.configure do |config|
     config.multiprocess_files_dir ||= Rails.root.join('tmp/prometheus_multiproc_dir')
   end
 end
+
+Sidekiq.configure_server do |config|
+  config.on(:startup) do
+    Gitlab::Metrics::SidekiqMetricsExporter.instance.start
+  end
+end
diff --git a/config/routes/admin.rb b/config/routes/admin.rb
index 5427bab93ce..c0748231813 100644
--- a/config/routes/admin.rb
+++ b/config/routes/admin.rb
@@ -67,7 +67,9 @@ namespace :admin do
   end
 
   resource :logs, only: [:show]
-  resource :health_check, controller: 'health_check', only: [:show]
+  resource :health_check, controller: 'health_check', only: [:show] do
+    post :reset_storage_health
+  end
   resource :background_jobs, controller: 'background_jobs', only: [:show]
   resource :system_info, controller: 'system_info', only: [:show]
   resources :requests_profiles, only: [:index, :show], param: :name, constraints: { name: /.+\.html/ }
diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml
index 7496bfa4fbb..83abc83c9f0 100644
--- a/config/sidekiq_queues.yml
+++ b/config/sidekiq_queues.yml
@@ -23,6 +23,8 @@
   - [update_merge_requests, 3]
   - [process_commit, 3]
   - [new_note, 2]
+  - [new_issue, 2]
+  - [new_merge_request, 2]
   - [build, 2]
   - [pipeline, 2]
   - [gitlab_shell, 2]
diff --git a/config/webpack.config.js b/config/webpack.config.js
index 30900db62e7..9625ffddb0a 100644
--- a/config/webpack.config.js
+++ b/config/webpack.config.js
@@ -3,7 +3,8 @@
 var fs = require('fs');
 var path = require('path');
 var webpack = require('webpack');
-var StatsPlugin = require('stats-webpack-plugin');
+var StatsWriterPlugin = require('webpack-stats-plugin').StatsWriterPlugin;
+var CopyWebpackPlugin = require('copy-webpack-plugin');
 var CompressionPlugin = require('compression-webpack-plugin');
 var NameAllModulesPlugin = require('name-all-modules-plugin');
 var BundleAnalyzerPlugin = require('webpack-bundle-analyzer').BundleAnalyzerPlugin;
@@ -61,10 +62,12 @@ var config = {
     pipelines_details:    './pipelines/pipeline_details_bundle.js',
     pipelines_times:      './pipelines/pipelines_times.js',
     profile:              './profile/profile_bundle.js',
+    project_import_gl:    './projects/project_import_gitlab_project.js',
     project_new:          './projects/project_new.js',
     prometheus_metrics:   './prometheus_metrics',
     protected_branches:   './protected_branches',
     protected_tags:       './protected_tags',
+    repo:                 './repo/index.js',
     sidebar:              './sidebar/sidebar_bundle.js',
     schedule_form:        './pipeline_schedules/pipeline_schedule_form_bundle.js',
     schedules_index:      './pipeline_schedules/pipeline_schedules_index_bundle.js',
@@ -122,18 +125,33 @@ var config = {
         test: /locale\/\w+\/(.*)\.js$/,
         loader: 'exports-loader?locales',
       },
-    ]
+      {
+        test: /monaco-editor\/\w+\/vs\/loader\.js$/,
+        use: [
+          { loader: 'exports-loader', options: 'l.global' },
+          { loader: 'imports-loader', options: 'l=>{},this=>l,AMDLoader=>this,module=>undefined' },
+        ],
+      }
+    ],
+
+    noParse: [/monaco-editor\/\w+\/vs\//],
   },
 
   plugins: [
     // manifest filename must match config.webpack.manifest_filename
     // webpack-rails only needs assetsByChunkName to function properly
-    new StatsPlugin('manifest.json', {
-      chunkModules: false,
-      source: false,
-      chunks: false,
-      modules: false,
-      assets: true
+    new StatsWriterPlugin({
+      filename: 'manifest.json',
+      transform: function(data, opts) {
+        var stats = opts.compiler.getStats().toJson({
+          chunkModules: false,
+          source: false,
+          chunks: false,
+          modules: false,
+          assets: true
+        });
+        return JSON.stringify(stats, null, 2);
+      }
     }),
 
     // prevent pikaday from including moment.js
@@ -182,6 +200,7 @@ var config = {
         'pdf_viewer',
         'pipelines',
         'pipelines_details',
+        'repo',
         'schedule_form',
         'schedules_index',
         'sidebar',
@@ -205,6 +224,26 @@ var config = {
     new webpack.optimize.CommonsChunkPlugin({
       names: ['main', 'locale', 'common', 'webpack_runtime'],
     }),
+
+    // copy pre-compiled vendor libraries verbatim
+    new CopyWebpackPlugin([
+      {
+        from: path.join(ROOT_PATH, `node_modules/monaco-editor/${IS_PRODUCTION ? 'min' : 'dev'}/vs`),
+        to: 'monaco-editor/vs',
+        transform: function(content, path) {
+          if (/\.js$/.test(path) && !/worker/i.test(path)) {
+            return (
+              '(function(){\n' +
+              'var define = this.define, require = this.require;\n' +
+              'window.define = define; window.require = require;\n' +
+              content +
+              '\n}.call(window.__monaco_context__ || (window.__monaco_context__ = {})));'
+            );
+          }
+          return content;
+        }
+      }
+    ]),
   ],
 
   resolve: {
@@ -253,6 +292,7 @@ if (IS_DEV_SERVER) {
   config.devServer = {
     host: DEV_SERVER_HOST,
     port: DEV_SERVER_PORT,
+    disableHostCheck: true,
     headers: { 'Access-Control-Allow-Origin': '*' },
     stats: 'errors-only',
     hot: DEV_SERVER_LIVERELOAD,