diff options
59 files changed, 846 insertions, 519 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index 630aef6751f..8c64e68967e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,20 @@ documentation](doc/development/changelog.md) for instructions on adding your own entry. +## 10.5.5 (2018-03-15) + +### Fixed (3 changes) + +- Fix missing uploads after group transfer. !17658 +- Fix code and wiki search results when filename is non-ASCII. +- Remove double caching of Repository#empty?. + +### Performance (2 changes) + +- Adding missing indexes on taggings table. +- Add index on section_name_id on ci_build_trace_sections table. + + ## 10.5.4 (2018-03-08) ### Fixed (11 changes) diff --git a/GITALY_SERVER_VERSION b/GITALY_SERVER_VERSION index 5aee1345c52..8f63f4f9a10 100644 --- a/GITALY_SERVER_VERSION +++ b/GITALY_SERVER_VERSION @@ -1 +1 @@ -0.89.0 +0.91.0 @@ -164,7 +164,7 @@ gem 'sidekiq-limit_fetch', '~> 3.4', require: false gem 'rufus-scheduler', '~> 3.4' # HTTP requests -gem 'httparty', '~> 0.15.6' +gem 'httparty', '~> 0.13.3' # Colored output to console gem 'rainbow', '~> 2.2' @@ -359,7 +359,7 @@ group :development, :test do gem 'benchmark-ips', '~> 2.3.0', require: false gem 'license_finder', '~> 3.1', require: false - gem 'knapsack', '~> 1.11.0' + gem 'knapsack', '~> 1.16' gem 'activerecord_sane_schema_dumper', '0.2' @@ -420,9 +420,9 @@ gem 'google-protobuf', '= 3.5.1' gem 'toml-rb', '~> 1.0.0', require: false # Feature toggles -gem 'flipper', '~> 0.11.0' -gem 'flipper-active_record', '~> 0.11.0' -gem 'flipper-active_support_cache_store', '~> 0.11.0' +gem 'flipper', '~> 0.13.0' +gem 'flipper-active_record', '~> 0.13.0' +gem 'flipper-active_support_cache_store', '~> 0.13.0' # Structured logging gem 'lograge', '~> 0.5' diff --git a/Gemfile.lock b/Gemfile.lock index fcc6ef8468c..1dd8576e30b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -218,13 +218,13 @@ GEM path_expander (~> 1.0) ruby_parser (~> 3.0) sexp_processor (~> 4.0) - flipper (0.11.0) - flipper-active_record (0.11.0) + flipper (0.13.0) + flipper-active_record (0.13.0) activerecord (>= 3.2, < 6) - flipper (~> 0.11.0) - flipper-active_support_cache_store (0.11.0) + flipper (~> 0.13.0) + flipper-active_support_cache_store (0.13.0) activesupport (>= 3.2, < 6) - flipper (~> 0.11.0) + flipper (~> 0.13.0) flowdock (0.7.1) httparty (~> 0.7) multi_json @@ -411,7 +411,8 @@ GEM domain_name (~> 0.5) http-form_data (1.0.1) http_parser.rb (0.6.0) - httparty (0.15.7) + httparty (0.13.7) + json (~> 1.8) multi_xml (>= 0.5.2) httpclient (2.8.2) i18n (0.9.5) @@ -449,9 +450,8 @@ GEM kaminari-core (= 1.0.1) kaminari-core (1.0.1) kgio (2.10.0) - knapsack (1.11.0) + knapsack (1.16.0) rake - timecop (>= 0.1.0) kubeclient (2.2.0) http (= 0.9.8) recursive-open-struct (= 1.0.0) @@ -1039,9 +1039,9 @@ DEPENDENCIES fast_blank ffaker (~> 2.4) flay (~> 2.10.0) - flipper (~> 0.11.0) - flipper-active_record (~> 0.11.0) - flipper-active_support_cache_store (~> 0.11.0) + flipper (~> 0.13.0) + flipper-active_record (~> 0.13.0) + flipper-active_support_cache_store (~> 0.13.0) fog-aliyun (~> 0.2.0) fog-aws (~> 2.0) fog-core (~> 1.44) @@ -1081,14 +1081,14 @@ DEPENDENCIES hipchat (~> 1.5.0) html-pipeline (~> 1.11.0) html2text - httparty (~> 0.15.6) + httparty (~> 0.13.3) influxdb (~> 0.2) jira-ruby (~> 1.4) jquery-atwho-rails (~> 1.3.2) json-schema (~> 2.8.0) jwt (~> 1.5.6) kaminari (~> 1.0) - knapsack (~> 1.11.0) + knapsack (~> 1.16) kubeclient (~> 2.2.0) letter_opener_web (~> 1.3.0) license_finder (~> 3.1) diff --git a/app/assets/javascripts/merge_request_tabs.js b/app/assets/javascripts/merge_request_tabs.js index f01aef45500..e77318fef46 100644 --- a/app/assets/javascripts/merge_request_tabs.js +++ b/app/assets/javascripts/merge_request_tabs.js @@ -73,6 +73,7 @@ export default class MergeRequestTabs { constructor({ action, setUrl, stubLocation } = {}) { const mergeRequestTabs = document.querySelector('.js-tabs-affix'); const navbar = document.querySelector('.navbar-gitlab'); + const peek = document.getElementById('peek'); const paddingTop = 16; this.diffsLoaded = false; @@ -86,6 +87,10 @@ export default class MergeRequestTabs { this.showTab = this.showTab.bind(this); this.stickyTop = navbar ? navbar.offsetHeight - paddingTop : 0; + if (peek) { + this.stickyTop += peek.offsetHeight; + } + if (mergeRequestTabs) { this.stickyTop += mergeRequestTabs.offsetHeight; } diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 0dee6df525d..3cbbf8b5dfa 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -347,15 +347,15 @@ class ApplicationSetting < ActiveRecord::Base end def home_page_url_column_exists? - ActiveRecord::Base.connection.column_exists?(:application_settings, :home_page_url) + ::Gitlab::Database.cached_column_exists?(:application_settings, :home_page_url) end def help_page_support_url_column_exists? - ActiveRecord::Base.connection.column_exists?(:application_settings, :help_page_support_url) + ::Gitlab::Database.cached_column_exists?(:application_settings, :help_page_support_url) end def sidekiq_throttling_column_exists? - ActiveRecord::Base.connection.column_exists?(:application_settings, :sidekiq_throttling_enabled) + ::Gitlab::Database.cached_column_exists?(:application_settings, :sidekiq_throttling_enabled) end def domain_whitelist_raw diff --git a/app/models/project.rb b/app/models/project.rb index a291ad7eed5..5487194ed3e 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1519,8 +1519,8 @@ class Project < ActiveRecord::Base @errors = original_errors end - def add_export_job(current_user:) - job_id = ProjectExportWorker.perform_async(current_user.id, self.id) + def add_export_job(current_user:, params: {}) + job_id = ProjectExportWorker.perform_async(current_user.id, self.id, params) if job_id Rails.logger.info "Export job started for project ID #{self.id} with job ID #{job_id}" diff --git a/app/models/project_services/jira_service.rb b/app/models/project_services/jira_service.rb index e5035c81df0..601a6a077f5 100644 --- a/app/models/project_services/jira_service.rb +++ b/app/models/project_services/jira_service.rb @@ -161,11 +161,6 @@ class JiraService < IssueTrackerService add_comment(data, jira_issue) end - # reason why service cannot be tested - def disabled_title - "Please fill in Password and Username." - end - def test(_) result = test_settings success = result.present? diff --git a/app/models/project_services/pipelines_email_service.rb b/app/models/project_services/pipelines_email_service.rb index 9c7b58dead5..4cf149ac044 100644 --- a/app/models/project_services/pipelines_email_service.rb +++ b/app/models/project_services/pipelines_email_service.rb @@ -39,10 +39,6 @@ class PipelinesEmailService < Service project.pipelines.any? end - def disabled_title - 'Please setup a pipeline on your repository.' - end - def test_data(project, user) data = Gitlab::DataBuilder::Pipeline.build(project.pipelines.last) data[:user] = user.hook_attrs diff --git a/app/models/service.rb b/app/models/service.rb index 99bf757ae44..2556db68146 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -162,11 +162,6 @@ class Service < ActiveRecord::Base true end - # reason why service cannot be tested - def disabled_title - "Please setup a project repository." - end - # Provide convenient accessor methods # for each serialized property. # Also keep track of updated properties in a similar way as ActiveModel::Dirty diff --git a/app/services/files/create_service.rb b/app/services/files/create_service.rb index 46acdc5406c..a954564946b 100644 --- a/app/services/files/create_service.rb +++ b/app/services/files/create_service.rb @@ -1,11 +1,11 @@ module Files class CreateService < Files::BaseService def create_commit! - handler = Lfs::FileModificationHandler.new(project, @branch_name) + transformer = Lfs::FileTransformer.new(project, @branch_name) - handler.new_file(@file_path, @file_content) do |content_or_lfs_pointer| - create_transformed_commit(content_or_lfs_pointer) - end + result = transformer.new_file(@file_path, @file_content) + + create_transformed_commit(result.content) end private diff --git a/app/services/files/multi_service.rb b/app/services/files/multi_service.rb index a03c59f569d..13a1dee4173 100644 --- a/app/services/files/multi_service.rb +++ b/app/services/files/multi_service.rb @@ -3,11 +3,33 @@ module Files UPDATE_FILE_ACTIONS = %w(update move delete).freeze def create_commit! + transformer = Lfs::FileTransformer.new(project, @branch_name) + + actions = actions_after_lfs_transformation(transformer, params[:actions]) + + commit_actions!(actions) + end + + private + + def actions_after_lfs_transformation(transformer, actions) + actions.map do |action| + if action[:action] == 'create' + result = transformer.new_file(action[:file_path], action[:content], encoding: action[:encoding]) + action[:content] = result.content + action[:encoding] = result.encoding + end + + action + end + end + + def commit_actions!(actions) repository.multi_action( current_user, message: @commit_message, branch_name: @branch_name, - actions: params[:actions], + actions: actions, author_email: @author_email, author_name: @author_name, start_project: @start_project, @@ -17,8 +39,6 @@ module Files raise_error(e) end - private - def validate! super diff --git a/app/services/lfs/file_modification_handler.rb b/app/services/lfs/file_modification_handler.rb deleted file mode 100644 index fe9091a6e5d..00000000000 --- a/app/services/lfs/file_modification_handler.rb +++ /dev/null @@ -1,42 +0,0 @@ -module Lfs - class FileModificationHandler - attr_reader :project, :branch_name - - delegate :repository, to: :project - - def initialize(project, branch_name) - @project = project - @branch_name = branch_name - end - - def new_file(file_path, file_content) - if project.lfs_enabled? && lfs_file?(file_path) - lfs_pointer_file = Gitlab::Git::LfsPointerFile.new(file_content) - lfs_object = create_lfs_object!(lfs_pointer_file, file_content) - content = lfs_pointer_file.pointer - - success = yield(content) - - link_lfs_object!(lfs_object) if success - else - yield(file_content) - end - end - - private - - def lfs_file?(file_path) - repository.attributes_at(branch_name, file_path)['filter'] == 'lfs' - end - - def create_lfs_object!(lfs_pointer_file, file_content) - LfsObject.find_or_create_by(oid: lfs_pointer_file.sha256, size: lfs_pointer_file.size) do |lfs_object| - lfs_object.file = CarrierWaveStringFile.new(file_content) - end - end - - def link_lfs_object!(lfs_object) - project.lfs_objects << lfs_object - end - end -end diff --git a/app/services/lfs/file_transformer.rb b/app/services/lfs/file_transformer.rb new file mode 100644 index 00000000000..69281ee3137 --- /dev/null +++ b/app/services/lfs/file_transformer.rb @@ -0,0 +1,66 @@ +module Lfs + # Usage: Calling `new_file` check to see if a file should be in LFS and + # return a transformed result with `content` and `encoding` to commit. + # + # For LFS an LfsObject linked to the project is stored and an LFS + # pointer returned. If the file isn't in LFS the untransformed content + # is returned to save in the commit. + # + # transformer = Lfs::FileTransformer.new(project, @branch_name) + # content_or_lfs_pointer = transformer.new_file(file_path, content).content + # create_transformed_commit(content_or_lfs_pointer) + # + class FileTransformer + attr_reader :project, :branch_name + + delegate :repository, to: :project + + def initialize(project, branch_name) + @project = project + @branch_name = branch_name + end + + def new_file(file_path, file_content, encoding: nil) + if project.lfs_enabled? && lfs_file?(file_path) + file_content = Base64.decode64(file_content) if encoding == 'base64' + lfs_pointer_file = Gitlab::Git::LfsPointerFile.new(file_content) + lfs_object = create_lfs_object!(lfs_pointer_file, file_content) + + link_lfs_object!(lfs_object) + + Result.new(content: lfs_pointer_file.pointer, encoding: 'text') + else + Result.new(content: file_content, encoding: encoding) + end + end + + class Result + attr_reader :content, :encoding + + def initialize(content:, encoding:) + @content = content + @encoding = encoding + end + end + + private + + def lfs_file?(file_path) + cached_attributes.attributes(file_path)['filter'] == 'lfs' + end + + def cached_attributes + @cached_attributes ||= Gitlab::Git::AttributesAtRefParser.new(repository, branch_name) + end + + def create_lfs_object!(lfs_pointer_file, file_content) + LfsObject.find_or_create_by(oid: lfs_pointer_file.sha256, size: lfs_pointer_file.size) do |lfs_object| + lfs_object.file = CarrierWaveStringFile.new(file_content) + end + end + + def link_lfs_object!(lfs_object) + project.lfs_objects << lfs_object + end + end +end diff --git a/app/services/projects/import_export/export_service.rb b/app/services/projects/import_export/export_service.rb index af41ce82f65..d16aa3de639 100644 --- a/app/services/projects/import_export/export_service.rb +++ b/app/services/projects/import_export/export_service.rb @@ -26,7 +26,7 @@ module Projects end def project_tree_saver - Gitlab::ImportExport::ProjectTreeSaver.new(project: project, current_user: @current_user, shared: @shared) + Gitlab::ImportExport::ProjectTreeSaver.new(project: project, current_user: @current_user, shared: @shared, params: @params) end def uploads_saver diff --git a/app/views/projects/services/_form.html.haml b/app/views/projects/services/_form.html.haml index 053ea24b848..684b082efbb 100644 --- a/app/views/projects/services/_form.html.haml +++ b/app/views/projects/services/_form.html.haml @@ -15,11 +15,6 @@ .footer-block.row-content-block = service_save_button(@service) - - if @service.valid? && @service.activated? - - unless @service.can_test? - - disabled_class = 'disabled' - - disabled_title = @service.disabled_title - = link_to 'Cancel', project_settings_integrations_path(@project), class: 'btn btn-cancel' - if lookup_context.template_exists?('show', "projects/services/#{@service.to_param}", true) diff --git a/app/workers/project_export_worker.rb b/app/workers/project_export_worker.rb index c100852374a..0b502143e5d 100644 --- a/app/workers/project_export_worker.rb +++ b/app/workers/project_export_worker.rb @@ -4,10 +4,11 @@ class ProjectExportWorker sidekiq_options retry: 3 - def perform(current_user_id, project_id) + def perform(current_user_id, project_id, params = {}) + params = params.with_indifferent_access current_user = User.find(current_user_id) project = Project.find(project_id) - ::Projects::ImportExport::ExportService.new(project, current_user).execute + ::Projects::ImportExport::ExportService.new(project, current_user, params).execute end end diff --git a/changelogs/unreleased/41902-add-api-option-to-overwrite-project-description-on-project-export.yml b/changelogs/unreleased/41902-add-api-option-to-overwrite-project-description-on-project-export.yml new file mode 100644 index 00000000000..60a649f22c9 --- /dev/null +++ b/changelogs/unreleased/41902-add-api-option-to-overwrite-project-description-on-project-export.yml @@ -0,0 +1,5 @@ +--- +title: Adds the option to the project export API to override the project description and display GitLab export description once imported +merge_request: 17744 +author: +type: added diff --git a/changelogs/unreleased/44235-update-knapsack-to-1-16-0.yml b/changelogs/unreleased/44235-update-knapsack-to-1-16-0.yml new file mode 100644 index 00000000000..265d36b763f --- /dev/null +++ b/changelogs/unreleased/44235-update-knapsack-to-1-16-0.yml @@ -0,0 +1,5 @@ +--- +title: Update knapsack to 1.16.0 +merge_request: 17735 +author: Takuya Noguchi +type: other diff --git a/changelogs/unreleased/fix-code-search-500-with-non-ascii-filename.yml b/changelogs/unreleased/fix-code-search-500-with-non-ascii-filename.yml deleted file mode 100644 index 29e3b7be985..00000000000 --- a/changelogs/unreleased/fix-code-search-500-with-non-ascii-filename.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Fix code and wiki search results when filename is non-ASCII -merge_request: -author: -type: fixed diff --git a/changelogs/unreleased/jej-commit-api-tracks-lfs.yml b/changelogs/unreleased/jej-commit-api-tracks-lfs.yml new file mode 100644 index 00000000000..8284abf9f28 --- /dev/null +++ b/changelogs/unreleased/jej-commit-api-tracks-lfs.yml @@ -0,0 +1,5 @@ +--- +title: Create commit API and Web IDE obey LFS filters +merge_request: 16718 +author: +type: fixed diff --git a/changelogs/unreleased/mk-fix-move-upload-files-on-group-transfer.yml b/changelogs/unreleased/mk-fix-move-upload-files-on-group-transfer.yml deleted file mode 100644 index ba366b81600..00000000000 --- a/changelogs/unreleased/mk-fix-move-upload-files-on-group-transfer.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Fix missing uploads after group transfer -merge_request: 17658 -author: -type: fixed diff --git a/changelogs/unreleased/sh-add-missing-acts-as-taggable-indices.yml b/changelogs/unreleased/sh-add-missing-acts-as-taggable-indices.yml deleted file mode 100644 index d9a1a0db9e8..00000000000 --- a/changelogs/unreleased/sh-add-missing-acts-as-taggable-indices.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Adding missing indexes on taggings table -merge_request: -author: -type: performance diff --git a/changelogs/unreleased/sh-add-section-name-index.yml b/changelogs/unreleased/sh-add-section-name-index.yml deleted file mode 100644 index c822b4e851b..00000000000 --- a/changelogs/unreleased/sh-add-section-name-index.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Add index on section_name_id on ci_build_trace_sections table -merge_request: -author: -type: performance diff --git a/changelogs/unreleased/sh-cache-column-exists.yml b/changelogs/unreleased/sh-cache-column-exists.yml new file mode 100644 index 00000000000..8bc648f2b32 --- /dev/null +++ b/changelogs/unreleased/sh-cache-column-exists.yml @@ -0,0 +1,5 @@ +--- +title: Cache column_exists? for application settings +merge_request: +author: +type: performance diff --git a/changelogs/unreleased/sh-cache-table-exists.yml b/changelogs/unreleased/sh-cache-table-exists.yml new file mode 100644 index 00000000000..37407b2a005 --- /dev/null +++ b/changelogs/unreleased/sh-cache-table-exists.yml @@ -0,0 +1,5 @@ +--- +title: Cache table_exists?('application_settings') to reduce repeated schema reloads +merge_request: +author: +type: performance diff --git a/changelogs/unreleased/sh-remove-double-caching-repo-empty.yml b/changelogs/unreleased/sh-remove-double-caching-repo-empty.yml deleted file mode 100644 index 1684be4e5e3..00000000000 --- a/changelogs/unreleased/sh-remove-double-caching-repo-empty.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Remove double caching of Repository#empty? -merge_request: -author: -type: fixed diff --git a/doc/administration/raketasks/check.md b/doc/administration/raketasks/check.md index 51c62742d01..7d34d35e7d1 100644 --- a/doc/administration/raketasks/check.md +++ b/doc/administration/raketasks/check.md @@ -84,9 +84,9 @@ checks using those checksums can be run. These checks also detect missing files. Currently, integrity checks are supported for the following types of file: -* CI artifacts -* LFS objects -* User uploads +* CI artifacts (Available from version 10.7.0) +* LFS objects (Available from version 10.6.0) +* User uploads (Available from version 10.6.0) **Omnibus Installation** diff --git a/doc/api/project_import_export.md b/doc/api/project_import_export.md index 677765368a8..de5207fc5e4 100644 --- a/doc/api/project_import_export.md +++ b/doc/api/project_import_export.md @@ -15,9 +15,10 @@ POST /projects/:id/export | Attribute | Type | Required | Description | | --------- | -------------- | -------- | ---------------------------------------- | | `id` | integer/string | yes | The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user | +| `description` | string | no | Overrides the project description | ```console -curl --request POST --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/projects/1/export +curl --request POST --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" --form "description=Foo Bar" https://gitlab.example.com/api/v4/projects/1/export ``` ```json diff --git a/doc/ci/examples/README.md b/doc/ci/examples/README.md index f69729f602d..c1e258aedca 100644 --- a/doc/ci/examples/README.md +++ b/doc/ci/examples/README.md @@ -47,8 +47,11 @@ There's also a collection of repositories with [example projects](https://gitlab ## Static Application Security Testing (SAST) -- **(Ultimate)** [Scan your code for vulnerabilities](https://docs.gitlab.com/ee/ci/examples/sast.html) -- [Scan your Docker images for vulnerabilities](sast_docker.md) +**(Ultimate)** [Scan your code for vulnerabilities](https://docs.gitlab.com/ee/ci/examples/sast.html) + +## Container Scanning + +[Scan your Docker images for vulnerabilities](container_scanning.md) ## Dynamic Application Security Testing (DAST) diff --git a/doc/ci/examples/code_climate.md b/doc/ci/examples/code_climate.md index d7df53494ed..ec5e5afb8c6 100644 --- a/doc/ci/examples/code_climate.md +++ b/doc/ci/examples/code_climate.md @@ -16,18 +16,26 @@ codequality: - docker:dind script: - docker pull codeclimate/codeclimate - - docker run --env CODECLIMATE_CODE="$PWD" --volume "$PWD":/code --volume /var/run/docker.sock:/var/run/docker.sock --volume /tmp/cc:/tmp/cc codeclimate/codeclimate:0.69.0 init - - docker run --env CODECLIMATE_CODE="$PWD" --volume "$PWD":/code --volume /var/run/docker.sock:/var/run/docker.sock --volume /tmp/cc:/tmp/cc codeclimate/codeclimate:0.69.0 analyze -f json > codeclimate.json || true + - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') + - docker run + --env SOURCE_CODE="$PWD" \ + --volume "$PWD":/code \ + --volume /var/run/docker.sock:/var/run/docker.sock \ + "registry.gitlab.com/gitlab-org/security-products/codequality:$SP_VERSION" /code artifacts: paths: [codeclimate.json] ``` -This will create a `codequality` job in your CI pipeline and will allow you to -download and analyze the report artifact in JSON format. +The above example will create a `codequality` job in your CI/CD pipeline which +will scan your source code for code quality issues. The report will be saved +as an artifact that you can later download and analyze. -For [GitLab Starter][ee] users, this information can be automatically -extracted and shown right in the merge request widget. [Learn more on code quality -diffs in merge requests](https://docs.gitlab.com/ee/user/project/merge_requests/code_quality_diff.html). +TIP: **Tip:** +Starting with [GitLab Starter][ee] 9.3, this information will +be automatically extracted and shown right in the merge request widget. To do +so, the CI/CD job must be named `codequality` and the artifact path must be +`codeclimate.json`. +[Learn more on code quality diffs in merge requests](https://docs.gitlab.com/ee/user/project/merge_requests/code_quality_diff.html). [cli]: https://github.com/codeclimate/codeclimate [dind]: ../docker/using_docker_build.md#use-docker-in-docker-executor diff --git a/doc/ci/examples/container_scanning.md b/doc/ci/examples/container_scanning.md new file mode 100644 index 00000000000..3437b63748a --- /dev/null +++ b/doc/ci/examples/container_scanning.md @@ -0,0 +1,55 @@ +# Container Scanning with GitLab CI/CD + +You can check your Docker images (or more precisely the containers) for known +vulnerabilities by using [Clair](https://github.com/coreos/clair) and +[clair-scanner](https://github.com/arminc/clair-scanner), two open source tools +for Vulnerability Static Analysis for containers. + +All you need is a GitLab Runner with the Docker executor (the shared Runners on +GitLab.com will work fine). You can then add a new job to `.gitlab-ci.yml`, +called `sast:container`: + +```yaml +sast:container: + image: docker:latest + variables: + DOCKER_DRIVER: overlay2 + ## Define two new variables based on GitLab's CI/CD predefined variables + ## https://docs.gitlab.com/ee/ci/variables/#predefined-variables-environment-variables + CI_APPLICATION_REPOSITORY: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG + CI_APPLICATION_TAG: $CI_COMMIT_SHA + allow_failure: true + services: + - docker:dind + script: + - docker run -d --name db arminc/clair-db:latest + - docker run -p 6060:6060 --link db:postgres -d --name clair arminc/clair-local-scan:v2.0.1 + - apk add -U wget ca-certificates + - docker pull ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG} + - wget https://github.com/arminc/clair-scanner/releases/download/v8/clair-scanner_linux_amd64 + - mv clair-scanner_linux_amd64 clair-scanner + - chmod +x clair-scanner + - touch clair-whitelist.yml + - ./clair-scanner -c http://docker:6060 --ip $(hostname -i) -r gl-sast-container-report.json -l clair.log -w clair-whitelist.yml ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG} || true + artifacts: + paths: [gl-sast-container-report.json] +``` + +The above example will create a `sast:container` job in your CI/CD pipeline, pull +the image from the [Container Registry](../../user/project/container_registry.md) +(whose name is defined from the two `CI_APPLICATION_` variables) and scan it +for possible vulnerabilities. The report will be saved as an artifact that you +can later download and analyze. + +If you want to whitelist some specific vulnerabilities, you can do so by defining +them in a [YAML file](https://github.com/arminc/clair-scanner/blob/master/README.md#example-whitelist-yaml-file), +in our case its named `clair-whitelist.yml`. + +TIP: **Tip:** +Starting with [GitLab Ultimate][ee] 10.4, this information will +be automatically extracted and shown right in the merge request widget. To do +so, the CI/CD job must be named `sast:container` and the artifact path must be +`gl-sast-container-report.json`. +[Learn more on container scanning results shown in merge requests](https://docs.gitlab.com/ee/user/project/merge_requests/container_scanning.html). + +[ee]: https://about.gitlab.com/products/ diff --git a/doc/ci/examples/sast_docker.md b/doc/ci/examples/sast_docker.md index 57a9c4bcfc1..9f4a63e296d 100644 --- a/doc/ci/examples/sast_docker.md +++ b/doc/ci/examples/sast_docker.md @@ -1,55 +1 @@ -# Static Application Security Testing for Docker containers with GitLab CI/CD - -You can check your Docker images (or more precisely the containers) for known -vulnerabilities by using [Clair](https://github.com/coreos/clair) and -[clair-scanner](https://github.com/arminc/clair-scanner), two open source tools -for Vulnerability Static Analysis for containers. - -All you need is a GitLab Runner with the Docker executor (the shared Runners on -GitLab.com will work fine). You can then add a new job to `.gitlab-ci.yml`, -called `sast:container`: - -```yaml -sast:container: - image: docker:latest - variables: - DOCKER_DRIVER: overlay2 - ## Define two new variables based on GitLab's CI/CD predefined variables - ## https://docs.gitlab.com/ee/ci/variables/#predefined-variables-environment-variables - CI_APPLICATION_REPOSITORY: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG - CI_APPLICATION_TAG: $CI_COMMIT_SHA - allow_failure: true - services: - - docker:dind - script: - - docker run -d --name db arminc/clair-db:latest - - docker run -p 6060:6060 --link db:postgres -d --name clair arminc/clair-local-scan:v2.0.1 - - apk add -U wget ca-certificates - - docker pull ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG} - - wget https://github.com/arminc/clair-scanner/releases/download/v8/clair-scanner_linux_amd64 - - mv clair-scanner_linux_amd64 clair-scanner - - chmod +x clair-scanner - - touch clair-whitelist.yml - - ./clair-scanner -c http://docker:6060 --ip $(hostname -i) -r gl-sast-container-report.json -l clair.log -w clair-whitelist.yml ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG} || true - artifacts: - paths: [gl-sast-container-report.json] -``` - -The above example will create a `sast:container` job in your CI/CD pipeline, pull -the image from the [Container Registry](../../user/project/container_registry.md) -(whose name is defined from the two `CI_APPLICATION_` variables) and scan it -for possible vulnerabilities. The report will be saved as an artifact that you -can later download and analyze. - -If you want to whitelist some specific vulnerabilities, you can do so by defining -them in a [YAML file](https://github.com/arminc/clair-scanner/blob/master/README.md#example-whitelist-yaml-file), -in our case its named `clair-whitelist.yml`. - -TIP: **Tip:** -Starting with [GitLab Ultimate][ee] 10.4, this information will -be automatically extracted and shown right in the merge request widget. To do -so, the CI/CD job must be named `sast:container` and the artifact path must be -`gl-sast-container-report.json`. -[Learn more on application security testing results shown in merge requests](https://docs.gitlab.com/ee/user/project/merge_requests/sast_docker.html). - -[ee]: https://about.gitlab.com/products/ +This document was moved to [another location](./container_scanning.md).
\ No newline at end of file diff --git a/doc/development/doc_styleguide.md b/doc/development/doc_styleguide.md index 6fe5f647d6c..41e3412c7ff 100644 --- a/doc/development/doc_styleguide.md +++ b/doc/development/doc_styleguide.md @@ -1,89 +1,28 @@ -# Documentation styleguide +# Documentation style guidelines -This styleguide recommends best practices to improve documentation and to keep -it organized and easy to find. +The documentation style guide defines the markup structure used in +GitLab documentation. Check the +[documentation guidelines](writing_documentation.md) for general development instructions. -See also [writing documentation](writing_documentation.md). - -## Location and naming of documents - ->**Note:** -These guidelines derive from the discussion taken place in issue [#3349][ce-3349]. - -The documentation hierarchy can be vastly improved by providing a better layout -and organization of directories. - -Having a structured document layout, we will be able to have meaningful URLs -like `docs.gitlab.com/user/project/merge_requests.html`. With this pattern, -you can immediately tell that you are navigating a user related documentation -and is about the project and its merge requests. - -Do not create summaries of similar types of content (e.g. an index of all articles, videos, etc.), -rather organise content by its subject (e.g. everything related to CI goes together) -and cross-link between any related content. - -The table below shows what kind of documentation goes where. - -| Directory | What belongs here | -| --------- | -------------- | -| `doc/user/` | User related documentation. Anything that can be done within the GitLab UI goes here including `/admin`. | -| `doc/administration/` | Documentation that requires the user to have access to the server where GitLab is installed. The admin settings that can be accessed via GitLab's interface go under `doc/user/admin_area/`. | -| `doc/api/` | API related documentation. | -| `doc/development/` | Documentation related to the development of GitLab. Any styleguides should go here. | -| `doc/legal/` | Legal documents about contributing to GitLab. | -| `doc/install/`| Probably the most visited directory, since `installation.md` is there. Ideally this should go under `doc/administration/`, but it's best to leave it as-is in order to avoid confusion (still debated though). | -| `doc/update/` | Same with `doc/install/`. Should be under `administration/`, but this is a well known location, better leave as-is, at least for now. | -| `doc/topics/` | Indexes per Topic (`doc/topics/topic-name/index.md`): all resources for that topic (user and admin documentation, articles, and third-party docs) | - ---- - -**General rules:** - -1. The correct naming and location of a new document, is a combination - of the relative URL of the document in question and the GitLab Map design - that is used for UX purposes ([source][graffle], [image][gitlab-map]). -1. When creating a new document and it has more than one word in its name, - make sure to use underscores instead of spaces or dashes (`-`). For example, - a proper naming would be `import_projects_from_github.md`. The same rule - applies to images. -1. There are four main directories, `user`, `administration`, `api` and `development`. -1. The `doc/user/` directory has five main subdirectories: `project/`, `group/`, - `profile/`, `dashboard/` and `admin_area/`. - 1. `doc/user/project/` should contain all project related documentation. - 1. `doc/user/group/` should contain all group related documentation. - 1. `doc/user/profile/` should contain all profile related documentation. - Every page you would navigate under `/profile` should have its own document, - i.e. `account.md`, `applications.md`, `emails.md`, etc. - 1. `doc/user/dashboard/` should contain all dashboard related documentation. - 1. `doc/user/admin_area/` should contain all admin related documentation - describing what can be achieved by accessing GitLab's admin interface - (_not to be confused with `doc/administration` where server access is - required_). - 1. Every category under `/admin/application_settings` should have its - own document located at `doc/user/admin_area/settings/`. For example, - the **Visibility and Access Controls** category should have a document - located at `doc/user/admin_area/settings/visibility_and_access_controls.md`. -1. The `doc/topics/` directory holds topic-related technical content. Create - `doc/topics/topic-name/subtopic-name/index.md` when subtopics become necessary. - General user- and admin- related documentation, should be placed accordingly. - ---- - -If you are unsure where a document should live, you can ping `@axil` or `@marcia` in your -merge request. +Check the GitLab hanbook for the [writing styles guidelines](https://about.gitlab.com/handbook/communication/#writing-style-guidelines). ## Text -- Split up long lines, this makes it much easier to review and edit. Only +- Split up long lines (wrap text), this makes it much easier to review and edit. Only double line breaks are shown as a full line break in [GitLab markdown][gfm]. 80-100 characters is a good line length -- Make sure that the documentation is added in the correct directory and that +- Make sure that the documentation is added in the correct + [directory](writing_documentation.md#documentation-directory-structure) and that there's a link to it somewhere useful - Do not duplicate information - Be brief and clear - Unless there's a logical reason not to, add documents in alphabetical order - Write in US English - Use [single spaces][] instead of double spaces +- Jump a line between different markups (e.g., after every paragraph, hearder, list, etc) +- Capitalize "G" and "L" in GitLab +- Capitalize feature, products, and methods names. E.g.: GitLab Runner, Geo, +Issue Boards, Git, Prometheus, Continuous Integration. ## Formatting @@ -103,6 +42,8 @@ merge request. links shift too, which eventually leads to dead links. If you think it is compelling to add numbers in headings, make sure to at least discuss it with someone in the Merge Request +- [Avoid using symbols and special chars](https://gitlab.com/gitlab-com/gitlab-docs/issues/84) + in headers. Whenever possible, they should be plain and short text. - Avoid adding things that show ephemeral statuses. For example, if a feature is considered beta or experimental, put this info in a note, not in the heading. - When introducing a new document, be careful for the headings to be @@ -121,71 +62,18 @@ merge request. you can use `[Text][identifier]` and at the bottom of the section or the document add: `[identifier]: https://example.com`, in which case, we do encourage you to also add an alternative text: `[identifier]: https://example.com "Alternative text"` that appears when hovering your mouse on a link. - -### Linking to inline docs - -Sometimes it's needed to link to the built-in documentation that GitLab provides -under `/help`. This is normally done in files inside the `app/views/` directory -with the help of the `help_page_path` helper method. - -In its simplest form, the HAML code to generate a link to the `/help` page is: - -```haml -= link_to 'Help page', help_page_path('user/permissions') -``` - -The `help_page_path` contains the path to the document you want to link to with -the following conventions: - -- it is relative to the `doc/` directory in the GitLab repository -- the `.md` extension must be omitted -- it must not end with a slash (`/`) - -Below are some special cases where should be used depending on the context. -You can combine one or more of the following: - -1. **Linking to an anchor link.** Use `anchor` as part of the `help_page_path` - method: - - ```haml - = link_to 'Help page', help_page_path('user/permissions', anchor: 'anchor-link') - ``` - -1. **Opening links in a new tab.** This should be the default behavior: - - ```haml - = link_to 'Help page', help_page_path('user/permissions'), target: '_blank' - ``` - -1. **Linking to a circle icon.** Usually used in settings where a long - description cannot be used, like near checkboxes. You can basically use - any font awesome icon, but prefer the `question-circle`: - - ```haml - = link_to icon('question-circle'), help_page_path('user/permissions') - ``` - -1. **Using a button link.** Useful in places where text would be out of context - with the rest of the page layout: - - ```haml - = link_to 'Help page', help_page_path('user/permissions'), class: 'btn btn-info' - ``` - -1. **Using links inline of some text.** - - ```haml - Description to #{link_to 'Help page', help_page_path('user/permissions')}. - ``` - -1. **Adding a period at the end of the sentence.** Useful when you don't want - the period to be part of the link: - - ```haml - = succeed '.' do - Learn more in the - = link_to 'Help page', help_page_path('user/permissions') - ``` +- To link to internal documentation, use relative links, not full URLs. Use `../` to + navigate tp high-level directories, and always add the file name `file.md` at the + end of the link with the `.md` extension, not `.html`. + Example: instead of `[text](../../merge_requests/)`, use + `[text](../../merge_requests/index.md)` or, `[text](../../ci/README.md)`, or, + for anchor links, `[text](../../ci/README.md#examples)`. + Using the markdown extension is necessary for the [`/help`](writing_documentation.md#gitlab-help) + section of GitLab. +- To link from CE to EE-only documentation, use the EE-only doc full URL. +- Use [meaningful anchor texts](https://www.futurehosting.com/blog/links-should-have-meaningful-anchor-text-heres-why/). + E.g., instead of writing something like `Read more about GitLab Issue Boards [here](LINK)`, + write `Read more about [GitLab Issue Boards](LINK)`. ## Images @@ -222,7 +110,7 @@ Inside the document: - Notes should be quoted with the word `Note:` being bold. Use this form: - ``` + ```md >**Note:** This is something to note. ``` @@ -234,25 +122,25 @@ Inside the document: If the note spans across multiple lines it's OK to split the line. -## New features +## Specific sections and terms -New features must be shipped with its accompanying documentation and the doc -reviewed by a technical writer. +To mention and/or reference specific terms in GitLab, please follow the styles +below. -### Mentioning GitLab versions and tiers +### GitLab versions and tiers - Every piece of documentation that comes with a new feature should declare the GitLab version that feature got introduced. Right below the heading add a note: - ``` + ```md > Introduced in GitLab 8.3. ``` - If possible every feature should have a link to the MR, issue, or epic that introduced it. The above note would be then transformed to: - ``` + ```md > [Introduced][ce-1242] in GitLab 8.3. ``` @@ -263,121 +151,41 @@ reviewed by a technical writer. the [paid tier](https://about.gitlab.com/handbook/marketing/product-marketing/#tiers) the feature is available in: - ``` + ```md > [Introduced][ee-1234] in [GitLab Starter](https://about.gitlab.com/products/) 8.3. ``` Otherwise, leave this mention out. -## References - -- **GitLab Restart:** - There are many cases that a restart/reconfigure of GitLab is required. To - avoid duplication, link to the special document that can be found in - [`doc/administration/restart_gitlab.md`][doc-restart]. Usually the text will - read like: - - ``` - Save the file and [reconfigure GitLab](../administration/restart_gitlab.md) - for the changes to take effect. - ``` - If the document you are editing resides in a place other than the GitLab CE/EE - `doc/` directory, instead of the relative link, use the full path: - `http://docs.gitlab.com/ce/administration/restart_gitlab.html`. - Replace `reconfigure` with `restart` where appropriate. - -## Installation guide - -- **Ruby:** - In [step 2 of the installation guide](../install/installation.md#2-ruby), - we install Ruby from source. Whenever there is a new version that needs to - be updated, remember to change it throughout the codeblock and also replace - the sha256sum (it can be found in the [downloads page][ruby-dl] of the Ruby - website). +### GitLab Restart -[ruby-dl]: https://www.ruby-lang.org/en/downloads/ "Ruby download website" - -## Changing document location - -Changing a document's location is not to be taken lightly. Remember that the -documentation is available to all installations under `help/` and not only to -GitLab.com or http://docs.gitlab.com. Make sure this is discussed with the -Documentation team beforehand. - -If you indeed need to change a document's location, do NOT remove the old -document, but rather replace all of its contents with a new line: - -``` -This document was moved to [another location](path/to/new_doc.md). -``` +There are many cases that a restart/reconfigure of GitLab is required. To +avoid duplication, link to the special document that can be found in +[`doc/administration/restart_gitlab.md`][doc-restart]. Usually the text will +read like: -where `path/to/new_doc.md` is the relative path to the root directory `doc/`. + ``` + Save the file and [reconfigure GitLab](../administration/restart_gitlab.md) + for the changes to take effect. + ``` ---- +If the document you are editing resides in a place other than the GitLab CE/EE +`doc/` directory, instead of the relative link, use the full path: +`http://docs.gitlab.com/ce/administration/restart_gitlab.html`. +Replace `reconfigure` with `restart` where appropriate. -For example, if you were to move `doc/workflow/lfs/lfs_administration.md` to -`doc/administration/lfs.md`, then the steps would be: +### Installation guide -1. Copy `doc/workflow/lfs/lfs_administration.md` to `doc/administration/lfs.md` -1. Replace the contents of `doc/workflow/lfs/lfs_administration.md` with: +**Ruby:** +In [step 2 of the installation guide](../install/installation.md#2-ruby), +we install Ruby from source. Whenever there is a new version that needs to +be updated, remember to change it throughout the codeblock and also replace +the sha256sum (it can be found in the [downloads page][ruby-dl] of the Ruby +website). - ``` - This document was moved to [another location](../../administration/lfs.md). - ``` - -1. Find and replace any occurrences of the old location with the new one. - A quick way to find them is to use `git grep`. First go to the root directory - where you cloned the `gitlab-ce` repository and then do: - - ``` - git grep -n "workflow/lfs/lfs_administration" - git grep -n "lfs/lfs_administration" - ``` - -NOTE: **Note:** -If the document being moved has any Disqus comments on it, there are extra steps -to follow documented just [below](#redirections-for-pages-with-disqus-comments). - -Things to note: - -- Since we also use inline documentation, except for the documentation itself, - the document might also be referenced in the views of GitLab (`app/`) which will - render when visiting `/help`, and sometimes in the testing suite (`spec/`). -- The above `git grep` command will search recursively in the directory you run - it in for `workflow/lfs/lfs_administration` and `lfs/lfs_administration` - and will print the file and the line where this file is mentioned. - You may ask why the two greps. Since we use relative paths to link to - documentation, sometimes it might be useful to search a path deeper. -- The `*.md` extension is not used when a document is linked to GitLab's - built-in help page, that's why we omit it in `git grep`. -- Use the checklist on the documentation MR description template. - -### Redirections for pages with Disqus comments - -If the documentation page being relocated already has any Disqus comments, -we need to preserve the Disqus thread. - -Disqus uses an identifier per page, and for docs.gitlab.com, the page identifier -is configured to be the page URL. Therefore, when we change the document location, -we need to preserve the old URL as the same Disqus identifier. - -To do that, add to the frontmatter the variable `redirect_from`, -using the old URL as value. For example, let's say I moved the document -available under `https://docs.gitlab.com/my-old-location/README.html` to a new location, -`https://docs.gitlab.com/my-new-location/index.html`. - -Into the **new document** frontmatter add the following: - -```yaml ---- -redirect_from: 'https://docs.gitlab.com/my-old-location/README.html' ---- -``` - -Note: it is necessary to include the file name in the `redirect_from` URL, -even if it's `index.html` or `README.html`. +[ruby-dl]: https://www.ruby-lang.org/en/downloads/ "Ruby download website" -## Configuration documentation for source and Omnibus installations +### Configuration documentation for source and Omnibus installations GitLab currently officially supports two installation methods: installations from source and Omnibus packages installations. @@ -394,7 +202,7 @@ When there is a list of steps to perform, usually that entails editing the configuration file and reconfiguring/restarting GitLab. In such case, follow the style below as a guide: -```` +```md **For Omnibus installations** 1. Edit `/etc/gitlab/gitlab.rb`: @@ -421,7 +229,7 @@ the style below as a guide: [reconfigure]: path/to/administration/restart_gitlab.md#omnibus-gitlab-reconfigure [restart]: path/to/administration/restart_gitlab.md#installations-from-source -```` +``` In this case: @@ -433,7 +241,7 @@ In this case: - different highlighting languages are used for each config in the code block - the [references](#references) guide is used for reconfigure/restart -## Fake tokens +### Fake tokens There may be times where a token is needed to demonstrate an API call using cURL or a secret variable used in CI. It is strongly advised not to use real @@ -456,7 +264,7 @@ You can use the following fake tokens as examples. | Health check token | `Tu7BgjR9qeZTEyRzGG2P` | | Request profile token | `7VgpS4Ax5utVD2esNstz` | -## API +### API Here is a list of must-have items. Use them in the exact order that appears on this document. Further explanation is given below. @@ -472,10 +280,10 @@ on this document. Further explanation is given below. - Every method must have a cURL example. - Every method must have a response body (in JSON format). -### Method description +#### Method description Use the following table headers to describe the methods. Attributes should -always be in code blocks using backticks (`). +always be in code blocks using backticks (``` ` ```). ``` | Attribute | Type | Required | Description | @@ -488,7 +296,7 @@ Rendered example: | --------- | ---- | -------- | ----------- | | `user` | string | yes | The GitLab username | -### cURL commands +#### cURL commands - Use `https://gitlab.example.com/api/v4/` as an endpoint. - Wherever needed use this personal access token: `9koXpg98eAheJpvBs5tK`. @@ -505,11 +313,11 @@ Rendered example: | `-X PUT` | Use this method when updating existing objects | | `-X DELETE` | Use this method when removing existing objects | -### cURL Examples +#### cURL Examples Below is a set of [cURL][] examples that you can use in the API documentation. -#### Simple cURL command +##### Simple cURL command Get the details of a group: @@ -517,7 +325,7 @@ Get the details of a group: curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/groups/gitlab-org ``` -#### cURL example with parameters passed in the URL +##### cURL example with parameters passed in the URL Create a new project under the authenticated user's namespace: @@ -525,7 +333,7 @@ Create a new project under the authenticated user's namespace: curl --request POST --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" "https://gitlab.example.com/api/v4/projects?name=foo" ``` -#### Post data using cURL's --data +##### Post data using cURL's --data Instead of using `-X POST` and appending the parameters to the URI, you can use cURL's `--data` option. The example below will create a new project `foo` under @@ -535,7 +343,7 @@ the authenticated user's namespace. curl --data "name=foo" --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" "https://gitlab.example.com/api/v4/projects" ``` -#### Post data using JSON content +##### Post data using JSON content > **Note:** In this example we create a new group. Watch carefully the single and double quotes. @@ -544,7 +352,7 @@ and double quotes. curl --request POST --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" --header "Content-Type: application/json" --data '{"path": "my-group", "name": "My group"}' https://gitlab.example.com/api/v4/groups ``` -#### Post data using form-data +##### Post data using form-data Instead of using JSON or urlencode you can use multipart/form-data which properly handles data encoding: @@ -556,7 +364,7 @@ curl --request POST --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" --form "title The above example is run by and administrator and will add an SSH public key titled ssh-key to user's account which has an id of 25. -#### Escape special characters +##### Escape special characters Spaces or slashes (`/`) may sometimes result to errors, thus it is recommended to escape them when possible. In the example below we create a new issue which @@ -569,7 +377,7 @@ curl --request POST --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" "https://gitl Use `%2F` for slashes (`/`). -#### Pass arrays to API calls +##### Pass arrays to API calls The GitLab API sometimes accepts arrays of strings or integers. For example, to restrict the sign-up e-mail domains of a GitLab instance to `*.example.com` and @@ -584,6 +392,3 @@ curl --request PUT --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" --data "domain [gfm]: http://docs.gitlab.com/ce/user/markdown.html#newlines "GitLab flavored markdown documentation" [ce-1242]: https://gitlab.com/gitlab-org/gitlab-ce/issues/1242 [doc-restart]: ../administration/restart_gitlab.md "GitLab restart documentation" -[ce-3349]: https://gitlab.com/gitlab-org/gitlab-ce/issues/3349 "Documentation restructure" -[graffle]: https://gitlab.com/gitlab-org/gitlab-design/blob/d8d39f4a87b90fb9ae89ca12dc565347b4900d5e/production/resources/gitlab-map.graffle -[gitlab-map]: https://gitlab.com/gitlab-org/gitlab-design/raw/master/production/resources/gitlab-map.png diff --git a/doc/development/new_fe_guide/index.md b/doc/development/new_fe_guide/index.md index 08c6a266e7f..78931defa24 100644 --- a/doc/development/new_fe_guide/index.md +++ b/doc/development/new_fe_guide/index.md @@ -19,7 +19,7 @@ Guidance on topics related to development. Learn about all the dependencies that make up our frontend, including some of our own custom built libraries. -## [Style](style/index.md) +## [Style guides](style/index.md) Style guides to keep our code consistent. diff --git a/doc/development/new_fe_guide/style/index.md b/doc/development/new_fe_guide/style/index.md index ebee57bebbf..335d9e66240 100644 --- a/doc/development/new_fe_guide/style/index.md +++ b/doc/development/new_fe_guide/style/index.md @@ -1,4 +1,4 @@ -# Style +# Style guides ## [HTML style guide](html.md) diff --git a/doc/development/writing_documentation.md b/doc/development/writing_documentation.md index 40c21e5355c..d6a13e7483a 100644 --- a/doc/development/writing_documentation.md +++ b/doc/development/writing_documentation.md @@ -1,13 +1,9 @@ -# Writing documentation +# GitLab Documentation guidelines - **General Documentation**: written by the [developers responsible by creating features](#contributing-to-docs). Should be submitted in the same merge request containing code. Feature proposals (by GitLab contributors) should also be accompanied by its respective documentation. They can be later improved by PMs and Technical Writers. - **[Technical Articles](#technical-articles)**: written by any [GitLab Team](https://about.gitlab.com/team/) member, GitLab contributors, or [Community Writers](https://about.gitlab.com/handbook/product/technical-writing/community-writers/). - **Indexes per topic**: initially prepared by the Technical Writing Team, and kept up-to-date by developers and PMs in the same merge request containing code. They gather all resources for that topic in a single page (user and admin documentation, articles, and third-party docs). -## Documentation style guidelines - -All the docs follow the same [styleguide](doc_styleguide.md). - ## Contributing to docs Whenever a feature is changed, updated, introduced, or deprecated, the merge @@ -29,23 +25,16 @@ Whenever you submit a merge request for the documentation, use the documentation Please check the [documentation workflow](https://about.gitlab.com/handbook/product/technical-writing/workflow/) before getting started. -### Documentation directory structure - -The documentation is structured based on the GitLab UI structure itself, -separated by [`user`](https://gitlab.com/gitlab-org/gitlab-ce/tree/master/doc/user), -[`administrator`](https://gitlab.com/gitlab-org/gitlab-ce/tree/master/doc/administration), and [`contributor`](https://gitlab.com/gitlab-org/gitlab-ce/tree/master/doc/development). - -To learn where to place a new document, check the [documentation style guide](doc_styleguide.md#location-and-naming-of-documents). +## Documentation structure -In order to have a [solid site structure](https://searchengineland.com/seo-benefits-developing-solid-site-structure-277456) for our documentation, -all docs should be linked. Every new document should be cross-linked to its related documentation, and linked from its topic-related index, when existent. +- Overview and use cases: what it is, why it is necessary, why one would use it +- Requirements: what do we need to get started +- Tutorial: how to set it up, how to use it -The directories `/workflow/`, `/gitlab-basics/`, `/university/`, and `/articles/` have -been deprecated and the majority their docs have been moved to their correct location -in small iterations. Don't create new docs in these folders. +Always link a new document from its topic-related index, otherwise, it will +not be included it in the documentation site search. -To move a document from its location to another directory, read the section -[changing document location](doc_styleguide.md#changing-document-location) of the doc style guide. +_Note: to be extended._ ### Feature overview and use cases @@ -75,16 +64,169 @@ overview there. > **Overview** and **use cases** are required to **every** Enterprise Edition feature, and for every **major** feature present in Community Edition. -### Markdown +## Markdown and styles Currently GitLab docs use Redcarpet as [markdown](../user/markdown.md) engine, but there's an [open discussion](https://gitlab.com/gitlab-com/gitlab-docs/issues/50) for implementing Kramdown in the near future. -### Previewing locally +All the docs follow the [documentation style guidelines](doc_styleguide.md). -To preview your changes to documentation locally, please follow -this [development guide](https://gitlab.com/gitlab-com/gitlab-docs/blob/master/README.md#development). +## Documentation directory structure + +The documentation is structured based on the GitLab UI structure itself, +separated by [`user`](https://gitlab.com/gitlab-org/gitlab-ce/tree/master/doc/user), +[`administrator`](https://gitlab.com/gitlab-org/gitlab-ce/tree/master/doc/administration), and [`contributor`](https://gitlab.com/gitlab-org/gitlab-ce/tree/master/doc/development). + +In order to have a [solid site structure](https://searchengineland.com/seo-benefits-developing-solid-site-structure-277456) for our documentation, +all docs should be linked. Every new document should be cross-linked to its related documentation, and linked from its topic-related index, when existent. + +The directories `/workflow/`, `/gitlab-basics/`, `/university/`, and `/articles/` have +been deprecated and the majority their docs have been moved to their correct location +in small iterations. Please don't create new docs in these folders. + +### Location and naming documents + +The documentation hierarchy can be vastly improved by providing a better layout +and organization of directories. + +Having a structured document layout, we will be able to have meaningful URLs +like `docs.gitlab.com/user/project/merge_requests/index.html`. With this pattern, +you can immediately tell that you are navigating a user related documentation +and is about the project and its merge requests. + +Do not create summaries of similar types of content (e.g. an index of all articles, videos, etc.), +rather organize content by its subject (e.g. everything related to CI goes together) +and cross-link between any related content. + +The table below shows what kind of documentation goes where. + +| Directory | What belongs here | +| --------- | -------------- | +| `doc/user/` | User related documentation. Anything that can be done within the GitLab UI goes here including `/admin`. | +| `doc/administration/` | Documentation that requires the user to have access to the server where GitLab is installed. The admin settings that can be accessed via GitLab's interface go under `doc/user/admin_area/`. | +| `doc/api/` | API related documentation. | +| `doc/development/` | Documentation related to the development of GitLab. Any styleguides should go here. | +| `doc/legal/` | Legal documents about contributing to GitLab. | +| `doc/install/`| Probably the most visited directory, since `installation.md` is there. Ideally this should go under `doc/administration/`, but it's best to leave it as-is in order to avoid confusion (still debated though). | +| `doc/update/` | Same with `doc/install/`. Should be under `administration/`, but this is a well known location, better leave as-is, at least for now. | +| `doc/topics/` | Indexes per Topic (`doc/topics/topic-name/index.md`): all resources for that topic (user and admin documentation, articles, and third-party docs) | + +--- + +**General rules:** + +1. The correct naming and location of a new document, is a combination + of the relative URL of the document in question and the GitLab Map design + that is used for UX purposes ([source][graffle], [image][gitlab-map]). +1. When creating a new document and it has more than one word in its name, + make sure to use underscores instead of spaces or dashes (`-`). For example, + a proper naming would be `import_projects_from_github.md`. The same rule + applies to images. +1. Start a new directory with an `index.md` file. +1. There are four main directories, `user`, `administration`, `api` and `development`. +1. The `doc/user/` directory has five main subdirectories: `project/`, `group/`, + `profile/`, `dashboard/` and `admin_area/`. + 1. `doc/user/project/` should contain all project related documentation. + 1. `doc/user/group/` should contain all group related documentation. + 1. `doc/user/profile/` should contain all profile related documentation. + Every page you would navigate under `/profile` should have its own document, + i.e. `account.md`, `applications.md`, `emails.md`, etc. + 1. `doc/user/dashboard/` should contain all dashboard related documentation. + 1. `doc/user/admin_area/` should contain all admin related documentation + describing what can be achieved by accessing GitLab's admin interface + (_not to be confused with `doc/administration` where server access is + required_). + 1. Every category under `/admin/application_settings` should have its + own document located at `doc/user/admin_area/settings/`. For example, + the **Visibility and Access Controls** category should have a document + located at `doc/user/admin_area/settings/visibility_and_access_controls.md`. +1. The `doc/topics/` directory holds topic-related technical content. Create + `doc/topics/topic-name/subtopic-name/index.md` when subtopics become necessary. + General user- and admin- related documentation, should be placed accordingly. + +If you are unsure where a document should live, you can ping `@axil` or `@marcia` in your +merge request. + +### Changing document location + +Changing a document's location is not to be taken lightly. Remember that the +documentation is available to all installations under `help/` and not only to +GitLab.com or http://docs.gitlab.com. Make sure this is discussed with the +Documentation team beforehand. + +If you indeed need to change a document's location, do NOT remove the old +document, but rather replace all of its contents with a new line: + +``` +This document was moved to [another location](path/to/new_doc.md). +``` + +where `path/to/new_doc.md` is the relative path to the root directory `doc/`. + +--- + +For example, if you were to move `doc/workflow/lfs/lfs_administration.md` to +`doc/administration/lfs.md`, then the steps would be: + +1. Copy `doc/workflow/lfs/lfs_administration.md` to `doc/administration/lfs.md` +1. Replace the contents of `doc/workflow/lfs/lfs_administration.md` with: + + ``` + This document was moved to [another location](../../administration/lfs.md). + ``` + +1. Find and replace any occurrences of the old location with the new one. + A quick way to find them is to use `git grep`. First go to the root directory + where you cloned the `gitlab-ce` repository and then do: -### Testing + ``` + git grep -n "workflow/lfs/lfs_administration" + git grep -n "lfs/lfs_administration" + ``` + +NOTE: **Note:** +If the document being moved has any Disqus comments on it, there are extra steps +to follow documented just [below](#redirections-for-pages-with-disqus-comments). + +Things to note: + +- Since we also use inline documentation, except for the documentation itself, + the document might also be referenced in the views of GitLab (`app/`) which will + render when visiting `/help`, and sometimes in the testing suite (`spec/`). +- The above `git grep` command will search recursively in the directory you run + it in for `workflow/lfs/lfs_administration` and `lfs/lfs_administration` + and will print the file and the line where this file is mentioned. + You may ask why the two greps. Since we use relative paths to link to + documentation, sometimes it might be useful to search a path deeper. +- The `*.md` extension is not used when a document is linked to GitLab's + built-in help page, that's why we omit it in `git grep`. +- Use the checklist on the documentation MR description template. + +### Redirections for pages with Disqus comments + +If the documentation page being relocated already has any Disqus comments, +we need to preserve the Disqus thread. + +Disqus uses an identifier per page, and for docs.gitlab.com, the page identifier +is configured to be the page URL. Therefore, when we change the document location, +we need to preserve the old URL as the same Disqus identifier. + +To do that, add to the frontmatter the variable `redirect_from`, +using the old URL as value. For example, let's say I moved the document +available under `https://docs.gitlab.com/my-old-location/README.html` to a new location, +`https://docs.gitlab.com/my-new-location/index.html`. + +Into the **new document** frontmatter add the following: + +```yaml +--- +redirect_from: 'https://docs.gitlab.com/my-old-location/README.html' +--- +``` + +Note: it is necessary to include the file name in the `redirect_from` URL, +even if it's `index.html` or `README.html`. + +## Testing We treat documentation as code, thus have implemented some testing. Currently, the following tests are in place: @@ -103,7 +245,7 @@ Currently, the following tests are in place: Submitting an EE-equivalent merge request cherry-picking all commits from CE to EE is essential to avoid them. -### Branch naming +## Branch naming If your contribution contains **only** documentation changes, you can speed up the CI process by following some branch naming conventions. You have three @@ -118,7 +260,7 @@ choices: If your branch name matches any of the above, it will run only the docs tests. If it doesn't, the whole test suite will run (including docs). -### Merge requests for GitLab documentation +## Merge requests for GitLab documentation Before getting started, make sure you read the introductory section "[contributing to docs](#contributing-to-docs)" above and the @@ -137,7 +279,7 @@ frozen or released, use the label `Pick into X.Y` to get it merged into the correct release. Avoid picking into a past release as much as you can, as it increases the work of the release managers. -#### Cherry-picking from CE to EE +### Cherry-picking from CE to EE As we have the `master` branch of CE merged into EE once a day, it's common to run into merge conflicts. To avoid them, we [test for merge conflicts against EE](#testing) @@ -161,7 +303,10 @@ a couple more commits to the EE branch, but ask the reviewer to review the EE me additionally to the CE MR. If there are many EE-only changes though, start a new MR to EE only. -### Previewing the changes live +## Previewing the changes live + +To preview your changes to documentation locally, please follow +this [development guide](https://gitlab.com/gitlab-com/gitlab-docs/blob/master/README.md#development). If you want to preview the doc changes of your merge request live, you can use the manual `review-docs-deploy` job in your merge request. You will need at @@ -221,7 +366,7 @@ working on. If you don't, the remote docs branch won't be removed either, and the server where the Review Apps are hosted will eventually be out of disk space. -#### Technical aspects +### Technical aspects If you want to know the hot details, here's what's really happening: @@ -256,6 +401,74 @@ The following GitLab features are used among others: - [Artifacts](../ci/yaml/README.md#artifacts) - [Specific Runner](../ci/runners/README.md#locking-a-specific-runner-from-being-enabled-for-other-projects) +## GitLab `/help` + +Every GitLab instance includes the documentation, which is available from `/help` +(`http://my-instance.com/help`), e.g., <https://gitlab.com/help>. + +When you're building a new feature, you may need to link the documentation +from GitLab, the application. This is normally done in files inside the +`app/views/` directory with the help of the `help_page_path` helper method. + +In its simplest form, the HAML code to generate a link to the `/help` page is: + +```haml += link_to 'Help page', help_page_path('user/permissions') +``` + +The `help_page_path` contains the path to the document you want to link to with +the following conventions: + +- it is relative to the `doc/` directory in the GitLab repository +- the `.md` extension must be omitted +- it must not end with a slash (`/`) + +Below are some special cases where should be used depending on the context. +You can combine one or more of the following: + +1. **Linking to an anchor link.** Use `anchor` as part of the `help_page_path` + method: + + ```haml + = link_to 'Help page', help_page_path('user/permissions', anchor: 'anchor-link') + ``` + +1. **Opening links in a new tab.** This should be the default behavior: + + ```haml + = link_to 'Help page', help_page_path('user/permissions'), target: '_blank' + ``` + +1. **Linking to a circle icon.** Usually used in settings where a long + description cannot be used, like near checkboxes. You can basically use + any font awesome icon, but prefer the `question-circle`: + + ```haml + = link_to icon('question-circle'), help_page_path('user/permissions') + ``` + +1. **Using a button link.** Useful in places where text would be out of context + with the rest of the page layout: + + ```haml + = link_to 'Help page', help_page_path('user/permissions'), class: 'btn btn-info' + ``` + +1. **Using links inline of some text.** + + ```haml + Description to #{link_to 'Help page', help_page_path('user/permissions')}. + ``` + +1. **Adding a period at the end of the sentence.** Useful when you don't want + the period to be part of the link: + + ```haml + = succeed '.' do + Learn more in the + = link_to 'Help page', help_page_path('user/permissions') + ``` + ## General Documentation vs Technical Articles ### General documentation @@ -270,7 +483,7 @@ They are topic-related documentation, written with an user-friendly approach and A technical article guides users and/or admins to achieve certain objectives (within guides and tutorials), or provide an overview of that particular topic or feature (within technical overviews). It can also describe the use, implementation, or integration of third-party tools with GitLab. -They should be placed in a new directory named `/article-title/index.md` under a topic-related folder, and their images should be placed in `/article-title/img/`. For example, a new article on GitLab Pages should be placed in `doc/user/project/pages/article-title/` and a new article on GitLab CI/CD should be placed in `doc/ci/article-title/`. +They should be placed in a new directory named `/article-title/index.md` under a topic-related folder, and their images should be placed in `/article-title/img/`. For example, a new article on GitLab Pages should be placed in `doc/user/project/pages/article-title/` and a new article on GitLab CI/CD should be placed in `doc/ci/examples/article-title/`. #### Types of Technical Articles @@ -324,3 +537,5 @@ date: 2017-02-01 Use the [writing method](https://about.gitlab.com/handbook/product/technical-writing/#writing-method) defined by the Technical Writing team. +[gitlab-map]: https://gitlab.com/gitlab-org/gitlab-design/raw/master/production/resources/gitlab-map.png +[graffle]: https://gitlab.com/gitlab-org/gitlab-design/blob/d8d39f4a87b90fb9ae89ca12dc565347b4900d5e/production/resources/gitlab-map.graffle diff --git a/doc/downgrade_ee_to_ce/README.md b/doc/downgrade_ee_to_ce/README.md index ff1ac94ac58..f656057e3da 100644 --- a/doc/downgrade_ee_to_ce/README.md +++ b/doc/downgrade_ee_to_ce/README.md @@ -70,7 +70,7 @@ To downgrade an Omnibus installation, it is sufficient to install the Community Edition package on top of the currently installed one. You can do this manually, by directly [downloading the package](https://packages.gitlab.com/gitlab/gitlab-ce) you need, or by adding our CE package repository and following the -[CE installation instructions](https://about.gitlab.com/downloads/?version=ce). +[CE installation instructions](https://about.gitlab.com/installation/?version=ce). **Source Installation** diff --git a/doc/topics/autodevops/index.md b/doc/topics/autodevops/index.md index ec091549c05..4dc3adc1441 100644 --- a/doc/topics/autodevops/index.md +++ b/doc/topics/autodevops/index.md @@ -20,7 +20,7 @@ project in an easy and automatic way: 1. [Auto Test](#auto-test) 1. [Auto Code Quality](#auto-code-quality) 1. [Auto SAST (Static Application Security Testing)](#auto-sast) -1. [Auto SAST for Docker images](#auto-sast-for-docker-images) +1. [Auto Container Scanning](#auto-container-scanning) 1. [Auto Review Apps](#auto-review-apps) 1. [Auto DAST (Dynamic Application Security Testing)](#auto-dast) 1. [Auto Deploy](#auto-deploy) @@ -217,7 +217,7 @@ check out. In GitLab Ultimate, any security warnings are also [shown in the merge request widget](https://docs.gitlab.com/ee/user/project/merge_requests/sast.html). -### Auto SAST for Docker images +### Auto Container Scanning > Introduced in GitLab 10.4. @@ -228,7 +228,7 @@ created, it's uploaded as an artifact which you can later download and check out. In GitLab Ultimate, any security warnings are also -[shown in the merge request widget](https://docs.gitlab.com/ee/user/project/merge_requests/sast_docker.html). +[shown in the merge request widget](https://docs.gitlab.com/ee/user/project/merge_requests/container_scanning.html). ### Auto Review Apps diff --git a/doc/user/project/repository/img/jupyter_notebook.png b/doc/user/project/repository/img/jupyter_notebook.png Binary files differnew file mode 100644 index 00000000000..52c5c5aea32 --- /dev/null +++ b/doc/user/project/repository/img/jupyter_notebook.png diff --git a/doc/user/project/repository/index.md b/doc/user/project/repository/index.md index e6aede7f46e..ae131d51305 100644 --- a/doc/user/project/repository/index.md +++ b/doc/user/project/repository/index.md @@ -53,6 +53,22 @@ To get started with the command line, please read through the Use GitLab's [file finder](../../../workflow/file_finder.md) to search for files in a repository. +### Jupyter Notebook files + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/issues/2508) in GitLab 9.1 + +[Jupyter][jupyter] Notebook (previously IPython Notebook) files are used for +interactive computing in many fields and contain a complete record of the +user's sessions and include code, narrative text, equations and rich output. + +When added to a repository, Jupyter Notebooks with a `.ipynb` extension will be +rendered to HTML when viewed. + + + +Interactive features, including JavaScript plots, will not work when viewed in +GitLab. + ## Branches When you submit changes in a new [branch](branches/index.md), you create a new version @@ -158,3 +174,5 @@ Lock your files to prevent any conflicting changes. ## Repository's API You can access your repos via [repository API](../../../api/repositories.md). + +[jupyter]: https://jupyter.org diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb index 6ec2626df1a..b0a7fd6f4ab 100644 --- a/lib/api/project_export.rb +++ b/lib/api/project_export.rb @@ -31,8 +31,13 @@ module API desc 'Start export' do detail 'This feature was introduced in GitLab 10.6.' end + params do + optional :description, type: String, desc: 'Override the project description' + end post ':id/export' do - user_project.add_export_job(current_user: current_user) + project_export_params = declared_params(include_missing: false) + + user_project.add_export_job(current_user: current_user, params: project_export_params) accepted! end diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index c7263f302ab..010ca1ec27b 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -52,6 +52,8 @@ module ContainerRegistry conn.request(:authorization, :bearer, options[:token].to_s) end + yield(conn) if block_given? + conn.adapter :net_http end @@ -80,8 +82,7 @@ module ContainerRegistry def faraday @faraday ||= Faraday.new(@base_uri) do |conn| - initialize_connection(conn, @options) - accept_manifest(conn) + initialize_connection(conn, @options, &method(:accept_manifest)) end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index b7c596a973d..e392a015b91 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -70,7 +70,7 @@ module Gitlab active_db_connection = ActiveRecord::Base.connection.active? rescue false active_db_connection && - ActiveRecord::Base.connection.table_exists?('application_settings') + Gitlab::Database.cached_table_exists?('application_settings') rescue ActiveRecord::NoDatabaseError false end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index e51794fef99..76501dd50e8 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -183,6 +183,15 @@ module Gitlab ActiveRecord::Base.connection end + def self.cached_column_exists?(table_name, column_name) + connection.schema_cache.columns_hash(table_name).has_key?(column_name.to_s) + end + + def self.cached_table_exists?(table_name) + # Rails 5 uses data_source_exists? instead of table_exists? + connection.schema_cache.table_exists?(table_name) + end + private_class_method :connection def self.database_version diff --git a/lib/gitlab/git/lfs_pointer_file.rb b/lib/gitlab/git/lfs_pointer_file.rb index da12ed7d125..2ae0a889590 100644 --- a/lib/gitlab/git/lfs_pointer_file.rb +++ b/lib/gitlab/git/lfs_pointer_file.rb @@ -1,13 +1,16 @@ module Gitlab module Git class LfsPointerFile + VERSION = "https://git-lfs.github.com/spec/v1".freeze + VERSION_LINE = "version #{VERSION}".freeze + def initialize(data) @data = data end def pointer @pointer ||= <<~FILE - version https://git-lfs.github.com/spec/v1 + #{VERSION_LINE} oid sha256:#{sha256} size #{size} FILE @@ -20,6 +23,10 @@ module Gitlab def sha256 @sha256 ||= Digest::SHA256.hexdigest(@data) end + + def inspect + "#<#{self.class}:#{object_id} @size=#{size}, @sha256=#{sha256.inspect}>" + end end end end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index fbc93542619..9811c447a01 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -1002,8 +1002,9 @@ module Gitlab # This only checks the root .gitattributes file, # it does not traverse subfolders to find additional .gitattributes files # - # This method is around 30 times slower than `attributes`, - # which uses `$GIT_DIR/info/attributes` + # This method is around 30 times slower than `attributes`, which uses + # `$GIT_DIR/info/attributes`. Consider caching AttributesAtRefParser + # and reusing that for multiple calls instead of this method. def attributes_at(ref, file_path) parser = AttributesAtRefParser.new(self, ref) parser.attributes(file_path) diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 4b5f9f3a926..8f5bb8f9597 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -35,6 +35,8 @@ module Gitlab end def restored_project + return @project unless @tree_hash + @restored_project ||= restore_project end @@ -81,9 +83,13 @@ module Gitlab end def restore_project - return @project unless @tree_hash + params = project_params + + if params[:description].present? + params[:description_html] = nil + end - @project.update_columns(project_params) + @project.update_columns(params) @project end diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb index 3473b466936..5510c0b8b2f 100644 --- a/lib/gitlab/import_export/project_tree_saver.rb +++ b/lib/gitlab/import_export/project_tree_saver.rb @@ -5,7 +5,8 @@ module Gitlab attr_reader :full_path - def initialize(project:, current_user:, shared:) + def initialize(project:, current_user:, shared:, params: {}) + @params = params @project = project @current_user = current_user @shared = shared @@ -25,6 +26,10 @@ module Gitlab private def project_json_tree + if @params[:description].present? + project_json['description'] = @params[:description] + end + project_json['project_members'] += group_members_json project_json.to_json diff --git a/spec/features/projects/import_export/import_file_spec.rb b/spec/features/projects/import_export/import_file_spec.rb index e8bb9c6a86c..b25f5161748 100644 --- a/spec/features/projects/import_export/import_file_spec.rb +++ b/spec/features/projects/import_export/import_file_spec.rb @@ -41,6 +41,7 @@ feature 'Import/Export - project import integration test', :js do project = Project.last expect(project).not_to be_nil + expect(project.description).to eq("Foo Bar") expect(project.issues).not_to be_empty expect(project.merge_requests).not_to be_empty expect(project_hook_exists?(project)).to be true diff --git a/spec/features/projects/import_export/test_project_export.tar.gz b/spec/features/projects/import_export/test_project_export.tar.gz Binary files differindex 0cc68aff494..ecb7651acad 100644 --- a/spec/features/projects/import_export/test_project_export.tar.gz +++ b/spec/features/projects/import_export/test_project_export.tar.gz diff --git a/spec/lib/gitlab/database_spec.rb b/spec/lib/gitlab/database_spec.rb index b2f13fae73f..1fe1d3926ad 100644 --- a/spec/lib/gitlab/database_spec.rb +++ b/spec/lib/gitlab/database_spec.rb @@ -287,6 +287,29 @@ describe Gitlab::Database do end end + describe '.cached_column_exists?' do + it 'only retrieves data once' do + expect(ActiveRecord::Base.connection).to receive(:columns).once.and_call_original + + 2.times do + expect(described_class.cached_column_exists?(:projects, :id)).to be_truthy + expect(described_class.cached_column_exists?(:projects, :bogus_column)).to be_falsey + end + end + end + + describe '.cached_table_exists?' do + it 'only retrieves data once per table' do + expect(ActiveRecord::Base.connection).to receive(:table_exists?).with(:projects).once.and_call_original + expect(ActiveRecord::Base.connection).to receive(:table_exists?).with(:bogus_table_name).once.and_call_original + + 2.times do + expect(described_class.cached_table_exists?(:projects)).to be_truthy + expect(described_class.cached_table_exists?(:bogus_table_name)).to be_falsey + end + end + end + describe '#true_value' do it 'returns correct value for PostgreSQL' do expect(described_class).to receive(:postgresql?).and_return(true) diff --git a/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb b/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb index 13e930bafe3..8e25cd26c2f 100644 --- a/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb +++ b/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb @@ -42,6 +42,10 @@ describe Gitlab::ImportExport::ProjectTreeRestorer do expect(project.project_feature.merge_requests_access_level).to eq(ProjectFeature::ENABLED) end + it 'has the project description' do + expect(Project.find_by_path('project').description).to eq('Nisi et repellendus ut enim quo accusamus vel magnam.') + end + it 'has the project html description' do expect(Project.find_by_path('project').description_html).to eq('description') end diff --git a/spec/lib/gitlab/import_export/project_tree_saver_spec.rb b/spec/lib/gitlab/import_export/project_tree_saver_spec.rb index 3049491f0ae..0d20a551e2a 100644 --- a/spec/lib/gitlab/import_export/project_tree_saver_spec.rb +++ b/spec/lib/gitlab/import_export/project_tree_saver_spec.rb @@ -29,8 +29,17 @@ describe Gitlab::ImportExport::ProjectTreeSaver do project_json(project_tree_saver.full_path) end + context 'with description override' do + let(:params) { { description: 'Foo Bar' } } + let(:project_tree_saver) { described_class.new(project: project, current_user: user, shared: shared, params: params) } + + it 'overrides the project description' do + expect(saved_project_json).to include({ 'description' => params[:description] }) + end + end + it 'saves the correct json' do - expect(saved_project_json).to include({ "visibility_level" => 20 }) + expect(saved_project_json).to include({ 'description' => 'description', 'visibility_level' => 20 }) end it 'has milestones' do @@ -259,6 +268,7 @@ describe Gitlab::ImportExport::ProjectTreeSaver do :issues_disabled, :wiki_enabled, :builds_private, + description: 'description', issues: [issue], snippets: [snippet], releases: [release], diff --git a/spec/requests/api/project_export_spec.rb b/spec/requests/api/project_export_spec.rb index fbed527963f..12583109b59 100644 --- a/spec/requests/api/project_export_spec.rb +++ b/spec/requests/api/project_export_spec.rb @@ -285,6 +285,17 @@ describe API::ProjectExport do context 'when user is not a member' do it_behaves_like 'post project export start not found' end + + context 'when overriding description' do + it 'starts' do + params = { description: "Foo" } + + expect_any_instance_of(Projects::ImportExport::ExportService).to receive(:execute) + post api(path, project.owner), params + + expect(response).to have_gitlab_http_status(202) + end + end end end end diff --git a/spec/services/files/create_service_spec.rb b/spec/services/files/create_service_spec.rb index 030263b1502..abe99b9e794 100644 --- a/spec/services/files/create_service_spec.rb +++ b/spec/services/files/create_service_spec.rb @@ -43,7 +43,7 @@ describe Files::CreateService do blob = repository.blob_at('lfs', file_path) - expect(blob.data).not_to start_with('version https://git-lfs.github.com/spec/v1') + expect(blob.data).not_to start_with(Gitlab::Git::LfsPointerFile::VERSION_LINE) expect(blob.data).to eq(file_content) end end @@ -58,7 +58,7 @@ describe Files::CreateService do blob = repository.blob_at('lfs', file_path) - expect(blob.data).to start_with('version https://git-lfs.github.com/spec/v1') + expect(blob.data).to start_with(Gitlab::Git::LfsPointerFile::VERSION_LINE) end it "creates an LfsObject with the file's content" do diff --git a/spec/services/files/multi_service_spec.rb b/spec/services/files/multi_service_spec.rb index b9971776b33..59984c10990 100644 --- a/spec/services/files/multi_service_spec.rb +++ b/spec/services/files/multi_service_spec.rb @@ -4,28 +4,30 @@ describe Files::MultiService do subject { described_class.new(project, user, commit_params) } let(:project) { create(:project, :repository) } + let(:repository) { project.repository } let(:user) { create(:user) } let(:branch_name) { project.default_branch } let(:original_file_path) { 'files/ruby/popen.rb' } let(:new_file_path) { 'files/ruby/popen.rb' } + let(:file_content) { 'New content' } let(:action) { 'update' } let!(:original_commit_id) do Gitlab::Git::Commit.last_for_path(project.repository, branch_name, original_file_path).sha end - let(:actions) do - [ - { - action: action, - file_path: new_file_path, - previous_path: original_file_path, - content: 'New content', - last_commit_id: original_commit_id - } - ] + let(:default_action) do + { + action: action, + file_path: new_file_path, + previous_path: original_file_path, + content: file_content, + last_commit_id: original_commit_id + } end + let(:actions) { [default_action] } + let(:commit_params) do { commit_message: "Update File", @@ -110,6 +112,56 @@ describe Files::MultiService do end end + context 'when creating a file matching an LFS filter' do + let(:action) { 'create' } + let(:branch_name) { 'lfs' } + let(:new_file_path) { 'test_file.lfs' } + + before do + allow(project).to receive(:lfs_enabled?).and_return(true) + end + + it 'creates an LFS pointer' do + subject.execute + + blob = repository.blob_at('lfs', new_file_path) + + expect(blob.data).to start_with(Gitlab::Git::LfsPointerFile::VERSION_LINE) + end + + it "creates an LfsObject with the file's content" do + subject.execute + + expect(LfsObject.last.file.read).to eq file_content + end + + context 'with base64 encoded content' do + let(:raw_file_content) { 'Raw content' } + let(:file_content) { Base64.encode64(raw_file_content) } + let(:actions) { [default_action.merge(encoding: 'base64')] } + + it 'creates an LFS pointer' do + subject.execute + + blob = repository.blob_at('lfs', new_file_path) + + expect(blob.data).to start_with(Gitlab::Git::LfsPointerFile::VERSION_LINE) + end + + it "creates an LfsObject with the file's content" do + subject.execute + + expect(LfsObject.last.file.read).to eq raw_file_content + end + end + + it 'links the LfsObject to the project' do + expect do + subject.execute + end.to change { project.lfs_objects.count }.by(1) + end + end + context 'when file status validation is skipped' do let(:action) { 'create' } let(:new_file_path) { 'files/ruby/new_file.rb' } diff --git a/spec/services/lfs/file_transformer_spec.rb b/spec/services/lfs/file_transformer_spec.rb new file mode 100644 index 00000000000..e8938338cb7 --- /dev/null +++ b/spec/services/lfs/file_transformer_spec.rb @@ -0,0 +1,97 @@ +require "spec_helper" + +describe Lfs::FileTransformer do + let(:project) { create(:project, :repository) } + let(:repository) { project.repository } + let(:file_content) { 'Test file content' } + let(:branch_name) { 'lfs' } + let(:file_path) { 'test_file.lfs' } + + subject { described_class.new(project, branch_name) } + + describe '#new_file' do + context 'with lfs disabled' do + it 'skips gitattributes check' do + expect(repository.raw).not_to receive(:blob_at) + + subject.new_file(file_path, file_content) + end + + it 'returns untransformed content' do + result = subject.new_file(file_path, file_content) + + expect(result.content).to eq(file_content) + end + + it 'returns untransformed encoding' do + result = subject.new_file(file_path, file_content, encoding: 'base64') + + expect(result.encoding).to eq('base64') + end + end + + context 'with lfs enabled' do + before do + allow(project).to receive(:lfs_enabled?).and_return(true) + end + + it 'reuses cached gitattributes' do + second_file = 'another_file.lfs' + + expect(repository.raw).to receive(:blob_at).with(branch_name, '.gitattributes').once + + subject.new_file(file_path, file_content) + subject.new_file(second_file, file_content) + end + + it "creates an LfsObject with the file's content" do + subject.new_file(file_path, file_content) + + expect(LfsObject.last.file.read).to eq file_content + end + + it 'returns an LFS pointer' do + result = subject.new_file(file_path, file_content) + + expect(result.content).to start_with(Gitlab::Git::LfsPointerFile::VERSION_LINE) + end + + it 'returns LFS pointer encoding as text' do + result = subject.new_file(file_path, file_content, encoding: 'base64') + + expect(result.encoding).to eq('text') + end + + context "when doesn't use LFS" do + let(:file_path) { 'other.filetype' } + + it "doesn't create LFS pointers" do + new_content = subject.new_file(file_path, file_content).content + + expect(new_content).not_to start_with(Gitlab::Git::LfsPointerFile::VERSION_LINE) + expect(new_content).to eq(file_content) + end + end + + it 'links LfsObjects to project' do + expect do + subject.new_file(file_path, file_content) + end.to change { project.lfs_objects.count }.by(1) + end + + context 'when LfsObject already exists' do + let(:lfs_pointer) { Gitlab::Git::LfsPointerFile.new(file_content) } + + before do + create(:lfs_object, oid: lfs_pointer.sha256, size: lfs_pointer.size) + end + + it 'links LfsObjects to project' do + expect do + subject.new_file(file_path, file_content) + end.to change { project.lfs_objects.count }.by(1) + end + end + end + end +end diff --git a/vendor/gitlab-ci-yml/Auto-DevOps.gitlab-ci.yml b/vendor/gitlab-ci-yml/Auto-DevOps.gitlab-ci.yml index 33f9efc1490..4223dc18933 100644 --- a/vendor/gitlab-ci-yml/Auto-DevOps.gitlab-ci.yml +++ b/vendor/gitlab-ci-yml/Auto-DevOps.gitlab-ci.yml @@ -285,8 +285,8 @@ production: export CI_APPLICATION_TAG=$CI_COMMIT_SHA export CI_CONTAINER_NAME=ci_job_build_${CI_JOB_ID} export TILLER_NAMESPACE=$KUBE_NAMESPACE - # Extract "MAJOR.MINOR" from CI_SERVER_VERSION and generate "MAJOR-MINOR-stable" for Static Code Analysis - export SCA_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') + # Extract "MAJOR.MINOR" from CI_SERVER_VERSION and generate "MAJOR-MINOR-stable" for Security Products + export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/') function sast_container() { if [[ -n "$CI_REGISTRY_USER" ]]; then @@ -307,11 +307,10 @@ production: } function codeclimate() { - docker run --env CODECLIMATE_CODE="$PWD" \ + docker run --env SOURCE_CODE="$PWD" \ --volume "$PWD":/code \ --volume /var/run/docker.sock:/var/run/docker.sock \ - --volume /tmp/cc:/tmp/cc \ - "registry.gitlab.com/gitlab-org/security-products/codequality/codeclimate:${SCA_VERSION}" analyze -f json > codeclimate.json + "registry.gitlab.com/gitlab-org/security-products/codequality:$SP_VERSION" /code } function sast() { @@ -328,7 +327,7 @@ production: --env SAST_DISABLE_REMOTE_CHECKS="${SAST_DISABLE_REMOTE_CHECKS:-false}" \ --volume "$PWD:/code" \ --volume /var/run/docker.sock:/var/run/docker.sock \ - "registry.gitlab.com/gitlab-org/security-products/sast:$SCA_VERSION" /app/bin/run /code + "registry.gitlab.com/gitlab-org/security-products/sast:$SP_VERSION" /app/bin/run /code ;; *) echo "GitLab EE is required" |
