diff options
27 files changed, 308 insertions, 21 deletions
diff --git a/app/assets/javascripts/vue_shared/components/file_row.vue b/app/assets/javascripts/vue_shared/components/file_row.vue index f54033efc54..0cbcdbf2eb4 100644 --- a/app/assets/javascripts/vue_shared/components/file_row.vue +++ b/app/assets/javascripts/vue_shared/components/file_row.vue @@ -136,6 +136,7 @@ export default { <div v-else :class="fileClass" + :title="file.name" class="file-row" role="button" @click="clickFile" diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index acef5d2e643..372f6d678f6 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -687,9 +687,18 @@ module Ci end end + # Returns the modified paths. + # + # The returned value is + # * Array: List of modified paths that should be evaluated + # * nil: Modified path can not be evaluated def modified_paths strong_memoize(:modified_paths) do - push_details.modified_paths + if merge_request? + merge_request.modified_paths + elsif branch_updated? + push_details.modified_paths + end end end diff --git a/app/models/releases/link.rb b/app/models/releases/link.rb index 6f639e5a7b2..6c507c47752 100644 --- a/app/models/releases/link.rb +++ b/app/models/releases/link.rb @@ -6,7 +6,7 @@ module Releases belongs_to :release - validates :url, presence: true, url: true, uniqueness: { scope: :release } + validates :url, presence: true, url: { protocols: %w(http https ftp) }, uniqueness: { scope: :release } validates :name, presence: true, uniqueness: { scope: :release } scope :sorted, -> { order(created_at: :desc) } diff --git a/changelogs/unreleased/50559-add-milestone-progress-to-api.yml b/changelogs/unreleased/50559-add-milestone-progress-to-api.yml new file mode 100644 index 00000000000..e68e4bd6059 --- /dev/null +++ b/changelogs/unreleased/50559-add-milestone-progress-to-api.yml @@ -0,0 +1,5 @@ +--- +title: 'API: Expose milestone progress' +merge_request: 25173 +author: Robert Schilling +type: added diff --git a/changelogs/unreleased/56237-api-truncated-commit-title.yml b/changelogs/unreleased/56237-api-truncated-commit-title.yml new file mode 100644 index 00000000000..1a48d0fda1b --- /dev/null +++ b/changelogs/unreleased/56237-api-truncated-commit-title.yml @@ -0,0 +1,5 @@ +--- +title: 'API: Expose full commit title' +merge_request: 25189 +author: Robert Schilling +type: fixed diff --git a/changelogs/unreleased/57410-api-create-release-link-with-ftp-address-return-400-bad-request.yml b/changelogs/unreleased/57410-api-create-release-link-with-ftp-address-return-400-bad-request.yml new file mode 100644 index 00000000000..6be6a2115b9 --- /dev/null +++ b/changelogs/unreleased/57410-api-create-release-link-with-ftp-address-return-400-bad-request.yml @@ -0,0 +1,5 @@ +--- +title: Add support for FTP assets for releases +merge_request: 25071 +author: Robert Schilling +type: added diff --git a/changelogs/unreleased/add-title-attribute-to-file-row.yml b/changelogs/unreleased/add-title-attribute-to-file-row.yml new file mode 100644 index 00000000000..c68d3d544e7 --- /dev/null +++ b/changelogs/unreleased/add-title-attribute-to-file-row.yml @@ -0,0 +1,5 @@ +--- +title: add title attribute to display file name +merge_request: 25154 +author: Satoshi Nakamatsu @satoshicano +type: added diff --git a/changelogs/unreleased/support-only-changes-on-mr-pipelines.yml b/changelogs/unreleased/support-only-changes-on-mr-pipelines.yml new file mode 100644 index 00000000000..fbab898b799 --- /dev/null +++ b/changelogs/unreleased/support-only-changes-on-mr-pipelines.yml @@ -0,0 +1,5 @@ +--- +title: 'Support `only: changes:` on MR pipelines' +merge_request: 24490 +author: Hiroyuki Sato +type: added diff --git a/doc/administration/pages/index.md b/doc/administration/pages/index.md index 10ae8c7dedf..5c809f25fbd 100644 --- a/doc/administration/pages/index.md +++ b/doc/administration/pages/index.md @@ -11,7 +11,7 @@ description: 'Learn how to administer GitLab Pages.' > - This guide is for Omnibus GitLab installations. If you have installed > GitLab from source, follow the [Pages source installation document](source.md). > - To learn how to use GitLab Pages, read the [user documentation][pages-userguide]. -> - Does NOT support subgroups. See [this issue](https://gitlab.com/gitlab-org/gitlab-ce/issues/30548) for more information and status. +> - Support for subgroup project's websites was [introduced](https://gitlab.com/gitlab-org/gitlab-ce/issues/30548) in GitLab 11.8. This document describes how to set up the _latest_ GitLab Pages feature. Make sure to read the [changelog](#changelog) if you are upgrading to a new GitLab diff --git a/doc/api/group_milestones.md b/doc/api/group_milestones.md index 7be01ce9c6d..920acaf3e49 100644 --- a/doc/api/group_milestones.md +++ b/doc/api/group_milestones.md @@ -42,6 +42,7 @@ Example Response: "due_date": "2013-11-29", "start_date": "2013-11-10", "state": "active", + "percentage_complete" : 66, "updated_at": "2013-10-02T09:24:18Z", "created_at": "2013-10-02T09:24:18Z" } diff --git a/doc/api/milestones.md b/doc/api/milestones.md index fa8f8a0bcf0..21a390442bd 100644 --- a/doc/api/milestones.md +++ b/doc/api/milestones.md @@ -39,6 +39,7 @@ Example Response: "due_date": "2013-11-29", "start_date": "2013-11-10", "state": "active", + "percentage_complete" : 66, "updated_at": "2013-10-02T09:24:18Z", "created_at": "2013-10-02T09:24:18Z" } diff --git a/doc/api/releases/links.md b/doc/api/releases/links.md index ae99f3bd8b6..fd7b9d6e6e2 100644 --- a/doc/api/releases/links.md +++ b/doc/api/releases/links.md @@ -3,6 +3,7 @@ > [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/issues/41766) in GitLab 11.7. Using this API you can manipulate GitLab's [Release](../../user/project/releases/index.md) links. For manipulating other Release assets, see [Release API](index.md). +GitLab supports links links to `http`, `https`, and `ftp` assets. ## Get links diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index 984878b6c9b..8c8a31e9323 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -423,10 +423,28 @@ connected with merge requests yet, and because GitLab is creating pipelines before an user can create a merge request we don't know a target branch at this point. -Without a target branch, it is not possible to know what the common ancestor is, -thus we always create a job in that case. This feature works best for stable -branches like `master` because in that case GitLab uses the previous commit -that is present in a branch to compare against the latest SHA that was pushed. +#### Using `changes` with `merge_requests` + +With [pipelines for merge requests](../merge_request_pipelines/index.md), +make it possible to define if a job should be created base on files modified +in a merge request. + +For example: + +``` +docker build service one: + script: docker build -t my-service-one-image:$CI_COMMIT_REF_SLUG . + only: + refs: + - merge_requests + changes: + - Dockerfile + - service-one/**/* +``` + +In the scenario above, if you create or update a merge request that changes +either files in `service-one` folder or `Dockerfile`, GitLab creates and triggers +the `docker build service one` job. ## `tags` diff --git a/doc/development/README.md b/doc/development/README.md index d5829e31343..13646cbfe48 100644 --- a/doc/development/README.md +++ b/doc/development/README.md @@ -49,6 +49,7 @@ description: 'Learn how to contribute to GitLab.' - [Working with the GitHub importer](github_importer.md) - [Import/Export development documentation](import_export.md) - [Working with Merge Request diffs](diffs.md) +- [Kubernetes integration guidelines](kubernetes.md) - [Permissions](permissions.md) - [Prometheus metrics](prometheus_metrics.md) - [Guidelines for reusing abstractions](reusing_abstractions.md) diff --git a/doc/development/documentation/styleguide.md b/doc/development/documentation/styleguide.md index cda66447c2c..7a3a8f25c2d 100644 --- a/doc/development/documentation/styleguide.md +++ b/doc/development/documentation/styleguide.md @@ -112,7 +112,7 @@ table_display_block: true ## Emphasis - Use double asterisks (`**`) to mark a word or text in bold (`**bold**`). -- Use undescore (`_`) for text in italics (`_italic_`). +- Use underscore (`_`) for text in italics (`_italic_`). - Use greater than (`>`) for blockquotes. ## Punctuation diff --git a/doc/development/kubernetes.md b/doc/development/kubernetes.md new file mode 100644 index 00000000000..4b2d48903ac --- /dev/null +++ b/doc/development/kubernetes.md @@ -0,0 +1,126 @@ +# Kubernetes integration - development guidelines + +This document provides various guidelines when developing for GitLab's +[Kubernetes integration](../user/project/clusters/index.md). + +## Development + +### Architecture + +Some Kubernetes operations, such as creating restricted project +namespaces are performed on the GitLab Rails application. These +operations are performed using a [client library](#client-library). +These operations will carry an element of risk as the operations will be +run as the same user running the GitLab Rails application, see the +[security](#security) section below. + +Some Kubernetes operations, such as installing cluster applications are +performed on one-off pods on the Kubernetes cluster itself. These +installation pods are currently named `install-<application_name>` and +are created within the `gitlab-managed-apps` namespace. + +In terms of code organization, we generally add objects that represent +Kubernetes resources in +[`lib/gitlab/kubernetes`](https://gitlab.com/gitlab-org/gitlab-ce/tree/master/lib/gitlab/kubernetes). + +### Client library + +We use the [`kubeclient`](https://rubygems.org/gems/kubeclient) gem to +perform Kubernetes API calls. As the `kubeclient` gem does not support +different API Groups (e.g. `apis/rbac.authorization.k8s.io`) from a +single client, we have created a wrapper class, +[`Gitlab::Kubernetes::KubeClient`](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/kubernetes/kube_client.rb) +that will enable you to achieve this. + +Selected Kubernetes API groups are currently supported. Do add support +for new API groups or methods to +[`Gitlab::Kubernetes::KubeClient`](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/kubernetes/kube_client.rb) +if you need to use them. New API groups or API group versions can be +added to `SUPPORTED_API_GROUPS` - internally, this will create an +internal client for that group. New methods can be added as a delegation +to the relevant internal client. + +### Performance considerations + +All calls to the Kubernetes API must be in a background process. Do not +perform Kubernetes API calls within a web request as this will block +unicorn and can easily lead to a Denial Of Service (DoS) attack in GitLab as +the Kubernetes cluster response times are outside of our control. + +The easiest way to ensure your calls happen a background process is to +delegate any such work to happen in a [sidekiq +worker](sidekiq_style_guide.md). + +There are instances where you would like to make calls to Kubernetes and +return the response and as such a background worker does not seem to be +a good fit. For such cases you should make use of [reactive +caching](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/app/models/concerns/reactive_caching.rb). +For example: + +```ruby + def calculate_reactive_cache! + { pods: cluster.platform_kubernetes.kubeclient.get_pods } + end + + def pods + with_reactive_cache do |data| + data[:pods] + end + end +``` + +### Testing + +We have some Webmock stubs in +[`KubernetesHelpers`](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/spec/support/helpers/kubernetes_helpers.rb) +which can help with mocking out calls to Kubernetes API in your tests. + +## Security + +### SSRF + +As URLs for Kubernetes clusters are user controlled it is easily +susceptible to Server Side Request Forgery (SSRF) attacks. You should +understand the mitigation strategies if you are adding more API calls to +a cluster. + +Mitigation strategies include: + +1. Not allowing redirects to attacker controller resources: + [`Kubeclient::KubeClient`](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/lib/gitlab/kubernetes/kube_client.rb#) + can be configured to disallow any redirects by passing in + `http_max_redirects: 0` as an option. +1. Not exposing error messages: by doing so, we + prevent attackers from triggering errors to expose results from + attacker controlled requests. For example, we do not expose (or store) + raw error messages: + + ```ruby + rescue Kubernetes::HttpError => e + # bad + # app.make_errored!("Kubernetes error: #{e.message}") + + # good + app.make_errored!("Kubernetes error: #{e.error_code}") + ``` + +## Debugging + +Logs related to the Kubernetes integration can be found in +[kubernetes.log](../administration/logs.md#kuberneteslog). On a local +GDK install, this will be present in `log/kubernetes.log`. + +Some services such as +[`Clusters::Applications::InstallService`](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/app/services/clusters/applications/install_service.rb#L18) +rescues `StandardError` which can make it harder to debug issues in an +development environment. The current workaround is to temporarily +comment out the `rescue` in your local development source. + +You can also follow the installation pod logs to debug issues related to +installation. Once the installation/upgrade is underway, wait for the +pod to be created. Then run the following to obtain the pods logs as +they are written: + +```bash +kubectl logs <pod_name> --follow -n gitlab-managed-apps +``` diff --git a/doc/user/project/pages/index.md b/doc/user/project/pages/index.md index 11f6165fcb4..2de3fb7e080 100644 --- a/doc/user/project/pages/index.md +++ b/doc/user/project/pages/index.md @@ -66,7 +66,7 @@ publish any website written directly in plain HTML, CSS, and JavaScript.</p> If you're using GitLab.com, your website will be publicly available to the internet. If you're using self-managed instances (Core, Starter, Premium, or Ultimate), your websites will be published on your own server, according to the -[Pages admin settings](../../../administration/pages/index.md) chosen by your sysdamin, +[Pages admin settings](../../../administration/pages/index.md) chosen by your sysadmin, who can opt for making them public or internal to your server. ### How it works diff --git a/doc/user/project/web_ide/index.md b/doc/user/project/web_ide/index.md index 9c095da5a4b..3e85e97d7a5 100644 --- a/doc/user/project/web_ide/index.md +++ b/doc/user/project/web_ide/index.md @@ -50,7 +50,7 @@ review the list of changed files. Click on each file to review the changes and click the tick icon to stage the file. Once you have staged some changes, you can add a commit message and commit the -staged changes. Unstaged changes will not be commited. +staged changes. Unstaged changes will not be committed.  diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 0246f36a685..0ef56067b95 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -369,8 +369,9 @@ module API end class Commit < Grape::Entity - expose :id, :short_id, :title, :created_at + expose :id, :short_id, :created_at expose :parent_ids + expose :full_title, as: :title expose :safe_message, as: :message expose :author_name, :author_email, :authored_date expose :committer_name, :committer_email, :committed_date @@ -501,6 +502,9 @@ module API expose :state, :created_at, :updated_at expose :due_date expose :start_date + expose :percentage_complete do |milestone, options| + milestone.percent_complete(options[:current_user]) + end expose :web_url do |milestone, _options| Gitlab::UrlBuilder.build(milestone) diff --git a/lib/api/milestone_responses.rb b/lib/api/milestone_responses.rb index a0ca39b69d4..4c68c568aaa 100644 --- a/lib/api/milestone_responses.rb +++ b/lib/api/milestone_responses.rb @@ -35,19 +35,19 @@ module API milestones = filter_by_iid(milestones, params[:iids]) if params[:iids].present? milestones = filter_by_search(milestones, params[:search]) if params[:search] - present paginate(milestones), with: Entities::Milestone + present paginate(milestones), with: Entities::Milestone, current_user: current_user end def get_milestone_for(parent) milestone = parent.milestones.find(params[:milestone_id]) - present milestone, with: Entities::Milestone + present milestone, with: Entities::Milestone, current_user: current_user end def create_milestone_for(parent) milestone = ::Milestones::CreateService.new(parent, current_user, declared_params).execute if milestone.valid? - present milestone, with: Entities::Milestone + present milestone, with: Entities::Milestone, current_user: current_user else render_api_error!("Failed to create milestone #{milestone.errors.messages}", 400) end @@ -60,7 +60,7 @@ module API milestone = ::Milestones::UpdateService.new(parent, current_user, milestone_params).execute(milestone) if milestone.valid? - present milestone, with: Entities::Milestone + present milestone, with: Entities::Milestone, current_user: current_user else render_api_error!("Failed to update milestone #{milestone.errors.messages}", 400) end diff --git a/lib/gitlab/ci/build/policy/changes.rb b/lib/gitlab/ci/build/policy/changes.rb index 1663c875426..9c705a1cd3e 100644 --- a/lib/gitlab/ci/build/policy/changes.rb +++ b/lib/gitlab/ci/build/policy/changes.rb @@ -10,7 +10,7 @@ module Gitlab end def satisfied_by?(pipeline, seed) - return true unless pipeline.branch_updated? + return true if pipeline.modified_paths.nil? pipeline.modified_paths.any? do |path| @globs.any? do |glob| diff --git a/spec/fixtures/api/schemas/public_api/v4/milestone.json b/spec/fixtures/api/schemas/public_api/v4/milestone.json index 6ca2e88ae91..971f7980f46 100644 --- a/spec/fixtures/api/schemas/public_api/v4/milestone.json +++ b/spec/fixtures/api/schemas/public_api/v4/milestone.json @@ -8,6 +8,7 @@ "title": { "type": "string" }, "description": { "type": ["string", "null"] }, "state": { "type": "string" }, + "percentage_complete": { "type": "integer" }, "created_at": { "type": "date" }, "updated_at": { "type": "date" }, "start_date": { "type": "date" }, diff --git a/spec/lib/gitlab/ci/build/policy/changes_spec.rb b/spec/lib/gitlab/ci/build/policy/changes_spec.rb index 5fee37bb43e..dc3329061d1 100644 --- a/spec/lib/gitlab/ci/build/policy/changes_spec.rb +++ b/spec/lib/gitlab/ci/build/policy/changes_spec.rb @@ -73,9 +73,9 @@ describe Gitlab::Ci::Build::Policy::Changes do expect(policy).not_to be_satisfied_by(pipeline, seed) end - context 'when pipelines does not run for a branch update' do + context 'when modified paths can not be evaluated' do before do - pipeline.before_sha = Gitlab::Git::BLANK_SHA + allow(pipeline).to receive(:modified_paths) { nil } end it 'is always satisfied' do @@ -115,5 +115,57 @@ describe Gitlab::Ci::Build::Policy::Changes do expect(policy).not_to be_satisfied_by(pipeline, seed) end end + + context 'when branch is created' do + let(:pipeline) do + create(:ci_empty_pipeline, project: project, + ref: 'feature', + source: source, + sha: '0b4bc9a4', + before_sha: Gitlab::Git::BLANK_SHA, + merge_request: merge_request) + end + + let(:ci_build) do + build(:ci_build, pipeline: pipeline, project: project, ref: 'feature') + end + + let(:seed) { double('build seed', to_resource: ci_build) } + + context 'when source is merge request' do + let(:source) { :merge_request } + + let(:merge_request) do + create(:merge_request, + source_project: project, + source_branch: 'feature', + target_project: project, + target_branch: 'master') + end + + it 'is satified by changes in the merge request' do + policy = described_class.new(%w[files/ruby/feature.rb]) + + expect(policy).to be_satisfied_by(pipeline, seed) + end + + it 'is not satified by changes not in the merge request' do + policy = described_class.new(%w[foo.rb]) + + expect(policy).not_to be_satisfied_by(pipeline, seed) + end + end + + context 'when source is push' do + let(:source) { :push } + let(:merge_request) { nil } + + it 'is always satified' do + policy = described_class.new(%w[foo.rb]) + + expect(policy).to be_satisfied_by(pipeline, seed) + end + end + end end end diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb index 72a0df96a80..460b5c8cd31 100644 --- a/spec/models/ci/pipeline_spec.rb +++ b/spec/models/ci/pipeline_spec.rb @@ -1172,8 +1172,26 @@ describe Ci::Pipeline, :mailer do pipeline.update_column(:before_sha, Gitlab::Git::BLANK_SHA) end - it 'raises an error' do - expect { pipeline.modified_paths }.to raise_error(ArgumentError) + it 'returns nil' do + expect(pipeline.modified_paths).to be_nil + end + end + + context 'when source is merge request' do + let(:pipeline) do + create(:ci_pipeline, source: :merge_request, merge_request: merge_request) + end + + let(:merge_request) do + create(:merge_request, + source_project: project, + source_branch: 'feature', + target_project: project, + target_branch: 'master') + end + + it 'returns merge request modified paths' do + expect(pipeline.modified_paths).to match(merge_request.modified_paths) end end end diff --git a/spec/models/releases/link_spec.rb b/spec/models/releases/link_spec.rb index 06ed1438688..4dd26c976cc 100644 --- a/spec/models/releases/link_spec.rb +++ b/spec/models/releases/link_spec.rb @@ -77,4 +77,28 @@ describe Releases::Link do it { is_expected.to be_truthy } end + + describe 'supported protocols' do + where(:protocol) do + %w(http https ftp) + end + + with_them do + let(:link) { build(:release_link, url: protocol + '://assets.com/download') } + + it 'will be valid' do + expect(link).to be_valid + end + end + end + + describe 'unsupported protocol' do + context 'for torrent' do + let(:link) { build(:release_link, url: 'torrent://assets.com/download') } + + it 'will be invalid' do + expect(link).to be_invalid + end + end + end end diff --git a/spec/requests/api/commits_spec.rb b/spec/requests/api/commits_spec.rb index 6b9bc6eda6a..bc892523cf8 100644 --- a/spec/requests/api/commits_spec.rb +++ b/spec/requests/api/commits_spec.rb @@ -237,7 +237,7 @@ describe API::Commits do end describe 'create' do - let(:message) { 'Created file' } + let(:message) { 'Created a new file with a very very looooooooooooooooooooooooooooooooooooooooooooooong commit message' } let(:invalid_c_params) do { branch: 'master', diff --git a/spec/support/api/milestones_shared_examples.rb b/spec/support/api/milestones_shared_examples.rb index 5f709831ce1..b426fadb001 100644 --- a/spec/support/api/milestones_shared_examples.rb +++ b/spec/support/api/milestones_shared_examples.rb @@ -8,12 +8,17 @@ shared_examples_for 'group and project milestones' do |route_definition| describe "GET #{route_definition}" do it 'returns milestones list' do + create(:issue, project: project, milestone: milestone) + create(:closed_issue, project: project, milestone: milestone) + create(:closed_issue, project: project, milestone: milestone) + get api(route, user) expect(response).to have_gitlab_http_status(200) expect(response).to include_pagination_headers expect(json_response).to be_an Array expect(json_response.first['title']).to eq(milestone.title) + expect(json_response.first['percentage_complete']).to eq(66) end it 'returns a 401 error if user not authenticated' do |