From 327e344417888abbbc0811a8068c96c1af6fd326 Mon Sep 17 00:00:00 2001 From: mhasbini Date: Wed, 12 Apr 2017 14:38:00 +0300 Subject: Add issues/:iid/closed_by api endpoint --- changelogs/unreleased/26437-closed-by.yml | 4 ++ doc/api/issues.md | 61 +++++++++++++++++++++++++ lib/api/issues.rb | 15 ++++++ spec/factories/merge_requests_closing_issues.rb | 6 +++ spec/features/issuables/issuable_list_spec.rb | 2 +- spec/requests/api/issues_spec.rb | 35 ++++++++++++++ 6 files changed, 122 insertions(+), 1 deletion(-) create mode 100644 changelogs/unreleased/26437-closed-by.yml create mode 100644 spec/factories/merge_requests_closing_issues.rb diff --git a/changelogs/unreleased/26437-closed-by.yml b/changelogs/unreleased/26437-closed-by.yml new file mode 100644 index 00000000000..6325d3576bc --- /dev/null +++ b/changelogs/unreleased/26437-closed-by.yml @@ -0,0 +1,4 @@ +--- +title: Add issues/:iid/closed_by api endpoint +merge_request: +author: mhasbini diff --git a/doc/api/issues.md b/doc/api/issues.md index 5f01fcdd396..6c10b5ab0e7 100644 --- a/doc/api/issues.md +++ b/doc/api/issues.md @@ -823,6 +823,67 @@ Example response: } ``` +## List merge requests that will close issue on merge + +Get all the merge requests that will close issue when merged. + +``` +GET /projects/:id/issues/:issue_iid/closed_by +``` + +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `id` | integer | yes | The ID of a project | +| `issue_iid` | integer | yes | The internal ID of a project issue | + +```bash +curl --header "PRIVATE-TOKEN: 9koXpg98eAheJpvBs5tK" https://gitlab.example.com/api/v4/projects/1/issues/11/closed_by +``` + +Example response: + +```json +[ + { + "id": 6471, + "iid": 6432, + "project_id": 1, + "title": "add a test for cgi lexer options", + "description": "closes #11", + "state": "opened", + "created_at": "2017-04-06T18:33:34.168Z", + "updated_at": "2017-04-09T20:10:24.983Z", + "target_branch": "master", + "source_branch": "feature.custom-highlighting", + "upvotes": 0, + "downvotes": 0, + "author": { + "name": "Administrator", + "username": "root", + "id": 1, + "state": "active", + "avatar_url": "http://www.gravatar.com/avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon", + "web_url": "https://gitlab.example.com/root" + }, + "assignee": null, + "source_project_id": 1, + "target_project_id": 1, + "labels": [], + "work_in_progress": false, + "milestone": null, + "merge_when_pipeline_succeeds": false, + "merge_status": "unchecked", + "sha": "5a62481d563af92b8e32d735f2fa63b94e806835", + "merge_commit_sha": null, + "user_notes_count": 1, + "should_remove_source_branch": null, + "force_remove_source_branch": false, + "web_url": "https://gitlab.example.com/gitlab-org/gitlab-test/merge_requests/6432" + } +] +``` + + ## Comments on issues Comments are done via the [notes](notes.md) resource. diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 09053e615cb..60b9a89d6a2 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -215,6 +215,21 @@ module API authorize!(:destroy_issue, issue) issue.destroy end + + desc 'List merge requests closing issue' do + success Entities::MergeRequestBasic + end + params do + requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue' + end + get ':id/issues/:issue_iid/closed_by' do + issue = find_project_issue(params[:issue_iid]) + + merge_request_ids = MergeRequestsClosingIssues.where(issue_id: issue).select(:merge_request_id) + merge_requests = MergeRequestsFinder.new(current_user, project_id: user_project.id).execute.where(id: merge_request_ids) + + present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project + end end end end diff --git a/spec/factories/merge_requests_closing_issues.rb b/spec/factories/merge_requests_closing_issues.rb new file mode 100644 index 00000000000..fdbdc00cad7 --- /dev/null +++ b/spec/factories/merge_requests_closing_issues.rb @@ -0,0 +1,6 @@ +FactoryGirl.define do + factory :merge_requests_closing_issues do + issue + merge_request + end +end diff --git a/spec/features/issuables/issuable_list_spec.rb b/spec/features/issuables/issuable_list_spec.rb index 3dc872ae520..f3ec80bb149 100644 --- a/spec/features/issuables/issuable_list_spec.rb +++ b/spec/features/issuables/issuable_list_spec.rb @@ -68,7 +68,7 @@ describe 'issuable list', feature: true do source_project: project, source_branch: generate(:branch)) - MergeRequestsClosingIssues.create!(issue: issue, merge_request: merge_request) + create(:merge_requests_closing_issues, issue: issue, merge_request: merge_request) end end end diff --git a/spec/requests/api/issues_spec.rb b/spec/requests/api/issues_spec.rb index 551aae7d701..e8d9f7f0f0c 100644 --- a/spec/requests/api/issues_spec.rb +++ b/spec/requests/api/issues_spec.rb @@ -1345,6 +1345,41 @@ describe API::Issues, api: true do include_examples 'time tracking endpoints', 'issue' end + describe 'GET :id/issues/:issue_iid/closed_by' do + let(:merge_request) do + create(:merge_request, + :simple, + author: user, + source_project: project, + target_project: project, + description: "closes #{issue.to_reference}") + end + + before do + create(:merge_requests_closing_issues, issue: issue, merge_request: merge_request) + end + + it 'returns merge requests that will close issue on merge' do + get api("/projects/#{project.id}/issues/#{issue.iid}/closed_by", user) + + expect_paginated_array_response(size: 1) + end + + context 'when no merge requests will close issue' do + it 'returns empty array' do + get api("/projects/#{project.id}/issues/#{closed_issue.iid}/closed_by", user) + + expect_paginated_array_response(size: 0) + end + end + + it "returns 404 when issue doesn't exists" do + get api("/projects/#{project.id}/issues/9999/closed_by", user) + + expect(response).to have_http_status(404) + end + end + def expect_paginated_array_response(size: nil) expect(response).to have_http_status(200) expect(response).to include_pagination_headers -- cgit v1.2.1 From d7a17107c867b82918c5188f6fc7c4bb6e8beb59 Mon Sep 17 00:00:00 2001 From: Ben Bodenmiller Date: Thu, 23 Mar 2017 22:33:16 +0000 Subject: don't require no_root_squash on NFS only recommend --- doc/administration/high_availability/nfs.md | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/doc/administration/high_availability/nfs.md b/doc/administration/high_availability/nfs.md index bf1aa6b9ac5..c5125dc6d5a 100644 --- a/doc/administration/high_availability/nfs.md +++ b/doc/administration/high_availability/nfs.md @@ -7,21 +7,20 @@ supported natively in NFS version 4. NFSv3 also supports locking as long as Linux Kernel 2.6.5+ is used. We recommend using version 4 and do not specifically test NFSv3. -**no_root_squash**: NFS normally changes the `root` user to `nobody`. This is -a good security measure when NFS shares will be accessed by many different -users. However, in this case only GitLab will use the NFS share so it -is safe. GitLab requires the `no_root_squash` setting because we need to -manage file permissions automatically. Without the setting you will receive -errors when the Omnibus package tries to alter permissions. Note that GitLab -and other bundled components do **not** run as `root` but as non-privileged -users. The requirement for `no_root_squash` is to allow the Omnibus package to -set ownership and permissions on files, as needed. - ### Recommended options When you define your NFS exports, we recommend you also add the following options: +- `no_root_squash` - NFS normally changes the `root` user to `nobody`. This is + a good security measure when NFS shares will be accessed by many different + users. However, in this case only GitLab will use the NFS share so it + is safe. GitLab recommends the `no_root_squash` setting because we need to + manage file permissions automatically. Without the setting you may receive + errors when the Omnibus package tries to alter permissions. Note that GitLab + and other bundled components do **not** run as `root` but as non-privileged + users. The recommendation for `no_root_squash` is to allow the Omnibus package + to set ownership and permissions on files, as needed. - `sync` - Force synchronous behavior. Default is asynchronous and under certain circumstances it could lead to data loss if a failure occurs before data has synced. -- cgit v1.2.1 From 7057fbf404968aed61d108967514e5ffc18fe91b Mon Sep 17 00:00:00 2001 From: Jeff Stubler Date: Mon, 2 Jan 2017 19:55:26 -0600 Subject: Add updated time to project list --- app/assets/stylesheets/pages/projects.scss | 1 + app/helpers/projects_helper.rb | 2 +- app/views/shared/projects/_project.html.haml | 34 +++++++++++++----------- changelogs/unreleased/26509-show-update-time.yml | 4 +++ spec/helpers/projects_helper_spec.rb | 2 +- 5 files changed, 26 insertions(+), 17 deletions(-) create mode 100644 changelogs/unreleased/26509-show-update-time.yml diff --git a/app/assets/stylesheets/pages/projects.scss b/app/assets/stylesheets/pages/projects.scss index 717ebb44a23..75b4b920b53 100644 --- a/app/assets/stylesheets/pages/projects.scss +++ b/app/assets/stylesheets/pages/projects.scss @@ -610,6 +610,7 @@ pre.light-well { .controls { margin-left: auto; + text-align: right; } .ci-status-link { diff --git a/app/helpers/projects_helper.rb b/app/helpers/projects_helper.rb index 6b9e4267281..7fbdbe507ef 100644 --- a/app/helpers/projects_helper.rb +++ b/app/helpers/projects_helper.rb @@ -160,7 +160,7 @@ module ProjectsHelper end def project_list_cache_key(project) - key = [project.namespace.cache_key, project.cache_key, controller.controller_name, controller.action_name, current_application_settings.cache_key, 'v2.3'] + key = [project.namespace.cache_key, project.cache_key, controller.controller_name, controller.action_name, current_application_settings.cache_key, 'v2.4'] key << pipeline_status_cache_key(project.pipeline_status) if project.pipeline_status.has_status? key diff --git a/app/views/shared/projects/_project.html.haml b/app/views/shared/projects/_project.html.haml index 761f0b606b5..dd130ac058c 100644 --- a/app/views/shared/projects/_project.html.haml +++ b/app/views/shared/projects/_project.html.haml @@ -7,6 +7,7 @@ - show_last_commit_as_description = false unless local_assigns[:show_last_commit_as_description] == true && project.commit - css_class += " no-description" if project.description.blank? && !show_last_commit_as_description - cache_key = project_list_cache_key(project) +- updated_tooltip = time_ago_with_tooltip(project.updated_at) %li.project-row{ class: css_class } = cache(cache_key) do @@ -36,18 +37,21 @@ = markdown_field(project, :description) .controls - - if project.archived - %span.prepend-left-10.label.label-warning archived - - if project.pipeline_status.has_status? - %span.prepend-left-10 - = render_project_pipeline_status(project.pipeline_status) - - if forks - %span.prepend-left-10 - = icon('code-fork') - = number_with_delimiter(project.forks_count) - - if stars - %span.prepend-left-10 - = icon('star') - = number_with_delimiter(project.star_count) - %span.prepend-left-10.visibility-icon.has-tooltip{ data: { container: 'body', placement: 'left' }, title: visibility_icon_description(project) } - = visibility_level_icon(project.visibility_level, fw: true) + .prepend-top-0 + - if project.archived + %span.prepend-left-10.label.label-warning archived + - if project.pipeline_status.has_status? + %span.prepend-left-10 + = render_project_pipeline_status(project.pipeline_status) + - if forks + %span.prepend-left-10 + = icon('code-fork') + = number_with_delimiter(project.forks_count) + - if stars + %span.prepend-left-10 + = icon('star') + = number_with_delimiter(project.star_count) + %span.prepend-left-10.visibility-icon.has-tooltip{ data: { container: 'body', placement: 'left' }, title: visibility_icon_description(project) } + = visibility_level_icon(project.visibility_level, fw: true) + .prepend-top-5 + updated #{updated_tooltip} diff --git a/changelogs/unreleased/26509-show-update-time.yml b/changelogs/unreleased/26509-show-update-time.yml new file mode 100644 index 00000000000..012fd00dd87 --- /dev/null +++ b/changelogs/unreleased/26509-show-update-time.yml @@ -0,0 +1,4 @@ +--- +title: Add update time to project lists. +merge_request: 8514 +author: Jeff Stubler diff --git a/spec/helpers/projects_helper_spec.rb b/spec/helpers/projects_helper_spec.rb index 40efab6e4f7..b44af79811f 100644 --- a/spec/helpers/projects_helper_spec.rb +++ b/spec/helpers/projects_helper_spec.rb @@ -93,7 +93,7 @@ describe ProjectsHelper do end it "includes a version" do - expect(helper.project_list_cache_key(project)).to include("v2.3") + expect(helper.project_list_cache_key(project).last).to start_with('v') end it "includes the pipeline status when there is a status" do -- cgit v1.2.1 From 7940a5d49ca7b0abbd0fed3bcc0184ebd33a7d0b Mon Sep 17 00:00:00 2001 From: TM Lee Date: Fri, 10 Mar 2017 09:58:48 +0800 Subject: [#22826] Ensure find_file_link and download button is part of file tree header - Move the find_file_link and download partial into the project tree_header partial - Added Changelog --- app/assets/stylesheets/pages/tree.scss | 1 - app/views/projects/tree/_tree_header.html.haml | 4 ++++ app/views/projects/tree/show.html.haml | 10 +------- ...ferent-files-views-find-file-button-missing.yml | 4 ++++ spec/features/projects/files/find_files_spec.rb | 28 ++++++++++++++++++++++ 5 files changed, 37 insertions(+), 10 deletions(-) create mode 100644 changelogs/unreleased/22826-ui-inconsistency-different-files-views-find-file-button-missing.yml create mode 100644 spec/features/projects/files/find_files_spec.rb diff --git a/app/assets/stylesheets/pages/tree.scss b/app/assets/stylesheets/pages/tree.scss index f3916622b6f..03c75ce61f5 100644 --- a/app/assets/stylesheets/pages/tree.scss +++ b/app/assets/stylesheets/pages/tree.scss @@ -160,7 +160,6 @@ .tree-controls { float: right; - margin-top: 11px; position: relative; z-index: 2; diff --git a/app/views/projects/tree/_tree_header.html.haml b/app/views/projects/tree/_tree_header.html.haml index 259207a6dfd..e7b3fe3ffda 100644 --- a/app/views/projects/tree/_tree_header.html.haml +++ b/app/views/projects/tree/_tree_header.html.haml @@ -1,3 +1,7 @@ +.tree-controls + = render 'projects/find_file_link' + = render 'projects/buttons/download', project: @project, ref: @ref + .tree-ref-holder = render 'shared/ref_switcher', destination: 'tree', path: @path diff --git a/app/views/projects/tree/show.html.haml b/app/views/projects/tree/show.html.haml index a2a26039220..910d765aed0 100644 --- a/app/views/projects/tree/show.html.haml +++ b/app/views/projects/tree/show.html.haml @@ -7,12 +7,4 @@ = render 'projects/last_push' %div{ class: container_class } - .tree-controls - = render 'projects/find_file_link' - = render 'projects/buttons/download', project: @project, ref: @ref - - #tree-holder.tree-holder.clearfix - .nav-block - = render 'projects/tree/tree_header', tree: @tree - - = render 'projects/tree/tree_content', tree: @tree + = render 'projects/files' diff --git a/changelogs/unreleased/22826-ui-inconsistency-different-files-views-find-file-button-missing.yml b/changelogs/unreleased/22826-ui-inconsistency-different-files-views-find-file-button-missing.yml new file mode 100644 index 00000000000..c42fbd4e1f1 --- /dev/null +++ b/changelogs/unreleased/22826-ui-inconsistency-different-files-views-find-file-button-missing.yml @@ -0,0 +1,4 @@ +--- +title: Fix UI inconsistency different files view (find file button missing) +merge_request: 9847 +author: TM Lee diff --git a/spec/features/projects/files/find_files_spec.rb b/spec/features/projects/files/find_files_spec.rb new file mode 100644 index 00000000000..06f52bef902 --- /dev/null +++ b/spec/features/projects/files/find_files_spec.rb @@ -0,0 +1,28 @@ +require 'spec_helper' + +feature 'Find files button in the tree header', feature: true do + given(:user) { create(:user) } + given(:project) { create(:project) } + + background do + login_as(user) + project.team << [user, :developer] + end + + scenario 'project main screen' do + visit namespace_project_path( + project.namespace, + project + ) + expect(page).to have_selector('.tree-controls .shortcuts-find-file') + end + + scenario 'project tree screen' do + visit namespace_project_tree_path( + project.namespace, + project, + project.default_branch + ) + expect(page).to have_selector('.tree-controls .shortcuts-find-file') + end +end -- cgit v1.2.1 From 7030eb3286613be919e12daf95c43061b49aa5f5 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 11 Apr 2017 22:29:30 -0300 Subject: Add faraday gem --- Gemfile | 8 ++++++-- Gemfile.lock | 22 ++++++++++++++-------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/Gemfile b/Gemfile index ad8db206da6..8654e486925 100644 --- a/Gemfile +++ b/Gemfile @@ -17,6 +17,10 @@ gem 'pg', '~> 0.18.2', group: :postgres gem 'rugged', '~> 0.25.1.1' +gem 'faraday', '~> 0.11.0' +gem 'net-http-persistent', '~> 2.9.4' +gem 'faraday-http-cache' + # Authentication libraries gem 'devise', '~> 4.2' gem 'doorkeeper', '~> 4.2.0' @@ -186,7 +190,7 @@ gem 'gemnasium-gitlab-service', '~> 0.2' gem 'slack-notifier', '~> 1.5.1' # Asana integration -gem 'asana', '~> 0.4.0' +gem 'asana', '~> 0.6.0' # FogBugz integration gem 'ruby-fogbugz', '~> 0.2.1' @@ -345,7 +349,7 @@ gem 'html2text' gem 'ruby-prof', '~> 0.16.2' # OAuth -gem 'oauth2', '~> 1.2.0' +gem 'oauth2', '~> 1.3.0' # Soft deletion gem 'paranoia', '~> 2.2' diff --git a/Gemfile.lock b/Gemfile.lock index bb91db1e805..7d9a950bfd3 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -47,7 +47,7 @@ GEM akismet (2.0.0) allocations (1.0.5) arel (6.0.4) - asana (0.4.0) + asana (0.6.0) faraday (~> 0.9) faraday_middleware (~> 0.9) faraday_middleware-multi_json (~> 0.0) @@ -193,10 +193,12 @@ GEM factory_girl_rails (4.7.0) factory_girl (~> 4.7.0) railties (>= 3.0.0) - faraday (0.9.2) + faraday (0.11.0) multipart-post (>= 1.2, < 3) - faraday_middleware (0.10.0) - faraday (>= 0.7.4, < 0.10) + faraday-http-cache (2.0.0) + faraday (~> 0.8) + faraday_middleware (0.11.0.1) + faraday (>= 0.7.4, < 1.0) faraday_middleware-multi_json (0.0.6) faraday_middleware multi_json @@ -447,6 +449,7 @@ GEM mustermann-grape (0.4.0) mustermann (= 0.4.0) mysql2 (0.3.20) + net-http-persistent (2.9.4) net-ldap (0.12.1) net-ssh (3.0.1) netrc (0.11.0) @@ -454,8 +457,8 @@ GEM mini_portile2 (~> 2.1.0) numerizer (0.1.1) oauth (0.5.1) - oauth2 (1.2.0) - faraday (>= 0.8, < 0.10) + oauth2 (1.3.1) + faraday (>= 0.8, < 0.12) jwt (~> 1.0) multi_json (~> 1.3) multi_xml (~> 0.5) @@ -853,7 +856,7 @@ DEPENDENCIES after_commit_queue (~> 1.3.0) akismet (~> 2.0) allocations (~> 1.0) - asana (~> 0.4.0) + asana (~> 0.6.0) asciidoctor (~> 1.5.2) asciidoctor-plantuml (= 0.0.7) attr_encrypted (~> 3.0.0) @@ -891,6 +894,8 @@ DEPENDENCIES email_reply_trimmer (~> 0.1) email_spec (~> 1.6.0) factory_girl_rails (~> 4.7.0) + faraday (~> 0.11.0) + faraday-http-cache ffaker (~> 2.4) flay (~> 2.8.0) fog-aws (~> 0.9) @@ -941,9 +946,10 @@ DEPENDENCIES minitest (~> 5.7.0) mousetrap-rails (~> 1.4.6) mysql2 (~> 0.3.16) + net-http-persistent (~> 2.9.4) net-ssh (~> 3.0.1) nokogiri (~> 1.6.7, >= 1.6.7.2) - oauth2 (~> 1.2.0) + oauth2 (~> 1.3.0) octokit (~> 4.6.2) oj (~> 2.17.4) omniauth (~> 1.4.2) -- cgit v1.2.1 From fc42f3dffa364a360c76ff2785813d74f42016c7 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 11 Apr 2017 22:33:54 -0300 Subject: Add basic client for the GitHub API --- lib/github/client.rb | 21 +++++++++++++++++++++ lib/github/collection.rb | 26 ++++++++++++++++++++++++++ lib/github/issues.rb | 20 ++++++++++++++++++++ lib/github/labels.rb | 20 ++++++++++++++++++++ lib/github/milestones.rb | 20 ++++++++++++++++++++ lib/github/pull_requests.rb | 20 ++++++++++++++++++++ lib/github/rate_limit.rb | 41 +++++++++++++++++++++++++++++++++++++++++ lib/github/releases.rb | 20 ++++++++++++++++++++ lib/github/repositories.rb | 17 +++++++++++++++++ lib/github/response.rb | 22 ++++++++++++++++++++++ lib/github/user.rb | 23 +++++++++++++++++++++++ 11 files changed, 250 insertions(+) create mode 100644 lib/github/client.rb create mode 100644 lib/github/collection.rb create mode 100644 lib/github/issues.rb create mode 100644 lib/github/labels.rb create mode 100644 lib/github/milestones.rb create mode 100644 lib/github/pull_requests.rb create mode 100644 lib/github/rate_limit.rb create mode 100644 lib/github/releases.rb create mode 100644 lib/github/repositories.rb create mode 100644 lib/github/response.rb create mode 100644 lib/github/user.rb diff --git a/lib/github/client.rb b/lib/github/client.rb new file mode 100644 index 00000000000..2511523bf6a --- /dev/null +++ b/lib/github/client.rb @@ -0,0 +1,21 @@ +module Github + class Client + attr_reader :connection + + def initialize(token) + @connection = Faraday.new(url: 'https://api.github.com') do |faraday| + faraday.adapter :net_http_persistent + faraday.response :json, content_type: /\bjson$/ + faraday.authorization 'token', token + faraday.response :logger + end + end + + def get(url, query = {}) + rate_limit = RateLimit.new(connection) + sleep rate_limit.reset_in if rate_limit.exceed? + + Github::Response.new(connection.get(url, query)) + end + end +end diff --git a/lib/github/collection.rb b/lib/github/collection.rb new file mode 100644 index 00000000000..bbca12a1c84 --- /dev/null +++ b/lib/github/collection.rb @@ -0,0 +1,26 @@ +module Github + class Collection + def initialize(url) + @url = url + end + + def fetch(query = {}) + return [] if @url.blank? + + Enumerator.new do |yielder| + loop do + response = client.get(@url, query) + response.body.each { |item| yielder << item } + raise StopIteration unless response.rels.key?(:next) + @url = response.rels[:next] + end + end.lazy + end + + private + + def client + @client ||= Github::Client.new + end + end +end diff --git a/lib/github/issues.rb b/lib/github/issues.rb new file mode 100644 index 00000000000..27843e1cdd8 --- /dev/null +++ b/lib/github/issues.rb @@ -0,0 +1,20 @@ +module Github + class Issues + attr_reader :owner, :repo + + def initialize(owner, repo) + @owner = owner + @repo = repo + end + + def fetch + Collection.new(issues_url).fetch(state: :all, sort: :created, direction: :asc, per_page: 10) + end + + private + + def issues_url + "/repos/#{owner}/#{repo}/issues" + end + end +end diff --git a/lib/github/labels.rb b/lib/github/labels.rb new file mode 100644 index 00000000000..0ea023888b3 --- /dev/null +++ b/lib/github/labels.rb @@ -0,0 +1,20 @@ +module Github + class Labels + attr_reader :owner, :repo + + def initialize(owner, repo) + @owner = owner + @repo = repo + end + + def fetch + Collection.new(labels_url).fetch + end + + private + + def labels_url + "/repos/#{owner}/#{repo}/labels" + end + end +end diff --git a/lib/github/milestones.rb b/lib/github/milestones.rb new file mode 100644 index 00000000000..d50278105db --- /dev/null +++ b/lib/github/milestones.rb @@ -0,0 +1,20 @@ +module Github + class Milestones + attr_reader :owner, :repo + + def initialize(owner, repo) + @owner = owner + @repo = repo + end + + def fetch + Collection.new(milestones_url).fetch + end + + private + + def milestones_url + "/repos/#{owner}/#{repo}/milestones" + end + end +end diff --git a/lib/github/pull_requests.rb b/lib/github/pull_requests.rb new file mode 100644 index 00000000000..af04c90d454 --- /dev/null +++ b/lib/github/pull_requests.rb @@ -0,0 +1,20 @@ +module Github + class PullRequests + attr_reader :owner, :repo + + def initialize(owner, repo) + @owner = owner + @repo = repo + end + + def fetch + Collection.new(pull_requests_url).fetch(state: :all, sort: :created, direction: :asc) + end + + private + + def pull_requests_url + "/repos/#{owner}/#{repo}/pulls" + end + end +end diff --git a/lib/github/rate_limit.rb b/lib/github/rate_limit.rb new file mode 100644 index 00000000000..30b000f8a9a --- /dev/null +++ b/lib/github/rate_limit.rb @@ -0,0 +1,41 @@ +module Github + class RateLimit + SAFE_REMAINING_REQUESTS = 100.freeze + SAFE_RESET_TIME = 500.freeze + + attr_reader :connection + + def initialize(connection) + @connection = connection + end + + def exceed? + return false unless enabled? + + remaining <= SAFE_REMAINING_REQUESTS + end + + def remaining + @remaining ||= response.body.dig('rate', 'remaining').to_i + end + + def reset_in + @reset ||= response.body.dig('rate', 'reset').to_i + end + + private + + def rate_limit_url + '/rate_limit' + end + + def response + @response ||= connection.get(rate_limit_url) + end + + # GitHub Rate Limit API returns 404 when the rate limit is disabled + def enabled? + response.status != 404 + end + end +end diff --git a/lib/github/releases.rb b/lib/github/releases.rb new file mode 100644 index 00000000000..026ef4e6853 --- /dev/null +++ b/lib/github/releases.rb @@ -0,0 +1,20 @@ +module Github + class Releases + attr_reader :owner, :repo + + def initialize(owner, repo) + @owner = owner + @repo = repo + end + + def fetch + Collection.new(releases_url).fetch + end + + private + + def releases_url + "/repos/#{owner}/#{repo}/releases" + end + end +end diff --git a/lib/github/repositories.rb b/lib/github/repositories.rb new file mode 100644 index 00000000000..42342471102 --- /dev/null +++ b/lib/github/repositories.rb @@ -0,0 +1,17 @@ +module Github + class Repositories + def initialize(username) + @username = username + end + + def fetch + Collection.new(repos_url).fetch + end + + private + + def repos_url + '/user/repos' + end + end +end diff --git a/lib/github/response.rb b/lib/github/response.rb new file mode 100644 index 00000000000..c34b69aa4ea --- /dev/null +++ b/lib/github/response.rb @@ -0,0 +1,22 @@ +module Github + class Response + attr_reader :raw, :headers, :body, :status + + def initialize(response) + @raw = response + @headers = response.headers + @body = response.body + @status = response.status + end + + def rels + links = headers['Link'].to_s.split(', ').map do |link| + href, name = link.match(/<(.*?)>; rel="(\w+)"/).captures + + [name.to_sym, href] + end + + Hash[*links.flatten] + end + end +end diff --git a/lib/github/user.rb b/lib/github/user.rb new file mode 100644 index 00000000000..19fe6230820 --- /dev/null +++ b/lib/github/user.rb @@ -0,0 +1,23 @@ +module Github + class User + attr_reader :username + + def initialize(username) + @username = username + end + + def get + client.get(user_url).body + end + + private + + def client + @client ||= Github::Client.new + end + + def user_url + "/users/#{username}" + end + end +end -- cgit v1.2.1 From b43ecca9060b3d7ffd84d700caecf5f35fd403a9 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 11 Apr 2017 22:34:59 -0300 Subject: Add basic representations for the Github API results --- lib/github/representation/base.rb | 13 ++++ lib/github/representation/branch.rb | 52 +++++++++++++ lib/github/representation/label.rb | 17 +++++ lib/github/representation/milestone.rb | 37 +++++++++ lib/github/representation/pull_request.rb | 123 ++++++++++++++++++++++++++++++ lib/github/representation/repo.rb | 9 +++ lib/github/representation/user.rb | 19 +++++ 7 files changed, 270 insertions(+) create mode 100644 lib/github/representation/base.rb create mode 100644 lib/github/representation/branch.rb create mode 100644 lib/github/representation/label.rb create mode 100644 lib/github/representation/milestone.rb create mode 100644 lib/github/representation/pull_request.rb create mode 100644 lib/github/representation/repo.rb create mode 100644 lib/github/representation/user.rb diff --git a/lib/github/representation/base.rb b/lib/github/representation/base.rb new file mode 100644 index 00000000000..6765bfb803b --- /dev/null +++ b/lib/github/representation/base.rb @@ -0,0 +1,13 @@ +module Github + module Representation + class Base + def initialize(raw) + @raw = raw + end + + private + + attr_reader :raw + end + end +end diff --git a/lib/github/representation/branch.rb b/lib/github/representation/branch.rb new file mode 100644 index 00000000000..7c65a948ede --- /dev/null +++ b/lib/github/representation/branch.rb @@ -0,0 +1,52 @@ +module Github + module Representation + class Branch < Representation::Base + attr_reader :repository + + def initialize(repository, raw) + @repository = repository + @raw = raw + end + + def user + raw.dig('user', 'login') || 'unknown' + end + + def repo + return @repo if defined?(@repo) + + @repo = Github::Representation::Repo.new(raw['repo']) if raw['repo'].present? + end + + def ref + raw['ref'] + end + + def sha + raw['sha'] + end + + def short_sha + Commit.truncate_sha(sha) + end + + def exists? + branch_exists? && commit_exists? + end + + def valid? + sha.present? && ref.present? + end + + private + + def branch_exists? + repository.branch_exists?(ref) + end + + def commit_exists? + repository.branch_names_contains(sha).include?(ref) + end + end + end +end diff --git a/lib/github/representation/label.rb b/lib/github/representation/label.rb new file mode 100644 index 00000000000..b3140ab76fc --- /dev/null +++ b/lib/github/representation/label.rb @@ -0,0 +1,17 @@ +module Github + module Representation + class Label < Representation::Base + def color + "##{raw['color']}" + end + + def title + raw['name'] + end + + def url + raw['url'] + end + end + end +end diff --git a/lib/github/representation/milestone.rb b/lib/github/representation/milestone.rb new file mode 100644 index 00000000000..5ea54eb178f --- /dev/null +++ b/lib/github/representation/milestone.rb @@ -0,0 +1,37 @@ +module Github + module Representation + class Milestone < Representation::Base + def iid + raw['number'] + end + + def title + raw['title'] + end + + def description + raw['description'] + end + + def due_date + raw['due_on'] + end + + def state + raw['state'] == 'closed' ? 'closed' : 'active' + end + + def url + raw['url'] + end + + def created_at + raw['created_at'] + end + + def updated_at + raw['updated_at'] + end + end + end +end diff --git a/lib/github/representation/pull_request.rb b/lib/github/representation/pull_request.rb new file mode 100644 index 00000000000..85f4e1bdac3 --- /dev/null +++ b/lib/github/representation/pull_request.rb @@ -0,0 +1,123 @@ +module Github + module Representation + class PullRequest < Representation::Base + attr_reader :project + + delegate :user, :repo, :ref, :sha, to: :source_branch, prefix: true + delegate :user, :exists?, :repo, :ref, :sha, :short_sha, to: :target_branch, prefix: true + + def initialize(project, raw) + @project = project + @raw = raw + end + + def iid + raw['number'] + end + + def title + raw['title'] + end + + def description + raw['body'] || '' + end + + def source_project + project + end + + def source_branch_exists? + !cross_project? && source_branch.exists? + end + + def source_branch_name + @source_branch_name ||= + if cross_project? || !source_branch_exists? + source_branch_name_prefixed + else + source_branch_ref + end + end + + def target_project + project + end + + def target_branch_name + @target_branch_name ||= target_branch_exists? ? target_branch_ref : target_branch_name_prefixed + end + + def milestone + return unless raw['milestone'].present? + + @milestone ||= Github::Representation::Milestone.new(raw['milestone']) + end + + def author + @author ||= Github::Representation::User.new(raw['user']) + end + + def assignee + return unless assigned? + + @assignee ||= Github::Representation::User.new(raw['assignee']) + end + + def state + return 'merged' if raw['state'] == 'closed' && raw['merged_at'].present? + return 'closed' if raw['state'] == 'closed' + + 'opened' + end + + def url + raw['url'] + end + + def created_at + raw['created_at'] + end + + def updated_at + raw['updated_at'] + end + + def assigned? + raw['assignee'].present? + end + + def opened? + state == 'opened' + end + + def valid? + source_branch.valid? && target_branch.valid? + end + + private + + def source_branch + @source_branch ||= Representation::Branch.new(project.repository, raw['head']) + end + + def source_branch_name_prefixed + "gh-#{target_branch_short_sha}/#{iid}/#{source_branch_user}/#{source_branch_ref}" + end + + def target_branch + @target_branch ||= Representation::Branch.new(project.repository, raw['base']) + end + + def target_branch_name_prefixed + "gl-#{target_branch_short_sha}/#{iid}/#{target_branch_user}/#{target_branch_ref}" + end + + def cross_project? + return true if source_branch_repo.nil? + + source_branch_repo.id != target_branch_repo.id + end + end + end +end diff --git a/lib/github/representation/repo.rb b/lib/github/representation/repo.rb new file mode 100644 index 00000000000..b9cae43450e --- /dev/null +++ b/lib/github/representation/repo.rb @@ -0,0 +1,9 @@ +module Github + module Representation + class Repo < Representation::Base + def id + raw['id'] + end + end + end +end diff --git a/lib/github/representation/user.rb b/lib/github/representation/user.rb new file mode 100644 index 00000000000..70a0ce000ae --- /dev/null +++ b/lib/github/representation/user.rb @@ -0,0 +1,19 @@ +module Github + module Representation + class User < Representation::Base + def id + raw['id'] + end + + def email + return @email if defined?(@email) + + @email = Github::User.new(username).get.fetch('email', nil) + end + + def username + raw['login'] + end + end + end +end -- cgit v1.2.1 From f76363e445153ea831fd508e366c0f31b24fa53e Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 11 Apr 2017 22:36:33 -0300 Subject: Add basic importer for GitHub repositories --- lib/github/import.rb | 166 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100644 lib/github/import.rb diff --git a/lib/github/import.rb b/lib/github/import.rb new file mode 100644 index 00000000000..89f597551f9 --- /dev/null +++ b/lib/github/import.rb @@ -0,0 +1,166 @@ +module Github + class Import + class MergeRequest < ::MergeRequest + self.table_name = 'merge_requests' + + self.reset_callbacks :save + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + + attr_reader :project, :repository, :cached_user_ids, :errors + + def initialize(project) + @project = project + @repository = project.repository + @cached_user_ids = {} + @errors = [] + end + + def execute(owner, repo, token) + # Fetch repository + begin + project.create_repository + project.repository.add_remote('github', "https://#{token}@github.com/#{owner}/#{repo}.git") + project.repository.set_remote_as_mirror('github') + project.repository.fetch_remote('github', forced: true) + project.repository.remove_remote('github') + rescue Gitlab::Shell::Error => e + error(:project, "https://github.com/#{owner}/#{repo}.git", e.message) + end + + # Fetch labels + labels = Github::Labels.new(owner, repo).fetch + + labels.each do |raw| + begin + label = Github::Representation::Label.new(raw) + project.labels.create!(title: label.title, color: label.color) + rescue => e + error(:label, label.url, e.message) + end + end + + # Fetch milestones + milestones = Github::Milestones.new(owner, repo).fetch + + milestones.each do |raw| + begin + milestone = Github::Representation::Milestone.new(raw) + + project.milestones.create!( + iid: milestone.iid, + title: milestone.title, + description: milestone.description, + due_date: milestone.due_date, + state: milestone.state, + created_at: milestone.created_at, + updated_at: milestone.updated_at + ) + rescue => e + error(:milestone, milestone.url, e.message) + end + end + + # Fetch pull requests + pull_requests = Github::PullRequests.new(owner, repo).fetch + + pull_requests.each do |raw| + pull_request = Github::Representation::PullRequest.new(project, raw) + next unless pull_request.valid? + + begin + restore_source_branch(pull_request) unless pull_request.source_branch_exists? + restore_target_branch(pull_request) unless pull_request.target_branch_exists? + + merge_request = MergeRequest.find_or_initialize_by(iid: pull_request.iid, source_project_id: project.id) do |record| + record.iid = pull_request.iid + record.title = pull_request.title + record.description = pull_request.description + record.source_project = pull_request.source_project + record.source_branch = pull_request.source_branch_name + record.source_branch_sha = pull_request.source_branch_sha + record.target_project = pull_request.target_project + record.target_branch = pull_request.target_branch_name + record.target_branch_sha = pull_request.target_branch_sha + record.state = pull_request.state + record.milestone_id = milestone_id(pull_request.milestone) + record.author_id = user_id(pull_request.author, project.creator_id) + record.assignee_id = user_id(pull_request.assignee) + record.created_at = pull_request.created_at + record.updated_at = pull_request.updated_at + end + + merge_request.save(validate: false) + merge_request.merge_request_diffs.create + rescue => e + error(:pull_request, pull_request.url, "#{e.message}\n\n#{e.exception.backtrace.join('\n')}") + ensure + clean_up_restored_branches(pull_request) + end + end + + repository.expire_content_cache + + errors + end + + private + + def restore_source_branch(pull_request) + repository.create_branch(pull_request.source_branch_name, pull_request.source_branch_sha) + end + + def restore_target_branch(pull_request) + repository.create_branch(pull_request.target_branch_name, pull_request.target_branch_sha) + end + + def remove_branch(name) + repository.delete_branch(name) + rescue Rugged::ReferenceError + errors << { type: :branch, url: nil, error: "Could not clean up restored branch: #{name}" } + end + + def clean_up_restored_branches(pull_request) + return if pull_request.opened? + + remove_branch(pull_request.source_branch_name) unless pull_request.source_branch_exists? + remove_branch(pull_request.target_branch_name) unless pull_request.target_branch_exists? + end + + def milestone_id(milestone) + return unless milestone.present? + + project.milestones.select(:id).find_by(iid: milestone.iid)&.id + end + + def user_id(user, fallback_id = nil) + return unless user.present? + return cached_user_ids[user.id] if cached_user_ids.key?(user.id) + + cached_user_ids[user.id] = find_by_external_uid(user.id) || find_by_email(user.email) || fallback_id + end + + def find_by_email(email) + return nil unless email + + ::User.find_by_any_email(email)&.id + end + + def find_by_external_uid(id) + return nil unless id + + identities = ::Identity.arel_table + + ::User.select(:id) + .joins(:identities) + .where(identities[:provider].eq(:github).and(identities[:extern_uid].eq(id))) + .first&.id + end + + def error(type, url, message) + errors << { type: type, url: Gitlab::UrlSanitizer.sanitize(url), error: message } + end + end +end -- cgit v1.2.1 From 9bcb1b27a93f71df3893eb883d4dc99b95cd71dd Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 12 Apr 2017 22:39:45 -0300 Subject: Remove unused gems --- Gemfile | 2 -- Gemfile.lock | 5 ----- 2 files changed, 7 deletions(-) diff --git a/Gemfile b/Gemfile index 8654e486925..58af6c51b77 100644 --- a/Gemfile +++ b/Gemfile @@ -18,8 +18,6 @@ gem 'pg', '~> 0.18.2', group: :postgres gem 'rugged', '~> 0.25.1.1' gem 'faraday', '~> 0.11.0' -gem 'net-http-persistent', '~> 2.9.4' -gem 'faraday-http-cache' # Authentication libraries gem 'devise', '~> 4.2' diff --git a/Gemfile.lock b/Gemfile.lock index 7d9a950bfd3..59cc67c46fb 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -195,8 +195,6 @@ GEM railties (>= 3.0.0) faraday (0.11.0) multipart-post (>= 1.2, < 3) - faraday-http-cache (2.0.0) - faraday (~> 0.8) faraday_middleware (0.11.0.1) faraday (>= 0.7.4, < 1.0) faraday_middleware-multi_json (0.0.6) @@ -449,7 +447,6 @@ GEM mustermann-grape (0.4.0) mustermann (= 0.4.0) mysql2 (0.3.20) - net-http-persistent (2.9.4) net-ldap (0.12.1) net-ssh (3.0.1) netrc (0.11.0) @@ -895,7 +892,6 @@ DEPENDENCIES email_spec (~> 1.6.0) factory_girl_rails (~> 4.7.0) faraday (~> 0.11.0) - faraday-http-cache ffaker (~> 2.4) flay (~> 2.8.0) fog-aws (~> 0.9) @@ -946,7 +942,6 @@ DEPENDENCIES minitest (~> 5.7.0) mousetrap-rails (~> 1.4.6) mysql2 (~> 0.3.16) - net-http-persistent (~> 2.9.4) net-ssh (~> 3.0.1) nokogiri (~> 1.6.7, >= 1.6.7.2) oauth2 (~> 1.3.0) -- cgit v1.2.1 From 4f68dc4f97d928aca8f477ce390c7189254bc4d0 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 12 Apr 2017 22:39:59 -0300 Subject: Bump oj gem to version 2.17.5 --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index 59cc67c46fb..44375230256 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -462,7 +462,7 @@ GEM rack (>= 1.2, < 3) octokit (4.6.2) sawyer (~> 0.8.0, >= 0.5.3) - oj (2.17.4) + oj (2.17.5) omniauth (1.4.2) hashie (>= 1.2, < 4) rack (>= 1.0, < 3) -- cgit v1.2.1 From 0a52ae8380d52bf697057d32a404c93f11c9f93d Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 12 Apr 2017 22:40:35 -0300 Subject: Remove unused GitHub endpoint wrappers --- lib/github/issues.rb | 20 -------------------- lib/github/labels.rb | 20 -------------------- lib/github/milestones.rb | 20 -------------------- lib/github/pull_requests.rb | 20 -------------------- lib/github/releases.rb | 20 -------------------- 5 files changed, 100 deletions(-) delete mode 100644 lib/github/issues.rb delete mode 100644 lib/github/labels.rb delete mode 100644 lib/github/milestones.rb delete mode 100644 lib/github/pull_requests.rb delete mode 100644 lib/github/releases.rb diff --git a/lib/github/issues.rb b/lib/github/issues.rb deleted file mode 100644 index 27843e1cdd8..00000000000 --- a/lib/github/issues.rb +++ /dev/null @@ -1,20 +0,0 @@ -module Github - class Issues - attr_reader :owner, :repo - - def initialize(owner, repo) - @owner = owner - @repo = repo - end - - def fetch - Collection.new(issues_url).fetch(state: :all, sort: :created, direction: :asc, per_page: 10) - end - - private - - def issues_url - "/repos/#{owner}/#{repo}/issues" - end - end -end diff --git a/lib/github/labels.rb b/lib/github/labels.rb deleted file mode 100644 index 0ea023888b3..00000000000 --- a/lib/github/labels.rb +++ /dev/null @@ -1,20 +0,0 @@ -module Github - class Labels - attr_reader :owner, :repo - - def initialize(owner, repo) - @owner = owner - @repo = repo - end - - def fetch - Collection.new(labels_url).fetch - end - - private - - def labels_url - "/repos/#{owner}/#{repo}/labels" - end - end -end diff --git a/lib/github/milestones.rb b/lib/github/milestones.rb deleted file mode 100644 index d50278105db..00000000000 --- a/lib/github/milestones.rb +++ /dev/null @@ -1,20 +0,0 @@ -module Github - class Milestones - attr_reader :owner, :repo - - def initialize(owner, repo) - @owner = owner - @repo = repo - end - - def fetch - Collection.new(milestones_url).fetch - end - - private - - def milestones_url - "/repos/#{owner}/#{repo}/milestones" - end - end -end diff --git a/lib/github/pull_requests.rb b/lib/github/pull_requests.rb deleted file mode 100644 index af04c90d454..00000000000 --- a/lib/github/pull_requests.rb +++ /dev/null @@ -1,20 +0,0 @@ -module Github - class PullRequests - attr_reader :owner, :repo - - def initialize(owner, repo) - @owner = owner - @repo = repo - end - - def fetch - Collection.new(pull_requests_url).fetch(state: :all, sort: :created, direction: :asc) - end - - private - - def pull_requests_url - "/repos/#{owner}/#{repo}/pulls" - end - end -end diff --git a/lib/github/releases.rb b/lib/github/releases.rb deleted file mode 100644 index 026ef4e6853..00000000000 --- a/lib/github/releases.rb +++ /dev/null @@ -1,20 +0,0 @@ -module Github - class Releases - attr_reader :owner, :repo - - def initialize(owner, repo) - @owner = owner - @repo = repo - end - - def fetch - Collection.new(releases_url).fetch - end - - private - - def releases_url - "/repos/#{owner}/#{repo}/releases" - end - end -end -- cgit v1.2.1 From 0b1d1931fb00f15987f695d76efb5dac9d3992d7 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 12 Apr 2017 22:41:58 -0300 Subject: Add issue representation --- lib/github/representation/issue.rb | 57 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 lib/github/representation/issue.rb diff --git a/lib/github/representation/issue.rb b/lib/github/representation/issue.rb new file mode 100644 index 00000000000..62c71cc191b --- /dev/null +++ b/lib/github/representation/issue.rb @@ -0,0 +1,57 @@ +module Github + module Representation + class Issue < Representation::Base + def iid + raw['number'] + end + + def title + raw['title'] + end + + def description + raw['body'] || '' + end + + def milestone + return unless raw['milestone'].present? + + @milestone ||= Github::Representation::Milestone.new(raw['milestone']) + end + + def author + @author ||= Github::Representation::User.new(raw['user']) + end + + def assignee + return unless assigned? + + @assignee ||= Github::Representation::User.new(raw['assignee']) + end + + def state + raw['state'] == 'closed' ? 'closed' : 'opened' + end + + def url + raw['url'] + end + + def created_at + raw['created_at'] + end + + def updated_at + raw['updated_at'] + end + + def assigned? + raw['assignee'].present? + end + + def pull_request? + raw['pull_request'].present? + end + end + end +end -- cgit v1.2.1 From 2c92cc52d74d41663e96a0a3eabf268db3f68947 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 12 Apr 2017 22:42:12 -0300 Subject: Add comment representation --- lib/github/representation/comment.rb | 58 ++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 lib/github/representation/comment.rb diff --git a/lib/github/representation/comment.rb b/lib/github/representation/comment.rb new file mode 100644 index 00000000000..ac7832ce28b --- /dev/null +++ b/lib/github/representation/comment.rb @@ -0,0 +1,58 @@ +module Github + module Representation + class Comment < Representation::Base + def note + raw['body'] || '' + end + + def author + @author ||= Github::Representation::User.new(raw['user']) + end + + def commit_id + raw['commit_id'] + end + + def line_code + return unless on_diff? + + parsed_lines = Gitlab::Diff::Parser.new.parse(diff_hunk.lines) + generate_line_code(parsed_lines.to_a.last) + end + + def type + 'LegacyDiffNote' if on_diff? + end + + def url + raw['url'] + end + + def created_at + raw['created_at'] + end + + def updated_at + raw['updated_at'] + end + + private + + def generate_line_code(line) + Gitlab::Diff::LineCode.generate(file_path, line.new_pos, line.old_pos) + end + + def on_diff? + diff_hunk.present? + end + + def diff_hunk + raw_data.diff_hunk + end + + def file_path + raw_data.path + end + end + end +end -- cgit v1.2.1 From 8538066e00d8bda542f219fb03d104f8364760bd Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 12 Apr 2017 22:43:38 -0300 Subject: Refactoring collection wrapper --- lib/github/collection.rb | 12 ++++-------- lib/github/repositories.rb | 2 +- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/lib/github/collection.rb b/lib/github/collection.rb index bbca12a1c84..1b0c00928c5 100644 --- a/lib/github/collection.rb +++ b/lib/github/collection.rb @@ -1,18 +1,14 @@ module Github class Collection - def initialize(url) - @url = url - end - - def fetch(query = {}) - return [] if @url.blank? + def fetch(url, query = {}) + return [] if url.blank? Enumerator.new do |yielder| loop do - response = client.get(@url, query) + response = client.get(url, query) response.body.each { |item| yielder << item } raise StopIteration unless response.rels.key?(:next) - @url = response.rels[:next] + url = response.rels[:next] end end.lazy end diff --git a/lib/github/repositories.rb b/lib/github/repositories.rb index 42342471102..b6a03173673 100644 --- a/lib/github/repositories.rb +++ b/lib/github/repositories.rb @@ -5,7 +5,7 @@ module Github end def fetch - Collection.new(repos_url).fetch + Collection.new.fetch(repos_url) end private -- cgit v1.2.1 From 104144f373a58069680dd3639e89eb1fce9a3a9b Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 12 Apr 2017 22:44:55 -0300 Subject: Refactoring client to not parse response body automatically --- lib/github/client.rb | 7 +++---- lib/github/rate_limit.rb | 10 +++++++--- lib/github/response.rb | 11 +++++------ 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/lib/github/client.rb b/lib/github/client.rb index 2511523bf6a..07cf264e8d7 100644 --- a/lib/github/client.rb +++ b/lib/github/client.rb @@ -4,10 +4,8 @@ module Github def initialize(token) @connection = Faraday.new(url: 'https://api.github.com') do |faraday| - faraday.adapter :net_http_persistent - faraday.response :json, content_type: /\bjson$/ faraday.authorization 'token', token - faraday.response :logger + faraday.adapter :net_http end end @@ -15,7 +13,8 @@ module Github rate_limit = RateLimit.new(connection) sleep rate_limit.reset_in if rate_limit.exceed? - Github::Response.new(connection.get(url, query)) + response = connection.get(url, query) + Github::Response.new(response.headers, response.body, response.status) end end end diff --git a/lib/github/rate_limit.rb b/lib/github/rate_limit.rb index 30b000f8a9a..9dbdf2f4c68 100644 --- a/lib/github/rate_limit.rb +++ b/lib/github/rate_limit.rb @@ -16,11 +16,11 @@ module Github end def remaining - @remaining ||= response.body.dig('rate', 'remaining').to_i + @remaining ||= body.dig('rate', 'remaining').to_i end def reset_in - @reset ||= response.body.dig('rate', 'reset').to_i + @reset ||= body.dig('rate', 'reset').to_i end private @@ -30,7 +30,11 @@ module Github end def response - @response ||= connection.get(rate_limit_url) + connection.get(rate_limit_url) + end + + def body + @body ||= Oj.load(response.body, class_cache: false, mode: :compat) end # GitHub Rate Limit API returns 404 when the rate limit is disabled diff --git a/lib/github/response.rb b/lib/github/response.rb index c34b69aa4ea..2fa852c9fdc 100644 --- a/lib/github/response.rb +++ b/lib/github/response.rb @@ -1,12 +1,11 @@ module Github class Response - attr_reader :raw, :headers, :body, :status + attr_reader :headers, :body, :status - def initialize(response) - @raw = response - @headers = response.headers - @body = response.body - @status = response.status + def initialize(headers, body, status) + @headers = headers + @body = Oj.load(body, class_cache: false, mode: :compat) + @status = status end def rels -- cgit v1.2.1 From 4a3b895d913191102d0c69a4917a8af76a4f9b67 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 12 Apr 2017 22:46:31 -0300 Subject: Refactoring Github import to avoid memory leak --- lib/github/import.rb | 187 +++++++++++++++++++++++++++++++++++---------------- 1 file changed, 129 insertions(+), 58 deletions(-) diff --git a/lib/github/import.rb b/lib/github/import.rb index 89f597551f9..19d366b0444 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -9,6 +9,24 @@ module Github self.reset_callbacks :validate end + class Issue < ::Issue + self.table_name = 'issues' + + self.reset_callbacks :save + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + + class Note < ::Note + self.table_name = 'notes' + + self.reset_callbacks :save + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + attr_reader :project, :repository, :cached_user_ids, :errors def initialize(project) @@ -22,7 +40,7 @@ module Github # Fetch repository begin project.create_repository - project.repository.add_remote('github', "https://#{token}@github.com/#{owner}/#{repo}.git") + project.repository.add_remote('github', "https://881a01d03026458e51285a4c7038c9fe4daa5561@github.com/#{owner}/#{repo}.git") project.repository.set_remote_as_mirror('github') project.repository.fetch_remote('github', forced: true) project.repository.remove_remote('github') @@ -31,74 +49,127 @@ module Github end # Fetch labels - labels = Github::Labels.new(owner, repo).fetch - - labels.each do |raw| - begin - label = Github::Representation::Label.new(raw) - project.labels.create!(title: label.title, color: label.color) - rescue => e - error(:label, label.url, e.message) + url = "/repos/#{owner}/#{repo}/labels" + + loop do + response = Github::Client.new.get(url) + + response.body.each do |raw| + begin + label = Github::Representation::Label.new(raw) + next if project.labels.where(title: label.title).exists? + + project.labels.create!(title: label.title, color: label.color) + rescue => e + error(:label, label.url, e.message) + end end + + break unless url = response.rels[:next] end # Fetch milestones - milestones = Github::Milestones.new(owner, repo).fetch - - milestones.each do |raw| - begin - milestone = Github::Representation::Milestone.new(raw) - - project.milestones.create!( - iid: milestone.iid, - title: milestone.title, - description: milestone.description, - due_date: milestone.due_date, - state: milestone.state, - created_at: milestone.created_at, - updated_at: milestone.updated_at - ) - rescue => e - error(:milestone, milestone.url, e.message) + url = "/repos/#{owner}/#{repo}/milestones" + + loop do + response = Github::Client.new.get(url, state: :all) + + response.body.each do |raw| + begin + milestone = Github::Representation::Milestone.new(raw) + next if project.milestones.where(iid: milestone.iid).exists? + + project.milestones.create!( + iid: milestone.iid, + title: milestone.title, + description: milestone.description, + due_date: milestone.due_date, + state: milestone.state, + created_at: milestone.created_at, + updated_at: milestone.updated_at + ) + rescue => e + error(:milestone, milestone.url, e.message) + end end + + break unless url = response.rels[:next] end # Fetch pull requests - pull_requests = Github::PullRequests.new(owner, repo).fetch - - pull_requests.each do |raw| - pull_request = Github::Representation::PullRequest.new(project, raw) - next unless pull_request.valid? - - begin - restore_source_branch(pull_request) unless pull_request.source_branch_exists? - restore_target_branch(pull_request) unless pull_request.target_branch_exists? - - merge_request = MergeRequest.find_or_initialize_by(iid: pull_request.iid, source_project_id: project.id) do |record| - record.iid = pull_request.iid - record.title = pull_request.title - record.description = pull_request.description - record.source_project = pull_request.source_project - record.source_branch = pull_request.source_branch_name - record.source_branch_sha = pull_request.source_branch_sha - record.target_project = pull_request.target_project - record.target_branch = pull_request.target_branch_name - record.target_branch_sha = pull_request.target_branch_sha - record.state = pull_request.state - record.milestone_id = milestone_id(pull_request.milestone) - record.author_id = user_id(pull_request.author, project.creator_id) - record.assignee_id = user_id(pull_request.assignee) - record.created_at = pull_request.created_at - record.updated_at = pull_request.updated_at + url = "/repos/#{owner}/#{repo}/pulls" + + loop do + response = Github::Client.new.get(url, state: :all, sort: :created, direction: :asc) + + response.body.each do |raw| + pull_request = Github::Representation::PullRequest.new(project, raw) + merge_request = MergeRequest.find_or_initialize_by(iid: pull_request.iid, source_project_id: project.id) + next unless merge_request.new_record? && pull_request.valid? + + begin + restore_source_branch(pull_request) unless pull_request.source_branch_exists? + restore_target_branch(pull_request) unless pull_request.target_branch_exists? + + merge_request.iid = pull_request.iid + merge_request.title = pull_request.title + merge_request.description = pull_request.description + merge_request.source_project = pull_request.source_project + merge_request.source_branch = pull_request.source_branch_name + merge_request.source_branch_sha = pull_request.source_branch_sha + merge_request.target_project = pull_request.target_project + merge_request.target_branch = pull_request.target_branch_name + merge_request.target_branch_sha = pull_request.target_branch_sha + merge_request.state = pull_request.state + merge_request.milestone_id = milestone_id(pull_request.milestone) + merge_request.author_id = user_id(pull_request.author, project.creator_id) + merge_request.assignee_id = user_id(pull_request.assignee) + merge_request.created_at = pull_request.created_at + merge_request.updated_at = pull_request.updated_at + merge_request.save(validate: false) + + merge_request.merge_request_diffs.create + rescue => e + error(:pull_request, pull_request.url, e.message) + ensure + clean_up_restored_branches(pull_request) end + end + + break unless url = response.rels[:next] + end - merge_request.save(validate: false) - merge_request.merge_request_diffs.create - rescue => e - error(:pull_request, pull_request.url, "#{e.message}\n\n#{e.exception.backtrace.join('\n')}") - ensure - clean_up_restored_branches(pull_request) + # Fetch issues + url = "/repos/#{owner}/#{repo}/issues" + + loop do + response = Github::Client.new.get(url, state: :all, sort: :created, direction: :asc) + + response.body.each do |raw| + representation = Github::Representation::Issue.new(raw) + + next if representation.pull_request? + next if Issue.where(iid: representation.iid, project_id: project.id).exists? + + begin + issue = Issue.new + issue.iid = representation.iid + issue.project_id = project.id + issue.title = representation.title + issue.description = representation.description + issue.state = representation.state + issue.milestone_id = milestone_id(representation.milestone) + issue.author_id = user_id(representation.author, project.creator_id) + issue.assignee_id = user_id(representation.assignee) + issue.created_at = representation.created_at + issue.updated_at = representation.updated_at + issue.save(validate: false) + rescue => e + error(:issue, representation.url, e.message) + end end + + break unless url = response.rels[:next] end repository.expire_content_cache -- cgit v1.2.1 From 00912ed963a3495d59d8632d8d195515e9686129 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Thu, 13 Apr 2017 13:59:26 -0300 Subject: Refactoring Github response --- lib/github/client.rb | 5 ++--- lib/github/response.rb | 14 +++++++++----- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/lib/github/client.rb b/lib/github/client.rb index 07cf264e8d7..1c24daba1ef 100644 --- a/lib/github/client.rb +++ b/lib/github/client.rb @@ -2,7 +2,7 @@ module Github class Client attr_reader :connection - def initialize(token) + def initialize(token = '881a01d03026458e51285a4c7038c9fe4daa5561') @connection = Faraday.new(url: 'https://api.github.com') do |faraday| faraday.authorization 'token', token faraday.adapter :net_http @@ -13,8 +13,7 @@ module Github rate_limit = RateLimit.new(connection) sleep rate_limit.reset_in if rate_limit.exceed? - response = connection.get(url, query) - Github::Response.new(response.headers, response.body, response.status) + Github::Response.new(connection.get(url, query)) end end end diff --git a/lib/github/response.rb b/lib/github/response.rb index 2fa852c9fdc..2fd07dd822e 100644 --- a/lib/github/response.rb +++ b/lib/github/response.rb @@ -1,11 +1,15 @@ module Github class Response - attr_reader :headers, :body, :status + attr_reader :raw, :headers, :status - def initialize(headers, body, status) - @headers = headers - @body = Oj.load(body, class_cache: false, mode: :compat) - @status = status + def initialize(response) + @raw = response + @headers = response.headers + @status = response.status + end + + def body + @body ||= Oj.load(raw.body, class_cache: false, mode: :compat) end def rels -- cgit v1.2.1 From eb95f0e5b2bc606eeffb2e214379082862d973d6 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Thu, 13 Apr 2017 16:24:55 -0300 Subject: Fix comment representation --- lib/github/representation/comment.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/github/representation/comment.rb b/lib/github/representation/comment.rb index ac7832ce28b..819e4107118 100644 --- a/lib/github/representation/comment.rb +++ b/lib/github/representation/comment.rb @@ -47,11 +47,11 @@ module Github end def diff_hunk - raw_data.diff_hunk + raw['diff_hunk'] end def file_path - raw_data.path + raw['path'] end end end -- cgit v1.2.1 From c26076664f557f697555ced7e97fff9ea8f88aab Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Thu, 13 Apr 2017 17:05:39 -0300 Subject: Import pull requests comments --- lib/github/import.rb | 62 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/lib/github/import.rb b/lib/github/import.rb index 19d366b0444..c4b03da9bc1 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -126,9 +126,69 @@ module Github merge_request.assignee_id = user_id(pull_request.assignee) merge_request.created_at = pull_request.created_at merge_request.updated_at = pull_request.updated_at - merge_request.save(validate: false) + merge_request.save!(validate: false) merge_request.merge_request_diffs.create + + # Fetch review comments + review_comments_url = "/repos/#{owner}/#{repo}/pulls/#{pull_request.iid}/comments" + + loop do + review_comments = Github::Client.new.get(review_comments_url) + + ActiveRecord::Base.no_touching do + review_comments.body.each do |raw| + begin + comment = Github::Representation::Comment.new(raw) + + note = Note.new + note.project_id = project.id + note.noteable = merge_request + note.note = comment.note + note.commit_id = comment.commit_id + note.line_code = comment.line_code + note.author_id = user_id(comment.author, project.creator_id) + note.type = comment.type + note.created_at = comment.created_at + note.updated_at = comment.updated_at + note.save!(validate: false) + rescue => e + error(:review_comment, comment.url, e.message) + end + end + end + + break unless review_comments_url = review_comments.rels[:next] + end + + # Fetch comments + comments_url = "/repos/#{owner}/#{repo}/issues/#{pull_request.iid}/comments" + + loop do + comments = Github::Client.new.get(comments_url) + + ActiveRecord::Base.no_touching do + comments.body.each do |raw| + begin + comment = Github::Representation::Comment.new(raw) + + note = Note.new + note.project_id = project.id + note.noteable = merge_request + note.note = comment.note + note.author_id = user_id(comment.author, project.creator_id) + note.created_at = comment.created_at + note.updated_at = comment.updated_at + note.save!(validate: false) + rescue => e + error(:comment, comment.url, e.message) + end + end + end + + break unless comments_url = comments.rels[:next] + end + rescue => e error(:pull_request, pull_request.url, e.message) ensure -- cgit v1.2.1 From db3220092ade9cb604b24e56a2c21092d8708f9c Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Thu, 13 Apr 2017 17:31:29 -0300 Subject: Import issues comments --- lib/github/import.rb | 30 ++++++++++++++++++++++++++++++ lib/github/representation/issue.rb | 4 ++++ 2 files changed, 34 insertions(+) diff --git a/lib/github/import.rb b/lib/github/import.rb index c4b03da9bc1..c1596e97b3a 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -224,6 +224,36 @@ module Github issue.created_at = representation.created_at issue.updated_at = representation.updated_at issue.save(validate: false) + + if issue.has_comments? + # Fetch comments + comments_url = "/repos/#{owner}/#{repo}/issues/#{issue.iid}/comments" + + loop do + comments = Github::Client.new.get(comments_url) + + ActiveRecord::Base.no_touching do + comments.body.each do |raw| + begin + comment = Github::Representation::Comment.new(raw) + + note = Note.new + note.project_id = project.id + note.noteable = issue + note.note = comment.note + note.author_id = user_id(comment.author, project.creator_id) + note.created_at = comment.created_at + note.updated_at = comment.updated_at + note.save!(validate: false) + rescue => e + error(:comment, comment.url, e.message) + end + end + end + + break unless comments_url = comments.rels[:next] + end + end rescue => e error(:issue, representation.url, e.message) end diff --git a/lib/github/representation/issue.rb b/lib/github/representation/issue.rb index 62c71cc191b..c73eefa0983 100644 --- a/lib/github/representation/issue.rb +++ b/lib/github/representation/issue.rb @@ -49,6 +49,10 @@ module Github raw['assignee'].present? end + def has_comments? + raw['comments'] > 0 + end + def pull_request? raw['pull_request'].present? end -- cgit v1.2.1 From 33c8f315b96a1460d5ff1885a728e0a033f82d7b Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Thu, 13 Apr 2017 18:21:03 -0300 Subject: Apply labels to issues/merge requests --- lib/github/import.rb | 108 ++++++++++++++++++++++--------------- lib/github/representation/issue.rb | 8 +++ 2 files changed, 74 insertions(+), 42 deletions(-) diff --git a/lib/github/import.rb b/lib/github/import.rb index c1596e97b3a..538d6f77351 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -27,11 +27,12 @@ module Github self.reset_callbacks :validate end - attr_reader :project, :repository, :cached_user_ids, :errors + attr_reader :project, :repository, :cached_label_ids, :cached_user_ids, :errors def initialize(project) @project = project @repository = project.repository + @cached_label_ids = {} @cached_user_ids = {} @errors = [] end @@ -40,7 +41,7 @@ module Github # Fetch repository begin project.create_repository - project.repository.add_remote('github', "https://881a01d03026458e51285a4c7038c9fe4daa5561@github.com/#{owner}/#{repo}.git") + project.repository.add_remote('github', "https://{token}@github.com/#{owner}/#{repo}.git") project.repository.set_remote_as_mirror('github') project.repository.fetch_remote('github', forced: true) project.repository.remove_remote('github') @@ -57,6 +58,8 @@ module Github response.body.each do |raw| begin label = Github::Representation::Label.new(raw) + + # TODO: we should take group labels in account next if project.labels.where(title: label.title).exists? project.labels.create!(title: label.title, color: label.color) @@ -68,6 +71,12 @@ module Github break unless url = response.rels[:next] end + # Cache labels + # TODO: we should take group labels in account + project.labels.select(:id, :title).find_each do |label| + @cached_label_ids[label.title] = label.id + end + # Fetch milestones url = "/repos/#{owner}/#{repo}/milestones" @@ -208,50 +217,61 @@ module Github response.body.each do |raw| representation = Github::Representation::Issue.new(raw) - next if representation.pull_request? - next if Issue.where(iid: representation.iid, project_id: project.id).exists? - begin - issue = Issue.new - issue.iid = representation.iid - issue.project_id = project.id - issue.title = representation.title - issue.description = representation.description - issue.state = representation.state - issue.milestone_id = milestone_id(representation.milestone) - issue.author_id = user_id(representation.author, project.creator_id) - issue.assignee_id = user_id(representation.assignee) - issue.created_at = representation.created_at - issue.updated_at = representation.updated_at - issue.save(validate: false) - - if issue.has_comments? - # Fetch comments - comments_url = "/repos/#{owner}/#{repo}/issues/#{issue.iid}/comments" - - loop do - comments = Github::Client.new.get(comments_url) - - ActiveRecord::Base.no_touching do - comments.body.each do |raw| - begin - comment = Github::Representation::Comment.new(raw) - - note = Note.new - note.project_id = project.id - note.noteable = issue - note.note = comment.note - note.author_id = user_id(comment.author, project.creator_id) - note.created_at = comment.created_at - note.updated_at = comment.updated_at - note.save!(validate: false) - rescue => e - error(:comment, comment.url, e.message) + # Every pull request is an issue, but not every issue + # is a pull request. For this reason, "shared" actions + # for both features, like manipulating assignees, labels + # and milestones, are provided within the Issues API. + if representation.pull_request? + next unless representation.has_labels? + + merge_request = MergeRequest.find_by!(target_project_id: project.id, iid: representation.iid) + merge_request.update_attribute(:label_ids, label_ids(representation.labels)) + else + next if Issue.where(iid: representation.iid, project_id: project.id).exists? + + issue = Issue.new + issue.iid = representation.iid + issue.project_id = project.id + issue.title = representation.title + issue.description = representation.description + issue.state = representation.state + issue.label_ids = label_ids(representation.labels) + issue.milestone_id = milestone_id(representation.milestone) + issue.author_id = user_id(representation.author, project.creator_id) + issue.assignee_id = user_id(representation.assignee) + issue.created_at = representation.created_at + issue.updated_at = representation.updated_at + issue.save(validate: false) + + if representation.has_comments? + # Fetch comments + comments_url = "/repos/#{owner}/#{repo}/issues/#{issue.iid}/comments" + + loop do + comments = Github::Client.new.get(comments_url) + + ActiveRecord::Base.no_touching do + comments.body.each do |raw| + begin + comment = Github::Representation::Comment.new(raw) + + note = Note.new + note.project_id = project.id + note.noteable = issue + note.note = comment.note + note.author_id = user_id(comment.author, project.creator_id) + note.created_at = comment.created_at + note.updated_at = comment.updated_at + note.save!(validate: false) + rescue => e + error(:comment, comment.url, e.message) + end end end - end - break unless comments_url = comments.rels[:next] + break unless comments_url = comments.rels[:next] + end end end rescue => e @@ -290,6 +310,10 @@ module Github remove_branch(pull_request.target_branch_name) unless pull_request.target_branch_exists? end + def label_ids(issuable) + issuable.map { |attrs| cached_label_ids[attrs.fetch('name')] }.compact + end + def milestone_id(milestone) return unless milestone.present? diff --git a/lib/github/representation/issue.rb b/lib/github/representation/issue.rb index c73eefa0983..f155880b984 100644 --- a/lib/github/representation/issue.rb +++ b/lib/github/representation/issue.rb @@ -13,6 +13,10 @@ module Github raw['body'] || '' end + def labels + raw['labels'] + end + def milestone return unless raw['milestone'].present? @@ -53,6 +57,10 @@ module Github raw['comments'] > 0 end + def has_labels? + labels.count > 0 + end + def pull_request? raw['pull_request'].present? end -- cgit v1.2.1 From a32adb82dc5b27069835219a5189f497686b8b04 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Thu, 13 Apr 2017 19:00:10 -0300 Subject: Remove sensitive information --- lib/github/client.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/github/client.rb b/lib/github/client.rb index 1c24daba1ef..1450a8d3cc0 100644 --- a/lib/github/client.rb +++ b/lib/github/client.rb @@ -2,7 +2,7 @@ module Github class Client attr_reader :connection - def initialize(token = '881a01d03026458e51285a4c7038c9fe4daa5561') + def initialize(token) @connection = Faraday.new(url: 'https://api.github.com') do |faraday| faraday.authorization 'token', token faraday.adapter :net_http -- cgit v1.2.1 From f35573f12eb579b31b014fa99509c694021c33c7 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 19 Apr 2017 19:14:40 -0300 Subject: Extract common attributes to Github::Representation::Base --- lib/github/representation/base.rb | 12 ++++++++++++ lib/github/representation/comment.rb | 12 ------------ lib/github/representation/issue.rb | 12 ------------ lib/github/representation/label.rb | 4 ---- lib/github/representation/milestone.rb | 12 ------------ lib/github/representation/pull_request.rb | 12 ------------ 6 files changed, 12 insertions(+), 52 deletions(-) diff --git a/lib/github/representation/base.rb b/lib/github/representation/base.rb index 6765bfb803b..5ea294ed49c 100644 --- a/lib/github/representation/base.rb +++ b/lib/github/representation/base.rb @@ -5,6 +5,18 @@ module Github @raw = raw end + def url + raw['url'] + end + + def created_at + raw['created_at'] + end + + def updated_at + raw['updated_at'] + end + private attr_reader :raw diff --git a/lib/github/representation/comment.rb b/lib/github/representation/comment.rb index 819e4107118..02bcd9eaa0e 100644 --- a/lib/github/representation/comment.rb +++ b/lib/github/representation/comment.rb @@ -24,18 +24,6 @@ module Github 'LegacyDiffNote' if on_diff? end - def url - raw['url'] - end - - def created_at - raw['created_at'] - end - - def updated_at - raw['updated_at'] - end - private def generate_line_code(line) diff --git a/lib/github/representation/issue.rb b/lib/github/representation/issue.rb index f155880b984..3bb767a5daa 100644 --- a/lib/github/representation/issue.rb +++ b/lib/github/representation/issue.rb @@ -37,18 +37,6 @@ module Github raw['state'] == 'closed' ? 'closed' : 'opened' end - def url - raw['url'] - end - - def created_at - raw['created_at'] - end - - def updated_at - raw['updated_at'] - end - def assigned? raw['assignee'].present? end diff --git a/lib/github/representation/label.rb b/lib/github/representation/label.rb index b3140ab76fc..60aa51f9569 100644 --- a/lib/github/representation/label.rb +++ b/lib/github/representation/label.rb @@ -8,10 +8,6 @@ module Github def title raw['name'] end - - def url - raw['url'] - end end end end diff --git a/lib/github/representation/milestone.rb b/lib/github/representation/milestone.rb index 5ea54eb178f..917e6394ad4 100644 --- a/lib/github/representation/milestone.rb +++ b/lib/github/representation/milestone.rb @@ -20,18 +20,6 @@ module Github def state raw['state'] == 'closed' ? 'closed' : 'active' end - - def url - raw['url'] - end - - def created_at - raw['created_at'] - end - - def updated_at - raw['updated_at'] - end end end end diff --git a/lib/github/representation/pull_request.rb b/lib/github/representation/pull_request.rb index 85f4e1bdac3..b33561565bf 100644 --- a/lib/github/representation/pull_request.rb +++ b/lib/github/representation/pull_request.rb @@ -71,18 +71,6 @@ module Github 'opened' end - def url - raw['url'] - end - - def created_at - raw['created_at'] - end - - def updated_at - raw['updated_at'] - end - def assigned? raw['assignee'].present? end -- cgit v1.2.1 From 00e3d60c3d0ad1b6c981e3069d8b815d5119aa90 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 19 Apr 2017 19:17:42 -0300 Subject: Extract Github::Representation::Issuable --- lib/github/representation/issuable.rb | 37 +++++++++++++++++++++++++++++++ lib/github/representation/issue.rb | 34 +--------------------------- lib/github/representation/pull_request.rb | 34 +--------------------------- 3 files changed, 39 insertions(+), 66 deletions(-) create mode 100644 lib/github/representation/issuable.rb diff --git a/lib/github/representation/issuable.rb b/lib/github/representation/issuable.rb new file mode 100644 index 00000000000..a55976f9019 --- /dev/null +++ b/lib/github/representation/issuable.rb @@ -0,0 +1,37 @@ +module Github + module Representation + class Issuable < Representation::Base + def iid + raw['number'] + end + + def title + raw['title'] + end + + def description + raw['body'] || '' + end + + def milestone + return unless raw['milestone'].present? + + @milestone ||= Github::Representation::Milestone.new(raw['milestone']) + end + + def author + @author ||= Github::Representation::User.new(raw['user']) + end + + def assignee + return unless assigned? + + @assignee ||= Github::Representation::User.new(raw['assignee']) + end + + def assigned? + raw['assignee'].present? + end + end + end +end diff --git a/lib/github/representation/issue.rb b/lib/github/representation/issue.rb index 3bb767a5daa..df3540a6e6c 100644 --- a/lib/github/representation/issue.rb +++ b/lib/github/representation/issue.rb @@ -1,46 +1,14 @@ module Github module Representation - class Issue < Representation::Base - def iid - raw['number'] - end - - def title - raw['title'] - end - - def description - raw['body'] || '' - end - + class Issue < Representation::Issuable def labels raw['labels'] end - def milestone - return unless raw['milestone'].present? - - @milestone ||= Github::Representation::Milestone.new(raw['milestone']) - end - - def author - @author ||= Github::Representation::User.new(raw['user']) - end - - def assignee - return unless assigned? - - @assignee ||= Github::Representation::User.new(raw['assignee']) - end - def state raw['state'] == 'closed' ? 'closed' : 'opened' end - def assigned? - raw['assignee'].present? - end - def has_comments? raw['comments'] > 0 end diff --git a/lib/github/representation/pull_request.rb b/lib/github/representation/pull_request.rb index b33561565bf..0596b0425a2 100644 --- a/lib/github/representation/pull_request.rb +++ b/lib/github/representation/pull_request.rb @@ -1,6 +1,6 @@ module Github module Representation - class PullRequest < Representation::Base + class PullRequest < Representation::Issuable attr_reader :project delegate :user, :repo, :ref, :sha, to: :source_branch, prefix: true @@ -11,18 +11,6 @@ module Github @raw = raw end - def iid - raw['number'] - end - - def title - raw['title'] - end - - def description - raw['body'] || '' - end - def source_project project end @@ -48,22 +36,6 @@ module Github @target_branch_name ||= target_branch_exists? ? target_branch_ref : target_branch_name_prefixed end - def milestone - return unless raw['milestone'].present? - - @milestone ||= Github::Representation::Milestone.new(raw['milestone']) - end - - def author - @author ||= Github::Representation::User.new(raw['user']) - end - - def assignee - return unless assigned? - - @assignee ||= Github::Representation::User.new(raw['assignee']) - end - def state return 'merged' if raw['state'] == 'closed' && raw['merged_at'].present? return 'closed' if raw['state'] == 'closed' @@ -71,10 +43,6 @@ module Github 'opened' end - def assigned? - raw['assignee'].present? - end - def opened? state == 'opened' end -- cgit v1.2.1 From ac1634fac9ef2891ef98d499fe6391d315b98b30 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 19 Apr 2017 19:41:17 -0300 Subject: Extract a method to import issues/pull requests comments --- lib/github/import.rb | 112 +++++++++++++++------------------------------------ 1 file changed, 33 insertions(+), 79 deletions(-) diff --git a/lib/github/import.rb b/lib/github/import.rb index 538d6f77351..333bfa0fe05 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -141,63 +141,11 @@ module Github # Fetch review comments review_comments_url = "/repos/#{owner}/#{repo}/pulls/#{pull_request.iid}/comments" - - loop do - review_comments = Github::Client.new.get(review_comments_url) - - ActiveRecord::Base.no_touching do - review_comments.body.each do |raw| - begin - comment = Github::Representation::Comment.new(raw) - - note = Note.new - note.project_id = project.id - note.noteable = merge_request - note.note = comment.note - note.commit_id = comment.commit_id - note.line_code = comment.line_code - note.author_id = user_id(comment.author, project.creator_id) - note.type = comment.type - note.created_at = comment.created_at - note.updated_at = comment.updated_at - note.save!(validate: false) - rescue => e - error(:review_comment, comment.url, e.message) - end - end - end - - break unless review_comments_url = review_comments.rels[:next] - end + fetch_comments(merge_request, :review_comment, review_comments_url) # Fetch comments comments_url = "/repos/#{owner}/#{repo}/issues/#{pull_request.iid}/comments" - - loop do - comments = Github::Client.new.get(comments_url) - - ActiveRecord::Base.no_touching do - comments.body.each do |raw| - begin - comment = Github::Representation::Comment.new(raw) - - note = Note.new - note.project_id = project.id - note.noteable = merge_request - note.note = comment.note - note.author_id = user_id(comment.author, project.creator_id) - note.created_at = comment.created_at - note.updated_at = comment.updated_at - note.save!(validate: false) - rescue => e - error(:comment, comment.url, e.message) - end - end - end - - break unless comments_url = comments.rels[:next] - end - + fetch_comments(merge_request, :comment, comments_url) rescue => e error(:pull_request, pull_request.url, e.message) ensure @@ -247,31 +195,7 @@ module Github if representation.has_comments? # Fetch comments comments_url = "/repos/#{owner}/#{repo}/issues/#{issue.iid}/comments" - - loop do - comments = Github::Client.new.get(comments_url) - - ActiveRecord::Base.no_touching do - comments.body.each do |raw| - begin - comment = Github::Representation::Comment.new(raw) - - note = Note.new - note.project_id = project.id - note.noteable = issue - note.note = comment.note - note.author_id = user_id(comment.author, project.creator_id) - note.created_at = comment.created_at - note.updated_at = comment.updated_at - note.save!(validate: false) - rescue => e - error(:comment, comment.url, e.message) - end - end - end - - break unless comments_url = comments.rels[:next] - end + fetch_comments(issue, :comment, comments_url) end end rescue => e @@ -289,6 +213,36 @@ module Github private + def fetch_comments(noteable, type, url) + loop do + comments = Github::Client.new.get(url) + + ActiveRecord::Base.no_touching do + comments.body.each do |raw| + begin + representation = Github::Representation::Comment.new(raw) + + note = Note.new + note.project_id = project.id + note.noteable = noteable + note.note = representation.note + note.commit_id = representation.commit_id + note.line_code = representation.line_code + note.author_id = user_id(representation.author, project.creator_id) + note.type = representation.type + note.created_at = representation.created_at + note.updated_at = representation.updated_at + note.save!(validate: false) + rescue => e + error(type, representation.url, e.message) + end + end + end + + break unless url = comments.rels[:next] + end + end + def restore_source_branch(pull_request) repository.create_branch(pull_request.source_branch_name, pull_request.source_branch_sha) end -- cgit v1.2.1 From 782aab1319bdcfbe1634d4b33444e8ce5b57d394 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 19 Apr 2017 20:04:58 -0300 Subject: Pass a options hash to Github::Client --- lib/github/client.rb | 6 +++--- lib/github/collection.rb | 8 +++++++- lib/github/import.rb | 23 ++++++++++++----------- lib/github/representation/base.rb | 7 ++++--- lib/github/representation/comment.rb | 2 +- lib/github/representation/issuable.rb | 4 ++-- lib/github/representation/pull_request.rb | 5 +++-- lib/github/representation/user.rb | 2 +- lib/github/user.rb | 7 ++++--- 9 files changed, 37 insertions(+), 27 deletions(-) diff --git a/lib/github/client.rb b/lib/github/client.rb index 1450a8d3cc0..95536cae57f 100644 --- a/lib/github/client.rb +++ b/lib/github/client.rb @@ -2,9 +2,9 @@ module Github class Client attr_reader :connection - def initialize(token) - @connection = Faraday.new(url: 'https://api.github.com') do |faraday| - faraday.authorization 'token', token + def initialize(options) + @connection = Faraday.new(url: options.fetch(:url)) do |faraday| + faraday.authorization 'token', options.fetch(:token) faraday.adapter :net_http end end diff --git a/lib/github/collection.rb b/lib/github/collection.rb index 1b0c00928c5..12d6703476b 100644 --- a/lib/github/collection.rb +++ b/lib/github/collection.rb @@ -1,5 +1,11 @@ module Github class Collection + attr_reader :options + + def initialize(options) + @options = options + end + def fetch(url, query = {}) return [] if url.blank? @@ -16,7 +22,7 @@ module Github private def client - @client ||= Github::Client.new + @client ||= Github::Client.new(options) end end end diff --git a/lib/github/import.rb b/lib/github/import.rb index 333bfa0fe05..17244d8aea1 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -27,17 +27,18 @@ module Github self.reset_callbacks :validate end - attr_reader :project, :repository, :cached_label_ids, :cached_user_ids, :errors + attr_reader :project, :repository, :options, :cached_label_ids, :cached_user_ids, :errors - def initialize(project) + def initialize(project, options) @project = project @repository = project.repository + @options = options @cached_label_ids = {} @cached_user_ids = {} @errors = [] end - def execute(owner, repo, token) + def execute(owner, repo) # Fetch repository begin project.create_repository @@ -53,7 +54,7 @@ module Github url = "/repos/#{owner}/#{repo}/labels" loop do - response = Github::Client.new.get(url) + response = Github::Client.new(options).get(url) response.body.each do |raw| begin @@ -81,7 +82,7 @@ module Github url = "/repos/#{owner}/#{repo}/milestones" loop do - response = Github::Client.new.get(url, state: :all) + response = Github::Client.new(options).get(url, state: :all) response.body.each do |raw| begin @@ -109,10 +110,10 @@ module Github url = "/repos/#{owner}/#{repo}/pulls" loop do - response = Github::Client.new.get(url, state: :all, sort: :created, direction: :asc) + response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc) response.body.each do |raw| - pull_request = Github::Representation::PullRequest.new(project, raw) + pull_request = Github::Representation::PullRequest.new(project, raw, options) merge_request = MergeRequest.find_or_initialize_by(iid: pull_request.iid, source_project_id: project.id) next unless merge_request.new_record? && pull_request.valid? @@ -160,10 +161,10 @@ module Github url = "/repos/#{owner}/#{repo}/issues" loop do - response = Github::Client.new.get(url, state: :all, sort: :created, direction: :asc) + response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc) response.body.each do |raw| - representation = Github::Representation::Issue.new(raw) + representation = Github::Representation::Issue.new(raw, options) begin # Every pull request is an issue, but not every issue @@ -215,12 +216,12 @@ module Github def fetch_comments(noteable, type, url) loop do - comments = Github::Client.new.get(url) + comments = Github::Client.new(options).get(url) ActiveRecord::Base.no_touching do comments.body.each do |raw| begin - representation = Github::Representation::Comment.new(raw) + representation = Github::Representation::Comment.new(raw, options) note = Note.new note.project_id = project.id diff --git a/lib/github/representation/base.rb b/lib/github/representation/base.rb index 5ea294ed49c..385e62ae99d 100644 --- a/lib/github/representation/base.rb +++ b/lib/github/representation/base.rb @@ -1,8 +1,9 @@ module Github module Representation class Base - def initialize(raw) - @raw = raw + def initialize(raw, options = {}) + @raw = raw + @options = options end def url @@ -19,7 +20,7 @@ module Github private - attr_reader :raw + attr_reader :raw, :options end end end diff --git a/lib/github/representation/comment.rb b/lib/github/representation/comment.rb index 02bcd9eaa0e..22cb98b0eff 100644 --- a/lib/github/representation/comment.rb +++ b/lib/github/representation/comment.rb @@ -6,7 +6,7 @@ module Github end def author - @author ||= Github::Representation::User.new(raw['user']) + @author ||= Github::Representation::User.new(raw['user'], options) end def commit_id diff --git a/lib/github/representation/issuable.rb b/lib/github/representation/issuable.rb index a55976f9019..9713b82615d 100644 --- a/lib/github/representation/issuable.rb +++ b/lib/github/representation/issuable.rb @@ -20,13 +20,13 @@ module Github end def author - @author ||= Github::Representation::User.new(raw['user']) + @author ||= Github::Representation::User.new(raw['user'], options) end def assignee return unless assigned? - @assignee ||= Github::Representation::User.new(raw['assignee']) + @assignee ||= Github::Representation::User.new(raw['assignee'], options) end def assigned? diff --git a/lib/github/representation/pull_request.rb b/lib/github/representation/pull_request.rb index 0596b0425a2..4119ca400c6 100644 --- a/lib/github/representation/pull_request.rb +++ b/lib/github/representation/pull_request.rb @@ -6,9 +6,10 @@ module Github delegate :user, :repo, :ref, :sha, to: :source_branch, prefix: true delegate :user, :exists?, :repo, :ref, :sha, :short_sha, to: :target_branch, prefix: true - def initialize(project, raw) + def initialize(project, raw, options) @project = project - @raw = raw + @raw = raw + @options = options end def source_project diff --git a/lib/github/representation/user.rb b/lib/github/representation/user.rb index 70a0ce000ae..79758555319 100644 --- a/lib/github/representation/user.rb +++ b/lib/github/representation/user.rb @@ -8,7 +8,7 @@ module Github def email return @email if defined?(@email) - @email = Github::User.new(username).get.fetch('email', nil) + @email = Github::User.new(username, options).get.fetch('email', nil) end def username diff --git a/lib/github/user.rb b/lib/github/user.rb index 19fe6230820..f88a29e590b 100644 --- a/lib/github/user.rb +++ b/lib/github/user.rb @@ -1,9 +1,10 @@ module Github class User - attr_reader :username + attr_reader :username, :options - def initialize(username) + def initialize(username, options) @username = username + @options = options end def get @@ -13,7 +14,7 @@ module Github private def client - @client ||= Github::Client.new + @client ||= Github::Client.new(options) end def user_url -- cgit v1.2.1 From 5691c9b0611645ac3a6f5e32d033d7bafb09d781 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 19 Apr 2017 20:31:52 -0300 Subject: Does not remove the GitHub remote --- lib/github/import.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/github/import.rb b/lib/github/import.rb index 17244d8aea1..17dd6dd0ccf 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -45,7 +45,6 @@ module Github project.repository.add_remote('github', "https://{token}@github.com/#{owner}/#{repo}.git") project.repository.set_remote_as_mirror('github') project.repository.fetch_remote('github', forced: true) - project.repository.remove_remote('github') rescue Gitlab::Shell::Error => e error(:project, "https://github.com/#{owner}/#{repo}.git", e.message) end -- cgit v1.2.1 From 181445307cbb4a8cb2904a3b969bc39568d133d5 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 19 Apr 2017 20:39:04 -0300 Subject: Add a method to format issues/pull requests/comments body --- lib/github/import.rb | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/lib/github/import.rb b/lib/github/import.rb index 17dd6dd0ccf..0552320a474 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -27,7 +27,8 @@ module Github self.reset_callbacks :validate end - attr_reader :project, :repository, :options, :cached_label_ids, :cached_user_ids, :errors + attr_reader :project, :repository, :options, :cached_label_ids, + :cached_gitlab_users, :cached_user_ids, :errors def initialize(project, options) @project = project @@ -35,6 +36,7 @@ module Github @options = options @cached_label_ids = {} @cached_user_ids = {} + @cached_gitlab_users = {} @errors = [] end @@ -120,9 +122,10 @@ module Github restore_source_branch(pull_request) unless pull_request.source_branch_exists? restore_target_branch(pull_request) unless pull_request.target_branch_exists? + author_id = user_id(pull_request.author, project.creator_id) merge_request.iid = pull_request.iid merge_request.title = pull_request.title - merge_request.description = pull_request.description + merge_request.description = format_description(pull_request.description, pull_request.author) merge_request.source_project = pull_request.source_project merge_request.source_branch = pull_request.source_branch_name merge_request.source_branch_sha = pull_request.source_branch_sha @@ -131,7 +134,7 @@ module Github merge_request.target_branch_sha = pull_request.target_branch_sha merge_request.state = pull_request.state merge_request.milestone_id = milestone_id(pull_request.milestone) - merge_request.author_id = user_id(pull_request.author, project.creator_id) + merge_request.author_id = author_id merge_request.assignee_id = user_id(pull_request.assignee) merge_request.created_at = pull_request.created_at merge_request.updated_at = pull_request.updated_at @@ -178,19 +181,20 @@ module Github else next if Issue.where(iid: representation.iid, project_id: project.id).exists? + author_id = user_id(representation.author, project.creator_id) issue = Issue.new issue.iid = representation.iid issue.project_id = project.id issue.title = representation.title - issue.description = representation.description + issue.description = format_description(representation.description, representation.author) issue.state = representation.state issue.label_ids = label_ids(representation.labels) issue.milestone_id = milestone_id(representation.milestone) - issue.author_id = user_id(representation.author, project.creator_id) + issue.author_id = author_id issue.assignee_id = user_id(representation.assignee) issue.created_at = representation.created_at issue.updated_at = representation.updated_at - issue.save(validate: false) + issue.save!(validate: false) if representation.has_comments? # Fetch comments @@ -220,15 +224,16 @@ module Github ActiveRecord::Base.no_touching do comments.body.each do |raw| begin - representation = Github::Representation::Comment.new(raw, options) + representation = Github::Representation::Comment.new(raw, options) + author_id = user_id(representation.author, project.creator_id) note = Note.new note.project_id = project.id note.noteable = noteable - note.note = representation.note + note.note = format_description(representation.note, representation.author) note.commit_id = representation.commit_id note.line_code = representation.line_code - note.author_id = user_id(representation.author, project.creator_id) + note.author_id = author_id note.type = representation.type note.created_at = representation.created_at note.updated_at = representation.updated_at @@ -278,7 +283,10 @@ module Github return unless user.present? return cached_user_ids[user.id] if cached_user_ids.key?(user.id) - cached_user_ids[user.id] = find_by_external_uid(user.id) || find_by_email(user.email) || fallback_id + gitlab_user_id = find_by_external_uid(user.id) || find_by_email(user.email) + + cached_gitlab_users[user.id] = gitlab_user_id.present? + cached_user_ids[user.id] = gitlab_user_id || fallback_id end def find_by_email(email) @@ -298,6 +306,12 @@ module Github .first&.id end + def format_description(body, author) + return body if cached_gitlab_users[author.id] + + "*Created by: #{author.username}*\n\n#{body}" + end + def error(type, url, message) errors << { type: type, url: Gitlab::UrlSanitizer.sanitize(url), error: message } end -- cgit v1.2.1 From a7cb336e55a56797919dddeef3978997962127d3 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 19 Apr 2017 20:45:49 -0300 Subject: Use while instead of loop/break --- lib/github/import.rb | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/lib/github/import.rb b/lib/github/import.rb index 0552320a474..e9632d3ee11 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -54,14 +54,12 @@ module Github # Fetch labels url = "/repos/#{owner}/#{repo}/labels" - loop do + while url response = Github::Client.new(options).get(url) response.body.each do |raw| begin label = Github::Representation::Label.new(raw) - - # TODO: we should take group labels in account next if project.labels.where(title: label.title).exists? project.labels.create!(title: label.title, color: label.color) @@ -70,7 +68,7 @@ module Github end end - break unless url = response.rels[:next] + url = response.rels[:next] end # Cache labels @@ -82,7 +80,7 @@ module Github # Fetch milestones url = "/repos/#{owner}/#{repo}/milestones" - loop do + while url response = Github::Client.new(options).get(url, state: :all) response.body.each do |raw| @@ -104,13 +102,13 @@ module Github end end - break unless url = response.rels[:next] + url = response.rels[:next] end # Fetch pull requests url = "/repos/#{owner}/#{repo}/pulls" - loop do + while url response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc) response.body.each do |raw| @@ -156,13 +154,13 @@ module Github end end - break unless url = response.rels[:next] + url = response.rels[:next] end # Fetch issues url = "/repos/#{owner}/#{repo}/issues" - loop do + while url response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc) response.body.each do |raw| @@ -207,7 +205,7 @@ module Github end end - break unless url = response.rels[:next] + url = response.rels[:next] end repository.expire_content_cache @@ -218,7 +216,7 @@ module Github private def fetch_comments(noteable, type, url) - loop do + while url comments = Github::Client.new(options).get(url) ActiveRecord::Base.no_touching do @@ -244,7 +242,7 @@ module Github end end - break unless url = comments.rels[:next] + url = comments.rels[:next] end end -- cgit v1.2.1 From 3aa89795568dc2fbc53f2fac8c4f8ac3499dadb0 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 19 Apr 2017 21:03:46 -0300 Subject: Refactoring Github import --- lib/github/import.rb | 63 +++++++++++++++++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 25 deletions(-) diff --git a/lib/github/import.rb b/lib/github/import.rb index e9632d3ee11..12f4dbc9e43 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -27,12 +27,13 @@ module Github self.reset_callbacks :validate end - attr_reader :project, :repository, :options, :cached_label_ids, - :cached_gitlab_users, :cached_user_ids, :errors + attr_reader :project, :repository, :repo, :options, :errors, + :cached_label_ids, :cached_gitlab_users, :cached_user_ids def initialize(project, options) @project = project @repository = project.repository + @repo = project.import_source @options = options @cached_label_ids = {} @cached_user_ids = {} @@ -40,19 +41,32 @@ module Github @errors = [] end - def execute(owner, repo) - # Fetch repository + def execute + fetch_repository + fetch_labels + fetch_milestones + fetch_pull_requests + fetch_issues + expire_repository_cache + + errors + end + + private + + def fetch_repository begin project.create_repository - project.repository.add_remote('github', "https://{token}@github.com/#{owner}/#{repo}.git") + project.repository.add_remote('github', "https://{token}@github.com/#{repo}.git") project.repository.set_remote_as_mirror('github') project.repository.fetch_remote('github', forced: true) rescue Gitlab::Shell::Error => e - error(:project, "https://github.com/#{owner}/#{repo}.git", e.message) + error(:project, "https://github.com/#{repo}.git", e.message) end + end - # Fetch labels - url = "/repos/#{owner}/#{repo}/labels" + def fetch_labels + url = "/repos/#{repo}/labels" while url response = Github::Client.new(options).get(url) @@ -72,13 +86,13 @@ module Github end # Cache labels - # TODO: we should take group labels in account project.labels.select(:id, :title).find_each do |label| @cached_label_ids[label.title] = label.id end + end - # Fetch milestones - url = "/repos/#{owner}/#{repo}/milestones" + def fetch_milestones + url = "/repos/#{repo}/milestones" while url response = Github::Client.new(options).get(url, state: :all) @@ -104,9 +118,10 @@ module Github url = response.rels[:next] end + end - # Fetch pull requests - url = "/repos/#{owner}/#{repo}/pulls" + def fetch_pull_requests + url = "/repos/#{repo}/pulls" while url response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc) @@ -141,14 +156,13 @@ module Github merge_request.merge_request_diffs.create # Fetch review comments - review_comments_url = "/repos/#{owner}/#{repo}/pulls/#{pull_request.iid}/comments" + review_comments_url = "/repos/#{repo}/pulls/#{pull_request.iid}/comments" fetch_comments(merge_request, :review_comment, review_comments_url) # Fetch comments - comments_url = "/repos/#{owner}/#{repo}/issues/#{pull_request.iid}/comments" + comments_url = "/repos/#{repo}/issues/#{pull_request.iid}/comments" fetch_comments(merge_request, :comment, comments_url) rescue => e - error(:pull_request, pull_request.url, e.message) ensure clean_up_restored_branches(pull_request) end @@ -156,9 +170,10 @@ module Github url = response.rels[:next] end + end - # Fetch issues - url = "/repos/#{owner}/#{repo}/issues" + def fetch_issues + url = "/repos/#{repo}/issues" while url response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc) @@ -196,7 +211,7 @@ module Github if representation.has_comments? # Fetch comments - comments_url = "/repos/#{owner}/#{repo}/issues/#{issue.iid}/comments" + comments_url = "/repos/#{repo}/issues/#{issue.iid}/comments" fetch_comments(issue, :comment, comments_url) end end @@ -207,14 +222,8 @@ module Github url = response.rels[:next] end - - repository.expire_content_cache - - errors end - private - def fetch_comments(noteable, type, url) while url comments = Github::Client.new(options).get(url) @@ -310,6 +319,10 @@ module Github "*Created by: #{author.username}*\n\n#{body}" end + def expire_repository_cache + repository.expire_content_cache + end + def error(type, url, message) errors << { type: type, url: Gitlab::UrlSanitizer.sanitize(url), error: message } end -- cgit v1.2.1 From 3c0a713a379025c079f8a684943e95af087c9e86 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 19 Apr 2017 21:18:11 -0300 Subject: Import Github releases --- lib/github/import.rb | 27 +++++++++++++++++++++++++++ lib/github/representation/release.rb | 17 +++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 lib/github/representation/release.rb diff --git a/lib/github/import.rb b/lib/github/import.rb index 12f4dbc9e43..4ef0ac3e40c 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -255,6 +255,33 @@ module Github end end + def fetch_releases + url = "/repos/#{repo}/releases" + + while url + response = Github::Client.new(options).get(url) + + response.body.each do |raw| + representation = Github::Representation::Release.new(raw) + next unless representation.valid? + + release = ::Release.find_or_initialize_by(project_id: project.id, tag: representation.tag) + next unless relese.new_record? + + begin + release.description = representation.description + release.created_at = representation.created_at + release.updated_at = representation.updated_at + release.save!(validate: false) + rescue => e + error(:release, representation.url, e.message) + end + end + + url = response.rels[:next] + end + end + def restore_source_branch(pull_request) repository.create_branch(pull_request.source_branch_name, pull_request.source_branch_sha) end diff --git a/lib/github/representation/release.rb b/lib/github/representation/release.rb new file mode 100644 index 00000000000..e7e4b428c1a --- /dev/null +++ b/lib/github/representation/release.rb @@ -0,0 +1,17 @@ +module Github + module Representation + class Release < Representation::Base + def description + raw['body'] + end + + def tag + raw['tag_name'] + end + + def valid? + !raw['draft'] + end + end + end +end -- cgit v1.2.1 From bd9e5c5ddf1810da33c65090c16f4e44e087cbcf Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 19 Apr 2017 22:01:14 -0300 Subject: Clone GitHub wiki --- lib/github/import.rb | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/lib/github/import.rb b/lib/github/import.rb index 4ef0ac3e40c..bf0df041f15 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -1,5 +1,6 @@ module Github class Import + class MergeRequest < ::MergeRequest self.table_name = 'merge_requests' @@ -47,6 +48,7 @@ module Github fetch_milestones fetch_pull_requests fetch_issues + fetch_wiki_repository expire_repository_cache errors @@ -57,7 +59,7 @@ module Github def fetch_repository begin project.create_repository - project.repository.add_remote('github', "https://{token}@github.com/#{repo}.git") + project.repository.add_remote('github', "https://{options.fetch(:token)}@github.com/#{repo}.git") project.repository.set_remote_as_mirror('github') project.repository.fetch_remote('github', forced: true) rescue Gitlab::Shell::Error => e @@ -65,6 +67,22 @@ module Github end end + def fetch_wiki_repository + wiki_url = "https://{options.fetch(:token)}@github.com/#{repo}.wiki.git" + wiki_path = "#{project.path_with_namespace}.wiki" + + unless project.wiki.repository_exists? + gitlab_shell.import_repository(project.repository_storage_path, wiki_path, wiki_url) + end + rescue Gitlab::Shell::Error => e + # GitHub error message when the wiki repo has not been created, + # this means that repo has wiki enabled, but have no pages. So, + # we can skip the import. + if e.message !~ /repository not exported/ + errors(:wiki, wiki_url, e.message) + end + end + def fetch_labels url = "/repos/#{repo}/labels" @@ -163,6 +181,7 @@ module Github comments_url = "/repos/#{repo}/issues/#{pull_request.iid}/comments" fetch_comments(merge_request, :comment, comments_url) rescue => e + error(:pull_request, pull_request.url, e.message) ensure clean_up_restored_branches(pull_request) end @@ -209,8 +228,8 @@ module Github issue.updated_at = representation.updated_at issue.save!(validate: false) + # Fetch comments if representation.has_comments? - # Fetch comments comments_url = "/repos/#{repo}/issues/#{issue.iid}/comments" fetch_comments(issue, :comment, comments_url) end -- cgit v1.2.1 From 09a6d32817f5f4da8ff9f3331a7046670a10beed Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 19 Apr 2017 22:01:37 -0300 Subject: Keep track of import errors --- lib/github/import.rb | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/github/import.rb b/lib/github/import.rb index bf0df041f15..a1854f7f235 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -50,6 +50,7 @@ module Github fetch_issues fetch_wiki_repository expire_repository_cache + track_errors errors end @@ -369,6 +370,15 @@ module Github repository.expire_content_cache end + def track_errors + return unless errors.any? + + project.update_column(:import_error, { + message: 'The remote data could not be fully imported.', + errors: errors + }.to_json) + end + def error(type, url, message) errors << { type: type, url: Gitlab::UrlSanitizer.sanitize(url), error: message } end -- cgit v1.2.1 From e50606cd2d0c5a010d3c139cbad1eaf16701d8fd Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Wed, 19 Apr 2017 22:54:09 -0300 Subject: Refactor rake task to to import GitHub repositores --- lib/github/import.rb | 1 + lib/github/repositories.rb | 8 ++-- lib/tasks/import.rake | 111 +++++++++------------------------------------ 3 files changed, 28 insertions(+), 92 deletions(-) diff --git a/lib/github/import.rb b/lib/github/import.rb index a1854f7f235..de987c263b6 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -1,5 +1,6 @@ module Github class Import + include Gitlab::ShellAdapter class MergeRequest < ::MergeRequest self.table_name = 'merge_requests' diff --git a/lib/github/repositories.rb b/lib/github/repositories.rb index b6a03173673..c1c9448f305 100644 --- a/lib/github/repositories.rb +++ b/lib/github/repositories.rb @@ -1,11 +1,13 @@ module Github class Repositories - def initialize(username) - @username = username + attr_reader :options + + def initialize(options) + @options = options end def fetch - Collection.new.fetch(repos_url) + Collection.new(options).fetch(repos_url) end private diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index a9dad6a1bf0..897b2f04ce1 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -1,67 +1,5 @@ require 'benchmark' require 'rainbow/ext/string' -require_relative '../gitlab/shell_adapter' -require_relative '../gitlab/github_import/importer' - -class NewImporter < ::Gitlab::GithubImport::Importer - def execute - # Same as ::Gitlab::GithubImport::Importer#execute, but showing some progress. - puts 'Importing repository...'.color(:aqua) - import_repository unless project.repository_exists? - - puts 'Importing labels...'.color(:aqua) - import_labels - - puts 'Importing milestones...'.color(:aqua) - import_milestones - - puts 'Importing pull requests...'.color(:aqua) - import_pull_requests - - puts 'Importing issues...'.color(:aqua) - import_issues - - puts 'Importing issue comments...'.color(:aqua) - import_comments(:issues) - - puts 'Importing pull request comments...'.color(:aqua) - import_comments(:pull_requests) - - puts 'Importing wiki...'.color(:aqua) - import_wiki - - # Gitea doesn't have a Release API yet - # See https://github.com/go-gitea/gitea/issues/330 - unless project.gitea_import? - import_releases - end - - handle_errors - - project.repository.after_import - project.import_finish - - true - end - - def import_repository - begin - raise 'Blocked import URL.' if Gitlab::UrlBlocker.blocked_url?(project.import_url) - - project.create_repository - project.repository.add_remote(project.import_type, project.import_url) - project.repository.set_remote_as_mirror(project.import_type) - project.repository.fetch_remote(project.import_type, forced: true) - rescue => e - # Expire cache to prevent scenarios such as: - # 1. First import failed, but the repo was imported successfully, so +exists?+ returns true - # 2. Retried import, repo is broken or not imported but +exists?+ still returns true - project.repository.expire_content_cache if project.repository_exists? - - raise "Error importing repository #{project.import_url} into #{project.path_with_namespace} - #{e.message}" - end - end -end class GithubImport def self.run!(*args) @@ -69,14 +7,14 @@ class GithubImport end def initialize(token, gitlab_username, project_path, extras) - @token = token + @options = { url: 'https://api.github.com', token: token } @project_path = project_path @current_user = User.find_by_username(gitlab_username) @github_repo = extras.empty? ? nil : extras.first end def run! - @repo = GithubRepos.new(@token, @current_user, @github_repo).choose_one! + @repo = GithubRepos.new(@options, @current_user, @github_repo).choose_one! raise 'No repo found!' unless @repo @@ -90,25 +28,24 @@ class GithubImport private def show_warning! - puts "This will import GH #{@repo.full_name.bright} into GL #{@project_path.bright} as #{@current_user.name}" + puts "This will import GH #{@repo['full_name'].bright} into GL #{@project_path.bright} as #{@current_user.name}" puts "Permission checks are ignored. Press any key to continue.".color(:red) STDIN.getch - puts 'Starting the import...'.color(:green) + puts 'Starting the import (this could take a while)'.color(:green) end def import! - import_url = @project.import_url.gsub(/\:\/\/(.*@)?/, "://#{@token}@") - @project.update(import_url: import_url) - @project.import_start timings = Benchmark.measure do - NewImporter.new(@project).execute + Github::Import.new(@project, @options).execute end puts "Import finished. Timings: #{timings}".color(:green) + + @project.import_finish end def new_project @@ -116,17 +53,17 @@ class GithubImport namespace_path, _sep, name = @project_path.rpartition('/') namespace = find_or_create_namespace(namespace_path) - Project.create!( - import_url: "https://#{@token}@github.com/#{@repo.full_name}.git", + Projects::CreateService.new( + @current_user, name: name, path: name, - description: @repo.description, - namespace: namespace, + description: @repo['description'], + namespace_id: namespace.id, visibility_level: visibility_level, import_type: 'github', - import_source: @repo.full_name, - creator: @current_user - ) + import_source: @repo['full_name'], + skip_wiki: @repo['has_wiki'] + ).execute end end @@ -159,13 +96,13 @@ class GithubImport end def visibility_level - @repo.private ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility + @repo['private'] ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility end end class GithubRepos - def initialize(token, current_user, github_repo) - @token = token + def initialize(options, current_user, github_repo) + @options = options @current_user = current_user @github_repo = github_repo end @@ -174,17 +111,17 @@ class GithubRepos return found_github_repo if @github_repo repos.each do |repo| - print "ID: #{repo[:id].to_s.bright} ".color(:green) - puts "- Name: #{repo[:full_name]}".color(:green) + print "ID: #{repo['id'].to_s.bright}".color(:green) + print "\tName: #{repo['full_name']}\n".color(:green) end print 'ID? '.bright - repos.find { |repo| repo[:id] == repo_id } + repos.find { |repo| repo['id'] == repo_id } end def found_github_repo - repos.find { |repo| repo[:full_name] == @github_repo } + repos.find { |repo| repo['full_name'] == @github_repo } end def repo_id @@ -192,11 +129,7 @@ class GithubRepos end def repos - @repos ||= client.repos - end - - def client - @client ||= Gitlab::GithubImport::Client.new(@token, {}) + Github::Repositories.new(@options).fetch end end -- cgit v1.2.1 From 2b7328c3171661ce8e4a5b3b840ff5f3df19d3e0 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 24 Apr 2017 18:42:50 -0300 Subject: Rename find_by_email/find_by_external_uid methods --- lib/github/import.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/github/import.rb b/lib/github/import.rb index de987c263b6..cbda4be2576 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -338,19 +338,19 @@ module Github return unless user.present? return cached_user_ids[user.id] if cached_user_ids.key?(user.id) - gitlab_user_id = find_by_external_uid(user.id) || find_by_email(user.email) + gitlab_user_id = user_id_by_external_uid(user.id) || user_id_by_email(user.email) cached_gitlab_users[user.id] = gitlab_user_id.present? cached_user_ids[user.id] = gitlab_user_id || fallback_id end - def find_by_email(email) + def user_id_by_email(email) return nil unless email ::User.find_by_any_email(email)&.id end - def find_by_external_uid(id) + def user_id_by_external_uid(id) return nil unless id identities = ::Identity.arel_table -- cgit v1.2.1 From 9bdde5796a6e6ed56a29865b19a910bdea5d078e Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 24 Apr 2017 19:04:39 -0300 Subject: Add Github::Representation::Base#id --- lib/github/representation/base.rb | 4 ++++ lib/github/representation/repo.rb | 3 --- lib/github/representation/user.rb | 4 ---- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/lib/github/representation/base.rb b/lib/github/representation/base.rb index 385e62ae99d..f26bdbdd546 100644 --- a/lib/github/representation/base.rb +++ b/lib/github/representation/base.rb @@ -6,6 +6,10 @@ module Github @options = options end + def id + raw['id'] + end + def url raw['url'] end diff --git a/lib/github/representation/repo.rb b/lib/github/representation/repo.rb index b9cae43450e..6938aa7db05 100644 --- a/lib/github/representation/repo.rb +++ b/lib/github/representation/repo.rb @@ -1,9 +1,6 @@ module Github module Representation class Repo < Representation::Base - def id - raw['id'] - end end end end diff --git a/lib/github/representation/user.rb b/lib/github/representation/user.rb index 79758555319..18591380e25 100644 --- a/lib/github/representation/user.rb +++ b/lib/github/representation/user.rb @@ -1,10 +1,6 @@ module Github module Representation class User < Representation::Base - def id - raw['id'] - end - def email return @email if defined?(@email) -- cgit v1.2.1 From c7935dcfaec18f4628101268b13b9509b6b412ba Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 24 Apr 2017 19:26:07 -0300 Subject: Does not freeze integer values --- lib/github/rate_limit.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/github/rate_limit.rb b/lib/github/rate_limit.rb index 9dbdf2f4c68..38beb617173 100644 --- a/lib/github/rate_limit.rb +++ b/lib/github/rate_limit.rb @@ -1,7 +1,7 @@ module Github class RateLimit - SAFE_REMAINING_REQUESTS = 100.freeze - SAFE_RESET_TIME = 500.freeze + SAFE_REMAINING_REQUESTS = 100 + SAFE_RESET_TIME = 500 attr_reader :connection -- cgit v1.2.1 From f73a0280c96f76b1b19200990ab07adc732dbb92 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 24 Apr 2017 19:28:08 -0300 Subject: Extract rate limit URL to a constant --- lib/github/rate_limit.rb | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/github/rate_limit.rb b/lib/github/rate_limit.rb index 38beb617173..ab2b9c707d8 100644 --- a/lib/github/rate_limit.rb +++ b/lib/github/rate_limit.rb @@ -2,6 +2,7 @@ module Github class RateLimit SAFE_REMAINING_REQUESTS = 100 SAFE_RESET_TIME = 500 + RATE_LIMIT_URL = '/rate_limit'.freeze attr_reader :connection @@ -25,12 +26,8 @@ module Github private - def rate_limit_url - '/rate_limit' - end - def response - connection.get(rate_limit_url) + connection.get(RATE_LIMIT_URL) end def body -- cgit v1.2.1 From 275f00ee88cb9ef6ae3ae7879498a0ed4f448681 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 24 Apr 2017 19:49:17 -0300 Subject: Refactoring Github::RateLimit --- lib/github/client.rb | 8 +++++--- lib/github/rate_limit.rb | 33 +++++++++------------------------ 2 files changed, 14 insertions(+), 27 deletions(-) diff --git a/lib/github/client.rb b/lib/github/client.rb index 95536cae57f..bf1b2b0a523 100644 --- a/lib/github/client.rb +++ b/lib/github/client.rb @@ -1,17 +1,19 @@ module Github class Client - attr_reader :connection + attr_reader :connection, :rate_limit def initialize(options) @connection = Faraday.new(url: options.fetch(:url)) do |faraday| faraday.authorization 'token', options.fetch(:token) faraday.adapter :net_http end + + @rate_limit = RateLimit.new(connection) end def get(url, query = {}) - rate_limit = RateLimit.new(connection) - sleep rate_limit.reset_in if rate_limit.exceed? + exceed, reset_in = rate_limit.get + sleep reset_in if exceed Github::Response.new(connection.get(url, query)) end diff --git a/lib/github/rate_limit.rb b/lib/github/rate_limit.rb index ab2b9c707d8..884693d093c 100644 --- a/lib/github/rate_limit.rb +++ b/lib/github/rate_limit.rb @@ -10,33 +10,18 @@ module Github @connection = connection end - def exceed? - return false unless enabled? + def get + response = connection.get(RATE_LIMIT_URL) - remaining <= SAFE_REMAINING_REQUESTS - end - - def remaining - @remaining ||= body.dig('rate', 'remaining').to_i - end - - def reset_in - @reset ||= body.dig('rate', 'reset').to_i - end + # GitHub Rate Limit API returns 404 when the rate limit is disabled + return false unless response.status != 404 - private - - def response - connection.get(RATE_LIMIT_URL) - end - - def body - @body ||= Oj.load(response.body, class_cache: false, mode: :compat) - end + body = Oj.load(response.body, class_cache: false, mode: :compat) + remaining = body.dig('rate', 'remaining').to_i + reset_in = body.dig('rate', 'reset').to_i + exceed = remaining <= SAFE_REMAINING_REQUESTS - # GitHub Rate Limit API returns 404 when the rate limit is disabled - def enabled? - response.status != 404 + [exceed, reset_in] end end end -- cgit v1.2.1 From 1f498b73dabfd1bb1f2beb9cc6639db4e433ad28 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 24 Apr 2017 20:00:11 -0300 Subject: Use only one cache hash with with a hash initializer by default --- lib/github/import.rb | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/lib/github/import.rb b/lib/github/import.rb index cbda4be2576..8ffbb21bac0 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -29,17 +29,14 @@ module Github self.reset_callbacks :validate end - attr_reader :project, :repository, :repo, :options, :errors, - :cached_label_ids, :cached_gitlab_users, :cached_user_ids + attr_reader :project, :repository, :repo, :options, :errors, :cached def initialize(project, options) @project = project @repository = project.repository @repo = project.import_source @options = options - @cached_label_ids = {} - @cached_user_ids = {} - @cached_gitlab_users = {} + @cached = Hash.new { |hash, key| hash[key] = Hash.new } @errors = [] end @@ -107,7 +104,7 @@ module Github # Cache labels project.labels.select(:id, :title).find_each do |label| - @cached_label_ids[label.title] = label.id + cached[:label_ids][label.title] = label.id end end @@ -325,7 +322,7 @@ module Github end def label_ids(issuable) - issuable.map { |attrs| cached_label_ids[attrs.fetch('name')] }.compact + issuable.map { |attrs| cached[:label_ids][attrs.fetch('name')] }.compact end def milestone_id(milestone) @@ -336,12 +333,12 @@ module Github def user_id(user, fallback_id = nil) return unless user.present? - return cached_user_ids[user.id] if cached_user_ids.key?(user.id) + return cached[:user_ids][user.id] if cached[:user_ids].key?(user.id) gitlab_user_id = user_id_by_external_uid(user.id) || user_id_by_email(user.email) - cached_gitlab_users[user.id] = gitlab_user_id.present? - cached_user_ids[user.id] = gitlab_user_id || fallback_id + cached[:gitlab_user_ids][user.id] = gitlab_user_id.present? + cached[:user_ids][user.id] = gitlab_user_id || fallback_id end def user_id_by_email(email) @@ -362,7 +359,7 @@ module Github end def format_description(body, author) - return body if cached_gitlab_users[author.id] + return body if cached[:gitlab_user_ids][author.id] "*Created by: #{author.username}*\n\n#{body}" end -- cgit v1.2.1 From 5c72ba0ff1be82f31c292765e8d6a76277327979 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 24 Apr 2017 20:28:06 -0300 Subject: Finish the import process if some error occurs when fetching the repo --- lib/github/error.rb | 6 ++++++ lib/github/import.rb | 11 ++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 lib/github/error.rb diff --git a/lib/github/error.rb b/lib/github/error.rb new file mode 100644 index 00000000000..41cd4c67847 --- /dev/null +++ b/lib/github/error.rb @@ -0,0 +1,6 @@ +module Github + class Error < StandardError + end + + class RepositoryFetchError < Error; end +end diff --git a/lib/github/import.rb b/lib/github/import.rb index 8ffbb21bac0..56b0b7a3d4f 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -1,3 +1,4 @@ +require_relative 'error' module Github class Import include Gitlab::ShellAdapter @@ -48,9 +49,12 @@ module Github fetch_issues fetch_wiki_repository expire_repository_cache - track_errors - errors + true + rescue Github::RepositoryFetchError + false + ensure + keep_track_of_errors end private @@ -63,6 +67,7 @@ module Github project.repository.fetch_remote('github', forced: true) rescue Gitlab::Shell::Error => e error(:project, "https://github.com/#{repo}.git", e.message) + raise Github::RepositoryFetchError end end @@ -368,7 +373,7 @@ module Github repository.expire_content_cache end - def track_errors + def keep_track_of_errors return unless errors.any? project.update_column(:import_error, { -- cgit v1.2.1 From 22a33d8256101b0a5f348ae93623d4b189a96d47 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 24 Apr 2017 20:45:51 -0300 Subject: Avoid unnecessary use of Arel to find users by external uid --- lib/github/import.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/github/import.rb b/lib/github/import.rb index 56b0b7a3d4f..ba30ea0e2cd 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -355,11 +355,9 @@ module Github def user_id_by_external_uid(id) return nil unless id - identities = ::Identity.arel_table - ::User.select(:id) .joins(:identities) - .where(identities[:provider].eq(:github).and(identities[:extern_uid].eq(id))) + .merge(::Identity.where(provider: :github, extern_uid: id)) .first&.id end -- cgit v1.2.1 From 05255631ae2dad3b22796343453bb3288ca3dbff Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 24 Apr 2017 21:03:42 -0300 Subject: Cache labels at the same time we fetch them from the GH API --- lib/github/import.rb | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/lib/github/import.rb b/lib/github/import.rb index ba30ea0e2cd..641147c40e4 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -95,10 +95,13 @@ module Github response.body.each do |raw| begin - label = Github::Representation::Label.new(raw) - next if project.labels.where(title: label.title).exists? + representation = Github::Representation::Label.new(raw) - project.labels.create!(title: label.title, color: label.color) + label = project.labels.find_or_create_by!(title: representation.title) do |label| + label.color = representation.color + end + + cached[:label_ids][label.title] = label.id rescue => e error(:label, label.url, e.message) end @@ -106,11 +109,6 @@ module Github url = response.rels[:next] end - - # Cache labels - project.labels.select(:id, :title).find_each do |label| - cached[:label_ids][label.title] = label.id - end end def fetch_milestones -- cgit v1.2.1 From 5d106f2597bb009f0e7b8d5ed02b6f2206237e7e Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 24 Apr 2017 21:13:51 -0300 Subject: Use the base initiliazer for representations --- lib/github/import.rb | 2 +- lib/github/representation/branch.rb | 9 ++++----- lib/github/representation/pull_request.rb | 14 ++++++-------- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/lib/github/import.rb b/lib/github/import.rb index 641147c40e4..ef0ef9adc7a 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -147,7 +147,7 @@ module Github response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc) response.body.each do |raw| - pull_request = Github::Representation::PullRequest.new(project, raw, options) + pull_request = Github::Representation::PullRequest.new(raw, options.merge(project: project)) merge_request = MergeRequest.find_or_initialize_by(iid: pull_request.iid, source_project_id: project.id) next unless merge_request.new_record? && pull_request.valid? diff --git a/lib/github/representation/branch.rb b/lib/github/representation/branch.rb index 7c65a948ede..d1dac6944f0 100644 --- a/lib/github/representation/branch.rb +++ b/lib/github/representation/branch.rb @@ -3,11 +3,6 @@ module Github class Branch < Representation::Base attr_reader :repository - def initialize(repository, raw) - @repository = repository - @raw = raw - end - def user raw.dig('user', 'login') || 'unknown' end @@ -47,6 +42,10 @@ module Github def commit_exists? repository.branch_names_contains(sha).include?(ref) end + + def repository + @repository ||= options.fetch(:repository) + end end end end diff --git a/lib/github/representation/pull_request.rb b/lib/github/representation/pull_request.rb index 4119ca400c6..ac9c8283b4b 100644 --- a/lib/github/representation/pull_request.rb +++ b/lib/github/representation/pull_request.rb @@ -6,12 +6,6 @@ module Github delegate :user, :repo, :ref, :sha, to: :source_branch, prefix: true delegate :user, :exists?, :repo, :ref, :sha, :short_sha, to: :target_branch, prefix: true - def initialize(project, raw, options) - @project = project - @raw = raw - @options = options - end - def source_project project end @@ -54,8 +48,12 @@ module Github private + def project + @project ||= options.fetch(:project) + end + def source_branch - @source_branch ||= Representation::Branch.new(project.repository, raw['head']) + @source_branch ||= Representation::Branch.new(raw['head'], repository: project.repository) end def source_branch_name_prefixed @@ -63,7 +61,7 @@ module Github end def target_branch - @target_branch ||= Representation::Branch.new(project.repository, raw['base']) + @target_branch ||= Representation::Branch.new(raw['base'], repository: project.repository) end def target_branch_name_prefixed -- cgit v1.2.1 From 30794972f46de4d3af1c3f1843d46f1162660c3e Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 24 Apr 2017 21:21:58 -0300 Subject: Set timeout options to the Github::Client --- lib/github/client.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/github/client.rb b/lib/github/client.rb index bf1b2b0a523..e65d908d232 100644 --- a/lib/github/client.rb +++ b/lib/github/client.rb @@ -4,6 +4,8 @@ module Github def initialize(options) @connection = Faraday.new(url: options.fetch(:url)) do |faraday| + faraday.options.open_timeout = options.fetch(:timeout, 60) + faraday.options.timeout = options.fetch(:timeout, 60) faraday.authorization 'token', options.fetch(:token) faraday.adapter :net_http end -- cgit v1.2.1 From d2765b453f1103721fef24581f8a24e7a0397252 Mon Sep 17 00:00:00 2001 From: TM Lee Date: Tue, 25 Apr 2017 08:45:30 +0800 Subject: [#22826] Add newline before expect in accordance to four phase thoughtbot style --- spec/features/projects/files/find_files_spec.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spec/features/projects/files/find_files_spec.rb b/spec/features/projects/files/find_files_spec.rb index 06f52bef902..716b7591b95 100644 --- a/spec/features/projects/files/find_files_spec.rb +++ b/spec/features/projects/files/find_files_spec.rb @@ -14,6 +14,7 @@ feature 'Find files button in the tree header', feature: true do project.namespace, project ) + expect(page).to have_selector('.tree-controls .shortcuts-find-file') end @@ -23,6 +24,7 @@ feature 'Find files button in the tree header', feature: true do project, project.default_branch ) + expect(page).to have_selector('.tree-controls .shortcuts-find-file') end end -- cgit v1.2.1 From dd1157c80b90728a71b1f3a723e2cdeb335ab1b3 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 24 Apr 2017 23:41:36 -0300 Subject: Use Class.new(SuperClass) to define an empty custom error class --- lib/github/error.rb | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/github/error.rb b/lib/github/error.rb index 41cd4c67847..dfca3a18f9c 100644 --- a/lib/github/error.rb +++ b/lib/github/error.rb @@ -1,6 +1,4 @@ module Github - class Error < StandardError - end - - class RepositoryFetchError < Error; end + Error = Class.new(StandardError) + RepositoryFetchError = Class.new(Github::Error) end -- cgit v1.2.1 From aeb1684c524628f7279eadce5e461880e9865349 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Mon, 24 Apr 2017 23:41:46 -0300 Subject: Fix small typo --- lib/github/import.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/github/import.rb b/lib/github/import.rb index ef0ef9adc7a..fddd52af260 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -324,8 +324,8 @@ module Github remove_branch(pull_request.target_branch_name) unless pull_request.target_branch_exists? end - def label_ids(issuable) - issuable.map { |attrs| cached[:label_ids][attrs.fetch('name')] }.compact + def label_ids(labels) + labels.map { |attrs| cached[:label_ids][attrs.fetch('name')] }.compact end def milestone_id(milestone) -- cgit v1.2.1 From 43f73107ca2011ac8ed3af5a79c378c81820eeb6 Mon Sep 17 00:00:00 2001 From: Achilleas Pipinellis Date: Tue, 25 Apr 2017 10:01:59 +0200 Subject: Expose GitLab version in backup file Instead of doing hacks like http://stackoverflow.com/a/26082612/974710 --- lib/backup/manager.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 7b4476fa4db..0562961c66b 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -15,7 +15,7 @@ module Backup s[:gitlab_version] = Gitlab::VERSION s[:tar_version] = tar_version s[:skipped] = ENV["SKIP"] - tar_file = "#{s[:backup_created_at].strftime('%s_%Y_%m_%d')}#{FILE_NAME_SUFFIX}" + tar_file = "#{s[:backup_created_at].strftime('%s_%Y_%m_%d_')}#{s[:gitlab_version]}#{FILE_NAME_SUFFIX}" Dir.chdir(Gitlab.config.backup.path) do File.open("#{Gitlab.config.backup.path}/backup_information.yml", -- cgit v1.2.1 From d2889b67708fb602aa2c7a52d274665581ef9021 Mon Sep 17 00:00:00 2001 From: Achilleas Pipinellis Date: Tue, 25 Apr 2017 10:57:37 +0200 Subject: Refactor backup/restore docs --- changelogs/unreleased/gl-version-backup-file.yml | 4 + doc/raketasks/backup_restore.md | 254 +++++++++++++---------- 2 files changed, 144 insertions(+), 114 deletions(-) create mode 100644 changelogs/unreleased/gl-version-backup-file.yml diff --git a/changelogs/unreleased/gl-version-backup-file.yml b/changelogs/unreleased/gl-version-backup-file.yml new file mode 100644 index 00000000000..9b5abd58ae7 --- /dev/null +++ b/changelogs/unreleased/gl-version-backup-file.yml @@ -0,0 +1,4 @@ +--- +title: Refactor backup/restore docs +merge_request: +author: diff --git a/doc/raketasks/backup_restore.md b/doc/raketasks/backup_restore.md index e680a560888..5be6053b76e 100644 --- a/doc/raketasks/backup_restore.md +++ b/doc/raketasks/backup_restore.md @@ -1,41 +1,52 @@ -# Backup restore +# Backing up and restoring GitLab ![backup banner](backup_hrz.png) An application data backup creates an archive file that contains the database, all repositories and all attachments. -This archive will be saved in `backup_path`, which is specified in the -`config/gitlab.yml` file. -The filename will be `[TIMESTAMP]_gitlab_backup.tar`, where `TIMESTAMP` -identifies the time at which each backup was created. - -> In GitLab 8.15 we changed the timestamp format from `EPOCH` (`1393513186`) -> to `EPOCH_YYYY_MM_DD` (`1393513186_2014_02_27`) -You can only restore a backup to exactly the same version of GitLab on which it -was created. The best way to migrate your repositories from one server to +You can only restore a backup to **exactly the same version** of GitLab on which +it was created. The best way to migrate your repositories from one server to another is through backup restore. -To restore a backup, you will also need to restore `/etc/gitlab/gitlab-secrets.json` -(for omnibus packages) or `/home/git/gitlab/.secret` (for installations -from source). This file contains the database encryption key, -[CI secret variables](../ci/variables/README.md#secret-variables), and -secret variables used for [two-factor authentication](../security/two_factor_authentication.md). -If you fail to restore this encryption key file along with the application data -backup, users with two-factor authentication enabled and GitLab Runners will -lose access to your GitLab server. +## Backup + +GitLab provides a simple command line interface to backup your whole installation, +and is flexible enough to fit your needs. -## Create a backup of the GitLab system +### Backup timestamp + +>**Note:** +In GitLab 9.2 the timestamp format was changed from `EPOCH_YYYY_MM_DD` to +`EPOCH_YYYY_MM_DD_GitLab version`, for example `1493107454_2017_04_25` +would become `1493107454_2017_04_25_9.1.0`. + +The backup archive will be saved in `backup_path`, which is specified in the +`config/gitlab.yml` file. +The filename will be `[TIMESTAMP]_gitlab_backup.tar`, where `TIMESTAMP` +identifies the time at which each backup was created, plus the GitLab version. +The timestamp is needed if you need to restore GitLab and multiple backups are +available. + +For example, if the backup name is `1493107454_2017_04_25_9.1.0_gitlab_backup.tar`, +then the timestamp is `1493107454_2017_04_25_9.1.0`. + +### Creating a backup of the GitLab system Use this command if you've installed GitLab with the Omnibus package: + ``` sudo gitlab-rake gitlab:backup:create ``` + Use this if you've installed GitLab from source: + ``` sudo -u git -H bundle exec rake gitlab:backup:create RAILS_ENV=production ``` + If you are running GitLab within a Docker container, you can run the backup from the host: + ``` docker exec -t gitlab-rake gitlab:backup:create ``` @@ -69,9 +80,9 @@ Deleting tmp directories...[DONE] Deleting old backups... [SKIPPING] ``` -## Backup Strategy Option +### Backup strategy option -> **Note:** Introduced as an option in 8.17 +> **Note:** Introduced as an option in GitLab 8.17. The default backup strategy is to essentially stream data from the respective data locations to the backup using the Linux command `tar` and `gzip`. This works @@ -91,7 +102,7 @@ To use the `copy` strategy instead of the default streaming strategy, specify `STRATEGY=copy` in the Rake task command. For example, `sudo gitlab-rake gitlab:backup:create STRATEGY=copy`. -## Exclude specific directories from the backup +### Excluding specific directories from the backup You can choose what should be backed up by adding the environment variable `SKIP`. The available options are: @@ -115,7 +126,7 @@ sudo gitlab-rake gitlab:backup:create SKIP=db,uploads sudo -u git -H bundle exec rake gitlab:backup:create SKIP=db,uploads RAILS_ENV=production ``` -## Upload backups to remote (cloud) storage +### Uploading backups to a remote (cloud) storage Starting with GitLab 7.4 you can let the backup script upload the '.tar' file it creates. It uses the [Fog library](http://fog.io/) to perform the upload. @@ -259,7 +270,7 @@ For installations from source: remote_directory: 'gitlab_backups' ``` -## Backup archive permissions +### Backup archive permissions The backup archives created by GitLab (`1393513186_2014_02_27_gitlab_backup.tar`) will have owner/group git:git and 0600 permissions by default. @@ -277,11 +288,11 @@ gitlab_rails['backup_archive_permissions'] = 0644 # Makes the backup archives wo archive_permissions: 0644 # Makes the backup archives world-readable ``` -## Storing configuration files +### Storing configuration files Please be informed that a backup does not store your configuration -files. One reason for this is that your database contains encrypted -information for two-factor authentication. Storing encrypted +files. One reason for this is that your database contains encrypted +information for two-factor authentication. Storing encrypted information along with its key in the same place defeats the purpose of using encryption in the first place! @@ -294,11 +305,74 @@ At the very **minimum** you should backup `/etc/gitlab/gitlab.rb` and `/home/git/gitlab/config/secrets.yml` (source) to preserve your database encryption key. -## Restore a previously created backup +### Configuring cron to make daily backups + +>**Note:** +The following cron jobs do not [backup your GitLab configuration files](#storing-configuration-files) +or [SSH host keys](https://superuser.com/questions/532040/copy-ssh-keys-from-one-server-to-another-server/532079#532079). + +**For Omnibus installations** + +To schedule a cron job that backs up your repositories and GitLab metadata, use the root user: + +``` +sudo su - +crontab -e +``` + +There, add the following line to schedule the backup for everyday at 2 AM: + +``` +0 2 * * * /opt/gitlab/bin/gitlab-rake gitlab:backup:create CRON=1 +``` + +You may also want to set a limited lifetime for backups to prevent regular +backups using all your disk space. To do this add the following lines to +`/etc/gitlab/gitlab.rb` and reconfigure: -You can only restore a backup to exactly the same version of GitLab that you created it on, for example 7.2.1. +``` +# limit backup lifetime to 7 days - 604800 seconds +gitlab_rails['backup_keep_time'] = 604800 +``` -### Prerequisites +Note that the `backup_keep_time` configuration option only manages local +files. GitLab does not automatically prune old files stored in a third-party +object storage (e.g., AWS S3) because the user may not have permission to list +and delete files. We recommend that you configure the appropriate retention +policy for your object storage. For example, you can configure [the S3 backup +policy as described here](http://stackoverflow.com/questions/37553070/gitlab-omnibus-delete-backup-from-amazon-s3). + +**For installation from source** + +``` +cd /home/git/gitlab +sudo -u git -H editor config/gitlab.yml # Enable keep_time in the backup section to automatically delete old backups +sudo -u git crontab -e # Edit the crontab for the git user +``` + +Add the following lines at the bottom: + +``` +# Create a full backup of the GitLab repositories and SQL database every day at 4am +0 4 * * * cd /home/git/gitlab && PATH=/usr/local/bin:/usr/bin:/bin bundle exec rake gitlab:backup:create RAILS_ENV=production CRON=1 +``` + +The `CRON=1` environment setting tells the backup script to suppress all progress output if there are no errors. +This is recommended to reduce cron spam. + +## Restore + +GitLab provides a simple command line interface to backup your whole installation, +and is flexible enough to fit your needs. + +The [restore prerequisites section](#restore-prerequisites) includes crucial +information. Make sure to read and test the whole restore process at least once +before attempting to perform it in a production environment. + +You can only restore a backup to **exactly the same version** of GitLab that +you created it on, for example 9.1.0. + +### Restore prerequisites You need to have a working GitLab installation before you can perform a restore. This is mainly because the system user performing the @@ -307,13 +381,23 @@ the SQL database it needs to import data into ('gitlabhq_production'). All existing data will be either erased (SQL) or moved to a separate directory (repositories, uploads). -If some or all of your GitLab users are using two-factor authentication (2FA) -then you must also make sure to restore `/etc/gitlab/gitlab.rb` and -`/etc/gitlab/gitlab-secrets.json` (Omnibus), or -`/home/git/gitlab/config/secrets.yml` (installations from source). Note that you -need to run `gitlab-ctl reconfigure` after changing `gitlab-secrets.json`. +To restore a backup, you will also need to restore `/etc/gitlab/gitlab-secrets.json` +(for Omnibus packages) or `/home/git/gitlab/.secret` (for installations +from source). This file contains the database encryption key, +[CI secret variables](../ci/variables/README.md#secret-variables), and +secret variables used for [two-factor authentication](../user/profile/account/two_factor_authentication.md). +If you fail to restore this encryption key file along with the application data +backup, users with two-factor authentication enabled and GitLab Runners will +lose access to your GitLab server. + +Depending on your case, you might want to run the restore command with one or +more of the following options: + +- `BACKUP=timestamp_of_backup` - Required if more than one backup exists. + Read what the [backup timestamp is about](#backup-timestamp). +- `force=yes` - Do not ask if the authorized_keys file should get regenerated. -### Installation from source +### Restore for installation from source ``` # Stop processes that are connected to the database @@ -322,13 +406,6 @@ sudo service gitlab stop bundle exec rake gitlab:backup:restore RAILS_ENV=production ``` -Options: - -``` -BACKUP=timestamp_of_backup (required if more than one backup exists) -force=yes (do not ask if the authorized_keys file should get regenerated) -``` - Example output: ``` @@ -360,13 +437,13 @@ Restoring repositories: Deleting tmp directories...[DONE] ``` -### Omnibus installations +### Restore for Omnibus installations This procedure assumes that: -- You have installed the exact same version of GitLab Omnibus with which the - backup was created -- You have run `sudo gitlab-ctl reconfigure` at least once +- You have installed the **exact same version** of GitLab Omnibus with which the + backup was created. +- You have run `sudo gitlab-ctl reconfigure` at least once. - GitLab is running. If not, start it using `sudo gitlab-ctl start`. First make sure your backup tar file is in the backup directory described in the @@ -374,7 +451,7 @@ First make sure your backup tar file is in the backup directory described in the `/var/opt/gitlab/backups`. ```shell -sudo cp 1393513186_2014_02_27_gitlab_backup.tar /var/opt/gitlab/backups/ +sudo cp 1493107454_2017_04_25_9.1.0_gitlab_backup.tar /var/opt/gitlab/backups/ ``` Stop the processes that are connected to the database. Leave the rest of GitLab @@ -392,7 +469,7 @@ restore: ```shell # This command will overwrite the contents of your GitLab database! -sudo gitlab-rake gitlab:backup:restore BACKUP=1393513186_2014_02_27 +sudo gitlab-rake gitlab:backup:restore BACKUP=1493107454_2017_04_25_9.1.0 ``` Restart and check GitLab: @@ -404,59 +481,7 @@ sudo gitlab-rake gitlab:check SANITIZE=true If there is a GitLab version mismatch between your backup tar file and the installed version of GitLab, the restore command will abort with an error. Install the -[correct GitLab version](https://about.gitlab.com/downloads/archives/) and try again. - -## Configure cron to make daily backups - -### For installation from source: -``` -cd /home/git/gitlab -sudo -u git -H editor config/gitlab.yml # Enable keep_time in the backup section to automatically delete old backups -sudo -u git crontab -e # Edit the crontab for the git user -``` - -Add the following lines at the bottom: - -``` -# Create a full backup of the GitLab repositories and SQL database every day at 4am -0 4 * * * cd /home/git/gitlab && PATH=/usr/local/bin:/usr/bin:/bin bundle exec rake gitlab:backup:create RAILS_ENV=production CRON=1 -``` - -The `CRON=1` environment setting tells the backup script to suppress all progress output if there are no errors. -This is recommended to reduce cron spam. - -### For omnibus installations - -To schedule a cron job that backs up your repositories and GitLab metadata, use the root user: - -``` -sudo su - -crontab -e -``` - -There, add the following line to schedule the backup for everyday at 2 AM: - -``` -0 2 * * * /opt/gitlab/bin/gitlab-rake gitlab:backup:create CRON=1 -``` - -You may also want to set a limited lifetime for backups to prevent regular -backups using all your disk space. To do this add the following lines to -`/etc/gitlab/gitlab.rb` and reconfigure: - -``` -# limit backup lifetime to 7 days - 604800 seconds -gitlab_rails['backup_keep_time'] = 604800 -``` - -Note that the `backup_keep_time` configuration option only manages local -files. GitLab does not automatically prune old files stored in a third-party -object storage (e.g. AWS S3) because the user may not have permission to list -and delete files. We recommend that you configure the appropriate retention -policy for your object storage. For example, you can configure [the S3 backup -policy here as described here](http://stackoverflow.com/questions/37553070/gitlab-omnibus-delete-backup-from-amazon-s3). - -NOTE: This cron job does not [backup your omnibus-gitlab configuration](#backup-and-restore-omnibus-gitlab-configuration) or [SSH host keys](https://superuser.com/questions/532040/copy-ssh-keys-from-one-server-to-another-server/532079#532079). +[correct GitLab version](https://packages.gitlab.com/gitlab/) and try again. ## Alternative backup strategies @@ -481,6 +506,19 @@ Example: LVM snapshots + rsync If you are running GitLab on a virtualized server you can possibly also create VM snapshots of the entire GitLab server. It is not uncommon however for a VM snapshot to require you to power down the server, so this approach is probably of limited practical use. +## Additional notes + +This documentation is for GitLab Community and Enterprise Edition. We backup +GitLab.com and make sure your data is secure, but you can't use these methods +to export / backup your data yourself from GitLab.com. + +Issues are stored in the database. They can't be stored in Git itself. + +To migrate your repositories from one server to another with an up-to-date version of +GitLab, you can use the [import rake task](import.md) to do a mass import of the +repository. Note that if you do an import rake task, rather than a backup restore, you +will have all your repositories, but not any other data. + ## Troubleshooting ### Restoring database backup using omnibus packages outputs warnings @@ -490,7 +528,6 @@ If you are using backup restore procedures you might encounter the following war psql:/var/opt/gitlab/backups/db/database.sql:22: ERROR: must be owner of extension plpgsql psql:/var/opt/gitlab/backups/db/database.sql:2931: WARNING: no privileges could be revoked for "public" (two occurrences) psql:/var/opt/gitlab/backups/db/database.sql:2933: WARNING: no privileges were granted for "public" (two occurrences) - ``` Be advised that, backup is successfully restored in spite of these warnings. @@ -499,14 +536,3 @@ The rake task runs this as the `gitlab` user which does not have the superuser a Those objects have no influence on the database backup/restore but they give this annoying warning. For more information see similar questions on postgresql issue tracker[here](http://www.postgresql.org/message-id/201110220712.30886.adrian.klaver@gmail.com) and [here](http://www.postgresql.org/message-id/2039.1177339749@sss.pgh.pa.us) as well as [stack overflow](http://stackoverflow.com/questions/4368789/error-must-be-owner-of-language-plpgsql). - -## Note -This documentation is for GitLab CE. -We backup GitLab.com and make sure your data is secure, but you can't use these methods to export / backup your data yourself from GitLab.com. - -Issues are stored in the database. They can't be stored in Git itself. - -To migrate your repositories from one server to another with an up-to-date version of -GitLab, you can use the [import rake task](import.md) to do a mass import of the -repository. Note that if you do an import rake task, rather than a backup restore, you -will have all your repositories, but not any other data. -- cgit v1.2.1 From 31c990cc41e2dc7576cf867a313862a07fd8c170 Mon Sep 17 00:00:00 2001 From: blackst0ne Date: Tue, 25 Apr 2017 21:06:24 +1100 Subject: Fix rendering emoji inside a string --- app/assets/javascripts/gfm_auto_complete.js | 11 ++++++++++- app/assets/javascripts/lib/utils/regexp.js | 10 ++++++++++ changelogs/unreleased/fix_emoji_parser.yml | 4 ++++ lib/banzai/filter/emoji_filter.rb | 5 ++++- spec/features/issues/gfm_autocomplete_spec.rb | 27 +++++++++++++++++++++++++++ spec/fixtures/markdown.md.erb | 2 +- spec/helpers/gitlab_markdown_helper_spec.rb | 4 ++-- spec/lib/banzai/filter/emoji_filter_spec.rb | 10 ++++++++-- 8 files changed, 66 insertions(+), 7 deletions(-) create mode 100644 app/assets/javascripts/lib/utils/regexp.js create mode 100644 changelogs/unreleased/fix_emoji_parser.yml diff --git a/app/assets/javascripts/gfm_auto_complete.js b/app/assets/javascripts/gfm_auto_complete.js index 9ac4c49d697..d25be803c0a 100644 --- a/app/assets/javascripts/gfm_auto_complete.js +++ b/app/assets/javascripts/gfm_auto_complete.js @@ -3,6 +3,7 @@ import emojiMap from 'emojis/digests.json'; import emojiAliases from 'emojis/aliases.json'; import { glEmojiTag } from '~/behaviors/gl_emoji'; +import glRegexp from '~/lib/utils/regexp'; // Creates the variables for setting up GFM auto-completion window.gl = window.gl || {}; @@ -127,7 +128,15 @@ window.gl.GfmAutoComplete = { callbacks: { sorter: this.DefaultOptions.sorter, beforeInsert: this.DefaultOptions.beforeInsert, - filter: this.DefaultOptions.filter + filter: this.DefaultOptions.filter, + + matcher: (flag, subtext) => { + const relevantText = subtext.trim().split(/\s/).pop(); + const regexp = new RegExp(`(?:[^${glRegexp.unicodeLetters}0-9:]|\n|^):([^:]*)$`, 'gi'); + const match = regexp.exec(relevantText); + + return match && match.length ? match[1] : null; + } } }); // Team Members diff --git a/app/assets/javascripts/lib/utils/regexp.js b/app/assets/javascripts/lib/utils/regexp.js new file mode 100644 index 00000000000..baa0b51d59b --- /dev/null +++ b/app/assets/javascripts/lib/utils/regexp.js @@ -0,0 +1,10 @@ +/** + * Regexp utility for the convenience of working with regular expressions. + * + */ + +// Inspired by https://github.com/mishoo/UglifyJS/blob/2bc1d02363db3798d5df41fb5059a19edca9b7eb/lib/parse-js.js#L203 +// Unicode 6.1 +const unicodeLetters = '\\u0041-\\u005A\\u0061-\\u007A\\u00AA\\u00B5\\u00BA\\u00C0-\\u00D6\\u00D8-\\u00F6\\u00F8-\\u02C1\\u02C6-\\u02D1\\u02E0-\\u02E4\\u02EC\\u02EE\\u0370-\\u0374\\u0376\\u0377\\u037A-\\u037D\\u0386\\u0388-\\u038A\\u038C\\u038E-\\u03A1\\u03A3-\\u03F5\\u03F7-\\u0481\\u048A-\\u0527\\u0531-\\u0556\\u0559\\u0561-\\u0587\\u05D0-\\u05EA\\u05F0-\\u05F2\\u0620-\\u064A\\u066E\\u066F\\u0671-\\u06D3\\u06D5\\u06E5\\u06E6\\u06EE\\u06EF\\u06FA-\\u06FC\\u06FF\\u0710\\u0712-\\u072F\\u074D-\\u07A5\\u07B1\\u07CA-\\u07EA\\u07F4\\u07F5\\u07FA\\u0800-\\u0815\\u081A\\u0824\\u0828\\u0840-\\u0858\\u08A0\\u08A2-\\u08AC\\u0904-\\u0939\\u093D\\u0950\\u0958-\\u0961\\u0971-\\u0977\\u0979-\\u097F\\u0985-\\u098C\\u098F\\u0990\\u0993-\\u09A8\\u09AA-\\u09B0\\u09B2\\u09B6-\\u09B9\\u09BD\\u09CE\\u09DC\\u09DD\\u09DF-\\u09E1\\u09F0\\u09F1\\u0A05-\\u0A0A\\u0A0F\\u0A10\\u0A13-\\u0A28\\u0A2A-\\u0A30\\u0A32\\u0A33\\u0A35\\u0A36\\u0A38\\u0A39\\u0A59-\\u0A5C\\u0A5E\\u0A72-\\u0A74\\u0A85-\\u0A8D\\u0A8F-\\u0A91\\u0A93-\\u0AA8\\u0AAA-\\u0AB0\\u0AB2\\u0AB3\\u0AB5-\\u0AB9\\u0ABD\\u0AD0\\u0AE0\\u0AE1\\u0B05-\\u0B0C\\u0B0F\\u0B10\\u0B13-\\u0B28\\u0B2A-\\u0B30\\u0B32\\u0B33\\u0B35-\\u0B39\\u0B3D\\u0B5C\\u0B5D\\u0B5F-\\u0B61\\u0B71\\u0B83\\u0B85-\\u0B8A\\u0B8E-\\u0B90\\u0B92-\\u0B95\\u0B99\\u0B9A\\u0B9C\\u0B9E\\u0B9F\\u0BA3\\u0BA4\\u0BA8-\\u0BAA\\u0BAE-\\u0BB9\\u0BD0\\u0C05-\\u0C0C\\u0C0E-\\u0C10\\u0C12-\\u0C28\\u0C2A-\\u0C33\\u0C35-\\u0C39\\u0C3D\\u0C58\\u0C59\\u0C60\\u0C61\\u0C85-\\u0C8C\\u0C8E-\\u0C90\\u0C92-\\u0CA8\\u0CAA-\\u0CB3\\u0CB5-\\u0CB9\\u0CBD\\u0CDE\\u0CE0\\u0CE1\\u0CF1\\u0CF2\\u0D05-\\u0D0C\\u0D0E-\\u0D10\\u0D12-\\u0D3A\\u0D3D\\u0D4E\\u0D60\\u0D61\\u0D7A-\\u0D7F\\u0D85-\\u0D96\\u0D9A-\\u0DB1\\u0DB3-\\u0DBB\\u0DBD\\u0DC0-\\u0DC6\\u0E01-\\u0E30\\u0E32\\u0E33\\u0E40-\\u0E46\\u0E81\\u0E82\\u0E84\\u0E87\\u0E88\\u0E8A\\u0E8D\\u0E94-\\u0E97\\u0E99-\\u0E9F\\u0EA1-\\u0EA3\\u0EA5\\u0EA7\\u0EAA\\u0EAB\\u0EAD-\\u0EB0\\u0EB2\\u0EB3\\u0EBD\\u0EC0-\\u0EC4\\u0EC6\\u0EDC-\\u0EDF\\u0F00\\u0F40-\\u0F47\\u0F49-\\u0F6C\\u0F88-\\u0F8C\\u1000-\\u102A\\u103F\\u1050-\\u1055\\u105A-\\u105D\\u1061\\u1065\\u1066\\u106E-\\u1070\\u1075-\\u1081\\u108E\\u10A0-\\u10C5\\u10C7\\u10CD\\u10D0-\\u10FA\\u10FC-\\u1248\\u124A-\\u124D\\u1250-\\u1256\\u1258\\u125A-\\u125D\\u1260-\\u1288\\u128A-\\u128D\\u1290-\\u12B0\\u12B2-\\u12B5\\u12B8-\\u12BE\\u12C0\\u12C2-\\u12C5\\u12C8-\\u12D6\\u12D8-\\u1310\\u1312-\\u1315\\u1318-\\u135A\\u1380-\\u138F\\u13A0-\\u13F4\\u1401-\\u166C\\u166F-\\u167F\\u1681-\\u169A\\u16A0-\\u16EA\\u16EE-\\u16F0\\u1700-\\u170C\\u170E-\\u1711\\u1720-\\u1731\\u1740-\\u1751\\u1760-\\u176C\\u176E-\\u1770\\u1780-\\u17B3\\u17D7\\u17DC\\u1820-\\u1877\\u1880-\\u18A8\\u18AA\\u18B0-\\u18F5\\u1900-\\u191C\\u1950-\\u196D\\u1970-\\u1974\\u1980-\\u19AB\\u19C1-\\u19C7\\u1A00-\\u1A16\\u1A20-\\u1A54\\u1AA7\\u1B05-\\u1B33\\u1B45-\\u1B4B\\u1B83-\\u1BA0\\u1BAE\\u1BAF\\u1BBA-\\u1BE5\\u1C00-\\u1C23\\u1C4D-\\u1C4F\\u1C5A-\\u1C7D\\u1CE9-\\u1CEC\\u1CEE-\\u1CF1\\u1CF5\\u1CF6\\u1D00-\\u1DBF\\u1E00-\\u1F15\\u1F18-\\u1F1D\\u1F20-\\u1F45\\u1F48-\\u1F4D\\u1F50-\\u1F57\\u1F59\\u1F5B\\u1F5D\\u1F5F-\\u1F7D\\u1F80-\\u1FB4\\u1FB6-\\u1FBC\\u1FBE\\u1FC2-\\u1FC4\\u1FC6-\\u1FCC\\u1FD0-\\u1FD3\\u1FD6-\\u1FDB\\u1FE0-\\u1FEC\\u1FF2-\\u1FF4\\u1FF6-\\u1FFC\\u2071\\u207F\\u2090-\\u209C\\u2102\\u2107\\u210A-\\u2113\\u2115\\u2119-\\u211D\\u2124\\u2126\\u2128\\u212A-\\u212D\\u212F-\\u2139\\u213C-\\u213F\\u2145-\\u2149\\u214E\\u2160-\\u2188\\u2C00-\\u2C2E\\u2C30-\\u2C5E\\u2C60-\\u2CE4\\u2CEB-\\u2CEE\\u2CF2\\u2CF3\\u2D00-\\u2D25\\u2D27\\u2D2D\\u2D30-\\u2D67\\u2D6F\\u2D80-\\u2D96\\u2DA0-\\u2DA6\\u2DA8-\\u2DAE\\u2DB0-\\u2DB6\\u2DB8-\\u2DBE\\u2DC0-\\u2DC6\\u2DC8-\\u2DCE\\u2DD0-\\u2DD6\\u2DD8-\\u2DDE\\u2E2F\\u3005-\\u3007\\u3021-\\u3029\\u3031-\\u3035\\u3038-\\u303C\\u3041-\\u3096\\u309D-\\u309F\\u30A1-\\u30FA\\u30FC-\\u30FF\\u3105-\\u312D\\u3131-\\u318E\\u31A0-\\u31BA\\u31F0-\\u31FF\\u3400-\\u4DB5\\u4E00-\\u9FCC\\uA000-\\uA48C\\uA4D0-\\uA4FD\\uA500-\\uA60C\\uA610-\\uA61F\\uA62A\\uA62B\\uA640-\\uA66E\\uA67F-\\uA697\\uA6A0-\\uA6EF\\uA717-\\uA71F\\uA722-\\uA788\\uA78B-\\uA78E\\uA790-\\uA793\\uA7A0-\\uA7AA\\uA7F8-\\uA801\\uA803-\\uA805\\uA807-\\uA80A\\uA80C-\\uA822\\uA840-\\uA873\\uA882-\\uA8B3\\uA8F2-\\uA8F7\\uA8FB\\uA90A-\\uA925\\uA930-\\uA946\\uA960-\\uA97C\\uA984-\\uA9B2\\uA9CF\\uAA00-\\uAA28\\uAA40-\\uAA42\\uAA44-\\uAA4B\\uAA60-\\uAA76\\uAA7A\\uAA80-\\uAAAF\\uAAB1\\uAAB5\\uAAB6\\uAAB9-\\uAABD\\uAAC0\\uAAC2\\uAADB-\\uAADD\\uAAE0-\\uAAEA\\uAAF2-\\uAAF4\\uAB01-\\uAB06\\uAB09-\\uAB0E\\uAB11-\\uAB16\\uAB20-\\uAB26\\uAB28-\\uAB2E\\uABC0-\\uABE2\\uAC00-\\uD7A3\\uD7B0-\\uD7C6\\uD7CB-\\uD7FB\\uF900-\\uFA6D\\uFA70-\\uFAD9\\uFB00-\\uFB06\\uFB13-\\uFB17\\uFB1D\\uFB1F-\\uFB28\\uFB2A-\\uFB36\\uFB38-\\uFB3C\\uFB3E\\uFB40\\uFB41\\uFB43\\uFB44\\uFB46-\\uFBB1\\uFBD3-\\uFD3D\\uFD50-\\uFD8F\\uFD92-\\uFDC7\\uFDF0-\\uFDFB\\uFE70-\\uFE74\\uFE76-\\uFEFC\\uFF21-\\uFF3A\\uFF41-\\uFF5A\\uFF66-\\uFFBE\\uFFC2-\\uFFC7\\uFFCA-\\uFFCF\\uFFD2-\\uFFD7\\uFFDA-\\uFFDC'; + +export default { unicodeLetters }; diff --git a/changelogs/unreleased/fix_emoji_parser.yml b/changelogs/unreleased/fix_emoji_parser.yml new file mode 100644 index 00000000000..2b1fffe2457 --- /dev/null +++ b/changelogs/unreleased/fix_emoji_parser.yml @@ -0,0 +1,4 @@ +--- +title: Fix rendering emoji inside a string +merge_request: 10647 +author: blackst0ne diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index d6138816e70..6255a611dbe 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -53,7 +53,10 @@ module Banzai # Build a regexp that matches all valid :emoji: names. def self.emoji_pattern - @emoji_pattern ||= /:(#{Gitlab::Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}):/ + @emoji_pattern ||= + /(?<=[^[:alnum:]:]|\n|^) + :(#{Gitlab::Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}): + (?=[^[:alnum:]:]|$)/x end # Build a regexp that matches all valid unicode emojis names. diff --git a/spec/features/issues/gfm_autocomplete_spec.rb b/spec/features/issues/gfm_autocomplete_spec.rb index 7135565294b..f4f24e1d12d 100644 --- a/spec/features/issues/gfm_autocomplete_spec.rb +++ b/spec/features/issues/gfm_autocomplete_spec.rb @@ -46,6 +46,33 @@ feature 'GFM autocomplete', feature: true, js: true do expect(find('#at-view-58')).not_to have_selector('.cur:first-of-type') end + it 'does not open autocomplete menu when ":" is prefixed by a number and letters' do + note = find('#note_note') + + # Number. + page.within '.timeline-content-form' do + note.native.send_keys('7:') + end + + expect(page).not_to have_selector('.atwho-view') + + # ASCII letter. + page.within '.timeline-content-form' do + note.set('') + note.native.send_keys('w:') + end + + expect(page).not_to have_selector('.atwho-view') + + # Non-ASCII letter. + page.within '.timeline-content-form' do + note.set('') + note.native.send_keys('Ё:') + end + + expect(page).not_to have_selector('.atwho-view') + end + it 'selects the first item for assignee dropdowns' do page.within '.timeline-content-form' do find('#note_note').native.send_keys('') diff --git a/spec/fixtures/markdown.md.erb b/spec/fixtures/markdown.md.erb index 0cdbc32431d..51a3e91d201 100644 --- a/spec/fixtures/markdown.md.erb +++ b/spec/fixtures/markdown.md.erb @@ -116,7 +116,7 @@ Linking to a file relative to this project's repository should work. Because life would be :zzz: without Emoji, right? :rocket: -Get ready for the Emoji :bomb:: :+1::-1::ok_hand::wave::v::raised_hand::muscle: +Get ready for the Emoji :bomb: : :+1: :-1: :ok_hand: :wave: :v: :raised_hand: :muscle: ### TableOfContentsFilter diff --git a/spec/helpers/gitlab_markdown_helper_spec.rb b/spec/helpers/gitlab_markdown_helper_spec.rb index 6cf3f86680a..b8fd54c66e4 100644 --- a/spec/helpers/gitlab_markdown_helper_spec.rb +++ b/spec/helpers/gitlab_markdown_helper_spec.rb @@ -111,9 +111,9 @@ describe GitlabMarkdownHelper do end it 'replaces commit message with emoji to link' do - actual = link_to_gfm(':book:Book', '/foo') + actual = link_to_gfm(':book: Book', '/foo') expect(actual). - to eq '📖Book' + to eq '📖 Book' end end diff --git a/spec/lib/banzai/filter/emoji_filter_spec.rb b/spec/lib/banzai/filter/emoji_filter_spec.rb index 707212e07fd..086a006c45f 100644 --- a/spec/lib/banzai/filter/emoji_filter_spec.rb +++ b/spec/lib/banzai/filter/emoji_filter_spec.rb @@ -68,9 +68,9 @@ describe Banzai::Filter::EmojiFilter, lib: true do expect(doc.css('gl-emoji').size).to eq 1 end - it 'matches multiple emoji in a row' do + it 'does not match multiple emoji in a row' do doc = filter(':see_no_evil::hear_no_evil::speak_no_evil:') - expect(doc.css('gl-emoji').size).to eq 3 + expect(doc.css('gl-emoji').size).to eq 0 end it 'unicode matches multiple emoji in a row' do @@ -83,6 +83,12 @@ describe Banzai::Filter::EmojiFilter, lib: true do expect(doc.css('gl-emoji').size).to eq 6 end + it 'does not match emoji in a string' do + doc = filter("'2a00:a4c0:100::1'") + + expect(doc.css('gl-emoji').size).to eq 0 + end + it 'has a data-name attribute' do doc = filter(':-1:') expect(doc.css('gl-emoji').first.attr('data-name')).to eq 'thumbsdown' -- cgit v1.2.1 From 82adef2c5a6edbfa43407d69b434c907436498d0 Mon Sep 17 00:00:00 2001 From: dixpac Date: Sat, 15 Apr 2017 13:43:00 +0200 Subject: Increase line-height in build-header so elements don't overlap --- app/assets/stylesheets/pages/builds.scss | 1 + changelogs/unreleased/fix_build_header_line_height.yml | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 changelogs/unreleased/fix_build_header_line_height.yml diff --git a/app/assets/stylesheets/pages/builds.scss b/app/assets/stylesheets/pages/builds.scss index 411f1c4442b..724b4080ee0 100644 --- a/app/assets/stylesheets/pages/builds.scss +++ b/app/assets/stylesheets/pages/builds.scss @@ -200,6 +200,7 @@ .header-content { flex: 1; + line-height: 1.8; a { color: $gl-text-color; diff --git a/changelogs/unreleased/fix_build_header_line_height.yml b/changelogs/unreleased/fix_build_header_line_height.yml new file mode 100644 index 00000000000..95b6221f8d2 --- /dev/null +++ b/changelogs/unreleased/fix_build_header_line_height.yml @@ -0,0 +1,4 @@ +--- +title: Change line-height on build-header so elements don't overlap +merge_request: +author: Dino Maric -- cgit v1.2.1 From ede9afbf3de9301bd7defaf4e5db47a534e87d0b Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Tue, 25 Apr 2017 19:23:13 +0800 Subject: Remove description for build_events, add pipeline_events --- app/helpers/services_helper.rb | 4 ++-- changelogs/unreleased/30645-show-pipeline-events-description.yml | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 changelogs/unreleased/30645-show-pipeline-events-description.yml diff --git a/app/helpers/services_helper.rb b/app/helpers/services_helper.rb index 715e5893a2c..3707bb5ba36 100644 --- a/app/helpers/services_helper.rb +++ b/app/helpers/services_helper.rb @@ -13,8 +13,8 @@ module ServicesHelper "Event will be triggered when a confidential issue is created/updated/closed" when "merge_request", "merge_request_events" "Event will be triggered when a merge request is created/updated/merged" - when "build", "build_events" - "Event will be triggered when a build status changes" + when "pipeline", "pipeline_events" + "Event will be triggered when a pipeline status changes" when "wiki_page", "wiki_page_events" "Event will be triggered when a wiki page is created/updated" when "commit", "commit_events" diff --git a/changelogs/unreleased/30645-show-pipeline-events-description.yml b/changelogs/unreleased/30645-show-pipeline-events-description.yml new file mode 100644 index 00000000000..fb75dde1d86 --- /dev/null +++ b/changelogs/unreleased/30645-show-pipeline-events-description.yml @@ -0,0 +1,4 @@ +--- +title: Fix pipeline events description for Slack and Mattermost integration +merge_request: 10908 +author: -- cgit v1.2.1 From cb1207af36d7c67d6db8e2155ae65b87b009bb36 Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Tue, 25 Apr 2017 14:46:06 +0100 Subject: Swap a before_save call with a before_create/before_update pair to avoid a confict with elasticsearch-model --- app/models/concerns/cache_markdown_field.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/models/concerns/cache_markdown_field.rb b/app/models/concerns/cache_markdown_field.rb index 2eedc143968..f033028c4e5 100644 --- a/app/models/concerns/cache_markdown_field.rb +++ b/app/models/concerns/cache_markdown_field.rb @@ -120,7 +120,9 @@ module CacheMarkdownField attrs end - before_save :refresh_markdown_cache!, if: :invalidated_markdown_cache? + # Using before_update here conflicts with elasticsearch-model somehow + before_create :refresh_markdown_cache!, if: :invalidated_markdown_cache? + before_update :refresh_markdown_cache!, if: :invalidated_markdown_cache? end class_methods do -- cgit v1.2.1 From 0e24b1280eca55bbe52931e530f3ce6ee38addc1 Mon Sep 17 00:00:00 2001 From: Nick Thomas Date: Tue, 25 Apr 2017 17:13:44 +0100 Subject: Fix a failing spec --- spec/models/concerns/cache_markdown_field_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/models/concerns/cache_markdown_field_spec.rb b/spec/models/concerns/cache_markdown_field_spec.rb index de0069bdcac..4edafbc4e32 100644 --- a/spec/models/concerns/cache_markdown_field_spec.rb +++ b/spec/models/concerns/cache_markdown_field_spec.rb @@ -18,7 +18,7 @@ describe CacheMarkdownField do end extend ActiveModel::Callbacks - define_model_callbacks :save + define_model_callbacks :create, :update include CacheMarkdownField cache_markdown_field :foo @@ -56,7 +56,7 @@ describe CacheMarkdownField do end def save - run_callbacks :save do + run_callbacks :update do changes_applied end end -- cgit v1.2.1 From 7df974c433a7d675a2dcb2dab1063e61686476ec Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 25 Apr 2017 13:30:08 -0300 Subject: Add blank line before the raise method on Github::Collection --- lib/github/collection.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/github/collection.rb b/lib/github/collection.rb index 12d6703476b..014b2038c4b 100644 --- a/lib/github/collection.rb +++ b/lib/github/collection.rb @@ -13,6 +13,7 @@ module Github loop do response = client.get(url, query) response.body.each { |item| yielder << item } + raise StopIteration unless response.rels.key?(:next) url = response.rels[:next] end -- cgit v1.2.1 From 2f934ce22f18d2fa12a849014db541e133e1b3d1 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 25 Apr 2017 13:30:30 -0300 Subject: Remove the Github::Error base class --- lib/github/error.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/github/error.rb b/lib/github/error.rb index dfca3a18f9c..66d7afaa787 100644 --- a/lib/github/error.rb +++ b/lib/github/error.rb @@ -1,4 +1,3 @@ module Github - Error = Class.new(StandardError) - RepositoryFetchError = Class.new(Github::Error) + RepositoryFetchError = Class.new(StandardError) end -- cgit v1.2.1 From 000a723d844c0bccfb04cbb32c683cd4d7e7b07e Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 25 Apr 2017 15:57:50 -0300 Subject: Fix small typo on GitHub::Import --- lib/github/import.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/github/import.rb b/lib/github/import.rb index fddd52af260..fefb713a29e 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -103,7 +103,7 @@ module Github cached[:label_ids][label.title] = label.id rescue => e - error(:label, label.url, e.message) + error(:label, representation.url, e.message) end end -- cgit v1.2.1 From 39ab842bc2ff1395ed1bf6768ff2a409a33bf3d3 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 25 Apr 2017 15:58:17 -0300 Subject: Create project repository only when it not exists --- lib/github/import.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/github/import.rb b/lib/github/import.rb index fefb713a29e..a0afd12f45f 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -61,7 +61,7 @@ module Github def fetch_repository begin - project.create_repository + project.create_repository unless project.repository.exists? project.repository.add_remote('github', "https://{options.fetch(:token)}@github.com/#{repo}.git") project.repository.set_remote_as_mirror('github') project.repository.fetch_remote('github', forced: true) -- cgit v1.2.1 From 44954c507ed99db892e9e4e40779bc6fc6f0a56f Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 25 Apr 2017 17:05:40 -0300 Subject: Fix import of notes on Pull Request diff --- lib/github/import.rb | 15 +++++++++++---- lib/github/representation/comment.rb | 4 ---- lib/github/response.rb | 2 +- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/lib/github/import.rb b/lib/github/import.rb index a0afd12f45f..96b69d78c55 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -30,6 +30,14 @@ module Github self.reset_callbacks :validate end + class LegacyDiffNote < ::LegacyDiffNote + self.table_name = 'notes' + + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + attr_reader :project, :repository, :repo, :options, :errors, :cached def initialize(project, options) @@ -177,7 +185,7 @@ module Github # Fetch review comments review_comments_url = "/repos/#{repo}/pulls/#{pull_request.iid}/comments" - fetch_comments(merge_request, :review_comment, review_comments_url) + fetch_comments(merge_request, :review_comment, review_comments_url, LegacyDiffNote) # Fetch comments comments_url = "/repos/#{repo}/issues/#{pull_request.iid}/comments" @@ -245,7 +253,7 @@ module Github end end - def fetch_comments(noteable, type, url) + def fetch_comments(noteable, type, url, klass = Note) while url comments = Github::Client.new(options).get(url) @@ -255,14 +263,13 @@ module Github representation = Github::Representation::Comment.new(raw, options) author_id = user_id(representation.author, project.creator_id) - note = Note.new + note = klass.new note.project_id = project.id note.noteable = noteable note.note = format_description(representation.note, representation.author) note.commit_id = representation.commit_id note.line_code = representation.line_code note.author_id = author_id - note.type = representation.type note.created_at = representation.created_at note.updated_at = representation.updated_at note.save!(validate: false) diff --git a/lib/github/representation/comment.rb b/lib/github/representation/comment.rb index 22cb98b0eff..1b5be91461b 100644 --- a/lib/github/representation/comment.rb +++ b/lib/github/representation/comment.rb @@ -20,10 +20,6 @@ module Github generate_line_code(parsed_lines.to_a.last) end - def type - 'LegacyDiffNote' if on_diff? - end - private def generate_line_code(line) diff --git a/lib/github/response.rb b/lib/github/response.rb index 2fd07dd822e..761c524b553 100644 --- a/lib/github/response.rb +++ b/lib/github/response.rb @@ -9,7 +9,7 @@ module Github end def body - @body ||= Oj.load(raw.body, class_cache: false, mode: :compat) + Oj.load(raw.body, class_cache: false, mode: :compat) end def rels -- cgit v1.2.1 From 2ea5d4b7af57037f10f42fddf96bb8c96875a8cb Mon Sep 17 00:00:00 2001 From: Robert Schilling Date: Tue, 25 Apr 2017 16:56:32 +0200 Subject: Ability to filter merge requests by labels and milestones --- app/finders/merge_requests_finder.rb | 2 +- doc/api/merge_requests.md | 15 ++++--- lib/api/merge_requests.rb | 25 ++++++----- spec/requests/api/merge_requests_spec.rb | 72 ++++++++++++++++++++++++++++++-- 4 files changed, 93 insertions(+), 21 deletions(-) diff --git a/app/finders/merge_requests_finder.rb b/app/finders/merge_requests_finder.rb index 42f0ebd774c..2fc34f186ad 100644 --- a/app/finders/merge_requests_finder.rb +++ b/app/finders/merge_requests_finder.rb @@ -6,7 +6,7 @@ # current_user - which user use # params: # scope: 'created-by-me' or 'assigned-to-me' or 'all' -# state: 'open' or 'closed' or 'all' +# state: 'open', 'closed', 'merged', or 'all' # group_id: integer # project_id: integer # milestone_title: string diff --git a/doc/api/merge_requests.md b/doc/api/merge_requests.md index ff956add348..23bf7234572 100644 --- a/doc/api/merge_requests.md +++ b/doc/api/merge_requests.md @@ -11,15 +11,20 @@ GET /projects/:id/merge_requests GET /projects/:id/merge_requests?state=opened GET /projects/:id/merge_requests?state=all GET /projects/:id/merge_requests?iids[]=42&iids[]=43 +GET /projects/:id/merge_requests?milestone=release ``` Parameters: -- `id` (required) - The ID or [URL-encoded path of the project](README.md#namespaced-path-encoding) owned by the authenticated user -- `iid` (optional) - Return the request having the given `iid` -- `state` (optional) - Return `all` requests or just those that are `merged`, `opened` or `closed` -- `order_by` (optional) - Return requests ordered by `created_at` or `updated_at` fields. Default is `created_at` -- `sort` (optional) - Return requests sorted in `asc` or `desc` order. Default is `desc` +| Attribute | Type | Required | Description | +| --------- | ---- | -------- | ----------- | +| `id` | integer | yes | The ID of a project | +| `iids` | Array[integer] | no | Return the request having the given `iid` | +| `state` | string | no | Return all merge requests or just those that are `opened`, `closed`, or `merged`| +| `order_by`| string | no | Return requests ordered by `created_at` or `updated_at` fields. Default is `created_at` | +| `sort` | string | no | Return requests sorted in `asc` or `desc` order. Default is `desc` | +| `milestone` | string | no | Return merge requests for a specific milestone | +| `labels` | string | no | Return merge requests matching a comma separated list of labels | ```json [ diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index c7dc2ea336f..e5793fbc5cb 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -33,6 +33,17 @@ module API end end + def find_merge_requests(args = {}) + args = params.merge(args) + + args[:milestone_title] = args.delete(:milestone) + args[:label_name] = args.delete(:labels) + + merge_requests = MergeRequestsFinder.new(current_user, args).execute.inc_notes_with_associations + + merge_requests.reorder(args[:order_by] => args[:sort]) + end + params :optional_params_ce do optional :description, type: String, desc: 'The description of the merge request' optional :assignee_id, type: Integer, desc: 'The ID of a user to assign the merge request' @@ -57,23 +68,15 @@ module API optional :sort, type: String, values: %w[asc desc], default: 'desc', desc: 'Return merge requests sorted in `asc` or `desc` order.' optional :iids, type: Array[Integer], desc: 'The IID array of merge requests' + optional :milestone, type: String, desc: 'Return merge requests for a specific milestone' + optional :labels, type: String, desc: 'Comma-separated list of label names' use :pagination end get ":id/merge_requests" do authorize! :read_merge_request, user_project - merge_requests = user_project.merge_requests.inc_notes_with_associations - merge_requests = filter_by_iid(merge_requests, params[:iids]) if params[:iids].present? - - merge_requests = - case params[:state] - when 'opened' then merge_requests.opened - when 'closed' then merge_requests.closed - when 'merged' then merge_requests.merged - else merge_requests - end + merge_requests = find_merge_requests(project_id: user_project.id) - merge_requests = merge_requests.reorder(params[:order_by] => params[:sort]) present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project end diff --git a/spec/requests/api/merge_requests_spec.rb b/spec/requests/api/merge_requests_spec.rb index 91b1e1aee4d..904e049d767 100644 --- a/spec/requests/api/merge_requests_spec.rb +++ b/spec/requests/api/merge_requests_spec.rb @@ -6,10 +6,17 @@ describe API::MergeRequests do let(:admin) { create(:user, :admin) } let(:non_member) { create(:user) } let!(:project) { create(:project, :public, :repository, creator: user, namespace: user.namespace, only_allow_merge_if_pipeline_succeeds: false) } - let!(:merge_request) { create(:merge_request, :simple, author: user, assignee: user, source_project: project, title: "Test", created_at: base_time) } - let!(:merge_request_closed) { create(:merge_request, state: "closed", author: user, assignee: user, source_project: project, title: "Closed test", created_at: base_time + 1.second) } - let!(:merge_request_merged) { create(:merge_request, state: "merged", author: user, assignee: user, source_project: project, title: "Merged test", created_at: base_time + 2.seconds, merge_commit_sha: '9999999999999999999999999999999999999999') } let(:milestone) { create(:milestone, title: '1.0.0', project: project) } + let(:milestone1) { create(:milestone, title: '0.9', project: project) } + let!(:merge_request) { create(:merge_request, :simple, milestone: milestone1, author: user, assignee: user, source_project: project, target_project: project, title: "Test", created_at: base_time) } + let!(:merge_request_closed) { create(:merge_request, state: "closed", milestone: milestone1, author: user, assignee: user, source_project: project, target_project: project, title: "Closed test", created_at: base_time + 1.second) } + let!(:merge_request_merged) { create(:merge_request, state: "merged", author: user, assignee: user, source_project: project, target_project: project, title: "Merged test", created_at: base_time + 2.seconds, merge_commit_sha: '9999999999999999999999999999999999999999') } + let!(:note) { create(:note_on_merge_request, author: user, project: project, noteable: merge_request, note: "a comment on a MR") } + let!(:note2) { create(:note_on_merge_request, author: user, project: project, noteable: merge_request, note: "another comment on a MR") } + let!(:label) do + create(:label, title: 'label', color: '#FFAABB', project: project) + end + let!(:label_link) { create(:label_link, label: label, target: merge_request) } before do project.team << [user, :reporter] @@ -99,6 +106,63 @@ describe API::MergeRequests do expect(response).to match_response_schema('public_api/v4/merge_requests') end + it 'returns an empty array if no issue matches milestone' do + get api("/projects/#{project.id}/merge_requests", user), milestone: '1.0.0' + + expect(response).to have_http_status(200) + expect(json_response).to be_an Array + expect(json_response.length).to eq(0) + end + + it 'returns an empty array if milestone does not exist' do + get api("/projects/#{project.id}/merge_requests", user), milestone: 'foo' + + expect(response).to have_http_status(200) + expect(json_response).to be_an Array + expect(json_response.length).to eq(0) + end + + it 'returns an array of merge requests in given milestone' do + get api("/projects/#{project.id}/merge_requests", user), milestone: '0.9' + + expect(json_response.first['title']).to eq merge_request_closed.title + expect(json_response.first['id']).to eq merge_request_closed.id + end + + it 'returns an array of merge requests matching state in milestone' do + get api("/projects/#{project.id}/merge_requests", user), milestone: '0.9', state: 'closed' + + expect(response).to have_http_status(200) + expect(json_response).to be_an Array + expect(json_response.length).to eq(1) + expect(json_response.first['id']).to eq(merge_request_closed.id) + end + + it 'returns an array of labeled merge requests' do + get api("/projects/#{project.id}/merge_requests?labels=#{label.title}", user) + + expect(response).to have_http_status(200) + expect(json_response).to be_an Array + expect(json_response.length).to eq(1) + expect(json_response.first['labels']).to eq([label.title]) + end + + it 'returns an array of labeled merge requests where all labels match' do + get api("/projects/#{project.id}/merge_requests?labels=#{label.title},foo,bar", user) + + expect(response).to have_http_status(200) + expect(json_response).to be_an Array + expect(json_response.length).to eq(0) + end + + it 'returns an empty array if no merge request matches labels' do + get api("/projects/#{project.id}/merge_requests?labels=foo,bar", user) + + expect(response).to have_http_status(200) + expect(json_response).to be_an Array + expect(json_response.length).to eq(0) + end + context "with ordering" do before do @mr_later = mr_with_later_created_and_updated_at_time @@ -166,7 +230,7 @@ describe API::MergeRequests do expect(json_response['created_at']).to be_present expect(json_response['updated_at']).to be_present expect(json_response['labels']).to eq(merge_request.label_names) - expect(json_response['milestone']).to be_nil + expect(json_response['milestone']).to be_a Hash expect(json_response['assignee']).to be_a Hash expect(json_response['author']).to be_a Hash expect(json_response['target_branch']).to eq(merge_request.target_branch) -- cgit v1.2.1 From 70d15ae1d968070e82028dd48c15c363ac9f8fa9 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 25 Apr 2017 17:08:32 -0300 Subject: Fix Rubocop offenses --- lib/github/import.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/github/import.rb b/lib/github/import.rb index 96b69d78c55..f12f979ae02 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -160,8 +160,7 @@ module Github next unless merge_request.new_record? && pull_request.valid? begin - restore_source_branch(pull_request) unless pull_request.source_branch_exists? - restore_target_branch(pull_request) unless pull_request.target_branch_exists? + restore_branches(pull_request) author_id = user_id(pull_request.author, project.creator_id) merge_request.iid = pull_request.iid @@ -310,6 +309,11 @@ module Github end end + def restore_branches(pull_request) + restore_source_branch(pull_request) unless pull_request.source_branch_exists? + restore_target_branch(pull_request) unless pull_request.target_branch_exists? + end + def restore_source_branch(pull_request) repository.create_branch(pull_request.source_branch_name, pull_request.source_branch_sha) end -- cgit v1.2.1 From f184cb433b90e5b03d8a2ecf9916e7a1bfda5ee9 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 25 Apr 2017 23:19:30 -0300 Subject: Fix undefined attribute params from import task --- lib/tasks/import.rake | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index 897b2f04ce1..e22424c98d3 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -71,7 +71,6 @@ class GithubImport return @current_user.namespace if names == @current_user.namespace_path return @current_user.namespace unless @current_user.can_create_group? - names = params[:target_namespace].presence || names full_path_namespace = Namespace.find_by_full_path(names) return full_path_namespace if full_path_namespace -- cgit v1.2.1 From d082b78998d43349f6d0cbaeb90bb448aa1188c3 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Tue, 25 Apr 2017 23:47:49 -0300 Subject: Add basic progress output to GitHub import --- lib/github/import.rb | 11 ++++++++++- lib/tasks/import.rake | 4 ++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/lib/github/import.rb b/lib/github/import.rb index f12f979ae02..3e9162ffb9d 100644 --- a/lib/github/import.rb +++ b/lib/github/import.rb @@ -38,24 +38,33 @@ module Github self.reset_callbacks :validate end - attr_reader :project, :repository, :repo, :options, :errors, :cached + attr_reader :project, :repository, :repo, :options, :errors, :cached, :verbose def initialize(project, options) @project = project @repository = project.repository @repo = project.import_source @options = options + @verbose = options.fetch(:verbose, false) @cached = Hash.new { |hash, key| hash[key] = Hash.new } @errors = [] end + # rubocop: disable Rails/Output def execute + puts 'Fetching repository...'.color(:aqua) if verbose fetch_repository + puts 'Fetching labels...'.color(:aqua) if verbose fetch_labels + puts 'Fetching milestones...'.color(:aqua) if verbose fetch_milestones + puts 'Fetching pull requests...'.color(:aqua) if verbose fetch_pull_requests + puts 'Fetching issues...'.color(:aqua) if verbose fetch_issues + puts 'Cloning wiki repository...'.color(:aqua) if verbose fetch_wiki_repository + puts 'Expiring repository cache...'.color(:aqua) if verbose expire_repository_cache true diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index e22424c98d3..bc76d7edc55 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -7,7 +7,7 @@ class GithubImport end def initialize(token, gitlab_username, project_path, extras) - @options = { url: 'https://api.github.com', token: token } + @options = { url: 'https://api.github.com', token: token, verbose: true } @project_path = project_path @current_user = User.find_by_username(gitlab_username) @github_repo = extras.empty? ? nil : extras.first @@ -28,7 +28,7 @@ class GithubImport private def show_warning! - puts "This will import GH #{@repo['full_name'].bright} into GL #{@project_path.bright} as #{@current_user.name}" + puts "This will import GitHub #{@repo['full_name'].bright} into GitLab #{@project_path.bright} as #{@current_user.name}" puts "Permission checks are ignored. Press any key to continue.".color(:red) STDIN.getch -- cgit v1.2.1 From 4dfdef2ddfc3cdeb6f6231e397543d120083a4c2 Mon Sep 17 00:00:00 2001 From: Timothy Andrew Date: Fri, 21 Apr 2017 06:44:47 +0000 Subject: Allow admins to sudo to blocked users. - Currently, (for example) admins can't delete snippets for blocked users, which is an unexpected limitation. - We modify `authenticate!` to conduct the `access_api` policy check against the `initial_current_user`, instead of the user being impersonated. - Update CHANGELOG for !10842 --- .../29505-allow-admins-sudo-to-blocked-users.yml | 4 ++++ lib/api/helpers.rb | 2 +- spec/requests/api/helpers_spec.rb | 28 +++++++++++++++++++++- 3 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 changelogs/unreleased/29505-allow-admins-sudo-to-blocked-users.yml diff --git a/changelogs/unreleased/29505-allow-admins-sudo-to-blocked-users.yml b/changelogs/unreleased/29505-allow-admins-sudo-to-blocked-users.yml new file mode 100644 index 00000000000..42fd71ccd5f --- /dev/null +++ b/changelogs/unreleased/29505-allow-admins-sudo-to-blocked-users.yml @@ -0,0 +1,4 @@ +--- +title: Allow admins to sudo to blocked users via the API +merge_request: 10842 +author: diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index ddff3c8c1e8..86bf567fe69 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -102,7 +102,7 @@ module API end def authenticate! - unauthorized! unless current_user && can?(current_user, :access_api) + unauthorized! unless current_user && can?(initial_current_user, :access_api) end def authenticate_non_get! diff --git a/spec/requests/api/helpers_spec.rb b/spec/requests/api/helpers_spec.rb index 988a57a80ea..6cbe1ffa1a4 100644 --- a/spec/requests/api/helpers_spec.rb +++ b/spec/requests/api/helpers_spec.rb @@ -427,6 +427,7 @@ describe API::Helpers, api: true do context 'current_user is nil' do before do expect_any_instance_of(self.class).to receive(:current_user).and_return(nil) + allow_any_instance_of(self.class).to receive(:initial_current_user).and_return(nil) end it 'returns a 401 response' do @@ -435,13 +436,38 @@ describe API::Helpers, api: true do end context 'current_user is present' do + let(:user) { build(:user) } + before do - expect_any_instance_of(self.class).to receive(:current_user).at_least(:once).and_return(User.new) + expect_any_instance_of(self.class).to receive(:current_user).at_least(:once).and_return(user) + expect_any_instance_of(self.class).to receive(:initial_current_user).and_return(user) end it 'does not raise an error' do expect { authenticate! }.not_to raise_error end end + + context 'current_user is blocked' do + let(:user) { build(:user, :blocked) } + + before do + expect_any_instance_of(self.class).to receive(:current_user).at_least(:once).and_return(user) + end + + it 'raises an error' do + expect_any_instance_of(self.class).to receive(:initial_current_user).and_return(user) + + expect { authenticate! }.to raise_error '401 - {"message"=>"401 Unauthorized"}' + end + + it "doesn't raise an error if an admin user is impersonating a blocked user (via sudo)" do + admin_user = build(:user, :admin) + + expect_any_instance_of(self.class).to receive(:initial_current_user).and_return(admin_user) + + expect { authenticate! }.not_to raise_error + end + end end end -- cgit v1.2.1 From e809729ece441a282e6928e6b875337bbb54bfb9 Mon Sep 17 00:00:00 2001 From: Robert Schilling Date: Tue, 25 Apr 2017 21:33:11 +0200 Subject: Add changelog entry --- changelogs/unreleased/more-mr-filters.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 changelogs/unreleased/more-mr-filters.yml diff --git a/changelogs/unreleased/more-mr-filters.yml b/changelogs/unreleased/more-mr-filters.yml new file mode 100644 index 00000000000..3c2114f6614 --- /dev/null +++ b/changelogs/unreleased/more-mr-filters.yml @@ -0,0 +1,4 @@ +--- +title: 'API: Filter merge requests by milestone and labels' +merge_request: Robert Schilling +author: 10924 -- cgit v1.2.1 From 51207344f8f26358998bfe4d1e1dd057f11d09f4 Mon Sep 17 00:00:00 2001 From: Robert Schilling Date: Wed, 26 Apr 2017 10:19:16 +0200 Subject: Add documentation example for labels --- doc/api/merge_requests.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/api/merge_requests.md b/doc/api/merge_requests.md index 23bf7234572..dde855b2bd4 100644 --- a/doc/api/merge_requests.md +++ b/doc/api/merge_requests.md @@ -12,6 +12,7 @@ GET /projects/:id/merge_requests?state=opened GET /projects/:id/merge_requests?state=all GET /projects/:id/merge_requests?iids[]=42&iids[]=43 GET /projects/:id/merge_requests?milestone=release +GET /projects/:id/merge_requests?labels=bug,reproduced ``` Parameters: -- cgit v1.2.1 From aa364386d2c08298cc4a201b4a7d2f779ff6fbb6 Mon Sep 17 00:00:00 2001 From: "Luke \"Jared\" Bennett" Date: Wed, 26 Apr 2017 09:35:41 +0100 Subject: Correct the resource counted for MR collapsed info --- app/views/shared/milestones/_sidebar.html.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/shared/milestones/_sidebar.html.haml b/app/views/shared/milestones/_sidebar.html.haml index ccc808ff43e..774d20fb5ba 100644 --- a/app/views/shared/milestones/_sidebar.html.haml +++ b/app/views/shared/milestones/_sidebar.html.haml @@ -89,7 +89,7 @@ .sidebar-collapsed-icon %strong = icon('exclamation', 'aria-hidden': 'true') - %span= milestone.issues_visible_to_user(current_user).count + %span= milestone.merge_requests.count .title.hide-collapsed Merge requests %span.badge= milestone.merge_requests.count -- cgit v1.2.1 From ccaf6ea59ab44e917125b411f17404e32f2d735a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Wed, 26 Apr 2017 11:19:20 +0200 Subject: Wait for AJAX after steps defined in SharedActiveTab MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- features/steps/shared/active_tab.rb | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/features/steps/shared/active_tab.rb b/features/steps/shared/active_tab.rb index 4eef7aff213..8bae80a8707 100644 --- a/features/steps/shared/active_tab.rb +++ b/features/steps/shared/active_tab.rb @@ -1,5 +1,10 @@ module SharedActiveTab include Spinach::DSL + include WaitForAjax + + after do + wait_for_ajax if javascript_test? + end def ensure_active_main_tab(content) expect(find('.layout-nav li.active')).to have_content(content) -- cgit v1.2.1 From 67974f1dfb413452ca6fc048c856360f4a843eb1 Mon Sep 17 00:00:00 2001 From: Tiago Botelho Date: Tue, 25 Apr 2017 16:47:57 +0100 Subject: remove invalid services --- app/models/service.rb | 1 + spec/models/service_spec.rb | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/app/models/service.rb b/app/models/service.rb index dc76bf925d3..cbb75186206 100644 --- a/app/models/service.rb +++ b/app/models/service.rb @@ -26,6 +26,7 @@ class Service < ActiveRecord::Base has_one :service_hook validates :project_id, presence: true, unless: proc { |service| service.template? } + validates :type, presence: true scope :visible, -> { where.not(type: 'GitlabIssueTrackerService') } scope :issue_trackers, -> { where(category: 'issue_tracker') } diff --git a/spec/models/service_spec.rb b/spec/models/service_spec.rb index 0e2f07e945f..f5ba8f76f40 100644 --- a/spec/models/service_spec.rb +++ b/spec/models/service_spec.rb @@ -6,6 +6,10 @@ describe Service, models: true do it { is_expected.to have_one :service_hook } end + describe 'Validations' do + it { is_expected.to validate_presence_of(:type).on(:create) } + end + describe "Test Button" do before do @service = Service.new -- cgit v1.2.1 From 27317d55b0a2ad7d18b324fa30496f0acd07f211 Mon Sep 17 00:00:00 2001 From: "Luke \"Jared\" Bennett" Date: Wed, 26 Apr 2017 11:02:32 +0100 Subject: Added test --- .../features/projects/milestones/milestone_spec.rb | 24 ++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/spec/features/projects/milestones/milestone_spec.rb b/spec/features/projects/milestones/milestone_spec.rb index dab78fd3571..5e19907eef9 100644 --- a/spec/features/projects/milestones/milestone_spec.rb +++ b/spec/features/projects/milestones/milestone_spec.rb @@ -63,4 +63,28 @@ feature 'Project milestone', :feature do expect(page).not_to have_content('Assign some issues to this milestone.') end end + + context 'when project has an issue' do + before do + create(:issue, project: project, milestone: milestone) + + visit namespace_project_milestone_path(project.namespace, project, milestone) + end + + describe 'the collapsed sidebar' do + before do + find('.milestone-sidebar .gutter-toggle').click + end + + it 'shows the total MR and issue counts' do + find('.milestone-sidebar .block', match: :first) + blocks = all('.milestone-sidebar .block') + + aggregate_failures 'MR and issue blocks' do + expect(blocks[3]).to have_content 1 + expect(blocks[4]).to have_content 0 + end + end + end + end end -- cgit v1.2.1 From 87327c5845a7feec92e546dc0da42282ac27de40 Mon Sep 17 00:00:00 2001 From: Jarka Kadlecova Date: Thu, 20 Apr 2017 10:56:41 +0200 Subject: Support preview_markdown action for personal_snippets --- app/controllers/concerns/markdown_preview.rb | 19 +++++++++++++++++++ app/controllers/projects/wikis_controller.rb | 21 +++++++-------------- app/controllers/projects_controller.rb | 19 +++++-------------- app/controllers/snippets_controller.rb | 5 +++++ app/helpers/gitlab_markdown_helper.rb | 2 +- .../unreleased/12910-personal-snippet-prep-2.yml | 4 ++++ config/routes/snippets.rb | 1 + spec/controllers/projects/wikis_controller_spec.rb | 16 ++++++++++++++++ spec/controllers/projects_controller_spec.rb | 10 ++++++++++ spec/controllers/snippets_controller_spec.rb | 12 ++++++++++++ 10 files changed, 80 insertions(+), 29 deletions(-) create mode 100644 app/controllers/concerns/markdown_preview.rb create mode 100644 changelogs/unreleased/12910-personal-snippet-prep-2.yml create mode 100644 spec/controllers/projects/wikis_controller_spec.rb diff --git a/app/controllers/concerns/markdown_preview.rb b/app/controllers/concerns/markdown_preview.rb new file mode 100644 index 00000000000..40eff267348 --- /dev/null +++ b/app/controllers/concerns/markdown_preview.rb @@ -0,0 +1,19 @@ +module MarkdownPreview + private + + def render_markdown_preview(text, markdown_context = {}) + render json: { + body: view_context.markdown(text, markdown_context), + references: { + users: preview_referenced_users(text) + } + } + end + + def preview_referenced_users(text) + extractor = Gitlab::ReferenceExtractor.new(@project, current_user) + extractor.analyze(text, author: current_user) + + extractor.users.map(&:username) + end +end diff --git a/app/controllers/projects/wikis_controller.rb b/app/controllers/projects/wikis_controller.rb index c5e24b9e365..96125684da0 100644 --- a/app/controllers/projects/wikis_controller.rb +++ b/app/controllers/projects/wikis_controller.rb @@ -1,4 +1,6 @@ class Projects::WikisController < Projects::ApplicationController + include MarkdownPreview + before_action :authorize_read_wiki! before_action :authorize_create_wiki!, only: [:edit, :create, :history] before_action :authorize_admin_wiki!, only: :destroy @@ -91,21 +93,13 @@ class Projects::WikisController < Projects::ApplicationController ) end - def preview_markdown - text = params[:text] - - ext = Gitlab::ReferenceExtractor.new(@project, current_user) - ext.analyze(text, author: current_user) - - render json: { - body: view_context.markdown(text, pipeline: :wiki, project_wiki: @project_wiki, page_slug: params[:id]), - references: { - users: ext.users.map(&:username) - } - } + def git_access end - def git_access + def preview_markdown + context = { pipeline: :wiki, project_wiki: @project_wiki, page_slug: params[:id] } + + render_markdown_preview(params[:text], context) end private @@ -115,7 +109,6 @@ class Projects::WikisController < Projects::ApplicationController # Call #wiki to make sure the Wiki Repo is initialized @project_wiki.wiki - @sidebar_wiki_entries = WikiPage.group_by_directory(@project_wiki.pages.first(15)) rescue ProjectWiki::CouldNotCreateWikiError flash[:notice] = "Could not create Wiki Repository at this time. Please try again later." diff --git a/app/controllers/projects_controller.rb b/app/controllers/projects_controller.rb index 6807c37f972..9f6ee4826e6 100644 --- a/app/controllers/projects_controller.rb +++ b/app/controllers/projects_controller.rb @@ -1,6 +1,7 @@ class ProjectsController < Projects::ApplicationController include IssuableCollections include ExtractsPath + include MarkdownPreview before_action :authenticate_user!, except: [:index, :show, :activity, :refs] before_action :project, except: [:index, :new, :create] @@ -216,20 +217,6 @@ class ProjectsController < Projects::ApplicationController } end - def preview_markdown - text = params[:text] - - ext = Gitlab::ReferenceExtractor.new(@project, current_user) - ext.analyze(text, author: current_user) - - render json: { - body: view_context.markdown(text), - references: { - users: ext.users.map(&:username) - } - } - end - def refs branches = BranchesFinder.new(@repository, params).execute.map(&:name) @@ -252,6 +239,10 @@ class ProjectsController < Projects::ApplicationController render json: options.to_json end + def preview_markdown + render_markdown_preview(params[:text]) + end + private # Render project landing depending of which features are available diff --git a/app/controllers/snippets_controller.rb b/app/controllers/snippets_controller.rb index f3fd3da8b20..056910fad67 100644 --- a/app/controllers/snippets_controller.rb +++ b/app/controllers/snippets_controller.rb @@ -2,6 +2,7 @@ class SnippetsController < ApplicationController include ToggleAwardEmoji include SpammableActions include SnippetsActions + include MarkdownPreview before_action :snippet, only: [:show, :edit, :destroy, :update, :raw, :download] @@ -77,6 +78,10 @@ class SnippetsController < ApplicationController ) end + def preview_markdown + render_markdown_preview(params[:text], skip_project_check: true) + end + protected def snippet diff --git a/app/helpers/gitlab_markdown_helper.rb b/app/helpers/gitlab_markdown_helper.rb index cd442237086..106feb87398 100644 --- a/app/helpers/gitlab_markdown_helper.rb +++ b/app/helpers/gitlab_markdown_helper.rb @@ -196,7 +196,7 @@ module GitlabMarkdownHelper end # Calls Banzai.post_process with some common context options - def banzai_postprocess(html, context) + def banzai_postprocess(html, context = {}) context.merge!( current_user: (current_user if defined?(current_user)), diff --git a/changelogs/unreleased/12910-personal-snippet-prep-2.yml b/changelogs/unreleased/12910-personal-snippet-prep-2.yml new file mode 100644 index 00000000000..bd9527c30c8 --- /dev/null +++ b/changelogs/unreleased/12910-personal-snippet-prep-2.yml @@ -0,0 +1,4 @@ +--- +title: Support Markdown previews for personal snippets +merge_request: 10810 +author: diff --git a/config/routes/snippets.rb b/config/routes/snippets.rb index ce0d1314292..56534f677be 100644 --- a/config/routes/snippets.rb +++ b/config/routes/snippets.rb @@ -3,6 +3,7 @@ resources :snippets, concerns: :awardable do get 'raw' get 'download' post :mark_as_spam + post :preview_markdown end end diff --git a/spec/controllers/projects/wikis_controller_spec.rb b/spec/controllers/projects/wikis_controller_spec.rb new file mode 100644 index 00000000000..92addf30307 --- /dev/null +++ b/spec/controllers/projects/wikis_controller_spec.rb @@ -0,0 +1,16 @@ +require 'spec_helper' + +describe Projects::WikisController do + let(:project) { create(:project_empty_repo, :public) } + let(:user) { create(:user) } + + describe 'POST #preview_markdown' do + it 'renders json in a correct format' do + sign_in(user) + + post :preview_markdown, namespace_id: project.namespace, project_id: project, id: 'page/path', text: '*Markdown* text' + + expect(JSON.parse(response.body).keys).to match_array(%w(body references)) + end + end +end diff --git a/spec/controllers/projects_controller_spec.rb b/spec/controllers/projects_controller_spec.rb index a88ffc1ea6a..eafc2154568 100644 --- a/spec/controllers/projects_controller_spec.rb +++ b/spec/controllers/projects_controller_spec.rb @@ -398,4 +398,14 @@ describe ProjectsController do expect(parsed_body["Commits"]).to include("123456") end end + + describe 'POST #preview_markdown' do + it 'renders json in a correct format' do + sign_in(user) + + post :preview_markdown, namespace_id: public_project.namespace, id: public_project, text: '*Markdown* text' + + expect(JSON.parse(response.body).keys).to match_array(%w(body references)) + end + end end diff --git a/spec/controllers/snippets_controller_spec.rb b/spec/controllers/snippets_controller_spec.rb index 5de3b9890ef..234f3edd3d8 100644 --- a/spec/controllers/snippets_controller_spec.rb +++ b/spec/controllers/snippets_controller_spec.rb @@ -521,4 +521,16 @@ describe SnippetsController do end end end + + describe 'POST #preview_markdown' do + let(:snippet) { create(:personal_snippet, :public) } + + it 'renders json in a correct format' do + sign_in(user) + + post :preview_markdown, id: snippet, text: '*Markdown* text' + + expect(JSON.parse(response.body).keys).to match_array(%w(body references)) + end + end end -- cgit v1.2.1 From 893b8bffb5a726619b9ff5b8e4cb410fb9e4b458 Mon Sep 17 00:00:00 2001 From: blackst0ne Date: Wed, 26 Apr 2017 21:56:27 +1100 Subject: Replace header merge request icon --- app/views/shared/icons/_mr_bold.svg | 3 ++- changelogs/unreleased/replace_header_mr_icon.yml | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 changelogs/unreleased/replace_header_mr_icon.yml diff --git a/app/views/shared/icons/_mr_bold.svg b/app/views/shared/icons/_mr_bold.svg index 2daa55a8652..5468545da2e 100644 --- a/app/views/shared/icons/_mr_bold.svg +++ b/app/views/shared/icons/_mr_bold.svg @@ -1 +1,2 @@ - + + diff --git a/changelogs/unreleased/replace_header_mr_icon.yml b/changelogs/unreleased/replace_header_mr_icon.yml new file mode 100644 index 00000000000..2ef6500f88a --- /dev/null +++ b/changelogs/unreleased/replace_header_mr_icon.yml @@ -0,0 +1,4 @@ +--- +title: Replace header merge request icon +merge_request: 10932 +author: blackst0ne -- cgit v1.2.1 From 93a698f9b25baa1d3a66326f3f9761103c5ffb87 Mon Sep 17 00:00:00 2001 From: Lin Jen-Shin Date: Wed, 26 Apr 2017 11:54:45 +0000 Subject: Update CHANGELOG.md for 9.1.1 [ci skip] --- CHANGELOG.md | 15 +++++++++++++++ ...0306-transaction-while-moving-issues-to-ghost-user.yml | 4 ---- .../31174-project-integration-service-sub-nav.yml | 4 ---- ...80-fix-showing-issues-from-pending-delete-projects.yml | 4 ---- .../unreleased/add_index_on_ci_builds_updated_at.yml | 4 ---- .../dm-fix-individual-notes-reply-attributes.yml | 5 ----- changelogs/unreleased/dm-fix-oauth-user-creation.yml | 4 ---- .../unreleased/fix-gb-fix-blocked-pipeline-duration.yml | 4 ---- .../fix-gb-fix-incorrect-commit-status-badge-text.yml | 4 ---- changelogs/unreleased/fix-usage-ping-doc-link.yml | 4 ---- .../unreleased/sh-optimize-milestone-polymorphic-url.yml | 4 ---- .../unreleased/tc-realtime-every-pipeline-on-mr.yml | 4 ---- 12 files changed, 15 insertions(+), 45 deletions(-) delete mode 100644 changelogs/unreleased/30306-transaction-while-moving-issues-to-ghost-user.yml delete mode 100644 changelogs/unreleased/31174-project-integration-service-sub-nav.yml delete mode 100644 changelogs/unreleased/31280-fix-showing-issues-from-pending-delete-projects.yml delete mode 100644 changelogs/unreleased/add_index_on_ci_builds_updated_at.yml delete mode 100644 changelogs/unreleased/dm-fix-individual-notes-reply-attributes.yml delete mode 100644 changelogs/unreleased/dm-fix-oauth-user-creation.yml delete mode 100644 changelogs/unreleased/fix-gb-fix-blocked-pipeline-duration.yml delete mode 100644 changelogs/unreleased/fix-gb-fix-incorrect-commit-status-badge-text.yml delete mode 100644 changelogs/unreleased/fix-usage-ping-doc-link.yml delete mode 100644 changelogs/unreleased/sh-optimize-milestone-polymorphic-url.yml delete mode 100644 changelogs/unreleased/tc-realtime-every-pipeline-on-mr.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 977a7927615..a47c43dd5d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,21 @@ documentation](doc/development/changelog.md) for instructions on adding your own entry. +## 9.1.1 (2017-04-26) + +- Add a transaction around move_issues_to_ghost_user. !10465 +- Properly expire cache for all MRs of a pipeline. !10770 +- Add sub-nav for Project Integration Services edit page. !10813 +- Fix missing duration for blocked pipelines. !10856 +- Fix lastest commit status text on main project page. !10863 +- Add index on ci_builds.updated_at. !10870 (blackst0ne) +- Fix 500 error due to trying to show issues from pending deleting projects. !10906 +- Ensures that OAuth/LDAP/SAML users don't need to be confirmed. +- Ensure replying to an individual note by email creates a note with its own discussion ID. +- Fix OAuth, LDAP and SAML SSO when regular sign-ups are disabled. +- Fix usage ping docs link from empty cohorts page. +- Eliminate N+1 queries in loading namespaces for every issuable in milestones. + ## 9.1.0 (2017-04-22) - Added merge requests empty state. !7342 diff --git a/changelogs/unreleased/30306-transaction-while-moving-issues-to-ghost-user.yml b/changelogs/unreleased/30306-transaction-while-moving-issues-to-ghost-user.yml deleted file mode 100644 index 5fc57e44be6..00000000000 --- a/changelogs/unreleased/30306-transaction-while-moving-issues-to-ghost-user.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -title: Add a transaction around move_issues_to_ghost_user -merge_request: 10465 -author: diff --git a/changelogs/unreleased/31174-project-integration-service-sub-nav.yml b/changelogs/unreleased/31174-project-integration-service-sub-nav.yml deleted file mode 100644 index f3f91f92428..00000000000 --- a/changelogs/unreleased/31174-project-integration-service-sub-nav.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -title: Add sub-nav for Project Integration Services edit page -merge_request: 10813 -author: diff --git a/changelogs/unreleased/31280-fix-showing-issues-from-pending-delete-projects.yml b/changelogs/unreleased/31280-fix-showing-issues-from-pending-delete-projects.yml deleted file mode 100644 index 410480de573..00000000000 --- a/changelogs/unreleased/31280-fix-showing-issues-from-pending-delete-projects.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -title: Fix 500 error due to trying to show issues from pending deleting projects -merge_request: 10906 -author: diff --git a/changelogs/unreleased/add_index_on_ci_builds_updated_at.yml b/changelogs/unreleased/add_index_on_ci_builds_updated_at.yml deleted file mode 100644 index ede9b946a70..00000000000 --- a/changelogs/unreleased/add_index_on_ci_builds_updated_at.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -title: Add index on ci_builds.updated_at -merge_request: 10870 -author: blackst0ne diff --git a/changelogs/unreleased/dm-fix-individual-notes-reply-attributes.yml b/changelogs/unreleased/dm-fix-individual-notes-reply-attributes.yml deleted file mode 100644 index e8c05092ea8..00000000000 --- a/changelogs/unreleased/dm-fix-individual-notes-reply-attributes.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Ensure replying to an individual note by email creates a note with its own - discussion ID -merge_request: -author: diff --git a/changelogs/unreleased/dm-fix-oauth-user-creation.yml b/changelogs/unreleased/dm-fix-oauth-user-creation.yml deleted file mode 100644 index 161b114394a..00000000000 --- a/changelogs/unreleased/dm-fix-oauth-user-creation.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -title: Fix OAuth, LDAP and SAML SSO when regular sign-ups are disabled -merge_request: -author: diff --git a/changelogs/unreleased/fix-gb-fix-blocked-pipeline-duration.yml b/changelogs/unreleased/fix-gb-fix-blocked-pipeline-duration.yml deleted file mode 100644 index 5c87b1fdbd5..00000000000 --- a/changelogs/unreleased/fix-gb-fix-blocked-pipeline-duration.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -title: Fix missing duration for blocked pipelines -merge_request: 10856 -author: diff --git a/changelogs/unreleased/fix-gb-fix-incorrect-commit-status-badge-text.yml b/changelogs/unreleased/fix-gb-fix-incorrect-commit-status-badge-text.yml deleted file mode 100644 index abe047af06f..00000000000 --- a/changelogs/unreleased/fix-gb-fix-incorrect-commit-status-badge-text.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -title: Fix lastest commit status text on main project page -merge_request: 10863 -author: diff --git a/changelogs/unreleased/fix-usage-ping-doc-link.yml b/changelogs/unreleased/fix-usage-ping-doc-link.yml deleted file mode 100644 index 5217a4e4e4b..00000000000 --- a/changelogs/unreleased/fix-usage-ping-doc-link.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -title: Fix usage ping docs link from empty cohorts page -merge_request: -author: diff --git a/changelogs/unreleased/sh-optimize-milestone-polymorphic-url.yml b/changelogs/unreleased/sh-optimize-milestone-polymorphic-url.yml deleted file mode 100644 index ad62c896b04..00000000000 --- a/changelogs/unreleased/sh-optimize-milestone-polymorphic-url.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -title: Eliminate N+1 queries in loading namespaces for every issuable in milestones -merge_request: -author: diff --git a/changelogs/unreleased/tc-realtime-every-pipeline-on-mr.yml b/changelogs/unreleased/tc-realtime-every-pipeline-on-mr.yml deleted file mode 100644 index 944baae257c..00000000000 --- a/changelogs/unreleased/tc-realtime-every-pipeline-on-mr.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- -title: Properly expire cache for all MRs of a pipeline -merge_request: 10770 -author: -- cgit v1.2.1 From 019b06b9d29162356a89be46f958a8c0cbd022fd Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Wed, 26 Apr 2017 12:04:22 +0000 Subject: Load a project's CI status in batch from redis --- app/helpers/projects_helper.rb | 5 + app/models/project.rb | 2 + app/views/shared/projects/_list.html.haml | 1 + .../27376-bvl-load-pipelinestatus-in-batch.yml | 4 + lib/gitlab/cache/ci/project_pipeline_status.rb | 53 ++++++-- spec/helpers/ci_status_helper_spec.rb | 15 ++- spec/helpers/projects_helper_spec.rb | 12 ++ .../cache/ci/project_pipeline_status_spec.rb | 142 +++++++++++++++++++-- 8 files changed, 204 insertions(+), 30 deletions(-) create mode 100644 changelogs/unreleased/27376-bvl-load-pipelinestatus-in-batch.yml diff --git a/app/helpers/projects_helper.rb b/app/helpers/projects_helper.rb index 5f97e6114ea..9ac852aa5ee 100644 --- a/app/helpers/projects_helper.rb +++ b/app/helpers/projects_helper.rb @@ -166,6 +166,11 @@ module ProjectsHelper key end + def load_pipeline_status(projects) + Gitlab::Cache::Ci::ProjectPipelineStatus. + load_in_batch_for_projects(projects) + end + private def repo_children_classes(field) diff --git a/app/models/project.rb b/app/models/project.rb index 73593f04283..c7dc562c238 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -74,6 +74,7 @@ class Project < ActiveRecord::Base attr_accessor :new_default_branch attr_accessor :old_path_with_namespace + attr_writer :pipeline_status alias_attribute :title, :name @@ -1181,6 +1182,7 @@ class Project < ActiveRecord::Base end end + # Lazy loading of the `pipeline_status` attribute def pipeline_status @pipeline_status ||= Gitlab::Cache::Ci::ProjectPipelineStatus.load_for_project(self) end diff --git a/app/views/shared/projects/_list.html.haml b/app/views/shared/projects/_list.html.haml index c0699b13719..aaffc0927eb 100644 --- a/app/views/shared/projects/_list.html.haml +++ b/app/views/shared/projects/_list.html.haml @@ -7,6 +7,7 @@ - skip_namespace = false unless local_assigns[:skip_namespace] == true - show_last_commit_as_description = false unless local_assigns[:show_last_commit_as_description] == true - remote = false unless local_assigns[:remote] == true +- load_pipeline_status(projects) .js-projects-list-holder - if projects.any? diff --git a/changelogs/unreleased/27376-bvl-load-pipelinestatus-in-batch.yml b/changelogs/unreleased/27376-bvl-load-pipelinestatus-in-batch.yml new file mode 100644 index 00000000000..3d615f5d8a7 --- /dev/null +++ b/changelogs/unreleased/27376-bvl-load-pipelinestatus-in-batch.yml @@ -0,0 +1,4 @@ +--- +title: Fetch pipeline status in batch from redis +merge_request: 10785 +author: diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index b358f2efa4f..4fc9a075edc 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -15,18 +15,51 @@ module Gitlab end end + def self.load_in_batch_for_projects(projects) + cached_results_for_projects(projects).zip(projects).each do |result, project| + project.pipeline_status = new(project, result) + project.pipeline_status.load_status + end + end + + def self.cached_results_for_projects(projects) + result = Gitlab::Redis.with do |redis| + redis.multi do + projects.each do |project| + cache_key = cache_key_for_project(project) + redis.exists(cache_key) + redis.hmget(cache_key, :sha, :status, :ref) + end + end + end + + result.each_slice(2).map do |(cache_key_exists, (sha, status, ref))| + pipeline_info = { sha: sha, status: status, ref: ref } + { loaded_from_cache: cache_key_exists, pipeline_info: pipeline_info } + end + end + + def self.cache_key_for_project(project) + "projects/#{project.id}/pipeline_status" + end + def self.update_for_pipeline(pipeline) - new(pipeline.project, - sha: pipeline.sha, - status: pipeline.status, - ref: pipeline.ref).store_in_cache_if_needed + pipeline_info = { + sha: pipeline.sha, + status: pipeline.status, + ref: pipeline.ref + } + + new(pipeline.project, pipeline_info: pipeline_info). + store_in_cache_if_needed end - def initialize(project, sha: nil, status: nil, ref: nil) + def initialize(project, pipeline_info: {}, loaded_from_cache: nil) @project = project - @sha = sha - @ref = ref - @status = status + @sha = pipeline_info[:sha] + @ref = pipeline_info[:ref] + @status = pipeline_info[:status] + @loaded = loaded_from_cache end def has_status? @@ -85,6 +118,8 @@ module Gitlab end def has_cache? + return self.loaded unless self.loaded.nil? + Gitlab::Redis.with do |redis| redis.exists(cache_key) end @@ -95,7 +130,7 @@ module Gitlab end def cache_key - "projects/#{project.id}/build_status" + self.class.cache_key_for_project(project) end end end diff --git a/spec/helpers/ci_status_helper_spec.rb b/spec/helpers/ci_status_helper_spec.rb index 3dc13001056..e6bb953e9d8 100644 --- a/spec/helpers/ci_status_helper_spec.rb +++ b/spec/helpers/ci_status_helper_spec.rb @@ -46,14 +46,15 @@ describe CiStatusHelper do end describe "#pipeline_status_cache_key" do - let(:pipeline_status) do - Gitlab::Cache::Ci::ProjectPipelineStatus - .new(build(:project), sha: '123abc', status: 'success') - end - it "builds a cache key for pipeline status" do - expect(helper.pipeline_status_cache_key(pipeline_status)) - .to eq("pipeline-status/123abc-success") + pipeline_status = Gitlab::Cache::Ci::ProjectPipelineStatus.new( + build(:project), + pipeline_info: { + sha: "123abc", + status: "success" + } + ) + expect(helper.pipeline_status_cache_key(pipeline_status)).to eq("pipeline-status/123abc-success") end end end diff --git a/spec/helpers/projects_helper_spec.rb b/spec/helpers/projects_helper_spec.rb index a7fc5d14859..d96d48a1977 100644 --- a/spec/helpers/projects_helper_spec.rb +++ b/spec/helpers/projects_helper_spec.rb @@ -103,6 +103,18 @@ describe ProjectsHelper do end end + describe '#load_pipeline_status' do + it 'loads the pipeline status in batch' do + project = build(:empty_project) + + helper.load_pipeline_status([project]) + # Skip lazy loading of the `pipeline_status` attribute + pipeline_status = project.instance_variable_get('@pipeline_status') + + expect(pipeline_status).to be_a(Gitlab::Cache::Ci::ProjectPipelineStatus) + end + end + describe 'link_to_member' do let(:group) { create(:group) } let(:project) { create(:empty_project, group: group) } diff --git a/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb b/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb index fced253dd01..b386852b196 100644 --- a/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb +++ b/spec/lib/gitlab/cache/ci/project_pipeline_status_spec.rb @@ -1,8 +1,9 @@ require 'spec_helper' -describe Gitlab::Cache::Ci::ProjectPipelineStatus do +describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do let(:project) { create(:project) } let(:pipeline_status) { described_class.new(project) } + let(:cache_key) { "projects/#{project.id}/pipeline_status" } describe '.load_for_project' do it "loads the status" do @@ -12,12 +13,110 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus do end end + describe 'loading in batches' do + let(:status) { 'success' } + let(:sha) { '424d1b73bc0d3cb726eb7dc4ce17a4d48552f8c6' } + let(:ref) { 'master' } + let(:pipeline_info) { { sha: sha, status: status, ref: ref } } + let(:project_without_status) { create(:project) } + + describe '.load_in_batch_for_projects' do + it 'preloads pipeline_status on projects' do + described_class.load_in_batch_for_projects([project]) + + # Don't call the accessor that would lazy load the variable + expect(project.instance_variable_get('@pipeline_status')).to be_a(described_class) + end + + describe 'without a status in redis' do + it 'loads the status from a commit when it was not in redis' do + empty_status = { sha: nil, status: nil, ref: nil } + fake_pipeline = described_class.new( + project_without_status, + pipeline_info: empty_status, + loaded_from_cache: false + ) + + expect(described_class).to receive(:new). + with(project_without_status, + pipeline_info: empty_status, + loaded_from_cache: false). + and_return(fake_pipeline) + expect(fake_pipeline).to receive(:load_from_project) + expect(fake_pipeline).to receive(:store_in_cache) + + described_class.load_in_batch_for_projects([project_without_status]) + end + + it 'only connects to redis twice' do + # Once to load, once to store in the cache + expect(Gitlab::Redis).to receive(:with).exactly(2).and_call_original + + described_class.load_in_batch_for_projects([project_without_status]) + + expect(project_without_status.pipeline_status).not_to be_nil + end + end + + describe 'when a status was cached in redis' do + before do + Gitlab::Redis.with do |redis| + redis.mapped_hmset(cache_key, + { sha: sha, status: status, ref: ref }) + end + end + + it 'loads the correct status' do + described_class.load_in_batch_for_projects([project]) + + pipeline_status = project.instance_variable_get('@pipeline_status') + + expect(pipeline_status.sha).to eq(sha) + expect(pipeline_status.status).to eq(status) + expect(pipeline_status.ref).to eq(ref) + end + + it 'only connects to redis once' do + expect(Gitlab::Redis).to receive(:with).exactly(1).and_call_original + + described_class.load_in_batch_for_projects([project]) + + expect(project.pipeline_status).not_to be_nil + end + + it "doesn't load the status separatly" do + expect_any_instance_of(described_class).not_to receive(:load_from_project) + expect_any_instance_of(described_class).not_to receive(:load_from_cache) + + described_class.load_in_batch_for_projects([project]) + end + end + end + + describe '.cached_results_for_projects' do + it 'loads a status from redis for all projects' do + Gitlab::Redis.with do |redis| + redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) + end + + result = [{ loaded_from_cache: false, pipeline_info: { sha: nil, status: nil, ref: nil } }, + { loaded_from_cache: true, pipeline_info: pipeline_info }] + + expect(described_class.cached_results_for_projects([project_without_status, project])).to eq(result) + end + end + end + describe '.update_for_pipeline' do it 'refreshes the cache if nescessary' do - pipeline = build_stubbed(:ci_pipeline, sha: '123456', status: 'success') + pipeline = build_stubbed(:ci_pipeline, + sha: '123456', status: 'success', ref: 'master') fake_status = double expect(described_class).to receive(:new). - with(pipeline.project, sha: '123456', status: 'success', ref: 'master'). + with(pipeline.project, + pipeline_info: { + sha: '123456', status: 'success', ref: 'master' + }). and_return(fake_status) expect(fake_status).to receive(:store_in_cache_if_needed) @@ -110,7 +209,7 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus do pipeline_status.status = 'failed' pipeline_status.store_in_cache - read_sha, read_status = Gitlab::Redis.with { |redis| redis.hmget("projects/#{project.id}/build_status", :sha, :status) } + read_sha, read_status = Gitlab::Redis.with { |redis| redis.hmget(cache_key, :sha, :status) } expect(read_sha).to eq('123456') expect(read_status).to eq('failed') @@ -120,10 +219,10 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus do describe '#store_in_cache_if_needed', :redis do it 'stores the state in the cache when the sha is the HEAD of the project' do create(:ci_pipeline, :success, project: project, sha: project.commit.sha) - build_status = described_class.load_for_project(project) + pipeline_status = described_class.load_for_project(project) - build_status.store_in_cache_if_needed - sha, status, ref = Gitlab::Redis.with { |redis| redis.hmget("projects/#{project.id}/build_status", :sha, :status, :ref) } + pipeline_status.store_in_cache_if_needed + sha, status, ref = Gitlab::Redis.with { |redis| redis.hmget(cache_key, :sha, :status, :ref) } expect(sha).not_to be_nil expect(status).not_to be_nil @@ -131,10 +230,13 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus do end it "doesn't store the status in redis when the sha is not the head of the project" do - other_status = described_class.new(project, sha: "123456", status: "failed") + other_status = described_class.new( + project, + pipeline_info: { sha: "123456", status: "failed" } + ) other_status.store_in_cache_if_needed - sha, status = Gitlab::Redis.with { |redis| redis.hmget("projects/#{project.id}/build_status", :sha, :status) } + sha, status = Gitlab::Redis.with { |redis| redis.hmget(cache_key, :sha, :status) } expect(sha).to be_nil expect(status).to be_nil @@ -142,11 +244,18 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus do it "deletes the cache if the repository doesn't have a head commit" do empty_project = create(:empty_project) - Gitlab::Redis.with { |redis| redis.mapped_hmset("projects/#{empty_project.id}/build_status", { sha: "sha", status: "pending", ref: 'master' }) } - other_status = described_class.new(empty_project, sha: "123456", status: "failed") + Gitlab::Redis.with do |redis| + redis.mapped_hmset(cache_key, + { sha: 'sha', status: 'pending', ref: 'master' }) + end + + other_status = described_class.new(empty_project, + pipeline_info: { + sha: "123456", status: "failed" + }) other_status.store_in_cache_if_needed - sha, status, ref = Gitlab::Redis.with { |redis| redis.hmget("projects/#{empty_project.id}/build_status", :sha, :status, :ref) } + sha, status, ref = Gitlab::Redis.with { |redis| redis.hmget("projects/#{empty_project.id}/pipeline_status", :sha, :status, :ref) } expect(sha).to be_nil expect(status).to be_nil @@ -157,9 +266,13 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus do describe "with a status in redis", :redis do let(:status) { 'success' } let(:sha) { '424d1b73bc0d3cb726eb7dc4ce17a4d48552f8c6' } + let(:ref) { 'master' } before do - Gitlab::Redis.with { |redis| redis.mapped_hmset("projects/#{project.id}/build_status", { sha: sha, status: status }) } + Gitlab::Redis.with do |redis| + redis.mapped_hmset(cache_key, + { sha: sha, status: status, ref: ref }) + end end describe '#load_from_cache' do @@ -168,6 +281,7 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus do expect(pipeline_status.sha).to eq(sha) expect(pipeline_status.status).to eq(status) + expect(pipeline_status.ref).to eq(ref) end end @@ -181,7 +295,7 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus do it 'deletes values from redis' do pipeline_status.delete_from_cache - key_exists = Gitlab::Redis.with { |redis| redis.exists("projects/#{project.id}/build_status") } + key_exists = Gitlab::Redis.with { |redis| redis.exists(cache_key) } expect(key_exists).to be_falsy end -- cgit v1.2.1 From 5234d181804d35dabe2e6d4d997e25653e6944fd Mon Sep 17 00:00:00 2001 From: Marcia Ramos Date: Wed, 26 Apr 2017 13:13:11 +0000 Subject: Reorg CE Docs landing page --- doc/README.md | 247 ++++++++++++++++++++++++++++++----------- doc/migrate_ci_to_ce/README.md | 58 +++++----- doc/topics/git/index.md | 4 + 3 files changed, 214 insertions(+), 95 deletions(-) diff --git a/doc/README.md b/doc/README.md index fb393aa09a1..6406040da4b 100644 --- a/doc/README.md +++ b/doc/README.md @@ -1,80 +1,195 @@ # GitLab Community Edition -All technical content published by GitLab lives in the documentation, including: +[GitLab](https://about.gitlab.com/) is a Git-based fully featured platform +for software development. -- **General Documentation** - - [User docs](#user-documentation): general documentation dedicated to regular users of GitLab - - [Admin docs](#administrator-documentation): general documentation dedicated to administrators of GitLab instances - - [Contributor docs](#contributor-documentation): general documentation on how to develop and contribute to GitLab -- [GitLab University](university/README.md): guides to learn Git and GitLab - through courses and videos. +**GitLab Community Edition (CE)** is an opensource product, self-hosted, free to use. +All [GitLab products](https://about.gitlab.com/products/) contain the features +available in GitLab CE. Premium features are available in +[GitLab Enterprise Edition (EE)](https://about.gitlab.com/gitlab-ee/). -## User documentation +---- -- [Account Security](user/profile/account/two_factor_authentication.md) Securing your account via two-factor authentication, etc. -- [API](api/README.md) Automate GitLab via a simple and powerful API. -- [CI/CD](ci/README.md) GitLab Continuous Integration (CI) and Continuous Delivery (CD) getting started, `.gitlab-ci.yml` options, and examples. -- [Container Registry](user/project/container_registry.md) Learn how to use GitLab Container Registry. -- [Discussions](user/discussions/index.md) Threads, comments, and resolvable discussions in issues, commits, and merge requests. -- [Git Attributes](user/project/git_attributes.md) Managing Git attributes using a `.gitattributes` file. -- [Git cheatsheet](https://gitlab.com/gitlab-com/marketing/raw/master/design/print/git-cheatsheet/print-pdf/git-cheatsheet.pdf) Download a PDF describing the most used Git operations. -- [GitLab as OAuth2 authentication service provider](integration/oauth_provider.md). It allows you to login to other applications from GitLab. -- [GitLab basics](gitlab-basics/README.md) Find step by step how to start working on your commandline and on GitLab. -- [GitLab Pages](user/project/pages/index.md) Using GitLab Pages. +Shortcuts to GitLab's most visited docs: + +| [GitLab CI](ci/README.md) | Other | +| :----- | :----- | +| [Quick start guide](ci/quick_start/README.md) | [API](api/README.md) | +| [Configuring `.gitlab-ci.yml`](ci/yaml/README.md) | [SSH authentication](ssh/README.md) | +| [Using Docker images](ci/docker/using_docker_images.md) | [GitLab Pages](user/project/pages/index.md) | + +## Getting started with GitLab + +- [GitLab Basics](gitlab-basics/README.md): Start working on your command line and on GitLab. +- [GitLab Workflow](workflow/README.md): Enhance your workflow with the best of GitLab Workflow. + - See also [GitLab Workflow - an overview](https://about.gitlab.com/2016/10/25/gitlab-workflow-an-overview/). +- [GitLab Markdown](user/markdown.md): GitLab's advanced formatting system (GitLab Flavored Markdown). +- [GitLab Slash Commands](user/project/slash_commands.md): Textual shortcuts for common actions on issues or merge requests that are usually done by clicking buttons or dropdowns in GitLab's UI. + +### User account + +- [Authentication](topics/authentication/index.md): Account security with two-factor authentication, setup your ssh keys and deploy keys for secure access to your projects. +- [Profile settings](profile/README.md): Manage your profile settings, two factor authentication and more. +- [User permissions](user/permissions.md): Learn what each role in a project (external/guest/reporter/developer/master/owner) can do. + +### Projects and groups + +- [Create a project](gitlab-basics/create-project.md) +- [Fork a project](gitlab-basics/fork-project.md) - [Importing and exporting projects between instances](user/project/settings/import_export.md). -- [Importing to GitLab](workflow/importing/README.md) Import your projects from GitHub, Bitbucket, GitLab.com, FogBugz and SVN into GitLab. -- [Markdown](user/markdown.md) GitLab's advanced formatting system. -- [Migrating from SVN](workflow/importing/migrating_from_svn.md) Convert a SVN repository to Git and GitLab. -- [Permissions](user/permissions.md) Learn what each role in a project (external/guest/reporter/developer/master/owner) can do. -- [Profile Settings](profile/README.md) -- [Project Services](user/project/integrations/project_services.md) Integrate a project with external services, such as CI and chat. -- [Public access](public_access/public_access.md) Learn how you can allow public and internal access to projects. +- [Project access](public_access/public_access.md): Setting up your project's visibility to public, internal, or private. +- [Groups](workflow/groups.md): Organize your projects in groups. + - [Create a group](gitlab-basics/create-group.md) + - [GitLab Subgroups](user/group/subgroups/index.md) - [Search through GitLab](user/search/index.md): Search for issues, merge requests, projects, groups, todos, and issues in Issue Boards. -- [Snippets](user/snippets.md) Snippets allow you to create little bits of code. -- [SSH](ssh/README.md) Setup your ssh keys and deploy keys for secure access to your projects. -- [Webhooks](user/project/integrations/webhooks.md) Let GitLab notify you when new code has been pushed to your project. -- [Workflow](workflow/README.md) Using GitLab functionality and importing projects from GitHub and SVN. + +### Repository + +Manage files and branches from the UI (user interface): + +- Files + - [Create a file](user/project/repository/web_editor.md#create-a-file) + - [Upload a file](user/project/repository/web_editor.md#upload-a-file) + - [File templates](user/project/repository/web_editor.md#template-dropdowns) + - [Create a directory](user/project/repository/web_editor.md#create-a-directory) + - [Start a merge request](user/project/repository/web_editor.md#tips) (when committing via UI) +- Branches + - [Create a branch](user/project/repository/web_editor.md#create-a-new-branch) + - [Protected branches](user/project/protected_branches.md#protected-branches) + +### Issues and Merge Requests (MRs) + +- [Discussions](user/discussions/index.md) Threads, comments, and resolvable discussions in issues, commits, and merge requests. +- Issues + - [Create an issue](gitlab-basics/create-issue.md#how-to-create-an-issue-in-gitlab) + - [Confidential Issues](user/project/issues/confidential_issues.md) + - [Automatic issue closing](user/project/issues/automatic_issue_closing.md) + - [Issue Boards](user/project/issue_board.md) +- [Issues and merge requests templates](user/project/description_templates.md): Create templates for submitting new issues and merge requests. +- [Labels](user/project/labels.md): Categorize your issues or merge requests based on descriptive titles. +- [Merge Requests](user/project/merge_requests/index.md) + - [Work In Progress Merge Requests](user/project/merge_requests/work_in_progress_merge_requests.md) + - [Merge Request discussion resolution](user/discussions/index.md#moving-a-single-discussion-to-a-new-issue): Resolve discussions, move discussions in a merge request to an issue, only allow merge requests to be merged if all discussions are resolved. + - [Checkout merge requests locally](user/project/merge_requests/index.md#checkout-merge-requests-locally) + - [Cherry-pick](user/project/merge_requests/cherry_pick_changes.md) +- [Milestones](user/project/milestones/index.md): Organize issues and merge requests into a cohesive group, optionally setting a due date. +- [Todos](workflow/todos.md): A chronological list of to-dos that are waiting for your input, all in a simple dashboard. + +### Git and GitLab + +- [Git](topics/git/index.md): Getting started with Git, branching strategies, Git LFS, advanced use. +- [Git cheatsheet](https://gitlab.com/gitlab-com/marketing/raw/master/design/print/git-cheatsheet/print-pdf/git-cheatsheet.pdf): Download a PDF describing the most used Git operations. +- [GitLab Flow](workflow/gitlab_flow.md): explore the best of Git with the GitLab Flow strategy. + +### Migrate and import your projects from other platforms + +- [Importing to GitLab](workflow/importing/README.md): Import your projects from GitHub, Bitbucket, GitLab.com, FogBugz and SVN into GitLab. +- [Migrating from SVN](workflow/importing/migrating_from_svn.md): Convert a SVN repository to Git and GitLab. + +## GitLab's superpowers + +Take a step ahead and dive into GitLab's advanced features. + +- [GitLab Pages](user/project/pages/index.md): Build, test, and deploy your static website with GitLab Pages. +- [Snippets](user/snippets.md): Snippets allow you to create little bits of code. +- [Wikis](workflow/project_features.md#wiki): Enhance your repository documentation with built-in wikis. + +### Continuous Integration, Delivery, and Deployment + +- [GitLab CI](ci/README.md): Explore the features and capabilities of Continuous Integration, Continuous Delivery, and Continuous Deployment with GitLab. + - [Auto Deploy](ci/autodeploy/index.md): Configure GitLab CI for the deployment of your application. + - [Review Apps](ci/review_apps/index.md): Preview changes to your app right from a merge request. +- [GitLab Cycle Analytics](user/project/cycle_analytics.md): Cycle Analytics measures the time it takes to go from an [idea to production](https://about.gitlab.com/2016/08/05/continuous-integration-delivery-and-deployment-with-gitlab/#from-idea-to-production-with-gitlab) for each project you have. +- [GitLab Container Registry](user/project/container_registry.md): Learn how to use GitLab's built-in Container Registry. + +### Automation + +- [API](api/README.md): Automate GitLab via a simple and powerful API. +- [GitLab Webhooks](user/project/integrations/webhooks.md): Let GitLab notify you when new code has been pushed to your project. + +### Integrations + +- [Project Services](user/project/integrations/project_services.md): Integrate a project with external services, such as CI and chat. +- [GitLab Integration](integration/README.md): Integrate with multiple third-party services with GitLab to allow external issue trackers and external authentication. + +---- ## Administrator documentation -- [Access restrictions](user/admin_area/settings/visibility_and_access_controls.md#enabled-git-access-protocols) Define which Git access protocols can be used to talk to GitLab -- [Authentication/Authorization](administration/auth/README.md) Configure external authentication with LDAP, SAML, CAS and additional Omniauth providers. -- [Container Registry](administration/container_registry.md) Configure Docker Registry with GitLab. -- [Custom Git hooks](administration/custom_hooks.md) Custom Git hooks (on the filesystem) for when webhooks aren't enough. -- [Debugging Tips](administration/troubleshooting/debug.md) Tips to debug problems when things go wrong -- [Environment Variables](administration/environment_variables.md) to configure GitLab. -- [Git LFS configuration](workflow/lfs/lfs_administration.md) -- [GitLab Pages configuration](administration/pages/index.md) Configure GitLab Pages. -- [GitLab performance monitoring with InfluxDB](administration/monitoring/performance/introduction.md) Configure GitLab and InfluxDB for measuring performance metrics. -- [GitLab performance monitoring with Prometheus](administration/monitoring/prometheus/index.md) Configure GitLab and Prometheus for measuring performance metrics. -- [Header logo](customization/branded_page_and_email_header.md) Change the logo on the overall page and email header. -- [High Availability](administration/high_availability/README.md) Configure multiple servers for scaling or high availability. -- [Housekeeping](administration/housekeeping.md) Keep your Git repository tidy and fast. -- [Install](install/README.md) Requirements, directory structures and installation from source. -- [Integration](integration/README.md) How to integrate with systems such as JIRA, Redmine, Twitter. -- [Issue closing pattern](administration/issue_closing_pattern.md) Customize how to close an issue from commit messages. -- [Koding](administration/integration/koding.md) Set up Koding to use with GitLab. -- [Libravatar](customization/libravatar.md) Use Libravatar instead of Gravatar for user avatars. -- [Log system](administration/logs.md) Log system. -- [Migrate GitLab CI to CE/EE](migrate_ci_to_ce/README.md) Follow this guide to migrate your existing GitLab CI data to GitLab CE/EE. -- [Monitoring uptime](user/admin_area/monitoring/health_check.md) Check the server status using the health check endpoint. -- [Operations](administration/operations.md) Keeping GitLab up and running. -- [Polling](administration/polling.md) Configure how often the GitLab UI polls for updates -- [Raketasks](raketasks/README.md) Backups, maintenance, automatic webhook setup and the importing of projects. -- [Reply by email](administration/reply_by_email.md) Allow users to comment on issues and merge requests by replying to notification emails. -- [Repository checks](administration/repository_checks.md) Periodic Git repository checks. -- [Repository storage paths](administration/repository_storage_paths.md) Manage the paths used to store repositories. -- [Request Profiling](administration/monitoring/performance/request_profiling.md) Get a detailed profile on slow requests. -- [Restart GitLab](administration/restart_gitlab.md) Learn how to restart GitLab and its components. -- [Security](security/README.md) Learn what you can do to further secure your GitLab instance. -- [Sidekiq Troubleshooting](administration/troubleshooting/sidekiq.md) Debug when Sidekiq appears hung and is not processing jobs. -- [System hooks](system_hooks/system_hooks.md) Notifications when users, projects and keys are changed. -- [Update](update/README.md) Update guides to upgrade your installation. +Learn how to administer your GitLab instance. Regular users don't +have access to GitLab administration tools and settings. + +### Install, update, upgrade, migrate + +- [Install](install/README.md): Requirements, directory structures and installation from source. +- [Mattermost](https://docs.gitlab.com/omnibus/gitlab-mattermost/): Integrate [Mattermost](https://about.mattermost.com/) with your GitLab installation. +- [Migrate GitLab CI to CE/EE](migrate_ci_to_ce/README.md): If you have an old GitLab installation (older than 8.0), follow this guide to migrate your existing GitLab CI data to GitLab CE/EE. +- [Restart GitLab](administration/restart_gitlab.md): Learn how to restart GitLab and its components. +- [Update](update/README.md): Update guides to upgrade your installation. + +### User permissions + +- [Access restrictions](user/admin_area/settings/visibility_and_access_controls.md#enabled-git-access-protocols): Define which Git access protocols can be used to talk to GitLab +- [Authentication/Authorization](topics/authentication/index.md#gitlab-administrators): Enforce 2FA, configure external authentication with LDAP, SAML, CAS and additional Omniauth providers. + +### GitLab admins' superpowers + +- [Container Registry](administration/container_registry.md): Configure Docker Registry with GitLab. +- [Custom Git hooks](administration/custom_hooks.md): Custom Git hooks (on the filesystem) for when webhooks aren't enough. +- [Git LFS configuration](workflow/lfs/lfs_administration.md): Learn how to use LFS under GitLab. +- [GitLab Pages configuration](administration/pages/index.md): Configure GitLab Pages. +- [High Availability](administration/high_availability/README.md): Configure multiple servers for scaling or high availability. - [User cohorts](user/admin_area/user_cohorts.md) View user activity over time. -- [Web terminals](administration/integration/terminal.md) Provide terminal access to environments from within GitLab. -- [Welcome message](customization/welcome_message.md) Add a custom welcome message to the sign-in page. +- [Web terminals](administration/integration/terminal.md): Provide terminal access to environments from within GitLab. +- GitLab CI + - [CI admin settings](user/admin_area/settings/continuous_integration.md): Define max artifacts size and expiration time. + +### Integrations + +- [Integrations](integration/README.md): How to integrate with systems such as JIRA, Redmine, Twitter. +- [Koding](administration/integration/koding.md): Set up Koding to use with GitLab. +- [Mattermost](user/project/integrations/mattermost.md): Set up GitLab with Mattermost. + +### Monitoring + +- [GitLab performance monitoring with InfluxDB](administration/monitoring/performance/introduction.md): Configure GitLab and InfluxDB for measuring performance metrics. +- [GitLab performance monitoring with Prometheus](administration/monitoring/prometheus/index.md): Configure GitLab and Prometheus for measuring performance metrics. +- [Monitoring uptime](user/admin_area/monitoring/health_check.md): Check the server status using the health check endpoint. + +### Performance + +- [Housekeeping](administration/housekeeping.md): Keep your Git repository tidy and fast. +- [Operations](administration/operations.md): Keeping GitLab up and running. +- [Polling](administration/polling.md): Configure how often the GitLab UI polls for updates. +- [Request Profiling](administration/monitoring/performance/request_profiling.md): Get a detailed profile on slow requests. + +### Customization + +- [Adjust your instance's timezone](workflow/timezone.md): Customize the default time zone of GitLab. +- [Environment variables](administration/environment_variables.md): Supported environment variables that can be used to override their defaults values in order to configure GitLab. +- [Header logo](customization/branded_page_and_email_header.md): Change the logo on the overall page and email header. +- [Issue closing pattern](administration/issue_closing_pattern.md): Customize how to close an issue from commit messages. +- [Libravatar](customization/libravatar.md): Use Libravatar instead of Gravatar for user avatars. +- [Welcome message](customization/welcome_message.md): Add a custom welcome message to the sign-in page. + +### Admin tools + +- [Raketasks](raketasks/README.md): Backups, maintenance, automatic webhook setup and the importing of projects. + - [Backup and restore](raketasks/backup_restore.md): Backup and restore your GitLab instance. +- [Reply by email](administration/reply_by_email.md): Allow users to comment on issues and merge requests by replying to notification emails. +- [Repository checks](administration/repository_checks.md): Periodic Git repository checks. +- [Repository storage paths](administration/repository_storage_paths.md): Manage the paths used to store repositories. +- [Security](security/README.md): Learn what you can do to further secure your GitLab instance. +- [System hooks](system_hooks/system_hooks.md): Notifications when users, projects and keys are changed. + +### Troubleshooting + +- [Debugging tips](administration/troubleshooting/debug.md): Tips to debug problems when things go wrong +- [Log system](administration/logs.md): Where to look for logs. +- [Sidekiq Troubleshooting](administration/troubleshooting/sidekiq.md): Debug when Sidekiq appears hung and is not processing jobs. ## Contributor documentation -- [Development](development/README.md) All styleguides and explanations how to contribute. -- [Legal](legal/README.md) Contributor license agreements. +- [Development](development/README.md): All styleguides and explanations how to contribute. +- [Legal](legal/README.md): Contributor license agreements. +- [Writing documentation](development/writing_documentation.md): Contributing to GitLab Docs. diff --git a/doc/migrate_ci_to_ce/README.md b/doc/migrate_ci_to_ce/README.md index 8f9ef054949..2e7782736ff 100644 --- a/doc/migrate_ci_to_ce/README.md +++ b/doc/migrate_ci_to_ce/README.md @@ -1,4 +1,4 @@ -## Migrate GitLab CI to GitLab CE or EE +# Migrate GitLab CI to GitLab CE or EE Beginning with version 8.0 of GitLab Community Edition (CE) and Enterprise Edition (EE), GitLab CI is no longer its own application, but is instead built @@ -12,7 +12,7 @@ is not possible.** We recommend that you read through the entire migration process in this document before beginning. -### Overview +## Overview In this document we assume you have a GitLab server and a GitLab CI server. It does not matter if these are the same machine. @@ -26,7 +26,7 @@ can be online for most of the procedure; the only GitLab downtime (if any) is during the upgrade to 8.0. Your CI service will be offline from the moment you upgrade to 8.0 until you finish the migration procedure. -### Before upgrading +## Before upgrading If you have GitLab CI installed using omnibus-gitlab packages but **you don't want to migrate your existing data**: @@ -38,12 +38,12 @@ run `sudo gitlab-ctl reconfigure` and you can reach CI at `gitlab.example.com/ci If you want to migrate your existing data, continue reading. -#### 0. Updating Omnibus from versions prior to 7.13 +### 0. Updating Omnibus from versions prior to 7.13 If you are updating from older versions you should first update to 7.14 and then to 8.0. Otherwise it's pretty likely that you will encounter problems described in the [Troubleshooting](#troubleshooting). -#### 1. Verify that backups work +### 1. Verify that backups work Make sure that the backup script on both servers can connect to the database. @@ -73,7 +73,7 @@ sudo -u git -H bundle exec rake gitlab:backup:create RAILS_ENV=production SKIP=r If this fails you need to fix it before upgrading to 8.0. Also see https://about.gitlab.com/getting-help/ -#### 2. Check source and target database types +### 2. Check source and target database types Check what databases you use on your GitLab server and your CI server. Look for the 'adapter:' line. If your CI server and your GitLab server use @@ -102,7 +102,7 @@ cd /home/git/gitlab sudo -u git -H bundle exec rake gitlab:env:info RAILS_ENV=production ``` -#### 3. Storage planning +### 3. Storage planning Decide where to store CI build traces on GitLab server. GitLab CI uses files on disk to store CI build traces. The default path for these build @@ -111,34 +111,34 @@ traces is `/var/opt/gitlab/gitlab-ci/builds` (Omnibus) or a special location, or if you are using NFS, you should make sure that you store build traces on the same storage as your Git repositories. -### I. Upgrading +## I. Upgrading From this point on, GitLab CI will be unavailable for your end users. -#### 1. Upgrade GitLab to 8.0 +### 1. Upgrade GitLab to 8.0 First upgrade your GitLab server to version 8.0: https://about.gitlab.com/update/ -#### 2. Disable CI on the GitLab server during the migration +### 2. Disable CI on the GitLab server during the migration After you update, go to the admin panel and temporarily disable CI. As an administrator, go to **Admin Area** -> **Settings**, and under **Continuous Integration** uncheck **Disable to prevent CI usage until rake ci:migrate is run (8.0 only)**. -#### 3. CI settings are now in GitLab +### 3. CI settings are now in GitLab If you want to use custom CI settings (e.g. change where builds are stored), please update `/etc/gitlab/gitlab.rb` (Omnibus) or `/home/git/gitlab/config/gitlab.yml` (Source). -#### 4. Upgrade GitLab CI to 8.0 +### 4. Upgrade GitLab CI to 8.0 Now upgrade GitLab CI to version 8.0. If you are using Omnibus packages, this may have already happened when you upgraded GitLab to 8.0. -#### 5. Disable GitLab CI on the CI server +### 5. Disable GitLab CI on the CI server Disable GitLab CI after upgrading to 8.0. @@ -154,9 +154,9 @@ cd /home/gitlab_ci/gitlab-ci sudo -u gitlab_ci -H bundle exec whenever --clear-crontab RAILS_ENV=production ``` -### II. Moving data +## II. Moving data -#### 1. Database encryption key +### 1. Database encryption key Move the database encryption key from your CI server to your GitLab server. The command below will show you what you need to copy-paste to your @@ -174,7 +174,7 @@ cd /home/gitlab_ci/gitlab-ci sudo -u gitlab_ci -H bundle exec rake backup:show_secrets RAILS_ENV=production ``` -#### 2. SQL data and build traces +### 2. SQL data and build traces Create your final CI data export. If you are converting from MySQL to PostgreSQL, add ` MYSQL_TO_POSTGRESQL=1` to the end of the rake command. When @@ -192,7 +192,7 @@ cd /home/gitlab_ci/gitlab-ci sudo -u gitlab_ci -H bundle exec rake backup:create RAILS_ENV=production ``` -#### 3. Copy data to the GitLab server +### 3. Copy data to the GitLab server If you were running GitLab and GitLab CI on the same server you can skip this step. @@ -209,7 +209,7 @@ ssh -A ci_admin@ci_server.example scp /path/to/12345_gitlab_ci_backup.tar gitlab_admin@gitlab_server.example:~ ``` -#### 4. Move data to the GitLab backups folder +### 4. Move data to the GitLab backups folder Make the CI data archive discoverable for GitLab. We assume below that you store backups in the default path, adjust the command if necessary. @@ -223,7 +223,7 @@ sudo mv /path/to/12345_gitlab_ci_backup.tar /var/opt/gitlab/backups/ sudo mv /path/to/12345_gitlab_ci_backup.tar /home/git/gitlab/tmp/backups/ ``` -#### 5. Import the CI data into GitLab. +### 5. Import the CI data into GitLab. This step will delete any existing CI data on your GitLab server. There should be no CI data yet because you turned CI on the GitLab server off earlier. @@ -239,7 +239,7 @@ cd /home/git/gitlab sudo -u git -H bundle exec rake ci:migrate RAILS_ENV=production ``` -#### 6. Restart GitLab +### 6. Restart GitLab ``` # On your GitLab server: @@ -251,7 +251,7 @@ sudo gitlab-ctl restart sidekiq sudo service gitlab reload ``` -### III. Redirecting traffic +## III. Redirecting traffic If you were running GitLab CI with Omnibus packages and you were using the internal NGINX configuration your CI service should now be available both at @@ -261,7 +261,7 @@ If you installed GitLab CI from source we now need to configure a redirect in NGINX so that existing CI runners can keep using the old CI server address, and so that existing links to your CI server keep working. -#### 1. Update Nginx configuration +### 1. Update Nginx configuration To ensure that your existing CI runners are able to communicate with the migrated installation, and that existing build triggers still work, you'll need @@ -317,22 +317,22 @@ You should also make sure that you can: 1. `curl https://YOUR_GITLAB_SERVER_FQDN/` from your previous GitLab CI server. 1. `curl https://YOUR_CI_SERVER_FQDN/` from your GitLab CE (or EE) server. -#### 2. Check Nginx configuration +### 2. Check Nginx configuration sudo nginx -t -#### 3. Restart Nginx +### 3. Restart Nginx sudo /etc/init.d/nginx restart -#### Restore from backup +### Restore from backup If something went wrong and you need to restore a backup, consult the [Backup restoration](../raketasks/backup_restore.md) guide. -### Troubleshooting +## Troubleshooting -#### show:secrets problem (Omnibus-only) +### show:secrets problem (Omnibus-only) If you see errors like this: ``` Missing `secret_key_base` or `db_key_base` for 'production' environment. The secrets will be generated and stored in `config/secrets.yml` @@ -343,7 +343,7 @@ Errno::EACCES: Permission denied @ rb_sysopen - config/secrets.yml This can happen if you are updating from versions prior to 7.13 straight to 8.0. The fix for this is to update to Omnibus 7.14 first and then update it to 8.0. -#### Permission denied when accessing /var/opt/gitlab/gitlab-ci/builds +### Permission denied when accessing /var/opt/gitlab/gitlab-ci/builds To fix that issue you have to change builds/ folder permission before doing final backup: ``` sudo chown -R gitlab-ci:gitlab-ci /var/opt/gitlab/gitlab-ci/builds @@ -354,7 +354,7 @@ Then before executing `ci:migrate` you need to fix builds folder permission: sudo chown git:git /var/opt/gitlab/gitlab-ci/builds ``` -#### Problems when importing CI database to GitLab +### Problems when importing CI database to GitLab If you were migrating CI database from MySQL to PostgreSQL manually you can see errors during import about missing sequences: ``` ALTER SEQUENCE diff --git a/doc/topics/git/index.md b/doc/topics/git/index.md index b99ba317a43..d13066c9015 100644 --- a/doc/topics/git/index.md +++ b/doc/topics/git/index.md @@ -17,6 +17,10 @@ We've gathered some resources to help you to get the best from Git with GitLab. - [Start using Git on the command line](../../gitlab-basics/start-using-git.md) - [Command Line basic commands](../../gitlab-basics/command-line-commands.md) - [GitLab Git Cheat Sheet (download)](https://gitlab.com/gitlab-com/marketing/raw/master/design/print/git-cheatsheet/print-pdf/git-cheatsheet.pdf) +- Commits + - [Revert a commit](../../user/project/merge_requests/revert_changes.md#reverting-a-commit) + - [Cherry-picking a commit](../../user/project/merge_requests/cherry_pick_changes.md#cherry-picking-a-commit) + - [Squashing commits](../../workflow/gitlab_flow.md#squashing-commits-with-rebase) - **Articles:** - [Git Tips & Tricks](https://about.gitlab.com/2016/12/08/git-tips-and-tricks/) - [Eight Tips to help you work better with Git](https://about.gitlab.com/2015/02/19/8-tips-to-help-you-work-better-with-git/) -- cgit v1.2.1 From 4a924a5e4edd009b1fe9df57e04d70d4e3c077b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Wed, 26 Apr 2017 16:27:41 +0200 Subject: Fix Rubocop CyclomaticComplexity offense MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- app/policies/project_policy.rb | 44 +++++++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/app/policies/project_policy.rb b/app/policies/project_policy.rb index f8594e29547..5baac9ebe4b 100644 --- a/app/policies/project_policy.rb +++ b/app/policies/project_policy.rb @@ -2,20 +2,13 @@ class ProjectPolicy < BasePolicy def rules team_access!(user) - owner = project.owner == user || - (project.group && project.group.has_owner?(user)) - - owner_access! if user.admin? || owner - team_member_owner_access! if owner + owner_access! if user.admin? || owner? + team_member_owner_access! if owner? if project.public? || (project.internal? && !user.external?) guest_access! public_access! - - if project.request_access_enabled && - !(owner || user.admin? || project.team.member?(user) || project_group_member?(user)) - can! :request_access - end + can! :request_access if access_requestable? end archived_access! if project.archived? @@ -27,6 +20,13 @@ class ProjectPolicy < BasePolicy @subject end + def owner? + return @owner if defined?(@owner) + + @owner = project.owner == user || + (project.group && project.group.has_owner?(user)) + end + def guest_access! can! :read_project can! :read_board @@ -226,14 +226,6 @@ class ProjectPolicy < BasePolicy disabled_features! end - def project_group_member?(user) - project.group && - ( - project.group.members_with_parents.exists?(user_id: user.id) || - project.group.requesters.exists?(user_id: user.id) - ) - end - def block_issues_abilities unless project.feature_available?(:issues, user) cannot! :read_issue if project.default_issues_tracker? @@ -254,6 +246,22 @@ class ProjectPolicy < BasePolicy private + def project_group_member?(user) + project.group && + ( + project.group.members_with_parents.exists?(user_id: user.id) || + project.group.requesters.exists?(user_id: user.id) + ) + end + + def access_requestable? + project.request_access_enabled && + !owner? && + !user.admin? && + !project.team.member?(user) && + !project_group_member?(user) + end + # A base set of abilities for read-only users, which # is then augmented as necessary for anonymous and other # read-only users. -- cgit v1.2.1 From 6f5b579f94c4bb3590c56e409434a15b6401afac Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Wed, 26 Apr 2017 14:46:26 +0000 Subject: Use rspec-set to speed up examples --- Gemfile | 1 + Gemfile.lock | 2 ++ doc/development/testing.md | 15 ++++++++++ spec/finders/issues_finder_spec.rb | 24 ++++++++-------- spec/requests/api/issues_spec.rb | 49 ++++++++++++++++++-------------- spec/requests/api/merge_requests_spec.rb | 1 + 6 files changed, 58 insertions(+), 34 deletions(-) diff --git a/Gemfile b/Gemfile index 58af6c51b77..decbcfe6375 100644 --- a/Gemfile +++ b/Gemfile @@ -293,6 +293,7 @@ group :development, :test do gem 'spinach-rails', '~> 0.2.1' gem 'spinach-rerun-reporter', '~> 0.0.2' gem 'rspec_profiling', '~> 0.0.5' + gem 'rspec-set', '~> 0.1.3' # Prevent occasions where minitest is not bundled in packaged versions of ruby (see #3826) gem 'minitest', '~> 5.7.0' diff --git a/Gemfile.lock b/Gemfile.lock index 83c9bb57c3b..fb11f590110 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -659,6 +659,7 @@ GEM rspec-support (~> 3.5.0) rspec-retry (0.4.5) rspec-core + rspec-set (0.1.3) rspec-support (3.5.0) rspec_profiling (0.0.5) activerecord @@ -989,6 +990,7 @@ DEPENDENCIES rqrcode-rails3 (~> 0.1.7) rspec-rails (~> 3.5.0) rspec-retry (~> 0.4.5) + rspec-set (~> 0.1.3) rspec_profiling (~> 0.0.5) rubocop (~> 0.47.1) rubocop-rspec (~> 1.15.0) diff --git a/doc/development/testing.md b/doc/development/testing.md index 2c7154f1dea..9b0b9808827 100644 --- a/doc/development/testing.md +++ b/doc/development/testing.md @@ -202,6 +202,7 @@ Please consult the [dedicated "Frontend testing" guide](./fe_guide/testing.md). - Try to follow the [Four-Phase Test][four-phase-test] pattern, using newlines to separate phases. - Try to use `Gitlab.config.gitlab.host` rather than hard coding `'localhost'` +- On `before` and `after` hooks, prefer it scoped to `:context` over `:all` [four-phase-test]: https://robots.thoughtbot.com/four-phase-test @@ -225,6 +226,20 @@ so we need to set some guidelines for their use going forward: [lets-not]: https://robots.thoughtbot.com/lets-not +#### `set` variables + +In some cases there is no need to recreate the same object for tests again for +each example. For example, a project is needed to test issues on the same +project, one project will do for the entire file. This can be achieved by using +`set` in the same way you would use `let`. + +`rspec-set` only works on ActiveRecord objects, and before new examples it +reloads or recreates the model, _only_ if needed. That is, when you changed +properties or destroyed the object. + +There is one gotcha; you can't reference a model defined in a `let` block in a +`set` block. + ### Time-sensitive tests [Timecop](https://github.com/travisjeffery/timecop) is available in our diff --git a/spec/finders/issues_finder_spec.rb b/spec/finders/issues_finder_spec.rb index 231fd85c464..a1ae1d746af 100644 --- a/spec/finders/issues_finder_spec.rb +++ b/spec/finders/issues_finder_spec.rb @@ -1,24 +1,24 @@ require 'spec_helper' describe IssuesFinder do - let(:user) { create(:user) } - let(:user2) { create(:user) } - let(:project1) { create(:empty_project) } - let(:project2) { create(:empty_project) } - let(:milestone) { create(:milestone, project: project1) } - let(:label) { create(:label, project: project2) } - let(:issue1) { create(:issue, author: user, assignee: user, project: project1, milestone: milestone, title: 'gitlab') } - let(:issue2) { create(:issue, author: user, assignee: user, project: project2, description: 'gitlab') } - let(:issue3) { create(:issue, author: user2, assignee: user2, project: project2, title: 'tanuki', description: 'tanuki') } + set(:user) { create(:user) } + set(:user2) { create(:user) } + set(:project1) { create(:empty_project) } + set(:project2) { create(:empty_project) } + set(:milestone) { create(:milestone, project: project1) } + set(:label) { create(:label, project: project2) } + set(:issue1) { create(:issue, author: user, assignee: user, project: project1, milestone: milestone, title: 'gitlab') } + set(:issue2) { create(:issue, author: user, assignee: user, project: project2, description: 'gitlab') } + set(:issue3) { create(:issue, author: user2, assignee: user2, project: project2, title: 'tanuki', description: 'tanuki') } describe '#execute' do - let(:closed_issue) { create(:issue, author: user2, assignee: user2, project: project2, state: 'closed') } - let!(:label_link) { create(:label_link, label: label, target: issue2) } + set(:closed_issue) { create(:issue, author: user2, assignee: user2, project: project2, state: 'closed') } + set(:label_link) { create(:label_link, label: label, target: issue2) } let(:search_user) { user } let(:params) { {} } let(:issues) { IssuesFinder.new(search_user, params.reverse_merge(scope: scope, state: 'opened')).execute } - before do + before(:context) do project1.team << [user, :master] project2.team << [user, :developer] project2.team << [user2, :developer] diff --git a/spec/requests/api/issues_spec.rb b/spec/requests/api/issues_spec.rb index 784fd1ff885..3ca13111acb 100644 --- a/spec/requests/api/issues_spec.rb +++ b/spec/requests/api/issues_spec.rb @@ -3,14 +3,17 @@ require 'spec_helper' describe API::Issues do include EmailHelpers - let(:user) { create(:user) } + set(:user) { create(:user) } + set(:project) do + create(:empty_project, :public, creator_id: user.id, namespace: user.namespace) + end + let(:user2) { create(:user) } let(:non_member) { create(:user) } - let(:guest) { create(:user) } - let(:author) { create(:author) } - let(:assignee) { create(:assignee) } + set(:guest) { create(:user) } + set(:author) { create(:author) } + set(:assignee) { create(:assignee) } let(:admin) { create(:user, :admin) } - let!(:project) { create(:empty_project, :public, creator_id: user.id, namespace: user.namespace ) } let(:issue_title) { 'foo' } let(:issue_description) { 'closed' } let!(:closed_issue) do @@ -43,19 +46,19 @@ describe API::Issues do title: issue_title, description: issue_description end - let!(:label) do + set(:label) do create(:label, title: 'label', color: '#FFAABB', project: project) end let!(:label_link) { create(:label_link, label: label, target: issue) } - let!(:milestone) { create(:milestone, title: '1.0.0', project: project) } - let!(:empty_milestone) do + set(:milestone) { create(:milestone, title: '1.0.0', project: project) } + set(:empty_milestone) do create(:milestone, title: '2.0.0', project: project) end let!(:note) { create(:note_on_issue, author: user, project: project, noteable: issue) } let(:no_milestone_title) { URI.escape(Milestone::None.title) } - before do + before(:all) do project.team << [user, :reporter] project.team << [guest, :guest] end @@ -70,6 +73,8 @@ describe API::Issues do end context "when authenticated" do + let(:first_issue) { json_response.first } + it "returns an array of issues" do get api("/issues", user) @@ -79,46 +84,46 @@ describe API::Issues do end it 'returns an array of closed issues' do - get api('/issues?state=closed', user) + get api('/issues', user), state: :closed expect_paginated_array_response(size: 1) - expect(json_response.first['id']).to eq(closed_issue.id) + expect(first_issue['id']).to eq(closed_issue.id) end it 'returns an array of opened issues' do - get api('/issues?state=opened', user) + get api('/issues', user), state: :opened expect_paginated_array_response(size: 1) - expect(json_response.first['id']).to eq(issue.id) + expect(first_issue['id']).to eq(issue.id) end it 'returns an array of all issues' do - get api('/issues?state=all', user) + get api('/issues', user), state: :all expect_paginated_array_response(size: 2) - expect(json_response.first['id']).to eq(issue.id) + expect(first_issue['id']).to eq(issue.id) expect(json_response.second['id']).to eq(closed_issue.id) end it 'returns issues matching given search string for title' do - get api("/issues?search=#{issue.title}", user) + get api("/issues", user), search: issue.title expect_paginated_array_response(size: 1) expect(json_response.first['id']).to eq(issue.id) end it 'returns issues matching given search string for description' do - get api("/issues?search=#{issue.description}", user) + get api("/issues", user), search: issue.description expect_paginated_array_response(size: 1) - expect(json_response.first['id']).to eq(issue.id) + expect(first_issue['id']).to eq(issue.id) end it 'returns an array of labeled issues' do - get api("/issues?labels=#{label.title}", user) + get api("/issues", user), labels: label.title expect_paginated_array_response(size: 1) - expect(json_response.first['labels']).to eq([label.title]) + expect(first_issue['labels']).to eq([label.title]) end it 'returns an array of labeled issues when all labels matches' do @@ -135,13 +140,13 @@ describe API::Issues do end it 'returns an empty array if no issue matches labels' do - get api('/issues?labels=foo,bar', user) + get api('/issues', user), labels: 'foo,bar' expect_paginated_array_response(size: 0) end it 'returns an array of labeled issues matching given state' do - get api("/issues?labels=#{label.title}&state=opened", user) + get api("/issues", user), labels: label.title, state: :opened expect_paginated_array_response(size: 1) expect(json_response.first['labels']).to eq([label.title]) diff --git a/spec/requests/api/merge_requests_spec.rb b/spec/requests/api/merge_requests_spec.rb index 904e049d767..c4bff1647b5 100644 --- a/spec/requests/api/merge_requests_spec.rb +++ b/spec/requests/api/merge_requests_spec.rb @@ -26,6 +26,7 @@ describe API::MergeRequests do context "when unauthenticated" do it "returns authentication error" do get api("/projects/#{project.id}/merge_requests") + expect(response).to have_http_status(401) end end -- cgit v1.2.1 From da9d787ac0ee3a193b70acf6535d9d5722db4f1a Mon Sep 17 00:00:00 2001 From: Douwe Maan Date: Wed, 26 Apr 2017 09:59:48 -0500 Subject: Bump Sidekiq to 5.0.0 --- Gemfile | 2 +- Gemfile.lock | 8 ++++---- changelogs/unreleased/dm-sidekiq-5.yml | 4 ++++ 3 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 changelogs/unreleased/dm-sidekiq-5.yml diff --git a/Gemfile b/Gemfile index decbcfe6375..41c2dcfd76e 100644 --- a/Gemfile +++ b/Gemfile @@ -144,7 +144,7 @@ gem 'after_commit_queue', '~> 1.3.0' gem 'acts-as-taggable-on', '~> 4.0' # Background jobs -gem 'sidekiq', '~> 4.2.7' +gem 'sidekiq', '~> 5.0' gem 'sidekiq-cron', '~> 0.4.4' gem 'redis-namespace', '~> 1.5.2' gem 'sidekiq-limit_fetch', '~> 3.4' diff --git a/Gemfile.lock b/Gemfile.lock index fb11f590110..4c3b2480b54 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -603,7 +603,7 @@ GEM json recursive-open-struct (1.0.0) redcarpet (3.4.0) - redis (3.2.2) + redis (3.3.3) redis-actionpack (5.0.1) actionpack (>= 4.0, < 6) redis-rack (>= 1, < 3) @@ -717,11 +717,11 @@ GEM rack shoulda-matchers (2.8.0) activesupport (>= 3.0.0) - sidekiq (4.2.10) + sidekiq (5.0.0) concurrent-ruby (~> 1.0) connection_pool (~> 2.2, >= 2.2.0) rack-protection (>= 1.5.0) - redis (~> 3.2, >= 3.2.1) + redis (~> 3.3, >= 3.3.3) sidekiq-cron (0.4.4) redis-namespace (>= 1.5.2) rufus-scheduler (>= 2.0.24) @@ -1007,7 +1007,7 @@ DEPENDENCIES settingslogic (~> 2.0.9) sham_rack (~> 1.3.6) shoulda-matchers (~> 2.8.0) - sidekiq (~> 4.2.7) + sidekiq (~> 5.0) sidekiq-cron (~> 0.4.4) sidekiq-limit_fetch (~> 3.4) simplecov (~> 0.14.0) diff --git a/changelogs/unreleased/dm-sidekiq-5.yml b/changelogs/unreleased/dm-sidekiq-5.yml new file mode 100644 index 00000000000..69c94b18929 --- /dev/null +++ b/changelogs/unreleased/dm-sidekiq-5.yml @@ -0,0 +1,4 @@ +--- +title: Bump Sidekiq to 5.0.0 +merge_request: +author: -- cgit v1.2.1 From 6824fcbf1c1a62b008dffe4a25e594f91cf1d4b2 Mon Sep 17 00:00:00 2001 From: Islam Wazery Date: Tue, 11 Apr 2017 12:11:49 +0000 Subject: Update bamboo.md, the troubleshooting section --- doc/user/project/integrations/bamboo.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/user/project/integrations/bamboo.md b/doc/user/project/integrations/bamboo.md index cad4757f287..1e28646bc97 100644 --- a/doc/user/project/integrations/bamboo.md +++ b/doc/user/project/integrations/bamboo.md @@ -51,9 +51,9 @@ service in GitLab. ## Troubleshooting -If builds are not triggered, these are a couple of things to keep in mind. +If builds are not triggered, ensure you entered the right GitLab IP address in +Bamboo under 'Trigger IP addresses'. + +>**Note:** +- Starting with GitLab 8.14.0, builds are triggered on push events. -1. Ensure you entered the right GitLab IP address in Bamboo under 'Trigger - IP addresses'. -1. Remember that GitLab only triggers builds on push events. A commit via the - web interface will not trigger CI currently. -- cgit v1.2.1 From b45b08cbbfeabdd0a7d0f1c03a2584e58ebe684b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Wed, 26 Apr 2017 17:37:01 +0200 Subject: Run `bundle check` after `bundle install` instead of having a dedicated job for it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- .gitlab-ci.yml | 7 ------- scripts/prepare_build.sh | 2 +- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4a16a0aaba0..5ea1d89e659 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -402,13 +402,6 @@ docs:check:links: # Check the internal links - bundle exec nanoc check internal_links -bundler:check: - stage: test - <<: *dedicated-runner - <<: *ruby-static-analysis - script: - - bundle check - bundler:audit: stage: test <<: *ruby-static-analysis diff --git a/scripts/prepare_build.sh b/scripts/prepare_build.sh index de7379425cf..fd173c0ba88 100755 --- a/scripts/prepare_build.sh +++ b/scripts/prepare_build.sh @@ -32,7 +32,7 @@ sed -i 's/localhost/redis/g' config/resque.yml cp config/gitlab.yml.example config/gitlab.yml if [ "$USE_BUNDLE_INSTALL" != "false" ]; then - retry bundle install --clean $BUNDLE_INSTALL_FLAGS + retry bundle install --clean $BUNDLE_INSTALL_FLAGS && bundle check fi # Only install knapsack after bundle install! Otherwise oddly some native -- cgit v1.2.1 From df85a7c00439f7a9636b6f46448336d76df7f7da Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Wed, 26 Apr 2017 15:57:44 +0000 Subject: Submodule Dockerfile templates --- .gitlab-ci.yml | 1 + changelogs/unreleased/zj-dockerfiles.yml | 4 ++++ lib/gitlab/template/dockerfile_template.rb | 4 ++-- lib/tasks/gitlab/update_templates.rake | 8 ++++++-- .../projects/files/dockerfile_dropdown_spec.rb | 6 ++++++ vendor/Dockerfile/CONTRIBUTING.md | 5 +++++ vendor/Dockerfile/HTTPd.Dockerfile | 3 +++ vendor/Dockerfile/LICENSE | 21 +++++++++++++++++++++ vendor/Dockerfile/PHP.Dockerfile | 14 ++++++++++++++ vendor/Dockerfile/Python2.Dockerfile | 11 +++++++++++ vendor/dockerfile/HTTPdDockerfile | 3 --- 11 files changed, 73 insertions(+), 7 deletions(-) create mode 100644 changelogs/unreleased/zj-dockerfiles.yml create mode 100644 vendor/Dockerfile/CONTRIBUTING.md create mode 100644 vendor/Dockerfile/HTTPd.Dockerfile create mode 100644 vendor/Dockerfile/LICENSE create mode 100644 vendor/Dockerfile/PHP.Dockerfile create mode 100644 vendor/Dockerfile/Python2.Dockerfile delete mode 100644 vendor/dockerfile/HTTPdDockerfile diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4a16a0aaba0..230ca698ad0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,6 +11,7 @@ variables: NODE_ENV: "test" SIMPLECOV: "true" GIT_DEPTH: "20" + GIT_SUBMODULE_STRATEGY: "none" PHANTOMJS_VERSION: "2.1.1" GET_SOURCES_ATTEMPTS: "3" KNAPSACK_RSPEC_SUITE_REPORT_PATH: knapsack/${CI_PROJECT_NAME}/rspec_report-master.json diff --git a/changelogs/unreleased/zj-dockerfiles.yml b/changelogs/unreleased/zj-dockerfiles.yml new file mode 100644 index 00000000000..40cb7dcfb76 --- /dev/null +++ b/changelogs/unreleased/zj-dockerfiles.yml @@ -0,0 +1,4 @@ +--- +title: Dockerfiles templates are imported from gitlab.com/gitlab-org/Dockerfile +merge_request: 10663 +author: diff --git a/lib/gitlab/template/dockerfile_template.rb b/lib/gitlab/template/dockerfile_template.rb index d5d3e045a42..20b054b0bd8 100644 --- a/lib/gitlab/template/dockerfile_template.rb +++ b/lib/gitlab/template/dockerfile_template.rb @@ -8,7 +8,7 @@ module Gitlab class << self def extension - 'Dockerfile' + '.Dockerfile' end def categories @@ -18,7 +18,7 @@ module Gitlab end def base_dir - Rails.root.join('vendor/dockerfile') + Rails.root.join('vendor/Dockerfile') end def finder(project = nil) diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake index cb2adc81c9d..1b04e1350ed 100644 --- a/lib/tasks/gitlab/update_templates.rake +++ b/lib/tasks/gitlab/update_templates.rake @@ -5,7 +5,7 @@ namespace :gitlab do end def update(template) - sub_dir = template.repo_url.match(/([a-z-]+)\.git\z/)[1] + sub_dir = template.repo_url.match(/([A-Za-z-]+)\.git\z/)[1] dir = File.join(vendor_directory, sub_dir) unless clone_repository(template.repo_url, dir) @@ -45,7 +45,11 @@ namespace :gitlab do Template.new( "https://gitlab.com/gitlab-org/gitlab-ci-yml.git", /(\.{1,2}|LICENSE|CONTRIBUTING.md|Pages|autodeploy|\.gitlab-ci.yml)\z/ - ) + ), + Template.new( + "https://gitlab.com/gitlab-org/Dockerfile.git", + /(\.{1,2}|LICENSE|CONTRIBUTING.md|\.Dockerfile)\z/ + ), ].freeze def vendor_directory diff --git a/spec/features/projects/files/dockerfile_dropdown_spec.rb b/spec/features/projects/files/dockerfile_dropdown_spec.rb index a7cc98a2059..548131c7cd4 100644 --- a/spec/features/projects/files/dockerfile_dropdown_spec.rb +++ b/spec/features/projects/files/dockerfile_dropdown_spec.rb @@ -1,11 +1,14 @@ require 'spec_helper' +require 'fileutils' feature 'User wants to add a Dockerfile file', feature: true do before do user = create(:user) project = create(:project) project.team << [user, :master] + login_as user + visit namespace_project_new_blob_path(project.namespace, project, 'master', file_name: 'Dockerfile') end @@ -15,11 +18,14 @@ feature 'User wants to add a Dockerfile file', feature: true do scenario 'user can pick a Dockerfile file from the dropdown', js: true do find('.js-dockerfile-selector').click + wait_for_ajax + within '.dockerfile-selector' do find('.dropdown-input-field').set('HTTPd') find('.dropdown-content li', text: 'HTTPd').click end + wait_for_ajax expect(page).to have_css('.dockerfile-selector .dropdown-toggle-text', text: 'HTTPd') diff --git a/vendor/Dockerfile/CONTRIBUTING.md b/vendor/Dockerfile/CONTRIBUTING.md new file mode 100644 index 00000000000..91b92eafa1b --- /dev/null +++ b/vendor/Dockerfile/CONTRIBUTING.md @@ -0,0 +1,5 @@ +The canonical repository for `Dockerfile` templates is +https://gitlab.com/gitlab-org/Dockerfile. + +GitLab only mirrors the templates. Please submit your merge requests to +https://gitlab.com/gitlab-org/Dockerfile. diff --git a/vendor/Dockerfile/HTTPd.Dockerfile b/vendor/Dockerfile/HTTPd.Dockerfile new file mode 100644 index 00000000000..2f05427323c --- /dev/null +++ b/vendor/Dockerfile/HTTPd.Dockerfile @@ -0,0 +1,3 @@ +FROM httpd:alpine + +COPY ./ /usr/local/apache2/htdocs/ diff --git a/vendor/Dockerfile/LICENSE b/vendor/Dockerfile/LICENSE new file mode 100644 index 00000000000..d6c93c6fcf7 --- /dev/null +++ b/vendor/Dockerfile/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016-2017 GitLab.org + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/Dockerfile/PHP.Dockerfile b/vendor/Dockerfile/PHP.Dockerfile new file mode 100644 index 00000000000..6b098efcd85 --- /dev/null +++ b/vendor/Dockerfile/PHP.Dockerfile @@ -0,0 +1,14 @@ +FROM php:7.0-apache + +# Customize any core extensions here +#RUN apt-get update && apt-get install -y \ +# libfreetype6-dev \ +# libjpeg62-turbo-dev \ +# libmcrypt-dev \ +# libpng12-dev \ +# && docker-php-ext-install -j$(nproc) iconv mcrypt \ +# && docker-php-ext-configure gd --with-freetype-dir=/usr/include/ --with-jpeg-dir=/usr/include/ \ +# && docker-php-ext-install -j$(nproc) gd + +COPY config/php.ini /usr/local/etc/php/ +COPY src/ /var/www/html/ diff --git a/vendor/Dockerfile/Python2.Dockerfile b/vendor/Dockerfile/Python2.Dockerfile new file mode 100644 index 00000000000..c9a03584d40 --- /dev/null +++ b/vendor/Dockerfile/Python2.Dockerfile @@ -0,0 +1,11 @@ +FROM python:2.7 + +RUN mkdir -p /usr/src/app +WORKDIR /usr/src/app + +COPY requirements.txt /usr/src/app/ +RUN pip install --no-cache-dir -r requirements.txt + +COPY . /usr/src/app + +CMD ["python", "app.py"] diff --git a/vendor/dockerfile/HTTPdDockerfile b/vendor/dockerfile/HTTPdDockerfile deleted file mode 100644 index 2f05427323c..00000000000 --- a/vendor/dockerfile/HTTPdDockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM httpd:alpine - -COPY ./ /usr/local/apache2/htdocs/ -- cgit v1.2.1 From c7289d3e15791d73e421ecf4b239f005d2670ac6 Mon Sep 17 00:00:00 2001 From: Job van der Voort Date: Wed, 26 Apr 2017 17:24:09 +0100 Subject: correct documentation on opt-out-ness of usage ping --- doc/user/admin_area/settings/usage_statistics.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/user/admin_area/settings/usage_statistics.md b/doc/user/admin_area/settings/usage_statistics.md index c3f3179d99e..733e70ca9bf 100644 --- a/doc/user/admin_area/settings/usage_statistics.md +++ b/doc/user/admin_area/settings/usage_statistics.md @@ -3,7 +3,7 @@ GitLab Inc. will periodically collect information about your instance in order to perform various actions. -All statistics are opt-in and you can always disable them from the admin panel. +All statistics are opt-out, you can disable them from the admin panel. ## Version check -- cgit v1.2.1 From 3bce12957037173011aec132acd1913fc702d48a Mon Sep 17 00:00:00 2001 From: Achilleas Pipinellis Date: Wed, 26 Apr 2017 14:08:35 +0200 Subject: Fix failing spec http://rubular.com/r/op7TI9UX6K --- spec/tasks/gitlab/backup_rake_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/tasks/gitlab/backup_rake_spec.rb b/spec/tasks/gitlab/backup_rake_spec.rb index daea0c6bb37..0a4a6ed8145 100644 --- a/spec/tasks/gitlab/backup_rake_spec.rb +++ b/spec/tasks/gitlab/backup_rake_spec.rb @@ -350,7 +350,7 @@ describe 'gitlab:app namespace rake task' do end it 'name has human readable time' do - expect(@backup_tar).to match(/\d+_\d{4}_\d{2}_\d{2}_gitlab_backup.tar$/) + expect(@backup_tar).to match(/\d+_\d{4}_\d{2}_\d{2}_\d+\.\d+\.\d+(-pre)?_gitlab_backup.tar$/) end end end # gitlab:app namespace -- cgit v1.2.1 From 1ac15de9f03509f7fd860faccbf49f57c760f546 Mon Sep 17 00:00:00 2001 From: Filipa Lacerda Date: Wed, 26 Apr 2017 16:37:54 +0000 Subject: Resolve "Jobs dropdown in mini graph should close when we receive an error" --- .../javascripts/mini_pipeline_graph_dropdown.js | 7 +- .../javascripts/pipelines/components/stage.js | 49 ++++++---- app/assets/stylesheets/framework/dropdowns.scss | 1 - .../mini_pipeline_graph_dropdown_spec.js | 106 ++++++++++++--------- spec/javascripts/pipelines/stage_spec.js | 15 +++ 5 files changed, 114 insertions(+), 64 deletions(-) diff --git a/app/assets/javascripts/mini_pipeline_graph_dropdown.js b/app/assets/javascripts/mini_pipeline_graph_dropdown.js index 9c58c465001..64c1447f427 100644 --- a/app/assets/javascripts/mini_pipeline_graph_dropdown.js +++ b/app/assets/javascripts/mini_pipeline_graph_dropdown.js @@ -28,7 +28,9 @@ export default class MiniPipelineGraph { * All dropdown events are fired at the .dropdown-menu's parent element. */ bindEvents() { - $(document).off('shown.bs.dropdown', this.container).on('shown.bs.dropdown', this.container, this.getBuildsList); + $(document) + .off('shown.bs.dropdown', this.container) + .on('shown.bs.dropdown', this.container, this.getBuildsList); } /** @@ -91,6 +93,9 @@ export default class MiniPipelineGraph { }, error: () => { this.toggleLoading(button); + if ($(button).parent().hasClass('open')) { + $(button).dropdown('toggle'); + } new Flash('An error occurred while fetching the builds.', 'alert'); }, }); diff --git a/app/assets/javascripts/pipelines/components/stage.js b/app/assets/javascripts/pipelines/components/stage.js index b8cc3630611..203485f2990 100644 --- a/app/assets/javascripts/pipelines/components/stage.js +++ b/app/assets/javascripts/pipelines/components/stage.js @@ -2,13 +2,6 @@ import StatusIconEntityMap from '../../ci_status_icons'; export default { - data() { - return { - builds: '', - spinner: '', - }; - }, - props: { stage: { type: Object, @@ -16,6 +9,13 @@ export default { }, }, + data() { + return { + builds: '', + spinner: '', + }; + }, + updated() { if (this.builds) { this.stopDropdownClickPropagation(); @@ -31,7 +31,13 @@ export default { return this.$http.get(this.stage.dropdown_path) .then((response) => { this.builds = JSON.parse(response.body).html; - }, () => { + }) + .catch(() => { + // If dropdown is opened we'll close it. + if (this.$el.classList.contains('open')) { + $(this.$refs.dropdown).dropdown('toggle'); + } + const flash = new Flash('Something went wrong on our end.'); return flash; }); @@ -46,9 +52,10 @@ export default { * target the click event of this component. */ stopDropdownClickPropagation() { - $(this.$el.querySelectorAll('.js-builds-dropdown-list a.mini-pipeline-graph-dropdown-item')).on('click', (e) => { - e.stopPropagation(); - }); + $(this.$el.querySelectorAll('.js-builds-dropdown-list a.mini-pipeline-graph-dropdown-item')) + .on('click', (e) => { + e.stopPropagation(); + }); }, }, computed: { @@ -81,12 +88,22 @@ export default { data-placement="top" data-toggle="dropdown" type="button" - :aria-label="stage.title"> - - + :aria-label="stage.title" + ref="dropdown"> + +