SERVER-24699: make all repository publishing synchronous

2 files changed, 351 insertions, 17 deletions

diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 05b6347c4a7..306fe85edb8 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -221,6 +221,10 @@ post:
       script: |
         ${killall_mci|pkill -9 mongod; pkill -9 mongos; pkill -9 mongo; pkill -9 bsondump; pkill -9 mongoimport; pkill -9 mongoexport; pkill -9 mongodump; pkill -9 mongorestore; pkill -9 mongostat; pkill -9 mongofiles; pkill -9 mongooplog; pkill -9 mongotop; pkill -9 mongobridge; pkill -9 mongod-2.6; pkill -9 mongos-2.6; pkill -9 mongo-2.6; pkill -9 bsondump-2.6; pkill -9 mongoimport-2.6; pkill -9 mongoexport-2.6; pkill -9 mongodump-2.6; pkill -9 mongorestore-2.6; pkill -9 mongostat-2.6; pkill -9 mongofiles-2.6; pkill -9 mongooplog-2.6; pkill -9 mongotop-2.6; pkill -9 mongobridge-2.6; pkill -9 mongod-2.4; pkill -9 mongos-2.4; pkill -9 mongo-2.4; pkill -9 bsondump-2.4; pkill -9 mongoimport-2.4; pkill -9 mongoexport-2.4; pkill -9 mongodump-2.4; pkill -9 mongorestore-2.4; pkill -9 mongostat-2.4; pkill -9 mongofiles-2.4; pkill -9 mongooplog-2.4; pkill -9 mongotop-2.4; pkill -9 buildlogger.py; pkill -9 smoke.py; pkill -9 python; pkill -9 cl; pkill -9 lock_mgr_test; pkill -9 background_job_test; pkill -9 repl_coordinator_impl_heartbeat_test} >/dev/null 2>&1
         exit 0
+  - command: shell.exec
+    params:
+      script: |
+        rm -rf ~/.aws
 
 timeout:
   - command: shell.exec
@@ -1934,13 +1938,35 @@ tasks:
         silent: true
         script: |
           set -o errexit
+
+          mkdir ~/.aws
+
+          cat <<EOF > ~/.aws/config
+          [default]
+          region = us-east-1
+          EOF
+
+          cat <<EOF > ~/.aws/credentials
+          [default]
+          aws_access_key_id = ${aws_key}
+          aws_secret_access_key = ${aws_secret}
+          EOF
+
+          cat <<EOF > notary_env.sh
+          export NOTARY_TOKEN=${signing_auth_token_30}
+          EOF
+
           echo "${signing_auth_token_30}" > signing_auth_token
+
     - command: shell.exec
       params:
         working_dir: src
         script: |
+          . notary_env.sh
+
           set -o errexit
           set -o verbose
+
           mv mongo-binaries.tgz mongodb-binaries.${ext|tgz}
           mv mongo-debugsymbols.tgz debugsymbols-*.${ext|tgz} || true
           cp mongodb-*.${ext|tgz} mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz}
@@ -1950,24 +1976,12 @@ tasks:
 
           notary-client.py --key-name "server-3.0" --auth-token-file ${workdir}/src/signing_auth_token --comment "Evergreen Automatic Signing ${revision} - ${build_variant} - ${branch_name}" --notary-url http://notary-service.build.10gen.cc:5000 --skip-missing mongodb-${push_name}-${push_arch}-${suffix}.${ext|tgz} mongodb-${push_name}-${push_arch}-debugsymbols-${suffix}.${ext|tgz} mongodb-win32-${push_arch}-${suffix}.msi mongodb-src-${src_suffix}.${ext|tar.gz}
 
-          if [ "${has_packages|}" = "true" ] ; then
-            for rpm in $(find repo/ -name \*.rpm) ; do
-              cd $(dirname $rpm)
-              # Use --package-file-suffix "" to overwrite existing RPM's instead of saving new "-signed.rpm" RPM files
-              #
-              notary-client.py --key-name "server-3.0" --auth-token-file ${workdir}/src/signing_auth_token --comment "Evergreen Automatic Signing ${revision} - ${build_variant} - ${branch_name}" --notary-url http://notary-service.build.10gen.cc:5000 --archive-file-ext gpg --outputs sig --package-file-suffix "" $(basename $rpm)
-
-              cd ${workdir}/src
-            done
-            for release_file in $(find repo/ -name Release) ; do
-              cd $(dirname $release_file)
-              notary-client.py --key-name "server-3.0" --auth-token-file ${workdir}/src/signing_auth_token --comment "Evergreen Automatic Signing ${revision} - ${build_variant} - ${branch_name}" --notary-url http://notary-service.build.10gen.cc:5000 --outputs sig Release
-
-              cd ${workdir}/src
-            done
+          if [ "${has_packages|}" = "true" ]; then
+             CURATOR_RELEASE="90080e97b430c197ec487f52de569b2adc028b4e"
+             curl -L -O http://boxes.10gen.com/build/curator/curator-dist-rhel70-$CURATOR_RELEASE.tar.gz
+             tar -zxvf curator-dist-rhel70-$CURATOR_RELEASE.tar.gz
+             ./curator repo --config ./etc/repo_config.yaml --distro ${packager_distro} --edition ${repo_edition} --version ${version} --arch ${packager_arch} --packages $(find repo \(-name "*.deb" -o -name "*.rpm"\) -printf "%h\n" | head -n 1)/\*
           fi
-          tar zcvf repo-${push_arch}.tgz repo || true
-          rm signing_auth_token
 
     # Put the binaries tarball/zipfile
     - command: s3.put
@@ -2587,6 +2601,7 @@ buildvariants:
     has_packages: true
     packager_script: packager.py
     packager_distro: ubuntu1204
+    repo_edition: org
   tasks:
   - name: compile
     distros:
@@ -2664,6 +2679,7 @@ buildvariants:
     has_packages: true
     packager_script: packager.py
     packager_distro: ubuntu1404
+    repo_edition: org
   tasks:
   - name: compile
     distros:
@@ -2742,6 +2758,7 @@ buildvariants:
     has_packages: true
     packager_script: packager-enterprise.py
     packager_distro: amazon
+    repo_edition: enterprise
   tasks:
   - name: compile
     distros:
@@ -2789,6 +2806,7 @@ buildvariants:
     has_packages: true
     packager_script: packager.py
     packager_distro: amazon
+    repo_edition: org
   tasks:
   - name: compile
     distros:
@@ -3462,6 +3480,7 @@ buildvariants:
     has_packages: true
     packager_script: packager-enterprise.py
     packager_distro: rhel57
+    repo_edition: enterprise
   tasks:
   - name: compile
     distros:
@@ -3499,6 +3518,7 @@ buildvariants:
     has_packages: true
     packager_script: packager-enterprise.py
     packager_distro: rhel62
+    repo_edition: enterprise
   tasks:
   - name: compile
     distros:
@@ -3537,6 +3557,7 @@ buildvariants:
     has_packages: true
     packager_script: packager-enterprise.py
     packager_distro: rhel70
+    repo_edition: enterprise
   tasks:
   - name: compile
   - name: audit
@@ -3570,6 +3591,7 @@ buildvariants:
     has_packages: true
     packager_script: packager.py
     packager_distro: rhel55
+    repo_edition: org
   tasks:
   - name: compile
     distros:
@@ -3648,6 +3670,7 @@ buildvariants:
     has_packages: true
     packager_script: packager.py
     packager_distro: rhel62
+    repo_edition: org
   tasks:
   - name: compile
     distros:
@@ -3728,6 +3751,7 @@ buildvariants:
     has_packages: true
     packager_script: packager.py
     packager_distro: rhel70
+    repo_edition: org
   tasks:
   - name: compile
     distros:
@@ -3811,6 +3835,7 @@ buildvariants:
     has_packages: true
     packager_script: packager-enterprise.py
     packager_distro: ubuntu1204
+    repo_edition: enterprise
   tasks:
   - name: compile
     distros:
@@ -3847,6 +3872,7 @@ buildvariants:
     has_packages: true
     packager_script: packager-enterprise.py
     packager_distro: ubuntu1404
+    repo_edition: enterprise
   tasks:
   - name: compile
     distros:
@@ -3887,6 +3913,7 @@ buildvariants:
     has_packages: true
     packager_script: packager-enterprise.py
     packager_distro: suse11
+    repo_edition: enterprise
   tasks:
   - name: compile
     distros:
@@ -3922,6 +3949,7 @@ buildvariants:
     has_packages: true
     packager_script: packager.py
     packager_distro: suse11
+    repo_edition: org
   tasks:
   - name: compile
     distros:
@@ -4054,6 +4082,7 @@ buildvariants:
     has_packages: true
     packager_script: packager-enterprise.py
     packager_distro: debian71
+    repo_edition: enterprise
   tasks:
   - name: compile
     distros:
@@ -4089,6 +4118,7 @@ buildvariants:
     has_packages: true
     packager_script: packager.py
     packager_distro: debian71
+    repo_edition: org
   tasks:
   - name: compile
     distros:
diff --git a/etc/repo_config.yaml b/etc/repo_config.yaml
new file mode 100644
index 00000000000..943d59d2f56
--- /dev/null
+++ b/etc/repo_config.yaml
@@ -0,0 +1,304 @@
+mirrors:
+  3.3: unstable
+  nightly: testing
+  rc: testing
+
+services:
+  notary_url: "http://notary-service.build.10gen.cc:5000"
+
+templates:
+  deb:
+    org: |
+      Origin: mongodb
+      Label: mongodb
+      Suite: {{ .CodeName }}
+      Codename: {{ .CodeName }}/mongodb-org
+      Architectures: {{ .Architectures }}
+      Components: {{ .Component }}
+      Description: MongoDB packages
+    enterprise: |
+      Origin: mongodb
+      Label: mongodb
+      Suite: {{ .CodeName }}
+      Codename: {{ .CodeName }}/mongodb-enterprise
+      Architectures: {{ .Architectures }}
+      Components: {{ .Component }}
+      Description: MongoDB packages
+  index_page: |
+    <!DOCTYPE html>
+    <html>
+     <head>
+       <title>{{ .Title }}</title>
+     </head>
+
+     <body>
+       <table>
+        <tr><td>
+          <h1>{{ .Title }}</h1>
+        </td></tr>
+
+        <tr><td>
+          <hr>
+        </td></tr>
+
+        <tr><td>
+          <a href='..'>Parent Directory</a>
+        </td></tr>
+
+        {{ range $fn := .Files }}
+        <tr><td>
+          <a href='{{ $fn }}'>{{ $fn }}</a>
+        </td></tr>
+        {{ end }}
+
+        <tr><td>
+          <hr>
+        </td></tr>
+
+        <tr><td>
+          <address>{{ .RepoName }}</address>
+        </td></tr>
+      </table>
+     </body>
+    </html>
+
+
+repos:
+
+####################
+#
+# Community Repos:
+#
+####################
+
+  - name: rhel55
+    type: rpm
+    edition: org
+    bucket: repo.mongodb.com
+    repos:
+      - yum/redhat/5/mongodb-enterprise
+      - yum/redhat/5Server/mongodb-enterprise
+
+  - name: rhel62
+    type: rpm
+    edition: org
+    bucket: repo.mongodb.org
+    repos:
+      - yum/redhat/6/mongodb-org
+      - yum/redhat/6Server/mongodb-org
+
+  - name: rhel70
+    type: rpm
+    edition: org
+    bucket: repo.mongodb.org
+    repos:
+      - yum/redhat/7/mongodb-org
+      - yum/redhat/7Server/mongodb-org
+
+  - name: amazon
+    type: rpm
+    edition: org
+    bucket: repo.mongodb.org
+    repos:
+      - yum/amazon/2013.03/mongodb-org
+
+  - name: suse11
+    type: rpm
+    edition: org
+    bucket: repo.mongodb.org
+    repos:
+      - zypper/suse/11/mongodb-org
+
+  - name: suse12
+    type: rpm
+    edition: org
+    bucket: repo.mongodb.org
+    repos:
+      - zypper/suse/12/mongodb-org
+
+  - name: debian81
+    type: deb
+    code_name: "jessie"
+    bucket: repo.mongodb.org
+    edition: org
+    component: main
+    architectures:
+      - amd64
+    repos:
+      - apt/debian/dists/jessie/mongodb-org
+
+  - name: debian71
+    type: deb
+    code_name: "wheezy"
+    bucket: repo.mongodb.org
+    edition: org
+    component: main
+    architectures:
+      - amd64
+    repos:
+      - apt/debian/dists/wheezy/mongodb-org
+
+  - name: ubuntu1204
+    type: deb
+    code_name: "precise"
+    edition: org
+    bucket: repo.mongodb.org
+    component: multiverse
+    architectures:
+      - amd64
+    repos:
+      - apt/ubuntu/dists/precise/mongodb-org
+
+  - name: ubuntu1404
+    type: deb
+    code_name: "trusty"
+    edition: org
+    bucket: repo.mongodb.org
+    component: multiverse
+    architectures:
+      - amd64
+    repos:
+      - apt/ubuntu/dists/trusty/mongodb-org
+
+  - name: ubuntu1604
+    type: deb
+    code_name: "xenial"
+    edition: org
+    bucket: repo.mongodb.org
+    component: multiverse
+    architectures:
+      - amd64
+    repos:
+      - apt/ubuntu/dists/xenial/mongodb-org
+
+####################
+#
+# Enterprise Repos:
+#
+####################
+
+  - name: rhel57
+    type: rpm
+    edition: enterprise
+    bucket: repo.mongodb.com
+    repos:
+      - yum/redhat/5/mongodb-enterprise
+      - yum/redhat/5Server/mongodb-enterprise
+
+  - name: rhel62
+    type: rpm
+    edition: enterprise
+    bucket: repo.mongodb.com
+    repos:
+      - yum/redhat/6/mongodb-enterprise
+      - yum/redhat/6Server/mongodb-enterprise
+
+  - name: rhel70
+    type: rpm
+    edition: enterprise
+    bucket: repo.mongodb.com
+    repos:
+      - yum/redhat/7/mongodb-enterprise
+      - yum/redhat/7Server/mongodb-enterprise
+
+  - name: rhel71
+    type: rpm
+    edition: enterprise
+    bucket: repo.mongodb.com
+    repos:
+      - yum/redhat/7/mongodb-enterprise
+      - yum/redhat/7Server/mongodb-enterprise
+
+  - name: rhel72
+    type: rpm
+    edition: enterprise
+    bucket: repo.mongodb.com
+    repos:
+      - yum/redhat/7/mongodb-enterprise
+      - yum/redhat/7Server/mongodb-enterprise
+
+  - name: amazon
+    type: rpm
+    edition: enterprise
+    bucket: repo.mongodb.com
+    repos:
+      - yum/amazon/2013.03/mongodb-enterprise
+
+  - name: suse11
+    type: rpm
+    edition: enterprise
+    bucket: repo.mongodb.com
+    repos:
+      - zypper/suse/11/mongodb-enterprise
+
+  - name: suse12
+    type: rpm
+    edition: enterprise
+    bucket: repo.mongodb.com
+    repos:
+      - zypper/suse/12/mongodb-enterprise
+
+  - name: debian81
+    type: deb
+    edition: enterprise
+    code_name: "jessie"
+    bucket: repo.mongodb.com
+    component: main
+    architectures:
+      - amd64
+      - ppc64el
+      - s390x
+    repos:
+      - apt/debian/dists/jessie/mongodb-enterprise
+
+  - name: debian71
+    type: deb
+    code_name: "wheezy"
+    edition: enterprise
+    bucket: repo.mongodb.com
+    component: main
+    architectures:
+      - amd64
+      - ppc64el
+      - s390x
+    repos:
+      - apt/debian/dists/wheezy/mongodb-enterprise
+
+  - name: ubuntu1204
+    type: deb
+    code_name: "precise"
+    edition: enterprise
+    bucket: repo.mongodb.com
+    component: multiverse
+    architectures:
+      - amd64
+      - ppc64el
+      - s390x
+    repos:
+      - apt/ubuntu/dists/precise/mongodb-enterprise
+
+  - name: ubuntu1404
+    type: deb
+    code_name: "trusty"
+    edition: enterprise
+    bucket: repo.mongodb.com
+    component: multiverse
+    architectures:
+      - amd64
+      - ppc64el
+      - s390x
+    repos:
+      - apt/ubuntu/dists/trusty/mongodb-enterprise
+
+  - name: ubuntu1604
+    type: deb
+    code_name: "xenial"
+    edition: enterprise
+    bucket: repo.mongodb.com
+    component: multiverse
+    architectures:
+      - amd64
+      - ppc64el
+      - s390x
+    repos:
+      - apt/ubuntu/dists/xenial/mongodb-enterprise