path: root/common.yml

#######################################
#    Tools Driver Config for MCI      #
#######################################

mongo_tools_variables:

## Common download URLs (merge in as hashes)
  mongo_download_urls:
    osx_108: &mongo_url_osx108
      mongo_url: "https://fastdl.mongodb.org/osx/mongodb-osx-x86_64-latest.tgz"
    solaris: &mongo_url_solaris
      mongo_url: "https://fastdl.mongodb.org/sunos5/mongodb-sunos5-x86_64-latest.tgz"
    rhel55_32: &mongo_url_rhell55_32
      mongo_url: "https://fastdl.mongodb.org/linux/mongodb-linux-i686-latest.tgz"
    linux_64_bit: &mongo_url_ubuntu1204
      mongo_url: "http://downloads.10gen.com/linux/mongodb-linux-x86_64-enterprise-ubuntu1204-latest.tgz"
    linux_64_bit_enterprise: &mongo_url_ubuntu1204_enterprise
      mongo_url: "http://downloads.10gen.com/linux/mongodb-linux-x86_64-enterprise-ubuntu1204-latest.tgz"
    windows_32_bit: &mongo_url_windows32
      mongo_url: "https://fastdl.mongodb.org/win32/mongodb-win32-i386-latest.zip"
    windows_64_bit: &mongo_url_windows64
      mongo_url: "https://fastdl.mongodb.org/win32/mongodb-win32-x86_64-2008plus-latest.zip"
    windows_64_bit_enterprise: &mongo_url_windows64_enterprise
      mongo_url: "http://downloads.10gen.com/win32/mongodb-win32-x86_64-enterprise-windows-64-latest.zip"

## List of tests to run on each buildvariant
  mongo_tools_task_lists:
    osx_108_task_list: &osx_108_tasks
      - name: db
      - name: dist
      - name: legacy
      - name: unit
    osx_108_ssl_task_list: &osx_108_ssl_tasks
      - name: dist
    rhel55_32_task_list: &rhel55_32_tasks
      - name: db
      - name: dist
      - name: legacy
      - name: unit
    solaris_task_list: &solaris_tasks
      - name: db
      - name: dist
      - name: legacy
      - name: unit
    ubuntu1204_task_list: &ubuntu1204_tasks
      - name: bsonutil
      - name: db
      - name: dist
      - name: integration
      - name: json
      - name: legacy
      - name: lint
      - name: log
      - name: pool
      - name: progress
      - name: text
      - name: util
      - name: vet
    ubuntu1204_ssl_task_list: &ubuntu1204_ssl_tasks
      - name: dist
      - name: integration
    windows_32_task_list: &windows_32_tasks
      - name: db
        distros:
        - windows-64-vs2013-test
      - name: dist
      - name: legacy
        distros:
        - windows-64-vs2013-test
      - name: unit
    windows_64_task_list: &windows_64_tasks
      - name: db
      - name: dist
      - name: legacy
        distros:
        - windows-64-vs2013-test
      - name: unit
    windows_64_ssl_task_list: &windows_64_ssl_tasks
      - name: dist

## Common mongodb arguments
  mongodb_arguments:
    default: &mongodb_default_startup_args
      mongodb_args: &default_startup_args_string "--dbpath mongodb/db_files --port 27017 --setParameter=enableTestCommands=1 &"
    ssl: &mongodb_ssl_startup_args
      mongodb_args: "--sslMode requireSSL --sslCAFile common/db/openssl/testdata/ca.pem --sslPEMKeyFile common/db/openssl/testdata/server.pem --dbpath mongodb/db_files --port 27017 --setParameter=enableTestCommands=1 &"
    auth: &mongodb_auth_startup_args
      mongodb_args: "--auth --dbpath mongodb/db_files --port 27017 --setParameter=enableTestCommands=1 &"

functions:
  "attach coverage reports" :
    command: attach.task_files
    params:
      "${package} coverage report (HTML)": https://s3.amazonaws.com/mciuploads/mongo-tools/coverage/${package}/${task_id}.html
      "${package} coverage report (TXT)": https://s3.amazonaws.com/mciuploads/mongo-tools/coverage/${package}/${task_id}.txt

  "attach tool":
    command: attach.task_files
    params:
      "${tool} (${edition|community})": https://s3.amazonaws.com/mciuploads/mongo-tools/binaries/${build_id}/${edition|community}/${tool}${extension}

  "build tool":
    command: shell.exec
    params:
      working_dir: src
      script: |
        echo "Building ${tool}..."
        if [ "Windows_NT" = "$OS" ]; then
            set -o igncr
        fi;
        if [ '${library_path}' != '' ]; then
            export ${library_path}
        fi
        . ./${set_gopath|set_gopath.sh}
        go build ${args} ${build_tags} -o bin/${tool} ${tool}/main/${tool}.go

  "download mongod":
    command: shell.exec
    params:
      working_dir: src
      script: |
        set -o verbose
        rm -rf mongodb
        mkdir mongodb
        cd mongodb
        curl -s ${mongo_url} --output mongodb.tgz
        ${decompress} mongodb.tgz
        chmod +x ./mongodb-*/bin/*
        mv ./mongodb-*/bin/* .

  "fetch tool" :
    command: s3.get
    params:
      bucket: mciuploads
      aws_key: ${aws_key}
      aws_secret: ${aws_secret}
      local_file: src/${tool}${extension}
      remote_file: mongo-tools/binaries/${build_id}/${edition|community}/${tool}${extension}

  "generate coverage html + text":
    command: shell.exec
    params:
      working_dir: src
      script: |
        set -o verbose
        if [ "${coverage}" = "true" ]; then
          if [ "Windows_NT" = "$OS" ]; then
            set -o igncr
          fi;
          . ./${set_gopath|set_gopath.sh}
          go tool cover -html=coverage.out -o coverage.html
          go tool cover -func=coverage.out -o coverage.txt
        fi;

  "get buildnumber":
    command: keyval.inc
    params:
      key: "${build_variant}_tools"
      destination: "builder_num"

  "move coverage data":
    command: shell.exec
    params:
      working_dir: src
      script: |
        set -o verbose
        if [ "${coverage}" = "true" ]; then
          mv ${package}/coverage.out . 
        fi

  "run unit test":
    command: shell.exec 
    params:
      working_dir: src
      script: |
        set -e
        if [ "Windows_NT" = "$OS" ]; then
            set -o igncr
        fi;
        . ./${set_gopath|set_gopath.sh}
        if [ '${library_path}' != '' ]; then
            export ${library_path}
        fi
        export MONGODB_KERBEROS_PASSWORD=${kerberos_password}
        cd ${package}
        go test ${coverage_args} ${args} ${build_tags}

  "setup integration test":
    command: shell.exec
    params:
      working_dir: src
      # Set up Kerberos stuff: run kinit if necessary, and add KDC to registry
      # on Windows (see https://wiki.mongodb.com/display/DH/Testing+Kerberos)
      script: |
        if [ '${run_kinit}' = 'true' ]
        then
          echo "${kerberos_password}" | kinit -p drivers@LDAPTEST.10GEN.CC;
        fi;
        if [ "Windows_NT" = "$OS" ]; then
          cmd /c "REG ADD HKLM\SYSTEM\ControlSet001\Control\Lsa\Kerberos\Domains\LDAPTEST.10GEN.CC /v KdcNames /d ldaptest.10gen.cc /t REG_MULTI_SZ /f"
        fi;

  "setup permissions":
    command: shell.exec
    params:
      working_dir: src
      script: |
        chmod 700 go.sh
        chmod 700 ${set_gopath|set_gopath.sh}

  "run tool unit tests":
    command: shell.exec
    params:
      working_dir: src
      script: | 
        if [ "Windows_NT" = "$OS" ]; then
            set -o igncr
        fi;
        . ./${set_gopath|set_gopath.sh}
        if [ '${library_path}' != '' ]; then
            export ${library_path}
        fi;

        # TODO bsondump needs tests
        # TODO mongodump needs tests
        # TODO mongotop needs tests

        # mongoimport
        cd mongoimport
        go test ${coverage_args} ${args} ${build_tags}
        cd ..

        # mongoexport
        cd mongoexport
        go test ${coverage_args} ${args} ${build_tags}
        cd ..

        # mongostat
        cd mongostat
        go test ${coverage_args} ${args} ${build_tags}
        cd ..

        # mongooplog
        cd mongooplog
        go test ${coverage_args} ${args} ${build_tags}
        cd ..

        # mongorestore
        cd mongorestore
        go test ${coverage_args} ${args} ${build_tags}
        cd ..

        # mongofiles
        cd mongofiles
        go test ${coverage_args} ${args} ${build_tags}
        cd ..

  "run tool integration tests":
    command: shell.exec
    params:
      working_dir: src
      script: |
        set -e
        if [ "Windows_NT" = "$OS" ]; then
            set -o igncr
        fi;
        . ./set_gopath.sh
        if [ "${library_path}" != "" ]; then
            export ${library_path}
        fi

        # mongodump
        cd mongodump
        go test -v ${coverage_args} ${args} ${build_tags}
        cd ..

        # mongofiles
        cd mongofiles
        go test -v ${coverage_args} ${args} ${build_tags}
        cd ..

        # mongoimport
        cd mongoimport
        go test -v ${coverage_args} ${args} ${build_tags}
        cd ..

        # mongooplog
        cd mongooplog
        go test -v ${coverage_args} ${args} ${build_tags}
        cd ..

  "setup credentials" :
    command: shell.exec
    params:
      working_dir: src
      silent: true
      script: |
        cat > mci.buildlogger <<END_OF_CREDS
        slavename='${slave}'
        passwd='${passwd}'
        END_OF_CREDS

  "start mongod":
    command: shell.exec
    params:
      working_dir: src
      background: true
      script: |
        set -o verbose
        rm -rf mongodb/${db_files_dir|db_files} mongodb/${logfile|run.log};
        mkdir mongodb/${db_files_dir|db_files};
        echo "Starting mongod...";
        ./mongodb/mongod${extension} ${mongodb_args} ${mongod_extra_options|}

  "upload coverage report":
    command: s3.put
    params:
      aws_key: ${aws_key}
      aws_secret: ${aws_secret}
      local_file: src/coverage.${format|txt}
      remote_file: mongo-tools/coverage/${package}/${task_id}.${format|txt}
      bucket: mciuploads
      permissions: public-read
      content_type: ${content_type|text/plain}
      build_variants: ["ubuntu"]

  "upload tool":
    command: s3.put
    params:
      aws_key: ${aws_key}
      aws_secret: ${aws_secret}
      local_file: src/bin/${tool}
      remote_file: mongo-tools/binaries/${build_id}/${edition|community}/${tool}${extension}
      bucket: mciuploads
      permissions: public-read
      content_type: application/octet-stream

pre:
  - command: expansions.fetch
    params:
      keys:
        - local_key: "aws_key"
          remote_key: "project_aws_key"
        - local_key: "aws_secret"
          remote_key: "project_aws_secret"
        - local_key: "slave"
          remote_key: "slave"
        - local_key: "passwd"
          remote_key: "passwd"
        - local_key: "kerberos_password"
          remote_key: "kerberos_password"
  - command: shell.exec
    params:
      script: |
        set -o verbose
        ${killall_mci|pkill -9 mongo; pkill -9 mongodump; pkill -9 mongoexport; pkill -9 mongoimport; pkill -9 mongofiles; pkill -9 mongooplog; pkill -9 mongorestore; pkill -9 mongostat; pkill -9 mongotop; pkill -9 mongod; pkill -9 mongos; pkill -f buildlogger.py; pkill -f smoke.py}
        rm -rf src || true
        rm -rf /data/db/*
        true

tasks:
#- name: auth
#  commands:
#    - command: git.get_project
#      params:
#        directory: src
#    - func: "setup go env"
#    - func: "download mongod"
#    - func: "run mongod test"
#      vars:
#        package: "common/db"
#        types: "auth"
#    - func: "generate coverage html + text"
#    - func: "upload coverage report"
#    - func: "upload coverage report"
#      vars:
#        format: "html"
#        content_type: "text/html"
#    - func: "attach coverage reports"
#
#- name: ssl
#  commands:
#    - command: git.get_project
#      params:
#        directory: src
#    - func: "setup go env"
#    - func: "download mongod"
#    - command: expansions.update
#      params:
#        updates:
#          - key: "package"
#            value: "common/db/openssl"
#          - key: "types"
#            value: "ssl"
#          - key: "port"
#            value: "20000"
#    - func: "run mongod test"
#    - command: expansions.update
#      params:
#        updates:
#          - key: "types"
#            value: "ssl_auth"
#          - key: "mongod_extra_options"
#            value: "--sslMode requireSSL --sslCAFile src/src/github.com/mongodb/mongo-tools/common/db/openssl/testdata/ca.pem --sslPEMKeyFile src/src/github.com/mongodb/mongo-tools/common/db/openssl/testdata/server.pem --auth"
#    - func: "run mongod test"
#
- name: bsonutil
  commands:
    - command: git.get_project
      params:
        directory: src
    - command: git.apply_patch
      params:
        directory: src
    - command: expansions.update
      params:
        updates:
          - key: "package"
            value: "common/bsonutil"
    - func: "run unit test"

- name: db
  commands:
    - command: git.get_project
      params:
        directory: src
    - command: git.apply_patch
      params:
        directory: src
    - command: expansions.update
      params:
        updates:
          - key: "package"
            value: "common/db"
          - key: "args"
            value: "-test.types=db"
          - key: "mongodb_args"
            value: *default_startup_args_string
    - func: "download mongod"
    - func: "start mongod"
    - func: "setup integration test"
    - func: "run unit test"

- name: dist
  depends_on:
  commands:
    - command: git.get_project
      params:
        directory: src
    - command: git.apply_patch
      params:
        directory: src
    - command: shell.exec
      params:
        working_dir: src
        script: |
          rm -rf bin/
          mkdir bin
    # bsondump
    - func: "build tool"
      vars:
        tool: bsondump
    - func: "upload tool"
      vars:
        tool: bsondump
    - func: "attach tool"
      vars:
        tool: bsondump
    # mongodump
    - func: "build tool"
      vars:
        tool: mongodump
    - func: "upload tool"
      vars:
        tool: mongodump
    - func: "attach tool"
      vars:
        tool: mongodump
    # mongoexport
    - func: "build tool"
      vars:
        tool: mongoexport
    - func: "upload tool"
      vars:
        tool: mongoexport
    - func: "attach tool"
      vars:
        tool: mongoexport
    # mongofiles
    - func: "build tool"
      vars:
        tool: mongofiles
    - func: "upload tool"
      vars:
        tool: mongofiles
    - func: "attach tool"
      vars:
        tool: mongofiles
    # mongoimport
    - func: "build tool"
      vars:
        tool: mongoimport
    - func: "upload tool"
      vars:
        tool: mongoimport
    - func: "attach tool"
      vars:
        tool: mongoimport
    # mongooplog
    - func: "build tool"
      vars:
        tool: mongooplog
    - func: "upload tool"
      vars:
        tool: mongooplog
    - func: "attach tool"
      vars:
        tool: mongooplog
    # mongorestore
    - func: "build tool"
      vars:
        tool: mongorestore
    - func: "upload tool"
      vars:
        tool: mongorestore
    - func: "attach tool"
      vars:
        tool: mongorestore
    # mongostat
    - func: "build tool"
      vars:
        tool: mongostat
    - func: "upload tool"
      vars:
        tool: mongostat
    - func: "attach tool"
      vars:
        tool: mongostat
    # mongotop
    - func: "build tool"
      vars:
        tool: mongotop
    - func: "upload tool"
      vars:
        tool: mongotop
    - func: "attach tool"
      vars:
        tool: mongotop

- name: integration
  commands:
    - command: git.get_project
      params:
        directory: src
    - command: git.apply_patch
      params:
        directory: src
    - command: expansions.update
      params:
        updates:
          - key: "args"
            value: "-test.types=${integration_test_args}"
    - func: "download mongod"
    - func: "start mongod"
    - command: shell.exec
      params:
        directory: src
        script: |
          # Give mongod a chance to stop
          sleep 2s
    - func: "run tool integration tests"

- name: json
  commands:
    - command: git.get_project
      params:
        directory: src
    - command: git.apply_patch
      params:
        directory: src
    - command: expansions.update
      params:
        updates:
          - key: "package"
            value: "common/json"
    - func: "run unit test"

- name: legacy
  depends_on:
  - name: dist
  commands:
    - command: git.get_project
      params:
        directory: src
    - command: git.apply_patch
      params:
        directory: src
    - func: "get buildnumber"
    - func: "setup credentials"
    - func: "download mongod"
    - func: "fetch tool"
      vars:
        tool: mongoimport
    - func: "fetch tool"
      vars:
        tool: mongoexport
    - func: "fetch tool"
      vars:
        tool: mongodump
    - func: "fetch tool"
      vars:
        tool: mongostat
    - func: "fetch tool"
      vars:
        tool: mongorestore
    - func: "fetch tool"
      vars:
        tool: mongooplog
    - func: "fetch tool"
      vars:
        tool: mongofiles
    - command: shell.exec
      params:
        working_dir: src
        script: |
          mv ./mongodb/mongod${extension} .
          mv ./mongodb/mongo${extension} .
          mv legacy/* .
          chmod +x mongo*
          python buildscripts/smoke.py --nopreallocj --with-cleanbb --mongod ./mongod --mongo ./mongo --report-file report.json --continue-on-failure --buildlogger-builder MCI_${build_variant} --buildlogger-buildnum ${builder_num|} --buildlogger-credentials ./mci.buildlogger --buildlogger-phase ${task_name}_${execution} tool
          echo $? > retVal
    - command: attach.results
      params:
        file_location: src/report.json
    - command: shell.exec
      params:
        script: |
          ${killall_mci|pkill -9 mongo; pkill -9 mongodump; pkill -9 mongoexport; pkill -9 mongoimport; pkill -9 mongofiles; pkill -9 mongooplog; pkill -9 mongorestore; pkill -9 mongostat; pkill -9 mongotop; pkill -9 mongod; pkill -9 mongos; pkill -f buildlogger.py; pkill -f smoke.py}
          exit 0

- name: lint
  commands:
    - command: git.get_project
      params:
        directory: src
    - command: git.apply_patch
      params:
        directory: src
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -o errexit
          set -o verbose
          retVal=$(./go.sh run vendor/src/github.com/3rf/mongo-lint/golint/golint.go src);
          if [ "$retVal" = "" ]; then exit 0; else echo $retVal; exit 1; fi;

- name: log
  commands:
    - command: git.get_project
      params:
        directory: src
    - command: git.apply_patch
      params:
        directory: src
    - command: expansions.update
      params:
        updates:
          - key: "package"
            value: "common/log"
    - func: "run unit test"

- name: pool
  commands:
    - command: git.get_project
      params:
        directory: src
    - command: git.apply_patch
      params:
        directory: src
    - command: expansions.update
      params:
        updates:
          - key: "package"
            value: "common/pool"
    - func: "run unit test"

- name: progress
  commands:
    - command: git.get_project
      params:
        directory: src
    - command: git.apply_patch
      params:
        directory: src
    - command: expansions.update
      params:
        updates:
          - key: "package"
            value: "common/progress"
    - func: "run unit test"

- name: text
  commands:
    - command: git.get_project
      params:
        directory: src
    - command: git.apply_patch
      params:
        directory: src
    - command: expansions.update
      params:
        updates:
          - key: "package"
            value: "common/text"
    - func: "run unit test"

- name: unit
  commands:
    - command: git.get_project
      params:
        directory: src
    - command: git.apply_patch
      params:
        directory: src
    - func: "run tool unit tests"

- name: unit-coverage
  commands:
    - command: git.get_project
      params:
        directory: src
    - command: git.apply_patch
      params:
        directory: src
    - func: "run tool unit tests"

- name: util
  commands:
    - command: git.get_project
      params:
        directory: src
    - command: git.apply_patch
      params:
        directory: src
    - command: expansions.update
      params:
        updates:
          - key: "package"
            value: "common/util"
    - func: "run unit test"

- name: vet
  commands:
    - command: git.get_project
      params:
        directory: src
    - command: git.apply_patch
      params:
        directory: src
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -o verbose
          retVal=$(./go.sh tool vet .); if [ "$retVal" = "" ]; then exit 0; else exit 1; fi;

buildvariants:
#######################################
#           OSX Buildvariant          #
#######################################
- name: osx-108
  display_name: OSX 10.8 64-bit
  run_on:
  - osx-108
  expansions:
    <<: *mongo_url_osx108
    <<: *mongodb_default_startup_args
  tasks: *osx_108_tasks

- name: osx-108-ssl
  display_name: OSX 10.8 64-bit SSL
  run_on:
  - osx-108
  expansions:
    build_tags: -tags "ssl"
    edition: ssl
  tasks: *osx_108_ssl_tasks

#######################################
#          RHEL Buildvariant          #
#######################################
- name: linux-32
  display_name: Linux 32-bit
  disabled: true
  run_on:
  - rhel55-32
  expansions:
    <<: *mongo_url_rhell55_32
    <<: *mongodb_default_startup_args
  tasks: *rhel55_32_tasks

#######################################
#          Ubuntu Buildvariants       #
#######################################

- name: ubuntu
  display_name: Linux 64-bit
  run_on:
  - ubuntu1204-test
  expansions:
    <<: *mongo_url_ubuntu1204
    <<: *mongodb_default_startup_args
    integration_test_args: integration
  tasks: *ubuntu1204_tasks

- name: ubuntu-ssl
  display_name: Linux 64-bit SSL
  run_on:
  - ubuntu1204-test
  expansions:
    <<: *mongo_url_ubuntu1204_enterprise
    <<: *mongodb_ssl_startup_args
    build_tags: -tags "ssl"
    edition: ssl
    integration_test_args: "integration,ssl"
  tasks: *ubuntu1204_ssl_tasks

- name: ubuntu-enterprise
  display_name: Linux 64-bit Enterprise
  run_on:
  - ubuntu1204-test
  expansions:
    <<: *mongo_url_ubuntu1204_enterprise
    <<: *mongodb_default_startup_args
    build_tags: -tags "ssl sasl"
    edition: enterprise
    run_kinit: true
    integration_test_args: integration
  tasks: *ubuntu1204_tasks

#######################################
#         Solaris Buildvariant        #
#######################################
- name: solaris
  display_name: Solaris 64-bit
  run_on:
  - solaris
  expansions:
    <<: *mongo_url_solaris
    <<: *mongodb_default_startup_args
  tasks: *solaris_tasks

#######################################
#        Windows Buildvariants        #
#######################################
- name: windows-32
  display_name: Windows 32-bit
  run_on:
  - windows-32
  expansions:
    <<: *mongo_url_windows32
    <<: *mongodb_default_startup_args
    extension: .exe
    preproc_gpm: "perl -pi -e 's/\\r\\n/\\n/g' "
    set_gopath: set_gopath_windows.sh
  tasks: *windows_32_tasks

- name: windows-64
  display_name: Windows 64-bit
  run_on:
  - windows-64-vs2013-test
  expansions:
    <<: *mongo_url_windows64
    <<: *mongodb_default_startup_args
    extension: .exe
    preproc_gpm: "perl -pi -e 's/\\r\\n/\\n/g' "
    set_gopath: set_gopath_windows.sh
  tasks: *windows_64_tasks

- name: windows-64-ssl
  display_name: Windows 64-bit SSL
  run_on:
  - windows-64-vs2013-compile
  expansions:
    build_tags: -tags "ssl"
    edition: ssl
    extension: .exe
    library_path: PATH="/cygdrive/c/mingw-w64/x86_64-4.9.1-posix-seh-rt_v3-rev1/mingw64/bin:/cygdrive/c/sasl/:$PATH"
    preproc_gpm: "perl -pi -e 's/\\r\\n/\\n/g' "
    set_gopath: set_gopath_windows.sh
  tasks: *windows_64_ssl_tasks

- name: windows-64-enterprise
  display_name: Windows 64-bit Enterprise
  run_on:
  - windows-64-vs2013-compile
  expansions:
    <<: *mongo_url_windows64_enterprise
    <<: *mongodb_ssl_startup_args
    build_tags: -tags "sasl ssl"
    edition: enterprise
    extension: .exe
    library_path: PATH="/cygdrive/c/mingw-w64/x86_64-4.9.1-posix-seh-rt_v3-rev1/mingw64/bin:/cygdrive/c/sasl/:$PATH"
    preproc_gpm: "perl -pi -e 's/\\r\\n/\\n/g' "
    set_gopath: set_gopath_windows.sh
  tasks: *windows_64_tasks