path: root/etc/perf.yml

command_type: system
stepback: false

pre:
    - command: shell.track
    - command: shell.exec
      params:
        silent: true
        script: |
          ${killall_mci|pkill -9 mongod; pkill -9 mongos; pkill -9 mongo; pkill -9 bsondump; pkill -9 mongoimport; pkill -9 mongoexport; pkill -9 mongodump; pkill -9 mongorestore; pkill -9 mongostat; pkill -9 mongofiles; pkill -9 mongooplog; pkill -9 mongotop; pkill -9 mongobridge; pkill -9 mongod-2.6; pkill -9 mongos-2.6; pkill -9 mongo-2.6; pkill -9 bsondump-2.6; pkill -9 mongoimport-2.6; pkill -9 mongoexport-2.6; pkill -9 mongodump-2.6; pkill -9 mongorestore-2.6; pkill -9 mongostat-2.6; pkill -9 mongofiles-2.6; pkill -9 mongooplog-2.6; pkill -9 mongotop-2.6; pkill -9 mongobridge-2.6; pkill -9 mongod-2.4; pkill -9 mongos-2.4; pkill -9 mongo-2.4; pkill -9 bsondump-2.4; pkill -9 mongoimport-2.4; pkill -9 mongoexport-2.4; pkill -9 mongodump-2.4; pkill -9 mongorestore-2.4; pkill -9 mongostat-2.4; pkill -9 mongofiles-2.4; pkill -9 mongooplog-2.4; pkill -9 mongotop-2.4; pkill -9 resmoke.py; pkill -9 python; pkill -9 lldb; pkill -9 _test} >/dev/null 2>&1
          exit 0

post:
    - command: attach.results
      params:
        file_location: src/report.json
    - command: shell.exec
      params:
        silent: true
        script: |
          ${killall_mci|pkill -9 mongod; pkill -9 mongos; pkill -9 mongo; pkill -9 bsondump; pkill -9 mongoimport; pkill -9 mongoexport; pkill -9 mongodump; pkill -9 mongorestore; pkill -9 mongostat; pkill -9 mongofiles; pkill -9 mongooplog; pkill -9 mongotop; pkill -9 mongobridge; pkill -9 mongod-2.6; pkill -9 mongos-2.6; pkill -9 mongo-2.6; pkill -9 bsondump-2.6; pkill -9 mongoimport-2.6; pkill -9 mongoexport-2.6; pkill -9 mongodump-2.6; pkill -9 mongorestore-2.6; pkill -9 mongostat-2.6; pkill -9 mongofiles-2.6; pkill -9 mongooplog-2.6; pkill -9 mongotop-2.6; pkill -9 mongobridge-2.6; pkill -9 mongod-2.4; pkill -9 mongos-2.4; pkill -9 mongo-2.4; pkill -9 bsondump-2.4; pkill -9 mongoimport-2.4; pkill -9 mongoexport-2.4; pkill -9 mongodump-2.4; pkill -9 mongorestore-2.4; pkill -9 mongostat-2.4; pkill -9 mongofiles-2.4; pkill -9 mongooplog-2.4; pkill -9 mongotop-2.4; pkill -9 resmoke.py; pkill -9 python; pkill -9 lldb; pkill -9 _test} >/dev/null 2>&1
          exit 0
    - command: shell.cleanup

functions:
  "download analysis scripts":
    - command: shell.exec
      params:
        script: |
          set -v
          rm -rf ./dsi
          mkdir -p ./src
          git clone git@github.com:10gen/dsi.git
  "start server":
    - command: shell.exec
      params:
        script: |
          rm -rf ./*
          mkdir src
    - command: s3.get
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        remote_file: ${project}/${version_id}/${revision}/mongod-${version_id}
        bucket: mciuploads
        local_file: src/mongod
    - command: s3.get
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        remote_file: ${project}/${version_id}/${revision}/mongo-${version_id}
        bucket: mciuploads
        local_file: src/mongo
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -e
          set -v
          chmod +x mongod
          chmod +x mongo
          git clone https://github.com/mongodb/mongo-perf perf
          cd perf
          git checkout r20160805
    - command: shell.exec
      params:
        background: true
        working_dir: src
        script: |
          set -e
          set -o verbose
          mkdir -p ./dbdata
          ${mongod_exec_wrapper} ./mongod --dbpath ./dbdata ${mongod_flags}
    - command : shell.exec
      params:
        working_dir: src
        script: |
          set -e
          set -o verbose
          sleep 5
          # if we started a replset, initiate it and wait for it to become primary
          ./mongo --eval "if(db.isMaster().isreplicaset){rs.initiate(); assert.soon(function(){return db.isMaster().ismaster}, 'no primary')}"
          echo "MONGOD STARTED."
  "compare":
    - command: json.get
      params:
        task: ${compare_task}
        variant : ${variant1}
        file: "src/stand.json"
        name: "perf"
    - command: json.get
      params:
        task: ${compare_task}
        variant : ${variant2}
        file: "src/node.json"
        name: "perf"
    - command: shell.exec
      type : test
      params:
        working_dir: src
        script: |
          set -o errexit
          set -o verbose
          virtualenv ./venv
          source ./venv/bin/activate
          pip install -r ../dsi/requirements/analysis.txt
          python -u ../dsi/analysis/compare.py -b stand.json -c node.json
    - command: "json.send"
      params:
        name: "perf"
        file: "src/perf.json"
  "analyze":
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -e
          set -v
          virtualenv ./venv
          source ./venv/bin/activate
          pip install -r ../dsi/requirements/analysis.txt
    - command: json.get_history
      params:
        task: ${task_name}
        file: "src/history.json"
        name: "perf"
    - command: json.get_history
      params:
        tags: true
        task: ${task_name}
        file: "src/tags.json"
        name: "perf"
    - command: shell.exec
      type : test
      params:
        working_dir: src
        script: |
          set -o errexit
          set -o verbose
          source ./venv/bin/activate
          # Any tasks that want the analysis scripts to analyze mongod.log files should pass in
          # `reports_analysis: true` as a var to this function. The following line will select the
          # appropriate flags if it's `true`.
          reports_analysis_flags="--reports-analysis . --perf-file perf/perf.json"
          cmd_flags=$([ "${reports_analysis}" = "true" ] && echo "$reports_analysis_flags" || echo "")
          REFTAG="3.0.12-Baseline"
          OVERRIDE="../dsi/analysis/v3.2/perf_override.json" # Note use of v3.2 here cannot use ${project} perf
          python ../dsi/analysis/perf_regression_check.py $cmd_flags -f history.json --rev ${revision} -t tags.json --refTag $REFTAG --overrideFile $OVERRIDE --variant ${build_variant} --threshold 0.10 --threadThreshold 0.15
  "run perf tests":
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -e
          set -v
          virtualenv ./venv
          source ./venv/bin/activate
          pip install argparse
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -e
          set -v
          source ./venv/bin/activate
          cd perf
          #  give mongod a few seconds to start up so that we can connect.
          sleep 5
          ${perf_exec_wrapper} python benchrun.py --shell ../mongo -t ${threads} --trialCount 5 -f testcases/*.js --includeFilter ${includeFilter1}  --includeFilter ${includeFilter2} --excludeFilter ${excludeFilter} --out perf.json --exclude-testbed
    - command: "json.send"
      params:
        name: "perf"
        file: "src/perf/perf.json"

tasks:
- name: compile
  commands:
    - command: git.get_project
      params:
        directory: src
    - command: git.apply_patch
      params:
        directory: src
    - command: shell.exec
      params:
        working_dir: src
        script: |
          set -o errexit
          set -o verbose
          ${scons|scons} ${compile_flags|} mongo
          ${scons|scons} ${compile_flags|} mongod
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: src/mongod
        remote_file: ${project}/${version_id}/${revision}/mongod-${version_id}
        bucket: mciuploads
        permissions: public-read
        content_type: ${content_type|application/octet-stream}
        display_name: mongod
    - command: s3.put
      params:
        aws_key: ${aws_key}
        aws_secret: ${aws_secret}
        local_file: src/mongo
        remote_file: ${project}/${version_id}/${revision}/mongo-${version_id}
        bucket: mciuploads
        permissions: public-read
        content_type: ${content_type|application/octet-stream}
        display_name: mongo
- name: query
  depends_on:
  - variant: linux-wt-standalone
    name: compile
  commands:
    - func: "start server"
    - func: "run perf tests"
      vars:
        includeFilter1: "query"
        includeFilter2 : "core regression"
        excludeFilter : "single_threaded"
        threads : "1 2 4 8"
    - func: "download analysis scripts"
    - func: "analyze"
      vars:
        reports_analysis: true
- name: where
  depends_on:
  - variant: linux-wt-standalone
    name: compile
  commands:
    - func: "start server"
    - func: "run perf tests"
      vars:
        includeFilter1: "where"
        includeFilter2 : "core regression"
        excludeFilter : "single_threaded"
        threads : "1 2 4 8"
    - func: "download analysis scripts"
    - func: "analyze"
      vars:
        reports_analysis: true
- name: update
  depends_on:
  - variant: linux-wt-standalone
    name: compile
  commands:
    - func: "start server"
    - func: "run perf tests"
      vars:
        includeFilter1: "update"
        includeFilter2 : "core regression"
        excludeFilter : "single_threaded"
        threads : "1 2 4 8"
    - func: "download analysis scripts"
    - func: "analyze"
      vars:
        reports_analysis: true
- name: insert
  depends_on:
  - variant: linux-wt-standalone
    name: compile
  commands:
    - func: "start server"
    - func: "run perf tests"
      vars:
        includeFilter1: "insert"
        includeFilter2 : "core regression"
        excludeFilter : "single_threaded"
        threads : "1 2 4 8"
    - func: "download analysis scripts"
    - func: "analyze"
      vars:
        reports_analysis: true
- name: geo
  depends_on:
  - variant: linux-wt-standalone
    name: compile
  commands:
    - func: "start server"
    - func: "run perf tests"
      vars:
        includeFilter1: "geo"
        includeFilter2 : "core regression"
        excludeFilter : "single_threaded"
        threads : "1 2 4 8"
    - func: "download analysis scripts"
    - func: "analyze"
      vars:
        reports_analysis: true
- name: misc
  depends_on:
  - variant: linux-wt-standalone
    name: compile
  commands:
    - func: "start server"
    - func: "run perf tests"
      vars:
        includeFilter1: "command multi remove mixed"
        includeFilter2 : "core regression"
        excludeFilter : "single_threaded"
        threads : "1 2 4 8"
    - func: "download analysis scripts"
    - func: "analyze"
      vars:
        reports_analysis: true
- name: singleThreaded
  depends_on:
  - variant: linux-wt-standalone
    name: compile
  commands:
    - func: "start server"
    - func: "run perf tests"
      vars:
        includeFilter1: "single_threaded"
        includeFilter2 : "core regression"
        excludeFilter : "none"
        threads : "1"
    - func: "download analysis scripts"
    - func: "analyze"
      vars:
        reports_analysis: true
- name: singleThreaded-wt-repl-comp
  depends_on:
    - name : compile
      variant : linux-wt-standalone
    - name : singleThreaded
      variant : linux-wt-standalone
      status : "*"
    - name : singleThreaded
      variant :  linux-wt-repl
      status : "*"
  commands:
    - func: "download analysis scripts"
    - func: "compare"
      vars:
        compare_task : "singleThreaded"
        variant1 : "linux-wt-standalone"
        variant2 : "linux-wt-repl"
    - func: "analyze"
- name: insert-wt-repl-comp
  depends_on:
    - name : compile
      variant : linux-wt-standalone
    - name : insert
      variant : linux-wt-standalone
      status : "*"
    - name : insert
      variant :  linux-wt-repl
      status : "*"
  commands:
    - func: "download analysis scripts"
    - func: "compare"
      vars:
        compare_task : "insert"
        variant1 : "linux-wt-standalone"
        variant2 : "linux-wt-repl"
    - func: "analyze"
- name: update-wt-repl-comp
  depends_on:
    - name : compile
      variant : linux-wt-standalone
    - name : update
      variant : linux-wt-standalone
      status : "*"
    - name : update
      variant :  linux-wt-repl
      status : "*"
  commands:
    - func: "download analysis scripts"
    - func: "compare"
      vars:
        compare_task : "update"
        variant1 : "linux-wt-standalone"
        variant2 : "linux-wt-repl"
    - func: "analyze"
- name: misc-wt-repl-comp
  depends_on:
    - name : compile
      variant : linux-wt-standalone
    - name : misc
      variant : linux-wt-standalone
      status : "*"
    - name : misc
      variant :  linux-wt-repl
      status : "*"
  commands:
    - func: "download analysis scripts"
    - func: "compare"
      vars:
        compare_task : "misc"
        variant1 : "linux-wt-standalone"
        variant2 : "linux-wt-repl"
    - func: "analyze"
- name: singleThreaded-mmap-repl-comp
  depends_on:
    - name : compile
      variant : linux-wt-standalone
    - name : singleThreaded
      variant : linux-mmap-standalone
      status : "*"
    - name : singleThreaded
      variant :  linux-mmap-repl
      status : "*"
  commands:
    - func: "download analysis scripts"
    - func: "compare"
      vars:
        compare_task : "singleThreaded"
        variant1 : "linux-mmap-standalone"
        variant2 : "linux-mmap-repl"
    - func: "analyze"
- name: insert-mmap-repl-comp
  depends_on:
    - name : compile
      variant : linux-wt-standalone
    - name : insert
      variant : linux-mmap-standalone
      status : "*"
    - name : insert
      variant :  linux-mmap-repl
      status : "*"
  commands:
    - func: "download analysis scripts"
    - func: "compare"
      vars:
        compare_task : "insert"
        variant1 : "linux-mmap-standalone"
        variant2 : "linux-mmap-repl"
    - func: "analyze"
- name: update-mmap-repl-comp
  depends_on:
    - name : compile
      variant : linux-wt-standalone
    - name : update
      variant : linux-mmap-standalone
      status : "*"
    - name : update
      variant :  linux-mmap-repl
      status : "*"
  commands:
    - func: "download analysis scripts"
    - func: "compare"
      vars:
        compare_task : "update"
        variant1 : "linux-mmap-standalone"
        variant2 : "linux-mmap-repl"
    - func: "analyze"
- name: misc-mmap-repl-comp
  depends_on:
    - name : compile
      variant : linux-wt-standalone
    - name : misc
      variant : linux-mmap-standalone
      status : "*"
    - name : misc
      variant :  linux-mmap-repl
      status : "*"
  commands:
    - func: "download analysis scripts"
    - func: "compare"
      vars:
        compare_task : "misc"
        variant1 : "linux-mmap-standalone"
        variant2 : "linux-mmap-repl"
    - func: "analyze"
- name: aggregation
  depends_on:
  - variant: linux-wt-standalone
    name: compile
  commands:
    - func: "start server"
    - func: "run perf tests"
      vars:
        includeFilter1: "aggregation"
        includeFilter2: "regression"
        excludeFilter: "none"
        threads : "1"
    - func: "download analysis scripts"
    - func: "analyze"
      vars:
        log_analysis: true

buildvariants:
- name: linux-wt-standalone
  display_name: Standalone Linux WT
  batchtime: 90 # 1.5 hours
  expansions:
    compile_flags: &compile_flags -j$(grep -c ^processor /proc/cpuinfo) CC=/opt/mongodbtoolchain/bin/gcc CXX=/opt/mongodbtoolchain/bin/g++ --release
    mongod_exec_wrapper: &exec_wrapper "numactl --physcpubind=4,5,6,7 -i 1"
    perf_exec_wrapper: &perf_wrapper "numactl --physcpubind=1,2,3 -i 0"
    mongod_flags: "--storageEngine=wiredTiger --logpath ./mongod.log --fork --syncdelay 0 --nojournal --setParameter ttlMonitorEnabled=false --setParameter diagnosticDataCollectionEnabled=false  --wiredTigerCacheSizeGB 16"
    project: &project perf-3.2
  run_on:
  - "centos6-perf"
  tasks:
  - name: compile
    distros:
    - rhel62-large
  - name: aggregation
  - name: query
  - name: where
  - name: update
  - name: insert
  - name: geo
  - name: misc
  - name: singleThreaded

- name: linux-mmap-standalone
  display_name: Standalone Linux MMAPv1
  batchtime: 90 # 1.5 hours
  expansions:
    compile_flags: *compile_flags
    mongod_exec_wrapper: *exec_wrapper
    perf_exec_wrapper: *perf_wrapper
    mongod_flags: "--storageEngine=mmapv1 --logpath ./mongod.log --fork --syncdelay 0 --nojournal --setParameter ttlMonitorEnabled=false --setParameter diagnosticDataCollectionEnabled=false"
    project: *project
  run_on:
  - "centos6-perf"
  tasks:
  - name: aggregation
  - name: query
  - name: where
  - name: update
  - name: insert
  - name: geo
  - name: misc
  - name: singleThreaded

- name: linux-wt-repl
  display_name: 1-Node ReplSet Linux WT
  batchtime: 360 # 6 hours
  expansions:
    compile_flags: *compile_flags
    mongod_exec_wrapper: *exec_wrapper
    perf_exec_wrapper: *perf_wrapper
    mongod_flags: "--replSet=test --storageEngine=wiredTiger --logpath ./mongod.log --fork --syncdelay 0 --nojournal --setParameter ttlMonitorEnabled=false --wiredTigerCacheSizeGB 16 --oplogSize 100000 --setParameter diagnosticDataCollectionEnabled=false "
    project: *project
  run_on:
  - "centos6-perf"
  tasks:
  - name: update
  - name: insert
  - name: misc
  - name: singleThreaded

- name: linux-mmap-repl
  display_name: 1-Node ReplSet Linux MMAPv1
  batchtime: 360 # 6 hours
  expansions:
    compile_flags: *compile_flags
    mongod_exec_wrapper: *exec_wrapper
    perf_exec_wrapper: *perf_wrapper
    mongod_flags: "--replSet=test --storageEngine=mmapv1 --logpath ./mongod.log --fork --syncdelay 0 --nojournal --setParameter ttlMonitorEnabled=false --oplogSize 100000 --setParameter diagnosticDataCollectionEnabled=false"
    project: *project
  run_on:
  - "centos6-perf"
  tasks:
  - name: update
  - name: insert
  - name: misc
  - name: singleThreaded

- name: linux-wt-repl-compare
  display_name: 1-Node ReplSet Linux WT Comparison Standalone
  batchtime: 360 # 6 hours
  expansions:
    project: *project
  run_on:
  - "centos6-perf"
#    - "ubuntu1404-docker"
  tasks:
  - name: singleThreaded-wt-repl-comp
  - name: insert-wt-repl-comp
  - name: update-wt-repl-comp
  - name: misc-wt-repl-comp

- name: linux-mmap-repl-compare
  display_name: 1-Node ReplSet Linux MMAPv1 Comparison Standalone
  batchtime: 360 # 6 hours
  expansions:
    project: *project
  run_on:
  - "centos6-perf"
#    - "ubuntu1404-docker"
  tasks:
  - name: singleThreaded-mmap-repl-comp
  - name: insert-mmap-repl-comp
  - name: update-mmap-repl-comp
  - name: misc-mmap-repl-comp