command_type: system stepback: false pre: - command: shell.exec params: silent: true script: | ${killall_mci|pkill -9 mongod; pkill -9 mongos; pkill -9 mongo; pkill -9 bsondump; pkill -9 mongoimport; pkill -9 mongoexport; pkill -9 mongodump; pkill -9 mongorestore; pkill -9 mongostat; pkill -9 mongofiles; pkill -9 mongotop; pkill -9 mongobridge; pkill -9 mongod-2.6; pkill -9 mongos-2.6; pkill -9 mongo-2.6; pkill -9 bsondump-2.6; pkill -9 mongoimport-2.6; pkill -9 mongoexport-2.6; pkill -9 mongodump-2.6; pkill -9 mongorestore-2.6; pkill -9 mongostat-2.6; pkill -9 mongofiles-2.6; pkill -9 mongotop-2.6; pkill -9 mongobridge-2.6; pkill -9 mongod-2.4; pkill -9 mongos-2.4; pkill -9 mongo-2.4; pkill -9 bsondump-2.4; pkill -9 mongoimport-2.4; pkill -9 mongoexport-2.4; pkill -9 mongodump-2.4; pkill -9 mongorestore-2.4; pkill -9 mongostat-2.4; pkill -9 mongofiles-2.4; pkill -9 mongotop-2.4; pkill -9 resmoke.py; pkill -9 python; pkill -9 python2; pkill -9 lldb; pkill -9 _test} >/dev/null 2>&1 exit 0 post: - command: attach.results params: file_location: src/report.json - command: s3.put params: aws_key: ${aws_key} aws_secret: ${aws_secret} local_file: src/mongod.log remote_file: ${project}/${build_variant}/${revision}/${task_id}/${version_id}/logs/mongod-${build_id}.log bucket: mciuploads permissions: public-read content_type: ${content_type|text/plain} display_name: mongod.log - command: shell.exec params: silent: true script: | ${killall_mci|pkill -9 mongod; pkill -9 mongos; pkill -9 mongo; pkill -9 bsondump; pkill -9 mongoimport; pkill -9 mongoexport; pkill -9 mongodump; pkill -9 mongorestore; pkill -9 mongostat; pkill -9 mongofiles; pkill -9 mongotop; pkill -9 mongobridge; pkill -9 mongod-2.6; pkill -9 mongos-2.6; pkill -9 mongo-2.6; pkill -9 bsondump-2.6; pkill -9 mongoimport-2.6; pkill -9 mongoexport-2.6; pkill -9 mongodump-2.6; pkill -9 mongorestore-2.6; pkill -9 mongostat-2.6; pkill -9 mongofiles-2.6; pkill -9 mongotop-2.6; pkill -9 mongobridge-2.6; pkill -9 mongod-2.4; pkill -9 mongos-2.4; pkill -9 mongo-2.4; pkill -9 bsondump-2.4; pkill -9 mongoimport-2.4; pkill -9 mongoexport-2.4; pkill -9 mongodump-2.4; pkill -9 mongorestore-2.4; pkill -9 mongostat-2.4; pkill -9 mongofiles-2.4; pkill -9 mongotop-2.4; pkill -9 resmoke.py; pkill -9 python; pkill -9 python2; pkill -9 lldb; pkill -9 _test} >/dev/null 2>&1 exit 0 - command: shell.exec params: working_dir: src script: | # removes files from the (local) scons cache when it's over a # threshold, to the $prune_ratio percentage. Ideally override # these default values in the distro config in evergreen. if [ -d "${scons_cache_path}" ]; then ${python|/opt/mongodbtoolchain/v2/bin/python2} buildscripts/scons_cache_prune.py --cache-dir ${scons_cache_path} --cache-size ${scons_cache_size|200} --prune-ratio ${scons_prune_ratio|0.8} fi modules: - name: enterprise repo: git@github.com:10gen/mongo-enterprise-modules.git prefix: src/mongo/db/modules branch: v3.6 functions: "download analysis scripts": - command: shell.exec params: script: | set -v rm -rf ./dsi mkdir -p ./src git clone git@github.com:10gen/dsi.git # get the mongo source, note the s3.get calls put the # exe files in their respective make locations. - command: git.get_project params: directory: src "start server": - command: shell.exec params: script: | rm -rf ./* mkdir src - command: s3.get params: aws_key: ${aws_key} aws_secret: ${aws_secret} remote_file: ${project}/${version_id}/${revision}/mongod-${version_id} bucket: mciuploads local_file: src/mongod - command: s3.get params: aws_key: ${aws_key} aws_secret: ${aws_secret} remote_file: ${project}/${version_id}/${revision}/mongo-${version_id} bucket: mciuploads local_file: src/mongo - command: shell.exec params: working_dir: src script: | set -e set -v chmod +x mongod chmod +x mongo git clone git@github.com:mongodb/mongo-perf.git perf cd perf git describe --tags - command: shell.exec params: background: true working_dir: src script: | set -e set -o verbose mkdir -p ./dbdata ${mongod_exec_wrapper} ./mongod --dbpath ./dbdata ${mongod_flags} - command : shell.exec params: working_dir: src script: | set -e set -o verbose sleep 5 # if we started a replset, initiate it and wait for it to become primary # # Note: This process is always currently started with --nojournal (not a recommended production configuration, see # https://docs.mongodb.com/manual/tutorial/manage-journaling/#disable-journaling). # As a result, writeConcernMajorityJournalDefault can be set to false. If this becomes configurable later # then the correct value should be passed to rs.initiate or getCmdLineOpts needs to interrogated (but # only after db.createUser). ./mongo --eval "if(db.isMaster().isreplicaset){\ rs.initiate({_id: 'test', version: 1, members: [ { _id: 0, host : 'localhost:27017' }], writeConcernMajorityJournalDefault:false});\ assert.soon(function(){return db.isMaster().ismaster}, 'no primary');\ }" # benchRun() authenticates against the admin db, with a user that must has admin access. # Note: This is possibly a legacy requirement from times when it would call serverStatus. # Btw, when mongod is started without --auth, these should be harmless no-ops ./mongo --eval "db.createUser({user: 'admin', pwd: 'password', roles:\ [ { role: 'root', db: 'admin' } ] })"\ admin # print the replset config unless this is a standalone ./mongo --eval "if( db.isMaster().hosts ) { printjson(rs.config()); }" --username admin --password password admin echo "MONGOD STARTED." "analyze": - command: shell.exec params: working_dir: src script: | set -e set -v ../dsi/run-dsi setup - command: shell.exec params: working_dir: src script: | cat > bootstrap.yml < runtime.yml < runtime_secret.yml <- --auth --fork --inMemoryEngineConfigString 'eviction=(threads_min=1),' --inMemorySizeGB 60 --logpath ./mongod.log --networkMessageCompressors noop --setParameter diagnosticDataCollectionEnabled=false --setParameter enableTestCommands=1 --setParameter ttlMonitorEnabled=false --storageEngine inMemory --syncdelay 0 use_scons_cache: true project: &project perf run_on: - "centos6-perf" tasks: - name: compile distros: - rhel62-large - name: aggregation - name: agg-query-comparison - name: query - name: views-aggregation - name: views-query - name: where - name: update - name: insert - name: geo - name: misc - name: singleThreaded - name: linux-mmap-standalone display_name: Standalone Linux MMAPv1 batchtime: 90 # 1.5 hours expansions: mongod_exec_wrapper: *exec_wrapper perf_exec_wrapper: *perf_wrapper mongod_flags: >- --auth --fork --logpath ./mongod.log --networkMessageCompressors noop --nojournal --setParameter diagnosticDataCollectionEnabled=false --setParameter enableTestCommands=1 --setParameter ttlMonitorEnabled=false --storageEngine mmapv1 --syncdelay 0 project: *project run_on: - "centos6-perf" tasks: - name: aggregation - name: agg-query-comparison - name: query - name: views-aggregation - name: views-query - name: where - name: update - name: insert - name: geo - name: misc - name: singleThreaded - name: linux-wt-repl display_name: 1-Node ReplSet Linux inMemory batchtime: 90 # 1.5 hours expansions: mongod_exec_wrapper: *exec_wrapper perf_exec_wrapper: *perf_wrapper mongod_flags: >- --auth --fork --inMemoryEngineConfigString 'eviction=(threads_min=1),' --inMemorySizeGB 60 --logpath ./mongod.log --networkMessageCompressors noop --oplogSize 30000 --replSet test --setParameter diagnosticDataCollectionEnabled=false --setParameter enableTestCommands=1 --setParameter ttlMonitorEnabled=false --storageEngine inMemory --syncdelay 0 project: *project run_on: - "centos6-perf" tasks: - name: update - name: insert - name: misc - name: singleThreaded - name: linux-mmap-repl display_name: 1-Node ReplSet Linux MMAPv1 batchtime: 90 # 1.5 hours expansions: mongod_exec_wrapper: *exec_wrapper perf_exec_wrapper: *perf_wrapper mongod_flags: >- --auth --fork --logpath ./mongod.log --networkMessageCompressors noop --nojournal --oplogSize 100000 --replSet test --setParameter diagnosticDataCollectionEnabled=false --setParameter enableTestCommands=1 --setParameter ttlMonitorEnabled=false --storageEngine mmapv1 --syncdelay 0 project: *project run_on: - "centos6-perf" tasks: - name: update - name: insert - name: misc - name: singleThreaded