summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--.pylintrc3
-rwxr-xr-xbuildscripts/generate-pip-constraints.sh101
-rw-r--r--buildscripts/linter/runner.py4
-rw-r--r--buildscripts/requirements.txt20
-rw-r--r--buildscripts/resmokelib/requirements.txt7
-rw-r--r--docs/building.md2
-rw-r--r--etc/evergreen.yml160
-rw-r--r--etc/pip/README.md78
-rw-r--r--etc/pip/compile-requirements.txt4
-rw-r--r--etc/pip/components/aws.req3
-rw-r--r--etc/pip/components/compile.req4
-rw-r--r--etc/pip/components/core.req5
-rw-r--r--etc/pip/components/jiraclient.req2
-rw-r--r--etc/pip/components/lint.req6
-rw-r--r--etc/pip/components/mypy.req1
-rw-r--r--etc/pip/components/platform.req4
-rw-r--r--etc/pip/components/resmoke.req3
-rw-r--r--etc/pip/constraints.txt45
-rw-r--r--etc/pip/core-requirements.txt2
-rw-r--r--etc/pip/dev-requirements.txt6
-rw-r--r--etc/pip/evgtest-requirements.txt5
-rw-r--r--etc/pip/jira-requirements.txt4
-rw-r--r--etc/pip/lint-requirements.txt5
-rw-r--r--etc/pip/powercycle-requirements.txt4
-rw-r--r--etc/pip/toolchain-requirements.txt14
-rw-r--r--pytests/requirements.txt4
26 files changed, 395 insertions, 101 deletions
diff --git a/.pylintrc b/.pylintrc
index 7bd38008874..9784ddd8105 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -23,3 +23,6 @@ variable-rgx=[a-z_][a-z0-9_]{1,50}$
# W0611 - unused-import - typing module is needed for mypy
disable=bad-continuation,fixme,import-error,line-too-long,no-member,locally-disabled,redefined-variable-type,too-few-public-methods,unused-import
+
+[IMPORTS]
+known-third-party=boto3,botocore,psutil,yaml
diff --git a/buildscripts/generate-pip-constraints.sh b/buildscripts/generate-pip-constraints.sh
new file mode 100755
index 00000000000..37dba0e4775
--- /dev/null
+++ b/buildscripts/generate-pip-constraints.sh
@@ -0,0 +1,101 @@
+#!/bin/bash
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname ${BASH_SOURCE[0]})" && pwd)"
+DEFAULT_WORKING_DIR="${SCRIPT_DIR}/../build/pip"
+
+showUsage() {
+ cat <<EOF
+USAGE:
+ generate-pip-constraints.sh [-o CON_FILE] ...
+ generate-pip-constraints.sh -h
+
+ -h, --help Show this message
+ -o CON_FILE Write constraints.txt to CON_FILE
+
+This command passes all unrecognized arguments to two pip invocations,
+one for a python2 virtual environment and one for a python3 virtual environment.
+It then forms a unified multi-version constraints file at constraints.txt in its working directory.
+
+This script's working directory currently defaults to '${DEFAULT_WORKING_DIR}'.
+This default can be overriden via \$WORKING_DIR.
+EOF
+}
+
+CON_FILE=""
+ARGS=()
+while [[ $# -gt 0 ]]
+do KEY="${1}"
+ case "$KEY" in
+ (-h|--help) showUsage; exit 0;;
+ (-o) CON_FILE="${2}"; shift 2;;
+ (*) ARGS+=("${KEY}"); shift;;
+ esac
+done
+
+if [[ ${#ARGS} -eq 0 ]]; then
+ 1>&2 echo "No pip arguments given. Failing..."
+ exit 2
+fi
+
+generateConstraints(){
+ EXE="${1}"
+ DIR="${2}"
+ if ! (
+ export VIRTUAL_ENV_DISABLE_PROMPT=yes
+ virtualenv --python "${EXE}" "${DIR}"
+ . "${DIR}"/*/activate
+ pip install "${ARGS[@]}"
+ pip freeze >"${DIR}/requirements.txt"
+ )
+ then RC=$?
+ 1>&2 echo "Errors occured while attempting
+ to install all requirements into '${DIR}'
+ with python executable '${EXE}'"
+ return $RC
+ fi
+}
+
+WORKING_DIR="${WORKING_DIR:-${DEFAULT_WORKING_DIR}}"
+if [[ -d $WORKING_DIR ]]; then
+ 1>&2 echo "Removing existing working dir at '$WORKING_DIR'..."
+ rm -r "${WORKING_DIR}"
+fi
+ABSOLUTE_WORKING_DIR="$(mkdir -p "${WORKING_DIR}" && cd "${WORKING_DIR}" && pwd)"
+
+PIP2_DIR="${ABSOLUTE_WORKING_DIR}/python2"
+PIP3_DIR="${ABSOLUTE_WORKING_DIR}/python3"
+
+generateConstraints python2 "${PIP2_DIR}"
+generateConstraints python3 "${PIP3_DIR}"
+
+if [[ -z $CON_FILE ]]; then
+ CON_FILE="${ABSOLUTE_WORKING_DIR}/constraints.txt"
+fi
+(
+ printf '# == PLEASE DO NOT MANUALLY EDIT THIS FILE =='
+ printf '\n# For more details, see etc/pip/README.md'
+ printf '\n#'
+ printf '\n# This file was generated via the following command:'
+ printf "\n# $ 'bash' 'buildscripts/generate-pip-constraints.sh'"
+ printf " '%s'" "${ARGS[@]}"
+ printf '\n'
+
+ printf '\n# Common requirements\n'
+ comm -12 "${PIP2_DIR}/requirements.txt" "${PIP3_DIR}/requirements.txt"
+
+ printf '\n# Python2 requirements\n'
+ comm -23 "${PIP2_DIR}/requirements.txt" "${PIP3_DIR}/requirements.txt" |
+ sed -e 's/$/; python_version < "3"/'
+
+ printf '\n# Python3 requirements\n'
+ comm -13 "${PIP2_DIR}/requirements.txt" "${PIP3_DIR}/requirements.txt" |
+ sed -e 's/$/; python_version > "3"/'
+
+ printf '\n'
+ cat "${SCRIPT_DIR}/../etc/pip/components/platform.req"
+) >"${CON_FILE}"
+
+1>&2 echo "All pip requirements were successfully installed in a virtual environment.
+See '${CON_FILE}' for all installed packages."
diff --git a/buildscripts/linter/runner.py b/buildscripts/linter/runner.py
index 67c69d25d02..e93ccd63a4c 100644
--- a/buildscripts/linter/runner.py
+++ b/buildscripts/linter/runner.py
@@ -131,8 +131,8 @@ Could not find the correct version of linter '%s', expected '%s'. Check your
PATH environment variable or re-run with --verbose for more information.
To fix, install the needed python modules for both Python 2.7, and Python 3.x:
- sudo pip2 install -r buildscripts/requirements.txt
- sudo pip3 install -r buildscripts/requirements.txt
+ sudo pip2 install -r etc/pip/lint-requirements.txt
+ sudo pip3 install -r etc/pip/lint-requirements.txt
These commands are typically available via packages with names like python-pip,
python2-pip, and python3-pip. See your OS documentation for help.
diff --git a/buildscripts/requirements.txt b/buildscripts/requirements.txt
deleted file mode 100644
index dac9d36b45f..00000000000
--- a/buildscripts/requirements.txt
+++ /dev/null
@@ -1,20 +0,0 @@
-# Jira integration
-cryptography == 2.0
-jira == 1.0.10
-pyjwt == 1.5.3
-# Other
-pyyaml == 3.13
-unittest-xml-reporting == 2.1.0
-# Linters
-yapf == 0.21.0
-mypy == 0.580 ; python_version > "3"
-# typing in Python 2 for mypy
-typing == 3.6.1; python_version < "3"
-pylint == 1.8.3
-pydocstyle == 2.1.1
-# resmoke.py
--r resmokelib/requirements.txt
-# generate_error_codes.py
-cheetah3 == 3.0.0; python_version < "3"
-jinja2 == 2.10
-mock == 2.0.0
diff --git a/buildscripts/resmokelib/requirements.txt b/buildscripts/resmokelib/requirements.txt
deleted file mode 100644
index f7be0432c6b..00000000000
--- a/buildscripts/resmokelib/requirements.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-mock == 2.0.0 ; python_version < "3"
-pymongo >= 3.0, ~= 3.6.0
-pypiwin32 == 219 ; sys_platform == "win32" and python_version < "3"
-pypiwin32 == 223 ; sys_platform == "win32" and python_version > "3"
-PyYAML == 3.13
-requests >= 2.16.1
-subprocess32 >= 3.2.7 ; os_name == "posix" and python_version < "3"
diff --git a/docs/building.md b/docs/building.md
index 8fdc2f8a766..960cd991ea6 100644
--- a/docs/building.md
+++ b/docs/building.md
@@ -33,7 +33,7 @@ Python Prerequisites
In order to build MongoDB, Python 2.7.x is required, and several Python modules. To install
the required Python modules, run:
- $ pip2 install -r buildscripts/requirements.txt
+ $ pip2 install -r etc/pip/compile-requirements.txt
Note: If the `pip2` command is not available, `pip` without a suffix may be the pip command
associated with Python 2.7.x.
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 0a438ffd8a8..6dc486bbd90 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -455,6 +455,7 @@ variables:
# spawning a large number of linker processes.
num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
python: python
+ python3: '/cygdrive/c/python/python36/python.exe'
num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
ext: zip
use_scons_cache: true
@@ -1295,22 +1296,38 @@ functions:
command: shell.exec
type: test
params:
+ shell: bash
script: |
# exit immediately if virtualenv is not found
set -o errexit
set -o verbose
- python_loc=$(which ${python|/opt/mongodbtoolchain/v2/bin/python2})
- python3_loc=$(which ${python|/opt/mongodbtoolchain/v2/bin/python3})
virtualenv_loc=$(which ${virtualenv|virtualenv})
- if [ "Windows_NT" = "$OS" ]; then
- python_loc=$(cygpath -w $python_loc)
- python3_loc=$(cygpath -w c:/python/Python36/python.exe)
+
+ python2_loc=$(which ${python|/opt/mongodbtoolchain/v2/bin/python2})
+ python3_loc=$(which ${python3|/opt/mongodbtoolchain/v2/bin/python3})
+ venv2_dir="${workdir}/venv"
+ venv3_dir="${workdir}/venv_3"
+ if command -V cygpath; then
+ # Sad note: We have to use the Windows path instead of the posix path here.
+ # Otherwise, virtualenv may mistakenly resolve paths relative to c:\cygdrive.
+ python2_loc=$(cygpath -w $python2_loc)
+ python3_loc=$(cygpath -w $python3_loc)
+ venv2_dir="$(cygpath -w "$venv2_dir")"
+ venv3_dir="$(cygpath -w "$venv3_dir")"
fi
- # Set up virtualenv in ${workdir}
- "$virtualenv_loc" --python "$python_loc" --system-site-packages "${workdir}/venv"
- # Add virtualenv for python3 in ${workdir}
- "$virtualenv_loc" --python "$python3_loc" --system-site-packages "${workdir}/venv_3"
+
+ # Set up virtualenvs in ${workdir}
+ "$virtualenv_loc" --python "$python2_loc" --system-site-packages "$venv2_dir"
+ "$virtualenv_loc" --python "$python3_loc" --system-site-packages "$venv3_dir"
+
+ # Link python2/python3 in the primary virtualenv bin dir
+ export VIRTUAL_ENV_DISABLE_PROMPT=yes
+ venv2_bin="$(dirname "$(cd "$venv2_dir"; . ./*/activate; which python)")"
+ py2_exe="$(cd "$venv2_dir"; . ./*/activate; which python)"
+ py3_exe="$(cd "$venv3_dir"; . ./*/activate; which python)"
+ if [[ ! -f $venv2_bin/python2 ]]; then ln -sfv "$py2_exe" "$venv2_bin/python2"; fi
+ if [[ ! -f $venv2_bin/python3 ]]; then ln -sfv "$py3_exe" "$venv2_bin/python3"; fi
"run tests" :
- *determine_task_timeout
@@ -1339,8 +1356,7 @@ functions:
# activate the virtualenv if it has been set up
${activate_virtualenv}
- pip install boto3==1.5.27
- pip install mock==2.0.0
+ python2 -m pip install -r etc/pip/evgtest-requirements.txt
# Set the TMPDIR environment variable to be a directory in the task's working
# directory so that temporary files created by processes spawned by resmoke.py get
@@ -1504,6 +1520,7 @@ functions:
type: test
params:
working_dir: src
+ shell: bash
script: |
set -o errexit
set -o verbose
@@ -1520,12 +1537,15 @@ functions:
fi
${activate_virtualenv}
- set +o errexit
- ${compile_env|} $python ./buildscripts/scons.py ${compile_flags|} ${task_compile_flags|} ${task_compile_flags_extra|} ${scons_cache_args|} $extra_args ${targets} ${additional_targets|} MONGO_VERSION=${version}
- exit_status=$?
+
+ python2 -m pip install -r ./etc/pip/compile-requirements.txt
+ ${compile_env|} $python ./buildscripts/scons.py \
+ ${compile_flags|} ${task_compile_flags|} ${task_compile_flags_extra|} \
+ ${scons_cache_args|} $extra_args \
+ ${targets} ${additional_targets|} MONGO_VERSION=${version} || exit_status=$?
# If compile fails we do not run any tests
- if [ $exit_status -ne 0 ]; then
- if [ ${dump_scons_config_on_failure} = 'true' ]; then
+ if [[ $exit_status -ne 0 ]]; then
+ if [[ "${dump_scons_config_on_failure}" == true ]]; then
echo "Dumping build/scons/config.log"
cat build/scons/config.log
fi
@@ -1929,7 +1949,7 @@ functions:
${activate_virtualenv}
# The Windows build variants are running python 2.7.3 and require TLS 1.2 from pyOpenSSL
- pip install 'pyOpenSSL ; sys_platform == "win32" or sys_platform == "cygwin"'
+ python2 -m pip install 'pyOpenSSL ; sys_platform == "win32" or sys_platform == "cygwin"'
rm -rf /data/install /data/multiversion
$python buildscripts/setup_multiversion_mongodb.py \
@@ -2132,8 +2152,7 @@ functions:
set -o errexit
${activate_virtualenv}
- pip install -r buildscripts/requirements.txt
- pip install -r pytests/requirements.txt
+ python2 -m pip install -r etc/pip/powercycle-requirements.txt
if [ ! -z "${subnet_id}" ]; then
subnet_id="-n ${subnet_id}"
@@ -2261,7 +2280,7 @@ functions:
script: |
set -o errexit
# Copy buildscripts, pytests and mongoDB executables to the remote host.
- file_param="--file buildscripts --file pytests"
+ file_param="--file etc --file buildscripts --file pytests"
mongo_executables="mongo mongod mongos"
for executable in $mongo_executables
do
@@ -2291,7 +2310,7 @@ functions:
cmds="$cmds; virtualenv --python \$python_loc --system-site-packages ${virtualenv_dir|venv}"
cmds="$cmds; activate=\$(find ${virtualenv_dir|venv} -name 'activate')"
cmds="$cmds; . \$activate"
- cmds="$cmds; pip install -r \$remote_dir/pytests/requirements.txt"
+ cmds="$cmds; pip2 install -r \$remote_dir/etc/pip/powercycle-requirements.txt"
ssh_connection_options="${ssh_identity} ${ssh_connection_options}"
${activate_virtualenv}
$python buildscripts/remote_operations.py \
@@ -2767,7 +2786,7 @@ functions:
python=python
else
if [ "Windows_NT" = "$OS" ]; then
- python=/cygdrive/c/python/Python36/python
+ python=/cygdrive/c/python/python36/python
else
python=${python3|/opt/mongodbtoolchain/v2/bin/python3}
fi
@@ -3466,6 +3485,8 @@ tasks:
- "*Test"
- "./**.pdb"
- "./**.msi"
+ - "./etc/pip/**"
+ - "./etc/scons/**"
- "./etc/*san.suppressions"
- "./etc/repo_config.yaml"
- "./etc/test_lifecycle.yml"
@@ -4182,19 +4203,58 @@ tasks:
set -o errexit
set -o verbose
- ### TODO: Remove python3 when mypy 0.580 is installed in the toolchain.
- # Since mypy requires python3, we need to activate the venv_3
- ${activate_virtualenv_3}
- pip install -r buildscripts/requirements.txt
- updated_mypy=$(PATH=$PATH:/opt/mongodbtoolchain/v2/bin which mypy)
- deactivate
- ###
${activate_virtualenv}
- # TODO: Remove once the linters have been updated on the variants.
- pip install -r buildscripts/requirements.txt
- # The linters require the modules be installed.
- pip install -r pytests/requirements.txt
- MYPY=$updated_mypy ${compile_env|} $python ./buildscripts/scons.py ${compile_flags|} --stack-size=1024 lint
+ python3 -m pip install -r etc/pip/lint-requirements.txt
+ python2 -m pip install -r etc/pip/lint-requirements.txt
+ export MYPY="$(
+ PATH+=':/opt/mongodbtoolchain/v2/bin'
+ if command -V cygpath 2>/dev/null; then
+ PATH+=":$(cypath "${workdir}")/venv_3/Scripts"
+ else
+ PATH+=":${workdir}/venv_3/bin"
+ fi
+ which mypy
+ )"
+ ${compile_env|} python2 ./buildscripts/scons.py ${compile_flags|} --stack-size=1024 lint
+
+- name: verify_pip
+ depends_on: []
+ commands:
+ - command: manifest.load
+ - func: "git get project"
+ - command: shell.exec
+ type: test
+ params:
+ working_dir: src
+ script: |
+ set -o errexit
+ set -o verbose
+
+ # Note: One pleasant side effect of this task is that it should pre warm pip's cache.
+ # For every pypa project that is part of our current toolchain, this will do nothing.
+ # For every pypa project that we are missing, the .whl file will be added to the pip cache.
+ # (See https://pip.pypa.io/en/latest/reference/pip_install/#caching)
+
+ ${activate_virtualenv}
+ # This installs the explicit project versions which would be installed in the toolchain
+ # from this patch
+ python2 -m pip install -r etc/pip/constraints.txt
+ python3 -m pip install -r etc/pip/constraints.txt
+
+ python2 -m pip freeze >requirements.txt.python2.old
+ python3 -m pip freeze >requirements.txt.python3.old
+
+ # This installs any requirements which are unsatisfied by the constraints.txt above
+ python2 -m pip install -r etc/pip/toolchain-requirements.txt
+ python3 -m pip install -r etc/pip/toolchain-requirements.txt
+
+ python2 -m pip freeze >requirements.txt.python2.new
+ python3 -m pip freeze >requirements.txt.python3.new
+
+ # Compare the old freezes to the new freezes.
+ # They should be the same if our constraints satisfy our toolchain requirements
+ diff -w requirements.txt.python2.old requirements.txt.python2.new
+ diff -w requirements.txt.python3.old requirements.txt.python3.new
- <<: *task_template
name: burn_in_tests
@@ -5061,16 +5121,6 @@ tasks:
name: ese
commands:
- func: "do setup"
- - command: shell.exec
- type: test
- params:
- working_dir: src
- script: |
- set -o errexit
- set -o verbose
-
- ${activate_virtualenv}
- python -m pip install -r src/mongo/db/modules/enterprise/jstests/encryptdb/libs/requirements.txt
- func: "run tests"
vars:
resmoke_args: --suites=ese --storageEngine=wiredTiger
@@ -7135,8 +7185,13 @@ tasks:
set -o verbose
${activate_virtualenv}
- if [ "Windows_NT" = "$OS" ]; then
- /cygdrive/c/python/python36/python.exe -m pip install -r buildscripts/requirements.txt
+ # Since the free monitoring tests use python 3 via C exec,
+ # they deliberately hit python3.6 on Windows and toolchain python otherwise.
+ # We happen to pre-seed the core requirements in the toolchain,
+ # but we have no way of doing so to Windows python3.6 installed via Chocolatey.
+ if command -V cygpath; then
+ echo "Installing core requirements into Python at '${python3}'"
+ "$(cygpath "${python3}")" -mpip install -r etc/pip/core-requirements.txt
fi
- func: "run tests"
vars:
@@ -7853,8 +7908,7 @@ tasks:
set -o verbose
${activate_virtualenv}
- # Install Python modules to support OAuth with pip until it is available in the toolchain.
- pip install cryptography==1.7.2 pyjwt==1.5.3
+ python2 -m pip install -r etc/pip/jira-requirements.txt
# We use a small batch size to avoid hitting the load balancer timeout if the Evergreen
# API query is not fast enough.
@@ -9492,6 +9546,7 @@ buildvariants:
# spawning a large number of linker processes.
num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
python: python
+ python3: '/cygdrive/c/python/python36/python.exe'
ext: zip
use_scons_cache: true
multiversion_platform: windows
@@ -9655,6 +9710,7 @@ buildvariants:
# spawning a large number of linker processes.
num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
python: python
+ python3: '/cygdrive/c/python/python36/python.exe'
num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
ext: zip
use_scons_cache: true
@@ -9675,12 +9731,14 @@ buildvariants:
- name: compile_TG
requires:
- name: burn_in_tests
+ - name: verify_pip
distros:
- windows-64-vs2015-large
- name: compile_benchmarks
distros:
- windows-64-vs2015-large
- name: burn_in_tests
+ - name: verify_pip
- name: audit
- name: auth_audit
- name: benchmarks_orphaned
@@ -9976,6 +10034,7 @@ buildvariants:
# spawning a large number of linker processes.
num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
python: python
+ python3: '/cygdrive/c/python/python36/python.exe'
num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
ext: zip
use_scons_cache: true
@@ -10008,6 +10067,7 @@ buildvariants:
# spawning a large number of linker processes.
num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
python: python
+ python3: '/cygdrive/c/python/python36/python.exe'
num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
test_flags: --storageEngine=inMemory --excludeWithAnyTags=requires_persistence,requires_journaling,uses_transactions
ext: zip
@@ -10083,6 +10143,7 @@ buildvariants:
# spawning a large number of linker processes.
num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
python: python
+ python3: '/cygdrive/c/python/python36/python.exe'
num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
ext: zip
use_scons_cache: true
@@ -10200,6 +10261,7 @@ buildvariants:
# spawning a large number of linker processes.
num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
python: python
+ python3: '/cygdrive/c/python/python36/python.exe'
num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
ext: zip
use_scons_cache: true
@@ -10891,6 +10953,7 @@ buildvariants:
- name: compile_all_run_unittests_TG
requires:
- name: burn_in_tests
+ - name: verify_pip
- name: lint
distros:
- rhel62-large
@@ -10901,6 +10964,7 @@ buildvariants:
- name: burn_in_tests
distros:
- rhel62-large
+ - name: verify_pip
- name: rollback_fuzzer
- name: rollback_fuzzer_clean_shutdowns
- name: rollback_fuzzer_unclean_shutdowns
diff --git a/etc/pip/README.md b/etc/pip/README.md
new file mode 100644
index 00000000000..473664eb6a4
--- /dev/null
+++ b/etc/pip/README.md
@@ -0,0 +1,78 @@
+## On requirements (`*-requirements.txt`) files
+
+MongoDB requires multiple pypa projects installed to build and test. To that end, we provide our own
+`*-requirements.txt` files for specific domains of use. Inside each requirements file, there are
+only include statements for component files. These files are the bare requirements for specific
+components of our python environment. This separation allows us to avoid repetition and conflict in
+our requirements across components.
+
+For most developers, if you pip-install `dev-requirements.txt`, you have the python requirements to
+lint, build, and test MongoDB.
+
+## On the constraints (`constraints.txt`) file
+
+Our requirements files are *minimally* constrained. For the majority of pypa projects, any
+given requirements file will install the newest version of the project. For increased stability,
+there is a `constraints.txt` file with explicit versions for each required project. This file
+represents a _somewhat_ portable manifest of the latest acceptable pypa projects version according
+to a set of local python environments. It is also used to install the site-packages in the MongoDB
+internal toolchain-builder.
+
+Please note that the `verify_pip` task in evergreen confirms that `constraints.txt` satisfies
+`toolchain-requirements.txt`. If you modify the pypa project requirements, you should regenerate the
+constraints file.
+
+## How to modify a pypa project requirement in a component
+
+The most common edit of our requirements is likely a change to the constraints on a pypa project
+that we already use. For example, say that we currently require `pymongo >= 3.0, < 3.6.0` in the
+component `core`. You would like to use PyMongo 3.7, so you instead modify the line in
+`etc/pip/components/core.req` to read `pymongo >= 3.0, != 3.6.0`. Since this is a modification to an
+existing component, you do not need to modify any requirements file. However, you do need to
+regenerate the constraints file. The workflow will usually look like:
+
+```
+$ # Make your changes to the component file
+$ $EDITOR etc/pip/components/core.req
+$ # Regenerate the constraints file
+$ bash buildscripts/generate-pip-constraints.sh -r etc/pip/toolchain-requirements.txt -o etc/pip/constraints.txt
+```
+
+## How to add a new component (`*.req`) file
+
+Occasionally, we will require a set of pypa projects for an entirely new piece of software in our
+repository. This usually implies adding a new component file. For example, say that we need to add
+a logging system to both local development and evergreen. This system requires the fictional pypa
+project `FooLog`. So we add a file `foolog.req` and require it from both `dev-requirements.txt` and
+`evgtest-requirements.txt`. Like the majority of our components, we want it in the toolchain, so we
+also add it to `toolchain-requirements.txt`. The workflow will usually look like:
+
+```
+$ # Make the component file
+$ echo "FooLog" >etc/pip/components/foolog.req
+$ # Require the component from the requirements files
+$ echo "-r components/foolog.req" >>etc/pip/dev-requirements.txt
+$ echo "-r components/foolog.req" >>etc/pip/evgtest-requirements.txt
+$ echo "-r components/foolog.req" >>etc/pip/toolchain-requirements.txt
+$ # Regenerate the constraints file
+$ bash buildscripts/generate-pip-constraints.sh -r etc/pip/toolchain-requirements.txt -o etc/pip/constraints.txt
+```
+
+## How to add a new requirements (`*-requirements.txt`) file
+
+Rarely, we will have an entirely new domain of requirements that is useful. In this case, we need to
+at least make a new requirements file. For example, say we want to make a requirements file for
+packaging our code. We would need most of the requirements for `dev-requirements.txt` but the
+testing has already been done in our continuous integration. So we create a new file
+`package-requirements.txt` and require a smaller subset of components. The new file at
+`etc/pip/package-requirements.txt` would look like this:
+```
+-r components/platform.req
+-r components/core.req
+
+-r components/compile.req
+-r components/lint.req
+```
+
+Notice that since we did not change any components files, we do not need to regenerate our
+constraints.
diff --git a/etc/pip/compile-requirements.txt b/etc/pip/compile-requirements.txt
new file mode 100644
index 00000000000..74b44861e27
--- /dev/null
+++ b/etc/pip/compile-requirements.txt
@@ -0,0 +1,4 @@
+-r components/platform.req
+-r components/core.req
+
+-r components/compile.req
diff --git a/etc/pip/components/aws.req b/etc/pip/components/aws.req
new file mode 100644
index 00000000000..3245054d700
--- /dev/null
+++ b/etc/pip/components/aws.req
@@ -0,0 +1,3 @@
+boto3
+botocore
+psutil
diff --git a/etc/pip/components/compile.req b/etc/pip/components/compile.req
new file mode 100644
index 00000000000..4f218e53ddf
--- /dev/null
+++ b/etc/pip/components/compile.req
@@ -0,0 +1,4 @@
+# Mongo compile
+Cheetah3; python_version < "3" # src/mongo/base/generate_error_codes.py
+regex
+typing; python_version < "3"
diff --git a/etc/pip/components/core.req b/etc/pip/components/core.req
new file mode 100644
index 00000000000..ac21ffd16a9
--- /dev/null
+++ b/etc/pip/components/core.req
@@ -0,0 +1,5 @@
+# Core (we need these for most builscripts)
+pip >= 18.0
+PyYAML >= 3.0.0
+requests >= 2.0.0
+pymongo >= 3.0, < 3.6.0 # See PYTHON-1434, SERVER-34820
diff --git a/etc/pip/components/jiraclient.req b/etc/pip/components/jiraclient.req
new file mode 100644
index 00000000000..54588334ace
--- /dev/null
+++ b/etc/pip/components/jiraclient.req
@@ -0,0 +1,2 @@
+# TIG jira integration
+jira
diff --git a/etc/pip/components/lint.req b/etc/pip/components/lint.req
new file mode 100644
index 00000000000..74f244638b7
--- /dev/null
+++ b/etc/pip/components/lint.req
@@ -0,0 +1,6 @@
+# Linters
+pylint
+yapf
+# typing in Python 2 for mypy
+typing; python_version < "3"
+pydocstyle
diff --git a/etc/pip/components/mypy.req b/etc/pip/components/mypy.req
new file mode 100644
index 00000000000..16bd01c175c
--- /dev/null
+++ b/etc/pip/components/mypy.req
@@ -0,0 +1 @@
+mypy == 0.620; python_version > "3.4"
diff --git a/etc/pip/components/platform.req b/etc/pip/components/platform.req
new file mode 100644
index 00000000000..3fb8a515910
--- /dev/null
+++ b/etc/pip/components/platform.req
@@ -0,0 +1,4 @@
+# Platform-specific components
+pypiwin32==219; sys_platform == "win32" and python_version < "3"
+pypiwin32==223; sys_platform == "win32" and python_version > "3"
+subprocess32==3.5.2; os_name == "posix" and platform_release != "2.6.18-194.el5xen" and platform_release != "2.6.18-274.el5xen" and python_version < "3"
diff --git a/etc/pip/components/resmoke.req b/etc/pip/components/resmoke.req
new file mode 100644
index 00000000000..9bcc7c6f684
--- /dev/null
+++ b/etc/pip/components/resmoke.req
@@ -0,0 +1,3 @@
+mock; python_version < "3"
+PyKMIP == 0.4.0; python_version < "3" # It's now 0.8.0. We're far enough back to have API conflicts.
+jinja2
diff --git a/etc/pip/constraints.txt b/etc/pip/constraints.txt
index 9fc75aab922..e7787ca71a4 100644
--- a/etc/pip/constraints.txt
+++ b/etc/pip/constraints.txt
@@ -1,59 +1,66 @@
+# == PLEASE DO NOT MANUALLY EDIT THIS FILE ==
+# For more details, see etc/pip/README.md
+#
+# This file was generated via the following command:
+# $ 'bash' 'buildscripts/generate-pip-constraints.sh' '-r' 'etc/pip/toolchain-requirements.txt'
+
# Common requirements
asn1crypto==0.24.0
-astroid==1.6.5
-boto3==1.5.27
-botocore==1.8.50
-certifi==2018.8.13
+boto3==1.9.13
+botocore==1.12.13
+certifi==2018.8.24
cffi==1.11.5
chardet==3.0.4
-cryptography==2.0
+cryptography==2.3.1
defusedxml==0.5.0
docutils==0.14
-enum34==1.1.6
idna==2.7
isort==4.3.4
Jinja2==2.10
-jira==1.0.10
+jira==2.0.0
jmespath==0.9.3
lazy-object-proxy==1.3.1
MarkupSafe==1.0
mccabe==0.6.1
-mock==2.0.0
oauthlib==2.1.0
pbr==4.2.0
-psutil==5.4.3
-pycparser==2.18
+psutil==5.4.7
+pycparser==2.19
pydocstyle==2.1.1
-PyJWT==1.5.3
-pylint==1.8.3
-pymongo==3.6.1
+PyJWT==1.6.4
+pymongo==3.5.1
python-dateutil==2.7.3
PyYAML==3.13
+regex==2018.8.29
requests==2.19.1
requests-oauthlib==1.0.0
requests-toolbelt==0.8.0
s3transfer==0.1.13
six==1.11.0
snowballstemmer==1.2.1
-unittest-xml-reporting==2.1.0
urllib3==1.23
wrapt==1.10.11
-yapf==0.21.0
+yapf==0.24.0
# Python2 requirements
+astroid==1.6.5; python_version < "3"
backports.functools-lru-cache==1.5; python_version < "3"
-Cheetah3==3.0.0; python_version < "3"
+Cheetah3==3.1.0; python_version < "3"
configparser==3.5.0; python_version < "3"
+enum34==1.1.6; python_version < "3"
funcsigs==1.0.2; python_version < "3"
futures==3.2.0; python_version < "3"
ipaddress==1.0.22; python_version < "3"
-ordereddict==1.1; python_version < "3"
+mock==2.0.0; python_version < "3"
PyKMIP==0.4.0; python_version < "3"
+pylint==1.9.3; python_version < "3"
singledispatch==3.4.0.3; python_version < "3"
-typing==3.6.1; python_version < "3"
+typing==3.6.6; python_version < "3"
# Python3 requirements
-mypy==0.580; python_version > "3"
+astroid==2.0.4; python_version > "3"
+mypy==0.620; python_version > "3"
+pylint==2.1.1; python_version > "3"
typed-ast==1.1.0; python_version > "3"
# Platform-specific components
diff --git a/etc/pip/core-requirements.txt b/etc/pip/core-requirements.txt
new file mode 100644
index 00000000000..daa77546015
--- /dev/null
+++ b/etc/pip/core-requirements.txt
@@ -0,0 +1,2 @@
+-r components/platform.req
+-r components/core.req
diff --git a/etc/pip/dev-requirements.txt b/etc/pip/dev-requirements.txt
new file mode 100644
index 00000000000..bd3471b61eb
--- /dev/null
+++ b/etc/pip/dev-requirements.txt
@@ -0,0 +1,6 @@
+-r components/platform.req
+-r components/core.req
+
+-r components/compile.req
+-r components/lint.req
+-r components/resmoke.req
diff --git a/etc/pip/evgtest-requirements.txt b/etc/pip/evgtest-requirements.txt
new file mode 100644
index 00000000000..42b183117d9
--- /dev/null
+++ b/etc/pip/evgtest-requirements.txt
@@ -0,0 +1,5 @@
+-r components/platform.req
+-r components/core.req
+
+-r components/resmoke.req
+-r components/aws.req
diff --git a/etc/pip/jira-requirements.txt b/etc/pip/jira-requirements.txt
new file mode 100644
index 00000000000..d18d5b36356
--- /dev/null
+++ b/etc/pip/jira-requirements.txt
@@ -0,0 +1,4 @@
+-r components/platform.req
+-r components/core.req
+
+-r components/jiraclient.req
diff --git a/etc/pip/lint-requirements.txt b/etc/pip/lint-requirements.txt
new file mode 100644
index 00000000000..8f136e14230
--- /dev/null
+++ b/etc/pip/lint-requirements.txt
@@ -0,0 +1,5 @@
+-r components/platform.req
+-r components/core.req
+
+-r components/lint.req
+-r components/mypy.req
diff --git a/etc/pip/powercycle-requirements.txt b/etc/pip/powercycle-requirements.txt
new file mode 100644
index 00000000000..f694454bc0b
--- /dev/null
+++ b/etc/pip/powercycle-requirements.txt
@@ -0,0 +1,4 @@
+-r components/platform.req
+-r components/core.req
+
+-r components/aws.req
diff --git a/etc/pip/toolchain-requirements.txt b/etc/pip/toolchain-requirements.txt
new file mode 100644
index 00000000000..a0f90baa433
--- /dev/null
+++ b/etc/pip/toolchain-requirements.txt
@@ -0,0 +1,14 @@
+# This file is specifically intended to pull in *all* of our requirements besides the platform
+# specific options in components/platform.req. The motivation for this structure is that pip
+# (including pip freeze) discards platform constraints. Thus generate-pip-constraints.sh always adds
+# platform.req to the end of any constraints.txt files it makes.
+
+-r components/core.req
+
+-r components/compile.req
+-r components/lint.req
+-r components/mypy.req
+-r components/resmoke.req
+
+-r components/aws.req
+-r components/jiraclient.req
diff --git a/pytests/requirements.txt b/pytests/requirements.txt
deleted file mode 100644
index 65703f4322e..00000000000
--- a/pytests/requirements.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-# powertest
-boto3 == 1.5.27
-psutil == 5.4.3
-pymongo >= 3.0, ~= 3.6.0