SERVER-36152 Update requirements.txt

This is the full upgrade and split of the majority of our pypa project
requirements.

26 files changed, 395 insertions, 101 deletions

diff --git a/.pylintrc b/.pylintrc
index 7bd38008874..9784ddd8105 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -23,3 +23,6 @@ variable-rgx=[a-z_][a-z0-9_]{1,50}$
 # W0611 - unused-import - typing module is needed for mypy
 
 disable=bad-continuation,fixme,import-error,line-too-long,no-member,locally-disabled,redefined-variable-type,too-few-public-methods,unused-import
+
+[IMPORTS]
+known-third-party=boto3,botocore,psutil,yaml
diff --git a/buildscripts/generate-pip-constraints.sh b/buildscripts/generate-pip-constraints.sh
new file mode 100755
index 00000000000..37dba0e4775
--- /dev/null
+++ b/buildscripts/generate-pip-constraints.sh
@@ -0,0 +1,101 @@
+#!/bin/bash
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname ${BASH_SOURCE[0]})" && pwd)"
+DEFAULT_WORKING_DIR="${SCRIPT_DIR}/../build/pip"
+
+showUsage() {
+    cat <<EOF
+USAGE:
+    generate-pip-constraints.sh [-o CON_FILE] ...
+    generate-pip-constraints.sh -h
+
+    -h, --help      Show this message
+    -o CON_FILE     Write constraints.txt to CON_FILE
+
+This command passes all unrecognized arguments to two pip invocations,
+one for a python2 virtual environment and one for a python3 virtual environment.
+It then forms a unified multi-version constraints file at constraints.txt in its working directory.
+
+This script's working directory currently defaults to '${DEFAULT_WORKING_DIR}'.
+This default can be overriden via \$WORKING_DIR.
+EOF
+}
+
+CON_FILE=""
+ARGS=()
+while [[ $# -gt 0 ]]
+do KEY="${1}"
+    case "$KEY" in
+        (-h|--help) showUsage; exit 0;;
+        (-o) CON_FILE="${2}"; shift 2;;
+        (*) ARGS+=("${KEY}"); shift;;
+    esac
+done
+
+if [[ ${#ARGS} -eq 0 ]]; then
+    1>&2 echo "No pip arguments given. Failing..."
+    exit 2
+fi
+
+generateConstraints(){
+    EXE="${1}"
+    DIR="${2}"
+    if ! (
+        export VIRTUAL_ENV_DISABLE_PROMPT=yes
+        virtualenv --python "${EXE}" "${DIR}"
+        . "${DIR}"/*/activate
+        pip install "${ARGS[@]}"
+        pip freeze >"${DIR}/requirements.txt"
+    )
+    then RC=$?
+        1>&2 echo "Errors occured while attempting
+          to install all requirements into '${DIR}'
+          with python executable '${EXE}'"
+        return $RC
+    fi
+}
+
+WORKING_DIR="${WORKING_DIR:-${DEFAULT_WORKING_DIR}}"
+if [[ -d $WORKING_DIR ]]; then
+    1>&2 echo "Removing existing working dir at '$WORKING_DIR'..."
+    rm -r "${WORKING_DIR}"
+fi
+ABSOLUTE_WORKING_DIR="$(mkdir -p "${WORKING_DIR}" && cd "${WORKING_DIR}" && pwd)"
+
+PIP2_DIR="${ABSOLUTE_WORKING_DIR}/python2"
+PIP3_DIR="${ABSOLUTE_WORKING_DIR}/python3"
+
+generateConstraints python2 "${PIP2_DIR}"
+generateConstraints python3 "${PIP3_DIR}"
+
+if [[ -z $CON_FILE ]]; then
+    CON_FILE="${ABSOLUTE_WORKING_DIR}/constraints.txt"
+fi
+(
+    printf '# == PLEASE DO NOT MANUALLY EDIT THIS FILE =='
+    printf '\n# For more details, see etc/pip/README.md'
+    printf '\n#'
+    printf '\n# This file was generated via the following command:'
+    printf "\n# $ 'bash' 'buildscripts/generate-pip-constraints.sh'"
+    printf " '%s'" "${ARGS[@]}"
+    printf '\n'
+
+    printf '\n# Common requirements\n'
+    comm -12 "${PIP2_DIR}/requirements.txt" "${PIP3_DIR}/requirements.txt"
+
+    printf '\n# Python2 requirements\n'
+    comm -23 "${PIP2_DIR}/requirements.txt" "${PIP3_DIR}/requirements.txt" |
+        sed -e 's/$/; python_version < "3"/'
+
+    printf '\n# Python3 requirements\n'
+    comm -13 "${PIP2_DIR}/requirements.txt" "${PIP3_DIR}/requirements.txt" |
+        sed -e 's/$/; python_version > "3"/'
+
+    printf '\n'
+    cat "${SCRIPT_DIR}/../etc/pip/components/platform.req"
+) >"${CON_FILE}"
+
+1>&2 echo "All pip requirements were successfully installed in a virtual environment.
+See '${CON_FILE}' for all installed packages."
diff --git a/buildscripts/linter/runner.py b/buildscripts/linter/runner.py
index 67c69d25d02..e93ccd63a4c 100644
--- a/buildscripts/linter/runner.py
+++ b/buildscripts/linter/runner.py
@@ -131,8 +131,8 @@ Could not find the correct version of linter '%s', expected '%s'. Check your
 PATH environment variable or re-run with --verbose for more information.
 
 To fix, install the needed python modules for both Python 2.7, and Python 3.x:
-   sudo pip2 install -r buildscripts/requirements.txt
-   sudo pip3 install -r buildscripts/requirements.txt
+   sudo pip2 install -r etc/pip/lint-requirements.txt
+   sudo pip3 install -r etc/pip/lint-requirements.txt
 
 These commands are typically available via packages with names like python-pip,
 python2-pip, and python3-pip. See your OS documentation for help.
diff --git a/buildscripts/requirements.txt b/buildscripts/requirements.txt
deleted file mode 100644
index dac9d36b45f..00000000000
--- a/buildscripts/requirements.txt
+++ /dev/null
@@ -1,20 +0,0 @@
-# Jira integration
-cryptography == 2.0
-jira == 1.0.10
-pyjwt == 1.5.3
-# Other
-pyyaml == 3.13
-unittest-xml-reporting == 2.1.0
-# Linters
-yapf == 0.21.0
-mypy == 0.580 ; python_version > "3"
-# typing in Python 2 for mypy
-typing == 3.6.1; python_version < "3"
-pylint == 1.8.3
-pydocstyle == 2.1.1
-# resmoke.py
--r resmokelib/requirements.txt
-# generate_error_codes.py
-cheetah3 == 3.0.0; python_version < "3"
-jinja2 == 2.10
-mock == 2.0.0
diff --git a/buildscripts/resmokelib/requirements.txt b/buildscripts/resmokelib/requirements.txt
deleted file mode 100644
index f7be0432c6b..00000000000
--- a/buildscripts/resmokelib/requirements.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-mock == 2.0.0 ; python_version < "3"
-pymongo >= 3.0, ~= 3.6.0
-pypiwin32 == 219 ; sys_platform == "win32" and python_version < "3"
-pypiwin32 == 223 ; sys_platform == "win32" and python_version > "3"
-PyYAML == 3.13
-requests >= 2.16.1
-subprocess32 >= 3.2.7 ; os_name == "posix" and python_version < "3"
diff --git a/docs/building.md b/docs/building.md
index 8fdc2f8a766..960cd991ea6 100644
--- a/docs/building.md
+++ b/docs/building.md
@@ -33,7 +33,7 @@ Python Prerequisites
 In order to build MongoDB, Python 2.7.x is required, and several Python modules. To install
 the required Python modules, run:
 
-    $ pip2 install -r buildscripts/requirements.txt
+    $ pip2 install -r etc/pip/compile-requirements.txt
 
 Note: If the `pip2` command is not available, `pip` without a suffix may be the pip command
 associated with Python 2.7.x.
diff --git a/etc/evergreen.yml b/etc/evergreen.yml
index 0a438ffd8a8..6dc486bbd90 100644
--- a/etc/evergreen.yml
+++ b/etc/evergreen.yml
@@ -455,6 +455,7 @@ variables:
     # spawning a large number of linker processes.
     num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
     python: python
+    python3: '/cygdrive/c/python/python36/python.exe'
     num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
     ext: zip
     use_scons_cache: true
@@ -1295,22 +1296,38 @@ functions:
     command: shell.exec
     type: test
     params:
+        shell: bash
         script: |
           # exit immediately if virtualenv is not found
           set -o errexit
           set -o verbose
 
-          python_loc=$(which ${python|/opt/mongodbtoolchain/v2/bin/python2})
-          python3_loc=$(which ${python|/opt/mongodbtoolchain/v2/bin/python3})
           virtualenv_loc=$(which ${virtualenv|virtualenv})
-          if [ "Windows_NT" = "$OS" ]; then
-            python_loc=$(cygpath -w $python_loc)
-            python3_loc=$(cygpath -w c:/python/Python36/python.exe)
+
+          python2_loc=$(which ${python|/opt/mongodbtoolchain/v2/bin/python2})
+          python3_loc=$(which ${python3|/opt/mongodbtoolchain/v2/bin/python3})
+          venv2_dir="${workdir}/venv"
+          venv3_dir="${workdir}/venv_3"
+          if command -V cygpath; then
+            # Sad note: We have to use the Windows path instead of the posix path here.
+            # Otherwise, virtualenv may mistakenly resolve paths relative to c:\cygdrive.
+            python2_loc=$(cygpath -w $python2_loc)
+            python3_loc=$(cygpath -w $python3_loc)
+            venv2_dir="$(cygpath -w "$venv2_dir")"
+            venv3_dir="$(cygpath -w "$venv3_dir")"
           fi
-          # Set up virtualenv in ${workdir}
-          "$virtualenv_loc" --python "$python_loc" --system-site-packages "${workdir}/venv"
-          # Add virtualenv for python3 in ${workdir}
-          "$virtualenv_loc" --python "$python3_loc" --system-site-packages "${workdir}/venv_3"
+
+          # Set up virtualenvs in ${workdir}
+          "$virtualenv_loc" --python "$python2_loc" --system-site-packages "$venv2_dir"
+          "$virtualenv_loc" --python "$python3_loc" --system-site-packages "$venv3_dir"
+
+          # Link python2/python3 in the primary virtualenv bin dir
+          export VIRTUAL_ENV_DISABLE_PROMPT=yes
+          venv2_bin="$(dirname "$(cd "$venv2_dir"; . ./*/activate; which python)")"
+          py2_exe="$(cd "$venv2_dir"; . ./*/activate; which python)"
+          py3_exe="$(cd "$venv3_dir"; . ./*/activate; which python)"
+          if [[ ! -f $venv2_bin/python2 ]]; then ln -sfv "$py2_exe" "$venv2_bin/python2"; fi
+          if [[ ! -f $venv2_bin/python3 ]]; then ln -sfv "$py3_exe" "$venv2_bin/python3"; fi
 
   "run tests" :
     - *determine_task_timeout
@@ -1339,8 +1356,7 @@ functions:
 
           # activate the virtualenv if it has been set up
           ${activate_virtualenv}
-          pip install boto3==1.5.27
-          pip install mock==2.0.0
+          python2 -m pip install -r etc/pip/evgtest-requirements.txt
 
           # Set the TMPDIR environment variable to be a directory in the task's working
           # directory so that temporary files created by processes spawned by resmoke.py get
@@ -1504,6 +1520,7 @@ functions:
     type: test
     params:
       working_dir: src
+      shell: bash
       script: |
         set -o errexit
         set -o verbose
@@ -1520,12 +1537,15 @@ functions:
         fi
 
         ${activate_virtualenv}
-        set +o errexit
-        ${compile_env|} $python ./buildscripts/scons.py ${compile_flags|} ${task_compile_flags|} ${task_compile_flags_extra|} ${scons_cache_args|} $extra_args ${targets} ${additional_targets|} MONGO_VERSION=${version}
-        exit_status=$?
+
+        python2 -m pip install -r ./etc/pip/compile-requirements.txt
+        ${compile_env|} $python ./buildscripts/scons.py                                     \
+            ${compile_flags|} ${task_compile_flags|} ${task_compile_flags_extra|}           \
+            ${scons_cache_args|} $extra_args                                                \
+            ${targets} ${additional_targets|} MONGO_VERSION=${version} || exit_status=$?
         # If compile fails we do not run any tests
-        if [ $exit_status -ne 0 ]; then
-          if [ ${dump_scons_config_on_failure} = 'true' ]; then
+        if [[ $exit_status -ne 0 ]]; then
+          if [[ "${dump_scons_config_on_failure}" == true ]]; then
             echo "Dumping build/scons/config.log"
             cat build/scons/config.log
           fi
@@ -1929,7 +1949,7 @@ functions:
 
         ${activate_virtualenv}
         # The Windows build variants are running python 2.7.3 and require TLS 1.2 from pyOpenSSL
-        pip install 'pyOpenSSL ; sys_platform == "win32" or sys_platform == "cygwin"'
+        python2 -m pip install 'pyOpenSSL ; sys_platform == "win32" or sys_platform == "cygwin"'
 
         rm -rf /data/install /data/multiversion
         $python buildscripts/setup_multiversion_mongodb.py   \
@@ -2132,8 +2152,7 @@ functions:
 
           set -o errexit
           ${activate_virtualenv}
-          pip install -r buildscripts/requirements.txt
-          pip install -r pytests/requirements.txt
+          python2 -m pip install -r etc/pip/powercycle-requirements.txt
 
           if [ ! -z "${subnet_id}" ]; then
             subnet_id="-n ${subnet_id}"
@@ -2261,7 +2280,7 @@ functions:
         script: |
           set -o errexit
           # Copy buildscripts, pytests and mongoDB executables to the remote host.
-          file_param="--file buildscripts --file pytests"
+          file_param="--file etc --file buildscripts --file pytests"
           mongo_executables="mongo mongod mongos"
           for executable in $mongo_executables
           do
@@ -2291,7 +2310,7 @@ functions:
           cmds="$cmds; virtualenv --python \$python_loc --system-site-packages ${virtualenv_dir|venv}"
           cmds="$cmds; activate=\$(find ${virtualenv_dir|venv} -name 'activate')"
           cmds="$cmds; . \$activate"
-          cmds="$cmds; pip install -r \$remote_dir/pytests/requirements.txt"
+          cmds="$cmds; pip2 install -r \$remote_dir/etc/pip/powercycle-requirements.txt"
           ssh_connection_options="${ssh_identity} ${ssh_connection_options}"
           ${activate_virtualenv}
           $python buildscripts/remote_operations.py          \
@@ -2767,7 +2786,7 @@ functions:
             python=python
           else
             if [ "Windows_NT" = "$OS" ]; then
-              python=/cygdrive/c/python/Python36/python
+              python=/cygdrive/c/python/python36/python
             else
               python=${python3|/opt/mongodbtoolchain/v2/bin/python3}
             fi
@@ -3466,6 +3485,8 @@ tasks:
           - "*Test"
           - "./**.pdb"
           - "./**.msi"
+          - "./etc/pip/**"
+          - "./etc/scons/**"
           - "./etc/*san.suppressions"
           - "./etc/repo_config.yaml"
           - "./etc/test_lifecycle.yml"
@@ -4182,19 +4203,58 @@ tasks:
           set -o errexit
           set -o verbose
 
-          ### TODO: Remove python3 when mypy 0.580 is installed in the toolchain.
-          # Since mypy requires python3, we need to activate the venv_3
-          ${activate_virtualenv_3}
-          pip install -r buildscripts/requirements.txt
-          updated_mypy=$(PATH=$PATH:/opt/mongodbtoolchain/v2/bin which mypy)
-          deactivate
-          ###
           ${activate_virtualenv}
-          # TODO: Remove once the linters have been updated on the variants.
-          pip install -r buildscripts/requirements.txt
-          # The linters require the modules be installed.
-          pip install -r pytests/requirements.txt
-          MYPY=$updated_mypy ${compile_env|} $python ./buildscripts/scons.py ${compile_flags|} --stack-size=1024 lint
+          python3 -m pip install -r etc/pip/lint-requirements.txt
+          python2 -m pip install -r etc/pip/lint-requirements.txt
+          export MYPY="$(
+            PATH+=':/opt/mongodbtoolchain/v2/bin'
+            if command -V cygpath 2>/dev/null; then
+                PATH+=":$(cypath "${workdir}")/venv_3/Scripts"
+            else
+                PATH+=":${workdir}/venv_3/bin"
+            fi
+            which mypy
+          )"
+          ${compile_env|} python2 ./buildscripts/scons.py ${compile_flags|} --stack-size=1024 lint
+
+- name: verify_pip
+  depends_on: []
+  commands:
+  - command: manifest.load
+  - func: "git get project"
+  - command: shell.exec
+    type: test
+    params:
+      working_dir: src
+      script: |
+        set -o errexit
+        set -o verbose
+
+        # Note: One pleasant side effect of this task is that it should pre warm pip's cache.
+        # For every pypa project that is part of our current toolchain, this will do nothing.
+        # For every pypa project that we are missing, the .whl file will be added to the pip cache.
+        # (See https://pip.pypa.io/en/latest/reference/pip_install/#caching)
+
+        ${activate_virtualenv}
+        # This installs the explicit project versions which would be installed in the toolchain
+        # from this patch
+        python2 -m pip install -r etc/pip/constraints.txt
+        python3 -m pip install -r etc/pip/constraints.txt
+
+        python2 -m pip freeze >requirements.txt.python2.old
+        python3 -m pip freeze >requirements.txt.python3.old
+
+        # This installs any requirements which are unsatisfied by the constraints.txt above
+        python2 -m pip install -r etc/pip/toolchain-requirements.txt
+        python3 -m pip install -r etc/pip/toolchain-requirements.txt
+
+        python2 -m pip freeze >requirements.txt.python2.new
+        python3 -m pip freeze >requirements.txt.python3.new
+
+        # Compare the old freezes to the new freezes.
+        # They should be the same if our constraints satisfy our toolchain requirements
+        diff -w requirements.txt.python2.old requirements.txt.python2.new
+        diff -w requirements.txt.python3.old requirements.txt.python3.new
 
 - <<: *task_template
   name: burn_in_tests
@@ -5061,16 +5121,6 @@ tasks:
   name: ese
   commands:
   - func: "do setup"
-  - command: shell.exec
-    type: test
-    params:
-      working_dir: src
-      script: |
-        set -o errexit
-        set -o verbose
-
-        ${activate_virtualenv}
-        python -m pip install -r src/mongo/db/modules/enterprise/jstests/encryptdb/libs/requirements.txt
   - func: "run tests"
     vars:
       resmoke_args: --suites=ese --storageEngine=wiredTiger
@@ -7135,8 +7185,13 @@ tasks:
         set -o verbose
 
         ${activate_virtualenv}
-        if [ "Windows_NT" = "$OS" ]; then
-          /cygdrive/c/python/python36/python.exe -m pip install -r buildscripts/requirements.txt
+        # Since the free monitoring tests use python 3 via C exec,
+        # they deliberately hit python3.6 on Windows and toolchain python otherwise.
+        # We happen to pre-seed the core requirements in the toolchain,
+        # but we have no way of doing so to Windows python3.6 installed via Chocolatey.
+        if command -V cygpath; then
+            echo "Installing core requirements into Python at '${python3}'"
+            "$(cygpath "${python3}")" -mpip install -r etc/pip/core-requirements.txt
         fi
   - func: "run tests"
     vars:
@@ -7853,8 +7908,7 @@ tasks:
         set -o verbose
 
         ${activate_virtualenv}
-        # Install Python modules to support OAuth with pip until it is available in the toolchain.
-        pip install cryptography==1.7.2 pyjwt==1.5.3
+        python2 -m pip install -r etc/pip/jira-requirements.txt
 
         # We use a small batch size to avoid hitting the load balancer timeout if the Evergreen
         # API query is not fast enough.
@@ -9492,6 +9546,7 @@ buildvariants:
     # spawning a large number of linker processes.
     num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
     python: python
+    python3: '/cygdrive/c/python/python36/python.exe'
     ext: zip
     use_scons_cache: true
     multiversion_platform: windows
@@ -9655,6 +9710,7 @@ buildvariants:
     # spawning a large number of linker processes.
     num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
     python: python
+    python3: '/cygdrive/c/python/python36/python.exe'
     num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
     ext: zip
     use_scons_cache: true
@@ -9675,12 +9731,14 @@ buildvariants:
   - name: compile_TG
     requires:
     - name: burn_in_tests
+    - name: verify_pip
     distros:
     - windows-64-vs2015-large
   - name: compile_benchmarks
     distros:
     - windows-64-vs2015-large
   - name: burn_in_tests
+  - name: verify_pip
   - name: audit
   - name: auth_audit
   - name: benchmarks_orphaned
@@ -9976,6 +10034,7 @@ buildvariants:
     # spawning a large number of linker processes.
     num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
     python: python
+    python3: '/cygdrive/c/python/python36/python.exe'
     num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
     ext: zip
     use_scons_cache: true
@@ -10008,6 +10067,7 @@ buildvariants:
     # spawning a large number of linker processes.
     num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
     python: python
+    python3: '/cygdrive/c/python/python36/python.exe'
     num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
     test_flags: --storageEngine=inMemory --excludeWithAnyTags=requires_persistence,requires_journaling,uses_transactions
     ext: zip
@@ -10083,6 +10143,7 @@ buildvariants:
     # spawning a large number of linker processes.
     num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
     python: python
+    python3: '/cygdrive/c/python/python36/python.exe'
     num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
     ext: zip
     use_scons_cache: true
@@ -10200,6 +10261,7 @@ buildvariants:
     # spawning a large number of linker processes.
     num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 ))
     python: python
+    python3: '/cygdrive/c/python/python36/python.exe'
     num_jobs_available: $(grep -c ^processor /proc/cpuinfo)
     ext: zip
     use_scons_cache: true
@@ -10891,6 +10953,7 @@ buildvariants:
   - name: compile_all_run_unittests_TG
     requires:
     - name: burn_in_tests
+    - name: verify_pip
     - name: lint
     distros:
     - rhel62-large
@@ -10901,6 +10964,7 @@ buildvariants:
   - name: burn_in_tests
     distros:
     - rhel62-large
+  - name: verify_pip
   - name: rollback_fuzzer
   - name: rollback_fuzzer_clean_shutdowns
   - name: rollback_fuzzer_unclean_shutdowns
diff --git a/etc/pip/README.md b/etc/pip/README.md
new file mode 100644
index 00000000000..473664eb6a4
--- /dev/null
+++ b/etc/pip/README.md
@@ -0,0 +1,78 @@
+## On requirements (`*-requirements.txt`) files
+
+MongoDB requires multiple pypa projects installed to build and test. To that end, we provide our own
+`*-requirements.txt` files for specific domains of use. Inside each requirements file, there are
+only include statements for component files. These files are the bare requirements for specific
+components of our python environment. This separation allows us to avoid repetition and conflict in
+our requirements across components.
+
+For most developers, if you pip-install `dev-requirements.txt`, you have the python requirements to
+lint, build, and test MongoDB.
+
+## On the constraints (`constraints.txt`) file
+
+Our requirements files are *minimally* constrained. For the majority of pypa projects, any
+given requirements file will install the newest version of the project. For increased stability,
+there is a `constraints.txt` file with explicit versions for each required project. This file
+represents a _somewhat_ portable manifest of the latest acceptable pypa projects version according
+to a set of local python environments. It is also used to install the site-packages in the MongoDB
+internal toolchain-builder.
+
+Please note that the `verify_pip` task in evergreen confirms that `constraints.txt` satisfies
+`toolchain-requirements.txt`. If you modify the pypa project requirements, you should regenerate the
+constraints file.
+
+## How to modify a pypa project requirement in a component
+
+The most common edit of our requirements is likely a change to the constraints on a pypa project
+that we already use. For example, say that we currently require `pymongo >= 3.0, < 3.6.0` in the
+component `core`. You would like to use PyMongo 3.7, so you instead modify the line in
+`etc/pip/components/core.req` to read `pymongo >= 3.0, != 3.6.0`. Since this is a modification to an
+existing component, you do not need to modify any requirements file. However, you do need to
+regenerate the constraints file. The workflow will usually look like:
+
+```
+$ # Make your changes to the component file
+$ $EDITOR etc/pip/components/core.req
+$ # Regenerate the constraints file
+$ bash buildscripts/generate-pip-constraints.sh -r etc/pip/toolchain-requirements.txt -o etc/pip/constraints.txt
+```
+
+## How to add a new component (`*.req`) file
+
+Occasionally, we will require a set of pypa projects for an entirely new piece of software in our
+repository. This usually implies adding a new component file. For example, say that we need to add
+a logging system to both local development and evergreen. This system requires the fictional pypa
+project `FooLog`. So we add a file `foolog.req` and require it from both `dev-requirements.txt` and
+`evgtest-requirements.txt`. Like the majority of our components, we want it in the toolchain, so we
+also add it to `toolchain-requirements.txt`. The workflow will usually look like:
+
+```
+$ # Make the component file
+$ echo "FooLog" >etc/pip/components/foolog.req
+$ # Require the component from the requirements files
+$ echo "-r components/foolog.req" >>etc/pip/dev-requirements.txt
+$ echo "-r components/foolog.req" >>etc/pip/evgtest-requirements.txt
+$ echo "-r components/foolog.req" >>etc/pip/toolchain-requirements.txt
+$ # Regenerate the constraints file
+$ bash buildscripts/generate-pip-constraints.sh -r etc/pip/toolchain-requirements.txt -o etc/pip/constraints.txt
+```
+
+## How to add a new requirements (`*-requirements.txt`) file
+
+Rarely, we will have an entirely new domain of requirements that is useful. In this case, we need to
+at least make a new requirements file. For example, say we want to make a requirements file for
+packaging our code. We would need most of the requirements for `dev-requirements.txt` but the
+testing has already been done in our continuous integration. So we create a new file
+`package-requirements.txt` and require a smaller subset of components. The new file at
+`etc/pip/package-requirements.txt` would look like this:
+```
+-r components/platform.req
+-r components/core.req
+
+-r components/compile.req
+-r components/lint.req
+```
+
+Notice that since we did not change any components files, we do not need to regenerate our
+constraints.
diff --git a/etc/pip/compile-requirements.txt b/etc/pip/compile-requirements.txt
new file mode 100644
index 00000000000..74b44861e27
--- /dev/null
+++ b/etc/pip/compile-requirements.txt
@@ -0,0 +1,4 @@
+-r components/platform.req
+-r components/core.req
+
+-r components/compile.req
diff --git a/etc/pip/components/aws.req b/etc/pip/components/aws.req
new file mode 100644
index 00000000000..3245054d700
--- /dev/null
+++ b/etc/pip/components/aws.req
@@ -0,0 +1,3 @@
+boto3
+botocore
+psutil
diff --git a/etc/pip/components/compile.req b/etc/pip/components/compile.req
new file mode 100644
index 00000000000..4f218e53ddf
--- /dev/null
+++ b/etc/pip/components/compile.req
@@ -0,0 +1,4 @@
+# Mongo compile
+Cheetah3; python_version < "3" # src/mongo/base/generate_error_codes.py
+regex
+typing; python_version < "3"
diff --git a/etc/pip/components/core.req b/etc/pip/components/core.req
new file mode 100644
index 00000000000..ac21ffd16a9
--- /dev/null
+++ b/etc/pip/components/core.req
@@ -0,0 +1,5 @@
+# Core (we need these for most builscripts)
+pip >= 18.0
+PyYAML >= 3.0.0
+requests >= 2.0.0
+pymongo >= 3.0, < 3.6.0 # See PYTHON-1434, SERVER-34820
diff --git a/etc/pip/components/jiraclient.req b/etc/pip/components/jiraclient.req
new file mode 100644
index 00000000000..54588334ace
--- /dev/null
+++ b/etc/pip/components/jiraclient.req
@@ -0,0 +1,2 @@
+# TIG jira integration
+jira
diff --git a/etc/pip/components/lint.req b/etc/pip/components/lint.req
new file mode 100644
index 00000000000..74f244638b7
--- /dev/null
+++ b/etc/pip/components/lint.req
@@ -0,0 +1,6 @@
+# Linters
+pylint
+yapf
+# typing in Python 2 for mypy
+typing; python_version < "3"
+pydocstyle
diff --git a/etc/pip/components/mypy.req b/etc/pip/components/mypy.req
new file mode 100644
index 00000000000..16bd01c175c
--- /dev/null
+++ b/etc/pip/components/mypy.req
@@ -0,0 +1 @@
+mypy == 0.620; python_version > "3.4"
diff --git a/etc/pip/components/platform.req b/etc/pip/components/platform.req
new file mode 100644
index 00000000000..3fb8a515910
--- /dev/null
+++ b/etc/pip/components/platform.req
@@ -0,0 +1,4 @@
+# Platform-specific components
+pypiwin32==219; sys_platform == "win32" and python_version < "3"
+pypiwin32==223; sys_platform == "win32" and python_version > "3"
+subprocess32==3.5.2; os_name == "posix" and platform_release != "2.6.18-194.el5xen" and platform_release != "2.6.18-274.el5xen" and python_version < "3"
diff --git a/etc/pip/components/resmoke.req b/etc/pip/components/resmoke.req
new file mode 100644
index 00000000000..9bcc7c6f684
--- /dev/null
+++ b/etc/pip/components/resmoke.req
@@ -0,0 +1,3 @@
+mock; python_version < "3"
+PyKMIP == 0.4.0; python_version < "3" # It's now 0.8.0. We're far enough back to have API conflicts.
+jinja2
diff --git a/etc/pip/constraints.txt b/etc/pip/constraints.txt
index 9fc75aab922..e7787ca71a4 100644
--- a/etc/pip/constraints.txt
+++ b/etc/pip/constraints.txt
@@ -1,59 +1,66 @@
+# == PLEASE DO NOT MANUALLY EDIT THIS FILE ==
+# For more details, see etc/pip/README.md
+#
+# This file was generated via the following command:
+# $ 'bash' 'buildscripts/generate-pip-constraints.sh' '-r' 'etc/pip/toolchain-requirements.txt'
+
 # Common requirements
 asn1crypto==0.24.0
-astroid==1.6.5
-boto3==1.5.27
-botocore==1.8.50
-certifi==2018.8.13
+boto3==1.9.13
+botocore==1.12.13
+certifi==2018.8.24
 cffi==1.11.5
 chardet==3.0.4
-cryptography==2.0
+cryptography==2.3.1
 defusedxml==0.5.0
 docutils==0.14
-enum34==1.1.6
 idna==2.7
 isort==4.3.4
 Jinja2==2.10
-jira==1.0.10
+jira==2.0.0
 jmespath==0.9.3
 lazy-object-proxy==1.3.1
 MarkupSafe==1.0
 mccabe==0.6.1
-mock==2.0.0
 oauthlib==2.1.0
 pbr==4.2.0
-psutil==5.4.3
-pycparser==2.18
+psutil==5.4.7
+pycparser==2.19
 pydocstyle==2.1.1
-PyJWT==1.5.3
-pylint==1.8.3
-pymongo==3.6.1
+PyJWT==1.6.4
+pymongo==3.5.1
 python-dateutil==2.7.3
 PyYAML==3.13
+regex==2018.8.29
 requests==2.19.1
 requests-oauthlib==1.0.0
 requests-toolbelt==0.8.0
 s3transfer==0.1.13
 six==1.11.0
 snowballstemmer==1.2.1
-unittest-xml-reporting==2.1.0
 urllib3==1.23
 wrapt==1.10.11
-yapf==0.21.0
+yapf==0.24.0
 
 # Python2 requirements
+astroid==1.6.5; python_version < "3"
 backports.functools-lru-cache==1.5; python_version < "3"
-Cheetah3==3.0.0; python_version < "3"
+Cheetah3==3.1.0; python_version < "3"
 configparser==3.5.0; python_version < "3"
+enum34==1.1.6; python_version < "3"
 funcsigs==1.0.2; python_version < "3"
 futures==3.2.0; python_version < "3"
 ipaddress==1.0.22; python_version < "3"
-ordereddict==1.1; python_version < "3"
+mock==2.0.0; python_version < "3"
 PyKMIP==0.4.0; python_version < "3"
+pylint==1.9.3; python_version < "3"
 singledispatch==3.4.0.3; python_version < "3"
-typing==3.6.1; python_version < "3"
+typing==3.6.6; python_version < "3"
 
 # Python3 requirements
-mypy==0.580; python_version > "3"
+astroid==2.0.4; python_version > "3"
+mypy==0.620; python_version > "3"
+pylint==2.1.1; python_version > "3"
 typed-ast==1.1.0; python_version > "3"
 
 # Platform-specific components
diff --git a/etc/pip/core-requirements.txt b/etc/pip/core-requirements.txt
new file mode 100644
index 00000000000..daa77546015
--- /dev/null
+++ b/etc/pip/core-requirements.txt
@@ -0,0 +1,2 @@
+-r components/platform.req
+-r components/core.req
diff --git a/etc/pip/dev-requirements.txt b/etc/pip/dev-requirements.txt
new file mode 100644
index 00000000000..bd3471b61eb
--- /dev/null
+++ b/etc/pip/dev-requirements.txt
@@ -0,0 +1,6 @@
+-r components/platform.req
+-r components/core.req
+
+-r components/compile.req
+-r components/lint.req
+-r components/resmoke.req
diff --git a/etc/pip/evgtest-requirements.txt b/etc/pip/evgtest-requirements.txt
new file mode 100644
index 00000000000..42b183117d9
--- /dev/null
+++ b/etc/pip/evgtest-requirements.txt
@@ -0,0 +1,5 @@
+-r components/platform.req
+-r components/core.req
+
+-r components/resmoke.req
+-r components/aws.req
diff --git a/etc/pip/jira-requirements.txt b/etc/pip/jira-requirements.txt
new file mode 100644
index 00000000000..d18d5b36356
--- /dev/null
+++ b/etc/pip/jira-requirements.txt
@@ -0,0 +1,4 @@
+-r components/platform.req
+-r components/core.req
+
+-r components/jiraclient.req
diff --git a/etc/pip/lint-requirements.txt b/etc/pip/lint-requirements.txt
new file mode 100644
index 00000000000..8f136e14230
--- /dev/null
+++ b/etc/pip/lint-requirements.txt
@@ -0,0 +1,5 @@
+-r components/platform.req
+-r components/core.req
+
+-r components/lint.req
+-r components/mypy.req
diff --git a/etc/pip/powercycle-requirements.txt b/etc/pip/powercycle-requirements.txt
new file mode 100644
index 00000000000..f694454bc0b
--- /dev/null
+++ b/etc/pip/powercycle-requirements.txt
@@ -0,0 +1,4 @@
+-r components/platform.req
+-r components/core.req
+
+-r components/aws.req
diff --git a/etc/pip/toolchain-requirements.txt b/etc/pip/toolchain-requirements.txt
new file mode 100644
index 00000000000..a0f90baa433
--- /dev/null
+++ b/etc/pip/toolchain-requirements.txt
@@ -0,0 +1,14 @@
+# This file is specifically intended to pull in *all* of our requirements besides the platform
+# specific options in components/platform.req. The motivation for this structure is that pip
+# (including pip freeze) discards platform constraints. Thus generate-pip-constraints.sh always adds
+# platform.req to the end of any constraints.txt files it makes.
+
+-r components/core.req
+
+-r components/compile.req
+-r components/lint.req
+-r components/mypy.req
+-r components/resmoke.req
+
+-r components/aws.req
+-r components/jiraclient.req
diff --git a/pytests/requirements.txt b/pytests/requirements.txt
deleted file mode 100644
index 65703f4322e..00000000000
--- a/pytests/requirements.txt
+++ /dev/null
@@ -1,4 +0,0 @@
-# powertest
-boto3 == 1.5.27
-psutil == 5.4.3
-pymongo >= 3.0, ~= 3.6.0