diff options
26 files changed, 395 insertions, 101 deletions
diff --git a/.pylintrc b/.pylintrc index 7bd38008874..9784ddd8105 100644 --- a/.pylintrc +++ b/.pylintrc @@ -23,3 +23,6 @@ variable-rgx=[a-z_][a-z0-9_]{1,50}$ # W0611 - unused-import - typing module is needed for mypy disable=bad-continuation,fixme,import-error,line-too-long,no-member,locally-disabled,redefined-variable-type,too-few-public-methods,unused-import + +[IMPORTS] +known-third-party=boto3,botocore,psutil,yaml diff --git a/buildscripts/generate-pip-constraints.sh b/buildscripts/generate-pip-constraints.sh new file mode 100755 index 00000000000..37dba0e4775 --- /dev/null +++ b/buildscripts/generate-pip-constraints.sh @@ -0,0 +1,101 @@ +#!/bin/bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname ${BASH_SOURCE[0]})" && pwd)" +DEFAULT_WORKING_DIR="${SCRIPT_DIR}/../build/pip" + +showUsage() { + cat <<EOF +USAGE: + generate-pip-constraints.sh [-o CON_FILE] ... + generate-pip-constraints.sh -h + + -h, --help Show this message + -o CON_FILE Write constraints.txt to CON_FILE + +This command passes all unrecognized arguments to two pip invocations, +one for a python2 virtual environment and one for a python3 virtual environment. +It then forms a unified multi-version constraints file at constraints.txt in its working directory. + +This script's working directory currently defaults to '${DEFAULT_WORKING_DIR}'. +This default can be overriden via \$WORKING_DIR. +EOF +} + +CON_FILE="" +ARGS=() +while [[ $# -gt 0 ]] +do KEY="${1}" + case "$KEY" in + (-h|--help) showUsage; exit 0;; + (-o) CON_FILE="${2}"; shift 2;; + (*) ARGS+=("${KEY}"); shift;; + esac +done + +if [[ ${#ARGS} -eq 0 ]]; then + 1>&2 echo "No pip arguments given. Failing..." + exit 2 +fi + +generateConstraints(){ + EXE="${1}" + DIR="${2}" + if ! ( + export VIRTUAL_ENV_DISABLE_PROMPT=yes + virtualenv --python "${EXE}" "${DIR}" + . "${DIR}"/*/activate + pip install "${ARGS[@]}" + pip freeze >"${DIR}/requirements.txt" + ) + then RC=$? + 1>&2 echo "Errors occured while attempting + to install all requirements into '${DIR}' + with python executable '${EXE}'" + return $RC + fi +} + +WORKING_DIR="${WORKING_DIR:-${DEFAULT_WORKING_DIR}}" +if [[ -d $WORKING_DIR ]]; then + 1>&2 echo "Removing existing working dir at '$WORKING_DIR'..." + rm -r "${WORKING_DIR}" +fi +ABSOLUTE_WORKING_DIR="$(mkdir -p "${WORKING_DIR}" && cd "${WORKING_DIR}" && pwd)" + +PIP2_DIR="${ABSOLUTE_WORKING_DIR}/python2" +PIP3_DIR="${ABSOLUTE_WORKING_DIR}/python3" + +generateConstraints python2 "${PIP2_DIR}" +generateConstraints python3 "${PIP3_DIR}" + +if [[ -z $CON_FILE ]]; then + CON_FILE="${ABSOLUTE_WORKING_DIR}/constraints.txt" +fi +( + printf '# == PLEASE DO NOT MANUALLY EDIT THIS FILE ==' + printf '\n# For more details, see etc/pip/README.md' + printf '\n#' + printf '\n# This file was generated via the following command:' + printf "\n# $ 'bash' 'buildscripts/generate-pip-constraints.sh'" + printf " '%s'" "${ARGS[@]}" + printf '\n' + + printf '\n# Common requirements\n' + comm -12 "${PIP2_DIR}/requirements.txt" "${PIP3_DIR}/requirements.txt" + + printf '\n# Python2 requirements\n' + comm -23 "${PIP2_DIR}/requirements.txt" "${PIP3_DIR}/requirements.txt" | + sed -e 's/$/; python_version < "3"/' + + printf '\n# Python3 requirements\n' + comm -13 "${PIP2_DIR}/requirements.txt" "${PIP3_DIR}/requirements.txt" | + sed -e 's/$/; python_version > "3"/' + + printf '\n' + cat "${SCRIPT_DIR}/../etc/pip/components/platform.req" +) >"${CON_FILE}" + +1>&2 echo "All pip requirements were successfully installed in a virtual environment. +See '${CON_FILE}' for all installed packages." diff --git a/buildscripts/linter/runner.py b/buildscripts/linter/runner.py index 67c69d25d02..e93ccd63a4c 100644 --- a/buildscripts/linter/runner.py +++ b/buildscripts/linter/runner.py @@ -131,8 +131,8 @@ Could not find the correct version of linter '%s', expected '%s'. Check your PATH environment variable or re-run with --verbose for more information. To fix, install the needed python modules for both Python 2.7, and Python 3.x: - sudo pip2 install -r buildscripts/requirements.txt - sudo pip3 install -r buildscripts/requirements.txt + sudo pip2 install -r etc/pip/lint-requirements.txt + sudo pip3 install -r etc/pip/lint-requirements.txt These commands are typically available via packages with names like python-pip, python2-pip, and python3-pip. See your OS documentation for help. diff --git a/buildscripts/requirements.txt b/buildscripts/requirements.txt deleted file mode 100644 index dac9d36b45f..00000000000 --- a/buildscripts/requirements.txt +++ /dev/null @@ -1,20 +0,0 @@ -# Jira integration -cryptography == 2.0 -jira == 1.0.10 -pyjwt == 1.5.3 -# Other -pyyaml == 3.13 -unittest-xml-reporting == 2.1.0 -# Linters -yapf == 0.21.0 -mypy == 0.580 ; python_version > "3" -# typing in Python 2 for mypy -typing == 3.6.1; python_version < "3" -pylint == 1.8.3 -pydocstyle == 2.1.1 -# resmoke.py --r resmokelib/requirements.txt -# generate_error_codes.py -cheetah3 == 3.0.0; python_version < "3" -jinja2 == 2.10 -mock == 2.0.0 diff --git a/buildscripts/resmokelib/requirements.txt b/buildscripts/resmokelib/requirements.txt deleted file mode 100644 index f7be0432c6b..00000000000 --- a/buildscripts/resmokelib/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -mock == 2.0.0 ; python_version < "3" -pymongo >= 3.0, ~= 3.6.0 -pypiwin32 == 219 ; sys_platform == "win32" and python_version < "3" -pypiwin32 == 223 ; sys_platform == "win32" and python_version > "3" -PyYAML == 3.13 -requests >= 2.16.1 -subprocess32 >= 3.2.7 ; os_name == "posix" and python_version < "3" diff --git a/docs/building.md b/docs/building.md index 8fdc2f8a766..960cd991ea6 100644 --- a/docs/building.md +++ b/docs/building.md @@ -33,7 +33,7 @@ Python Prerequisites In order to build MongoDB, Python 2.7.x is required, and several Python modules. To install the required Python modules, run: - $ pip2 install -r buildscripts/requirements.txt + $ pip2 install -r etc/pip/compile-requirements.txt Note: If the `pip2` command is not available, `pip` without a suffix may be the pip command associated with Python 2.7.x. diff --git a/etc/evergreen.yml b/etc/evergreen.yml index 0a438ffd8a8..6dc486bbd90 100644 --- a/etc/evergreen.yml +++ b/etc/evergreen.yml @@ -455,6 +455,7 @@ variables: # spawning a large number of linker processes. num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 )) python: python + python3: '/cygdrive/c/python/python36/python.exe' num_jobs_available: $(grep -c ^processor /proc/cpuinfo) ext: zip use_scons_cache: true @@ -1295,22 +1296,38 @@ functions: command: shell.exec type: test params: + shell: bash script: | # exit immediately if virtualenv is not found set -o errexit set -o verbose - python_loc=$(which ${python|/opt/mongodbtoolchain/v2/bin/python2}) - python3_loc=$(which ${python|/opt/mongodbtoolchain/v2/bin/python3}) virtualenv_loc=$(which ${virtualenv|virtualenv}) - if [ "Windows_NT" = "$OS" ]; then - python_loc=$(cygpath -w $python_loc) - python3_loc=$(cygpath -w c:/python/Python36/python.exe) + + python2_loc=$(which ${python|/opt/mongodbtoolchain/v2/bin/python2}) + python3_loc=$(which ${python3|/opt/mongodbtoolchain/v2/bin/python3}) + venv2_dir="${workdir}/venv" + venv3_dir="${workdir}/venv_3" + if command -V cygpath; then + # Sad note: We have to use the Windows path instead of the posix path here. + # Otherwise, virtualenv may mistakenly resolve paths relative to c:\cygdrive. + python2_loc=$(cygpath -w $python2_loc) + python3_loc=$(cygpath -w $python3_loc) + venv2_dir="$(cygpath -w "$venv2_dir")" + venv3_dir="$(cygpath -w "$venv3_dir")" fi - # Set up virtualenv in ${workdir} - "$virtualenv_loc" --python "$python_loc" --system-site-packages "${workdir}/venv" - # Add virtualenv for python3 in ${workdir} - "$virtualenv_loc" --python "$python3_loc" --system-site-packages "${workdir}/venv_3" + + # Set up virtualenvs in ${workdir} + "$virtualenv_loc" --python "$python2_loc" --system-site-packages "$venv2_dir" + "$virtualenv_loc" --python "$python3_loc" --system-site-packages "$venv3_dir" + + # Link python2/python3 in the primary virtualenv bin dir + export VIRTUAL_ENV_DISABLE_PROMPT=yes + venv2_bin="$(dirname "$(cd "$venv2_dir"; . ./*/activate; which python)")" + py2_exe="$(cd "$venv2_dir"; . ./*/activate; which python)" + py3_exe="$(cd "$venv3_dir"; . ./*/activate; which python)" + if [[ ! -f $venv2_bin/python2 ]]; then ln -sfv "$py2_exe" "$venv2_bin/python2"; fi + if [[ ! -f $venv2_bin/python3 ]]; then ln -sfv "$py3_exe" "$venv2_bin/python3"; fi "run tests" : - *determine_task_timeout @@ -1339,8 +1356,7 @@ functions: # activate the virtualenv if it has been set up ${activate_virtualenv} - pip install boto3==1.5.27 - pip install mock==2.0.0 + python2 -m pip install -r etc/pip/evgtest-requirements.txt # Set the TMPDIR environment variable to be a directory in the task's working # directory so that temporary files created by processes spawned by resmoke.py get @@ -1504,6 +1520,7 @@ functions: type: test params: working_dir: src + shell: bash script: | set -o errexit set -o verbose @@ -1520,12 +1537,15 @@ functions: fi ${activate_virtualenv} - set +o errexit - ${compile_env|} $python ./buildscripts/scons.py ${compile_flags|} ${task_compile_flags|} ${task_compile_flags_extra|} ${scons_cache_args|} $extra_args ${targets} ${additional_targets|} MONGO_VERSION=${version} - exit_status=$? + + python2 -m pip install -r ./etc/pip/compile-requirements.txt + ${compile_env|} $python ./buildscripts/scons.py \ + ${compile_flags|} ${task_compile_flags|} ${task_compile_flags_extra|} \ + ${scons_cache_args|} $extra_args \ + ${targets} ${additional_targets|} MONGO_VERSION=${version} || exit_status=$? # If compile fails we do not run any tests - if [ $exit_status -ne 0 ]; then - if [ ${dump_scons_config_on_failure} = 'true' ]; then + if [[ $exit_status -ne 0 ]]; then + if [[ "${dump_scons_config_on_failure}" == true ]]; then echo "Dumping build/scons/config.log" cat build/scons/config.log fi @@ -1929,7 +1949,7 @@ functions: ${activate_virtualenv} # The Windows build variants are running python 2.7.3 and require TLS 1.2 from pyOpenSSL - pip install 'pyOpenSSL ; sys_platform == "win32" or sys_platform == "cygwin"' + python2 -m pip install 'pyOpenSSL ; sys_platform == "win32" or sys_platform == "cygwin"' rm -rf /data/install /data/multiversion $python buildscripts/setup_multiversion_mongodb.py \ @@ -2132,8 +2152,7 @@ functions: set -o errexit ${activate_virtualenv} - pip install -r buildscripts/requirements.txt - pip install -r pytests/requirements.txt + python2 -m pip install -r etc/pip/powercycle-requirements.txt if [ ! -z "${subnet_id}" ]; then subnet_id="-n ${subnet_id}" @@ -2261,7 +2280,7 @@ functions: script: | set -o errexit # Copy buildscripts, pytests and mongoDB executables to the remote host. - file_param="--file buildscripts --file pytests" + file_param="--file etc --file buildscripts --file pytests" mongo_executables="mongo mongod mongos" for executable in $mongo_executables do @@ -2291,7 +2310,7 @@ functions: cmds="$cmds; virtualenv --python \$python_loc --system-site-packages ${virtualenv_dir|venv}" cmds="$cmds; activate=\$(find ${virtualenv_dir|venv} -name 'activate')" cmds="$cmds; . \$activate" - cmds="$cmds; pip install -r \$remote_dir/pytests/requirements.txt" + cmds="$cmds; pip2 install -r \$remote_dir/etc/pip/powercycle-requirements.txt" ssh_connection_options="${ssh_identity} ${ssh_connection_options}" ${activate_virtualenv} $python buildscripts/remote_operations.py \ @@ -2767,7 +2786,7 @@ functions: python=python else if [ "Windows_NT" = "$OS" ]; then - python=/cygdrive/c/python/Python36/python + python=/cygdrive/c/python/python36/python else python=${python3|/opt/mongodbtoolchain/v2/bin/python3} fi @@ -3466,6 +3485,8 @@ tasks: - "*Test" - "./**.pdb" - "./**.msi" + - "./etc/pip/**" + - "./etc/scons/**" - "./etc/*san.suppressions" - "./etc/repo_config.yaml" - "./etc/test_lifecycle.yml" @@ -4182,19 +4203,58 @@ tasks: set -o errexit set -o verbose - ### TODO: Remove python3 when mypy 0.580 is installed in the toolchain. - # Since mypy requires python3, we need to activate the venv_3 - ${activate_virtualenv_3} - pip install -r buildscripts/requirements.txt - updated_mypy=$(PATH=$PATH:/opt/mongodbtoolchain/v2/bin which mypy) - deactivate - ### ${activate_virtualenv} - # TODO: Remove once the linters have been updated on the variants. - pip install -r buildscripts/requirements.txt - # The linters require the modules be installed. - pip install -r pytests/requirements.txt - MYPY=$updated_mypy ${compile_env|} $python ./buildscripts/scons.py ${compile_flags|} --stack-size=1024 lint + python3 -m pip install -r etc/pip/lint-requirements.txt + python2 -m pip install -r etc/pip/lint-requirements.txt + export MYPY="$( + PATH+=':/opt/mongodbtoolchain/v2/bin' + if command -V cygpath 2>/dev/null; then + PATH+=":$(cypath "${workdir}")/venv_3/Scripts" + else + PATH+=":${workdir}/venv_3/bin" + fi + which mypy + )" + ${compile_env|} python2 ./buildscripts/scons.py ${compile_flags|} --stack-size=1024 lint + +- name: verify_pip + depends_on: [] + commands: + - command: manifest.load + - func: "git get project" + - command: shell.exec + type: test + params: + working_dir: src + script: | + set -o errexit + set -o verbose + + # Note: One pleasant side effect of this task is that it should pre warm pip's cache. + # For every pypa project that is part of our current toolchain, this will do nothing. + # For every pypa project that we are missing, the .whl file will be added to the pip cache. + # (See https://pip.pypa.io/en/latest/reference/pip_install/#caching) + + ${activate_virtualenv} + # This installs the explicit project versions which would be installed in the toolchain + # from this patch + python2 -m pip install -r etc/pip/constraints.txt + python3 -m pip install -r etc/pip/constraints.txt + + python2 -m pip freeze >requirements.txt.python2.old + python3 -m pip freeze >requirements.txt.python3.old + + # This installs any requirements which are unsatisfied by the constraints.txt above + python2 -m pip install -r etc/pip/toolchain-requirements.txt + python3 -m pip install -r etc/pip/toolchain-requirements.txt + + python2 -m pip freeze >requirements.txt.python2.new + python3 -m pip freeze >requirements.txt.python3.new + + # Compare the old freezes to the new freezes. + # They should be the same if our constraints satisfy our toolchain requirements + diff -w requirements.txt.python2.old requirements.txt.python2.new + diff -w requirements.txt.python3.old requirements.txt.python3.new - <<: *task_template name: burn_in_tests @@ -5061,16 +5121,6 @@ tasks: name: ese commands: - func: "do setup" - - command: shell.exec - type: test - params: - working_dir: src - script: | - set -o errexit - set -o verbose - - ${activate_virtualenv} - python -m pip install -r src/mongo/db/modules/enterprise/jstests/encryptdb/libs/requirements.txt - func: "run tests" vars: resmoke_args: --suites=ese --storageEngine=wiredTiger @@ -7135,8 +7185,13 @@ tasks: set -o verbose ${activate_virtualenv} - if [ "Windows_NT" = "$OS" ]; then - /cygdrive/c/python/python36/python.exe -m pip install -r buildscripts/requirements.txt + # Since the free monitoring tests use python 3 via C exec, + # they deliberately hit python3.6 on Windows and toolchain python otherwise. + # We happen to pre-seed the core requirements in the toolchain, + # but we have no way of doing so to Windows python3.6 installed via Chocolatey. + if command -V cygpath; then + echo "Installing core requirements into Python at '${python3}'" + "$(cygpath "${python3}")" -mpip install -r etc/pip/core-requirements.txt fi - func: "run tests" vars: @@ -7853,8 +7908,7 @@ tasks: set -o verbose ${activate_virtualenv} - # Install Python modules to support OAuth with pip until it is available in the toolchain. - pip install cryptography==1.7.2 pyjwt==1.5.3 + python2 -m pip install -r etc/pip/jira-requirements.txt # We use a small batch size to avoid hitting the load balancer timeout if the Evergreen # API query is not fast enough. @@ -9492,6 +9546,7 @@ buildvariants: # spawning a large number of linker processes. num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 )) python: python + python3: '/cygdrive/c/python/python36/python.exe' ext: zip use_scons_cache: true multiversion_platform: windows @@ -9655,6 +9710,7 @@ buildvariants: # spawning a large number of linker processes. num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 )) python: python + python3: '/cygdrive/c/python/python36/python.exe' num_jobs_available: $(grep -c ^processor /proc/cpuinfo) ext: zip use_scons_cache: true @@ -9675,12 +9731,14 @@ buildvariants: - name: compile_TG requires: - name: burn_in_tests + - name: verify_pip distros: - windows-64-vs2015-large - name: compile_benchmarks distros: - windows-64-vs2015-large - name: burn_in_tests + - name: verify_pip - name: audit - name: auth_audit - name: benchmarks_orphaned @@ -9976,6 +10034,7 @@ buildvariants: # spawning a large number of linker processes. num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 )) python: python + python3: '/cygdrive/c/python/python36/python.exe' num_jobs_available: $(grep -c ^processor /proc/cpuinfo) ext: zip use_scons_cache: true @@ -10008,6 +10067,7 @@ buildvariants: # spawning a large number of linker processes. num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 )) python: python + python3: '/cygdrive/c/python/python36/python.exe' num_jobs_available: $(grep -c ^processor /proc/cpuinfo) test_flags: --storageEngine=inMemory --excludeWithAnyTags=requires_persistence,requires_journaling,uses_transactions ext: zip @@ -10083,6 +10143,7 @@ buildvariants: # spawning a large number of linker processes. num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 )) python: python + python3: '/cygdrive/c/python/python36/python.exe' num_jobs_available: $(grep -c ^processor /proc/cpuinfo) ext: zip use_scons_cache: true @@ -10200,6 +10261,7 @@ buildvariants: # spawning a large number of linker processes. num_scons_link_jobs_available: $(( $(grep -c ^processor /proc/cpuinfo) / 4 )) python: python + python3: '/cygdrive/c/python/python36/python.exe' num_jobs_available: $(grep -c ^processor /proc/cpuinfo) ext: zip use_scons_cache: true @@ -10891,6 +10953,7 @@ buildvariants: - name: compile_all_run_unittests_TG requires: - name: burn_in_tests + - name: verify_pip - name: lint distros: - rhel62-large @@ -10901,6 +10964,7 @@ buildvariants: - name: burn_in_tests distros: - rhel62-large + - name: verify_pip - name: rollback_fuzzer - name: rollback_fuzzer_clean_shutdowns - name: rollback_fuzzer_unclean_shutdowns diff --git a/etc/pip/README.md b/etc/pip/README.md new file mode 100644 index 00000000000..473664eb6a4 --- /dev/null +++ b/etc/pip/README.md @@ -0,0 +1,78 @@ +## On requirements (`*-requirements.txt`) files + +MongoDB requires multiple pypa projects installed to build and test. To that end, we provide our own +`*-requirements.txt` files for specific domains of use. Inside each requirements file, there are +only include statements for component files. These files are the bare requirements for specific +components of our python environment. This separation allows us to avoid repetition and conflict in +our requirements across components. + +For most developers, if you pip-install `dev-requirements.txt`, you have the python requirements to +lint, build, and test MongoDB. + +## On the constraints (`constraints.txt`) file + +Our requirements files are *minimally* constrained. For the majority of pypa projects, any +given requirements file will install the newest version of the project. For increased stability, +there is a `constraints.txt` file with explicit versions for each required project. This file +represents a _somewhat_ portable manifest of the latest acceptable pypa projects version according +to a set of local python environments. It is also used to install the site-packages in the MongoDB +internal toolchain-builder. + +Please note that the `verify_pip` task in evergreen confirms that `constraints.txt` satisfies +`toolchain-requirements.txt`. If you modify the pypa project requirements, you should regenerate the +constraints file. + +## How to modify a pypa project requirement in a component + +The most common edit of our requirements is likely a change to the constraints on a pypa project +that we already use. For example, say that we currently require `pymongo >= 3.0, < 3.6.0` in the +component `core`. You would like to use PyMongo 3.7, so you instead modify the line in +`etc/pip/components/core.req` to read `pymongo >= 3.0, != 3.6.0`. Since this is a modification to an +existing component, you do not need to modify any requirements file. However, you do need to +regenerate the constraints file. The workflow will usually look like: + +``` +$ # Make your changes to the component file +$ $EDITOR etc/pip/components/core.req +$ # Regenerate the constraints file +$ bash buildscripts/generate-pip-constraints.sh -r etc/pip/toolchain-requirements.txt -o etc/pip/constraints.txt +``` + +## How to add a new component (`*.req`) file + +Occasionally, we will require a set of pypa projects for an entirely new piece of software in our +repository. This usually implies adding a new component file. For example, say that we need to add +a logging system to both local development and evergreen. This system requires the fictional pypa +project `FooLog`. So we add a file `foolog.req` and require it from both `dev-requirements.txt` and +`evgtest-requirements.txt`. Like the majority of our components, we want it in the toolchain, so we +also add it to `toolchain-requirements.txt`. The workflow will usually look like: + +``` +$ # Make the component file +$ echo "FooLog" >etc/pip/components/foolog.req +$ # Require the component from the requirements files +$ echo "-r components/foolog.req" >>etc/pip/dev-requirements.txt +$ echo "-r components/foolog.req" >>etc/pip/evgtest-requirements.txt +$ echo "-r components/foolog.req" >>etc/pip/toolchain-requirements.txt +$ # Regenerate the constraints file +$ bash buildscripts/generate-pip-constraints.sh -r etc/pip/toolchain-requirements.txt -o etc/pip/constraints.txt +``` + +## How to add a new requirements (`*-requirements.txt`) file + +Rarely, we will have an entirely new domain of requirements that is useful. In this case, we need to +at least make a new requirements file. For example, say we want to make a requirements file for +packaging our code. We would need most of the requirements for `dev-requirements.txt` but the +testing has already been done in our continuous integration. So we create a new file +`package-requirements.txt` and require a smaller subset of components. The new file at +`etc/pip/package-requirements.txt` would look like this: +``` +-r components/platform.req +-r components/core.req + +-r components/compile.req +-r components/lint.req +``` + +Notice that since we did not change any components files, we do not need to regenerate our +constraints. diff --git a/etc/pip/compile-requirements.txt b/etc/pip/compile-requirements.txt new file mode 100644 index 00000000000..74b44861e27 --- /dev/null +++ b/etc/pip/compile-requirements.txt @@ -0,0 +1,4 @@ +-r components/platform.req +-r components/core.req + +-r components/compile.req diff --git a/etc/pip/components/aws.req b/etc/pip/components/aws.req new file mode 100644 index 00000000000..3245054d700 --- /dev/null +++ b/etc/pip/components/aws.req @@ -0,0 +1,3 @@ +boto3 +botocore +psutil diff --git a/etc/pip/components/compile.req b/etc/pip/components/compile.req new file mode 100644 index 00000000000..4f218e53ddf --- /dev/null +++ b/etc/pip/components/compile.req @@ -0,0 +1,4 @@ +# Mongo compile +Cheetah3; python_version < "3" # src/mongo/base/generate_error_codes.py +regex +typing; python_version < "3" diff --git a/etc/pip/components/core.req b/etc/pip/components/core.req new file mode 100644 index 00000000000..ac21ffd16a9 --- /dev/null +++ b/etc/pip/components/core.req @@ -0,0 +1,5 @@ +# Core (we need these for most builscripts) +pip >= 18.0 +PyYAML >= 3.0.0 +requests >= 2.0.0 +pymongo >= 3.0, < 3.6.0 # See PYTHON-1434, SERVER-34820 diff --git a/etc/pip/components/jiraclient.req b/etc/pip/components/jiraclient.req new file mode 100644 index 00000000000..54588334ace --- /dev/null +++ b/etc/pip/components/jiraclient.req @@ -0,0 +1,2 @@ +# TIG jira integration +jira diff --git a/etc/pip/components/lint.req b/etc/pip/components/lint.req new file mode 100644 index 00000000000..74f244638b7 --- /dev/null +++ b/etc/pip/components/lint.req @@ -0,0 +1,6 @@ +# Linters +pylint +yapf +# typing in Python 2 for mypy +typing; python_version < "3" +pydocstyle diff --git a/etc/pip/components/mypy.req b/etc/pip/components/mypy.req new file mode 100644 index 00000000000..16bd01c175c --- /dev/null +++ b/etc/pip/components/mypy.req @@ -0,0 +1 @@ +mypy == 0.620; python_version > "3.4" diff --git a/etc/pip/components/platform.req b/etc/pip/components/platform.req new file mode 100644 index 00000000000..3fb8a515910 --- /dev/null +++ b/etc/pip/components/platform.req @@ -0,0 +1,4 @@ +# Platform-specific components +pypiwin32==219; sys_platform == "win32" and python_version < "3" +pypiwin32==223; sys_platform == "win32" and python_version > "3" +subprocess32==3.5.2; os_name == "posix" and platform_release != "2.6.18-194.el5xen" and platform_release != "2.6.18-274.el5xen" and python_version < "3" diff --git a/etc/pip/components/resmoke.req b/etc/pip/components/resmoke.req new file mode 100644 index 00000000000..9bcc7c6f684 --- /dev/null +++ b/etc/pip/components/resmoke.req @@ -0,0 +1,3 @@ +mock; python_version < "3" +PyKMIP == 0.4.0; python_version < "3" # It's now 0.8.0. We're far enough back to have API conflicts. +jinja2 diff --git a/etc/pip/constraints.txt b/etc/pip/constraints.txt index 9fc75aab922..e7787ca71a4 100644 --- a/etc/pip/constraints.txt +++ b/etc/pip/constraints.txt @@ -1,59 +1,66 @@ +# == PLEASE DO NOT MANUALLY EDIT THIS FILE == +# For more details, see etc/pip/README.md +# +# This file was generated via the following command: +# $ 'bash' 'buildscripts/generate-pip-constraints.sh' '-r' 'etc/pip/toolchain-requirements.txt' + # Common requirements asn1crypto==0.24.0 -astroid==1.6.5 -boto3==1.5.27 -botocore==1.8.50 -certifi==2018.8.13 +boto3==1.9.13 +botocore==1.12.13 +certifi==2018.8.24 cffi==1.11.5 chardet==3.0.4 -cryptography==2.0 +cryptography==2.3.1 defusedxml==0.5.0 docutils==0.14 -enum34==1.1.6 idna==2.7 isort==4.3.4 Jinja2==2.10 -jira==1.0.10 +jira==2.0.0 jmespath==0.9.3 lazy-object-proxy==1.3.1 MarkupSafe==1.0 mccabe==0.6.1 -mock==2.0.0 oauthlib==2.1.0 pbr==4.2.0 -psutil==5.4.3 -pycparser==2.18 +psutil==5.4.7 +pycparser==2.19 pydocstyle==2.1.1 -PyJWT==1.5.3 -pylint==1.8.3 -pymongo==3.6.1 +PyJWT==1.6.4 +pymongo==3.5.1 python-dateutil==2.7.3 PyYAML==3.13 +regex==2018.8.29 requests==2.19.1 requests-oauthlib==1.0.0 requests-toolbelt==0.8.0 s3transfer==0.1.13 six==1.11.0 snowballstemmer==1.2.1 -unittest-xml-reporting==2.1.0 urllib3==1.23 wrapt==1.10.11 -yapf==0.21.0 +yapf==0.24.0 # Python2 requirements +astroid==1.6.5; python_version < "3" backports.functools-lru-cache==1.5; python_version < "3" -Cheetah3==3.0.0; python_version < "3" +Cheetah3==3.1.0; python_version < "3" configparser==3.5.0; python_version < "3" +enum34==1.1.6; python_version < "3" funcsigs==1.0.2; python_version < "3" futures==3.2.0; python_version < "3" ipaddress==1.0.22; python_version < "3" -ordereddict==1.1; python_version < "3" +mock==2.0.0; python_version < "3" PyKMIP==0.4.0; python_version < "3" +pylint==1.9.3; python_version < "3" singledispatch==3.4.0.3; python_version < "3" -typing==3.6.1; python_version < "3" +typing==3.6.6; python_version < "3" # Python3 requirements -mypy==0.580; python_version > "3" +astroid==2.0.4; python_version > "3" +mypy==0.620; python_version > "3" +pylint==2.1.1; python_version > "3" typed-ast==1.1.0; python_version > "3" # Platform-specific components diff --git a/etc/pip/core-requirements.txt b/etc/pip/core-requirements.txt new file mode 100644 index 00000000000..daa77546015 --- /dev/null +++ b/etc/pip/core-requirements.txt @@ -0,0 +1,2 @@ +-r components/platform.req +-r components/core.req diff --git a/etc/pip/dev-requirements.txt b/etc/pip/dev-requirements.txt new file mode 100644 index 00000000000..bd3471b61eb --- /dev/null +++ b/etc/pip/dev-requirements.txt @@ -0,0 +1,6 @@ +-r components/platform.req +-r components/core.req + +-r components/compile.req +-r components/lint.req +-r components/resmoke.req diff --git a/etc/pip/evgtest-requirements.txt b/etc/pip/evgtest-requirements.txt new file mode 100644 index 00000000000..42b183117d9 --- /dev/null +++ b/etc/pip/evgtest-requirements.txt @@ -0,0 +1,5 @@ +-r components/platform.req +-r components/core.req + +-r components/resmoke.req +-r components/aws.req diff --git a/etc/pip/jira-requirements.txt b/etc/pip/jira-requirements.txt new file mode 100644 index 00000000000..d18d5b36356 --- /dev/null +++ b/etc/pip/jira-requirements.txt @@ -0,0 +1,4 @@ +-r components/platform.req +-r components/core.req + +-r components/jiraclient.req diff --git a/etc/pip/lint-requirements.txt b/etc/pip/lint-requirements.txt new file mode 100644 index 00000000000..8f136e14230 --- /dev/null +++ b/etc/pip/lint-requirements.txt @@ -0,0 +1,5 @@ +-r components/platform.req +-r components/core.req + +-r components/lint.req +-r components/mypy.req diff --git a/etc/pip/powercycle-requirements.txt b/etc/pip/powercycle-requirements.txt new file mode 100644 index 00000000000..f694454bc0b --- /dev/null +++ b/etc/pip/powercycle-requirements.txt @@ -0,0 +1,4 @@ +-r components/platform.req +-r components/core.req + +-r components/aws.req diff --git a/etc/pip/toolchain-requirements.txt b/etc/pip/toolchain-requirements.txt new file mode 100644 index 00000000000..a0f90baa433 --- /dev/null +++ b/etc/pip/toolchain-requirements.txt @@ -0,0 +1,14 @@ +# This file is specifically intended to pull in *all* of our requirements besides the platform +# specific options in components/platform.req. The motivation for this structure is that pip +# (including pip freeze) discards platform constraints. Thus generate-pip-constraints.sh always adds +# platform.req to the end of any constraints.txt files it makes. + +-r components/core.req + +-r components/compile.req +-r components/lint.req +-r components/mypy.req +-r components/resmoke.req + +-r components/aws.req +-r components/jiraclient.req diff --git a/pytests/requirements.txt b/pytests/requirements.txt deleted file mode 100644 index 65703f4322e..00000000000 --- a/pytests/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -# powertest -boto3 == 1.5.27 -psutil == 5.4.3 -pymongo >= 3.0, ~= 3.6.0 |