Merge from 3.1: Issue #13703: add a way to randomize the hash values of basic types (str, bytes, datetime)

in order to make algorithmic complexity attacks on (e.g.) web apps much more complicated.

The environment variable PYTHONHASHSEED and the new command line flag -R control this
behavior.

1 files changed, 190 insertions, 111 deletions

diff --git a/Modules/main.c b/Modules/main.c
index 9607cb334d..ed84aa0286 100644
--- a/Modules/main.c
+++ b/Modules/main.c
@@ -2,7 +2,6 @@
 
 #include "Python.h"
 #include "osdefs.h"
-#include "import.h"
 
 #include <locale.h>
 
@@ -47,7 +46,7 @@ static wchar_t **orig_argv;
 static int  orig_argc;
 
 /* command line options */
-#define BASE_OPTS L"bBc:dEhiJm:ORsStuvVW:xX?"
+#define BASE_OPTS L"bBc:dEhiJm:OqRsStuvVW:xX:?"
 
 #define PROGRAM_OPTS BASE_OPTS
 
@@ -72,6 +71,7 @@ static char *usage_2 = "\
 -m mod : run library module as a script (terminates option list)\n\
 -O     : optimize generated bytecode slightly; also PYTHONOPTIMIZE=x\n\
 -OO    : remove doc-strings in addition to the -O optimizations\n\
+-q     : don't print version and copyright messages on interactive startup\n\
 -R     : use a pseudo-random salt to make hash() values of various types be\n\
          unpredictable between separate invocations of the interpreter, as\n\
          a defence against denial-of-service attacks\n\
@@ -85,7 +85,9 @@ static char *usage_3 = "\
          can be supplied multiple times to increase verbosity\n\
 -V     : print the Python version number and exit (also --version)\n\
 -W arg : warning control; arg is action:message:category:module:lineno\n\
+         also PYTHONWARNINGS=arg\n\
 -x     : skip first line of source, allowing use of non-Unix forms of #!cmd\n\
+-X opt : set implementation-specific option\n\
 ";
 static char *usage_4 = "\
 file   : program read from script file\n\
@@ -96,11 +98,11 @@ PYTHONSTARTUP: file executed on interactive startup (no default)\n\
 PYTHONPATH   : '%c'-separated list of directories prefixed to the\n\
                default module search path.  The result is sys.path.\n\
 ";
-static char *usage_5 = "\
-PYTHONHOME   : alternate <prefix> directory (or <prefix>%c<exec_prefix>).\n\
-               The default module search path uses %s.\n\
-PYTHONCASEOK : ignore case in 'import' statements (Windows).\n\
-PYTHONIOENCODING: Encoding[:errors] used for stdin/stdout/stderr.\n\
+static char *usage_5 =
+"PYTHONHOME   : alternate <prefix> directory (or <prefix>%c<exec_prefix>).\n"
+"               The default module search path uses %s.\n"
+"PYTHONCASEOK : ignore case in 'import' statements (Windows).\n"
+"PYTHONIOENCODING: Encoding[:errors] used for stdin/stdout/stderr.\n\
 ";
 static char *usage_6 = "\
 PYTHONHASHSEED: if this variable is set to ``random``, the effect is the same \n\
@@ -109,28 +111,6 @@ PYTHONHASHSEED: if this variable is set to ``random``, the effect is the same \n
    in the range [0,4294967295] to get hash values with a predictable seed.\n\
 ";
 
-#ifndef MS_WINDOWS
-static FILE*
-_wfopen(const wchar_t *path, const wchar_t *mode)
-{
-    char cpath[PATH_MAX];
-    char cmode[10];
-    size_t r;
-    r = wcstombs(cpath, path, PATH_MAX);
-    if (r == (size_t)-1 || r >= PATH_MAX) {
-        errno = EINVAL;
-        return NULL;
-    }
-    r = wcstombs(cmode, mode, 10);
-    if (r == (size_t)-1 || r >= 10) {
-        errno = EINVAL;
-        return NULL;
-    }
-    return fopen(cpath, cmode);
-}
-#endif
-
-
 static int
 usage(int exitcode, wchar_t* program)
 {
@@ -231,35 +211,102 @@ static int RunModule(wchar_t *modname, int set_argv0)
     return 0;
 }
 
-static int RunMainFromImporter(wchar_t *filename)
+static int
+RunMainFromImporter(wchar_t *filename)
 {
-    PyObject *argv0 = NULL, *importer = NULL;
+    PyObject *argv0 = NULL, *importer, *sys_path;
+    int sts;
 
-    if ((argv0 = PyUnicode_FromWideChar(filename,wcslen(filename))) &&
-        (importer = PyImport_GetImporter(argv0)) &&
-        (importer->ob_type != &PyNullImporter_Type))
-    {
-             /* argv0 is usable as an import source, so
-                    put it in sys.path[0] and import __main__ */
-        PyObject *sys_path = NULL;
-        if ((sys_path = PySys_GetObject("path")) &&
-            !PyList_SetItem(sys_path, 0, argv0))
-        {
-            Py_INCREF(argv0);
-            Py_DECREF(importer);
-            sys_path = NULL;
-            return RunModule(L"__main__", 0) != 0;
-        }
+    argv0 = PyUnicode_FromWideChar(filename, wcslen(filename));
+    if (argv0 == NULL)
+        goto error;
+
+    importer = PyImport_GetImporter(argv0);
+    if (importer == NULL)
+        goto error;
+
+    if (importer->ob_type == &PyNullImporter_Type) {
+        Py_DECREF(argv0);
+        Py_DECREF(importer);
+        return -1;
     }
+    Py_DECREF(importer);
+
+    /* argv0 is usable as an import source, so put it in sys.path[0]
+       and import __main__ */
+    sys_path = PySys_GetObject("path");
+    if (sys_path == NULL)
+        goto error;
+    if (PyList_SetItem(sys_path, 0, argv0)) {
+        argv0 = NULL;
+        goto error;
+    }
+    Py_INCREF(argv0);
+
+    sts = RunModule(L"__main__", 0);
+    return sts != 0;
+
+error:
     Py_XDECREF(argv0);
-    Py_XDECREF(importer);
-    if (PyErr_Occurred()) {
+    PyErr_Print();
+    return 1;
+}
+
+static int
+run_command(wchar_t *command, PyCompilerFlags *cf)
+{
+    PyObject *unicode, *bytes;
+    int ret;
+
+    unicode = PyUnicode_FromWideChar(command, -1);
+    if (unicode == NULL)
+        goto error;
+    bytes = PyUnicode_AsUTF8String(unicode);
+    Py_DECREF(unicode);
+    if (bytes == NULL)
+        goto error;
+    ret = PyRun_SimpleStringFlags(PyBytes_AsString(bytes), cf);
+    Py_DECREF(bytes);
+    return ret != 0;
+
+error:
+    PySys_WriteStderr("Unable to decode the command from the command line:\n");
+    PyErr_Print();
+    return 1;
+}
+
+static int
+run_file(FILE *fp, const wchar_t *filename, PyCompilerFlags *p_cf)
+{
+    PyObject *unicode, *bytes = NULL;
+    char *filename_str;
+    int run;
+
+    /* call pending calls like signal handlers (SIGINT) */
+    if (Py_MakePendingCalls() == -1) {
         PyErr_Print();
         return 1;
     }
-    else {
-        return -1;
+
+    if (filename) {
+        unicode = PyUnicode_FromWideChar(filename, wcslen(filename));
+        if (unicode != NULL) {
+            bytes = PyUnicode_EncodeFSDefault(unicode);
+            Py_DECREF(unicode);
+        }
+        if (bytes != NULL)
+            filename_str = PyBytes_AsString(bytes);
+        else {
+            PyErr_Clear();
+            filename_str = "<encoding error>";
+        }
     }
+    else
+        filename_str = "<stdin>";
+
+    run = PyRun_AnyFileExFlags(fp, filename_str, filename != NULL, p_cf);
+    Py_XDECREF(bytes);
+    return run != 0;
 }
 
 
@@ -275,6 +322,9 @@ Py_Main(int argc, wchar_t **argv)
     wchar_t *module = NULL;
     FILE *fp = stdin;
     char *p;
+#ifdef MS_WINDOWS
+    wchar_t *wp;
+#endif
     int skipfirstline = 0;
     int stdin_is_interactive = 0;
     int help = 0;
@@ -368,8 +418,6 @@ Py_Main(int argc, wchar_t **argv)
             skipfirstline = 1;
             break;
 
-        /* case 'X': reserved for implementation-specific arguments */
-
         case 'h':
         case '?':
             help++;
@@ -383,6 +431,14 @@ Py_Main(int argc, wchar_t **argv)
             PySys_AddWarnOption(_PyOS_optarg);
             break;
 
+        case 'X':
+            PySys_AddXOption(_PyOS_optarg);
+            break;
+
+        case 'q':
+            Py_QuietFlag++;
+            break;
+
         case 'R':
             Py_HashRandomizationFlag++;
             break;
@@ -415,6 +471,61 @@ Py_Main(int argc, wchar_t **argv)
         (p = Py_GETENV("PYTHONNOUSERSITE")) && *p != '\0')
         Py_NoUserSiteDirectory = 1;
 
+#ifdef MS_WINDOWS
+    if (!Py_IgnoreEnvironmentFlag && (wp = _wgetenv(L"PYTHONWARNINGS")) &&
+        *wp != L'\0') {
+        wchar_t *buf, *warning;
+
+        buf = (wchar_t *)malloc((wcslen(wp) + 1) * sizeof(wchar_t));
+        if (buf == NULL)
+            Py_FatalError(
+               "not enough memory to copy PYTHONWARNINGS");
+        wcscpy(buf, wp);
+        for (warning = wcstok(buf, L",");
+             warning != NULL;
+             warning = wcstok(NULL, L",")) {
+            PySys_AddWarnOption(warning);
+        }
+        free(buf);
+    }
+#else
+    if ((p = Py_GETENV("PYTHONWARNINGS")) && *p != '\0') {
+        char *buf, *oldloc;
+        PyObject *unicode;
+
+        /* settle for strtok here as there's no one standard
+           C89 wcstok */
+        buf = (char *)malloc(strlen(p) + 1);
+        if (buf == NULL)
+            Py_FatalError(
+               "not enough memory to copy PYTHONWARNINGS");
+        strcpy(buf, p);
+        oldloc = strdup(setlocale(LC_ALL, NULL));
+        setlocale(LC_ALL, "");
+        for (p = strtok(buf, ","); p != NULL; p = strtok(NULL, ",")) {
+#ifdef __APPLE__
+            /* Use utf-8 on Mac OS X */
+            unicode = PyUnicode_FromString(p);
+#else
+            wchar_t *wchar;
+            size_t len;
+            wchar = _Py_char2wchar(p, &len);
+            if (wchar == NULL)
+                continue;
+            unicode = PyUnicode_FromWideChar(wchar, len);
+            PyMem_Free(wchar);
+#endif
+            if (unicode == NULL)
+                continue;
+            PySys_AddWarnOptionUnicode(unicode);
+            Py_DECREF(unicode);
+        }
+        setlocale(LC_ALL, oldloc);
+        free(oldloc);
+        free(buf);
+    }
+#endif
+
     if (command == NULL && module == NULL && _PyOS_optind < argc &&
         wcscmp(argv[_PyOS_optind], L"-") != 0)
     {
@@ -430,11 +541,14 @@ Py_Main(int argc, wchar_t **argv)
 
     stdin_is_interactive = Py_FdIsInteractive(stdin, (char *)0);
 
-    if (Py_UnbufferedStdioFlag) {
 #if defined(MS_WINDOWS) || defined(__CYGWIN__)
-        _setmode(fileno(stdin), O_BINARY);
-        _setmode(fileno(stdout), O_BINARY);
+    /* don't translate newlines (\r\n <=> \n) */
+    _setmode(fileno(stdin), O_BINARY);
+    _setmode(fileno(stdout), O_BINARY);
+    _setmode(fileno(stderr), O_BINARY);
 #endif
+
+    if (Py_UnbufferedStdioFlag) {
 #ifdef HAVE_SETVBUF
         setvbuf(stdin,  (char *)NULL, _IONBF, BUFSIZ);
         setvbuf(stdout, (char *)NULL, _IONBF, BUFSIZ);
@@ -496,8 +610,9 @@ Py_Main(int argc, wchar_t **argv)
 #endif
     Py_Initialize();
 
-    if (Py_VerboseFlag ||
-        (command == NULL && filename == NULL && module == NULL && stdin_is_interactive)) {
+    if (!Py_QuietFlag && (Py_VerboseFlag ||
+                        (command == NULL && filename == NULL &&
+                         module == NULL && stdin_is_interactive))) {
         fprintf(stderr, "Python %s on %s\n",
             Py_GetVersion(), Py_GetPlatform());
         if (!Py_NoSiteFlag)
@@ -511,10 +626,9 @@ Py_Main(int argc, wchar_t **argv)
     }
 
     if (module != NULL) {
-        /* Backup _PyOS_optind and force sys.argv[0] = '-c'
-           so that PySys_SetArgv correctly sets sys.path[0] to ''*/
+        /* Backup _PyOS_optind and force sys.argv[0] = '-m'*/
         _PyOS_optind--;
-        argv[_PyOS_optind] = L"-c";
+        argv[_PyOS_optind] = L"-m";
     }
 
     PySys_SetArgv(argc-_PyOS_optind, argv+_PyOS_optind);
@@ -530,22 +644,8 @@ Py_Main(int argc, wchar_t **argv)
     }
 
     if (command) {
-        char *commandStr;
-        PyObject *commandObj = PyUnicode_FromWideChar(
-            command, wcslen(command));
+        sts = run_command(command, &cf);
         free(command);
-        if (commandObj != NULL)
-            commandStr = _PyUnicode_AsString(commandObj);
-        else
-            commandStr = NULL;
-        if (commandStr != NULL) {
-            sts = PyRun_SimpleStringFlags(commandStr, &cf) != 0;
-            Py_DECREF(commandObj);
-        }
-        else {
-            PyErr_Print();
-            sts = 1;
-        }
     } else if (module) {
         sts = RunModule(module, 1);
     }
@@ -564,19 +664,20 @@ Py_Main(int argc, wchar_t **argv)
         }
 
         if (sts==-1 && filename!=NULL) {
-            if ((fp = _wfopen(filename, L"r")) == NULL) {
-                char cfilename[PATH_MAX];
-                size_t r = wcstombs(cfilename, filename, PATH_MAX);
-                if (r == PATH_MAX)
-                    /* cfilename is not null-terminated;
-                     * forcefully null-terminating it
-                     * might break the shift state */
-                    strcpy(cfilename, "<file name too long>");
-                if (r == ((size_t)-1))
-                    strcpy(cfilename, "<unprintable file name>");
+            fp = _Py_wfopen(filename, L"r");
+            if (fp == NULL) {
+                char *cfilename_buffer;
+                const char *cfilename;
+                int err = errno;
+                cfilename_buffer = _Py_wchar2char(filename, NULL);
+                if (cfilename_buffer != NULL)
+                    cfilename = cfilename_buffer;
+                else
+                    cfilename = "<unprintable file name>";
                 fprintf(stderr, "%ls: can't open file '%s': [Errno %d] %s\n",
-                    argv[0], cfilename, errno, strerror(errno));
-
+                    argv[0], cfilename, err, strerror(err));
+                if (cfilename_buffer)
+                    PyMem_Free(cfilename_buffer);
                 return 2;
             }
             else if (skipfirstline) {
@@ -602,30 +703,8 @@ Py_Main(int argc, wchar_t **argv)
             }
         }
 
-        if (sts==-1) {
-            PyObject *filenameObj = NULL;
-            char *p_cfilename = "<stdin>";
-            if (filename) {
-                filenameObj = PyUnicode_FromWideChar(
-                    filename, wcslen(filename));
-                if (filenameObj != NULL)
-                    p_cfilename = _PyUnicode_AsString(filenameObj);
-                else
-                    p_cfilename = "<decoding error>";
-            }
-            /* call pending calls like signal handlers (SIGINT) */
-            if (Py_MakePendingCalls() == -1) {
-                PyErr_Print();
-                sts = 1;
-            } else {
-                sts = PyRun_AnyFileExFlags(
-                    fp,
-                    p_cfilename,
-                    filename != NULL, &cf) != 0;
-            }
-            Py_XDECREF(filenameObj);
-        }
-
+        if (sts == -1)
+            sts = run_file(fp, filename, &cf);
     }
 
     /* Check this environment variable at the end, to give programs the