diff options
| author | Benjamin Peterson <benjamin@python.org> | 2010-04-03 22:55:48 +0000 |
|---|---|---|
| committer | Benjamin Peterson <benjamin@python.org> | 2010-04-03 22:55:48 +0000 |
| commit | 18d31dfb08d053685c29c63536872aa65b6a20ea (patch) | |
| tree | 30d81d862adcfc58913c0d314cfb14ec288f6bb5 /Parser | |
| parent | b9aac80fe0b18b60b6129c51e3237910291bb985 (diff) | |
| download | cpython-18d31dfb08d053685c29c63536872aa65b6a20ea.tar.gz | |
Merged revisions 79723 via svnmerge from
svn+ssh://pythondev@svn.python.org/python/trunk
........
r79723 | benjamin.peterson | 2010-04-03 17:48:51 -0500 (Sat, 03 Apr 2010) | 1 line
ensure that the locale does not affect the tokenization of identifiers
........
Diffstat (limited to 'Parser')
| -rw-r--r-- | Parser/tokenizer.c | 12 |
1 files changed, 11 insertions, 1 deletions
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 0e2dc3a52d..be2940cd67 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -179,6 +179,16 @@ decode_str(const char *str, int exec_input, struct tok_state *tok) #else /* PGEN */ +/* Ensure that the locale does not interfere with tokenization. */ + +static int +ascii_isalnum(int c) +{ + return (('a' <= c && c <= 'z') || + ('A' <= c && c <= 'Z') || + ('0' <= c && c <= '9')); +} + static char * error_ret(struct tok_state *tok) /* XXX */ { @@ -245,7 +255,7 @@ get_coding_spec(const char *s, Py_ssize_t size) } while (t[0] == '\x20' || t[0] == '\t'); begin = t; - while (isalnum(Py_CHARMASK(t[0])) || + while (ascii_isalnum(Py_CHARMASK(t[0])) || t[0] == '-' || t[0] == '_' || t[0] == '.') t++; |