Add unspecified-encoding checker #3826 (#4753)

* Add unspecified-encoding checker #3826
This adds an unspecified-encoding checker that adds a warning
whenever open() is called without an explicit encoding argument.
This closes #3826

* Update tests to conform to unspecified-encoding
With addition of the unspecified-encoding checker calls of open()
need an encoding argument.
Where necessary this argument has been added, or the message has been
disabled.
This also includes small linting changes to a small number
of tests. Their test-data has been updated to reflect new line numbers.

* Update scripts to conform to unspecified-encoding
With addition of the unspecified-encoding checker calls of open()
need an encoding argument.
Where necessary this argument has been added.

* Update pylint to conform to unspecified-encoding
With addition of the unspecified-encoding checker calls of open()
need an encoding argument.
Where necessary this argument has been added.

Co-authored-by: Marc Mueller <30130371+cdce8p@users.noreply.github.com>
Co-authored-by: Marc Mueller <30130371+cdce8p@users.noreply.github.com>

15 files changed, 131 insertions, 56 deletions

diff --git a/tests/functional/b/bad_open_mode_py3.py b/tests/functional/b/bad_open_mode_py3.py
index 27aa2e6a2..b68238013 100644
--- a/tests/functional/b/bad_open_mode_py3.py
+++ b/tests/functional/b/bad_open_mode_py3.py
@@ -3,22 +3,22 @@
 

 NAME = "foo.bar"

 open(NAME, "wb")

-open(NAME, "w")

+open(NAME, "w", encoding="utf-8")

 open(NAME, "rb")

-open(NAME, "x")

+open(NAME, "x", encoding="utf-8")

 open(NAME, "br")

-open(NAME, "+r")

+open(NAME, "+r", encoding="utf-8")

 open(NAME, "xb")

-open(NAME, "rwx")  # [bad-open-mode]

-open(NAME, "rr")  # [bad-open-mode]

-open(NAME, "+")  # [bad-open-mode]

-open(NAME, "xw")  # [bad-open-mode]

+open(NAME, "rwx", encoding="utf-8")  # [bad-open-mode]

+open(NAME, "rr", encoding="utf-8")  # [bad-open-mode]

+open(NAME, "+", encoding="utf-8")  # [bad-open-mode]

+open(NAME, "xw", encoding="utf-8")  # [bad-open-mode]

 open(NAME, "ab+")

 open(NAME, "a+b")

 open(NAME, "+ab")

 open(NAME, "+rUb")

 open(NAME, "x+b")

-open(NAME, "Ua")  # [bad-open-mode]

-open(NAME, "Ur++")  # [bad-open-mode]

-open(NAME, "Ut")

+open(NAME, "Ua", encoding="utf-8")  # [bad-open-mode]

+open(NAME, "Ur++", encoding="utf-8")  # [bad-open-mode]

+open(NAME, "Ut", encoding="utf-8")

 open(NAME, "Ubr")

diff --git a/tests/functional/c/consider/consider_using_with.py b/tests/functional/c/consider/consider_using_with.py
index fc2804455..d05866aaa 100644
--- a/tests/functional/c/consider/consider_using_with.py
+++ b/tests/functional/c/consider/consider_using_with.py
@@ -151,7 +151,7 @@ def test_suppress_in_exit_stack():
     """Regression test for issue #4654 (false positive)"""
     with contextlib.ExitStack() as stack:
         _ = stack.enter_context(
-            open("/sys/firmware/devicetree/base/hwid,location", "r")
+            open("/sys/firmware/devicetree/base/hwid,location", "r", encoding="utf-8")
         )  # must not trigger
 
 
diff --git a/tests/functional/c/consider/consider_using_with_open.py b/tests/functional/c/consider/consider_using_with_open.py
index e93b619b7..30be92def 100644
--- a/tests/functional/c/consider/consider_using_with_open.py
+++ b/tests/functional/c/consider/consider_using_with_open.py
@@ -1,5 +1,5 @@
 # pylint: disable=missing-function-docstring, missing-module-docstring, invalid-name, import-outside-toplevel
-# pylint: disable=missing-class-docstring, too-few-public-methods, unused-variable, multiple-statements
+# pylint: disable=missing-class-docstring, too-few-public-methods, unused-variable, multiple-statements, line-too-long
 """
 The functional test for the standard ``open()`` function has to be moved in a separate file,
 because PyPy has to be excluded for the tests as the ``open()`` function is uninferable in PyPy.
@@ -8,26 +8,26 @@ PyPy from ALL functional tests.
 """
 from contextlib import contextmanager
 
-
-myfile = open("test.txt")  # [consider-using-with]
+myfile = open("test.txt", encoding="utf-8")  # [consider-using-with]
 
 
 def test_open():
-    fh = open("test.txt")  # [consider-using-with]
+    fh = open("test.txt", encoding="utf-8")  # [consider-using-with]
     fh.close()
 
-    with open("test.txt") as fh:  # must not trigger
+    with open("test.txt", encoding="utf-8") as fh:  # must not trigger
         fh.read()
 
 
 def test_open_in_enter():
     """Message must not trigger if the resource is allocated in a context manager."""
+
     class MyContextManager:
         def __init__(self):
             self.file_handle = None
 
         def __enter__(self):
-            self.file_handle = open("foo.txt", "w")  # must not trigger
+            self.file_handle = open("foo.txt", "w", encoding="utf-8")  # must not trigger
 
         def __exit__(self, exc_type, exc_value, traceback):
             self.file_handle.close()
@@ -36,31 +36,31 @@ def test_open_in_enter():
 @contextmanager
 def test_open_in_with_contextlib():
     """Message must not trigger if the resource is allocated in a context manager."""
-    file_handle = open("foo.txt", "w")  # must not trigger
+    file_handle = open("foo.txt", "w", encoding="utf-8")  # must not trigger
     yield file_handle
     file_handle.close()
 
 
 def test_open_outside_assignment():
-    open("foo").read()  # [consider-using-with]
-    content = open("foo").read()  # [consider-using-with]
+    open("foo", encoding="utf-8").read()  # [consider-using-with]
+    content = open("foo", encoding="utf-8").read()  # [consider-using-with]
 
 
 def test_open_inside_with_block():
-    with open("foo") as fh:
-        open("bar")  # [consider-using-with]
+    with open("foo", encoding="utf-8") as fh:
+        open("bar", encoding="utf-8")  # [consider-using-with]
 
 
 def test_ternary_if_in_with_block(file1, file2, which):
     """Regression test for issue #4676 (false positive)"""
-    with (open(file1) if which else open(file2)) as input_file:  # must not trigger
+    with (open(file1, encoding="utf-8") if which else open(file2, encoding="utf-8")) as input_file:  # must not trigger
         return input_file.read()
 
 
 def test_single_line_with(file1):
-    with open(file1): return file1.read()  # must not trigger
+    with open(file1, encoding="utf-8"): return file1.read()  # must not trigger
 
 
 def test_multiline_with_items(file1, file2, which):
-    with (open(file1) if which
-          else open(file2)) as input_file:  return input_file.read()
+    with (open(file1, encoding="utf-8") if which
+        else open(file2, encoding="utf-8")) as input_file: return input_file.read()
diff --git a/tests/functional/c/consider/consider_using_with_open.txt b/tests/functional/c/consider/consider_using_with_open.txt
index 63cd08b7c..ea3503123 100644
--- a/tests/functional/c/consider/consider_using_with_open.txt
+++ b/tests/functional/c/consider/consider_using_with_open.txt
@@ -1,5 +1,5 @@
-consider-using-with:12:9::Consider using 'with' for resource-allocating operations
-consider-using-with:16:9:test_open:Consider using 'with' for resource-allocating operations
+consider-using-with:11:9::Consider using 'with' for resource-allocating operations
+consider-using-with:15:9:test_open:Consider using 'with' for resource-allocating operations
 consider-using-with:45:4:test_open_outside_assignment:Consider using 'with' for resource-allocating operations
 consider-using-with:46:14:test_open_outside_assignment:Consider using 'with' for resource-allocating operations
 consider-using-with:51:8:test_open_inside_with_block:Consider using 'with' for resource-allocating operations
diff --git a/tests/functional/d/defined_and_used_on_same_line.py b/tests/functional/d/defined_and_used_on_same_line.py
index a9b9eb7d5..d5bdb4db0 100644
--- a/tests/functional/d/defined_and_used_on_same_line.py
+++ b/tests/functional/d/defined_and_used_on_same_line.py
@@ -1,5 +1,5 @@
 """Check for definitions and usage happening on the same line."""
-#pylint: disable=missing-docstring,multiple-statements,no-absolute-import,parameter-unpacking,wrong-import-position,unnecessary-comprehension
+#pylint: disable=missing-docstring,multiple-statements,no-absolute-import,parameter-unpacking,wrong-import-position,unnecessary-comprehension,unspecified-encoding
 from __future__ import print_function
 print([index
        for index in range(10)])
diff --git a/tests/functional/d/disabled_msgid_in_pylintrc.py b/tests/functional/d/disabled_msgid_in_pylintrc.py
index 09ddf1147..91cb155e1 100644
--- a/tests/functional/d/disabled_msgid_in_pylintrc.py
+++ b/tests/functional/d/disabled_msgid_in_pylintrc.py
@@ -1,6 +1,6 @@
 """https://github.com/PyCQA/pylint/issues/4265"""
 
 try:
-    f = open('test')
+    f = open('test', encoding="utf-8")
 except Exception:
     pass
diff --git a/tests/functional/n/non/non_iterator_returned.py b/tests/functional/n/non/non_iterator_returned.py
index 94c3601f5..4d0dd2a39 100644
--- a/tests/functional/n/non/non_iterator_returned.py
+++ b/tests/functional/n/non/non_iterator_returned.py
@@ -67,7 +67,7 @@ class FileBasedIterator(object):
     def __iter__(self):
         if self.file is not None:
             self.file.close()
-        self.file = open(self.path)
+        self.file = open(self.path, encoding="utf-8")
         # self file has two infered values: None and <instance of 'file'>
         # we don't want to emit error in this case
         return self.file
diff --git a/tests/functional/r/redefined_argument_from_local.py b/tests/functional/r/redefined_argument_from_local.py
index 820252781..7ccff1d18 100644
--- a/tests/functional/r/redefined_argument_from_local.py
+++ b/tests/functional/r/redefined_argument_from_local.py
@@ -1,31 +1,33 @@
 # pylint: disable=missing-docstring, unused-variable, unused-argument
 # pylint: disable=redefined-outer-name, invalid-name
 
+
 def test_redefined_in_with(name):
-    with open('something') as name: # [redefined-argument-from-local]
+    with open("something", encoding="utf-8") as name:  # [redefined-argument-from-local]
         pass
-    with open('something') as (second, name): # [redefined-argument-from-local]
+    with open("something", encoding="utf-8") as (second, name):  # [redefined-argument-from-local]
         pass
-    with open('something') as (second, (name, third)): # [redefined-argument-from-local]
+    with open("something", encoding="utf-8") as (
+        second,
+        (name, third),  # [redefined-argument-from-local]
+    ):
         pass
     other = None
-    with open('something') as other:
+    with open("something", encoding="utf-8") as other:
         pass
 
 
-
 def test_not_redefined_in_with(name):
-    with open('something') as test_redefined_in_with:
+    with open("something", encoding="utf-8") as test_redefined_in_with:
         pass
 
 
-
 def test_redefined_in_for(name):
-    for name in []: # [redefined-argument-from-local]
+    for name in []:  # [redefined-argument-from-local]
         pass
-    for (name, is_) in []: # [redefined-argument-from-local]
+    for (name, is_) in []:  # [redefined-argument-from-local]
         pass
-    for (is_, (name, _)) in []: # [redefined-argument-from-local]
+    for (is_, (name, _)) in []:  # [redefined-argument-from-local]
         pass
     for _ in []:
         pass
@@ -45,7 +47,7 @@ def test_not_redefined_in_for(name):
 def test_redefined_in_except_handler(name):
     try:
         1 / 0
-    except ZeroDivisionError as name: # [redefined-argument-from-local]
+    except ZeroDivisionError as name:  # [redefined-argument-from-local]
         pass
 
 
@@ -58,7 +60,7 @@ def test_not_redefined_in_except_handler(name):
 
 def test_not_redefined(name):
     if not name:
-        name = ''
+        name = ""
 
 
 def apply_filter(objects, filt=lambda obj: True):
diff --git a/tests/functional/r/redefined_argument_from_local.txt b/tests/functional/r/redefined_argument_from_local.txt
index 28ec50da1..b811971c6 100644
--- a/tests/functional/r/redefined_argument_from_local.txt
+++ b/tests/functional/r/redefined_argument_from_local.txt
@@ -1,7 +1,7 @@
-redefined-argument-from-local:5:30:test_redefined_in_with:Redefining argument with the local name 'name'
-redefined-argument-from-local:7:39:test_redefined_in_with:Redefining argument with the local name 'name'
-redefined-argument-from-local:9:40:test_redefined_in_with:Redefining argument with the local name 'name'
-redefined-argument-from-local:24:8:test_redefined_in_for:Redefining argument with the local name 'name'
-redefined-argument-from-local:26:9:test_redefined_in_for:Redefining argument with the local name 'name'
-redefined-argument-from-local:28:15:test_redefined_in_for:Redefining argument with the local name 'name'
-redefined-argument-from-local:48:4:test_redefined_in_except_handler:Redefining argument with the local name 'name'
+redefined-argument-from-local:6:48:test_redefined_in_with:Redefining argument with the local name 'name'
+redefined-argument-from-local:8:57:test_redefined_in_with:Redefining argument with the local name 'name'
+redefined-argument-from-local:12:9:test_redefined_in_with:Redefining argument with the local name 'name'
+redefined-argument-from-local:26:8:test_redefined_in_for:Redefining argument with the local name 'name'
+redefined-argument-from-local:28:9:test_redefined_in_for:Redefining argument with the local name 'name'
+redefined-argument-from-local:30:15:test_redefined_in_for:Redefining argument with the local name 'name'
+redefined-argument-from-local:50:4:test_redefined_in_except_handler:Redefining argument with the local name 'name'
diff --git a/tests/functional/r/regression/regression_4612_crash_pytest_fixture.py b/tests/functional/r/regression/regression_4612_crash_pytest_fixture.py
index f0aa0f5d1..f7680f1fb 100644
--- a/tests/functional/r/regression/regression_4612_crash_pytest_fixture.py
+++ b/tests/functional/r/regression/regression_4612_crash_pytest_fixture.py
@@ -5,5 +5,5 @@ import pytest
 
 @pytest.fixture
 def qm_file():
-    qm_file = open("src/test/resources/example_qm_file.csv").read()
+    qm_file = open("src/test/resources/example_qm_file.csv", encoding="utf-8").read()
     return qm_file
diff --git a/tests/functional/u/unspecified_encoding_py38.py b/tests/functional/u/unspecified_encoding_py38.py
new file mode 100644
index 000000000..20d2d7be1
--- /dev/null
+++ b/tests/functional/u/unspecified_encoding_py38.py
@@ -0,0 +1,55 @@
+"""Warnings for using open() without specifying an encoding"""
+# pylint: disable=consider-using-with
+import io
+import locale
+
+FILENAME = "foo.bar"
+open(FILENAME, "w", encoding="utf-8")
+open(FILENAME, "wb")
+open(FILENAME, "w+b")
+open(FILENAME)  # [unspecified-encoding]
+open(FILENAME, "wt")  # [unspecified-encoding]
+open(FILENAME, "w+")  # [unspecified-encoding]
+open(FILENAME, "w", encoding=None)  # [unspecified-encoding]
+open(FILENAME, "r")  # [unspecified-encoding]
+
+with open(FILENAME, encoding="utf8", errors="surrogateescape") as f:
+    pass
+
+LOCALE_ENCODING = locale.getlocale()[1]
+with open(FILENAME, encoding=LOCALE_ENCODING) as f:
+    pass
+
+with open(FILENAME) as f:  # [unspecified-encoding]
+    pass
+
+with open(FILENAME, encoding=None) as f:  # [unspecified-encoding]
+    pass
+
+LOCALE_ENCODING = None
+with open(FILENAME, encoding=LOCALE_ENCODING) as f:  # [unspecified-encoding]
+    pass
+
+io.open(FILENAME, "w+b")
+io.open_code(FILENAME)
+io.open(FILENAME)  # [unspecified-encoding]
+io.open(FILENAME, "wt")  # [unspecified-encoding]
+io.open(FILENAME, "w+")  # [unspecified-encoding]
+io.open(FILENAME, "w", encoding=None)  # [unspecified-encoding]
+
+with io.open(FILENAME, encoding="utf8", errors="surrogateescape") as f:
+    pass
+
+LOCALE_ENCODING = locale.getlocale()[1]
+with io.open(FILENAME, encoding=LOCALE_ENCODING) as f:
+    pass
+
+with io.open(FILENAME) as f:  # [unspecified-encoding]
+    pass
+
+with io.open(FILENAME, encoding=None) as f:  # [unspecified-encoding]
+    pass
+
+LOCALE_ENCODING = None
+with io.open(FILENAME, encoding=LOCALE_ENCODING) as f:  # [unspecified-encoding]
+    pass
diff --git a/tests/functional/u/unspecified_encoding_py38.rc b/tests/functional/u/unspecified_encoding_py38.rc
new file mode 100644
index 000000000..85fc502b3
--- /dev/null
+++ b/tests/functional/u/unspecified_encoding_py38.rc
@@ -0,0 +1,2 @@
+[testoptions]
+min_pyver=3.8
diff --git a/tests/functional/u/unspecified_encoding_py38.txt b/tests/functional/u/unspecified_encoding_py38.txt
new file mode 100644
index 000000000..cde7f2238
--- /dev/null
+++ b/tests/functional/u/unspecified_encoding_py38.txt
@@ -0,0 +1,15 @@
+unspecified-encoding:10:0::"Using open without explicitly specifying an encoding"
+unspecified-encoding:11:0::"Using open without explicitly specifying an encoding"
+unspecified-encoding:12:0::"Using open without explicitly specifying an encoding"
+unspecified-encoding:13:0::"Using open without explicitly specifying an encoding"
+unspecified-encoding:14:0::"Using open without explicitly specifying an encoding"
+unspecified-encoding:23:5::"Using open without explicitly specifying an encoding"
+unspecified-encoding:26:5::"Using open without explicitly specifying an encoding"
+unspecified-encoding:30:5::"Using open without explicitly specifying an encoding"
+unspecified-encoding:35:0::"Using open without explicitly specifying an encoding"
+unspecified-encoding:36:0::"Using open without explicitly specifying an encoding"
+unspecified-encoding:37:0::"Using open without explicitly specifying an encoding"
+unspecified-encoding:38:0::"Using open without explicitly specifying an encoding"
+unspecified-encoding:47:5::"Using open without explicitly specifying an encoding"
+unspecified-encoding:50:5::"Using open without explicitly specifying an encoding"
+unspecified-encoding:54:5::"Using open without explicitly specifying an encoding"
diff --git a/tests/functional/w/with_used_before_assign.py b/tests/functional/w/with_used_before_assign.py
index 64a475af1..ebe7d3093 100644
--- a/tests/functional/w/with_used_before_assign.py
+++ b/tests/functional/w/with_used_before_assign.py
@@ -1,11 +1,12 @@
-'''
+"""
 Regression test for
 https://bitbucket.org/logilab/pylint/issue/128/attributeerror-when-parsing
-'''
+"""
 from __future__ import with_statement
 
+
 def do_nothing():
     """ empty """
-    with open("") as ctx.obj:  # [undefined-variable]
+    with open("", encoding="utf-8") as ctx.obj:  # [undefined-variable]
         context.do()  # [used-before-assignment]
         context = None
diff --git a/tests/functional/w/with_used_before_assign.txt b/tests/functional/w/with_used_before_assign.txt
index 4a98cec60..8e1fe1be8 100644
--- a/tests/functional/w/with_used_before_assign.txt
+++ b/tests/functional/w/with_used_before_assign.txt
@@ -1,2 +1,2 @@
-undefined-variable:9:21:do_nothing:Undefined variable 'ctx'
-used-before-assignment:10:8:do_nothing:Using variable 'context' before assignment
+undefined-variable:10:39:do_nothing:Undefined variable 'ctx'
+used-before-assignment:11:8:do_nothing:Using variable 'context' before assignment