Add ignore_signatures to similarity checker (#4474)

* Add ignore_signatures to similarity checker

Co-authored-by: Pierre Sassoulas <pierre.sassoulas@gmail.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

7 files changed, 129 insertions, 13 deletions

diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt
index 7d1cd868b..a5e18e823 100644
--- a/CONTRIBUTORS.txt
+++ b/CONTRIBUTORS.txt
@@ -486,3 +486,6 @@ contributors:
 * Andrew Haigh (nelfin): contributor
 
 * Pang Yu Shao (yushao2): contributor
+
+* Aditya Gupta (adityagupta1089) : contributor
+  - Added ignore_signatures to duplicate checker
diff --git a/ChangeLog b/ChangeLog
index a78f63f0f..a8a7323e9 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -51,6 +51,10 @@ modules are added.
 
   Closes #4412
 
+* Add ignore_signatures to duplicate code checker
+
+  Closes #3619
+
 
 What's New in Pylint 2.8.2?
 ===========================
diff --git a/doc/whatsnew/2.9.rst b/doc/whatsnew/2.9.rst
index fe78c6fd2..14669ca27 100644
--- a/doc/whatsnew/2.9.rst
+++ b/doc/whatsnew/2.9.rst
@@ -15,6 +15,8 @@ New checkers
 * ``consider-using-dict-items``: Emitted when iterating over dictionary keys and then
   indexing the same dictionary with the key within loop body.
 
+* An ``ignore_signatures`` option has been added to the similarity checker. It will permits  to reduce false positives when multiple functions have the same parameters.
+
 Other Changes
 =============
 
diff --git a/pylint/checkers/similar.py b/pylint/checkers/similar.py
index 780db07ef..01caf91a3 100644
--- a/pylint/checkers/similar.py
+++ b/pylint/checkers/similar.py
@@ -30,7 +30,9 @@ import re
 import sys
 from collections import defaultdict
 from getopt import getopt
-from itertools import groupby
+from io import TextIOWrapper
+from itertools import chain, groupby
+from typing import List
 
 import astroid
 
@@ -47,18 +49,22 @@ class Similar:
 
     def __init__(
         self,
-        min_lines=4,
-        ignore_comments=False,
-        ignore_docstrings=False,
-        ignore_imports=False,
-    ):
+        min_lines: int = 4,
+        ignore_comments: bool = False,
+        ignore_docstrings: bool = False,
+        ignore_imports: bool = False,
+        ignore_signatures: bool = False,
+    ) -> None:
         self.min_lines = min_lines
         self.ignore_comments = ignore_comments
         self.ignore_docstrings = ignore_docstrings
         self.ignore_imports = ignore_imports
-        self.linesets = []
+        self.ignore_signatures = ignore_signatures
+        self.linesets: List["LineSet"] = []
 
-    def append_stream(self, streamid, stream, encoding=None):
+    def append_stream(
+        self, streamid: str, stream: TextIOWrapper, encoding=None
+    ) -> None:
         """append a file to search for similarities"""
         if encoding is None:
             readlines = stream.readlines
@@ -72,6 +78,7 @@ class Similar:
                     self.ignore_comments,
                     self.ignore_docstrings,
                     self.ignore_imports,
+                    self.ignore_signatures,
                 )
             )
         except UnicodeDecodeError:
@@ -178,12 +185,19 @@ class Similar:
         self.linesets = [line for lineset in linesets_collection for line in lineset]
 
 
-def stripped_lines(lines, ignore_comments, ignore_docstrings, ignore_imports):
+def stripped_lines(
+    lines,
+    ignore_comments: bool,
+    ignore_docstrings: bool,
+    ignore_imports: bool,
+    ignore_signatures: bool,
+):
     """return lines with leading/trailing whitespace and any ignored code
     features removed
     """
-    if ignore_imports:
+    if ignore_imports or ignore_signatures:
         tree = astroid.parse("".join(lines))
+    if ignore_imports:
         node_is_import_by_lineno = (
             (node.lineno, isinstance(node, (astroid.Import, astroid.ImportFrom)))
             for node in tree.body
@@ -195,6 +209,15 @@ def stripped_lines(lines, ignore_comments, ignore_docstrings, ignore_imports):
             )
         }
         current_line_is_import = False
+    if ignore_signatures:
+        functions = [
+            n
+            for n in tree.body
+            if isinstance(n, (astroid.FunctionDef, astroid.AsyncFunctionDef))
+        ]
+        signature_lines = set(
+            chain(*(range(func.fromlineno, func.body[0].lineno) for func in functions))
+        )
 
     strippedlines = []
     docstring = None
@@ -220,6 +243,8 @@ def stripped_lines(lines, ignore_comments, ignore_docstrings, ignore_imports):
                 line = ""
         if ignore_comments:
             line = line.split("#", 1)[0].strip()
+        if ignore_signatures and lineno in signature_lines:
+            line = ""
         strippedlines.append(line)
     return strippedlines
 
@@ -235,11 +260,12 @@ class LineSet:
         ignore_comments=False,
         ignore_docstrings=False,
         ignore_imports=False,
+        ignore_signatures=False,
     ):
         self.name = name
         self._real_lines = lines
         self._stripped_lines = stripped_lines(
-            lines, ignore_comments, ignore_docstrings, ignore_imports
+            lines, ignore_comments, ignore_docstrings, ignore_imports, ignore_signatures
         )
         self._index = self._mk_index()
 
@@ -361,6 +387,15 @@ class SimilarChecker(BaseChecker, Similar, MapReduceMixin):
                 "help": "Ignore imports when computing similarities.",
             },
         ),
+        (
+            "ignore-signatures",
+            {
+                "default": False,
+                "type": "yn",
+                "metavar": "<y or n>",
+                "help": "Ignore function signatures when computing similarities.",
+            },
+        ),
     )
     # reports
     reports = (("RP0801", "Duplication", report_similarities),)  # type: ignore
@@ -386,6 +421,8 @@ class SimilarChecker(BaseChecker, Similar, MapReduceMixin):
             self.ignore_docstrings = self.config.ignore_docstrings
         elif optname == "ignore-imports":
             self.ignore_imports = self.config.ignore_imports
+        elif optname == "ignore-signatures":
+            self.ignore_signatures = self.config.ignore_signatures
 
     def open(self):
         """init the checkers: reset linesets and statistics information"""
@@ -451,7 +488,7 @@ def usage(status=0):
     print()
     print(
         "Usage: symilar [-d|--duplicates min_duplicated_lines] \
-[-i|--ignore-comments] [--ignore-docstrings] [--ignore-imports] file1..."
+[-i|--ignore-comments] [--ignore-docstrings] [--ignore-imports] [--ignore-signatures] file1..."
     )
     sys.exit(status)
 
@@ -468,11 +505,13 @@ def Run(argv=None):
         "ignore-comments",
         "ignore-imports",
         "ignore-docstrings",
+        "ignore-signatures",
     )
     min_lines = 4
     ignore_comments = False
     ignore_docstrings = False
     ignore_imports = False
+    ignore_signatures = False
     opts, args = getopt(argv, s_opts, l_opts)
     for opt, val in opts:
         if opt in ("-d", "--duplicates"):
@@ -485,9 +524,13 @@ def Run(argv=None):
             ignore_docstrings = True
         elif opt in ("--ignore-imports",):
             ignore_imports = True
+        elif opt in ("--ignore-signatures",):
+            ignore_signatures = True
     if not args:
         usage(1)
-    sim = Similar(min_lines, ignore_comments, ignore_docstrings, ignore_imports)
+    sim = Similar(
+        min_lines, ignore_comments, ignore_docstrings, ignore_imports, ignore_signatures
+    )
     for filename in args:
         with open(filename) as stream:
             sim.append_stream(filename, stream)
diff --git a/tests/checkers/unittest_similar.py b/tests/checkers/unittest_similar.py
index 01b2bab6e..fc0035861 100644
--- a/tests/checkers/unittest_similar.py
+++ b/tests/checkers/unittest_similar.py
@@ -31,6 +31,8 @@ SIMILAR1 = str(INPUT / "similar1")
 SIMILAR2 = str(INPUT / "similar2")
 SIMILAR3 = str(INPUT / "similar3")
 SIMILAR4 = str(INPUT / "similar4")
+SIMILAR5 = str(INPUT / "similar5")
+SIMILAR6 = str(INPUT / "similar6")
 MULTILINE = str(INPUT / "multiline-import")
 HIDE_CODE_WITH_IMPORTS = str(INPUT / "hide_code_with_imports.py")
 
@@ -153,6 +155,45 @@ TOTAL lines=16 duplicates=0 percent=0.00
     )
 
 
+def test_ignore_signatures_fail():
+    output = StringIO()
+    with redirect_stdout(output), pytest.raises(SystemExit) as ex:
+        similar.Run([SIMILAR5, SIMILAR6])
+    assert ex.value.code == 0
+    assert (
+        output.getvalue().strip()
+        == (
+            """
+7 similar lines in 2 files
+==%s:1
+==%s:8
+       arg1: int = 3,
+       arg2: Class1 = val1,
+       arg3: Class2 = func3(val2),
+       arg4: int = 4,
+       arg5: int = 5
+   ) -> Ret1:
+       pass
+TOTAL lines=23 duplicates=7 percent=30.43
+"""
+            % (SIMILAR5, SIMILAR6)
+        ).strip()
+    )
+
+
+def test_ignore_signatures_pass():
+    output = StringIO()
+    with redirect_stdout(output), pytest.raises(SystemExit) as ex:
+        similar.Run(["--ignore-signatures", SIMILAR5, SIMILAR6])
+    assert ex.value.code == 0
+    assert (
+        output.getvalue().strip()
+        == """
+TOTAL lines=23 duplicates=0 percent=0.00
+""".strip()
+    )
+
+
 def test_no_hide_code_with_imports():
     output = StringIO()
     with redirect_stdout(output), pytest.raises(SystemExit) as ex:
diff --git a/tests/input/similar5 b/tests/input/similar5
new file mode 100644
index 000000000..d87610b1e
--- /dev/null
+++ b/tests/input/similar5
@@ -0,0 +1,8 @@
+def func1(
+    arg1: int = 3,
+    arg2: Class1 = val1,
+    arg3: Class2 = func3(val2),
+    arg4: int = 4,
+    arg5: int = 5
+) -> Ret1:
+    pass
diff --git a/tests/input/similar6 b/tests/input/similar6
new file mode 100644
index 000000000..0377e9284
--- /dev/null
+++ b/tests/input/similar6
@@ -0,0 +1,15 @@
+
+@deco1(dval1)
+@deco2(dval2)
+@deco3(
+    dval3,
+    dval4
+)
+async def func2(
+    arg1: int = 3,
+    arg2: Class1 = val1,
+    arg3: Class2 = func3(val2),
+    arg4: int = 4,
+    arg5: int = 5
+) -> Ret1:
+    pass