diff options
-rw-r--r-- | pygments/util.py | 16 | ||||
-rw-r--r-- | tests/test_util.py | 16 |
2 files changed, 32 insertions, 0 deletions
diff --git a/pygments/util.py b/pygments/util.py index 8376a67f..1f54c291 100644 --- a/pygments/util.py +++ b/pygments/util.py @@ -263,6 +263,22 @@ def format_lines(var_name, seq, raw=False, indent_level=0): return '\n'.join(lines) +def duplicates_removed(it, already_seen=()): + """ + Returns a list with duplicates removed from the iterable `it`. + + Order is preserved. + """ + lst = [] + seen = set() + for i in it: + if i in seen or i in already_seen: + continue + lst.append(i) + seen.add(i) + return lst + + class Future(object): """Generic class to defer some work. diff --git a/tests/test_util.py b/tests/test_util.py index 7bf03409..7fbbba2d 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -141,3 +141,19 @@ class UtilTest(unittest.TestCase): exec(output, d) self.assertTrue(isinstance(d['var'], tuple)) self.assertEqual(('cat', 'dog'), d['var']) + + def test_duplicates_removed_seq_types(self): + # tuple + x = util.duplicates_removed(('a', 'a', 'b')) + self.assertSequenceEqual(('a', 'b'), x) + # list + x = util.duplicates_removed(['a', 'a', 'b']) + self.assertSequenceEqual(('a', 'b'), x) + # iterator + x = util.duplicates_removed(iter(('a', 'a', 'b'))) + self.assertSequenceEqual(('a', 'b'), x) + + def test_duplicates_removed_nonconsecutive(self): + # keeps first + x = util.duplicates_removed(('a', 'b', 'a')) + self.assertSequenceEqual(('a', 'b'), x) |