Add 'duplicates_removed' function (for *_builtins.py generation).

2 files changed, 32 insertions, 0 deletions

diff --git a/pygments/util.py b/pygments/util.py
index 8376a67f..1f54c291 100644
--- a/pygments/util.py
+++ b/pygments/util.py
@@ -263,6 +263,22 @@ def format_lines(var_name, seq, raw=False, indent_level=0):
     return '\n'.join(lines)
 
 
+def duplicates_removed(it, already_seen=()):
+    """
+    Returns a list with duplicates removed from the iterable `it`.
+
+    Order is preserved.
+    """
+    lst = []
+    seen = set()
+    for i in it:
+        if i in seen or i in already_seen:
+            continue
+        lst.append(i)
+        seen.add(i)
+    return lst
+
+
 class Future(object):
     """Generic class to defer some work.
 
diff --git a/tests/test_util.py b/tests/test_util.py
index 7bf03409..7fbbba2d 100644
--- a/tests/test_util.py
+++ b/tests/test_util.py
@@ -141,3 +141,19 @@ class UtilTest(unittest.TestCase):
         exec(output, d)
         self.assertTrue(isinstance(d['var'], tuple))
         self.assertEqual(('cat', 'dog'), d['var'])
+
+    def test_duplicates_removed_seq_types(self):
+        # tuple
+        x = util.duplicates_removed(('a', 'a', 'b'))
+        self.assertSequenceEqual(('a', 'b'), x)
+        # list
+        x = util.duplicates_removed(['a', 'a', 'b'])
+        self.assertSequenceEqual(('a', 'b'), x)
+        # iterator
+        x = util.duplicates_removed(iter(('a', 'a', 'b')))
+        self.assertSequenceEqual(('a', 'b'), x)
+
+    def test_duplicates_removed_nonconsecutive(self):
+        # keeps first
+        x = util.duplicates_removed(('a', 'b', 'a'))
+        self.assertSequenceEqual(('a', 'b'), x)