fix: Replace skip_last_empty with IncludeDelimiter (#1046)

5 files changed, 12 insertions, 35 deletions

diff --git a/src/Depfile.cpp b/src/Depfile.cpp
index 5a13cc22..3fcc0bcd 100644
--- a/src/Depfile.cpp
+++ b/src/Depfile.cpp
@@ -72,8 +72,11 @@ rewrite_paths(const Context& ctx, const std::string& file_content)
   adjusted_file_content.reserve(file_content.size());
 
   bool content_rewritten = false;
-  for (const auto line : util::Tokenizer(
-         file_content, "\n", util::Tokenizer::Mode::skip_last_empty)) {
+  using util::Tokenizer;
+  for (const auto line : Tokenizer(file_content,
+                                   "\n",
+                                   Tokenizer::Mode::include_empty,
+                                   Tokenizer::IncludeDelimiter::yes)) {
     const auto tokens = Util::split_into_views(line, " \t");
     for (size_t i = 0; i < tokens.size(); ++i) {
       DEBUG_ASSERT(!line.empty()); // line.empty() -> no tokens
@@ -96,7 +99,6 @@ rewrite_paths(const Context& ctx, const std::string& file_content)
         adjusted_file_content.append(token.begin(), token.end());
       }
     }
-    adjusted_file_content.push_back('\n');
   }
 
   if (content_rewritten) {
diff --git a/src/Util.cpp b/src/Util.cpp
index e68ef79f..870a2ae1 100644
--- a/src/Util.cpp
+++ b/src/Util.cpp
@@ -178,8 +178,11 @@ rewrite_stderr_to_absolute_paths(string_view text)
   static const std::string in_file_included_from = "In file included from ";
 
   std::string result;
-  for (auto line :
-       util::Tokenizer(text, "\n", util::Tokenizer::Mode::skip_last_empty)) {
+  using util::Tokenizer;
+  for (auto line : Tokenizer(text,
+                             "\n",
+                             Tokenizer::Mode::include_empty,
+                             Tokenizer::IncludeDelimiter::yes)) {
     // Rewrite <path> to <absolute path> in the following two cases, where X may
     // be optional ANSI CSI sequences:
     //
@@ -208,7 +211,6 @@ rewrite_stderr_to_absolute_paths(string_view text)
         result.append(line.data(), line.length());
       }
     }
-    result += '\n';
   }
   return result;
 }
diff --git a/src/util/Tokenizer.cpp b/src/util/Tokenizer.cpp
index b1d3e7e8..6300cd5c 100644
--- a/src/util/Tokenizer.cpp
+++ b/src/util/Tokenizer.cpp
@@ -44,10 +44,6 @@ Tokenizer::Iterator::advance(bool initial)
       m_right = delim_pos == npos ? string.length() : delim_pos;
     }
   } while (mode == Mode::skip_empty && m_left == m_right);
-
-  if (mode == Mode::skip_last_empty && m_left == string.length()) {
-    m_left = npos;
-  }
 }
 
 nonstd::sv_lite::string_view
diff --git a/src/util/Tokenizer.hpp b/src/util/Tokenizer.hpp
index e57d4c2c..d139832e 100644
--- a/src/util/Tokenizer.hpp
+++ b/src/util/Tokenizer.hpp
@@ -32,9 +32,8 @@ class Tokenizer
 {
 public:
   enum class Mode {
-    include_empty,   // Include empty tokens.
-    skip_empty,      // Skip empty tokens.
-    skip_last_empty, // Include empty tokens except the last one.
+    include_empty, // Include empty tokens.
+    skip_empty,    // Skip empty tokens.
   };
 
   enum class IncludeDelimiter { no, yes };
diff --git a/unittest/test_util_Tokenizer.cpp b/unittest/test_util_Tokenizer.cpp
index 2fa516f6..c2fa94e2 100644
--- a/unittest/test_util_Tokenizer.cpp
+++ b/unittest/test_util_Tokenizer.cpp
@@ -76,17 +76,6 @@ TEST_CASE("util::Tokenizer")
           {"0", "1", "2", "3", "4", "5", "6", "7", "8", "9"});
   }
 
-  SUBCASE("skip last empty token")
-  {
-    SplitTest split(Mode::skip_last_empty);
-    split("", "/", {});
-    split("/", "/", {""});
-    split("a/", "/", {"a"});
-    split("/b", "/", {"", "b"});
-    split("a/b", "/", {"a", "b"});
-    split("/a:", "/:", {"", "a"});
-  }
-
   SUBCASE("include empty and delimiter")
   {
     SplitTest split(Mode::include_empty, IncludeDelimiter::yes);
@@ -113,15 +102,4 @@ TEST_CASE("util::Tokenizer")
           "/:.+_abcdef",
           {"0.", "1.", "2.", "3.", "4.", "5.", "6.", "7.", "8.", "9."});
   }
-
-  SUBCASE("skip last empty and include delimiter")
-  {
-    SplitTest split(Mode::skip_last_empty, IncludeDelimiter::yes);
-    split("", "/", {});
-    split("/", "/", {"/"});
-    split("a/", "/", {"a/"});
-    split("/b", "/", {"/", "b"});
-    split("a/b", "/", {"a/", "b"});
-    split("/a:", "/:", {"/", "a:"});
-  }
 }