summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOrgad Shaneh <orgad.shaneh@audiocodes.com>2022-04-12 22:09:57 +0300
committerGitHub <noreply@github.com>2022-04-12 21:09:57 +0200
commit2ba8445b358a40753809495aae07ac1b3169d746 (patch)
treec4b4bc0caf16e5cb8603948bc063422a85bcfe51
parenta9b30fd3b9cf67b1c937579478f30c6e6d4a4325 (diff)
downloadccache-2ba8445b358a40753809495aae07ac1b3169d746.tar.gz
fix: Replace skip_last_empty with IncludeDelimiter (#1046)
-rw-r--r--src/Depfile.cpp8
-rw-r--r--src/Util.cpp8
-rw-r--r--src/util/Tokenizer.cpp4
-rw-r--r--src/util/Tokenizer.hpp5
-rw-r--r--unittest/test_util_Tokenizer.cpp22
5 files changed, 12 insertions, 35 deletions
diff --git a/src/Depfile.cpp b/src/Depfile.cpp
index 5a13cc22..3fcc0bcd 100644
--- a/src/Depfile.cpp
+++ b/src/Depfile.cpp
@@ -72,8 +72,11 @@ rewrite_paths(const Context& ctx, const std::string& file_content)
adjusted_file_content.reserve(file_content.size());
bool content_rewritten = false;
- for (const auto line : util::Tokenizer(
- file_content, "\n", util::Tokenizer::Mode::skip_last_empty)) {
+ using util::Tokenizer;
+ for (const auto line : Tokenizer(file_content,
+ "\n",
+ Tokenizer::Mode::include_empty,
+ Tokenizer::IncludeDelimiter::yes)) {
const auto tokens = Util::split_into_views(line, " \t");
for (size_t i = 0; i < tokens.size(); ++i) {
DEBUG_ASSERT(!line.empty()); // line.empty() -> no tokens
@@ -96,7 +99,6 @@ rewrite_paths(const Context& ctx, const std::string& file_content)
adjusted_file_content.append(token.begin(), token.end());
}
}
- adjusted_file_content.push_back('\n');
}
if (content_rewritten) {
diff --git a/src/Util.cpp b/src/Util.cpp
index e68ef79f..870a2ae1 100644
--- a/src/Util.cpp
+++ b/src/Util.cpp
@@ -178,8 +178,11 @@ rewrite_stderr_to_absolute_paths(string_view text)
static const std::string in_file_included_from = "In file included from ";
std::string result;
- for (auto line :
- util::Tokenizer(text, "\n", util::Tokenizer::Mode::skip_last_empty)) {
+ using util::Tokenizer;
+ for (auto line : Tokenizer(text,
+ "\n",
+ Tokenizer::Mode::include_empty,
+ Tokenizer::IncludeDelimiter::yes)) {
// Rewrite <path> to <absolute path> in the following two cases, where X may
// be optional ANSI CSI sequences:
//
@@ -208,7 +211,6 @@ rewrite_stderr_to_absolute_paths(string_view text)
result.append(line.data(), line.length());
}
}
- result += '\n';
}
return result;
}
diff --git a/src/util/Tokenizer.cpp b/src/util/Tokenizer.cpp
index b1d3e7e8..6300cd5c 100644
--- a/src/util/Tokenizer.cpp
+++ b/src/util/Tokenizer.cpp
@@ -44,10 +44,6 @@ Tokenizer::Iterator::advance(bool initial)
m_right = delim_pos == npos ? string.length() : delim_pos;
}
} while (mode == Mode::skip_empty && m_left == m_right);
-
- if (mode == Mode::skip_last_empty && m_left == string.length()) {
- m_left = npos;
- }
}
nonstd::sv_lite::string_view
diff --git a/src/util/Tokenizer.hpp b/src/util/Tokenizer.hpp
index e57d4c2c..d139832e 100644
--- a/src/util/Tokenizer.hpp
+++ b/src/util/Tokenizer.hpp
@@ -32,9 +32,8 @@ class Tokenizer
{
public:
enum class Mode {
- include_empty, // Include empty tokens.
- skip_empty, // Skip empty tokens.
- skip_last_empty, // Include empty tokens except the last one.
+ include_empty, // Include empty tokens.
+ skip_empty, // Skip empty tokens.
};
enum class IncludeDelimiter { no, yes };
diff --git a/unittest/test_util_Tokenizer.cpp b/unittest/test_util_Tokenizer.cpp
index 2fa516f6..c2fa94e2 100644
--- a/unittest/test_util_Tokenizer.cpp
+++ b/unittest/test_util_Tokenizer.cpp
@@ -76,17 +76,6 @@ TEST_CASE("util::Tokenizer")
{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9"});
}
- SUBCASE("skip last empty token")
- {
- SplitTest split(Mode::skip_last_empty);
- split("", "/", {});
- split("/", "/", {""});
- split("a/", "/", {"a"});
- split("/b", "/", {"", "b"});
- split("a/b", "/", {"a", "b"});
- split("/a:", "/:", {"", "a"});
- }
-
SUBCASE("include empty and delimiter")
{
SplitTest split(Mode::include_empty, IncludeDelimiter::yes);
@@ -113,15 +102,4 @@ TEST_CASE("util::Tokenizer")
"/:.+_abcdef",
{"0.", "1.", "2.", "3.", "4.", "5.", "6.", "7.", "8.", "9."});
}
-
- SUBCASE("skip last empty and include delimiter")
- {
- SplitTest split(Mode::skip_last_empty, IncludeDelimiter::yes);
- split("", "/", {});
- split("/", "/", {"/"});
- split("a/", "/", {"a/"});
- split("/b", "/", {"/", "b"});
- split("a/b", "/", {"a/", "b"});
- split("/a:", "/:", {"/", "a:"});
- }
}