C++: Small fixes to the token buffer.

- Fixed blocking macro check
- Enabled buffer "compression": when tokens are generated and no new
macro is being blocked, then prepend the tokens to the previous buffer.
This happens a lot when undo-ing look-ahead.
- Added documentation

Change-Id: I6fa816d94ce4696e473bdbc4f3bf477d77e4dd51
Reviewed-by: Nikolai Kosjar <nikolai.kosjar@digia.com>

1 files changed, 25 insertions, 8 deletions

diff --git a/src/libs/cplusplus/pp-engine.cpp b/src/libs/cplusplus/pp-engine.cpp
index 4866a8f100..9938735fba 100644
--- a/src/libs/cplusplus/pp-engine.cpp
+++ b/src/libs/cplusplus/pp-engine.cpp
@@ -106,6 +106,14 @@ typedef ScopedSwap<unsigned> ScopedUnsignedSwap;
 namespace CPlusPlus {
 
 namespace Internal {
+/// Buffers tokens for the Preprocessor::lex() to read next. Do not use  this
+/// class directly, but use Preprocessor::State::pushTokenBuffer .
+///
+/// New tokens are added when undoing look-ahead, or after expanding a macro.
+/// When macro expansion happened, the macro is passed in, and blocked until
+/// all tokens generated by it (and by subsequent expansion of those generated
+/// tokens) are read from the buffer. See Preprocessor::lex() for details on
+/// exactly when the buffer (and subsequently a blocking macro) is removed.
 struct TokenBuffer
 {
     std::deque<PPToken> tokens;
@@ -121,8 +129,9 @@ struct TokenBuffer
             return false;
 
         for (const TokenBuffer *it = this; it; it = it->next)
-            if (it->macro == macro && it->macro->name() == macro->name())
-                return true;
+            if (it->macro)
+                if (it->macro == macro || (it->macro->name() == macro->name()))
+                    return true;
         return false;
     }
 };
@@ -555,22 +564,21 @@ Preprocessor::State::State()
     m_expansionResult.reserve(256);
 }
 
-//#define COMPRESS_TOKEN_BUFFER
+#define COMPRESS_TOKEN_BUFFER
 void Preprocessor::State::pushTokenBuffer(const PPToken *start, const PPToken *end, const Macro *macro)
 {
     if (m_tokenBufferDepth <= MAX_TOKEN_BUFFER_DEPTH) {
 #ifdef COMPRESS_TOKEN_BUFFER
-        // This does not work correctly for boost's preprocessor library, or that library exposes a bug in the code.
         if (macro || !m_tokenBuffer) {
+            // If there is a new blocking macro (or no token buffer yet), create
+            // one.
             m_tokenBuffer = new TokenBuffer(start, end, macro, m_tokenBuffer);
             ++m_tokenBufferDepth;
         } else {
+            // No new blocking macro is passed in, so tokens can be prepended to
+            // the existing buffer.
             m_tokenBuffer->tokens.insert(m_tokenBuffer->tokens.begin(), start, end);
         }
-        unsigned tkCount = 0;
-        for (TokenBuffer *it = m_tokenBuffer; it; it = m_tokenBuffer->next)
-            tkCount += it->tokens.size();
-        qDebug()<<"New depth:" << m_tokenBufferDepth << "with total token count:" << tkCount;
 #else
         m_tokenBuffer = new TokenBuffer(start, end, macro, m_tokenBuffer);
         ++m_tokenBufferDepth;
@@ -693,13 +701,22 @@ void Preprocessor::lex(PPToken *tk)
 {
 _Lagain:
     if (m_state.m_tokenBuffer) {
+        // There is a token buffer, so read from there.
         if (m_state.m_tokenBuffer->tokens.empty()) {
+            // The token buffer is empty, so pop it, and start over.
             m_state.popTokenBuffer();
             goto _Lagain;
         }
         *tk = m_state.m_tokenBuffer->tokens.front();
         m_state.m_tokenBuffer->tokens.pop_front();
+        // The token buffer might now be empty. We leave it in, because the
+        // token we just read might expand into new tokens, or might be a call
+        // to the macro that generated this token. In either case, the macro
+        // that generated the token still needs to be blocked (!), which is
+        // recorded in the token buffer. Removing the blocked macro and the
+        // empty token buffer happens the next time that this method is called.
     } else {
+        // No token buffer, so have the lexer scan the next token.
         tk->setSource(m_state.m_source);
         m_state.m_lexer->scan(tk);
     }