[C++] Rewrite of the preprocessor.

This rewrite fixes a couple of issues with the pre-processor. It now
supports:
- macros in macro bodies
- stringification of parameters [cpp.stringize]
- the concatenation operator [cpp.concat]
- #include MACRO_HERE
- defined() inside macro bodies used in pp-conditions.

Change-Id: Ifdb78041fb6afadf44f939a4bd66ce2832b8601f
Reviewed-by: Roberto Raggi <roberto.raggi@nokia.com>

24 files changed, 1146 insertions, 1538 deletions

diff --git a/src/libs/cplusplus/CppDocument.cpp b/src/libs/cplusplus/CppDocument.cpp
index ec617ff929..0d39e3b1db 100644
--- a/src/libs/cplusplus/CppDocument.cpp
+++ b/src/libs/cplusplus/CppDocument.cpp
@@ -198,6 +198,8 @@ protected:
 };
 
 
+#define DO_NOT_DUMP_ALL_PARSER_ERRORS
+
 class DocumentDiagnosticClient : public DiagnosticClient
 {
     enum { MAX_MESSAGE_COUNT = 10 };
@@ -217,8 +219,10 @@ public:
         if (level == Error) {
             ++errorCount;
 
+#ifdef DO_NOT_DUMP_ALL_PARSER_ERRORS
             if (errorCount >= MAX_MESSAGE_COUNT)
                 return; // ignore the error
+#endif // DO_NOT_DUMP_ALL_PARSER_ERRORS
         }
 
         const QString fileName = QString::fromUtf8(fileId->chars(), fileId->size());
@@ -229,6 +233,16 @@ public:
         QString message;
         message.vsprintf(format, ap);
 
+#ifndef DO_NOT_DUMP_ALL_PARSER_ERRORS
+        {
+            const char *levelStr = "Unknown level";
+            if (level == Document::DiagnosticMessage::Warning) levelStr = "Warning";
+            if (level == Document::DiagnosticMessage::Error) levelStr = "Error";
+            if (level == Document::DiagnosticMessage::Fatal) levelStr = "Fatal";
+            qDebug("%s:%u:%u: %s: %s", fileId->chars(), line, column, levelStr, message.toUtf8().constData());
+        }
+#endif // DO_NOT_DUMP_ALL_PARSER_ERRORS
+
         Document::DiagnosticMessage m(convertLevel(level), doc->fileName(),
                                       line, column, message);
         messages->append(m);
@@ -341,10 +355,9 @@ void Document::appendMacro(const Macro &macro)
 
 void Document::addMacroUse(const Macro &macro, unsigned offset, unsigned length,
                            unsigned beginLine,
-                           const QVector<MacroArgumentReference> &actuals, bool inCondition)
+                           const QVector<MacroArgumentReference> &actuals)
 {
     MacroUse use(macro, offset, offset + length, beginLine);
-    use.setInCondition(inCondition);
 
     foreach (const MacroArgumentReference &actual, actuals) {
         const Block arg(actual.position(), actual.position() + actual.length());
diff --git a/src/libs/cplusplus/CppDocument.h b/src/libs/cplusplus/CppDocument.h
index 258d86138b..4147a95c34 100644
--- a/src/libs/cplusplus/CppDocument.h
+++ b/src/libs/cplusplus/CppDocument.h
@@ -77,8 +77,7 @@ public:
 
     void appendMacro(const Macro &macro);
     void addMacroUse(const Macro &macro, unsigned offset, unsigned length,
-                     unsigned beginLine, const QVector<MacroArgumentReference> &range,
-                     bool inCondition);
+                     unsigned beginLine, const QVector<MacroArgumentReference> &range);
     void addUndefinedMacroUse(const QByteArray &name, unsigned offset);
 
     Control *control() const;
@@ -247,7 +246,6 @@ public:
     class MacroUse: public Block {
         Macro _macro;
         QVector<Block> _arguments;
-        bool _inCondition;
         unsigned _beginLine;
 
     public:
@@ -255,7 +253,6 @@ public:
                         unsigned begin, unsigned end, unsigned beginLine)
             : Block(begin, end),
               _macro(macro),
-              _inCondition(false),
               _beginLine(beginLine)
         { }
 
@@ -268,9 +265,6 @@ public:
         QVector<Block> arguments() const
         { return _arguments; }
 
-        bool isInCondition() const
-        { return _inCondition; }
-
         unsigned beginLine() const
         { return _beginLine; }
 
@@ -281,9 +275,6 @@ public:
         void addArgument(const Block &block)
         { _arguments.append(block); }
 
-        void setInCondition(bool set)
-        { _inCondition = set; }
-
         friend class Document;
     };
 
diff --git a/src/libs/cplusplus/FastPreprocessor.h b/src/libs/cplusplus/FastPreprocessor.h
index b032b6af27..85a8e8c630 100644
--- a/src/libs/cplusplus/FastPreprocessor.h
+++ b/src/libs/cplusplus/FastPreprocessor.h
@@ -69,7 +69,6 @@ public:
     virtual void startExpandingMacro(unsigned,
                                      const Macro &,
                                      const QByteArray &,
-                                     bool,
                                      const QVector<MacroArgumentReference> &) {}
 
     virtual void stopExpandingMacro(unsigned, const Macro &) {}
diff --git a/src/libs/cplusplus/Macro.cpp b/src/libs/cplusplus/Macro.cpp
index 7020f5f01c..c3390f06fd 100644
--- a/src/libs/cplusplus/Macro.cpp
+++ b/src/libs/cplusplus/Macro.cpp
@@ -92,19 +92,11 @@ QString Macro::decoratedName() const
 QString Macro::toString() const
 {
     QString text = decoratedName();
-    text.append(QString::fromUtf8(_definition.constData(), _definition.size()));
+    text.append(QString::fromUtf8(_definitionText.constData(), _definitionText.size()));
     return text;
 }
 
 QString Macro::toStringWithLineBreaks() const
 {
-    if (_lineBreaks.isEmpty())
-        return toString();
-
-    QString text = decoratedName();
-    QString definitionWithBreaks = QString::fromUtf8(_definition.constData(), _definition.size());
-    foreach (unsigned pos, _lineBreaks)
-        definitionWithBreaks[pos] = '\n';
-    text.append(definitionWithBreaks);
-    return text;
+    return toString();
 }
diff --git a/src/libs/cplusplus/Macro.h b/src/libs/cplusplus/Macro.h
index 004969333f..56de90fcf4 100644
--- a/src/libs/cplusplus/Macro.h
+++ b/src/libs/cplusplus/Macro.h
@@ -52,6 +52,8 @@
 #ifndef CPLUSPLUS_PP_MACRO_H
 #define CPLUSPLUS_PP_MACRO_H
 
+#include "PPToken.h"
+
 #include <CPlusPlusForwardDeclarations.h>
 
 #include <QByteArray>
@@ -60,8 +62,12 @@
 
 namespace CPlusPlus {
 
+class Environment;
+
 class CPLUSPLUS_EXPORT Macro
 {
+    typedef Internal::PPToken PPToken;
+
 public:
     Macro();
 
@@ -71,13 +77,16 @@ public:
     void setName(const QByteArray &name)
     { _name = name; }
 
-    QByteArray definition() const
-    { return _definition; }
+    const QByteArray definitionText() const
+    { return _definitionText; }
+
+    const QVector<PPToken> &definitionTokens() const
+    { return _definitionTokens; }
 
-    void setDefinition(const QByteArray &definition)
-    { _definition = definition; }
+    void setDefinition(const QByteArray &definitionText, const QVector<PPToken> &definitionTokens)
+    { _definitionText = definitionText; _definitionTokens = definitionTokens; }
 
-    QVector<QByteArray> formals() const
+    const QVector<QByteArray> &formals() const
     { return _formals; }
 
     void addFormal(const QByteArray &formal)
@@ -125,16 +134,11 @@ public:
     void setVariadic(bool isVariadic)
     { f._variadic = isVariadic; }
 
-    void setLineBreaks(const QList<unsigned> &breaks)
-    { _lineBreaks = breaks; }
-
-    const QList<unsigned> &lineBreaks() const
-    { return _lineBreaks; }
-
     QString toString() const;
     QString toStringWithLineBreaks() const;
 
-// ### private
+private:
+    friend class Environment;
     Macro *_next;
     unsigned _hashcode;
 
@@ -149,10 +153,10 @@ private:
     };
 
     QByteArray _name;
-    QByteArray _definition;
+    QByteArray _definitionText;
+    QVector<PPToken> _definitionTokens;
     QVector<QByteArray> _formals;
     QString _fileName;
-    QList<unsigned> _lineBreaks;
     unsigned _line;
     unsigned _offset;
     unsigned _length;
diff --git a/src/libs/cplusplus/PPToken.cpp b/src/libs/cplusplus/PPToken.cpp
new file mode 100644
index 0000000000..80577ab6cf
--- /dev/null
+++ b/src/libs/cplusplus/PPToken.cpp
@@ -0,0 +1,45 @@
+#include "PPToken.h"
+
+#include <cstring>
+
+using namespace CPlusPlus::Internal;
+
+ByteArrayRef::ByteArrayRef()
+    : m_ref(0)
+    , m_offset(0)
+    , m_length(0)
+{}
+
+bool ByteArrayRef::startsWith(const char *s) const
+{
+    int l = std::strlen(s);
+    if (l > m_length)
+        return false;
+    return !qstrncmp(start(), s, l);
+}
+
+int ByteArrayRef::count(char ch) const
+{
+    if (!m_ref)
+        return 0;
+
+    int num = 0;
+    const char *b = start();
+    const char *i = b + m_length;
+    while (i != b)
+        if (*--i == ch)
+            ++num;
+    return num;
+}
+
+PPToken::PPToken()
+{}
+
+void PPToken::squeeze()
+{
+    if (isValid()) {
+        m_src = m_src.mid(offset, length());
+        m_src.squeeze();
+        offset = 0;
+    }
+}
diff --git a/src/libs/cplusplus/PPToken.h b/src/libs/cplusplus/PPToken.h
new file mode 100644
index 0000000000..82076872ef
--- /dev/null
+++ b/src/libs/cplusplus/PPToken.h
@@ -0,0 +1,106 @@
+#ifndef CPLUSPLUS_INTERNAL_PPTOKEN_H
+#define CPLUSPLUS_INTERNAL_PPTOKEN_H
+
+#include <CPlusPlus.h>
+#include <Token.h>
+
+#include <QByteArray>
+
+namespace CPlusPlus {
+namespace Internal {
+
+class CPLUSPLUS_EXPORT ByteArrayRef
+{
+public:
+    ByteArrayRef();
+
+    ByteArrayRef(const QByteArray *ref)
+        : m_ref(ref)
+        , m_offset(0)
+        , m_length(ref->length())
+    {}
+
+    ByteArrayRef(const QByteArray *ref, int offset, int length)
+        : m_ref(ref)
+        , m_offset(offset)
+        , m_length(length)
+    {
+        Q_ASSERT(ref);
+        Q_ASSERT(offset >= 0);
+        Q_ASSERT(length >= 0);
+        Q_ASSERT(offset + length <= ref->size());
+    }
+
+    inline const char *start() const
+    { return m_ref ? m_ref->constData() + m_offset : 0; }
+
+    inline int length() const
+    { return m_length; }
+
+    inline int size() const
+    { return length(); }
+
+    inline char at(int pos) const
+    { return m_ref && pos >= 0 && pos < m_length ? m_ref->at(m_offset + pos) : '\0'; }
+
+    inline char operator[](int pos) const
+    { return at(pos); }
+
+    QByteArray toByteArray() const
+    { return m_ref ? QByteArray(m_ref->constData() + m_offset, m_length) : QByteArray(); }
+
+    bool operator==(const QByteArray &other) const
+    { return m_ref ? (m_length == other.length() && !qstrncmp(m_ref->constData() + m_offset, other.constData(), m_length)) : false; }
+    bool operator!=(const QByteArray &other) const
+    { return !this->operator==(other); }
+
+    bool startsWith(const char *ch) const;
+
+    int count(char c) const;
+
+private:
+    const QByteArray *m_ref;
+    int m_offset;
+    int m_length;
+};
+
+inline bool operator==(const QByteArray &other, const ByteArrayRef &ref)
+{ return ref == other; }
+
+inline bool operator!=(const QByteArray &other, const ByteArrayRef &ref)
+{ return ref != other; }
+
+class CPLUSPLUS_EXPORT PPToken: public Token
+{
+public:
+    PPToken();
+
+    PPToken(const QByteArray &src)
+        : m_src(src)
+    {}
+
+    void setSource(const QByteArray &src)
+    { m_src = src; }
+
+    const QByteArray &source() const
+    { return m_src; }
+
+    const char *start() const
+    { return m_src.constData() + offset; }
+
+    ByteArrayRef asByteArrayRef() const
+    { return ByteArrayRef(&m_src, offset, length()); }
+
+    bool isValid() const
+    { return !m_src.isEmpty(); }
+
+    void squeeze();
+
+private:
+    QByteArray m_src;
+};
+
+} // namespace Internal
+} // namespace CPlusPlus
+
+#endif // CPLUSPLUS_INTERNAL_PPTOKEN_H
diff --git a/src/libs/cplusplus/PreprocessorClient.h b/src/libs/cplusplus/PreprocessorClient.h
index 0056a6d869..c3c1dd9462 100644
--- a/src/libs/cplusplus/PreprocessorClient.h
+++ b/src/libs/cplusplus/PreprocessorClient.h
@@ -75,7 +75,7 @@ public:
 
 public:
   Client();
-  virtual ~Client();
+  virtual ~Client() = 0;
 
   virtual void macroAdded(const Macro &macro) = 0;
 
@@ -85,13 +85,13 @@ public:
   virtual void startExpandingMacro(unsigned offset,
                                    const Macro &macro,
                                    const QByteArray &originalText,
-                                   bool inCondition = false,
                                    const QVector<MacroArgumentReference> &actuals
                                             = QVector<MacroArgumentReference>()) = 0;
 
   virtual void stopExpandingMacro(unsigned offset,
                                   const Macro &macro) = 0;
 
+  /// Start skipping from the given offset.
   virtual void startSkippingBlocks(unsigned offset) = 0;
   virtual void stopSkippingBlocks(unsigned offset) = 0;
 
diff --git a/src/libs/cplusplus/PreprocessorEnvironment.cpp b/src/libs/cplusplus/PreprocessorEnvironment.cpp
index 42c3bc04f4..174afe66a7 100644
--- a/src/libs/cplusplus/PreprocessorEnvironment.cpp
+++ b/src/libs/cplusplus/PreprocessorEnvironment.cpp
@@ -150,7 +150,7 @@ void Environment::reset()
     _hash_count = 401;
 }
 
-bool Environment::isBuiltinMacro(const QByteArray &s)
+bool Environment::isBuiltinMacro(const Internal::ByteArrayRef &s)
 {
     if (s.length() != 8)
         return false;
@@ -236,6 +236,22 @@ Macro *Environment::resolve(const QByteArray &name) const
     return it;
 }
 
+Macro *Environment::resolve(const Internal::ByteArrayRef &name) const
+{
+    if (! _macros)
+        return 0;
+
+    Macro *it = _hash[hashCode(name) % _hash_count];
+    for (; it; it = it->_next) {
+        if (it->name() != name)
+            continue;
+        else if (it->isHidden())
+            return 0;
+        else break;
+    }
+    return it;
+}
+
 unsigned Environment::hashCode(const QByteArray &s)
 {
     unsigned hash_value = 0;
@@ -246,6 +262,16 @@ unsigned Environment::hashCode(const QByteArray &s)
     return hash_value;
 }
 
+unsigned Environment::hashCode(const Internal::ByteArrayRef &s)
+{
+    unsigned hash_value = 0;
+
+    for (int i = 0; i < s.length(); ++i)
+        hash_value = (hash_value << 5) - hash_value + s.at(i);
+
+    return hash_value;
+}
+
 void Environment::rehash()
 {
     if (_hash) {
diff --git a/src/libs/cplusplus/PreprocessorEnvironment.h b/src/libs/cplusplus/PreprocessorEnvironment.h
index a904f9c8fc..b03261fa89 100644
--- a/src/libs/cplusplus/PreprocessorEnvironment.h
+++ b/src/libs/cplusplus/PreprocessorEnvironment.h
@@ -53,6 +53,7 @@
 #define CPLUSPLUS_PP_ENVIRONMENT_H
 
 #include "CPlusPlusForwardDeclarations.h"
+#include "PPToken.h"
 
 #include <QList>
 #include <QByteArray>
@@ -78,6 +79,7 @@ public:
     Macro *remove(const QByteArray &name);
 
     Macro *resolve(const QByteArray &name) const;
+    Macro *resolve(const Internal::ByteArrayRef &name) const;
 
     iterator firstMacro() const;
     iterator lastMacro() const;
@@ -85,10 +87,11 @@ public:
     void reset();
     void addMacros(const QList<Macro> &macros);
 
-    static bool isBuiltinMacro(const QByteArray &name);
+    static bool isBuiltinMacro(const Internal::ByteArrayRef &name);
 
 private:
     static unsigned hashCode(const QByteArray &s);
+    static unsigned hashCode(const Internal::ByteArrayRef &s);
     void rehash();
 
 public:
diff --git a/src/libs/cplusplus/cplusplus-lib.pri b/src/libs/cplusplus/cplusplus-lib.pri
index a48a23d11b..677dff2556 100644
--- a/src/libs/cplusplus/cplusplus-lib.pri
+++ b/src/libs/cplusplus/cplusplus-lib.pri
@@ -51,9 +51,9 @@ HEADERS += \
     $$PWD/pp.h \
     $$PWD/pp-cctype.h \
     $$PWD/pp-engine.h \
-    $$PWD/pp-macro-expander.h \
     $$PWD/pp-scanner.h \
-    $$PWD/findcdbbreakpoint.h
+    $$PWD/findcdbbreakpoint.h \
+    $$PWD/PPToken.h
 
 SOURCES += \
     $$PWD/SimpleLexer.cpp \
@@ -78,8 +78,8 @@ SOURCES += \
     $$PWD/FastPreprocessor.cpp \
     $$PWD/Macro.cpp \
     $$PWD/pp-engine.cpp \
-    $$PWD/pp-macro-expander.cpp \
     $$PWD/pp-scanner.cpp \
-    $$PWD/findcdbbreakpoint.cpp
+    $$PWD/findcdbbreakpoint.cpp \
+    $$PWD/PPToken.cpp
 
 RESOURCES += $$PWD/cplusplus.qrc
diff --git a/src/libs/cplusplus/cplusplus.pro b/src/libs/cplusplus/cplusplus.pro
index 82ee8bfded..1df6385e91 100644
--- a/src/libs/cplusplus/cplusplus.pro
+++ b/src/libs/cplusplus/cplusplus.pro
@@ -1,9 +1,9 @@
 TEMPLATE = lib
 TARGET = CPlusPlus
 
-DEFINES += NDEBUG
-unix:QMAKE_CXXFLAGS_DEBUG += -O2
-
+#DEFINES += NDEBUG
+#unix:QMAKE_CXXFLAGS_DEBUG += -O2
+QMAKE_CXXFLAGS_DEBUG += -ggdb
 include(../../qtcreatorlibrary.pri)
 include(cplusplus-lib.pri)
 include(../languageutils/languageutils.pri)
diff --git a/src/libs/cplusplus/cplusplus.qbs b/src/libs/cplusplus/cplusplus.qbs
index 7b1aa6e254..d79a5e0148 100644
--- a/src/libs/cplusplus/cplusplus.qbs
+++ b/src/libs/cplusplus/cplusplus.qbs
@@ -149,8 +149,8 @@ DynamicLibrary {
         "pp-cctype.h",
         "pp-engine.cpp",
         "pp-engine.h",
-        "pp-macro-expander.cpp",
-        "pp-macro-expander.h",
+        "PPToken.cpp",
+        "PPToken.h",
         "pp-scanner.cpp",
         "pp-scanner.h",
         "pp.h",
diff --git a/src/libs/cplusplus/pp-engine.cpp b/src/libs/cplusplus/pp-engine.cpp
index 4479ee8d96..634f5789d7 100644
--- a/src/libs/cplusplus/pp-engine.cpp
+++ b/src/libs/cplusplus/pp-engine.cpp
@@ -52,6 +52,7 @@
 #include "pp.h"
 #include "pp-cctype.h"
 
+#include <Control.h>
 #include <Lexer.h>
 #include <Token.h>
 #include <Literals.h>
@@ -60,9 +61,72 @@
 #include <QtDebug>
 #include <algorithm>
 #include <QList>
+#include <QDate>
+#include <QTime>
+
+#define NO_DEBUG
+
+#ifndef NO_DEBUG
+#  include <iostream>
+#endif // NO_DEBUG
+
+namespace {
+enum {
+    eagerExpansion = 1,
+    MAX_TOKEN_EXPANSION_COUNT = 5000
+};
+}
+
+namespace {
+template<typename _T>
+class ScopedSwap
+{
+    _T oldValue;
+    _T &ref;
+
+public:
+    ScopedSwap(_T &var, _T newValue)
+        : oldValue(newValue)
+        , ref(var)
+    {
+        std::swap(ref, oldValue);
+    }
+
+    ~ScopedSwap()
+    {
+        std::swap(ref, oldValue);
+    }
+};
+typedef ScopedSwap<bool> ScopedBoolSwap;
+typedef ScopedSwap<unsigned> ScopedUnsignedSwap;
+} // anonymous namespace
 
 namespace CPlusPlus {
 
+namespace Internal {
+struct TokenBuffer
+{
+    std::list<PPToken> tokens;
+    const Macro *macro;
+    TokenBuffer *next;
+    QVector<QByteArray> blockedMacros;
+
+    template <typename _Iterator>
+    TokenBuffer(_Iterator firstToken, _Iterator lastToken, const Macro *macro, TokenBuffer *next)
+        : tokens(firstToken, lastToken), macro(macro), next(next)
+    {}
+
+    bool isBlocked(const QByteArray &macroName) const {
+        for (const TokenBuffer *it = this; it; it = it->next)
+            if (it->blockedMacros.contains(macroName))
+                return true;
+        return false;
+    }
+
+    void blockMacro(const QByteArray &macroName)
+    { blockedMacros.append(macroName); }
+};
+
 struct Value
 {
     enum Kind {
@@ -133,15 +197,20 @@ struct Value
 #undef PP_DEFINE_BIN_OP
 };
 
+} // namespace Internal
 } // namespace CPlusPlus
 
-
 using namespace CPlusPlus;
-
+using namespace CPlusPlus::Internal;
 
 namespace {
 
-Macro *macroDefinition(QByteArray name, unsigned offset, Environment *env, Client *client)
+inline bool isValidToken(const PPToken &tk)
+{
+    return tk.isNot(T_EOF_SYMBOL) && (! tk.newline() || tk.joined());
+}
+
+Macro *macroDefinition(const QByteArray &name, unsigned offset, Environment *env, Client *client)
 {
     Macro *m = env->resolve(name);
     if (client) {
@@ -452,757 +521,664 @@ private:
 
 } // end of anonymous namespace
 
-
-Preprocessor::Preprocessor(Client *client, Environment *env)
-    : client(client),
-      env(env),
-      _expand(env),
-      _skipping(MAX_LEVEL),
-      _trueTest(MAX_LEVEL),
-      _dot(_tokens.end()),
-      _result(0),
-      _markGeneratedTokens(false),
-      _expandMacros(true),
-      _keepComments(false)
+Preprocessor::State::State()
+    : m_lexer(0)
+    , m_skipping(MAX_LEVEL)
+    , m_trueTest(MAX_LEVEL)
+    , m_ifLevel(0)
+    , m_tokenBuffer(0)
+    , m_inPreprocessorDirective(false)
+    , m_result(0)
+    , m_markGeneratedTokens(true)
+    , m_noLines(false)
+    , m_inCondition(false)
+    , m_inDefine(false)
 {
-    resetIfLevel ();
+    m_skipping[m_ifLevel] = false;
+    m_trueTest[m_ifLevel] = false;
 }
 
-void Preprocessor::pushState(const State &s)
+
+Preprocessor::Preprocessor(Client *client, Environment *env)
+    : m_client(client)
+    , m_env(env)
+    , m_expandMacros(true)
+    , m_keepComments(false)
 {
-    _savedStates.append(state());
-    _source = s.source;
-    _tokens = s.tokens;
-    _dot = s.dot;
 }
 
-Preprocessor::State Preprocessor::state() const
+void Preprocessor::pushState(const State &newState)
 {
-    State state;
-    state.source = _source;
-    state.tokens = _tokens;
-    state.dot = _dot;
-    return state;
+    m_savedStates.append(m_state);
+    m_state = newState;
 }
 
 void Preprocessor::popState()
 {
-    const State &state = _savedStates.last();
-    _source = state.source;
-    _tokens = state.tokens;
-    _dot = state.dot;
-    _savedStates.removeLast();
+    const State &s = m_savedStates.last();
+    delete m_state.m_lexer;
+    m_state = s;
+    m_savedStates.removeLast();
 }
 
 QByteArray Preprocessor::operator()(const QString &fileName, const QString &source)
 {
-    const QString previousOriginalSource = _originalSource;
-    _originalSource = source;
+    const QString previousOriginalSource = m_originalSource;
+    m_originalSource = source;
     const QByteArray bytes = source.toLatin1();
     const QByteArray preprocessedCode = operator()(fileName, bytes);
-    _originalSource = previousOriginalSource;
+    m_originalSource = previousOriginalSource;
     return preprocessedCode;
 }
 
 QByteArray Preprocessor::operator()(const QString &fileName,
-                                    const QByteArray &source)
+                                    const QByteArray &source,
+                                    bool noLines,
+                                    bool markGeneratedTokens)
 {
     QByteArray preprocessed;
-    preprocess(fileName, source, &preprocessed);
+//    qDebug()<<"running" << fileName<<"with"<<source.count('\n')<<"lines...";
+    preprocess(fileName, source, &preprocessed, noLines, markGeneratedTokens, false);
     return preprocessed;
 }
 
-QByteArray Preprocessor::expand(const QByteArray &source)
-{
-    QByteArray result;
-    result.reserve(256);
-    expand(source, &result);
-    return result;
-}
-
-void Preprocessor::expand(const QByteArray &source, QByteArray *result)
-{
-    if (result)
-        _expand(source, result);
-}
-
-void Preprocessor::expand(const char *first, const char *last, QByteArray *result)
-{
-    const QByteArray source = QByteArray::fromRawData(first, last - first);
-    return expand(source, result);
-}
-
-void Preprocessor::out(const QByteArray &text)
-{
-    if (_result)
-        _result->append(text);
-}
-
-void Preprocessor::out(char ch)
-{
-    if (_result)
-        _result->append(ch);
-}
-
-void Preprocessor::out(const char *s)
-{
-    if (_result)
-        _result->append(s);
-}
-
 bool Preprocessor::expandMacros() const
 {
-    return _expandMacros;
+    return m_expandMacros;
 }
 
 void Preprocessor::setExpandMacros(bool expandMacros)
 {
-    _expandMacros = expandMacros;
+    m_expandMacros = expandMacros;
 }
 
 bool Preprocessor::keepComments() const
 {
-    return _keepComments;
+    return m_keepComments;
 }
 
 void Preprocessor::setKeepComments(bool keepComments)
 {
-    _keepComments = keepComments;
+    m_keepComments = keepComments;
 }
 
-Preprocessor::State Preprocessor::createStateFromSource(const QByteArray &source) const
+Preprocessor::State Preprocessor::createStateFromSource(const QString &fileName,
+                                                        const QByteArray &source,
+                                                        QByteArray *result,
+                                                        bool noLines,
+                                                        bool markGeneratedTokens,
+                                                        bool inCondition) const
 {
     State state;
-    state.source = source;
-    Lexer lex(state.source.constBegin(), state.source.constEnd());
-    lex.setScanKeywords(false);
-    if (_keepComments)
-        lex.setScanCommentTokens(true);
-    Token tok;
-    do {
-        lex(&tok);
-        state.tokens.append(tok);
-    } while (tok.isNot(T_EOF_SYMBOL));
-    state.dot = state.tokens.constBegin();
+    state.m_currentFileName = fileName;
+    state.m_source = source;
+    state.m_lexer = new Lexer(source.constBegin(), source.constEnd());
+    state.m_lexer->setScanKeywords(false);
+    state.m_lexer->setScanAngleStringLiteralTokens(false);
+    if (m_keepComments)
+        state.m_lexer->setScanCommentTokens(true);
+    state.m_result = result;
+    state.m_noLines = noLines;
+    state.m_markGeneratedTokens = markGeneratedTokens;
+    state.m_inCondition = inCondition;
     return state;
 }
 
-void Preprocessor::processNewline(bool force, int extraLines)
+void Preprocessor::handleDefined(PPToken *tk)
 {
-    if (_dot != _tokens.constBegin()) {
-        TokenIterator prevTok = _dot - 1;
-
-        // Line changes due to multi-line tokens that we assume got printed
-        // to the preprocessed source. See interaction with skipToNextLine.
-        if (maybeMultilineToken(prevTok)) {
-            const char *ptr = _source.constBegin() + prevTok->begin();
-            const char *end = ptr + prevTok->length();
-
-            for (; ptr != end; ++ptr) {
-                if (*ptr == '\n')
-                    ++env->currentLine;
-            }
-        }
-    }
-
-    unsigned lineno = _dot->lineno + extraLines;
-
-    if (! force && env->currentLine == lineno)
+    unsigned lineno = tk->lineno;
+    lex(tk); // consume "defined" token
+    bool lparenSeen = tk->is(T_LPAREN);
+    if (lparenSeen)
+        lex(tk); // consume "(" token
+    if (tk->isNot(T_IDENTIFIER))
+        //### TODO: generate error message
         return;
-
-    if (force || env->currentLine > lineno) {
-        out("\n# ");
-        out(QByteArray::number(lineno));
-        out(' ');
-        out('"');
-        out(env->currentFile.toUtf8());
-        out('"');
-        out('\n');
-    } else {
-        for (unsigned i = env->currentLine; i < lineno; ++i)
-            out('\n');
-    }
-
-    env->currentLine = lineno;
+    PPToken idToken = *tk;
+    do {
+        lex(tk);
+        if (tk->isNot(T_POUND_POUND))
+            break;
+        lex(tk);
+        if (tk->is(T_IDENTIFIER))
+            idToken = generateConcatenated(idToken, *tk);
+        else
+            break;
+    } while (isValidToken(*tk));
+    pushToken(tk);
+    QByteArray result(1, '0');
+    if (m_env->resolve(idToken.asByteArrayRef()))
+        result[0] = '1';
+    *tk = generateToken(T_NUMERIC_LITERAL, ByteArrayRef(&result), lineno, false);
 }
 
-void Preprocessor::processSkippingBlocks(bool skippingBlocks,
-                                         TokenIterator start, TokenIterator /*end*/)
+void Preprocessor::pushToken(Preprocessor::PPToken *tk)
 {
-    if (! client)
-        return;
-
-    if (skippingBlocks != _skipping[iflevel]) {
-        unsigned offset = start->offset;
-
-        if (_skipping[iflevel]) {
-            if (_dot->f.newline)
-                ++offset;
-
-            client->startSkippingBlocks(offset);
+    const PPToken currentTokenBuffer[] = { *tk };
+    m_state.m_tokenBuffer = new TokenBuffer(currentTokenBuffer,
+                                            currentTokenBuffer + 1,
+                                            /*macro */ 0,
+                                            m_state.m_tokenBuffer);
+}
 
-        } else {
-            if (offset)
-                --offset;
+void Preprocessor::lex(PPToken *tk)
+{
+_Lagain:
+    if (m_state.m_tokenBuffer) {
+        if (m_state.m_tokenBuffer->tokens.empty()) {
+            TokenBuffer *r = m_state.m_tokenBuffer;
+            m_state.m_tokenBuffer = m_state.m_tokenBuffer->next;
+            delete r;
+            goto _Lagain;
+        }
+        *tk = m_state.m_tokenBuffer->tokens.front();
+        m_state.m_tokenBuffer->tokens.pop_front();
+    } else {
+        tk->setSource(m_state.m_source);
+        m_state.m_lexer->scan(tk);
+    }
 
-            client->stopSkippingBlocks(offset);
+    if (tk->isValid() && !tk->generated() && !tk->is(T_EOF_SYMBOL))
+        m_env->currentLine = tk->lineno;
+
+_Lclassify:
+    if (! m_state.m_inPreprocessorDirective) {
+        if (tk->newline() && tk->is(T_POUND)) {
+            handlePreprocessorDirective(tk);
+            goto _Lclassify;
+        } else if (tk->newline() && skipping()) {
+            ScopedBoolSwap s(m_state.m_inPreprocessorDirective, true);
+            do {
+                lex(tk);
+            } while (isValidToken(*tk));
+            goto _Lclassify;
+        } else if (tk->is(T_IDENTIFIER) && !isQtReservedWord(tk->asByteArrayRef())) {
+            static const QByteArray ppDefined("defined");
+            if (m_state.m_inCondition && tk->asByteArrayRef() == ppDefined)
+                handleDefined(tk);
+            else if (handleIdentifier(tk))
+                goto _Lagain;
         }
     }
 }
 
-bool Preprocessor::markGeneratedTokens(bool markGeneratedTokens,
-                                       TokenIterator dot)
+void Preprocessor::skipPreprocesorDirective(PPToken *tk)
 {
-    bool previous = _markGeneratedTokens;
-    if (previous != markGeneratedTokens) {
-        if (! dot)
-            dot = _dot;
-        const int pos = markGeneratedTokens ? dot->begin() : (dot - 1)->end();
-        this->markGeneratedTokens(markGeneratedTokens, pos, dot->lineno - _dot->lineno, dot->f.newline);
+    ScopedBoolSwap s(m_state.m_inPreprocessorDirective, true);
+
+    while (isValidToken(*tk)) {
+        lex(tk);
     }
-    return previous;
 }
 
-bool Preprocessor::markGeneratedTokens(bool markGeneratedTokens, int position, int extraLines, bool newline)
+bool Preprocessor::handleIdentifier(PPToken *tk)
 {
-    bool previous = _markGeneratedTokens;
-    _markGeneratedTokens = markGeneratedTokens;
-
-    if (previous != _markGeneratedTokens) {
-
-        if (_markGeneratedTokens)
-            out("\n#gen true");
-        else
-            out("\n#gen false");
-
-        processNewline(/*force = */ true, extraLines);
-
-        const char *begin = _source.constBegin();
-        const char *end   = begin + position;
-
-        const char *it = end - 1;
-        for (; it != begin - 1; --it) {
-            if (*it == '\n')
-                break;
+    ScopedBoolSwap s(m_state.m_inPreprocessorDirective, true);
+
+    static const QByteArray ppLine("__LINE__");
+    static const QByteArray ppFile("__FILE__");
+    static const QByteArray ppDate("__DATE__");
+    static const QByteArray ppTime("__TIME__");
+
+    ByteArrayRef macroNameRef = tk->asByteArrayRef();
+    bool newline = tk->newline();
+
+    if (!m_state.m_inDefine && macroNameRef.size() == 8 && macroNameRef[0] == '_' && macroNameRef[1] == '_') {
+        PPToken newTk;
+        if (macroNameRef == ppLine) {
+            QByteArray txt = QByteArray::number(tk->lineno);
+            newTk = generateToken(T_STRING_LITERAL, &txt, tk->lineno, false);
+        } else if (macroNameRef == ppFile) {
+            QByteArray txt;
+            txt.append('"');
+            txt.append(m_env->currentFile.toUtf8());
+            txt.append('"');
+            newTk = generateToken(T_STRING_LITERAL, &txt, tk->lineno, false);
+        } else if (macroNameRef == ppDate) {
+            QByteArray txt;
+            txt.append('"');
+            txt.append(QDate::currentDate().toString().toUtf8());
+            txt.append('"');
+            newTk = generateToken(T_STRING_LITERAL, &txt, tk->lineno, false);
+        } else if (macroNameRef == ppTime) {
+            QByteArray txt;
+            txt.append('"');
+            txt.append(QTime::currentTime().toString().toUtf8());
+            txt.append('"');
+            newTk = generateToken(T_STRING_LITERAL, &txt, tk->lineno, false);
         }
-        ++it;
-
-        for (; it != end; ++it) {
-            if (! pp_isspace(*it))
-                out(' ');
 
-            else
-                out(*it);
+        if (newTk.isValid()) {
+            newTk.f.newline = newline;
+            newTk.f.whitespace = tk->whitespace();
+            *tk = newTk;
+            return false;
         }
-
-        if (!markGeneratedTokens && newline)
-            processNewline(/*force = */ true);
     }
 
-    return previous;
-}
+    const QByteArray macroName = macroNameRef.toByteArray();
+    if (tk->generated() && m_state.m_tokenBuffer && m_state.m_tokenBuffer->isBlocked(macroName))
+        return false;
 
-bool Preprocessor::maybeAfterComment() const
-{
-    unsigned endOfPreviousToken = 0;
-
-    if (_dot != _tokens.constBegin())
-        endOfPreviousToken = (_dot - 1)->end();
+    Macro *macro = m_env->resolve(macroName);
+    if (!macro)
+        return false;
+//    qDebug() << "expanding" << macro->name() << "on line" << tk->lineno;
 
-    const char *start = _source.constBegin() + endOfPreviousToken;
+    if (m_client)
+        m_client->startExpandingMacro(tk->offset, *macro, macroName);
+    QVector<PPToken> body = macro->definitionTokens();
 
-    if (*start == '/')
-        return true;
-
-    return false;
-}
+    if (macro->isFunctionLike()) {
+        if (!expandMacros() || !handleFunctionLikeMacro(tk, macro, body, !m_state.m_inDefine))
+            // the call is not function like or expandMacros() returns false, so stop
+            return false;
 
-bool Preprocessor::maybeMultilineToken(Preprocessor::TokenIterator tok)
-{
-    return tok->isLiteral()
-            || (_keepComments
-                && (tok->kind() == T_COMMENT
-                    || tok->kind() == T_DOXY_COMMENT));
-}
+    }
 
-void Preprocessor::skipToNextLine()
-{
-    do {
-        if (maybeMultilineToken(_dot)) {
-            const char *ptr = _source.constBegin() + _dot->begin();
-            const char *end = ptr + _dot->length();
-
-            int newlines = 0;
-            for (; ptr != end; ++ptr) {
-                if (*ptr == '\n')
-                    ++newlines;
-            }
-            if (newlines) {
-                // This function does not output tokens it skips. We need to offset
-                // the currentLine so it gets correctly adjusted by processNewline.
-                env->currentLine -= newlines;
-                ++_dot;
-                return;
-            }
+    if (body.isEmpty()) {
+        if (!m_state.m_inDefine) {
+            // macro expanded to empty, so characters disappeared, hence force a re-indent.
+            PPToken forceWhitespacingToken;
+            // special case: for a macro that expanded to empty, we do not want
+            // to generate a new #line and re-indent, but just generate the
+            // amount of spaces that the macro name took up.
+            forceWhitespacingToken.f.length = tk->length() + (tk->whitespace() ? 1 : 0);
+            body.push_front(forceWhitespacingToken);
         }
+    } else {
+        PPToken &firstNewTk = body.first();
+        firstNewTk.f.newline = newline;
+        firstNewTk.f.whitespace = true; // the macro call is removed, so space the first token correctly.
+    }
 
-        ++_dot;
+    m_state.m_tokenBuffer = new TokenBuffer(body.begin(), body.end(),
+                                           macro, m_state.m_tokenBuffer);
+    m_state.m_tokenBuffer->blockMacro(macroName);
 
-    } while (_dot->isNot(T_EOF_SYMBOL)
-             && (_dot->f.joined || !_dot->f.newline));
+    if (m_client)
+        m_client->stopExpandingMacro(tk->offset, *macro);
+
+    return true;
 }
 
-void Preprocessor::preprocess(const QString &fileName, const QByteArray &source,
-                              QByteArray *result)
+bool Preprocessor::handleFunctionLikeMacro(PPToken *tk, const Macro *macro, QVector<PPToken> &body, bool addWhitespaceMarker)
 {
-    const int previousIfLevel = iflevel;
-
-    QByteArray *previousResult = _result;
-    _result = result;
-
-    pushState(createStateFromSource(source));
-
-    const QString previousFileName = env->currentFile;
-    env->currentFile = fileName;
-
-    const unsigned previousCurrentLine = env->currentLine;
-    env->currentLine = 0;
-
-    while (true) {
-
-        if (_dot->f.joined)
-            out("\\");
-
-        processNewline();
-
-        if (_dot->is(T_EOF_SYMBOL)) {
-            break;
-
-        } else if (_dot->is(T_POUND) && (! _dot->f.joined && _dot->f.newline)) {
-            // handle the preprocessor directive
-
-            TokenIterator start = _dot;
-            skipToNextLine();
-
-            const bool skippingBlocks = _skipping[iflevel];
-
-            processDirective(start, _dot);
-            processSkippingBlocks(skippingBlocks, start, _dot);
-
-        } else if (skipping()) {
-            // skip the current line
-            skipToNextLine();
-
-        } else {
-
-            if (_dot->f.whitespace || maybeAfterComment()) {
-                unsigned endOfPreviousToken = 0;
-
-                if (_dot != _tokens.constBegin())
-                    endOfPreviousToken = (_dot - 1)->end();
-
-                const unsigned beginOfToken = _dot->begin();
-
-                const char *start = _source.constBegin() + endOfPreviousToken;
-                const char *end = _source.constBegin() + beginOfToken;
-
-                const char *it = end - 1;
-                for (; it != start - 1; --it) {
-                    if (*it == '\n')
-                        break;
-                }
-                ++it;
-
-                for (; it != end; ++it) {
-                    if (pp_isspace(*it))
-                        out(*it);
-
-                    else
-                        out(' ');
-                }
-            }
-
-            if (_dot->isNot(T_IDENTIFIER)) {
-                out(tokenSpell(*_dot));
-                ++_dot;
+    static const QByteArray ppVaArgs("__VA_ARGS__");
+
+    QVector<QVector<PPToken> > actuals;
+    PPToken idToken = *tk;
+    if (!collectActualArguments(tk, &actuals)) {
+        pushToken(tk);
+        *tk = idToken;
+        return false;
+    }
 
-            } else {
-                const TokenIterator identifierToken = _dot;
-                ++_dot; // skip T_IDENTIFIER
-
-                const QByteArray spell = tokenSpell(*identifierToken);
-                if (! _expandMacros) {
-                    if (! env->isBuiltinMacro(spell)) {
-                        Macro *m = env->resolve(spell);
-                        if (m && ! m->isFunctionLike()) {
-                            // expand object-like macros.
-                            processObjectLikeMacro(identifierToken, spell, m);
-                            continue;
-                        }
+    QVector<PPToken> expanded;
+    for (size_t i = 0, bodySize = body.size(); i < bodySize && expanded.size() < MAX_TOKEN_EXPANSION_COUNT; ++i) {
+        int expandedSize = expanded.size();
+        const PPToken &token = body[i];
+
+        if (token.is(T_IDENTIFIER)) {
+            const ByteArrayRef id = token.asByteArrayRef();
+            const QVector<QByteArray> &formals = macro->formals();
+            int j = 0;
+            for (; j < formals.size() && expanded.size() < MAX_TOKEN_EXPANSION_COUNT; ++j) {
+                if (formals[j] == id) {
+                    if (actuals.size() <= j) {
+                        // too few actual parameters
+                        //### TODO: error message
+                        goto exitNicely;
                     }
-                    out(spell);
-                    continue;
-                }
-
-                else if (env->isBuiltinMacro(spell))
-                    expandBuiltinMacro(identifierToken, spell);
-
-                else {
-                    if (Macro *m = env->resolve(spell)) {
-                        if (! m->isFunctionLike()) {
-                            if (0 == (m = processObjectLikeMacro(identifierToken, spell, m)))
-                                continue;
 
-                            // the macro expansion generated something that looks like
-                            // a function-like macro.
+                    QVector<PPToken> actualsForThisParam = actuals[j];
+                    if (id == ppVaArgs || (macro->isVariadic() && j + 1 == formals.size())) {
+                        unsigned lineno = 0;
+                        QByteArray comma(",");
+                        ByteArrayRef commaRef(&comma);
+                        for (int k = j + 1; k < actuals.size(); ++k) {
+                            if (!actualsForThisParam.isEmpty())
+                                lineno = actualsForThisParam.last().lineno;
+                            actualsForThisParam.append(generateToken(T_COMMA, commaRef, lineno, true));
+                            actualsForThisParam += actuals[k];
                         }
+                    }
 
-                        // `m' is function-like macro.
-                        if (_dot->is(T_LPAREN)) {
-                            QVector<MacroArgumentReference> actuals;
-                            collectActualArguments(&actuals);
-
-                            if (_dot->is(T_RPAREN)) {
-                                expandFunctionLikeMacro(identifierToken, m, actuals);
-                                continue;
-                            }
+                    if (i > 0  && body[i - 1].is(T_POUND)) {
+                        QByteArray newText;
+                        newText.reserve(256);
+                        unsigned lineno = 0;
+                        for (int i = 0, ei = actualsForThisParam.size(); i < ei; ++i) {
+                            const PPToken &t = actualsForThisParam.at(i);
+                            if (i == 0)
+                                lineno = t.lineno;
+                            else if (t.whitespace())
+                                newText.append(' ');
+                            newText.append(t.start(), t.length());
                         }
+                        newText.replace("\\", "\\\\");
+                        newText.replace("\"", "\\\"");
+                        expanded.push_back(generateToken(T_STRING_LITERAL, ByteArrayRef(&newText), lineno, true));
+                    } else  {
+                        expanded += actualsForThisParam;
                     }
-
-                    // it's not a function or object-like macro.
-                    out(spell);
+                    break;
                 }
             }
-        }
-    }
 
-    popState();
+            if (j == formals.size())
+                expanded.push_back(token);
+        } else if (token.isNot(T_POUND) && token.isNot(T_POUND_POUND)) {
+            expanded.push_back(token);
+        }
 
-    env->currentFile = previousFileName;
-    env->currentLine = previousCurrentLine;
-    _result = previousResult;
+        if (i > 1 && body[i - 1].is(T_POUND_POUND)) {
+            if (expandedSize < 1 || expanded.size() == expandedSize) //### TODO: [cpp.concat] placemarkers
+                continue;
+            const PPToken &leftTk = expanded[expandedSize - 1];
+            const PPToken &rightTk = expanded[expandedSize];
+            expanded[expandedSize - 1] = generateConcatenated(leftTk, rightTk);
+            expanded.remove(expandedSize);
+        }
+    }
 
-    iflevel = previousIfLevel;
+exitNicely:
+    pushToken(tk);
+    if (addWhitespaceMarker) {
+        PPToken forceWhitespacingToken;
+        expanded.push_front(forceWhitespacingToken);
+    }
+    body = expanded;
+    return true;
 }
 
-void Preprocessor::collectActualArguments(QVector<MacroArgumentReference> *actuals)
+/// invalid pp-tokens are used as markers to force whitespace checks.
+void Preprocessor::preprocess(const QString &fileName, const QByteArray &source,
+                              QByteArray *result, bool noLines,
+                              bool markGeneratedTokens, bool inCondition)
 {
-    if (_dot->isNot(T_LPAREN))
+    if (source.isEmpty())
         return;
 
-    ++_dot;
+    pushState(createStateFromSource(fileName, source, result, noLines, markGeneratedTokens, inCondition));
 
-    if (_dot->is(T_RPAREN))
-        return;
+    const QString previousFileName = m_env->currentFile;
+    m_env->currentFile = fileName;
 
-    actuals->append(collectOneActualArgument());
+    const unsigned previousCurrentLine = m_env->currentLine;
+    m_env->currentLine = 0;
 
-    while (_dot->is(T_COMMA)) {
-        ++_dot;
+    const QByteArray fn = fileName.toUtf8();
 
-        actuals->append(collectOneActualArgument());
-    }
-}
-
-MacroArgumentReference Preprocessor::collectOneActualArgument()
-{
-    const unsigned position = _dot->begin();
-
-    while (_dot->isNot(T_EOF_SYMBOL)) {
-        if (_dot->is(T_COMMA) || _dot->is(T_RPAREN))
-            break;
-
-        if (_dot->isNot(T_LPAREN))
-            ++_dot;
+    PPToken tk(m_state.m_source), prevTk;
+    unsigned lineno = 1;
+    do {
+_Lrestart:
+        bool forceLine = false;
+        lex(&tk);
 
-        else {
-            int count = 0;
+        if (!tk.isValid()) {
+            bool wasGenerated = prevTk.generated();
+            prevTk = tk;
+            prevTk.f.generated = wasGenerated;
+            goto _Lrestart;
+        }
 
-            for (; _dot->isNot(T_EOF_SYMBOL); ++_dot) {
-                if (_dot->is(T_LPAREN))
-                    ++count;
+        if (m_state.m_markGeneratedTokens && tk.generated() && !prevTk.generated()) {
+            startNewOutputLine();
+            out("#gen true\n");
+            ++lineno;
+            forceLine = true;
+        } else if (m_state.m_markGeneratedTokens && !tk.generated() && prevTk.generated()) {
+            startNewOutputLine();
+            out("#gen false\n");
+            ++lineno;
+            forceLine = true;
+        }
 
-                else if (_dot->is(T_RPAREN)) {
-                    if (! --count) {
-                        ++_dot;
-                        break;
-                    }
+        if (forceLine || lineno != tk.lineno) {
+            if (forceLine || lineno > tk.lineno || tk.lineno - lineno > 3) {
+                if (m_state.m_noLines) {
+                    if (!m_state.m_markGeneratedTokens)
+                        out(' ');
+                } else {
+                    startNewOutputLine();
+                    out("# ");
+                    out(QByteArray::number(tk.lineno));
+                    out(" \"");
+                    out(fn);
+                    out("\"\n");
                 }
+            } else {
+                for (unsigned i = lineno; i < tk.lineno; ++i)
+                    out('\n');
             }
+        } else {
+            if (tk.newline() && prevTk.isValid())
+                out('\n');
         }
-    }
-
-    const unsigned end = _dot->begin();
-
-    return MacroArgumentReference(position, end - position);
-}
 
-Macro *Preprocessor::processObjectLikeMacro(TokenIterator identifierToken,
-                                            const QByteArray &spell,
-                                            Macro *m)
-{
-    QByteArray tmp;
-    expandObjectLikeMacro(identifierToken, spell, m, &tmp);
-
-    if (_dot->is(T_LPAREN)) {
-        // check if the expension generated a function-like macro.
-
-        m = 0; // reset the active the macro
-
-        pushState(createStateFromSource(tmp));
-
-        if (_dot->is(T_IDENTIFIER)) {
-            const QByteArray id = tokenSpell(*_dot);
-
-            if (Macro *macro = env->resolve(id)) {
-                if (macro->isFunctionLike())
-                    m = macro;
+        if (tk.whitespace() || prevTk.generated() != tk.generated() || !prevTk.isValid()) {
+            if (prevTk.generated() && tk.generated()) {
+                out(' ');
+            } else if (tk.isValid() && !prevTk.isValid() && tk.lineno == lineno) {
+                out(QByteArray(prevTk.length() + (tk.whitespace() ? 1 : 0), ' '));
+            } else if (prevTk.generated() != tk.generated() || !prevTk.isValid()) {
+                const char *begin = tk.source().constBegin();
+                const char *end = begin + tk.offset;
+                const char *it = end - 1;
+                for (; it >= begin; --it)
+                    if (*it == '\n')
+                        break;
+                ++it;
+                for (; it < end; ++it)
+                    out(' ');
+            } else {
+                const char *begin = tk.source().constBegin();
+                const char *end = begin + tk.offset;
+                const char *it = end - 1;
+                for (; it >= begin; --it)
+                    if (!pp_isspace(*it) || *it == '\n')
+                        break;
+                ++it;
+                for (; it < end; ++it)
+                    out(*it);
             }
         }
 
-        popState();
+        const ByteArrayRef tkBytes = tk.asByteArrayRef();
+        out(tkBytes);
+        lineno = tk.lineno;
+        if (tk.is(T_COMMENT) || tk.is(T_DOXY_COMMENT))
+            lineno += tkBytes.count('\n');
+        prevTk = tk;
+    } while (tk.isNot(T_EOF_SYMBOL));
 
-        if (m != 0)
-            return m;
-    }
+    popState();
 
-    const bool was = markGeneratedTokens(true, identifierToken);
-    out(tmp);
-    (void) markGeneratedTokens(was);
-    return 0;
+    m_env->currentFile = previousFileName;
+    m_env->currentLine = previousCurrentLine;
 }
 
-void Preprocessor::expandBuiltinMacro(TokenIterator identifierToken,
-                                      const QByteArray &spell)
+bool Preprocessor::collectActualArguments(PPToken *tk, QVector<QVector<PPToken> > *actuals)
 {
-    const bool was = markGeneratedTokens(true, identifierToken);
-    expand(spell, _result);
-    (void) markGeneratedTokens(was);
-}
+    Q_ASSERT(tk);
+    Q_ASSERT(actuals);
 
-void Preprocessor::expandObjectLikeMacro(TokenIterator identifierToken,
-                                         const QByteArray &spell,
-                                         Macro *m,
-                                         QByteArray *result)
-{
-    if (client)
-        client->startExpandingMacro(identifierToken->offset,
-                                    *m, spell, false);
+    lex(tk); // consume the identifier
 
-    m->setHidden(true);
-    expand(m->definition(), result);
-    m->setHidden(false);
+    if (tk->isNot(T_LPAREN))
+        //### TODO: error message
+        return false;
 
-    if (client)
-        client->stopExpandingMacro(_dot->offset, *m);
-}
+    QVector<PPToken> tokens;
+    lex(tk);
+    scanActualArgument(tk, &tokens);
 
-void Preprocessor::expandFunctionLikeMacro(TokenIterator identifierToken,
-                                           Macro *m,
-                                           const QVector<MacroArgumentReference> &actuals)
-{
-    const char *beginOfText = startOfToken(*identifierToken);
-    const char *endOfText = endOfToken(*_dot);
-    ++_dot; // skip T_RPAREN
+    actuals->append(tokens);
 
-    if (client) {
-        const QByteArray text =
-                QByteArray::fromRawData(beginOfText,
-                                        endOfText - beginOfText);
+    while (tk->is(T_COMMA)) {
+        lex(tk);
 
-        client->startExpandingMacro(identifierToken->offset,
-                                    *m, text, false, actuals);
+        QVector<PPToken> tokens;
+        scanActualArgument(tk, &tokens);
+        actuals->append(tokens);
     }
 
-    const bool was = markGeneratedTokens(true, identifierToken);
-    expand(beginOfText, endOfText, _result);
-    (void) markGeneratedTokens(was);
-
-    if (client)
-        client->stopExpandingMacro(_dot->offset, *m);
-}
-
-const char *Preprocessor::startOfToken(const Token &token) const
-{ return _source.constBegin() + token.begin(); }
-
-const char *Preprocessor::endOfToken(const Token &token) const
-{ return _source.constBegin() + token.end(); }
-
-QByteArray Preprocessor::tokenSpell(const Token &token) const
-{
-    const QByteArray text = QByteArray::fromRawData(_source.constBegin() + token.offset,
-                                                     token.f.length);
-    return text;
+    if (tk->is(T_RPAREN))
+        lex(tk);
+    //###TODO: else error message
+    return true;
 }
 
-QByteArray Preprocessor::tokenText(const Token &token) const
+void Preprocessor::scanActualArgument(PPToken *tk, QVector<PPToken> *tokens)
 {
-    const QByteArray text(_source.constBegin() + token.offset,
-                          token.f.length);
-    return text;
-}
+    Q_ASSERT(tokens);
 
-void Preprocessor::processDirective(TokenIterator firstToken, TokenIterator lastToken)
-{
-    RangeLexer tk(firstToken, lastToken);
-    ++tk; // skip T_POUND
+    int count = 0;
 
-    if (tk->is(T_IDENTIFIER)) {
-        const QByteArray directive = tokenSpell(*tk);
-        switch (PP_DIRECTIVE_TYPE d = classifyDirective(directive)) {
-        case PP_DEFINE:
-            if (! skipping())
-                processDefine(firstToken, lastToken);
-            break;
-
-        case PP_INCLUDE:
-        case PP_INCLUDE_NEXT:
-        case PP_IMPORT:
-            if (! skipping())
-                processInclude(d == PP_INCLUDE_NEXT, firstToken, lastToken);
-            break;
-
-        case PP_UNDEF:
-            if (! skipping())
-                processUndef(firstToken, lastToken);
-            break;
-
-        case PP_ELIF:
-            processElif(firstToken, lastToken);
-            break;
-
-        case PP_ELSE:
-            processElse(firstToken, lastToken);
-            break;
-
-        case PP_ENDIF:
-            processEndif(firstToken, lastToken);
-            break;
-
-        case PP_IF:
-            processIf(firstToken, lastToken);
-            break;
-
-        case PP_IFDEF:
-        case PP_IFNDEF:
-            processIfdef(d == PP_IFNDEF, firstToken, lastToken);
+    while (tk->isNot(T_EOF_SYMBOL)) {
+        if (tk->is(T_LPAREN)) {
+            ++count;
+        } else if (tk->is(T_RPAREN)) {
+            if (! count)
+                break;
+            --count;
+        } else if (! count && tk->is(T_COMMA)) {
             break;
+        }
 
-        default:
-            break;
-        } // switch
+        tokens->append(*tk);
+        lex(tk);
     }
 }
 
-QVector<Token> Preprocessor::tokenize(const QByteArray &text) const
-{
-    QVector<Token> tokens;
-    Lexer lex(text.constBegin(), text.constEnd());
-    lex.setScanKeywords(false);
-    Token tk;
-    do {
-        lex(&tk);
-        tokens.append(tk);
-    } while (tk.isNot(T_EOF_SYMBOL));
-    return tokens;
-}
-
-void Preprocessor::processInclude(bool, TokenIterator firstToken,
-                                  TokenIterator lastToken, bool acceptMacros)
+void Preprocessor::handlePreprocessorDirective(PPToken *tk)
 {
-    if (! client)
-        return; // nothing to do.
+    ScopedBoolSwap s(m_state.m_inPreprocessorDirective, true);
 
-    RangeLexer tk(firstToken, lastToken);
-    ++tk; // skip T_POUND
-    ++tk; // skip `include|nclude_next'
-
-    if (acceptMacros && tk->is(T_IDENTIFIER)) {
-        // ### TODO: implement me
-#if 0
-        QByteArray name;
-        name.reserve(256);
-        MacroExpander expandInclude(env);
-        expandInclude(startOfToken(tokens.at(2)),
-                      startOfToken(tokens.last()),
-                      &name);
-        const QByteArray previousSource = switchSource(name);
-        //processInclude(skipCurentPath, tokenize(name), /*accept macros=*/ false);
-        (void) switchSource(previousSource);
-#endif
-
-    } else if (tk->is(T_LESS)) {
-
-        TokenIterator start = tk.dot();
-
-        for (; tk->isNot(T_EOF_SYMBOL); ++tk) {
-            if (tk->is(T_GREATER))
-                break;
-        }
+    PPToken poundToken = *tk;
+    lex(tk); // scan the directive
 
-        const char *beginOfPath = endOfToken(*start);
-        const char *endOfPath = startOfToken(*tk);
+    if (tk->newline() && ! tk->joined())
+        return; // nothing to do.
 
-        QString fn = string(beginOfPath, endOfPath - beginOfPath);
-        client->sourceNeeded(fn, Client::IncludeGlobal, firstToken->lineno);
+    static const QByteArray ppDefine("define");
+    static const QByteArray ppIf("if");
+    static const QByteArray ppIfDef("ifdef");
+    static const QByteArray ppIfNDef("ifndef");
+    static const QByteArray ppEndIf("endif");
+    static const QByteArray ppElse("else");
+    static const QByteArray ppUndef("undef");
+    static const QByteArray ppElif("elif");
+    static const QByteArray ppInclude("include");
+    static const QByteArray ppIncludeNext("include_next");
+    static const QByteArray ppImport("import");
+    //### TODO:
+    // line
+    // error
+    // pragma
 
-    } else if (tk->is(T_ANGLE_STRING_LITERAL) || tk->is(T_STRING_LITERAL)) {
+    if (tk->is(T_IDENTIFIER)) {
+        const ByteArrayRef directive = tk->asByteArrayRef();
+
+        if (!skipping() && directive == ppDefine)
+            handleDefineDirective(tk);
+        else if (!skipping() && directive == ppUndef)
+            handleUndefDirective(tk);
+        else if (!skipping() && (directive == ppInclude
+                                 || directive == ppIncludeNext
+                                 || directive == ppImport))
+            handleIncludeDirective(tk);
+        else if (directive == ppIf)
+            handleIfDirective(tk);
+        else if (directive == ppIfDef)
+            handleIfDefDirective(false, tk);
+        else if (directive == ppIfNDef)
+            handleIfDefDirective(true, tk);
+        else if (directive == ppEndIf)
+            handleEndIfDirective(tk, poundToken);
+        else if (directive == ppElse)
+            handleElseDirective(tk, poundToken);
+        else if (directive == ppElif)
+            handleElifDirective(tk, poundToken);
+
+        skipPreprocesorDirective(tk);
+    }
+}
 
-        const QByteArray spell = tokenSpell(*tk);
-        const char *beginOfPath = spell.constBegin();
-        const char *endOfPath = spell.constEnd();
-        const char quote = *beginOfPath;
 
-        if (beginOfPath + 1 != endOfPath && ((quote == '"' && endOfPath[-1] == '"') ||
-                                              (quote == '<' && endOfPath[-1] == '>'))) {
+void Preprocessor::handleIncludeDirective(PPToken *tk)
+{
+    m_state.m_lexer->setScanAngleStringLiteralTokens(true);
+    lex(tk); // consume "include" token
+    m_state.m_lexer->setScanAngleStringLiteralTokens(false);
+    QByteArray included;
 
-            QString fn = string(beginOfPath + 1, spell.length() - 2);
-            client->sourceNeeded(fn, Client::IncludeLocal, firstToken->lineno);
-        }
+    if (tk->is(T_STRING_LITERAL) || tk->is(T_ANGLE_STRING_LITERAL)) {
+        included = tk->asByteArrayRef().toByteArray();
+    } else {
+        included = expand(tk);
     }
+    included = included.trimmed();
+    const unsigned line = tk->lineno;
+    lex(tk); // consume string token
+
+//    qDebug("include [[%s]]", included.toUtf8().constData());
+    Client::IncludeType mode;
+    if (included.at(0) == '"')
+        mode = Client::IncludeLocal;
+    else if (included.at(0) == '<')
+        mode = Client::IncludeGlobal;
+    else
+        return; //### TODO: add error message?
+
+    included = included.mid(1, included.size() - 2);
+    QString inc = QString::fromUtf8(included.constData());
+    if (m_client)
+        m_client->sourceNeeded(inc, mode, line);
 }
 
-void Preprocessor::processDefine(TokenIterator firstToken, TokenIterator lastToken)
+void Preprocessor::handleDefineDirective(PPToken *tk)
 {
-    RangeLexer tk(firstToken, lastToken);
-
-    if (tk.size() < 3)
-        return; // nothing to do
+    const unsigned defineOffset = tk->offset;
+    lex(tk); // consume "define" token
 
-    ++tk; // skip T_POUND
-    ++tk; // skip T_DEFINE
-
-    if (tk->isNot(T_IDENTIFIER)) {
-        // ### warning expected an `identifier'
+    bool hasIdentifier = false;
+    if (tk->isNot(T_IDENTIFIER))
         return;
-    }
+
+    ScopedBoolSwap inDefine(m_state.m_inDefine, true);
 
     Macro macro;
-    macro.setFileName(env->currentFile);
-    macro.setLine(env->currentLine);
-    macro.setName(tokenText(*tk));
-    macro.setOffset(firstToken->offset);
-    macro.setLength(endOfToken(lastToken[- 1]) - startOfToken(*firstToken));
-    ++tk; // skip T_IDENTIFIER
+    macro.setFileName(m_env->currentFile);
+    macro.setLine(m_env->currentLine);
+    QByteArray macroName = tk->asByteArrayRef().toByteArray();
+    macro.setName(macroName);
+    macro.setOffset(tk->offset);
 
-    bool hasIdentifier = false;
-    if (tk->is(T_LPAREN) && ! tk->f.whitespace) {
-        // a function-like macro definition
+    lex(tk);
+
+    if (isValidToken(*tk) && tk->is(T_LPAREN) && ! tk->whitespace()) {
         macro.setFunctionLike(true);
 
-        ++tk; // skip T_LPAREN
-        if (tk->is(T_IDENTIFIER)) {
+        lex(tk); // skip `('
+
+        if (isValidToken(*tk) && tk->is(T_IDENTIFIER)) {
             hasIdentifier = true;
-            macro.addFormal(tokenText(*tk));
-            ++tk; // skip T_IDENTIFIER
-            while (tk->is(T_COMMA)) {
-                ++tk;// skip T_COMMA
-                if (tk->isNot(T_IDENTIFIER)) {
+            macro.addFormal(tk->asByteArrayRef().toByteArray());
+
+            lex(tk);
+
+            while (isValidToken(*tk) && tk->is(T_COMMA)) {
+                lex(tk);
+
+                if (isValidToken(*tk) && tk->is(T_IDENTIFIER)) {
+                    macro.addFormal(tk->asByteArrayRef().toByteArray());
+                    lex(tk);
+                } else {
                     hasIdentifier = false;
-                    break;
                 }
-                macro.addFormal(tokenText(*tk));
-                ++tk; // skip T_IDENTIFIER
             }
         }
 
@@ -1210,18 +1186,24 @@ void Preprocessor::processDefine(TokenIterator firstToken, TokenIterator lastTok
             macro.setVariadic(true);
             if (!hasIdentifier)
                 macro.addFormal("__VA_ARGS__");
-            ++tk; // skip T_DOT_DOT_DOT
-        }
-
-        if (tk->isNot(T_RPAREN)) {
-            // ### warning expected `)'
-            return;
+            lex(tk); // consume elipsis token
         }
+        if (isValidToken(*tk) && tk->is(T_RPAREN))
+            lex(tk); // consume ")" token
+    }
 
-        ++tk; // skip T_RPAREN
+    QVector<PPToken> bodyTokens;
+    PPToken firstBodyToken = *tk;
+    while (isValidToken(*tk)) {
+        tk->f.generated = true;
+        bodyTokens.push_back(*tk);
+        lex(tk);
+        if (eagerExpansion)
+            while (tk->is(T_IDENTIFIER) && !isQtReservedWord(tk->asByteArrayRef()) && handleIdentifier(tk))
+                lex(tk);
     }
 
-    if (isQtReservedWord(macro.name())) {
+    if (isQtReservedWord(ByteArrayRef(&macroName))) {
         QByteArray macroId = macro.name();
 
         if (macro.isFunctionLike()) {
@@ -1236,265 +1218,231 @@ void Preprocessor::processDefine(TokenIterator firstToken, TokenIterator lastTok
             macroId += ')';
         }
 
-        macro.setDefinition(macroId);
+        bodyTokens.clear();
+        macro.setDefinition(macroId, bodyTokens);
     } else {
-        // ### make me fast!
-        const char *startOfDefinition = startOfToken(*tk);
-        const char *endOfDefinition = endOfToken(lastToken[- 1]);
-        // It could be that the start is not really before that end, so the check...
-        if (startOfDefinition < endOfDefinition) {
-            QList<unsigned> lineBreaks;
-            lineBreaks.reserve(4); // A reasonable guess...?
-            QByteArray definition;
-            definition.reserve(endOfDefinition - startOfDefinition);
-            while (startOfDefinition != endOfDefinition) {
-                bool replace = false;
-                if (*startOfDefinition == '\n'
-                        || (startOfDefinition + 1 != endOfDefinition
-                            && *startOfDefinition == '\\'
-                            && *(startOfDefinition + 1) == '\n')) {
-                    replace = true;
-                    if (*startOfDefinition != '\n')
-                        ++startOfDefinition;
-                }
-                if (replace) {
-                    definition.append(' ');
-                    lineBreaks.append(definition.length() - 1);
-                } else {
-                    definition.append(*startOfDefinition);
-                }
-                ++startOfDefinition;
-            }
-            macro.setDefinition(definition.trimmed());
-            macro.setLineBreaks(lineBreaks);
+        int start = firstBodyToken.offset;
+        int len = tk->offset - start;
+        QByteArray bodyText = firstBodyToken.source().mid(start, len).trimmed();
+        for (int i = 0, count = bodyTokens.size(); i < count; ++i) {
+            PPToken &t = bodyTokens[i];
+            if (t.isValid())
+                t.squeeze();
         }
+        macro.setDefinition(bodyText, bodyTokens);
     }
 
-    env->bind(macro);
+    macro.setLength(tk->offset - defineOffset);
+    m_env->bind(macro);
+
+//    qDebug() << "adding macro" << macro.name() << "defined at" << macro.fileName() << ":"<<macro.line();
 
-    if (client)
-        client->macroAdded(macro);
+    if (m_client)
+        m_client->macroAdded(macro);
 }
 
-void Preprocessor::processIf(TokenIterator firstToken, TokenIterator lastToken)
+QByteArray Preprocessor::expand(PPToken *tk, PPToken *lastConditionToken)
 {
-    RangeLexer tk(firstToken, lastToken);
-
-    ++tk; // skip T_POUND
-    ++tk; // skipt `if'
-
-    if (testIfLevel()) {
-        const char *first = startOfToken(*tk);
-        const char *last = startOfToken(*lastToken);
+    QByteArray condition;
+    condition.reserve(256);
+    while (isValidToken(*tk)) {
+        const ByteArrayRef s = tk->asByteArrayRef();
+        condition.append(s.start(), s.length());
+        condition += ' ';
+        *lastConditionToken = *tk;
+        lex(tk);
+    }
+//    qDebug("*** Condition before: [%s]", condition.constData());
 
-        MacroExpander expandCondition (env, 0, client, tk.dot()->offset);
-        QByteArray condition;
-        condition.reserve(256);
-        expandCondition(first, last, &condition);
+    QByteArray result;
+    result.reserve(256);
 
-        QVector<Token> tokens = tokenize(condition);
+    preprocess(m_state.m_currentFileName, condition, &result, true, false, true);
+    result.squeeze();
+//    qDebug("*** Condition after: [%s]", result.constData());
+    return result;
+}
 
-        const Value result = evalExpression(tokens.constBegin(),
-                                            tokens.constEnd() - 1,
-                                            condition);
+const PPToken Preprocessor::evalExpression(PPToken *tk, Value &result)
+{
+    PPToken lastConditionToken;
+    const QByteArray expanded = expand(tk, &lastConditionToken);
+    Lexer lexer(expanded.constData(), expanded.constData() + expanded.size());
+    std::vector<Token> buf;
+    Token t;
+    do {
+        lexer.scan(&t);
+        buf.push_back(t);
+    } while (t.isNot(T_EOF_SYMBOL));
+    ExpressionEvaluator eval(m_client, m_env);
+    result = eval(&buf[0], &buf[buf.size() - 1], expanded);
+    return lastConditionToken;
+}
 
-        _trueTest[iflevel] = ! result.is_zero ();
-        _skipping[iflevel]  =   result.is_zero ();
+void Preprocessor::handleIfDirective(PPToken *tk)
+{
+    lex(tk); // consume "if" token
+    Value result;
+    const PPToken lastExpressionToken = evalExpression(tk, result);
+    const bool value = !result.is_zero();
+
+    const bool wasSkipping = m_state.m_skipping[m_state.m_ifLevel];
+    ++m_state.m_ifLevel;
+    m_state.m_trueTest[m_state.m_ifLevel] = value;
+    if (wasSkipping) {
+        m_state.m_skipping[m_state.m_ifLevel] = wasSkipping;
+    } else {
+        bool startSkipping = !value;
+        m_state.m_skipping[m_state.m_ifLevel] = startSkipping;
+        if (startSkipping && m_client)
+            startSkippingBlocks(lastExpressionToken);
     }
+
 }
 
-void Preprocessor::processElse(TokenIterator firstToken, TokenIterator lastToken)
+void Preprocessor::handleElifDirective(PPToken *tk, const PPToken &poundToken)
 {
-    RangeLexer tk(firstToken, lastToken);
-
-    if (iflevel == 0 && !skipping ()) {
-        // std::cerr << "*** WARNING #else without #if" << std::endl;
-    } else if (iflevel > 0 && _skipping[iflevel - 1]) {
-        _skipping[iflevel] = true;
+    if (m_state.m_ifLevel == 0) {
+//        std::cerr << "*** WARNING #elif without #if" << std::endl;
+        handleIfDirective(tk);
     } else {
-        _skipping[iflevel] = _trueTest[iflevel];
+        lex(tk); // consume "elif" token
+        if (m_state.m_skipping[m_state.m_ifLevel - 1]) {
+            // we keep on skipping because we are nested in a skipped block
+            m_state.m_skipping[m_state.m_ifLevel] = true;
+        } else if (m_state.m_trueTest[m_state.m_ifLevel]) {
+            if (!m_state.m_skipping[m_state.m_ifLevel]) {
+                // start skipping because the preceeding then-part was not skipped
+                m_state.m_skipping[m_state.m_ifLevel] = true;
+                if (m_client)
+                    startSkippingBlocks(poundToken);
+            }
+        } else {
+            // preceeding then-part was skipped, so calculate if we should start
+            // skipping, depending on the condition
+            Value result;
+            evalExpression(tk, result);
+
+            bool startSkipping = result.is_zero();
+            m_state.m_trueTest[m_state.m_ifLevel] = !startSkipping;
+            m_state.m_skipping[m_state.m_ifLevel] = startSkipping;
+            if (m_client && !startSkipping)
+                m_client->stopSkippingBlocks(poundToken.offset - 1);
+        }
     }
 }
 
-void Preprocessor::processElif(TokenIterator firstToken, TokenIterator lastToken)
+void Preprocessor::handleElseDirective(PPToken *tk, const PPToken &poundToken)
 {
-    RangeLexer tk(firstToken, lastToken);
-    ++tk; // skip T_POUND
-    ++tk; // skipt `elif'
-
-    if (! (iflevel > 0)) {
-        // std::cerr << "*** WARNING: " << __FILE__ << __LINE__ << std::endl;
-    } else if (iflevel == 0 && !skipping()) {
-        // std::cerr << "*** WARNING #else without #if" << std::endl;
-    } else if (!_trueTest[iflevel] && !_skipping[iflevel - 1]) {
-
-        const char *first = startOfToken(*tk);
-        const char *last = startOfToken(*lastToken);
+    lex(tk); // consume "else" token
 
-        MacroExpander expandCondition (env, 0, client, tk.dot()->offset);
-        QByteArray condition;
-        condition.reserve(256);
-        expandCondition(first, last, &condition);
-
-        QVector<Token> tokens = tokenize(condition);
-
-        const Value result = evalExpression(tokens.constBegin(),
-                                            tokens.constEnd() - 1,
-                                            condition);
-
-        _trueTest[iflevel] = ! result.is_zero ();
-        _skipping[iflevel]  =   result.is_zero ();
-    } else {
-        _skipping[iflevel] = true;
+    if (m_state.m_ifLevel != 0) {
+        if (m_state.m_skipping[m_state.m_ifLevel - 1]) {
+            // we keep on skipping because we are nested in a skipped block
+            m_state.m_skipping[m_state.m_ifLevel] = true;
+        } else {
+            bool wasSkipping = m_state.m_skipping[m_state.m_ifLevel];
+            bool startSkipping = m_state.m_trueTest[m_state.m_ifLevel];
+            m_state.m_skipping[m_state.m_ifLevel] = startSkipping;
+
+            if (m_client && wasSkipping && !startSkipping)
+                m_client->stopSkippingBlocks(poundToken.offset - 1);
+            else if (m_client && !wasSkipping && startSkipping)
+                startSkippingBlocks(poundToken);
+        }
+    }
+#ifndef NO_DEBUG
+    else {
+        std::cerr << "*** WARNING #else without #if" << std::endl;
     }
+#endif // NO_DEBUG
 }
 
-void Preprocessor::processEndif(TokenIterator, TokenIterator)
+void Preprocessor::handleEndIfDirective(PPToken *tk, const PPToken &poundToken)
 {
-    if (iflevel == 0 && !skipping()) {
-        // std::cerr << "*** WARNING #endif without #if" << std::endl;
+    if (m_state.m_ifLevel == 0) {
+#ifndef NO_DEBUG
+        std::cerr << "*** WARNING #endif without #if";
+        if (!tk->generated())
+            std::cerr << " on line " << tk->lineno << " of file " << m_state.m_currentFileName.toUtf8().constData();
+        std::cerr << std::endl;
+#endif // NO_DEBUG
     } else {
-        _skipping[iflevel] = false;
-        _trueTest[iflevel] = false;
-
-        --iflevel;
+        bool wasSkipping = m_state.m_skipping[m_state.m_ifLevel];
+        m_state.m_skipping[m_state.m_ifLevel] = false;
+        m_state.m_trueTest[m_state.m_ifLevel] = false;
+        --m_state.m_ifLevel;
+        if (m_client && wasSkipping && !m_state.m_skipping[m_state.m_ifLevel])
+            m_client->stopSkippingBlocks(poundToken.offset - 1);
     }
+
+    lex(tk); // consume "endif" token
 }
 
-void Preprocessor::processIfdef(bool checkUndefined,
-                                TokenIterator firstToken,
-                                TokenIterator lastToken)
+void Preprocessor::handleIfDefDirective(bool checkUndefined, PPToken *tk)
 {
-    RangeLexer tk(firstToken, lastToken);
-
-    ++tk; // skip T_POUND
-    ++tk; // skip `ifdef'
-    if (testIfLevel()) {
-        if (tk->is(T_IDENTIFIER)) {
-            const QByteArray macroName = tokenSpell(*tk);
+    static const QByteArray qCreatorRun("Q_CREATOR_RUN");
 
-            bool value = false;
-            if (Macro *macro = macroDefinition(macroName, tk->offset, env, client)) {
-                value = true;
+    lex(tk); // consume "ifdef" token
+    if (tk->is(T_IDENTIFIER)) {
+        bool value = false;
+        const ByteArrayRef macroName = tk->asByteArrayRef();
+        if (Macro *macro = macroDefinition(macroName.toByteArray(), tk->offset, m_env, m_client)) {
+            value = true;
 
-                // the macro is a feature constraint(e.g. QT_NO_XXX)
-                if (checkUndefined && macroName.startsWith("QT_NO_")) {
-                    if (macro->fileName() == QLatin1String("<configuration>")) {
-                        // and it' defined in a pro file (e.g. DEFINES += QT_NO_QOBJECT)
+            // the macro is a feature constraint(e.g. QT_NO_XXX)
+            if (checkUndefined && macroName.startsWith("QT_NO_")) {
+                if (macro->fileName() == QLatin1String("<configuration>")) {
+                    // and it' defined in a pro file (e.g. DEFINES += QT_NO_QOBJECT)
 
-                        value = false; // take the branch
-                    }
+                    value = false; // take the branch
                 }
-
-            } else if (env->isBuiltinMacro(macroName)) {
-                value = true;
-            } else if (macroName == "Q_CREATOR_RUN") {
-                value = true;
             }
-
-            if (checkUndefined)
-                value = ! value;
-
-            _trueTest[iflevel] =   value;
-            _skipping [iflevel] = ! value;
+        } else if (m_env->isBuiltinMacro(macroName)) {
+            value = true;
+        } else if (macroName == qCreatorRun) {
+            value = true;
         }
-    }
-}
 
-void Preprocessor::processUndef(TokenIterator firstToken, TokenIterator lastToken)
-{
-    RangeLexer tk(firstToken, lastToken);
+        if (checkUndefined)
+            value = !value;
 
-    ++tk; // skip T_POUND
-    ++tk; // skip `undef'
+        const bool wasSkipping = m_state.m_skipping[m_state.m_ifLevel];
+        ++m_state.m_ifLevel;
+        m_state.m_trueTest[m_state.m_ifLevel] = value;
+        m_state.m_skipping[m_state.m_ifLevel] = wasSkipping ? wasSkipping : !value;
 
-    if (tk->is(T_IDENTIFIER)) {
-        const QByteArray macroName = tokenText(*tk);
-        const Macro *macro = env->remove(macroName);
+        if (m_client && !wasSkipping && !value)
+            startSkippingBlocks(*tk);
 
-        if (client && macro)
-            client->macroAdded(*macro);
+        lex(tk); // consume the identifier
     }
-}
-
-void Preprocessor::resetIfLevel ()
-{
-    iflevel = 0;
-    _skipping[iflevel] = false;
-    _trueTest[iflevel] = false;
-}
-
-Preprocessor::PP_DIRECTIVE_TYPE Preprocessor::classifyDirective(const QByteArray &directive) const
-{
-    switch (directive.size())
-    {
-    case 2:
-        if (directive[0] == 'i' && directive[1] == 'f')
-            return PP_IF;
-        break;
-
-    case 4:
-        if (directive[0] == 'e' && directive == "elif")
-            return PP_ELIF;
-        else if (directive[0] == 'e' && directive == "else")
-            return PP_ELSE;
-        break;
-
-    case 5:
-        if (directive[0] == 'i' && directive == "ifdef")
-            return PP_IFDEF;
-        else if (directive[0] == 'u' && directive == "undef")
-            return PP_UNDEF;
-        else if (directive[0] == 'e' && directive == "endif")
-            return PP_ENDIF;
-        break;
-
-    case 6:
-        if (directive[0] == 'i' && directive == "ifndef")
-            return PP_IFNDEF;
-        else if (directive[0] == 'i' && directive == "import")
-            return PP_IMPORT;
-        else if (directive[0] == 'd' && directive == "define")
-            return PP_DEFINE;
-        break;
-
-    case 7:
-        if (directive[0] == 'i' && directive == "include")
-            return PP_INCLUDE;
-        break;
-
-    case 12:
-        if (directive[0] == 'i' && directive == "include_next")
-            return PP_INCLUDE_NEXT;
-        break;
-
-    default:
-        break;
+#ifndef NO_DEBUG
+    else {
+        std::cerr << "*** WARNING #ifdef without identifier" << std::endl;
     }
-
-    return PP_UNKNOWN_DIRECTIVE;
+#endif // NO_DEBUG
 }
 
-bool Preprocessor::testIfLevel()
+void Preprocessor::handleUndefDirective(PPToken *tk)
 {
-    const bool result = !_skipping[iflevel++];
-    _skipping[iflevel] = _skipping[iflevel - 1];
-    _trueTest[iflevel] = false;
-    return result;
-}
-
-int Preprocessor::skipping() const
-{ return _skipping[iflevel]; }
+    lex(tk); // consume "undef" token
+    if (tk->is(T_IDENTIFIER)) {
+        const ByteArrayRef macroName = tk->asByteArrayRef();
+        const Macro *macro = m_env->remove(macroName.toByteArray());
 
-Value Preprocessor::evalExpression(TokenIterator firstToken, TokenIterator lastToken,
-                                   const QByteArray &source) const
-{
-    ExpressionEvaluator eval(client, env);
-    const Value result = eval(firstToken, lastToken, source);
-    return result;
+        if (m_client && macro)
+            m_client->macroAdded(*macro);
+        lex(tk); // consume macro name
+    }
+#ifndef NO_DEBUG
+    else {
+        std::cerr << "*** WARNING #undef without identifier" << std::endl;
+    }
+#endif // NO_DEBUG
 }
 
-bool Preprocessor::isQtReservedWord(const QByteArray &macroId) const
+bool Preprocessor::isQtReservedWord(const ByteArrayRef &macroId)
 {
     const int size = macroId.size();
     if      (size == 9 && macroId.at(0) == 'Q' && macroId == "Q_SIGNALS")
@@ -1538,9 +1486,63 @@ bool Preprocessor::isQtReservedWord(const QByteArray &macroId) const
 
 QString Preprocessor::string(const char *first, int length) const
 {
-    if (_originalSource.isEmpty())
+    if (m_originalSource.isEmpty())
         return QString::fromUtf8(first, length);
 
-    const int position = first - _source.constData();
-    return _originalSource.mid(position, length);
+    const int position = first - m_state.m_source.constData();
+    return m_originalSource.mid(position, length);
+}
+
+PPToken Preprocessor::generateToken(enum Kind kind, const ByteArrayRef &content, unsigned lineno, bool addQuotes)
+{
+    size_t len = content.size();
+    const size_t pos = m_scratchBuffer.size();
+
+    if (kind == T_STRING_LITERAL && addQuotes)
+        m_scratchBuffer.append('"');
+    m_scratchBuffer.append(content.start(), content.length());
+    if (kind == T_STRING_LITERAL && addQuotes) {
+        m_scratchBuffer.append('"');
+        len += 2;
+    }
+
+    PPToken tok(m_scratchBuffer);
+    tok.f.kind = kind;
+    if (m_state.m_lexer->control()) {
+        if (kind == T_STRING_LITERAL)
+            tok.string = m_state.m_lexer->control()->stringLiteral(m_scratchBuffer.constData() + pos, len);
+        else if (kind == T_IDENTIFIER)
+            tok.identifier = m_state.m_lexer->control()->identifier(m_scratchBuffer.constData() + pos, len);
+        else if (kind == T_NUMERIC_LITERAL)
+            tok.number = m_state.m_lexer->control()->numericLiteral(m_scratchBuffer.constData() + pos, len);
+    }
+    tok.offset = pos;
+    tok.f.length = len;
+    tok.f.generated = true;
+    tok.lineno = lineno;
+    return tok;
+}
+
+PPToken Preprocessor::generateConcatenated(const PPToken &leftTk, const PPToken &rightTk)
+{
+    QByteArray newText;
+    newText.reserve(leftTk.length() + rightTk.length());
+    newText.append(leftTk.start(), leftTk.length());
+    newText.append(rightTk.start(), rightTk.length());
+    return generateToken(T_IDENTIFIER, ByteArrayRef(&newText), leftTk.lineno, true);
+}
+
+void Preprocessor::startSkippingBlocks(const Preprocessor::PPToken &tk) const
+{
+    if (!m_client)
+        return;
+
+    int iter = tk.end();
+    const QByteArray &txt = tk.source();
+    for (; iter < txt.size(); ++iter) {
+        if (txt.at(iter) == '\n') {
+            m_client->startSkippingBlocks(iter + 1);
+            return;
+        }
+    }
 }
diff --git a/src/libs/cplusplus/pp-engine.h b/src/libs/cplusplus/pp-engine.h
index 40c3cabaa8..a2f4d6415c 100644
--- a/src/libs/cplusplus/pp-engine.h
+++ b/src/libs/cplusplus/pp-engine.h
@@ -52,29 +52,39 @@
 #ifndef CPLUSPLUS_PP_ENGINE_H
 #define CPLUSPLUS_PP_ENGINE_H
 
+#include "PPToken.h"
 #include "PreprocessorClient.h"
-#include "pp-macro-expander.h"
 
+#include <Lexer.h>
 #include <Token.h>
 #include <QVector>
 #include <QBitArray>
+#include <QByteArray>
 
 namespace CPlusPlus {
 
-struct Value;
 class Environment;
 
+namespace Internal {
+class PPToken;
+struct TokenBuffer;
+struct Value;
+}
+
 class CPLUSPLUS_EXPORT Preprocessor
 {
+    typedef Internal::PPToken PPToken;
+    typedef Internal::Value Value;
+
 public:
     Preprocessor(Client *client, Environment *env);
 
     QByteArray operator()(const QString &filename, const QString &source);
-    QByteArray operator()(const QString &filename, const QByteArray &source);
+    QByteArray operator()(const QString &filename, const QByteArray &source, bool noLines = false, bool markGeneratedTokens = true);
 
     void preprocess(const QString &filename,
                     const QByteArray &source,
-                    QByteArray *result);
+                    QByteArray *result, bool noLines, bool markGeneratedTokens, bool inCondition);
 
     bool expandMacros() const;
     void setExpandMacros(bool expandMacros);
@@ -85,126 +95,101 @@ public:
 private:
     enum { MAX_LEVEL = 512 };
 
-    enum PP_DIRECTIVE_TYPE
-    {
-        PP_UNKNOWN_DIRECTIVE,
-        PP_DEFINE,
-        PP_IMPORT,
-        PP_INCLUDE,
-        PP_INCLUDE_NEXT,
-        PP_ELIF,
-        PP_ELSE,
-        PP_ENDIF,
-        PP_IF,
-        PP_IFDEF,
-        PP_IFNDEF,
-        PP_UNDEF
-    };
-
-    typedef const CPlusPlus::Token *TokenIterator;
-
     struct State {
-        QByteArray source;
-        QVector<CPlusPlus::Token> tokens;
-        TokenIterator dot;
-    };
+        State();
 
-    bool markGeneratedTokens(bool markGeneratedTokens, TokenIterator dot = 0);
-    bool markGeneratedTokens(bool markGeneratedTokens, int position, int extraLines=0, bool newline=false);
+        QString m_currentFileName;
 
-    QByteArray expand(const QByteArray &source);
-    void expand(const QByteArray &source, QByteArray *result);
-    void expand(const char *first, const char *last, QByteArray *result);
-    void expandBuiltinMacro(TokenIterator identifierToken,
-                            const QByteArray &spell);
-    void expandObjectLikeMacro(TokenIterator identifierToken,
-                               const QByteArray &spell,
-                               Macro *m, QByteArray *result);
-    void expandFunctionLikeMacro(TokenIterator identifierToken, Macro *m,
-                                 const QVector<MacroArgumentReference> &actuals);
+        QByteArray m_source;
+        Lexer *m_lexer;
+        QBitArray m_skipping;
+        QBitArray m_trueTest;
+        int m_ifLevel;
+        Internal::TokenBuffer *m_tokenBuffer;
+        bool m_inPreprocessorDirective;
 
-    void resetIfLevel();
-    bool testIfLevel();
-    int skipping() const;
+        QByteArray *m_result;
+        bool m_markGeneratedTokens;
 
-    PP_DIRECTIVE_TYPE classifyDirective(const QByteArray &directive) const;
+        bool m_noLines;
+        bool m_inCondition;
+        bool m_inDefine;
+    };
+
+    void handleDefined(PPToken *tk);
+    void pushToken(PPToken *tk);
+    void lex(PPToken *tk);
+    void skipPreprocesorDirective(PPToken *tk);
+    bool handleIdentifier(PPToken *tk);
+    bool handleFunctionLikeMacro(PPToken *tk, const Macro *macro, QVector<PPToken> &body, bool addWhitespaceMarker);
 
-    Value evalExpression(TokenIterator firstToken,
-                         TokenIterator lastToken,
-                         const QByteArray &source) const;
+    bool skipping() const
+    { return m_state.m_skipping[m_state.m_ifLevel]; }
 
     QVector<CPlusPlus::Token> tokenize(const QByteArray &text) const;
 
-    const char *startOfToken(const CPlusPlus::Token &token) const;
-    const char *endOfToken(const CPlusPlus::Token &token) const;
+    bool collectActualArguments(PPToken *tk, QVector<QVector<PPToken> > *actuals);
+    void scanActualArgument(PPToken *tk, QVector<PPToken> *tokens);
 
-    QByteArray tokenSpell(const CPlusPlus::Token &token) const;
-    QByteArray tokenText(const CPlusPlus::Token &token) const; // does a deep copy
+    void handlePreprocessorDirective(PPToken *tk);
+    void handleIncludeDirective(PPToken *tk);
+    void handleDefineDirective(PPToken *tk);
+    QByteArray expand(PPToken *tk, PPToken *lastConditionToken = 0);
+    const Internal::PPToken evalExpression(PPToken *tk, Value &result);
+    void handleIfDirective(PPToken *tk);
+    void handleElifDirective(PPToken *tk, const PPToken &poundToken);
+    void handleElseDirective(PPToken *tk, const PPToken &poundToken);
+    void handleEndIfDirective(PPToken *tk, const PPToken &poundToken);
+    void handleIfDefDirective(bool checkUndefined, PPToken *tk);
+    void handleUndefDirective(PPToken *tk);
 
-    void collectActualArguments(QVector<MacroArgumentReference> *actuals);
-    MacroArgumentReference collectOneActualArgument();
+    static bool isQtReservedWord(const Internal::ByteArrayRef &name);
 
-    void processNewline(bool force = false, int extraLines = 0);
+    void pushState(const State &newState);
+    void popState();
 
-    void processSkippingBlocks(bool skippingBlocks,
-                               TokenIterator dot, TokenIterator lastToken);
+    State createStateFromSource(const QString &fileName, const QByteArray &source, QByteArray *result, bool noLines, bool markGeneratedTokens, bool inCondition) const;
 
-    Macro *processObjectLikeMacro(TokenIterator identifierToken,
-                                  const QByteArray &spell,
-                                  Macro *m);
+    inline bool atStartOfOutputLine() const
+    { return (m_state.m_result && !m_state.m_result->isEmpty()) ? m_state.m_result->end()[-1] == '\n' : true; }
 
-    void processDirective(TokenIterator dot, TokenIterator lastToken);
-    void processInclude(bool skipCurrentPath,
-                        TokenIterator dot, TokenIterator lastToken,
-                        bool acceptMacros = true);
-    void processDefine(TokenIterator dot, TokenIterator lastToken);
-    void processIf(TokenIterator dot, TokenIterator lastToken);
-    void processElse(TokenIterator dot, TokenIterator lastToken);
-    void processElif(TokenIterator dot, TokenIterator lastToken);
-    void processEndif(TokenIterator dot, TokenIterator lastToken);
-    void processIfdef(bool checkUndefined,
-                      TokenIterator dot, TokenIterator lastToken);
-    void processUndef(TokenIterator dot, TokenIterator lastToken);
+    inline void startNewOutputLine() const
+    {
+        if (m_state.m_result && !m_state.m_result->isEmpty() && m_state.m_result->end()[-1] != '\n')
+            out('\n');
+    }
 
-    bool isQtReservedWord(const QByteArray &name) const;
+    inline void out(const QByteArray &text) const
+    { if (m_state.m_result) m_state.m_result->append(text); }
 
-    State state() const;
-    void pushState(const State &state);
-    void popState();
+    inline void out(char ch) const
+    { if (m_state.m_result) m_state.m_result->append(ch); }
 
-    State createStateFromSource(const QByteArray &source) const;
+    inline void out(const char *s) const
+    { if (m_state.m_result) m_state.m_result->append(s); }
 
-    void out(const QByteArray &text);
-    void out(char ch);
-    void out(const char *s);
+    inline void out(const Internal::ByteArrayRef &ref) const
+    { if (m_state.m_result) m_state.m_result->append(ref.start(), ref.length()); }
 
     QString string(const char *first, int len) const;
-    bool maybeAfterComment() const;
-
-    bool maybeMultilineToken(TokenIterator tok);
-    void skipToNextLine();
 
-private:
-    Client *client;
-    Environment *env;
-    MacroExpander _expand;
+    PPToken generateToken(enum Kind kind, const Internal::ByteArrayRef &content, unsigned lineno, bool addQuotes);
+    PPToken generateConcatenated(const PPToken &leftTk, const PPToken &rightTk);
 
-    QBitArray _skipping; // ### move in state
-    QBitArray _trueTest; // ### move in state
-    int iflevel; // ### move in state
+    void startSkippingBlocks(const PPToken &tk) const;
 
-    QList<State> _savedStates;
+private:
+    Client *m_client;
+    Environment *m_env;
+    QByteArray m_scratchBuffer;
 
-    QByteArray _source;
-    QVector<CPlusPlus::Token> _tokens;
-    TokenIterator _dot;
+    QList<State> m_savedStates;
 
-    QByteArray *_result;
-    bool _markGeneratedTokens;
+    QString m_originalSource;
+    bool m_expandMacros;
+    bool m_keepComments;
 
-    QString _originalSource;
-    bool _expandMacros;
-    bool _keepComments;
+    State m_state;
 };
 
 } // namespace CPlusPlus
diff --git a/src/libs/cplusplus/pp-macro-expander.cpp b/src/libs/cplusplus/pp-macro-expander.cpp
deleted file mode 100644
index cde82884c0..0000000000
--- a/src/libs/cplusplus/pp-macro-expander.cpp
+++ /dev/null
@@ -1,449 +0,0 @@
-/**************************************************************************
-**
-** This file is part of Qt Creator
-**
-** Copyright (c) 2012 Nokia Corporation and/or its subsidiary(-ies).
-**
-** Contact: Nokia Corporation (qt-info@nokia.com)
-**
-**
-** GNU Lesser General Public License Usage
-**
-** This file may be used under the terms of the GNU Lesser General Public
-** License version 2.1 as published by the Free Software Foundation and
-** appearing in the file LICENSE.LGPL included in the packaging of this file.
-** Please review the following information to ensure the GNU Lesser General
-** Public License version 2.1 requirements will be met:
-** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
-**
-** In addition, as a special exception, Nokia gives you certain additional
-** rights. These rights are described in the Nokia Qt LGPL Exception
-** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
-**
-** Other Usage
-**
-** Alternatively, this file may be used in accordance with the terms and
-** conditions contained in a signed written agreement between you and Nokia.
-**
-** If you have questions regarding the use of this file, please contact
-** Nokia at qt-info@nokia.com.
-**
-**************************************************************************/
-
-#include "pp-macro-expander.h"
-
-#include "pp.h"
-#include "pp-cctype.h"
-#include <QDateTime>
-
-namespace CPlusPlus {
-
-
-
-struct pp_frame
-{
-    Macro *expanding_macro;
-    const QVector<QByteArray> actuals;
-
-    pp_frame(Macro *expanding_macro, const QVector<QByteArray> &actuals)
-        : expanding_macro (expanding_macro),
-          actuals (actuals)
-    { }
-};
-
-
-} // namespace CPlusPlus
-
-using namespace CPlusPlus;
-
-inline static bool comment_p (const char *__first, const char *__last)
-{
-    if (__first == __last)
-        return false;
-
-    if (*__first != '/')
-        return false;
-
-    if (++__first == __last)
-        return false;
-
-    return (*__first == '/' || *__first == '*');
-}
-
-MacroExpander::MacroExpander(Environment *env, pp_frame *frame, Client *client, unsigned start_offset)
-    : env(env),
-      frame(frame),
-      client(client),
-      start_offset(start_offset),
-      lines(0)
-{ }
-
-const QByteArray *MacroExpander::resolve_formal(const QByteArray &__name)
-{
-    if (! (frame && frame->expanding_macro))
-        return 0;
-
-    const QVector<QByteArray> formals = frame->expanding_macro->formals();
-    for (int index = 0; index < formals.size(); ++index) {
-        const QByteArray formal = formals.at(index);
-
-        if (formal == __name && index < frame->actuals.size())
-            return &frame->actuals.at(index);
-    }
-
-    return 0;
-}
-
-const char *MacroExpander::operator()(const char *first, const char *last,
-                                      QByteArray *result)
-{
-    return expand(first, last, result);
-}
-
-const char *MacroExpander::expand(const char *__first, const char *__last,
-                                  QByteArray *__result)
-{
-    __first = skip_blanks (__first, __last);
-    lines = skip_blanks.lines;
-
-    while (__first != __last)
-    {
-        if (*__first == '\n')
-        {
-            __result->append("\n# ");
-            __result->append(QByteArray::number(env->currentLine));
-            __result->append(' ');
-            __result->append('"');
-            __result->append(env->currentFile.toUtf8());
-            __result->append('"');
-            __result->append('\n');
-            ++lines;
-
-            __first = skip_blanks (++__first, __last);
-            lines += skip_blanks.lines;
-
-            if (__first != __last && *__first == '#')
-                break;
-        }
-        else if (*__first == '#')
-        {
-            __first = skip_blanks (++__first, __last);
-            lines += skip_blanks.lines;
-
-            const char *end_id = skip_identifier (__first, __last);
-            const QByteArray fast_name(__first, end_id - __first);
-            __first = end_id;
-
-            if (const QByteArray *actual = resolve_formal (fast_name))
-            {
-                __result->append('\"');
-
-                const char *actual_begin = actual->constData ();
-                const char *actual_end = actual_begin + actual->size ();
-
-                for (const char *it = skip_whitespaces (actual_begin, actual_end);
-                        it != actual_end; ++it)
-                {
-                    if (*it == '"' || *it == '\\')
-                    {
-                        __result->append('\\');
-                        __result->append(*it);
-                    }
-                    else if (*it == '\n')
-                    {
-                        __result->append('"');
-                        __result->append('\n');
-                        __result->append('"');
-                    }
-                    else
-                        __result->append(*it);
-                }
-
-                __result->append('\"');
-            }
-            else {
-                // ### warning message?
-            }
-        }
-        else if (*__first == '\"')
-        {
-            const char *next_pos = skip_string_literal (__first, __last);
-            lines += skip_string_literal.lines;
-            __result->append(__first, next_pos - __first);
-            __first = next_pos;
-        }
-        else if (*__first == '\'')
-        {
-            const char *next_pos = skip_char_literal (__first, __last);
-            lines += skip_char_literal.lines;
-            __result->append(__first, next_pos - __first);
-            __first = next_pos;
-        }
-        else if (*__first == '\\')
-        {
-            ++__first;
-            if (__first != __last && *__first == '\n')
-            {
-                ++lines;
-                ++__first;
-            } else {
-                __result->append('\\');
-            }
-        }
-        else if (comment_p (__first, __last))
-        {
-            __first = skip_comment_or_divop (__first, __last);
-            int n = skip_comment_or_divop.lines;
-            lines += n;
-
-            while (n-- > 0)
-                __result->append('\n');
-        }
-        else if (pp_isspace (*__first))
-        {
-            for (; __first != __last; ++__first)
-            {
-                if (*__first == '\n' || !pp_isspace (*__first))
-                    break;
-            }
-
-            __result->append(' ');
-        }
-        else if (pp_isdigit (*__first))
-        {
-            const char *next_pos = skip_number (__first, __last);
-            lines += skip_number.lines;
-            __result->append(__first, next_pos - __first);
-            __first = next_pos;
-        }
-        else if (pp_isalpha (*__first) || *__first == '_')
-        {
-            const char *name_begin = __first;
-            const char *name_end = skip_identifier (__first, __last);
-            __first = name_end; // advance
-
-            // search for the paste token
-            const char *next = skip_blanks (__first, __last);
-            bool paste = false;
-
-            bool need_comma = false;
-            if (next != __last && *next == ',') {
-                const char *x = skip_blanks(__first + 1, __last);
-                if (x != __last && *x == '#' && (x + 1) != __last && x[1] == '#') {
-                    need_comma = true;
-                    paste = true;
-                    __first = skip_blanks(x + 2, __last);
-                }
-            }
-
-            if (next != __last && *next == '#')
-            {
-                paste = true;
-                ++next;
-                if (next != __last && *next == '#')
-                    __first = skip_blanks(++next, __last);
-            }
-
-            const QByteArray fast_name(name_begin, name_end - name_begin);
-            const QByteArray *actual = resolve_formal (fast_name);
-            if (actual)
-            {
-                const char *begin = actual->constData ();
-                const char *end = begin + actual->size ();
-                if (paste) {
-                    for (--end; end != begin - 1; --end) {
-                        if (! pp_isspace(*end))
-                            break;
-                    }
-                    ++end;
-                }
-                __result->append(begin, end - begin);
-                if (need_comma)
-                    __result->append(',');
-                continue;
-            }
-
-            Macro *macro = env->resolve (fast_name);
-            if (! macro || macro->isHidden() || env->hideNext)
-            {
-                if (fast_name.size () == 7 && fast_name [0] == 'd' && fast_name == "defined")
-                    env->hideNext = true;
-                else
-                    env->hideNext = false;
-
-                if (fast_name.size () == 8 && fast_name [0] == '_' && fast_name [1] == '_')
-                {
-                    if (fast_name == "__LINE__")
-                    {
-                        __result->append(QByteArray::number(env->currentLine + lines));
-                        continue;
-                    }
-
-                    else if (fast_name == "__FILE__")
-                    {
-                        __result->append('"');
-                        __result->append(env->currentFile.toUtf8());
-                        __result->append('"');
-                        continue;
-                    }
-
-                    else if (fast_name == "__DATE__")
-                    {
-                        __result->append('"');
-                        __result->append(QDate::currentDate().toString().toUtf8());
-                        __result->append('"');
-                        continue;
-                    }
-
-                    else if (fast_name == "__TIME__")
-                    {
-                        __result->append('"');
-                        __result->append(QTime::currentTime().toString().toUtf8());
-                        __result->append('"');
-                        continue;
-                    }
-
-                }
-
-                __result->append(name_begin, name_end - name_begin);
-                continue;
-            }
-
-            if (! macro->isFunctionLike())
-            {
-                Macro *m = 0;
-
-                if (! macro->definition().isEmpty())
-                {
-                    macro->setHidden(true);
-
-                    QByteArray __tmp;
-                    __tmp.reserve (256);
-
-                    MacroExpander expand_macro (env);
-                    expand_macro(macro->definition(), &__tmp);
-
-                    if (! __tmp.isEmpty ())
-                    {
-                        const char *__tmp_begin = __tmp.constBegin();
-                        const char *__tmp_end = __tmp.constEnd();
-                        const char *__begin_id = skip_whitespaces (__tmp_begin, __tmp_end);
-                        const char *__end_id = skip_identifier (__begin_id, __tmp_end);
-
-                        if (__end_id == __tmp_end)
-                        {
-                            const QByteArray __id (__begin_id, __end_id - __begin_id);
-                            m = env->resolve (__id);
-                        }
-
-                        if (! m)
-                            *__result += __tmp;
-                    }
-
-                    macro->setHidden(false);
-                }
-
-                if (! m)
-                    continue;
-
-                macro = m;
-            }
-
-            // function like macro
-            const char *arg_it = skip_whitespaces (__first, __last);
-
-            if (arg_it == __last || *arg_it != '(')
-            {
-                __result->append(name_begin, name_end - name_begin);
-                lines += skip_whitespaces.lines;
-                __first = arg_it;
-                continue;
-            }
-
-            QVector<QByteArray> actuals;
-            actuals.reserve(macro->formals().size());
-            ++arg_it; // skip '('
-
-            MacroExpander expand_actual (env, frame);
-
-            const char *arg_end = skip_argument(arg_it, __last);
-            if (arg_it != arg_end || (arg_end != __last && *arg_end == ','))
-            {
-                const QByteArray actual(arg_it, arg_end - arg_it);
-                QByteArray expanded;
-                expand_actual(actual.constBegin (), actual.constEnd (), &expanded);
-                pushActuals(actuals, macro, expanded);
-                arg_it = arg_end;
-            }
-
-            while (arg_it != __last && *arg_end == ',')
-            {
-                ++arg_it; // skip ','
-
-                arg_end = skip_argument(arg_it, __last);
-                const QByteArray actual(arg_it, arg_end - arg_it);
-                QByteArray expanded;
-                expand_actual(actual.constBegin (), actual.constEnd (), &expanded);
-                pushActuals(actuals, macro, expanded);
-                arg_it = arg_end;
-            }
-
-            if (! (arg_it != __last && *arg_it == ')'))
-                return __last;
-
-            ++arg_it; // skip ')'
-            __first = arg_it;
-
-            pp_frame frame (macro, actuals);
-            MacroExpander expand_macro (env, &frame);
-            macro->setHidden(true);
-            expand_macro (macro->definition(), __result);
-            macro->setHidden(false);
-        }
-        else
-            __result->append(*__first++);
-    }
-
-    return __first;
-}
-
-const char *MacroExpander::skip_argument_variadics (QVector<QByteArray> const &__actuals,
-                                                    Macro *__macro,
-                                                    const char *__first, const char *__last)
-{
-    const char *arg_end = skip_argument (__first, __last);
-
-    while (__macro->isVariadic() && __first != arg_end && arg_end != __last && *arg_end == ','
-           && (__actuals.size () + 1) == __macro->formals().size ())
-    {
-        arg_end = skip_argument (++arg_end, __last);
-    }
-
-    return arg_end;
-}
-
-void MacroExpander::pushActuals(QVector<QByteArray> & actuals, Macro *__macro, const QByteArray& expanded)
-{
-    if (__macro->isVariadic() && actuals.count() == __macro->formals().count()) {
-        //already enough params --> append to the last one
-        QByteArray& b = actuals.last();
-        b.append(",");
-        b.append(expanded.trimmed());
-    }
-    else {
-        const char * __first = expanded.constData();
-        const char * __last = __first + expanded.length();
-        const char * arg_it = __first;
-
-        const char *arg_end = skip_argument_variadics(actuals, __macro, arg_it, __last);
-        actuals.push_back(QByteArray(arg_it, arg_end - arg_it).trimmed());
-        arg_it = arg_end;
-
-        while (arg_it != __last) {
-            ++arg_it; // skip ','
-            const char *arg_end = skip_argument_variadics(actuals, __macro, arg_it, __last);
-            actuals.push_back(QByteArray(arg_it, arg_end - arg_it).trimmed());
-            arg_it = arg_end;
-        }
-    }
-}
diff --git a/src/libs/cplusplus/pp-macro-expander.h b/src/libs/cplusplus/pp-macro-expander.h
deleted file mode 100644
index 97080a9f20..0000000000
--- a/src/libs/cplusplus/pp-macro-expander.h
+++ /dev/null
@@ -1,110 +0,0 @@
-/**************************************************************************
-**
-** This file is part of Qt Creator
-**
-** Copyright (c) 2012 Nokia Corporation and/or its subsidiary(-ies).
-**
-** Contact: Nokia Corporation (qt-info@nokia.com)
-**
-**
-** GNU Lesser General Public License Usage
-**
-** This file may be used under the terms of the GNU Lesser General Public
-** License version 2.1 as published by the Free Software Foundation and
-** appearing in the file LICENSE.LGPL included in the packaging of this file.
-** Please review the following information to ensure the GNU Lesser General
-** Public License version 2.1 requirements will be met:
-** http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
-**
-** In addition, as a special exception, Nokia gives you certain additional
-** rights. These rights are described in the Nokia Qt LGPL Exception
-** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
-**
-** Other Usage
-**
-** Alternatively, this file may be used in accordance with the terms and
-** conditions contained in a signed written agreement between you and Nokia.
-**
-** If you have questions regarding the use of this file, please contact
-** Nokia at qt-info@nokia.com.
-**
-**************************************************************************/
-/*
-  Copyright 2005 Roberto Raggi <roberto@kdevelop.org>
-
-  Permission to use, copy, modify, distribute, and sell this software and its
-  documentation for any purpose is hereby granted without fee, provided that
-  the above copyright notice appear in all copies and that both that
-  copyright notice and this permission notice appear in supporting
-  documentation.
-
-  The above copyright notice and this permission notice shall be included in
-  all copies or substantial portions of the Software.
-
-  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
-  KDEVELOP TEAM BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
-  AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-  CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-*/
-
-#ifndef CPLUSPLUS_PP_MACRO_EXPANDER_H
-#define CPLUSPLUS_PP_MACRO_EXPANDER_H
-
-#include "Macro.h"
-#include "pp-scanner.h"
-#include <QVector>
-#include <QByteArray>
-
-namespace CPlusPlus {
-
-class Environment;
-class Client;
-
-struct pp_frame;
-
-class MacroExpander
-{
-    Environment *env;
-    pp_frame *frame;
-    Client *client;
-    unsigned start_offset;
-
-    pp_skip_number skip_number;
-    pp_skip_identifier skip_identifier;
-    pp_skip_string_literal skip_string_literal;
-    pp_skip_char_literal skip_char_literal;
-    pp_skip_argument skip_argument;
-    pp_skip_comment_or_divop skip_comment_or_divop;
-    pp_skip_blanks skip_blanks;
-    pp_skip_whitespaces skip_whitespaces;
-
-    const QByteArray *resolve_formal(const QByteArray &name);
-
-public:
-    explicit MacroExpander(Environment *env, pp_frame *frame = 0, Client *client = 0, unsigned start_offset = 0);
-
-    const char *operator()(const char *first, const char *last,
-                             QByteArray *result);
-
-    const char *operator()(const QByteArray &source,
-                             QByteArray *result)
-    { return operator()(source.constBegin(), source.constEnd(), result); }
-
-    const char *expand(const char *first, const char *last,
-                       QByteArray *result);
-
-    const char *skip_argument_variadics(const QVector<QByteArray> &actuals,
-                                         Macro *macro,
-                                         const char *first, const char *last);
-    void pushActuals(QVector<QByteArray> & actuals, Macro *__macro, const QByteArray& expanded);
-
-public: // attributes
-    int lines;
-};
-
-} // namespace CPlusPlus
-
-#endif // CPLUSPLUS_PP_MACRO_EXPANDER_H
-
diff --git a/src/libs/cplusplus/pp.h b/src/libs/cplusplus/pp.h
index bb4d622df4..08f50ecea3 100644
--- a/src/libs/cplusplus/pp.h
+++ b/src/libs/cplusplus/pp.h
@@ -56,7 +56,6 @@
 #include "PreprocessorClient.h"
 #include "PreprocessorEnvironment.h"
 #include "pp-scanner.h"
-#include "pp-macro-expander.h"
 #include "pp-engine.h"
 
 #endif // CPLUSPLUS_PREPROCESSOR_H
diff --git a/src/plugins/cppeditor/cppcompleteswitch.cpp b/src/plugins/cppeditor/cppcompleteswitch.cpp
index c0deb7a5e2..99b92e7fa5 100644
--- a/src/plugins/cppeditor/cppcompleteswitch.cpp
+++ b/src/plugins/cppeditor/cppcompleteswitch.cpp
@@ -103,7 +103,7 @@ public:
 class Operation: public CppQuickFixOperation
 {
 public:
-    Operation(const QSharedPointer<const Internal::CppQuickFixAssistInterface> &interface,
+    Operation(const QSharedPointer<const CppEditor::Internal::CppQuickFixAssistInterface> &interface,
               int priority,
               CompoundStatementAST *compoundStatement,
               const QStringList &values)
@@ -156,7 +156,7 @@ static Enum *findEnum(const QList<LookupItem> &results,
     return 0;
 }
 
-static Enum *conditionEnum(const QSharedPointer<const Internal::CppQuickFixAssistInterface> &interface,
+static Enum *conditionEnum(const QSharedPointer<const CppEditor::Internal::CppQuickFixAssistInterface> &interface,
                            SwitchStatementAST *statement)
 {
     Block *block = statement->symbol;
diff --git a/src/plugins/cppeditor/cppfunctiondecldeflink.cpp b/src/plugins/cppeditor/cppfunctiondecldeflink.cpp
index 2534df0d97..7f3ec478d2 100644
--- a/src/plugins/cppeditor/cppfunctiondecldeflink.cpp
+++ b/src/plugins/cppeditor/cppfunctiondecldeflink.cpp
@@ -993,7 +993,7 @@ class ApplyDeclDefLinkOperation : public CppQuickFixOperation
 {
 public:
     explicit ApplyDeclDefLinkOperation(
-            const QSharedPointer<const Internal::CppQuickFixAssistInterface> &interface,
+            const QSharedPointer<const CppEditor::Internal::CppQuickFixAssistInterface> &interface,
             const QSharedPointer<FunctionDeclDefLink> &link)
         : CppQuickFixOperation(interface, 10)
         , m_link(link)
diff --git a/src/plugins/cppeditor/cppinsertdecldef.cpp b/src/plugins/cppeditor/cppinsertdecldef.cpp
index 87cb345dba..4221da624f 100644
--- a/src/plugins/cppeditor/cppinsertdecldef.cpp
+++ b/src/plugins/cppeditor/cppinsertdecldef.cpp
@@ -64,7 +64,7 @@ namespace {
 class InsertDeclOperation: public CppQuickFixOperation
 {
 public:
-    InsertDeclOperation(const QSharedPointer<const Internal::CppQuickFixAssistInterface> &interface,
+    InsertDeclOperation(const QSharedPointer<const CppEditor::Internal::CppQuickFixAssistInterface> &interface,
                         const QString &targetFileName, const Class *targetSymbol,
                         InsertionPointLocator::AccessSpec xsSpec,
                         const QString &decl)
@@ -229,7 +229,7 @@ namespace {
 class InsertDefOperation: public CppQuickFixOperation
 {
 public:
-    InsertDefOperation(const QSharedPointer<const Internal::CppQuickFixAssistInterface> &interface,
+    InsertDefOperation(const QSharedPointer<const CppEditor::Internal::CppQuickFixAssistInterface> &interface,
                        Declaration *decl, const InsertionLocation &loc)
         : CppQuickFixOperation(interface, 0)
         , m_decl(decl)
diff --git a/src/plugins/cpptools/cppcompletionassist.cpp b/src/plugins/cpptools/cppcompletionassist.cpp
index e7f41b2b73..c40b6306fc 100644
--- a/src/plugins/cpptools/cppcompletionassist.cpp
+++ b/src/plugins/cpptools/cppcompletionassist.cpp
@@ -79,7 +79,7 @@
 using namespace CPlusPlus;
 using namespace CppEditor;
 using namespace CppTools;
-using namespace Internal;
+using namespace CppTools::Internal;
 using namespace TextEditor;
 
 namespace CppTools {
diff --git a/src/plugins/cpptools/cppmodelmanager.cpp b/src/plugins/cpptools/cppmodelmanager.cpp
index f5a721adf8..8611e4f5f3 100644
--- a/src/plugins/cpptools/cppmodelmanager.cpp
+++ b/src/plugins/cpptools/cppmodelmanager.cpp
@@ -503,7 +503,7 @@ void CppPreprocessor::passedMacroDefinitionCheck(unsigned offset, const Macro &m
         return;
 
     m_currentDoc->addMacroUse(macro, offset, macro.name().length(), env.currentLine,
-                              QVector<MacroArgumentReference>(), true);
+                              QVector<MacroArgumentReference>());
 }
 
 void CppPreprocessor::failedMacroDefinitionCheck(unsigned offset, const QByteArray &name)
@@ -517,7 +517,6 @@ void CppPreprocessor::failedMacroDefinitionCheck(unsigned offset, const QByteArr
 void CppPreprocessor::startExpandingMacro(unsigned offset,
                                           const Macro &macro,
                                           const QByteArray &originalText,
-                                          bool inCondition,
                                           const QVector<MacroArgumentReference> &actuals)
 {
     if (! m_currentDoc)
@@ -525,7 +524,7 @@ void CppPreprocessor::startExpandingMacro(unsigned offset,
 
     //qDebug() << "start expanding:" << macro.name() << "text:" << originalText;
     m_currentDoc->addMacroUse(macro, offset, originalText.length(), env.currentLine,
-                              actuals, inCondition);
+                              actuals);
 }
 
 void CppPreprocessor::stopExpandingMacro(unsigned, const Macro &)
@@ -600,7 +599,9 @@ void CppPreprocessor::sourceNeeded(QString &fileName, IncludeType type, unsigned
         }
     }
 
-    //qDebug() << "parse file:" << fileName << "contents:" << contents.size();
+//    qDebug() << "parse file:" << fileName
+//             << "contents:" << contents.size()
+//                ;
 
     Document::Ptr doc = snapshot.document(fileName);
     if (doc) {
@@ -620,6 +621,8 @@ void CppPreprocessor::sourceNeeded(QString &fileName, IncludeType type, unsigned
 
     const QByteArray preprocessedCode = preprocess(fileName, contents);
 
+//    { QByteArray b(preprocessedCode); b.replace("\n", "<<<\n"); qDebug("Preprocessed code for \"%s\": [[%s]]", fileName.toUtf8().constData(), b.constData()); }
+
     doc->setUtf8Source(preprocessedCode);
     doc->keepSourceAndAST();
     doc->tokenize();
diff --git a/src/plugins/cpptools/cppmodelmanager.h b/src/plugins/cpptools/cppmodelmanager.h
index d1380556b9..d8a8e750e8 100644
--- a/src/plugins/cpptools/cppmodelmanager.h
+++ b/src/plugins/cpptools/cppmodelmanager.h
@@ -303,7 +303,6 @@ protected:
     virtual void startExpandingMacro(unsigned offset,
                                      const CPlusPlus::Macro &macro,
                                      const QByteArray &originalText,
-                                     bool inCondition,
                                      const QVector<CPlusPlus::MacroArgumentReference> &actuals);
     virtual void stopExpandingMacro(unsigned offset, const CPlusPlus::Macro &macro);
     virtual void startSkippingBlocks(unsigned offset);