C++: Add utf16 indices to Macro and Document::MacroUse

In most cases we need to work with the utf16 indices. Only in
cppfindreferences the byte interface is still needed since there we read
in files and work on a QByteArray to save memory.

Change-Id: I6ef6a93fc1875a8c9a305c075d51a9ca034c41bb
Reviewed-by: Erik Verbruggen <erik.verbruggen@digia.com>

7 files changed, 74 insertions, 54 deletions

diff --git a/src/plugins/cpptools/cppcodemodelinspectordumper.cpp b/src/plugins/cpptools/cppcodemodelinspectordumper.cpp
index 2aef1c21cf..680944b677 100644
--- a/src/plugins/cpptools/cppcodemodelinspectordumper.cpp
+++ b/src/plugins/cpptools/cppcodemodelinspectordumper.cpp
@@ -660,8 +660,9 @@ void Dumper::dumpDocuments(const QList<CPlusPlus::Document::Ptr> &documents, boo
                 const QString type = use.isFunctionLike()
                         ? QLatin1String("function-like") : QLatin1String("object-like");
                 m_out << i4 << "at line " << use.beginLine() << ", "
-                      << QString::fromUtf8(use.macro().name()) << ", begin=" << use.begin()
-                      << ", end=" << use.end() << ", " << type << ", args="
+                      << use.macro().nameToQString().size()
+                      << ", begin=" << use.utf16charsBegin() << ", end=" << use.utf16charsEnd()
+                      << ", " << type << ", args="
                       << use.arguments().size() << "\n";
             }
         }
diff --git a/src/plugins/cpptools/cppcompletionassist.cpp b/src/plugins/cpptools/cppcompletionassist.cpp
index 77a25b1d5c..29ddb68223 100644
--- a/src/plugins/cpptools/cppcompletionassist.cpp
+++ b/src/plugins/cpptools/cppcompletionassist.cpp
@@ -1751,7 +1751,7 @@ void CppCompletionAssistProcessor::addMacros_helper(const CPlusPlus::Snapshot &s
         addMacros_helper(snapshot, i.resolvedFileName(), processed, definedMacros);
 
     foreach (const Macro &macro, doc->definedMacros()) {
-        const QString macroName = QString::fromUtf8(macro.name().constData(), macro.name().length());
+        const QString macroName = macro.nameToQString();
         if (!macro.isHidden())
             definedMacros->insert(macroName);
         else
diff --git a/src/plugins/cpptools/cppfindreferences.cpp b/src/plugins/cpptools/cppfindreferences.cpp
index 6f353cb520..63b3b6a587 100644
--- a/src/plugins/cpptools/cppfindreferences.cpp
+++ b/src/plugins/cpptools/cppfindreferences.cpp
@@ -572,10 +572,10 @@ restart_search:
                 }
 
                 if (macro.name() == useMacro.name()) {
-                    unsigned lineStart;
-                    const QByteArray &lineSource = matchingLine(use.begin(), source, &lineStart);
-                    usages.append(Usage(fileName, QString::fromUtf8(lineSource), use.beginLine(),
-                                        use.begin() - lineStart, useMacro.name().length()));
+                    unsigned column;
+                    const QString &lineSource = matchingLine(use.bytesBegin(), source, &column);
+                    usages.append(Usage(fileName, lineSource, use.beginLine(), column,
+                                        useMacro.nameToQString().size()));
                 }
             }
         }
@@ -585,19 +585,26 @@ restart_search:
         return usages;
     }
 
-    static QByteArray matchingLine(unsigned position, const QByteArray &source,
-                                   unsigned *lineStart = 0)
+    static QString matchingLine(unsigned bytesOffsetOfUseStart, const QByteArray &utf8Source,
+                                unsigned *columnOfUseStart = 0)
     {
-        int lineBegin = source.lastIndexOf('\n', position) + 1;
-        int lineEnd = source.indexOf('\n', position);
+        int lineBegin = utf8Source.lastIndexOf('\n', bytesOffsetOfUseStart) + 1;
+        int lineEnd = utf8Source.indexOf('\n', bytesOffsetOfUseStart);
         if (lineEnd == -1)
-            lineEnd = source.length();
-
-        if (lineStart)
-            *lineStart = lineBegin;
+            lineEnd = utf8Source.length();
+
+        if (columnOfUseStart) {
+            *columnOfUseStart = 0;
+            const char *startOfUse = utf8Source.constData() + bytesOffsetOfUseStart;
+            QTC_ASSERT(startOfUse < utf8Source.constData() + lineEnd, return QString());
+            const char *currentSourceByte = utf8Source.constData() + lineBegin;
+            unsigned char yychar = *currentSourceByte;
+            while (currentSourceByte != startOfUse)
+                Lexer::yyinp_utf8(currentSourceByte, yychar, *columnOfUseStart);
+        }
 
-        const QByteArray matchingLine = source.mid(lineBegin, lineEnd - lineBegin);
-        return matchingLine;
+        const QByteArray matchingLine = utf8Source.mid(lineBegin, lineEnd - lineBegin);
+        return QString::fromUtf8(matchingLine, matchingLine.size());
     }
 };
 
@@ -638,7 +645,7 @@ void CppFindReferences::findMacroUses(const Macro &macro, const QString &replace
     Core::SearchResult *search = Core::SearchResultWindow::instance()->startNewSearch(
                 tr("C++ Macro Usages:"),
                 QString(),
-                QString::fromUtf8(macro.name()),
+                macro.nameToQString(),
                 replace ? Core::SearchResultWindow::SearchAndReplace
                         : Core::SearchResultWindow::SearchOnly,
                 Core::SearchResultWindow::PreserveCaseDisabled,
@@ -661,11 +668,11 @@ void CppFindReferences::findMacroUses(const Macro &macro, const QString &replace
     // add the macro definition itself
     {
         const QByteArray &source = getSource(macro.fileName(), workingCopy);
-        unsigned lineStart;
-        const QByteArray line = FindMacroUsesInFile::matchingLine(macro.offset(), source,
-                                                                  &lineStart);
-        search->addResult(macro.fileName(), macro.line(), QString::fromUtf8(line),
-                          macro.offset() - lineStart, macro.name().length());
+        unsigned column;
+        const QString line = FindMacroUsesInFile::matchingLine(macro.bytesOffset(), source,
+                                                               &column);
+        search->addResult(macro.fileName(), macro.line(), line, column,
+                          macro.nameToQString().length());
     }
 
     QFuture<Usage> result;
@@ -679,7 +686,7 @@ void CppFindReferences::findMacroUses(const Macro &macro, const QString &replace
 
 void CppFindReferences::renameMacroUses(const Macro &macro, const QString &replacement)
 {
-    const QString textToReplace = replacement.isEmpty() ? QString::fromUtf8(macro.name()) : replacement;
+    const QString textToReplace = replacement.isEmpty() ? macro.nameToQString() : replacement;
     findMacroUses(macro, textToReplace, true);
 }
 
diff --git a/src/plugins/cpptools/cpphighlightingsupportinternal.cpp b/src/plugins/cpptools/cpphighlightingsupportinternal.cpp
index 0fe3351d94..29bf316082 100644
--- a/src/plugins/cpptools/cpphighlightingsupportinternal.cpp
+++ b/src/plugins/cpptools/cpphighlightingsupportinternal.cpp
@@ -59,15 +59,15 @@ QFuture<TextEditor::HighlightingResult> CppHighlightingSupportInternal::highligh
     // Get macro definitions
     foreach (const CPlusPlus::Macro& macro, doc->definedMacros()) {
         int line, column;
-        editor()->convertPosition(macro.offset(), &line, &column);
+        editor()->convertPosition(macro.utf16CharOffset(), &line, &column);
         ++column; //Highlighting starts at (column-1) --> compensate here
-        Result use(line, column, macro.name().size(), MacroUse);
+        Result use(line, column, macro.nameToQString().size(), MacroUse);
         macroUses.append(use);
     }
 
     // Get macro uses
     foreach (const Document::MacroUse &macro, doc->macroUses()) {
-        const QString name = QString::fromUtf8(macro.macro().name());
+        const QString name = macro.macro().nameToQString();
 
         //Filter out QtKeywords
         if (isQtKeyword(QStringRef(&name)))
@@ -86,7 +86,7 @@ QFuture<TextEditor::HighlightingResult> CppHighlightingSupportInternal::highligh
             continue;
 
         int line, column;
-        editor()->convertPosition(macro.begin(), &line, &column);
+        editor()->convertPosition(macro.utf16charsBegin(), &line, &column);
         ++column; //Highlighting starts at (column-1) --> compensate here
         Result use(line, column, name.size(), MacroUse);
         macroUses.append(use);
diff --git a/src/plugins/cpptools/cpppreprocessor.cpp b/src/plugins/cpptools/cpppreprocessor.cpp
index c6dfc5012b..4e809989ad 100644
--- a/src/plugins/cpptools/cpppreprocessor.cpp
+++ b/src/plugins/cpptools/cpppreprocessor.cpp
@@ -291,41 +291,51 @@ static inline const Macro revision(const CppModelManagerInterface::WorkingCopy &
     return newMacro;
 }
 
-void CppPreprocessor::passedMacroDefinitionCheck(unsigned offset, unsigned line, const Macro &macro)
+void CppPreprocessor::passedMacroDefinitionCheck(unsigned bytesOffset, unsigned utf16charsOffset,
+                                                 unsigned line, const Macro &macro)
 {
     if (!m_currentDoc)
         return;
 
-    m_currentDoc->addMacroUse(revision(m_workingCopy, macro), offset, macro.name().length(), line,
-                              QVector<MacroArgumentReference>());
+    m_currentDoc->addMacroUse(revision(m_workingCopy, macro),
+                              bytesOffset, macro.name().length(),
+                              utf16charsOffset, macro.nameToQString().size(),
+                              line, QVector<MacroArgumentReference>());
 }
 
-void CppPreprocessor::failedMacroDefinitionCheck(unsigned offset, const ByteArrayRef &name)
+void CppPreprocessor::failedMacroDefinitionCheck(unsigned bytesOffset, unsigned utf16charOffset,
+                                                 const ByteArrayRef &name)
 {
     if (!m_currentDoc)
         return;
 
-    m_currentDoc->addUndefinedMacroUse(QByteArray(name.start(), name.size()), offset);
+    m_currentDoc->addUndefinedMacroUse(QByteArray(name.start(), name.size()),
+                                       bytesOffset, utf16charOffset);
 }
 
-void CppPreprocessor::notifyMacroReference(unsigned offset, unsigned line, const Macro &macro)
+void CppPreprocessor::notifyMacroReference(unsigned bytesOffset, unsigned utf16charOffset,
+                                           unsigned line, const Macro &macro)
 {
     if (!m_currentDoc)
         return;
 
-    m_currentDoc->addMacroUse(revision(m_workingCopy, macro), offset, macro.name().length(), line,
-                              QVector<MacroArgumentReference>());
+    m_currentDoc->addMacroUse(revision(m_workingCopy, macro),
+                              bytesOffset, macro.name().length(),
+                              utf16charOffset, macro.nameToQString().size(),
+                              line, QVector<MacroArgumentReference>());
 }
 
-void CppPreprocessor::startExpandingMacro(unsigned offset, unsigned line,
-                                          const Macro &macro,
+void CppPreprocessor::startExpandingMacro(unsigned bytesOffset, unsigned utf16charOffset,
+                                          unsigned line, const Macro &macro,
                                           const QVector<MacroArgumentReference> &actuals)
 {
     if (!m_currentDoc)
         return;
 
-    m_currentDoc->addMacroUse(revision(m_workingCopy, macro), offset, macro.name().length(), line,
-                              actuals);
+    m_currentDoc->addMacroUse(revision(m_workingCopy, macro),
+                              bytesOffset, macro.name().length(),
+                              utf16charOffset, macro.nameToQString().size(),
+                              line, actuals);
 }
 
 void CppPreprocessor::stopExpandingMacro(unsigned, const Macro &)
@@ -366,16 +376,16 @@ void CppPreprocessor::mergeEnvironment(Document::Ptr doc)
     m_env.addMacros(doc->definedMacros());
 }
 
-void CppPreprocessor::startSkippingBlocks(unsigned offset)
+void CppPreprocessor::startSkippingBlocks(unsigned utf16charsOffset)
 {
     if (m_currentDoc)
-        m_currentDoc->startSkippingBlocks(offset);
+        m_currentDoc->startSkippingBlocks(utf16charsOffset);
 }
 
-void CppPreprocessor::stopSkippingBlocks(unsigned offset)
+void CppPreprocessor::stopSkippingBlocks(unsigned utf16charsOffset)
 {
     if (m_currentDoc)
-        m_currentDoc->stopSkippingBlocks(offset);
+        m_currentDoc->stopSkippingBlocks(utf16charsOffset);
 }
 
 void CppPreprocessor::sourceNeeded(unsigned line, const QString &fileName, IncludeType type)
diff --git a/src/plugins/cpptools/cpppreprocessor.h b/src/plugins/cpptools/cpppreprocessor.h
index f8b076be8a..a250074f94 100644
--- a/src/plugins/cpptools/cpppreprocessor.h
+++ b/src/plugins/cpptools/cpppreprocessor.h
@@ -64,19 +64,21 @@ protected:
     void mergeEnvironment(CPlusPlus::Document::Ptr doc);
 
     virtual void macroAdded(const CPlusPlus::Macro &macro);
-    virtual void passedMacroDefinitionCheck(unsigned offset, unsigned line,
-                                            const CPlusPlus::Macro &macro);
-    virtual void failedMacroDefinitionCheck(unsigned offset, const CPlusPlus::ByteArrayRef &name);
-    virtual void notifyMacroReference(unsigned offset, unsigned line,
-                                      const CPlusPlus::Macro &macro);
-    virtual void startExpandingMacro(unsigned offset,
+    virtual void passedMacroDefinitionCheck(unsigned bytesOffset, unsigned utf16charsOffset,
+                                            unsigned line, const CPlusPlus::Macro &macro);
+    virtual void failedMacroDefinitionCheck(unsigned bytesOffset, unsigned utf16charOffset,
+                                            const CPlusPlus::ByteArrayRef &name);
+    virtual void notifyMacroReference(unsigned bytesOffset, unsigned utf16charOffset,
+                                      unsigned line, const CPlusPlus::Macro &macro);
+    virtual void startExpandingMacro(unsigned bytesOffset,
+                                     unsigned utf16charOffset,
                                      unsigned line,
                                      const CPlusPlus::Macro &macro,
                                      const QVector<CPlusPlus::MacroArgumentReference> &actuals);
-    virtual void stopExpandingMacro(unsigned offset, const CPlusPlus::Macro &macro);
+    virtual void stopExpandingMacro(unsigned bytesOffset, const CPlusPlus::Macro &macro);
     virtual void markAsIncludeGuard(const QByteArray &macroName);
-    virtual void startSkippingBlocks(unsigned offset);
-    virtual void stopSkippingBlocks(unsigned offset);
+    virtual void startSkippingBlocks(unsigned utf16charsOffset);
+    virtual void stopSkippingBlocks(unsigned utf16charsOffset);
     virtual void sourceNeeded(unsigned line, const QString &fileName, IncludeType type);
 
 private:
diff --git a/src/plugins/cpptools/cpptoolseditorsupport.cpp b/src/plugins/cpptools/cpptoolseditorsupport.cpp
index 1f18b1db31..3866f774f5 100644
--- a/src/plugins/cpptools/cpptoolseditorsupport.cpp
+++ b/src/plugins/cpptools/cpptoolseditorsupport.cpp
@@ -333,7 +333,7 @@ void CppEditorSupport::onDocumentUpdated(Document::Ptr doc)
         QList<BlockRange> ifdefedOutBlocks;
         ifdefedOutBlocks.reserve(skippedBlocks.size());
         foreach (const Document::Block &block, skippedBlocks)
-            ifdefedOutBlocks.append(BlockRange(block.begin(), block.end()));
+            ifdefedOutBlocks.append(BlockRange(block.utf16charsBegin(), block.utf16charsEnd()));
         setIfdefedOutBlocks(ifdefedOutBlocks);
     }