WIP: Add support for generating compile_commands.json

CONFIG += create_compile_commands will generate a per-target file that
contains all command lines.

TODO: rename CONFIG+=create_compile_commands to CONFIG+=compiledb or
compilation_database to allow for a future change of format

Change-Id: I99780dea806a6e97e31bc93d3b6bb7e9f6099a9e

17 files changed, 120 insertions, 44 deletions

diff --git a/qmake/Makefile.unix b/qmake/Makefile.unix
index 0f69b6b487..90ba013f9a 100644
--- a/qmake/Makefile.unix
+++ b/qmake/Makefile.unix
@@ -25,7 +25,7 @@ QOBJS = \
 	qfilesystementry.o qfsfileengine.o qfsfileengine_iterator.o \
 	qiodevice.o qsettings.o qtemporaryfile.o qtextstream.o \
 	qjsonarray.o qjson.o qjsondocument.o qjsonobject.o qjsonparser.o qjsonvalue.o \
-	qmetatype.o qsystemerror.o qvariant.o \
+	qjsonwriter.o qmetatype.o qsystemerror.o qvariant.o \
 	quuid.o \
 	qarraydata.o qbitarray.o qbytearray.o qbytearraymatcher.o \
 	qcryptographichash.o qdatetime.o qhash.o qlinkedlist.o qlist.o \
@@ -99,6 +99,7 @@ DEPEND_SRC = \
 	   $(SOURCE_PATH)/src/corelib/serialization/qjsonarray.cpp \
 	   $(SOURCE_PATH)/src/corelib/serialization/qjson.cpp \
 	   $(SOURCE_PATH)/src/corelib/serialization/qjsondocument.cpp \
+	   $(SOURCE_PATH)/src/corelib/serialization/qjsonwriter.cpp \
 	   $(SOURCE_PATH)/src/corelib/serialization/qjsonobject.cpp \
 	   $(SOURCE_PATH)/src/corelib/serialization/qjsonparser.cpp \
 	   $(SOURCE_PATH)/src/corelib/serialization/qjsonvalue.cpp \
@@ -463,6 +464,9 @@ qjson.o: $(SOURCE_PATH)/src/corelib/serialization/qjson.cpp
 qjsondocument.o: $(SOURCE_PATH)/src/corelib/serialization/qjsondocument.cpp
 	$(CXX) -c -o $@ $(CXXFLAGS) $<
 
+qjsonwriter.o: $(SOURCE_PATH)/src/corelib/serialization/qjsonwriter.cpp
+	$(CXX) -c -o $@ $(CXXFLAGS) $<
+
 qjsonparser.o: $(SOURCE_PATH)/src/corelib/serialization/qjsonparser.cpp
 	$(CXX) -c -o $@ $(CXXFLAGS) $<
 
diff --git a/qmake/generators/mac/pbuilder_pbx.cpp b/qmake/generators/mac/pbuilder_pbx.cpp
index 5407ed6c69..23a105d212 100644
--- a/qmake/generators/mac/pbuilder_pbx.cpp
+++ b/qmake/generators/mac/pbuilder_pbx.cpp
@@ -66,7 +66,7 @@ ProjectBuilderMakefileGenerator::ProjectBuilderMakefileGenerator() : UnixMakefil
 }
 
 bool
-ProjectBuilderMakefileGenerator::writeMakefile(QTextStream &t)
+ProjectBuilderMakefileGenerator::writeMakefile(QTextStream &t, CompileCommandRecorder *)
 {
     writingUnixMakefileGenerator = false;
     if(!project->values("QMAKE_FAILED_REQUIREMENTS").isEmpty()) {
diff --git a/qmake/generators/mac/pbuilder_pbx.h b/qmake/generators/mac/pbuilder_pbx.h
index f15c814cb4..5dc1b90676 100644
--- a/qmake/generators/mac/pbuilder_pbx.h
+++ b/qmake/generators/mac/pbuilder_pbx.h
@@ -40,7 +40,7 @@ class ProjectBuilderMakefileGenerator : public UnixMakefileGenerator
     int pbuilderVersion() const;
     bool writeSubDirs(QTextStream &);
     bool writeMakeParts(QTextStream &);
-    bool writeMakefile(QTextStream &) override;
+    bool writeMakefile(QTextStream &, CompileCommandRecorder *) override;
 
     QString pbxbuild();
     QHash<QString, QString> keys;
diff --git a/qmake/generators/makefile.cpp b/qmake/generators/makefile.cpp
index 7364353183..2918a2ee58 100644
--- a/qmake/generators/makefile.cpp
+++ b/qmake/generators/makefile.cpp
@@ -41,6 +41,9 @@
 #include <qdebug.h>
 #include <qbuffer.h>
 #include <qdatetime.h>
+#include <qjsondocument.h>
+#include <qjsonobject.h>
+#include <qjsonvalue.h>
 
 #if defined(Q_OS_UNIX)
 #include <unistd.h>
@@ -1087,7 +1090,12 @@ MakefileGenerator::write()
     if(Option::qmake_mode == Option::QMAKE_GENERATE_MAKEFILE || //write makefile
        Option::qmake_mode == Option::QMAKE_GENERATE_PROJECT) {
         QTextStream t(&Option::output);
-        if(!writeMakefile(t)) {
+
+        QScopedPointer<CompileCommandRecorder> compileCommands;
+        if (project->isActiveConfig("create_compile_commands"))
+            compileCommands.reset(new CompileCommandRecorder);
+
+        if(!writeMakefile(t, compileCommands.data())) {
 #if 1
             warn_msg(WarnLogic, "Unable to generate output for: %s [TEMPLATE %s]",
                      Option::output.fileName().toLatin1().constData(),
@@ -1096,6 +1104,24 @@ MakefileGenerator::write()
                 Option::output.remove();
 #endif
         }
+
+        if (compileCommands) {
+            QFileInfo outputMakeFileInfo(fileInfo(Option::output.fileName()));
+            QString compileCommandsFileName = "compile_commands";
+            QString infix = outputMakeFileInfo.completeSuffix();
+            if (!infix.isEmpty()) {
+                compileCommandsFileName += '_';
+                compileCommandsFileName += infix;
+            }
+            compileCommandsFileName += ".json";
+            QFile f(compileCommandsFileName);
+            if (f.open(QIODevice::WriteOnly | QIODevice::Truncate)) {
+                f.write(QJsonDocument(compileCommands->commands).toJson());
+            } else {
+                warn_msg(WarnLogic, "Unable to generate compile commands output %s",
+                         compileCommandsFileName.toLatin1().constData());
+            }
+        }
     }
     return true;
 }
@@ -1137,8 +1163,10 @@ MakefileGenerator::writePrlFile()
 }
 
 void
-MakefileGenerator::writeObj(QTextStream &t, const char *src)
+MakefileGenerator::writeObj(QTextStream &t, CompileCommandRecorder *compileCommands, const char *src)
 {
+    static QRegExp compileCommandVariable("\\$\\((\\w+)\\)");
+
     const ProStringList &srcl = project->values(src);
     const ProStringList objl = createObjectList(srcl);
 
@@ -1185,6 +1213,22 @@ MakefileGenerator::writeObj(QTextStream &t, const char *src)
             p.replace(stringSrc, escapeFilePath(srcf));
             p.replace(stringObj, escapeFilePath(dstf));
             t << "\n\t" << p;
+            if (compileCommands) {
+                QJsonObject command;
+                command["directory"] = Option::output_dir;
+                command["file"] = srcf;
+
+                QString expandedCommand = p;
+                int pos = 0;
+                while ((pos = compileCommandVariable.indexIn(expandedCommand, 0)) != -1) {
+                    QString variableName = compileCommandVariable.cap(1);
+                    QString expansion = compileCommands->compilerVariables.value(variableName);
+                    expandedCommand.replace(pos, compileCommandVariable.matchedLength(), expansion);
+                }
+
+                command["command"] = expandedCommand;
+                compileCommands->commands.append(command);
+            }
         }
         t << endl << endl;
     }
@@ -2277,11 +2321,11 @@ MakefileGenerator::writeStubMakefile(QTextStream &t)
 }
 
 bool
-MakefileGenerator::writeMakefile(QTextStream &t)
+MakefileGenerator::writeMakefile(QTextStream &t, CompileCommandRecorder *compileCommands)
 {
     t << "####### Compile\n\n";
-    writeObj(t, "SOURCES");
-    writeObj(t, "GENERATED_SOURCES");
+    writeObj(t, compileCommands, "SOURCES");
+    writeObj(t, compileCommands, "GENERATED_SOURCES");
 
     t << "####### Install\n\n";
     writeInstalls(t);
diff --git a/qmake/generators/makefile.h b/qmake/generators/makefile.h
index 350ebd377a..f81fc3cb54 100644
--- a/qmake/generators/makefile.h
+++ b/qmake/generators/makefile.h
@@ -36,9 +36,19 @@
 #include <qlist.h>
 #include <qhash.h>
 #include <qfileinfo.h>
+#include <qjsonarray.h>
 
 QT_BEGIN_NAMESPACE
 
+struct CompileCommandRecorder
+{
+    QJsonArray commands;
+    // QMAKE_RUN_CC etc. command lines may contain references to variables such as $(DEFINES). Their
+    // values are coded in the makefile and also supplied here, for expansion before writing
+    // compile_commands.json.
+    QHash<QString, QString> compilerVariables;
+};
+
 #ifdef Q_OS_WIN32
 #define QT_POPEN _popen
 #define QT_POPEN_READ "rb"
@@ -75,7 +85,7 @@ protected:
     ProStringList createObjectList(const ProStringList &sources);
 
     //makefile style generator functions
-    void writeObj(QTextStream &, const char *src);
+    void writeObj(QTextStream &, CompileCommandRecorder *compileCommands, const char *src);
     void writeInstalls(QTextStream &t, bool noBuild=false);
     void writeHeader(QTextStream &t);
     void writeSubDirs(QTextStream &t);
@@ -88,7 +98,7 @@ protected:
     void writeExtraCompilerVariables(QTextStream &t);
     bool writeDummyMakefile(QTextStream &t);
     virtual bool writeStubMakefile(QTextStream &t);
-    virtual bool writeMakefile(QTextStream &t);
+    virtual bool writeMakefile(QTextStream &t, CompileCommandRecorder *compileCommands);
     virtual void writeDefaultVariables(QTextStream &t);
 
     QString pkgConfigPrefix() const;
diff --git a/qmake/generators/projectgenerator.cpp b/qmake/generators/projectgenerator.cpp
index ef34955eb1..06092003e8 100644
--- a/qmake/generators/projectgenerator.cpp
+++ b/qmake/generators/projectgenerator.cpp
@@ -322,7 +322,7 @@ ProjectGenerator::init()
 }
 
 bool
-ProjectGenerator::writeMakefile(QTextStream &t)
+ProjectGenerator::writeMakefile(QTextStream &t, CompileCommandRecorder *)
 {
     t << "######################################################################" << endl;
     t << "# Automatically generated by qmake (" QMAKE_VERSION_STR ") " << QDateTime::currentDateTime().toString() << endl;
diff --git a/qmake/generators/projectgenerator.h b/qmake/generators/projectgenerator.h
index cbc9f371ab..1792d249d6 100644
--- a/qmake/generators/projectgenerator.h
+++ b/qmake/generators/projectgenerator.h
@@ -41,7 +41,7 @@ class ProjectGenerator : public MakefileGenerator
     QString fixPathToQmake(const QString &file);
 protected:
     void init() override;
-    bool writeMakefile(QTextStream &) override;
+    bool writeMakefile(QTextStream &, CompileCommandRecorder *) override;
 
     QString escapeFilePath(const QString &path) const override { Q_ASSERT(false); return QString(); }
 
diff --git a/qmake/generators/unix/unixmake.h b/qmake/generators/unix/unixmake.h
index 5b0766855b..e6a06c65b4 100644
--- a/qmake/generators/unix/unixmake.h
+++ b/qmake/generators/unix/unixmake.h
@@ -61,8 +61,8 @@ protected:
 
     void writeDefaultVariables(QTextStream &t) override;
     void writeSubTargets(QTextStream &t, QList<SubTarget*> subtargets, int flags) override;
-    void writeMakeParts(QTextStream &);
-    bool writeMakefile(QTextStream &) override;
+    void writeMakeParts(QTextStream &, CompileCommandRecorder *compileCommands);
+    bool writeMakefile(QTextStream &, CompileCommandRecorder *compileCommands) override;
 
 private:
     void init2();
diff --git a/qmake/generators/unix/unixmake2.cpp b/qmake/generators/unix/unixmake2.cpp
index fa283d0291..5caa7ac090 100644
--- a/qmake/generators/unix/unixmake2.cpp
+++ b/qmake/generators/unix/unixmake2.cpp
@@ -58,7 +58,7 @@ UnixMakefileGenerator::writePrlFile(QTextStream &t)
 }
 
 bool
-UnixMakefileGenerator::writeMakefile(QTextStream &t)
+UnixMakefileGenerator::writeMakefile(QTextStream &t, CompileCommandRecorder *compileCommands)
 {
 
     writeHeader(t);
@@ -70,8 +70,8 @@ UnixMakefileGenerator::writeMakefile(QTextStream &t)
         project->first("TEMPLATE") == "aux") {
         if(Option::mkfile::do_stub_makefile && MakefileGenerator::writeStubMakefile(t))
             return true;
-        writeMakeParts(t);
-        return MakefileGenerator::writeMakefile(t);
+        writeMakeParts(t, compileCommands);
+        return MakefileGenerator::writeMakefile(t, compileCommands);
     } else if (project->first("TEMPLATE") == "subdirs") {
         MakefileGenerator::writeSubDirs(t);
         return true;
@@ -170,7 +170,7 @@ static QString rfc1034Identifier(const QString &str)
 }
 
 void
-UnixMakefileGenerator::writeMakeParts(QTextStream &t)
+UnixMakefileGenerator::writeMakeParts(QTextStream &t, CompileCommandRecorder *compileCommands)
 {
     bool do_incremental = (project->isActiveConfig("incremental") &&
                            !project->values("QMAKE_INCREMENTAL").isEmpty() &&
@@ -183,14 +183,17 @@ UnixMakefileGenerator::writeMakeParts(QTextStream &t)
     writeExportedVariables(t);
 
     t << "####### Compiler, tools and options\n\n";
-    t << "CC            = " << var("QMAKE_CC") << endl;
-    t << "CXX           = " << var("QMAKE_CXX") << endl;
-    t << "DEFINES       = "
-      << varGlue("PRL_EXPORT_DEFINES","-D"," -D"," ")
-      << varGlue("DEFINES","-D"," -D","") << endl;
-    t << "CFLAGS        = " << var("QMAKE_CFLAGS") << " $(DEFINES)\n";
-    t << "CXXFLAGS      = " << var("QMAKE_CXXFLAGS") << " $(DEFINES)\n";
-    t << "INCPATH       =";
+    const ProString cc = var("QMAKE_CC");
+    t << "CC            = " << cc << endl;
+    const ProString cxx = var("QMAKE_CXX");
+    t << "CXX           = " << cxx << endl;
+    const ProString defines = varGlue("PRL_EXPORT_DEFINES","-D"," -D"," ") + varGlue("DEFINES","-D"," -D","");
+    t << "DEFINES       = " << defines << endl;
+    const ProString cFlags = var("QMAKE_CFLAGS") + " $(DEFINES)";
+    t << "CFLAGS        = " << cFlags << endl;
+    const ProString cxxFlags = var("QMAKE_CXXFLAGS") + " $(DEFINES)";
+    t << "CXXFLAGS      = " << cxxFlags << endl;
+    ProString incPath;
     {
         QString isystem = var("QMAKE_CFLAGS_ISYSTEM");
         const ProStringList &incs = project->values("INCLUDEPATH");
@@ -199,16 +202,30 @@ UnixMakefileGenerator::writeMakeParts(QTextStream &t)
             if (inc.isEmpty())
                 continue;
 
-            if (!isystem.isEmpty() && isSystemInclude(inc.toQString()))
-                t << ' ' << isystem << ' ';
-            else
-                t << " -I";
-            t << escapeFilePath(inc);
+            if (!isystem.isEmpty() && isSystemInclude(inc.toQString())) {
+                incPath += ' ';
+                incPath += isystem;
+                incPath += ' ';
+            } else {
+                incPath += " -I";
+            }
+            incPath += escapeFilePath(inc);
         }
     }
-    if(!project->isEmpty("QMAKE_FRAMEWORKPATH_FLAGS"))
-       t << " " << var("QMAKE_FRAMEWORKPATH_FLAGS");
-    t << endl;
+    if(!project->isEmpty("QMAKE_FRAMEWORKPATH_FLAGS")) {
+       incPath += ' ';
+       incPath += var("QMAKE_FRAMEWORKPATH_FLAGS");
+    }
+    t << "INCPATH       =" << incPath << endl;
+
+    if (compileCommands) {
+        compileCommands->compilerVariables["CC"] = cc.toQString();
+        compileCommands->compilerVariables["CXX"] = cxx.toQString();
+        compileCommands->compilerVariables["DEFINES"] = defines.toQString();
+        compileCommands->compilerVariables["CFLAGS"] = cFlags.toQString();
+        compileCommands->compilerVariables["CXXFLAGS"] = cxxFlags.toQString();
+        compileCommands->compilerVariables["INCPATH"] = incPath.toQString();
+    }
 
     writeDefaultVariables(t);
 
diff --git a/qmake/generators/win32/mingw_make.cpp b/qmake/generators/win32/mingw_make.cpp
index de7363e51b..93805d21c5 100644
--- a/qmake/generators/win32/mingw_make.cpp
+++ b/qmake/generators/win32/mingw_make.cpp
@@ -86,7 +86,7 @@ bool MingwMakefileGenerator::processPrlFileBase(QString &origFile, const QString
     return Win32MakefileGenerator::processPrlFileBase(origFile, origName, fixedBase, slashOff);
 }
 
-bool MingwMakefileGenerator::writeMakefile(QTextStream &t)
+bool MingwMakefileGenerator::writeMakefile(QTextStream &t, CompileCommandRecorder *compileCommands)
 {
     writeHeader(t);
     if (writeDummyMakefile(t))
@@ -110,7 +110,7 @@ bool MingwMakefileGenerator::writeMakefile(QTextStream &t)
             return true;
         }
         writeMingwParts(t);
-        return MakefileGenerator::writeMakefile(t);
+        return MakefileGenerator::writeMakefile(t, compileCommands);
     }
     else if(project->first("TEMPLATE") == "subdirs") {
         writeSubDirs(t);
diff --git a/qmake/generators/win32/mingw_make.h b/qmake/generators/win32/mingw_make.h
index 5da5b24088..6b1a1830b5 100644
--- a/qmake/generators/win32/mingw_make.h
+++ b/qmake/generators/win32/mingw_make.h
@@ -45,7 +45,7 @@ protected:
     bool processPrlFileBase(QString &origFile, const QStringRef &origName,
                             const QStringRef &fixedBase, int slashOff) override;
     QString getManifestFileForRcFile() const override;
-    bool writeMakefile(QTextStream &) override;
+    bool writeMakefile(QTextStream &, CompileCommandRecorder *) override;
     void init() override;
     QString installRoot() const override;
 private:
diff --git a/qmake/generators/win32/msvc_nmake.cpp b/qmake/generators/win32/msvc_nmake.cpp
index a6e850a036..0814ac22c2 100644
--- a/qmake/generators/win32/msvc_nmake.cpp
+++ b/qmake/generators/win32/msvc_nmake.cpp
@@ -44,7 +44,7 @@ NmakeMakefileGenerator::NmakeMakefileGenerator() : usePCH(false), usePCHC(false)
 }
 
 bool
-NmakeMakefileGenerator::writeMakefile(QTextStream &t)
+NmakeMakefileGenerator::writeMakefile(QTextStream &t, CompileCommandRecorder *compileCommands)
 {
     writeHeader(t);
     if (writeDummyMakefile(t))
@@ -58,7 +58,7 @@ NmakeMakefileGenerator::writeMakefile(QTextStream &t)
             return MakefileGenerator::writeStubMakefile(t);
 #endif
         writeNmakeParts(t);
-        return MakefileGenerator::writeMakefile(t);
+        return MakefileGenerator::writeMakefile(t, compileCommands);
     }
     else if(project->first("TEMPLATE") == "subdirs") {
         writeSubDirs(t);
diff --git a/qmake/generators/win32/msvc_nmake.h b/qmake/generators/win32/msvc_nmake.h
index 5bfdba2bbc..f887623413 100644
--- a/qmake/generators/win32/msvc_nmake.h
+++ b/qmake/generators/win32/msvc_nmake.h
@@ -36,7 +36,7 @@ QT_BEGIN_NAMESPACE
 class NmakeMakefileGenerator : public Win32MakefileGenerator
 {
     void writeNmakeParts(QTextStream &);
-    bool writeMakefile(QTextStream &) override;
+    bool writeMakefile(QTextStream &, CompileCommandRecorder *) override;
     void writeImplicitRulesPart(QTextStream &t) override;
     void writeBuildRulesPart(QTextStream &t) override;
     void writeLinkCommand(QTextStream &t, const QString &extraFlags = QString(), const QString &extraInlineFileContent = QString());
diff --git a/qmake/generators/win32/msvc_vcproj.cpp b/qmake/generators/win32/msvc_vcproj.cpp
index 06a96f7538..67c2ea0a38 100644
--- a/qmake/generators/win32/msvc_vcproj.cpp
+++ b/qmake/generators/win32/msvc_vcproj.cpp
@@ -123,7 +123,7 @@ VcprojGenerator::~VcprojGenerator()
     delete projectWriter;
 }
 
-bool VcprojGenerator::writeMakefile(QTextStream &t)
+bool VcprojGenerator::writeMakefile(QTextStream &t, CompileCommandRecorder *)
 {
     initProject(); // Fills the whole project with proper data
 
@@ -202,7 +202,7 @@ bool VcprojGenerator::writeProjectMakefile()
         projectWriter->write(xmlOut, mergedProject);
         return true;
     } else if(project->first("TEMPLATE") == "vcsubdirs") {
-        return writeMakefile(t);
+        return writeMakefile(t, /*compile commands*/nullptr);
     }
     return false;
 }
diff --git a/qmake/generators/win32/msvc_vcproj.h b/qmake/generators/win32/msvc_vcproj.h
index 55d36c3762..fc10b1dc6a 100644
--- a/qmake/generators/win32/msvc_vcproj.h
+++ b/qmake/generators/win32/msvc_vcproj.h
@@ -47,7 +47,7 @@ class VcprojGenerator : public Win32MakefileGenerator
     bool is64Bit;
     bool writeVcprojParts(QTextStream &);
 
-    bool writeMakefile(QTextStream &) override;
+    bool writeMakefile(QTextStream &, CompileCommandRecorder *) override;
     bool writeProjectMakefile() override;
 
     void init() override;
diff --git a/qmake/qmake.pro b/qmake/qmake.pro
index 6a6116c8db..9b73b9da31 100644
--- a/qmake/qmake.pro
+++ b/qmake/qmake.pro
@@ -132,6 +132,7 @@ SOURCES += \
     qjson.cpp \
     qjsonarray.cpp \
     qjsondocument.cpp \
+    qjsonwriter.cpp \
     qjsonobject.cpp \
     qjsonparser.cpp \
     qjsonvalue.cpp \
@@ -185,6 +186,7 @@ HEADERS += \
     qjson_p.h \
     qjsonarray.h \
     qjsondocument.h \
+    qjsonwriter_p.h \
     qjsonobject.h \
     qjsonparser_p.h \
     qjsonvalue.h \
diff --git a/src/corelib/global/qconfig-bootstrapped.h b/src/corelib/global/qconfig-bootstrapped.h
index 10458e41d7..80d33271cb 100644
--- a/src/corelib/global/qconfig-bootstrapped.h
+++ b/src/corelib/global/qconfig-bootstrapped.h
@@ -130,7 +130,6 @@
 #ifdef QT_BUILD_QMAKE
 #define QT_FEATURE_commandlineparser -1
 #define QT_NO_COMPRESS
-#define QT_JSON_READONLY
 #define QT_FEATURE_settings 1
 #define QT_NO_STANDARDPATHS
 #define QT_FEATURE_textcodec -1