Diagnose object library self-reference

The code

    add_library(A OBJECT a.c)
    target_sources(A PRIVATE $<TARGET_OBJECTS:A>)

used to crash CMake via infinite recursion while evaluating the
generator expression.  Then the change in commit v3.9.0-rc1~266^2~1
(cmGeneratorTarget: Replace source classifier implementation,
2017-04-07) avoided the infinite recursion because GetKindedSources now
creates a map entry and initializes it once.  If it is called again on
the same target during that initialization, the partially computed
results are returned.  This is still wrong but does not crash.
Detect and diagnose this case instead.

Co-Author: Ben Boeckel <ben.boeckel@kitware.com>
Fixes: #16578

6 files changed, 25 insertions, 0 deletions

diff --git a/Source/cmGeneratorTarget.cxx b/Source/cmGeneratorTarget.cxx
index b93442dc05..e8ecac4493 100644
--- a/Source/cmGeneratorTarget.cxx
+++ b/Source/cmGeneratorTarget.cxx
@@ -1025,12 +1025,23 @@ cmGeneratorTarget::KindedSources const& cmGeneratorTarget::GetKindedSources(
   std::string const key = cmSystemTools::UpperCase(config);
   KindedSourcesMapType::iterator it = this->KindedSourcesMap.find(key);
   if (it != this->KindedSourcesMap.end()) {
+    if (!it->second.Initialized) {
+      std::ostringstream e;
+      e << "The SOURCES of \"" << this->GetName()
+        << "\" use a generator expression that depends on the "
+           "SOURCES themselves.";
+      this->GlobalGenerator->GetCMakeInstance()->IssueMessage(
+        cmake::FATAL_ERROR, e.str(), this->GetBacktrace());
+      static KindedSources empty;
+      return empty;
+    }
     return it->second;
   }
 
   // Add an entry to the map for this configuration.
   KindedSources& files = this->KindedSourcesMap[key];
   this->ComputeKindedSources(files, config);
+  files.Initialized = true;
   return files;
 }
 
diff --git a/Source/cmGeneratorTarget.h b/Source/cmGeneratorTarget.h
index c04d62b547..52147e3197 100644
--- a/Source/cmGeneratorTarget.h
+++ b/Source/cmGeneratorTarget.h
@@ -107,6 +107,11 @@ public:
     std::set<std::string> ExpectedResxHeaders;
     std::set<std::string> ExpectedXamlHeaders;
     std::set<std::string> ExpectedXamlSources;
+    bool Initialized;
+    KindedSources()
+      : Initialized(false)
+    {
+    }
   };
 
   /** Get all sources needed for a configuration with kinds assigned.  */
diff --git a/Tests/RunCMake/ObjectLibrary/OwnSources-result.txt b/Tests/RunCMake/ObjectLibrary/OwnSources-result.txt
new file mode 100644
index 0000000000..d00491fd7e
--- /dev/null
+++ b/Tests/RunCMake/ObjectLibrary/OwnSources-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/ObjectLibrary/OwnSources-stderr.txt b/Tests/RunCMake/ObjectLibrary/OwnSources-stderr.txt
new file mode 100644
index 0000000000..40d650e214
--- /dev/null
+++ b/Tests/RunCMake/ObjectLibrary/OwnSources-stderr.txt
@@ -0,0 +1,5 @@
+^CMake Error at OwnSources.cmake:[0-9]+ \(add_library\):
+  The SOURCES of "A" use a generator expression that depends on the SOURCES
+  themselves.
+Call Stack \(most recent call first\):
+  CMakeLists.txt:[0-9]+ \(include\)$
diff --git a/Tests/RunCMake/ObjectLibrary/OwnSources.cmake b/Tests/RunCMake/ObjectLibrary/OwnSources.cmake
new file mode 100644
index 0000000000..e7bdf8d83b
--- /dev/null
+++ b/Tests/RunCMake/ObjectLibrary/OwnSources.cmake
@@ -0,0 +1,2 @@
+add_library(A OBJECT a.c)
+target_sources(A PRIVATE $<TARGET_OBJECTS:A>)
diff --git a/Tests/RunCMake/ObjectLibrary/RunCMakeTest.cmake b/Tests/RunCMake/ObjectLibrary/RunCMakeTest.cmake
index fe708ce979..b8eed73bb6 100644
--- a/Tests/RunCMake/ObjectLibrary/RunCMakeTest.cmake
+++ b/Tests/RunCMake/ObjectLibrary/RunCMakeTest.cmake
@@ -19,6 +19,7 @@ run_cmake(LinkObjRHS1)
 run_cmake(LinkObjRHS2)
 run_cmake(MissingSource)
 run_cmake(ObjWithObj)
+run_cmake(OwnSources)
 run_cmake(PostBuild)
 run_cmake(PreBuild)
 run_cmake(PreLink)