diff options
Diffstat (limited to 'Source')
| -rw-r--r-- | Source/CMakeLists.txt | 2 | ||||
| -rw-r--r-- | Source/cmQtAutoGenerator.cxx | 232 | ||||
| -rw-r--r-- | Source/cmQtAutoGenerator.h | 102 | ||||
| -rw-r--r-- | Source/cmQtAutoGeneratorMocUic.cxx | 1170 | ||||
| -rw-r--r-- | Source/cmQtAutoGeneratorMocUic.h | 284 | ||||
| -rw-r--r-- | Source/cmWorkerPool.cxx | 770 | ||||
| -rw-r--r-- | Source/cmWorkerPool.h | 219 |
7 files changed, 1545 insertions, 1234 deletions
diff --git a/Source/CMakeLists.txt b/Source/CMakeLists.txt index 924d9977e1..fcea2e3278 100644 --- a/Source/CMakeLists.txt +++ b/Source/CMakeLists.txt @@ -391,6 +391,8 @@ set(SRCS cmVariableWatch.h cmVersion.cxx cmVersion.h + cmWorkerPool.cxx + cmWorkerPool.h cmWorkingDirectory.cxx cmWorkingDirectory.h cmXMLParser.cxx diff --git a/Source/cmQtAutoGenerator.cxx b/Source/cmQtAutoGenerator.cxx index f115016d65..6fbea8226e 100644 --- a/Source/cmQtAutoGenerator.cxx +++ b/Source/cmQtAutoGenerator.cxx @@ -14,12 +14,6 @@ #include "cmSystemTools.h" #include "cmake.h" -#include <algorithm> -#include <sstream> -#include <utility> - -// -- Class methods - cmQtAutoGenerator::Logger::Logger() { // Initialize logger @@ -431,232 +425,6 @@ bool cmQtAutoGenerator::FileSystem::MakeParentDirectory( return cmQtAutoGenerator::MakeParentDirectory(filename); } -int cmQtAutoGenerator::ReadOnlyProcessT::PipeT::init(uv_loop_t* uv_loop, - ReadOnlyProcessT* process) -{ - Process_ = process; - Target_ = nullptr; - return UVPipe_.init(*uv_loop, 0, this); -} - -int cmQtAutoGenerator::ReadOnlyProcessT::PipeT::startRead(std::string* target) -{ - Target_ = target; - return uv_read_start(uv_stream(), &PipeT::UVAlloc, &PipeT::UVData); -} - -void cmQtAutoGenerator::ReadOnlyProcessT::PipeT::reset() -{ - Process_ = nullptr; - Target_ = nullptr; - UVPipe_.reset(); - Buffer_.clear(); - Buffer_.shrink_to_fit(); -} - -void cmQtAutoGenerator::ReadOnlyProcessT::PipeT::UVAlloc(uv_handle_t* handle, - size_t suggestedSize, - uv_buf_t* buf) -{ - auto& pipe = *reinterpret_cast<PipeT*>(handle->data); - pipe.Buffer_.resize(suggestedSize); - buf->base = pipe.Buffer_.data(); - buf->len = pipe.Buffer_.size(); -} - -void cmQtAutoGenerator::ReadOnlyProcessT::PipeT::UVData(uv_stream_t* stream, - ssize_t nread, - const uv_buf_t* buf) -{ - auto& pipe = *reinterpret_cast<PipeT*>(stream->data); - if (nread > 0) { - // Append data to merged output - if ((buf->base != nullptr) && (pipe.Target_ != nullptr)) { - pipe.Target_->append(buf->base, nread); - } - } else if (nread < 0) { - // EOF or error - auto* proc = pipe.Process_; - // Check it this an unusual error - if (nread != UV_EOF) { - if (!proc->Result()->error()) { - proc->Result()->ErrorMessage = - "libuv reading from pipe failed with error code "; - proc->Result()->ErrorMessage += std::to_string(nread); - } - } - // Clear libuv pipe handle and try to finish - pipe.reset(); - proc->UVTryFinish(); - } -} - -void cmQtAutoGenerator::ProcessResultT::reset() -{ - ExitStatus = 0; - TermSignal = 0; - if (!StdOut.empty()) { - StdOut.clear(); - StdOut.shrink_to_fit(); - } - if (!StdErr.empty()) { - StdErr.clear(); - StdErr.shrink_to_fit(); - } - if (!ErrorMessage.empty()) { - ErrorMessage.clear(); - ErrorMessage.shrink_to_fit(); - } -} - -void cmQtAutoGenerator::ReadOnlyProcessT::setup( - ProcessResultT* result, bool mergedOutput, - std::vector<std::string> const& command, std::string const& workingDirectory) -{ - Setup_.WorkingDirectory = workingDirectory; - Setup_.Command = command; - Setup_.Result = result; - Setup_.MergedOutput = mergedOutput; -} - -static std::string getUVError(const char* prefixString, int uvErrorCode) -{ - std::ostringstream ost; - ost << prefixString << ": " << uv_strerror(uvErrorCode); - return ost.str(); -} - -bool cmQtAutoGenerator::ReadOnlyProcessT::start( - uv_loop_t* uv_loop, std::function<void()>&& finishedCallback) -{ - if (IsStarted() || (Result() == nullptr)) { - return false; - } - - // Reset result before the start - Result()->reset(); - - // Fill command string pointers - if (!Setup().Command.empty()) { - CommandPtr_.reserve(Setup().Command.size() + 1); - for (std::string const& arg : Setup().Command) { - CommandPtr_.push_back(arg.c_str()); - } - CommandPtr_.push_back(nullptr); - } else { - Result()->ErrorMessage = "Empty command"; - } - - if (!Result()->error()) { - if (UVPipeOut_.init(uv_loop, this) != 0) { - Result()->ErrorMessage = "libuv stdout pipe initialization failed"; - } - } - if (!Result()->error()) { - if (UVPipeErr_.init(uv_loop, this) != 0) { - Result()->ErrorMessage = "libuv stderr pipe initialization failed"; - } - } - if (!Result()->error()) { - // -- Setup process stdio options - // stdin - UVOptionsStdIO_[0].flags = UV_IGNORE; - UVOptionsStdIO_[0].data.stream = nullptr; - // stdout - UVOptionsStdIO_[1].flags = - static_cast<uv_stdio_flags>(UV_CREATE_PIPE | UV_WRITABLE_PIPE); - UVOptionsStdIO_[1].data.stream = UVPipeOut_.uv_stream(); - // stderr - UVOptionsStdIO_[2].flags = - static_cast<uv_stdio_flags>(UV_CREATE_PIPE | UV_WRITABLE_PIPE); - UVOptionsStdIO_[2].data.stream = UVPipeErr_.uv_stream(); - - // -- Setup process options - std::fill_n(reinterpret_cast<char*>(&UVOptions_), sizeof(UVOptions_), 0); - UVOptions_.exit_cb = &ReadOnlyProcessT::UVExit; - UVOptions_.file = CommandPtr_[0]; - UVOptions_.args = const_cast<char**>(CommandPtr_.data()); - UVOptions_.cwd = Setup_.WorkingDirectory.c_str(); - UVOptions_.flags = UV_PROCESS_WINDOWS_HIDE; - UVOptions_.stdio_count = static_cast<int>(UVOptionsStdIO_.size()); - UVOptions_.stdio = UVOptionsStdIO_.data(); - - // -- Spawn process - int uvErrorCode = UVProcess_.spawn(*uv_loop, UVOptions_, this); - if (uvErrorCode != 0) { - Result()->ErrorMessage = - getUVError("libuv process spawn failed ", uvErrorCode); - } - } - // -- Start reading from stdio streams - if (!Result()->error()) { - if (UVPipeOut_.startRead(&Result()->StdOut) != 0) { - Result()->ErrorMessage = "libuv start reading from stdout pipe failed"; - } - } - if (!Result()->error()) { - if (UVPipeErr_.startRead(Setup_.MergedOutput ? &Result()->StdOut - : &Result()->StdErr) != 0) { - Result()->ErrorMessage = "libuv start reading from stderr pipe failed"; - } - } - - if (!Result()->error()) { - IsStarted_ = true; - FinishedCallback_ = std::move(finishedCallback); - } else { - // Clear libuv handles and finish - UVProcess_.reset(); - UVPipeOut_.reset(); - UVPipeErr_.reset(); - CommandPtr_.clear(); - } - - return IsStarted(); -} - -void cmQtAutoGenerator::ReadOnlyProcessT::UVExit(uv_process_t* handle, - int64_t exitStatus, - int termSignal) -{ - auto& proc = *reinterpret_cast<ReadOnlyProcessT*>(handle->data); - if (proc.IsStarted() && !proc.IsFinished()) { - // Set error message on demand - proc.Result()->ExitStatus = exitStatus; - proc.Result()->TermSignal = termSignal; - if (!proc.Result()->error()) { - if (termSignal != 0) { - proc.Result()->ErrorMessage = "Process was terminated by signal "; - proc.Result()->ErrorMessage += - std::to_string(proc.Result()->TermSignal); - } else if (exitStatus != 0) { - proc.Result()->ErrorMessage = "Process failed with return value "; - proc.Result()->ErrorMessage += - std::to_string(proc.Result()->ExitStatus); - } - } - - // Reset process handle and try to finish - proc.UVProcess_.reset(); - proc.UVTryFinish(); - } -} - -void cmQtAutoGenerator::ReadOnlyProcessT::UVTryFinish() -{ - // There still might be data in the pipes after the process has finished. - // Therefore check if the process is finished AND all pipes are closed - // before signaling the worker thread to continue. - if (UVProcess_.get() == nullptr) { - if (UVPipeOut_.uv_pipe() == nullptr) { - if (UVPipeErr_.uv_pipe() == nullptr) { - IsFinished_ = true; - FinishedCallback_(); - } - } - } -} - cmQtAutoGenerator::cmQtAutoGenerator() = default; cmQtAutoGenerator::~cmQtAutoGenerator() = default; diff --git a/Source/cmQtAutoGenerator.h b/Source/cmQtAutoGenerator.h index 479d35760c..437fa20987 100644 --- a/Source/cmQtAutoGenerator.h +++ b/Source/cmQtAutoGenerator.h @@ -7,14 +7,8 @@ #include "cmFilePathChecksum.h" #include "cmQtAutoGen.h" -#include "cmUVHandlePtr.h" -#include "cm_uv.h" -#include <array> -#include <functional> #include <mutex> -#include <stddef.h> -#include <stdint.h> #include <string> #include <vector> @@ -137,102 +131,6 @@ public: cmFilePathChecksum FilePathChecksum_; }; - /// @brief Return value and output of an external process - struct ProcessResultT - { - void reset(); - bool error() const - { - return (ExitStatus != 0) || (TermSignal != 0) || !ErrorMessage.empty(); - } - - std::int64_t ExitStatus = 0; - int TermSignal = 0; - std::string StdOut; - std::string StdErr; - std::string ErrorMessage; - }; - - /// @brief External process management class - struct ReadOnlyProcessT - { - // -- Types - - /// @brief libuv pipe buffer class - class PipeT - { - public: - int init(uv_loop_t* uv_loop, ReadOnlyProcessT* process); - int startRead(std::string* target); - void reset(); - - // -- Libuv casts - uv_pipe_t* uv_pipe() { return UVPipe_.get(); } - uv_stream_t* uv_stream() - { - return reinterpret_cast<uv_stream_t*>(uv_pipe()); - } - uv_handle_t* uv_handle() - { - return reinterpret_cast<uv_handle_t*>(uv_pipe()); - } - - // -- Libuv callbacks - static void UVAlloc(uv_handle_t* handle, size_t suggestedSize, - uv_buf_t* buf); - static void UVData(uv_stream_t* stream, ssize_t nread, - const uv_buf_t* buf); - - private: - ReadOnlyProcessT* Process_ = nullptr; - std::string* Target_ = nullptr; - std::vector<char> Buffer_; - cm::uv_pipe_ptr UVPipe_; - }; - - /// @brief Process settings - struct SetupT - { - std::string WorkingDirectory; - std::vector<std::string> Command; - ProcessResultT* Result = nullptr; - bool MergedOutput = false; - }; - - // -- Const accessors - const SetupT& Setup() const { return Setup_; } - ProcessResultT* Result() const { return Setup_.Result; } - bool IsStarted() const { return IsStarted_; } - bool IsFinished() const { return IsFinished_; } - - // -- Runtime - void setup(ProcessResultT* result, bool mergedOutput, - std::vector<std::string> const& command, - std::string const& workingDirectory = std::string()); - bool start(uv_loop_t* uv_loop, std::function<void()>&& finishedCallback); - - private: - // -- Friends - friend class PipeT; - // -- Libuv callbacks - static void UVExit(uv_process_t* handle, int64_t exitStatus, - int termSignal); - void UVTryFinish(); - - // -- Setup - SetupT Setup_; - // -- Runtime - bool IsStarted_ = false; - bool IsFinished_ = false; - std::function<void()> FinishedCallback_; - std::vector<const char*> CommandPtr_; - std::array<uv_stdio_container_t, 3> UVOptionsStdIO_; - uv_process_options_t UVOptions_; - cm::uv_process_ptr UVProcess_; - PipeT UVPipeOut_; - PipeT UVPipeErr_; - }; - public: // -- Constructors cmQtAutoGenerator(); diff --git a/Source/cmQtAutoGeneratorMocUic.cxx b/Source/cmQtAutoGeneratorMocUic.cxx index ec1a1aa797..80684b6033 100644 --- a/Source/cmQtAutoGeneratorMocUic.cxx +++ b/Source/cmQtAutoGeneratorMocUic.cxx @@ -4,7 +4,7 @@ #include <algorithm> #include <array> -#include <cstddef> +#include <deque> #include <list> #include <memory> #include <set> @@ -153,11 +153,118 @@ bool cmQtAutoGeneratorMocUic::UicSettingsT::skipped( return (!Enabled || (SkipList.find(fileName) != SkipList.end())); } -void cmQtAutoGeneratorMocUic::JobParseT::Process(WorkerT& wrk) +void cmQtAutoGeneratorMocUic::JobT::LogError(GenT genType, + std::string const& message) const +{ + Gen()->AbortError(); + Gen()->Log().Error(genType, message); +} + +void cmQtAutoGeneratorMocUic::JobT::LogFileError( + GenT genType, std::string const& filename, std::string const& message) const +{ + Gen()->AbortError(); + Gen()->Log().ErrorFile(genType, filename, message); +} + +void cmQtAutoGeneratorMocUic::JobT::LogCommandError( + GenT genType, std::string const& message, + std::vector<std::string> const& command, std::string const& output) const +{ + Gen()->AbortError(); + Gen()->Log().ErrorCommand(genType, message, command, output); +} + +bool cmQtAutoGeneratorMocUic::JobT::RunProcess( + GenT genType, cmWorkerPool::ProcessResultT& result, + std::vector<std::string> const& command) +{ + // Log command + if (Log().Verbose()) { + std::string msg = "Running command:\n"; + msg += QuotedCommand(command); + msg += '\n'; + Log().Info(genType, msg); + } + return cmWorkerPool::JobT::RunProcess(result, command, + Gen()->Base().AutogenBuildDir); +} + +void cmQtAutoGeneratorMocUic::JobMocPredefsT::Process() +{ + // (Re)generate moc_predefs.h on demand + bool generate(false); + bool fileExists(FileSys().FileExists(Gen()->Moc().PredefsFileAbs)); + if (!fileExists) { + if (Log().Verbose()) { + std::string reason = "Generating "; + reason += Quoted(Gen()->Moc().PredefsFileRel); + reason += " because it doesn't exist"; + Log().Info(GenT::MOC, reason); + } + generate = true; + } else if (Gen()->Moc().SettingsChanged) { + if (Log().Verbose()) { + std::string reason = "Generating "; + reason += Quoted(Gen()->Moc().PredefsFileRel); + reason += " because the settings changed."; + Log().Info(GenT::MOC, reason); + } + generate = true; + } + if (generate) { + cmWorkerPool::ProcessResultT result; + { + // Compose command + std::vector<std::string> cmd = Gen()->Moc().PredefsCmd; + // Add includes + cmd.insert(cmd.end(), Gen()->Moc().Includes.begin(), + Gen()->Moc().Includes.end()); + // Add definitions + for (std::string const& def : Gen()->Moc().Definitions) { + cmd.push_back("-D" + def); + } + // Execute command + if (!RunProcess(GenT::MOC, result, cmd)) { + std::string emsg = "The content generation command for "; + emsg += Quoted(Gen()->Moc().PredefsFileRel); + emsg += " failed.\n"; + emsg += result.ErrorMessage; + LogCommandError(GenT::MOC, emsg, cmd, result.StdOut); + } + } + + // (Re)write predefs file only on demand + if (!result.error()) { + if (!fileExists || + FileSys().FileDiffers(Gen()->Moc().PredefsFileAbs, result.StdOut)) { + if (FileSys().FileWrite(Gen()->Moc().PredefsFileAbs, result.StdOut)) { + // Success + } else { + std::string emsg = "Writing "; + emsg += Quoted(Gen()->Moc().PredefsFileRel); + emsg += " failed."; + LogFileError(GenT::MOC, Gen()->Moc().PredefsFileAbs, emsg); + } + } else { + // Touch to update the time stamp + if (Log().Verbose()) { + std::string msg = "Touching "; + msg += Quoted(Gen()->Moc().PredefsFileRel); + msg += "."; + Log().Info(GenT::MOC, msg); + } + FileSys().Touch(Gen()->Moc().PredefsFileAbs); + } + } + } +} + +void cmQtAutoGeneratorMocUic::JobParseT::Process() { if (AutoMoc && Header) { // Don't parse header for moc if the file is included by a source already - if (wrk.Gen().ParallelMocIncluded(FileName)) { + if (Gen()->ParallelMocIncluded(FileName)) { AutoMoc = false; } } @@ -165,35 +272,32 @@ void cmQtAutoGeneratorMocUic::JobParseT::Process(WorkerT& wrk) if (AutoMoc || AutoUic) { std::string error; MetaT meta; - if (wrk.FileSys().FileRead(meta.Content, FileName, &error)) { + if (FileSys().FileRead(meta.Content, FileName, &error)) { if (!meta.Content.empty()) { - meta.FileDir = wrk.FileSys().SubDirPrefix(FileName); - meta.FileBase = - wrk.FileSys().GetFilenameWithoutLastExtension(FileName); + meta.FileDir = FileSys().SubDirPrefix(FileName); + meta.FileBase = FileSys().GetFilenameWithoutLastExtension(FileName); bool success = true; if (AutoMoc) { if (Header) { - success = ParseMocHeader(wrk, meta); + success = ParseMocHeader(meta); } else { - success = ParseMocSource(wrk, meta); + success = ParseMocSource(meta); } } if (AutoUic && success) { - ParseUic(wrk, meta); + ParseUic(meta); } } else { - wrk.LogFileWarning(GenT::GEN, FileName, "The source file is empty"); + Log().WarningFile(GenT::GEN, FileName, "The source file is empty"); } } else { - wrk.LogFileError(GenT::GEN, FileName, - "Could not read the file: " + error); + LogFileError(GenT::GEN, FileName, "Could not read the file: " + error); } } } -bool cmQtAutoGeneratorMocUic::JobParseT::ParseMocSource(WorkerT& wrk, - MetaT const& meta) +bool cmQtAutoGeneratorMocUic::JobParseT::ParseMocSource(MetaT const& meta) { struct JobPre { @@ -211,7 +315,7 @@ bool cmQtAutoGeneratorMocUic::JobParseT::ParseMocSource(WorkerT& wrk, }; // Check if this source file contains a relevant macro - std::string const ownMacro = wrk.Moc().FindMacro(meta.Content); + std::string const ownMacro = Gen()->Moc().FindMacro(meta.Content); // Extract moc includes from file std::deque<MocInclude> mocIncsUsc; @@ -220,11 +324,11 @@ bool cmQtAutoGeneratorMocUic::JobParseT::ParseMocSource(WorkerT& wrk, if (meta.Content.find("moc") != std::string::npos) { const char* contentChars = meta.Content.c_str(); cmsys::RegularExpressionMatch match; - while (wrk.Moc().RegExpInclude.find(contentChars, match)) { + while (Gen()->Moc().RegExpInclude.find(contentChars, match)) { std::string incString = match.match(2); - std::string incDir(wrk.FileSys().SubDirPrefix(incString)); + std::string incDir(FileSys().SubDirPrefix(incString)); std::string incBase = - wrk.FileSys().GetFilenameWithoutLastExtension(incString); + FileSys().GetFilenameWithoutLastExtension(incString); if (cmHasLiteralPrefix(incBase, "moc_")) { // moc_<BASE>.cxx // Remove the moc_ part from the base name @@ -253,10 +357,10 @@ bool cmQtAutoGeneratorMocUic::JobParseT::ParseMocSource(WorkerT& wrk, // Process moc_<BASE>.cxx includes for (const MocInclude& mocInc : mocIncsUsc) { std::string const header = - MocFindIncludedHeader(wrk, meta.FileDir, mocInc.Dir + mocInc.Base); + MocFindIncludedHeader(meta.FileDir, mocInc.Dir + mocInc.Base); if (!header.empty()) { // Check if header is skipped - if (wrk.Moc().skipped(header)) { + if (Gen()->Moc().skipped(header)) { continue; } // Register moc job @@ -271,9 +375,9 @@ bool cmQtAutoGeneratorMocUic::JobParseT::ParseMocSource(WorkerT& wrk, std::string emsg = "The file includes the moc file "; emsg += Quoted(mocInc.Inc); emsg += ", but the header "; - emsg += Quoted(MocStringHeaders(wrk, mocInc.Base)); + emsg += Quoted(MocStringHeaders(mocInc.Base)); emsg += " could not be found."; - wrk.LogFileError(GenT::MOC, FileName, emsg); + LogFileError(GenT::MOC, FileName, emsg); } return false; } @@ -282,7 +386,7 @@ bool cmQtAutoGeneratorMocUic::JobParseT::ParseMocSource(WorkerT& wrk, // Process <BASE>.moc includes for (const MocInclude& mocInc : mocIncsDot) { const bool ownMoc = (mocInc.Base == meta.FileBase); - if (wrk.Moc().RelaxedMode) { + if (Gen()->Moc().RelaxedMode) { // Relaxed mode if (!ownMacro.empty() && ownMoc) { // Add self @@ -292,10 +396,10 @@ bool cmQtAutoGeneratorMocUic::JobParseT::ParseMocSource(WorkerT& wrk, // In relaxed mode try to find a header instead but issue a warning. // This is for KDE4 compatibility std::string const header = - MocFindIncludedHeader(wrk, meta.FileDir, mocInc.Dir + mocInc.Base); + MocFindIncludedHeader(meta.FileDir, mocInc.Dir + mocInc.Base); if (!header.empty()) { // Check if header is skipped - if (wrk.Moc().skipped(header)) { + if (Gen()->Moc().skipped(header)) { continue; } // Register moc job @@ -305,14 +409,14 @@ bool cmQtAutoGeneratorMocUic::JobParseT::ParseMocSource(WorkerT& wrk, std::string emsg = "The file includes the moc file "; emsg += Quoted(mocInc.Inc); emsg += ", but does not contain a "; - emsg += wrk.Moc().MacrosString(); + emsg += Gen()->Moc().MacrosString(); emsg += " macro.\nRunning moc on\n "; emsg += Quoted(header); emsg += "!\nBetter include "; emsg += Quoted("moc_" + mocInc.Base + ".cpp"); emsg += " for a compatibility with strict mode.\n" "(CMAKE_AUTOMOC_RELAXED_MODE warning)\n"; - wrk.LogFileWarning(GenT::MOC, FileName, emsg); + Log().WarningFile(GenT::MOC, FileName, emsg); } else { std::string emsg = "The file includes the moc file "; emsg += Quoted(mocInc.Inc); @@ -324,7 +428,7 @@ bool cmQtAutoGeneratorMocUic::JobParseT::ParseMocSource(WorkerT& wrk, emsg += Quoted("moc_" + mocInc.Base + ".cpp"); emsg += " for compatibility with strict mode.\n" "(CMAKE_AUTOMOC_RELAXED_MODE warning)\n"; - wrk.LogFileWarning(GenT::MOC, FileName, emsg); + Log().WarningFile(GenT::MOC, FileName, emsg); } } } else { @@ -334,9 +438,9 @@ bool cmQtAutoGeneratorMocUic::JobParseT::ParseMocSource(WorkerT& wrk, emsg += ", which seems to be the moc file from a different " "source file.\nCMAKE_AUTOMOC_RELAXED_MODE: Also a " "matching header "; - emsg += Quoted(MocStringHeaders(wrk, mocInc.Base)); + emsg += Quoted(MocStringHeaders(mocInc.Base)); emsg += " could not be found."; - wrk.LogFileError(GenT::MOC, FileName, emsg); + LogFileError(GenT::MOC, FileName, emsg); } return false; } @@ -352,9 +456,9 @@ bool cmQtAutoGeneratorMocUic::JobParseT::ParseMocSource(WorkerT& wrk, std::string emsg = "The file includes the moc file "; emsg += Quoted(mocInc.Inc); emsg += ", but does not contain a "; - emsg += wrk.Moc().MacrosString(); + emsg += Gen()->Moc().MacrosString(); emsg += " macro."; - wrk.LogFileWarning(GenT::MOC, FileName, emsg); + Log().WarningFile(GenT::MOC, FileName, emsg); } } else { // Don't allow <BASE>.moc include other than self in strict mode @@ -365,7 +469,7 @@ bool cmQtAutoGeneratorMocUic::JobParseT::ParseMocSource(WorkerT& wrk, "source file.\nThis is not supported. Include "; emsg += Quoted(meta.FileBase + ".moc"); emsg += " to run moc on this source file."; - wrk.LogFileError(GenT::MOC, FileName, emsg); + LogFileError(GenT::MOC, FileName, emsg); } return false; } @@ -379,7 +483,7 @@ bool cmQtAutoGeneratorMocUic::JobParseT::ParseMocSource(WorkerT& wrk, // foo.cpp instead of foo.h, because otherwise it won't build. // But warn, since this is not how it is supposed to be used. // This is for KDE4 compatibility. - if (wrk.Moc().RelaxedMode && ownMocUscIncluded) { + if (Gen()->Moc().RelaxedMode && ownMocUscIncluded) { JobPre uscJobPre; // Remove underscore job request { @@ -408,7 +512,7 @@ bool cmQtAutoGeneratorMocUic::JobParseT::ParseMocSource(WorkerT& wrk, emsg += Quoted(meta.FileBase + ".moc"); emsg += " for compatibility with strict mode.\n" "(CMAKE_AUTOMOC_RELAXED_MODE warning)"; - wrk.LogFileWarning(GenT::MOC, FileName, emsg); + Log().WarningFile(GenT::MOC, FileName, emsg); } // Add own source job jobs.emplace_back( @@ -423,7 +527,7 @@ bool cmQtAutoGeneratorMocUic::JobParseT::ParseMocSource(WorkerT& wrk, emsg += "!\nConsider to\n - add #include \""; emsg += meta.FileBase; emsg += ".moc\"\n - enable SKIP_AUTOMOC for this file"; - wrk.LogFileError(GenT::MOC, FileName, emsg); + LogFileError(GenT::MOC, FileName, emsg); } return false; } @@ -431,76 +535,80 @@ bool cmQtAutoGeneratorMocUic::JobParseT::ParseMocSource(WorkerT& wrk, // Convert pre jobs to actual jobs for (JobPre& jobPre : jobs) { - JobHandleT jobHandle = cm::make_unique<JobMocT>( + cmWorkerPool::JobHandleT jobHandle = cm::make_unique<JobMocT>( std::move(jobPre.SourceFile), FileName, std::move(jobPre.IncludeString)); if (jobPre.self) { // Read dependencies from this source - static_cast<JobMocT&>(*jobHandle).FindDependencies(wrk, meta.Content); + JobMocT& jobMoc = static_cast<JobMocT&>(*jobHandle); + Gen()->Moc().FindDependencies(meta.Content, jobMoc.Depends); + jobMoc.DependsValid = true; } - if (!wrk.Gen().ParallelJobPushMoc(jobHandle)) { + if (!Gen()->ParallelJobPushMoc(std::move(jobHandle))) { return false; } } return true; } -bool cmQtAutoGeneratorMocUic::JobParseT::ParseMocHeader(WorkerT& wrk, - MetaT const& meta) +bool cmQtAutoGeneratorMocUic::JobParseT::ParseMocHeader(MetaT const& meta) { bool success = true; - std::string const macroName = wrk.Moc().FindMacro(meta.Content); + std::string const macroName = Gen()->Moc().FindMacro(meta.Content); if (!macroName.empty()) { - JobHandleT jobHandle = cm::make_unique<JobMocT>( + cmWorkerPool::JobHandleT jobHandle = cm::make_unique<JobMocT>( std::string(FileName), std::string(), std::string()); // Read dependencies from this source - static_cast<JobMocT&>(*jobHandle).FindDependencies(wrk, meta.Content); - success = wrk.Gen().ParallelJobPushMoc(jobHandle); + { + JobMocT& jobMoc = static_cast<JobMocT&>(*jobHandle); + Gen()->Moc().FindDependencies(meta.Content, jobMoc.Depends); + jobMoc.DependsValid = true; + } + success = Gen()->ParallelJobPushMoc(std::move(jobHandle)); } return success; } std::string cmQtAutoGeneratorMocUic::JobParseT::MocStringHeaders( - WorkerT& wrk, std::string const& fileBase) const + std::string const& fileBase) const { std::string res = fileBase; res += ".{"; - res += cmJoin(wrk.Base().HeaderExtensions, ","); + res += cmJoin(Gen()->Base().HeaderExtensions, ","); res += "}"; return res; } std::string cmQtAutoGeneratorMocUic::JobParseT::MocFindIncludedHeader( - WorkerT& wrk, std::string const& includerDir, std::string const& includeBase) + std::string const& includerDir, std::string const& includeBase) { std::string header; // Search in vicinity of the source - if (!wrk.Base().FindHeader(header, includerDir + includeBase)) { + if (!Gen()->Base().FindHeader(header, includerDir + includeBase)) { // Search in include directories - for (std::string const& path : wrk.Moc().IncludePaths) { + for (std::string const& path : Gen()->Moc().IncludePaths) { std::string fullPath = path; fullPath.push_back('/'); fullPath += includeBase; - if (wrk.Base().FindHeader(header, fullPath)) { + if (Gen()->Base().FindHeader(header, fullPath)) { break; } } } // Sanitize if (!header.empty()) { - header = wrk.FileSys().GetRealPath(header); + header = FileSys().GetRealPath(header); } return header; } -bool cmQtAutoGeneratorMocUic::JobParseT::ParseUic(WorkerT& wrk, - MetaT const& meta) +bool cmQtAutoGeneratorMocUic::JobParseT::ParseUic(MetaT const& meta) { bool success = true; if (meta.Content.find("ui_") != std::string::npos) { const char* contentChars = meta.Content.c_str(); cmsys::RegularExpressionMatch match; - while (wrk.Uic().RegExpInclude.find(contentChars, match)) { - if (!ParseUicInclude(wrk, meta, match.match(2))) { + while (Gen()->Uic().RegExpInclude.find(contentChars, match)) { + if (!ParseUicInclude(meta, match.match(2))) { success = false; break; } @@ -511,15 +619,15 @@ bool cmQtAutoGeneratorMocUic::JobParseT::ParseUic(WorkerT& wrk, } bool cmQtAutoGeneratorMocUic::JobParseT::ParseUicInclude( - WorkerT& wrk, MetaT const& meta, std::string&& includeString) + MetaT const& meta, std::string&& includeString) { bool success = false; - std::string uiInputFile = UicFindIncludedFile(wrk, meta, includeString); + std::string uiInputFile = UicFindIncludedFile(meta, includeString); if (!uiInputFile.empty()) { - if (!wrk.Uic().skipped(uiInputFile)) { - JobHandleT jobHandle = cm::make_unique<JobUicT>( + if (!Gen()->Uic().skipped(uiInputFile)) { + cmWorkerPool::JobHandleT jobHandle = cm::make_unique<JobUicT>( std::move(uiInputFile), FileName, std::move(includeString)); - success = wrk.Gen().ParallelJobPushUic(jobHandle); + success = Gen()->ParallelJobPushUic(std::move(jobHandle)); } else { // A skipped file is successful success = true; @@ -529,16 +637,16 @@ bool cmQtAutoGeneratorMocUic::JobParseT::ParseUicInclude( } std::string cmQtAutoGeneratorMocUic::JobParseT::UicFindIncludedFile( - WorkerT& wrk, MetaT const& meta, std::string const& includeString) + MetaT const& meta, std::string const& includeString) { std::string res; std::string searchFile = - wrk.FileSys().GetFilenameWithoutLastExtension(includeString).substr(3); + FileSys().GetFilenameWithoutLastExtension(includeString).substr(3); searchFile += ".ui"; // Collect search paths list std::deque<std::string> testFiles; { - std::string const searchPath = wrk.FileSys().SubDirPrefix(includeString); + std::string const searchPath = FileSys().SubDirPrefix(includeString); std::string searchFileFull; if (!searchPath.empty()) { @@ -554,12 +662,12 @@ std::string cmQtAutoGeneratorMocUic::JobParseT::UicFindIncludedFile( } } // AUTOUIC search paths - if (!wrk.Uic().SearchPaths.empty()) { - for (std::string const& sPath : wrk.Uic().SearchPaths) { + if (!Gen()->Uic().SearchPaths.empty()) { + for (std::string const& sPath : Gen()->Uic().SearchPaths) { testFiles.push_back((sPath + "/").append(searchFile)); } if (!searchPath.empty()) { - for (std::string const& sPath : wrk.Uic().SearchPaths) { + for (std::string const& sPath : Gen()->Uic().SearchPaths) { testFiles.push_back((sPath + "/").append(searchFileFull)); } } @@ -568,8 +676,8 @@ std::string cmQtAutoGeneratorMocUic::JobParseT::UicFindIncludedFile( // Search for the .ui file! for (std::string const& testFile : testFiles) { - if (wrk.FileSys().FileExists(testFile)) { - res = wrk.FileSys().GetRealPath(testFile); + if (FileSys().FileExists(testFile)) { + res = FileSys().GetRealPath(testFile); break; } } @@ -584,162 +692,141 @@ std::string cmQtAutoGeneratorMocUic::JobParseT::UicFindIncludedFile( emsg += Quoted(testFile); emsg += "\n"; } - wrk.LogFileError(GenT::UIC, FileName, emsg); + LogFileError(GenT::UIC, FileName, emsg); } return res; } -void cmQtAutoGeneratorMocUic::JobMocPredefsT::Process(WorkerT& wrk) +void cmQtAutoGeneratorMocUic::JobPostParseT::Process() { - // (Re)generate moc_predefs.h on demand - bool generate(false); - bool fileExists(wrk.FileSys().FileExists(wrk.Moc().PredefsFileAbs)); - if (!fileExists) { - if (wrk.Log().Verbose()) { - std::string reason = "Generating "; - reason += Quoted(wrk.Moc().PredefsFileRel); - reason += " because it doesn't exist"; - wrk.LogInfo(GenT::MOC, reason); - } - generate = true; - } else if (wrk.Moc().SettingsChanged) { - if (wrk.Log().Verbose()) { - std::string reason = "Generating "; - reason += Quoted(wrk.Moc().PredefsFileRel); - reason += " because the settings changed."; - wrk.LogInfo(GenT::MOC, reason); - } - generate = true; + if (Gen()->Moc().Enabled) { + // Add mocs compilations fence job + Gen()->WorkerPool().EmplaceJob<JobMocsCompilationT>(); } - if (generate) { - ProcessResultT result; - { - // Compose command - std::vector<std::string> cmd = wrk.Moc().PredefsCmd; - // Add includes - cmd.insert(cmd.end(), wrk.Moc().Includes.begin(), - wrk.Moc().Includes.end()); - // Add definitions - for (std::string const& def : wrk.Moc().Definitions) { - cmd.push_back("-D" + def); - } - // Execute command - if (!wrk.RunProcess(GenT::MOC, result, cmd)) { - std::string emsg = "The content generation command for "; - emsg += Quoted(wrk.Moc().PredefsFileRel); - emsg += " failed.\n"; - emsg += result.ErrorMessage; - wrk.LogCommandError(GenT::MOC, emsg, cmd, result.StdOut); - } + // Add finish job + Gen()->WorkerPool().EmplaceJob<JobFinishT>(); +} + +void cmQtAutoGeneratorMocUic::JobMocsCompilationT::Process() +{ + // Compose mocs compilation file content + std::string content = + "// This file is autogenerated. Changes will be overwritten.\n"; + if (Gen()->MocAutoFiles().empty()) { + // Placeholder content + content += "// No files found that require moc or the moc files are " + "included\n"; + content += "enum some_compilers { need_more_than_nothing };\n"; + } else { + // Valid content + char const sbeg = Gen()->Base().MultiConfig ? '<' : '"'; + char const send = Gen()->Base().MultiConfig ? '>' : '"'; + for (std::string const& mocfile : Gen()->MocAutoFiles()) { + content += "#include "; + content += sbeg; + content += mocfile; + content += send; + content += '\n'; } + } - // (Re)write predefs file only on demand - if (!result.error()) { - if (!fileExists || - wrk.FileSys().FileDiffers(wrk.Moc().PredefsFileAbs, result.StdOut)) { - std::string error; - if (wrk.FileSys().FileWrite(wrk.Moc().PredefsFileAbs, result.StdOut, - &error)) { - // Success - } else { - std::string emsg = "Writing "; - emsg += Quoted(wrk.Moc().PredefsFileRel); - emsg += " failed. "; - emsg += error; - wrk.LogFileError(GenT::MOC, wrk.Moc().PredefsFileAbs, emsg); - } - } else { - // Touch to update the time stamp - if (wrk.Log().Verbose()) { - std::string msg = "Touching "; - msg += Quoted(wrk.Moc().PredefsFileRel); - msg += "."; - wrk.LogInfo(GenT::MOC, msg); - } - wrk.FileSys().Touch(wrk.Moc().PredefsFileAbs); - } + std::string const& compAbs = Gen()->Moc().CompFileAbs; + if (FileSys().FileDiffers(compAbs, content)) { + // Actually write mocs compilation file + if (Log().Verbose()) { + Log().Info(GenT::MOC, "Generating MOC compilation " + compAbs); } + if (!FileSys().FileWrite(compAbs, content)) { + LogFileError(GenT::MOC, compAbs, + "mocs compilation file writing failed."); + } + } else if (Gen()->MocAutoFileUpdated()) { + // Only touch mocs compilation file + if (Log().Verbose()) { + Log().Info(GenT::MOC, "Touching mocs compilation " + compAbs); + } + FileSys().Touch(compAbs); } } void cmQtAutoGeneratorMocUic::JobMocT::FindDependencies( - WorkerT& wrk, std::string const& content) + std::string const& content) { - wrk.Moc().FindDependencies(content, Depends); + Gen()->Moc().FindDependencies(content, Depends); DependsValid = true; } -void cmQtAutoGeneratorMocUic::JobMocT::Process(WorkerT& wrk) +void cmQtAutoGeneratorMocUic::JobMocT::Process() { // Compute build file name if (!IncludeString.empty()) { - BuildFile = wrk.Base().AutogenIncludeDir; + BuildFile = Gen()->Base().AutogenIncludeDir; BuildFile += '/'; BuildFile += IncludeString; } else { // Relative build path - std::string relPath = wrk.FileSys().GetFilePathChecksum(SourceFile); + std::string relPath = FileSys().GetFilePathChecksum(SourceFile); relPath += "/moc_"; - relPath += wrk.FileSys().GetFilenameWithoutLastExtension(SourceFile); + relPath += FileSys().GetFilenameWithoutLastExtension(SourceFile); // Register relative file path with duplication check - relPath = wrk.Gen().ParallelMocAutoRegister(relPath); + relPath = Gen()->ParallelMocAutoRegister(relPath); // Absolute build path - if (wrk.Base().MultiConfig) { - BuildFile = wrk.Base().AutogenIncludeDir; + if (Gen()->Base().MultiConfig) { + BuildFile = Gen()->Base().AutogenIncludeDir; BuildFile += '/'; BuildFile += relPath; } else { - BuildFile = wrk.Base().AbsoluteBuildPath(relPath); + BuildFile = Gen()->Base().AbsoluteBuildPath(relPath); } } - if (UpdateRequired(wrk)) { - GenerateMoc(wrk); + if (UpdateRequired()) { + GenerateMoc(); } } -bool cmQtAutoGeneratorMocUic::JobMocT::UpdateRequired(WorkerT& wrk) +bool cmQtAutoGeneratorMocUic::JobMocT::UpdateRequired() { - bool const verbose = wrk.Gen().Log().Verbose(); + bool const verbose = Log().Verbose(); // Test if the build file exists - if (!wrk.FileSys().FileExists(BuildFile)) { + if (!FileSys().FileExists(BuildFile)) { if (verbose) { std::string reason = "Generating "; reason += Quoted(BuildFile); reason += " from its source file "; reason += Quoted(SourceFile); reason += " because it doesn't exist"; - wrk.LogInfo(GenT::MOC, reason); + Log().Info(GenT::MOC, reason); } return true; } // Test if any setting changed - if (wrk.Moc().SettingsChanged) { + if (Gen()->Moc().SettingsChanged) { if (verbose) { std::string reason = "Generating "; reason += Quoted(BuildFile); reason += " from "; reason += Quoted(SourceFile); reason += " because the MOC settings changed"; - wrk.LogInfo(GenT::MOC, reason); + Log().Info(GenT::MOC, reason); } return true; } // Test if the moc_predefs file is newer - if (!wrk.Moc().PredefsFileAbs.empty()) { + if (!Gen()->Moc().PredefsFileAbs.empty()) { bool isOlder = false; { std::string error; - isOlder = wrk.FileSys().FileIsOlderThan( - BuildFile, wrk.Moc().PredefsFileAbs, &error); + isOlder = FileSys().FileIsOlderThan(BuildFile, + Gen()->Moc().PredefsFileAbs, &error); if (!isOlder && !error.empty()) { - wrk.LogError(GenT::MOC, error); + LogError(GenT::MOC, error); return false; } } @@ -748,8 +835,8 @@ bool cmQtAutoGeneratorMocUic::JobMocT::UpdateRequired(WorkerT& wrk) std::string reason = "Generating "; reason += Quoted(BuildFile); reason += " because it's older than: "; - reason += Quoted(wrk.Moc().PredefsFileAbs); - wrk.LogInfo(GenT::MOC, reason); + reason += Quoted(Gen()->Moc().PredefsFileAbs); + Log().Info(GenT::MOC, reason); } return true; } @@ -760,9 +847,9 @@ bool cmQtAutoGeneratorMocUic::JobMocT::UpdateRequired(WorkerT& wrk) bool isOlder = false; { std::string error; - isOlder = wrk.FileSys().FileIsOlderThan(BuildFile, SourceFile, &error); + isOlder = FileSys().FileIsOlderThan(BuildFile, SourceFile, &error); if (!isOlder && !error.empty()) { - wrk.LogError(GenT::MOC, error); + LogError(GenT::MOC, error); return false; } } @@ -772,7 +859,7 @@ bool cmQtAutoGeneratorMocUic::JobMocT::UpdateRequired(WorkerT& wrk) reason += Quoted(BuildFile); reason += " because it's older than its source file "; reason += Quoted(SourceFile); - wrk.LogInfo(GenT::MOC, reason); + Log().Info(GenT::MOC, reason); } return true; } @@ -785,7 +872,7 @@ bool cmQtAutoGeneratorMocUic::JobMocT::UpdateRequired(WorkerT& wrk) std::string content; { std::string error; - if (!wrk.FileSys().FileRead(content, SourceFile, &error)) { + if (!FileSys().FileRead(content, SourceFile, &error)) { std::string emsg = "Could not read file\n "; emsg += Quoted(SourceFile); emsg += "\nrequired by moc include "; @@ -794,20 +881,20 @@ bool cmQtAutoGeneratorMocUic::JobMocT::UpdateRequired(WorkerT& wrk) emsg += Quoted(IncluderFile); emsg += ".\n"; emsg += error; - wrk.LogError(GenT::MOC, emsg); + LogError(GenT::MOC, emsg); return false; } } - FindDependencies(wrk, content); + FindDependencies(content); } // Check dependency timestamps std::string error; - std::string sourceDir = wrk.FileSys().SubDirPrefix(SourceFile); + std::string sourceDir = FileSys().SubDirPrefix(SourceFile); for (std::string const& depFileRel : Depends) { std::string depFileAbs = - wrk.Moc().FindIncludedFile(sourceDir, depFileRel); + Gen()->Moc().FindIncludedFile(sourceDir, depFileRel); if (!depFileAbs.empty()) { - if (wrk.FileSys().FileIsOlderThan(BuildFile, depFileAbs, &error)) { + if (FileSys().FileIsOlderThan(BuildFile, depFileAbs, &error)) { if (verbose) { std::string reason = "Generating "; reason += Quoted(BuildFile); @@ -815,18 +902,18 @@ bool cmQtAutoGeneratorMocUic::JobMocT::UpdateRequired(WorkerT& wrk) reason += Quoted(SourceFile); reason += " because it is older than it's dependency file "; reason += Quoted(depFileAbs); - wrk.LogInfo(GenT::MOC, reason); + Log().Info(GenT::MOC, reason); } return true; } if (!error.empty()) { - wrk.LogError(GenT::MOC, error); + LogError(GenT::MOC, error); return false; } } else { std::string message = "Could not find dependency file "; message += Quoted(depFileRel); - wrk.LogFileWarning(GenT::MOC, SourceFile, message); + Log().WarningFile(GenT::MOC, SourceFile, message); } } } @@ -834,41 +921,40 @@ bool cmQtAutoGeneratorMocUic::JobMocT::UpdateRequired(WorkerT& wrk) return false; } -void cmQtAutoGeneratorMocUic::JobMocT::GenerateMoc(WorkerT& wrk) +void cmQtAutoGeneratorMocUic::JobMocT::GenerateMoc() { // Make sure the parent directory exists - if (!wrk.FileSys().MakeParentDirectory(BuildFile)) { - wrk.LogFileError(GenT::MOC, BuildFile, - "Could not create parent directory."); + if (!FileSys().MakeParentDirectory(BuildFile)) { + LogFileError(GenT::MOC, BuildFile, "Could not create parent directory."); return; } { // Compose moc command std::vector<std::string> cmd; - cmd.push_back(wrk.Moc().Executable); + cmd.push_back(Gen()->Moc().Executable); // Add options - cmd.insert(cmd.end(), wrk.Moc().AllOptions.begin(), - wrk.Moc().AllOptions.end()); + cmd.insert(cmd.end(), Gen()->Moc().AllOptions.begin(), + Gen()->Moc().AllOptions.end()); // Add predefs include - if (!wrk.Moc().PredefsFileAbs.empty()) { + if (!Gen()->Moc().PredefsFileAbs.empty()) { cmd.emplace_back("--include"); - cmd.push_back(wrk.Moc().PredefsFileAbs); + cmd.push_back(Gen()->Moc().PredefsFileAbs); } cmd.emplace_back("-o"); cmd.push_back(BuildFile); cmd.push_back(SourceFile); // Execute moc command - ProcessResultT result; - if (wrk.RunProcess(GenT::MOC, result, cmd)) { + cmWorkerPool::ProcessResultT result; + if (RunProcess(GenT::MOC, result, cmd)) { // Moc command success // Print moc output if (!result.StdOut.empty()) { - wrk.LogInfo(GenT::MOC, result.StdOut); + Log().Info(GenT::MOC, result.StdOut); } // Notify the generator that a not included file changed (on demand) if (IncludeString.empty()) { - wrk.Gen().ParallelMocAutoUpdated(); + Gen()->ParallelMocAutoUpdated(); } } else { // Moc command failed @@ -879,51 +965,51 @@ void cmQtAutoGeneratorMocUic::JobMocT::GenerateMoc(WorkerT& wrk) emsg += Quoted(BuildFile); emsg += ".\n"; emsg += result.ErrorMessage; - wrk.LogCommandError(GenT::MOC, emsg, cmd, result.StdOut); + LogCommandError(GenT::MOC, emsg, cmd, result.StdOut); } - wrk.FileSys().FileRemove(BuildFile); + FileSys().FileRemove(BuildFile); } } } -void cmQtAutoGeneratorMocUic::JobUicT::Process(WorkerT& wrk) +void cmQtAutoGeneratorMocUic::JobUicT::Process() { // Compute build file name - BuildFile = wrk.Base().AutogenIncludeDir; + BuildFile = Gen()->Base().AutogenIncludeDir; BuildFile += '/'; BuildFile += IncludeString; - if (UpdateRequired(wrk)) { - GenerateUic(wrk); + if (UpdateRequired()) { + GenerateUic(); } } -bool cmQtAutoGeneratorMocUic::JobUicT::UpdateRequired(WorkerT& wrk) +bool cmQtAutoGeneratorMocUic::JobUicT::UpdateRequired() { - bool const verbose = wrk.Gen().Log().Verbose(); + bool const verbose = Log().Verbose(); // Test if the build file exists - if (!wrk.FileSys().FileExists(BuildFile)) { + if (!FileSys().FileExists(BuildFile)) { if (verbose) { std::string reason = "Generating "; reason += Quoted(BuildFile); reason += " from its source file "; reason += Quoted(SourceFile); reason += " because it doesn't exist"; - wrk.LogInfo(GenT::UIC, reason); + Log().Info(GenT::UIC, reason); } return true; } // Test if the uic settings changed - if (wrk.Uic().SettingsChanged) { + if (Gen()->Uic().SettingsChanged) { if (verbose) { std::string reason = "Generating "; reason += Quoted(BuildFile); reason += " from "; reason += Quoted(SourceFile); reason += " because the UIC settings changed"; - wrk.LogInfo(GenT::UIC, reason); + Log().Info(GenT::UIC, reason); } return true; } @@ -933,9 +1019,9 @@ bool cmQtAutoGeneratorMocUic::JobUicT::UpdateRequired(WorkerT& wrk) bool isOlder = false; { std::string error; - isOlder = wrk.FileSys().FileIsOlderThan(BuildFile, SourceFile, &error); + isOlder = FileSys().FileIsOlderThan(BuildFile, SourceFile, &error); if (!isOlder && !error.empty()) { - wrk.LogError(GenT::UIC, error); + LogError(GenT::UIC, error); return false; } } @@ -945,7 +1031,7 @@ bool cmQtAutoGeneratorMocUic::JobUicT::UpdateRequired(WorkerT& wrk) reason += Quoted(BuildFile); reason += " because it's older than its source file "; reason += Quoted(SourceFile); - wrk.LogInfo(GenT::UIC, reason); + Log().Info(GenT::UIC, reason); } return true; } @@ -954,37 +1040,36 @@ bool cmQtAutoGeneratorMocUic::JobUicT::UpdateRequired(WorkerT& wrk) return false; } -void cmQtAutoGeneratorMocUic::JobUicT::GenerateUic(WorkerT& wrk) +void cmQtAutoGeneratorMocUic::JobUicT::GenerateUic() { // Make sure the parent directory exists - if (!wrk.FileSys().MakeParentDirectory(BuildFile)) { - wrk.LogFileError(GenT::UIC, BuildFile, - "Could not create parent directory."); + if (!FileSys().MakeParentDirectory(BuildFile)) { + LogFileError(GenT::UIC, BuildFile, "Could not create parent directory."); return; } { // Compose uic command std::vector<std::string> cmd; - cmd.push_back(wrk.Uic().Executable); + cmd.push_back(Gen()->Uic().Executable); { - std::vector<std::string> allOpts = wrk.Uic().TargetOptions; - auto optionIt = wrk.Uic().Options.find(SourceFile); - if (optionIt != wrk.Uic().Options.end()) { + std::vector<std::string> allOpts = Gen()->Uic().TargetOptions; + auto optionIt = Gen()->Uic().Options.find(SourceFile); + if (optionIt != Gen()->Uic().Options.end()) { UicMergeOptions(allOpts, optionIt->second, - (wrk.Base().QtVersionMajor == 5)); + (Gen()->Base().QtVersionMajor == 5)); } cmd.insert(cmd.end(), allOpts.begin(), allOpts.end()); } cmd.emplace_back("-o"); - cmd.push_back(BuildFile); - cmd.push_back(SourceFile); + cmd.emplace_back(BuildFile); + cmd.emplace_back(SourceFile); - ProcessResultT result; - if (wrk.RunProcess(GenT::UIC, result, cmd)) { + cmWorkerPool::ProcessResultT result; + if (RunProcess(GenT::UIC, result, cmd)) { // Uic command success // Print uic output if (!result.StdOut.empty()) { - wrk.LogInfo(GenT::UIC, result.StdOut); + Log().Info(GenT::UIC, result.StdOut); } } else { // Uic command failed @@ -997,144 +1082,16 @@ void cmQtAutoGeneratorMocUic::JobUicT::GenerateUic(WorkerT& wrk) emsg += Quoted(IncluderFile); emsg += ".\n"; emsg += result.ErrorMessage; - wrk.LogCommandError(GenT::UIC, emsg, cmd, result.StdOut); + LogCommandError(GenT::UIC, emsg, cmd, result.StdOut); } - wrk.FileSys().FileRemove(BuildFile); - } - } -} - -cmQtAutoGeneratorMocUic::WorkerT::WorkerT(cmQtAutoGeneratorMocUic* gen, - uv_loop_t* uvLoop) - : Gen_(gen) -{ - // Initialize uv asynchronous callback for process starting - ProcessRequest_.init(*uvLoop, &WorkerT::UVProcessStart, this); - // Start thread - Thread_ = std::thread(&WorkerT::Loop, this); -} - -cmQtAutoGeneratorMocUic::WorkerT::~WorkerT() -{ - // Join thread - if (Thread_.joinable()) { - Thread_.join(); - } -} - -void cmQtAutoGeneratorMocUic::WorkerT::LogInfo( - GenT genType, std::string const& message) const -{ - Log().Info(genType, message); -} - -void cmQtAutoGeneratorMocUic::WorkerT::LogWarning( - GenT genType, std::string const& message) const -{ - Log().Warning(genType, message); -} - -void cmQtAutoGeneratorMocUic::WorkerT::LogFileWarning( - GenT genType, std::string const& filename, std::string const& message) const -{ - Log().WarningFile(genType, filename, message); -} - -void cmQtAutoGeneratorMocUic::WorkerT::LogError( - GenT genType, std::string const& message) const -{ - Gen().ParallelRegisterJobError(); - Log().Error(genType, message); -} - -void cmQtAutoGeneratorMocUic::WorkerT::LogFileError( - GenT genType, std::string const& filename, std::string const& message) const -{ - Gen().ParallelRegisterJobError(); - Log().ErrorFile(genType, filename, message); -} - -void cmQtAutoGeneratorMocUic::WorkerT::LogCommandError( - GenT genType, std::string const& message, - std::vector<std::string> const& command, std::string const& output) const -{ - Gen().ParallelRegisterJobError(); - Log().ErrorCommand(genType, message, command, output); -} - -bool cmQtAutoGeneratorMocUic::WorkerT::RunProcess( - GenT genType, ProcessResultT& result, - std::vector<std::string> const& command) -{ - if (command.empty()) { - return false; - } - - // Create process instance - { - std::lock_guard<std::mutex> lock(ProcessMutex_); - Process_ = cm::make_unique<ReadOnlyProcessT>(); - Process_->setup(&result, true, command, Gen().Base().AutogenBuildDir); - } - - // Send asynchronous process start request to libuv loop - ProcessRequest_.send(); - - // Log command - if (this->Log().Verbose()) { - std::string msg = "Running command:\n"; - msg += QuotedCommand(command); - msg += '\n'; - this->LogInfo(genType, msg); - } - - // Wait until the process has been finished and destroyed - { - std::unique_lock<std::mutex> ulock(ProcessMutex_); - while (Process_) { - ProcessCondition_.wait(ulock); + FileSys().FileRemove(BuildFile); } } - return !result.error(); } -void cmQtAutoGeneratorMocUic::WorkerT::Loop() +void cmQtAutoGeneratorMocUic::JobFinishT::Process() { - while (true) { - Gen().WorkerSwapJob(JobHandle_); - if (JobHandle_) { - JobHandle_->Process(*this); - } else { - break; - } - } -} - -void cmQtAutoGeneratorMocUic::WorkerT::UVProcessStart(uv_async_t* handle) -{ - auto& wrk = *reinterpret_cast<WorkerT*>(handle->data); - { - std::lock_guard<std::mutex> lock(wrk.ProcessMutex_); - if (wrk.Process_ && !wrk.Process_->IsStarted()) { - wrk.Process_->start(handle->loop, [&wrk] { wrk.UVProcessFinished(); }); - } - } - - if (!wrk.Process_->IsStarted()) { - wrk.UVProcessFinished(); - } -} - -void cmQtAutoGeneratorMocUic::WorkerT::UVProcessFinished() -{ - { - std::lock_guard<std::mutex> lock(ProcessMutex_); - if (Process_ && (Process_->IsFinished() || !Process_->IsStarted())) { - Process_.reset(); - } - } - // Notify idling thread - ProcessCondition_.notify_one(); + Gen()->AbortSuccess(); } cmQtAutoGeneratorMocUic::cmQtAutoGeneratorMocUic() @@ -1147,24 +1104,9 @@ cmQtAutoGeneratorMocUic::cmQtAutoGeneratorMocUic() "[\"<](([^ \">]+/)?moc_[^ \">/]+\\.cpp|[^ \">]+\\.moc)[\">]"); Uic_.RegExpInclude.compile("(^|\n)[ \t]*#[ \t]*include[ \t]+" "[\"<](([^ \">]+/)?ui_[^ \">/]+\\.h)[\">]"); - - // Initialize libuv loop - uv_disable_stdio_inheritance(); -#ifdef CMAKE_UV_SIGNAL_HACK - UVHackRAII_ = cm::make_unique<cmUVSignalHackRAII>(); -#endif - UVLoop_ = cm::make_unique<uv_loop_t>(); - uv_loop_init(UVLoop()); - - // Initialize libuv asynchronous iteration request - UVRequest().init(*UVLoop(), &cmQtAutoGeneratorMocUic::UVPollStage, this); } -cmQtAutoGeneratorMocUic::~cmQtAutoGeneratorMocUic() -{ - // Close libuv loop - uv_loop_close(UVLoop()); -} +cmQtAutoGeneratorMocUic::~cmQtAutoGeneratorMocUic() = default; bool cmQtAutoGeneratorMocUic::Init(cmMakefile* makefile) { @@ -1364,7 +1306,7 @@ bool cmQtAutoGeneratorMocUic::Init(cmMakefile* makefile) Moc_.PredefsCmd = InfoGetList("AM_MOC_PREDEFS_CMD"); // Install moc predefs job if (!Moc().PredefsCmd.empty()) { - JobQueues_.MocPredefs.emplace_back(cm::make_unique<JobMocPredefsT>()); + WorkerPool().EmplaceJob<JobMocPredefsT>(); } } @@ -1400,46 +1342,48 @@ bool cmQtAutoGeneratorMocUic::Init(cmMakefile* makefile) } // - Headers and sources + // Add sources { - auto addHeader = [this](std::string&& hdr, bool moc, bool uic) { - this->JobQueues_.Headers.emplace_back( - cm::make_unique<JobParseT>(std::move(hdr), moc, uic, true)); - }; auto addSource = [this](std::string&& src, bool moc, bool uic) { - this->JobQueues_.Sources.emplace_back( - cm::make_unique<JobParseT>(std::move(src), moc, uic, false)); + WorkerPool().EmplaceJob<JobParseT>(std::move(src), moc, uic, false); }; - - // Add headers - for (std::string& hdr : InfoGetList("AM_HEADERS")) { - addHeader(std::move(hdr), true, true); + for (std::string& src : InfoGetList("AM_SOURCES")) { + addSource(std::move(src), true, true); } if (Moc().Enabled) { - for (std::string& hdr : InfoGetList("AM_MOC_HEADERS")) { - addHeader(std::move(hdr), true, false); + for (std::string& src : InfoGetList("AM_MOC_SOURCES")) { + addSource(std::move(src), true, false); } } if (Uic().Enabled) { - for (std::string& hdr : InfoGetList("AM_UIC_HEADERS")) { - addHeader(std::move(hdr), false, true); + for (std::string& src : InfoGetList("AM_UIC_SOURCES")) { + addSource(std::move(src), false, true); } } - - // Add sources - for (std::string& src : InfoGetList("AM_SOURCES")) { - addSource(std::move(src), true, true); + } + // Add Fence job + WorkerPool().EmplaceJob<JobFenceT>(); + // Add headers + { + auto addHeader = [this](std::string&& hdr, bool moc, bool uic) { + WorkerPool().EmplaceJob<JobParseT>(std::move(hdr), moc, uic, true); + }; + for (std::string& hdr : InfoGetList("AM_HEADERS")) { + addHeader(std::move(hdr), true, true); } if (Moc().Enabled) { - for (std::string& src : InfoGetList("AM_MOC_SOURCES")) { - addSource(std::move(src), true, false); + for (std::string& hdr : InfoGetList("AM_MOC_HEADERS")) { + addHeader(std::move(hdr), true, false); } } if (Uic().Enabled) { - for (std::string& src : InfoGetList("AM_UIC_SOURCES")) { - addSource(std::move(src), false, true); + for (std::string& hdr : InfoGetList("AM_UIC_HEADERS")) { + addHeader(std::move(hdr), false, true); } } } + // Addpost parse fence job + WorkerPool().EmplaceJob<JobPostParseT>(); // Init derived information // ------------------------ @@ -1536,93 +1480,20 @@ bool cmQtAutoGeneratorMocUic::Init(cmMakefile* makefile) bool cmQtAutoGeneratorMocUic::Process() { - // Run libuv event loop - UVRequest().send(); - if (uv_run(UVLoop(), UV_RUN_DEFAULT) == 0) { - if (JobError_) { - return false; - } - } else { + SettingsFileRead(); + if (!CreateDirectories()) { return false; } - return true; -} - -void cmQtAutoGeneratorMocUic::UVPollStage(uv_async_t* handle) -{ - reinterpret_cast<cmQtAutoGeneratorMocUic*>(handle->data)->PollStage(); -} -void cmQtAutoGeneratorMocUic::PollStage() -{ - switch (Stage_) { - case StageT::SETTINGS_READ: - SettingsFileRead(); - SetStage(StageT::CREATE_DIRECTORIES); - break; - case StageT::CREATE_DIRECTORIES: - CreateDirectories(); - SetStage(StageT::PARSE_SOURCES); - break; - case StageT::PARSE_SOURCES: - if (ThreadsStartJobs(JobQueues_.Sources)) { - SetStage(StageT::PARSE_HEADERS); - } - break; - case StageT::PARSE_HEADERS: - if (ThreadsStartJobs(JobQueues_.Headers)) { - SetStage(StageT::MOC_PREDEFS); - } - break; - case StageT::MOC_PREDEFS: - if (ThreadsStartJobs(JobQueues_.MocPredefs)) { - SetStage(StageT::MOC_PROCESS); - } - break; - case StageT::MOC_PROCESS: - if (ThreadsStartJobs(JobQueues_.Moc)) { - SetStage(StageT::MOCS_COMPILATION); - } - break; - case StageT::MOCS_COMPILATION: - if (ThreadsJobsDone()) { - MocGenerateCompilation(); - SetStage(StageT::UIC_PROCESS); - } - break; - case StageT::UIC_PROCESS: - if (ThreadsStartJobs(JobQueues_.Uic)) { - SetStage(StageT::SETTINGS_WRITE); - } - break; - case StageT::SETTINGS_WRITE: - SettingsFileWrite(); - SetStage(StageT::FINISH); - break; - case StageT::FINISH: - if (ThreadsJobsDone()) { - // Clear all libuv handles - ThreadsStop(); - UVRequest().reset(); - // Set highest END stage manually - Stage_ = StageT::END; - } - break; - case StageT::END: - break; + if (!WorkerPool_.Process(Base().NumThreads, this)) { + return false; } -} -void cmQtAutoGeneratorMocUic::SetStage(StageT stage) -{ if (JobError_) { - stage = StageT::FINISH; - } - // Only allow to increase the stage - if (Stage_ < stage) { - Stage_ = stage; - UVRequest().send(); + return false; } + + return SettingsFileWrite(); } void cmQtAutoGeneratorMocUic::SettingsFileRead() @@ -1691,11 +1562,10 @@ void cmQtAutoGeneratorMocUic::SettingsFileRead() } } -void cmQtAutoGeneratorMocUic::SettingsFileWrite() +bool cmQtAutoGeneratorMocUic::SettingsFileWrite() { - std::lock_guard<std::mutex> jobsLock(JobsMutex_); // Only write if any setting changed - if (!JobError_ && (Moc().SettingsChanged || Uic().SettingsChanged)) { + if (Moc().SettingsChanged || Uic().SettingsChanged) { if (Log().Verbose()) { Log().Info(GenT::GEN, "Writing settings file " + Quoted(SettingsFile_)); } @@ -1721,246 +1591,136 @@ void cmQtAutoGeneratorMocUic::SettingsFileWrite() "Settings file writing failed. " + error); // Remove old settings file to trigger a full rebuild on the next run FileSys().FileRemove(SettingsFile_); - RegisterJobError(); + return false; } } + return true; } -void cmQtAutoGeneratorMocUic::CreateDirectories() +bool cmQtAutoGeneratorMocUic::CreateDirectories() { // Create AUTOGEN include directory if (!FileSys().MakeDirectory(Base().AutogenIncludeDir)) { Log().ErrorFile(GenT::GEN, Base().AutogenIncludeDir, "Could not create directory."); - RegisterJobError(); + return false; } + return true; } -bool cmQtAutoGeneratorMocUic::ThreadsStartJobs(JobQueueT& queue) +// Private method that requires cmQtAutoGeneratorMocUic::JobsMutex_ to be +// locked +void cmQtAutoGeneratorMocUic::Abort(bool error) { - bool done = false; - std::size_t queueSize = queue.size(); - - // Change the active queue - { - std::lock_guard<std::mutex> jobsLock(JobsMutex_); - // Check if there are still unfinished jobs from the previous queue - if (JobsRemain_ == 0) { - if (!JobThreadsAbort_) { - JobQueue_.swap(queue); - JobsRemain_ = queueSize; - } else { - // Abort requested - queue.clear(); - queueSize = 0; - } - done = true; - } + if (error) { + JobError_.store(true); } + WorkerPool_.Abort(); +} - if (done && (queueSize != 0)) { - // Start new threads on demand - if (Workers_.empty()) { - Workers_.resize(Base().NumThreads); - for (auto& item : Workers_) { - item = cm::make_unique<WorkerT>(this, UVLoop()); +bool cmQtAutoGeneratorMocUic::ParallelJobPushMoc( + cmWorkerPool::JobHandleT&& jobHandle) +{ + JobMocT const& mocJob(static_cast<JobMocT&>(*jobHandle)); + // Do additional tests if this is an included moc job + if (!mocJob.IncludeString.empty()) { + std::lock_guard<std::mutex> guard(MocMetaMutex_); + // Register included moc file + MocIncludedFiles_.emplace(mocJob.SourceFile); + + // Check if the same moc file would be generated from a different + // source file. + auto const range = MocIncludes_.equal_range(mocJob.IncludeString); + for (auto it = range.first; it != range.second; ++it) { + if (it->second[0] == mocJob.SourceFile) { + // The output file already gets generated + return true; } - } else { - // Notify threads - if (queueSize == 1) { - JobsConditionRead_.notify_one(); - } else { - JobsConditionRead_.notify_all(); + { + // The output file already gets generated - from a different source + // file! + std::string error = "The two source files\n "; + error += Quoted(mocJob.IncluderFile); + error += " and\n "; + error += Quoted(it->second[1]); + error += "\ncontain the same moc include string "; + error += Quoted(mocJob.IncludeString); + error += "\nbut the moc file would be generated from different " + "source files\n "; + error += Quoted(mocJob.SourceFile); + error += " and\n "; + error += Quoted(it->second[0]); + error += ".\nConsider to\n" + "- not include the \"moc_<NAME>.cpp\" file\n" + "- add a directory prefix to a \"<NAME>.moc\" include " + "(e.g \"sub/<NAME>.moc\")\n" + "- rename the source file(s)\n"; + Log().Error(GenT::MOC, error); + AbortError(); + return false; } } - } - - return done; -} -void cmQtAutoGeneratorMocUic::ThreadsStop() -{ - if (!Workers_.empty()) { - // Clear all jobs - { - std::lock_guard<std::mutex> jobsLock(JobsMutex_); - JobThreadsAbort_ = true; - JobsRemain_ -= JobQueue_.size(); - JobQueue_.clear(); - - JobQueues_.Sources.clear(); - JobQueues_.Headers.clear(); - JobQueues_.MocPredefs.clear(); - JobQueues_.Moc.clear(); - JobQueues_.Uic.clear(); - } - // Wake threads - JobsConditionRead_.notify_all(); - // Join and clear threads - Workers_.clear(); + // We're still here so register this job + MocIncludes_.emplace_hint(range.first, mocJob.IncludeString, + std::array<std::string, 2>{ + { mocJob.SourceFile, mocJob.IncluderFile } }); } + return WorkerPool_.PushJob(std::move(jobHandle)); } -bool cmQtAutoGeneratorMocUic::ThreadsJobsDone() +bool cmQtAutoGeneratorMocUic::ParallelJobPushUic( + cmWorkerPool::JobHandleT&& jobHandle) { - std::lock_guard<std::mutex> jobsLock(JobsMutex_); - return (JobsRemain_ == 0); -} - -void cmQtAutoGeneratorMocUic::WorkerSwapJob(JobHandleT& jobHandle) -{ - bool const jobProcessed(jobHandle); - if (jobProcessed) { - jobHandle.reset(); - } + const JobUicT& uicJob(static_cast<JobUicT&>(*jobHandle)); { - std::unique_lock<std::mutex> jobsLock(JobsMutex_); - // Reduce the remaining job count and notify the libuv loop - // when all jobs are done - if (jobProcessed) { - --JobsRemain_; - if (JobsRemain_ == 0) { - UVRequest().send(); + std::lock_guard<std::mutex> guard(UicMetaMutex_); + // Check if the same uic file would be generated from a different + // source file. + auto const range = UicIncludes_.equal_range(uicJob.IncludeString); + for (auto it = range.first; it != range.second; ++it) { + if (it->second[0] == uicJob.SourceFile) { + // The output file already gets generated + return true; } - } - // Wait for new jobs - while (!JobThreadsAbort_ && JobQueue_.empty()) { - JobsConditionRead_.wait(jobsLock); - } - // Try to pick up a new job handle - if (!JobThreadsAbort_ && !JobQueue_.empty()) { - jobHandle = std::move(JobQueue_.front()); - JobQueue_.pop_front(); - } - } -} - -void cmQtAutoGeneratorMocUic::ParallelRegisterJobError() -{ - std::lock_guard<std::mutex> jobsLock(JobsMutex_); - RegisterJobError(); -} - -// Private method that requires cmQtAutoGeneratorMocUic::JobsMutex_ to be -// locked -void cmQtAutoGeneratorMocUic::RegisterJobError() -{ - JobError_ = true; - if (!JobThreadsAbort_) { - JobThreadsAbort_ = true; - // Clear remaining jobs - if (JobsRemain_ != 0) { - JobsRemain_ -= JobQueue_.size(); - JobQueue_.clear(); - } - } -} - -bool cmQtAutoGeneratorMocUic::ParallelJobPushMoc(JobHandleT& jobHandle) -{ - std::lock_guard<std::mutex> jobsLock(JobsMutex_); - if (!JobThreadsAbort_) { - bool pushJobHandle = true; - // Do additional tests if this is an included moc job - const JobMocT& mocJob(static_cast<JobMocT&>(*jobHandle)); - if (!mocJob.IncludeString.empty()) { - // Register included moc file and look for collisions - MocIncludedFiles_.emplace(mocJob.SourceFile); - if (!MocIncludedStrings_.emplace(mocJob.IncludeString).second) { - // Another source file includes the same moc file! - for (const JobHandleT& otherHandle : JobQueues_.Moc) { - const JobMocT& otherJob(static_cast<JobMocT&>(*otherHandle)); - if (otherJob.IncludeString == mocJob.IncludeString) { - // Check if the same moc file would be generated from different - // source files which is an error. - if (otherJob.SourceFile != mocJob.SourceFile) { - // Include string collision - std::string error = "The two source files\n "; - error += Quoted(mocJob.IncluderFile); - error += " and\n "; - error += Quoted(otherJob.IncluderFile); - error += "\ncontain the same moc include string "; - error += Quoted(mocJob.IncludeString); - error += "\nbut the moc file would be generated from different " - "source files\n "; - error += Quoted(mocJob.SourceFile); - error += " and\n "; - error += Quoted(otherJob.SourceFile); - error += ".\nConsider to\n" - "- not include the \"moc_<NAME>.cpp\" file\n" - "- add a directory prefix to a \"<NAME>.moc\" include " - "(e.g \"sub/<NAME>.moc\")\n" - "- rename the source file(s)\n"; - Log().Error(GenT::MOC, error); - RegisterJobError(); - } - // Do not push this job in since the included moc file already - // gets generated by an other job. - pushJobHandle = false; - break; - } - } + { + // The output file already gets generated - from a different .ui + // file! + std::string error = "The two source files\n "; + error += Quoted(uicJob.IncluderFile); + error += " and\n "; + error += Quoted(it->second[1]); + error += "\ncontain the same uic include string "; + error += Quoted(uicJob.IncludeString); + error += "\nbut the uic file would be generated from different " + "source files\n "; + error += Quoted(uicJob.SourceFile); + error += " and\n "; + error += Quoted(it->second[0]); + error += + ".\nConsider to\n" + "- add a directory prefix to a \"ui_<NAME>.h\" include " + "(e.g \"sub/ui_<NAME>.h\")\n" + "- rename the <NAME>.ui file(s) and adjust the \"ui_<NAME>.h\" " + "include(s)\n"; + Log().Error(GenT::UIC, error); + AbortError(); + return false; } } - // Push job on demand - if (pushJobHandle) { - JobQueues_.Moc.emplace_back(std::move(jobHandle)); - } - } - return !JobError_; -} -bool cmQtAutoGeneratorMocUic::ParallelJobPushUic(JobHandleT& jobHandle) -{ - std::lock_guard<std::mutex> jobsLock(JobsMutex_); - if (!JobThreadsAbort_) { - bool pushJobHandle = true; - // Look for include collisions. - const JobUicT& uicJob(static_cast<JobUicT&>(*jobHandle)); - for (const JobHandleT& otherHandle : JobQueues_.Uic) { - const JobUicT& otherJob(static_cast<JobUicT&>(*otherHandle)); - if (otherJob.IncludeString == uicJob.IncludeString) { - // Check if the same uic file would be generated from different - // source files which would be an error. - if (otherJob.SourceFile != uicJob.SourceFile) { - // Include string collision - std::string error = "The two source files\n "; - error += Quoted(uicJob.IncluderFile); - error += " and\n "; - error += Quoted(otherJob.IncluderFile); - error += "\ncontain the same uic include string "; - error += Quoted(uicJob.IncludeString); - error += "\nbut the uic file would be generated from different " - "source files\n "; - error += Quoted(uicJob.SourceFile); - error += " and\n "; - error += Quoted(otherJob.SourceFile); - error += - ".\nConsider to\n" - "- add a directory prefix to a \"ui_<NAME>.h\" include " - "(e.g \"sub/ui_<NAME>.h\")\n" - "- rename the <NAME>.ui file(s) and adjust the \"ui_<NAME>.h\" " - "include(s)\n"; - Log().Error(GenT::UIC, error); - RegisterJobError(); - } - // Do not push this job in since the uic file already - // gets generated by an other job. - pushJobHandle = false; - break; - } - } - if (pushJobHandle) { - JobQueues_.Uic.emplace_back(std::move(jobHandle)); - } + // We're still here so register this job + UicIncludes_.emplace_hint(range.first, uicJob.IncludeString, + std::array<std::string, 2>{ + { uicJob.SourceFile, uicJob.IncluderFile } }); } - return !JobError_; + return WorkerPool_.PushJob(std::move(jobHandle)); } bool cmQtAutoGeneratorMocUic::ParallelMocIncluded( std::string const& sourceFile) { - std::lock_guard<std::mutex> mocLock(JobsMutex_); + std::lock_guard<std::mutex> guard(MocMetaMutex_); return (MocIncludedFiles_.find(sourceFile) != MocIncludedFiles_.end()); } @@ -1969,7 +1729,7 @@ std::string cmQtAutoGeneratorMocUic::ParallelMocAutoRegister( { std::string res; { - std::lock_guard<std::mutex> mocLock(JobsMutex_); + std::lock_guard<std::mutex> mocLock(MocMetaMutex_); res = baseName; res += ".cpp"; if (MocAutoFiles_.find(res) == MocAutoFiles_.end()) { @@ -1990,63 +1750,3 @@ std::string cmQtAutoGeneratorMocUic::ParallelMocAutoRegister( } return res; } - -void cmQtAutoGeneratorMocUic::ParallelMocAutoUpdated() -{ - std::lock_guard<std::mutex> mocLock(JobsMutex_); - MocAutoFileUpdated_ = true; -} - -void cmQtAutoGeneratorMocUic::MocGenerateCompilation() -{ - std::lock_guard<std::mutex> mocLock(JobsMutex_); - if (!JobError_ && Moc().Enabled) { - // Write mocs compilation build file - { - // Compose mocs compilation file content - std::string content = - "// This file is autogenerated. Changes will be overwritten.\n"; - if (MocAutoFiles_.empty()) { - // Placeholder content - content += "// No files found that require moc or the moc files are " - "included\n"; - content += "enum some_compilers { need_more_than_nothing };\n"; - } else { - // Valid content - char const sbeg = Base().MultiConfig ? '<' : '"'; - char const send = Base().MultiConfig ? '>' : '"'; - for (std::string const& mocfile : MocAutoFiles_) { - content += "#include "; - content += sbeg; - content += mocfile; - content += send; - content += '\n'; - } - } - - std::string const& compAbs = Moc().CompFileAbs; - if (FileSys().FileDiffers(compAbs, content)) { - // Actually write mocs compilation file - if (Log().Verbose()) { - Log().Info(GenT::MOC, "Generating MOC compilation " + compAbs); - } - std::string error; - if (!FileSys().FileWrite(compAbs, content, &error)) { - Log().ErrorFile(GenT::MOC, compAbs, - "mocs compilation file writing failed. " + error); - RegisterJobError(); - return; - } - } else if (MocAutoFileUpdated_) { - // Only touch mocs compilation file - if (Log().Verbose()) { - Log().Info(GenT::MOC, "Touching mocs compilation " + compAbs); - } - FileSys().Touch(compAbs); - } - } - // Write mocs compilation wrapper file - if (Base().MultiConfig) { - } - } -} diff --git a/Source/cmQtAutoGeneratorMocUic.h b/Source/cmQtAutoGeneratorMocUic.h index 27d73a741b..4efc2c62f2 100644 --- a/Source/cmQtAutoGeneratorMocUic.h +++ b/Source/cmQtAutoGeneratorMocUic.h @@ -7,20 +7,16 @@ #include "cmQtAutoGen.h" #include "cmQtAutoGenerator.h" -#include "cmUVHandlePtr.h" -#include "cmUVSignalHackRAII.h" // IWYU pragma: keep -#include "cm_uv.h" +#include "cmWorkerPool.h" #include "cmsys/RegularExpression.hxx" -#include <condition_variable> -#include <cstddef> -#include <deque> +#include <array> +#include <atomic> #include <map> #include <memory> // IWYU pragma: keep #include <mutex> #include <set> #include <string> -#include <thread> #include <unordered_set> #include <utility> #include <vector> @@ -39,7 +35,7 @@ public: public: // -- Types - class WorkerT; + typedef std::multimap<std::string, std::array<std::string, 2>> IncludesMap; /// @brief Search key plus regular expression pair /// @@ -173,31 +169,71 @@ public: cmsys::RegularExpression RegExpInclude; }; - /// @brief Abstract job class for threaded processing + /// @brief Abstract job class for concurrent job processing /// - class JobT + class JobT : public cmWorkerPool::JobT { - public: - JobT() = default; - virtual ~JobT() = default; + protected: + /** + * @brief Protected default constructor + */ + JobT(bool fence = false) + : cmWorkerPool::JobT(fence) + { + } + + //! Get the generator. Only valid during Process() call! + cmQtAutoGeneratorMocUic* Gen() const + { + return static_cast<cmQtAutoGeneratorMocUic*>(UserData()); + }; + + //! Get the file system interface. Only valid during Process() call! + FileSystem& FileSys() { return Gen()->FileSys(); } + //! Get the logger. Only valid during Process() call! + Logger& Log() { return Gen()->Log(); } + + // -- Error logging with automatic abort + void LogError(GenT genType, std::string const& message) const; + void LogFileError(GenT genType, std::string const& filename, + std::string const& message) const; + void LogCommandError(GenT genType, std::string const& message, + std::vector<std::string> const& command, + std::string const& output) const; - JobT(JobT const&) = delete; - JobT& operator=(JobT const&) = delete; + /** + * @brief Run an external process. Use only during Process() call! + */ + bool RunProcess(GenT genType, cmWorkerPool::ProcessResultT& result, + std::vector<std::string> const& command); + }; - // -- Abstract processing interface - virtual void Process(WorkerT& wrk) = 0; + /// @brief Fence job utility class + /// + class JobFenceT : public JobT + { + public: + JobFenceT() + : JobT(true) + { + } + void Process() override{}; }; - // Job management types - typedef std::unique_ptr<JobT> JobHandleT; - typedef std::deque<JobHandleT> JobQueueT; + /// @brief Generate moc_predefs.h + /// + class JobMocPredefsT : public JobT + { + private: + void Process() override; + }; - /// @brief Parse source job + /// @brief Parses a source file /// class JobParseT : public JobT { public: - JobParseT(std::string&& fileName, bool moc, bool uic, bool header = false) + JobParseT(std::string fileName, bool moc, bool uic, bool header = false) : FileName(std::move(fileName)) , AutoMoc(moc) , AutoUic(uic) @@ -213,18 +249,15 @@ public: std::string FileBase; }; - void Process(WorkerT& wrk) override; - bool ParseMocSource(WorkerT& wrk, MetaT const& meta); - bool ParseMocHeader(WorkerT& wrk, MetaT const& meta); - std::string MocStringHeaders(WorkerT& wrk, - std::string const& fileBase) const; - std::string MocFindIncludedHeader(WorkerT& wrk, - std::string const& includerDir, + void Process() override; + bool ParseMocSource(MetaT const& meta); + bool ParseMocHeader(MetaT const& meta); + std::string MocStringHeaders(std::string const& fileBase) const; + std::string MocFindIncludedHeader(std::string const& includerDir, std::string const& includeBase); - bool ParseUic(WorkerT& wrk, MetaT const& meta); - bool ParseUicInclude(WorkerT& wrk, MetaT const& meta, - std::string&& includeString); - std::string UicFindIncludedFile(WorkerT& wrk, MetaT const& meta, + bool ParseUic(MetaT const& meta); + bool ParseUicInclude(MetaT const& meta, std::string&& includeString); + std::string UicFindIncludedFile(MetaT const& meta, std::string const& includeString); private: @@ -234,12 +267,20 @@ public: bool Header = false; }; - /// @brief Generate moc_predefs + /// @brief Generates additional jobs after all files have been parsed /// - class JobMocPredefsT : public JobT + class JobPostParseT : public JobFenceT { private: - void Process(WorkerT& wrk) override; + void Process() override; + }; + + /// @brief Generate mocs_compilation.cpp + /// + class JobMocsCompilationT : public JobFenceT + { + private: + void Process() override; }; /// @brief Moc a file job @@ -247,20 +288,20 @@ public: class JobMocT : public JobT { public: - JobMocT(std::string&& sourceFile, std::string includerFile, - std::string&& includeString) + JobMocT(std::string sourceFile, std::string includerFile, + std::string includeString) : SourceFile(std::move(sourceFile)) , IncluderFile(std::move(includerFile)) , IncludeString(std::move(includeString)) { } - void FindDependencies(WorkerT& wrk, std::string const& content); + void FindDependencies(std::string const& content); private: - void Process(WorkerT& wrk) override; - bool UpdateRequired(WorkerT& wrk); - void GenerateMoc(WorkerT& wrk); + void Process() override; + bool UpdateRequired(); + void GenerateMoc(); public: std::string SourceFile; @@ -276,8 +317,8 @@ public: class JobUicT : public JobT { public: - JobUicT(std::string&& sourceFile, std::string includerFile, - std::string&& includeString) + JobUicT(std::string sourceFile, std::string includerFile, + std::string includeString) : SourceFile(std::move(sourceFile)) , IncluderFile(std::move(includerFile)) , IncludeString(std::move(includeString)) @@ -285,9 +326,9 @@ public: } private: - void Process(WorkerT& wrk) override; - bool UpdateRequired(WorkerT& wrk); - void GenerateUic(WorkerT& wrk); + void Process() override; + bool UpdateRequired(); + void GenerateUic(); public: std::string SourceFile; @@ -296,80 +337,12 @@ public: std::string BuildFile; }; - /// @brief Worker Thread + /// @brief The last job /// - class WorkerT + class JobFinishT : public JobFenceT { - public: - WorkerT(cmQtAutoGeneratorMocUic* gen, uv_loop_t* uvLoop); - ~WorkerT(); - - WorkerT(WorkerT const&) = delete; - WorkerT& operator=(WorkerT const&) = delete; - - // -- Const accessors - cmQtAutoGeneratorMocUic& Gen() const { return *Gen_; } - Logger& Log() const { return Gen_->Log(); } - FileSystem& FileSys() const { return Gen_->FileSys(); } - const BaseSettingsT& Base() const { return Gen_->Base(); } - const MocSettingsT& Moc() const { return Gen_->Moc(); } - const UicSettingsT& Uic() const { return Gen_->Uic(); } - - // -- Log info - void LogInfo(GenT genType, std::string const& message) const; - // -- Log warning - void LogWarning(GenT genType, std::string const& message) const; - void LogFileWarning(GenT genType, std::string const& filename, - std::string const& message) const; - // -- Log error - void LogError(GenT genType, std::string const& message) const; - void LogFileError(GenT genType, std::string const& filename, - std::string const& message) const; - void LogCommandError(GenT genType, std::string const& message, - std::vector<std::string> const& command, - std::string const& output) const; - - // -- External processes - /// @brief Verbose logging version - bool RunProcess(GenT genType, ProcessResultT& result, - std::vector<std::string> const& command); - private: - /// @brief Thread main loop - void Loop(); - - // -- Libuv callbacks - static void UVProcessStart(uv_async_t* handle); - void UVProcessFinished(); - - private: - // -- Generator - cmQtAutoGeneratorMocUic* Gen_; - // -- Job handle - JobHandleT JobHandle_; - // -- Process management - std::mutex ProcessMutex_; - cm::uv_async_ptr ProcessRequest_; - std::condition_variable ProcessCondition_; - std::unique_ptr<ReadOnlyProcessT> Process_; - // -- System thread - std::thread Thread_; - }; - - /// @brief Processing stage - enum class StageT - { - SETTINGS_READ, - CREATE_DIRECTORIES, - PARSE_SOURCES, - PARSE_HEADERS, - MOC_PREDEFS, - MOC_PROCESS, - MOCS_COMPILATION, - UIC_PROCESS, - SETTINGS_WRITE, - FINISH, - END + void Process() override; }; // -- Const settings interface @@ -377,41 +350,39 @@ public: const MocSettingsT& Moc() const { return this->Moc_; } const UicSettingsT& Uic() const { return this->Uic_; } - // -- Worker thread interface - void WorkerSwapJob(JobHandleT& jobHandle); // -- Parallel job processing interface - void ParallelRegisterJobError(); - bool ParallelJobPushMoc(JobHandleT& jobHandle); - bool ParallelJobPushUic(JobHandleT& jobHandle); - bool ParallelMocIncluded(std::string const& sourceFile); + cmWorkerPool& WorkerPool() { return WorkerPool_; } + void AbortError() { Abort(true); } + void AbortSuccess() { Abort(false); } + bool ParallelJobPushMoc(cmWorkerPool::JobHandleT&& jobHandle); + bool ParallelJobPushUic(cmWorkerPool::JobHandleT&& jobHandle); + + // -- Mocs compilation include file updated flag + void ParallelMocAutoUpdated() { MocAutoFileUpdated_.store(true); } + bool MocAutoFileUpdated() const { return MocAutoFileUpdated_.load(); } + + // -- Mocs compilation file register std::string ParallelMocAutoRegister(std::string const& baseName); - void ParallelMocAutoUpdated(); + bool ParallelMocIncluded(std::string const& sourceFile); + std::set<std::string> const& MocAutoFiles() const + { + return this->MocAutoFiles_; + } private: // -- Utility accessors Logger& Log() { return Logger_; } FileSystem& FileSys() { return FileSys_; } - // -- libuv loop accessors - uv_loop_t* UVLoop() { return UVLoop_.get(); } - cm::uv_async_ptr& UVRequest() { return UVRequest_; } // -- Abstract processing interface bool Init(cmMakefile* makefile) override; bool Process() override; - // -- Process stage - static void UVPollStage(uv_async_t* handle); - void PollStage(); - void SetStage(StageT stage); // -- Settings file void SettingsFileRead(); - void SettingsFileWrite(); + bool SettingsFileWrite(); // -- Thread processing - bool ThreadsStartJobs(JobQueueT& queue); - bool ThreadsJobsDone(); - void ThreadsStop(); - void RegisterJobError(); + void Abort(bool error); // -- Generation - void CreateDirectories(); - void MocGenerateCompilation(); + bool CreateDirectories(); private: // -- Utility @@ -421,39 +392,22 @@ private: BaseSettingsT Base_; MocSettingsT Moc_; UicSettingsT Uic_; - // -- libuv loop -#ifdef CMAKE_UV_SIGNAL_HACK - std::unique_ptr<cmUVSignalHackRAII> UVHackRAII_; -#endif - std::unique_ptr<uv_loop_t> UVLoop_; - cm::uv_async_ptr UVRequest_; - StageT Stage_ = StageT::SETTINGS_READ; - // -- Job queues - std::mutex JobsMutex_; - struct - { - JobQueueT Sources; - JobQueueT Headers; - JobQueueT MocPredefs; - JobQueueT Moc; - JobQueueT Uic; - } JobQueues_; - JobQueueT JobQueue_; - std::size_t volatile JobsRemain_ = 0; - bool volatile JobError_ = false; - bool volatile JobThreadsAbort_ = false; - std::condition_variable JobsConditionRead_; // -- Moc meta - std::set<std::string> MocIncludedStrings_; + std::mutex MocMetaMutex_; std::set<std::string> MocIncludedFiles_; + IncludesMap MocIncludes_; std::set<std::string> MocAutoFiles_; - bool volatile MocAutoFileUpdated_ = false; + std::atomic<bool> MocAutoFileUpdated_ = ATOMIC_VAR_INIT(false); + // -- Uic meta + std::mutex UicMetaMutex_; + IncludesMap UicIncludes_; // -- Settings file std::string SettingsFile_; std::string SettingsStringMoc_; std::string SettingsStringUic_; - // -- Threads and loops - std::vector<std::unique_ptr<WorkerT>> Workers_; + // -- Thread pool and job queue + std::atomic<bool> JobError_ = ATOMIC_VAR_INIT(false); + cmWorkerPool WorkerPool_; }; #endif diff --git a/Source/cmWorkerPool.cxx b/Source/cmWorkerPool.cxx new file mode 100644 index 0000000000..464182c558 --- /dev/null +++ b/Source/cmWorkerPool.cxx @@ -0,0 +1,770 @@ +/* Distributed under the OSI-approved BSD 3-Clause License. See accompanying + file Copyright.txt or https://cmake.org/licensing for details. */ +#include "cmWorkerPool.h" + +#include "cmRange.h" +#include "cmUVHandlePtr.h" +#include "cmUVSignalHackRAII.h" // IWYU pragma: keep +#include "cm_uv.h" + +#include <algorithm> +#include <array> +#include <condition_variable> +#include <deque> +#include <functional> +#include <mutex> +#include <stddef.h> +#include <thread> + +/** + * @brief libuv pipe buffer class + */ +class cmUVPipeBuffer +{ +public: + typedef cmRange<char const*> DataRange; + typedef std::function<void(DataRange)> DataFunction; + /// On error the ssize_t argument is a non zero libuv error code + typedef std::function<void(ssize_t)> EndFunction; + +public: + /** + * Reset to construction state + */ + void reset(); + + /** + * Initializes uv_pipe(), uv_stream() and uv_handle() + * @return true on success + */ + bool init(uv_loop_t* uv_loop); + + /** + * Start reading + * @return true on success + */ + bool startRead(DataFunction dataFunction, EndFunction endFunction); + + //! libuv pipe + uv_pipe_t* uv_pipe() const { return UVPipe_.get(); } + //! uv_pipe() casted to libuv stream + uv_stream_t* uv_stream() const { return static_cast<uv_stream_t*>(UVPipe_); } + //! uv_pipe() casted to libuv handle + uv_handle_t* uv_handle() { return static_cast<uv_handle_t*>(UVPipe_); } + +private: + // -- Libuv callbacks + static void UVAlloc(uv_handle_t* handle, size_t suggestedSize, + uv_buf_t* buf); + static void UVData(uv_stream_t* stream, ssize_t nread, const uv_buf_t* buf); + +private: + cm::uv_pipe_ptr UVPipe_; + std::vector<char> Buffer_; + DataFunction DataFunction_; + EndFunction EndFunction_; +}; + +void cmUVPipeBuffer::reset() +{ + if (UVPipe_.get() != nullptr) { + EndFunction_ = nullptr; + DataFunction_ = nullptr; + Buffer_.clear(); + Buffer_.shrink_to_fit(); + UVPipe_.reset(); + } +} + +bool cmUVPipeBuffer::init(uv_loop_t* uv_loop) +{ + reset(); + if (uv_loop == nullptr) { + return false; + } + int ret = UVPipe_.init(*uv_loop, 0, this); + return (ret == 0); +} + +bool cmUVPipeBuffer::startRead(DataFunction dataFunction, + EndFunction endFunction) +{ + if (UVPipe_.get() == nullptr) { + return false; + } + if (!dataFunction || !endFunction) { + return false; + } + DataFunction_ = std::move(dataFunction); + EndFunction_ = std::move(endFunction); + int ret = uv_read_start(uv_stream(), &cmUVPipeBuffer::UVAlloc, + &cmUVPipeBuffer::UVData); + return (ret == 0); +} + +void cmUVPipeBuffer::UVAlloc(uv_handle_t* handle, size_t suggestedSize, + uv_buf_t* buf) +{ + auto& pipe = *reinterpret_cast<cmUVPipeBuffer*>(handle->data); + pipe.Buffer_.resize(suggestedSize); + buf->base = pipe.Buffer_.data(); + buf->len = static_cast<unsigned long>(pipe.Buffer_.size()); +} + +void cmUVPipeBuffer::UVData(uv_stream_t* stream, ssize_t nread, + const uv_buf_t* buf) +{ + auto& pipe = *reinterpret_cast<cmUVPipeBuffer*>(stream->data); + if (nread > 0) { + if (buf->base != nullptr) { + // Call data function + pipe.DataFunction_(DataRange(buf->base, buf->base + nread)); + } + } else if (nread < 0) { + // Save the end function on the stack before resetting the pipe + EndFunction efunc; + efunc.swap(pipe.EndFunction_); + // Reset pipe before calling the end function + pipe.reset(); + // Call end function + efunc((nread == UV_EOF) ? 0 : nread); + } +} + +/** + * @brief External process management class + */ +class cmUVReadOnlyProcess +{ +public: + // -- Types + //! @brief Process settings + struct SetupT + { + std::string WorkingDirectory; + std::vector<std::string> Command; + cmWorkerPool::ProcessResultT* Result = nullptr; + bool MergedOutput = false; + }; + +public: + // -- Const accessors + SetupT const& Setup() const { return Setup_; } + cmWorkerPool::ProcessResultT* Result() const { return Setup_.Result; } + bool IsStarted() const { return IsStarted_; } + bool IsFinished() const { return IsFinished_; } + + // -- Runtime + void setup(cmWorkerPool::ProcessResultT* result, bool mergedOutput, + std::vector<std::string> const& command, + std::string const& workingDirectory = std::string()); + bool start(uv_loop_t* uv_loop, std::function<void()> finishedCallback); + +private: + // -- Libuv callbacks + static void UVExit(uv_process_t* handle, int64_t exitStatus, int termSignal); + void UVPipeOutData(cmUVPipeBuffer::DataRange data); + void UVPipeOutEnd(ssize_t error); + void UVPipeErrData(cmUVPipeBuffer::DataRange data); + void UVPipeErrEnd(ssize_t error); + void UVTryFinish(); + +private: + // -- Setup + SetupT Setup_; + // -- Runtime + bool IsStarted_ = false; + bool IsFinished_ = false; + std::function<void()> FinishedCallback_; + std::vector<const char*> CommandPtr_; + std::array<uv_stdio_container_t, 3> UVOptionsStdIO_; + uv_process_options_t UVOptions_; + cm::uv_process_ptr UVProcess_; + cmUVPipeBuffer UVPipeOut_; + cmUVPipeBuffer UVPipeErr_; +}; + +void cmUVReadOnlyProcess::setup(cmWorkerPool::ProcessResultT* result, + bool mergedOutput, + std::vector<std::string> const& command, + std::string const& workingDirectory) +{ + Setup_.WorkingDirectory = workingDirectory; + Setup_.Command = command; + Setup_.Result = result; + Setup_.MergedOutput = mergedOutput; +} + +bool cmUVReadOnlyProcess::start(uv_loop_t* uv_loop, + std::function<void()> finishedCallback) +{ + if (IsStarted() || (Result() == nullptr)) { + return false; + } + + // Reset result before the start + Result()->reset(); + + // Fill command string pointers + if (!Setup().Command.empty()) { + CommandPtr_.reserve(Setup().Command.size() + 1); + for (std::string const& arg : Setup().Command) { + CommandPtr_.push_back(arg.c_str()); + } + CommandPtr_.push_back(nullptr); + } else { + Result()->ErrorMessage = "Empty command"; + } + + if (!Result()->error()) { + if (!UVPipeOut_.init(uv_loop)) { + Result()->ErrorMessage = "libuv stdout pipe initialization failed"; + } + } + if (!Result()->error()) { + if (!UVPipeErr_.init(uv_loop)) { + Result()->ErrorMessage = "libuv stderr pipe initialization failed"; + } + } + if (!Result()->error()) { + // -- Setup process stdio options + // stdin + UVOptionsStdIO_[0].flags = UV_IGNORE; + UVOptionsStdIO_[0].data.stream = nullptr; + // stdout + UVOptionsStdIO_[1].flags = + static_cast<uv_stdio_flags>(UV_CREATE_PIPE | UV_WRITABLE_PIPE); + UVOptionsStdIO_[1].data.stream = UVPipeOut_.uv_stream(); + // stderr + UVOptionsStdIO_[2].flags = + static_cast<uv_stdio_flags>(UV_CREATE_PIPE | UV_WRITABLE_PIPE); + UVOptionsStdIO_[2].data.stream = UVPipeErr_.uv_stream(); + + // -- Setup process options + std::fill_n(reinterpret_cast<char*>(&UVOptions_), sizeof(UVOptions_), 0); + UVOptions_.exit_cb = &cmUVReadOnlyProcess::UVExit; + UVOptions_.file = CommandPtr_[0]; + UVOptions_.args = const_cast<char**>(CommandPtr_.data()); + UVOptions_.cwd = Setup_.WorkingDirectory.c_str(); + UVOptions_.flags = UV_PROCESS_WINDOWS_HIDE; + UVOptions_.stdio_count = static_cast<int>(UVOptionsStdIO_.size()); + UVOptions_.stdio = UVOptionsStdIO_.data(); + + // -- Spawn process + int uvErrorCode = UVProcess_.spawn(*uv_loop, UVOptions_, this); + if (uvErrorCode != 0) { + Result()->ErrorMessage = "libuv process spawn failed"; + if (const char* uvErr = uv_strerror(uvErrorCode)) { + Result()->ErrorMessage += ": "; + Result()->ErrorMessage += uvErr; + } + } + } + // -- Start reading from stdio streams + if (!Result()->error()) { + if (!UVPipeOut_.startRead( + [this](cmUVPipeBuffer::DataRange range) { + this->UVPipeOutData(range); + }, + [this](ssize_t error) { this->UVPipeOutEnd(error); })) { + Result()->ErrorMessage = "libuv start reading from stdout pipe failed"; + } + } + if (!Result()->error()) { + if (!UVPipeErr_.startRead( + [this](cmUVPipeBuffer::DataRange range) { + this->UVPipeErrData(range); + }, + [this](ssize_t error) { this->UVPipeErrEnd(error); })) { + Result()->ErrorMessage = "libuv start reading from stderr pipe failed"; + } + } + + if (!Result()->error()) { + IsStarted_ = true; + FinishedCallback_ = std::move(finishedCallback); + } else { + // Clear libuv handles and finish + UVProcess_.reset(); + UVPipeOut_.reset(); + UVPipeErr_.reset(); + CommandPtr_.clear(); + } + + return IsStarted(); +} + +void cmUVReadOnlyProcess::UVExit(uv_process_t* handle, int64_t exitStatus, + int termSignal) +{ + auto& proc = *reinterpret_cast<cmUVReadOnlyProcess*>(handle->data); + if (proc.IsStarted() && !proc.IsFinished()) { + // Set error message on demand + proc.Result()->ExitStatus = exitStatus; + proc.Result()->TermSignal = termSignal; + if (!proc.Result()->error()) { + if (termSignal != 0) { + proc.Result()->ErrorMessage = "Process was terminated by signal "; + proc.Result()->ErrorMessage += + std::to_string(proc.Result()->TermSignal); + } else if (exitStatus != 0) { + proc.Result()->ErrorMessage = "Process failed with return value "; + proc.Result()->ErrorMessage += + std::to_string(proc.Result()->ExitStatus); + } + } + + // Reset process handle + proc.UVProcess_.reset(); + // Try finish + proc.UVTryFinish(); + } +} + +void cmUVReadOnlyProcess::UVPipeOutData(cmUVPipeBuffer::DataRange data) +{ + Result()->StdOut.append(data.begin(), data.end()); +} + +void cmUVReadOnlyProcess::UVPipeOutEnd(ssize_t error) +{ + // Process pipe error + if ((error != 0) && !Result()->error()) { + Result()->ErrorMessage = + "Reading from stdout pipe failed with libuv error code "; + Result()->ErrorMessage += std::to_string(error); + } + // Try finish + UVTryFinish(); +} + +void cmUVReadOnlyProcess::UVPipeErrData(cmUVPipeBuffer::DataRange data) +{ + std::string* str = + Setup_.MergedOutput ? &Result()->StdOut : &Result()->StdErr; + str->append(data.begin(), data.end()); +} + +void cmUVReadOnlyProcess::UVPipeErrEnd(ssize_t error) +{ + // Process pipe error + if ((error != 0) && !Result()->error()) { + Result()->ErrorMessage = + "Reading from stderr pipe failed with libuv error code "; + Result()->ErrorMessage += std::to_string(error); + } + // Try finish + UVTryFinish(); +} + +void cmUVReadOnlyProcess::UVTryFinish() +{ + // There still might be data in the pipes after the process has finished. + // Therefore check if the process is finished AND all pipes are closed + // before signaling the worker thread to continue. + if ((UVProcess_.get() != nullptr) || (UVPipeOut_.uv_pipe() != nullptr) || + (UVPipeErr_.uv_pipe() != nullptr)) { + return; + } + IsFinished_ = true; + FinishedCallback_(); +} + +/** + * @brief Private worker pool internals + */ +class cmWorkerPoolInternal +{ +public: + // -- Types + + /** + * @brief Worker thread + */ + class WorkerT + { + public: + WorkerT(unsigned int index); + ~WorkerT(); + + WorkerT(WorkerT const&) = delete; + WorkerT& operator=(WorkerT const&) = delete; + + /** + * Start the thread + */ + void Start(cmWorkerPoolInternal* internal); + + /** + * @brief Run an external process + */ + bool RunProcess(cmWorkerPool::ProcessResultT& result, + std::vector<std::string> const& command, + std::string const& workingDirectory); + + // -- Accessors + unsigned int Index() const { return Index_; } + cmWorkerPool::JobHandleT& JobHandle() { return JobHandle_; } + + private: + // -- Libuv callbacks + static void UVProcessStart(uv_async_t* handle); + void UVProcessFinished(); + + private: + //! @brief Job handle + cmWorkerPool::JobHandleT JobHandle_; + //! @brief Worker index + unsigned int Index_; + // -- Process management + struct + { + std::mutex Mutex; + cm::uv_async_ptr Request; + std::condition_variable Condition; + std::unique_ptr<cmUVReadOnlyProcess> ROP; + } Proc_; + // -- System thread + std::thread Thread_; + }; + +public: + // -- Constructors + cmWorkerPoolInternal(cmWorkerPool* pool); + ~cmWorkerPoolInternal(); + + /** + * @brief Runs the libuv loop + */ + bool Process(); + + /** + * @brief Clear queue and abort threads + */ + void Abort(); + + /** + * @brief Push a job to the queue and notify a worker + */ + bool PushJob(cmWorkerPool::JobHandleT&& jobHandle); + + /** + * @brief Worker thread main loop method + */ + void Work(WorkerT* worker); + + // -- Request slots + static void UVSlotBegin(uv_async_t* handle); + static void UVSlotEnd(uv_async_t* handle); + +public: + // -- UV loop +#ifdef CMAKE_UV_SIGNAL_HACK + std::unique_ptr<cmUVSignalHackRAII> UVHackRAII; +#endif + std::unique_ptr<uv_loop_t> UVLoop; + cm::uv_async_ptr UVRequestBegin; + cm::uv_async_ptr UVRequestEnd; + + // -- Thread pool and job queue + std::mutex Mutex; + bool Aborting = false; + bool FenceProcessing = false; + unsigned int WorkersRunning = 0; + unsigned int WorkersIdle = 0; + unsigned int JobsProcessing = 0; + std::deque<cmWorkerPool::JobHandleT> Queue; + std::condition_variable Condition; + std::vector<std::unique_ptr<WorkerT>> Workers; + + // -- References + cmWorkerPool* Pool = nullptr; +}; + +cmWorkerPoolInternal::WorkerT::WorkerT(unsigned int index) + : Index_(index) +{ +} + +cmWorkerPoolInternal::WorkerT::~WorkerT() +{ + if (Thread_.joinable()) { + Thread_.join(); + } +} + +void cmWorkerPoolInternal::WorkerT::Start(cmWorkerPoolInternal* internal) +{ + Proc_.Request.init(*(internal->UVLoop), &WorkerT::UVProcessStart, this); + Thread_ = std::thread(&cmWorkerPoolInternal::Work, internal, this); +} + +bool cmWorkerPoolInternal::WorkerT::RunProcess( + cmWorkerPool::ProcessResultT& result, + std::vector<std::string> const& command, std::string const& workingDirectory) +{ + if (command.empty()) { + return false; + } + // Create process instance + { + std::lock_guard<std::mutex> lock(Proc_.Mutex); + Proc_.ROP = cm::make_unique<cmUVReadOnlyProcess>(); + Proc_.ROP->setup(&result, true, command, workingDirectory); + } + // Send asynchronous process start request to libuv loop + Proc_.Request.send(); + // Wait until the process has been finished and destroyed + { + std::unique_lock<std::mutex> ulock(Proc_.Mutex); + while (Proc_.ROP) { + Proc_.Condition.wait(ulock); + } + } + return !result.error(); +} + +void cmWorkerPoolInternal::WorkerT::UVProcessStart(uv_async_t* handle) +{ + auto* wrk = reinterpret_cast<WorkerT*>(handle->data); + bool startFailed = false; + { + auto& Proc = wrk->Proc_; + std::lock_guard<std::mutex> lock(Proc.Mutex); + if (Proc.ROP && !Proc.ROP->IsStarted()) { + startFailed = + !Proc.ROP->start(handle->loop, [wrk] { wrk->UVProcessFinished(); }); + } + } + // Clean up if starting of the process failed + if (startFailed) { + wrk->UVProcessFinished(); + } +} + +void cmWorkerPoolInternal::WorkerT::UVProcessFinished() +{ + { + std::lock_guard<std::mutex> lock(Proc_.Mutex); + if (Proc_.ROP && (Proc_.ROP->IsFinished() || !Proc_.ROP->IsStarted())) { + Proc_.ROP.reset(); + } + } + // Notify idling thread + Proc_.Condition.notify_one(); +} + +void cmWorkerPool::ProcessResultT::reset() +{ + ExitStatus = 0; + TermSignal = 0; + if (!StdOut.empty()) { + StdOut.clear(); + StdOut.shrink_to_fit(); + } + if (!StdErr.empty()) { + StdErr.clear(); + StdErr.shrink_to_fit(); + } + if (!ErrorMessage.empty()) { + ErrorMessage.clear(); + ErrorMessage.shrink_to_fit(); + } +} + +cmWorkerPoolInternal::cmWorkerPoolInternal(cmWorkerPool* pool) + : Pool(pool) +{ + // Initialize libuv loop + uv_disable_stdio_inheritance(); +#ifdef CMAKE_UV_SIGNAL_HACK + UVHackRAII = cm::make_unique<cmUVSignalHackRAII>(); +#endif + UVLoop = cm::make_unique<uv_loop_t>(); + uv_loop_init(UVLoop.get()); +} + +cmWorkerPoolInternal::~cmWorkerPoolInternal() +{ + uv_loop_close(UVLoop.get()); +} + +bool cmWorkerPoolInternal::Process() +{ + // Reset state + Aborting = false; + // Initialize libuv asynchronous request + UVRequestBegin.init(*UVLoop, &cmWorkerPoolInternal::UVSlotBegin, this); + UVRequestEnd.init(*UVLoop, &cmWorkerPoolInternal::UVSlotEnd, this); + // Send begin request + UVRequestBegin.send(); + // Run libuv loop + return (uv_run(UVLoop.get(), UV_RUN_DEFAULT) == 0); +} + +void cmWorkerPoolInternal::Abort() +{ + bool firstCall = false; + // Clear all jobs and set abort flag + { + std::lock_guard<std::mutex> guard(Mutex); + if (!Aborting) { + // Register abort and clear queue + Aborting = true; + Queue.clear(); + firstCall = true; + } + } + if (firstCall) { + // Wake threads + Condition.notify_all(); + } +} + +inline bool cmWorkerPoolInternal::PushJob(cmWorkerPool::JobHandleT&& jobHandle) +{ + std::lock_guard<std::mutex> guard(Mutex); + if (Aborting) { + return false; + } + + // Append the job to the queue + Queue.emplace_back(std::move(jobHandle)); + + // Notify an idle worker if there's one + if (WorkersIdle != 0) { + Condition.notify_one(); + } + + return true; +} + +void cmWorkerPoolInternal::UVSlotBegin(uv_async_t* handle) +{ + auto& gint = *reinterpret_cast<cmWorkerPoolInternal*>(handle->data); + // Create worker threads + { + unsigned int const num = gint.Pool->ThreadCount(); + // Create workers + gint.Workers.reserve(num); + for (unsigned int ii = 0; ii != num; ++ii) { + gint.Workers.emplace_back(cm::make_unique<WorkerT>(ii)); + } + // Start workers + for (auto& wrk : gint.Workers) { + wrk->Start(&gint); + } + } + // Destroy begin request + gint.UVRequestBegin.reset(); +} + +void cmWorkerPoolInternal::UVSlotEnd(uv_async_t* handle) +{ + auto& gint = *reinterpret_cast<cmWorkerPoolInternal*>(handle->data); + // Join and destroy worker threads + gint.Workers.clear(); + // Destroy end request + gint.UVRequestEnd.reset(); +} + +void cmWorkerPoolInternal::Work(WorkerT* worker) +{ + std::unique_lock<std::mutex> uLock(Mutex); + // Increment running workers count + ++WorkersRunning; + // Enter worker main loop + while (true) { + // Abort on request + if (Aborting) { + break; + } + // Wait for new jobs + if (Queue.empty()) { + ++WorkersIdle; + Condition.wait(uLock); + --WorkersIdle; + continue; + } + + // Check for fence jobs + if (FenceProcessing || Queue.front()->IsFence()) { + if (JobsProcessing != 0) { + Condition.wait(uLock); + continue; + } + // No jobs get processed. Set the fence job processing flag. + FenceProcessing = true; + } + + // Pop next job from queue + worker->JobHandle() = std::move(Queue.front()); + Queue.pop_front(); + + // Unlocked scope for job processing + ++JobsProcessing; + { + uLock.unlock(); + worker->JobHandle()->Work(Pool, worker->Index()); // Process job + worker->JobHandle().reset(); // Destroy job + uLock.lock(); + } + --JobsProcessing; + + // Was this a fence job? + if (FenceProcessing) { + FenceProcessing = false; + Condition.notify_all(); + } + } + + // Decrement running workers count + if (--WorkersRunning == 0) { + // Last worker thread about to finish. Send libuv event. + UVRequestEnd.send(); + } +} + +cmWorkerPool::JobT::~JobT() = default; + +bool cmWorkerPool::JobT::RunProcess(ProcessResultT& result, + std::vector<std::string> const& command, + std::string const& workingDirectory) +{ + // Get worker by index + auto* wrk = Pool_->Int_->Workers.at(WorkerIndex_).get(); + return wrk->RunProcess(result, command, workingDirectory); +} + +cmWorkerPool::cmWorkerPool() + : Int_(cm::make_unique<cmWorkerPoolInternal>(this)) +{ +} + +cmWorkerPool::~cmWorkerPool() = default; + +bool cmWorkerPool::Process(unsigned int threadCount, void* userData) +{ + // Setup user data + UserData_ = userData; + ThreadCount_ = (threadCount > 0) ? threadCount : 1u; + + // Run libuv loop + bool success = Int_->Process(); + + // Clear user data + UserData_ = nullptr; + ThreadCount_ = 0; + + return success; +} + +bool cmWorkerPool::PushJob(JobHandleT&& jobHandle) +{ + return Int_->PushJob(std::move(jobHandle)); +} + +void cmWorkerPool::Abort() +{ + Int_->Abort(); +} diff --git a/Source/cmWorkerPool.h b/Source/cmWorkerPool.h new file mode 100644 index 0000000000..71c7d84f0e --- /dev/null +++ b/Source/cmWorkerPool.h @@ -0,0 +1,219 @@ +/* Distributed under the OSI-approved BSD 3-Clause License. See accompanying + file Copyright.txt or https://cmake.org/licensing for details. */ +#ifndef cmWorkerPool_h +#define cmWorkerPool_h + +#include "cmConfigure.h" // IWYU pragma: keep + +#include "cmAlgorithms.h" // IWYU pragma: keep + +#include <memory> // IWYU pragma: keep +#include <stdint.h> +#include <string> +#include <utility> +#include <vector> + +// -- Types +class cmWorkerPoolInternal; + +/** @class cmWorkerPool + * @brief Thread pool with job queue + */ +class cmWorkerPool +{ +public: + /** + * Return value and output of an external process. + */ + struct ProcessResultT + { + void reset(); + bool error() const + { + return (ExitStatus != 0) || (TermSignal != 0) || !ErrorMessage.empty(); + } + + std::int64_t ExitStatus = 0; + int TermSignal = 0; + std::string StdOut; + std::string StdErr; + std::string ErrorMessage; + }; + + /** + * Abstract job class for concurrent job processing. + */ + class JobT + { + public: + JobT(JobT const&) = delete; + JobT& operator=(JobT const&) = delete; + + /** + * @brief Virtual destructor. + */ + virtual ~JobT(); + + /** + * @brief Fence job flag + * + * Fence jobs require that: + * - all jobs before in the queue have been processed + * - no jobs later in the queue will be processed before this job was + * processed + */ + bool IsFence() const { return Fence_; } + + protected: + /** + * @brief Protected default constructor + */ + JobT(bool fence = false) + : Fence_(fence) + { + } + + /** + * Abstract processing interface that must be implement in derived classes. + */ + virtual void Process() = 0; + + /** + * Get the worker pool. + * Only valid during the JobT::Process() call! + */ + cmWorkerPool* Pool() const { return Pool_; } + + /** + * Get the user data. + * Only valid during the JobT::Process() call! + */ + void* UserData() const { return Pool_->UserData(); }; + + /** + * Get the worker index. + * This is the index of the thread processing this job and is in the range + * [0..ThreadCount). + * Concurrently processing jobs will never have the same WorkerIndex(). + * Only valid during the JobT::Process() call! + */ + unsigned int WorkerIndex() const { return WorkerIndex_; } + + /** + * Run an external read only process. + * Use only during JobT::Process() call! + */ + bool RunProcess(ProcessResultT& result, + std::vector<std::string> const& command, + std::string const& workingDirectory); + + private: + //! Needs access to Work() + friend class cmWorkerPoolInternal; + //! Worker thread entry method. + void Work(cmWorkerPool* pool, unsigned int workerIndex) + { + Pool_ = pool; + WorkerIndex_ = workerIndex; + this->Process(); + } + + private: + cmWorkerPool* Pool_ = nullptr; + unsigned int WorkerIndex_ = 0; + bool Fence_ = false; + }; + + /** + * @brief Job handle type + */ + typedef std::unique_ptr<JobT> JobHandleT; + + /** + * @brief Fence job base class + */ + class JobFenceT : public JobT + { + public: + JobFenceT() + : JobT(true) + { + } + //! Does nothing + void Process() override{}; + }; + + /** + * @brief Fence job that aborts the worker pool. + * This class is useful as the last job in the job queue. + */ + class JobEndT : JobFenceT + { + public: + //! Does nothing + void Process() override { Pool()->Abort(); } + }; + +public: + // -- Methods + cmWorkerPool(); + ~cmWorkerPool(); + + /** + * @brief Blocking function that starts threads to process all Jobs in + * the queue. + * + * This method blocks until a job calls the Abort() method. + * @arg threadCount Number of threads to process jobs. + * @arg userData Common user data pointer available in all Jobs. + */ + bool Process(unsigned int threadCount, void* userData = nullptr); + + /** + * Number of worker threads passed to Process(). + * Only valid during Process(). + */ + unsigned int ThreadCount() const { return ThreadCount_; } + + /** + * User data reference passed to Process(). + * Only valid during Process(). + */ + void* UserData() const { return UserData_; } + + // -- Job processing interface + + /** + * @brief Clears the job queue and aborts all worker threads. + * + * This method is thread safe and can be called from inside a job. + */ + void Abort(); + + /** + * @brief Push job to the queue. + * + * This method is thread safe and can be called from inside a job or before + * Process(). + */ + bool PushJob(JobHandleT&& jobHandle); + + /** + * @brief Push job to the queue + * + * This method is thread safe and can be called from inside a job or before + * Process(). + */ + template <class T, typename... Args> + bool EmplaceJob(Args&&... args) + { + return PushJob(cm::make_unique<T>(std::forward<Args>(args)...)); + } + +private: + void* UserData_ = nullptr; + unsigned int ThreadCount_ = 0; + std::unique_ptr<cmWorkerPoolInternal> Int_; +}; + +#endif |