diff options
author | Dennis Oberst <dennis.oberst@qt.io> | 2023-02-28 18:47:05 +0100 |
---|---|---|
committer | Dennis Oberst <dennis.oberst@qt.io> | 2023-03-20 20:58:08 +0100 |
commit | d795dfaee7093a30b278e6e8b9dea30c2e7a2106 (patch) | |
tree | 7797d1c53dca0d33a6618a0f55eaa8fdeeef4fcd | |
parent | 5c2245cd66894cc27d6d4afcf13499db6434ee2e (diff) | |
download | qtbase-d795dfaee7093a30b278e6e8b9dea30c2e7a2106.tar.gz |
Example: update wordcount example
Added a QFileDialog to let the user select a path. Before, the path
was statically assigned with "../../" , which is not optimal.
I also modified the findFiles function to check for text files in
general and not only *.cpp and *.h files. Lastly the result of the
word counting is now displayed on the console, as I think this is an
informative output from this example.
Task-number: QTBUG-111165
Pick-to: 6.5 6.5.0
Change-Id: Ie27c6acb4f79a78e3bef141edb92de08901fde71
Reviewed-by: MÃ¥rten Nordheim <marten.nordheim@qt.io>
-rw-r--r-- | doc/global/manifest-meta.qdocconf | 2 | ||||
-rw-r--r-- | examples/qtconcurrent/wordcount/doc/src/qtconcurrent-wordcount.qdoc | 43 | ||||
-rw-r--r-- | examples/qtconcurrent/wordcount/main.cpp | 146 |
3 files changed, 127 insertions, 64 deletions
diff --git a/doc/global/manifest-meta.qdocconf b/doc/global/manifest-meta.qdocconf index 63c8c4befb..633888c3d1 100644 --- a/doc/global/manifest-meta.qdocconf +++ b/doc/global/manifest-meta.qdocconf @@ -211,7 +211,7 @@ manifestmeta.thumbnail.names = "QtCore/Contiguous Cache Example" \ "QtCore/Semaphores Example" \ "QtCore/Wait Conditions Example" \ "QtConcurrent/Map Example" \ - "QtConcurrent/QtConcurrent Word Count Example" \ + "QtConcurrent/Word Count" \ "QtConcurrent/Run Function Example" \ "QtGui/Raster Window Example" \ "QtNetwork/Network Download*" \ diff --git a/examples/qtconcurrent/wordcount/doc/src/qtconcurrent-wordcount.qdoc b/examples/qtconcurrent/wordcount/doc/src/qtconcurrent-wordcount.qdoc index 2c5724bc85..50b6fc2964 100644 --- a/examples/qtconcurrent/wordcount/doc/src/qtconcurrent-wordcount.qdoc +++ b/examples/qtconcurrent/wordcount/doc/src/qtconcurrent-wordcount.qdoc @@ -1,15 +1,44 @@ -// Copyright (C) 2016 The Qt Company Ltd. +// Copyright (C) 2023 The Qt Company Ltd. // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GFDL-1.3-no-invariants-only /*! \example wordcount - \title QtConcurrent Word Count Example - \brief Demonstrates how to use the map-reduce algorithm. + \meta tags {threads, console} + \title Word Count \ingroup qtconcurrentexamples + \brief Demonstrates how to use the map-reduce algorithm. + + The Qt Concurrent \e {Word Count} example demonstrates the use of the + map-reduce algorithm when applied to the problem of counting words in a + collection of files. + + First, the Application starts a QFileDialog to select a starting + path, and then prints the output to the console. + + \include examples-run.qdocinc + + \section1 Comparing the operations + + Compare a single-threaded, sequential approach to counting the words in + the text files to a multithreaded approach with mappedReduced(): + + \dots + \snippet wordcount/main.cpp 1 + \dots + \snippet wordcount/main.cpp 2 + \dots + + The first argument to the \l {QtConcurrent::}{mappedReduced} function is the + container to operate on. The second argument is the mapping function + \c {countWords()}. It is called in parallel by multiple threads. The + third argument is the reducing function \c {reduce()}. It is called + once for each result returned by the mapping function, and generates the + final computation result. - The QtConcurrent Word Count example demonstrates the use of the map-reduce - algorithm when applied to the problem of counting words in a collection - of files. + The function returns a QFuture object of type \c WordCount. Call the + \l {QFuture::}{result} function immediately on this QFuture to block further + execution until the result becomes available. - This is a command-line application. + \note The mapping function must be thread-safe since it is called from + multiple threads. */ diff --git a/examples/qtconcurrent/wordcount/main.cpp b/examples/qtconcurrent/wordcount/main.cpp index 6cd50b2022..4175641ce9 100644 --- a/examples/qtconcurrent/wordcount/main.cpp +++ b/examples/qtconcurrent/wordcount/main.cpp @@ -1,43 +1,18 @@ -// Copyright (C) 2016 The Qt Company Ltd. +// Copyright (C) 2023 The Qt Company Ltd. // SPDX-License-Identifier: LicenseRef-Qt-Commercial OR BSD-3-Clause -#include <QList> -#include <QMap> -#include <QTextStream> -#include <QString> -#include <QStringList> -#include <QDir> -#include <QElapsedTimer> -#include <QApplication> -#include <QDebug> - -#include <qtconcurrentmap.h> - -using namespace QtConcurrent; - -/* - Utility function that recursivily searches for files. -*/ -QStringList findFiles(const QString &startDir, const QStringList &filters) -{ - QStringList names; - QDir dir(startDir); - - const auto files = dir.entryList(filters, QDir::Files); - for (const QString &file : files) - names += startDir + '/' + file; - - const auto subdirs = dir.entryList(QDir::AllDirs | QDir::NoDotAndDotDot); - for (const QString &subdir : subdirs) - names += findFiles(startDir + '/' + subdir, filters); - return names; -} +#include <QtWidgets/qfiledialog.h> +#include <QtWidgets/qapplication.h> +#include <QtCore/qmimedatabase.h> +#include <QtCore/qelapsedtimer.h> +#include <QtConcurrent/qtconcurrentmap.h> typedef QMap<QString, int> WordCount; -/* - Single threaded word counter function. -*/ +void printHighestResult(const WordCount &, qsizetype); +QStringList findFiles(const QString &); + +// Single threaded word counter function. WordCount singleThreadedWordCount(const QStringList &files) { WordCount wordCount; @@ -46,7 +21,7 @@ WordCount singleThreadedWordCount(const QStringList &files) f.open(QIODevice::ReadOnly); QTextStream textStream(&f); while (!textStream.atEnd()) { - const auto words = textStream.readLine().split(' '); + const auto words = textStream.readLine().split(' ', Qt::SkipEmptyParts); for (const QString &word : words) wordCount[word] += 1; } @@ -54,7 +29,6 @@ WordCount singleThreadedWordCount(const QStringList &files) return wordCount; } - // countWords counts the words in a single file. This function is // called in parallel by several threads and must be thread // safe. @@ -66,7 +40,7 @@ WordCount countWords(const QString &file) WordCount wordCount; while (!textStream.atEnd()) { - const auto words = textStream.readLine().split(' '); + const auto words = textStream.readLine().split(' ', Qt::SkipEmptyParts); for (const QString &word : words) wordCount[word] += 1; } @@ -86,33 +60,93 @@ void reduce(WordCount &result, const WordCount &w) int main(int argc, char** argv) { QApplication app(argc, argv); - qDebug() << "finding files..."; - QStringList files = findFiles("../../", QStringList() << "*.cpp" << "*.h"); - qDebug() << files.count() << "files"; - - qDebug() << "warmup"; - { - WordCount total = singleThreadedWordCount(files); - } - - qDebug() << "warmup done"; - - int singleThreadTime = 0; + app.setOrganizationName("QtProject"); + app.setApplicationName(QCoreApplication::translate("main", "Word Count")); + + QFileDialog fileDialog; + fileDialog.setOption(QFileDialog::ReadOnly); + // Grab the directory path from the dialog + auto dirPath = QFileDialog::getExistingDirectory(nullptr, + QCoreApplication::translate("main","Select a Folder"), + QDir::currentPath()); + + QStringList files = findFiles(dirPath); + qDebug() << QCoreApplication::translate("main", "Indexing %1 files in %2") + .arg(files.size()).arg(dirPath); + + // Start the single threaded operation + qint64 singleThreadTime; { QElapsedTimer timer; timer.start(); + //! [1] WordCount total = singleThreadedWordCount(files); + //! [1] singleThreadTime = timer.elapsed(); - qDebug() << "single thread" << singleThreadTime; + qDebug() << QCoreApplication::translate("main", "Single threaded scanning took %1 ms") + .arg(singleThreadTime); } - - int mapReduceTime = 0; + // Start the multithreaded mappedReduced operation. + qint64 mapReduceTime; { QElapsedTimer timer; timer.start(); - WordCount total = mappedReduced(files, countWords, reduce).result(); + //! [2] + WordCount total = QtConcurrent::mappedReduced(files, countWords, reduce).result(); + //! [2] mapReduceTime = timer.elapsed(); - qDebug() << "MapReduce" << mapReduceTime; + qDebug() << QCoreApplication::translate("main", "MapReduce scanning took %1 ms") + .arg(mapReduceTime); + qDebug() << QCoreApplication::translate("main", "MapReduce speedup: %1") + .arg(((double)singleThreadTime - (double)mapReduceTime) / (double)mapReduceTime + 1); + printHighestResult(total, 10); } - qDebug() << "MapReduce speedup x" << ((double)singleThreadTime - (double)mapReduceTime) / (double)mapReduceTime + 1; +} + +// Utility function that recursively searches for text files. +QStringList findFiles(const QString &startDir) +{ + QStringList names; + QDir dir(startDir); + static const QMimeDatabase db; + + const auto files = dir.entryList(QDir::Files); + for (const QString &file : files) { + const auto path = startDir + QDir::separator() + file; + const QMimeType mime = db.mimeTypeForFile(QFileInfo(path)); + const auto mimeTypesForFile = mime.parentMimeTypes(); + + for (const auto &i : mimeTypesForFile) { + if (i.contains("text", Qt::CaseInsensitive) + || mime.comment().contains("text", Qt::CaseInsensitive)) { + names += startDir + QDir::separator() + file; + } + } + } + + const auto subdirs = dir.entryList(QDir::AllDirs | QDir::NoDotAndDotDot); + for (const QString &subdir : subdirs) { + if (names.length() >= 20000) { + qDebug() << QCoreApplication::translate("main", "Too many files! Aborting ..."); + exit(-1); + } + names += findFiles(startDir + QDir::separator() + subdir); + } + return names; +} + +// Utility function that prints the results of the map in decreasing order based on the value. +void printHighestResult(const WordCount &countedWords, qsizetype nResults) +{ + using pair = QPair<QString, int>; + QList<pair> vec; + + std::copy(countedWords.keyValueBegin(), countedWords.keyValueEnd(), + std::back_inserter<QList<pair>>(vec)); + std::sort(vec.begin(), vec.end(), + [](const pair &l, const pair &r) { return l.second > r.second; }); + + qDebug() << QCoreApplication::translate("main", "Most occurring words are:"); + for (qsizetype i = 0; i < qMin(vec.size(), nResults); ++i) + qDebug() << vec[i].first << " : " << vec[i].second; } |