diff options
author | Mikko Pulkki <mikko.pulkki@mapbox.com> | 2019-10-21 16:27:56 +0300 |
---|---|---|
committer | Mikko Pulkki <mikko.pulkki@mapbox.com> | 2019-10-21 18:17:20 +0300 |
commit | 4e38f10c6a1806541713d60036fc1ee0aa98fe11 (patch) | |
tree | e003ca709b9c4de2247b1488dc6f04076caf9f06 | |
parent | 86610ffdd3e816a9eb5ad1076290e9a7dce21a92 (diff) | |
download | qtlocation-mapboxgl-upstream/mpulkki-render-test-fps.tar.gz |
Add test runner operation for benchmarking fpsupstream/mpulkki-render-test-fps
-rw-r--r-- | platform/default/include/mbgl/gfx/headless_frontend.hpp | 1 | ||||
-rw-r--r-- | platform/default/src/mbgl/gfx/headless_frontend.cpp | 6 | ||||
-rw-r--r-- | render-test/metadata.hpp | 7 | ||||
-rw-r--r-- | render-test/parser.cpp | 34 | ||||
-rw-r--r-- | render-test/runner.cpp | 132 | ||||
-rw-r--r-- | render-test/runner.hpp | 3 |
6 files changed, 175 insertions, 8 deletions
diff --git a/platform/default/include/mbgl/gfx/headless_frontend.hpp b/platform/default/include/mbgl/gfx/headless_frontend.hpp index b777cc7b7f..353452123d 100644 --- a/platform/default/include/mbgl/gfx/headless_frontend.hpp +++ b/platform/default/include/mbgl/gfx/headless_frontend.hpp @@ -49,6 +49,7 @@ public: PremultipliedImage readStillImage(); PremultipliedImage render(Map&); + void renderOnce(Map&); optional<TransformState> getTransformState() const; diff --git a/platform/default/src/mbgl/gfx/headless_frontend.cpp b/platform/default/src/mbgl/gfx/headless_frontend.cpp index b7156cc2b7..87d09911a2 100644 --- a/platform/default/src/mbgl/gfx/headless_frontend.cpp +++ b/platform/default/src/mbgl/gfx/headless_frontend.cpp @@ -158,10 +158,14 @@ PremultipliedImage HeadlessFrontend::render(Map& map) { if (error) { std::rethrow_exception(error); } - + return result; } +void HeadlessFrontend::renderOnce(Map&) { + util::RunLoop::Get()->runOnce(); +} + optional<TransformState> HeadlessFrontend::getTransformState() const { if (updateParameters) { return updateParameters->transformState; diff --git a/render-test/metadata.hpp b/render-test/metadata.hpp index cb21d3e804..548210f953 100644 --- a/render-test/metadata.hpp +++ b/render-test/metadata.hpp @@ -68,6 +68,12 @@ struct MemoryProbe { } }; +struct FpsProbe { + float average = 0.0; + float minOnePc = 0.0; + float tolerance = 0.0f; +}; + struct NetworkProbe { NetworkProbe() = default; NetworkProbe(size_t requests_, size_t transferred_) : requests(requests_), transferred(transferred_) {} @@ -82,6 +88,7 @@ public: std::map<std::string, FileSizeProbe> fileSize; std::map<std::string, MemoryProbe> memory; std::map<std::string, NetworkProbe> network; + std::map<std::string, FpsProbe> fps; }; struct TestMetadata { diff --git a/render-test/parser.cpp b/render-test/parser.cpp index 02eabe5a60..69d6981c02 100644 --- a/render-test/parser.cpp +++ b/render-test/parser.cpp @@ -303,6 +303,22 @@ std::string serializeMetrics(const TestMetrics& metrics) { writer.EndArray(); } + if (!metrics.fps.empty()) { + // Start fps section + writer.Key("fps"); + writer.StartArray(); + for (const auto& fpsProbe : metrics.fps) { + assert(!fpsProbe.first.empty()); + writer.StartArray(); + writer.String(fpsProbe.first.c_str()); + writer.Double(fpsProbe.second.average); + writer.Double(fpsProbe.second.minOnePc); + writer.EndArray(); + } + writer.EndArray(); + // End fps section + } + writer.EndObject(); return s.GetString(); @@ -433,6 +449,7 @@ ArgumentsTuple parseArguments(int argc, char** argv) { if (testFilterValue && !std::regex_match(testPath.path().string(), args::get(testFilterValue))) { continue; } + if (testPath.path().filename() == "style.json") { testPaths.emplace_back(testPath, getTestExpectations(testPath, path, expectationsPaths)); } @@ -549,6 +566,23 @@ TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path) { } } + if (document.HasMember("fps")) { + const mbgl::JSValue& fpsValue = document["fps"]; + assert(fpsValue.IsArray()); + for (auto& probeValue : fpsValue.GetArray()) { + assert(probeValue.IsArray()); + assert(probeValue.Size() >= 4u); + assert(probeValue[0].IsString()); + assert(probeValue[1].IsNumber()); // Average + assert(probeValue[2].IsNumber()); // Minimum + assert(probeValue[3].IsNumber()); // Tolerance + const std::string mark{probeValue[0].GetString(), probeValue[0].GetStringLength()}; + assert(!mark.empty()); + result.fps.insert( + {std::move(mark), {probeValue[1].GetFloat(), probeValue[2].GetFloat(), probeValue[3].GetFloat()}}); + } + } + return result; } diff --git a/render-test/runner.cpp b/render-test/runner.cpp index 96d0031148..46b890b76c 100644 --- a/render-test/runner.cpp +++ b/render-test/runner.cpp @@ -1,6 +1,7 @@ #include <mbgl/map/camera.hpp> #include <mbgl/map/map_observer.hpp> #include <mbgl/renderer/renderer.hpp> +#include <mbgl/renderer/renderer_observer.hpp> #include <mbgl/style/conversion/filter.hpp> #include <mbgl/style/conversion/layer.hpp> #include <mbgl/style/conversion/light.hpp> @@ -33,7 +34,30 @@ using namespace mbgl; -// static +class TestRunnerMapObserver : public MapObserver { +public: + TestRunnerMapObserver() : mapLoadFailure(false), finishRenderingMap(false), idle(false) {} + + void onDidFailLoadingMap(MapLoadError, const std::string&) override { mapLoadFailure = true; } + + void onDidFinishRenderingMap(RenderMode mode) override final { + if (!finishRenderingMap) finishRenderingMap = mode == RenderMode::Full; + } + + void onDidBecomeIdle() override final { idle = true; } + + void reset() { + mapLoadFailure = false; + finishRenderingMap = false; + idle = false; + } + + bool mapLoadFailure; + bool finishRenderingMap; + bool idle; +}; + +// static const std::string& TestRunner::getBasePath() { const static std::string result = std::string(TEST_RUNNER_ROOT_PATH).append("/mapbox-gl-js/test/integration"); return result; @@ -339,16 +363,38 @@ bool TestRunner::checkRenderTestResults(mbgl::PremultipliedImage&& actualImage, } } #endif // !defined(SANITIZE) + // Check fps metrics + for (const auto& expected : metadata.expectedMetrics.fps) { + auto actual = metadata.metrics.fps.find(expected.first); + if (actual == metadata.metrics.fps.end()) { + metadata.errorMessage = "Failed to find fps probe: " + expected.first; + return false; + } + auto result = checkValue(expected.second.average, actual->second.average, expected.second.tolerance); + if (!std::get<bool>(result)) { + std::stringstream ss; + ss << "Average fps at probe \"" << expected.first << "\" is " << actual->second.average + << ", expected to be " << expected.second.average << " with tolerance of " << expected.second.tolerance; + metadata.errorMessage = ss.str(); + return false; + } + result = checkValue(expected.second.minOnePc, actual->second.minOnePc, expected.second.tolerance); + if (!std::get<bool>(result)) { + std::stringstream ss; + ss << "Minimum(1%) fps at probe \"" << expected.first << "\" is " << actual->second.minOnePc + << ", expected to be " << expected.second.minOnePc << " with tolerance of " << expected.second.tolerance; + metadata.errorMessage = ss.str(); + return false; + } + } return true; } bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { - if (!metadata.document.HasMember("metadata") || - !metadata.document["metadata"].HasMember("test") || + if (!metadata.document.HasMember("metadata") || !metadata.document["metadata"].HasMember("test") || !metadata.document["metadata"]["test"].HasMember("operations")) { return true; } - assert(metadata.document["metadata"]["test"]["operations"].IsArray()); const auto& operationsArray = metadata.document["metadata"]["test"]["operations"].GetArray(); @@ -364,6 +410,7 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { auto& frontend = maps[key]->frontend; auto& map = maps[key]->map; + auto& observer = maps[key]->observer; static const std::string waitOp("wait"); static const std::string sleepOp("sleep"); @@ -394,6 +441,7 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { static const std::string setFeatureStateOp("setFeatureState"); static const std::string getFeatureStateOp("getFeatureState"); static const std::string removeFeatureStateOp("removeFeatureState"); + static const std::string panGestureOp("panGesture"); if (operationArray[0].GetString() == waitOp) { // wait @@ -835,9 +883,76 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { return false; } frontend.getRenderer()->removeFeatureState(sourceID, sourceLayer, featureID, stateKey); + } else if (operationArray[0].GetString() == panGestureOp) { + // benchmarkPanGesture + assert(operationArray.Size() >= 4u); + assert(operationArray[1].IsString()); // identifier + assert(operationArray[2].IsNumber()); // duration + assert(operationArray[3].IsArray()); // start [lat, lng, zoom] + assert(operationArray[4].IsArray()); // end [lat, lng, zoom] + + if (metadata.mapMode != mbgl::MapMode::Continuous) { + metadata.errorMessage = "Map mode must be Continous for " + panGestureOp + " operation"; + return false; + } + + std::string mark = operationArray[1].GetString(); + int duration = operationArray[2].GetFloat(); + LatLng startPos, endPos; + double startZoom, endZoom; + std::vector<float> samples; + + auto parsePosition = [](auto arr) -> std::tuple<LatLng, double> { + assert(arr.Size() >= 3); + return {{arr[1].GetDouble(), arr[0].GetDouble()}, arr[2].GetDouble()}; + }; + + std::tie(startPos, startZoom) = parsePosition(operationArray[3].GetArray()); + std::tie(endPos, endZoom) = parsePosition(operationArray[4].GetArray()); + + // Jump to the starting point of the segment and make sure there's something to render + map.jumpTo(mbgl::CameraOptions().withCenter(startPos).withZoom(startZoom)); + + observer->reset(); + while (!observer->finishRenderingMap) { + frontend.renderOnce(map); + } + + if (observer->mapLoadFailure) return false; + + size_t frames = 0; + float totalTime = 0.0; + bool transitionFinished = false; + + mbgl::AnimationOptions animationOptions(mbgl::Milliseconds(duration * 1000)); + animationOptions.minZoom = util::min(startZoom, endZoom); + animationOptions.transitionFinishFn = [&]() { transitionFinished = true; }; + + map.flyTo(mbgl::CameraOptions().withCenter(endPos).withZoom(endZoom), animationOptions); + + for (; !transitionFinished; frames++) { + frontend.renderOnce(map); + float frameTime = (float)frontend.getFrameTime(); + totalTime += frameTime; + + samples.push_back(frameTime); + } + + float averageFps = totalTime > 0.0 ? frames / totalTime : 0.0; + float minFrameTime = 0.0; + + // Use 1% of the longest frames to compute the minimum fps + std::sort(samples.begin(), samples.end()); + + int sampleCount = util::max(1, (int)samples.size() / 100); + for (auto it = samples.rbegin(); it != samples.rbegin() + sampleCount; it++) minFrameTime += *it; + + float minOnePcFps = sampleCount / minFrameTime; + + metadata.metrics.fps.insert({std::move(mark), {averageFps, minOnePcFps, 0.0f}}); } else { - metadata.errorMessage = std::string("Unsupported operation: ") + operationArray[0].GetString(); + metadata.errorMessage = std::string("Unsupported operation: ") + operationArray[0].GetString(); return false; } @@ -846,9 +961,10 @@ bool TestRunner::runOperations(const std::string& key, TestMetadata& metadata) { } TestRunner::Impl::Impl(const TestMetadata& metadata) - : frontend(metadata.size, metadata.pixelRatio, swapBehavior(metadata.mapMode)), + : observer(std::make_unique<TestRunnerMapObserver>()), + frontend(metadata.size, metadata.pixelRatio, swapBehavior(metadata.mapMode)), map(frontend, - mbgl::MapObserver::nullObserver(), + *observer.get(), mbgl::MapOptions() .withMapMode(metadata.mapMode) .withSize(metadata.size) @@ -856,6 +972,8 @@ TestRunner::Impl::Impl(const TestMetadata& metadata) .withCrossSourceCollisions(metadata.crossSourceCollisions), mbgl::ResourceOptions().withCacheOnlyRequestsSupport(false)) {} +TestRunner::Impl::~Impl() {} + bool TestRunner::run(TestMetadata& metadata) { AllocationIndex::setActive(false); AllocationIndex::reset(); diff --git a/render-test/runner.hpp b/render-test/runner.hpp index ea593bcc61..6aba2d2391 100644 --- a/render-test/runner.hpp +++ b/render-test/runner.hpp @@ -5,6 +5,7 @@ #include <memory> +class TestRunnerMapObserver; struct TestMetadata; class TestRunner { @@ -26,7 +27,9 @@ private: struct Impl { Impl(const TestMetadata&); + ~Impl(); + std::unique_ptr<TestRunnerMapObserver> observer; mbgl::HeadlessFrontend frontend; mbgl::Map map; }; |