From b2d06b8a17cd012d0c3accc95645780e6de6732a Mon Sep 17 00:00:00 2001 From: Mikhail Pozdnyakov Date: Thu, 12 Dec 2019 20:11:51 +0200 Subject: [test runner] Split operations parsing and execution --- render-test/metadata.hpp | 23 ++ render-test/parser.cpp | 722 ++++++++++++++++++++++++++++++++++++++++++++++- render-test/parser.hpp | 37 +++ render-test/runner.cpp | 656 +----------------------------------------- render-test/runner.hpp | 30 +- 5 files changed, 805 insertions(+), 663 deletions(-) (limited to 'render-test') diff --git a/render-test/metadata.hpp b/render-test/metadata.hpp index ba4a875b0c..918758c3e8 100644 --- a/render-test/metadata.hpp +++ b/render-test/metadata.hpp @@ -9,14 +9,19 @@ #include "filesystem.hpp" +#include #include namespace mbgl { + +class Map; +class HeadlessFrontend; namespace gfx { struct RenderingStats; } } // namespace mbgl +class TestRunnerMapObserver; struct TestStatistics { TestStatistics() = default; @@ -168,3 +173,21 @@ struct TestMetadata { TestMetrics metrics; TestMetrics expectedMetrics; }; + +class TestContext { +public: + virtual mbgl::HeadlessFrontend& getFrontend() = 0; + virtual mbgl::Map& getMap() = 0; + virtual TestRunnerMapObserver& getObserver() = 0; + virtual TestMetadata& getMetadata() = 0; + + GfxProbe activeGfxProbe; + GfxProbe baselineGfxProbe; + bool gfxProbeActive = false; + +protected: + virtual ~TestContext() = default; +}; + +using TestOperation = std::function; +using TestOperations = std::list; \ No newline at end of file diff --git a/render-test/parser.cpp b/render-test/parser.cpp index 1c1f8b2798..2373413da5 100644 --- a/render-test/parser.cpp +++ b/render-test/parser.cpp @@ -1,26 +1,41 @@ -#include +#include "parser.hpp" + +#include "allocation_index.hpp" +#include "file_source.hpp" +#include "filesystem.hpp" +#include "metadata.hpp" +#include "runner.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include +#include #include +#include #include +#include #include #include #include #include -#include -#include #include #include #include #include -#include "filesystem.hpp" -#include "metadata.hpp" -#include "parser.hpp" -#include "runner.hpp" - #include #include @@ -571,6 +586,697 @@ TestMetadata parseTestMetadata(const TestPaths& paths, const Manifest& manifest) return metadata; } +namespace TestOperationNames { +const std::string waitOp("wait"); +const std::string sleepOp("sleep"); +const std::string addImageOp("addImage"); +const std::string updateImageOp("updateImage"); +const std::string removeImageOp("removeImage"); +const std::string setStyleOp("setStyle"); +const std::string setCenterOp("setCenter"); +const std::string setZoomOp("setZoom"); +const std::string setBearingOp("setBearing"); +const std::string setPitchOp("setPitch"); +const std::string setFilterOp("setFilter"); +const std::string setLayerZoomRangeOp("setLayerZoomRange"); +const std::string setLightOp("setLight"); +const std::string addLayerOp("addLayer"); +const std::string removeLayerOp("removeLayer"); +const std::string addSourceOp("addSource"); +const std::string removeSourceOp("removeSource"); +const std::string setPaintPropertyOp("setPaintProperty"); +const std::string setLayoutPropertyOp("setLayoutProperty"); +const std::string fileSizeProbeOp("probeFileSize"); +const std::string memoryProbeOp("probeMemory"); +const std::string memoryProbeStartOp("probeMemoryStart"); +const std::string memoryProbeEndOp("probeMemoryEnd"); +const std::string networkProbeOp("probeNetwork"); +const std::string networkProbeStartOp("probeNetworkStart"); +const std::string networkProbeEndOp("probeNetworkEnd"); +const std::string setFeatureStateOp("setFeatureState"); +const std::string getFeatureStateOp("getFeatureState"); +const std::string removeFeatureStateOp("removeFeatureState"); +const std::string panGestureOp("panGesture"); +const std::string gfxProbeOp("probeGFX"); +const std::string gfxProbeStartOp("probeGFXStart"); +const std::string gfxProbeEndOp("probeGFXEnd"); +} // namespace TestOperationNames + +using namespace TestOperationNames; + +TestOperations parseTestOperations(TestMetadata& metadata, const Manifest& manifest) { + TestOperations result; + if (!metadata.document.HasMember("metadata") || !metadata.document["metadata"].HasMember("test") || + !metadata.document["metadata"]["test"].HasMember("operations")) { + return result; + } + assert(metadata.document["metadata"]["test"]["operations"].IsArray()); + + const auto& operationsArray = metadata.document["metadata"]["test"]["operations"].GetArray(); + if (operationsArray.Empty()) { + return result; + } + for (auto& operationArray : operationsArray) { + assert(operationArray.Size() >= 1u); + + if (operationArray[0].GetString() == waitOp) { + // wait + result.emplace_back([](TestContext& ctx) { + try { + ctx.getFrontend().render(ctx.getMap()); + return true; + } catch (const std::exception&) { + return false; + } + }); + } else if (operationArray[0].GetString() == sleepOp) { + // sleep + mbgl::Duration duration = mbgl::Seconds(3); + if (operationArray.Size() >= 2u) { + duration = mbgl::Milliseconds(operationArray[1].GetUint()); + } + result.emplace_back([duration](TestContext&) { + mbgl::util::Timer sleepTimer; + bool sleeping = true; + + sleepTimer.start(duration, mbgl::Duration::zero(), [&]() { sleeping = false; }); + + while (sleeping) { + mbgl::util::RunLoop::Get()->runOnce(); + } + return true; + }); + + } else if (operationArray[0].GetString() == addImageOp || operationArray[0].GetString() == updateImageOp) { + // addImage | updateImage + assert(operationArray.Size() >= 3u); + + float pixelRatio = 1.0f; + bool sdf = false; + + if (operationArray.Size() == 4u) { + assert(operationArray[3].IsObject()); + const auto& imageOptions = operationArray[3].GetObject(); + if (imageOptions.HasMember("pixelRatio")) { + pixelRatio = imageOptions["pixelRatio"].GetFloat(); + } + if (imageOptions.HasMember("sdf")) { + sdf = imageOptions["sdf"].GetBool(); + } + } + + std::string imageName = operationArray[1].GetString(); + imageName.erase(std::remove(imageName.begin(), imageName.end(), '"'), imageName.end()); + + std::string imagePath = operationArray[2].GetString(); + imagePath.erase(std::remove(imagePath.begin(), imagePath.end(), '"'), imagePath.end()); + + const mbgl::filesystem::path filePath = (mbgl::filesystem::path(manifest.getAssetPath()) / imagePath); + + result.emplace_back([filePath = filePath.string(), imageName, sdf, pixelRatio](TestContext& ctx) { + mbgl::optional maybeImage = mbgl::util::readFile(filePath); + if (!maybeImage) { + ctx.getMetadata().errorMessage = std::string("Failed to load expected image ") + filePath; + return false; + } + + ctx.getMap().getStyle().addImage( + std::make_unique(imageName, mbgl::decodeImage(*maybeImage), pixelRatio, sdf)); + return true; + }); + + } else if (operationArray[0].GetString() == removeImageOp) { + // removeImage + assert(operationArray.Size() >= 2u); + assert(operationArray[1].IsString()); + + std::string imageName{operationArray[1].GetString(), operationArray[1].GetStringLength()}; + result.emplace_back([imageName](TestContext& ctx) { + ctx.getMap().getStyle().removeImage(imageName); + return true; + }); + } else if (operationArray[0].GetString() == setStyleOp) { + // setStyle + assert(operationArray.Size() >= 2u); + std::string json; + if (operationArray[1].IsString()) { + std::string stylePath = manifest.localizeURL(operationArray[1].GetString()); + auto maybeStyle = readJson(stylePath); + if (maybeStyle.is()) { + auto& style = maybeStyle.get(); + manifest.localizeStyleURLs(static_cast(style), style); + json = serializeJsonValue(style); + } + } else { + manifest.localizeStyleURLs(operationArray[1], metadata.document); + json = serializeJsonValue(operationArray[1]); + } + result.emplace_back([json](TestContext& ctx) { + ctx.getMap().getStyle().loadJSON(json); + return true; + }); + } else if (operationArray[0].GetString() == setCenterOp) { + // setCenter + assert(operationArray.Size() >= 2u); + assert(operationArray[1].IsArray()); + + const auto& centerArray = operationArray[1].GetArray(); + assert(centerArray.Size() == 2u); + mbgl::LatLng center{centerArray[1].GetDouble(), centerArray[0].GetDouble()}; + result.emplace_back([center](TestContext& ctx) { + ctx.getMap().jumpTo(mbgl::CameraOptions().withCenter(center)); + return true; + }); + } else if (operationArray[0].GetString() == setZoomOp) { + // setZoom + assert(operationArray.Size() >= 2u); + assert(operationArray[1].IsNumber()); + double zoom = operationArray[1].GetDouble(); + result.emplace_back([zoom](TestContext& ctx) { + ctx.getMap().jumpTo(mbgl::CameraOptions().withZoom(zoom)); + return true; + }); + } else if (operationArray[0].GetString() == setBearingOp) { + // setBearing + assert(operationArray.Size() >= 2u); + assert(operationArray[1].IsNumber()); + double bearing = operationArray[1].GetDouble(); + result.emplace_back([bearing](TestContext& ctx) { + ctx.getMap().jumpTo(mbgl::CameraOptions().withBearing(bearing)); + return true; + }); + } else if (operationArray[0].GetString() == setPitchOp) { + // setPitch + assert(operationArray.Size() >= 2u); + assert(operationArray[1].IsNumber()); + double pitch = operationArray[1].GetDouble(); + result.emplace_back([pitch](TestContext& ctx) { + ctx.getMap().jumpTo(mbgl::CameraOptions().withPitch(pitch)); + return true; + }); + } else if (operationArray[0].GetString() == setFilterOp) { + // setFilter + assert(operationArray.Size() >= 3u); + assert(operationArray[1].IsString()); + + std::string layerName{operationArray[1].GetString(), operationArray[1].GetStringLength()}; + mbgl::style::conversion::Error error; + auto converted = mbgl::style::conversion::convert(operationArray[2], error); + result.emplace_back([converted, layerName, error](TestContext& ctx) { + if (!converted) { + ctx.getMetadata().errorMessage = std::string("Unable to convert filter: ") + error.message; + return false; + } + auto layer = ctx.getMap().getStyle().getLayer(layerName); + if (!layer) { + ctx.getMetadata().errorMessage = std::string("Layer not found: ") + layerName; + return false; + } + layer->setFilter(std::move(*converted)); + return true; + }); + + } else if (operationArray[0].GetString() == setLayerZoomRangeOp) { + // setLayerZoomRange + assert(operationArray.Size() >= 4u); + assert(operationArray[1].IsString()); + assert(operationArray[2].IsNumber()); + assert(operationArray[3].IsNumber()); + + std::string layerName{operationArray[1].GetString(), operationArray[1].GetStringLength()}; + float minZoom = operationArray[2].GetFloat(); + float maxZoom = operationArray[3].GetFloat(); + result.emplace_back([layerName, minZoom, maxZoom](TestContext& ctx) { + auto layer = ctx.getMap().getStyle().getLayer(layerName); + if (!layer) { + ctx.getMetadata().errorMessage = std::string("Layer not found: ") + layerName; + return false; + } + layer->setMinZoom(minZoom); + layer->setMaxZoom(maxZoom); + return true; + }); + } else if (operationArray[0].GetString() == setLightOp) { + // setLight + assert(operationArray.Size() >= 2u); + assert(operationArray[1].IsObject()); + mbgl::style::conversion::Error error; + auto converted = mbgl::style::conversion::convert(operationArray[1], error); + if (!converted) { + metadata.errorMessage = std::string("Unable to convert light: ") + error.message; + return {}; + } + result.emplace_back([impl = converted->impl](TestContext& ctx) { + ctx.getMap().getStyle().setLight(std::make_unique(impl)); + return true; + }); + } else if (operationArray[0].GetString() == addLayerOp) { + // addLayer + assert(operationArray.Size() >= 2u); + assert(operationArray[1].IsObject()); + result.emplace_back([json = serializeJsonValue(operationArray[1])](TestContext& ctx) { + mbgl::style::conversion::Error error; + auto converted = mbgl::style::conversion::convertJSON>(json, error); + if (!converted) { + ctx.getMetadata().errorMessage = std::string("Unable to convert layer: ") + error.message; + return false; + } + ctx.getMap().getStyle().addLayer(std::move(*converted)); + return true; + }); + } else if (operationArray[0].GetString() == removeLayerOp) { + // removeLayer + assert(operationArray.Size() >= 2u); + assert(operationArray[1].IsString()); + std::string layerName = operationArray[1].GetString(); + result.emplace_back( + [layerName](TestContext& ctx) { return bool(ctx.getMap().getStyle().removeLayer(layerName)); }); + } else if (operationArray[0].GetString() == addSourceOp) { + // addSource + assert(operationArray.Size() >= 3u); + assert(operationArray[1].IsString()); + assert(operationArray[2].IsObject()); + std::string sourceName = operationArray[1].GetString(); + + manifest.localizeSourceURLs(operationArray[2], metadata.document); + result.emplace_back([sourceName, json = serializeJsonValue(operationArray[2])](TestContext& ctx) { + mbgl::style::conversion::Error error; + auto converted = + mbgl::style::conversion::convertJSON>(json, error, sourceName); + if (!converted) { + ctx.getMetadata().errorMessage = std::string("Unable to convert source: ") + error.message; + return false; + } + ctx.getMap().getStyle().addSource(std::move(*converted)); + return true; + }); + } else if (operationArray[0].GetString() == removeSourceOp) { + // removeSource + assert(operationArray.Size() >= 2u); + assert(operationArray[1].IsString()); + std::string sourceName = operationArray[1].GetString(); + result.emplace_back( + [sourceName](TestContext& ctx) { return bool(ctx.getMap().getStyle().removeSource(sourceName)); }); + } else if (operationArray[0].GetString() == setLayoutPropertyOp || + operationArray[0].GetString() == setPaintPropertyOp) { + // set{Paint|Layout}Property + assert(operationArray.Size() >= 4u); + assert(operationArray[1].IsString()); + assert(operationArray[2].IsString()); + + std::string layerName{operationArray[1].GetString(), operationArray[1].GetStringLength()}; + std::string propertyName{operationArray[2].GetString(), operationArray[2].GetStringLength()}; + result.emplace_back( + [layerName, propertyName, json = serializeJsonValue(operationArray[3])](TestContext& ctx) { + auto layer = ctx.getMap().getStyle().getLayer(layerName); + if (!layer) { + ctx.getMetadata().errorMessage = std::string("Layer not found: ") + layerName; + return false; + } + mbgl::JSDocument d; + d.Parse(json.c_str(), json.length()); + const mbgl::JSValue* propertyValue = &d; + layer->setProperty(propertyName, propertyValue); + return true; + }); + } else if (operationArray[0].GetString() == fileSizeProbeOp) { + // probeFileSize + assert(operationArray.Size() >= 4u); + assert(operationArray[1].IsString()); + assert(operationArray[2].IsString()); + assert(operationArray[3].IsNumber()); + + std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength()); + std::string path = std::string(operationArray[2].GetString(), operationArray[2].GetStringLength()); + assert(!path.empty()); + + float tolerance = operationArray[3].GetDouble(); + mbgl::filesystem::path filePath(path); + + bool compressed = false; + if (operationArray.Size() == 5) { + assert(operationArray[4].IsString()); + assert(std::string(operationArray[4].GetString(), operationArray[4].GetStringLength()) == "compressed"); + compressed = true; + } + + if (!filePath.is_absolute()) { + filePath = metadata.paths.defaultExpectations() / filePath; + } + result.emplace_back([filePath, path, mark, tolerance, compressed](TestContext& ctx) { + if (!mbgl::filesystem::exists(filePath)) { + ctx.getMetadata().errorMessage = std::string("File not found: ") + path; + return false; + } + size_t size = 0; + if (compressed) { + size = mbgl::util::compress(*mbgl::util::readFile(filePath)).size(); + } else { + size = mbgl::filesystem::file_size(filePath); + } + + ctx.getMetadata().metrics.fileSize.emplace(std::piecewise_construct, + std::forward_as_tuple(std::move(mark)), + std::forward_as_tuple(std::move(path), size, tolerance)); + return true; + }); + } else if (operationArray[0].GetString() == memoryProbeStartOp) { + // probeMemoryStart + result.emplace_back([](TestContext&) { + assert(!AllocationIndex::isActive()); + AllocationIndex::setActive(true); + return true; + }); + } else if (operationArray[0].GetString() == memoryProbeOp) { + // probeMemory + assert(AllocationIndex::isActive()); + assert(operationArray.Size() >= 2u); + assert(operationArray[1].IsString()); + std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength()); + float tolerance = -1.0f; + if (operationArray.Size() >= 3u) { + assert(operationArray[2].IsNumber()); + tolerance = float(operationArray[2].GetDouble()); + } + result.emplace_back([mark, tolerance](TestContext& ctx) { + auto emplaced = ctx.getMetadata().metrics.memory.emplace( + std::piecewise_construct, + std::forward_as_tuple(std::move(mark)), + std::forward_as_tuple(AllocationIndex::getAllocatedSizePeak(), + AllocationIndex::getAllocationsCount())); + if (tolerance >= 0.0f) emplaced.first->second.tolerance = tolerance; + return true; + }); + } else if (operationArray[0].GetString() == memoryProbeEndOp) { + // probeMemoryEnd + result.emplace_back([](TestContext&) { + assert(AllocationIndex::isActive()); + AllocationIndex::setActive(false); + AllocationIndex::reset(); + return true; + }); + } else if (operationArray[0].GetString() == networkProbeStartOp) { + // probeNetworkStart + result.emplace_back([](TestContext&) { + assert(!mbgl::ProxyFileSource::isTrackingActive()); + mbgl::ProxyFileSource::setTrackingActive(true); + return true; + }); + } else if (operationArray[0].GetString() == networkProbeOp) { + // probeNetwork + assert(operationArray.Size() >= 2u); + assert(operationArray[1].IsString()); + std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength()); + result.emplace_back([mark](TestContext& ctx) { + assert(mbgl::ProxyFileSource::isTrackingActive()); + ctx.getMetadata().metrics.network.emplace( + std::piecewise_construct, + std::forward_as_tuple(std::move(mark)), + std::forward_as_tuple(mbgl::ProxyFileSource::getRequestCount(), + mbgl::ProxyFileSource::getTransferredSize())); + return true; + }); + } else if (operationArray[0].GetString() == networkProbeEndOp) { + // probeNetworkEnd + result.emplace_back([](TestContext&) { + assert(mbgl::ProxyFileSource::isTrackingActive()); + mbgl::ProxyFileSource::setTrackingActive(false); + return true; + }); + } else if (operationArray[0].GetString() == setFeatureStateOp) { + // setFeatureState + assert(operationArray.Size() >= 3u); + assert(operationArray[1].IsObject()); + assert(operationArray[2].IsObject()); + + using namespace mbgl; + using namespace mbgl::style::conversion; + + std::string sourceID; + mbgl::optional sourceLayer; + std::string featureID; + std::string stateKey; + Value stateValue; + bool valueParsed = false; + FeatureState parsedState; + + const auto& featureOptions = operationArray[1].GetObject(); + if (featureOptions.HasMember("source")) { + sourceID = featureOptions["source"].GetString(); + } + if (featureOptions.HasMember("sourceLayer")) { + sourceLayer = {featureOptions["sourceLayer"].GetString()}; + } + if (featureOptions.HasMember("id")) { + if (featureOptions["id"].IsString()) { + featureID = featureOptions["id"].GetString(); + } else if (featureOptions["id"].IsNumber()) { + featureID = mbgl::util::toString(featureOptions["id"].GetUint64()); + } + } + const JSValue* state = &operationArray[2]; + + const std::function(const std::string&, const Convertible&)> convertFn = + [&](const std::string& k, const Convertible& v) -> optional { + optional value = toValue(v); + if (value) { + stateValue = std::move(*value); + valueParsed = true; + } else if (isArray(v)) { + std::vector array; + std::size_t length = arrayLength(v); + array.reserve(length); + for (size_t i = 0; i < length; ++i) { + optional arrayVal = toValue(arrayMember(v, i)); + if (arrayVal) { + array.emplace_back(*arrayVal); + } + } + std::unordered_map result; + result[k] = std::move(array); + stateValue = std::move(result); + valueParsed = true; + return nullopt; + + } else if (isObject(v)) { + eachMember(v, convertFn); + } + + if (!valueParsed) { + metadata.errorMessage = std::string("Could not get feature state value, state key: ") + k; + return nullopt; + } + stateKey = k; + parsedState[stateKey] = stateValue; + return nullopt; + }; + + eachMember(state, convertFn); + result.emplace_back([sourceID, sourceLayer, featureID, parsedState](TestContext& ctx) { + auto& frontend = ctx.getFrontend(); + try { + frontend.render(ctx.getMap()); + } catch (const std::exception&) { + return false; + } + frontend.getRenderer()->setFeatureState(sourceID, sourceLayer, featureID, parsedState); + return true; + }); + } else if (operationArray[0].GetString() == getFeatureStateOp) { + // getFeatureState + assert(operationArray.Size() >= 2u); + assert(operationArray[1].IsObject()); + + std::string sourceID; + mbgl::optional sourceLayer; + std::string featureID; + + const auto& featureOptions = operationArray[1].GetObject(); + if (featureOptions.HasMember("source")) { + sourceID = featureOptions["source"].GetString(); + } + if (featureOptions.HasMember("sourceLayer")) { + sourceLayer = {featureOptions["sourceLayer"].GetString()}; + } + if (featureOptions.HasMember("id")) { + if (featureOptions["id"].IsString()) { + featureID = featureOptions["id"].GetString(); + } else if (featureOptions["id"].IsNumber()) { + featureID = mbgl::util::toString(featureOptions["id"].GetUint64()); + } + } + result.emplace_back([sourceID, sourceLayer, featureID](TestContext& ctx) { + auto& frontend = ctx.getFrontend(); + try { + frontend.render(ctx.getMap()); + } catch (const std::exception&) { + return false; + } + mbgl::FeatureState state; + frontend.getRenderer()->getFeatureState(state, sourceID, sourceLayer, featureID); + return true; + }); + } else if (operationArray[0].GetString() == removeFeatureStateOp) { + // removeFeatureState + assert(operationArray.Size() >= 2u); + assert(operationArray[1].IsObject()); + + std::string sourceID; + mbgl::optional sourceLayer; + std::string featureID; + mbgl::optional stateKey; + + const auto& featureOptions = operationArray[1].GetObject(); + if (featureOptions.HasMember("source")) { + sourceID = featureOptions["source"].GetString(); + } + if (featureOptions.HasMember("sourceLayer")) { + sourceLayer = {featureOptions["sourceLayer"].GetString()}; + } + if (featureOptions.HasMember("id")) { + if (featureOptions["id"].IsString()) { + featureID = featureOptions["id"].GetString(); + } else if (featureOptions["id"].IsNumber()) { + featureID = mbgl::util::toString(featureOptions["id"].GetUint64()); + } + } + + if (operationArray.Size() >= 3u) { + assert(operationArray[2].IsString()); + stateKey = {operationArray[2].GetString()}; + } + + result.emplace_back([sourceID, sourceLayer, featureID, stateKey](TestContext& ctx) { + auto& frontend = ctx.getFrontend(); + try { + frontend.render(ctx.getMap()); + } catch (const std::exception&) { + return false; + } + frontend.getRenderer()->removeFeatureState(sourceID, sourceLayer, featureID, stateKey); + return true; + }); + } else if (operationArray[0].GetString() == panGestureOp) { + // benchmarkPanGesture + assert(operationArray.Size() >= 4u); + assert(operationArray[1].IsString()); // identifier + assert(operationArray[2].IsNumber()); // duration + assert(operationArray[3].IsArray()); // start [lat, lng, zoom] + assert(operationArray[4].IsArray()); // end [lat, lng, zoom] + + if (metadata.mapMode != mbgl::MapMode::Continuous) { + metadata.errorMessage = "Map mode must be Continous for " + panGestureOp + " operation"; + return {}; + } + + std::string mark = operationArray[1].GetString(); + int duration = operationArray[2].GetFloat(); + mbgl::LatLng startPos, endPos; + double startZoom, endZoom; + + auto parsePosition = [](auto arr) -> std::tuple { + assert(arr.Size() >= 3); + return {{arr[1].GetDouble(), arr[0].GetDouble()}, arr[2].GetDouble()}; + }; + + std::tie(startPos, startZoom) = parsePosition(operationArray[3].GetArray()); + std::tie(endPos, endZoom) = parsePosition(operationArray[4].GetArray()); + + result.emplace_back([mark, duration, startPos, endPos, startZoom, endZoom](TestContext& ctx) { + auto& map = ctx.getMap(); + auto& observer = ctx.getObserver(); + auto& frontend = ctx.getFrontend(); + std::vector samples; + // Jump to the starting point of the segment and make sure there's something to render + map.jumpTo(mbgl::CameraOptions().withCenter(startPos).withZoom(startZoom)); + + observer.reset(); + while (!observer.finishRenderingMap) { + frontend.renderOnce(map); + } + + if (observer.mapLoadFailure) return false; + + size_t frames = 0; + float totalTime = 0.0; + bool transitionFinished = false; + + mbgl::AnimationOptions animationOptions(mbgl::Milliseconds(duration * 1000)); + animationOptions.minZoom = mbgl::util::min(startZoom, endZoom); + animationOptions.transitionFinishFn = [&]() { transitionFinished = true; }; + + map.flyTo(mbgl::CameraOptions().withCenter(endPos).withZoom(endZoom), animationOptions); + + while (!transitionFinished) { + frames++; + frontend.renderOnce(map); + float frameTime = (float)frontend.getFrameTime(); + totalTime += frameTime; + + samples.push_back(frameTime); + } + + float averageFps = totalTime > 0.0 ? frames / totalTime : 0.0; + float minFrameTime = 0.0; + + // Use 1% of the longest frames to compute the minimum fps + std::sort(samples.begin(), samples.end()); + + int sampleCount = mbgl::util::max(1, (int)samples.size() / 100); + for (auto it = samples.rbegin(); it != samples.rbegin() + sampleCount; it++) minFrameTime += *it; + + float minOnePcFps = sampleCount / minFrameTime; + + ctx.getMetadata().metrics.fps.insert({mark, {averageFps, minOnePcFps, 0.0f}}); + return true; + }); + } else if (operationArray[0].GetString() == gfxProbeStartOp) { + // probeGFXStart + result.emplace_back([](TestContext& ctx) { + assert(!ctx.gfxProbeActive); + ctx.gfxProbeActive = true; + ctx.baselineGfxProbe = ctx.activeGfxProbe; + return true; + }); + } else if (operationArray[0].GetString() == gfxProbeEndOp) { + // probeGFXEnd + result.emplace_back([](TestContext& ctx) { + assert(ctx.gfxProbeActive); + ctx.gfxProbeActive = false; + return true; + }); + } else if (operationArray[0].GetString() == gfxProbeOp) { + // probeGFX + assert(operationArray.Size() >= 2u); + assert(operationArray[1].IsString()); + + std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength()); + result.emplace_back([mark](TestContext& ctx) { + auto& frontend = ctx.getFrontend(); + // Render the map and fetch rendering stats + try { + mbgl::gfx::RenderingStats stats = frontend.render(ctx.getMap()).stats; + ctx.activeGfxProbe = GfxProbe(stats, ctx.activeGfxProbe); + } catch (const std::exception&) { + return false; + } + // Compare memory allocations to the baseline probe + GfxProbe metricProbe = ctx.activeGfxProbe; + metricProbe.memIndexBuffers.peak -= ctx.baselineGfxProbe.memIndexBuffers.peak; + metricProbe.memVertexBuffers.peak -= ctx.baselineGfxProbe.memVertexBuffers.peak; + metricProbe.memTextures.peak -= ctx.baselineGfxProbe.memTextures.peak; + ctx.getMetadata().metrics.gfx.insert({mark, metricProbe}); + return true; + }); + } else { + metadata.errorMessage = std::string("Unsupported operation: ") + operationArray[0].GetString(); + return {}; + } + } + + return result; +} + // https://stackoverflow.com/questions/7053538/how-do-i-encode-a-string-to-base64-using-only-boost std::string encodeBase64(const std::string& data) { using namespace boost::archive::iterators; diff --git a/render-test/parser.hpp b/render-test/parser.hpp index b7b4e9201d..905118b401 100644 --- a/render-test/parser.hpp +++ b/render-test/parser.hpp @@ -25,8 +25,45 @@ std::vector readExpectedJSONEntries(const mbgl::filesystem::path& b TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path); TestMetadata parseTestMetadata(const TestPaths& paths, const Manifest& manifest); +TestOperations parseTestOperations(TestMetadata& metadata, const Manifest& manifest); std::string createResultPage(const TestStatistics&, const std::vector&, bool shuffle, uint32_t seed); std::string toJSON(const mbgl::Value& value, unsigned indent, bool singleLine); std::string toJSON(const std::vector& features, unsigned indent, bool singleLine); + +namespace TestOperationNames { +extern const std::string waitOp; +extern const std::string sleepOp; +extern const std::string addImageOp; +extern const std::string updateImageOp; +extern const std::string removeImageOp; +extern const std::string setStyleOp; +extern const std::string setCenterOp; +extern const std::string setZoomOp; +extern const std::string setBearingOp; +extern const std::string setPitchOp; +extern const std::string setFilterOp; +extern const std::string setLayerZoomRangeOp; +extern const std::string setLightOp; +extern const std::string addLayerOp; +extern const std::string removeLayerOp; +extern const std::string addSourceOp; +extern const std::string removeSourceOp; +extern const std::string setPaintPropertyOp; +extern const std::string setLayoutPropertyOp; +extern const std::string fileSizeProbeOp; +extern const std::string memoryProbeOp; +extern const std::string memoryProbeStartOp; +extern const std::string memoryProbeEndOp; +extern const std::string networkProbeOp; +extern const std::string networkProbeStartOp; +extern const std::string networkProbeEndOp; +extern const std::string setFeatureStateOp; +extern const std::string getFeatureStateOp; +extern const std::string removeFeatureStateOp; +extern const std::string panGestureOp; +extern const std::string gfxProbeOp; +extern const std::string gfxProbeStartOp; +extern const std::string gfxProbeEndOp; +} // namespace TestOperationNames \ No newline at end of file diff --git a/render-test/runner.cpp b/render-test/runner.cpp index 4037d42e45..8f47977f31 100644 --- a/render-test/runner.cpp +++ b/render-test/runner.cpp @@ -2,22 +2,16 @@ #include #include #include -#include -#include -#include -#include #include #include #include #include #include #include -#include #include #include #include #include -#include #include @@ -34,42 +28,7 @@ #include using namespace mbgl; - -namespace { -static const std::string waitOp("wait"); -static const std::string sleepOp("sleep"); -static const std::string addImageOp("addImage"); -static const std::string updateImageOp("updateImage"); -static const std::string removeImageOp("removeImage"); -static const std::string setStyleOp("setStyle"); -static const std::string setCenterOp("setCenter"); -static const std::string setZoomOp("setZoom"); -static const std::string setBearingOp("setBearing"); -static const std::string setPitchOp("setPitch"); -static const std::string setFilterOp("setFilter"); -static const std::string setLayerZoomRangeOp("setLayerZoomRange"); -static const std::string setLightOp("setLight"); -static const std::string addLayerOp("addLayer"); -static const std::string removeLayerOp("removeLayer"); -static const std::string addSourceOp("addSource"); -static const std::string removeSourceOp("removeSource"); -static const std::string setPaintPropertyOp("setPaintProperty"); -static const std::string setLayoutPropertyOp("setLayoutProperty"); -static const std::string fileSizeProbeOp("probeFileSize"); -static const std::string memoryProbeOp("probeMemory"); -static const std::string memoryProbeStartOp("probeMemoryStart"); -static const std::string memoryProbeEndOp("probeMemoryEnd"); -static const std::string networkProbeOp("probeNetwork"); -static const std::string networkProbeStartOp("probeNetworkStart"); -static const std::string networkProbeEndOp("probeNetworkEnd"); -static const std::string setFeatureStateOp("setFeatureState"); -static const std::string getFeatureStateOp("getFeatureState"); -static const std::string removeFeatureStateOp("removeFeatureState"); -static const std::string panGestureOp("panGesture"); -static const std::string gfxProbeOp("probeGFX"); -static const std::string gfxProbeStartOp("probeGFXStart"); -static const std::string gfxProbeEndOp("probeGFXEnd"); -} // namespace +using namespace TestOperationNames; GfxProbe::GfxProbe(const mbgl::gfx::RenderingStats& stats, const GfxProbe& prev) : numBuffers(stats.numBuffers), @@ -80,29 +39,6 @@ GfxProbe::GfxProbe(const mbgl::gfx::RenderingStats& stats, const GfxProbe& prev) memVertexBuffers(stats.memVertexBuffers, std::max(stats.memVertexBuffers, prev.memVertexBuffers.peak)), memTextures(stats.memTextures, std::max(stats.memTextures, prev.memTextures.peak)) {} -class TestRunnerMapObserver : public MapObserver { -public: - TestRunnerMapObserver() : mapLoadFailure(false), finishRenderingMap(false), idle(false) {} - - void onDidFailLoadingMap(MapLoadError, const std::string&) override { mapLoadFailure = true; } - - void onDidFinishRenderingMap(RenderMode mode) override final { - if (!finishRenderingMap) finishRenderingMap = mode == RenderMode::Full; - } - - void onDidBecomeIdle() override final { idle = true; } - - void reset() { - mapLoadFailure = false; - finishRenderingMap = false; - idle = false; - } - - bool mapLoadFailure; - bool finishRenderingMap; - bool idle; -}; - // static gfx::HeadlessBackend::SwapBehaviour swapBehavior(MapMode mode) { return mode == MapMode::Continuous ? gfx::HeadlessBackend::SwapBehaviour::Flush @@ -580,578 +516,6 @@ bool TestRunner::checkProbingResults(TestMetadata& resultMetadata) { return checkResult; } -bool TestRunner::runOperations(TestContext& ctx) { - TestMetadata& metadata = ctx.getMetadata(); - if (!metadata.document.HasMember("metadata") || !metadata.document["metadata"].HasMember("test") || - !metadata.document["metadata"]["test"].HasMember("operations")) { - return true; - } - assert(metadata.document["metadata"]["test"]["operations"].IsArray()); - - const auto& operationsArray = metadata.document["metadata"]["test"]["operations"].GetArray(); - if (operationsArray.Empty()) { - return true; - } - - const auto& operationIt = operationsArray.Begin(); - assert(operationIt->IsArray()); - - const auto& operationArray = operationIt->GetArray(); - assert(operationArray.Size() >= 1u); - - HeadlessFrontend& frontend = ctx.getFrontend(); - Map& map = ctx.getMap(); - TestRunnerMapObserver& observer = ctx.getObserver(); - - if (operationArray[0].GetString() == waitOp) { - // wait - try { - frontend.render(map); - } catch (const std::exception&) { - return false; - } - } else if (operationArray[0].GetString() == sleepOp) { - // sleep - mbgl::util::Timer sleepTimer; - bool sleeping = true; - - mbgl::Duration duration = mbgl::Seconds(3); - if (operationArray.Size() >= 2u) { - duration = mbgl::Milliseconds(operationArray[1].GetUint()); - } - - sleepTimer.start(duration, mbgl::Duration::zero(), [&]() { - sleeping = false; - }); - - while (sleeping) { - mbgl::util::RunLoop::Get()->runOnce(); - } - } else if (operationArray[0].GetString() == addImageOp || operationArray[0].GetString() == updateImageOp) { - // addImage | updateImage - assert(operationArray.Size() >= 3u); - - float pixelRatio = 1.0f; - bool sdf = false; - - if (operationArray.Size() == 4u) { - assert(operationArray[3].IsObject()); - const auto& imageOptions = operationArray[3].GetObject(); - if (imageOptions.HasMember("pixelRatio")) { - pixelRatio = imageOptions["pixelRatio"].GetFloat(); - } - if (imageOptions.HasMember("sdf")) { - sdf = imageOptions["sdf"].GetBool(); - } - } - - std::string imageName = operationArray[1].GetString(); - imageName.erase(std::remove(imageName.begin(), imageName.end(), '"'), imageName.end()); - - std::string imagePath = operationArray[2].GetString(); - imagePath.erase(std::remove(imagePath.begin(), imagePath.end(), '"'), imagePath.end()); - - const mbgl::filesystem::path filePath = (mbgl::filesystem::path(manifest.getAssetPath()) / imagePath); - - mbgl::optional maybeImage = mbgl::util::readFile(filePath.string()); - if (!maybeImage) { - metadata.errorMessage = std::string("Failed to load expected image ") + filePath.string(); - return false; - } - - map.getStyle().addImage(std::make_unique(imageName, mbgl::decodeImage(*maybeImage), pixelRatio, sdf)); - } else if (operationArray[0].GetString() == removeImageOp) { - // removeImage - assert(operationArray.Size() >= 2u); - assert(operationArray[1].IsString()); - - const std::string imageName { operationArray[1].GetString(), operationArray[1].GetStringLength() }; - map.getStyle().removeImage(imageName); - } else if (operationArray[0].GetString() == setStyleOp) { - // setStyle - assert(operationArray.Size() >= 2u); - if (operationArray[1].IsString()) { - std::string stylePath = manifest.localizeURL(operationArray[1].GetString()); - auto maybeStyle = readJson(stylePath); - if (maybeStyle.is()) { - auto& style = maybeStyle.get(); - manifest.localizeStyleURLs((mbgl::JSValue&)style, style); - map.getStyle().loadJSON(serializeJsonValue(style)); - } - } else { - manifest.localizeStyleURLs(operationArray[1], metadata.document); - map.getStyle().loadJSON(serializeJsonValue(operationArray[1])); - } - } else if (operationArray[0].GetString() == setCenterOp) { - // setCenter - assert(operationArray.Size() >= 2u); - assert(operationArray[1].IsArray()); - - const auto& centerArray = operationArray[1].GetArray(); - assert(centerArray.Size() == 2u); - - map.jumpTo(mbgl::CameraOptions().withCenter(mbgl::LatLng(centerArray[1].GetDouble(), centerArray[0].GetDouble()))); - } else if (operationArray[0].GetString() == setZoomOp) { - // setZoom - assert(operationArray.Size() >= 2u); - assert(operationArray[1].IsNumber()); - map.jumpTo(mbgl::CameraOptions().withZoom(operationArray[1].GetDouble())); - } else if (operationArray[0].GetString() == setBearingOp) { - // setBearing - assert(operationArray.Size() >= 2u); - assert(operationArray[1].IsNumber()); - map.jumpTo(mbgl::CameraOptions().withBearing(operationArray[1].GetDouble())); - } else if (operationArray[0].GetString() == setPitchOp) { - // setPitch - assert(operationArray.Size() >= 2u); - assert(operationArray[1].IsNumber()); - map.jumpTo(mbgl::CameraOptions().withPitch(operationArray[1].GetDouble())); - } else if (operationArray[0].GetString() == setFilterOp) { - // setFilter - assert(operationArray.Size() >= 3u); - assert(operationArray[1].IsString()); - - const std::string layerName { operationArray[1].GetString(), operationArray[1].GetStringLength() }; - - mbgl::style::conversion::Error error; - auto converted = mbgl::style::conversion::convert(operationArray[2], error); - if (!converted) { - metadata.errorMessage = std::string("Unable to convert filter: ") + error.message; - return false; - } else { - auto layer = map.getStyle().getLayer(layerName); - if (!layer) { - metadata.errorMessage = std::string("Layer not found: ") + layerName; - return false; - } else { - layer->setFilter(std::move(*converted)); - } - } - } else if (operationArray[0].GetString() == setLayerZoomRangeOp) { - // setLayerZoomRange - assert(operationArray.Size() >= 4u); - assert(operationArray[1].IsString()); - assert(operationArray[2].IsNumber()); - assert(operationArray[3].IsNumber()); - - const std::string layerName { operationArray[1].GetString(), operationArray[1].GetStringLength() }; - auto layer = map.getStyle().getLayer(layerName); - if (!layer) { - metadata.errorMessage = std::string("Layer not found: ") + layerName; - return false; - } else { - layer->setMinZoom(operationArray[2].GetFloat()); - layer->setMaxZoom(operationArray[3].GetFloat()); - } - } else if (operationArray[0].GetString() == setLightOp) { - // setLight - assert(operationArray.Size() >= 2u); - assert(operationArray[1].IsObject()); - - mbgl::style::conversion::Error error; - auto converted = mbgl::style::conversion::convert(operationArray[1], error); - if (!converted) { - metadata.errorMessage = std::string("Unable to convert light: ") + error.message; - return false; - } else { - map.getStyle().setLight(std::make_unique(std::move(*converted))); - } - } else if (operationArray[0].GetString() == addLayerOp) { - // addLayer - assert(operationArray.Size() >= 2u); - assert(operationArray[1].IsObject()); - - mbgl::style::conversion::Error error; - auto converted = mbgl::style::conversion::convert>(operationArray[1], error); - if (!converted) { - metadata.errorMessage = std::string("Unable to convert layer: ") + error.message; - return false; - } else { - map.getStyle().addLayer(std::move(*converted)); - } - } else if (operationArray[0].GetString() == removeLayerOp) { - // removeLayer - assert(operationArray.Size() >= 2u); - assert(operationArray[1].IsString()); - map.getStyle().removeLayer(operationArray[1].GetString()); - } else if (operationArray[0].GetString() == addSourceOp) { - // addSource - assert(operationArray.Size() >= 3u); - assert(operationArray[1].IsString()); - assert(operationArray[2].IsObject()); - - manifest.localizeSourceURLs(operationArray[2], metadata.document); - - mbgl::style::conversion::Error error; - auto converted = mbgl::style::conversion::convert>(operationArray[2], error, operationArray[1].GetString()); - if (!converted) { - metadata.errorMessage = std::string("Unable to convert source: ") + error.message; - return false; - } else { - map.getStyle().addSource(std::move(*converted)); - } - } else if (operationArray[0].GetString() == removeSourceOp) { - // removeSource - assert(operationArray.Size() >= 2u); - assert(operationArray[1].IsString()); - map.getStyle().removeSource(operationArray[1].GetString()); - } else if (operationArray[0].GetString() == setLayoutPropertyOp || - operationArray[0].GetString() == setPaintPropertyOp) { - // set{Paint|Layout}Property - assert(operationArray.Size() >= 4u); - assert(operationArray[1].IsString()); - assert(operationArray[2].IsString()); - - const std::string layerName { operationArray[1].GetString(), operationArray[1].GetStringLength() }; - const std::string propertyName { operationArray[2].GetString(), operationArray[2].GetStringLength() }; - - auto layer = map.getStyle().getLayer(layerName); - if (!layer) { - metadata.errorMessage = std::string("Layer not found: ") + layerName; - return false; - } else { - const mbgl::JSValue* propertyValue = &operationArray[3]; - layer->setProperty(propertyName, propertyValue); - } - } else if (operationArray[0].GetString() == fileSizeProbeOp) { - // probeFileSize - assert(operationArray.Size() >= 4u); - assert(operationArray[1].IsString()); - assert(operationArray[2].IsString()); - assert(operationArray[3].IsNumber()); - - std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength()); - std::string path = std::string(operationArray[2].GetString(), operationArray[2].GetStringLength()); - assert(!path.empty()); - - float tolerance = operationArray[3].GetDouble(); - mbgl::filesystem::path filePath(path); - - bool compressed = false; - if (operationArray.Size() == 5) { - assert(operationArray[4].IsString()); - assert(std::string(operationArray[4].GetString(), operationArray[4].GetStringLength()) == "compressed"); - compressed = true; - } - - if (!filePath.is_absolute()) { - filePath = metadata.paths.defaultExpectations() / filePath; - } - - if (mbgl::filesystem::exists(filePath)) { - size_t size = 0; - if (compressed) { - size = mbgl::util::compress(*mbgl::util::readFile(filePath)).size(); - } else { - size = mbgl::filesystem::file_size(filePath); - } - - metadata.metrics.fileSize.emplace(std::piecewise_construct, - std::forward_as_tuple(std::move(mark)), - std::forward_as_tuple(std::move(path), size, tolerance)); - } else { - metadata.errorMessage = std::string("File not found: ") + path; - return false; - } - } else if (operationArray[0].GetString() == memoryProbeStartOp) { - // probeMemoryStart - assert(!AllocationIndex::isActive()); - AllocationIndex::setActive(true); - } else if (operationArray[0].GetString() == memoryProbeOp) { - // probeMemory - assert(AllocationIndex::isActive()); - assert(operationArray.Size() >= 2u); - assert(operationArray[1].IsString()); - std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength()); - - auto emplaced = metadata.metrics.memory.emplace( - std::piecewise_construct, - std::forward_as_tuple(std::move(mark)), - std::forward_as_tuple(AllocationIndex::getAllocatedSizePeak(), AllocationIndex::getAllocationsCount())); - assert(emplaced.second); - if (operationArray.Size() >= 3u) { - assert(operationArray[2].IsNumber()); - emplaced.first->second.tolerance = float(operationArray[2].GetDouble()); - } - } else if (operationArray[0].GetString() == memoryProbeEndOp) { - // probeMemoryEnd - assert(AllocationIndex::isActive()); - AllocationIndex::setActive(false); - AllocationIndex::reset(); - } else if (operationArray[0].GetString() == networkProbeStartOp) { - // probeNetworkStart - assert(!ProxyFileSource::isTrackingActive()); - ProxyFileSource::setTrackingActive(true); - } else if (operationArray[0].GetString() == networkProbeOp) { - // probeNetwork - assert(ProxyFileSource::isTrackingActive()); - assert(operationArray.Size() >= 2u); - assert(operationArray[1].IsString()); - std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength()); - - metadata.metrics.network.emplace( - std::piecewise_construct, - std::forward_as_tuple(std::move(mark)), - std::forward_as_tuple(ProxyFileSource::getRequestCount(), ProxyFileSource::getTransferredSize())); - } else if (operationArray[0].GetString() == networkProbeEndOp) { - // probeNetworkEnd - assert(ProxyFileSource::isTrackingActive()); - ProxyFileSource::setTrackingActive(false); - } else if (operationArray[0].GetString() == setFeatureStateOp) { - // setFeatureState - assert(operationArray.Size() >= 3u); - assert(operationArray[1].IsObject()); - assert(operationArray[2].IsObject()); - - using namespace mbgl; - using namespace mbgl::style::conversion; - - std::string sourceID; - mbgl::optional sourceLayer; - std::string featureID; - std::string stateKey; - Value stateValue; - bool valueParsed = false; - FeatureState parsedState; - - const auto& featureOptions = operationArray[1].GetObject(); - if (featureOptions.HasMember("source")) { - sourceID = featureOptions["source"].GetString(); - } - if (featureOptions.HasMember("sourceLayer")) { - sourceLayer = {featureOptions["sourceLayer"].GetString()}; - } - if (featureOptions.HasMember("id")) { - if (featureOptions["id"].IsString()) { - featureID = featureOptions["id"].GetString(); - } else if (featureOptions["id"].IsNumber()) { - featureID = mbgl::util::toString(featureOptions["id"].GetUint64()); - } - } - const JSValue* state = &operationArray[2]; - - const std::function(const std::string&, const Convertible&)> convertFn = - [&](const std::string& k, const Convertible& v) -> optional { - optional value = toValue(v); - if (value) { - stateValue = std::move(*value); - valueParsed = true; - } else if (isArray(v)) { - std::vector array; - std::size_t length = arrayLength(v); - array.reserve(length); - for (size_t i = 0; i < length; ++i) { - optional arrayVal = toValue(arrayMember(v, i)); - if (arrayVal) { - array.emplace_back(*arrayVal); - } - } - std::unordered_map result; - result[k] = std::move(array); - stateValue = std::move(result); - valueParsed = true; - return nullopt; - - } else if (isObject(v)) { - eachMember(v, convertFn); - } - - if (!valueParsed) { - metadata.errorMessage = std::string("Could not get feature state value, state key: ") + k; - return nullopt; - } - stateKey = k; - parsedState[stateKey] = stateValue; - return nullopt; - }; - - eachMember(state, convertFn); - - try { - frontend.render(map); - } catch (const std::exception&) { - return false; - } - frontend.getRenderer()->setFeatureState(sourceID, sourceLayer, featureID, parsedState); - } else if (operationArray[0].GetString() == getFeatureStateOp) { - // getFeatureState - assert(operationArray.Size() >= 2u); - assert(operationArray[1].IsObject()); - - std::string sourceID; - mbgl::optional sourceLayer; - std::string featureID; - - const auto& featureOptions = operationArray[1].GetObject(); - if (featureOptions.HasMember("source")) { - sourceID = featureOptions["source"].GetString(); - } - if (featureOptions.HasMember("sourceLayer")) { - sourceLayer = {featureOptions["sourceLayer"].GetString()}; - } - if (featureOptions.HasMember("id")) { - if (featureOptions["id"].IsString()) { - featureID = featureOptions["id"].GetString(); - } else if (featureOptions["id"].IsNumber()) { - featureID = mbgl::util::toString(featureOptions["id"].GetUint64()); - } - } - - try { - frontend.render(map); - } catch (const std::exception&) { - return false; - } - mbgl::FeatureState state; - frontend.getRenderer()->getFeatureState(state, sourceID, sourceLayer, featureID); - } else if (operationArray[0].GetString() == removeFeatureStateOp) { - // removeFeatureState - assert(operationArray.Size() >= 2u); - assert(operationArray[1].IsObject()); - - std::string sourceID; - mbgl::optional sourceLayer; - std::string featureID; - mbgl::optional stateKey; - - const auto& featureOptions = operationArray[1].GetObject(); - if (featureOptions.HasMember("source")) { - sourceID = featureOptions["source"].GetString(); - } - if (featureOptions.HasMember("sourceLayer")) { - sourceLayer = {featureOptions["sourceLayer"].GetString()}; - } - if (featureOptions.HasMember("id")) { - if (featureOptions["id"].IsString()) { - featureID = featureOptions["id"].GetString(); - } else if (featureOptions["id"].IsNumber()) { - featureID = mbgl::util::toString(featureOptions["id"].GetUint64()); - } - } - - if (operationArray.Size() >= 3u) { - assert(operationArray[2].IsString()); - stateKey = {operationArray[2].GetString()}; - } - - try { - frontend.render(map); - } catch (const std::exception&) { - return false; - } - frontend.getRenderer()->removeFeatureState(sourceID, sourceLayer, featureID, stateKey); - } else if (operationArray[0].GetString() == panGestureOp) { - // benchmarkPanGesture - assert(operationArray.Size() >= 4u); - assert(operationArray[1].IsString()); // identifier - assert(operationArray[2].IsNumber()); // duration - assert(operationArray[3].IsArray()); // start [lat, lng, zoom] - assert(operationArray[4].IsArray()); // end [lat, lng, zoom] - - if (metadata.mapMode != mbgl::MapMode::Continuous) { - metadata.errorMessage = "Map mode must be Continous for " + panGestureOp + " operation"; - return false; - } - - std::string mark = operationArray[1].GetString(); - int duration = operationArray[2].GetFloat(); - LatLng startPos, endPos; - double startZoom, endZoom; - std::vector samples; - - auto parsePosition = [](auto arr) -> std::tuple { - assert(arr.Size() >= 3); - return {{arr[1].GetDouble(), arr[0].GetDouble()}, arr[2].GetDouble()}; - }; - - std::tie(startPos, startZoom) = parsePosition(operationArray[3].GetArray()); - std::tie(endPos, endZoom) = parsePosition(operationArray[4].GetArray()); - - // Jump to the starting point of the segment and make sure there's something to render - map.jumpTo(mbgl::CameraOptions().withCenter(startPos).withZoom(startZoom)); - - observer.reset(); - while (!observer.finishRenderingMap) { - frontend.renderOnce(map); - } - - if (observer.mapLoadFailure) return false; - - size_t frames = 0; - float totalTime = 0.0; - bool transitionFinished = false; - - mbgl::AnimationOptions animationOptions(mbgl::Milliseconds(duration * 1000)); - animationOptions.minZoom = util::min(startZoom, endZoom); - animationOptions.transitionFinishFn = [&]() { transitionFinished = true; }; - - map.flyTo(mbgl::CameraOptions().withCenter(endPos).withZoom(endZoom), animationOptions); - - while (!transitionFinished) { - frames++; - frontend.renderOnce(map); - float frameTime = (float)frontend.getFrameTime(); - totalTime += frameTime; - - samples.push_back(frameTime); - } - - float averageFps = totalTime > 0.0 ? frames / totalTime : 0.0; - float minFrameTime = 0.0; - - // Use 1% of the longest frames to compute the minimum fps - std::sort(samples.begin(), samples.end()); - - int sampleCount = util::max(1, (int)samples.size() / 100); - for (auto it = samples.rbegin(); it != samples.rbegin() + sampleCount; it++) minFrameTime += *it; - - float minOnePcFps = sampleCount / minFrameTime; - - metadata.metrics.fps.insert({std::move(mark), {averageFps, minOnePcFps, 0.0f}}); - - } else if (operationArray[0].GetString() == gfxProbeStartOp) { - // probeGFXStart - assert(!ctx.gfxProbeActive); - ctx.gfxProbeActive = true; - ctx.baselineGfxProbe = ctx.activeGfxProbe; - } else if (operationArray[0].GetString() == gfxProbeEndOp) { - // probeGFXEnd - assert(ctx.gfxProbeActive); - ctx.gfxProbeActive = false; - } else if (operationArray[0].GetString() == gfxProbeOp) { - // probeGFX - assert(operationArray.Size() >= 2u); - assert(operationArray[1].IsString()); - - std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength()); - - // Render the map and fetch rendering stats - gfx::RenderingStats stats; - - try { - stats = frontend.render(map).stats; - } catch (const std::exception&) { - return false; - } - - ctx.activeGfxProbe = GfxProbe(stats, ctx.activeGfxProbe); - - // Compare memory allocations to the baseline probe - GfxProbe metricProbe = ctx.activeGfxProbe; - metricProbe.memIndexBuffers.peak -= ctx.baselineGfxProbe.memIndexBuffers.peak; - metricProbe.memVertexBuffers.peak -= ctx.baselineGfxProbe.memVertexBuffers.peak; - metricProbe.memTextures.peak -= ctx.baselineGfxProbe.memTextures.peak; - metadata.metrics.gfx.insert({mark, metricProbe}); - - } else { - metadata.errorMessage = std::string("Unsupported operation: ") + operationArray[0].GetString(); - return false; - } - - operationsArray.Erase(operationIt); - return runOperations(ctx); -} - TestRunner::Impl::Impl(const TestMetadata& metadata) : observer(std::make_unique()), frontend(metadata.size, metadata.pixelRatio, swapBehavior(metadata.mapMode)), @@ -1197,9 +561,9 @@ bool TestRunner::run(TestMetadata& metadata) { return false; } - std::string key = mbgl::util::toString(uint32_t(metadata.mapMode)) - + "/" + mbgl::util::toString(metadata.pixelRatio) - + "/" + mbgl::util::toString(uint32_t(metadata.crossSourceCollisions)); + std::string key = mbgl::util::toString(uint32_t(metadata.mapMode)) + "/" + + mbgl::util::toString(metadata.pixelRatio) + "/" + + mbgl::util::toString(uint32_t(metadata.crossSourceCollisions)); if (maps.find(key) == maps.end()) { maps[key] = std::make_unique(metadata); @@ -1212,13 +576,19 @@ bool TestRunner::run(TestMetadata& metadata) { frontend.setSize(metadata.size); map.setSize(metadata.size); - map.setProjectionMode(mbgl::ProjectionMode().withAxonometric(metadata.axonometric).withXSkew(metadata.xSkew).withYSkew(metadata.ySkew)); + map.setProjectionMode(mbgl::ProjectionMode() + .withAxonometric(metadata.axonometric) + .withXSkew(metadata.xSkew) + .withYSkew(metadata.ySkew)); map.setDebug(metadata.debug); map.getStyle().loadJSON(serializeJsonValue(metadata.document)); map.jumpTo(map.getStyle().getDefaultCamera()); - if (!runOperations(ctx)) return false; + TestOperations operations = parseTestOperations(metadata, manifest); + for (const auto& operation : operations) { + if (!operation(ctx)) return false; + } HeadlessFrontend::RenderResult result; try { @@ -1245,6 +615,7 @@ bool TestRunner::run(TestMetadata& metadata) { } } +namespace { using InjectedProbeMap = std::map; bool runInjectedProbe(const std::set& probes, TestContext& ctx, const InjectedProbeMap& probeMap) { @@ -1258,6 +629,7 @@ bool runInjectedProbe(const std::set& probes, TestContext& ctx, con } return true; } +} // namespace bool TestRunner::runInjectedProbesBegin(TestContext& ctx) { static const std::string mark = " - default - start"; diff --git a/render-test/runner.hpp b/render-test/runner.hpp index 7076cdfd78..8a1d00bc97 100644 --- a/render-test/runner.hpp +++ b/render-test/runner.hpp @@ -11,22 +11,27 @@ class TestRunnerMapObserver; struct TestMetadata; -class TestContext { +class TestRunnerMapObserver : public mbgl::MapObserver { public: - virtual mbgl::HeadlessFrontend& getFrontend() = 0; - virtual mbgl::Map& getMap() = 0; - virtual TestRunnerMapObserver& getObserver() = 0; - virtual TestMetadata& getMetadata() = 0; + TestRunnerMapObserver() = default; + void onDidFailLoadingMap(mbgl::MapLoadError, const std::string&) override { mapLoadFailure = true; } - GfxProbe activeGfxProbe; - GfxProbe baselineGfxProbe; - bool gfxProbeActive = false; + void onDidFinishRenderingMap(RenderMode mode) override final { + if (!finishRenderingMap) finishRenderingMap = mode == RenderMode::Full; + } -protected: - virtual ~TestContext() = default; -}; + void onDidBecomeIdle() override final { idle = true; } + + void reset() { + mapLoadFailure = false; + finishRenderingMap = false; + idle = false; + } -using TestOperation = std::function; + bool mapLoadFailure; + bool finishRenderingMap; + bool idle; +}; class TestRunner { public: @@ -40,7 +45,6 @@ public: void doShuffle(uint32_t seed); private: - bool runOperations(TestContext&); bool runInjectedProbesBegin(TestContext&); bool runInjectedProbesEnd(TestContext&, mbgl::gfx::RenderingStats); -- cgit v1.2.1