From 6dbda4158776172a2b29b5ff7c9f6d1843677a8c Mon Sep 17 00:00:00 2001 From: Viktor Govako Date: Fri, 28 Jul 2023 19:05:30 -0300 Subject: [PATCH] [generator] Updated tests with search::CitiesBoundariesTable. Signed-off-by: Viktor Govako --- generator/final_processor_world.hpp | 4 +- .../generator_tests/raw_generator_test.cpp | 70 +++++++++++++++++-- .../test_generator.cpp | 36 +++++++++- .../test_generator.hpp | 4 ++ generator/raw_generator.cpp | 17 +++-- generator/raw_generator.hpp | 4 +- generator/water_boundary_checker.hpp | 12 ++-- generator/world_map_generator.hpp | 18 +++-- 8 files changed, 133 insertions(+), 32 deletions(-) diff --git a/generator/final_processor_world.hpp b/generator/final_processor_world.hpp index 8437a7ca38..f393067ba5 100644 --- a/generator/final_processor_world.hpp +++ b/generator/final_processor_world.hpp @@ -13,8 +13,8 @@ class WorldFinalProcessor : public FinalProcessorIntermediateMwmInterface public: using WorldGenerator = WorldMapGenerator; - explicit WorldFinalProcessor(std::string const & temporaryMwmPath, - std::string const & coastlineGeomFilename); + /// @param[in] coastGeomFilename Can be empty if you don't care about cutting borders by water. + WorldFinalProcessor(std::string const & temporaryMwmPath, std::string const & coastGeomFilename); void SetPopularPlaces(std::string const & filename); void SetCitiesAreas(std::string const & filename); diff --git a/generator/generator_tests/raw_generator_test.cpp b/generator/generator_tests/raw_generator_test.cpp index e4937163a0..ea413d849a 100644 --- a/generator/generator_tests/raw_generator_test.cpp +++ b/generator/generator_tests/raw_generator_test.cpp @@ -2,28 +2,42 @@ #include "generator/generator_tests_support/test_generator.hpp" +#include "search/cities_boundaries_table.hpp" + #include "indexer/classificator.hpp" +#include "indexer/data_source.hpp" namespace raw_generator_tests { using TestRawGenerator = generator::tests_support::TestRawGenerator; +uint32_t GetFeatureType(FeatureType & ft) +{ + uint32_t res = 0; + ft.ForEachType([&res](uint32_t t) + { + TEST_EQUAL(res, 0, ()); + res = t; + }); + return res; +} + // https://github.com/organicmaps/organicmaps/issues/2035 UNIT_CLASS_TEST(TestRawGenerator, Towns) { + uint32_t const townType = classif().GetTypeByPath({"place", "town"}); + uint32_t const villageType = classif().GetTypeByPath({"place", "village"}); + std::string const mwmName = "Towns"; BuildFB("./data/osm_test_data/towns.osm", mwmName); size_t count = 0; - ForEachFB(mwmName, [&count](feature::FeatureBuilder const & fb) + ForEachFB(mwmName, [&](feature::FeatureBuilder const & fb) { ++count; //LOG(LINFO, (fb)); - static uint32_t const townType = classif().GetTypeByPath({"place", "town"}); - static uint32_t const villageType = classif().GetTypeByPath({"place", "village"}); - bool const isTown = (fb.GetName() == "El Dorado"); TEST_EQUAL(isTown, fb.HasType(townType), ()); TEST_NOT_EQUAL(isTown, fb.HasType(villageType), ()); @@ -32,6 +46,54 @@ UNIT_CLASS_TEST(TestRawGenerator, Towns) }); TEST_EQUAL(count, 4, ()); + + // Prepare features data source. + FrozenDataSource dataSource; + std::vector mwmIDs; + for (auto const & name : { mwmName, std::string(WORLD_FILE_NAME) }) + { + BuildFeatures(name); + BuildSearch(name); + + platform::LocalCountryFile localFile(platform::LocalCountryFile::MakeTemporary(GetMwmPath(name))); + auto res = dataSource.RegisterMap(localFile); + TEST_EQUAL(res.second, MwmSet::RegResult::Success, ()); + mwmIDs.push_back(std::move(res.first)); + } + + /// @todo We should have only 1 boundary here for the "El Dorado" town, because all other palces are villages. + /// Now we have 2 boundaries + "Taylor" village, because it was transformed from place=city boundary above. + /// This is not a blocker, but good to fix World generator for this case in future. + + // Load boundaries. + search::CitiesBoundariesTable table(dataSource); + TEST(table.Load(), ()); + TEST_EQUAL(table.GetSize(), 2, ()); + + // Iterate for features in World. + count = 0; + FeaturesLoaderGuard guard(dataSource, mwmIDs[1]); + for (size_t id = 0; id < guard.GetNumFeatures(); ++id) + { + auto ft = guard.GetFeatureByIndex(id); + + std::string_view const name = ft->GetName(StringUtf8Multilang::kDefaultCode); + if (!name.empty()) + { + TEST_EQUAL(ft->GetGeomType(), feature::GeomType::Point, ()); + + search::CitiesBoundariesTable::Boundaries boundary; + TEST(table.Get(id, boundary), ()); + TEST(boundary.HasPoint(ft->GetCenter()), ()); + + if (name == "El Dorado") + TEST_EQUAL(GetFeatureType(*ft), townType, ()); + + ++count; + } + } + + TEST_EQUAL(count, 2, ()); } } // namespace raw_generator_tests diff --git a/generator/generator_tests_support/test_generator.cpp b/generator/generator_tests_support/test_generator.cpp index 91e9025eff..c0ee9253a9 100644 --- a/generator/generator_tests_support/test_generator.cpp +++ b/generator/generator_tests_support/test_generator.cpp @@ -4,6 +4,7 @@ #include "generator/feature_sorter.hpp" #include "generator/osm_source.hpp" #include "generator/raw_generator.hpp" +#include "generator/search_index_builder.hpp" #include "indexer/classificator_loader.hpp" #include "indexer/features_offsets_table.hpp" @@ -65,25 +66,54 @@ void TestRawGenerator::BuildFB(std::string const & osmFilePath, std::string cons m_genInfo.m_tmpDir = m_genInfo.m_targetDir = GetTmpPath(); m_genInfo.m_fileName = mwmName; + m_genInfo.m_citiesBoundariesFilename = GetCitiesBoundariesPath(); + RawGenerator rawGenerator(m_genInfo); rawGenerator.ForceReloadCache(); + + rawGenerator.GenerateWorld(false /* cutBordersByWater */); rawGenerator.GenerateCountries(true /* isTests */); + CHECK(rawGenerator.Execute(), ("Error generating", mwmName)); } void TestRawGenerator::BuildFeatures(std::string const & mwmName) { - CHECK(feature::GenerateFinalFeatures(m_genInfo, mwmName, feature::DataHeader::MapType::Country), ()); + using namespace feature; + auto const type = IsWorld(mwmName) ? DataHeader::MapType::World : DataHeader::MapType::Country; + CHECK(GenerateFinalFeatures(m_genInfo, mwmName, type), ()); std::string const mwmPath = GetMwmPath(mwmName); - CHECK(feature::BuildOffsetsTable(mwmPath), ()); + CHECK(BuildOffsetsTable(mwmPath), ()); CHECK(indexer::BuildIndexFromDataFile(mwmPath, mwmPath), ()); } +void TestRawGenerator::BuildSearch(std::string const & mwmName) +{ + CHECK(indexer::BuildSearchIndexFromDataFile(mwmName, m_genInfo, true /* forceRebuild */, 1 /* threadsCount */), ()); + + if (IsWorld(mwmName)) + { + generator::OsmIdToBoundariesTable table; + CHECK(generator::DeserializeBoundariesTable(GetCitiesBoundariesPath(), table), ()); + CHECK(generator::BuildCitiesBoundaries(GetMwmPath(mwmName), table), ()); + } +} + std::string TestRawGenerator::GetMwmPath(std::string const & mwmName) const { - return base::JoinPath(m_genInfo.m_targetDir, mwmName + DATA_FILE_EXTENSION); + return m_genInfo.GetTargetFileName(mwmName, DATA_FILE_EXTENSION); +} + +std::string TestRawGenerator::GetCitiesBoundariesPath() const +{ + return m_genInfo.GetTmpFileName(CITIES_BOUNDARIES_FILE_TAG, ".bin"); +} + +bool TestRawGenerator::IsWorld(std::string const & mwmName) const +{ + return (mwmName == WORLD_FILE_NAME); } } // namespace tests_support diff --git a/generator/generator_tests_support/test_generator.hpp b/generator/generator_tests_support/test_generator.hpp index 5e0366c1fb..573a726f6c 100644 --- a/generator/generator_tests_support/test_generator.hpp +++ b/generator/generator_tests_support/test_generator.hpp @@ -23,6 +23,7 @@ public: void BuildFB(std::string const & osmFilePath, std::string const & mwmName); void BuildFeatures(std::string const & mwmName); + void BuildSearch(std::string const & mwmName); template void ForEachFB(std::string const & mwmName, FnT && fn) { @@ -34,7 +35,10 @@ public: } std::string GetMwmPath(std::string const & mwmName) const; + std::string GetCitiesBoundariesPath() const; + feature::GenerateInfo const & GetGenInfo() const { return m_genInfo; } + bool IsWorld(std::string const & mwmName) const; }; } // namespace tests_support diff --git a/generator/raw_generator.cpp b/generator/raw_generator.cpp index 31cd1d4cc8..35cc4387a2 100644 --- a/generator/raw_generator.cpp +++ b/generator/raw_generator.cpp @@ -120,11 +120,11 @@ void RawGenerator::GenerateCountries(bool isTests/* = false*/) m_finalProcessors.emplace(CreateCountryFinalProcessor(affiliation, false)); } -void RawGenerator::GenerateWorld() +void RawGenerator::GenerateWorld(bool cutBordersByWater/* = true */) { auto processor = CreateProcessor(ProcessorType::World, m_queue, m_genInfo.m_popularPlacesFilename); m_translators->Append(CreateTranslator(TranslatorType::World, processor, m_cache, m_genInfo)); - m_finalProcessors.emplace(CreateWorldFinalProcessor()); + m_finalProcessors.emplace(CreateWorldFinalProcessor(cutBordersByWater)); } void RawGenerator::GenerateCoasts() @@ -202,11 +202,16 @@ RawGenerator::FinalProcessorPtr RawGenerator::CreateCountryFinalProcessor( return finalProcessor; } -RawGenerator::FinalProcessorPtr RawGenerator::CreateWorldFinalProcessor() +RawGenerator::FinalProcessorPtr RawGenerator::CreateWorldFinalProcessor(bool cutBordersByWater) { - auto finalProcessor = std::make_shared( - m_genInfo.m_tmpDir, - m_genInfo.GetIntermediateFileName(WORLD_COASTS_FILE_NAME, RAW_GEOM_FILE_EXTENSION)); + std::string coastlineGeom; + if (cutBordersByWater) + { + // This file should exist or read exception will be thrown otherwise. + coastlineGeom = m_genInfo.GetIntermediateFileName(WORLD_COASTS_FILE_NAME, RAW_GEOM_FILE_EXTENSION); + } + auto finalProcessor = std::make_shared(m_genInfo.m_tmpDir, coastlineGeom); + finalProcessor->SetPopularPlaces(m_genInfo.m_popularPlacesFilename); finalProcessor->SetCitiesAreas(m_genInfo.GetIntermediateFileName(CITIES_AREAS_TMP_FILENAME)); return finalProcessor; diff --git a/generator/raw_generator.hpp b/generator/raw_generator.hpp index f06049f319..0fbb060c6f 100644 --- a/generator/raw_generator.hpp +++ b/generator/raw_generator.hpp @@ -23,7 +23,7 @@ public: size_t chunkSize = 1024); void GenerateCountries(bool isTests = false); - void GenerateWorld(); + void GenerateWorld(bool cutBordersByWater = true); void GenerateCoasts(); void GenerateCustom(std::shared_ptr const & translator); void GenerateCustom( @@ -44,7 +44,7 @@ private: FinalProcessorPtr CreateCoslineFinalProcessor(); FinalProcessorPtr CreateCountryFinalProcessor(AffiliationInterfacePtr const & affiliations, bool needMixNodes); - FinalProcessorPtr CreateWorldFinalProcessor(); + FinalProcessorPtr CreateWorldFinalProcessor(bool cutBordersByWater); bool GenerateFilteredFeatures(); feature::GenerateInfo & m_genInfo; diff --git a/generator/water_boundary_checker.hpp b/generator/water_boundary_checker.hpp index fce3a125fa..be3728cd03 100644 --- a/generator/water_boundary_checker.hpp +++ b/generator/water_boundary_checker.hpp @@ -1,4 +1,11 @@ #pragma once +#include "generator/feature_builder.hpp" + +#include "indexer/classificator.hpp" + +#include "geometry/region2d.hpp" +#include "geometry/tree4d.hpp" + class WaterBoundaryChecker { @@ -14,11 +21,6 @@ class WaterBoundaryChecker size_t m_selectedPolygons = 0; public: - WaterBoundaryChecker(std::string const & rawGeometryFileName) - { - LoadWaterGeometry(rawGeometryFileName); - } - ~WaterBoundaryChecker() { LOG_SHORT(LINFO, ("Features checked:", m_totalFeatures, "borders checked:", m_totalBorders, diff --git a/generator/world_map_generator.hpp b/generator/world_map_generator.hpp index c76338a057..38085261ae 100644 --- a/generator/world_map_generator.hpp +++ b/generator/world_map_generator.hpp @@ -14,8 +14,6 @@ #include "coding/point_coding.hpp" #include "geometry/polygon.hpp" -#include "geometry/region2d.hpp" -#include "geometry/tree4d.hpp" #include "base/logging.hpp" @@ -26,8 +24,6 @@ #include #include -#include "defines.hpp" - #include "water_boundary_checker.hpp" /// Process FeatureBuilder for world map. Main functions: @@ -94,11 +90,12 @@ class WorldMapGenerator std::string m_popularPlacesFilename; public: - explicit WorldMapGenerator(std::string const & worldFilename, std::string const & rawGeometryFileName, - std::string const & popularPlacesFilename) + /// @param[in] coastGeomFilename Can be empty if no need to cut borders by water. + /// @param[in] popularPlacesFilename Can be empty if no need in popular places. + WorldMapGenerator(std::string const & worldFilename, std::string const & coastGeomFilename, + std::string const & popularPlacesFilename) : m_worldBucket(worldFilename) , m_merger(kPointCoordBits - (scales::GetUpperScale() - scales::GetUpperWorldScale()) / 2) - , m_boundaryChecker(rawGeometryFileName) , m_popularPlacesFilename(popularPlacesFilename) { // Do not strip last types for given tags, @@ -110,11 +107,12 @@ public: for (size_t i = 0; i < ARRAY_SIZE(arr1); ++i) m_typesCorrector.SetDontNormalizeType(arr1[i]); - char const * arr2[] = {"boundary", "administrative", "4", "state"}; - m_typesCorrector.SetDontNormalizeType(arr2); - if (popularPlacesFilename.empty()) LOG(LWARNING, ("popular_places_data option not set. Popular atractions will not be added to World.mwm")); + + // Can be empty in tests. + if (!coastGeomFilename.empty()) + m_boundaryChecker.LoadWaterGeometry(coastGeomFilename); } void Process(feature::FeatureBuilder & fb)