diff --git a/generator/generator_integration_tests/features_tests.cpp b/generator/generator_integration_tests/features_tests.cpp index baac265d82..f452822271 100644 --- a/generator/generator_integration_tests/features_tests.cpp +++ b/generator/generator_integration_tests/features_tests.cpp @@ -1,4 +1,3 @@ -// ~/Projects/omim_copy/data/ #include "testing/testing.hpp" #include "generator/generator_integration_tests/helpers.hpp" @@ -28,6 +27,8 @@ class FeatureIntegrationTests public: FeatureIntegrationTests() { + // You can get features-2019_07_17__13_39_20 by running: + // rsync -v -p testdata.mapsme.cloud.devmail.ru::testdata/features-2019_07_17__13_39_20.zip . Init("features-2019_07_17__13_39_20" /* arhiveName */); } @@ -101,11 +102,11 @@ public: ++cityTownOrVillageCnt; } - TEST_EQUAL(fbs.size(), 949, ()); - TEST_EQUAL(pointsCnt, 364410, ()); + TEST_EQUAL(fbs.size(), 938, ()); + TEST_EQUAL(pointsCnt, 364399, ()); TEST_EQUAL(airportCnt, 3, ()); TEST_EQUAL(cityTownOrVillageCnt, 172, ()); - TEST_EQUAL(popularAttractionCnt, 146, ()); + TEST_EQUAL(popularAttractionCnt, 135, ()); } void BuildCountries() @@ -126,18 +127,18 @@ public: rawGenerator.GenerateCountries(); TEST(rawGenerator.Execute(), ()); - TestCountry(northAuckland, 1811915 /* fbsCnt */, 12195892 /* pointsCnt */, 1007316 /* pointCnt */, + TestCountry(northAuckland, 1811971 /* fbsCnt */, 12195948 /* pointsCnt */, 1007372 /* pointCnt */, 205469 /* lineCnt */, 599130 /* areaCnt */, 212087 /* poiCnt */, - 521 /* cityTownOrVillageCnt */, 47 /* popularAttractionCnt */, 3557 /* bookingHotelsCnt */); - TestCountry(northWellington, 797769 /* fbsCnt */, 7772261 /* pointsCnt */, 460437 /* pointCnt */, + 521 /* cityTownOrVillageCnt */, 43 /* popularAttractionCnt */, 3557 /* bookingHotelsCnt */); + TestCountry(northWellington, 797778 /* fbsCnt */, 7772270 /* pointsCnt */, 460446 /* pointCnt */, 87058 /* lineCnt */, 250274 /* areaCnt */, 95651 /* poiCnt */, - 297 /* cityTownOrVillageCnt */, 17 /* popularAttractionCnt */, 1062 /* bookingHotelsCnt */); - TestCountry(southCanterbury, 636922 /* fbsCnt */, 6984348 /* pointsCnt */, 397622 /* pointCnt */, + 297 /* cityTownOrVillageCnt */, 16 /* popularAttractionCnt */, 1062 /* bookingHotelsCnt */); + TestCountry(southCanterbury, 636934 /* fbsCnt */, 6984360 /* pointsCnt */, 397634 /* pointCnt */, 81712 /* lineCnt */, 157588 /* areaCnt */, 89249 /* poiCnt */, - 331 /* cityTownOrVillageCnt */, 42 /* popularAttractionCnt */, 2085 /* bookingHotelsCnt */); + 331 /* cityTownOrVillageCnt */, 40 /* popularAttractionCnt */, 2085 /* bookingHotelsCnt */); TestCountry(southSouthland, 340491 /* fbsCnt */, 5342804 /* pointsCnt */, 185845 /* pointCnt */, 40124 /* lineCnt */, 114522 /* areaCnt */, 40497 /* poiCnt */, - 297 /* cityTownOrVillageCnt */, 41 /* popularAttractionCnt */, 1621 /* bookingHotelsCnt */); + 297 /* cityTownOrVillageCnt */, 37 /* popularAttractionCnt */, 1621 /* bookingHotelsCnt */); } void CheckGeneratedData() diff --git a/generator/generator_tests/restriction_collector_test.cpp b/generator/generator_tests/restriction_collector_test.cpp index 0b8ac5f10d..86f7fe3619 100644 --- a/generator/generator_tests/restriction_collector_test.cpp +++ b/generator/generator_tests/restriction_collector_test.cpp @@ -237,9 +237,7 @@ UNIT_TEST(RestrictionWriter_Case1) stream.exceptions(std::fstream::failbit | std::fstream::badbit); stream.open(filename); std::stringstream buffer; - auto * buf = stream.rdbuf(); - if (buf->in_avail()) - buffer << stream.rdbuf(); + buffer << stream.rdbuf(); std::string const correctAnswer = "No,way,11,1,21\n" "No,way,13,3,23\n" diff --git a/generator/generator_tests/road_access_test.cpp b/generator/generator_tests/road_access_test.cpp index 179bbe0bf2..1bf52be5c9 100644 --- a/generator/generator_tests/road_access_test.cpp +++ b/generator/generator_tests/road_access_test.cpp @@ -209,9 +209,7 @@ UNIT_TEST(RoadAccessWriter_Case1) stream.exceptions(fstream::failbit | fstream::badbit); stream.open(filename); stringstream buffer; - auto * buf = stream.rdbuf(); - if (buf->in_avail()) - buffer << stream.rdbuf(); + buffer << stream.rdbuf(); string const correctAnswer = "Car Private 1 2\n" "Car Private 2 3\n"; diff --git a/generator/generator_tool/generator_tool.cpp b/generator/generator_tool/generator_tool.cpp index 378c04b23c..f820881a1c 100644 --- a/generator/generator_tool/generator_tool.cpp +++ b/generator/generator_tool/generator_tool.cpp @@ -306,7 +306,7 @@ int GeneratorToolMain(int argc, char ** argv) return EXIT_FAILURE; } - DataVersion{FLAGS_osm_file_name}.DumpToPath(genInfo.m_intermediateDir); + DataVersion{FLAGS_osm_file_name}.DumpToPath(genInfo.m_intermediateDir); // Generate .mwm.tmp files. if (FLAGS_generate_features || @@ -361,12 +361,8 @@ int GeneratorToolMain(int argc, char ** argv) if (!FLAGS_geo_objects_key_value.empty()) { if (!geo_objects::GenerateGeoObjects(FLAGS_regions_index, FLAGS_regions_key_value, - FLAGS_regions_key_value, - FLAGS_geo_objects_features, - FLAGS_ids_without_addresses, - FLAGS_geo_objects_key_value, - FLAGS_verbose, - threadsCount}; + FLAGS_geo_objects_features, FLAGS_ids_without_addresses, + FLAGS_geo_objects_key_value, FLAGS_verbose, threadsCount)) return EXIT_FAILURE; } @@ -391,8 +387,8 @@ int GeneratorToolMain(int argc, char ** argv) LOG(LINFO, ("Saving geo objects index to", outFile)); if (!indexer::BuildGeoObjectsIndexFromDataFile( - locDataFile, outFile, DataVersion::LoadFromPath(path).GetVersionJson(), - DataVersion::kFileTag)) + locDataFile, outFile, DataVersion::LoadFromPath(path).GetVersionJson(), + DataVersion::kFileTag)) { LOG(LCRITICAL, ("Error generating geo objects index.")); return EXIT_FAILURE; diff --git a/generator/osm_source.cpp b/generator/osm_source.cpp index 6936543c17..8a3856e6dd 100644 --- a/generator/osm_source.cpp +++ b/generator/osm_source.cpp @@ -244,50 +244,6 @@ bool ProcessorOsmElementsFromXml::TryRead(OsmElement & element) // Generate functions implementations. /////////////////////////////////////////////////////////////////////////////////////////////////// -bool GenerateRaw(feature::GenerateInfo & info, TranslatorInterface & translators) -{ - auto const fn = [&](OsmElement * element) { - CHECK(element, ()); - translators.Emit(*element); - }; - - SourceReader reader = info.m_osmFileName.empty() ? SourceReader() : SourceReader(info.m_osmFileName); - switch (info.m_osmFileType) - { - case feature::GenerateInfo::OsmSourceType::XML: - ProcessOsmElementsFromXML(reader, fn); - break; - case feature::GenerateInfo::OsmSourceType::O5M: - ProcessOsmElementsFromO5M(reader, fn); - break; - } - - LOG(LINFO, ("Processing", info.m_osmFileName, "done.")); - if (!translators.Finish()) - return false; - - translators.GetNames(info.m_bucketNames); - return true; -} - -LoaderWrapper::LoaderWrapper(feature::GenerateInfo & info) - : m_reader(std::make_unique(info, true /* forceReload */)) {} - -cache::IntermediateDataReader & LoaderWrapper::GetReader() -{ - return *m_reader->GetCache(); -} - -CacheLoader::CacheLoader(feature::GenerateInfo & info) : m_info(info) {} - -cache::IntermediateDataReader & CacheLoader::GetCache() -{ - if (!m_loader) - m_loader = std::make_unique(m_info); - - return m_loader->GetReader(); -} - bool GenerateIntermediateData(feature::GenerateInfo & info) { auto nodes = cache::CreatePointStorageWriter(info.m_nodeStorageType, diff --git a/generator/osm_source.hpp b/generator/osm_source.hpp index eb51ee4eda..51cc128a2b 100644 --- a/generator/osm_source.hpp +++ b/generator/osm_source.hpp @@ -43,77 +43,6 @@ public: uint64_t Read(char * buffer, uint64_t bufferSize); }; -class LoaderWrapper -{ -public: - LoaderWrapper(feature::GenerateInfo & info); - cache::IntermediateDataReader & GetReader(); - -private: - std::unique_ptr m_reader; -}; - -class CacheLoader -{ -public: - CacheLoader(feature::GenerateInfo & info); - cache::IntermediateDataReader & GetCache(); - -private: - feature::GenerateInfo & m_info; - std::unique_ptr m_loader; - - DISALLOW_COPY(CacheLoader); -}; -// This function is needed to generate intermediate data from OSM elements. -// The translators collection contains translators that translate the OSM element into -// some intermediate representation. -// -// To better understand the generation process of this step, -// we are looking at generation using the example of generation for countries. -// -// To start the generation we make the following calls: -// 1. feature::GenerateInfo genInfo; -// ... -// 2. CacheLoader cacheLoader(genInfo); -// 3. TranslatorCollection translators; -// 4. auto emitter = CreateEmitter(EmitterType::Country, genInfo); -// 5. translators.Append(CreateTranslator(TranslatorType::Country, emitter, cacheLoader.GetCache(), -// genInfo)); -// 6. GenerateRaw(genInfo, translators); -// -// In line 5, we create and add a translator for countries to the translator collection. -// TranslatorCountry is inheritor of Translator. -// -// Translator contains several important entities: FeatureMaker, FilterCollection, -// CollectorCollection and Emitter. In short, -// * FeatureMaker - an object that can create FeatureBuilder1 from OSM element, -// * FilterCollection - an object that contains a group of filters that may or may not pass OSM -// elements and FeatureBuilder1s, -// * CollectorCollection - an object that contains a group of collectors that collect additional -// information about OSM elements and FeatureBuilder1s (most often it is information that cannot -// be saved in FeatureBuilder1s from OSM element), -// * Emitter - an object that converts an element and saves it. -// -// The most important method is Translator::Emit. Please read it to understand how generation works. -// The order of calls is very important. First, the FilterCollection will filter the OSM elements, -// then call the CollectorCollection for OSM elements, then build the FeatureBuilder1 element -// form OSM element, then FilterCollection will filter the FeatureBuilder1, then call the -// CollectorCollection for the FeatureBuilder1, and then FeatureBuilder1 will fall into the emitter. -// -// TranslatorCountry contains for it specific filters, collectors, emitter and FeatureMaker. -// For example, there are FilterPlanet, which only passes relations with types multipolygon or -// boundary, and CameraNodeProcessor, which collects information about the cameras on the roads. -// -// In line 4, we create emitter for countries. -// The emitter is an important entity that needs to transform FeatureBuilder1 and save them in some -// way. The emitter can filter objects and change the representation of an object based on drawing -// rules and other application rules. In EmitterCountry stages are divided into layers. The layers -// are connected in a chain. For example, there are RepresentationLayer, which may change the -// presentation of the FeatureBuilder1 depending on the rules of the application, and BookingLayer, -// which mixes information from booking. You can read a more detailed look into the appropriate -// class code. -bool GenerateRaw(feature::GenerateInfo & info, TranslatorInterface & translators); bool GenerateIntermediateData(feature::GenerateInfo & info); void ProcessOsmElementsFromO5M(SourceReader & stream, std::function processor); diff --git a/generator/raw_generator.cpp b/generator/raw_generator.cpp index ba2447cb80..b6b4d3a18f 100644 --- a/generator/raw_generator.cpp +++ b/generator/raw_generator.cpp @@ -170,7 +170,7 @@ bool RawGenerator::GenerateFilteredFeatures() } CHECK(sourceProcessor, ()); - TranslatorsPool translators(m_translators, m_cache, m_threadsCount); + TranslatorsPool translators(m_translators, m_threadsCount); RawGeneratorWriter rawGeneratorWriter(m_queue, m_genInfo.m_tmpDir); rawGeneratorWriter.Run(); diff --git a/generator/translators_pool.cpp b/generator/translators_pool.cpp index a4b6dc6ebd..137e4381db 100644 --- a/generator/translators_pool.cpp +++ b/generator/translators_pool.cpp @@ -5,15 +5,14 @@ namespace generator { TranslatorsPool::TranslatorsPool(std::shared_ptr const & original, - std::shared_ptr const & cache, size_t threadCount) : m_threadPool(threadCount) { - m_translators.Push(original); - for (size_t i = 0; i < threadCount; ++i) - m_translators.Push(original->Clone()); + CHECK_GREATER_OR_EQUAL(threadCount, 1, ()); - CHECK_EQUAL(m_translators.Size(), threadCount + 1, ()); + m_translators.Push(original); + for (size_t i = 1; i < threadCount; ++i) + m_translators.Push(original->Clone()); } void TranslatorsPool::Emit(std::vector && elements) diff --git a/generator/translators_pool.hpp b/generator/translators_pool.hpp index 2e1801bec4..f2e973cd24 100644 --- a/generator/translators_pool.hpp +++ b/generator/translators_pool.hpp @@ -16,7 +16,6 @@ class TranslatorsPool { public: explicit TranslatorsPool(std::shared_ptr const & original, - std::shared_ptr const & cache, size_t threadCount); void Emit(std::vector && elements); diff --git a/generator/world_map_generator.hpp b/generator/world_map_generator.hpp index 0c2f352851..8f0c4cc140 100644 --- a/generator/world_map_generator.hpp +++ b/generator/world_map_generator.hpp @@ -316,7 +316,7 @@ public: { case feature::GeomType::Line: { - MergedFeatureBuilder1 * p = m_typesCorrector(fb); + MergedFeatureBuilder * p = m_typesCorrector(fb); if (p) m_merger(p); return false;