organicmaps/geocore
13
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity Actions

[generator] Refactor threads count parameter passing

Browse source
This commit is contained in:
Anatoly Serdtcev 2019-10-29 16:38:39 +03:00 • committed by LaGrunge
parent 765344777f
commit fdba6d7338
18 changed files with 45 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
base/logging.cpp
View file

@ -50,7 +50,7 @@ array<char const *, NUM_LOG_LEVELS> const & GetLogLevelNames()
class LogHelper
{
int m_threadsCount;
unsigned int m_threadsCount{0};
map<threads::ThreadID, int> m_threadID;
int GetThreadID()
@ -67,7 +67,7 @@ class LogHelper
array<size_t, NUM_LOG_LEVELS> m_lens;
public:
LogHelper() : m_threadsCount(0)
LogHelper()
{
// This code highly depends on the fact that GetLogLevelNames()
// always returns the same constant array of strings.

2
generator/feature_builder.hpp
View file

@ -338,7 +338,7 @@ void ForEachFromDatRawFormat(std::string const & filename, ToDo && toDo)
/// Parallel process features in .dat file.
template <class SerializationPolicy = serialization_policy::MinSize, class ToDo>
void ForEachParallelFromDatRawFormat(size_t threadsCount, std::string const & filename,
void ForEachParallelFromDatRawFormat(unsigned int threadsCount, std::string const & filename,
ToDo && toDo)
{
CHECK_GREATER_OR_EQUAL(threadsCount, 1, ());

2
generator/generate_info.hpp
View file

@ -39,6 +39,8 @@ struct GenerateInfo
OsmSourceType m_osmFileType;
std::string m_osmFileName;
unsigned int m_threadsCount;
bool m_preloadCache = false;
bool m_verbose = false;

14
generator/generator_tool/generator_tool.cpp
View file

@ -209,7 +209,6 @@ int GeneratorToolMain(int argc, char ** argv)
options = DefineOptions(argc, argv);
Platform & pl = GetPlatform();
auto threadsCount = pl.CpuCores();
if (options.m_user_resource_path.empty())
{
@ -229,6 +228,7 @@ int GeneratorToolMain(int argc, char ** argv)
string const path = base::AddSlashIfNeeded(options.m_data_path);
feature::GenerateInfo genInfo;
genInfo.m_threadsCount = pl.CpuCores();
genInfo.m_verbose = options.m_verbose;
genInfo.m_dataPath = path;
genInfo.m_targetDir = path;
@ -270,7 +270,7 @@ int GeneratorToolMain(int argc, char ** argv)
if (options.m_generate_features || options.m_generate_region_features ||
options.m_generate_streets_features || options.m_generate_geo_objects_features)
{
RawGenerator rawGenerator(genInfo, threadsCount);
RawGenerator rawGenerator(genInfo);
if (options.m_generate_region_features)
rawGenerator.GenerateRegionFeatures(options.m_regions_features, regionsInfoPath);
if (options.m_generate_streets_features)
@ -286,7 +286,8 @@ int GeneratorToolMain(int argc, char ** argv)
{
streets::GenerateStreets(options.m_regions_index, options.m_regions_key_value,
options.m_streets_features, options.m_geo_objects_features,
options.m_streets_key_value, options.m_verbose, threadsCount);
options.m_streets_key_value, options.m_verbose,
genInfo.m_threadsCount);
}
if (!options.m_geo_objects_key_value.empty())
@ -294,7 +295,7 @@ int GeneratorToolMain(int argc, char ** argv)
if (!geo_objects::GenerateGeoObjects(
options.m_regions_index, options.m_regions_key_value, options.m_geo_objects_features,
options.m_ids_without_addresses, options.m_geo_objects_key_value, options.m_verbose,
threadsCount))
genInfo.m_threadsCount))
return EXIT_FAILURE;
}
@ -363,7 +364,8 @@ int GeneratorToolMain(int argc, char ** argv)
if (options.m_generate_regions_kv)
{
regions::GenerateRegions(options.m_regions_features, regionsInfoPath,
options.m_regions_key_value, options.m_verbose, threadsCount);
options.m_regions_key_value, options.m_verbose,
genInfo.m_threadsCount);
}
if (options.m_generate_geocoder_token_index)
@ -380,7 +382,7 @@ int GeneratorToolMain(int argc, char ** argv)
}
geocoder::Geocoder geocoder;
geocoder.LoadFromJsonl(options.m_key_value, threadsCount);
geocoder.LoadFromJsonl(options.m_key_value, genInfo.m_threadsCount);
geocoder.SaveToBinaryIndex(options.m_geocoder_token_index);
}

14
generator/geo_objects/geo_objects.cpp
View file

@ -41,7 +41,7 @@ namespace
{
NullBuildingsInfo GetHelpfulNullBuildings(GeoObjectMaintainer & geoObjectMaintainer,
std::string const & pathInGeoObjectsTmpMwm,
size_t threadsCount)
unsigned int threadsCount)
{
NullBuildingsInfo result;
static int64_t counter = 0;
@ -82,7 +82,7 @@ using BuildingsGeometries =
BuildingsGeometries GetBuildingsGeometry(std::string const & pathInGeoObjectsTmpMwm,
NullBuildingsInfo const & buildingsInfo,
size_t threadsCount)
unsigned int threadsCount)
{
BuildingsGeometries result;
std::mutex updateMutex;
@ -114,7 +114,7 @@ BuildingsGeometries GetBuildingsGeometry(std::string const & pathInGeoObjectsTmp
size_t AddBuildingGeometriesToAddressPoints(std::string const & pathInGeoObjectsTmpMwm,
NullBuildingsInfo const & buildingsInfo,
BuildingsGeometries const & geometries,
size_t threadsCount)
unsigned int threadsCount)
{
auto const path = GetPlatform().TmpPathForFile();
FeaturesCollector collector(path);
@ -232,7 +232,7 @@ boost::optional<indexer::GeoObjectsIndex<IndexReader>> MakeTempGeoObjectsIndex(
void AddBuildingsAndThingsWithHousesThenEnrichAllWithRegionAddresses(
GeoObjectMaintainer & geoObjectMaintainer, std::string const & pathInGeoObjectsTmpMwm,
bool /*verbose*/, size_t threadsCount)
bool /*verbose*/, unsigned int threadsCount)
{
auto const concurrentTransformer = [&](FeatureBuilder & fb, uint64_t /* currPos */) {
geoObjectMaintainer.StoreAndEnrich(fb);
@ -244,7 +244,7 @@ void AddBuildingsAndThingsWithHousesThenEnrichAllWithRegionAddresses(
NullBuildingsInfo EnrichPointsWithOuterBuildingGeometry(GeoObjectMaintainer & geoObjectMaintainer,
std::string const & pathInGeoObjectsTmpMwm,
size_t threadsCount)
unsigned int threadsCount)
{
auto const buildingInfo =
GetHelpfulNullBuildings(geoObjectMaintainer, pathInGeoObjectsTmpMwm, threadsCount);
@ -268,7 +268,7 @@ void AddPoisEnrichedWithHouseAddresses(GeoObjectMaintainer & geoObjectMaintainer
NullBuildingsInfo const & buildingsInfo,
std::string const & pathInGeoObjectsTmpMwm,
std::ostream & streamPoiIdsToAddToLocalityIndex,
bool /*verbose*/, size_t threadsCount)
bool /*verbose*/, unsigned int threadsCount)
{
std::atomic_size_t counter{0};
std::mutex streamMutex;
@ -304,7 +304,7 @@ void AddPoisEnrichedWithHouseAddresses(GeoObjectMaintainer & geoObjectMaintainer
void FilterAddresslessThanGaveTheirGeometryToInnerPoints(std::string const & pathInGeoObjectsTmpMwm,
NullBuildingsInfo const & buildingsInfo,
size_t threadsCount)
unsigned int threadsCount)
{
auto const path = GetPlatform().TmpPathForFile();
FeaturesCollector collector(path);

8
generator/geo_objects/geo_objects.hpp
View file

@ -29,7 +29,7 @@ bool JsonHasBuilding(JsonValue const & json);
void AddBuildingsAndThingsWithHousesThenEnrichAllWithRegionAddresses(
GeoObjectMaintainer & geoObjectMaintainer, std::string const & pathInGeoObjectsTmpMwm,
bool verbose, size_t threadsCount);
bool verbose, unsigned int threadsCount);
struct NullBuildingsInfo
{
@ -42,18 +42,18 @@ struct NullBuildingsInfo
NullBuildingsInfo EnrichPointsWithOuterBuildingGeometry(
GeoObjectMaintainer & geoObjectMaintainer, std::string const & pathInGeoObjectsTmpMwm,
size_t threadsCount);
unsigned int threadsCount);
void AddPoisEnrichedWithHouseAddresses(GeoObjectMaintainer & geoObjectMaintainer,
NullBuildingsInfo const & buildingsInfo,
std::string const & pathInGeoObjectsTmpMwm,
std::ostream & streamPoiIdsToAddToLocalityIndex,
bool verbose, size_t threadsCount);
bool verbose, unsigned int threadsCount);
void FilterAddresslessThanGaveTheirGeometryToInnerPoints(std::string const & pathInGeoObjectsTmpMwm,
NullBuildingsInfo const & buildingsInfo,
size_t threadsCount);
unsigned int threadsCount);
} // namespace geo_objects
} // namespace generator

4
generator/geo_objects/geo_objects_generator.cpp
View file

@ -28,7 +28,7 @@ GeoObjectsGenerator::GeoObjectsGenerator(
GeoObjectMaintainer::RegionInfoGetter && regionInfoGetter,
GeoObjectMaintainer::RegionIdGetter && regionIdGetter, std::string pathInGeoObjectsTmpMwm,
std::string pathOutIdsWithoutAddress, std::string pathOutGeoObjectsKv, bool verbose,
size_t threadsCount)
unsigned int threadsCount)
: m_pathInGeoObjectsTmpMwm(std::move(pathInGeoObjectsTmpMwm))
, m_pathOutPoiIdsToAddToLocalityIndex(std::move(pathOutIdsWithoutAddress))
, m_pathOutGeoObjectsKv(std::move(pathOutGeoObjectsKv))
@ -86,7 +86,7 @@ bool GeoObjectsGenerator::GenerateGeoObjectsPrivate()
bool GenerateGeoObjects(std::string const & regionsIndex, std::string const & regionsKeyValue,
std::string const & geoObjectsFeatures,
std::string const & nodesListToIndex, std::string const & geoObjectKeyValue,
bool verbose, size_t threadsCount)
bool verbose, unsigned int threadsCount)
{
auto regionInfoGetter = regions::RegionInfoGetter(regionsIndex, regionsKeyValue);

7
generator/geo_objects/geo_objects_generator.hpp
View file

@ -21,7 +21,8 @@ public:
GeoObjectsGenerator(GeoObjectMaintainer::RegionInfoGetter && regionInfoGetter,
GeoObjectMaintainer::RegionIdGetter && regionIdGetter,
std::string pathInGeoObjectsTmpMwm, std::string pathOutIdsWithoutAddress,
std::string pathOutGeoObjectsKv, bool verbose, size_t threadsCount);
std::string pathOutGeoObjectsKv,
bool verbose, unsigned int threadsCount);
// This function generates key-value pairs for geo objects.
// First, we try to generate key-value pairs only for houses, since we cannot say anything about
@ -42,13 +43,13 @@ private:
std::string m_pathOutGeoObjectsKv;
bool m_verbose = false;
size_t m_threadsCount = 1;
unsigned int m_threadsCount = 1;
GeoObjectMaintainer m_geoObjectMaintainer;
};
bool GenerateGeoObjects(std::string const & regionsIndex, std::string const & regionsKeyValue,
std::string const & geoObjectsFeatures,
std::string const & nodesListToIndex, std::string const & geoObjectKeyValue,
bool verbose, size_t threadsCount);
bool verbose, unsigned int threadsCount);
} // namespace geo_objects
} // namespace generator

7
generator/raw_generator.cpp
View file

@ -16,9 +16,8 @@ using namespace std;
namespace generator
{
RawGenerator::RawGenerator(feature::GenerateInfo & genInfo, size_t threadsCount, size_t chunkSize)
RawGenerator::RawGenerator(feature::GenerateInfo & genInfo, size_t chunkSize)
: m_genInfo(genInfo)
, m_threadsCount(threadsCount)
, m_chunkSize(chunkSize)
, m_cache(std::make_shared<generator::cache::IntermediateData>(genInfo))
, m_queue(std::make_shared<FeatureProcessorQueue>())
@ -75,7 +74,7 @@ bool RawGenerator::Execute()
while (!m_finalProcessors.empty())
{
base::thread_pool::computational::ThreadPool threadPool(m_threadsCount);
base::thread_pool::computational::ThreadPool threadPool(m_genInfo.m_threadsCount);
while (true)
{
auto const finalProcessor = m_finalProcessors.top();
@ -113,7 +112,7 @@ bool RawGenerator::GenerateFilteredFeatures()
}
CHECK(sourceProcessor, ());
TranslatorsPool translators(m_translators, m_threadsCount);
TranslatorsPool translators(m_translators, m_genInfo.m_threadsCount);
RawGeneratorWriter rawGeneratorWriter(m_queue);
rawGeneratorWriter.Run();

4
generator/raw_generator.hpp
View file

@ -17,8 +17,7 @@ namespace generator
class RawGenerator
{
public:
explicit RawGenerator(feature::GenerateInfo & genInfo, size_t threadsCount = 1,
size_t chunkSize = 1024);
explicit RawGenerator(feature::GenerateInfo & genInfo, size_t chunkSize = 1024);
void GenerateRegionFeatures(
std::string const & regionsFeaturesPath, std::string const & regionsInfoPath);
@ -46,7 +45,6 @@ private:
bool GenerateFilteredFeatures();
feature::GenerateInfo & m_genInfo;
size_t m_threadsCount;
size_t m_chunkSize;
std::shared_ptr<cache::IntermediateData> m_cache;
std::shared_ptr<FeatureProcessorQueue> m_queue;

4
generator/regions/regions.cpp
View file

@ -40,7 +40,7 @@ public:
RegionsGenerator(std::string const & pathRegionsTmpMwm,
std::string const & pathInRegionsCollector,
std::string const & pathOutRegionsKv,
bool verbose, size_t threadsCount)
bool verbose, unsigned int threadsCount)
: m_pathRegionsTmpMwm{pathRegionsTmpMwm}
, m_pathOutRegionsKv{pathOutRegionsKv}
, m_verbose{verbose}
@ -325,7 +325,7 @@ private:
void GenerateRegions(std::string const & pathRegionsTmpMwm,
std::string const & pathInRegionsCollector,
std::string const & pathOutRegionsKv,
bool verbose, size_t threadsCount)
bool verbose, unsigned int threadsCount)
{
RegionsGenerator(pathRegionsTmpMwm, pathInRegionsCollector, pathOutRegionsKv,
verbose, threadsCount);

2
generator/regions/regions.hpp
View file

@ -10,6 +10,6 @@ void GenerateRegions(std::string const & pathRegionsTmpMwm,
std::string const & pathInRegionsCollector,
std::string const & pathOutRegionsKv,
bool verbose,
size_t threadsCount = 1);
unsigned int threadsCount = 1);
} // namespace regions
} // namespace generator

2
generator/regions/regions_builder.cpp
View file

@ -22,7 +22,7 @@ namespace generator
namespace regions
{
RegionsBuilder::RegionsBuilder(Regions && regions, PlacePointsMap && placePointsMap,
size_t threadsCount)
unsigned int threadsCount)
: m_threadsCount(threadsCount)
{
ASSERT(m_threadsCount != 0, ());

4
generator/regions/regions_builder.hpp
View file

@ -27,7 +27,7 @@ public:
using CountryFn = std::function<void(std::string const &, Node::PtrList const &)>;
explicit RegionsBuilder(Regions && regions, PlacePointsMap && placePointsMap,
size_t threadsCount = 1);
unsigned int threadsCount = 1);
Regions const & GetCountriesOuters() const;
StringsList GetCountryInternationalNames() const;
@ -71,7 +71,7 @@ private:
Regions m_countriesOuters;
Regions m_regionsInAreaOrder;
PlacePointsMap m_placePointsMap;
size_t m_threadsCount;
unsigned int m_threadsCount;
};
} // namespace regions
} // namespace generator

2
generator/streets/streets.cpp
View file

@ -16,7 +16,7 @@ void GenerateStreets(std::string const & pathInRegionsIndex, std::string const &
std::string const & pathInStreetsTmpMwm,
std::string const & pathInGeoObjectsTmpMwm,
std::string const & pathOutStreetsKv,
bool /*verbose*/, size_t threadsCount)
bool /*verbose*/, unsigned int threadsCount)
{
LOG(LINFO, ("Start generating streets..."));
auto timer = base::Timer();

2
generator/streets/streets.hpp
View file

@ -13,6 +13,6 @@ void GenerateStreets(std::string const & pathInRegionsIndex, std::string const &
std::string const & pathInStreetsTmpMwm,
std::string const & pathInGeoObjectsTmpMwm,
std::string const & pathOutStreetsKv,
bool verbose, size_t threadsCount);
bool verbose, unsigned int threadsCount);
} // namespace streets
} // namespace generator

2
generator/streets/streets_builder.cpp
View file

@ -27,7 +27,7 @@ namespace generator
namespace streets
{
StreetsBuilder::StreetsBuilder(RegionFinder const & regionFinder,
size_t threadsCount)
unsigned int threadsCount)
: m_regionFinder{regionFinder}, m_threadsCount{threadsCount}
{
}

4
generator/streets/streets_builder.hpp
View file

@ -33,7 +33,7 @@ public:
m2::PointD const & point, std::function<bool(KeyValue const & json)> const & selector)>;
using RegionGetter = std::function<std::shared_ptr<JsonValue>(uint64_t key)>;
explicit StreetsBuilder(RegionFinder const & regionFinder, size_t threadsCount = 1);
explicit StreetsBuilder(RegionFinder const & regionFinder, unsigned int threadsCount = 1);
void AssembleStreets(std::string const & pathInStreetsTmpMwm);
void AssembleBindings(std::string const & pathInGeoObjectsTmpMwm);
@ -81,7 +81,7 @@ private:
std::unordered_multimap<base::GeoObjectId, Street const *> m_streetFeatures2Streets;
RegionFinder m_regionFinder;
uint64_t m_osmSurrogateCounter{0};
size_t m_threadsCount;
unsigned int m_threadsCount;
std::mutex m_updateMutex;
};
} // namespace streets