AndrewShkrob/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

Review fixes.

Browse source
This commit is contained in:
Maksim Andrianov 2019-02-18 15:14:53 +03:00 • committed by mpimenov
parent 58021db34c
commit 35330f858b
10 changed files with 55 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
generator/collector_tag.hpp
View file

@ -14,7 +14,7 @@ class GeoObjectId;
namespace generator
{
// CollectorTag class collects validated value of a tag and to save it to file with following
// CollectorTag class collects validated value of a tag and saves it to file with following
// format: osmId<tab>tagValue.
class CollectorTag : public CollectorInterface
{

22
generator/descriptions_section_builder.cpp
View file

@ -30,9 +30,9 @@ std::string GetFileName(std::string path)
namespace generator
{
WikidataHelper::WikidataHelper(std::string const & mwmPath, std::string const & id2wikidataPath)
WikidataHelper::WikidataHelper(std::string const & mwmPath, std::string const & idToWikidataPath)
: m_mwmPath(mwmPath)
, m_id2wikidataPath(id2wikidataPath)
, m_idToWikidataPath(idToWikidataPath)
{
std::string const osmIdsToFeatureIdsPath = m_mwmPath + OSM2FEATURE_FILE_EXTENSION;
if (!routing::ParseFeatureIdToOsmIdMapping(osmIdsToFeatureIdsPath, m_featureIdToOsmId))
@ -40,7 +40,7 @@ WikidataHelper::WikidataHelper(std::string const & mwmPath, std::string const &
std::ifstream stream;
stream.exceptions(std::fstream::failbit | std::fstream::badbit);
stream.open(m_id2wikidataPath);
stream.open(m_idToWikidataPath);
stream.exceptions(std::fstream::badbit);
uint64_t id;
std::string wikidataId;
@ -48,7 +48,7 @@ WikidataHelper::WikidataHelper(std::string const & mwmPath, std::string const &
{
stream >> id >> wikidataId;
strings::Trim(wikidataId);
m_osmIdToFeatureId.emplace(base::GeoObjectId(id), wikidataId);
m_osmIdToWikidataId.emplace(base::GeoObjectId(id), wikidataId);
}
}
@ -58,9 +58,9 @@ boost::optional<std::string> WikidataHelper::GetWikidataId(uint32_t featureId) c
if (itFeatureIdToOsmId == std::end(m_featureIdToOsmId))
return {};
auto const itOsmIdToFeatureId = m_osmIdToFeatureId.find(itFeatureIdToOsmId->second);
return itOsmIdToFeatureId == std::end(m_osmIdToFeatureId) ?
boost::optional<std::string>() : itOsmIdToFeatureId->second;
auto const itOsmIdToWikidataId = m_osmIdToWikidataId.find(itFeatureIdToOsmId->second);
return itOsmIdToWikidataId == std::end(m_osmIdToWikidataId) ?
boost::optional<std::string>() : itOsmIdToWikidataId->second;
}
std::string DescriptionsCollectionBuilderStat::LangStatisticsToString() const
@ -84,8 +84,8 @@ std::string DescriptionsCollectionBuilderStat::LangStatisticsToString() const
DescriptionsCollectionBuilder::DescriptionsCollectionBuilder(std::string const & wikipediaDir,
std::string const & mwmFile,
std::string const & id2wikidataPath)
: m_wikidataHelper(mwmFile, id2wikidataPath), m_wikipediaDir(wikipediaDir), m_mwmFile(mwmFile) {}
std::string const & idToWikidataPath)
: m_wikidataHelper(mwmFile, idToWikidataPath), m_wikipediaDir(wikipediaDir), m_mwmFile(mwmFile) {}
DescriptionsCollectionBuilder::DescriptionsCollectionBuilder(std::string const & wikipediaDir,
std::string const & mwmFile)
@ -173,9 +173,9 @@ size_t DescriptionsCollectionBuilder::GetFeatureDescription(std::string const &
}
void BuildDescriptionsSection(std::string const & wikipediaDir, std::string const & mwmFile,
std::string const & id2wikidataPath)
std::string const & idToWikidataPath)
{
DescriptionsSectionBuilder<FeatureType>::Build(wikipediaDir, mwmFile, id2wikidataPath);
DescriptionsSectionBuilder<FeatureType>::Build(wikipediaDir, mwmFile, idToWikidataPath);
}
void BuildDescriptionsSection(std::string const & wikipediaDir, std::string const & mwmFile)

30
generator/descriptions_section_builder.hpp
View file

@ -38,15 +38,15 @@ class WikidataHelper
{
public:
WikidataHelper() = default;
explicit WikidataHelper(std::string const & mwmPath, std::string const & id2wikidataPath);
explicit WikidataHelper(std::string const & mwmPath, std::string const & idToWikidataPath);
boost::optional<std::string> GetWikidataId(uint32_t featureId) const;
private:
std::string m_mwmPath;
std::string m_id2wikidataPath;
std::string m_idToWikidataPath;
std::map<uint32_t, base::GeoObjectId> m_featureIdToOsmId;
std::map<base::GeoObjectId, std::string> m_osmIdToFeatureId;
std::map<base::GeoObjectId, std::string> m_osmIdToWikidataId;
};
template <class T>
@ -81,10 +81,10 @@ public:
void IncPage() { ++m_pages; }
void IncNumberWikipediaUrls() { ++m_numberWikipediaUrls; }
void IncNumberWikidataIds() { ++m_numberWikidataIds; }
size_t GetSize() const { return m_size; }
size_t GetPages() const { return m_pages; }
size_t GetNumberWikipediaUrls() const { return m_numberWikipediaUrls; }
size_t GetNumberWikidataIds() const { return m_numberWikidataIds; }
size_t GetTotalSize() const { return m_size; }
size_t GetNumberOfPages() const { return m_pages; }
size_t GetNumberOfWikipediaUrls() const { return m_numberWikipediaUrls; }
size_t GetNumberOfWikidataIds() const { return m_numberWikidataIds; }
LangStatistics const & GetLangStatistics() const { return m_langsStat; }
private:
@ -101,7 +101,7 @@ public:
friend class generator_tests::TestDescriptionSectionBuilder;
DescriptionsCollectionBuilder(std::string const & wikipediaDir, std::string const & mwmFile,
std::string const & id2wikidataPath);
std::string const & idToWikidataPath);
DescriptionsCollectionBuilder(std::string const & wikipediaDir, std::string const & mwmFile);
template <typename Ft, template <typename> class ForEachFromDatAdapter>
@ -172,9 +172,9 @@ template <typename Ft, template <typename> class ForEachFromDatAdapter = ForEach
struct DescriptionsSectionBuilder
{
static void Build(std::string const & wikipediaDir, std::string const & mwmFile,
std::string const & id2wikidataPath)
std::string const & idToWikidataPath)
{
DescriptionsCollectionBuilder descriptionsCollectionBuilder(wikipediaDir, mwmFile, id2wikidataPath);
DescriptionsCollectionBuilder descriptionsCollectionBuilder(wikipediaDir, mwmFile, idToWikidataPath);
Build(mwmFile, descriptionsCollectionBuilder);
}
@ -189,10 +189,10 @@ private:
{
auto descriptionList = builder.MakeDescriptions<Ft, ForEachFromDatAdapter>();
auto const & stat = builder.GetStat();
auto const size = stat.GetSize();
LOG(LINFO, ("Added", stat.GetNumberWikipediaUrls(), "pages form wikipedia urls for", mwmFile));
LOG(LINFO, ("Added", stat.GetNumberWikidataIds(), "pages form wikidata ids for", mwmFile));
LOG(LINFO, ("Added", stat.GetPages(), "pages for", mwmFile));
auto const size = stat.GetTotalSize();
LOG(LINFO, ("Added", stat.GetNumberOfWikipediaUrls(), "pages from wikipedia urls for", mwmFile));
LOG(LINFO, ("Added", stat.GetNumberOfWikidataIds(), "pages from wikidata ids for", mwmFile));
LOG(LINFO, ("Added", stat.GetNumberOfPages(), "pages for", mwmFile));
LOG(LINFO, ("Total size of added pages (before writing to section):", size));
CHECK_GREATER_OR_EQUAL(size, 0, ());
if (size == 0)
@ -212,7 +212,7 @@ private:
};
void BuildDescriptionsSection(std::string const & wikipediaDir, std::string const & mwmFile,
std::string const & id2wikidataPath);
std::string const & idToWikidataPath);
void BuildDescriptionsSection(std::string const & wikipediaDir, std::string const & mwmFile);
} // namespace generator

2
generator/generate_info.hpp
View file

@ -53,7 +53,7 @@ struct GenerateInfo
std::string m_popularPlacesFilename;
std::string m_id2wikidataFilename;
std::string m_idToWikidataFilename;
std::shared_ptr<generator::OsmIdToBoundariesTable> m_boundariesTable;

10
generator/generator_tests/descriptions_section_builder_tests.cpp
View file

@ -101,8 +101,8 @@ public:
auto const descriptionList = b.MakeDescriptions<Feature, ForEachFromDatMockAdapt>();
auto const & stat = b.GetStat();
TEST_EQUAL(GetTestDataPages(), descriptionList.size(), ());
TEST_EQUAL(GetTestDataPages(), stat.GetPages(), ());
TEST_EQUAL(GetTestDataSize(), stat.GetSize(), ());
TEST_EQUAL(GetTestDataPages(), stat.GetNumberOfPages(), ());
TEST_EQUAL(GetTestDataSize(), stat.GetTotalSize(), ());
TEST(CheckLangs(stat.GetLangStatistics()), ());
}
@ -133,7 +133,7 @@ public:
auto const path = DescriptionsCollectionBuilder::MakePathForWikipedia(m_wikiDir, first.m_url);
auto const size = b.FindPageAndFill(path, str);
TEST(size, ());
TEST_EQUAL(*size, GetPageSize(first.m_pages), ());
TEST_EQUAL(*size, GetNumberOfPagesize(first.m_pages), ());
TEST(CheckLangs(str, first.m_pages), ());
}
{
@ -177,7 +177,7 @@ public:
auto const path = DescriptionsCollectionBuilder::MakePathForWikipedia(m_wikiDir, wikiUrl);
auto const size = b.GetFeatureDescription(path, featureId, description);
TEST_EQUAL(size, GetPageSize(first.m_pages), ());
TEST_EQUAL(size, GetNumberOfPagesize(first.m_pages), ());
CHECK_NOT_EQUAL(size, 0, ());
TEST_EQUAL(description.m_featureIndex, featureId, ());
TEST(CheckLangs(description.m_description, first.m_pages), ());
@ -285,7 +285,7 @@ private:
return StringUtf8Multilang::GetLangIndex(lang) != StringUtf8Multilang::kUnsupportedLanguageCode;
}
static size_t GetPageSize(std::vector<std::pair<std::string, std::string>> const & p)
static size_t GetNumberOfPagesize(std::vector<std::pair<std::string, std::string>> const & p)
{
return std::accumulate(std::begin(p), std::end(p), size_t{0}, [] (size_t acc, auto const & n) {
return acc + n.second.size();

14
generator/generator_tool/generator_tool.cpp
View file

@ -153,7 +153,7 @@ DEFINE_string(opentable_data, "", "Path to opentable data in .tsv format.");
DEFINE_string(ugc_data, "", "Input UGC source database file name.");
DEFINE_string(wikipedia_pages, "", "Input dir with wikipedia pages.");
DEFINE_string(id2wikidata, "", "Path to file with id to wikidata mapping.");
DEFINE_string(idToWikidata, "", "Path to file with id to wikidata mapping.");
DEFINE_string(dump_wikipedia_urls, "", "Output file with wikipedia urls.");
DEFINE_bool(generate_popular_places, false, "Generate popular places section.");
@ -307,7 +307,7 @@ int GeneratorToolMain(int argc, char ** argv)
genInfo.m_emitCoasts = FLAGS_emit_coasts;
genInfo.m_fileName = FLAGS_output;
genInfo.m_genAddresses = FLAGS_generate_addresses_file;
genInfo.m_id2wikidataFilename = FLAGS_id2wikidata;
genInfo.m_idToWikidataFilename = FLAGS_idToWikidata;
auto emitter = CreateEmitter(EmitterType::Planet, genInfo);
if (!GenerateFeatures(genInfo, emitter))
@ -439,9 +439,9 @@ int GeneratorToolMain(int argc, char ** argv)
WikiUrlDumper wikiUrlDumper(FLAGS_dump_wikipedia_urls, datFiles);
wikiUrlDumper.Dump(threadsCount);
if (!FLAGS_id2wikidata.empty())
if (!FLAGS_idToWikidata.empty())
{
WikiDataFilter wikiDataFilter(FLAGS_id2wikidata, datFiles);
WikiDataFilter wikiDataFilter(FLAGS_idToWikidata, datFiles);
wikiDataFilter.Filter(threadsCount);
}
}
@ -611,8 +611,8 @@ int GeneratorToolMain(int argc, char ** argv)
if (!FLAGS_wikipedia_pages.empty())
{
if (!FLAGS_id2wikidata.empty())
BuildDescriptionsSection(FLAGS_wikipedia_pages, datFile, FLAGS_id2wikidata);
if (!FLAGS_idToWikidata.empty())
BuildDescriptionsSection(FLAGS_wikipedia_pages, datFile, FLAGS_idToWikidata);
else
BuildDescriptionsSection(FLAGS_wikipedia_pages, datFile);
}
@ -688,7 +688,7 @@ int GeneratorToolMain(int argc, char ** argv)
int main(int argc, char ** argv)
{
{
try
{
return GeneratorToolMain(argc, argv);

2
generator/translator_planet.cpp
View file

@ -56,7 +56,7 @@ TranslatorPlanet::TranslatorPlanet(std::shared_ptr<EmitterInterface> emitter,
, m_nodeRelations(m_routingTagsProcessor)
, m_wayRelations(m_routingTagsProcessor)
, m_metalinesBuilder(info.GetIntermediateFileName(METALINES_FILENAME))
, m_wikiDataCollector(info.m_id2wikidataFilename, "wikidata", WikiDataValidator, true /* ignoreIfNotOpen */)
, m_wikiDataCollector(info.m_idToWikidataFilename, "wikidata", WikiDataValidator, true /* ignoreIfNotOpen */)
{
auto const addrFilePath = info.GetAddressesFileName();
if (!addrFilePath.empty())

14
generator/wiki_url_dumper.cpp
View file

@ -66,8 +66,8 @@ void WikiUrlDumper::DumpOne(std::string const & path, std::ostream & stream)
});
}
WikiDataFilter::WikiDataFilter(std::string const & path, std::vector<std::string> const & datFiles)
: m_path(path), m_dataFiles(datFiles)
WikiDataFilter::WikiDataFilter(std::string const & path, std::vector<std::string> const & dataFiles)
: m_path(path), m_dataFiles(dataFiles)
{
std::ifstream stream;
stream.exceptions(std::fstream::failbit | std::fstream::badbit);
@ -78,12 +78,12 @@ WikiDataFilter::WikiDataFilter(std::string const & path, std::vector<std::string
while (stream)
{
stream >> id >> wikidata;
m_id2wikiData.emplace(base::GeoObjectId(id), wikidata);
m_idToWikiData.emplace(base::GeoObjectId(id), wikidata);
}
}
// static
void WikiDataFilter::FilterOne(std::string const & path, std::map<base::GeoObjectId, std::string> const & id2wikiData,
void WikiDataFilter::FilterOne(std::string const & path, std::map<base::GeoObjectId, std::string> const & idToWikiData,
std::ostream & stream)
{
auto const & needWikiUrl = ftypes::WikiChecker::Instance();
@ -91,8 +91,8 @@ void WikiDataFilter::FilterOne(std::string const & path, std::map<base::GeoObjec
if (!needWikiUrl(feature.GetTypesHolder()))
return;
auto const it = id2wikiData.find(feature.GetMostGenericOsmId());
if (it == std::end(id2wikiData))
auto const it = idToWikiData.find(feature.GetMostGenericOsmId());
if (it == std::end(idToWikiData))
return;
stream << it->first.GetEncodedId() << "\t" << it->second << "\n";
@ -109,7 +109,7 @@ void WikiDataFilter::Filter(size_t cpuCount)
auto const fn = [&](std::string const & filename) {
std::stringstream stringStream;
FilterOne(filename, m_id2wikiData, stringStream);
FilterOne(filename, m_idToWikiData, stringStream);
return stringStream.str();
};

6
generator/wiki_url_dumper.hpp
View file

@ -26,16 +26,16 @@ private:
class WikiDataFilter
{
public:
explicit WikiDataFilter(std::string const & path, std::vector<std::string> const & datFiles);
explicit WikiDataFilter(std::string const & path, std::vector<std::string> const & dataFiles);
static void FilterOne(std::string const & path, std::map<base::GeoObjectId, std::string> const & id2wikiData,
static void FilterOne(std::string const & path, std::map<base::GeoObjectId, std::string> const & idToWikiData,
std::ostream & stream);
void Filter(size_t cpuCount);
private:
std::string m_path;
std::map<base::GeoObjectId, std::string> m_id2wikiData;
std::map<base::GeoObjectId, std::string> m_idToWikiData;
std::vector<std::string> m_dataFiles;
};
} // namespace generator

8
tools/unix/generate_planet.sh
View file

@ -185,7 +185,7 @@ DESCRIPTIONS_DOWNLOADER="$PYTHON_SCRIPTS_PATH/descriptions_downloader.py"
LOCALADS_SCRIPT="$PYTHON_SCRIPTS_PATH/local_ads/mwm_to_csv_4localads.py"
UGC_FILE="${UGC_FILE:-$INTDIR/ugc_db.sqlite3}"
POPULAR_PLACES_FILE="${POPULAR_PLACES_FILE:-$INTDIR/popular_places.csv}"
WIKIDATA_FILE="${WIKIDATA_FILE:-$INTDIR/id2wikidata.csv}"
WIKIDATA_FILE="${WIKIDATA_FILE:-$INTDIR/idToWikidata.csv}"
BOOKING_SCRIPT="$PYTHON_SCRIPTS_PATH/booking_hotels.py"
BOOKING_FILE="${BOOKING_FILE:-$INTDIR/hotels.csv}"
OPENTABLE_SCRIPT="$PYTHON_SCRIPTS_PATH/opentable_restaurants.py"
@ -454,7 +454,7 @@ if [ "$MODE" == "features" ]; then
[ -f "$BOOKING_FILE" ] && PARAMS_SPLIT="$PARAMS_SPLIT --booking_data=$BOOKING_FILE"
[ -f "$OPENTABLE_FILE" ] && PARAMS_SPLIT="$PARAMS_SPLIT --opentable_data=$OPENTABLE_FILE"
[ -f "$POPULAR_PLACES_FILE" ] && PARAMS_SPLIT="$PARAMS_SPLIT --popular_places_data=$POPULAR_PLACES_FILE"
[ -n "$OPT_DESCRIPTIONS" ] && PARAMS_SPLIT="$PARAMS_SPLIT --id2wikidata=$WIKIDATA_FILE"
[ -n "$OPT_DESCRIPTIONS" ] && PARAMS_SPLIT="$PARAMS_SPLIT --idToWikidata=$WIKIDATA_FILE"
"$GENERATOR_TOOL" --intermediate_data_path="$INTDIR/" \
@ -560,7 +560,7 @@ if [ "$MODE" == "descriptions" ]; then
LANGS="en ru es"
"$GENERATOR_TOOL" --intermediate_data_path="$INTDIR/" --user_resource_path="$DATA_PATH/" \
--dump_wikipedia_urls="$URLS_PATH" --id2wikidata="$WIKIDATA_FILE" 2>> $LOG
--dump_wikipedia_urls="$URLS_PATH" --idToWikidata="$WIKIDATA_FILE" 2>> $LOG
PARAMS="--wikipedia $URLS_PATH --wikidata $WIKIDATA_FILE --output_dir $WIKI_PAGES_PATH"
[ -f "$POPULAR_PLACES_FILE" ] && PARAMS="$PARAMS --popularity=$POPULAR_PLACES_FILE"
@ -569,7 +569,7 @@ if [ "$MODE" == "descriptions" ]; then
for file in "$TARGET"/*.mwm; do
if [[ "$file" != *minsk-pass* && "$file" != *World* ]]; then
BASENAME="$(basename "$file" .mwm)"
"$GENERATOR_TOOL" --wikipedia_pages="$WIKI_PAGES_PATH/" --id2wikidata="$WIKIDATA_FILE" \
"$GENERATOR_TOOL" --wikipedia_pages="$WIKI_PAGES_PATH/" --idToWikidata="$WIKIDATA_FILE" \
--data_path="$TARGET" --user_resource_path="$DATA_PATH/" --output="$BASENAME" 2>> "$LOG_PATH/$BASENAME.log" &
forky
fi