mnalis/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

[generator] [search] Minor fixes.

Browse source
This commit is contained in:
Maxim Pimenov 2018-07-25 17:29:46 +03:00 • committed by Maksim Andrianov
parent 7a1c013f93
commit d39089f03f
5 changed files with 91 additions and 71 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
generator/generator_tests_support/test_mwm_builder.cpp
View file

@ -131,7 +131,7 @@ void TestMwmBuilder::Finish()
CHECK(indexer::BuildIndexFromDataFile(path, path), ("Can't build geometry index."));
CHECK(indexer::BuildSearchIndexFromDataFile(1 /* threadsCount */, path, true /* forceRebuild */),
CHECK(indexer::BuildSearchIndexFromDataFile(path, true /* forceRebuild */, 1 /* threadsCount */),
("Can't build search index."));
if (m_type == feature::DataHeader::world)

3
generator/generator_tool/generator_tool.cpp
View file

@ -383,7 +383,8 @@ int main(int argc, char ** argv)
LOG(LINFO, ("Generating search index for", datFile));
/// @todo Make threads count according to environment (single mwm build or planet build).
if (!indexer::BuildSearchIndexFromDataFile(8, datFile, true))
if (!indexer::BuildSearchIndexFromDataFile(datFile, true /* forceRebuild */,
1 /* threadsCount */))
LOG(LCRITICAL, ("Error generating search index."));
LOG(LINFO, ("Generating rank table for", datFile));

103
generator/osm_xml_source.hpp
View file

@ -2,19 +2,18 @@
#include "generator/osm_element.hpp"
#include "base/assert.hpp"
#include "base/string_utils.hpp"
#include <functional>
#include <string>
class XMLSource
{
OsmElement m_parent;
OsmElement m_child;
size_t m_depth = 0;
OsmElement * m_current = nullptr;
using TEmmiterFn = function<void(OsmElement *)>;
TEmmiterFn m_EmmiterFn;
public:
XMLSource(TEmmiterFn fn) : m_EmmiterFn(fn) {}
using Emitter = std::function<void(OsmElement *)>;
XMLSource(Emitter fn) : m_emitter(fn) {}
void CharData(std::string const &) {}
@ -24,13 +23,13 @@ public:
return;
if (key == "id")
CHECK ( strings::to_uint64(value, m_current->id), ("Unknown element with invalid id : ", value) );
CHECK(strings::to_uint64(value, m_current->id), ("Unknown element with invalid id:", value));
else if (key == "lon")
CHECK ( strings::to_double(value, m_current->lon), ("Bad node lon : ", value) );
CHECK(strings::to_double(value, m_current->lon), ("Bad node lon:", value));
else if (key == "lat")
CHECK ( strings::to_double(value, m_current->lat), ("Bad node lat : ", value) );
CHECK(strings::to_double(value, m_current->lat), ("Bad node lat:", value));
else if (key == "ref")
CHECK ( strings::to_uint64(value, m_current->ref), ("Bad node ref in way : ", value) );
CHECK(strings::to_uint64(value, m_current->ref), ("Bad node ref in way:", value));
else if (key == "k")
m_current->k = value;
else if (key == "v")
@ -46,20 +45,22 @@ public:
ASSERT_GREATER_OR_EQUAL(tagName.size(), 2, ());
// As tagKey we use first two char of tag name.
OsmElement::EntityType tagKey = OsmElement::EntityType(*reinterpret_cast<uint16_t const *>(tagName.data()));
OsmElement::EntityType tagKey =
OsmElement::EntityType(*reinterpret_cast<uint16_t const *>(tagName.data()));
switch (++m_depth)
{
case 1:
m_current = nullptr;
break;
case 2:
m_current = &m_parent;
m_current->type = tagKey;
break;
default:
m_current = &m_child;
m_current->type = tagKey;
case 1:
m_current = nullptr;
break;
case 2:
m_current = &m_parent;
m_current->type = tagKey;
break;
default:
m_current = &m_child;
m_current->type = tagKey;
break;
}
return true;
}
@ -68,30 +69,38 @@ public:
{
switch (--m_depth)
{
case 0:
break;
case 0:
break;
case 1:
m_EmmiterFn(m_current);
m_parent.Clear();
break;
case 1:
m_emitter(m_current);
m_parent.Clear();
break;
default:
switch (m_child.type)
{
case OsmElement::EntityType::Member:
m_parent.AddMember(m_child.ref, m_child.memberType, m_child.role);
break;
case OsmElement::EntityType::Tag:
m_parent.AddTag(m_child.k, m_child.v);
break;
case OsmElement::EntityType::Nd:
m_parent.AddNd(m_child.ref);
default:
break;
}
m_current = &m_parent;
m_child.Clear();
default:
switch (m_child.type)
{
case OsmElement::EntityType::Member:
m_parent.AddMember(m_child.ref, m_child.memberType, m_child.role);
break;
case OsmElement::EntityType::Tag:
m_parent.AddTag(m_child.k, m_child.v);
break;
case OsmElement::EntityType::Nd:
m_parent.AddNd(m_child.ref);
break;
}
m_current = &m_parent;
m_child.Clear();
}
}
private:
OsmElement m_parent;
OsmElement m_child;
size_t m_depth = 0;
OsmElement * m_current = nullptr;
Emitter m_emitter;
};

51
generator/search_index_builder.cpp
View file

@ -48,7 +48,6 @@ using namespace std;
#define SYNONYMS_FILE "synonyms.txt"
namespace
{
class SynonymsHolder
@ -319,8 +318,8 @@ void AddFeatureNameIndexPairs(FeaturesVectorTest const & features,
synonyms.get(), keyValuePairs, categoriesHolder, header.GetScaleRange(), valueBuilder));
}
pair<uint32_t, bool> GetStreetIndex(search::MwmContext & ctx, uint32_t featureID,
string const & streetName)
bool GetStreetIndex(search::MwmContext & ctx, uint32_t featureID, string const & streetName,
uint32_t & result)
{
size_t streetIndex = 0;
strings::UniString const street = search::GetStreetNameAsKey(streetName);
@ -337,13 +336,17 @@ pair<uint32_t, bool> GetStreetIndex(search::MwmContext & ctx, uint32_t featureID
streetIndex = search::ReverseGeocoder::GetMatchedStreetIndex(street, streets);
if (streetIndex < streets.size())
return { base::checked_cast<uint32_t>(streetIndex), true };
{
result = base::checked_cast<uint32_t>(streetIndex);
return true;
}
}
return { hasStreet ? 1 : 0, false };
result = hasStreet ? 1 : 0;
return false;
}
void BuildAddressTable(uint32_t threadsCount, FilesContainerR & container, Writer & writer)
void BuildAddressTable(FilesContainerR & container, Writer & writer, uint32_t threadsCount)
{
// Read all street names to memory.
ReaderSource<ModelReaderPtr> src(container.GetReader(SEARCH_TOKENS_FILE_TAG));
@ -357,8 +360,13 @@ void BuildAddressTable(uint32_t threadsCount, FilesContainerR & container, Write
// Initialize temporary source for the current mwm file.
FrozenDataSource dataSource;
auto const res = dataSource.RegisterMap(platform::LocalCountryFile::MakeTemporary(container.GetFileName()));
ASSERT_EQUAL(res.second, MwmSet::RegResult::Success, ());
MwmSet::MwmId mwmId;
{
auto const regResult =
dataSource.RegisterMap(platform::LocalCountryFile::MakeTemporary(container.GetFileName()));
ASSERT_EQUAL(regResult.second, MwmSet::RegResult::Success, ());
mwmId = regResult.first;
}
vector<unique_ptr<search::MwmContext>> contexts(threadsCount);
@ -371,28 +379,29 @@ void BuildAddressTable(uint32_t threadsCount, FilesContainerR & container, Write
mutex resMutex;
// Thread working function.
auto const fn = [&](uint32_t threadIdx)
{
uint64_t const fc = featuresCount;
auto const fn = [&](uint32_t threadIdx) {
uint64_t const fc = static_cast<uint64_t>(featuresCount);
uint32_t const beg = static_cast<uint32_t>(fc * threadIdx / threadsCount);
uint32_t const end = static_cast<uint32_t>(fc * (threadIdx + 1) / threadsCount);
for (uint32_t i = beg; i < end; ++i)
{
auto const res = GetStreetIndex(*(contexts[threadIdx]), i, addrs[i].Get(feature::AddressData::STREET));
uint32_t streetIndex;
bool const found = GetStreetIndex(*(contexts[threadIdx]), i,
addrs[i].Get(feature::AddressData::STREET), streetIndex);
lock_guard<mutex> guard(resMutex);
if (res.second)
if (found)
{
results[i] = res.first;
++bounds[res.first];
results[i] = streetIndex;
++bounds[streetIndex];
++address;
}
else if (res.first > 0)
else if (streetIndex > 0)
{
++missing;
++address;
++missing;
++address;
}
}
};
@ -401,7 +410,7 @@ void BuildAddressTable(uint32_t threadsCount, FilesContainerR & container, Write
vector<thread> threads;
for (size_t i = 0; i < threadsCount; ++i)
{
auto handle = dataSource.GetMwmHandleById(res.first);
auto handle = dataSource.GetMwmHandleById(mwmId);
contexts[i] = make_unique<search::MwmContext>(move(handle));
threads.emplace_back(fn, i);
}
@ -434,7 +443,7 @@ void BuildAddressTable(uint32_t threadsCount, FilesContainerR & container, Write
namespace indexer
{
bool BuildSearchIndexFromDataFile(uint32_t threadsCount, string const & filename, bool forceRebuild)
bool BuildSearchIndexFromDataFile(string const & filename, bool forceRebuild, uint32_t threadsCount)
{
Platform & platform = GetPlatform();
@ -457,7 +466,7 @@ bool BuildSearchIndexFromDataFile(uint32_t threadsCount, string const & filename
if (filename != WORLD_FILE_NAME && filename != WORLD_COASTS_FILE_NAME)
{
FileWriter writer(addrFilePath);
BuildAddressTable(threadsCount, readContainer, writer);
BuildAddressTable(readContainer, writer, threadsCount);
LOG(LINFO, ("Search address table size =", writer.Size()));
}
{

3
generator/search_index_builder.hpp
View file

@ -11,7 +11,8 @@ namespace indexer
// An attempt to rewrite the search index of an old mwm may result in a future crash
// when using search because this function does not update mwm's version. This results
// in version mismatch when trying to read the index.
bool BuildSearchIndexFromDataFile(uint32_t threadsCount, std::string const & filename, bool forceRebuild = false);
bool BuildSearchIndexFromDataFile(std::string const & filename, bool forceRebuild,
uint32_t threadsCount);
void BuildSearchIndex(FilesContainerR & container, Writer & indexWriter);
} // namespace indexer