kirylkaveryn/organicmaps
1
Fork 0
forked from organicmaps/organicmaps-tmp
Code Pull requests Activity

Extract index path generation from offsets table class

Browse source
This commit is contained in:
Lev Dragunov 2015-03-06 19:19:04 +03:00 • committed by Alex Zolotarev
parent fa062c45a4
commit cbc137f240
8 changed files with 55 additions and 62 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
indexer/features_offsets_table.cpp
View file

@ -45,9 +45,8 @@ namespace feature
}
// static
unique_ptr<FeaturesOffsetsTable> FeaturesOffsetsTable::Load(string const & countryName)
unique_ptr<FeaturesOffsetsTable> FeaturesOffsetsTable::Load(string const & fileName)
{
string const fileName = GetIndexFileName(countryName);
uint64_t size;
if (!GetPlatform().GetFileSizeByFullPath(fileName, size))
return unique_ptr<FeaturesOffsetsTable>();
@ -56,34 +55,30 @@ namespace feature
// static
unique_ptr<FeaturesOffsetsTable> FeaturesOffsetsTable::CreateIfNotExistsAndLoad(
string const & countryName)
string const & fileName, FilesContainerR const & mwmFileContainer)
{
string const fileName = GetIndexFileName(countryName);
uint64_t size;
if (GetPlatform().GetFileSizeByFullPath(fileName,size))
return Load(countryName);
return Load(fileName);
string const mwmName = GetPlatform().WritablePathForFile(countryName + DATA_FILE_EXTENSION);
FilesContainerR cont(mwmName);
if (!cont.IsExist(HEADER_FILE_TAG))
if (!mwmFileContainer.IsExist(HEADER_FILE_TAG))
return unique_ptr<FeaturesOffsetsTable>();
DataHeader header;
header.Load(cont.GetReader(HEADER_FILE_TAG));
header.Load(mwmFileContainer.GetReader(HEADER_FILE_TAG));
Builder builder;
FeaturesVector(cont, header).ForEachOffset([&builder] (FeatureType const &, uint32_t offset)
FeaturesVector(mwmFileContainer, header).ForEachOffset([&builder] (FeatureType const &, uint32_t offset)
{
builder.PushOffset(offset);
});
unique_ptr<FeaturesOffsetsTable> table(Build(builder));
table->Save(countryName);
table->Save(fileName);
return table;
}
void FeaturesOffsetsTable::Save(string const & countryName)
void FeaturesOffsetsTable::Save(string const & fileName)
{
string const fileName = GetIndexFileName(countryName);
string const fileNameTmp = fileName + EXTENSION_TMP;
succinct::mapper::freeze(m_table, fileNameTmp.c_str());
my::RenameFileX(fileNameTmp, fileName);
@ -111,12 +106,7 @@ namespace feature
else
rightBound = middle;
}
ASSERT_EQUAL(offset, m_table.select(leftBound), ("Can't find offset", offset, "in the table"));
return leftBound;
}
string FeaturesOffsetsTable::GetIndexFileName(string const & countryName)
{
return GetPlatform().WritablePathForCountryIndexes(countryName) + countryName + FEATURES_OFFSETS_TABLE_FILE_EXT;
}
} // namespace feature

21
indexer/features_offsets_table.hpp
View file

@ -53,10 +53,10 @@ namespace feature
/// mapped to the memory and used by internal structures of
/// FeaturesOffsetsTable.
///
/// \param countryName a countryName to save index file to
/// \param fileName a full path of the file to load or store data
/// \return a pointer to an instance of FeaturesOffsetsTable or nullptr
/// when it's not possible to load FeaturesOffsetsTable.
static unique_ptr<FeaturesOffsetsTable> Load(string const & countryName);
static unique_ptr<FeaturesOffsetsTable> Load(string const & fileName);
/// Loads FeaturesOffsetsTable from FilesMappingContainer. Note
/// that some part of a file referenced by container will be
@ -67,18 +67,19 @@ namespace feature
///
/// \warning May take a lot of time if there is no precomputed section
///
/// \param countryName a country to create index to
/// \param fileName a full path of the file to load or store data
/// \param mwmFileContainer mwm container to read features data (if we need to construct them)
/// \return a pointer to an instance of FeaturesOffsetsTable or nullptr
/// when it's not possible to create FeaturesOffsetsTable.
static unique_ptr<FeaturesOffsetsTable> CreateIfNotExistsAndLoad(string const & countryName);
static unique_ptr<FeaturesOffsetsTable> CreateIfNotExistsAndLoad(string const & fileName, FilesContainerR const & mwmFileContainer);
FeaturesOffsetsTable(FeaturesOffsetsTable const &) = delete;
FeaturesOffsetsTable const & operator=(FeaturesOffsetsTable const &) = delete;
/// Serializes current instance to a section in container.
///
/// \param countryName a name of the country to create data
void Save(string const & countryName);
/// \param fileName a full path of the file to store data
void Save(string const & fileName);
/// \param index index of a feature
/// \return offset a feature
@ -102,15 +103,7 @@ namespace feature
return succinct::mapper::size_of(m_table);
}
/// Delete temporary index file (only for features offsets table)
static void CleanIndexFile(string const & countryName)
{
FileWriter::DeleteFileX(GetIndexFileName(countryName));
}
private:
static string GetIndexFileName(string const & countryName);
FeaturesOffsetsTable(succinct::elias_fano::elias_fano_builder & builder);
FeaturesOffsetsTable(string const &);

18
indexer/indexer_tests/features_offsets_table_test.cpp
View file

@ -61,9 +61,10 @@ namespace feature
string const testFileName = "minsk-pass";
Platform & p = GetPlatform();
FilesContainerR baseContainer(p.GetReader(testFileName + DATA_FILE_EXTENSION));
FeaturesOffsetsTable::CleanIndexFile(testFileName);
unique_ptr<FeaturesOffsetsTable> table(FeaturesOffsetsTable::CreateIfNotExistsAndLoad(testFileName));
MY_SCOPE_GUARD(deleteTestFileIndexGuard, bind(&FeaturesOffsetsTable::CleanIndexFile, cref(testFileName)));
const string indexFile = p.GetIndexFileName(testFileName, FEATURES_OFFSETS_TABLE_FILE_EXT);
FileWriter::DeleteFileX(indexFile);
unique_ptr<FeaturesOffsetsTable> table(FeaturesOffsetsTable::CreateIfNotExistsAndLoad(indexFile, baseContainer));
MY_SCOPE_GUARD(deleteTestFileIndexGuard, bind(&FileWriter::DeleteFileX, cref(indexFile)));
TEST(table.get(), ());
feature::DataHeader header;
@ -77,7 +78,7 @@ namespace feature
TEST_EQUAL(builderSize, table->size(), ());
table = unique_ptr<FeaturesOffsetsTable>();
table = unique_ptr<FeaturesOffsetsTable>(FeaturesOffsetsTable::Load(testFileName));
table = unique_ptr<FeaturesOffsetsTable>(FeaturesOffsetsTable::Load(indexFile));
TEST_EQUAL(builderSize, table->size(), ());
}
@ -86,6 +87,7 @@ namespace feature
string const testFileName = "test_file";
Platform & p = GetPlatform();
FilesContainerR baseContainer(p.GetReader("minsk-pass" DATA_FILE_EXTENSION));
const string indexFile = p.GetIndexFileName(testFileName, FEATURES_OFFSETS_TABLE_FILE_EXT);
feature::DataHeader header;
header.Load(baseContainer.GetReader(HEADER_FILE_TAG));
@ -102,7 +104,7 @@ namespace feature
string const testFile = p.WritablePathForFile(testFileName + DATA_FILE_EXTENSION);
MY_SCOPE_GUARD(deleteTestFileGuard, bind(&FileWriter::DeleteFileX, cref(testFile)));
MY_SCOPE_GUARD(deleteTestFileIndexGuard, bind(&FeaturesOffsetsTable::CleanIndexFile, cref(testFileName)));
MY_SCOPE_GUARD(deleteTestFileIndexGuard, bind(&FileWriter::DeleteFileX, cref(indexFile)));
// Store table in a temporary data file.
{
@ -114,14 +116,14 @@ namespace feature
{
testContainer.Write(baseContainer.GetReader(tag), tag);
});
table->Save(testFileName);
table->Save(indexFile);
testContainer.Finish();
}
// Load table from the temporary data file.
{
MY_SCOPE_GUARD(testTableGuard, bind(&FeaturesOffsetsTable::CleanIndexFile, cref(testFileName)));
unique_ptr<FeaturesOffsetsTable> loadedTable(FeaturesOffsetsTable::Load(testFileName));
MY_SCOPE_GUARD(testTableGuard, bind(&FileWriter::DeleteFileX, cref(indexFile)));
unique_ptr<FeaturesOffsetsTable> loadedTable(FeaturesOffsetsTable::Load(indexFile));
TEST(loadedTable.get(), ());
TEST_EQUAL(table->size(), loadedTable->size(), ());

9
platform/platform.cpp
View file

@ -95,10 +95,15 @@ string Platform::DeviceName() const
return OMIM_OS_NAME;
}
string Platform::WritablePathForCountryIndexes(string const & fileName) const
string Platform::WritablePathForCountryIndexes(string const & mwmName) const
{
string dir = WritableDir() + fileName + my::GetNativeSeparator();
string dir = WritableDir() + mwmName + my::GetNativeSeparator();
if (!IsFileExistsByFullPath(dir))
MkDir(dir);
return dir;
}
string Platform::GetIndexFileName(string const & mwmName, string const & extension) const
{
return GetPlatform().WritablePathForCountryIndexes(mwmName) + mwmName + extension;
}

4
platform/platform.hpp
View file

@ -61,7 +61,9 @@ public:
/// @return full path to file in user's writable directory
string WritablePathForFile(string const & file) const { return WritableDir() + file; }
/// @return full path to indexes directory for country file. Creates directory if it's not exists.
string WritablePathForCountryIndexes(string const & country_name) const;
string WritablePathForCountryIndexes(string const & mwmName) const;
/// @return generate full path to index based on mwmName and index extension
string GetIndexFileName(string const & mwmName, string const & extension) const;
/// @return resource dir (on some platforms it's differ from Writable dir)
string ResourcesDir() const { return m_resourcesDir; }

3
platform/platform_android.cpp
View file

@ -246,7 +246,8 @@ bool Platform::GetFileSizeByName(string const & fileName, uint64_t & size) const
void Platform::MkDir(string const & dirName) const
{
::mkdir(dirName.c_str(), 0755);
if (mkdir(dirName.c_str(), 0755))
LOG(LWARNING, ("Can't create directory: ", dirName));
}
namespace

30
routing/osrm2feature_map.cpp
View file

@ -13,6 +13,7 @@
#include "../std/fstream.hpp"
#include "../std/sstream.hpp"
#include "../std/unordered_map.hpp"
#include "../3party/succinct/mapper.hpp"
@ -321,33 +322,27 @@ void OsrmFtSegMappingBuilder::Save(FilesContainerW & cont) const
cont.Write(fName, ROUTING_FTSEG_FILE_TAG);
}
void OsrmFtSegBackwardIndex::Save(string const & countryName)
void OsrmFtSegBackwardIndex::Save(string const & nodesFileName, string const & bitsFileName)
{
string const dir = GetPlatform().WritablePathForCountryIndexes(countryName);
{
string const nodesFileName = dir + countryName + FTSEG_MAPPING_BACKWARD_INDEX_NODES_EXT;
string const nodesFileNameTmp = nodesFileName + EXTENSION_TMP;
succinct::mapper::freeze(m_nodeIds, nodesFileNameTmp.c_str());
my::RenameFileX(nodesFileNameTmp, nodesFileName);
}
{
string const bitsFileName = dir + countryName + FTSEG_MAPPING_BACKWARD_INDEX_BITS_EXT;
string const bitsFileNameTmp = bitsFileName + EXTENSION_TMP;
succinct::mapper::freeze(m_rankIndex, bitsFileNameTmp.c_str());
my::RenameFileX(bitsFileNameTmp, bitsFileName);
}
}
bool OsrmFtSegBackwardIndex::Load(string const & countryName)
bool OsrmFtSegBackwardIndex::Load(string const & nodesFileName, string const & bitsFileName)
{
string const dir = GetPlatform().WritablePathForCountryIndexes(countryName);
string const nodesName = dir + countryName + FTSEG_MAPPING_BACKWARD_INDEX_NODES_EXT;
string const bitsName = dir + countryName + FTSEG_MAPPING_BACKWARD_INDEX_BITS_EXT;
uint64_t size;
if (!GetPlatform().GetFileSizeByFullPath(nodesName, size) || !GetPlatform().GetFileSizeByFullPath(bitsName, size))
if (!GetPlatform().GetFileSizeByFullPath(nodesFileName, size) || !GetPlatform().GetFileSizeByFullPath(bitsFileName, size))
return false;
m_pMappedNodes.reset(new MmapReader(nodesName));
m_pMappedBits.reset(new MmapReader(bitsName));
m_pMappedNodes.reset(new MmapReader(nodesFileName));
m_pMappedBits.reset(new MmapReader(bitsFileName));
succinct::mapper::map(m_nodeIds, reinterpret_cast<char const *>(m_pMappedNodes->Data()));
succinct::mapper::map(m_rankIndex, reinterpret_cast<char const *>(m_pMappedBits->Data()));
@ -359,11 +354,16 @@ void OsrmFtSegBackwardIndex::Construct(const OsrmFtSegMapping & mapping, const u
{
Clear();
// Calculate data file pathes
Platform const & p = GetPlatform();
string const routingName = routingFile.GetName();
string const name(routingName, routingName.rfind(my::GetNativeSeparator())+1, routingName.find(DATA_FILE_EXTENSION) -routingName.rfind(my::GetNativeSeparator())-1);
m_table = feature::FeaturesOffsetsTable::CreateIfNotExistsAndLoad(name);
string const name(routingName, routingName.rfind(my::GetNativeSeparator()) + 1,
routingName.find(DATA_FILE_EXTENSION) - routingName.rfind(my::GetNativeSeparator()) - 1);
string const offsetsIndexName = p.GetIndexFileName(name, FEATURES_OFFSETS_TABLE_FILE_EXT);
string const bitsFileName = p.GetIndexFileName(name, FTSEG_MAPPING_BACKWARD_INDEX_BITS_EXT);
string const nodesFileName = p.GetIndexFileName(name, FTSEG_MAPPING_BACKWARD_INDEX_NODES_EXT);
m_table = feature::FeaturesOffsetsTable::CreateIfNotExistsAndLoad(offsetsIndexName, FilesContainerR(name + DATA_FILE_EXTENSION));
if (Load(name))
if (Load(bitsFileName, nodesFileName))
return;
// Generate temporary index to speedup processing
@ -399,7 +399,7 @@ void OsrmFtSegBackwardIndex::Construct(const OsrmFtSegMapping & mapping, const u
succinct::rs_bit_vector(inIndex).swap(m_rankIndex);
LOG(LINFO, ("Writing section to data file", routingName));
Save(name);
Save(bitsFileName, nodesFileName);
}
uint32_t OsrmFtSegBackwardIndex::GetNodeIdByFid(const uint32_t fid) const

4
routing/osrm2feature_map.hpp
View file

@ -107,9 +107,9 @@ class OsrmFtSegBackwardIndex
T().swap(t);
}
void Save(string const & countryName);
void Save(string const & nodesFileName, string const & bitsFileName);
bool Load(string const & container);
bool Load(string const & nodesFileName, string const & bitsFileName);
public:
void Construct(OsrmFtSegMapping const & mapping, uint32_t const maxNodeId, FilesMappingContainer & routingFile);