From cbc137f24089dac4f4540e23747bb7b0259c936b Mon Sep 17 00:00:00 2001 From: Lev Dragunov Date: Fri, 6 Mar 2015 19:19:04 +0300 Subject: [PATCH] Extract index path generation from offsets table class --- indexer/features_offsets_table.cpp | 28 ++++++----------- indexer/features_offsets_table.hpp | 21 +++++-------- .../features_offsets_table_test.cpp | 18 ++++++----- platform/platform.cpp | 9 ++++-- platform/platform.hpp | 4 ++- platform/platform_android.cpp | 3 +- routing/osrm2feature_map.cpp | 30 +++++++++---------- routing/osrm2feature_map.hpp | 4 +-- 8 files changed, 55 insertions(+), 62 deletions(-) diff --git a/indexer/features_offsets_table.cpp b/indexer/features_offsets_table.cpp index 23015bfd7f..87be9bab18 100644 --- a/indexer/features_offsets_table.cpp +++ b/indexer/features_offsets_table.cpp @@ -45,9 +45,8 @@ namespace feature } // static - unique_ptr FeaturesOffsetsTable::Load(string const & countryName) + unique_ptr FeaturesOffsetsTable::Load(string const & fileName) { - string const fileName = GetIndexFileName(countryName); uint64_t size; if (!GetPlatform().GetFileSizeByFullPath(fileName, size)) return unique_ptr(); @@ -56,34 +55,30 @@ namespace feature // static unique_ptr FeaturesOffsetsTable::CreateIfNotExistsAndLoad( - string const & countryName) + string const & fileName, FilesContainerR const & mwmFileContainer) { - string const fileName = GetIndexFileName(countryName); uint64_t size; if (GetPlatform().GetFileSizeByFullPath(fileName,size)) - return Load(countryName); + return Load(fileName); - string const mwmName = GetPlatform().WritablePathForFile(countryName + DATA_FILE_EXTENSION); - FilesContainerR cont(mwmName); - if (!cont.IsExist(HEADER_FILE_TAG)) + if (!mwmFileContainer.IsExist(HEADER_FILE_TAG)) return unique_ptr(); DataHeader header; - header.Load(cont.GetReader(HEADER_FILE_TAG)); + header.Load(mwmFileContainer.GetReader(HEADER_FILE_TAG)); Builder builder; - FeaturesVector(cont, header).ForEachOffset([&builder] (FeatureType const &, uint32_t offset) + FeaturesVector(mwmFileContainer, header).ForEachOffset([&builder] (FeatureType const &, uint32_t offset) { builder.PushOffset(offset); }); unique_ptr table(Build(builder)); - table->Save(countryName); + table->Save(fileName); return table; } - void FeaturesOffsetsTable::Save(string const & countryName) + void FeaturesOffsetsTable::Save(string const & fileName) { - string const fileName = GetIndexFileName(countryName); string const fileNameTmp = fileName + EXTENSION_TMP; succinct::mapper::freeze(m_table, fileNameTmp.c_str()); my::RenameFileX(fileNameTmp, fileName); @@ -111,12 +106,7 @@ namespace feature else rightBound = middle; } + ASSERT_EQUAL(offset, m_table.select(leftBound), ("Can't find offset", offset, "in the table")); return leftBound; } - - string FeaturesOffsetsTable::GetIndexFileName(string const & countryName) - { - return GetPlatform().WritablePathForCountryIndexes(countryName) + countryName + FEATURES_OFFSETS_TABLE_FILE_EXT; - } - } // namespace feature diff --git a/indexer/features_offsets_table.hpp b/indexer/features_offsets_table.hpp index 8f89c91ef2..5fa778560d 100644 --- a/indexer/features_offsets_table.hpp +++ b/indexer/features_offsets_table.hpp @@ -53,10 +53,10 @@ namespace feature /// mapped to the memory and used by internal structures of /// FeaturesOffsetsTable. /// - /// \param countryName a countryName to save index file to + /// \param fileName a full path of the file to load or store data /// \return a pointer to an instance of FeaturesOffsetsTable or nullptr /// when it's not possible to load FeaturesOffsetsTable. - static unique_ptr Load(string const & countryName); + static unique_ptr Load(string const & fileName); /// Loads FeaturesOffsetsTable from FilesMappingContainer. Note /// that some part of a file referenced by container will be @@ -67,18 +67,19 @@ namespace feature /// /// \warning May take a lot of time if there is no precomputed section /// - /// \param countryName a country to create index to + /// \param fileName a full path of the file to load or store data + /// \param mwmFileContainer mwm container to read features data (if we need to construct them) /// \return a pointer to an instance of FeaturesOffsetsTable or nullptr /// when it's not possible to create FeaturesOffsetsTable. - static unique_ptr CreateIfNotExistsAndLoad(string const & countryName); + static unique_ptr CreateIfNotExistsAndLoad(string const & fileName, FilesContainerR const & mwmFileContainer); FeaturesOffsetsTable(FeaturesOffsetsTable const &) = delete; FeaturesOffsetsTable const & operator=(FeaturesOffsetsTable const &) = delete; /// Serializes current instance to a section in container. /// - /// \param countryName a name of the country to create data - void Save(string const & countryName); + /// \param fileName a full path of the file to store data + void Save(string const & fileName); /// \param index index of a feature /// \return offset a feature @@ -102,15 +103,7 @@ namespace feature return succinct::mapper::size_of(m_table); } - /// Delete temporary index file (only for features offsets table) - static void CleanIndexFile(string const & countryName) - { - FileWriter::DeleteFileX(GetIndexFileName(countryName)); - } - private: - static string GetIndexFileName(string const & countryName); - FeaturesOffsetsTable(succinct::elias_fano::elias_fano_builder & builder); FeaturesOffsetsTable(string const &); diff --git a/indexer/indexer_tests/features_offsets_table_test.cpp b/indexer/indexer_tests/features_offsets_table_test.cpp index 010d40e0c3..9c1679fed5 100644 --- a/indexer/indexer_tests/features_offsets_table_test.cpp +++ b/indexer/indexer_tests/features_offsets_table_test.cpp @@ -61,9 +61,10 @@ namespace feature string const testFileName = "minsk-pass"; Platform & p = GetPlatform(); FilesContainerR baseContainer(p.GetReader(testFileName + DATA_FILE_EXTENSION)); - FeaturesOffsetsTable::CleanIndexFile(testFileName); - unique_ptr table(FeaturesOffsetsTable::CreateIfNotExistsAndLoad(testFileName)); - MY_SCOPE_GUARD(deleteTestFileIndexGuard, bind(&FeaturesOffsetsTable::CleanIndexFile, cref(testFileName))); + const string indexFile = p.GetIndexFileName(testFileName, FEATURES_OFFSETS_TABLE_FILE_EXT); + FileWriter::DeleteFileX(indexFile); + unique_ptr table(FeaturesOffsetsTable::CreateIfNotExistsAndLoad(indexFile, baseContainer)); + MY_SCOPE_GUARD(deleteTestFileIndexGuard, bind(&FileWriter::DeleteFileX, cref(indexFile))); TEST(table.get(), ()); feature::DataHeader header; @@ -77,7 +78,7 @@ namespace feature TEST_EQUAL(builderSize, table->size(), ()); table = unique_ptr(); - table = unique_ptr(FeaturesOffsetsTable::Load(testFileName)); + table = unique_ptr(FeaturesOffsetsTable::Load(indexFile)); TEST_EQUAL(builderSize, table->size(), ()); } @@ -86,6 +87,7 @@ namespace feature string const testFileName = "test_file"; Platform & p = GetPlatform(); FilesContainerR baseContainer(p.GetReader("minsk-pass" DATA_FILE_EXTENSION)); + const string indexFile = p.GetIndexFileName(testFileName, FEATURES_OFFSETS_TABLE_FILE_EXT); feature::DataHeader header; header.Load(baseContainer.GetReader(HEADER_FILE_TAG)); @@ -102,7 +104,7 @@ namespace feature string const testFile = p.WritablePathForFile(testFileName + DATA_FILE_EXTENSION); MY_SCOPE_GUARD(deleteTestFileGuard, bind(&FileWriter::DeleteFileX, cref(testFile))); - MY_SCOPE_GUARD(deleteTestFileIndexGuard, bind(&FeaturesOffsetsTable::CleanIndexFile, cref(testFileName))); + MY_SCOPE_GUARD(deleteTestFileIndexGuard, bind(&FileWriter::DeleteFileX, cref(indexFile))); // Store table in a temporary data file. { @@ -114,14 +116,14 @@ namespace feature { testContainer.Write(baseContainer.GetReader(tag), tag); }); - table->Save(testFileName); + table->Save(indexFile); testContainer.Finish(); } // Load table from the temporary data file. { - MY_SCOPE_GUARD(testTableGuard, bind(&FeaturesOffsetsTable::CleanIndexFile, cref(testFileName))); - unique_ptr loadedTable(FeaturesOffsetsTable::Load(testFileName)); + MY_SCOPE_GUARD(testTableGuard, bind(&FileWriter::DeleteFileX, cref(indexFile))); + unique_ptr loadedTable(FeaturesOffsetsTable::Load(indexFile)); TEST(loadedTable.get(), ()); TEST_EQUAL(table->size(), loadedTable->size(), ()); diff --git a/platform/platform.cpp b/platform/platform.cpp index ab89c3d7c1..87504f8108 100644 --- a/platform/platform.cpp +++ b/platform/platform.cpp @@ -95,10 +95,15 @@ string Platform::DeviceName() const return OMIM_OS_NAME; } -string Platform::WritablePathForCountryIndexes(string const & fileName) const +string Platform::WritablePathForCountryIndexes(string const & mwmName) const { - string dir = WritableDir() + fileName + my::GetNativeSeparator(); + string dir = WritableDir() + mwmName + my::GetNativeSeparator(); if (!IsFileExistsByFullPath(dir)) MkDir(dir); return dir; } + +string Platform::GetIndexFileName(string const & mwmName, string const & extension) const +{ + return GetPlatform().WritablePathForCountryIndexes(mwmName) + mwmName + extension; +} diff --git a/platform/platform.hpp b/platform/platform.hpp index f90f32a01e..82208d1402 100644 --- a/platform/platform.hpp +++ b/platform/platform.hpp @@ -61,7 +61,9 @@ public: /// @return full path to file in user's writable directory string WritablePathForFile(string const & file) const { return WritableDir() + file; } /// @return full path to indexes directory for country file. Creates directory if it's not exists. - string WritablePathForCountryIndexes(string const & country_name) const; + string WritablePathForCountryIndexes(string const & mwmName) const; + /// @return generate full path to index based on mwmName and index extension + string GetIndexFileName(string const & mwmName, string const & extension) const; /// @return resource dir (on some platforms it's differ from Writable dir) string ResourcesDir() const { return m_resourcesDir; } diff --git a/platform/platform_android.cpp b/platform/platform_android.cpp index aa330bc70a..6e83a93c5b 100644 --- a/platform/platform_android.cpp +++ b/platform/platform_android.cpp @@ -246,7 +246,8 @@ bool Platform::GetFileSizeByName(string const & fileName, uint64_t & size) const void Platform::MkDir(string const & dirName) const { - ::mkdir(dirName.c_str(), 0755); + if (mkdir(dirName.c_str(), 0755)) + LOG(LWARNING, ("Can't create directory: ", dirName)); } namespace diff --git a/routing/osrm2feature_map.cpp b/routing/osrm2feature_map.cpp index 536e4795fc..4ebb87f51d 100644 --- a/routing/osrm2feature_map.cpp +++ b/routing/osrm2feature_map.cpp @@ -13,6 +13,7 @@ #include "../std/fstream.hpp" #include "../std/sstream.hpp" +#include "../std/unordered_map.hpp" #include "../3party/succinct/mapper.hpp" @@ -321,33 +322,27 @@ void OsrmFtSegMappingBuilder::Save(FilesContainerW & cont) const cont.Write(fName, ROUTING_FTSEG_FILE_TAG); } -void OsrmFtSegBackwardIndex::Save(string const & countryName) +void OsrmFtSegBackwardIndex::Save(string const & nodesFileName, string const & bitsFileName) { - string const dir = GetPlatform().WritablePathForCountryIndexes(countryName); { - string const nodesFileName = dir + countryName + FTSEG_MAPPING_BACKWARD_INDEX_NODES_EXT; string const nodesFileNameTmp = nodesFileName + EXTENSION_TMP; succinct::mapper::freeze(m_nodeIds, nodesFileNameTmp.c_str()); my::RenameFileX(nodesFileNameTmp, nodesFileName); } { - string const bitsFileName = dir + countryName + FTSEG_MAPPING_BACKWARD_INDEX_BITS_EXT; string const bitsFileNameTmp = bitsFileName + EXTENSION_TMP; succinct::mapper::freeze(m_rankIndex, bitsFileNameTmp.c_str()); my::RenameFileX(bitsFileNameTmp, bitsFileName); } } -bool OsrmFtSegBackwardIndex::Load(string const & countryName) +bool OsrmFtSegBackwardIndex::Load(string const & nodesFileName, string const & bitsFileName) { - string const dir = GetPlatform().WritablePathForCountryIndexes(countryName); - string const nodesName = dir + countryName + FTSEG_MAPPING_BACKWARD_INDEX_NODES_EXT; - string const bitsName = dir + countryName + FTSEG_MAPPING_BACKWARD_INDEX_BITS_EXT; uint64_t size; - if (!GetPlatform().GetFileSizeByFullPath(nodesName, size) || !GetPlatform().GetFileSizeByFullPath(bitsName, size)) + if (!GetPlatform().GetFileSizeByFullPath(nodesFileName, size) || !GetPlatform().GetFileSizeByFullPath(bitsFileName, size)) return false; - m_pMappedNodes.reset(new MmapReader(nodesName)); - m_pMappedBits.reset(new MmapReader(bitsName)); + m_pMappedNodes.reset(new MmapReader(nodesFileName)); + m_pMappedBits.reset(new MmapReader(bitsFileName)); succinct::mapper::map(m_nodeIds, reinterpret_cast(m_pMappedNodes->Data())); succinct::mapper::map(m_rankIndex, reinterpret_cast(m_pMappedBits->Data())); @@ -359,11 +354,16 @@ void OsrmFtSegBackwardIndex::Construct(const OsrmFtSegMapping & mapping, const u { Clear(); // Calculate data file pathes + Platform const & p = GetPlatform(); string const routingName = routingFile.GetName(); - string const name(routingName, routingName.rfind(my::GetNativeSeparator())+1, routingName.find(DATA_FILE_EXTENSION) -routingName.rfind(my::GetNativeSeparator())-1); - m_table = feature::FeaturesOffsetsTable::CreateIfNotExistsAndLoad(name); + string const name(routingName, routingName.rfind(my::GetNativeSeparator()) + 1, + routingName.find(DATA_FILE_EXTENSION) - routingName.rfind(my::GetNativeSeparator()) - 1); + string const offsetsIndexName = p.GetIndexFileName(name, FEATURES_OFFSETS_TABLE_FILE_EXT); + string const bitsFileName = p.GetIndexFileName(name, FTSEG_MAPPING_BACKWARD_INDEX_BITS_EXT); + string const nodesFileName = p.GetIndexFileName(name, FTSEG_MAPPING_BACKWARD_INDEX_NODES_EXT); + m_table = feature::FeaturesOffsetsTable::CreateIfNotExistsAndLoad(offsetsIndexName, FilesContainerR(name + DATA_FILE_EXTENSION)); - if (Load(name)) + if (Load(bitsFileName, nodesFileName)) return; // Generate temporary index to speedup processing @@ -399,7 +399,7 @@ void OsrmFtSegBackwardIndex::Construct(const OsrmFtSegMapping & mapping, const u succinct::rs_bit_vector(inIndex).swap(m_rankIndex); LOG(LINFO, ("Writing section to data file", routingName)); - Save(name); + Save(bitsFileName, nodesFileName); } uint32_t OsrmFtSegBackwardIndex::GetNodeIdByFid(const uint32_t fid) const diff --git a/routing/osrm2feature_map.hpp b/routing/osrm2feature_map.hpp index 3e7889c8ad..8f0275195d 100644 --- a/routing/osrm2feature_map.hpp +++ b/routing/osrm2feature_map.hpp @@ -107,9 +107,9 @@ class OsrmFtSegBackwardIndex T().swap(t); } - void Save(string const & countryName); + void Save(string const & nodesFileName, string const & bitsFileName); - bool Load(string const & container); + bool Load(string const & nodesFileName, string const & bitsFileName); public: void Construct(OsrmFtSegMapping const & mapping, uint32_t const maxNodeId, FilesMappingContainer & routingFile);