From c30ae8eab34ab5de5e7684400f5e9032fa8f3736 Mon Sep 17 00:00:00 2001 From: Lev Dragunov Date: Thu, 5 Mar 2015 19:14:29 +0300 Subject: [PATCH] different file index pathes --- android/jni/Android.mk | 7 +- .../jni/com/mapswithme/platform/Platform.cpp | 9 +++ defines.hpp | 8 ++- indexer/features_offsets_table.cpp | 51 ++++++++------ indexer/features_offsets_table.hpp | 28 +++++--- .../features_offsets_table_test.cpp | 24 +++---- map/active_maps_layout.cpp | 1 + map/framework.cpp | 14 ++++ map/framework.hpp | 2 + platform/platform.hpp | 2 + platform/platform_ios.mm | 10 +++ platform/platform_linux.cpp | 9 +++ platform/platform_mac.mm | 9 +++ platform/platform_tizen.cpp | 9 +++ platform/platform_win.cpp | 7 ++ routing/osrm2feature_map.cpp | 66 +++++++++++-------- routing/osrm2feature_map.hpp | 41 +++--------- 17 files changed, 189 insertions(+), 108 deletions(-) diff --git a/android/jni/Android.mk b/android/jni/Android.mk index 9d30d9bba0..95769e01c6 100644 --- a/android/jni/Android.mk +++ b/android/jni/Android.mk @@ -28,6 +28,11 @@ LOCAL_MODULE := protobuf LOCAL_SRC_FILES := $(MY_PREBUILT_LIBS_PATH)/libprotobuf.a include $(PREBUILT_STATIC_LIBRARY) +include $(CLEAR_VARS) +LOCAL_MODULE := succinct +LOCAL_SRC_FILES := $(MY_PREBUILT_LIBS_PATH)/libsuccinct.a +include $(PREBUILT_STATIC_LIBRARY) + include $(CLEAR_VARS) LOCAL_MODULE := tomcrypt LOCAL_SRC_FILES := $(MY_PREBUILT_LIBS_PATH)/libtomcrypt.a @@ -137,7 +142,7 @@ include $(CLEAR_VARS) LOCAL_CPP_FEATURES += exceptions rtti LOCAL_MODULE := mapswithme -LOCAL_STATIC_LIBRARIES := map gui routing search storage indexer graphics platform anim geometry coding base expat freetype fribidi zlib bzip2 jansson tomcrypt protobuf osrm stats_client +LOCAL_STATIC_LIBRARIES := map gui routing search storage indexer graphics platform anim geometry coding base expat freetype fribidi zlib bzip2 jansson tomcrypt protobuf osrm stats_client succinct LOCAL_CFLAGS := -ffunction-sections -fdata-sections -Wno-extern-c-compat ifneq ($(NDK_DEBUG),1) diff --git a/android/jni/com/mapswithme/platform/Platform.cpp b/android/jni/com/mapswithme/platform/Platform.cpp index 2117cdde8a..7aaa4b8b6b 100644 --- a/android/jni/com/mapswithme/platform/Platform.cpp +++ b/android/jni/com/mapswithme/platform/Platform.cpp @@ -11,6 +11,7 @@ #include "../../../../../std/algorithm.hpp" +#include string Platform::UniqueClientId() const { @@ -55,6 +56,14 @@ string Platform::UniqueClientId() const return res; } +string Platform::WritablePathForFileIndexes(string const & country_name) const +{ + string dir = WritableDir() + country_name + '/'; + if (!IsFileExistsByFullPath(dir)) + mkdir(dir.c_str(), 0755); + return dir; +} + void Platform::RunOnGuiThread(TFunctor const & fn) { android::Platform::RunOnGuiThreadImpl(fn); diff --git a/defines.hpp b/defines.hpp index 4a2000c7df..554df8e6cd 100644 --- a/defines.hpp +++ b/defines.hpp @@ -4,6 +4,7 @@ #define DATA_FILE_EXTENSION_TMP ".mwm.tmp" #define FONT_FILE_EXTENSION ".ttf" #define OSM2FEATURE_FILE_EXTENSION ".osm2ft" +#define EXTENSION_TMP ".tmp" #define DATA_FILE_TAG "dat" #define GEOMETRY_FILE_TAG "geom" @@ -14,7 +15,6 @@ #define VERSION_FILE_TAG "version" #define METADATA_FILE_TAG "meta" #define METADATA_INDEX_FILE_TAG "metaidx" -#define FEATURES_OFFSETS_TABLE_FILE_TAG "offsets" #define COMPRESSED_SEARCH_INDEX_FILE_TAG "csdx" #define ROUTING_MATRIX_FILE_TAG "mercedes" @@ -25,8 +25,10 @@ #define ROUTING_FTSEG_FILE_TAG "ftseg" #define ROUTING_NODEIND_TO_FTSEGIND_FILE_TAG "node2ftseg" -#define FTSEG_MAPPING_BACKWARD_INDEX_NODES_TAG "bftsegnodes" -#define FTSEG_MAPPING_BACKWARD_INDEX_BITS_TAG "bftsegbits" + +#define FTSEG_MAPPING_BACKWARD_INDEX_NODES_EXT ".bftsegnodes" +#define FTSEG_MAPPING_BACKWARD_INDEX_BITS_EXT ".bftsegbits" +#define FEATURES_OFFSETS_TABLE_FILE_EXT ".offsets" //Secret word to unlock experimental features in production builds #define ROUTING_SECRET_UNLOCKING_WORD "?ariadna" diff --git a/indexer/features_offsets_table.cpp b/indexer/features_offsets_table.cpp index c39a691834..96eda2fbb8 100644 --- a/indexer/features_offsets_table.cpp +++ b/indexer/features_offsets_table.cpp @@ -3,10 +3,12 @@ #include "../indexer/data_header.hpp" #include "../indexer/features_vector.hpp" #include "../coding/file_writer.hpp" +#include "../coding/internal/file_data.hpp" +#include "../platform/platform.hpp" #include "../base/assert.hpp" #include "../base/scope_guard.hpp" #include "../std/string.hpp" -#include "../defines.hpp" + namespace feature { @@ -21,10 +23,10 @@ namespace feature { } - FeaturesOffsetsTable::FeaturesOffsetsTable(FilesMappingContainer::Handle && handle) - : m_handle(move(handle)) + FeaturesOffsetsTable::FeaturesOffsetsTable(string const & fileName) { - succinct::mapper::map(m_table, m_handle.GetData()); + m_pSrc = unique_ptr(new MmapReader(fileName)); + succinct::mapper::map(m_table, reinterpret_cast(m_pSrc->Data())); } // static @@ -43,26 +45,29 @@ namespace feature } // static - unique_ptr FeaturesOffsetsTable::Load( - FilesMappingContainer const & container) + unique_ptr FeaturesOffsetsTable::Load(string const & countryName) { - FilesMappingContainer::Handle handle(container.Map(FEATURES_OFFSETS_TABLE_FILE_TAG)); - if (!handle.IsValid()) + string const fileName = GetIndexFileName(countryName); + uint64_t size; + if (!GetPlatform().GetFileSizeByFullPath(fileName, size)) return unique_ptr(); - return unique_ptr(new FeaturesOffsetsTable(std::move(handle))); + return unique_ptr(new FeaturesOffsetsTable(fileName)); } // static unique_ptr FeaturesOffsetsTable::CreateIfNotExistsAndLoad( - FilesMappingContainer const & container) + string const & countryName) { - if (container.IsExist(FEATURES_OFFSETS_TABLE_FILE_TAG)) - return Load(container); + string const fileName = GetIndexFileName(countryName); + uint64_t size; + if (GetPlatform().GetFileSizeByFullPath(fileName,size)) + return Load(countryName); - if (!container.IsExist(HEADER_FILE_TAG)) + string const mwmName = GetPlatform().WritablePathForFile(countryName + DATA_FILE_EXTENSION); + FilesContainerR cont(mwmName); + if (!cont.IsExist(HEADER_FILE_TAG)) return unique_ptr(); - FilesContainerR cont(container.GetName()); DataHeader header; header.Load(cont.GetReader(HEADER_FILE_TAG)); @@ -72,17 +77,16 @@ namespace feature builder.PushOffset(offset); }); unique_ptr table(Build(builder)); - FilesContainerW writeCont(container.GetName(), FileWriter::OP_WRITE_EXISTING); - table->Save(writeCont); + table->Save(countryName); return table; } - void FeaturesOffsetsTable::Save(FilesContainerW & container) + void FeaturesOffsetsTable::Save(string const & countryName) { - string const fileName = container.GetFileName() + "." FEATURES_OFFSETS_TABLE_FILE_TAG; - MY_SCOPE_GUARD(deleteFileGuard, bind(&FileWriter::DeleteFileX, cref(fileName))); - succinct::mapper::freeze(m_table, fileName.c_str()); - container.Write(fileName, FEATURES_OFFSETS_TABLE_FILE_TAG); + string const fileName = GetIndexFileName(countryName); + string const fileNameTmp = fileName + EXTENSION_TMP; + succinct::mapper::freeze(m_table, fileNameTmp.c_str()); + my::RenameFileX(fileNameTmp, fileName); } uint64_t FeaturesOffsetsTable::GetFeatureOffset(size_t index) const @@ -115,4 +119,9 @@ namespace feature } + string FeaturesOffsetsTable::GetIndexFileName(string const & countryName) + { + return GetPlatform().WritablePathForFileIndexes(countryName) + countryName + FEATURES_OFFSETS_TABLE_FILE_EXT; + } + } // namespace feature diff --git a/indexer/features_offsets_table.hpp b/indexer/features_offsets_table.hpp index ceff65f7e0..eb0537c6ba 100644 --- a/indexer/features_offsets_table.hpp +++ b/indexer/features_offsets_table.hpp @@ -1,6 +1,8 @@ #pragma once #include "../coding/file_container.hpp" +#include "../coding/mmap_reader.hpp" +#include "../defines.hpp" #include "../std/stdint.hpp" #include "../std/unique_ptr.hpp" #include "../std/vector.hpp" @@ -51,11 +53,10 @@ namespace feature /// mapped to the memory and used by internal structures of /// FeaturesOffsetsTable. /// - /// \param container a container with a section devoted to - /// FeaturesOffsetsTable + /// \param countryName a countryName to save index file to /// \return a pointer to an instance of FeaturesOffsetsTable or nullptr /// when it's not possible to load FeaturesOffsetsTable. - static unique_ptr Load(FilesMappingContainer const & container); + static unique_ptr Load(string const & countryName); /// Loads FeaturesOffsetsTable from FilesMappingContainer. Note /// that some part of a file referenced by container will be @@ -66,19 +67,18 @@ namespace feature /// /// \warning May take a lot of time if there is no precomputed section /// - /// \param container a container with a section devoted to - /// FeaturesOffsetsTable + /// \param countryName a country to create index to /// \return a pointer to an instance of FeaturesOffsetsTable or nullptr /// when it's not possible to create FeaturesOffsetsTable. - static unique_ptr CreateIfNotExistsAndLoad(FilesMappingContainer const & container); + static unique_ptr CreateIfNotExistsAndLoad(string const & countryName); FeaturesOffsetsTable(FeaturesOffsetsTable const &) = delete; FeaturesOffsetsTable const & operator=(FeaturesOffsetsTable const &) = delete; /// Serializes current instance to a section in container. /// - /// \param container a container current instance will be serialized to - void Save(FilesContainerW & container); + /// \param countryName a name of the country to create data + void Save(string const & countryName); /// \param index index of a feature /// \return offset a feature @@ -102,13 +102,21 @@ namespace feature return succinct::mapper::size_of(m_table); } + /// Delete temporary index file + static void CleanIndexFiles(string const & countryName) + { + FileWriter::DeleteFileX(GetIndexFileName(countryName)); + } + private: + static string GetIndexFileName(string const & countryName); + FeaturesOffsetsTable(succinct::elias_fano::elias_fano_builder & builder); - FeaturesOffsetsTable(FilesMappingContainer::Handle && handle); + FeaturesOffsetsTable(string const &); succinct::elias_fano m_table; - FilesMappingContainer::Handle m_handle; + unique_ptr m_pSrc; }; } // namespace feature diff --git a/indexer/indexer_tests/features_offsets_table_test.cpp b/indexer/indexer_tests/features_offsets_table_test.cpp index fc321bb12d..50e34e27a0 100644 --- a/indexer/indexer_tests/features_offsets_table_test.cpp +++ b/indexer/indexer_tests/features_offsets_table_test.cpp @@ -58,12 +58,11 @@ namespace feature UNIT_TEST(FeaturesOffsetsTable_CreateIfNotExistsAndLoad) { + string const testFileName = "minsk-pass"; Platform & p = GetPlatform(); - FilesContainerR baseContainer(p.GetReader("minsk-pass" DATA_FILE_EXTENSION)); - if (baseContainer.IsExist(FEATURES_OFFSETS_TABLE_FILE_TAG)) - FilesContainerW(baseContainer.GetFileName(), FileWriter::OP_WRITE_EXISTING).DeleteSection(FEATURES_OFFSETS_TABLE_FILE_TAG); - FilesMappingContainer mappingContainer(baseContainer.GetFileName()); - unique_ptr table(FeaturesOffsetsTable::CreateIfNotExistsAndLoad(mappingContainer)); + FilesContainerR baseContainer(p.GetReader(testFileName + DATA_FILE_EXTENSION)); + FeaturesOffsetsTable::CleanIndexFiles(testFileName); + unique_ptr table(FeaturesOffsetsTable::CreateIfNotExistsAndLoad(testFileName)); TEST(table.get(), ()); feature::DataHeader header; @@ -77,13 +76,13 @@ namespace feature TEST_EQUAL(builderSize, table->size(), ()); table = unique_ptr(); - FilesMappingContainer newReadContainer(baseContainer.GetFileName()); - table = unique_ptr(FeaturesOffsetsTable::Load(newReadContainer)); + table = unique_ptr(FeaturesOffsetsTable::Load(testFileName)); TEST_EQUAL(builderSize, table->size(), ()); } UNIT_TEST(FeaturesOffsetsTable_ReadWrite) { + string const testFileName = "test_file"; Platform & p = GetPlatform(); FilesContainerR baseContainer(p.GetReader("minsk-pass" DATA_FILE_EXTENSION)); @@ -100,7 +99,7 @@ namespace feature TEST(table.get(), ()); TEST_EQUAL(builder.size(), table->size(), ()); - string const testFile = p.WritablePathForFile("test_file" DATA_FILE_EXTENSION); + string const testFile = p.WritablePathForFile(testFileName + DATA_FILE_EXTENSION); MY_SCOPE_GUARD(deleteTestFileGuard, bind(&FileWriter::DeleteFileX, cref(testFile))); // Store table in a temporary data file. @@ -111,19 +110,16 @@ namespace feature // table section. baseContainer.ForEachTag([&baseContainer, &testContainer](string const & tag) { - if (tag != FEATURES_OFFSETS_TABLE_FILE_TAG) testContainer.Write(baseContainer.GetReader(tag), tag); }); - table->Save(testContainer); + table->Save("test_file"); testContainer.Finish(); } // Load table from the temporary data file. { - FilesMappingContainer testContainer(testFile); - MY_SCOPE_GUARD(testContainerGuard, bind(&FilesMappingContainer::Close, &testContainer)); - - unique_ptr loadedTable(FeaturesOffsetsTable::Load(testContainer)); + MY_SCOPE_GUARD(testTableGuard, [&testFileName](){FeaturesOffsetsTable::CleanIndexFiles(testFileName);}); + unique_ptr loadedTable(FeaturesOffsetsTable::Load(testFileName)); TEST(loadedTable.get(), ()); TEST_EQUAL(table->size(), loadedTable->size(), ()); diff --git a/map/active_maps_layout.cpp b/map/active_maps_layout.cpp index f1e271fecc..58882fb194 100644 --- a/map/active_maps_layout.cpp +++ b/map/active_maps_layout.cpp @@ -355,6 +355,7 @@ void ActiveMapsLayout::StatusChangedCallback(TIndex const & index) if (newStatus == TStatus::EOnDisk) { + m_framework.DeleteCountryIndexes(index); if (group != TGroup::EUpToDate) { // Here we handle diff --git a/map/framework.cpp b/map/framework.cpp index 9a450b416f..64dff2ab42 100644 --- a/map/framework.cpp +++ b/map/framework.cpp @@ -307,6 +307,20 @@ void Framework::DeleteCountry(TIndex const & index, TMapOptions opt) m_storage.NotifyStatusChanged(index); } + + DeleteCountryIndexes(index); +} + +void Framework::DeleteCountryIndexes(TIndex const & index) +{ + string const & file = m_storage.CountryByIndex(index).Name(); + //Remove all indexes + m_routingSession.Reset(); + Platform::FilesList files; + Platform const & pl = GetPlatform(); + pl.GetFilesByRegExp(pl.WritablePathForFileIndexes(file), "*", files); + for (auto const & file : files) + my::DeleteFileX(file); } void Framework::DownloadCountry(TIndex const & index, TMapOptions opt) diff --git a/map/framework.hpp b/map/framework.hpp index c1c0d256bb..4040a43b46 100644 --- a/map/framework.hpp +++ b/map/framework.hpp @@ -170,6 +170,8 @@ public: void DeleteCountry(storage::TIndex const & index, storage::TMapOptions opt); /// options - flags that signal about parts of map that must be downloaded void DownloadCountry(storage::TIndex const & index, storage::TMapOptions opt); + /// Deletes user calculated indexes on country updates + void DeleteCountryIndexes(storage::TIndex const & index); storage::TStatus GetCountryStatus(storage::TIndex const & index) const; string GetCountryName(storage::TIndex const & index) const; diff --git a/platform/platform.hpp b/platform/platform.hpp index e1d26d430b..6605e7b852 100644 --- a/platform/platform.hpp +++ b/platform/platform.hpp @@ -60,6 +60,8 @@ public: string WritableDir() const { return m_writableDir; } /// @return full path to file in user's writable directory string WritablePathForFile(string const & file) const { return WritableDir() + file; } + /// @return full path to indexes directory for country file. Creates directory if it's not exists. + string WritablePathForFileIndexes(string const & country_name) const; /// @return resource dir (on some platforms it's differ from Writable dir) string ResourcesDir() const { return m_resourcesDir; } diff --git a/platform/platform_ios.mm b/platform/platform_ios.mm index c83e10c29c..302013ab53 100644 --- a/platform/platform_ios.mm +++ b/platform/platform_ios.mm @@ -3,6 +3,7 @@ #include "constants.hpp" #include "../coding/file_reader.hpp" +#include "../coding/file_name_utils.hpp" #include #include @@ -49,6 +50,15 @@ Platform::Platform() [pool release]; } + +string Platform::WritablePathForFileIndexes(string const & country_name) const +{ + string dir = m_writableDir + country_name.c_str() + '/'; + if (!IsFileExistsByFullPath(dir)) + ::mkdir(dir.c_str(), 0755); + return dir; +} + void Platform::GetFilesByRegExp(string const & directory, string const & regexp, FilesList & res) { pl::EnumerateFilesByRegExp(directory, regexp, res); diff --git a/platform/platform_linux.cpp b/platform/platform_linux.cpp index f99bdb027f..52d054d4c8 100644 --- a/platform/platform_linux.cpp +++ b/platform/platform_linux.cpp @@ -2,6 +2,7 @@ #include "../base/logging.hpp" #include "../coding/file_reader.hpp" +#include "../coding/file_name_utils.hpp" #include #include @@ -90,6 +91,14 @@ Platform::Platform() LOG(LDEBUG, ("Client ID:", UniqueClientId())); } +string Platform::WritablePathForFileIndexes(string const & country_name) const +{ + string dir = WritableDir() + country_name + my::GetNativeSeparator(); + if (!IsFileExistsByFullPath(dir)) + mkdir(dir.c_str(), 0755); + return dir; +} + int Platform::CpuCores() const { const long numCPU = sysconf(_SC_NPROCESSORS_ONLN); diff --git a/platform/platform_mac.mm b/platform/platform_mac.mm index fcbdc6d081..cd5eea67c7 100644 --- a/platform/platform_mac.mm +++ b/platform/platform_mac.mm @@ -78,6 +78,15 @@ Platform::Platform() [pool release]; } +string Platform::WritablePathForFileIndexes(string const & country_name) const +{ + + string dir = m_writableDir + country_name.c_str() + '/'; + if (!IsFileExistsByFullPath(dir)) + ::mkdir(dir.c_str(), 0755); + return dir; +} + int Platform::CpuCores() const { int mib[2], numCPU = 0; diff --git a/platform/platform_tizen.cpp b/platform/platform_tizen.cpp index 56bd4aaa93..47a8f01b60 100644 --- a/platform/platform_tizen.cpp +++ b/platform/platform_tizen.cpp @@ -10,6 +10,7 @@ #include #include "../coding/file_reader.hpp" +#include "../coding/file_name_utils.hpp" #include "../base/logging.hpp" @@ -43,6 +44,14 @@ Platform::Platform() m_flags[HAS_ROUTING] = true; } +string Platform::WritablePathForFileIndexes(string const & country_name) const +{ + string dir = WritableDir() + country_name + my::GetNativeSeparator(); + if (!IsFileExistsByFullPath(dir)) + Tizen::Io::Directory::Create(dir.c_str(), true); + return dir; +} + int Platform::CpuCores() const { /// @todo diff --git a/platform/platform_win.cpp b/platform/platform_win.cpp index 271a5f1a07..ddddc4ebfa 100644 --- a/platform/platform_win.cpp +++ b/platform/platform_win.cpp @@ -4,6 +4,7 @@ #include "../base/logging.hpp" #include "../coding/file_writer.hpp" +#include "../coding/file_name_utils.hpp" #include "../std/windows.hpp" #include "../std/bind.hpp" @@ -86,6 +87,12 @@ Platform::Platform() LOG(LDEBUG, ("Settings Directory:", m_settingsDir)); } +string Platform::WritablePathForFileIndexes(string const & country_name) const +{ + string dir = WritableDir() + country_name + my::GetNativeSeparator(); + return dir; +} + bool Platform::IsFileExistsByFullPath(string const & filePath) { return ::GetFileAttributesA(filePath.c_str()) != INVALID_FILE_ATTRIBUTES; diff --git a/routing/osrm2feature_map.cpp b/routing/osrm2feature_map.cpp index 924cd20d75..865a6d4361 100644 --- a/routing/osrm2feature_map.cpp +++ b/routing/osrm2feature_map.cpp @@ -3,6 +3,8 @@ #include "../defines.hpp" #include "../coding/varint.hpp" +#include "../coding/internal/file_data.hpp" +#include "../coding/file_name_utils.hpp" #include "../base/assert.hpp" #include "../base/logging.hpp" @@ -320,17 +322,49 @@ void OsrmFtSegMappingBuilder::Save(FilesContainerW & cont) const cont.Write(fName, ROUTING_FTSEG_FILE_TAG); } +void OsrmFtSegBackwardIndex::Save(string const & countryName) +{ + string dir = GetPlatform().WritablePathForFileIndexes(countryName); + { + string const nodesFileName = dir + countryName + FTSEG_MAPPING_BACKWARD_INDEX_NODES_EXT; + string const nodesFileNameTmp = nodesFileName + EXTENSION_TMP; + succinct::mapper::freeze(m_nodeIds, nodesFileNameTmp.c_str()); + my::RenameFileX(nodesFileNameTmp, nodesFileName); + } + { + string const bitsFileName = dir + countryName + FTSEG_MAPPING_BACKWARD_INDEX_BITS_EXT; + string const bitsFileNameTmp = bitsFileName + EXTENSION_TMP; + succinct::mapper::freeze(m_rankIndex, bitsFileNameTmp.c_str()); + my::RenameFileX(bitsFileNameTmp, bitsFileName); + } +} + +bool OsrmFtSegBackwardIndex::Load(string const & countryName) +{ + string dir = GetPlatform().WritablePathForFileIndexes(countryName); + string const nodesName = dir + countryName + FTSEG_MAPPING_BACKWARD_INDEX_NODES_EXT; + string const bitsName = dir + countryName + FTSEG_MAPPING_BACKWARD_INDEX_BITS_EXT; + uint64_t size; + if (! GetPlatform().GetFileSizeByFullPath(nodesName, size) || ! GetPlatform().GetFileSizeByFullPath(bitsName, size)) + return false; + m_pMappedNodes = unique_ptr(new MmapReader(nodesName)); + m_pMappedBits = unique_ptr(new MmapReader(bitsName)); + + succinct::mapper::map(m_nodeIds, reinterpret_cast(m_pMappedNodes->Data())); + succinct::mapper::map(m_rankIndex, reinterpret_cast(m_pMappedBits->Data())); + + return true; +} + void OsrmFtSegBackwardIndex::Construct(const OsrmFtSegMapping & mapping, const uint32_t maxNodeId, FilesMappingContainer & routingFile) { Clear(); // Calculate data file pathes string const routingName = routingFile.GetName(); - string const mwmName(routingName, 0, routingName.find(ROUTING_FILE_EXTENSION)); - FilesMappingContainer mwmContainer(mwmName); - mwmContainer.Open(mwmName); - m_table = feature::FeaturesOffsetsTable::CreateIfNotExistsAndLoad(mwmContainer); + string const name(routingName, routingName.rfind(my::GetNativeSeparator())+1, routingName.find(DATA_FILE_EXTENSION) -routingName.rfind(my::GetNativeSeparator())-1); + m_table = feature::FeaturesOffsetsTable::CreateIfNotExistsAndLoad(name); - if (Load(routingFile)) + if (Load(name)) return; // Generate temporary index to speedup processing @@ -365,28 +399,8 @@ void OsrmFtSegBackwardIndex::Construct(const OsrmFtSegMapping & mapping, const u succinct::elias_fano_compressed_list(nodeIds).swap(m_nodeIds); succinct::rs_bit_vector(inIndex).swap(m_rankIndex); LOG(LINFO, ("Writing section to data file", routingName)); - routingFile.Close(); - FilesContainerW writer(routingName); - Save(writer); - writer.Finish(); - routingFile.Open(routingName); - LOG(LINFO, ("Writing section to data file DONE", routingName)); - - // if (Load(parentCont)) - // return; - // size_t const count = segments.size(); - // m_index.resize(count); - // for (size_t i = 0; i < count; ++i) - // { - // OsrmMappingTypes::FtSeg s(segments[i]); - // m_index[i] = make_pair(s.m_fid, i); - // } - // sort(m_index.begin(), m_index.end(), [](IndexRecordTypeT const & a, IndexRecordTypeT const & b) - // { - // return a.first < b.first; - // }); - // Save(parentCont); + Save(name); } } diff --git a/routing/osrm2feature_map.hpp b/routing/osrm2feature_map.hpp index fb3139a8ec..33c66a0a65 100644 --- a/routing/osrm2feature_map.hpp +++ b/routing/osrm2feature_map.hpp @@ -1,9 +1,12 @@ #pragma once #include "../coding/file_container.hpp" +#include "../coding/mmap_reader.hpp" #include "../base/scope_guard.hpp" +#include "../platform/platform.hpp" + #include "../indexer/features_offsets_table.hpp" #include "../3party/succinct/rs_bit_vector.hpp" @@ -97,44 +100,16 @@ class OsrmFtSegBackwardIndex succinct::elias_fano_compressed_list m_nodeIds; unique_ptr m_table; - FilesMappingContainer::Handle m_handleNodes; - FilesMappingContainer::Handle m_handleBits; + unique_ptr m_pMappedNodes, m_pMappedBits; template void ClearContainer(T & t) { T().swap(t); } - void Save(FilesContainerW & container) - { - { - string const nodesFileName = container.GetFileName() + "." FTSEG_MAPPING_BACKWARD_INDEX_NODES_TAG; - MY_SCOPE_GUARD(deleteFileGuard, bind(&FileWriter::DeleteFileX, cref(nodesFileName))); - succinct::mapper::freeze(m_nodeIds, nodesFileName.c_str()); - container.Write(nodesFileName, FTSEG_MAPPING_BACKWARD_INDEX_NODES_TAG); - } - { - string const bitsFileName = container.GetFileName() + "." FTSEG_MAPPING_BACKWARD_INDEX_BITS_TAG; - MY_SCOPE_GUARD(deleteFileGuard, bind(&FileWriter::DeleteFileX, cref(bitsFileName))); - succinct::mapper::freeze(m_nodeIds, bitsFileName.c_str()); - container.Write(bitsFileName, FTSEG_MAPPING_BACKWARD_INDEX_BITS_TAG); - } - } + void Save(string const & countryName); - bool Load(FilesMappingContainer const & container) - { - if (!container.IsExist(FTSEG_MAPPING_BACKWARD_INDEX_NODES_TAG) || !container.IsExist(FTSEG_MAPPING_BACKWARD_INDEX_BITS_TAG)) - return false; - m_handleNodes.Assign(container.Map(FTSEG_MAPPING_BACKWARD_INDEX_NODES_TAG)); - ASSERT(m_handleNodes.IsValid(), ()); - succinct::mapper::map(m_nodeIds, m_handleNodes.GetData()); - - m_handleBits.Assign(container.Map(FTSEG_MAPPING_BACKWARD_INDEX_BITS_TAG)); - ASSERT(m_handleBits.IsValid(), ()); - succinct::mapper::map(m_rankIndex, m_handleBits.GetData()); - - return true; - } + bool Load(string const & container); public: void Construct(OsrmFtSegMapping const & mapping, uint32_t const maxNodeId, FilesMappingContainer & routingFile); @@ -156,8 +131,8 @@ public: ClearContainer(m_nodeIds); ClearContainer(m_rankIndex); m_table = unique_ptr(); - m_handleNodes.Unmap(); - m_handleBits.Unmap(); + m_pMappedBits = nullptr; + m_pMappedNodes = nullptr; } };