From 45faef6be3270c5d7a4fce25d04b06a5308ec8cc Mon Sep 17 00:00:00 2001 From: Lev Dragunov Date: Thu, 16 Jul 2015 13:07:19 +0300 Subject: [PATCH] Routing indexes new file API support. --- indexer/features_offsets_table.cpp | 4 ++- indexer/features_offsets_table.hpp | 6 ++-- .../features_offsets_table_test.cpp | 11 +++++-- routing/osrm2feature_map.cpp | 33 +++++++++---------- routing/osrm2feature_map.hpp | 5 +-- routing/routing_mapping.cpp | 2 +- routing/routing_tests/osrm_router_test.cpp | 15 +++++++-- 7 files changed, 46 insertions(+), 30 deletions(-) diff --git a/indexer/features_offsets_table.cpp b/indexer/features_offsets_table.cpp index e18824136d..a519a3baf6 100644 --- a/indexer/features_offsets_table.cpp +++ b/indexer/features_offsets_table.cpp @@ -56,12 +56,14 @@ namespace feature // static unique_ptr FeaturesOffsetsTable::CreateIfNotExistsAndLoad( - string const & fileName, FilesContainerR const & mwmFileContainer) + string const & fileName, platform::LocalCountryFile const & localFile) { uint64_t size; if (GetPlatform().GetFileSizeByFullPath(fileName,size)) return Load(fileName); + FilesContainerR mwmFileContainer(localFile.GetPath(TMapOptions::EMap)); + LOG(LINFO, ("Features offsets table is absent! Creating a new one.")); if (!mwmFileContainer.IsExist(HEADER_FILE_TAG)) diff --git a/indexer/features_offsets_table.hpp b/indexer/features_offsets_table.hpp index 25017a463f..72c28fcbb5 100644 --- a/indexer/features_offsets_table.hpp +++ b/indexer/features_offsets_table.hpp @@ -1,8 +1,8 @@ #pragma once -#include "coding/file_container.hpp" #include "coding/mmap_reader.hpp" #include "defines.hpp" +#include "platform/local_country_file.hpp" #include "std/cstdint.hpp" #include "std/unique_ptr.hpp" #include "std/vector.hpp" @@ -65,10 +65,10 @@ namespace feature /// \warning May take a lot of time if there is no precomputed section /// /// \param fileName a full path of the file to load or store data - /// \param mwmFileContainer mwm container to read features data (if we need to construct them) + /// \param localFile Representation of the map files with features data ( uses only if we need to construct them) /// \return a pointer to an instance of FeaturesOffsetsTable or nullptr /// when it's not possible to create FeaturesOffsetsTable. - static unique_ptr CreateIfNotExistsAndLoad(string const & fileName, FilesContainerR const & mwmFileContainer); + static unique_ptr CreateIfNotExistsAndLoad(string const & fileName, platform::LocalCountryFile const & localFile); FeaturesOffsetsTable(FeaturesOffsetsTable const &) = delete; FeaturesOffsetsTable const & operator=(FeaturesOffsetsTable const &) = delete; diff --git a/indexer/indexer_tests/features_offsets_table_test.cpp b/indexer/indexer_tests/features_offsets_table_test.cpp index 07e04b9b0e..5fe762717a 100644 --- a/indexer/indexer_tests/features_offsets_table_test.cpp +++ b/indexer/indexer_tests/features_offsets_table_test.cpp @@ -5,12 +5,16 @@ #include "indexer/features_vector.hpp" #include "platform/platform.hpp" +#include "platform/local_country_file_utils.hpp" + #include "coding/file_container.hpp" #include "base/scope_guard.hpp" #include "std/bind.hpp" #include "std/string.hpp" #include "defines.hpp" +using platform::CountryIndexes; + namespace feature { UNIT_TEST(FeaturesOffsetsTable_Empty) @@ -60,10 +64,13 @@ namespace feature { string const testFileName = "minsk-pass"; Platform & p = GetPlatform(); + platform::CountryFile country(testFileName); + platform::LocalCountryFile localFile(GetPlatform().WritableDir(), country, 0 /* version */); + localFile.SyncWithDisk(); FilesContainerR baseContainer(p.GetReader(testFileName + DATA_FILE_EXTENSION)); - const string indexFile = p.GetIndexFileName(testFileName, FEATURES_OFFSETS_TABLE_FILE_EXT); + const string indexFile = CountryIndexes::GetPath(localFile, CountryIndexes::Index::Offsets); FileWriter::DeleteFileX(indexFile); - unique_ptr table(FeaturesOffsetsTable::CreateIfNotExistsAndLoad(indexFile, baseContainer)); + unique_ptr table(FeaturesOffsetsTable::CreateIfNotExistsAndLoad(indexFile, localFile)); MY_SCOPE_GUARD(deleteTestFileIndexGuard, bind(&FileWriter::DeleteFileX, cref(indexFile))); TEST(table.get(), ()); diff --git a/routing/osrm2feature_map.cpp b/routing/osrm2feature_map.cpp index 3e21270a0f..c405a76fdc 100644 --- a/routing/osrm2feature_map.cpp +++ b/routing/osrm2feature_map.cpp @@ -7,6 +7,8 @@ #include "coding/read_write_utils.hpp" #include "coding/varint.hpp" +#include "platform/local_country_file_utils.hpp" + #include "base/assert.hpp" #include "base/logging.hpp" #include "base/math.hpp" @@ -18,6 +20,8 @@ #include "3party/succinct/mapper.hpp" +using platform::CountryIndexes; + namespace { const routing::TNodesList kEmptyList; @@ -109,7 +113,7 @@ void OsrmFtSegMapping::Clear() m_handle.Unmap(); } -void OsrmFtSegMapping::Load(FilesMappingContainer & cont) +void OsrmFtSegMapping::Load(FilesMappingContainer & cont, platform::LocalCountryFile const & localFile) { Clear(); @@ -124,7 +128,8 @@ void OsrmFtSegMapping::Load(FilesMappingContainer & cont) } } - m_backwardIndex.Construct(*this, m_offsets.back().m_nodeId, cont); + CountryIndexes::PreparePlaceOnDisk(localFile); + m_backwardIndex.Construct(*this, m_offsets.back().m_nodeId, cont, localFile); } void OsrmFtSegMapping::Map(FilesMappingContainer & cont) @@ -371,25 +376,17 @@ bool OsrmFtSegBackwardIndex::Load(string const & nodesFileName, string const & b } void OsrmFtSegBackwardIndex::Construct(OsrmFtSegMapping & mapping, uint32_t maxNodeId, - FilesMappingContainer & routingFile) + FilesMappingContainer & routingFile, + platform::LocalCountryFile const & localFile) { Clear(); - // Calculate data file pathes - Platform const & p = GetPlatform(); + string const offsetsIndexName = CountryIndexes::GetPath(localFile, CountryIndexes::Index::Offsets); + string const bitsFileName = CountryIndexes::GetPath(localFile, CountryIndexes::Index::Bits); + string const nodesFileName = CountryIndexes::GetPath(localFile, CountryIndexes::Index::Nodes); - /// @todo Find more suitable way to get mwm file name. - string const routingName = routingFile.GetName(); - string const mwmName(routingName, 0, routingName.rfind(ROUTING_FILE_EXTENSION)); - size_t const sepIndex = routingName.rfind(my::GetNativeSeparator()); - string const name(routingName, sepIndex + 1, routingName.rfind(DATA_FILE_EXTENSION) - sepIndex - 1); - - string const offsetsIndexName = p.GetIndexFileName(name, FEATURES_OFFSETS_TABLE_FILE_EXT); - string const bitsFileName = p.GetIndexFileName(name, FTSEG_MAPPING_BACKWARD_INDEX_BITS_EXT); - string const nodesFileName = p.GetIndexFileName(name, FTSEG_MAPPING_BACKWARD_INDEX_NODES_EXT); - - m_table = feature::FeaturesOffsetsTable::CreateIfNotExistsAndLoad(offsetsIndexName, FilesContainerR(mwmName)); - CHECK(m_table.get(), ("Can't get FeaturesOffsetsTable for", mwmName)); + m_table = feature::FeaturesOffsetsTable::CreateIfNotExistsAndLoad(offsetsIndexName, localFile); + CHECK(m_table.get(), ("Can't get FeaturesOffsetsTable for", offsetsIndexName)); if (Load(bitsFileName, nodesFileName)) return; @@ -441,7 +438,7 @@ void OsrmFtSegBackwardIndex::Construct(OsrmFtSegMapping & mapping, uint32_t maxN // Pack and save index succinct::rs_bit_vector(inIndex).swap(m_rankIndex); - LOG(LINFO, ("Writing section to data file", routingName)); + LOG(LINFO, ("Writing additional indexes to data files", bitsFileName, nodesFileName)); Save(bitsFileName, nodesFileName); } diff --git a/routing/osrm2feature_map.hpp b/routing/osrm2feature_map.hpp index c3e23f8560..b75d0012c9 100644 --- a/routing/osrm2feature_map.hpp +++ b/routing/osrm2feature_map.hpp @@ -113,7 +113,8 @@ class OsrmFtSegBackwardIndex public: void Construct(OsrmFtSegMapping & mapping, uint32_t maxNodeId, - FilesMappingContainer & routingFile); + FilesMappingContainer & routingFile, + platform::LocalCountryFile const & localFile); TNodesList const & GetNodeIdByFid(uint32_t fid) const; @@ -126,7 +127,7 @@ public: typedef set FtSegSetT; void Clear(); - void Load(FilesMappingContainer & cont); + void Load(FilesMappingContainer & cont, platform::LocalCountryFile const & localFile); void Map(FilesMappingContainer & cont); void Unmap(); diff --git a/routing/routing_mapping.cpp b/routing/routing_mapping.cpp index 734f277678..bed5d60193 100644 --- a/routing/routing_mapping.cpp +++ b/routing/routing_mapping.cpp @@ -97,7 +97,7 @@ void RoutingMapping::Map() ++m_mapCounter; if (!m_segMapping.IsMapped()) { - m_segMapping.Load(m_container); + m_segMapping.Load(m_container, m_handle.GetInfo()->GetLocalFile()); m_segMapping.Map(m_container); } } diff --git a/routing/routing_tests/osrm_router_test.cpp b/routing/routing_tests/osrm_router_test.cpp index b5d6d342b8..a687e36320 100644 --- a/routing/routing_tests/osrm_router_test.cpp +++ b/routing/routing_tests/osrm_router_test.cpp @@ -7,7 +7,11 @@ #include "coding/file_writer.hpp" +#include "platform/country_file.hpp" +#include "platform/local_country_file.hpp" +#include "platform/local_country_file_utils.hpp" #include "platform/platform.hpp" +#include "platform/platform_tests/file_utils.hpp" #include "defines.hpp" @@ -18,6 +22,7 @@ #include "std/vector.hpp" using namespace routing; +using platform::CountryIndexes; namespace { @@ -50,9 +55,13 @@ void TestMapping(InputDataT const & data, NodeIdDataT const & nodeIds, RangeDataT const & ranges) { + + platform::CountryFile country("TestCountry"); + platform::LocalCountryFile localFile(GetPlatform().WritableDir(), country, 0 /* version */); + localFile.SyncWithDisk(); static char const ftSegsPath[] = "test1.tmp"; - string const featuresOffsetsTablePath = - GetPlatform().GetIndexFileName(ftSegsPath, FEATURES_OFFSETS_TABLE_FILE_EXT); + string const & featuresOffsetsTablePath = + CountryIndexes::GetPath(localFile, CountryIndexes::Index::Offsets); MY_SCOPE_GUARD(ftSegsFileDeleter, bind(FileWriter::DeleteFileX, ftSegsPath)); MY_SCOPE_GUARD(featuresOffsetsTableFileDeleter, bind(FileWriter::DeleteFileX, featuresOffsetsTablePath)); @@ -94,7 +103,7 @@ void TestMapping(InputDataT const & data, { FilesMappingContainer cont(ftSegsPath); OsrmFtSegMapping mapping; - mapping.Load(cont); + mapping.Load(cont, localFile); mapping.Map(cont); TestNodeId(mapping, nodeIds);