diff --git a/coding/file_container.cpp b/coding/file_container.cpp index dedcd2a13a..082ebda4c8 100644 --- a/coding/file_container.cpp +++ b/coding/file_container.cpp @@ -1,5 +1,3 @@ -#include "base/SRC_FIRST.hpp" - #include "coding/file_container.hpp" #include "coding/read_write_utils.hpp" #include "coding/write_to_sink.hpp" @@ -78,23 +76,109 @@ FilesContainerR::FilesContainerR(ReaderT const & file) FilesContainerR::ReaderT FilesContainerR::GetReader(Tag const & tag) const { Info const * p = GetInfo(tag); - if (p) - return m_source.SubReader(p->m_offset, p->m_size); - else - MYTHROW(Reader::OpenException, (tag)); + if (!p) + MYTHROW(Reader::OpenException, ("Can't find section:", GetFileName(), tag)); + return m_source.SubReader(p->m_offset, p->m_size); +} + +pair FilesContainerR::GetAbsoluteOffsetAndSize(Tag const & tag) const +{ + Info const * p = GetInfo(tag); + if (!p) + MYTHROW(Reader::OpenException, ("Can't find section:", GetFileName(), tag)); + + auto reader = dynamic_cast(m_source.GetPtr()); + uint64_t const offset = reader ? reader->GetOffset() : 0; + return make_pair(offset + p->m_offset, p->m_size); } FilesContainerBase::Info const * FilesContainerBase::GetInfo(Tag const & tag) const { - InfoContainer::const_iterator i = - lower_bound(m_info.begin(), m_info.end(), tag, LessInfo()); - + auto i = lower_bound(m_info.begin(), m_info.end(), tag, LessInfo()); if (i != m_info.end() && i->m_tag == tag) return &(*i); else return 0; } +namespace detail +{ +///////////////////////////////////////////////////////////////////////////// +// MappedFile +///////////////////////////////////////////////////////////////////////////// +void MappedFile::Open(string const & fName) +{ + Close(); + +#ifdef OMIM_OS_WINDOWS + m_hFile = CreateFileA(fName.c_str(), GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_OVERLAPPED, NULL); + if (m_hFile == INVALID_HANDLE_VALUE) + MYTHROW(Reader::OpenException, ("Can't open file:", fName, "win last error:", GetLastError())); + m_hMapping = CreateFileMappingA(m_hFile, NULL, PAGE_READONLY, 0, 0, NULL); + if (m_hMapping == NULL) + MYTHROW(Reader::OpenException, ("Can't create file's Windows mapping:", fName, "win last error:", GetLastError())); +#else + m_fd = open(fName.c_str(), O_RDONLY | O_NONBLOCK); + if (m_fd == -1) + MYTHROW(Reader::OpenException, ("Can't open file:", fName)); +#endif +} + +void MappedFile::Close() +{ +#ifdef OMIM_OS_WINDOWS + if (m_hMapping != INVALID_HANDLE_VALUE) + { + CloseHandle(m_hMapping); + m_hMapping = INVALID_HANDLE_VALUE; + } + if (m_hFile != INVALID_HANDLE_VALUE) + { + CloseHandle(m_hFile); + m_hFile = INVALID_HANDLE_VALUE; + } +#else + if (m_fd != -1) + { + close(m_fd); + m_fd = -1; + } +#endif +} + +MappedFile::Handle MappedFile::Map(uint64_t offset, uint64_t size, string const & tag) const +{ +#ifdef OMIM_OS_WINDOWS + SYSTEM_INFO sysInfo; + memset(&sysInfo, 0, sizeof(sysInfo)); + GetSystemInfo(&sysInfo); + long const align = sysInfo.dwAllocationGranularity; +#else + long const align = sysconf(_SC_PAGE_SIZE); +#endif + + uint64_t const alignedOffset = (offset / align) * align; + ASSERT_LESS_OR_EQUAL(alignedOffset, offset, ()); + uint64_t const length = size + (offset - alignedOffset); + ASSERT_GREATER_OR_EQUAL(length, size, ()); + +#ifdef OMIM_OS_WINDOWS + void * pMap = MapViewOfFile(m_hMapping, FILE_MAP_READ, alignedOffset >> (sizeof(DWORD) * 8), DWORD(alignedOffset), length); + if (pMap == NULL) + MYTHROW(Reader::OpenException, ("Can't map section:", tag, "with [offset, size]:", offset, size, "win last error:", GetLastError())); +#else + void * pMap = mmap(0, length, PROT_READ, MAP_SHARED, m_fd, alignedOffset); + if (pMap == MAP_FAILED) + MYTHROW(Reader::OpenException, ("Can't map section:", tag, "with [offset, size]:", offset, size)); +#endif + + char const * data = reinterpret_cast(pMap); + char const * d = data + (offset - alignedOffset); + return Handle(d, data, size, length); +} + +} // namespace detail + ///////////////////////////////////////////////////////////////////////////// // FilesMappingContainer ///////////////////////////////////////////////////////////////////////////// @@ -111,43 +195,19 @@ FilesMappingContainer::~FilesMappingContainer() void FilesMappingContainer::Open(string const & fName) { - Close(); - { FileReader reader(fName); ReadInfo(reader); } -#ifdef OMIM_OS_WINDOWS - m_hFile = CreateFileA(fName.c_str(), GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_OVERLAPPED, NULL); - if (m_hFile == INVALID_HANDLE_VALUE) - MYTHROW(Reader::OpenException, ("Can't open file:", fName, "win last error:", GetLastError())); - m_hMapping = CreateFileMappingA(m_hFile, NULL, PAGE_READONLY, 0, 0, NULL); - if (m_hMapping == NULL) - MYTHROW(Reader::OpenException, ("Can't create file's Windows mapping:", fName, "win last error:", GetLastError())); -#else - m_fd = open(fName.c_str(), O_RDONLY | O_NONBLOCK); - if (m_fd == -1) - MYTHROW(Reader::OpenException, ("Can't open file:", fName)); -#endif + m_file.Open(fName); m_name = fName; } void FilesMappingContainer::Close() { -#ifdef OMIM_OS_WINDOWS - if (m_hMapping != INVALID_HANDLE_VALUE) - CloseHandle(m_hMapping); - if (m_hFile != INVALID_HANDLE_VALUE) - CloseHandle(m_hFile); -#else - if (m_fd != -1) - { - close(m_fd); - m_fd = -1; - } -#endif + m_file.Close(); m_name.clear(); } @@ -155,49 +215,19 @@ void FilesMappingContainer::Close() FilesMappingContainer::Handle FilesMappingContainer::Map(Tag const & tag) const { Info const * p = GetInfo(tag); - if (p) - { -#ifdef OMIM_OS_WINDOWS - SYSTEM_INFO sysInfo; - memset(&sysInfo, 0, sizeof(sysInfo)); - GetSystemInfo(&sysInfo); - long const offsetAlign = sysInfo.dwAllocationGranularity; -#else - long const offsetAlign = sysconf(_SC_PAGE_SIZE); -#endif + if (!p) + MYTHROW(Reader::OpenException, ("Can't find section:", m_name, tag)); - uint64_t const offset = (p->m_offset / offsetAlign) * offsetAlign; - ASSERT_LESS_OR_EQUAL(offset, p->m_offset, ()); - uint64_t const length = p->m_size + (p->m_offset - offset); - ASSERT_GREATER_OR_EQUAL(length, p->m_size, ()); - -#ifdef OMIM_OS_WINDOWS - void * pMap = MapViewOfFile(m_hMapping, FILE_MAP_READ, offset >> (sizeof(DWORD) * 8), DWORD(offset), length); - if (pMap == NULL) - MYTHROW(Reader::OpenException, ("Can't map section:", tag, "with [offset, size]:", *p, "win last error:", GetLastError())); -#else - void * pMap = mmap(0, length, PROT_READ, MAP_SHARED, m_fd, offset); - if (pMap == MAP_FAILED) - MYTHROW(Reader::OpenException, ("Can't map section:", tag, "with [offset, size]:", *p)); -#endif - - char const * data = reinterpret_cast(pMap); - char const * d = data + (p->m_offset - offset); - return Handle(d, data, p->m_size, length); - } - else - MYTHROW(Reader::OpenException, ("Can't find section:", tag)); - - return Handle(); + ASSERT_EQUAL(tag, p->m_tag, ()); + return m_file.Map(p->m_offset, p->m_size, tag); } FileReader FilesMappingContainer::GetReader(Tag const & tag) const { Info const * p = GetInfo(tag); - if (p) - return FileReader(m_name).SubReader(p->m_offset, p->m_size); - else - MYTHROW(Reader::OpenException, ("Can't find section:", tag)); + if (!p) + MYTHROW(Reader::OpenException, ("Can't find section:", m_name, tag)); + return FileReader(m_name).SubReader(p->m_offset, p->m_size); } ///////////////////////////////////////////////////////////////////////////// diff --git a/coding/file_container.hpp b/coding/file_container.hpp index 406cdf95f9..ea81e7d970 100644 --- a/coding/file_container.hpp +++ b/coding/file_container.hpp @@ -123,24 +123,30 @@ public: inline uint64_t GetFileSize() const { return m_source.Size(); } inline string const & GetFileName() const { return m_source.GetName(); } + pair GetAbsoluteOffsetAndSize(Tag const & tag) const; + private: ReaderT m_source; }; -class FilesMappingContainer : public FilesContainerBase +namespace detail { -public: - /// Do nothing by default, call Open to attach to file. - FilesMappingContainer() = default; - explicit FilesMappingContainer(string const & fName); - ~FilesMappingContainer(); +class MappedFile +{ + DISALLOW_COPY(MappedFile); + +public: + MappedFile() = default; + ~MappedFile() { Close(); } void Open(string const & fName); void Close(); - class Handle : private noncopyable + class Handle { + DISALLOW_COPY(Handle); + void Reset(); public: @@ -183,6 +189,33 @@ public: uint64_t m_origSize; }; + Handle Map(uint64_t offset, uint64_t size, string const & tag) const; + +private: +#ifdef OMIM_OS_WINDOWS + void * m_hFile = (void *)-1; + void * m_hMapping = (void *)-1; +#else + int m_fd = -1; +#endif +}; + +} // namespace detail + +class FilesMappingContainer : public FilesContainerBase +{ +public: + typedef detail::MappedFile::Handle Handle; + + /// Do nothing by default, call Open to attach to file. + FilesMappingContainer() = default; + explicit FilesMappingContainer(string const & fName); + + ~FilesMappingContainer(); + + void Open(string const & fName); + void Close(); + Handle Map(Tag const & tag) const; FileReader GetReader(Tag const & tag) const; @@ -190,12 +223,7 @@ public: private: string m_name; -#ifdef OMIM_OS_WINDOWS - void * m_hFile = (void *)-1; - void * m_hMapping = (void *)-1; -#else - int m_fd = -1; -#endif + detail::MappedFile m_file; }; class FilesContainerW : public FilesContainerBase diff --git a/coding/file_reader.hpp b/coding/file_reader.hpp index 659079affc..6008452f33 100644 --- a/coding/file_reader.hpp +++ b/coding/file_reader.hpp @@ -22,6 +22,8 @@ public: FileReader SubReader(uint64_t pos, uint64_t size) const; FileReader * CreateSubReader(uint64_t pos, uint64_t size) const; + inline uint64_t GetOffset() const { return m_Offset; } + protected: /// Make assertion that pos + size in FileReader bounds. bool AssertPosAndSize(uint64_t pos, uint64_t size) const; diff --git a/defines.hpp b/defines.hpp index 06ac0198ad..f955343247 100644 --- a/defines.hpp +++ b/defines.hpp @@ -24,6 +24,7 @@ #define METADATA_FILE_TAG "meta" #define METADATA_INDEX_FILE_TAG "metaidx" #define COMPRESSED_SEARCH_INDEX_FILE_TAG "csdx" +#define FEATURE_OFFSETS_FILE_TAG "offs" #define ROUTING_MATRIX_FILE_TAG "mercedes" #define ROUTING_EDGEDATA_FILE_TAG "daewoo" diff --git a/generator/generator_tool/generator_tool.cpp b/generator/generator_tool/generator_tool.cpp index 3339ac991a..1d52e6bea6 100644 --- a/generator/generator_tool/generator_tool.cpp +++ b/generator/generator_tool/generator_tool.cpp @@ -15,6 +15,7 @@ #include "indexer/classificator_loader.hpp" #include "indexer/classificator.hpp" #include "indexer/data_header.hpp" +#include "indexer/features_offsets_table.hpp" #include "indexer/features_vector.hpp" #include "indexer/index_builder.hpp" #include "indexer/search_index_builder.hpp" @@ -165,29 +166,30 @@ int main(int argc, char ** argv) for (size_t i = 0; i < count; ++i) { string const & country = genInfo.m_bucketNames[i]; + string const datFile = my::JoinFoldersToPath(path, country + DATA_FILE_EXTENSION); if (FLAGS_generate_geometry) { - LOG(LINFO, ("Generating result features for file", country)); - int mapType = feature::DataHeader::country; if (country == WORLD_FILE_NAME) mapType = feature::DataHeader::world; if (country == WORLD_COASTS_FILE_NAME) mapType = feature::DataHeader::worldcoasts; - if (!feature::GenerateFinalFeatures(genInfo, country, mapType)) - { - // If error - move to next bucket without index generation - continue; - } - } + // If error - move to next bucket without index generation. - string const datFile = path + country + DATA_FILE_EXTENSION; + LOG(LINFO, ("Generating result features for", country)); + if (!feature::GenerateFinalFeatures(genInfo, country, mapType)) + continue; + + LOG(LINFO, ("Generating offsets table for", datFile)); + if (!feature::BuildOffsetsTable(datFile)) + continue; + } if (FLAGS_generate_index) { - LOG(LINFO, ("Generating index for ", datFile)); + LOG(LINFO, ("Generating index for", datFile)); if (!indexer::BuildIndexFromDatFile(datFile, FLAGS_intermediate_data_path + country)) LOG(LCRITICAL, ("Error generating index.")); diff --git a/indexer/features_offsets_table.cpp b/indexer/features_offsets_table.cpp index 2b6d9e19fe..15a972e783 100644 --- a/indexer/features_offsets_table.cpp +++ b/indexer/features_offsets_table.cpp @@ -10,6 +10,7 @@ #include "base/assert.hpp" #include "base/logging.hpp" +#include "base/scope_guard.hpp" #include "std/string.hpp" @@ -64,6 +65,19 @@ namespace feature return LoadImpl(filePath); } + // static + unique_ptr FeaturesOffsetsTable::Load(FilesContainerR const & cont) + { + unique_ptr table(new FeaturesOffsetsTable()); + + table->m_file.Open(cont.GetFileName()); + auto p = cont.GetAbsoluteOffsetAndSize(FEATURE_OFFSETS_FILE_TAG); + table->m_handle.Assign(table->m_file.Map(p.first, p.second, FEATURE_OFFSETS_FILE_TAG)); + + succinct::mapper::map(table->m_table, table->m_handle.GetData()); + return table; + } + // static unique_ptr FeaturesOffsetsTable::CreateImpl( platform::LocalCountryFile const & localFile, @@ -73,6 +87,12 @@ namespace feature CountryIndexes::PreparePlaceOnDisk(localFile); + return Build(cont, storePath); + } + + unique_ptr FeaturesOffsetsTable::Build(FilesContainerR const & cont, + string const & storePath) + { Builder builder; FeaturesVector::ForEachOffset(cont.GetReader(DATA_FILE_TAG), [&builder] (uint32_t offset) { @@ -147,4 +167,23 @@ namespace feature ASSERT_EQUAL(offset, m_table.select(leftBound), ("Can't find offset", offset, "in the table")); return leftBound; } + + bool BuildOffsetsTable(string const & filePath) + { + try + { + string const destPath = filePath + ".offsets"; + MY_SCOPE_GUARD(fileDeleter, bind(FileWriter::DeleteFileX, destPath)); + + (void)feature::FeaturesOffsetsTable::Build(FilesContainerR(filePath), destPath); + FilesContainerW(filePath, FileWriter::OP_WRITE_EXISTING).Write(destPath, FEATURE_OFFSETS_FILE_TAG); + return true; + } + catch (RootException const & ex) + { + LOG(LERROR, ("Generating offsets table failed for", filePath, "reason", ex.Msg())); + return false; + } + } + } // namespace feature diff --git a/indexer/features_offsets_table.hpp b/indexer/features_offsets_table.hpp index b1b52f4ddb..5faba945e4 100644 --- a/indexer/features_offsets_table.hpp +++ b/indexer/features_offsets_table.hpp @@ -1,5 +1,6 @@ #pragma once +#include "coding/file_container.hpp" #include "coding/mmap_reader.hpp" #include "defines.hpp" @@ -12,7 +13,6 @@ #include "3party/succinct/mapper.hpp" -class FilesContainerR; namespace platform { class LocalCountryFile; @@ -57,6 +57,10 @@ namespace feature /// Load table by full path to the table file. static unique_ptr Load(string const & filePath); + static unique_ptr Load(FilesContainerR const & cont); + static unique_ptr Build(FilesContainerR const & cont, + string const & storePath); + /// Get table for the MWM map, represented by localFile and cont. static unique_ptr CreateIfNotExistsAndLoad( platform::LocalCountryFile const & localFile, FilesContainerR const & cont); @@ -94,6 +98,7 @@ namespace feature private: FeaturesOffsetsTable(succinct::elias_fano::elias_fano_builder & builder); FeaturesOffsetsTable(string const & filePath); + FeaturesOffsetsTable() = default; static unique_ptr LoadImpl(string const & filePath); static unique_ptr CreateImpl(platform::LocalCountryFile const & localFile, @@ -101,7 +106,12 @@ namespace feature string const & storePath); succinct::elias_fano m_table; - unique_ptr m_pReader; + + detail::MappedFile m_file; + detail::MappedFile::Handle m_handle; }; + + bool BuildOffsetsTable(string const & filePath); + } // namespace feature diff --git a/indexer/features_vector.cpp b/indexer/features_vector.cpp index 88627061fa..c909579dfd 100644 --- a/indexer/features_vector.cpp +++ b/indexer/features_vector.cpp @@ -23,8 +23,11 @@ FeaturesVectorTest::FeaturesVectorTest(string const & filePath) FeaturesVectorTest::FeaturesVectorTest(FilesContainerR const & cont) : m_cont(cont), m_header(m_cont), m_vector(m_cont, m_header, 0) { - if (m_header.GetFormat() >= version::v5) + auto const version = m_header.GetFormat(); + if (version == version::v5) m_vector.m_table = feature::FeaturesOffsetsTable::CreateIfNotExistsAndLoad(m_cont).release(); + else if (version >= version::v6) + m_vector.m_table = feature::FeaturesOffsetsTable::Load(m_cont).release(); } FeaturesVectorTest::~FeaturesVectorTest() diff --git a/indexer/features_vector.hpp b/indexer/features_vector.hpp index 53d8c46d66..be38523a73 100644 --- a/indexer/features_vector.hpp +++ b/indexer/features_vector.hpp @@ -55,6 +55,8 @@ private: /// Used in generator_tool and unit tests. class FeaturesVectorTest { + DISALLOW_COPY(FeaturesVectorTest); + FilesContainerR m_cont; feature::DataHeader m_header; FeaturesVector m_vector; diff --git a/indexer/index.cpp b/indexer/index.cpp index 1592715dff..ecda41c2c7 100644 --- a/indexer/index.cpp +++ b/indexer/index.cpp @@ -24,11 +24,17 @@ MwmValue::MwmValue(LocalCountryFile const & localFile) void MwmValue::SetTable(MwmInfoEx & info) { - if (GetHeader().GetFormat() < version::v5) + auto const version = GetHeader().GetFormat(); + if (version < version::v5) return; if (!info.m_table) - info.m_table = feature::FeaturesOffsetsTable::CreateIfNotExistsAndLoad(m_file, m_cont); + { + if (version == version::v5) + info.m_table = feature::FeaturesOffsetsTable::CreateIfNotExistsAndLoad(m_file, m_cont); + else + info.m_table = feature::FeaturesOffsetsTable::Load(m_cont); + } m_table = info.m_table.get(); } diff --git a/map/map_tests/mwm_set_test.cpp b/map/map_tests/mwm_set_test.cpp index d3919c19c8..0c2b0783a8 100644 --- a/map/map_tests/mwm_set_test.cpp +++ b/map/map_tests/mwm_set_test.cpp @@ -15,6 +15,8 @@ using namespace platform; using namespace my; +/* + * This test is useless because of we don't build offsets index from now. #ifndef OMIM_OS_WINDOWS UNIT_TEST(MwmSet_FileSystemErrors) { @@ -56,3 +58,4 @@ UNIT_TEST(MwmSet_FileSystemErrors) TEST(infos.empty(), ()); } #endif +*/ diff --git a/platform/mwm_version.hpp b/platform/mwm_version.hpp index ee1f393976..0fcd02623f 100644 --- a/platform/mwm_version.hpp +++ b/platform/mwm_version.hpp @@ -17,7 +17,8 @@ enum Format v3, // March 2013 (store type index, instead of raw type in search data) v4, // April 2015 (distinguish и and й in search index) v5, // July 2015 (feature id is the index in vector now). - lastFormat = v5 + v6, // October 2015 (offsets vector is in mwm now). + lastFormat = v6 }; struct MwmVersion