From d8f3008ecc7300e87262308e753c7bf3d850c9e9 Mon Sep 17 00:00:00 2001 From: Denis Koronchik Date: Tue, 16 Sep 2014 11:05:39 +0300 Subject: [PATCH] [coding] Implement FileMappingContainer --- coding/coding_tests/file_container_test.cpp | 87 +++++++++++++++++++++ coding/file_container.cpp | 87 +++++++++++++++++++-- coding/file_container.hpp | 50 +++++++++++- indexer/data_factory.cpp | 2 +- indexer/search_index_builder.cpp | 2 +- search/search_query.cpp | 4 +- 6 files changed, 218 insertions(+), 14 deletions(-) diff --git a/coding/coding_tests/file_container_test.cpp b/coding/coding_tests/file_container_test.cpp index 4757ed2a06..40360981b0 100644 --- a/coding/coding_tests/file_container_test.cpp +++ b/coding/coding_tests/file_container_test.cpp @@ -204,3 +204,90 @@ UNIT_TEST(FilesContainer_RewriteExisting) FileWriter::DeleteFileX(fName); } + +UNIT_TEST(FilesMappingContainer_Smoke) +{ + string const fName = "file_container.tmp"; + char const * key[] = { "3", "2", "1" }; + uint32_t const count = 1000000; + + // fill container + { + FilesContainerW writer(fName); + + for (size_t i = 0; i < ARRAY_SIZE(key); ++i) + { + FileWriter w = writer.GetWriter(key[i]); + for (uint32_t j = 0; j < count; ++j) + { + uint32_t v = j + i; + w.Write(&v, sizeof(v)); + } + } + } + + { + FilesMappingContainer reader(fName); + for (size_t i = 0; i < ARRAY_SIZE(key); ++i) + { + FilesMappingContainer::Handle h = reader.Map(key[i]); + uint32_t const * data = reinterpret_cast(h.GetData()); + for (uint32_t j = 0; j < count; ++j) + { + TEST_EQUAL(j + i, *data, ()); + ++data; + } + + h.Unmap(); + } + } + + FileWriter::DeleteFileX(fName); +} + +UNIT_TEST(FilesMappingContainer_PageSize) +{ + string const fName = "file_container.tmp"; + + size_t const pageSize = sysconf(_SC_PAGE_SIZE); + LOG(LINFO, ("Page size:", pageSize)); + + char const * key[] = { "3", "2", "1" }; + char const byte[] = { 'a', 'b', 'c', 'd', 'e', 'f', 'g' }; + size_t count[] = { pageSize-1, pageSize, pageSize+1 }; + size_t const sz = ARRAY_SIZE(key); + + { + FilesContainerW writer(fName); + + for (size_t i = 0; i < sz; ++i) + { + FileWriter w = writer.GetWriter(key[i]); + for (size_t j = 0; j < count[i]; ++j) + w.Write(&byte[j % ARRAY_SIZE(byte)], 1); + } + } + + { + FilesMappingContainer reader(fName); + FilesMappingContainer::Handle handle[sz]; + + for (size_t i = 0; i < sz; ++i) + { + handle[i] = reader.Map(key[i]); + TEST_EQUAL(handle[i].GetSize(), count[i], ()); + } + + for (size_t i = 0; i < sz; ++i) + { + char const * data = handle[i].GetData(); + for (size_t j = 0; j < count[i]; ++j) + TEST_EQUAL(*data++, byte[j % ARRAY_SIZE(byte)], ()); + } + + for (size_t i = 0; i < sz; ++i) + handle[i].Unmap(); + } + + FileWriter::DeleteFileX(fName); +} diff --git a/coding/file_container.cpp b/coding/file_container.cpp index 28716334e1..40c64f7e64 100644 --- a/coding/file_container.cpp +++ b/coding/file_container.cpp @@ -5,6 +5,17 @@ #include "write_to_sink.hpp" #include "internal/file_data.hpp" +#ifndef OMIM_OS_WINDOWS + #include + #include + #include + #ifdef OMIM_OS_ANDROID + #include + #else + #include + #endif +#endif + template void Read(TSource & src, FilesContainerBase::Info & i) { @@ -64,21 +75,83 @@ FilesContainerR::FilesContainerR(ReaderT const & file) FilesContainerR::ReaderT FilesContainerR::GetReader(Tag const & tag) const { - InfoContainer::const_iterator i = - lower_bound(m_info.begin(), m_info.end(), tag, LessInfo()); - - if (i != m_info.end() && i->m_tag == tag) - return m_source.SubReader(i->m_offset, i->m_size); + Info const * p = GetInfo(tag); + if (p) + return m_source.SubReader(p->m_offset, p->m_size); else MYTHROW(Reader::OpenException, (tag)); } -bool FilesContainerR::IsReaderExist(Tag const & tag) const +FilesContainerBase::Info const * FilesContainerBase::GetInfo(Tag const & tag) const { InfoContainer::const_iterator i = lower_bound(m_info.begin(), m_info.end(), tag, LessInfo()); - return (i != m_info.end() && i->m_tag == tag); + if (i != m_info.end() && i->m_tag == tag) + return &(*i); + else + return 0; +} + +///////////////////////////////////////////////////////////////////////////// +// FilesMappingContainer +///////////////////////////////////////////////////////////////////////////// + +FilesMappingContainer::FilesMappingContainer(string const & fName) +{ + { + FileReader reader(fName, 10, 1); + ReadInfo(reader); + } + + m_fd = open(fName.c_str(), O_RDONLY | O_NONBLOCK); + if (m_fd == -1) + MYTHROW(Reader::OpenException, ("Can't open file:", fName)); +} + +FilesMappingContainer::~FilesMappingContainer() +{ + close(m_fd); +} + +FilesMappingContainer::Handle FilesMappingContainer::Map(Tag const & tag) const +{ + Info const * p = GetInfo(tag); + if (p) + { + long const offsetAlign = sysconf(_SC_PAGE_SIZE); + + uint64_t const offset = (p->m_offset / offsetAlign) * offsetAlign; + ASSERT_LESS_OR_EQUAL(offset, p->m_offset, ()); + uint64_t const length = p->m_size + (p->m_offset - offset); + ASSERT_GREATER_OR_EQUAL(length, p->m_size, ()); + + char const * data = reinterpret_cast(mmap(0, length, PROT_READ, MAP_SHARED, m_fd, offset)); + + if (data == reinterpret_cast(-1)) + MYTHROW(Reader::OpenException, ("Can't map section:", tag, "with [offset, size]:", *p)); + + char const * d = data + (p->m_offset - offset); + return Handle(d, data, p->m_size, length); + } + else + MYTHROW(Reader::OpenException, ("Can't find section:", tag)); + + return Handle(); +} + +FilesMappingContainer::Handle::~Handle() +{ +// CHECK(!IsValid(), ()); +} + +void FilesMappingContainer::Handle::Unmap() +{ + ASSERT(IsValid(), ()); + VERIFY(0 == munmap((void*)m_origBase, m_origSize), ()); + + m_origBase = m_base = 0; + m_origSize = m_size = 0; } ///////////////////////////////////////////////////////////////////////////// diff --git a/coding/file_container.hpp b/coding/file_container.hpp index aa42a0f4a9..a9d8011098 100644 --- a/coding/file_container.hpp +++ b/coding/file_container.hpp @@ -23,6 +23,13 @@ public: friend string DebugPrint(Info const & info); }; + Info const * GetInfo(Tag const & tag) const; + + bool IsExist(Tag const & tag) const + { + return GetInfo(tag) != 0; + } + protected: struct LessInfo { @@ -74,7 +81,6 @@ protected: } }; - typedef vector InfoContainer; InfoContainer m_info; @@ -94,8 +100,6 @@ public: ReaderT GetReader(Tag const & tag) const; - bool IsReaderExist(Tag const & tag) const; - template void ForEachTag(F f) const { for (size_t i = 0; i < m_info.size(); ++i) @@ -109,6 +113,46 @@ private: ReaderT m_source; }; +class FilesMappingContainer : public FilesContainerBase +{ +public: + explicit FilesMappingContainer(string const & fName); + ~FilesMappingContainer(); + + class Handle + { + public: + Handle() + : m_base(0), m_origBase(0), m_size(0), m_origSize(0) + { + } + + Handle(char const * base, char const * alignBase, uint64_t size, uint64_t origSize) + : m_base(base), m_origBase(alignBase), m_size(size), m_origSize(origSize) + { + } + + ~Handle(); + + void Unmap(); + + bool IsValid() const { return (m_base != 0 && m_size > 0); } + uint64_t GetSize() const { return m_size; } + char const * GetData() const { return m_base; } + + private: + char const * m_base; + char const * m_origBase; + uint64_t m_size; + uint64_t m_origSize; + }; + + Handle Map(Tag const & tag) const; + +private: + int m_fd; +}; + class FilesContainerW : public FilesContainerBase { public: diff --git a/indexer/data_factory.cpp b/indexer/data_factory.cpp index 34011ad3ea..b1acc286cf 100644 --- a/indexer/data_factory.cpp +++ b/indexer/data_factory.cpp @@ -17,7 +17,7 @@ void LoadMapHeader(FilesContainerR const & cont, FHeaderT & header) { ModelReaderPtr headerReader = cont.GetReader(HEADER_FILE_TAG); - if (!cont.IsReaderExist(VERSION_FILE_TAG)) + if (!cont.IsExist(VERSION_FILE_TAG)) header.LoadVer1(headerReader); else { diff --git a/indexer/search_index_builder.cpp b/indexer/search_index_builder.cpp index 82f3817662..5b8373e9d8 100644 --- a/indexer/search_index_builder.cpp +++ b/indexer/search_index_builder.cpp @@ -409,7 +409,7 @@ bool indexer::BuildSearchIndexFromDatFile(string const & fName, bool forceRebuil { FilesContainerR readCont(datFile); - if (!forceRebuild && readCont.IsReaderExist(SEARCH_INDEX_FILE_TAG)) + if (!forceRebuild && readCont.IsExist(SEARCH_INDEX_FILE_TAG)) return true; FileWriter writer(tmpFile); diff --git a/search/search_query.cpp b/search/search_query.cpp index c6db240692..a467f3002f 100644 --- a/search/search_query.cpp +++ b/search/search_query.cpp @@ -1616,7 +1616,7 @@ void Query::SearchAddress(Results & res) Index::MwmLock mwmLock(*m_pIndex, mwmId); MwmValue * pMwm = mwmLock.GetValue(); if (pMwm && - pMwm->m_cont.IsReaderExist(SEARCH_INDEX_FILE_TAG) && + pMwm->m_cont.IsExist(SEARCH_INDEX_FILE_TAG) && pMwm->GetHeader().GetType() == FHeaderT::world) { impl::Locality city; @@ -2069,7 +2069,7 @@ void Query::SearchInMWM(Index::MwmLock const & mwmLock, Params const & params, { if (MwmValue * pMwm = mwmLock.GetValue()) { - if (pMwm->m_cont.IsReaderExist(SEARCH_INDEX_FILE_TAG)) + if (pMwm->m_cont.IsExist(SEARCH_INDEX_FILE_TAG)) { FHeaderT const & header = pMwm->GetHeader();