map-per/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

[coding] Implement FileMappingContainer

Browse source
This commit is contained in:
Denis Koronchik 2014-09-16 11:05:39 +03:00 • committed by Alex Zolotarev
parent b353a8adae
commit d8f3008ecc
6 changed files with 218 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

87
coding/coding_tests/file_container_test.cpp
View file

@ -204,3 +204,90 @@ UNIT_TEST(FilesContainer_RewriteExisting)
FileWriter::DeleteFileX(fName);
}
UNIT_TEST(FilesMappingContainer_Smoke)
{
string const fName = "file_container.tmp";
char const * key[] = { "3", "2", "1" };
uint32_t const count = 1000000;
// fill container
{
FilesContainerW writer(fName);
for (size_t i = 0; i < ARRAY_SIZE(key); ++i)
{
FileWriter w = writer.GetWriter(key[i]);
for (uint32_t j = 0; j < count; ++j)
{
uint32_t v = j + i;
w.Write(&v, sizeof(v));
}
}
}
{
FilesMappingContainer reader(fName);
for (size_t i = 0; i < ARRAY_SIZE(key); ++i)
{
FilesMappingContainer::Handle h = reader.Map(key[i]);
uint32_t const * data = reinterpret_cast<uint32_t const *>(h.GetData());
for (uint32_t j = 0; j < count; ++j)
{
TEST_EQUAL(j + i, *data, ());
++data;
}
h.Unmap();
}
}
FileWriter::DeleteFileX(fName);
}
UNIT_TEST(FilesMappingContainer_PageSize)
{
string const fName = "file_container.tmp";
size_t const pageSize = sysconf(_SC_PAGE_SIZE);
LOG(LINFO, ("Page size:", pageSize));
char const * key[] = { "3", "2", "1" };
char const byte[] = { 'a', 'b', 'c', 'd', 'e', 'f', 'g' };
size_t count[] = { pageSize-1, pageSize, pageSize+1 };
size_t const sz = ARRAY_SIZE(key);
{
FilesContainerW writer(fName);
for (size_t i = 0; i < sz; ++i)
{
FileWriter w = writer.GetWriter(key[i]);
for (size_t j = 0; j < count[i]; ++j)
w.Write(&byte[j % ARRAY_SIZE(byte)], 1);
}
}
{
FilesMappingContainer reader(fName);
FilesMappingContainer::Handle handle[sz];
for (size_t i = 0; i < sz; ++i)
{
handle[i] = reader.Map(key[i]);
TEST_EQUAL(handle[i].GetSize(), count[i], ());
}
for (size_t i = 0; i < sz; ++i)
{
char const * data = handle[i].GetData();
for (size_t j = 0; j < count[i]; ++j)
TEST_EQUAL(*data++, byte[j % ARRAY_SIZE(byte)], ());
}
for (size_t i = 0; i < sz; ++i)
handle[i].Unmap();
}
FileWriter::DeleteFileX(fName);
}

87
coding/file_container.cpp
View file

@ -5,6 +5,17 @@
#include "write_to_sink.hpp"
#include "internal/file_data.hpp"
#ifndef OMIM_OS_WINDOWS
#include <unistd.h>
#include <sys/mman.h>
#include <sys/stat.h>
#ifdef OMIM_OS_ANDROID
#include <fcntl.h>
#else
#include <sys/fcntl.h>
#endif
#endif
template <class TSource> void Read(TSource & src, FilesContainerBase::Info & i)
{
@ -64,21 +75,83 @@ FilesContainerR::FilesContainerR(ReaderT const & file)
FilesContainerR::ReaderT FilesContainerR::GetReader(Tag const & tag) const
{
InfoContainer::const_iterator i =
lower_bound(m_info.begin(), m_info.end(), tag, LessInfo());
if (i != m_info.end() && i->m_tag == tag)
return m_source.SubReader(i->m_offset, i->m_size);
Info const * p = GetInfo(tag);
if (p)
return m_source.SubReader(p->m_offset, p->m_size);
else
MYTHROW(Reader::OpenException, (tag));
}
bool FilesContainerR::IsReaderExist(Tag const & tag) const
FilesContainerBase::Info const * FilesContainerBase::GetInfo(Tag const & tag) const
{
InfoContainer::const_iterator i =
lower_bound(m_info.begin(), m_info.end(), tag, LessInfo());
return (i != m_info.end() && i->m_tag == tag);
if (i != m_info.end() && i->m_tag == tag)
return &(*i);
else
return 0;
}
/////////////////////////////////////////////////////////////////////////////
// FilesMappingContainer
/////////////////////////////////////////////////////////////////////////////
FilesMappingContainer::FilesMappingContainer(string const & fName)
{
{
FileReader reader(fName, 10, 1);
ReadInfo(reader);
}
m_fd = open(fName.c_str(), O_RDONLY | O_NONBLOCK);
if (m_fd == -1)
MYTHROW(Reader::OpenException, ("Can't open file:", fName));
}
FilesMappingContainer::~FilesMappingContainer()
{
close(m_fd);
}
FilesMappingContainer::Handle FilesMappingContainer::Map(Tag const & tag) const
{
Info const * p = GetInfo(tag);
if (p)
{
long const offsetAlign = sysconf(_SC_PAGE_SIZE);
uint64_t const offset = (p->m_offset / offsetAlign) * offsetAlign;
ASSERT_LESS_OR_EQUAL(offset, p->m_offset, ());
uint64_t const length = p->m_size + (p->m_offset - offset);
ASSERT_GREATER_OR_EQUAL(length, p->m_size, ());
char const * data = reinterpret_cast<char const *>(mmap(0, length, PROT_READ, MAP_SHARED, m_fd, offset));
if (data == reinterpret_cast<char const *>(-1))
MYTHROW(Reader::OpenException, ("Can't map section:", tag, "with [offset, size]:", *p));
char const * d = data + (p->m_offset - offset);
return Handle(d, data, p->m_size, length);
}
else
MYTHROW(Reader::OpenException, ("Can't find section:", tag));
return Handle();
}
FilesMappingContainer::Handle::~Handle()
{
// CHECK(!IsValid(), ());
}
void FilesMappingContainer::Handle::Unmap()
{
ASSERT(IsValid(), ());
VERIFY(0 == munmap((void*)m_origBase, m_origSize), ());
m_origBase = m_base = 0;
m_origSize = m_size = 0;
}
/////////////////////////////////////////////////////////////////////////////

50
coding/file_container.hpp
View file

@ -23,6 +23,13 @@ public:
friend string DebugPrint(Info const & info);
};
Info const * GetInfo(Tag const & tag) const;
bool IsExist(Tag const & tag) const
{
return GetInfo(tag) != 0;
}
protected:
struct LessInfo
{
@ -74,7 +81,6 @@ protected:
}
};
typedef vector<Info> InfoContainer;
InfoContainer m_info;
@ -94,8 +100,6 @@ public:
ReaderT GetReader(Tag const & tag) const;
bool IsReaderExist(Tag const & tag) const;
template <typename F> void ForEachTag(F f) const
{
for (size_t i = 0; i < m_info.size(); ++i)
@ -109,6 +113,46 @@ private:
ReaderT m_source;
};
class FilesMappingContainer : public FilesContainerBase
{
public:
explicit FilesMappingContainer(string const & fName);
~FilesMappingContainer();
class Handle
{
public:
Handle()
: m_base(0), m_origBase(0), m_size(0), m_origSize(0)
{
}
Handle(char const * base, char const * alignBase, uint64_t size, uint64_t origSize)
: m_base(base), m_origBase(alignBase), m_size(size), m_origSize(origSize)
{
}
~Handle();
void Unmap();
bool IsValid() const { return (m_base != 0 && m_size > 0); }
uint64_t GetSize() const { return m_size; }
char const * GetData() const { return m_base; }
private:
char const * m_base;
char const * m_origBase;
uint64_t m_size;
uint64_t m_origSize;
};
Handle Map(Tag const & tag) const;
private:
int m_fd;
};
class FilesContainerW : public FilesContainerBase
{
public:

2
indexer/data_factory.cpp
View file

@ -17,7 +17,7 @@ void LoadMapHeader(FilesContainerR const & cont, FHeaderT & header)
{
ModelReaderPtr headerReader = cont.GetReader(HEADER_FILE_TAG);
if (!cont.IsReaderExist(VERSION_FILE_TAG))
if (!cont.IsExist(VERSION_FILE_TAG))
header.LoadVer1(headerReader);
else
{

2
indexer/search_index_builder.cpp
View file

@ -409,7 +409,7 @@ bool indexer::BuildSearchIndexFromDatFile(string const & fName, bool forceRebuil
{
FilesContainerR readCont(datFile);
if (!forceRebuild && readCont.IsReaderExist(SEARCH_INDEX_FILE_TAG))
if (!forceRebuild && readCont.IsExist(SEARCH_INDEX_FILE_TAG))
return true;
FileWriter writer(tmpFile);

4
search/search_query.cpp
View file

@ -1616,7 +1616,7 @@ void Query::SearchAddress(Results & res)
Index::MwmLock mwmLock(*m_pIndex, mwmId);
MwmValue * pMwm = mwmLock.GetValue();
if (pMwm &&
pMwm->m_cont.IsReaderExist(SEARCH_INDEX_FILE_TAG) &&
pMwm->m_cont.IsExist(SEARCH_INDEX_FILE_TAG) &&
pMwm->GetHeader().GetType() == FHeaderT::world)
{
impl::Locality city;
@ -2069,7 +2069,7 @@ void Query::SearchInMWM(Index::MwmLock const & mwmLock, Params const & params,
{
if (MwmValue * pMwm = mwmLock.GetValue())
{
if (pMwm->m_cont.IsReaderExist(SEARCH_INDEX_FILE_TAG))
if (pMwm->m_cont.IsExist(SEARCH_INDEX_FILE_TAG))
{
FHeaderT const & header = pMwm->GetHeader();