RedAuburn/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

Merge pull request #179 from vng/generator

Browse source 
[mwm] Put feature offsets table into MWM container.
This commit is contained in:
Ilya Zverev 2015-10-12 16:18:54 +03:00
commit 52c1b02959
12 changed files with 228 additions and 101 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

174
coding/file_container.cpp
View file

@ -1,5 +1,3 @@
#include "base/SRC_FIRST.hpp"
#include "coding/file_container.hpp"
#include "coding/read_write_utils.hpp"
#include "coding/write_to_sink.hpp"
@ -78,23 +76,109 @@ FilesContainerR::FilesContainerR(ReaderT const & file)
FilesContainerR::ReaderT FilesContainerR::GetReader(Tag const & tag) const
{
Info const * p = GetInfo(tag);
if (p)
return m_source.SubReader(p->m_offset, p->m_size);
else
MYTHROW(Reader::OpenException, (tag));
if (!p)
MYTHROW(Reader::OpenException, ("Can't find section:", GetFileName(), tag));
return m_source.SubReader(p->m_offset, p->m_size);
}
pair<uint64_t, uint64_t> FilesContainerR::GetAbsoluteOffsetAndSize(Tag const & tag) const
{
Info const * p = GetInfo(tag);
if (!p)
MYTHROW(Reader::OpenException, ("Can't find section:", GetFileName(), tag));
auto reader = dynamic_cast<FileReader const *>(m_source.GetPtr());
uint64_t const offset = reader ? reader->GetOffset() : 0;
return make_pair(offset + p->m_offset, p->m_size);
}
FilesContainerBase::Info const * FilesContainerBase::GetInfo(Tag const & tag) const
{
InfoContainer::const_iterator i =
lower_bound(m_info.begin(), m_info.end(), tag, LessInfo());
auto i = lower_bound(m_info.begin(), m_info.end(), tag, LessInfo());
if (i != m_info.end() && i->m_tag == tag)
return &(*i);
else
return 0;
}
namespace detail
{
/////////////////////////////////////////////////////////////////////////////
// MappedFile
/////////////////////////////////////////////////////////////////////////////
void MappedFile::Open(string const & fName)
{
Close();
#ifdef OMIM_OS_WINDOWS
m_hFile = CreateFileA(fName.c_str(), GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_OVERLAPPED, NULL);
if (m_hFile == INVALID_HANDLE_VALUE)
MYTHROW(Reader::OpenException, ("Can't open file:", fName, "win last error:", GetLastError()));
m_hMapping = CreateFileMappingA(m_hFile, NULL, PAGE_READONLY, 0, 0, NULL);
if (m_hMapping == NULL)
MYTHROW(Reader::OpenException, ("Can't create file's Windows mapping:", fName, "win last error:", GetLastError()));
#else
m_fd = open(fName.c_str(), O_RDONLY | O_NONBLOCK);
if (m_fd == -1)
MYTHROW(Reader::OpenException, ("Can't open file:", fName));
#endif
}
void MappedFile::Close()
{
#ifdef OMIM_OS_WINDOWS
if (m_hMapping != INVALID_HANDLE_VALUE)
{
CloseHandle(m_hMapping);
m_hMapping = INVALID_HANDLE_VALUE;
}
if (m_hFile != INVALID_HANDLE_VALUE)
{
CloseHandle(m_hFile);
m_hFile = INVALID_HANDLE_VALUE;
}
#else
if (m_fd != -1)
{
close(m_fd);
m_fd = -1;
}
#endif
}
MappedFile::Handle MappedFile::Map(uint64_t offset, uint64_t size, string const & tag) const
{
#ifdef OMIM_OS_WINDOWS
SYSTEM_INFO sysInfo;
memset(&sysInfo, 0, sizeof(sysInfo));
GetSystemInfo(&sysInfo);
long const align = sysInfo.dwAllocationGranularity;
#else
long const align = sysconf(_SC_PAGE_SIZE);
#endif
uint64_t const alignedOffset = (offset / align) * align;
ASSERT_LESS_OR_EQUAL(alignedOffset, offset, ());
uint64_t const length = size + (offset - alignedOffset);
ASSERT_GREATER_OR_EQUAL(length, size, ());
#ifdef OMIM_OS_WINDOWS
void * pMap = MapViewOfFile(m_hMapping, FILE_MAP_READ, alignedOffset >> (sizeof(DWORD) * 8), DWORD(alignedOffset), length);
if (pMap == NULL)
MYTHROW(Reader::OpenException, ("Can't map section:", tag, "with [offset, size]:", offset, size, "win last error:", GetLastError()));
#else
void * pMap = mmap(0, length, PROT_READ, MAP_SHARED, m_fd, alignedOffset);
if (pMap == MAP_FAILED)
MYTHROW(Reader::OpenException, ("Can't map section:", tag, "with [offset, size]:", offset, size));
#endif
char const * data = reinterpret_cast<char const *>(pMap);
char const * d = data + (offset - alignedOffset);
return Handle(d, data, size, length);
}
} // namespace detail
/////////////////////////////////////////////////////////////////////////////
// FilesMappingContainer
/////////////////////////////////////////////////////////////////////////////
@ -111,43 +195,19 @@ FilesMappingContainer::~FilesMappingContainer()
void FilesMappingContainer::Open(string const & fName)
{
Close();
{
FileReader reader(fName);
ReadInfo(reader);
}
#ifdef OMIM_OS_WINDOWS
m_hFile = CreateFileA(fName.c_str(), GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_OVERLAPPED, NULL);
if (m_hFile == INVALID_HANDLE_VALUE)
MYTHROW(Reader::OpenException, ("Can't open file:", fName, "win last error:", GetLastError()));
m_hMapping = CreateFileMappingA(m_hFile, NULL, PAGE_READONLY, 0, 0, NULL);
if (m_hMapping == NULL)
MYTHROW(Reader::OpenException, ("Can't create file's Windows mapping:", fName, "win last error:", GetLastError()));
#else
m_fd = open(fName.c_str(), O_RDONLY | O_NONBLOCK);
if (m_fd == -1)
MYTHROW(Reader::OpenException, ("Can't open file:", fName));
#endif
m_file.Open(fName);
m_name = fName;
}
void FilesMappingContainer::Close()
{
#ifdef OMIM_OS_WINDOWS
if (m_hMapping != INVALID_HANDLE_VALUE)
CloseHandle(m_hMapping);
if (m_hFile != INVALID_HANDLE_VALUE)
CloseHandle(m_hFile);
#else
if (m_fd != -1)
{
close(m_fd);
m_fd = -1;
}
#endif
m_file.Close();
m_name.clear();
}
@ -155,49 +215,19 @@ void FilesMappingContainer::Close()
FilesMappingContainer::Handle FilesMappingContainer::Map(Tag const & tag) const
{
Info const * p = GetInfo(tag);
if (p)
{
#ifdef OMIM_OS_WINDOWS
SYSTEM_INFO sysInfo;
memset(&sysInfo, 0, sizeof(sysInfo));
GetSystemInfo(&sysInfo);
long const offsetAlign = sysInfo.dwAllocationGranularity;
#else
long const offsetAlign = sysconf(_SC_PAGE_SIZE);
#endif
if (!p)
MYTHROW(Reader::OpenException, ("Can't find section:", m_name, tag));
uint64_t const offset = (p->m_offset / offsetAlign) * offsetAlign;
ASSERT_LESS_OR_EQUAL(offset, p->m_offset, ());
uint64_t const length = p->m_size + (p->m_offset - offset);
ASSERT_GREATER_OR_EQUAL(length, p->m_size, ());
#ifdef OMIM_OS_WINDOWS
void * pMap = MapViewOfFile(m_hMapping, FILE_MAP_READ, offset >> (sizeof(DWORD) * 8), DWORD(offset), length);
if (pMap == NULL)
MYTHROW(Reader::OpenException, ("Can't map section:", tag, "with [offset, size]:", *p, "win last error:", GetLastError()));
#else
void * pMap = mmap(0, length, PROT_READ, MAP_SHARED, m_fd, offset);
if (pMap == MAP_FAILED)
MYTHROW(Reader::OpenException, ("Can't map section:", tag, "with [offset, size]:", *p));
#endif
char const * data = reinterpret_cast<char const *>(pMap);
char const * d = data + (p->m_offset - offset);
return Handle(d, data, p->m_size, length);
}
else
MYTHROW(Reader::OpenException, ("Can't find section:", tag));
return Handle();
ASSERT_EQUAL(tag, p->m_tag, ());
return m_file.Map(p->m_offset, p->m_size, tag);
}
FileReader FilesMappingContainer::GetReader(Tag const & tag) const
{
Info const * p = GetInfo(tag);
if (p)
return FileReader(m_name).SubReader(p->m_offset, p->m_size);
else
MYTHROW(Reader::OpenException, ("Can't find section:", tag));
if (!p)
MYTHROW(Reader::OpenException, ("Can't find section:", m_name, tag));
return FileReader(m_name).SubReader(p->m_offset, p->m_size);
}
/////////////////////////////////////////////////////////////////////////////

54
coding/file_container.hpp
View file

@ -123,24 +123,30 @@ public:
inline uint64_t GetFileSize() const { return m_source.Size(); }
inline string const & GetFileName() const { return m_source.GetName(); }
pair<uint64_t, uint64_t> GetAbsoluteOffsetAndSize(Tag const & tag) const;
private:
ReaderT m_source;
};
class FilesMappingContainer : public FilesContainerBase
namespace detail
{
public:
/// Do nothing by default, call Open to attach to file.
FilesMappingContainer() = default;
explicit FilesMappingContainer(string const & fName);
~FilesMappingContainer();
class MappedFile
{
DISALLOW_COPY(MappedFile);
public:
MappedFile() = default;
~MappedFile() { Close(); }
void Open(string const & fName);
void Close();
class Handle : private noncopyable
class Handle
{
DISALLOW_COPY(Handle);
void Reset();
public:
@ -183,6 +189,33 @@ public:
uint64_t m_origSize;
};
Handle Map(uint64_t offset, uint64_t size, string const & tag) const;
private:
#ifdef OMIM_OS_WINDOWS
void * m_hFile = (void *)-1;
void * m_hMapping = (void *)-1;
#else
int m_fd = -1;
#endif
};
} // namespace detail
class FilesMappingContainer : public FilesContainerBase
{
public:
typedef detail::MappedFile::Handle Handle;
/// Do nothing by default, call Open to attach to file.
FilesMappingContainer() = default;
explicit FilesMappingContainer(string const & fName);
~FilesMappingContainer();
void Open(string const & fName);
void Close();
Handle Map(Tag const & tag) const;
FileReader GetReader(Tag const & tag) const;
@ -190,12 +223,7 @@ public:
private:
string m_name;
#ifdef OMIM_OS_WINDOWS
void * m_hFile = (void *)-1;
void * m_hMapping = (void *)-1;
#else
int m_fd = -1;
#endif
detail::MappedFile m_file;
};
class FilesContainerW : public FilesContainerBase

2
coding/file_reader.hpp
View file

@ -22,6 +22,8 @@ public:
FileReader SubReader(uint64_t pos, uint64_t size) const;
FileReader * CreateSubReader(uint64_t pos, uint64_t size) const;
inline uint64_t GetOffset() const { return m_Offset; }
protected:
/// Make assertion that pos + size in FileReader bounds.
bool AssertPosAndSize(uint64_t pos, uint64_t size) const;

1
defines.hpp
View file

@ -24,6 +24,7 @@
#define METADATA_FILE_TAG "meta"
#define METADATA_INDEX_FILE_TAG "metaidx"
#define COMPRESSED_SEARCH_INDEX_FILE_TAG "csdx"
#define FEATURE_OFFSETS_FILE_TAG "offs"
#define ROUTING_MATRIX_FILE_TAG "mercedes"
#define ROUTING_EDGEDATA_FILE_TAG "daewoo"

22
generator/generator_tool/generator_tool.cpp
View file

@ -15,6 +15,7 @@
#include "indexer/classificator_loader.hpp"
#include "indexer/classificator.hpp"
#include "indexer/data_header.hpp"
#include "indexer/features_offsets_table.hpp"
#include "indexer/features_vector.hpp"
#include "indexer/index_builder.hpp"
#include "indexer/search_index_builder.hpp"
@ -165,29 +166,30 @@ int main(int argc, char ** argv)
for (size_t i = 0; i < count; ++i)
{
string const & country = genInfo.m_bucketNames[i];
string const datFile = my::JoinFoldersToPath(path, country + DATA_FILE_EXTENSION);
if (FLAGS_generate_geometry)
{
LOG(LINFO, ("Generating result features for file", country));
int mapType = feature::DataHeader::country;
if (country == WORLD_FILE_NAME)
mapType = feature::DataHeader::world;
if (country == WORLD_COASTS_FILE_NAME)
mapType = feature::DataHeader::worldcoasts;
if (!feature::GenerateFinalFeatures(genInfo, country, mapType))
{
// If error - move to next bucket without index generation
continue;
}
}
// If error - move to next bucket without index generation.
string const datFile = path + country + DATA_FILE_EXTENSION;
LOG(LINFO, ("Generating result features for", country));
if (!feature::GenerateFinalFeatures(genInfo, country, mapType))
continue;
LOG(LINFO, ("Generating offsets table for", datFile));
if (!feature::BuildOffsetsTable(datFile))
continue;
}
if (FLAGS_generate_index)
{
LOG(LINFO, ("Generating index for ", datFile));
LOG(LINFO, ("Generating index for", datFile));
if (!indexer::BuildIndexFromDatFile(datFile, FLAGS_intermediate_data_path + country))
LOG(LCRITICAL, ("Error generating index."));

39
indexer/features_offsets_table.cpp
View file

@ -10,6 +10,7 @@
#include "base/assert.hpp"
#include "base/logging.hpp"
#include "base/scope_guard.hpp"
#include "std/string.hpp"
@ -64,6 +65,19 @@ namespace feature
return LoadImpl(filePath);
}
// static
unique_ptr<FeaturesOffsetsTable> FeaturesOffsetsTable::Load(FilesContainerR const & cont)
{
unique_ptr<FeaturesOffsetsTable> table(new FeaturesOffsetsTable());
table->m_file.Open(cont.GetFileName());
auto p = cont.GetAbsoluteOffsetAndSize(FEATURE_OFFSETS_FILE_TAG);
table->m_handle.Assign(table->m_file.Map(p.first, p.second, FEATURE_OFFSETS_FILE_TAG));
succinct::mapper::map(table->m_table, table->m_handle.GetData<char>());
return table;
}
// static
unique_ptr<FeaturesOffsetsTable> FeaturesOffsetsTable::CreateImpl(
platform::LocalCountryFile const & localFile,
@ -73,6 +87,12 @@ namespace feature
CountryIndexes::PreparePlaceOnDisk(localFile);
return Build(cont, storePath);
}
unique_ptr<FeaturesOffsetsTable> FeaturesOffsetsTable::Build(FilesContainerR const & cont,
string const & storePath)
{
Builder builder;
FeaturesVector::ForEachOffset(cont.GetReader(DATA_FILE_TAG), [&builder] (uint32_t offset)
{
@ -147,4 +167,23 @@ namespace feature
ASSERT_EQUAL(offset, m_table.select(leftBound), ("Can't find offset", offset, "in the table"));
return leftBound;
}
bool BuildOffsetsTable(string const & filePath)
{
try
{
string const destPath = filePath + ".offsets";
MY_SCOPE_GUARD(fileDeleter, bind(FileWriter::DeleteFileX, destPath));
(void)feature::FeaturesOffsetsTable::Build(FilesContainerR(filePath), destPath);
FilesContainerW(filePath, FileWriter::OP_WRITE_EXISTING).Write(destPath, FEATURE_OFFSETS_FILE_TAG);
return true;
}
catch (RootException const & ex)
{
LOG(LERROR, ("Generating offsets table failed for", filePath, "reason", ex.Msg()));
return false;
}
}
} // namespace feature

14
indexer/features_offsets_table.hpp
View file

@ -1,5 +1,6 @@
#pragma once
#include "coding/file_container.hpp"
#include "coding/mmap_reader.hpp"
#include "defines.hpp"
@ -12,7 +13,6 @@
#include "3party/succinct/mapper.hpp"
class FilesContainerR;
namespace platform
{
class LocalCountryFile;
@ -57,6 +57,10 @@ namespace feature
/// Load table by full path to the table file.
static unique_ptr<FeaturesOffsetsTable> Load(string const & filePath);
static unique_ptr<FeaturesOffsetsTable> Load(FilesContainerR const & cont);
static unique_ptr<FeaturesOffsetsTable> Build(FilesContainerR const & cont,
string const & storePath);
/// Get table for the MWM map, represented by localFile and cont.
static unique_ptr<FeaturesOffsetsTable> CreateIfNotExistsAndLoad(
platform::LocalCountryFile const & localFile, FilesContainerR const & cont);
@ -94,6 +98,7 @@ namespace feature
private:
FeaturesOffsetsTable(succinct::elias_fano::elias_fano_builder & builder);
FeaturesOffsetsTable(string const & filePath);
FeaturesOffsetsTable() = default;
static unique_ptr<FeaturesOffsetsTable> LoadImpl(string const & filePath);
static unique_ptr<FeaturesOffsetsTable> CreateImpl(platform::LocalCountryFile const & localFile,
@ -101,7 +106,12 @@ namespace feature
string const & storePath);
succinct::elias_fano m_table;
unique_ptr<MmapReader> m_pReader;
detail::MappedFile m_file;
detail::MappedFile::Handle m_handle;
};
bool BuildOffsetsTable(string const & filePath);
} // namespace feature

5
indexer/features_vector.cpp
View file

@ -23,8 +23,11 @@ FeaturesVectorTest::FeaturesVectorTest(string const & filePath)
FeaturesVectorTest::FeaturesVectorTest(FilesContainerR const & cont)
: m_cont(cont), m_header(m_cont), m_vector(m_cont, m_header, 0)
{
if (m_header.GetFormat() >= version::v5)
auto const version = m_header.GetFormat();
if (version == version::v5)
m_vector.m_table = feature::FeaturesOffsetsTable::CreateIfNotExistsAndLoad(m_cont).release();
else if (version >= version::v6)
m_vector.m_table = feature::FeaturesOffsetsTable::Load(m_cont).release();
}
FeaturesVectorTest::~FeaturesVectorTest()

2
indexer/features_vector.hpp
View file

@ -55,6 +55,8 @@ private:
/// Used in generator_tool and unit tests.
class FeaturesVectorTest
{
DISALLOW_COPY(FeaturesVectorTest);
FilesContainerR m_cont;
feature::DataHeader m_header;
FeaturesVector m_vector;

10
indexer/index.cpp
View file

@ -24,11 +24,17 @@ MwmValue::MwmValue(LocalCountryFile const & localFile)
void MwmValue::SetTable(MwmInfoEx & info)
{
if (GetHeader().GetFormat() < version::v5)
auto const version = GetHeader().GetFormat();
if (version < version::v5)
return;
if (!info.m_table)
info.m_table = feature::FeaturesOffsetsTable::CreateIfNotExistsAndLoad(m_file, m_cont);
{
if (version == version::v5)
info.m_table = feature::FeaturesOffsetsTable::CreateIfNotExistsAndLoad(m_file, m_cont);
else
info.m_table = feature::FeaturesOffsetsTable::Load(m_cont);
}
m_table = info.m_table.get();
}

3
map/map_tests/mwm_set_test.cpp
View file

@ -15,6 +15,8 @@
using namespace platform;
using namespace my;
/*
* This test is useless because of we don't build offsets index from now.
#ifndef OMIM_OS_WINDOWS
UNIT_TEST(MwmSet_FileSystemErrors)
{
@ -56,3 +58,4 @@ UNIT_TEST(MwmSet_FileSystemErrors)
TEST(infos.empty(), ());
}
#endif
*/

3
platform/mwm_version.hpp
View file

@ -17,7 +17,8 @@ enum Format
v3, // March 2013 (store type index, instead of raw type in search data)
v4, // April 2015 (distinguish и and й in search index)
v5, // July 2015 (feature id is the index in vector now).
lastFormat = v5
v6, // October 2015 (offsets vector is in mwm now).
lastFormat = v6
};
struct MwmVersion