pastk/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

Refactor template parameter for set cache read/write mode

Browse source
This commit is contained in:
Sergey Yershov 2015-08-24 15:11:28 +03:00 • committed by Alex Zolotarev
parent 7216d84b86
commit 17b0dcb3e8
4 changed files with 75 additions and 92 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
generator/data_cache_file.hpp
View file

@ -2,22 +2,25 @@
#include "generator/osm_decl.hpp"
#include "coding/file_name_utils.hpp"
#include "coding/file_reader_stream.hpp"
#include "coding/file_writer_stream.hpp"
#include "coding/file_name_utils.hpp"
#include "base/logging.hpp"
#include "std/algorithm.hpp"
#include "std/exception.hpp"
#include "std/limits.hpp"
#include "std/utility.hpp"
#include "std/vector.hpp"
#include "std/algorithm.hpp"
#include "std/limits.hpp"
#include "std/exception.hpp"
/// Classes for reading and writing any data in file with map of offsets for
/// fast searching in memory by some key.
namespace cache
{
enum class EMode { Write = true, Read = false };
namespace detail
{
template <class TFile, class TValue>
@ -122,13 +125,13 @@ public:
} // namespace detail
template <bool TModeWrite>
template <EMode TMode>
class OSMElementCache
{
public:
using TKey = uint64_t;
using TStream = typename conditional<TModeWrite, FileWriterStream, FileReaderStream>::type;
using TOffsetFile = typename conditional<TModeWrite, FileWriter, FileReader>::type;
using TStream = typename conditional<TMode == EMode::Write, FileWriterStream, FileReaderStream>::type;
using TOffsetFile = typename conditional<TMode == EMode::Write, FileWriter, FileReader>::type;
protected:
TStream m_stream;
@ -164,4 +167,4 @@ public:
inline void SaveOffsets() { m_offsets.WriteAll(); }
inline void LoadOffsets() { m_offsets.ReadAll(); }
};
}
} // namespace cache

39
generator/osm_decl.hpp
View file

@ -1,14 +1,13 @@
#pragma once
#include "base/std_serialization.hpp"
#include "base/assert.hpp"
#include "base/std_serialization.hpp"
#include "std/utility.hpp"
#include "std/vector.hpp"
#include "std/string.hpp"
#include "std/algorithm.hpp"
#include "std/bind.hpp"
#include "std/string.hpp"
#include "std/utility.hpp"
#include "std/vector.hpp"
#define NODES_FILE "nodes.dat"
#define WAYS_FILE "ways.dat"
@ -17,7 +16,6 @@
#define ID2REL_EXT ".id2rel"
#define MAPPED_WAYS "mapped_ways.n2w"
struct WayElement
{
vector<uint64_t> nodes;
@ -32,16 +30,18 @@ struct WayElement
if (id == nodes.front())
return nodes.back();
ASSERT ( id == nodes.back(), () );
ASSERT(id == nodes.back(), ());
return nodes.front();
}
template <class ToDo> void ForEachPoint(ToDo & toDo) const
template <class ToDo>
void ForEachPoint(ToDo & toDo) const
{
for_each(nodes.begin(), nodes.end(), ref(toDo));
}
template <class ToDo> void ForEachPointOrdered(uint64_t start, ToDo & toDo)
template <class ToDo>
void ForEachPointOrdered(uint64_t start, ToDo & toDo)
{
if (start == nodes.front())
for_each(nodes.begin(), nodes.end(), ref(toDo));
@ -49,7 +49,6 @@ struct WayElement
for_each(nodes.rbegin(), nodes.rend(), ref(toDo));
}
template <class TArchive>
void Write(TArchive & ar) const
{
@ -81,17 +80,11 @@ public:
return ((it != tags.end()) ? it->second : string());
}
bool FindWay(uint64_t id, string & role) const
{
return FindRoleImpl(ways, id, role);
}
bool FindWay(uint64_t id, string & role) const { return FindRoleImpl(ways, id, role); }
bool FindNode(uint64_t id, string & role) const { return FindRoleImpl(nodes, id, role); }
bool FindNode(uint64_t id, string & role) const
{
return FindRoleImpl(nodes, id, role);
}
template <class ToDo> void ForEachWay(ToDo & toDo) const
template <class ToDo>
void ForEachWay(ToDo & toDo) const
{
for (size_t i = 0; i < ways.size(); ++i)
toDo(ways[i].first, ways[i].second);
@ -117,14 +110,16 @@ public:
}
protected:
bool FindRoleImpl(TMembers const & cnt, uint64_t id, string & role) const
bool FindRoleImpl(TMembers const & container, uint64_t id, string & role) const
{
for (auto const & e : cnt)
for (auto const & e : container)
{
if (e.first == id)
{
role = e.second;
return true;
}
}
return false;
}
};

23
generator/osm_source.cpp
View file

@ -48,13 +48,12 @@ uint64_t SourceReader::Read(char * buffer, uint64_t bufferSize)
namespace
{
template <class TNodesHolder, bool TModeWrite>
template <class TNodesHolder, cache::EMode TMode>
class IntermediateData
{
using TReader = cache::OSMElementCache<TModeWrite>;
using TFile = typename conditional<TModeWrite, FileWriter, FileReader>::type;
using TReader = cache::OSMElementCache<TMode>;
using TFile = typename conditional<TMode == cache::EMode::Write, FileWriter, FileReader>::type;
using TKey = uint64_t;
static_assert(is_integral<TKey>::value, "TKey is not integral type");
@ -480,7 +479,7 @@ bool GenerateFeaturesImpl(feature::GenerateInfo & info)
{
TNodesHolder nodes(info.GetIntermediateFileName(NODES_FILE, ""));
using TDataCache = IntermediateData<TNodesHolder, /*WriteMode=*/false>;
using TDataCache = IntermediateData<TNodesHolder, cache::EMode::Read>;
TDataCache cache(nodes, info.m_intermediateDir);
cache.LoadIndex();
@ -526,7 +525,7 @@ bool GenerateIntermediateDataImpl(feature::GenerateInfo & info)
try
{
TNodesHolder nodes(info.GetIntermediateFileName(NODES_FILE, ""));
using TDataCache = IntermediateData<TNodesHolder, /*WriteMode=*/true>;
using TDataCache = IntermediateData<TNodesHolder, cache::EMode::Write>;
TDataCache cache(nodes, info.m_intermediateDir);
SourceReader reader = info.m_osmFileName.empty() ? SourceReader() : SourceReader(info.m_osmFileName);
@ -558,11 +557,11 @@ bool GenerateFeatures(feature::GenerateInfo & info)
switch (info.m_nodeStorageType)
{
case feature::GenerateInfo::NodeStorageType::File:
return GenerateFeaturesImpl<RawFilePointStorage<BasePointStorage::MODE_READ>>(info);
return GenerateFeaturesImpl<RawFilePointStorage<BasePointStorage::EMode::Read>>(info);
case feature::GenerateInfo::NodeStorageType::Index:
return GenerateFeaturesImpl<MapFilePointStorage<BasePointStorage::MODE_READ>>(info);
return GenerateFeaturesImpl<MapFilePointStorage<BasePointStorage::EMode::Read>>(info);
case feature::GenerateInfo::NodeStorageType::Memory:
return GenerateFeaturesImpl<RawMemPointStorage<BasePointStorage::MODE_READ>>(info);
return GenerateFeaturesImpl<RawMemPointStorage<BasePointStorage::EMode::Read>>(info);
}
return false;
}
@ -572,11 +571,11 @@ bool GenerateIntermediateData(feature::GenerateInfo & info)
switch (info.m_nodeStorageType)
{
case feature::GenerateInfo::NodeStorageType::File:
return GenerateIntermediateDataImpl<RawFilePointStorage<BasePointStorage::MODE_WRITE>>(info);
return GenerateIntermediateDataImpl<RawFilePointStorage<BasePointStorage::EMode::Write>>(info);
case feature::GenerateInfo::NodeStorageType::Index:
return GenerateIntermediateDataImpl<MapFilePointStorage<BasePointStorage::MODE_WRITE>>(info);
return GenerateIntermediateDataImpl<MapFilePointStorage<BasePointStorage::EMode::Write>>(info);
case feature::GenerateInfo::NodeStorageType::Memory:
return GenerateIntermediateDataImpl<RawMemPointStorage<BasePointStorage::MODE_WRITE>>(info);
return GenerateIntermediateDataImpl<RawMemPointStorage<BasePointStorage::EMode::Write>>(info);
}
return false;
}

86
generator/point_storage.hpp
View file

@ -23,19 +23,18 @@ struct LatLonPos
};
static_assert(sizeof(LatLonPos) == 16, "Invalid structure size");
class BasePointStorage
{
size_t m_processedPoint = 0;
public:
enum EStorageMode {MODE_READ = false, MODE_WRITE = true};
enum class EMode { Read = false, Write = true };
inline size_t GetProcessedPoint() const { return m_processedPoint; }
inline void IncProcessedPoint() { ++m_processedPoint; }
};
template < BasePointStorage::EStorageMode TMode >
template <BasePointStorage::EMode TMode>
class RawFilePointStorage : public BasePointStorage
{
#ifdef OMIM_OS_WINDOWS
@ -44,15 +43,15 @@ class RawFilePointStorage : public BasePointStorage
using TFileReader = MmapReader;
#endif
typename conditional<TMode, FileWriter, TFileReader>::type m_file;
typename conditional<TMode == EMode::Write, FileWriter, TFileReader>::type m_file;
constexpr static double const kValueOrder = 1E+7;
public:
RawFilePointStorage(string const & name) : m_file(name) {}
template <bool T = TMode>
typename enable_if<T == MODE_WRITE, void>::type AddPoint(uint64_t id, double lat, double lng)
template <EMode T = TMode>
typename enable_if<T == EMode::Write, void>::type AddPoint(uint64_t id, double lat, double lng)
{
int64_t const lat64 = lat * kValueOrder;
int64_t const lng64 = lng * kValueOrder;
@ -69,8 +68,9 @@ public:
IncProcessedPoint();
}
template <bool T = TMode>
typename enable_if<T == MODE_READ, bool>::type GetPoint(uint64_t id, double & lat, double & lng) const
template <EMode T = TMode>
typename enable_if<T == EMode::Read, bool>::type GetPoint(uint64_t id, double & lat,
double & lng) const
{
LatLon ll;
m_file.Read(id * sizeof(ll), &ll, sizeof(ll));
@ -82,57 +82,48 @@ public:
lng = static_cast<double>(ll.lon) / kValueOrder;
return true;
}
else
{
LOG(LERROR, ("Node with id = ", id, " not found!"));
return false;
}
LOG(LERROR, ("Node with id = ", id, " not found!"));
return false;
}
};
template < BasePointStorage::EStorageMode TMode >
template <BasePointStorage::EMode TMode>
class RawMemPointStorage : public BasePointStorage
{
typename conditional<TMode, FileWriter, FileReader>::type m_file;
typename conditional<TMode == EMode::Write, FileWriter, FileReader>::type m_file;
constexpr static double const kValueOrder = 1E+7;
vector<LatLon> m_data;
public:
RawMemPointStorage(string const & name)
: m_file(name)
, m_data((size_t)0xFFFFFFFF)
RawMemPointStorage(string const & name) : m_file(name), m_data((size_t)0xFFFFFFFF)
{
InitStorage<TMode>();
}
~RawMemPointStorage()
{
DoneStorage<TMode>();
}
~RawMemPointStorage() { DoneStorage<TMode>(); }
template <bool T>
typename enable_if<T == MODE_WRITE, void>::type InitStorage() {}
template <EMode T>
typename enable_if<T == EMode::Write, void>::type InitStorage() {}
template <bool T>
typename enable_if<T == MODE_READ, void>::type InitStorage()
template <EMode T>
typename enable_if<T == EMode::Read, void>::type InitStorage()
{
m_file.Read(0, m_data.data(), m_data.size() * sizeof(LatLon));
}
template <bool T>
typename enable_if<T == MODE_WRITE, void>::type DoneStorage()
template <EMode T>
typename enable_if<T == EMode::Write, void>::type DoneStorage()
{
m_file.Write(m_data.data(), m_data.size() * sizeof(LatLon));
}
template <bool T>
typename enable_if<T == MODE_READ, void>::type DoneStorage() {}
template <EMode T>
typename enable_if<T == EMode::Read, void>::type DoneStorage() {}
template <bool T = TMode>
typename enable_if<T == MODE_WRITE, void>::type AddPoint(uint64_t id, double lat, double lng)
template <EMode T = TMode>
typename enable_if<T == EMode::Write, void>::type AddPoint(uint64_t id, double lat, double lng)
{
int64_t const lat64 = lat * kValueOrder;
int64_t const lng64 = lng * kValueOrder;
@ -146,8 +137,9 @@ public:
IncProcessedPoint();
}
template <bool T = TMode>
typename enable_if<T == MODE_READ, bool>::type GetPoint(uint64_t id, double & lat, double & lng) const
template <EMode T = TMode>
typename enable_if<T == EMode::Read, bool>::type GetPoint(uint64_t id, double & lat,
double & lng) const
{
LatLon const & ll = m_data[id];
// assume that valid coordinate is not (0, 0)
@ -157,33 +149,27 @@ public:
lng = static_cast<double>(ll.lon) / kValueOrder;
return true;
}
else
{
LOG(LERROR, ("Node with id = ", id, " not found!"));
return false;
}
LOG(LERROR, ("Node with id = ", id, " not found!"));
return false;
}
};
template < BasePointStorage::EStorageMode TMode >
template <BasePointStorage::EMode TMode>
class MapFilePointStorage : public BasePointStorage
{
typename conditional<TMode, FileWriter, FileReader>::type m_file;
typename conditional<TMode == EMode::Write, FileWriter, FileReader>::type m_file;
unordered_map<uint64_t, pair<int32_t, int32_t>> m_map;
constexpr static double const kValueOrder = 1E+7;
public:
MapFilePointStorage(string const & name) : m_file(name+".short")
{
InitStorage<TMode>();
}
MapFilePointStorage(string const & name) : m_file(name + ".short") { InitStorage<TMode>(); }
template <bool T>
typename enable_if<T == MODE_WRITE, void>::type InitStorage() {}
template <EMode T>
typename enable_if<T == EMode::Write, void>::type InitStorage() {}
template <bool T>
typename enable_if<T == MODE_READ, void>::type InitStorage()
template <EMode T>
typename enable_if<T == EMode::Read, void>::type InitStorage()
{
LOG(LINFO, ("Nodes reading is started"));