organicmaps/organicmaps-tmp
Archived
13
Fork 1
Code Issues 2.7k Pull requests 182 Projects Releases 65 Wiki Activity Actions

Move all classes for work with intermediate cache to imtermediate_data.hpp

Browse source
This commit is contained in:
Sergey Yershov 2015-08-25 13:40:53 +03:00 • committed by Alex Zolotarev
parent 770fd9da5a
commit 4ca9859adf
5 changed files with 225 additions and 233 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
generator/feature_generator.cpp
View file

@ -1,13 +1,13 @@
#include "generator/feature_generator.hpp"
#include "generator/data_cache_file.hpp"
#include "generator/intermediate_data.hpp"
#include "generator/osm_element.hpp"
#include "generator/osm_decl.hpp"
#include "generator/generate_info.hpp"
#include "generator/osm_decl.hpp"
#include "indexer/cell_id.hpp"
#include "indexer/data_header.hpp"
#include "indexer/mercator.hpp"
#include "indexer/cell_id.hpp"
#include "coding/varint.hpp"
@ -16,8 +16,8 @@
#include "base/stl_add.hpp"
#include "std/bind.hpp"
#include "std/unordered_map.hpp"
#include "std/target_os.hpp"
#include "std/unordered_map.hpp"
///////////////////////////////////////////////////////////////////////////////////////////////////
// FeaturesCollector implementation

216
generator/intermediate_data.hpp
View file

@ -3,6 +3,9 @@
#include "generator/osm_decl.hpp"
#include "coding/file_name_utils.hpp"
#include "coding/file_reader.hpp"
#include "coding/file_writer.hpp"
#include "coding/mmap_reader.hpp"
#include "base/logging.hpp"
@ -126,7 +129,6 @@ public:
};
} // namespace detail
template <EMode TMode>
class OSMElementCache
{
@ -212,4 +214,214 @@ public:
inline void SaveOffsets() { m_offsets.WriteAll(); }
inline void LoadOffsets() { m_offsets.ReadAll(); }
};
} // namespace cache
/// Used to store all world nodes inside temporary index file.
/// To find node by id, just calculate offset inside index file:
/// offset_in_file = sizeof(LatLon) * node_ID
class PointStorage
{
size_t m_processedPoint = 0;
public:
struct LatLon
{
int32_t lat;
int32_t lon;
};
static_assert(sizeof(LatLon) == 8, "Invalid structure size");
struct LatLonPos
{
uint64_t pos;
int32_t lat;
int32_t lon;
};
static_assert(sizeof(LatLonPos) == 16, "Invalid structure size");
inline size_t GetProcessedPoint() const { return m_processedPoint; }
inline void IncProcessedPoint() { ++m_processedPoint; }
};
template <EMode TMode>
class RawFilePointStorage : public PointStorage
{
#ifdef OMIM_OS_WINDOWS
using TFileReader = FileReader;
#else
using TFileReader = MmapReader;
#endif
typename conditional<TMode == EMode::Write, FileWriter, TFileReader>::type m_file;
constexpr static double const kValueOrder = 1E+7;
public:
RawFilePointStorage(string const & name) : m_file(name) {}
template <EMode T = TMode>
typename enable_if<T == EMode::Write, void>::type AddPoint(uint64_t id, double lat, double lng)
{
int64_t const lat64 = lat * kValueOrder;
int64_t const lng64 = lng * kValueOrder;
LatLon ll;
ll.lat = static_cast<int32_t>(lat64);
ll.lon = static_cast<int32_t>(lng64);
CHECK_EQUAL(static_cast<int64_t>(ll.lat), lat64, ("Latitude is out of 32bit boundary!"));
CHECK_EQUAL(static_cast<int64_t>(ll.lon), lng64, ("Longtitude is out of 32bit boundary!"));
m_file.Seek(id * sizeof(ll));
m_file.Write(&ll, sizeof(ll));
IncProcessedPoint();
}
template <EMode T = TMode>
typename enable_if<T == EMode::Read, bool>::type GetPoint(uint64_t id, double & lat,
double & lng) const
{
LatLon ll;
m_file.Read(id * sizeof(ll), &ll, sizeof(ll));
// assume that valid coordinate is not (0, 0)
if (ll.lat != 0.0 || ll.lon != 0.0)
{
lat = static_cast<double>(ll.lat) / kValueOrder;
lng = static_cast<double>(ll.lon) / kValueOrder;
return true;
}
LOG(LERROR, ("Node with id = ", id, " not found!"));
return false;
}
};
template <EMode TMode>
class RawMemPointStorage : public PointStorage
{
typename conditional<TMode == EMode::Write, FileWriter, FileReader>::type m_file;
constexpr static double const kValueOrder = 1E+7;
vector<LatLon> m_data;
public:
RawMemPointStorage(string const & name) : m_file(name), m_data((size_t)0xFFFFFFFF)
{
InitStorage<TMode>();
}
~RawMemPointStorage() { DoneStorage<TMode>(); }
template <EMode T>
typename enable_if<T == EMode::Write, void>::type InitStorage() {}
template <EMode T>
typename enable_if<T == EMode::Read, void>::type InitStorage()
{
m_file.Read(0, m_data.data(), m_data.size() * sizeof(LatLon));
}
template <EMode T>
typename enable_if<T == EMode::Write, void>::type DoneStorage()
{
m_file.Write(m_data.data(), m_data.size() * sizeof(LatLon));
}
template <EMode T>
typename enable_if<T == EMode::Read, void>::type DoneStorage() {}
template <EMode T = TMode>
typename enable_if<T == EMode::Write, void>::type AddPoint(uint64_t id, double lat, double lng)
{
int64_t const lat64 = lat * kValueOrder;
int64_t const lng64 = lng * kValueOrder;
LatLon & ll = m_data[id];
ll.lat = static_cast<int32_t>(lat64);
ll.lon = static_cast<int32_t>(lng64);
CHECK_EQUAL(static_cast<int64_t>(ll.lat), lat64, ("Latitude is out of 32bit boundary!"));
CHECK_EQUAL(static_cast<int64_t>(ll.lon), lng64, ("Longtitude is out of 32bit boundary!"));
IncProcessedPoint();
}
template <EMode T = TMode>
typename enable_if<T == EMode::Read, bool>::type GetPoint(uint64_t id, double & lat,
double & lng) const
{
LatLon const & ll = m_data[id];
// assume that valid coordinate is not (0, 0)
if (ll.lat != 0.0 || ll.lon != 0.0)
{
lat = static_cast<double>(ll.lat) / kValueOrder;
lng = static_cast<double>(ll.lon) / kValueOrder;
return true;
}
LOG(LERROR, ("Node with id = ", id, " not found!"));
return false;
}
};
template <EMode TMode>
class MapFilePointStorage : public PointStorage
{
typename conditional<TMode == EMode::Write, FileWriter, FileReader>::type m_file;
unordered_map<uint64_t, pair<int32_t, int32_t>> m_map;
constexpr static double const kValueOrder = 1E+7;
public:
MapFilePointStorage(string const & name) : m_file(name + ".short") { InitStorage<TMode>(); }
template <EMode T>
typename enable_if<T == EMode::Write, void>::type InitStorage() {}
template <EMode T>
typename enable_if<T == EMode::Read, void>::type InitStorage()
{
LOG(LINFO, ("Nodes reading is started"));
uint64_t const count = m_file.Size();
uint64_t pos = 0;
while (pos < count)
{
LatLonPos ll;
m_file.Read(pos, &ll, sizeof(ll));
m_map.emplace(make_pair(ll.pos, make_pair(ll.lat, ll.lon)));
pos += sizeof(ll);
}
LOG(LINFO, ("Nodes reading is finished"));
}
void AddPoint(uint64_t id, double lat, double lng)
{
int64_t const lat64 = lat * kValueOrder;
int64_t const lng64 = lng * kValueOrder;
LatLonPos ll;
ll.pos = id;
ll.lat = static_cast<int32_t>(lat64);
ll.lon = static_cast<int32_t>(lng64);
CHECK_EQUAL(static_cast<int64_t>(ll.lat), lat64, ("Latitude is out of 32bit boundary!"));
CHECK_EQUAL(static_cast<int64_t>(ll.lon), lng64, ("Longtitude is out of 32bit boundary!"));
m_file.Write(&ll, sizeof(ll));
IncProcessedPoint();
}
bool GetPoint(uint64_t id, double & lat, double & lng) const
{
auto i = m_map.find(id);
if (i == m_map.end())
return false;
lat = static_cast<double>(i->second.first) / kValueOrder;
lng = static_cast<double>(i->second.second) / kValueOrder;
return true;
}
};
} // namespace cache

15
generator/osm_source.cpp
View file

@ -1,11 +1,10 @@
#include "generator/coastlines_generator.hpp"
#include "generator/data_cache_file.hpp"
#include "generator/feature_generator.hpp"
#include "generator/intermediate_data.hpp"
#include "generator/osm_decl.hpp"
#include "generator/osm_element.hpp"
#include "generator/osm_o5m_source.hpp"
#include "generator/osm_source.hpp"
#include "generator/point_storage.hpp"
#include "generator/polygonizer.hpp"
#include "generator/world_map_generator.hpp"
#include "generator/xml_element.hpp"
@ -556,11 +555,11 @@ bool GenerateFeatures(feature::GenerateInfo & info)
switch (info.m_nodeStorageType)
{
case feature::GenerateInfo::NodeStorageType::File:
return GenerateFeaturesImpl<RawFilePointStorage<BasePointStorage::EMode::Read>>(info);
return GenerateFeaturesImpl<cache::RawFilePointStorage<cache::EMode::Read>>(info);
case feature::GenerateInfo::NodeStorageType::Index:
return GenerateFeaturesImpl<MapFilePointStorage<BasePointStorage::EMode::Read>>(info);
return GenerateFeaturesImpl<cache::MapFilePointStorage<cache::EMode::Read>>(info);
case feature::GenerateInfo::NodeStorageType::Memory:
return GenerateFeaturesImpl<RawMemPointStorage<BasePointStorage::EMode::Read>>(info);
return GenerateFeaturesImpl<cache::RawMemPointStorage<cache::EMode::Read>>(info);
}
return false;
}
@ -570,11 +569,11 @@ bool GenerateIntermediateData(feature::GenerateInfo & info)
switch (info.m_nodeStorageType)
{
case feature::GenerateInfo::NodeStorageType::File:
return GenerateIntermediateDataImpl<RawFilePointStorage<BasePointStorage::EMode::Write>>(info);
return GenerateIntermediateDataImpl<cache::RawFilePointStorage<cache::EMode::Write>>(info);
case feature::GenerateInfo::NodeStorageType::Index:
return GenerateIntermediateDataImpl<MapFilePointStorage<BasePointStorage::EMode::Write>>(info);
return GenerateIntermediateDataImpl<cache::MapFilePointStorage<cache::EMode::Write>>(info);
case feature::GenerateInfo::NodeStorageType::Memory:
return GenerateIntermediateDataImpl<RawMemPointStorage<BasePointStorage::EMode::Write>>(info);
return GenerateIntermediateDataImpl<cache::RawMemPointStorage<cache::EMode::Write>>(info);
}
return false;
}

217
generator/point_storage.hpp
View file

@ -1,217 +0,0 @@
#pragma once
#include "coding/mmap_reader.hpp"
#include "std/iostream.hpp"
#include "std/type_traits.hpp"
/// Used to store all world nodes inside temporary index file.
/// To find node by id, just calculate offset inside index file:
/// offset_in_file = sizeof(LatLon) * node_ID
struct LatLon
{
int32_t lat;
int32_t lon;
};
static_assert(sizeof(LatLon) == 8, "Invalid structure size");
struct LatLonPos
{
uint64_t pos;
int32_t lat;
int32_t lon;
};
static_assert(sizeof(LatLonPos) == 16, "Invalid structure size");
class BasePointStorage
{
size_t m_processedPoint = 0;
public:
enum class EMode { Read = false, Write = true };
inline size_t GetProcessedPoint() const { return m_processedPoint; }
inline void IncProcessedPoint() { ++m_processedPoint; }
};
template <BasePointStorage::EMode TMode>
class RawFilePointStorage : public BasePointStorage
{
#ifdef OMIM_OS_WINDOWS
using TFileReader = FileReader;
#else
using TFileReader = MmapReader;
#endif
typename conditional<TMode == EMode::Write, FileWriter, TFileReader>::type m_file;
constexpr static double const kValueOrder = 1E+7;
public:
RawFilePointStorage(string const & name) : m_file(name) {}
template <EMode T = TMode>
typename enable_if<T == EMode::Write, void>::type AddPoint(uint64_t id, double lat, double lng)
{
int64_t const lat64 = lat * kValueOrder;
int64_t const lng64 = lng * kValueOrder;
LatLon ll;
ll.lat = static_cast<int32_t>(lat64);
ll.lon = static_cast<int32_t>(lng64);
CHECK_EQUAL(static_cast<int64_t>(ll.lat), lat64, ("Latitude is out of 32bit boundary!"));
CHECK_EQUAL(static_cast<int64_t>(ll.lon), lng64, ("Longtitude is out of 32bit boundary!"));
m_file.Seek(id * sizeof(ll));
m_file.Write(&ll, sizeof(ll));
IncProcessedPoint();
}
template <EMode T = TMode>
typename enable_if<T == EMode::Read, bool>::type GetPoint(uint64_t id, double & lat,
double & lng) const
{
LatLon ll;
m_file.Read(id * sizeof(ll), &ll, sizeof(ll));
// assume that valid coordinate is not (0, 0)
if (ll.lat != 0.0 || ll.lon != 0.0)
{
lat = static_cast<double>(ll.lat) / kValueOrder;
lng = static_cast<double>(ll.lon) / kValueOrder;
return true;
}
LOG(LERROR, ("Node with id = ", id, " not found!"));
return false;
}
};
template <BasePointStorage::EMode TMode>
class RawMemPointStorage : public BasePointStorage
{
typename conditional<TMode == EMode::Write, FileWriter, FileReader>::type m_file;
constexpr static double const kValueOrder = 1E+7;
vector<LatLon> m_data;
public:
RawMemPointStorage(string const & name) : m_file(name), m_data((size_t)0xFFFFFFFF)
{
InitStorage<TMode>();
}
~RawMemPointStorage() { DoneStorage<TMode>(); }
template <EMode T>
typename enable_if<T == EMode::Write, void>::type InitStorage() {}
template <EMode T>
typename enable_if<T == EMode::Read, void>::type InitStorage()
{
m_file.Read(0, m_data.data(), m_data.size() * sizeof(LatLon));
}
template <EMode T>
typename enable_if<T == EMode::Write, void>::type DoneStorage()
{
m_file.Write(m_data.data(), m_data.size() * sizeof(LatLon));
}
template <EMode T>
typename enable_if<T == EMode::Read, void>::type DoneStorage() {}
template <EMode T = TMode>
typename enable_if<T == EMode::Write, void>::type AddPoint(uint64_t id, double lat, double lng)
{
int64_t const lat64 = lat * kValueOrder;
int64_t const lng64 = lng * kValueOrder;
LatLon & ll = m_data[id];
ll.lat = static_cast<int32_t>(lat64);
ll.lon = static_cast<int32_t>(lng64);
CHECK_EQUAL(static_cast<int64_t>(ll.lat), lat64, ("Latitude is out of 32bit boundary!"));
CHECK_EQUAL(static_cast<int64_t>(ll.lon), lng64, ("Longtitude is out of 32bit boundary!"));
IncProcessedPoint();
}
template <EMode T = TMode>
typename enable_if<T == EMode::Read, bool>::type GetPoint(uint64_t id, double & lat,
double & lng) const
{
LatLon const & ll = m_data[id];
// assume that valid coordinate is not (0, 0)
if (ll.lat != 0.0 || ll.lon != 0.0)
{
lat = static_cast<double>(ll.lat) / kValueOrder;
lng = static_cast<double>(ll.lon) / kValueOrder;
return true;
}
LOG(LERROR, ("Node with id = ", id, " not found!"));
return false;
}
};
template <BasePointStorage::EMode TMode>
class MapFilePointStorage : public BasePointStorage
{
typename conditional<TMode == EMode::Write, FileWriter, FileReader>::type m_file;
unordered_map<uint64_t, pair<int32_t, int32_t>> m_map;
constexpr static double const kValueOrder = 1E+7;
public:
MapFilePointStorage(string const & name) : m_file(name + ".short") { InitStorage<TMode>(); }
template <EMode T>
typename enable_if<T == EMode::Write, void>::type InitStorage() {}
template <EMode T>
typename enable_if<T == EMode::Read, void>::type InitStorage()
{
LOG(LINFO, ("Nodes reading is started"));
uint64_t const count = m_file.Size();
uint64_t pos = 0;
while (pos < count)
{
LatLonPos ll;
m_file.Read(pos, &ll, sizeof(ll));
m_map.emplace(make_pair(ll.pos, make_pair(ll.lat, ll.lon)));
pos += sizeof(ll);
}
LOG(LINFO, ("Nodes reading is finished"));
}
void AddPoint(uint64_t id, double lat, double lng)
{
int64_t const lat64 = lat * kValueOrder;
int64_t const lng64 = lng * kValueOrder;
LatLonPos ll;
ll.pos = id;
ll.lat = static_cast<int32_t>(lat64);
ll.lon = static_cast<int32_t>(lng64);
CHECK_EQUAL(static_cast<int64_t>(ll.lat), lat64, ("Latitude is out of 32bit boundary!"));
CHECK_EQUAL(static_cast<int64_t>(ll.lon), lng64, ("Longtitude is out of 32bit boundary!"));
m_file.Write(&ll, sizeof(ll));
IncProcessedPoint();
}
bool GetPoint(uint64_t id, double & lat, double & lng) const
{
auto i = m_map.find(id);
if (i == m_map.end())
return false;
lat = static_cast<double>(i->second.first) / kValueOrder;
lng = static_cast<double>(i->second.second) / kValueOrder;
return true;
}
};

2
xcode/generator/generator.xcodeproj/project.pbxproj
View file

@ -113,7 +113,6 @@
6753405A1A3F2A7400A0A8C3 /* xml_element.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = xml_element.cpp; sourceTree = "<group>"; };
6753405B1A3F2A7400A0A8C3 /* xml_element.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = xml_element.hpp; sourceTree = "<group>"; };
6764B8921ADD6A3300DD8B15 /* osm_o5m_source.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = osm_o5m_source.hpp; sourceTree = "<group>"; };
67EA644A1A7A4CF500872A69 /* point_storage.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = point_storage.hpp; sourceTree = "<group>"; };
/* End PBXFileReference section */
/* Begin PBXFrameworksBuildPhase section */
@ -180,7 +179,6 @@
6753404C1A3F2A7400A0A8C3 /* osm2type.hpp */,
6726C1D31A4AFEF4005EEA39 /* osm2meta.cpp */,
6726C1D41A4AFEF4005EEA39 /* osm2meta.hpp */,
67EA644A1A7A4CF500872A69 /* point_storage.hpp */,
6753404D1A3F2A7400A0A8C3 /* polygonizer.hpp */,
6753404E1A3F2A7400A0A8C3 /* routing_generator.cpp */,
6753404F1A3F2A7400A0A8C3 /* routing_generator.hpp */,