diff --git a/generator/feature_builder.cpp b/generator/feature_builder.cpp index 452c76ee9e..cdb7e59356 100644 --- a/generator/feature_builder.cpp +++ b/generator/feature_builder.cpp @@ -5,6 +5,7 @@ #include "../indexer/feature_impl.hpp" #include "../indexer/feature_visibility.hpp" #include "../indexer/geometry_serialization.hpp" +#include "../indexer/coding_params.hpp" #include "../geometry/region2d.hpp" @@ -208,21 +209,23 @@ void FeatureBuilder1::Serialize(buffer_t & data) const data.clear(); - SerializeBase(data, serial::CodingParams()); + serial::CodingParams cp; + + SerializeBase(data, cp); PushBackByteSink sink(data); EGeomType const type = m_Params.GetGeomType(); if (type != GEOM_POINT) - serial::SaveOuterPath(m_Geometry, serial::CodingParams(), sink); + serial::SaveOuterPath(m_Geometry, cp, sink); if (type == GEOM_AREA) { WriteVarUint(sink, uint32_t(m_Holes.size())); for (list::const_iterator i = m_Holes.begin(); i != m_Holes.end(); ++i) - serial::SaveOuterPath(*i, serial::CodingParams(), sink); + serial::SaveOuterPath(*i, cp, sink); } // check for correct serialization @@ -236,27 +239,33 @@ void FeatureBuilder1::Serialize(buffer_t & data) const void FeatureBuilder1::Deserialize(buffer_t & data) { - FeatureBase f; - f.Deserialize(data, 0, serial::CodingParams()); - InitFeatureBuilder(f); + serial::CodingParams cp; - ArrayByteSource src(f.DataPtr() + f.m_Header2Offset); + ArrayByteSource source(&data[0]); + m_Params.Read(source); EGeomType const type = m_Params.GetGeomType(); - if (type != GEOM_POINT) + if (type == GEOM_POINT) { - serial::LoadOuterPath(src, serial::CodingParams(), m_Geometry); - CalcRect(m_Geometry, m_LimitRect); + CoordPointT const center = PointU2PointD( + DecodeDelta(ReadVarUint(source), cp.GetBasePoint()), cp.GetCoordBits()); + + m_Center = m2::PointD(center.first, center.second); + m_LimitRect.Add(m_Center); + return; } + serial::LoadOuterPath(source, cp, m_Geometry); + CalcRect(m_Geometry, m_LimitRect); + if (type == GEOM_AREA) { - uint32_t const count = ReadVarUint(src); + uint32_t const count = ReadVarUint(source); for (uint32_t i = 0; i < count; ++i) { m_Holes.push_back(points_t()); - serial::LoadOuterPath(src, serial::CodingParams(), m_Holes.back()); + serial::LoadOuterPath(source, cp, m_Holes.back()); } } @@ -403,16 +412,3 @@ void FeatureBuilder2::Serialize(buffers_holder_t & data, serial::CodingParams co } } } - -void FeatureBuilder1::InitFeatureBuilder(FeatureBase const & ft) -{ - ft.ParseAll(); - - m_Params = ft.GetFeatureParams(); - - if (ft.GetFeatureType() == GEOM_POINT) - { - SetCenter(ft.GetCenter()); - m_Params.SetGeomType(GEOM_POINT); - } -} diff --git a/generator/feature_builder.hpp b/generator/feature_builder.hpp index 5c57c5418a..5e533faf7f 100644 --- a/generator/feature_builder.hpp +++ b/generator/feature_builder.hpp @@ -1,6 +1,12 @@ #pragma once #include "../indexer/feature.hpp" +#include "../coding/file_reader.hpp" + +#include "../std/bind.hpp" + + +namespace serial { class CodingParams; } /// Used for serialization\deserialization of features during --generate_features. class FeatureBuilder1 @@ -73,7 +79,6 @@ public: inline void SetParams(FeatureParams const & params) { m_Params = params; } protected: - void InitFeatureBuilder(FeatureBase const & ft); /// @name For diagnostic use only. //@{ @@ -150,12 +155,12 @@ namespace feature { /// Read feature from feature source. template - void ReadFromSourceRowFormat(TSource & src, FeatureBuilder1 & f) + void ReadFromSourceRowFormat(TSource & src, FeatureBuilder1 & fb) { uint32_t const sz = ReadVarUint(src); typename FeatureBuilder1::buffer_t buffer(sz); src.Read(&buffer[0], sz); - f.Deserialize(buffer); + fb.Deserialize(buffer); } /// Process features in .dat file. @@ -171,9 +176,9 @@ namespace feature // read features one by one while (currPos < fSize) { - FeatureBuilder1 f; - ReadFromSourceRowFormat(src, f); - toDo(f, currPos); + FeatureBuilder1 fb; + ReadFromSourceRowFormat(src, fb); + toDo(fb, currPos); currPos = src.Pos(); } } diff --git a/indexer/data_header.cpp b/indexer/data_header.cpp index 22486c90a1..7ff6d0202e 100644 --- a/indexer/data_header.cpp +++ b/indexer/data_header.cpp @@ -2,8 +2,11 @@ #include "point_to_int64.hpp" #include "scales.hpp" +#include "../defines.hpp" + #include "../coding/file_reader.hpp" #include "../coding/file_writer.hpp" +#include "../coding/file_container.hpp" #include "../coding/write_to_sink.hpp" #include "../coding/varint.hpp" @@ -12,15 +15,6 @@ namespace feature { - DataHeader::DataHeader() - { - Reset(); - } - - void DataHeader::Reset() - { - } - m2::RectD const DataHeader::GetBounds() const { return Int64ToRect(m_bounds, m_codingParams.GetCoordBits()); @@ -77,4 +71,9 @@ namespace feature src.Read(m_scales.data(), m_scales.size()); } + + void DataHeader::LoadForVersion(FilesContainerR const & cont) + { + Load(cont.GetReader(HEADER_FILE_TAG)); + } } diff --git a/indexer/data_header.hpp b/indexer/data_header.hpp index 2726508688..83fe37e7a5 100644 --- a/indexer/data_header.hpp +++ b/indexer/data_header.hpp @@ -6,15 +6,13 @@ #include "../std/array.hpp" -#include "../base/start_mem_debug.hpp" - class ModelReaderPtr; class FileWriter; +class FilesContainerR; namespace feature { - /// All file sizes are in bytes class DataHeader { serial::CodingParams m_codingParams; @@ -24,10 +22,6 @@ namespace feature array m_scales; public: - DataHeader(); - - /// Zero all fields - void Reset(); void SetCodingParams(serial::CodingParams const & params) { m_codingParams = params; } serial::CodingParams const & GetCodingParams() const { return m_codingParams; } @@ -36,16 +30,19 @@ namespace feature void SetBounds(m2::RectD const & r); void SetScales(int * arr); - size_t GetScalesCount() const { return m_scales.size(); } - int GetScale(int i) const { return m_scales[i]; } + + inline size_t GetScalesCount() const { return m_scales.size(); } + inline int GetScale(int i) const { return m_scales[i]; } + inline int GetLastScale() const { return GetScale(GetScalesCount() - 1); } + pair GetScaleRange() const; /// @name Serialization //@{ void Save(FileWriter & w) const; void Load(ModelReaderPtr const & r); + + void LoadForVersion(FilesContainerR const & cont); //@} }; } - -#include "../base/stop_mem_debug.hpp" diff --git a/indexer/feature.cpp b/indexer/feature.cpp index 880acc9e8d..4e79cba02c 100644 --- a/indexer/feature.cpp +++ b/indexer/feature.cpp @@ -1,25 +1,11 @@ #include "../base/SRC_FIRST.hpp" #include "feature.hpp" -#include "feature_impl.hpp" #include "feature_visibility.hpp" -#include "scales.hpp" -#include "geometry_serialization.hpp" -#include "coding_params.hpp" +#include "feature_loader_base.hpp" #include "../defines.hpp" // just for file extensions -#include "../coding/byte_stream.hpp" - -#include "../geometry/pointu_to_uint64.hpp" -#include "../geometry/rect2d.hpp" -#include "../geometry/region2d.hpp" - -#include "../base/logging.hpp" -#include "../base/stl_add.hpp" - -#include "../std/algorithm.hpp" - #include "../base/start_mem_debug.hpp" @@ -29,30 +15,34 @@ using namespace feature; // FeatureBase implementation /////////////////////////////////////////////////////////////////////////////////////////////////// -void FeatureBase::Deserialize(buffer_t & data, uint32_t offset, serial::CodingParams const & params) +void FeatureBase::Deserialize(feature::LoaderBase * pLoader, BufferT buffer) { - m_Offset = offset; - m_Data.swap(data); + m_pLoader.reset(pLoader); + m_pLoader->Deserialize(buffer); - m_CodingParams = params; - - m_CommonOffset = m_Header2Offset = 0; - m_bTypesParsed = m_bCommonParsed = false; - - m_Params = FeatureParamsBase(); m_LimitRect = m2::RectD::GetEmptyRect(); + m_bTypesParsed = m_bCommonParsed = false; + m_Header = m_pLoader->GetHeader(); } -uint32_t FeatureBase::CalcOffset(ArrayByteSource const & source) const +void FeatureBase::ParseTypes() const { - return static_cast(static_cast(source.Ptr()) - DataPtr()); + if (!m_bTypesParsed) + { + m_pLoader->ParseTypes(); + m_bTypesParsed = true; + } } -void FeatureBase::SetHeader(uint8_t h) +void FeatureBase::ParseCommon() const { - ASSERT ( m_Offset == 0, (m_Offset) ); - m_Data.resize(1); - m_Data[0] = h; + if (!m_bCommonParsed) + { + ParseTypes(); + + m_pLoader->ParseCommon(); + m_bCommonParsed = true; + } } feature::EGeomType FeatureBase::GetFeatureType() const @@ -70,56 +60,6 @@ feature::EGeomType FeatureBase::GetFeatureType() const } } -void FeatureBase::ParseTypes() const -{ - ASSERT(!m_bTypesParsed, ()); - - Classificator & c = classif(); - - ArrayByteSource source(DataPtr() + m_TypesOffset); - for (size_t i = 0; i < GetTypesCount(); ++i) - { - m_Types[i] = c.GetTypeForIndex(ReadVarUint(source)); - //m_Types[i] = ReadVarUint(source); - } - - m_bTypesParsed = true; - m_CommonOffset = CalcOffset(source); -} - -void FeatureBase::ParseCommon() const -{ - ASSERT(!m_bCommonParsed, ()); - if (!m_bTypesParsed) - ParseTypes(); - - ArrayByteSource source(DataPtr() + m_CommonOffset); - - uint8_t const h = Header(); - - EGeomType const type = GetFeatureType(); - - m_Params.Read(source, h, type); - - if (type == GEOM_POINT) - { - CoordPointT center = PointU2PointD(DecodeDelta(ReadVarUint(source), - m_CodingParams.GetBasePoint()), - m_CodingParams.GetCoordBits()); - m_Center = m2::PointD(center.first, center.second); - m_LimitRect.Add(m_Center); - } - - m_bCommonParsed = true; - m_Header2Offset = CalcOffset(source); -} - -void FeatureBase::ParseAll() const -{ - if (!m_bCommonParsed) - ParseCommon(); -} - string FeatureBase::DebugString() const { ASSERT(m_bCommonParsed, ()); @@ -137,98 +77,55 @@ string FeatureBase::DebugString() const return res; } -FeatureParams FeatureBase::GetFeatureParams() const -{ - FeatureParams params(m_Params); - params.AssignTypes(m_Types, m_Types + GetTypesCount()); - - uint8_t const h = (Header() & HEADER_GEOTYPE_MASK); - if (h & HEADER_GEOM_LINE) params.SetGeomType(GEOM_LINE); - if (h & HEADER_GEOM_AREA) params.SetGeomType(GEOM_AREA); - - return params; -} /////////////////////////////////////////////////////////////////////////////////////////////////// // FeatureType implementation /////////////////////////////////////////////////////////////////////////////////////////////////// -FeatureType::FeatureType(read_source_t & src) +void FeatureType::Deserialize(feature::LoaderBase * pLoader, BufferT buffer) { - Deserialize(src); -} + base_type::Deserialize(pLoader, buffer); -void FeatureType::Deserialize(read_source_t & src) -{ - m_cont = &src.m_cont; - m_header = &src.m_header; - - m_Points.clear(); - m_Triangles.clear(); + m_pLoader->AssignFeature(this); m_bHeader2Parsed = m_bPointsParsed = m_bTrianglesParsed = false; - m_ptsSimpMask = 0; - - m_InnerStats.MakeZero(); - - base_type::Deserialize(src.m_data, src.m_offset, m_header->GetCodingParams()); } -namespace +void FeatureType::ParseHeader2() const { - uint32_t const kInvalidOffset = uint32_t(-1); -} - -int FeatureType::GetScaleIndex(int scale) const -{ - int const count = m_header->GetScalesCount(); - - switch (scale) + if (!m_bHeader2Parsed) { - case -2: return 0; - case -1: return count-1; - default: - for (int i = 0; i < count; ++i) - if (scale <= m_header->GetScale(i)) - return i; - return -1; + ParseCommon(); + + m_pLoader->ParseHeader2(); + m_bHeader2Parsed = true; } } -int FeatureType::GetScaleIndex(int scale, offsets_t const & offsets) const +uint32_t FeatureType::ParseGeometry(int scale) const { - int ind = -1; - int const count = static_cast(offsets.size()); - - switch (scale) + uint32_t sz = 0; + if (!m_bPointsParsed) { - case -2: - // Choose the worst existing geometry. - ind = count-1; - while (ind >= 0 && offsets[ind] == kInvalidOffset) --ind; - break; + ParseHeader2(); - case -1: - // Choose the best geometry for the last visible scale. - ind = 0; - while (ind < count && offsets[ind] == kInvalidOffset) ++ind; - break; - - default: - for (size_t i = 0; i < m_header->GetScalesCount(); ++i) - { - if (scale <= m_header->GetScale(i)) - return (offsets[i] != kInvalidOffset ? i : -1); - } + sz = m_pLoader->ParseGeometry(scale); + m_bPointsParsed = true; } + return sz; +} - if (ind >= 0 && ind < count) - return ind; - else +uint32_t FeatureType::ParseTriangles(int scale) const +{ + uint32_t sz = 0; + if (!m_bTrianglesParsed) { - CHECK ( false, ("Feature should have any geometry ...") ); - return -1; + ParseHeader2(); + + sz = m_pLoader->ParseTriangles(scale); + m_bTrianglesParsed = true; } + return sz; } namespace @@ -283,241 +180,10 @@ m2::RectD FeatureType::GetLimitRect(int scale) const return m_LimitRect; } -namespace -{ - class BitSource - { - char const * m_ptr; - uint8_t m_pos; - - public: - BitSource(char const * p) : m_ptr(p), m_pos(0) {} - - uint8_t Read(uint8_t count) - { - ASSERT_LESS ( count, 9, () ); - - uint8_t v = *m_ptr; - v >>= m_pos; - v &= ((1 << count) - 1); - - m_pos += count; - if (m_pos >= 8) - { - ASSERT_EQUAL ( m_pos, 8, () ); - ++m_ptr; - m_pos = 0; - } - - return v; - } - - char const * RoundPtr() - { - if (m_pos > 0) - { - ++m_ptr; - m_pos = 0; - } - return m_ptr; - } - }; - - template uint8_t ReadByte(TSource & src) - { - return ReadPrimitiveFromSource(src); - } -} - -void FeatureType::ParseHeader2() const -{ - ASSERT(!m_bHeader2Parsed, ()); - if (!m_bCommonParsed) - ParseCommon(); - - uint8_t ptsCount, ptsMask, trgCount, trgMask; - - BitSource bitSource(DataPtr() + m_Header2Offset); - - uint8_t const h = (Header() & HEADER_GEOTYPE_MASK); - - if (h & HEADER_GEOM_LINE) - { - ptsCount = bitSource.Read(4); - if (ptsCount == 0) - ptsMask = bitSource.Read(4); - else - { - ASSERT_GREATER ( ptsCount, 1, () ); - } - } - - if (h & HEADER_GEOM_AREA) - { - trgCount = bitSource.Read(4); - if (trgCount == 0) - trgMask = bitSource.Read(4); - } - - ArrayByteSource src(bitSource.RoundPtr()); - - if (h & HEADER_GEOM_LINE) - { - if (ptsCount > 0) - { - int const count = (ptsCount - 2 + 3) / 4; - ASSERT_LESS ( count, 4, () ); - - for (int i = 0; i < count; ++i) - { - uint32_t mask = ReadByte(src); - m_ptsSimpMask += (mask << (i << 3)); - } - - char const * start = static_cast(src.Ptr()); - - src = ArrayByteSource(serial::LoadInnerPath(src.Ptr(), ptsCount, m_CodingParams, m_Points)); - - m_InnerStats.m_Points = static_cast(src.Ptr()) - start; - } - else - ReadOffsets(src, ptsMask, m_ptsOffsets); - } - - if (h & HEADER_GEOM_AREA) - { - if (trgCount > 0) - { - trgCount += 2; - - char const * start = static_cast(src.Ptr()); - - points_t points; - src = ArrayByteSource(serial::LoadInnerTriangles(src.Ptr(), trgCount, m_CodingParams, points)); - - m_InnerStats.m_Strips = static_cast(src.Ptr()) - start; - - for (uint8_t i = 2; i < trgCount; ++i) - { - m_Triangles.push_back(points[i-2]); - m_Triangles.push_back(points[i-1]); - m_Triangles.push_back(points[i]); - } - } - else - ReadOffsets(src, trgMask, m_trgOffsets); - } - - m_bHeader2Parsed = true; - m_InnerStats.m_Size = static_cast(src.Ptr()) - DataPtr(); -} - -uint32_t FeatureType::ParseGeometry(int scale) const -{ - ASSERT(!m_bPointsParsed, ()); - if (!m_bHeader2Parsed) - ParseHeader2(); - - uint32_t sz = 0; - if (Header() & HEADER_GEOM_LINE) - { - if (m_Points.empty()) - { - // outer geometry - int const ind = GetScaleIndex(scale, m_ptsOffsets); - if (ind != -1) - { - ReaderSource src( - m_cont->GetReader(feature::GetTagForIndex(GEOMETRY_FILE_TAG, ind))); - src.Skip(m_ptsOffsets[ind]); - serial::LoadOuterPath(src, m_CodingParams, m_Points); - - sz = static_cast(src.Pos() - m_ptsOffsets[ind]); - } - } - else - { - // filter inner geometry - - size_t const count = m_Points.size(); - points_t points; - points.reserve(count); - - uint32_t const scaleIndex = GetScaleIndex(scale); - ASSERT_LESS ( scaleIndex, m_header->GetScalesCount(), () ); - - points.push_back(m_Points.front()); - for (size_t i = 1; i < count-1; ++i) - { - // check for point visibility in needed scaleIndex - if (((m_ptsSimpMask >> (2*(i-1))) & 0x3) <= scaleIndex) - points.push_back(m_Points[i]); - } - points.push_back(m_Points.back()); - - m_Points.swap(points); - } - - CalcRect(m_Points, m_LimitRect); - } - - m_bPointsParsed = true; - return sz; -} - -uint32_t FeatureType::ParseTriangles(int scale) const -{ - ASSERT(!m_bTrianglesParsed, ()); - if (!m_bHeader2Parsed) - ParseHeader2(); - - uint32_t sz = 0; - if (Header() & HEADER_GEOM_AREA) - { - if (m_Triangles.empty()) - { - uint32_t const ind = GetScaleIndex(scale, m_trgOffsets); - if (ind != -1) - { - ReaderSource src( - m_cont->GetReader(feature::GetTagForIndex(TRIANGLE_FILE_TAG, ind))); - src.Skip(m_trgOffsets[ind]); - serial::LoadOuterTriangles(src, m_CodingParams, m_Triangles); - - sz = static_cast(src.Pos() - m_trgOffsets[ind]); - } - } - - CalcRect(m_Triangles, m_LimitRect); - } - - m_bTrianglesParsed = true; - return sz; -} - -void FeatureType::ReadOffsets(ArrayByteSource & src, uint8_t mask, offsets_t & offsets) const -{ - ASSERT_GREATER ( mask, 0, () ); - - size_t index = 0; - while (mask > 0) - { - ASSERT_LESS ( index, m_header->GetScalesCount(), () ); - offsets[index++] = (mask & 0x01) ? ReadVarUint(src) : kInvalidOffset; - mask = mask >> 1; - } - - while (index < offsets.size()) - offsets[index++] = kInvalidOffset; -} - void FeatureType::ParseAll(int scale) const { - if (!m_bPointsParsed) - ParseGeometry(scale); - - if (!m_bTrianglesParsed) - ParseTriangles(scale); + ParseGeometry(scale); + ParseTriangles(scale); } FeatureType::geom_stat_t FeatureType::GetGeometrySize(int scale) const @@ -545,9 +211,11 @@ class BestMatchedLangName int m_minPriority; public: - BestMatchedLangName(int8_t const * priorities, string & result) - : m_priorities(priorities), m_result(result), m_minPriority(256) {} + : m_priorities(priorities), m_result(result), m_minPriority(256) + { + } + bool operator() (int8_t lang, string const & utf8s) { ASSERT(lang >= 0 && lang < MAX_SUPPORTED_LANGUAGES, ()); @@ -568,8 +236,7 @@ public: string FeatureType::GetPreferredDrawableName(int8_t const * priorities) const { - if (!m_bCommonParsed) - ParseCommon(); + ParseCommon(); string res; if (priorities) @@ -588,8 +255,7 @@ string FeatureType::GetPreferredDrawableName(int8_t const * priorities) const uint32_t FeatureType::GetPopulation() const { - if (!m_bCommonParsed) - ParseCommon(); + ParseCommon(); if (m_Params.rank == 0) return 1; @@ -606,12 +272,14 @@ double FeatureType::GetPopulationDrawRank() const if (feature::IsCountry(m_Types, m_Types + GetTypesCount())) return 0.0; else - return min(3.0E6, static_cast(n)) / 3.0E6; + { + double const upperBound = 3.0E6; + return min(upperBound, static_cast(n)) / upperBound; + } } uint8_t FeatureType::GetSearchRank() const { - if (!m_bCommonParsed) - ParseCommon(); + ParseCommon(); return m_Params.rank; } diff --git a/indexer/feature.hpp b/indexer/feature.hpp index 49193d8e43..ff34c711d6 100644 --- a/indexer/feature.hpp +++ b/indexer/feature.hpp @@ -1,22 +1,21 @@ #pragma once - #include "cell_id.hpp" -#include "data_header.hpp" #include "feature_data.hpp" #include "../geometry/point2d.hpp" #include "../geometry/rect2d.hpp" -#include "../coding/file_container.hpp" - -#include "../base/base.hpp" #include "../base/buffer_vector.hpp" #include "../std/string.hpp" -#include "../std/bind.hpp" +#include "../std/shared_ptr.hpp" -class ArrayByteSource; +namespace feature +{ + class LoaderBase; + class LoaderCurrent; +} /// Base feature class for storing common data (without geometry). class FeatureBase @@ -24,9 +23,16 @@ class FeatureBase static const int m_maxTypesCount = feature::max_types_count; public: - FeatureBase() : m_Offset(0) {} - typedef vector buffer_t; + typedef char const * BufferT; + + void Deserialize(feature::LoaderBase * pLoader, BufferT buffer); + + /// @name Parse functions. Do simple dispatching to m_pLoader. + //@{ + void ParseTypes() const; + void ParseCommon() const; + //@} feature::EGeomType GetFeatureType() const; @@ -40,9 +46,7 @@ public: if (!(Header() & feature::HEADER_HAS_LAYER)) return 0; - if (!m_bCommonParsed) - ParseCommon(); - + ParseCommon(); return m_Params.layer; } @@ -73,9 +77,7 @@ public: if (!HasName()) return false; - if (!m_bCommonParsed) - ParseCommon(); - + ParseCommon(); m_Params.name.ForEachRef(functor); return true; } @@ -102,37 +104,28 @@ public: template void ForEachTypeRef(FunctorT & f) const { - if (!m_bTypesParsed) - ParseTypes(); + ParseTypes(); - uint32_t const typeCount = GetTypesCount(); - for (size_t i = 0; i < typeCount; ++i) + uint32_t const count = GetTypesCount(); + for (size_t i = 0; i < count; ++i) f(m_Types[i]); } - /// @name Statistic functions. +protected: + /// @name Need for FeatureBuilder. //@{ - uint32_t GetTypesSize() const { return m_CommonOffset - m_TypesOffset; } + friend class FeatureBuilder1; + inline void SetHeader(uint8_t h) { m_Header = h; } //@} - FeatureParams GetFeatureParams() const; - inline m2::PointD GetCenter() const { return m_Center; } - -protected: - void Deserialize(buffer_t & data, uint32_t offset, serial::CodingParams const & params); string DebugString() const; + inline uint8_t Header() const { return m_Header; } + protected: - buffer_t m_Data; - uint32_t m_Offset; + shared_ptr m_pLoader; - friend class FeatureBuilder1; - - void SetHeader(uint8_t h); - - inline char const * DataPtr() const { return &m_Data[m_Offset]; } - inline uint8_t Header() const { return static_cast(*DataPtr()); } - uint32_t CalcOffset(ArrayByteSource const & source) const; + uint8_t m_Header; mutable uint32_t m_Types[m_maxTypesCount]; @@ -142,17 +135,9 @@ protected: mutable m2::RectD m_LimitRect; - serial::CodingParams m_CodingParams; - - static uint32_t const m_TypesOffset = 1; - mutable uint32_t m_CommonOffset, m_Header2Offset; - mutable bool m_bTypesParsed, m_bCommonParsed; - void ParseTypes() const; - void ParseCommon() const; - - void ParseAll() const; + friend class feature::LoaderCurrent; }; /// Working feature class with geometry. @@ -161,34 +146,14 @@ class FeatureType : public FeatureBase typedef FeatureBase base_type; public: - struct read_source_t - { - FilesContainerR m_cont; + void Deserialize(feature::LoaderBase * pLoader, BufferT buffer); - buffer_t m_data; - uint32_t m_offset; - - feature::DataHeader m_header; - - read_source_t(FilesContainerR const & cont) - : m_cont(cont), m_offset(0) - { - } - - void assign(char const * data, uint32_t size) - { - m_data.assign(data, data + size); - } - }; - - FeatureType() {} - FeatureType(read_source_t & src); - - /// @param priorities optional array of languages priorities - /// if NULL, default (0) lang will be used - string GetPreferredDrawableName(int8_t const * priorities = NULL) const; - - void Deserialize(read_source_t & src); + /// @name Parse functions. Do simple dispatching to m_pLoader. + //@{ + void ParseHeader2() const; + uint32_t ParseGeometry(int scale) const; + uint32_t ParseTriangles(int scale) const; + //@} /// @name Geometry. //@{ @@ -199,8 +164,7 @@ public: template void ForEachPointRef(FunctorT & f, int scale) const { - if (!m_bPointsParsed) - ParseGeometry(scale); + ParseGeometry(scale); if (m_Points.empty()) { @@ -224,8 +188,7 @@ public: template void ForEachTriangleRef(FunctorT & f, int scale) const { - if (!m_bTrianglesParsed) - ParseTriangles(scale); + ParseTriangles(scale); for (size_t i = 0; i < m_Triangles.size();) { @@ -246,18 +209,21 @@ public: /// For test cases only. string DebugString(int scale) const; + /// @param priorities optional array of languages priorities + /// if NULL, default (0) lang will be used + string GetPreferredDrawableName(int8_t const * priorities = NULL) const; + uint32_t GetPopulation() const; double GetPopulationDrawRank() const; uint8_t GetSearchRank() const; - string GetRoadNumber() const { return m_Params.ref; } + inline string GetRoadNumber() const { return m_Params.ref; } /// @name Statistic functions. //@{ - void ParseBeforeStatistic() const + inline void ParseBeforeStatistic() const { - if (!m_bHeader2Parsed) - ParseHeader2(); + ParseHeader2(); } struct inner_geom_stat_t @@ -289,10 +255,6 @@ public: //@} private: - void ParseHeader2() const; - uint32_t ParseGeometry(int scale) const; - uint32_t ParseTriangles(int scale) const; - void ParseAll(int scale) const; // For better result this value should be greater than 17 @@ -302,29 +264,9 @@ private: typedef buffer_vector points_t; mutable points_t m_Points, m_Triangles; - FilesContainerR * m_cont; - - feature::DataHeader const * m_header; - mutable bool m_bHeader2Parsed, m_bPointsParsed, m_bTrianglesParsed; mutable inner_geom_stat_t m_InnerStats; - mutable uint32_t m_ptsSimpMask; - - typedef array offsets_t; // should be synchronized with ARRAY_SIZE(g_arrScales) - - void ReadOffsets(ArrayByteSource & src, uint8_t mask, offsets_t & offsets) const; - - /// Get the index for geometry serialization. - /// @param[in] scale: - /// -1 : index for the best geometry - /// -2 : index for the worst geometry - /// default : needed geometry - //@{ - int GetScaleIndex(int scale) const; - int GetScaleIndex(int scale, offsets_t const & offset) const; - //@} - - mutable offsets_t m_ptsOffsets, m_trgOffsets; + friend class feature::LoaderCurrent; }; diff --git a/indexer/feature_data.hpp b/indexer/feature_data.hpp index 5e0d3dc53f..380c0f2edd 100644 --- a/indexer/feature_data.hpp +++ b/indexer/feature_data.hpp @@ -127,6 +127,8 @@ struct FeatureParamsBase class FeatureParams : public FeatureParamsBase { + typedef FeatureParamsBase BaseT; + bool m_geomTypes[3]; public: @@ -182,6 +184,26 @@ public: for (size_t i = 0; i < m_Types.size(); ++i) WriteVarUint(sink, c.GetIndexForType(m_Types[i])); - FeatureParamsBase::Write(sink, header, GetGeomType()); + BaseT::Write(sink, header, GetGeomType()); + } + + template void Read(TSrc & src) + { + using namespace feature; + + uint8_t const header = ReadPrimitiveFromSource(src); + + uint8_t const type = (header & HEADER_GEOTYPE_MASK); + if (type & HEADER_GEOM_LINE) SetGeomType(GEOM_LINE); + if (type & HEADER_GEOM_AREA) SetGeomType(GEOM_AREA); + if (type == HEADER_GEOM_POINT) SetGeomType(GEOM_POINT); + + Classificator & c = classif(); + + size_t const count = (header & HEADER_TYPE_MASK) + 1; + for (size_t i = 0; i < count; ++i) + m_Types.push_back(c.GetTypeForIndex(ReadVarUint(src))); + + BaseT::Read(src, header, GetGeomType()); } }; diff --git a/indexer/feature_loader.cpp b/indexer/feature_loader.cpp new file mode 100644 index 0000000000..17c9357bf8 --- /dev/null +++ b/indexer/feature_loader.cpp @@ -0,0 +1,335 @@ +#include "../base/SRC_FIRST.hpp" + +#include "feature_loader.hpp" +#include "feature.hpp" +#include "feature_impl.hpp" +#include "scales.hpp" +#include "geometry_serialization.hpp" +#include "classificator.hpp" + +#include "../defines.hpp" // just for file extensions + +#include "../geometry/pointu_to_uint64.hpp" + +#include "../coding/byte_stream.hpp" + +#include "../base/start_mem_debug.hpp" + + +namespace feature +{ + +uint8_t LoaderCurrent::GetHeader() +{ + return Header(); +} + +void LoaderCurrent::ParseTypes() +{ + Classificator & c = classif(); + + ArrayByteSource source(DataPtr() + m_TypesOffset); + + size_t const count = m_pF->GetTypesCount(); + for (size_t i = 0; i < count; ++i) + m_pF->m_Types[i] = c.GetTypeForIndex(ReadVarUint(source)); + + m_CommonOffset = CalcOffset(source); +} + +void LoaderCurrent::ParseCommon() +{ + ArrayByteSource source(DataPtr() + m_CommonOffset); + + uint8_t const h = Header(); + + EGeomType const type = m_pF->GetFeatureType(); + + m_pF->m_Params.Read(source, h, type); + + if (type == GEOM_POINT) + { + serial::CodingParams const & cp = GetCodingParams(); + + CoordPointT const center = PointU2PointD( + DecodeDelta(ReadVarUint(source), cp.GetBasePoint()), cp.GetCoordBits()); + + m_pF->m_Center = m2::PointD(center.first, center.second); + m_pF->m_LimitRect.Add(m_pF->m_Center); + } + + m_Header2Offset = CalcOffset(source); +} + +namespace +{ + class BitSource + { + char const * m_ptr; + uint8_t m_pos; + + public: + BitSource(char const * p) : m_ptr(p), m_pos(0) {} + + uint8_t Read(uint8_t count) + { + ASSERT_LESS ( count, 9, () ); + + uint8_t v = *m_ptr; + v >>= m_pos; + v &= ((1 << count) - 1); + + m_pos += count; + if (m_pos >= 8) + { + ASSERT_EQUAL ( m_pos, 8, () ); + ++m_ptr; + m_pos = 0; + } + + return v; + } + + char const * RoundPtr() + { + if (m_pos > 0) + { + ++m_ptr; + m_pos = 0; + } + return m_ptr; + } + }; + + template uint8_t ReadByte(TSource & src) + { + return ReadPrimitiveFromSource(src); + } +} + +void LoaderCurrent::ParseHeader2() +{ + uint8_t ptsCount, ptsMask, trgCount, trgMask; + + BitSource bitSource(DataPtr() + m_Header2Offset); + + uint8_t const h = (Header() & HEADER_GEOTYPE_MASK); + + if (h & HEADER_GEOM_LINE) + { + ptsCount = bitSource.Read(4); + if (ptsCount == 0) + ptsMask = bitSource.Read(4); + else + { + ASSERT_GREATER ( ptsCount, 1, () ); + } + } + + if (h & HEADER_GEOM_AREA) + { + trgCount = bitSource.Read(4); + if (trgCount == 0) + trgMask = bitSource.Read(4); + } + + ArrayByteSource src(bitSource.RoundPtr()); + + if (h & HEADER_GEOM_LINE) + { + if (ptsCount > 0) + { + int const count = (ptsCount - 2 + 3) / 4; + ASSERT_LESS ( count, 4, () ); + + for (int i = 0; i < count; ++i) + { + uint32_t mask = ReadByte(src); + m_ptsSimpMask += (mask << (i << 3)); + } + + char const * start = static_cast(src.Ptr()); + + src = ArrayByteSource(serial::LoadInnerPath(src.Ptr(), ptsCount, GetCodingParams(), m_pF->m_Points)); + + m_pF->m_InnerStats.m_Points = static_cast(src.Ptr()) - start; + } + else + ReadOffsets(src, ptsMask, m_ptsOffsets); + } + + if (h & HEADER_GEOM_AREA) + { + if (trgCount > 0) + { + trgCount += 2; + + char const * start = static_cast(src.Ptr()); + + FeatureType::points_t points; + src = ArrayByteSource(serial::LoadInnerTriangles(src.Ptr(), trgCount, GetCodingParams(), points)); + + m_pF->m_InnerStats.m_Strips = static_cast(src.Ptr()) - start; + + for (uint8_t i = 2; i < trgCount; ++i) + { + m_pF->m_Triangles.push_back(points[i-2]); + m_pF->m_Triangles.push_back(points[i-1]); + m_pF->m_Triangles.push_back(points[i]); + } + } + else + ReadOffsets(src, trgMask, m_trgOffsets); + } + + m_pF->m_InnerStats.m_Size = static_cast(src.Ptr()) - DataPtr(); +} + +uint32_t LoaderCurrent::ParseGeometry(int scale) +{ + uint32_t sz = 0; + if (Header() & HEADER_GEOM_LINE) + { + if (m_pF->m_Points.empty()) + { + // outer geometry + int const ind = GetScaleIndex(scale, m_ptsOffsets); + if (ind != -1) + { + ReaderSource src(m_Info.GetGeometryReader(ind)); + src.Skip(m_ptsOffsets[ind]); + serial::LoadOuterPath(src, GetCodingParams(), m_pF->m_Points); + + sz = static_cast(src.Pos() - m_ptsOffsets[ind]); + } + } + else + { + // filter inner geometry + + size_t const count = m_pF->m_Points.size(); + FeatureType::points_t points; + points.reserve(count); + + int const scaleIndex = GetScaleIndex(scale); + ASSERT_LESS ( scaleIndex, m_Info.GetScalesCount(), () ); + + points.push_back(m_pF->m_Points.front()); + for (int i = 1; i < count-1; ++i) + { + // check for point visibility in needed scaleIndex + if (((m_ptsSimpMask >> (2*(i-1))) & 0x3) <= scaleIndex) + points.push_back(m_pF->m_Points[i]); + } + points.push_back(m_pF->m_Points.back()); + + m_pF->m_Points.swap(points); + } + + CalcRect(m_pF->m_Points, m_pF->m_LimitRect); + } + + return sz; +} + +uint32_t LoaderCurrent::ParseTriangles(int scale) +{ + uint32_t sz = 0; + if (Header() & HEADER_GEOM_AREA) + { + if (m_pF->m_Triangles.empty()) + { + uint32_t const ind = GetScaleIndex(scale, m_trgOffsets); + if (ind != -1) + { + ReaderSource src(m_Info.GetTrianglesReader(ind)); + src.Skip(m_trgOffsets[ind]); + serial::LoadOuterTriangles(src, GetCodingParams(), m_pF->m_Triangles); + + sz = static_cast(src.Pos() - m_trgOffsets[ind]); + } + } + + CalcRect(m_pF->m_Triangles, m_pF->m_LimitRect); + } + + return sz; +} + +namespace +{ + uint32_t const kInvalidOffset = uint32_t(-1); +} + +void LoaderCurrent::ReadOffsets(ArrayByteSource & src, uint8_t mask, offsets_t & offsets) const +{ + ASSERT_GREATER ( mask, 0, () ); + + int index = 0; + while (mask > 0) + { + ASSERT_LESS ( index, m_Info.GetScalesCount(), () ); + offsets[index++] = (mask & 0x01) ? ReadVarUint(src) : kInvalidOffset; + mask = mask >> 1; + } + + while (index < offsets.size()) + offsets[index++] = kInvalidOffset; +} + +int LoaderCurrent::GetScaleIndex(int scale) const +{ + int const count = m_Info.GetScalesCount(); + + switch (scale) + { + case -2: return 0; + case -1: return count-1; + default: + for (int i = 0; i < count; ++i) + if (scale <= m_Info.GetScale(i)) + return i; + return -1; + } +} + +int LoaderCurrent::GetScaleIndex(int scale, offsets_t const & offsets) const +{ + int ind = -1; + int const count = static_cast(offsets.size()); + + switch (scale) + { + case -2: + // Choose the worst existing geometry. + ind = count-1; + while (ind >= 0 && offsets[ind] == kInvalidOffset) --ind; + break; + + case -1: + // Choose the best geometry for the last visible scale. + ind = 0; + while (ind < count && offsets[ind] == kInvalidOffset) ++ind; + break; + + default: + { + int const count = m_Info.GetScalesCount(); + for (int i = 0; i < count; ++i) + { + if (scale <= m_Info.GetScale(i)) + return (offsets[i] != kInvalidOffset ? i : -1); + } + } + } + + if (ind >= 0 && ind < count) + return ind; + else + { + CHECK ( false, ("Feature should have any geometry ...") ); + return -1; + } +} + +} diff --git a/indexer/feature_loader.hpp b/indexer/feature_loader.hpp new file mode 100644 index 0000000000..5d0df0092a --- /dev/null +++ b/indexer/feature_loader.hpp @@ -0,0 +1,34 @@ +#pragma once +#include "feature_loader_base.hpp" + + +namespace feature +{ + class LoaderCurrent : public LoaderBase + { + typedef LoaderBase BaseT; + + void ReadOffsets(ArrayByteSource & src, uint8_t mask, offsets_t & offsets) const; + + /// Get the index for geometry serialization. + /// @param[in] scale: + /// -1 : index for the best geometry + /// -2 : index for the worst geometry + /// default : needed geometry + //@{ + int GetScaleIndex(int scale) const; + int GetScaleIndex(int scale, offsets_t const & offsets) const; + //@} + + public: + LoaderCurrent(SharedLoadInfo const & info) : BaseT(info) {} + + virtual uint8_t GetHeader(); + + virtual void ParseTypes(); + virtual void ParseCommon(); + virtual void ParseHeader2(); + virtual uint32_t ParseGeometry(int scale); + virtual uint32_t ParseTriangles(int scale); + }; +} diff --git a/indexer/feature_loader_base.cpp b/indexer/feature_loader_base.cpp new file mode 100644 index 0000000000..ce289297f3 --- /dev/null +++ b/indexer/feature_loader_base.cpp @@ -0,0 +1,68 @@ +#include "../base/SRC_FIRST.hpp" + +#include "feature_loader_base.hpp" +#include "feature_loader.hpp" +#include "feature_impl.hpp" + +#include "../defines.hpp" + +#include "../coding/byte_stream.hpp" + +#include "../base/start_mem_debug.hpp" + + +namespace feature +{ + +//////////////////////////////////////////////////////////////////////////////////////////// +// SharedLoadInfo implementation. +//////////////////////////////////////////////////////////////////////////////////////////// + +SharedLoadInfo::SharedLoadInfo(FilesContainerR const & cont, DataHeader const & header) + : m_cont(cont), m_header(header) +{ +} + +SharedLoadInfo::ReaderT SharedLoadInfo::GetDataReader() const +{ + return m_cont.GetReader(DATA_FILE_TAG); +} + +SharedLoadInfo::ReaderT SharedLoadInfo::GetGeometryReader(int ind) const +{ + return m_cont.GetReader(GetTagForIndex(GEOMETRY_FILE_TAG, ind)); +} + +SharedLoadInfo::ReaderT SharedLoadInfo::GetTrianglesReader(int ind) const +{ + return m_cont.GetReader(GetTagForIndex(TRIANGLE_FILE_TAG, ind)); +} + +LoaderBase * SharedLoadInfo::CreateLoader() const +{ + return new LoaderCurrent(*this); +} + + +//////////////////////////////////////////////////////////////////////////////////////////// +// LoaderBase implementation. +//////////////////////////////////////////////////////////////////////////////////////////// + +LoaderBase::LoaderBase(SharedLoadInfo const & info) + : m_Info(info), m_pF(0), m_Data(0) +{ +} + +void LoaderBase::Deserialize(BufferT data) +{ + m_Data = data; + m_CommonOffset = m_Header2Offset = 0; + m_ptsSimpMask = 0; +} + +uint32_t LoaderBase::CalcOffset(ArrayByteSource const & source) const +{ + return static_cast(static_cast(source.Ptr()) - DataPtr()); +} + +} diff --git a/indexer/feature_loader_base.hpp b/indexer/feature_loader_base.hpp new file mode 100644 index 0000000000..7ed75e1e5d --- /dev/null +++ b/indexer/feature_loader_base.hpp @@ -0,0 +1,90 @@ +#pragma once +#include "coding_params.hpp" +#include "data_header.hpp" + +#include "../coding/file_container.hpp" + + +class FeatureType; +class ArrayByteSource; + +namespace feature +{ + class LoaderBase; + + /// This info is created once. + class SharedLoadInfo + { + FilesContainerR m_cont; + DataHeader const & m_header; + + typedef FilesContainerR::ReaderT ReaderT; + + public: + SharedLoadInfo(FilesContainerR const & cont, DataHeader const & header); + + ReaderT GetDataReader() const; + ReaderT GetGeometryReader(int ind) const; + ReaderT GetTrianglesReader(int ind) const; + + LoaderBase * CreateLoader() const; + + inline serial::CodingParams const & GetCodingParams() const + { + return m_header.GetCodingParams(); + } + inline int GetScalesCount() const { return m_header.GetScalesCount(); } + inline int GetScale(int i) const { return m_header.GetScale(i); } + }; + + class LoaderBase + { + public: + LoaderBase(SharedLoadInfo const & info); + virtual ~LoaderBase() {} + + inline void AssignFeature(FeatureType * p) { m_pF = p; } + + // It seems like no need to store a copy of buffer (see FeaturesVector). + typedef char const * BufferT; + + virtual uint8_t GetHeader() = 0; + + virtual void ParseTypes() = 0; + virtual void ParseCommon() = 0; + virtual void ParseHeader2() = 0; + virtual uint32_t ParseGeometry(int scale) = 0; + virtual uint32_t ParseTriangles(int scale) = 0; + + void Deserialize(BufferT data); + + inline uint32_t GetTypesSize() const { return m_CommonOffset - m_TypesOffset; } + + protected: + inline char const * DataPtr() const { return m_Data; } + + uint32_t CalcOffset(ArrayByteSource const & source) const; + + inline serial::CodingParams const & GetCodingParams() const + { + return m_Info.GetCodingParams(); + } + + uint8_t Header() const { return static_cast(*DataPtr()); } + + protected: + SharedLoadInfo const & m_Info; + FeatureType * m_pF; + + BufferT m_Data; + + static uint32_t const m_TypesOffset = 1; + uint32_t m_CommonOffset, m_Header2Offset; + + uint32_t m_ptsSimpMask; + + typedef array offsets_t; // should be synchronized with ARRAY_SIZE(g_arrScales) + + offsets_t m_ptsOffsets, m_trgOffsets; + }; +} diff --git a/indexer/feature_processor.hpp b/indexer/feature_processor.hpp index f3bdf5d662..00ad6f503e 100644 --- a/indexer/feature_processor.hpp +++ b/indexer/feature_processor.hpp @@ -1,7 +1,10 @@ #pragma once +#include "data_header.hpp" #include "features_vector.hpp" +#include "../defines.hpp" + #include "../coding/file_reader.hpp" #include "../coding/file_container.hpp" @@ -14,7 +17,11 @@ namespace feature void ForEachFromDat(ModelReaderPtr reader, ToDo & toDo) { FilesContainerR container(reader); - FeaturesVector featureSource(container); + + DataHeader header; + header.Load(container.GetReader(HEADER_FILE_TAG)); + + FeaturesVector featureSource(container, header); featureSource.ForEachOffset(bind(ref(toDo), _1, _2)); } diff --git a/indexer/features_vector.hpp b/indexer/features_vector.hpp index 9fcd5cedc5..c66bbd1bee 100644 --- a/indexer/features_vector.hpp +++ b/indexer/features_vector.hpp @@ -1,76 +1,56 @@ #pragma once #include "feature.hpp" - -#include "../defines.hpp" +#include "feature_loader_base.hpp" #include "../coding/var_record_reader.hpp" -#include "../base/logging.hpp" - - -struct FeatureReaders -{ - FilesContainerR m_cont; - FilesContainerR::ReaderT m_datR; - - FeatureReaders(FilesContainerR const & cont) - : m_cont(cont), m_datR(cont.GetReader(DATA_FILE_TAG)) - { - } -}; +/// Note! This class is NOT Thread-Safe. +/// You should have separate instance of Vector for every thread. class FeaturesVector { public: - FeaturesVector(FeatureReaders const & dataR) - : m_RecordReader(dataR.m_datR, 256), m_source(dataR.m_cont) + FeaturesVector(FilesContainerR const & cont, feature::DataHeader const & header) + : m_LoadInfo(cont, header), m_RecordReader(m_LoadInfo.GetDataReader(), 256) { - m_source.m_header.Load(dataR.m_cont.GetReader(HEADER_FILE_TAG)); } - void Get(uint64_t pos, FeatureType & feature) const + void Get(uint64_t pos, FeatureType & ft) const { - m_RecordReader.ReadRecord(pos, m_source.m_data, m_source.m_offset); - feature.Deserialize(m_source); + uint32_t offset; + m_RecordReader.ReadRecord(pos, m_buffer, offset); + + ft.Deserialize(m_LoadInfo.CreateLoader(), &m_buffer[offset]); } template void ForEachOffset(ToDo toDo) const { - m_RecordReader.ForEachRecord(feature_getter(toDo, m_source)); + m_RecordReader.ForEachRecord(DoGetFeatures(m_LoadInfo, toDo)); } - //template void ForEach(TDo const & toDo) const - //{ - // FeatureType f; - // m_RecordReader.ForEachRecord( - // bind(toDo, bind(&FeaturesVector::DeserializeFeature, this, _2, _3, &f))); - //} - - //bool IsMyData(string const & fName) const - //{ - // return m_RecordReader.IsEqual(fName); - //} - private: - template class feature_getter + template class DoGetFeatures { + feature::SharedLoadInfo const & m_loadInfo; ToDo & m_toDo; - FeatureType::read_source_t & m_source; public: - feature_getter(ToDo & toDo, FeatureType::read_source_t & src) - : m_toDo(toDo), m_source(src) + DoGetFeatures(feature::SharedLoadInfo const & loadInfo, ToDo & toDo) + : m_loadInfo(loadInfo), m_toDo(toDo) { } - void operator() (uint32_t pos, char const * data, uint32_t size) const + + void operator() (uint32_t pos, char const * data, uint32_t /*size*/) const { - FeatureType f; - m_source.assign(data, size); - f.Deserialize(m_source); - m_toDo(f, pos); + FeatureType ft; + ft.Deserialize(m_loadInfo.CreateLoader(), data); + + m_toDo(ft, pos); } }; +private: + feature::SharedLoadInfo m_LoadInfo; VarRecordReader m_RecordReader; - mutable FeatureType::read_source_t m_source; + mutable vector m_buffer; }; diff --git a/indexer/index.hpp b/indexer/index.hpp index 95bcff847f..642cf5a08f 100644 --- a/indexer/index.hpp +++ b/indexer/index.hpp @@ -267,11 +267,10 @@ private: : m_action(INDEX_DO_NOTHING), m_file(file), m_lockCount(0), m_queriesSkipped(0) { - feature::DataHeader header; - header.Load(FilesContainerR(GetPlatform().GetReader(m_file)).GetReader(HEADER_FILE_TAG)); + m_header.LoadForVersion(FilesContainerR(GetPlatform().GetReader(m_file))); - m_rect = header.GetBounds(); - m_scaleRange = header.GetScaleRange(); + m_rect = m_header.GetBounds(); + m_scaleRange = m_header.GetScaleRange(); } IndexT * Lock(int scale, m2::RectD const & occlusionRect) @@ -322,7 +321,7 @@ private: search::SearchInfo * GetSearchInfo() const { - return new search::SearchInfo(FilesContainerR(GetPlatform().GetReader(m_file))); + return new search::SearchInfo(FilesContainerR(GetPlatform().GetReader(m_file)), m_header); } void CloseIfUnlocked() @@ -357,8 +356,7 @@ private: { if (p == 0) { - FilesContainerR container(GetPlatform().GetReader(m_file)); - p = new IndexT(container); + p = new IndexT(FilesContainerR(GetPlatform().GetReader(m_file)), m_header); } } @@ -383,6 +381,8 @@ private: } string m_file; + + feature::DataHeader m_header; m2::RectD m_rect; pair m_scaleRange; @@ -425,9 +425,8 @@ template class OffsetToFeatureAdapter : publ public: typedef typename BaseT::Query Query; - explicit OffsetToFeatureAdapter(FilesContainerR const & container) - : BaseT(container.GetReader(INDEX_FILE_TAG)), - m_FeatureVector(container) + OffsetToFeatureAdapter(FilesContainerR const & cont, feature::DataHeader const & header) + : BaseT(cont.GetReader(INDEX_FILE_TAG)), m_FeatureVector(cont, header) { } diff --git a/indexer/index_builder.cpp b/indexer/index_builder.cpp index dab305396b..4bb9966adf 100644 --- a/indexer/index_builder.cpp +++ b/indexer/index_builder.cpp @@ -3,7 +3,7 @@ #include "../defines.hpp" -#include "../coding/file_container.hpp" +#include "../base/logging.hpp" namespace indexer @@ -13,16 +13,16 @@ namespace indexer try { FilesContainerR readCont(datFile); - FeaturesVector featuresVector(readCont); - - FilesContainerW writeCont(datFile, FileWriter::OP_APPEND); - - FileWriter writer = writeCont.GetWriter(INDEX_FILE_TAG); feature::DataHeader header; header.Load(readCont.GetReader(HEADER_FILE_TAG)); - BuildIndex(header.GetScale(header.GetScalesCount()-1) + 1, featuresVector, writer, tmpFile); + FeaturesVector featuresVector(readCont, header); + + FilesContainerW writeCont(datFile, FileWriter::OP_APPEND); + FileWriter writer = writeCont.GetWriter(INDEX_FILE_TAG); + + BuildIndex(header.GetLastScale() + 1, featuresVector, writer, tmpFile); } catch (Reader::Exception const & e) { diff --git a/indexer/indexer.pro b/indexer/indexer.pro index a6299f5049..b3398fe921 100644 --- a/indexer/indexer.pro +++ b/indexer/indexer.pro @@ -33,6 +33,8 @@ SOURCES += \ types_mapping.cpp \ search_index_builder.cpp \ coding_params.cpp \ + feature_loader_base.cpp \ + feature_loader.cpp \ HEADERS += \ feature.hpp \ @@ -70,3 +72,5 @@ HEADERS += \ types_mapping.hpp \ search_index_builder.hpp \ coding_params.hpp \ + feature_loader_base.hpp \ + feature_loader.hpp \ diff --git a/indexer/indexer_tests/index_builder_test.cpp b/indexer/indexer_tests/index_builder_test.cpp index 96f86cf48b..a54b115715 100644 --- a/indexer/indexer_tests/index_builder_test.cpp +++ b/indexer/indexer_tests/index_builder_test.cpp @@ -27,7 +27,11 @@ UNIT_TEST(BuildIndexTest) // Build index. vector serialIndex; { - FeaturesVector featuresVector(originalContainer); + feature::DataHeader header; + header.Load(originalContainer.GetReader(HEADER_FILE_TAG)); + + FeaturesVector featuresVector(originalContainer, header); + MemWriter > serialWriter(serialIndex); indexer::BuildIndex(ScaleIndexBase::NUM_BUCKETS, featuresVector, serialWriter, "build_index_test"); diff --git a/indexer/search_index_builder.cpp b/indexer/search_index_builder.cpp index 179601313a..445b51c4f1 100644 --- a/indexer/search_index_builder.cpp +++ b/indexer/search_index_builder.cpp @@ -1,13 +1,21 @@ #include "search_index_builder.hpp" #include "features_vector.hpp" + +#include "../defines.hpp" + #include "../search/search_trie.hpp" #include "../search/string_search_utils.hpp" + #include "../coding/trie_builder.hpp" #include "../coding/writer.hpp" + #include "../base/string_utils.hpp" +#include "../base/logging.hpp" + #include "../std/algorithm.hpp" #include "../std/vector.hpp" + namespace { @@ -89,7 +97,11 @@ bool indexer::BuildSearchIndexFromDatFile(string const & datFile) try { FilesContainerR readCont(datFile); - FeaturesVector featuresVector(readCont); + + feature::DataHeader header; + header.Load(readCont.GetReader(HEADER_FILE_TAG)); + + FeaturesVector featuresVector(readCont, header); FilesContainerW writeCont(datFile, FileWriter::OP_APPEND); @@ -114,4 +126,3 @@ bool indexer::BuildSearchIndexFromDatFile(string const & datFile) return true; } - diff --git a/search/engine.cpp b/search/engine.cpp index f47bf2e1aa..0d56401722 100644 --- a/search/engine.cpp +++ b/search/engine.cpp @@ -5,6 +5,8 @@ #include "../platform/concurrent_runner.hpp" +#include "../base/logging.hpp" + #include "../std/function.hpp" #include "../std/string.hpp" #include "../std/vector.hpp" diff --git a/search/query.cpp b/search/query.cpp index 5f76e452f9..644e1e814a 100644 --- a/search/query.cpp +++ b/search/query.cpp @@ -4,12 +4,17 @@ #include "latlon_match.hpp" #include "string_search_utils.hpp" #include "search_trie_matching.hpp" + #include "../indexer/feature_visibility.hpp" + #include "../base/exception.hpp" #include "../base/stl_add.hpp" +#include "../base/logging.hpp" + #include "../std/algorithm.hpp" #include "../std/scoped_ptr.hpp" + namespace search { namespace impl diff --git a/search/search_trie.hpp b/search/search_trie.hpp index bf515cc481..4b62760373 100644 --- a/search/search_trie.hpp +++ b/search/search_trie.hpp @@ -1,4 +1,6 @@ #pragma once +#include "../defines.hpp" + #include "../indexer/features_vector.hpp" #include "../coding/reader.hpp" @@ -50,8 +52,8 @@ struct EdgeValueReader scoped_ptr m_iterator; public: - SearchInfo(FilesContainerR const & cont) - : m_features(cont), + SearchInfo(FilesContainerR const & cont, feature::DataHeader const & header) + : m_features(cont, header), m_iterator(::trie::reader::ReadTrie( cont.GetReader(SEARCH_INDEX_FILE_TAG), trie::ValueReader(),