diff --git a/coding/coding_tests/var_record_reader_test.cpp b/coding/coding_tests/var_record_reader_test.cpp index 81b05c07a0..1195b6c71b 100644 --- a/coding/coding_tests/var_record_reader_test.cpp +++ b/coding/coding_tests/var_record_reader_test.cpp @@ -20,10 +20,12 @@ namespace struct SaveForEachParams { explicit SaveForEachParams(vector > & data) : m_Data(data) {} - void operator () (uint64_t pos, char const * pData, uint32_t size) const + + void operator()(uint64_t pos, vector && data) const { - m_Data.push_back(make_pair(pos, string(pData, pData + size))); + m_Data.emplace_back(pos, string(data.begin(), data.end())); } + vector > & m_Data; }; @@ -37,7 +39,7 @@ UNIT_TEST(VarRecordReader_Simple) size_t const longStringSize = sizeof(longString) - 1; TEST_GREATER(longStringSize, 128, ()); { - MemWriter > writer(data); + MemWriter> writer(data); WriteVarUint(writer, 3U); // 0 writer.Write("abc", 3); // 1 WriteVarUint(writer, longStringSize); // 4 @@ -47,34 +49,22 @@ UNIT_TEST(VarRecordReader_Simple) // 11 + longStringSize } - uint32_t chunkSizes[] = {4, 5, 63, 64, 65, 1000}; - for (uint32_t chunkSize = 0; chunkSize < ARRAY_SIZE(chunkSizes); ++chunkSize) - { - MemReader reader(&data[0], data.size()); - VarRecordReader recordReader( - reader, chunkSizes[chunkSize]); + MemReader reader(&data[0], data.size()); + VarRecordReader recordReader(reader); - vector r; - uint32_t offset, size; + auto r = recordReader.ReadRecord(0); + TEST_EQUAL(string(r.begin(), r.end()), "abc", ()); - TEST_EQUAL(4, recordReader.ReadRecord(0, r, offset, size), ()); - r.resize(size); - TEST_EQUAL(string(r.begin() + offset, r.end()), "abc", ()); + r = recordReader.ReadRecord(6 + longStringSize); + TEST_EQUAL(string(r.begin(), r.end()), "defg", ()); - TEST_EQUAL(11 + longStringSize, recordReader.ReadRecord(6 + longStringSize, r, offset, size), ()); - r.resize(size); - TEST_EQUAL(string(r.begin() + offset, r.end()), "defg", ()); + r = recordReader.ReadRecord(4); + TEST_EQUAL(string(r.begin(), r.end()), longString, ()); - TEST_EQUAL(6 + longStringSize, recordReader.ReadRecord(4, r, offset, size), ()); - r.resize(size); - TEST_EQUAL(string(r.begin() + offset, r.end()), longString, ()); - - vector > forEachCalls; - recordReader.ForEachRecord(SaveForEachParams(forEachCalls)); - vector > expectedForEachCalls; - expectedForEachCalls.push_back(pair(0, "abc")); - expectedForEachCalls.push_back(pair(4, longString)); - expectedForEachCalls.push_back(pair(6 + longStringSize, "defg")); - TEST_EQUAL(forEachCalls, expectedForEachCalls, ()); - } + vector> forEachCalls; + recordReader.ForEachRecord(SaveForEachParams(forEachCalls)); + vector> expectedForEachCalls = {{0, "abc"}, + {4, longString}, + {6 + longStringSize, "defg"}}; + TEST_EQUAL(forEachCalls, expectedForEachCalls, ()); } diff --git a/coding/var_record_reader.hpp b/coding/var_record_reader.hpp index 0be5879ac6..c72561931f 100644 --- a/coding/var_record_reader.hpp +++ b/coding/var_record_reader.hpp @@ -11,77 +11,40 @@ #include #include -inline uint32_t VarRecordSizeReaderVarint(ArrayByteSource & source) -{ - return ReadVarUint(source); -} - -inline uint32_t VarRecordSizeReaderFixed(ArrayByteSource & source) -{ - return ReadPrimitiveFromSource(source); -} - -// Efficiently reads records, encoded as [VarUint size] [Data] .. [VarUint size] [Data]. -// If size of a record is less than expectedRecordSize, exactly 1 Reader.Read() call is made, -// otherwise exactly 2 Reader.Read() calls are made. -// Second template parameter is strategy for reading record size, -// either &VarRecordSizeReaderVarint or &VarRecordSizeReaderFixed. -template +// Reads records, encoded as [VarUint size] [Data] .. [VarUint size] [Data]. +template class VarRecordReader { public: - VarRecordReader(ReaderT const & reader, uint32_t expectedRecordSize) - : m_Reader(reader), m_ReaderSize(reader.Size()), m_ExpectedRecordSize(expectedRecordSize) + VarRecordReader(ReaderT const & reader) : m_reader(reader), m_readerSize(reader.Size()) {} + + std::vector ReadRecord(uint64_t const pos) const { - ASSERT_GREATER_OR_EQUAL(expectedRecordSize, 4, ()); + ASSERT_LESS(pos, m_readerSize, ()); + ReaderSource source(m_reader); + source.Skip(pos); + uint32_t const recordSize = ReadVarUint(source); + std::vector buffer(recordSize); + source.Read(buffer.data(), recordSize); + return buffer; } - uint64_t ReadRecord(uint64_t const pos, std::vector & buffer, uint32_t & recordOffset, - uint32_t & actualSize) const - { - ASSERT_LESS(pos, m_ReaderSize, ()); - uint32_t const initialSize = static_cast( - std::min(static_cast(m_ExpectedRecordSize), m_ReaderSize - pos)); - - if (buffer.size() < initialSize) - buffer.resize(initialSize); - - m_Reader.Read(pos, &buffer[0], initialSize); - ArrayByteSource source(&buffer[0]); - uint32_t const recordSize = VarRecordSizeReaderFn(source); - uint32_t const recordSizeSize = static_cast(source.PtrC() - &buffer[0]); - uint32_t const fullSize = recordSize + recordSizeSize; - ASSERT_LESS_OR_EQUAL(pos + fullSize, m_ReaderSize, ()); - if (buffer.size() < fullSize) - buffer.resize(fullSize); - if (initialSize < fullSize) - m_Reader.Read(pos + initialSize, &buffer[initialSize], fullSize - initialSize); - - recordOffset = recordSizeSize; - actualSize = fullSize; - return pos + fullSize; - } - - template - void ForEachRecord(F const & f) const + void ForEachRecord(std::function &&)> const & f) const { uint64_t pos = 0; - std::vector buffer; - while (pos < m_ReaderSize) + ReaderSource source(m_reader); + while (pos < m_readerSize) { - uint32_t offset = 0, size = 0; - uint64_t nextPos = ReadRecord(pos, buffer, offset, size); - // uint64_t -> uint32_t : assume that feature dat file not more than 4Gb - f(static_cast(pos), &buffer[offset], static_cast(size - offset)); - pos = nextPos; + uint32_t const recordSize = ReadVarUint(source); + std::vector buffer(recordSize); + source.Read(buffer.data(), recordSize); + f(static_cast(pos), std::move(buffer)); + pos = source.Pos(); } ASSERT_EQUAL(pos, m_ReaderSize, ()); } - bool IsEqual(std::string const & fName) const { return m_Reader.IsEqual(fName); } - protected: - ReaderT m_Reader; - uint64_t m_ReaderSize; - uint32_t m_ExpectedRecordSize; // Expected size of a record. + ReaderT m_reader; + uint64_t m_readerSize; }; diff --git a/indexer/feature.cpp b/indexer/feature.cpp index c9d335743f..8f0d8b100c 100644 --- a/indexer/feature.cpp +++ b/indexer/feature.cpp @@ -109,13 +109,13 @@ int GetScaleIndex(SharedLoadInfo const & loadInfo, int scale, return -1; } -uint32_t CalcOffset(ArrayByteSource const & source, FeatureType::Buffer const data) +uint32_t CalcOffset(ArrayByteSource const & source, const char * start) { - ASSERT_GREATER_OR_EQUAL(source.PtrC(), data, ()); - return static_cast(source.PtrC() - data); + ASSERT_GREATER_OR_EQUAL(source.PtrC(), start, ()); + return static_cast(source.PtrC() - start); } -uint8_t Header(FeatureType::Buffer const data) { return static_cast(*data); } +uint8_t Header(FeatureType::Buffer const & data) { return static_cast(data[0]); } void ReadOffsets(SharedLoadInfo const & loadInfo, ArrayByteSource & src, uint8_t mask, FeatureType::GeometryOffsets & offsets) @@ -181,11 +181,10 @@ uint8_t ReadByte(TSource & src) } } // namespace -FeatureType::FeatureType(SharedLoadInfo const * loadInfo, Buffer buffer) +FeatureType::FeatureType(SharedLoadInfo const * loadInfo, Buffer && buffer) : m_data(buffer) { CHECK(loadInfo, ()); m_loadInfo = loadInfo; - m_data = buffer; m_header = Header(m_data); m_offsets.Reset(); @@ -270,7 +269,7 @@ void FeatureType::ParseTypes() auto const typesOffset = sizeof(m_header); Classificator & c = classif(); - ArrayByteSource source(m_data + typesOffset); + ArrayByteSource source(m_data.data() + typesOffset); size_t const count = GetTypesCount(); uint32_t index = 0; @@ -290,7 +289,7 @@ void FeatureType::ParseTypes() throw; } - m_offsets.m_common = CalcOffset(source, m_data); + m_offsets.m_common = CalcOffset(source, m_data.data()); m_parsed.m_types = true; } @@ -302,7 +301,7 @@ void FeatureType::ParseCommon() CHECK(m_loadInfo, ()); ParseTypes(); - ArrayByteSource source(m_data + m_offsets.m_common); + ArrayByteSource source(m_data.data() + m_offsets.m_common); uint8_t const h = Header(m_data); m_params.Read(source, h); @@ -312,7 +311,7 @@ void FeatureType::ParseCommon() m_limitRect.Add(m_center); } - m_offsets.m_header2 = CalcOffset(source, m_data); + m_offsets.m_header2 = CalcOffset(source, m_data.data()); m_parsed.m_common = true; } @@ -341,7 +340,7 @@ void FeatureType::ParseHeader2() ParseCommon(); uint8_t ptsCount = 0, ptsMask = 0, trgCount = 0, trgMask = 0; - BitSource bitSource(m_data + m_offsets.m_header2); + BitSource bitSource(m_data.data() + m_offsets.m_header2); auto const headerGeomType = static_cast(Header(m_data) & HEADER_MASK_GEOMTYPE); if (headerGeomType == HeaderGeomType::Line) @@ -400,7 +399,7 @@ void FeatureType::ParseHeader2() ReadOffsets(*m_loadInfo, src, trgMask, m_offsets.m_trg); } } - m_innerStats.m_size = CalcOffset(src, m_data); + m_innerStats.m_size = CalcOffset(src, m_data.data()); m_parsed.m_header2 = true; } diff --git a/indexer/feature.hpp b/indexer/feature.hpp index 76d1d01cbf..d659e023fa 100644 --- a/indexer/feature.hpp +++ b/indexer/feature.hpp @@ -31,10 +31,10 @@ class MapObject; class FeatureType { public: - using Buffer = char const *; + using Buffer = std::vector; using GeometryOffsets = buffer_vector; - FeatureType(feature::SharedLoadInfo const * loadInfo, Buffer buffer); + FeatureType(feature::SharedLoadInfo const * loadInfo, Buffer && buffer); FeatureType(osm::MapObject const & emo); feature::GeomType GetGeomType() const; @@ -244,8 +244,7 @@ private: // Non-owning pointer to shared load info. SharedLoadInfo created once per FeaturesVector. feature::SharedLoadInfo const * m_loadInfo = nullptr; - // Raw pointer to data buffer. - Buffer m_data = nullptr; + Buffer m_data; ParsedFlags m_parsed; Offsets m_offsets; diff --git a/indexer/features_vector.cpp b/indexer/features_vector.cpp index eabe22303a..5c9c29febd 100644 --- a/indexer/features_vector.cpp +++ b/indexer/features_vector.cpp @@ -7,10 +7,8 @@ std::unique_ptr FeaturesVector::GetByIndex(uint32_t index) const { - uint32_t offset = 0, size = 0; auto const ftOffset = m_table ? m_table->GetFeatureOffset(index) : index; - m_recordReader.ReadRecord(ftOffset, m_buffer, offset, size); - return std::make_unique(&m_loadInfo, &m_buffer[offset]); + return std::make_unique(&m_loadInfo, m_recordReader.ReadRecord(ftOffset)); } size_t FeaturesVector::GetNumFeatures() const diff --git a/indexer/features_vector.hpp b/indexer/features_vector.hpp index 14147800df..1e27adc82f 100644 --- a/indexer/features_vector.hpp +++ b/indexer/features_vector.hpp @@ -20,7 +20,7 @@ class FeaturesVector public: FeaturesVector(FilesContainerR const & cont, feature::DataHeader const & header, feature::FeaturesOffsetsTable const * table) - : m_loadInfo(cont, header), m_recordReader(m_loadInfo.GetDataReader(), 256), m_table(table) + : m_loadInfo(cont, header), m_recordReader(m_loadInfo.GetDataReader()), m_table(table) { } @@ -31,8 +31,8 @@ public: template void ForEach(ToDo && toDo) const { uint32_t index = 0; - m_recordReader.ForEachRecord([&](uint32_t pos, char const * data, uint32_t /*size*/) { - FeatureType ft(&m_loadInfo, data); + m_recordReader.ForEachRecord([&](uint32_t pos, std::vector && data) { + FeatureType ft(&m_loadInfo, std::move(data)); // We can't properly set MwmId here, because FeaturesVector // works with FileContainerR, not with MwmId/MwmHandle/MwmValue. @@ -45,18 +45,15 @@ public: template static void ForEachOffset(ModelReaderPtr reader, ToDo && toDo) { - VarRecordReader recordReader(reader, 256); - recordReader.ForEachRecord([&] (uint32_t pos, char const * /*data*/, uint32_t /*size*/) - { - toDo(pos); - }); + VarRecordReader recordReader(reader); + recordReader.ForEachRecord([&](uint32_t pos, std::vector && /* data */) { toDo(pos); }); } private: friend class FeaturesVectorTest; feature::SharedLoadInfo m_loadInfo; - VarRecordReader m_recordReader; + VarRecordReader m_recordReader; mutable std::vector m_buffer; feature::FeaturesOffsetsTable const * m_table; }; diff --git a/indexer/indexer_tests/CMakeLists.txt b/indexer/indexer_tests/CMakeLists.txt index f6aeec2c6a..2b051e2312 100644 --- a/indexer/indexer_tests/CMakeLists.txt +++ b/indexer/indexer_tests/CMakeLists.txt @@ -27,6 +27,7 @@ set( postcodes_matcher_tests.cpp postcodes_tests.cpp rank_table_test.cpp + read_features_test.cpp scale_index_reading_tests.cpp scales_test.cpp search_string_utils_test.cpp diff --git a/indexer/indexer_tests/read_features_test.cpp b/indexer/indexer_tests/read_features_test.cpp new file mode 100644 index 0000000000..f2aa92e20a --- /dev/null +++ b/indexer/indexer_tests/read_features_test.cpp @@ -0,0 +1,38 @@ +#include "testing/testing.hpp" + +#include "indexer/classificator_loader.hpp" +#include "indexer/data_source.hpp" + +#include "platform/local_country_file.hpp" + +#include +#include +#include + +using namespace std; + +UNIT_TEST(ReadFeatures_Smoke) +{ + classificator::Load(); + + FrozenDataSource dataSource; + dataSource.RegisterMap(platform::LocalCountryFile::MakeForTesting("minsk-pass")); + + vector> infos; + dataSource.GetMwmsInfo(infos); + CHECK_EQUAL(infos.size(), 1, ()); + + auto handle = dataSource.GetMwmHandleById(MwmSet::MwmId(infos[0])); + + FeaturesLoaderGuard const guard(dataSource, handle.GetId()); + LOG(LINFO, (guard.GetNumFeatures())); + for (uint32_t i = 0; i < guard.GetNumFeatures() - 1; ++i) + { + LOG(LINFO, ("Trying", i, i + 1)); + auto ft1 = guard.GetFeatureByIndex(i); + auto ft2 = guard.GetFeatureByIndex(i + 1); + + ft2->ForEachType([](auto const t) {}); + ft1->ForEachType([](auto const t) {}); + } +}