diff --git a/coding/coding_tests/csv_reader_test.cpp b/coding/coding_tests/csv_reader_test.cpp index 60d3f77085..48518c1139 100644 --- a/coding/coding_tests/csv_reader_test.cpp +++ b/coding/coding_tests/csv_reader_test.cpp @@ -25,7 +25,8 @@ UNIT_TEST(CSVReaderSmoke) { auto const fileName = "test.csv"; ScopedFile sf(fileName, kCSV1); - FileReader fileReader(sf.GetFullPath()); + FileReader fileReader(sf.GetFullPath(), FileReader::kDefaultLogPageSize, + FileReader::kDefaultLogPageCount); CSVReader reader; reader.Read(fileReader, [](File const & file) { TEST_EQUAL(file.size(), 1, ()); @@ -49,7 +50,8 @@ UNIT_TEST(CSVReaderCustomDelimiter) { auto const fileName = "test.csv"; ScopedFile sf(fileName, kCSV2); - FileReader fileReader(sf.GetFullPath()); + FileReader fileReader(sf.GetFullPath(), FileReader::kDefaultLogPageSize, + FileReader::kDefaultLogPageCount); CSVReader reader; CSVReader::Params p; p.m_readHeader = true; @@ -67,7 +69,8 @@ UNIT_TEST(CSVReaderEmptyFile) { auto const fileName = "test.csv"; ScopedFile sf(fileName, kCSV2); - FileReader fileReader(sf.GetFullPath()); + FileReader fileReader(sf.GetFullPath(), FileReader::kDefaultLogPageSize, + FileReader::kDefaultLogPageCount); CSVReader reader; reader.Read(fileReader, [](File const & file) { TEST_EQUAL(file.size(), 0, ()); }); diff --git a/coding/coding_tests/reader_test.cpp b/coding/coding_tests/reader_test.cpp index 0da69bb07a..d751fea5e2 100644 --- a/coding/coding_tests/reader_test.cpp +++ b/coding/coding_tests/reader_test.cpp @@ -30,7 +30,8 @@ UNIT_TEST(FileReaderSmokeTest) } { - FileReader fileReader("reader_test_tmp.dat"); + FileReader fileReader("reader_test_tmp.dat", FileReader::kDefaultLogPageSize, + FileReader::kDefaultLogPageCount); TestReader(fileReader); } FileWriter::DeleteFileX("reader_test_tmp.dat"); @@ -47,7 +48,9 @@ UNIT_TEST(BufferReaderSmokeTest) writer.Write(&data[0], data.size()); } - BufferReader r2(FileReader("reader_test_tmp.dat"), 7); + BufferReader r2(FileReader("reader_test_tmp.dat", FileReader::kDefaultLogPageSize, + FileReader::kDefaultLogPageCount), + 7); TestReader(r2); FileWriter::DeleteFileX("reader_test_tmp.dat"); } @@ -63,7 +66,8 @@ UNIT_TEST(FileReaderNonExistentFileTest) { try { - FileReader reader("skjhfaxniauiuq2bmnszmn093sklsd"); + FileReader reader("skjhfaxniauiuq2bmnszmn093sklsd", FileReader::kDefaultLogPageSize, + FileReader::kDefaultLogPageCount); TEST(false, ("Exception should be thrown!")); } catch (FileReader::OpenException &) @@ -81,7 +85,8 @@ UNIT_TEST(FileReaderReadAsText) { string text; - FileReader(fName).ReadAsString(text); + FileReader(fName, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount) + .ReadAsString(text); TEST_EQUAL(text, fName, ()); } @@ -99,7 +104,8 @@ UNIT_TEST(ReaderStreamBuf) } { - ReaderStreamBuf buffer(make_unique(name)); + ReaderStreamBuf buffer(make_unique(name, FileReader::kDefaultLogPageSize, + FileReader::kDefaultLogPageCount)); istream s(&buffer); std::string str; diff --git a/coding/coding_tests/reader_writer_ops_test.cpp b/coding/coding_tests/reader_writer_ops_test.cpp index ecf6ff2081..dc4c4526b9 100644 --- a/coding/coding_tests/reader_writer_ops_test.cpp +++ b/coding/coding_tests/reader_writer_ops_test.cpp @@ -21,7 +21,7 @@ namespace } { - FileReader reader(tmpFile); + FileReader reader(tmpFile, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount); buffer.clear(); MemWriter > writer(buffer); rw_ops::Reverse(reader, writer); @@ -64,7 +64,7 @@ UNIT_TEST(Reverse_Smoke) { { FillRandFile(tmpFile, 10 * 1024 + 527); - FileReader reader(tmpFile); + FileReader reader(tmpFile, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount); vector buffer; GetReverseForReaderAndTmpFile(reader, buffer); diff --git a/coding/coding_tests/writer_test.cpp b/coding/coding_tests/writer_test.cpp index 7d5277f6c2..c08afb79f1 100644 --- a/coding/coding_tests/writer_test.cpp +++ b/coding/coding_tests/writer_test.cpp @@ -50,7 +50,7 @@ UNIT_TEST(FileWriter_Smoke) } vector s; { - FileReader reader(fileName); + FileReader reader(fileName, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount); s.resize(reader.Size()); reader.Read(0, &s[0], reader.Size()); } @@ -85,7 +85,7 @@ UNIT_TEST(SubWriter_FileWriter_Smoke) } vector s; { - FileReader reader(fileName); + FileReader reader(fileName, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount); s.resize(reader.Size()); reader.Read(0, &s[0], reader.Size()); } @@ -101,13 +101,13 @@ UNIT_TEST(FileWriter_DeleteFile) writer.Write("123", 3); } { - FileReader reader(fileName); + FileReader reader(fileName, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount); TEST_EQUAL(reader.Size(), 3, ()); } FileWriter::DeleteFileX(fileName); try { - FileReader reader(fileName); + FileReader reader(fileName, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount); TEST(false, ("Exception should be thrown!")); } catch (FileReader::OpenException & ) @@ -127,7 +127,7 @@ UNIT_TEST(FileWriter_AppendAndOpenExisting) writer.Write("abcd", 4); } { - FileReader reader(fileName); + FileReader reader(fileName, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount); TEST_EQUAL(reader.Size(), 4, ()); string s(static_cast(reader.Size()), 0); reader.Read(0, &s[0], s.size()); @@ -138,7 +138,7 @@ UNIT_TEST(FileWriter_AppendAndOpenExisting) writer.Write("123", 3); } { - FileReader reader(fileName); + FileReader reader(fileName, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount); TEST_EQUAL(reader.Size(), 3, ()); } { @@ -146,7 +146,7 @@ UNIT_TEST(FileWriter_AppendAndOpenExisting) writer.Write("4", 1); } { - FileReader reader(fileName); + FileReader reader(fileName, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount); TEST_EQUAL(reader.Size(), 4, ()); string s(static_cast(reader.Size()), 0); reader.Read(0, &s[0], s.size()); @@ -158,7 +158,7 @@ UNIT_TEST(FileWriter_AppendAndOpenExisting) writer.Write("56", 2); } { - FileReader reader(fileName); + FileReader reader(fileName, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount); TEST_EQUAL(reader.Size(), 4, ()); string s(static_cast(reader.Size()), 0); reader.Read(0, &s[0], 4); @@ -219,7 +219,7 @@ UNIT_TEST(FileWriter_Chunks) WriteTestData2(fileWriter); } { - FileReader r(TEST_FILE); + FileReader r(TEST_FILE, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount); ReadTestData(r); } FileWriter::DeleteFileX(TEST_FILE); diff --git a/coding/coding_tests/zip_creator_test.cpp b/coding/coding_tests/zip_creator_test.cpp index 781f6ff9f4..2c3592731c 100644 --- a/coding/coding_tests/zip_creator_test.cpp +++ b/coding/coding_tests/zip_creator_test.cpp @@ -19,7 +19,10 @@ void CreateAndTestZip(string const & filePath, string const & zipPath) ZipFileReader::FileListT files; ZipFileReader::FilesList(zipPath, files); - TEST_EQUAL(files[0].second, FileReader(filePath).Size(), ()); + TEST_EQUAL(files[0].second, + FileReader(filePath, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount) + .Size(), + ()); string const unzippedFile = "unzipped.tmp"; ZipFileReader::UnzipFile(zipPath, files[0].first, unzippedFile); diff --git a/coding/coding_tests/zip_reader_test.cpp b/coding/coding_tests/zip_reader_test.cpp index bb8781dfc7..e7ac65d061 100644 --- a/coding/coding_tests/zip_reader_test.cpp +++ b/coding/coding_tests/zip_reader_test.cpp @@ -175,13 +175,15 @@ UNIT_TEST(ZipExtract) string s; ZipFileReader::UnzipFile(ZIPFILE, files[0].first, OUTFILE); { - FileReader(OUTFILE).ReadAsString(s); + FileReader(OUTFILE, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount) + .ReadAsString(s); } TEST_EQUAL(s, "aaaaaaaaaa\x0A", ()); // OUTFILE should be rewritten correctly in the next lines ZipFileReader::UnzipFile(ZIPFILE, files[1].first, OUTFILE); { - FileReader(OUTFILE).ReadAsString(s); + FileReader(OUTFILE, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount) + .ReadAsString(s); } TEST_EQUAL(s, "Holalala\x0A", ()); FileWriter::DeleteFileX(OUTFILE); diff --git a/coding/file_container.cpp b/coding/file_container.cpp index 69d4a698f7..55845d6834 100644 --- a/coding/file_container.cpp +++ b/coding/file_container.cpp @@ -211,7 +211,7 @@ FilesMappingContainer::~FilesMappingContainer() void FilesMappingContainer::Open(string const & fName) { { - FileReader reader(fName); + FileReader reader(fName, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount); ReadInfo(reader); } @@ -242,7 +242,8 @@ FileReader FilesMappingContainer::GetReader(Tag const & tag) const Info const * p = GetInfo(tag); if (!p) MYTHROW(Reader::OpenException, ("Can't find section:", m_name, tag)); - return FileReader(m_name).SubReader(p->m_offset, p->m_size); + return FileReader(m_name, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount) + .SubReader(p->m_offset, p->m_size); } ///////////////////////////////////////////////////////////////////////////// @@ -307,7 +308,7 @@ void FilesContainerW::Open(FileWriter::Op op) case FileWriter::OP_WRITE_EXISTING: { // read an existing service info - FileReader reader(m_name); + FileReader reader(m_name, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount); ReadInfo(reader); } @@ -350,7 +351,8 @@ FilesContainerW::~FilesContainerW() uint64_t FilesContainerW::SaveCurrentSize() { ASSERT(!m_bFinished, ()); - uint64_t const curr = FileReader(m_name).Size(); + uint64_t const curr = + FileReader(m_name, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount).Size(); if (!m_info.empty()) m_info.back().m_size = curr - m_info.back().m_offset; return curr; @@ -426,7 +428,9 @@ FileWriter FilesContainerW::GetWriter(Tag const & tag) void FilesContainerW::Write(string const & fPath, Tag const & tag) { - Write(ModelReaderPtr(make_unique(fPath)), tag); + Write(ModelReaderPtr(make_unique(fPath, FileReader::kDefaultLogPageSize, + FileReader::kDefaultLogPageCount)), + tag); } void FilesContainerW::Write(ModelReaderPtr reader, Tag const & tag) diff --git a/coding/file_reader.cpp b/coding/file_reader.cpp index 500d0e9fbd..3980d39703 100644 --- a/coding/file_reader.cpp +++ b/coding/file_reader.cpp @@ -1,4 +1,5 @@ #include "coding/file_reader.hpp" + #include "coding/reader_cache.hpp" #include "coding/internal/file_data.hpp" @@ -10,86 +11,92 @@ #define LOG_FILE_READER_EVERY_N_READS_MASK 0xFFFFFFFF #endif +using namespace std; + namespace { - class FileDataWithCachedSize : public my::FileData +class FileDataWithCachedSize : public my::FileData +{ + using base_t = my::FileData; + +public: + explicit FileDataWithCachedSize(string const & fileName) + : base_t(fileName, FileData::OP_READ), m_Size(FileData::Size()) { - typedef my::FileData base_t; + } - public: - explicit FileDataWithCachedSize(string const & fileName) - : base_t(fileName, FileData::OP_READ), m_Size(FileData::Size()) {} + uint64_t Size() const { return m_Size; } - uint64_t Size() const { return m_Size; } +private: + uint64_t m_Size; +}; +} // namespace - private: - uint64_t m_Size; - }; -} +// static +const uint32_t FileReader::kDefaultLogPageSize = 10; +// static +const uint32_t FileReader::kDefaultLogPageCount = 4; class FileReader::FileReaderData { public: FileReaderData(string const & fileName, uint32_t logPageSize, uint32_t logPageCount) - : m_FileData(fileName), m_ReaderCache(logPageSize, logPageCount) + : m_fileData(fileName), m_readerCache(logPageSize, logPageCount) { #if LOG_FILE_READER_STATS - m_ReadCallCount = 0; + m_readCallCount = 0; #endif } ~FileReaderData() { #if LOG_FILE_READER_STATS - LOG(LINFO, ("FileReader", GetName(), m_ReaderCache.GetStatsStr())); + LOG(LINFO, ("FileReader", GetName(), m_readerCache.GetStatsStr())); #endif } - uint64_t Size() const { return m_FileData.Size(); } + uint64_t Size() const { return m_fileData.Size(); } void Read(uint64_t pos, void * p, size_t size) { #if LOG_FILE_READER_STATS - if (((++m_ReadCallCount) & LOG_FILE_READER_EVERY_N_READS_MASK) == 0) + if (((++m_readCallCount) & LOG_FILE_READER_EVERY_N_READS_MASK) == 0) { - LOG(LINFO, ("FileReader", GetName(), m_ReaderCache.GetStatsStr())); + LOG(LINFO, ("FileReader", GetName(), m_readerCache.GetStatsStr())); } #endif - return m_ReaderCache.Read(m_FileData, pos, p, size); + return m_readerCache.Read(m_fileData, pos, p, size); } private: - FileDataWithCachedSize m_FileData; - ReaderCache m_ReaderCache; + FileDataWithCachedSize m_fileData; + ReaderCache m_readerCache; #if LOG_FILE_READER_STATS - uint32_t m_ReadCallCount; + uint32_t m_readCallCount; #endif }; -FileReader::FileReader(string const & fileName, bool withExceptions, - uint32_t logPageSize, uint32_t logPageCount) +FileReader::FileReader(string const & fileName, uint32_t logPageSize, uint32_t logPageCount) : BaseType(fileName) + , m_logPageSize(logPageSize) + , m_logPageCount(logPageCount) , m_fileData(new FileReaderData(fileName, logPageSize, logPageCount)) , m_offset(0) , m_size(m_fileData->Size()) - , m_withExceptions(withExceptions) {} -FileReader::FileReader(FileReader const & reader, uint64_t offset, uint64_t size) +FileReader::FileReader(FileReader const & reader, uint64_t offset, uint64_t size, + uint32_t logPageSize, uint32_t logPageCount) : BaseType(reader.GetName()) + , m_logPageSize(logPageSize) + , m_logPageCount(logPageCount) , m_fileData(reader.m_fileData) , m_offset(offset) , m_size(size) - , m_withExceptions(reader.m_withExceptions) {} -uint64_t FileReader::Size() const -{ - return m_size; -} - void FileReader::Read(uint64_t pos, void * p, size_t size) const { CheckPosAndSize(pos, size); @@ -99,37 +106,29 @@ void FileReader::Read(uint64_t pos, void * p, size_t size) const FileReader FileReader::SubReader(uint64_t pos, uint64_t size) const { CheckPosAndSize(pos, size); - return FileReader(*this, m_offset + pos, size); + return FileReader(*this, m_offset + pos, size, m_logPageSize, m_logPageCount); } unique_ptr FileReader::CreateSubReader(uint64_t pos, uint64_t size) const { CheckPosAndSize(pos, size); // Can't use make_unique with private constructor. - return unique_ptr(new FileReader(*this, m_offset + pos, size)); -} - -bool FileReader::AssertPosAndSize(uint64_t pos, uint64_t size) const -{ - uint64_t const allSize1 = Size(); - bool const ret1 = (pos + size <= allSize1); - if (!m_withExceptions) - ASSERT(ret1, (pos, size, allSize1)); - - uint64_t const allSize2 = m_fileData->Size(); - bool const ret2 = (m_offset + pos + size <= allSize2); - if (!m_withExceptions) - ASSERT(ret2, (m_offset, pos, size, allSize2)); - - return (ret1 && ret2); + return unique_ptr( + new FileReader(*this, m_offset + pos, size, m_logPageSize, m_logPageCount)); } void FileReader::CheckPosAndSize(uint64_t pos, uint64_t size) const { - if (m_withExceptions && !AssertPosAndSize(pos, size)) + uint64_t const allSize1 = Size(); + bool const ret1 = (pos + size <= allSize1); + ASSERT(ret1, (pos, size, allSize1)); + + uint64_t const allSize2 = m_fileData->Size(); + bool const ret2 = (m_offset + pos + size <= allSize2); + ASSERT(ret2, (m_offset, pos, size, allSize2)); + + if (!ret1 || !ret2) MYTHROW(Reader::SizeException, (pos, size)); - else - ASSERT(AssertPosAndSize(pos, size), ()); } void FileReader::SetOffsetAndSize(uint64_t offset, uint64_t size) diff --git a/coding/file_reader.hpp b/coding/file_reader.hpp index 77e0123b6a..9e1cf4781c 100644 --- a/coding/file_reader.hpp +++ b/coding/file_reader.hpp @@ -1,43 +1,51 @@ #pragma once + #include "coding/reader.hpp" + #include "base/base.hpp" -#include "std/shared_ptr.hpp" + +#include +#include +#include +#include // FileReader, cheap to copy, not thread safe. // It is assumed that file is not modified during FireReader lifetime, // because of caching and assumption that Size() is constant. class FileReader : public ModelReader { - using BaseType = ModelReader; - public: - explicit FileReader(string const & fileName, - bool withExceptions = false, - uint32_t logPageSize = 10, - uint32_t logPageCount = 4); + static const uint32_t kDefaultLogPageSize; + static const uint32_t kDefaultLogPageCount; + + explicit FileReader(std::string const & fileName, uint32_t logPageSize, uint32_t logPageCount); class FileReaderData; - uint64_t Size() const override; + // Reader overrides: + uint64_t Size() const override { return m_size; } void Read(uint64_t pos, void * p, size_t size) const override; - FileReader SubReader(uint64_t pos, uint64_t size) const; - unique_ptr CreateSubReader(uint64_t pos, uint64_t size) const override; + std::unique_ptr CreateSubReader(uint64_t pos, uint64_t size) const override; - inline uint64_t GetOffset() const { return m_offset; } + FileReader SubReader(uint64_t pos, uint64_t size) const; + uint64_t GetOffset() const { return m_offset; } protected: + // Throws an exception if a (pos, size) read would result in an out-of-bounds access. void CheckPosAndSize(uint64_t pos, uint64_t size) const; - /// Make assertion that pos + size in FileReader bounds. - bool AssertPosAndSize(uint64_t pos, uint64_t size) const; - /// Used in special derived readers. + // Used in special derived readers. void SetOffsetAndSize(uint64_t offset, uint64_t size); private: - FileReader(FileReader const & reader, uint64_t offset, uint64_t size); + using BaseType = ModelReader; - shared_ptr m_fileData; + FileReader(FileReader const & reader, uint64_t offset, uint64_t size, uint32_t logPageSize, + uint32_t logPageCount); + + uint32_t m_logPageSize; + uint32_t m_logPageCount; + std::shared_ptr m_fileData; uint64_t m_offset; uint64_t m_size; - bool m_withExceptions; }; diff --git a/coding/file_sort.hpp b/coding/file_sort.hpp index fd7b8a17e8..237c692f9f 100644 --- a/coding/file_sort.hpp +++ b/coding/file_sort.hpp @@ -63,7 +63,8 @@ public: // Write output. { m_pTmpWriter.reset(); - FileReader reader(m_TmpFileName); + FileReader reader(m_TmpFileName, FileReader::kDefaultLogPageSize, + FileReader::kDefaultLogPageCount); ItemIndexPairGreater fGreater(m_Less); PriorityQueueType q(fGreater); for (uint32_t i = 0; i < m_ItemCount; i += m_BufferCapacity) diff --git a/generator/cities_boundaries_builder.cpp b/generator/cities_boundaries_builder.cpp index 33373cfaed..bacca11519 100644 --- a/generator/cities_boundaries_builder.cpp +++ b/generator/cities_boundaries_builder.cpp @@ -181,7 +181,7 @@ bool DeserializeBoundariesTable(std::string const & path, OsmIdToBoundariesTable try { - FileReader reader(path, true /* with exceptions */); + FileReader reader(path, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount); NonOwningReaderSource source(reader); double precision; diff --git a/generator/dumper.cpp b/generator/dumper.cpp index 259e4caaff..6ff62ca25f 100644 --- a/generator/dumper.cpp +++ b/generator/dumper.cpp @@ -195,7 +195,8 @@ namespace feature { using TValue = FeatureIndexValue; - FilesContainerR container(make_unique(fPath)); + FilesContainerR container(make_unique(fPath, FileReader::kDefaultLogPageSize, + FileReader::kDefaultLogPageCount)); feature::DataHeader header(container); serial::GeometryCodingParams codingParams( trie::GetGeometryCodingParams(header.GetDefGeometryCodingParams())); diff --git a/generator/feature_builder.hpp b/generator/feature_builder.hpp index 1f5d2213f1..65c747870a 100644 --- a/generator/feature_builder.hpp +++ b/generator/feature_builder.hpp @@ -284,7 +284,7 @@ namespace feature template void ForEachFromDatRawFormat(std::string const & fName, ToDo && toDo) { - FileReader reader(fName); + FileReader reader(fName, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount); ReaderSource src(reader); uint64_t currPos = 0; diff --git a/generator/feature_sorter.cpp b/generator/feature_sorter.cpp index 86f2422869..d2966b4d93 100644 --- a/generator/feature_sorter.cpp +++ b/generator/feature_sorter.cpp @@ -326,7 +326,8 @@ bool GenerateFinalFeatures(feature::GenerateInfo const & info, string const & na // store sorted features { - FileReader reader(srcFilePath); + FileReader reader(srcFilePath, FileReader::kDefaultLogPageSize, + FileReader::kDefaultLogPageCount); bool const isWorld = (mapType != DataHeader::country); diff --git a/generator/gen_mwm_info.hpp b/generator/gen_mwm_info.hpp index 6861bd79a8..36910a358c 100644 --- a/generator/gen_mwm_info.hpp +++ b/generator/gen_mwm_info.hpp @@ -74,7 +74,8 @@ public: { try { - FileReader reader(filename, true /* with exceptions */); + FileReader reader(filename, FileReader::kDefaultLogPageSize, + FileReader::kDefaultLogPageCount); NonOwningReaderSource src(reader); Read(src); } diff --git a/generator/intermediate_data.cpp b/generator/intermediate_data.cpp index 358f1b4aa2..9199ef50cf 100644 --- a/generator/intermediate_data.cpp +++ b/generator/intermediate_data.cpp @@ -56,7 +56,10 @@ namespace generator namespace cache { // IndexFileReader --------------------------------------------------------------------------------- -IndexFileReader::IndexFileReader(string const & name) : m_fileReader(name.c_str()) {} +IndexFileReader::IndexFileReader(string const & name) + : m_fileReader(name.c_str(), FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount) +{ +} void IndexFileReader::ReadAll() { @@ -117,7 +120,10 @@ void IndexFileWriter::Add(Key k, Value const & v) // OSMElementCacheReader --------------------------------------------------------------------------- OSMElementCacheReader::OSMElementCacheReader(string const & name, bool preload) - : m_fileReader(name), m_offsets(name + OFFSET_EXT), m_name(name), m_preload(preload) + : m_fileReader(name, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount) + , m_offsets(name + OFFSET_EXT) + , m_name(name) + , m_preload(preload) { if (!m_preload) return; @@ -171,7 +177,8 @@ void RawFilePointStorageWriter::AddPoint(uint64_t id, double lat, double lon) // RawMemPointStorageReader ------------------------------------------------------------------------ RawMemPointStorageReader::RawMemPointStorageReader(string const & name) - : m_fileReader(name), m_data(kMaxNodesInOSM) + : m_fileReader(name, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount) + , m_data(kMaxNodesInOSM) { static_assert(sizeof(size_t) == 8, "This code is only for 64-bit architectures"); m_fileReader.Read(0, m_data.data(), m_data.size() * sizeof(LatLon)); @@ -210,7 +217,8 @@ void RawMemPointStorageWriter::AddPoint(uint64_t id, double lat, double lon) // MapFilePointStorageReader ----------------------------------------------------------------------- MapFilePointStorageReader::MapFilePointStorageReader(string const & name) - : m_fileReader(name + kShortExtension) + : m_fileReader(name + kShortExtension, FileReader::kDefaultLogPageSize, + FileReader::kDefaultLogPageCount) { LOG(LINFO, ("Nodes reading is started")); diff --git a/generator/locality_sorter.cpp b/generator/locality_sorter.cpp index 4c4b00c46f..15fc0c7f71 100644 --- a/generator/locality_sorter.cpp +++ b/generator/locality_sorter.cpp @@ -210,7 +210,7 @@ bool GenerateLocalityDataImpl(FeaturesCollector & collector, NeedSerialize const // Sort features by their middle point. midPoints.Sort(); - FileReader reader(file, true /* with exceptions */); + FileReader reader(file, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount); for (auto const & point : midPoints.GetVector()) { ReaderSource src(reader); diff --git a/generator/metalines_builder.cpp b/generator/metalines_builder.cpp index 41e7c3db43..7585860cb9 100644 --- a/generator/metalines_builder.cpp +++ b/generator/metalines_builder.cpp @@ -194,7 +194,8 @@ bool WriteMetalinesSection(std::string const & mwmPath, std::string const & meta if (!routing::ParseOsmIdToFeatureIdMapping(osmIdsToFeatureIdsPath, osmIdToFeatureId)) return false; - FileReader reader(metalinesPath); + FileReader reader(metalinesPath, FileReader::kDefaultLogPageSize, + FileReader::kDefaultLogPageCount); ReaderSource src(reader); std::vector buffer; MemWriter> memWriter(buffer); diff --git a/generator/mwm_diff/diff.cpp b/generator/mwm_diff/diff.cpp index ca9ecc8bdc..f9c01e71a3 100644 --- a/generator/mwm_diff/diff.cpp +++ b/generator/mwm_diff/diff.cpp @@ -86,8 +86,10 @@ bool MakeDiff(string const & oldMwmPath, string const & newMwmPath, string const { try { - FileReader oldReader(oldMwmPath); - FileReader newReader(newMwmPath); + FileReader oldReader(oldMwmPath, FileReader::kDefaultLogPageSize, + FileReader::kDefaultLogPageCount); + FileReader newReader(newMwmPath, FileReader::kDefaultLogPageSize, + FileReader::kDefaultLogPageCount); FileWriter diffFileWriter(diffPath); switch (VERSION_LATEST) @@ -116,9 +118,11 @@ bool ApplyDiff(string const & oldMwmPath, string const & newMwmPath, string cons { try { - FileReader oldReader(oldMwmPath); + FileReader oldReader(oldMwmPath, FileReader::kDefaultLogPageSize, + FileReader::kDefaultLogPageCount); FileWriter newWriter(newMwmPath); - FileReader diffFileReader(diffPath); + FileReader diffFileReader(diffPath, FileReader::kDefaultLogPageSize, + FileReader::kDefaultLogPageCount); ReaderSource diffFileSource(diffFileReader); auto const version = ReadPrimitiveFromSource(diffFileSource); diff --git a/generator/search_index_builder.cpp b/generator/search_index_builder.cpp index 48441fd5f5..c298a38d03 100644 --- a/generator/search_index_builder.cpp +++ b/generator/search_index_builder.cpp @@ -427,7 +427,9 @@ bool BuildSearchIndexFromDataFile(string const & filename, bool forceRebuild) { FilesContainerW writeContainer(readContainer.GetFileName(), FileWriter::OP_WRITE_EXISTING); FileWriter writer = writeContainer.GetWriter(SEARCH_INDEX_FILE_TAG); - rw_ops::Reverse(FileReader(indexFilePath, true /* with exceptions */), writer); + rw_ops::Reverse(FileReader(indexFilePath, FileReader::kDefaultLogPageSize, + FileReader::kDefaultLogPageCount), + writer); } { diff --git a/generator/utils.hpp b/generator/utils.hpp index aabd800028..86e13ecfed 100644 --- a/generator/utils.hpp +++ b/generator/utils.hpp @@ -43,7 +43,7 @@ bool ForEachOsmId2FeatureId(std::string const & path, ToDo && toDo) gen::OsmID2FeatureID mapping; try { - FileReader reader(path, true /* with exceptions */); + FileReader reader(path, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount); NonOwningReaderSource source(reader); mapping.Read(source); } diff --git a/generator/world_map_generator.hpp b/generator/world_map_generator.hpp index 1ac1b4aca7..18f8348f79 100644 --- a/generator/world_map_generator.hpp +++ b/generator/world_map_generator.hpp @@ -49,7 +49,8 @@ public: void LoadWaterGeometry(std::string const & rawGeometryFileName) { LOG_SHORT(LINFO, ("Loading water geometry:", rawGeometryFileName)); - FileReader reader(rawGeometryFileName); + FileReader reader(rawGeometryFileName, FileReader::kDefaultLogPageSize, + FileReader::kDefaultLogPageCount); ReaderSource file(reader); size_t total = 0; diff --git a/indexer/feature_processor.hpp b/indexer/feature_processor.hpp index e8b7b636d6..e06714d96a 100644 --- a/indexer/feature_processor.hpp +++ b/indexer/feature_processor.hpp @@ -23,6 +23,8 @@ void ForEachFromDat(ModelReaderPtr reader, ToDo && toDo) template void ForEachFromDat(string const & fPath, ToDo && toDo) { - ForEachFromDat(make_unique(fPath), toDo); + ForEachFromDat(make_unique(fPath, FileReader::kDefaultLogPageSize, + FileReader::kDefaultLogPageCount), + toDo); } } diff --git a/indexer/locality_index_builder.hpp b/indexer/locality_index_builder.hpp index c8e931e067..3cf1a326bd 100644 --- a/indexer/locality_index_builder.hpp +++ b/indexer/locality_index_builder.hpp @@ -47,7 +47,8 @@ void BuildLocalityIndex(ObjectsVector const & objects, Writer & writer, sorter.SortAndFinish(); } - FileReader reader(cellsToValueFile); + FileReader reader(cellsToValueFile, FileReader::kDefaultLogPageSize, + FileReader::kDefaultLogPageCount); DDVector, FileReader, uint64_t> cellsToValue(reader); { diff --git a/indexer/scale_index_builder.hpp b/indexer/scale_index_builder.hpp index 5629dd25a9..868991ec89 100644 --- a/indexer/scale_index_builder.hpp +++ b/indexer/scale_index_builder.hpp @@ -161,7 +161,8 @@ void IndexScales(feature::DataHeader const & header, TFeaturesVector const & fea } } - FileReader reader(cellsToFeatureAllBucketsFile); + FileReader reader(cellsToFeatureAllBucketsFile, FileReader::kDefaultLogPageSize, + FileReader::kDefaultLogPageCount); DDVector cellsToFeaturesAllBuckets(reader); VarSerialVectorWriter recordWriter(writer, bucketsCount); @@ -182,7 +183,8 @@ void IndexScales(feature::DataHeader const & header, TFeaturesVector const & fea } { - FileReader reader(cellsToFeatureFile); + FileReader reader(cellsToFeatureFile, FileReader::kDefaultLogPageSize, + FileReader::kDefaultLogPageCount); DDVector cellsToFeatures( reader); SubWriter subWriter(writer); diff --git a/kml/kml_tests/serdes_tests.cpp b/kml/kml_tests/serdes_tests.cpp index 64eafc8203..eee987d25a 100644 --- a/kml/kml_tests/serdes_tests.cpp +++ b/kml/kml_tests/serdes_tests.cpp @@ -518,7 +518,7 @@ UNIT_TEST(Kml_Deserialization_Text_File) try { kml::DeserializerKml des(dataFromFile); - FileReader reader(kmlFile, true /* with exceptions */); + FileReader reader(kmlFile, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount); des.Deserialize(reader); } catch (FileReader::Exception const & exc) @@ -555,7 +555,7 @@ UNIT_TEST(Kml_Deserialization_Bin_File) try { kml::binary::DeserializerKml des(dataFromFile); - FileReader reader(kmbFile, true /* with exceptions */); + FileReader reader(kmbFile, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount); des.Deserialize(reader); } catch (FileReader::Exception const & exc) @@ -596,7 +596,7 @@ UNIT_TEST(Kml_Serialization_Bin_File) try { kml::binary::DeserializerKml des(dataFromFile); - FileReader reader(kmbFile, true /* with exceptions */); + FileReader reader(kmbFile, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount); des.Deserialize(reader); } catch (FileReader::Exception const & exc) @@ -647,7 +647,7 @@ UNIT_TEST(Kml_Serialization_Text_File) try { kml::DeserializerKml des(dataFromFile); - FileReader reader(kmlFile, true /* with exceptions */); + FileReader reader(kmlFile, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount); des.Deserialize(reader); } catch (FileReader::Exception const & exc) diff --git a/local_ads/local_ads_tests/file_helpers_tests.cpp b/local_ads/local_ads_tests/file_helpers_tests.cpp index 408b677abf..9ffa141e28 100644 --- a/local_ads/local_ads_tests/file_helpers_tests.cpp +++ b/local_ads/local_ads_tests/file_helpers_tests.cpp @@ -22,7 +22,8 @@ UNIT_TEST(LocalAdsHelpers_Read_Write_Country_Name) string result; { - FileReader reader(testFile.GetFullPath()); + FileReader reader(testFile.GetFullPath(), FileReader::kDefaultLogPageSize, + FileReader::kDefaultLogPageCount); ReaderSource src(reader); result = ReadCountryName(src); } @@ -44,7 +45,8 @@ UNIT_TEST(LocalAdsHelpers_Read_Write_Timestamp) local_ads::Timestamp resultInHours; local_ads::Timestamp resultInSeconds; { - FileReader reader(testFile.GetFullPath()); + FileReader reader(testFile.GetFullPath(), FileReader::kDefaultLogPageSize, + FileReader::kDefaultLogPageCount); ReaderSource src(reader); resultInHours = ReadTimestamp(src); resultInSeconds = ReadTimestamp(src); @@ -66,7 +68,8 @@ UNIT_TEST(LocalAdsHelpers_Read_Write_RawData) vector result; { - FileReader reader(testFile.GetFullPath()); + FileReader reader(testFile.GetFullPath(), FileReader::kDefaultLogPageSize, + FileReader::kDefaultLogPageCount); ReaderSource src(reader); result = ReadRawData(src); } diff --git a/local_ads/statistics.cpp b/local_ads/statistics.cpp index 4ec9529b91..b582042d7a 100644 --- a/local_ads/statistics.cpp +++ b/local_ads/statistics.cpp @@ -157,7 +157,7 @@ std::list ReadEvents(std::string const & fileName) try { - FileReader reader(fileName, true /* withExceptions */); + FileReader reader(fileName, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount); ReaderSource src(reader); ReadPackedData(src, [&result](local_ads::Statistics::PackedData && data, std::string const & countryId, int64_t mwmVersion, @@ -464,7 +464,8 @@ void Statistics::ExtractMetadata(std::string const & fileName) int64_t mwmVersion; Timestamp baseTimestamp; { - FileReader reader(fileName, true /* withExceptions */); + FileReader reader(fileName, FileReader::kDefaultLogPageSize, + FileReader::kDefaultLogPageCount); ReaderSource src(reader); ReadMetadata(src, countryId, mwmVersion, baseTimestamp); } @@ -492,7 +493,8 @@ void Statistics::BalanceMemory() uint64_t totalSize = 0; for (auto const & metadata : m_metadataCache) { - FileReader reader(metadata.second.m_fileName); + FileReader reader(metadata.second.m_fileName, FileReader::kDefaultLogPageSize, + FileReader::kDefaultLogPageCount); sizeInBytes[metadata.first] = reader.Size(); totalSize += reader.Size(); } diff --git a/map/bookmark_helpers.cpp b/map/bookmark_helpers.cpp index d824876bf7..19fd599b91 100644 --- a/map/bookmark_helpers.cpp +++ b/map/bookmark_helpers.cpp @@ -257,7 +257,9 @@ std::unique_ptr LoadKmlFile(std::string const & file, KmlFileType std::unique_ptr kmlData; try { - kmlData = LoadKmlData(FileReader(file, true /* withExceptions */), fileType); + kmlData = LoadKmlData( + FileReader(file, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount), + fileType); } catch (std::exception const & e) { diff --git a/map/cloud.cpp b/map/cloud.cpp index be0a162e11..aaebb88659 100644 --- a/map/cloud.cpp +++ b/map/cloud.cpp @@ -304,7 +304,7 @@ Cloud::SnapshotResponseData ReadSnapshotFile(std::string const & filename) std::string jsonStr; try { - FileReader r(filename, true /* with exceptions */); + FileReader r(filename, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount); r.ReadAsString(jsonStr); } catch (FileReader::Exception const & exception) @@ -458,7 +458,7 @@ bool Cloud::ReadIndex() std::string jsonStr; try { - FileReader r(indexFilePath, true /* with exceptions */); + FileReader r(indexFilePath, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount); r.ReadAsString(jsonStr); } catch (FileReader::Exception const & exception) diff --git a/map/local_ads_manager.cpp b/map/local_ads_manager.cpp index c1f38339a2..5d4385b8bd 100644 --- a/map/local_ads_manager.cpp +++ b/map/local_ads_manager.cpp @@ -518,7 +518,8 @@ void LocalAdsManager::ReadCampaignFile(std::string const & campaignFile) std::lock_guard lock(m_campaignsMutex); try { - FileReader reader(campaignFile, true /* withExceptions */); + FileReader reader(campaignFile, FileReader::kDefaultLogPageSize, + FileReader::kDefaultLogPageCount); ReaderSource src(reader); while (src.Size() > 0) { diff --git a/map/map_tests/bookmarks_test.cpp b/map/map_tests/bookmarks_test.cpp index 2cfeba2f1a..da609d0bc8 100644 --- a/map/map_tests/bookmarks_test.cpp +++ b/map/map_tests/bookmarks_test.cpp @@ -241,7 +241,10 @@ UNIT_CLASS_TEST(Runner, Bookmarks_ExportKML) TEST_EQUAL(bmManager.GetBmGroupsIdList().size(), 0, ()); BookmarkManager::KMLDataCollection kmlDataCollection2; - kmlDataCollection2.emplace_back("", LoadKmlData(FileReader(fileName), GetActiveKmlFileType())); + kmlDataCollection2.emplace_back("", + LoadKmlData(FileReader(fileName, FileReader::kDefaultLogPageSize, + FileReader::kDefaultLogPageCount), + GetActiveKmlFileType())); TEST(kmlDataCollection2.back().second, ()); bmManager.CreateCategories(std::move(kmlDataCollection2), false /* autoSave */); diff --git a/platform/chunks_download_strategy.cpp b/platform/chunks_download_strategy.cpp index fc38efafcd..0f3cc70d8f 100644 --- a/platform/chunks_download_strategy.cpp +++ b/platform/chunks_download_strategy.cpp @@ -91,7 +91,7 @@ int64_t ChunksDownloadStrategy::LoadOrInitChunks(string const & fName, int64_t f try { - FileReader r(fName, true /* with exceptions */); + FileReader r(fName, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount); ReaderSource src(r); int64_t const readSize = ReadVarInt(src); diff --git a/platform/string_storage_base.cpp b/platform/string_storage_base.cpp index a1f430462e..7ed038f8b1 100644 --- a/platform/string_storage_base.cpp +++ b/platform/string_storage_base.cpp @@ -24,7 +24,8 @@ StringStorageBase::StringStorageBase(string const & path) : m_path(path) try { LOG(LINFO, ("Settings path:", m_path)); - ReaderStreamBuf buffer(make_unique(m_path, true /* withExceptions */)); + ReaderStreamBuf buffer(make_unique(m_path, FileReader::kDefaultLogPageSize, + FileReader::kDefaultLogPageCount)); istream stream(&buffer); string line; diff --git a/routing/geometry.cpp b/routing/geometry.cpp index 0cb7d20809..f0306a61e1 100644 --- a/routing/geometry.cpp +++ b/routing/geometry.cpp @@ -82,7 +82,8 @@ private: FileGeometryLoader::FileGeometryLoader(string const & fileName, shared_ptr vehicleModel) - : m_featuresVector(FilesContainerR(make_unique(fileName))) + : m_featuresVector(FilesContainerR(make_unique( + fileName, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount))) , m_vehicleModel(vehicleModel) { CHECK(m_vehicleModel, ()); diff --git a/track_analyzing/track_analyzer/cmd_table.cpp b/track_analyzing/track_analyzer/cmd_table.cpp index 4fa130b228..a69d4a9429 100644 --- a/track_analyzing/track_analyzer/cmd_table.cpp +++ b/track_analyzing/track_analyzer/cmd_table.cpp @@ -15,6 +15,7 @@ #include "storage/storage.hpp" #include "coding/file_name_utils.hpp" +#include "coding/file_reader.hpp" #include @@ -168,7 +169,8 @@ class MatchedTrackPointToMoveType final { public: MatchedTrackPointToMoveType(string const & mwmFile) - : m_featuresVector(FilesContainerR(make_unique(mwmFile))) + : m_featuresVector(FilesContainerR(make_unique( + mwmFile, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount))) { } diff --git a/track_analyzing/track_analyzer/cmd_track.cpp b/track_analyzing/track_analyzer/cmd_track.cpp index a0bbea8e74..e1e5611388 100644 --- a/track_analyzing/track_analyzer/cmd_track.cpp +++ b/track_analyzing/track_analyzer/cmd_track.cpp @@ -13,6 +13,7 @@ #include "storage/storage.hpp" #include "coding/file_name_utils.hpp" +#include "coding/file_reader.hpp" #include "geometry/mercator.hpp" @@ -35,7 +36,8 @@ void CmdTrack(string const & trackFile, string const & mwmName, string const & u string const mwmFile = GetCurrentVersionMwmFile(storage, mwmName); shared_ptr vehicleModel = CarModelFactory({}).GetVehicleModelForCountry(mwmName); - FeaturesVectorTest featuresVector(FilesContainerR(make_unique(mwmFile))); + FeaturesVectorTest featuresVector(FilesContainerR(make_unique( + mwmFile, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount))); Geometry geometry(GeometryLoader::CreateFromFile(mwmFile, vehicleModel)); uint64_t const duration = diff --git a/track_analyzing/utils.cpp b/track_analyzing/utils.cpp index 1860f7064b..1ecdf43e51 100644 --- a/track_analyzing/utils.cpp +++ b/track_analyzing/utils.cpp @@ -52,7 +52,7 @@ double CalcSpeedKMpH(double meters, uint64_t secondsElapsed) void ReadTracks(shared_ptr numMwmIds, string const & filename, MwmToMatchedTracks & mwmToMatchedTracks) { - FileReader reader(filename); + FileReader reader(filename, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount); ReaderSource src(reader); MwmToMatchedTracksSerializer serializer(numMwmIds); serializer.Deserialize(mwmToMatchedTracks, src); diff --git a/ugc/storage.cpp b/ugc/storage.cpp index 6d49985487..abed9cd9ac 100644 --- a/ugc/storage.cpp +++ b/ugc/storage.cpp @@ -40,7 +40,8 @@ bool GetUGCFileSize(uint64_t & size) { try { - FileReader reader(GetUGCFilePath(), true /* with exceptions */); + FileReader reader(GetUGCFilePath(), FileReader::kDefaultLogPageSize, + FileReader::kDefaultLogPageCount); size = reader.Size(); } catch (RootException const &) @@ -165,7 +166,7 @@ UGCUpdate Storage::GetUGCUpdate(FeatureID const & id) const auto const ugcFilePath = GetUGCFilePath(); try { - FileReader r(ugcFilePath, true /* withExceptions */); + FileReader r(ugcFilePath, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount); r.Read(offset, buf.data(), size); } catch (FileReader::Exception const & exception) @@ -194,7 +195,7 @@ void Storage::Load() auto const indexFilePath = GetIndexFilePath(); try { - FileReader r(indexFilePath, true /* with exceptions */); + FileReader r(indexFilePath, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount); r.ReadAsString(data); } catch (FileReader::Exception const & exception) @@ -240,7 +241,7 @@ void Storage::Defragmentation() try { - FileReader r(ugcFilePath, true /* withExceptions */); + FileReader r(ugcFilePath, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount); FileWriter w(tmpUGCFilePath, FileWriter::Op::OP_APPEND); uint64_t actualOffset = 0; for (size_t i = 0; i < indexesSize; ++i) @@ -287,7 +288,7 @@ string Storage::GetUGCToSend() const auto array = my::NewJSONArray(); auto const indexesSize = m_UGCIndexes.size(); auto const ugcFilePath = GetUGCFilePath(); - FileReader r(ugcFilePath, true /* withExceptions */); + FileReader r(ugcFilePath, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount); vector buf; for (size_t i = 0; i < indexesSize; ++i) { @@ -424,7 +425,7 @@ size_t GetNumberOfUnsentUGC() string data; try { - FileReader r(indexFilePath, true /* with exceptions */); + FileReader r(indexFilePath, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount); r.ReadAsString(data); } catch (FileReader::Exception const & exception)