gpesquero/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

[coding] FileReader now always throws exception on an out-of-bounds access.

Browse source
This commit is contained in:
Maxim Pimenov 2018-06-14 16:33:04 +03:00 • committed by Arsentiy Milchakov
parent 1728f1af3d
commit 222a75ff67
39 changed files with 212 additions and 144 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
coding/coding_tests/csv_reader_test.cpp
View file

@ -25,7 +25,8 @@ UNIT_TEST(CSVReaderSmoke)
{
auto const fileName = "test.csv";
ScopedFile sf(fileName, kCSV1);
FileReader fileReader(sf.GetFullPath());
FileReader fileReader(sf.GetFullPath(), FileReader::kDefaultLogPageSize,
FileReader::kDefaultLogPageCount);
CSVReader reader;
reader.Read(fileReader, [](File const & file) {
TEST_EQUAL(file.size(), 1, ());
@ -49,7 +50,8 @@ UNIT_TEST(CSVReaderCustomDelimiter)
{
auto const fileName = "test.csv";
ScopedFile sf(fileName, kCSV2);
FileReader fileReader(sf.GetFullPath());
FileReader fileReader(sf.GetFullPath(), FileReader::kDefaultLogPageSize,
FileReader::kDefaultLogPageCount);
CSVReader reader;
CSVReader::Params p;
p.m_readHeader = true;
@ -67,7 +69,8 @@ UNIT_TEST(CSVReaderEmptyFile)
{
auto const fileName = "test.csv";
ScopedFile sf(fileName, kCSV2);
FileReader fileReader(sf.GetFullPath());
FileReader fileReader(sf.GetFullPath(), FileReader::kDefaultLogPageSize,
FileReader::kDefaultLogPageCount);
CSVReader reader;
reader.Read(fileReader, [](File const & file) { TEST_EQUAL(file.size(), 0, ()); });

16
coding/coding_tests/reader_test.cpp
View file

@ -30,7 +30,8 @@ UNIT_TEST(FileReaderSmokeTest)
}
{
FileReader fileReader("reader_test_tmp.dat");
FileReader fileReader("reader_test_tmp.dat", FileReader::kDefaultLogPageSize,
FileReader::kDefaultLogPageCount);
TestReader(fileReader);
}
FileWriter::DeleteFileX("reader_test_tmp.dat");
@ -47,7 +48,9 @@ UNIT_TEST(BufferReaderSmokeTest)
writer.Write(&data[0], data.size());
}
BufferReader r2(FileReader("reader_test_tmp.dat"), 7);
BufferReader r2(FileReader("reader_test_tmp.dat", FileReader::kDefaultLogPageSize,
FileReader::kDefaultLogPageCount),
7);
TestReader(r2);
FileWriter::DeleteFileX("reader_test_tmp.dat");
}
@ -63,7 +66,8 @@ UNIT_TEST(FileReaderNonExistentFileTest)
{
try
{
FileReader reader("skjhfaxniauiuq2bmnszmn093sklsd");
FileReader reader("skjhfaxniauiuq2bmnszmn093sklsd", FileReader::kDefaultLogPageSize,
FileReader::kDefaultLogPageCount);
TEST(false, ("Exception should be thrown!"));
}
catch (FileReader::OpenException &)
@ -81,7 +85,8 @@ UNIT_TEST(FileReaderReadAsText)
{
string text;
FileReader(fName).ReadAsString(text);
FileReader(fName, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount)
.ReadAsString(text);
TEST_EQUAL(text, fName, ());
}
@ -99,7 +104,8 @@ UNIT_TEST(ReaderStreamBuf)
}
{
ReaderStreamBuf buffer(make_unique<FileReader>(name));
ReaderStreamBuf buffer(make_unique<FileReader>(name, FileReader::kDefaultLogPageSize,
FileReader::kDefaultLogPageCount));
istream s(&buffer);
std::string str;

4
coding/coding_tests/reader_writer_ops_test.cpp
View file

@ -21,7 +21,7 @@ namespace
}
{
FileReader reader(tmpFile);
FileReader reader(tmpFile, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount);
buffer.clear();
MemWriter<vector<char> > writer(buffer);
rw_ops::Reverse(reader, writer);
@ -64,7 +64,7 @@ UNIT_TEST(Reverse_Smoke)
{
{
FillRandFile(tmpFile, 10 * 1024 + 527);
FileReader reader(tmpFile);
FileReader reader(tmpFile, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount);
vector<char> buffer;
GetReverseForReaderAndTmpFile(reader, buffer);

18
coding/coding_tests/writer_test.cpp
View file

@ -50,7 +50,7 @@ UNIT_TEST(FileWriter_Smoke)
}
vector<char> s;
{
FileReader reader(fileName);
FileReader reader(fileName, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount);
s.resize(reader.Size());
reader.Read(0, &s[0], reader.Size());
}
@ -85,7 +85,7 @@ UNIT_TEST(SubWriter_FileWriter_Smoke)
}
vector<char> s;
{
FileReader reader(fileName);
FileReader reader(fileName, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount);
s.resize(reader.Size());
reader.Read(0, &s[0], reader.Size());
}
@ -101,13 +101,13 @@ UNIT_TEST(FileWriter_DeleteFile)
writer.Write("123", 3);
}
{
FileReader reader(fileName);
FileReader reader(fileName, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount);
TEST_EQUAL(reader.Size(), 3, ());
}
FileWriter::DeleteFileX(fileName);
try
{
FileReader reader(fileName);
FileReader reader(fileName, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount);
TEST(false, ("Exception should be thrown!"));
}
catch (FileReader::OpenException & )
@ -127,7 +127,7 @@ UNIT_TEST(FileWriter_AppendAndOpenExisting)
writer.Write("abcd", 4);
}
{
FileReader reader(fileName);
FileReader reader(fileName, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount);
TEST_EQUAL(reader.Size(), 4, ());
string s(static_cast<uint32_t>(reader.Size()), 0);
reader.Read(0, &s[0], s.size());
@ -138,7 +138,7 @@ UNIT_TEST(FileWriter_AppendAndOpenExisting)
writer.Write("123", 3);
}
{
FileReader reader(fileName);
FileReader reader(fileName, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount);
TEST_EQUAL(reader.Size(), 3, ());
}
{
@ -146,7 +146,7 @@ UNIT_TEST(FileWriter_AppendAndOpenExisting)
writer.Write("4", 1);
}
{
FileReader reader(fileName);
FileReader reader(fileName, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount);
TEST_EQUAL(reader.Size(), 4, ());
string s(static_cast<uint32_t>(reader.Size()), 0);
reader.Read(0, &s[0], s.size());
@ -158,7 +158,7 @@ UNIT_TEST(FileWriter_AppendAndOpenExisting)
writer.Write("56", 2);
}
{
FileReader reader(fileName);
FileReader reader(fileName, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount);
TEST_EQUAL(reader.Size(), 4, ());
string s(static_cast<uint32_t>(reader.Size()), 0);
reader.Read(0, &s[0], 4);
@ -219,7 +219,7 @@ UNIT_TEST(FileWriter_Chunks)
WriteTestData2(fileWriter);
}
{
FileReader r(TEST_FILE);
FileReader r(TEST_FILE, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount);
ReadTestData(r);
}
FileWriter::DeleteFileX(TEST_FILE);

5
coding/coding_tests/zip_creator_test.cpp
View file

@ -19,7 +19,10 @@ void CreateAndTestZip(string const & filePath, string const & zipPath)
ZipFileReader::FileListT files;
ZipFileReader::FilesList(zipPath, files);
TEST_EQUAL(files[0].second, FileReader(filePath).Size(), ());
TEST_EQUAL(files[0].second,
FileReader(filePath, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount)
.Size(),
());
string const unzippedFile = "unzipped.tmp";
ZipFileReader::UnzipFile(zipPath, files[0].first, unzippedFile);

6
coding/coding_tests/zip_reader_test.cpp
View file

@ -175,13 +175,15 @@ UNIT_TEST(ZipExtract)
string s;
ZipFileReader::UnzipFile(ZIPFILE, files[0].first, OUTFILE);
{
FileReader(OUTFILE).ReadAsString(s);
FileReader(OUTFILE, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount)
.ReadAsString(s);
}
TEST_EQUAL(s, "aaaaaaaaaa\x0A", ());
// OUTFILE should be rewritten correctly in the next lines
ZipFileReader::UnzipFile(ZIPFILE, files[1].first, OUTFILE);
{
FileReader(OUTFILE).ReadAsString(s);
FileReader(OUTFILE, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount)
.ReadAsString(s);
}
TEST_EQUAL(s, "Holalala\x0A", ());
FileWriter::DeleteFileX(OUTFILE);

14
coding/file_container.cpp
View file

@ -211,7 +211,7 @@ FilesMappingContainer::~FilesMappingContainer()
void FilesMappingContainer::Open(string const & fName)
{
{
FileReader reader(fName);
FileReader reader(fName, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount);
ReadInfo(reader);
}
@ -242,7 +242,8 @@ FileReader FilesMappingContainer::GetReader(Tag const & tag) const
Info const * p = GetInfo(tag);
if (!p)
MYTHROW(Reader::OpenException, ("Can't find section:", m_name, tag));
return FileReader(m_name).SubReader(p->m_offset, p->m_size);
return FileReader(m_name, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount)
.SubReader(p->m_offset, p->m_size);
}
/////////////////////////////////////////////////////////////////////////////
@ -307,7 +308,7 @@ void FilesContainerW::Open(FileWriter::Op op)
case FileWriter::OP_WRITE_EXISTING:
{
// read an existing service info
FileReader reader(m_name);
FileReader reader(m_name, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount);
ReadInfo(reader);
}
@ -350,7 +351,8 @@ FilesContainerW::~FilesContainerW()
uint64_t FilesContainerW::SaveCurrentSize()
{
ASSERT(!m_bFinished, ());
uint64_t const curr = FileReader(m_name).Size();
uint64_t const curr =
FileReader(m_name, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount).Size();
if (!m_info.empty())
m_info.back().m_size = curr - m_info.back().m_offset;
return curr;
@ -426,7 +428,9 @@ FileWriter FilesContainerW::GetWriter(Tag const & tag)
void FilesContainerW::Write(string const & fPath, Tag const & tag)
{
Write(ModelReaderPtr(make_unique<FileReader>(fPath)), tag);
Write(ModelReaderPtr(make_unique<FileReader>(fPath, FileReader::kDefaultLogPageSize,
FileReader::kDefaultLogPageCount)),
tag);
}
void FilesContainerW::Write(ModelReaderPtr reader, Tag const & tag)

99
coding/file_reader.cpp
View file

@ -1,4 +1,5 @@
#include "coding/file_reader.hpp"
#include "coding/reader_cache.hpp"
#include "coding/internal/file_data.hpp"
@ -10,86 +11,92 @@
#define LOG_FILE_READER_EVERY_N_READS_MASK 0xFFFFFFFF
#endif
using namespace std;
namespace
{
class FileDataWithCachedSize : public my::FileData
class FileDataWithCachedSize : public my::FileData
{
using base_t = my::FileData;
public:
explicit FileDataWithCachedSize(string const & fileName)
: base_t(fileName, FileData::OP_READ), m_Size(FileData::Size())
{
typedef my::FileData base_t;
}
public:
explicit FileDataWithCachedSize(string const & fileName)
: base_t(fileName, FileData::OP_READ), m_Size(FileData::Size()) {}
uint64_t Size() const { return m_Size; }
uint64_t Size() const { return m_Size; }
private:
uint64_t m_Size;
};
} // namespace
private:
uint64_t m_Size;
};
}
// static
const uint32_t FileReader::kDefaultLogPageSize = 10;
// static
const uint32_t FileReader::kDefaultLogPageCount = 4;
class FileReader::FileReaderData
{
public:
FileReaderData(string const & fileName, uint32_t logPageSize, uint32_t logPageCount)
: m_FileData(fileName), m_ReaderCache(logPageSize, logPageCount)
: m_fileData(fileName), m_readerCache(logPageSize, logPageCount)
{
#if LOG_FILE_READER_STATS
m_ReadCallCount = 0;
m_readCallCount = 0;
#endif
}
~FileReaderData()
{
#if LOG_FILE_READER_STATS
LOG(LINFO, ("FileReader", GetName(), m_ReaderCache.GetStatsStr()));
LOG(LINFO, ("FileReader", GetName(), m_readerCache.GetStatsStr()));
#endif
}
uint64_t Size() const { return m_FileData.Size(); }
uint64_t Size() const { return m_fileData.Size(); }
void Read(uint64_t pos, void * p, size_t size)
{
#if LOG_FILE_READER_STATS
if (((++m_ReadCallCount) & LOG_FILE_READER_EVERY_N_READS_MASK) == 0)
if (((++m_readCallCount) & LOG_FILE_READER_EVERY_N_READS_MASK) == 0)
{
LOG(LINFO, ("FileReader", GetName(), m_ReaderCache.GetStatsStr()));
LOG(LINFO, ("FileReader", GetName(), m_readerCache.GetStatsStr()));
}
#endif
return m_ReaderCache.Read(m_FileData, pos, p, size);
return m_readerCache.Read(m_fileData, pos, p, size);
}
private:
FileDataWithCachedSize m_FileData;
ReaderCache<FileDataWithCachedSize, LOG_FILE_READER_STATS> m_ReaderCache;
FileDataWithCachedSize m_fileData;
ReaderCache<FileDataWithCachedSize, LOG_FILE_READER_STATS> m_readerCache;
#if LOG_FILE_READER_STATS
uint32_t m_ReadCallCount;
uint32_t m_readCallCount;
#endif
};
FileReader::FileReader(string const & fileName, bool withExceptions,
uint32_t logPageSize, uint32_t logPageCount)
FileReader::FileReader(string const & fileName, uint32_t logPageSize, uint32_t logPageCount)
: BaseType(fileName)
, m_logPageSize(logPageSize)
, m_logPageCount(logPageCount)
, m_fileData(new FileReaderData(fileName, logPageSize, logPageCount))
, m_offset(0)
, m_size(m_fileData->Size())
, m_withExceptions(withExceptions)
{}
FileReader::FileReader(FileReader const & reader, uint64_t offset, uint64_t size)
FileReader::FileReader(FileReader const & reader, uint64_t offset, uint64_t size,
uint32_t logPageSize, uint32_t logPageCount)
: BaseType(reader.GetName())
, m_logPageSize(logPageSize)
, m_logPageCount(logPageCount)
, m_fileData(reader.m_fileData)
, m_offset(offset)
, m_size(size)
, m_withExceptions(reader.m_withExceptions)
{}
uint64_t FileReader::Size() const
{
return m_size;
}
void FileReader::Read(uint64_t pos, void * p, size_t size) const
{
CheckPosAndSize(pos, size);
@ -99,37 +106,29 @@ void FileReader::Read(uint64_t pos, void * p, size_t size) const
FileReader FileReader::SubReader(uint64_t pos, uint64_t size) const
{
CheckPosAndSize(pos, size);
return FileReader(*this, m_offset + pos, size);
return FileReader(*this, m_offset + pos, size, m_logPageSize, m_logPageCount);
}
unique_ptr<Reader> FileReader::CreateSubReader(uint64_t pos, uint64_t size) const
{
CheckPosAndSize(pos, size);
// Can't use make_unique with private constructor.
return unique_ptr<Reader>(new FileReader(*this, m_offset + pos, size));
}
bool FileReader::AssertPosAndSize(uint64_t pos, uint64_t size) const
{
uint64_t const allSize1 = Size();
bool const ret1 = (pos + size <= allSize1);
if (!m_withExceptions)
ASSERT(ret1, (pos, size, allSize1));
uint64_t const allSize2 = m_fileData->Size();
bool const ret2 = (m_offset + pos + size <= allSize2);
if (!m_withExceptions)
ASSERT(ret2, (m_offset, pos, size, allSize2));
return (ret1 && ret2);
return unique_ptr<Reader>(
new FileReader(*this, m_offset + pos, size, m_logPageSize, m_logPageCount));
}
void FileReader::CheckPosAndSize(uint64_t pos, uint64_t size) const
{
if (m_withExceptions && !AssertPosAndSize(pos, size))
uint64_t const allSize1 = Size();
bool const ret1 = (pos + size <= allSize1);
ASSERT(ret1, (pos, size, allSize1));
uint64_t const allSize2 = m_fileData->Size();
bool const ret2 = (m_offset + pos + size <= allSize2);
ASSERT(ret2, (m_offset, pos, size, allSize2));
if (!ret1 || !ret2)
MYTHROW(Reader::SizeException, (pos, size));
else
ASSERT(AssertPosAndSize(pos, size), ());
}
void FileReader::SetOffsetAndSize(uint64_t offset, uint64_t size)

42
coding/file_reader.hpp
View file

@ -1,43 +1,51 @@
#pragma once
#include "coding/reader.hpp"
#include "base/base.hpp"
#include "std/shared_ptr.hpp"
#include <cstddef>
#include <cstdint>
#include <memory>
#include <string>
// FileReader, cheap to copy, not thread safe.
// It is assumed that file is not modified during FireReader lifetime,
// because of caching and assumption that Size() is constant.
class FileReader : public ModelReader
{
using BaseType = ModelReader;
public:
explicit FileReader(string const & fileName,
bool withExceptions = false,
uint32_t logPageSize = 10,
uint32_t logPageCount = 4);
static const uint32_t kDefaultLogPageSize;
static const uint32_t kDefaultLogPageCount;
explicit FileReader(std::string const & fileName, uint32_t logPageSize, uint32_t logPageCount);
class FileReaderData;
uint64_t Size() const override;
// Reader overrides:
uint64_t Size() const override { return m_size; }
void Read(uint64_t pos, void * p, size_t size) const override;
FileReader SubReader(uint64_t pos, uint64_t size) const;
unique_ptr<Reader> CreateSubReader(uint64_t pos, uint64_t size) const override;
std::unique_ptr<Reader> CreateSubReader(uint64_t pos, uint64_t size) const override;
inline uint64_t GetOffset() const { return m_offset; }
FileReader SubReader(uint64_t pos, uint64_t size) const;
uint64_t GetOffset() const { return m_offset; }
protected:
// Throws an exception if a (pos, size) read would result in an out-of-bounds access.
void CheckPosAndSize(uint64_t pos, uint64_t size) const;
/// Make assertion that pos + size in FileReader bounds.
bool AssertPosAndSize(uint64_t pos, uint64_t size) const;
/// Used in special derived readers.
// Used in special derived readers.
void SetOffsetAndSize(uint64_t offset, uint64_t size);
private:
FileReader(FileReader const & reader, uint64_t offset, uint64_t size);
using BaseType = ModelReader;
shared_ptr<FileReaderData> m_fileData;
FileReader(FileReader const & reader, uint64_t offset, uint64_t size, uint32_t logPageSize,
uint32_t logPageCount);
uint32_t m_logPageSize;
uint32_t m_logPageCount;
std::shared_ptr<FileReaderData> m_fileData;
uint64_t m_offset;
uint64_t m_size;
bool m_withExceptions;
};

3
coding/file_sort.hpp
View file

@ -63,7 +63,8 @@ public:
// Write output.
{
m_pTmpWriter.reset();
FileReader reader(m_TmpFileName);
FileReader reader(m_TmpFileName, FileReader::kDefaultLogPageSize,
FileReader::kDefaultLogPageCount);
ItemIndexPairGreater fGreater(m_Less);
PriorityQueueType q(fGreater);
for (uint32_t i = 0; i < m_ItemCount; i += m_BufferCapacity)

2
generator/cities_boundaries_builder.cpp
View file

@ -181,7 +181,7 @@ bool DeserializeBoundariesTable(std::string const & path, OsmIdToBoundariesTable
try
{
FileReader reader(path, true /* with exceptions */);
FileReader reader(path, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount);
NonOwningReaderSource source(reader);
double precision;

3
generator/dumper.cpp
View file

@ -195,7 +195,8 @@ namespace feature
{
using TValue = FeatureIndexValue;
FilesContainerR container(make_unique<FileReader>(fPath));
FilesContainerR container(make_unique<FileReader>(fPath, FileReader::kDefaultLogPageSize,
FileReader::kDefaultLogPageCount));
feature::DataHeader header(container);
serial::GeometryCodingParams codingParams(
trie::GetGeometryCodingParams(header.GetDefGeometryCodingParams()));

2
generator/feature_builder.hpp
View file

@ -284,7 +284,7 @@ namespace feature
template <class ToDo>
void ForEachFromDatRawFormat(std::string const & fName, ToDo && toDo)
{
FileReader reader(fName);
FileReader reader(fName, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount);
ReaderSource<FileReader> src(reader);
uint64_t currPos = 0;

3
generator/feature_sorter.cpp
View file

@ -326,7 +326,8 @@ bool GenerateFinalFeatures(feature::GenerateInfo const & info, string const & na
// store sorted features
{
FileReader reader(srcFilePath);
FileReader reader(srcFilePath, FileReader::kDefaultLogPageSize,
FileReader::kDefaultLogPageCount);
bool const isWorld = (mapType != DataHeader::country);

3
generator/gen_mwm_info.hpp
View file

@ -74,7 +74,8 @@ public:
{
try
{
FileReader reader(filename, true /* with exceptions */);
FileReader reader(filename, FileReader::kDefaultLogPageSize,
FileReader::kDefaultLogPageCount);
NonOwningReaderSource src(reader);
Read(src);
}

16
generator/intermediate_data.cpp
View file

@ -56,7 +56,10 @@ namespace generator
namespace cache
{
// IndexFileReader ---------------------------------------------------------------------------------
IndexFileReader::IndexFileReader(string const & name) : m_fileReader(name.c_str()) {}
IndexFileReader::IndexFileReader(string const & name)
: m_fileReader(name.c_str(), FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount)
{
}
void IndexFileReader::ReadAll()
{
@ -117,7 +120,10 @@ void IndexFileWriter::Add(Key k, Value const & v)
// OSMElementCacheReader ---------------------------------------------------------------------------
OSMElementCacheReader::OSMElementCacheReader(string const & name, bool preload)
: m_fileReader(name), m_offsets(name + OFFSET_EXT), m_name(name), m_preload(preload)
: m_fileReader(name, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount)
, m_offsets(name + OFFSET_EXT)
, m_name(name)
, m_preload(preload)
{
if (!m_preload)
return;
@ -171,7 +177,8 @@ void RawFilePointStorageWriter::AddPoint(uint64_t id, double lat, double lon)
// RawMemPointStorageReader ------------------------------------------------------------------------
RawMemPointStorageReader::RawMemPointStorageReader(string const & name)
: m_fileReader(name), m_data(kMaxNodesInOSM)
: m_fileReader(name, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount)
, m_data(kMaxNodesInOSM)
{
static_assert(sizeof(size_t) == 8, "This code is only for 64-bit architectures");
m_fileReader.Read(0, m_data.data(), m_data.size() * sizeof(LatLon));
@ -210,7 +217,8 @@ void RawMemPointStorageWriter::AddPoint(uint64_t id, double lat, double lon)
// MapFilePointStorageReader -----------------------------------------------------------------------
MapFilePointStorageReader::MapFilePointStorageReader(string const & name)
: m_fileReader(name + kShortExtension)
: m_fileReader(name + kShortExtension, FileReader::kDefaultLogPageSize,
FileReader::kDefaultLogPageCount)
{
LOG(LINFO, ("Nodes reading is started"));

2
generator/locality_sorter.cpp
View file

@ -210,7 +210,7 @@ bool GenerateLocalityDataImpl(FeaturesCollector & collector, NeedSerialize const
// Sort features by their middle point.
midPoints.Sort();
FileReader reader(file, true /* with exceptions */);
FileReader reader(file, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount);
for (auto const & point : midPoints.GetVector())
{
ReaderSource<FileReader> src(reader);

3
generator/metalines_builder.cpp
View file

@ -194,7 +194,8 @@ bool WriteMetalinesSection(std::string const & mwmPath, std::string const & meta
if (!routing::ParseOsmIdToFeatureIdMapping(osmIdsToFeatureIdsPath, osmIdToFeatureId))
return false;
FileReader reader(metalinesPath);
FileReader reader(metalinesPath, FileReader::kDefaultLogPageSize,
FileReader::kDefaultLogPageCount);
ReaderSource<FileReader> src(reader);
std::vector<uint8_t> buffer;
MemWriter<std::vector<uint8_t>> memWriter(buffer);

12
generator/mwm_diff/diff.cpp
View file

@ -86,8 +86,10 @@ bool MakeDiff(string const & oldMwmPath, string const & newMwmPath, string const
{
try
{
FileReader oldReader(oldMwmPath);
FileReader newReader(newMwmPath);
FileReader oldReader(oldMwmPath, FileReader::kDefaultLogPageSize,
FileReader::kDefaultLogPageCount);
FileReader newReader(newMwmPath, FileReader::kDefaultLogPageSize,
FileReader::kDefaultLogPageCount);
FileWriter diffFileWriter(diffPath);
switch (VERSION_LATEST)
@ -116,9 +118,11 @@ bool ApplyDiff(string const & oldMwmPath, string const & newMwmPath, string cons
{
try
{
FileReader oldReader(oldMwmPath);
FileReader oldReader(oldMwmPath, FileReader::kDefaultLogPageSize,
FileReader::kDefaultLogPageCount);
FileWriter newWriter(newMwmPath);
FileReader diffFileReader(diffPath);
FileReader diffFileReader(diffPath, FileReader::kDefaultLogPageSize,
FileReader::kDefaultLogPageCount);
ReaderSource<FileReader> diffFileSource(diffFileReader);
auto const version = ReadPrimitiveFromSource<uint32_t>(diffFileSource);

4
generator/search_index_builder.cpp
View file

@ -427,7 +427,9 @@ bool BuildSearchIndexFromDataFile(string const & filename, bool forceRebuild)
{
FilesContainerW writeContainer(readContainer.GetFileName(), FileWriter::OP_WRITE_EXISTING);
FileWriter writer = writeContainer.GetWriter(SEARCH_INDEX_FILE_TAG);
rw_ops::Reverse(FileReader(indexFilePath, true /* with exceptions */), writer);
rw_ops::Reverse(FileReader(indexFilePath, FileReader::kDefaultLogPageSize,
FileReader::kDefaultLogPageCount),
writer);
}
{

2
generator/utils.hpp
View file

@ -43,7 +43,7 @@ bool ForEachOsmId2FeatureId(std::string const & path, ToDo && toDo)
gen::OsmID2FeatureID mapping;
try
{
FileReader reader(path, true /* with exceptions */);
FileReader reader(path, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount);
NonOwningReaderSource source(reader);
mapping.Read(source);
}

3
generator/world_map_generator.hpp
View file

@ -49,7 +49,8 @@ public:
void LoadWaterGeometry(std::string const & rawGeometryFileName)
{
LOG_SHORT(LINFO, ("Loading water geometry:", rawGeometryFileName));
FileReader reader(rawGeometryFileName);
FileReader reader(rawGeometryFileName, FileReader::kDefaultLogPageSize,
FileReader::kDefaultLogPageCount);
ReaderSource<FileReader> file(reader);
size_t total = 0;

4
indexer/feature_processor.hpp
View file

@ -23,6 +23,8 @@ void ForEachFromDat(ModelReaderPtr reader, ToDo && toDo)
template <class ToDo>
void ForEachFromDat(string const & fPath, ToDo && toDo)
{
ForEachFromDat(make_unique<FileReader>(fPath), toDo);
ForEachFromDat(make_unique<FileReader>(fPath, FileReader::kDefaultLogPageSize,
FileReader::kDefaultLogPageCount),
toDo);
}
}

3
indexer/locality_index_builder.hpp
View file

@ -47,7 +47,8 @@ void BuildLocalityIndex(ObjectsVector const & objects, Writer & writer,
sorter.SortAndFinish();
}
FileReader reader(cellsToValueFile);
FileReader reader(cellsToValueFile, FileReader::kDefaultLogPageSize,
FileReader::kDefaultLogPageCount);
DDVector<CellValuePair<uint64_t>, FileReader, uint64_t> cellsToValue(reader);
{

6
indexer/scale_index_builder.hpp
View file

@ -161,7 +161,8 @@ void IndexScales(feature::DataHeader const & header, TFeaturesVector const & fea
}
}
FileReader reader(cellsToFeatureAllBucketsFile);
FileReader reader(cellsToFeatureAllBucketsFile, FileReader::kDefaultLogPageSize,
FileReader::kDefaultLogPageCount);
DDVector<CellFeatureBucketTuple, FileReader, uint64_t> cellsToFeaturesAllBuckets(reader);
VarSerialVectorWriter<TWriter> recordWriter(writer, bucketsCount);
@ -182,7 +183,8 @@ void IndexScales(feature::DataHeader const & header, TFeaturesVector const & fea
}
{
FileReader reader(cellsToFeatureFile);
FileReader reader(cellsToFeatureFile, FileReader::kDefaultLogPageSize,
FileReader::kDefaultLogPageCount);
DDVector<CellFeatureBucketTuple::CellFeaturePair, FileReader, uint64_t> cellsToFeatures(
reader);
SubWriter<TWriter> subWriter(writer);

8
kml/kml_tests/serdes_tests.cpp
View file

@ -518,7 +518,7 @@ UNIT_TEST(Kml_Deserialization_Text_File)
try
{
kml::DeserializerKml des(dataFromFile);
FileReader reader(kmlFile, true /* with exceptions */);
FileReader reader(kmlFile, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount);
des.Deserialize(reader);
}
catch (FileReader::Exception const & exc)
@ -555,7 +555,7 @@ UNIT_TEST(Kml_Deserialization_Bin_File)
try
{
kml::binary::DeserializerKml des(dataFromFile);
FileReader reader(kmbFile, true /* with exceptions */);
FileReader reader(kmbFile, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount);
des.Deserialize(reader);
}
catch (FileReader::Exception const & exc)
@ -596,7 +596,7 @@ UNIT_TEST(Kml_Serialization_Bin_File)
try
{
kml::binary::DeserializerKml des(dataFromFile);
FileReader reader(kmbFile, true /* with exceptions */);
FileReader reader(kmbFile, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount);
des.Deserialize(reader);
}
catch (FileReader::Exception const & exc)
@ -647,7 +647,7 @@ UNIT_TEST(Kml_Serialization_Text_File)
try
{
kml::DeserializerKml des(dataFromFile);
FileReader reader(kmlFile, true /* with exceptions */);
FileReader reader(kmlFile, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount);
des.Deserialize(reader);
}
catch (FileReader::Exception const & exc)

9
local_ads/local_ads_tests/file_helpers_tests.cpp
View file

@ -22,7 +22,8 @@ UNIT_TEST(LocalAdsHelpers_Read_Write_Country_Name)
string result;
{
FileReader reader(testFile.GetFullPath());
FileReader reader(testFile.GetFullPath(), FileReader::kDefaultLogPageSize,
FileReader::kDefaultLogPageCount);
ReaderSource<FileReader> src(reader);
result = ReadCountryName(src);
}
@ -44,7 +45,8 @@ UNIT_TEST(LocalAdsHelpers_Read_Write_Timestamp)
local_ads::Timestamp resultInHours;
local_ads::Timestamp resultInSeconds;
{
FileReader reader(testFile.GetFullPath());
FileReader reader(testFile.GetFullPath(), FileReader::kDefaultLogPageSize,
FileReader::kDefaultLogPageCount);
ReaderSource<FileReader> src(reader);
resultInHours = ReadTimestamp<chrono::hours>(src);
resultInSeconds = ReadTimestamp<chrono::seconds>(src);
@ -66,7 +68,8 @@ UNIT_TEST(LocalAdsHelpers_Read_Write_RawData)
vector<uint8_t> result;
{
FileReader reader(testFile.GetFullPath());
FileReader reader(testFile.GetFullPath(), FileReader::kDefaultLogPageSize,
FileReader::kDefaultLogPageCount);
ReaderSource<FileReader> src(reader);
result = ReadRawData(src);
}

8
local_ads/statistics.cpp
View file

@ -157,7 +157,7 @@ std::list<local_ads::Event> ReadEvents(std::string const & fileName)
try
{
FileReader reader(fileName, true /* withExceptions */);
FileReader reader(fileName, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount);
ReaderSource<FileReader> src(reader);
ReadPackedData(src, [&result](local_ads::Statistics::PackedData && data,
std::string const & countryId, int64_t mwmVersion,
@ -464,7 +464,8 @@ void Statistics::ExtractMetadata(std::string const & fileName)
int64_t mwmVersion;
Timestamp baseTimestamp;
{
FileReader reader(fileName, true /* withExceptions */);
FileReader reader(fileName, FileReader::kDefaultLogPageSize,
FileReader::kDefaultLogPageCount);
ReaderSource<FileReader> src(reader);
ReadMetadata(src, countryId, mwmVersion, baseTimestamp);
}
@ -492,7 +493,8 @@ void Statistics::BalanceMemory()
uint64_t totalSize = 0;
for (auto const & metadata : m_metadataCache)
{
FileReader reader(metadata.second.m_fileName);
FileReader reader(metadata.second.m_fileName, FileReader::kDefaultLogPageSize,
FileReader::kDefaultLogPageCount);
sizeInBytes[metadata.first] = reader.Size();
totalSize += reader.Size();
}

4
map/bookmark_helpers.cpp
View file

@ -257,7 +257,9 @@ std::unique_ptr<kml::FileData> LoadKmlFile(std::string const & file, KmlFileType
std::unique_ptr<kml::FileData> kmlData;
try
{
kmlData = LoadKmlData(FileReader(file, true /* withExceptions */), fileType);
kmlData = LoadKmlData(
FileReader(file, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount),
fileType);
}
catch (std::exception const & e)
{

4
map/cloud.cpp
View file

@ -304,7 +304,7 @@ Cloud::SnapshotResponseData ReadSnapshotFile(std::string const & filename)
std::string jsonStr;
try
{
FileReader r(filename, true /* with exceptions */);
FileReader r(filename, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount);
r.ReadAsString(jsonStr);
}
catch (FileReader::Exception const & exception)
@ -458,7 +458,7 @@ bool Cloud::ReadIndex()
std::string jsonStr;
try
{
FileReader r(indexFilePath, true /* with exceptions */);
FileReader r(indexFilePath, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount);
r.ReadAsString(jsonStr);
}
catch (FileReader::Exception const & exception)

3
map/local_ads_manager.cpp
View file

@ -518,7 +518,8 @@ void LocalAdsManager::ReadCampaignFile(std::string const & campaignFile)
std::lock_guard<std::mutex> lock(m_campaignsMutex);
try
{
FileReader reader(campaignFile, true /* withExceptions */);
FileReader reader(campaignFile, FileReader::kDefaultLogPageSize,
FileReader::kDefaultLogPageCount);
ReaderSource<FileReader> src(reader);
while (src.Size() > 0)
{

5
map/map_tests/bookmarks_test.cpp
View file

@ -241,7 +241,10 @@ UNIT_CLASS_TEST(Runner, Bookmarks_ExportKML)
TEST_EQUAL(bmManager.GetBmGroupsIdList().size(), 0, ());
BookmarkManager::KMLDataCollection kmlDataCollection2;
kmlDataCollection2.emplace_back("", LoadKmlData(FileReader(fileName), GetActiveKmlFileType()));
kmlDataCollection2.emplace_back("",
LoadKmlData(FileReader(fileName, FileReader::kDefaultLogPageSize,
FileReader::kDefaultLogPageCount),
GetActiveKmlFileType()));
TEST(kmlDataCollection2.back().second, ());
bmManager.CreateCategories(std::move(kmlDataCollection2), false /* autoSave */);

2
platform/chunks_download_strategy.cpp
View file

@ -91,7 +91,7 @@ int64_t ChunksDownloadStrategy::LoadOrInitChunks(string const & fName, int64_t f
try
{
FileReader r(fName, true /* with exceptions */);
FileReader r(fName, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount);
ReaderSource<FileReader> src(r);
int64_t const readSize = ReadVarInt<int64_t>(src);

3
platform/string_storage_base.cpp
View file

@ -24,7 +24,8 @@ StringStorageBase::StringStorageBase(string const & path) : m_path(path)
try
{
LOG(LINFO, ("Settings path:", m_path));
ReaderStreamBuf buffer(make_unique<FileReader>(m_path, true /* withExceptions */));
ReaderStreamBuf buffer(make_unique<FileReader>(m_path, FileReader::kDefaultLogPageSize,
FileReader::kDefaultLogPageCount));
istream stream(&buffer);
string line;

3
routing/geometry.cpp
View file

@ -82,7 +82,8 @@ private:
FileGeometryLoader::FileGeometryLoader(string const & fileName,
shared_ptr<VehicleModelInterface> vehicleModel)
: m_featuresVector(FilesContainerR(make_unique<FileReader>(fileName)))
: m_featuresVector(FilesContainerR(make_unique<FileReader>(
fileName, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount)))
, m_vehicleModel(vehicleModel)
{
CHECK(m_vehicleModel, ());

4
track_analyzing/track_analyzer/cmd_table.cpp
View file

@ -15,6 +15,7 @@
#include "storage/storage.hpp"
#include "coding/file_name_utils.hpp"
#include "coding/file_reader.hpp"
#include <iostream>
@ -168,7 +169,8 @@ class MatchedTrackPointToMoveType final
{
public:
MatchedTrackPointToMoveType(string const & mwmFile)
: m_featuresVector(FilesContainerR(make_unique<FileReader>(mwmFile)))
: m_featuresVector(FilesContainerR(make_unique<FileReader>(
mwmFile, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount)))
{
}

4
track_analyzing/track_analyzer/cmd_track.cpp
View file

@ -13,6 +13,7 @@
#include "storage/storage.hpp"
#include "coding/file_name_utils.hpp"
#include "coding/file_reader.hpp"
#include "geometry/mercator.hpp"
@ -35,7 +36,8 @@ void CmdTrack(string const & trackFile, string const & mwmName, string const & u
string const mwmFile = GetCurrentVersionMwmFile(storage, mwmName);
shared_ptr<VehicleModelInterface> vehicleModel =
CarModelFactory({}).GetVehicleModelForCountry(mwmName);
FeaturesVectorTest featuresVector(FilesContainerR(make_unique<FileReader>(mwmFile)));
FeaturesVectorTest featuresVector(FilesContainerR(make_unique<FileReader>(
mwmFile, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount)));
Geometry geometry(GeometryLoader::CreateFromFile(mwmFile, vehicleModel));
uint64_t const duration =

2
track_analyzing/utils.cpp
View file

@ -52,7 +52,7 @@ double CalcSpeedKMpH(double meters, uint64_t secondsElapsed)
void ReadTracks(shared_ptr<NumMwmIds> numMwmIds, string const & filename,
MwmToMatchedTracks & mwmToMatchedTracks)
{
FileReader reader(filename);
FileReader reader(filename, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount);
ReaderSource<FileReader> src(reader);
MwmToMatchedTracksSerializer serializer(numMwmIds);
serializer.Deserialize(mwmToMatchedTracks, src);

13
ugc/storage.cpp
View file

@ -40,7 +40,8 @@ bool GetUGCFileSize(uint64_t & size)
{
try
{
FileReader reader(GetUGCFilePath(), true /* with exceptions */);
FileReader reader(GetUGCFilePath(), FileReader::kDefaultLogPageSize,
FileReader::kDefaultLogPageCount);
size = reader.Size();
}
catch (RootException const &)
@ -165,7 +166,7 @@ UGCUpdate Storage::GetUGCUpdate(FeatureID const & id) const
auto const ugcFilePath = GetUGCFilePath();
try
{
FileReader r(ugcFilePath, true /* withExceptions */);
FileReader r(ugcFilePath, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount);
r.Read(offset, buf.data(), size);
}
catch (FileReader::Exception const & exception)
@ -194,7 +195,7 @@ void Storage::Load()
auto const indexFilePath = GetIndexFilePath();
try
{
FileReader r(indexFilePath, true /* with exceptions */);
FileReader r(indexFilePath, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount);
r.ReadAsString(data);
}
catch (FileReader::Exception const & exception)
@ -240,7 +241,7 @@ void Storage::Defragmentation()
try
{
FileReader r(ugcFilePath, true /* withExceptions */);
FileReader r(ugcFilePath, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount);
FileWriter w(tmpUGCFilePath, FileWriter::Op::OP_APPEND);
uint64_t actualOffset = 0;
for (size_t i = 0; i < indexesSize; ++i)
@ -287,7 +288,7 @@ string Storage::GetUGCToSend() const
auto array = my::NewJSONArray();
auto const indexesSize = m_UGCIndexes.size();
auto const ugcFilePath = GetUGCFilePath();
FileReader r(ugcFilePath, true /* withExceptions */);
FileReader r(ugcFilePath, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount);
vector<uint8_t> buf;
for (size_t i = 0; i < indexesSize; ++i)
{
@ -424,7 +425,7 @@ size_t GetNumberOfUnsentUGC()
string data;
try
{
FileReader r(indexFilePath, true /* with exceptions */);
FileReader r(indexFilePath, FileReader::kDefaultLogPageSize, FileReader::kDefaultLogPageCount);
r.ReadAsString(data);
}
catch (FileReader::Exception const & exception)