From 3b2242a81cb36c20f2cb0b5558ce7481cfb326b4 Mon Sep 17 00:00:00 2001 From: Maksim Andrianov Date: Mon, 22 Oct 2018 12:17:16 +0300 Subject: [PATCH] [generator] Fixed checking file offset. Fixed Set::Add. --- generator/feature_generator.cpp | 38 ++++++++------------- generator/feature_generator.hpp | 59 +++++++++++++++++---------------- generator/feature_sorter.cpp | 12 +++++-- generator/filter_elements.cpp | 4 +-- generator/geometry_holder.hpp | 8 +++-- 5 files changed, 62 insertions(+), 59 deletions(-) diff --git a/generator/feature_generator.cpp b/generator/feature_generator.cpp index 9d89e701cd..59b0683252 100644 --- a/generator/feature_generator.cpp +++ b/generator/feature_generator.cpp @@ -26,27 +26,11 @@ namespace feature { FeaturesCollector::FeaturesCollector(std::string const & fName) - : m_datFile(fName) -{ - CHECK_EQUAL(GetFileSize(m_datFile), 0, ()); -} + : m_datFile(fName), m_writeBuffer(kBufferSize) {} FeaturesCollector::~FeaturesCollector() { FlushBuffer(); - // Check file size - (void)GetFileSize(m_datFile); -} - -// static -uint32_t FeaturesCollector::GetFileSize(FileWriter const & f) -{ - // .dat file should be less than 4Gb - uint64_t const pos = f.Pos(); - uint32_t const ret = static_cast(pos); - - CHECK_EQUAL(static_cast(ret), pos, ("Feature offset is out of 32bit boundary!")); - return ret; } template @@ -68,7 +52,7 @@ pair PackValue(ValueT v) void FeaturesCollector::FlushBuffer() { - m_datFile.Write(m_writeBuffer, m_writePosition); + m_datFile.Write(m_writeBuffer.data(), m_writePosition); m_writePosition = 0; } @@ -82,9 +66,10 @@ void FeaturesCollector::Write(char const * src, size_t size) { do { - if (m_writePosition == sizeof(m_writeBuffer)) + if (m_writePosition == kBufferSize) FlushBuffer(); - size_t const part_size = min(size, sizeof(m_writeBuffer) - m_writePosition); + + size_t const part_size = min(size, kBufferSize - m_writePosition); memcpy(&m_writeBuffer[m_writePosition], src, part_size); m_writePosition += part_size; size -= part_size; @@ -116,10 +101,7 @@ uint32_t FeaturesCollector::operator()(FeatureBuilder1 const & fb) FeaturesAndRawGeometryCollector::FeaturesAndRawGeometryCollector(std::string const & featuresFileName, std::string const & rawGeometryFileName) - : FeaturesCollector(featuresFileName), m_rawGeometryFileStream(rawGeometryFileName) -{ - CHECK_EQUAL(GetFileSize(m_rawGeometryFileStream), 0, ()); -} + : FeaturesCollector(featuresFileName), m_rawGeometryFileStream(rawGeometryFileName) {} FeaturesAndRawGeometryCollector::~FeaturesAndRawGeometryCollector() { @@ -148,4 +130,12 @@ uint32_t FeaturesAndRawGeometryCollector::operator()(FeatureBuilder1 const & fb) } return featureId; } + +uint32_t CheckedFilePosCast(FileWriter const & f) +{ + uint64_t pos = f.Pos(); + CHECK_LESS_OR_EQUAL(pos, static_cast(std::numeric_limits::max()), + ("Feature offset is out of 32bit boundary!")); + return static_cast(pos); +} } diff --git a/generator/feature_generator.hpp b/generator/feature_generator.hpp index 376199cebe..d0e874c06d 100644 --- a/generator/feature_generator.hpp +++ b/generator/feature_generator.hpp @@ -4,6 +4,7 @@ #include "coding/file_writer.hpp" +#include #include #include #include @@ -15,13 +16,33 @@ namespace feature // Writes features to dat file. class FeaturesCollector { - char m_writeBuffer[48000]; - size_t m_writePosition = 0; - uint32_t m_featureID = 0; +public: + static size_t constexpr kBufferSize = 48000; + + FeaturesCollector(std::string const & fName); + virtual ~FeaturesCollector(); + + static uint64_t GetCurrentPosition(); + std::string const & GetFilePath() const { return m_datFile.GetName(); } + /// \brief Serializes |f|. + /// \returns Feature id of serialized feature if |f| is serialized after the call + /// and |kInvalidFeatureId| if not. + /// \note See implementation operator() in derived class for cases when |f| cannot be + /// serialized. + virtual uint32_t operator()(FeatureBuilder1 const & f); + virtual uint32_t operator()(FeatureBuilder1 & f) + { + return (*this)(const_cast(f)); + } + virtual void Finish() {} protected: static uint32_t constexpr kInvalidFeatureId = std::numeric_limits::max(); + /// \return Feature offset in the file, which is used as an ID later + uint32_t WriteFeatureBase(std::vector const & bytes, FeatureBuilder1 const & fb); + void Flush(); + FileWriter m_datFile; m2::RectD m_bounds; @@ -29,31 +50,9 @@ private: void Write(char const * src, size_t size); void FlushBuffer(); -protected: - - /// @return feature offset in the file, which is used as an ID later - uint32_t WriteFeatureBase(std::vector const & bytes, FeatureBuilder1 const & fb); - - void Flush(); - -public: - FeaturesCollector(std::string const & fName); - virtual ~FeaturesCollector(); - - static uint32_t GetFileSize(FileWriter const & f); - std::string const & GetFilePath() const { return m_datFile.GetName(); } - /// \brief Serializes |f|. - /// \returns feature id of serialized feature if |f| is serialized after the call - /// and |kInvalidFeatureId| if not. - /// \note See implementation operator() in derived class for cases when |f| cannot be - /// serialized. - virtual uint32_t operator()(FeatureBuilder1 const & f); - virtual uint32_t operator()(FeatureBuilder1 & f) - { - auto const & f1 = f; - return (*this)(f1); - }; - virtual void Finish() {} + std::vector m_writeBuffer; + size_t m_writePosition = 0; + uint32_t m_featureID = 0; }; class FeaturesAndRawGeometryCollector : public FeaturesCollector @@ -64,8 +63,10 @@ class FeaturesAndRawGeometryCollector : public FeaturesCollector public: FeaturesAndRawGeometryCollector(std::string const & featuresFileName, std::string const & rawGeometryFileName); - ~FeaturesAndRawGeometryCollector(); + ~FeaturesAndRawGeometryCollector() override; uint32_t operator()(FeatureBuilder1 const & f) override; }; + +uint32_t CheckedFilePosCast(FileWriter const & f); } diff --git a/generator/feature_sorter.cpp b/generator/feature_sorter.cpp index 8a7b3f333f..63d6536ae7 100644 --- a/generator/feature_sorter.cpp +++ b/generator/feature_sorter.cpp @@ -32,6 +32,7 @@ #include "defines.hpp" #include +#include #include #include @@ -62,6 +63,13 @@ public: m_helperFile[SEARCH_TOKENS] = make_unique(fName + SEARCH_TOKENS_FILE_TAG); } + ~FeaturesCollector2() + { + // Check file size. + auto const unused = CheckedFilePosCast(m_datFile); + UNUSED_VALUE(unused); + } + void Finish() { // write version information @@ -90,7 +98,7 @@ public: // File Writer finalization function with appending to the main mwm file. auto const finalizeFn = [this](unique_ptr w, string const & tag, - string const & postfix = string()) { + string const & postfix = string()) { w->Flush(); m_writer.Write(w->GetName(), tag + postfix); }; @@ -129,7 +137,7 @@ public: uint32_t operator()(FeatureBuilder2 & fb) { GeometryHolder holder([this](int i) -> FileWriter & { return *m_geoFile[i]; }, - [this](int i) -> FileWriter & { return *m_trgFile[i]; }, fb, m_header); + [this](int i) -> FileWriter & { return *m_trgFile[i]; }, fb, m_header); bool const isLine = fb.IsLine(); bool const isArea = fb.IsArea(); diff --git a/generator/filter_elements.cpp b/generator/filter_elements.cpp index 8a3ba5f487..ce87770a69 100644 --- a/generator/filter_elements.cpp +++ b/generator/filter_elements.cpp @@ -15,10 +15,10 @@ public: bool Add(T const & key) { if (Contains(key)) - return true; + return false; m_vec.push_back(key); - return false; + return true; } bool Contains(T const & key) const diff --git a/generator/geometry_holder.hpp b/generator/geometry_holder.hpp index 21ec7d8406..e3e6e4a143 100644 --- a/generator/geometry_holder.hpp +++ b/generator/geometry_holder.hpp @@ -14,6 +14,7 @@ #include #include +#include #include #include @@ -163,7 +164,9 @@ private: Points toSave(points.begin() + 1, points.end()); m_buffer.m_ptsMask |= (1 << i); - m_buffer.m_ptsOffset.push_back(feature::FeaturesCollector::GetFileSize(m_geoFileGetter(i))); + auto const pos = feature::CheckedFilePosCast(m_geoFileGetter(i)); + m_buffer.m_ptsOffset.push_back(pos); + serial::SaveOuterPath(toSave, cp, m_geoFileGetter(i)); } @@ -200,7 +203,8 @@ private: // saving to file m_buffer.m_trgMask |= (1 << i); - m_buffer.m_trgOffset.push_back(feature::FeaturesCollector::GetFileSize(m_trgFileGetter(i))); + auto const pos = feature::CheckedFilePosCast(m_trgFileGetter(i)); + m_buffer.m_trgOffset.push_back(pos); saver.Save(m_trgFileGetter(i)); }