diff --git a/map/gps_track.cpp b/map/gps_track.cpp index 34fcf3afa3..7d231ccf15 100644 --- a/map/gps_track.cpp +++ b/map/gps_track.cpp @@ -170,30 +170,6 @@ void GpsTrack::LazyInitFile() } } } - catch (GpsTrackFile::CorruptedFileException &) - { - // File has been corrupted. - // Drop any data from the file. - // If file exception happens, then drop file. - try - { - LOG(LINFO, ("File is corrupted, create new:", m_filePath)); - if (!m_file->Create(m_filePath, m_maxItemCount)) - { - LOG(LINFO, ("Cannot create GpsTrackFile:", m_filePath)); - m_file.reset(); - } - else - { - LOG(LINFO, ("GpsTrackFile has been created:", m_filePath)); - } - } - catch (RootException & e) - { - LOG(LINFO, ("GpsTrackFile.Create has caused exception:", e.Msg())); - m_file.reset(); - } - } catch (RootException & e) { LOG(LINFO, ("GpsTrackFile has caused exception:", e.Msg())); @@ -229,32 +205,21 @@ void GpsTrack::InitCollection(hours duration) if (!m_file) return; - // Read points from the file - // If CorruptedFileException happens, the clear the file - // If file exception happens, then drop file. try { - // Read points from file to the collection - m_file->ForEach([this](TItem const & info, size_t /* id */)->bool + // Get all data from the file + m_file->ForEach(0 /* timestampSince */ , [this](TItem const & info)->bool { pair evictedIds; m_collection->Add(info, evictedIds); return true; }); } - catch (GpsTrackFile::CorruptedFileException &) + catch (GpsTrackFile::ReadFileException & e) { - LOG(LINFO, ("GpsTrackFile is corrupted, clear it:", m_filePath)); + LOG(LINFO, ("GpsTrackFile.ForEach has caused exception:", e.Msg())); m_collection->Clear(); - try - { - m_file->Clear(); - } - catch (RootException & e) - { - LOG(LINFO, ("GpsTrackFile.Clear caused exception:", e.Msg())); - m_file.reset(); - } + m_file.reset(); } } @@ -302,16 +267,10 @@ void GpsTrack::UpdateFile(bool needClear, vector const & points) try { - // clear points from file, if need if (needClear) m_file->Clear(); - // add points to file if need - for (auto const & point : points) - { - size_t evictedId; - m_file->Append(point, evictedId); - } + m_file->Append(points); } catch (RootException & e) { diff --git a/map/gps_track_file.cpp b/map/gps_track_file.cpp index 7b37d5c287..e1f67148ec 100644 --- a/map/gps_track_file.cpp +++ b/map/gps_track_file.cpp @@ -1,125 +1,48 @@ #include "map/gps_track_file.hpp" +#include "coding/internal/file_data.hpp" + #include "base/assert.hpp" #include "base/logging.hpp" -#include "std/algorithm.hpp" - -namespace -{ - -size_t constexpr kLinearSearchDistance = 10; -size_t constexpr kLazyWriteHeaderMaxCount = 100; - -} // namespace - -size_t const GpsTrackFile::kInvalidId = numeric_limits::max(); - GpsTrackFile::GpsTrackFile() - : m_lazyWriteHeaderCount(0) + : m_maxItemCount(0) + , m_itemCount(0) { } GpsTrackFile::~GpsTrackFile() { - ASSERT(!m_stream.is_open(), ("Don't forget to close file")); - - if (m_stream.is_open()) - { - try - { - Close(); - } - catch (RootException const & e) - { - LOG(LINFO, ("Close caused exception", e.Msg())); - } - } } bool GpsTrackFile::Open(string const & filePath, size_t maxItemCount) { - ASSERT(!m_stream.is_open(), ("File must not be open on OpenFile")); + ASSERT(!m_stream.is_open(), ()); ASSERT(maxItemCount > 0, ()); - m_stream = fstream(filePath, ios::in|ios::out|ios::binary); + m_stream = fstream(filePath, ios::in|ios::out|ios::binary|ios::ate); if (!m_stream) return false; - try - { - m_filePath = filePath; + m_filePath = filePath; - // Check file integrity: - // - file size if correct; - // - Header fields m_first, m_last, m_lastId and m_timestamp are correct; - // - front and back items are correct, if exist. + size_t const fileSize = m_stream.tellp(); - m_stream.seekg(0, ios::end); - size_t const fileSize = m_stream.tellg(); - m_stream.seekg(0, ios::beg); + m_itemCount = fileSize / sizeof(TItem); + m_maxItemCount = maxItemCount; - if (fileSize < sizeof(Header)) - MYTHROW(CorruptedFileException, ("File:", m_filePath)); - - if ((fileSize - sizeof(Header)) % sizeof(TItem) != 0) - MYTHROW(CorruptedFileException, ("File:", m_filePath)); - - size_t const itemCount = (fileSize - sizeof(Header)) / sizeof(TItem); - - if (!ReadHeader(m_header)) - MYTHROW(CorruptedFileException, ("File:", m_filePath)); - - if (m_header.m_maxItemCount != (1 + maxItemCount)) - MYTHROW(CorruptedFileException, ("File:", m_filePath)); - - if (itemCount > m_header.m_maxItemCount) - MYTHROW(CorruptedFileException, ("File:", m_filePath)); - - if (itemCount == 0) - { - if (m_header.m_first != 0 && m_header.m_last != 0) - MYTHROW(CorruptedFileException, ("File:", m_filePath)); - } - else - { - if (m_header.m_first >= itemCount || m_header.m_last > itemCount) - MYTHROW(CorruptedFileException, ("File:", m_filePath)); - } - - if (m_header.m_lastId < itemCount) - MYTHROW(CorruptedFileException, ("File:", m_filePath)); - - if (m_header.m_first != m_header.m_last) - { - TItem frontItem; - if (!ReadItem(m_header.m_first, frontItem)) - MYTHROW(CorruptedFileException, ("File:", m_filePath)); - - TItem backItem; - size_t backIndex = (m_header.m_first + Distance(m_header.m_first, m_header.m_last, m_header.m_maxItemCount) - 1) % m_header.m_maxItemCount; - if (!ReadItem(backIndex, backItem)) - MYTHROW(CorruptedFileException, ("File:", m_filePath)); - - if (frontItem.m_timestamp > backItem.m_timestamp || m_header.m_timestamp != backItem.m_timestamp) - MYTHROW(CorruptedFileException, ("File:", m_filePath)); - } - } - catch (RootException &) - { - m_header = Header(); - m_filePath.clear(); - m_stream.close(); - throw; - } + // Set write position after last item position + m_stream.seekp(m_itemCount * sizeof(TItem), ios::beg); + if (0 != (m_stream.rdstate() & (ios::failbit | ios::badbit))) + MYTHROW(WriteFileException, ("File:", m_filePath)); return true; } bool GpsTrackFile::Create(string const & filePath, size_t maxItemCount) { - ASSERT(!m_stream.is_open(), ("File must not be open on CreateFile")); + ASSERT(!m_stream.is_open(), ()); ASSERT(maxItemCount > 0, ()); m_stream = fstream(filePath, ios::in|ios::out|ios::binary|ios::trunc); @@ -127,22 +50,13 @@ bool GpsTrackFile::Create(string const & filePath, size_t maxItemCount) if (!m_stream) return false; - try - { - m_filePath = filePath; + m_filePath = filePath; - m_header = Header(); - m_header.m_maxItemCount = maxItemCount + 1; + m_itemCount = 0; + m_maxItemCount = maxItemCount; - WriteHeader(m_header); - } - catch (RootException &) - { - m_header = Header(); - m_filePath.clear(); - m_stream.close(); - throw; - } + // Write position is set to the begin of file + ASSERT_EQUAL(m_stream.tellp(), 0, ()); return true; } @@ -154,272 +68,133 @@ bool GpsTrackFile::IsOpen() const void GpsTrackFile::Close() { - ASSERT(m_stream.is_open(), ("File is not open")); - - if (m_lazyWriteHeaderCount != 0) - { - m_lazyWriteHeaderCount = 0; - WriteHeader(m_header); - } + ASSERT(m_stream.is_open(), ()); m_stream.close(); if (0 != (m_stream.rdstate() & (ios::failbit | ios::badbit))) MYTHROW(WriteFileException, ("File:", m_filePath)); - m_header = Header(); m_filePath.clear(); + m_itemCount = 0; + m_maxItemCount = 0; } void GpsTrackFile::Flush() { - ASSERT(m_stream.is_open(), ("File is not open")); - - if (m_lazyWriteHeaderCount != 0) - { - m_lazyWriteHeaderCount = 0; - WriteHeader(m_header); - } + ASSERT(m_stream.is_open(), ()); m_stream.flush(); if (0 != (m_stream.rdstate() & (ios::failbit | ios::badbit))) MYTHROW(WriteFileException, ("File:", m_filePath)); } -size_t GpsTrackFile::Append(TItem const & item, size_t & evictedId) +void GpsTrackFile::Append(vector const & items) { - ASSERT(m_stream.is_open(), ("File is not open")); + ASSERT(m_stream.is_open(), ()); - if (item.m_timestamp < m_header.m_timestamp) - { - evictedId = kInvalidId; // nothing was evicted - return kInvalidId; // nothing was added - } + bool const needTrunc = (m_itemCount + items.size()) > (m_maxItemCount * 2); // see NOTE in class declaration - size_t const newLast = (m_header.m_last + 1) % m_header.m_maxItemCount; - size_t const newFirst = (newLast == m_header.m_first) ? ((m_header.m_first + 1) % m_header.m_maxItemCount) : m_header.m_first; + if (needTrunc) + TruncFile(); - WriteItem(m_header.m_last, item); + // Write position must be after last item position + ASSERT_EQUAL(m_stream.tellp(), m_itemCount * sizeof(TItem), ()); - size_t const addedId = m_header.m_lastId; + m_stream.write(reinterpret_cast(&items[0]), items.size() * sizeof(TItem)); + if (0 != (m_stream.rdstate() & (ios::failbit | ios::badbit ))) + MYTHROW(WriteFileException, ("File:", m_filePath)); - if (m_header.m_first == newFirst) - evictedId = kInvalidId; // nothing was evicted - else - evictedId = m_header.m_lastId - GetCount(); // the id of the first item - - m_header.m_first = newFirst; - m_header.m_last = newLast; - m_header.m_timestamp = item.m_timestamp; - m_header.m_lastId += 1; - - LazyWriteHeader(); - - return addedId; + m_itemCount += items.size(); } -void GpsTrackFile::ForEach(function const & fn) +void GpsTrackFile::Clear() { - ASSERT(m_stream.is_open(), ("File is not open")); + m_itemCount = 0; - double prevTimestamp = 0; + m_stream.close(); + m_stream = fstream(m_filePath, ios::in|ios::out|ios::binary|ios::trunc); - size_t id = m_header.m_lastId - GetCount(); // the id of the first item + if (!m_stream) + MYTHROW(WriteFileException, ("File:", m_filePath)); - for (size_t i = m_header.m_first; i != m_header.m_last; i = (i + 1) % m_header.m_maxItemCount) - { - TItem item; - if (!ReadItem(i, item)) - MYTHROW(CorruptedFileException, ("File:", m_filePath)); - - if (prevTimestamp > item.m_timestamp) - MYTHROW(CorruptedFileException, ("File:", m_filePath)); - - size_t itemId = id; - if (!fn(item, itemId)) - break; - - prevTimestamp = item.m_timestamp; - ++id; - } + // Write position is set to the begin of file + ASSERT_EQUAL(m_stream.tellp(), 0, ()); } -pair GpsTrackFile::DropEarlierThan(double timestamp) +void GpsTrackFile::ForEach(double timestampSince, std::function const & fn) { - ASSERT(m_stream.is_open(), ("File is not open")); + ASSERT(m_stream.is_open(), ()); - if (IsEmpty()) - return make_pair(kInvalidId, kInvalidId); // nothing was dropped + size_t i = GetFirstItemIndex(); - if (m_header.m_timestamp <= timestamp) - return Clear(); - - size_t const n = GetCount(); - - ASSERT_GREATER_OR_EQUAL(m_header.m_lastId, n, ()); - - // Try linear search for short distance - // In common case elements will be removed from the tail by small pieces - size_t const linearSearchCount = min(kLinearSearchDistance, n); - for (size_t i = m_header.m_first, j = 0; i != m_header.m_last; i = (i + 1) % m_header.m_maxItemCount, ++j) - { - if (j >= linearSearchCount) - break; - - TItem item; - if (!ReadItem(i, item)) - MYTHROW(CorruptedFileException, ("File:", m_filePath)); - - if (item.m_timestamp >= timestamp) - { - // Dropped range is - pair const res = make_pair(m_header.m_lastId - n, - m_header.m_lastId - n + j - 1); - - // Update header.first, if need - if (i != m_header.m_first) - { - m_header.m_first = i; - - LazyWriteHeader(); - } - - return res; - } - } - - // By nature items are sorted by timestamp. - // Use lower_bound algorithm to find first element later than timestamp. - size_t len = n, first = m_header.m_first; - while (len > 0) - { - size_t const step = len / 2; - size_t const index = (first + step) % m_header.m_maxItemCount; - - TItem item; - if (!ReadItem(index, item)) - MYTHROW(CorruptedFileException, ("File:", m_filePath)); - - if (item.m_timestamp < timestamp) - { - first = (index + 1) % m_header.m_maxItemCount; - len -= step + 1; - } - else - len = step; - } - - // Dropped range is - pair const res = - make_pair(m_header.m_lastId - n, - m_header.m_lastId - n + Distance(m_header.m_first, first, m_header.m_maxItemCount) - 1); - - // Update header.first, if need - if (first != m_header.m_first) - { - m_header.m_first = first; - - LazyWriteHeader(); - } - - return res; -} - -pair GpsTrackFile::Clear() -{ - ASSERT(m_stream.is_open(), ("File is not open")); - - if (m_header.m_first == 0 && m_header.m_last == 0 && - m_header.m_lastId == 0 && m_header.m_timestamp == 0) - { - return make_pair(kInvalidId, kInvalidId); // nothing was dropped - } - - // Dropped range is - pair const res = make_pair(m_header.m_lastId - GetCount(), - m_header.m_lastId - 1); - - m_header.m_first = 0; - m_header.m_last = 0; - m_header.m_lastId = 0; - m_header.m_timestamp = 0; - - LazyWriteHeader(); - - return res; -} - -size_t GpsTrackFile::GetMaxCount() const -{ - return m_header.m_maxItemCount == 0 ? 0 : m_header.m_maxItemCount - 1; -} - -size_t GpsTrackFile::GetCount() const -{ - return Distance(m_header.m_first, m_header.m_last, m_header.m_maxItemCount); -} - -bool GpsTrackFile::IsEmpty() const -{ - return m_header.m_first == m_header.m_last; -} - -double GpsTrackFile::GetTimestamp() const -{ - return m_header.m_timestamp; -} - -void GpsTrackFile::LazyWriteHeader() -{ - ++m_lazyWriteHeaderCount; - if (m_lazyWriteHeaderCount < kLazyWriteHeaderMaxCount) - return; - - m_lazyWriteHeaderCount = 0; - WriteHeader(m_header); -} - -bool GpsTrackFile::ReadHeader(Header & header) -{ - m_stream.seekg(0, ios::beg); - m_stream.read(reinterpret_cast(&header), sizeof(header)); + // Set read position to the first item + m_stream.seekg(i * sizeof(TItem), ios::beg); if (0 != (m_stream.rdstate() & (ios::failbit | ios::badbit))) MYTHROW(ReadFileException, ("File:", m_filePath)); - return ((m_stream.rdstate() & ios::eofbit) == 0); + + for (; i < m_itemCount; ++i) + { + TItem item; + m_stream.read(reinterpret_cast(&item), sizeof(TItem)); + if (0 != (m_stream.rdstate() & (ios::failbit | ios::badbit | ios::eofbit))) + MYTHROW(ReadFileException, ("File:", m_filePath)); + + if (item.m_timestamp >= timestampSince) + { + if (!fn(item)) + break; + } + } } -void GpsTrackFile::WriteHeader(Header const & header) +void GpsTrackFile::TruncFile() { - m_stream.seekp(0, ios::beg); - m_stream.write(reinterpret_cast(&header), sizeof(header)); - if (0 != (m_stream.rdstate() & (ios::failbit | ios::badbit))) + string const tmpFilePath = m_filePath + ".tmp"; + + fstream tmp = fstream(tmpFilePath, ios::in|ios::out|ios::binary|ios::trunc); + if (!tmp) MYTHROW(WriteFileException, ("File:", m_filePath)); -} -bool GpsTrackFile::ReadItem(size_t index, TItem & item) -{ - size_t const offset = ItemOffset(index); - m_stream.seekg(offset, ios::beg); - m_stream.read(reinterpret_cast(&item), sizeof(item)); + size_t i = GetFirstItemIndex(); + + // Set read position to the first item + m_stream.seekg(i * sizeof(TItem), ios::beg); if (0 != (m_stream.rdstate() & (ios::failbit | ios::badbit))) MYTHROW(ReadFileException, ("File:", m_filePath)); - return ((m_stream.rdstate() & ios::eofbit) == 0); -} -void GpsTrackFile::WriteItem(size_t index, TItem const & item) -{ - size_t const offset = ItemOffset(index); - m_stream.seekp(offset, ios::beg); - m_stream.write(reinterpret_cast(&item), sizeof(item)); - if (0 != (m_stream.rdstate() & (ios::failbit | ios::badbit))) + size_t itemCount = 0; + for (; i < m_itemCount; ++i, ++itemCount) + { + TItem item; + m_stream.read(reinterpret_cast(&item), sizeof(TItem)); + if (0 != (m_stream.rdstate() & (ios::failbit | ios::badbit | ios::eofbit))) + MYTHROW(ReadFileException, ("File:", m_filePath)); + + tmp.write(reinterpret_cast(&item), sizeof(TItem)); + if (0 != (tmp.rdstate() & (ios::failbit | ios::badbit))) + MYTHROW(WriteFileException, ("File:", tmpFilePath)); + } + + tmp.close(); + m_stream.close(); + + if (!my::DeleteFileX(m_filePath) || + !my::RenameFileX(tmpFilePath, m_filePath)) + { MYTHROW(WriteFileException, ("File:", m_filePath)); + } + + m_stream = fstream(m_filePath, ios::in|ios::out|ios::binary|ios::ate); + if (!m_stream) + MYTHROW(WriteFileException, ("File:", m_filePath)); + + m_itemCount = itemCount; + + // Write position must be after last item position (end of file) + ASSERT_EQUAL(m_stream.tellp(), m_itemCount * sizeof(TItem), ()); } -size_t GpsTrackFile::ItemOffset(size_t index) +size_t GpsTrackFile::GetFirstItemIndex() const { - return sizeof(Header) + index * sizeof(TItem); -} - -size_t GpsTrackFile::Distance(size_t first, size_t last, size_t maxItemCount) -{ - return (first <= last) ? (last - first) : (last + maxItemCount - first); + return (m_itemCount > m_maxItemCount) ? (m_itemCount - m_maxItemCount) : 0; // see NOTE in class declaration } diff --git a/map/gps_track_file.hpp b/map/gps_track_file.hpp index b6f9d14d38..9c3afc3ac9 100644 --- a/map/gps_track_file.hpp +++ b/map/gps_track_file.hpp @@ -14,9 +14,6 @@ class GpsTrackFile final public: DECLARE_EXCEPTION(WriteFileException, RootException); DECLARE_EXCEPTION(ReadFileException, RootException); - DECLARE_EXCEPTION(CorruptedFileException, RootException); - - static size_t const kInvalidId; // = numeric_limits::max(); using TItem = location::GpsTrackInfo; @@ -27,100 +24,59 @@ public: /// @param filePath - path to the file on disk /// @param maxItemCount - max number of items in recycling file /// @return If file does not exist then returns false, if everything is ok then returns true. - /// @exception If file corrupted then throws CorruptedFileException file, or ReadFileException if read fails. + /// @exception WriteFileException if seek fails. bool Open(string const & filePath, size_t maxItemCount); /// Creates new file /// @param filePath - path to the file on disk /// @param maxItemCount - max number of items in recycling file - /// @return If file cannot be created then false is returned, if everything is ok then returns true. - /// @exceptions WriteFileException if write fails. + /// @return If file cannot be created then returns false, if everything is ok then returns true. bool Create(string const & filePath, size_t maxItemCount); /// Returns true if file is open, otherwise returns false bool IsOpen() const; /// Flushes all changes and closes file - /// @exceptions WriteFileException if write fails. + /// @exception WriteFileException if write fails void Close(); /// Flushes all changes in file - /// @exceptions WriteFileException if write fails. + /// @exception WriteFileException if write fails void Flush(); /// Appends new point in the file - /// @param item - gps track point. - /// @param evictedId - identifier of evicted point due to recycling, kInvalidId if there is no evicted point. - /// @returns identifier of point, kInvalidId if point was not added. - /// @note Timestamp must be not less than GetTimestamp(), otherwise function returns false. - /// @note File is circular, when GetMaxItemCount() limit is reached, old point is evicted from file. + /// @param items - collection of gps track points. /// @exceptions WriteFileException if write fails. - size_t Append(TItem const & item, size_t & evictedId); + void Append(vector const & items); - /// Remove all points from the file - /// @returns range of identifiers of evicted points, or pair(kInvalidId,kInvalidId) if nothing was evicted. + /// Removes all data from file /// @exceptions WriteFileException if write fails. - pair Clear(); + void Clear(); - /// Returns max number of points in recycling file - size_t GetMaxCount() const; - - /// Returns number of items in the file, this values is <= GetMaxItemCount() - size_t GetCount() const; - - /// Returns true if file does not contain points - bool IsEmpty() const; - - /// Returns upper bound timestamp, or zero if there is no points - double GetTimestamp() const; - - /// Enumerates all points from the file in timestamp ascending order - /// @param fn - callable object which receives points. If fn returns false then enumeration is stopped. - /// @exceptions CorruptedFileException if file is corrupted or ReadFileException if read fails - void ForEach(function const & fn); - - /// Drops points earlier than specified date - /// @param timestamp - timestamp of lower bound, number of seconds since 1.1.1970. - /// @returns range of identifiers of evicted points, or pair(kInvalidId,kInvalidId) if nothing was evicted. - /// @exceptions CorruptedFileException if file is corrupted, ReadFileException if read fails - /// or WriteFileException if write fails. - pair DropEarlierThan(double timestamp); + /// Reads file and calls functor for each item with timestamp not earlier than specified + /// @param fn - callable function, return true to stop ForEach + /// @param timestampSince - timestamp to read data since + /// @exceptions ReadFileException if read fails. + void ForEach(double timestampSince, std::function const & fn); private: - /// Header, stored in beginning of file - /// @note Due to recycling, indexes of items are reused, but idendifiers are unique - /// until Clear or create new file - struct Header - { - size_t m_maxItemCount; // max number of items in recycling file (n + 1 for end element) - double m_timestamp; // upper bound timestamp - size_t m_first; // index of first item - size_t m_last; // index of last item, items are in range [first,last) - size_t m_lastId; // identifier of the last item + void TruncFile(); + size_t GetFirstItemIndex() const; - Header() - : m_maxItemCount(0) - , m_timestamp(0) - , m_first(0) - , m_last(0) - , m_lastId(0) - {} - }; + string m_filePath; + fstream m_stream; + size_t m_maxItemCount; // max count valid items in file, read note + size_t m_itemCount; // current number of items in file, read note - void LazyWriteHeader(); - - bool ReadHeader(Header & header); - void WriteHeader(Header const & header); - - bool ReadItem(size_t index, TItem & item); - void WriteItem(size_t index, TItem const & item); - - static size_t ItemOffset(size_t index); - static size_t Distance(size_t first, size_t last, size_t maxItemCount); - - string m_filePath; // file name - fstream m_stream; // buffered file stream - Header m_header; // file header - - uint32_t m_lazyWriteHeaderCount; // request count for write header + // NOTE + // New items append to the end of file, when file become too big, it is truncated. + // Here 'silly window sindrome' is possible: for example max file size is 100k items, + // and 99k items are filled, when 2k items is coming, then 98k items is copying to the tmp file + // which replaces origin file, and 2k added to the new file. Next time, when 1k items + // is comming, then again file truncated. That means that trunc will happens every time. + // Similar problem in TCP called 'silly window sindrome'. + // To prevent that problem, max file size can be 2 x m_maxItemCount items, when current number of items m_itemCount + // exceed 2 x m_maxItemCount, then second half of file - m_maxItemCount items is copying to the tmp file, + // which replaces origin file. That means that trunc will happens only then new m_maxItemCount items will be + // added but not every time. }; diff --git a/map/map_tests/gps_track_file_test.cpp b/map/map_tests/gps_track_file_test.cpp index 2814bfb1ce..c362cf91cb 100644 --- a/map/map_tests/gps_track_file_test.cpp +++ b/map/map_tests/gps_track_file_test.cpp @@ -34,378 +34,193 @@ inline string GetGpsTrackFilePath() } // namespace -UNIT_TEST(GpsTrackFile_SimpleWriteRead) +UNIT_TEST(GpsTrackFile_WriteReadWithoutTrunc) { - string const filePath = GetGpsTrackFilePath(); - MY_SCOPE_GUARD(gpsTestFileDeleter, bind(FileWriter::DeleteFileX, filePath)); - time_t const t = system_clock::to_time_t(system_clock::now()); double const timestamp = t; LOG(LINFO, ("Timestamp", ctime(&t), timestamp)); - size_t const fileMaxItemCount = 100000; - - // Write GPS tracks. - // (write only half of max items to do not do recycling) - { - GpsTrackFile file; - file.Create(filePath, fileMaxItemCount); - - TEST(file.IsOpen(), ()); - - TEST_EQUAL(fileMaxItemCount, file.GetMaxCount(), ()); - - TEST_EQUAL(0, file.GetCount(), ()); - - for (size_t i = 0; i < fileMaxItemCount / 2; ++i) - { - size_t evictedId; - size_t addedId = file.Append(Make(timestamp + i, ms::LatLon(i + 1000, i + 2000), i + 3000), evictedId); - TEST_EQUAL(i, addedId, ()); - TEST_EQUAL(GpsTrackFile::kInvalidId, evictedId, ()); - } - - TEST_EQUAL(fileMaxItemCount / 2, file.GetCount(), ()); - - file.Close(); - - TEST(!file.IsOpen(), ()); - } - - // Read GPS tracks. - { - GpsTrackFile file; - file.Open(filePath, fileMaxItemCount); - - TEST(file.IsOpen(), ()); - TEST(!file.IsEmpty(), ()); - - TEST_EQUAL(fileMaxItemCount, file.GetMaxCount(), ()); - - TEST_EQUAL(fileMaxItemCount/2, file.GetCount(), ()); - - size_t i = 0; - file.ForEach([&i,timestamp](location::GpsTrackInfo const & info, size_t id)->bool - { - TEST_EQUAL(id, i, ()); - TEST_EQUAL(info.m_timestamp, timestamp + i, ()); - TEST_EQUAL(info.m_latitude, i + 1000, ()); - TEST_EQUAL(info.m_longitude, i + 2000, ()); - TEST_EQUAL(info.m_speed, i + 3000, ()); - ++i; - return true; - }); - - TEST_EQUAL(i, fileMaxItemCount / 2, ()); - - auto res = file.Clear(); - TEST_EQUAL(res.first, 0, ()); - TEST_EQUAL(res.second, fileMaxItemCount / 2 - 1, ()); - - TEST(file.IsEmpty(), ()); - - file.Close(); - - TEST(!file.IsOpen(), ()); - } -} - -UNIT_TEST(GpsTrackFile_WriteReadWithEvicting) -{ string const filePath = GetGpsTrackFilePath(); MY_SCOPE_GUARD(gpsTestFileDeleter, bind(FileWriter::DeleteFileX, filePath)); - time_t const t = system_clock::to_time_t(system_clock::now()); - double const timestamp = t; - LOG(LINFO, ("Timestamp", ctime(&t), timestamp)); - size_t const fileMaxItemCount = 100000; - // Write GPS tracks. - // 2 x fileMaxItemCount more items are written in cyclic file, first half will be evicted + vector points; + points.reserve(fileMaxItemCount); + for (size_t i = 0; i < fileMaxItemCount; ++i) + points.emplace_back(Make(timestamp + i, ms::LatLon(-90 + i, -180 + i), 60 + i)); + + // Create file, write data and check written data { GpsTrackFile file; - file.Create(filePath, fileMaxItemCount); - + TEST(file.Create(filePath, fileMaxItemCount), ()); TEST(file.IsOpen(), ()); - TEST_EQUAL(fileMaxItemCount, file.GetMaxCount(), ()); - - TEST_EQUAL(0, file.GetCount(), ()); - - for (size_t i = 0; i < 2 * fileMaxItemCount; ++i) - { - size_t evictedId; - size_t addedId = file.Append(Make(timestamp + i, ms::LatLon(i + 1000, i + 2000), i + 3000), evictedId); - TEST_EQUAL(i, addedId, ()); - if (i >= fileMaxItemCount) - { - TEST_EQUAL(i - fileMaxItemCount, evictedId, ()); - } - else - { - TEST_EQUAL(GpsTrackFile::kInvalidId, evictedId, ()); - } - } - - TEST_EQUAL(fileMaxItemCount, file.GetCount(), ()); - - file.Close(); - - TEST(!file.IsOpen(), ()); - } - - // Read GPS tracks - // Only last fileMaxItemCount must be in cyclic buffer - { - GpsTrackFile file; - file.Open(filePath, fileMaxItemCount); - - TEST(file.IsOpen(), ()); - TEST(!file.IsEmpty(), ()); - - TEST_EQUAL(fileMaxItemCount, file.GetMaxCount(), ()); - TEST_EQUAL(fileMaxItemCount, file.GetCount(), ()); + file.Append(points); size_t i = 0; - file.ForEach([&i,timestamp](location::GpsTrackInfo const & info, size_t id)->bool + file.ForEach(0, [&](location::GpsTrackInfo const & point)->bool { - TEST_EQUAL(id, i + fileMaxItemCount, ()); - TEST_EQUAL(info.m_timestamp, timestamp + i + fileMaxItemCount, ()); - TEST_EQUAL(info.m_latitude, i + 1000 + fileMaxItemCount, ()); - TEST_EQUAL(info.m_longitude, i + 2000 + fileMaxItemCount, ()); - TEST_EQUAL(info.m_speed, i + 3000 + fileMaxItemCount, ()); + TEST_EQUAL(point.m_latitude, points[i].m_latitude, ()); + TEST_EQUAL(point.m_longitude, points[i].m_longitude, ()); + TEST_EQUAL(point.m_timestamp, points[i].m_timestamp, ()); + TEST_EQUAL(point.m_speed, points[i].m_speed, ()); ++i; return true; }); - TEST_EQUAL(i, fileMaxItemCount, ()); file.Close(); - TEST(!file.IsOpen(), ()); } -} - -UNIT_TEST(GpsTrackFile_DropInTail) -{ - string const filePath = GetGpsTrackFilePath(); - MY_SCOPE_GUARD(gpsTestFileDeleter, bind(FileWriter::DeleteFileX, filePath)); - - time_t const t = system_clock::to_time_t(system_clock::now()); - double const timestamp = t; - LOG(LINFO, ("Timestamp", ctime(&t), timestamp)); - - GpsTrackFile file; - file.Create(filePath, 100); - - TEST(file.IsOpen(), ()); - - TEST_EQUAL(100, file.GetMaxCount(), ()); - - for (size_t i = 0; i < 50; ++i) - { - size_t evictedId; - size_t addedId = file.Append(Make(timestamp + i, ms::LatLon(i + 1000, i + 2000), i + 3000), evictedId); - TEST_EQUAL(i, addedId, ()); - TEST_EQUAL(GpsTrackFile::kInvalidId, evictedId, ()); - } - - TEST_EQUAL(50, file.GetCount(), ()); - - auto res = file.DropEarlierThan(timestamp + 4.5); // drop points 0,1,2,3,4 - TEST_EQUAL(res.first, 0, ()); - TEST_EQUAL(res.second, 4, ()); - - TEST_EQUAL(45, file.GetCount(), ()); - - size_t i = 5; // new first - file.ForEach([&i,timestamp](location::GpsTrackInfo const & info, size_t id)->bool - { - TEST_EQUAL(info.m_timestamp, timestamp + i, ()); - TEST_EQUAL(info.m_latitude, i + 1000, ()); - TEST_EQUAL(info.m_longitude, i + 2000, ()); - TEST_EQUAL(info.m_speed, i + 3000, ()); - ++i; - return true; - }); - - res = file.Clear(); - TEST_EQUAL(res.first, 5, ()); - TEST_EQUAL(res.second, 49, ()); - - TEST(file.IsEmpty(), ()) - - file.Close(); - - TEST(!file.IsOpen(), ()); -} - -UNIT_TEST(GpsTrackFile_DropInMiddle) -{ - string const filePath = GetGpsTrackFilePath(); - MY_SCOPE_GUARD(gpsTestFileDeleter, bind(FileWriter::DeleteFileX, filePath)); - - time_t const t = system_clock::to_time_t(system_clock::now()); - double const timestamp = t; - LOG(LINFO, ("Timestamp", ctime(&t), timestamp)); - - GpsTrackFile file; - file.Create(filePath, 100); - - TEST(file.IsOpen(), ()); - - TEST_EQUAL(100, file.GetMaxCount(), ()); - - for (size_t i = 0; i < 50; ++i) - { - size_t evictedId; - size_t addedId = file.Append(Make(timestamp + i, ms::LatLon(i + 1000,i + 2000), i + 3000), evictedId); - TEST_EQUAL(i, addedId, ()); - TEST_EQUAL(GpsTrackFile::kInvalidId, evictedId, ()); - } - - TEST_EQUAL(50, file.GetCount(), ()); - - auto res = file.DropEarlierThan(timestamp + 48.5); // drop all except last - TEST_EQUAL(res.first, 0, ()); - TEST_EQUAL(res.second, 48, ()); - - TEST_EQUAL(1, file.GetCount(), ()); - - size_t i = 49; // new first - file.ForEach([&i,timestamp](location::GpsTrackInfo const & info, size_t id)->bool - { - TEST_EQUAL(info.m_timestamp, timestamp + i, ()); - TEST_EQUAL(info.m_latitude, i + 1000, ()); - TEST_EQUAL(info.m_longitude, i + 2000, ()); - TEST_EQUAL(info.m_speed, i + 3000, ()); - ++i; - return true; - }); - - file.Close(); - - TEST(!file.IsOpen(), ()); -} - -UNIT_TEST(GpsTrackFile_DropAll) -{ - string const filePath = GetGpsTrackFilePath(); - MY_SCOPE_GUARD(gpsTestFileDeleter, bind(FileWriter::DeleteFileX, filePath)); - - time_t const t = system_clock::to_time_t(system_clock::now()); - double const timestamp = t; - LOG(LINFO, ("Timestamp", ctime(&t), timestamp)); - - GpsTrackFile file; - file.Create(filePath, 100); - - TEST(file.IsOpen(), ()); - - TEST_EQUAL(100, file.GetMaxCount(), ()); - - for (size_t i = 0; i < 50; ++i) - { - size_t evictedId; - size_t addedId = file.Append(Make(timestamp + i, ms::LatLon(i + 1000, i + 2000), i + 3000), evictedId); - TEST_EQUAL(i, addedId, ()); - TEST_EQUAL(GpsTrackFile::kInvalidId, evictedId, ()); - } - - TEST_EQUAL(50, file.GetCount(), ()); - - auto res = file.DropEarlierThan(timestamp + 51); // drop all - TEST_EQUAL(res.first, 0, ()); - TEST_EQUAL(res.second, 49, ()); - - TEST(file.IsEmpty(), ()); - - TEST_EQUAL(0, file.GetCount(), ()); - - file.Close(); - - TEST(!file.IsOpen(), ()); -} - -UNIT_TEST(GpsTrackFile_Clear) -{ - string const filePath = GetGpsTrackFilePath(); - MY_SCOPE_GUARD(gpsTestFileDeleter, bind(FileWriter::DeleteFileX, filePath)); - - time_t const t = system_clock::to_time_t(system_clock::now()); - double const timestamp = t; - LOG(LINFO, ("Timestamp", ctime(&t), timestamp)); - - GpsTrackFile file; - file.Create(filePath, 100); - - TEST(file.IsOpen(), ()); - - TEST_EQUAL(100, file.GetMaxCount(), ()); - - for (size_t i = 0; i < 50; ++i) - { - size_t evictedId; - size_t addedId = file.Append(Make(timestamp + i, ms::LatLon(i + 1000, i + 2000), i + 3000), evictedId); - TEST_EQUAL(i, addedId, ()); - TEST_EQUAL(GpsTrackFile::kInvalidId, evictedId, ()); - } - - TEST_EQUAL(50, file.GetCount(), ()); - - auto res = file.Clear(); - TEST_EQUAL(res.first, 0, ()); - TEST_EQUAL(res.second, 49, ()); - - TEST(file.IsEmpty(), ()); - - TEST_EQUAL(0, file.GetCount(), ()); - - file.Close(); - - TEST(!file.IsOpen(), ()); -} - -UNIT_TEST(GpsTrackFile_CreateOpenClose) -{ - string const filePath = GetGpsTrackFilePath(); - MY_SCOPE_GUARD(gpsTestFileDeleter, bind(FileWriter::DeleteFileX, filePath)); - - time_t const t = system_clock::to_time_t(system_clock::now()); - double const timestamp = t; - LOG(LINFO, ("Timestamp", ctime(&t), timestamp)); + // Open file and check previously written data { GpsTrackFile file; - file.Create(filePath, 100); - + TEST(file.Open(filePath, fileMaxItemCount), ()); TEST(file.IsOpen(), ()); - TEST_EQUAL(100, file.GetMaxCount(), ()); - TEST_EQUAL(0, file.GetCount(), ()); + size_t i = 0; + file.ForEach(0, [&](location::GpsTrackInfo const & point)->bool + { + TEST_EQUAL(point.m_latitude, points[i].m_latitude, ()); + TEST_EQUAL(point.m_longitude, points[i].m_longitude, ()); + TEST_EQUAL(point.m_timestamp, points[i].m_timestamp, ()); + TEST_EQUAL(point.m_speed, points[i].m_speed, ()); + ++i; + return true; + }); + TEST_EQUAL(i, fileMaxItemCount, ()); + + // Clear data + file.Clear(); file.Close(); - TEST(!file.IsOpen(), ()); } + // Open file and check there is no data { GpsTrackFile file; - file.Open(filePath, 100); - + TEST(file.Open(filePath, fileMaxItemCount), ()); TEST(file.IsOpen(), ()); - TEST(file.IsEmpty(), ()); - TEST_EQUAL(100, file.GetMaxCount(), ()); - TEST_EQUAL(0, file.GetCount(), ()); - - auto res = file.Clear(); - TEST_EQUAL(res.first, GpsTrackFile::kInvalidId, ()); - TEST_EQUAL(res.second, GpsTrackFile::kInvalidId, ()); + size_t i = 0; + file.ForEach(0, [&](location::GpsTrackInfo const & point)->bool{ ++i; return true; }); + TEST_EQUAL(i, 0, ()); + + file.Close(); + TEST(!file.IsOpen(), ()); + } +} + +UNIT_TEST(GpsTrackFile_WriteReadWithTrunc) +{ + time_t const t = system_clock::to_time_t(system_clock::now()); + double const timestamp = t; + LOG(LINFO, ("Timestamp", ctime(&t), timestamp)); + + string const filePath = GetGpsTrackFilePath(); + MY_SCOPE_GUARD(gpsTestFileDeleter, bind(FileWriter::DeleteFileX, filePath)); + + size_t const fileMaxItemCount = 100000; + + vector points1, points2; + points1.reserve(fileMaxItemCount); + points2.reserve(fileMaxItemCount); + for (size_t i = 0; i < fileMaxItemCount; ++i) + { + points1.emplace_back(Make(timestamp + i, ms::LatLon(-90 + i, -180 + i), 60 + i)); + points2.emplace_back(Make(timestamp + i + fileMaxItemCount, ms::LatLon(-45 + i, -30 + i), 15 + i)); + } + + vector points3; + points3.reserve(fileMaxItemCount/2); + for (size_t i = 0; i < fileMaxItemCount/2; ++i) + points3.emplace_back(Make(timestamp + i, ms::LatLon(-30 + i, -60 + i), 90 + i)); + + // Create file and write blob 1 + { + GpsTrackFile file; + TEST(file.Create(filePath, fileMaxItemCount), ()); + TEST(file.IsOpen(), ()); + + file.Append(points1); + + file.Close(); + TEST(!file.IsOpen(), ()); + } + + // Create file and write blob 2 + { + GpsTrackFile file; + TEST(file.Open(filePath, fileMaxItemCount), ()); + TEST(file.IsOpen(), ()); + + file.Append(points2); + + file.Close(); + TEST(!file.IsOpen(), ()); + } + + // Create file and write blob 3 + { + GpsTrackFile file; + TEST(file.Open(filePath, fileMaxItemCount), ()); + TEST(file.IsOpen(), ()); + + file.Append(points3); // trunc happens here + + file.Close(); + TEST(!file.IsOpen(), ()); + } + + // Check file must contain second half of points2 and points3 + { + GpsTrackFile file; + TEST(file.Open(filePath, fileMaxItemCount), ()); + TEST(file.IsOpen(), ()); + + size_t i = 0; + file.ForEach(0, [&](location::GpsTrackInfo const & point)->bool + { + if (i < fileMaxItemCount/2) + { + TEST_EQUAL(point.m_latitude, points2[fileMaxItemCount/2 + i].m_latitude, ()); + TEST_EQUAL(point.m_longitude, points2[fileMaxItemCount/2 + i].m_longitude, ()); + TEST_EQUAL(point.m_timestamp, points2[fileMaxItemCount/2 + i].m_timestamp, ()); + TEST_EQUAL(point.m_speed, points2[fileMaxItemCount/2 + i].m_speed, ()); + } + else + { + TEST_EQUAL(point.m_latitude, points3[i - fileMaxItemCount/2].m_latitude, ()); + TEST_EQUAL(point.m_longitude, points3[i - fileMaxItemCount/2].m_longitude, ()); + TEST_EQUAL(point.m_timestamp, points3[i - fileMaxItemCount/2].m_timestamp, ()); + TEST_EQUAL(point.m_speed, points3[i - fileMaxItemCount/2].m_speed, ()); + } + ++i; + return true; + }); + TEST_EQUAL(i, fileMaxItemCount, ()); + + // Clear data + file.Clear(); + + file.Close(); + TEST(!file.IsOpen(), ()); + } + + // Check no data in file + { + GpsTrackFile file; + TEST(file.Open(filePath, fileMaxItemCount), ()); + TEST(file.IsOpen(), ()); + + size_t i = 0; + file.ForEach(0, [&](location::GpsTrackInfo const & point)->bool{ ++i; return true; }); + TEST_EQUAL(i, 0, ()); file.Close(); - TEST(!file.IsOpen(), ()); } }