pastk/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

Implemented GpsTrackStorage

Browse source
This commit is contained in:
Constantin Shalnev 2015-12-10 18:02:55 +03:00
parent 206ec64ce5
commit 77cf1357d0
7 changed files with 123 additions and 267 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

75
map/gps_track.cpp
View file

@ -139,7 +139,7 @@ void GpsTrack::ScheduleTask()
ProcessPoints();
}
CloseFile();
m_storage.reset();
});
}
@ -147,50 +147,18 @@ void GpsTrack::ScheduleTask()
m_cv.notify_one();
}
void GpsTrack::LazyInitFile()
void GpsTrack::InitStorageIfNeed()
{
if (m_file)
return;
m_file = make_unique<GpsTrackFile>();
// Open or create gps track file
try
{
if (!m_file->Open(m_filePath, m_maxItemCount))
{
if (!m_file->Create(m_filePath, m_maxItemCount))
{
LOG(LINFO, ("Cannot open or create GpsTrackFile:", m_filePath));
m_file.reset();
}
else
{
LOG(LINFO, ("GpsTrackFile has been created:", m_filePath));
}
}
}
catch (RootException & e)
{
LOG(LINFO, ("GpsTrackFile has caused exception:", e.Msg()));
m_file.reset();
}
}
void GpsTrack::CloseFile()
{
if (!m_file)
if (m_storage)
return;
try
{
m_file->Close();
m_file.reset();
m_storage = make_unique<GpsTrackStorage>(m_filePath, m_maxItemCount);
}
catch (RootException & e)
{
LOG(LINFO, ("GpsTrackFile.Close has caused exception:", e.Msg()));
m_file.reset();
LOG(LINFO, ("Storage has not been created:", e.Msg()));
}
}
@ -200,25 +168,24 @@ void GpsTrack::InitCollection(hours duration)
m_collection = make_unique<GpsTrackCollection>(m_maxItemCount, duration);
LazyInitFile();
if (!m_file)
InitStorageIfNeed();
if (!m_storage)
return;
try
{
m_file->ForEach([this](TItem const & info)->bool
m_storage->ForEach([this](TItem const & info)->bool
{
pair<size_t, size_t> evictedIds;
m_collection->Add(info, evictedIds);
return true;
});
}
catch (GpsTrackFile::ReadFileException & e)
catch (RootException & e)
{
LOG(LINFO, ("GpsTrackFile.ForEach has caused exception:", e.Msg()));
LOG(LINFO, ("Storage has caused exception:", e.Msg()));
m_collection->Clear();
m_file.reset();
m_storage.reset();
}
}
@ -240,7 +207,7 @@ void GpsTrack::ProcessPoints()
if (!m_collection && HasCallback())
InitCollection(duration);
UpdateFile(needClear, points);
UpdateStorage(needClear, points);
if (!m_collection)
return;
@ -254,27 +221,23 @@ bool GpsTrack::HasCallback()
return m_callback != nullptr;
}
void GpsTrack::UpdateFile(bool needClear, vector<TItem> const & points)
void GpsTrack::UpdateStorage(bool needClear, vector<TItem> const & points)
{
// Update file, if need
// If file exception happens, then drop the file.
LazyInitFile();
if (!m_file)
InitStorageIfNeed();
if (!m_storage)
return;
try
{
if (needClear)
m_file->Clear();
m_storage->Clear();
m_file->Append(points);
m_storage->Append(points);
}
catch (RootException & e)
{
LOG(LINFO, ("GpsTrackFile.Append has caused exception:", e.Msg()));
m_file.reset();
LOG(LINFO, ("Storage has caused exception:", e.Msg()));
m_storage.reset();
}
}

9
map/gps_track.hpp
View file

@ -1,7 +1,7 @@
#pragma once
#include "map/gps_track_collection.hpp"
#include "map/gps_track_file.hpp"
#include "map/gps_track_storage.hpp"
#include "std/condition_variable.hpp"
#include "std/mutex.hpp"
@ -56,10 +56,9 @@ private:
void ScheduleTask();
void ProcessPoints(); // called on the worker thread
bool HasCallback();
void LazyInitFile();
void CloseFile();
void InitStorageIfNeed();
void InitCollection(hours duration);
void UpdateFile(bool needClear, vector<TItem> const & points);
void UpdateStorage(bool needClear, vector<TItem> const & points);
void UpdateCollection(hours duration, bool needClear, vector<TItem> const & points);
size_t const m_maxItemCount;
@ -77,7 +76,7 @@ private:
TGpsTrackDiffCallback m_callback;
bool m_needSendSnapshop; // need send initial snapshot
unique_ptr<GpsTrackFile> m_file; // used in the worker thread
unique_ptr<GpsTrackStorage> m_storage; // used in the worker thread
unique_ptr<GpsTrackCollection> m_collection; // used in the worker thread
mutex m_threadGuard;

145
map/gps_track_file.cpp → map/gps_track_storage.cpp
View file

@ -1,4 +1,4 @@
#include "map/gps_track_file.hpp"
#include "map/gps_track_storage.hpp"
#include "coding/internal/file_data.hpp"
@ -12,90 +12,38 @@ namespace
size_t constexpr kItemBlockSize = 1000;
} // namespace
GpsTrackFile::GpsTrackFile()
: m_maxItemCount(0)
GpsTrackStorage::GpsTrackStorage(string const & filePath, size_t maxItemCount)
: m_filePath(filePath)
, m_maxItemCount(maxItemCount)
, m_itemCount(0)
{
ASSERT_GREATER(m_maxItemCount, 0, ());
m_stream = fstream(m_filePath, ios::in|ios::out|ios::binary|ios::ate);
if (m_stream)
{
size_t const fileSize = m_stream.tellp();
m_itemCount = fileSize / sizeof(TItem);
// Set write position after last item position
m_stream.seekp(m_itemCount * sizeof(TItem), ios::beg);
if (0 != (m_stream.rdstate() & (ios::failbit | ios::badbit)))
MYTHROW(OpenException, ("File:", m_filePath));
}
else
{
m_stream = fstream(m_filePath, ios::in|ios::out|ios::binary|ios::trunc);
if (!m_stream)
MYTHROW(OpenException, ("File:", m_filePath));
m_itemCount = 0;
}
}
GpsTrackFile::~GpsTrackFile()
{
}
bool GpsTrackFile::Open(string const & filePath, size_t maxItemCount)
{
ASSERT(!m_stream.is_open(), ());
ASSERT(maxItemCount > 0, ());
m_stream = fstream(filePath, ios::in|ios::out|ios::binary|ios::ate);
if (!m_stream)
return false;
m_filePath = filePath;
size_t const fileSize = m_stream.tellp();
m_itemCount = fileSize / sizeof(TItem);
m_maxItemCount = maxItemCount;
// Set write position after last item position
m_stream.seekp(m_itemCount * sizeof(TItem), ios::beg);
if (0 != (m_stream.rdstate() & (ios::failbit | ios::badbit)))
MYTHROW(WriteFileException, ("File:", m_filePath));
return true;
}
bool GpsTrackFile::Create(string const & filePath, size_t maxItemCount)
{
ASSERT(!m_stream.is_open(), ());
ASSERT(maxItemCount > 0, ());
m_stream = fstream(filePath, ios::in|ios::out|ios::binary|ios::trunc);
if (!m_stream)
return false;
m_filePath = filePath;
m_itemCount = 0;
m_maxItemCount = maxItemCount;
// Write position is set to the begin of file
ASSERT_EQUAL(m_stream.tellp(), 0, ());
return true;
}
bool GpsTrackFile::IsOpen() const
{
return m_stream.is_open();
}
void GpsTrackFile::Close()
{
ASSERT(m_stream.is_open(), ());
m_stream.close();
if (0 != (m_stream.rdstate() & (ios::failbit | ios::badbit)))
MYTHROW(WriteFileException, ("File:", m_filePath));
m_filePath.clear();
m_itemCount = 0;
m_maxItemCount = 0;
}
void GpsTrackFile::Flush()
{
ASSERT(m_stream.is_open(), ());
m_stream.flush();
if (0 != (m_stream.rdstate() & (ios::failbit | ios::badbit)))
MYTHROW(WriteFileException, ("File:", m_filePath));
}
void GpsTrackFile::Append(vector<TItem> const & items)
void GpsTrackStorage::Append(vector<TItem> const & items)
{
ASSERT(m_stream.is_open(), ());
@ -109,26 +57,32 @@ void GpsTrackFile::Append(vector<TItem> const & items)
m_stream.write(reinterpret_cast<char const *>(&items[0]), items.size() * sizeof(TItem));
if (0 != (m_stream.rdstate() & (ios::failbit | ios::badbit )))
MYTHROW(WriteFileException, ("File:", m_filePath));
MYTHROW(WriteException, ("File:", m_filePath));
m_stream.flush();
if (0 != (m_stream.rdstate() & (ios::failbit | ios::badbit )))
MYTHROW(WriteException, ("File:", m_filePath));
m_itemCount += items.size();
}
void GpsTrackFile::Clear()
void GpsTrackStorage::Clear()
{
ASSERT(m_stream.is_open(), ());
m_itemCount = 0;
m_stream.close();
m_stream = fstream(m_filePath, ios::in|ios::out|ios::binary|ios::trunc);
if (!m_stream)
MYTHROW(WriteFileException, ("File:", m_filePath));
MYTHROW(WriteException, ("File:", m_filePath));
// Write position is set to the begin of file
ASSERT_EQUAL(m_stream.tellp(), 0, ());
}
void GpsTrackFile::ForEach(std::function<bool(TItem const & item)> const & fn)
void GpsTrackStorage::ForEach(std::function<bool(TItem const & item)> const & fn)
{
ASSERT(m_stream.is_open(), ());
@ -137,7 +91,7 @@ void GpsTrackFile::ForEach(std::function<bool(TItem const & item)> const & fn)
// Set read position to the first item
m_stream.seekg(i * sizeof(TItem), ios::beg);
if (0 != (m_stream.rdstate() & (ios::failbit | ios::badbit)))
MYTHROW(ReadFileException, ("File:", m_filePath));
MYTHROW(ReadException, ("File:", m_filePath));
vector<TItem> items(kItemBlockSize);
for (; i < m_itemCount;)
@ -145,7 +99,7 @@ void GpsTrackFile::ForEach(std::function<bool(TItem const & item)> const & fn)
size_t const n = min(m_itemCount - i, items.size());
m_stream.read(reinterpret_cast<char *>(&items[0]), n * sizeof(TItem));
if (0 != (m_stream.rdstate() & (ios::failbit | ios::badbit | ios::eofbit)))
MYTHROW(ReadFileException, ("File:", m_filePath));
MYTHROW(ReadException, ("File:", m_filePath));
for (size_t j = 0; j < n; ++j)
{
@ -158,20 +112,20 @@ void GpsTrackFile::ForEach(std::function<bool(TItem const & item)> const & fn)
}
}
void GpsTrackFile::TruncFile()
void GpsTrackStorage::TruncFile()
{
string const tmpFilePath = m_filePath + ".tmp";
fstream tmp = fstream(tmpFilePath, ios::in|ios::out|ios::binary|ios::trunc);
if (!tmp)
MYTHROW(WriteFileException, ("Unable to create temporary file:", m_filePath));
MYTHROW(WriteException, ("Unable to create temporary file:", m_filePath));
size_t i = GetFirstItemIndex();
// Set read position to the first item
m_stream.seekg(i * sizeof(TItem), ios::beg);
if (0 != (m_stream.rdstate() & (ios::failbit | ios::badbit)))
MYTHROW(ReadFileException, ("File:", m_filePath));
MYTHROW(ReadException, ("File:", m_filePath));
size_t newItemCount = 0;
@ -183,11 +137,11 @@ void GpsTrackFile::TruncFile()
m_stream.read(reinterpret_cast<char *>(&items[0]), n * sizeof(TItem));
if (0 != (m_stream.rdstate() & (ios::failbit | ios::badbit | ios::eofbit)))
MYTHROW(ReadFileException, ("File:", m_filePath));
MYTHROW(ReadException, ("File:", m_filePath));
tmp.write(reinterpret_cast<char const *>(&items[0]), n * sizeof(TItem));
if (0 != (tmp.rdstate() & (ios::failbit | ios::badbit)))
MYTHROW(WriteFileException, ("File:", tmpFilePath));
MYTHROW(WriteException, ("File:", tmpFilePath));
i += n;
newItemCount += n;
@ -202,12 +156,13 @@ void GpsTrackFile::TruncFile()
if (!my::DeleteFileX(m_filePath) ||
!my::RenameFileX(tmpFilePath, m_filePath))
{
MYTHROW(WriteFileException, ("File:", m_filePath));
MYTHROW(WriteException, ("File:", m_filePath));
}
// Reopen stream
m_stream = fstream(m_filePath, ios::in|ios::out|ios::binary|ios::ate);
if (!m_stream)
MYTHROW(WriteFileException, ("File:", m_filePath));
MYTHROW(WriteException, ("File:", m_filePath));
m_itemCount = newItemCount;
@ -215,7 +170,7 @@ void GpsTrackFile::TruncFile()
ASSERT_EQUAL(m_stream.tellp(), m_itemCount * sizeof(TItem), ());
}
size_t GpsTrackFile::GetFirstItemIndex() const
size_t GpsTrackStorage::GetFirstItemIndex() const
{
return (m_itemCount > m_maxItemCount) ? (m_itemCount - m_maxItemCount) : 0; // see NOTE in declaration
}

50
map/gps_track_file.hpp → map/gps_track_storage.hpp
View file

@ -9,62 +9,42 @@
#include "std/limits.hpp"
#include "std/string.hpp"
class GpsTrackFile final
class GpsTrackStorage final
{
public:
DECLARE_EXCEPTION(WriteFileException, RootException);
DECLARE_EXCEPTION(ReadFileException, RootException);
DECLARE_EXCEPTION(OpenException, RootException);
DECLARE_EXCEPTION(WriteException, RootException);
DECLARE_EXCEPTION(ReadException, RootException);
using TItem = location::GpsTrackInfo;
GpsTrackFile();
~GpsTrackFile();
/// Opens file with track data.
/// Opens storage with track data.
/// @param filePath - path to the file on disk
/// @param maxItemCount - max number of items in recycling file
/// @return If file does not exist then returns false, if everything is ok then returns true.
/// @exception WriteFileException if seek fails.
bool Open(string const & filePath, size_t maxItemCount);
/// @exception OpenException if seek fails.
GpsTrackStorage(string const & filePath, size_t maxItemCount);
/// Creates new file
/// @param filePath - path to the file on disk
/// @param maxItemCount - max number of items in recycling file
/// @return If file cannot be created then returns false, if everything is ok then returns true.
bool Create(string const & filePath, size_t maxItemCount);
/// Returns true if file is open, otherwise returns false
bool IsOpen() const;
/// Flushes all changes and closes file
/// @exception WriteFileException if write fails
void Close();
/// Flushes all changes in file
/// @exception WriteFileException if write fails
void Flush();
/// Appends new point in the file
/// Appends new point to the storage
/// @param items - collection of gps track points.
/// @exceptions WriteFileException if write fails.
/// @exceptions WriteException if write fails or ReadException if read fails.
void Append(vector<TItem> const & items);
/// Removes all data from file
/// @exceptions WriteFileException if write fails.
/// Removes all data from the storage
/// @exceptions WriteException if write fails.
void Clear();
/// Reads file and calls functor for each item with timestamp not earlier than specified
/// Reads the storage and calls functor for each item
/// @param fn - callable function, return true to stop ForEach
/// @exceptions ReadFileException if read fails.
/// @exceptions ReadException if read fails.
void ForEach(std::function<bool(TItem const & item)> const & fn);
private:
void TruncFile();
size_t GetFirstItemIndex() const;
string m_filePath;
string const m_filePath;
size_t const m_maxItemCount;
fstream m_stream;
size_t m_maxItemCount; // max count valid items in file, read note
size_t m_itemCount; // current number of items in file, read note
// NOTE

4
map/map.pro
View file

@ -23,7 +23,7 @@ HEADERS += \
geourl_process.hpp \
gps_track.hpp \
gps_track_collection.hpp \
gps_track_file.hpp \
gps_track_storage.hpp \
mwm_url.hpp \
storage_bridge.hpp \
styled_point.hpp \
@ -45,7 +45,7 @@ SOURCES += \
geourl_process.cpp \
gps_track.cpp \
gps_track_collection.cpp \
gps_track_file.cpp \
gps_track_storage.cpp \
mwm_url.cpp \
storage_bridge.cpp \
styled_point.cpp \

105
map/map_tests/gps_track_file_test.cpp → map/map_tests/gps_track_storage_test.cpp
View file

@ -1,6 +1,6 @@
#include "testing/testing.hpp"
#include "map/gps_track_file.hpp"
#include "map/gps_track_storage.hpp"
#include "platform/platform.hpp"
@ -34,7 +34,7 @@ inline string GetGpsTrackFilePath()
} // namespace
UNIT_TEST(GpsTrackFile_WriteReadWithoutTrunc)
UNIT_TEST(GpsTrackStorage_WriteReadWithoutTrunc)
{
time_t const t = system_clock::to_time_t(system_clock::now());
double const timestamp = t;
@ -42,6 +42,7 @@ UNIT_TEST(GpsTrackFile_WriteReadWithoutTrunc)
string const filePath = GetGpsTrackFilePath();
MY_SCOPE_GUARD(gpsTestFileDeleter, bind(FileWriter::DeleteFileX, filePath));
FileWriter::DeleteFileX(filePath);
size_t const fileMaxItemCount = 100000;
@ -50,16 +51,14 @@ UNIT_TEST(GpsTrackFile_WriteReadWithoutTrunc)
for (size_t i = 0; i < fileMaxItemCount; ++i)
points.emplace_back(Make(timestamp + i, ms::LatLon(-90 + i, -180 + i), 60 + i));
// Create file, write data and check written data
// Open storage, write data and check written data
{
GpsTrackFile file;
TEST(file.Create(filePath, fileMaxItemCount), ());
TEST(file.IsOpen(), ());
GpsTrackStorage stg(filePath, fileMaxItemCount);
file.Append(points);
stg.Append(points);
size_t i = 0;
file.ForEach([&](location::GpsTrackInfo const & point)->bool
stg.ForEach([&](location::GpsTrackInfo const & point)->bool
{
TEST_EQUAL(point.m_latitude, points[i].m_latitude, ());
TEST_EQUAL(point.m_longitude, points[i].m_longitude, ());
@ -69,19 +68,14 @@ UNIT_TEST(GpsTrackFile_WriteReadWithoutTrunc)
return true;
});
TEST_EQUAL(i, fileMaxItemCount, ());
file.Close();
TEST(!file.IsOpen(), ());
}
// Open file and check previously written data
// Open storage and check previously written data
{
GpsTrackFile file;
TEST(file.Open(filePath, fileMaxItemCount), ());
TEST(file.IsOpen(), ());
GpsTrackStorage stg(filePath, fileMaxItemCount);
size_t i = 0;
file.ForEach([&](location::GpsTrackInfo const & point)->bool
stg.ForEach([&](location::GpsTrackInfo const & point)->bool
{
TEST_EQUAL(point.m_latitude, points[i].m_latitude, ());
TEST_EQUAL(point.m_longitude, points[i].m_longitude, ());
@ -93,28 +87,20 @@ UNIT_TEST(GpsTrackFile_WriteReadWithoutTrunc)
TEST_EQUAL(i, fileMaxItemCount, ());
// Clear data
file.Clear();
file.Close();
TEST(!file.IsOpen(), ());
stg.Clear();
}
// Open file and check there is no data
// Open storage and check there is no data
{
GpsTrackFile file;
TEST(file.Open(filePath, fileMaxItemCount), ());
TEST(file.IsOpen(), ());
GpsTrackStorage stg(filePath, fileMaxItemCount);
size_t i = 0;
file.ForEach([&](location::GpsTrackInfo const & point)->bool{ ++i; return true; });
stg.ForEach([&](location::GpsTrackInfo const & point)->bool{ ++i; return true; });
TEST_EQUAL(i, 0, ());
file.Close();
TEST(!file.IsOpen(), ());
}
}
UNIT_TEST(GpsTrackFile_WriteReadWithTrunc)
UNIT_TEST(GpsTrackStorage_WriteReadWithTrunc)
{
time_t const t = system_clock::to_time_t(system_clock::now());
double const timestamp = t;
@ -122,6 +108,7 @@ UNIT_TEST(GpsTrackFile_WriteReadWithTrunc)
string const filePath = GetGpsTrackFilePath();
MY_SCOPE_GUARD(gpsTestFileDeleter, bind(FileWriter::DeleteFileX, filePath));
FileWriter::DeleteFileX(filePath);
size_t const fileMaxItemCount = 100000;
@ -139,50 +126,30 @@ UNIT_TEST(GpsTrackFile_WriteReadWithTrunc)
for (size_t i = 0; i < fileMaxItemCount/2; ++i)
points3.emplace_back(Make(timestamp + i, ms::LatLon(-30 + i, -60 + i), 90 + i));
// Create file and write blob 1
// Open storage and write blob 1
{
GpsTrackFile file;
TEST(file.Create(filePath, fileMaxItemCount), ());
TEST(file.IsOpen(), ());
file.Append(points1);
file.Close();
TEST(!file.IsOpen(), ());
GpsTrackStorage stg(filePath, fileMaxItemCount);
stg.Append(points1);
}
// Create file and write blob 2
// Open storage and write blob 2
{
GpsTrackFile file;
TEST(file.Open(filePath, fileMaxItemCount), ());
TEST(file.IsOpen(), ());
file.Append(points2);
file.Close();
TEST(!file.IsOpen(), ());
GpsTrackStorage stg(filePath, fileMaxItemCount);
stg.Append(points2);
}
// Create file and write blob 3
// Open storage and write blob 3
{
GpsTrackFile file;
TEST(file.Open(filePath, fileMaxItemCount), ());
TEST(file.IsOpen(), ());
file.Append(points3); // trunc happens here
file.Close();
TEST(!file.IsOpen(), ());
GpsTrackStorage stg(filePath, fileMaxItemCount);
stg.Append(points3);
}
// Check file must contain second half of points2 and points3
// Check storage must contain second half of points2 and points3
{
GpsTrackFile file;
TEST(file.Open(filePath, fileMaxItemCount), ());
TEST(file.IsOpen(), ());
GpsTrackStorage stg(filePath, fileMaxItemCount);
size_t i = 0;
file.ForEach([&](location::GpsTrackInfo const & point)->bool
stg.ForEach([&](location::GpsTrackInfo const & point)->bool
{
if (i < fileMaxItemCount/2)
{
@ -204,23 +171,15 @@ UNIT_TEST(GpsTrackFile_WriteReadWithTrunc)
TEST_EQUAL(i, fileMaxItemCount, ());
// Clear data
file.Clear();
file.Close();
TEST(!file.IsOpen(), ());
stg.Clear();
}
// Check no data in file
// Check no data in storage
{
GpsTrackFile file;
TEST(file.Open(filePath, fileMaxItemCount), ());
TEST(file.IsOpen(), ());
GpsTrackStorage stg(filePath, fileMaxItemCount);
size_t i = 0;
file.ForEach([&](location::GpsTrackInfo const & point)->bool{ ++i; return true; });
stg.ForEach([&](location::GpsTrackInfo const & point)->bool{ ++i; return true; });
TEST_EQUAL(i, 0, ());
file.Close();
TEST(!file.IsOpen(), ());
}
}

2
map/map_tests/map_tests.pro
View file

@ -32,7 +32,7 @@ SOURCES += \
ge0_parser_tests.cpp \
geourl_test.cpp \
gps_track_collection_test.cpp \
gps_track_file_test.cpp \
gps_track_storage_test.cpp \
gps_track_test.cpp \
kmz_unarchive_test.cpp \
mwm_url_tests.cpp \