From ae88aa1971d63e506ba54000dc3aa53135534eb2 Mon Sep 17 00:00:00 2001 From: vng Date: Sun, 3 Jul 2011 20:16:43 +0300 Subject: [PATCH] [Refactoring] Use mwm-file descriptor (string name) in Index. Open FileReader only when necessary. --- coding/file_container.cpp | 10 ++++--- coding/file_container.hpp | 3 ++- coding/reader.cpp | 6 ++--- coding/reader.hpp | 13 +++------ generator/dumper.cpp | 4 +-- generator/dumper.hpp | 2 +- generator/statistics.cpp | 8 +++--- generator/statistics.hpp | 4 +-- indexer/feature_processor.hpp | 11 ++++++-- indexer/index.hpp | 28 ++++++++------------ indexer/indexer_tests/index_builder_test.cpp | 26 ++++++++---------- indexer/indexer_tests/index_test.cpp | 7 +++-- map/feature_vec_model.cpp | 2 +- map/feature_vec_model.hpp | 3 +-- map/framework.cpp | 20 ++++++-------- map/framework.hpp | 4 +-- map/map_tests/map_foreach_test.cpp | 13 ++++----- storage/storage.cpp | 18 ++++++++----- storage/storage.hpp | 5 ++-- 19 files changed, 91 insertions(+), 96 deletions(-) diff --git a/coding/file_container.cpp b/coding/file_container.cpp index 808f056e93..eeb5c0ef1f 100644 --- a/coding/file_container.cpp +++ b/coding/file_container.cpp @@ -147,14 +147,18 @@ FileWriter FilesContainerW::GetExistingWriter(Tag const & tag) MYTHROW(Writer::OpenException, (tag)); } -void FilesContainerW::Append(string const & fName, Tag const & tag) +void FilesContainerW::Append(string const & fPath, Tag const & tag) +{ + Append(new FileReader(fPath), tag); +} + +void FilesContainerW::Append(ModelReaderPtr reader, Tag const & tag) { ASSERT(!m_bFinished, ()); uint64_t const bufferSize = 4*1024; char buffer[bufferSize]; - FileReader reader(fName); - ReaderSource src(reader); + ReaderSource src(reader); FileWriter writer = GetWriter(tag); uint64_t size = reader.Size(); diff --git a/coding/file_container.hpp b/coding/file_container.hpp index db80621bdd..7a1df9ac9c 100644 --- a/coding/file_container.hpp +++ b/coding/file_container.hpp @@ -83,7 +83,8 @@ public: /// @name Append to existing container. /// @precondition Container should be constructed with OP_APPEND. //@{ - void Append(string const & fName, Tag const & tag); + void Append(string const & fPath, Tag const & tag); + void Append(ModelReaderPtr reader, Tag const & tag); void Append(vector const & buffer, Tag const & tag); //@} diff --git a/coding/reader.cpp b/coding/reader.cpp index dc839d76db..4119bb2e0b 100644 --- a/coding/reader.cpp +++ b/coding/reader.cpp @@ -11,11 +11,11 @@ void Reader::ReadAsString(string & s) const Read(0, &s[0], sz); } -bool ModelReader::IsEqual(string const & fName) const +bool Reader::IsEqual(string const & name1, string const & name2) { #if defined(OMIM_OS_WINDOWS) - return strings::EqualNoCase(fName, m_name); + return strings::EqualNoCase(name1, name2); #else - return (fName == m_name); + return (name1 == name2); #endif } diff --git a/coding/reader.hpp b/coding/reader.hpp index 6f95401a63..7db2627892 100644 --- a/coding/reader.hpp +++ b/coding/reader.hpp @@ -26,6 +26,8 @@ public: virtual Reader * CreateSubReader(uint64_t pos, uint64_t size) const = 0; void ReadAsString(string & s) const; + + static bool IsEqual(string const & name1, string const & name2); }; // Reader from memory. @@ -122,18 +124,9 @@ public: { return m_p->CreateSubReader(pos, size); } - - inline bool IsEqual(string const & name) const - { - return m_p->IsEqual(name); - } - - inline bool IsEqual(ModelReaderPtr const & file) const - { - return m_p->IsEqual(file.m_p->GetName()); - } }; + // Source that reads from a reader. template class ReaderSource { diff --git a/generator/dumper.cpp b/generator/dumper.cpp index 6693bde894..7161e22080 100644 --- a/generator/dumper.cpp +++ b/generator/dumper.cpp @@ -48,10 +48,10 @@ namespace feature return first.second > second.second; } - void DumpTypes(string const & datFile) + void DumpTypes(string const & fPath) { TypesCollector doClass; - feature::ForEachFromDat(new FileReader(datFile), doClass); + feature::ForEachFromDat(fPath, doClass); typedef vector vec_to_sort; vec_to_sort vecToSort(doClass.m_stats.begin(), doClass.m_stats.end()); diff --git a/generator/dumper.hpp b/generator/dumper.hpp index 48a4bfcd57..01d852a7b3 100644 --- a/generator/dumper.hpp +++ b/generator/dumper.hpp @@ -4,5 +4,5 @@ namespace feature { - void DumpTypes(string const & datFile); + void DumpTypes(string const & fPath); } diff --git a/generator/statistics.cpp b/generator/statistics.cpp index be4a07f761..c0f85e3583 100644 --- a/generator/statistics.cpp +++ b/generator/statistics.cpp @@ -17,9 +17,9 @@ using namespace feature; namespace stats { - void FileContainerStatistic(string const & fName) + void FileContainerStatistic(string const & fPath) { - FilesContainerR cont(fName); + FilesContainerR cont(fPath); vector tags; tags.push_back(DATA_FILE_TAG); @@ -79,10 +79,10 @@ namespace stats } }; - void CalcStatistic(string const & fName, MapInfo & info) + void CalcStatistic(string const & fPath, MapInfo & info) { AccumulateStatistic doProcess(info); - feature::ForEachFromDat(new FileReader(fName), doProcess); + feature::ForEachFromDat(fPath, doProcess); } void PrintInfo(char const * prefix, GeneralInfo const & info) diff --git a/generator/statistics.hpp b/generator/statistics.hpp index 2f4956fd20..74e6622967 100644 --- a/generator/statistics.hpp +++ b/generator/statistics.hpp @@ -69,8 +69,8 @@ namespace stats } }; - void FileContainerStatistic(string const & fName); + void FileContainerStatistic(string const & fPath); - void CalcStatistic(string const & fName, MapInfo & info); + void CalcStatistic(string const & fPath, MapInfo & info); void PrintStatistic(MapInfo & info); } diff --git a/indexer/feature_processor.hpp b/indexer/feature_processor.hpp index 575223f3f4..c471d73583 100644 --- a/indexer/feature_processor.hpp +++ b/indexer/feature_processor.hpp @@ -2,6 +2,7 @@ #include "features_vector.hpp" +#include "../coding/file_reader.hpp" #include "../coding/file_container.hpp" #include "../std/bind.hpp" @@ -10,13 +11,19 @@ namespace feature { template - void ForEachFromDat(ModelReaderPtr const & file, ToDo & toDo) + void ForEachFromDat(ModelReaderPtr reader, ToDo & toDo) { - FilesContainerR container(file); + FilesContainerR container(reader); FeaturesVector featureSource(container); featureSource.ForEachOffset(bind(ref(toDo), _1, _2)); } + template + void ForEachFromDat(string const & fPath, ToDo & toDo) + { + ForEachFromDat(new FileReader(fPath), toDo); + } + /// Read feature from feature source. template void ReadFromSourceRowFormat(TSource & src, FeatureBuilder1 & f) diff --git a/indexer/index.hpp b/indexer/index.hpp index 8f5264abfc..eeb21b3db1 100644 --- a/indexer/index.hpp +++ b/indexer/index.hpp @@ -8,6 +8,8 @@ #include "../../defines.hpp" +#include "../platform/platform.hpp" + #include "../geometry/rect2d.hpp" #include "../coding/file_container.hpp" @@ -134,7 +136,7 @@ public: } } - void Add(ModelReaderPtr const & file) + void Add(string const & file) { threads::MutexGuard mutexGuard(m_mutex); UNUSED_VALUE(mutexGuard); @@ -148,14 +150,14 @@ public: UpdateIndexes(); } - void Remove(string const & path) + void Remove(string const & file) { threads::MutexGuard mutexGuard(m_mutex); UNUSED_VALUE(mutexGuard); for (size_t i = 0; i < m_indexes.size(); ++i) { - if (m_indexes[i]->IsMyData(path)) + if (m_indexes[i]->IsMyData(file)) m_indexes[i]->m_action = IndexProxy::INDEX_REMOVE; } @@ -218,15 +220,12 @@ private: class IndexProxy { public: - typedef ModelReaderPtr ReaderT; - - explicit IndexProxy(ReaderT const & file) + explicit IndexProxy(string const & file) : m_action(INDEX_DO_NOTHING), m_file(file), m_pIndex(NULL), m_lockCount(0), m_queriesSkipped(0) { - // TODO: If path is cellid-style-square, make rect from cellid and don't open the file. feature::DataHeader header; - header.Load(FilesContainerR(m_file).GetReader(HEADER_FILE_TAG)); + header.Load(FilesContainerR(GetPlatform().GetReader(m_file)).GetReader(HEADER_FILE_TAG)); m_rect = header.GetBounds(); m_scaleRange = header.GetScaleRange(); @@ -266,13 +265,9 @@ private: return m_lockCount == 0; } - bool IsMyData(string const & path) const + bool IsMyData(string const & file) const { - return m_file.IsEqual(path); - } - bool IsMyData(ReaderT const & file) const - { - return m_file.IsEqual(file); + return Reader::IsEqual(m_file, file); } void CloseIfUnlocked() @@ -312,8 +307,7 @@ private: { if (!m_pIndex) { - // LOG(LINFO, (m_Path)); - FilesContainerR container(m_file); + FilesContainerR container(GetPlatform().GetReader(m_file)); m_pIndex = new IndexT(container); } } @@ -329,7 +323,7 @@ private: } } - ReaderT m_file; + string m_file; m2::RectD m_rect; pair m_scaleRange; diff --git a/indexer/indexer_tests/index_builder_test.cpp b/indexer/indexer_tests/index_builder_test.cpp index 25d43af4f1..54221c6136 100644 --- a/indexer/indexer_tests/index_builder_test.cpp +++ b/indexer/indexer_tests/index_builder_test.cpp @@ -4,9 +4,13 @@ #include "../index_builder.hpp" #include "../classificator_loader.hpp" #include "../features_vector.hpp" + #include "../../defines.hpp" + #include "../../platform/platform.hpp" + #include "../../coding/file_container.hpp" + #include "../../base/stl_add.hpp" @@ -17,7 +21,7 @@ UNIT_TEST(BuildIndexTest) p.GetReader("classificator.txt"), p.GetReader("visibility.txt")); - FilesContainerR originalContainer(p.WritablePathForFile("minsk-pass" DATA_FILE_EXTENSION)); + FilesContainerR originalContainer(p.GetReader("minsk-pass" DATA_FILE_EXTENSION)); // Build index. vector serialIndex; @@ -30,26 +34,18 @@ UNIT_TEST(BuildIndexTest) // Create a new mwm file. string const fileName = "build_index_test" DATA_FILE_EXTENSION; - FileWriter::DeleteFileX(fileName); + string const filePath = p.WritablePathForFile(fileName); + FileWriter::DeleteFileX(filePath); // Copy original mwm file and replace index in it. { - FilesContainerW containerWriter(fileName); + FilesContainerW containerWriter(filePath); vector tags; originalContainer.ForEachTag(MakeBackInsertFunctor(tags)); for (size_t i = 0; i < tags.size(); ++i) { if (tags[i] != INDEX_FILE_TAG) - { - FilesContainerR::ReaderT reader = originalContainer.GetReader(tags[i]); - size_t const sz = static_cast(reader.Size()); - if (sz > 0) - { - vector data(sz); - reader.Read(0, &data[0], sz); - containerWriter.Append(data, tags[i]); - } - } + containerWriter.Append(originalContainer.GetReader(tags[i]), tags[i]); } containerWriter.Append(serialIndex, INDEX_FILE_TAG); } @@ -57,12 +53,12 @@ UNIT_TEST(BuildIndexTest) { // Check that index actually works. Index::Type index; - index.Add(new FileReader(fileName)); + index.Add(fileName); // Make sure that index is actually parsed. index.ForEachInScale(NoopFunctor(), 15); } // Clean after the test. - FileWriter::DeleteFileX(fileName); + FileWriter::DeleteFileX(filePath); } diff --git a/indexer/indexer_tests/index_test.cpp b/indexer/indexer_tests/index_test.cpp index ee14da7fc2..2c37c029c3 100644 --- a/indexer/indexer_tests/index_test.cpp +++ b/indexer/indexer_tests/index_test.cpp @@ -1,17 +1,16 @@ #include "../../testing/testing.hpp" #include "../index.hpp" -#include "../index_builder.hpp" -#include "../../platform/platform.hpp" -#include "../../coding/file_container.hpp" + #include "../../base/macros.hpp" #include "../../base/stl_add.hpp" + #include "../../std/string.hpp" UNIT_TEST(IndexParseTest) { Index::Type index; - index.Add(GetPlatform().GetReader("minsk-pass" DATA_FILE_EXTENSION)); + index.Add("minsk-pass" DATA_FILE_EXTENSION); // Make sure that index is actually parsed. index.ForEachInScale(NoopFunctor(), 15); diff --git a/map/feature_vec_model.cpp b/map/feature_vec_model.cpp index fa275398e9..132e28aeed 100644 --- a/map/feature_vec_model.cpp +++ b/map/feature_vec_model.cpp @@ -37,7 +37,7 @@ void FeaturesFetcher::InitClassificator() } } -void FeaturesFetcher::AddMap(ReaderT const & file) +void FeaturesFetcher::AddMap(string const & file) { try { diff --git a/map/feature_vec_model.hpp b/map/feature_vec_model.hpp index 03d553984e..fa205ef6c7 100644 --- a/map/feature_vec_model.hpp +++ b/map/feature_vec_model.hpp @@ -37,10 +37,9 @@ namespace model mutable index_t::Query m_multiIndexQuery; public: - void InitClassificator(); - void AddMap(ReaderT const & file); + void AddMap(string const & file); void RemoveMap(string const & fName); void Clean(); void ClearCaches(); diff --git a/map/framework.cpp b/map/framework.cpp index cbd92f7a20..ce636f0294 100644 --- a/map/framework.cpp +++ b/map/framework.cpp @@ -286,19 +286,15 @@ void FrameWork::AddRedrawCommandSure() } template - void FrameWork::AddMap(ReaderT const & file) + void FrameWork::AddMap(string const & file) { // update rect for Show All button feature::DataHeader header; - header.Load(FilesContainerR(file).GetReader(HEADER_FILE_TAG)); + header.Load(FilesContainerR(GetPlatform().GetReader(file)).GetReader(HEADER_FILE_TAG)); + m_model.AddWorldRect(header.GetBounds()); - m2::RectD bounds = header.GetBounds(); - - m_model.AddWorldRect(bounds); - { - threads::MutexGuard lock(m_modelSyn); - m_model.AddMap(file); - } + threads::MutexGuard lock(m_modelSyn); + m_model.AddMap(file); } template @@ -508,7 +504,7 @@ void FrameWork::AddRedrawCommandSure() class ReadersAdder { protected: - typedef vector maps_list_t; + typedef vector maps_list_t; private: Platform & m_pl; @@ -517,9 +513,9 @@ void FrameWork::AddRedrawCommandSure() public: ReadersAdder(Platform & pl, maps_list_t & lst) : m_pl(pl), m_lst(lst) {} - void operator() (string const & f) + void operator() (string const & name) { - m_lst.push_back(m_pl.GetReader(f)); + m_lst.push_back(name); } }; diff --git a/map/framework.hpp b/map/framework.hpp index 98364bec7f..f63af9cd74 100644 --- a/map/framework.hpp +++ b/map/framework.hpp @@ -183,7 +183,7 @@ class FrameWork typedef typename TModel::ReaderT ReaderT; - void AddMap(ReaderT const & file); + void AddMap(string const & file); void RemoveMap(string const & datFile); void OnGpsUpdate(location::GpsInfo const & info); @@ -202,7 +202,7 @@ public: model_t & get_model(); - typedef vector maps_list_t; + typedef vector maps_list_t; void EnumLocalMaps(maps_list_t & filesList); void EnumBenchmarkMaps(maps_list_t & filesList); diff --git a/map/map_tests/map_foreach_test.cpp b/map/map_tests/map_foreach_test.cpp index 0bb4f33efe..20f1cee755 100644 --- a/map/map_tests/map_foreach_test.cpp +++ b/map/map_tests/map_foreach_test.cpp @@ -2,8 +2,6 @@ #include "../../testing/testing.hpp" -#include "../../geometry/rect_intersect.hpp" - #include "../../platform/platform.hpp" #include "../../map/feature_vec_model.hpp" @@ -14,6 +12,8 @@ #include "../../indexer/feature_processor.hpp" #include "../../indexer/classificator.hpp" +#include "../../geometry/rect_intersect.hpp" + #include "../../base/logging.hpp" #include "../../std/string.hpp" @@ -227,14 +227,15 @@ namespace } }; - void RunTest(ModelReaderPtr const & file) + void RunTest(string const & file) { model::FeaturesFetcher src1; src1.InitClassificator(); src1.AddMap(file); feature::DataHeader mapInfo; - mapInfo.Load(FilesContainerR(file).GetReader(HEADER_FILE_TAG)); + ModelReaderPtr reader = GetPlatform().GetReader(file); + mapInfo.Load(FilesContainerR(reader).GetReader(HEADER_FILE_TAG)); vector rects; rects.push_back(mapInfo.GetBounds()); @@ -247,7 +248,7 @@ namespace feature_cont_t v1, v2; for_each_in_rect(src1, v1, r); - file_source_t src2(file); + file_source_t src2(reader); for_each_in_rect(src2, v2, r); int const level = scales::GetScaleLevel(r); @@ -278,7 +279,7 @@ namespace char c; cin >> c; if (c == 'y') - RunTest(GetPlatform().GetReader(fName + DATA_FILE_EXTENSION)); + RunTest(fName + DATA_FILE_EXTENSION); } } diff --git a/storage/storage.cpp b/storage/storage.cpp index c756acca1e..9ecbfd0dcd 100644 --- a/storage/storage.cpp +++ b/storage/storage.cpp @@ -53,7 +53,6 @@ namespace storage m_removeMap = removeFunc; m_updateRect = updateRectFunc; - typedef vector map_list_t; map_list_t filesList; enumMapsFunc(filesList); @@ -170,8 +169,7 @@ namespace storage } void operator()(TTile const & tile) { - string const file = m_workingDir + tile.first; - m_removeFn(file); + m_removeFn(tile.first); } }; @@ -353,12 +351,18 @@ namespace storage if (size.second != 0) m_countryProgress.m_current = size.first; - /// @todo Get file reader from download framework. - // activate downloaded map piece - m_addMap(new FileReader(result.m_file)); + // get file descriptor + string file = result.m_file; + string::size_type const i = file.find_last_of('/'); + if (i != string::npos) + file = file.substr(i+1); + // activate downloaded map piece + m_addMap(file); + + // update rect from downloaded file feature::DataHeader header; - header.Load(FilesContainerR(result.m_file).GetReader(HEADER_FILE_TAG)); + header.Load(FilesContainerR(GetPlatform().GetReader(file)).GetReader(HEADER_FILE_TAG)); m_updateRect(header.GetBounds()); } DownloadNextCountryFromQueue(); diff --git a/storage/storage.hpp b/storage/storage.hpp index 9e8a482ebe..a691cde90b 100644 --- a/storage/storage.hpp +++ b/storage/storage.hpp @@ -88,11 +88,12 @@ namespace storage /// @name Communicate with Framework //@{ + typedef vector map_list_t; public: - typedef function TAddMapFunction; + typedef function TAddMapFunction; typedef function TRemoveMapFunction; typedef function TUpdateRectFunction; - typedef function &)> TEnumMapsFunction; + typedef function TEnumMapsFunction; private: TAddMapFunction m_addMap;