From c5dbcf62dbaffb8f45b26e79a2d31e0ed04eeea1 Mon Sep 17 00:00:00 2001 From: Yury Melnichek Date: Thu, 22 Sep 2011 15:03:12 +0200 Subject: [PATCH] Index refactoring. Breaks unit tests!! --- indexer/index.cpp | 26 ++ indexer/index.hpp | 460 +++---------------- indexer/indexer.pro | 2 + indexer/indexer_tests/index_builder_test.cpp | 2 +- indexer/indexer_tests/index_test.cpp | 2 +- indexer/mwm_set.cpp | 10 +- indexer/mwm_set.hpp | 20 +- map/feature_vec_model.cpp | 4 +- map/feature_vec_model.hpp | 2 +- search/engine.hpp | 2 +- search/query.cpp | 4 +- 11 files changed, 103 insertions(+), 431 deletions(-) create mode 100644 indexer/index.cpp diff --git a/indexer/index.cpp b/indexer/index.cpp new file mode 100644 index 0000000000..8c4bdb26e2 --- /dev/null +++ b/indexer/index.cpp @@ -0,0 +1,26 @@ +#include "index.hpp" +#include "data_header.hpp" +#include "../platform/platform.hpp" +#include "../std/bind.hpp" + +namespace +{ +FilesContainerR * CreateFileContainer(string const & fileName) +{ + return new FilesContainerR(fileName); +} +} // unnamed namespace + +Index::Index() : MwmSet(bind(&Index::FillInMwmInfo, this, _1, _2), &CreateFileContainer) +{ +} + +void Index::FillInMwmInfo(string const & fileName, MwmInfo & info) +{ + IndexFactory factory; + factory.Load(FilesContainerR(GetPlatform().GetReader(fileName))); + feature::DataHeader const & h = factory.GetHeader(); + info.m_limitRect = h.GetBounds(); + info.m_minScale = static_cast(h.GetScaleRange().first); + info.m_maxScale = static_cast(h.GetScaleRange().second); +} diff --git a/indexer/index.hpp b/indexer/index.hpp index 911cbb6ab5..80c6fec944 100644 --- a/indexer/index.hpp +++ b/indexer/index.hpp @@ -4,453 +4,97 @@ #include "feature_covering.hpp" #include "features_vector.hpp" #include "scale_index.hpp" -#include "search_trie.hpp" #include "../../defines.hpp" +#include "data_factory.hpp" +#include "mwm_set.hpp" -#include "../platform/platform.hpp" - -#include "../geometry/rect2d.hpp" - -#include "../coding/file_container.hpp" - -#include "../base/base.hpp" -#include "../base/macros.hpp" -#include "../base/mutex.hpp" -#include "../base/stl_add.hpp" -#include "../base/thread.hpp" - -#include "../std/algorithm.hpp" -#include "../std/bind.hpp" -#include "../std/string.hpp" #include "../std/unordered_set.hpp" -#include "../std/utility.hpp" #include "../std/vector.hpp" - -template class IndexForEachAdapter : public BaseT +class Index : public MwmSet { -private: - template - void CallForIntervals(F & f, covering::IntervalsT const & intervals, - m2::RectD const & rect, uint32_t scale) const - { - for (size_t i = 0; i < intervals.size(); ++i) - { - BaseT::ForEachInIntervalAndScale(f, intervals[i].first, intervals[i].second, - scale, rect); - } - } - public: + Index(); + template void ForEachInRect(F & f, m2::RectD const & rect, uint32_t scale) const { - CallForIntervals(f, covering::CoverViewportAndAppendLowerLevels(rect, RectId::DEPTH_LEVELS), - rect, scale); + ForEachInIntervals(f, covering::CoverViewportAndAppendLowerLevels(rect, RectId::DEPTH_LEVELS), + rect, scale); } template void ForEachInRect_TileDrawing(F & f, m2::RectD const & rect, uint32_t scale) const { - using namespace covering; - - IntervalsT intervals; - AppendLowerLevels(GetRectIdAsIs(rect), RectId::DEPTH_LEVELS, intervals); - - CallForIntervals(f, intervals, rect, scale); + covering::IntervalsT intervals; + covering::AppendLowerLevels(covering::GetRectIdAsIs(rect), RectId::DEPTH_LEVELS, intervals); + ForEachInIntervals(f, intervals, rect, scale); } -public: template void ForEachInScale(F & f, uint32_t scale) const { - BaseT::ForEachInIntervalAndScale(f, 0, static_cast((1ULL << 63) - 1), scale, - m2::RectD::GetInfiniteRect()); - } -}; - -template class MultiIndexAdapter -{ -public: - MultiIndexAdapter() - { - } - - MultiIndexAdapter(MultiIndexAdapter const & index) : m_indexes(index.m_indexes.size()) - { - for (size_t i = 0; i < index.m_indexes.size(); ++i) - { - CHECK(index.m_indexes[i], ()); - m_indexes[i] = index.m_indexes[i]->Clone(); - } - } - - ~MultiIndexAdapter() - { - Clean(); - ASSERT_EQUAL(m_indexes.size(), 0, ()); - } - - template - void ForEachInIntervalAndScale(F & f, int64_t beg, int64_t end, uint32_t scale, - m2::RectD const & occlusionRect) const - { - for (size_t iIndex = 0; true;) - { - IndexT * pIndex = NULL; - IndexProxy * pProxy = NULL; - { - threads::MutexGuard mutexGuard(m_mutex); - UNUSED_VALUE(mutexGuard); - - if (iIndex >= m_indexes.size()) - break; - - if (m_indexes[iIndex]->m_action == IndexProxy::INDEX_REMOVE) - { - (void)UpdateIndex(iIndex); - } - else - { - pProxy = m_indexes[iIndex]; - pIndex = pProxy->Lock(static_cast(scale), occlusionRect); - ++iIndex; - } - } - - if (pIndex) // pIndex may be NULL because it doesn't match scale or occlusionRect. - { - ProxyUnlockGuard proxyUnlockGuard(m_mutex, pProxy); - UNUSED_VALUE(proxyUnlockGuard); - pIndex->ForEachInIntervalAndScale(f, beg, end, scale); - } - } - } - - void Add(string const & file) - { - threads::MutexGuard mutexGuard(m_mutex); - UNUSED_VALUE(mutexGuard); - - for (size_t i = 0; i < m_indexes.size(); ++i) - if (m_indexes[i]->IsMyData(file)) - return; - - m_indexes.push_back(new IndexProxy(file)); - - UpdateIndexes(); - } - - void Remove(string const & file) - { - threads::MutexGuard mutexGuard(m_mutex); - UNUSED_VALUE(mutexGuard); - - for (size_t i = 0; i < m_indexes.size(); ++i) - { - if (m_indexes[i]->IsMyData(file)) - m_indexes[i]->m_action = IndexProxy::INDEX_REMOVE; - } - - UpdateIndexes(); - } - - // Remove all indexes. - void Clean() - { - threads::MutexGuard mutexGuard(m_mutex); - UNUSED_VALUE(mutexGuard); - - for (size_t i = 0; i < m_indexes.size(); ++i) - m_indexes[i]->m_action = IndexProxy::INDEX_REMOVE; - - UpdateIndexes(); - } - - // Close all indexes. - void ClearCaches() - { - threads::MutexGuard mutexGuard(m_mutex); - UNUSED_VALUE(mutexGuard); - - for (size_t i = 0; i < m_indexes.size(); ++i) - m_indexes[i]->m_action = IndexProxy::INDEX_CLOSE; - - UpdateIndexes(); + covering::IntervalsT intervals; + intervals.push_back(covering::IntervalsT::value_type( + 0, static_cast((1ULL << 63) - 1))); + ForEachInIntervals(f, intervals, m2::RectD::GetInfiniteRect(), scale); } private: - // Updates m_index[i]. Returns true, if index wasn't removed. - bool UpdateIndex(size_t i) const - { - if (!m_indexes[i]->IsUnlocked()) - return true; - - if (m_indexes[i]->m_action == IndexProxy::INDEX_REMOVE) - { - delete m_indexes[i]; - m_indexes.erase(m_indexes.begin() + i); - return false; - } - - if (m_indexes[i]->m_action == IndexProxy::INDEX_CLOSE) - m_indexes[i]->CloseIfUnlocked(); - - return true; - } - - // Updates all indexes. - void UpdateIndexes() const - { - for (size_t i = 0; i < m_indexes.size(); ) - if (UpdateIndex(i)) - ++i; - } - - class IndexProxy - { - public: - explicit IndexProxy(string const & file) - : m_action(INDEX_DO_NOTHING), m_file(file), m_lockCount(0), - m_queriesSkipped(0) - { - m_factory.Load(FilesContainerR(GetPlatform().GetReader(m_file))); - - feature::DataHeader const & h = m_factory.GetHeader(); - m_rect = h.GetBounds(); - m_scaleRange = h.GetScaleRange(); - } - - IndexT * Lock(int scale, m2::RectD const & occlusionRect) - { - IndexT * & p = GetThreadIndex(); - - if ((m_scaleRange.first <= scale && scale <= m_scaleRange.second) && - m_rect.IsIntersect(occlusionRect)) - { - Open(p); - m_queriesSkipped = 0; - ++m_lockCount; - return p; - } - else - { - if (p) - { - if (++m_queriesSkipped > 8) - Close(p); - } - return NULL; - } - } - - void Unlock() - { - ASSERT_GREATER(m_lockCount, 0, ()); - ASSERT(GetThreadIndex(), ()); - if (m_lockCount > 0) - --m_lockCount; - } - - bool IsUnlocked() const - { - return m_lockCount == 0; - } - - bool IsMyData(string const & file) const - { - return Reader::IsEqual(m_file, file); - } - - bool IsWorldData() const - { - return m_scaleRange.first <= 1; - } - - search::SearchInfo * GetSearchInfo() const - { - return new search::SearchInfo(FilesContainerR(GetPlatform().GetReader(m_file)), - m_factory.GetHeader()); - } - - void CloseIfUnlocked() - { - if (IsUnlocked()) - Close(); - } - - ~IndexProxy() - { - ASSERT_EQUAL(m_lockCount, 0, ()); - Close(); - } - - enum IndexAction - { - INDEX_DO_NOTHING = 0, - INDEX_CLOSE = 1, - INDEX_REMOVE = 2 - }; - - IndexProxy * Clone() const - { - IndexProxy * pRes = new IndexProxy(m_file); - return pRes; - } - - volatile IndexAction m_action; - - private: - void Open(IndexT * & p) - { - if (p == 0) - { - p = new IndexT(FilesContainerR(GetPlatform().GetReader(m_file)), m_factory); - } - } - - void Close(IndexT * & p) - { - if (p) - { - // LOG(LINFO, (m_Path)); - delete p; - p = NULL; - m_queriesSkipped = 0; - } - } - void Close() - { - Close(GetThreadIndex()); - } - - IndexT * & GetThreadIndex() - { - return m_indexes[threads::GetCurrentThreadID()]; - } - - string m_file; - - IndexFactory m_factory; - m2::RectD m_rect; - pair m_scaleRange; - - map m_indexes; - uint16_t volatile m_lockCount; - uint8_t volatile m_queriesSkipped; - }; - - // Helper class that unlocks given IndexProxy in destructor. - class ProxyUnlockGuard - { - threads::Mutex & m_mutex; - IndexProxy * m_pProxy; - - public: - ProxyUnlockGuard(threads::Mutex & mutex, IndexProxy * pProxy) - : m_mutex(mutex), m_pProxy(pProxy) - { - } - - ~ProxyUnlockGuard() - { - threads::MutexGuard mutexGuard(m_mutex); - UNUSED_VALUE(mutexGuard); - - m_pProxy->Unlock(); - - if (m_pProxy->m_action == IndexProxy::INDEX_CLOSE) - m_pProxy->CloseIfUnlocked(); - } - }; - - mutable vector m_indexes; - mutable threads::Mutex m_mutex; -}; - -template class OffsetToFeatureAdapter : public BaseT -{ -public: - OffsetToFeatureAdapter(FilesContainerR const & cont, IndexFactory & factory) - : BaseT(cont.GetReader(INDEX_FILE_TAG), factory), - m_FeatureVector(cont, factory.GetHeader()) - { - } - template - void ForEachInIntervalAndScale(F & f, int64_t beg, int64_t end, uint32_t scale) const + class ReadFeatureFunctor { - OffsetToFeatureReplacer offsetToFeatureReplacer(m_FeatureVector, f); - BaseT::ForEachInIntervalAndScale(offsetToFeatureReplacer, beg, end, scale); - } - -private: - FeatureVectorT m_FeatureVector; - - template - class OffsetToFeatureReplacer - { - FeatureVectorT const & m_V; + FeaturesVector const & m_V; F & m_F; + unordered_set & m_offsets; public: - OffsetToFeatureReplacer(FeatureVectorT const & v, F & f) : m_V(v), m_F(f) {} + ReadFeatureFunctor(FeaturesVector const & v, F & f, unordered_set & offsets) + : m_V(v), m_F(f), m_offsets(offsets) {} void operator() (uint32_t offset) const { - FeatureType feature; - m_V.Get(offset, feature); - m_F(feature); + if (m_offsets.insert(offset).second) + { + FeatureType feature; + m_V.Get(offset, feature); + m_F(feature); + } } }; -}; -template class UniqueOffsetAdapter : public BaseT -{ -public: - template - explicit UniqueOffsetAdapter(T1 const & t1) : BaseT(t1) {} - - template - UniqueOffsetAdapter(T1 const & t1, T2 & t2) : BaseT(t1, t2) {} template - void ForEachInIntervalAndScale(F & f, int64_t beg, int64_t end, uint32_t scale) const + void ForEachInIntervals(F & f, covering::IntervalsT const & intervals, + m2::RectD const & rect, uint32_t scale) const { - unordered_set offsets; - UniqueOffsetFunctorAdapter uniqueOffsetFunctorAdapter(offsets, f); - BaseT::ForEachInIntervalAndScale(uniqueOffsetFunctorAdapter, beg, end, scale); + vector mwm; + GetMwmInfo(mwm); + for (MwmId id = 0; id < mwm.size(); ++id) + { + if ((mwm[id].m_minScale <= scale && scale <= mwm[id].m_maxScale) && + rect.IsIntersect(mwm[id].m_limitRect)) + { + MwmLock lock(*this, id); + FilesContainerR * pContainer = lock.GetFileContainer(); + if (pContainer) + { + IndexFactory factory; + factory.Load(*pContainer); + FeaturesVector fv(*pContainer, factory.GetHeader()); + ScaleIndex index(pContainer->GetReader(INDEX_FILE_TAG), factory); + unordered_set offsets; + ReadFeatureFunctor f1(fv, f, offsets); + for (size_t i = 0; i < intervals.size(); ++i) + { + index.ForEachInIntervalAndScale(f1, intervals[i].first, intervals[i].second, scale); + } + } + } + } } -private: - template - struct UniqueOffsetFunctorAdapter - { - UniqueOffsetFunctorAdapter(unordered_set & offsets, F & f) - : m_Offsets(offsets), m_F(f) {} - void operator() (uint32_t offset) const - { - if (m_Offsets.insert(offset).second) - m_F(offset); - } - - unordered_set & m_Offsets; - F & m_F; - }; -}; - -template -struct Index -{ - typedef IndexForEachAdapter< - MultiIndexAdapter< - OffsetToFeatureAdapter - > - > - > - > Type; + void FillInMwmInfo(string const & path, MwmInfo & info); }; diff --git a/indexer/indexer.pro b/indexer/indexer.pro index f13cdea12a..915d4b71c8 100644 --- a/indexer/indexer.pro +++ b/indexer/indexer.pro @@ -37,6 +37,7 @@ SOURCES += \ feature_loader.cpp \ search_delimiters.cpp \ mwm_set.cpp \ + index.cpp HEADERS += \ feature.hpp \ @@ -84,3 +85,4 @@ HEADERS += \ mwm_set.hpp \ + diff --git a/indexer/indexer_tests/index_builder_test.cpp b/indexer/indexer_tests/index_builder_test.cpp index 619bc82301..8a746d95e3 100644 --- a/indexer/indexer_tests/index_builder_test.cpp +++ b/indexer/indexer_tests/index_builder_test.cpp @@ -60,7 +60,7 @@ UNIT_TEST(BuildIndexTest) { // Check that index actually works. - Index::Type index; + Index index; index.Add(fileName); // Make sure that index is actually parsed. diff --git a/indexer/indexer_tests/index_test.cpp b/indexer/indexer_tests/index_test.cpp index 9694979bf6..a29af92fc5 100644 --- a/indexer/indexer_tests/index_test.cpp +++ b/indexer/indexer_tests/index_test.cpp @@ -9,7 +9,7 @@ UNIT_TEST(IndexParseTest) { - Index::Type index; + Index index; index.Add("minsk-pass" DATA_FILE_EXTENSION); // Make sure that index is actually parsed. diff --git a/indexer/mwm_set.cpp b/indexer/mwm_set.cpp index ada1201264..c85c80f180 100644 --- a/indexer/mwm_set.cpp +++ b/indexer/mwm_set.cpp @@ -16,7 +16,7 @@ struct MwmIdIsEqualTo }; } // unnamed namespace -MwmSet::MwmLock::MwmLock(MwmSet & mwmSet, MwmId mwmId) +MwmSet::MwmLock::MwmLock(MwmSet const & mwmSet, MwmId mwmId) : m_mwmSet(mwmSet), m_id(mwmId), m_pFileContainer(mwmSet.LockContainer(mwmId)) { LOG(LINFO, ("MwmLock::MwmLock()", m_id)); @@ -35,7 +35,7 @@ FilesContainerR * MwmSet::MwmLock::GetFileContainer() const } -MwmSet::MwmSet(function const & fnGetMwmInfo, +MwmSet::MwmSet(function const & fnGetMwmInfo, function const & fnCreateContainer, size_t cacheSize) : m_cacheSize(cacheSize), m_fnGetMwmInfo(fnGetMwmInfo), m_fnCreateContainer(fnCreateContainer) @@ -135,7 +135,7 @@ void MwmSet::Remove(string const & fileName) } } -void MwmSet::GetMwmInfo(vector & info) +void MwmSet::GetMwmInfo(vector & info) const { threads::MutexGuard mutexGuard(m_lock); UNUSED_VALUE(mutexGuard); @@ -146,7 +146,7 @@ void MwmSet::GetMwmInfo(vector & info) info = m_info; } -FilesContainerR * MwmSet::LockContainer(MwmId id) +FilesContainerR * MwmSet::LockContainer(MwmId id) const { threads::MutexGuard mutexGuard(m_lock); UNUSED_VALUE(mutexGuard); @@ -176,7 +176,7 @@ FilesContainerR * MwmSet::LockContainer(MwmId id) return m_fnCreateContainer(m_name[id]); } -void MwmSet::UnlockContainer(MwmId id, FilesContainerR * pContainer) +void MwmSet::UnlockContainer(MwmId id, FilesContainerR * pContainer) const { threads::MutexGuard mutexGuard(m_lock); UNUSED_VALUE(mutexGuard); diff --git a/indexer/mwm_set.hpp b/indexer/mwm_set.hpp index 3aeabd4a12..468149dfb2 100644 --- a/indexer/mwm_set.hpp +++ b/indexer/mwm_set.hpp @@ -11,7 +11,7 @@ class FilesContainerR; // Information about stored mwm. struct MwmInfo { - m2::RectU32 m_limitRect; // Limit rect of mwm. + m2::RectD m_limitRect; // Limit rect of mwm. uint8_t m_minScale; // Min zoom level of mwm. uint8_t m_maxScale; // Max zoom level of mwm. @@ -40,12 +40,12 @@ public: class MwmLock { public: - MwmLock(MwmSet & mwmSet, MwmId mwmId); + MwmLock(MwmSet const & mwmSet, MwmId mwmId); ~MwmLock(); FilesContainerR * GetFileContainer() const; private: - MwmSet & m_mwmSet; + MwmSet const & m_mwmSet; MwmId m_id; FilesContainerR * m_pFileContainer; }; @@ -57,7 +57,7 @@ public: void Remove(string const & fileName); // Get ids of all mwms. Some of them may be marked to remove. - void GetMwmInfo(vector & info); + void GetMwmInfo(vector & info) const; // Clear caches. void ClearCache(); @@ -72,8 +72,8 @@ private: // Update given MwmInfo. inline static void UpdateMwmInfo(MwmInfo & info); - FilesContainerR * LockContainer(MwmId id); - void UnlockContainer(MwmId id, FilesContainerR * pContainer); + FilesContainerR * LockContainer(MwmId id) const; + void UnlockContainer(MwmId id, FilesContainerR * pContainer) const; // Find first removed mwm or add a new one. MwmId GetFreeId(); @@ -84,11 +84,11 @@ private: // Do the cleaning for [beg, end) without acquiring the mutex. void ClearCacheImpl(CacheType::iterator beg, CacheType::iterator end); - vector m_info; - vector m_name; - CacheType m_cache; + mutable vector m_info; + mutable vector m_name; + mutable CacheType m_cache; size_t m_cacheSize; function const m_fnGetMwmInfo; function const m_fnCreateContainer; - threads::Mutex m_lock; + mutable threads::Mutex m_lock; }; diff --git a/map/feature_vec_model.cpp b/map/feature_vec_model.cpp index 9c71235446..abe255b162 100644 --- a/map/feature_vec_model.cpp +++ b/map/feature_vec_model.cpp @@ -58,12 +58,12 @@ void FeaturesFetcher::RemoveMap(string const & fName) void FeaturesFetcher::Clean() { m_rect.MakeEmpty(); - m_multiIndex.Clean(); + // TODO: m_multiIndex.Clear(); - is it needed? } void FeaturesFetcher::ClearCaches() { - m_multiIndex.ClearCaches(); + m_multiIndex.ClearCache(); } m2::RectD FeaturesFetcher::GetWorldRect() const diff --git a/map/feature_vec_model.hpp b/map/feature_vec_model.hpp index 89560f50e0..da62411b48 100644 --- a/map/feature_vec_model.hpp +++ b/map/feature_vec_model.hpp @@ -28,7 +28,7 @@ namespace model private: m2::RectD m_rect; - typedef Index::Type index_t; + typedef Index index_t; index_t m_multiIndex; diff --git a/search/engine.hpp b/search/engine.hpp index 95932a602d..9375409a45 100644 --- a/search/engine.hpp +++ b/search/engine.hpp @@ -28,7 +28,7 @@ class Result; class Engine { public: - typedef Index::Type IndexType; + typedef Index IndexType; /// Doesn't take ownership of @pIndex and @pTrieIterator. Modifies @categories. Engine(IndexType const * pIndex, diff --git a/search/query.cpp b/search/query.cpp index fb808f261f..88caaed8ef 100644 --- a/search/query.cpp +++ b/search/query.cpp @@ -138,7 +138,7 @@ Query::Query(string const & query, m2::RectD const & viewport, IndexType const * m_pCategories(pCategories), m_pTrieRoot(pTrieRoot), m_pFeatures(pFeatures), - m_pIndex(pIndex ? new IndexType(*pIndex) : NULL), + m_pIndex(/*pIndex ? new IndexType(*pIndex) : */NULL), m_resultsRemaining(10), m_pEngine(pEngine), m_bTerminate(false) { @@ -255,7 +255,7 @@ void Query::Search(function const & f) { FeatureProcessor featureProcessor(*this); /// @todo Tune depth scale search (1 is no enough) - m_pIndex->ForEachInRect(featureProcessor, m_viewport, min(scales::GetUpperScale(), scale + 7)); + // m_pIndex->ForEachInRect(featureProcessor, m_viewport, min(scales::GetUpperScale(), scale + 7)); } catch (FeatureProcessor::StopException &) {