pastk/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

Index refactoring. Breaks unit tests!!

Browse source
This commit is contained in:
Yury Melnichek 2011-09-22 15:03:12 +02:00 • committed by Alex Zolotarev
parent 36ecd0526d
commit c5dbcf62db
11 changed files with 103 additions and 431 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
indexer/index.cpp Normal file
View file

@ -0,0 +1,26 @@
#include "index.hpp"
#include "data_header.hpp"
#include "../platform/platform.hpp"
#include "../std/bind.hpp"
namespace
{
FilesContainerR * CreateFileContainer(string const & fileName)
{
return new FilesContainerR(fileName);
}
} // unnamed namespace
Index::Index() : MwmSet(bind(&Index::FillInMwmInfo, this, _1, _2), &CreateFileContainer)
{
}
void Index::FillInMwmInfo(string const & fileName, MwmInfo & info)
{
IndexFactory factory;
factory.Load(FilesContainerR(GetPlatform().GetReader(fileName)));
feature::DataHeader const & h = factory.GetHeader();
info.m_limitRect = h.GetBounds();
info.m_minScale = static_cast<uint8_t>(h.GetScaleRange().first);
info.m_maxScale = static_cast<uint8_t>(h.GetScaleRange().second);
}

460
indexer/index.hpp
View file

@ -4,453 +4,97 @@
#include "feature_covering.hpp"
#include "features_vector.hpp"
#include "scale_index.hpp"
#include "search_trie.hpp"
#include "../../defines.hpp"
#include "data_factory.hpp"
#include "mwm_set.hpp"
#include "../platform/platform.hpp"
#include "../geometry/rect2d.hpp"
#include "../coding/file_container.hpp"
#include "../base/base.hpp"
#include "../base/macros.hpp"
#include "../base/mutex.hpp"
#include "../base/stl_add.hpp"
#include "../base/thread.hpp"
#include "../std/algorithm.hpp"
#include "../std/bind.hpp"
#include "../std/string.hpp"
#include "../std/unordered_set.hpp"
#include "../std/utility.hpp"
#include "../std/vector.hpp"
template <class BaseT> class IndexForEachAdapter : public BaseT
class Index : public MwmSet
{
private:
template <typename F>
void CallForIntervals(F & f, covering::IntervalsT const & intervals,
m2::RectD const & rect, uint32_t scale) const
{
for (size_t i = 0; i < intervals.size(); ++i)
{
BaseT::ForEachInIntervalAndScale(f, intervals[i].first, intervals[i].second,
scale, rect);
}
}
public:
Index();
template <typename F>
void ForEachInRect(F & f, m2::RectD const & rect, uint32_t scale) const
{
CallForIntervals(f, covering::CoverViewportAndAppendLowerLevels(rect, RectId::DEPTH_LEVELS),
rect, scale);
ForEachInIntervals(f, covering::CoverViewportAndAppendLowerLevels(rect, RectId::DEPTH_LEVELS),
rect, scale);
}
template <typename F>
void ForEachInRect_TileDrawing(F & f, m2::RectD const & rect, uint32_t scale) const
{
using namespace covering;
IntervalsT intervals;
AppendLowerLevels(GetRectIdAsIs(rect), RectId::DEPTH_LEVELS, intervals);
CallForIntervals(f, intervals, rect, scale);
covering::IntervalsT intervals;
covering::AppendLowerLevels(covering::GetRectIdAsIs(rect), RectId::DEPTH_LEVELS, intervals);
ForEachInIntervals(f, intervals, rect, scale);
}
public:
template <typename F>
void ForEachInScale(F & f, uint32_t scale) const
{
BaseT::ForEachInIntervalAndScale(f, 0, static_cast<int64_t>((1ULL << 63) - 1), scale,
m2::RectD::GetInfiniteRect());
}
};
template <class IndexT> class MultiIndexAdapter
{
public:
MultiIndexAdapter()
{
}
MultiIndexAdapter(MultiIndexAdapter const & index) : m_indexes(index.m_indexes.size())
{
for (size_t i = 0; i < index.m_indexes.size(); ++i)
{
CHECK(index.m_indexes[i], ());
m_indexes[i] = index.m_indexes[i]->Clone();
}
}
~MultiIndexAdapter()
{
Clean();
ASSERT_EQUAL(m_indexes.size(), 0, ());
}
template <typename F>
void ForEachInIntervalAndScale(F & f, int64_t beg, int64_t end, uint32_t scale,
m2::RectD const & occlusionRect) const
{
for (size_t iIndex = 0; true;)
{
IndexT * pIndex = NULL;
IndexProxy * pProxy = NULL;
{
threads::MutexGuard mutexGuard(m_mutex);
UNUSED_VALUE(mutexGuard);
if (iIndex >= m_indexes.size())
break;
if (m_indexes[iIndex]->m_action == IndexProxy::INDEX_REMOVE)
{
(void)UpdateIndex(iIndex);
}
else
{
pProxy = m_indexes[iIndex];
pIndex = pProxy->Lock(static_cast<int>(scale), occlusionRect);
++iIndex;
}
}
if (pIndex) // pIndex may be NULL because it doesn't match scale or occlusionRect.
{
ProxyUnlockGuard proxyUnlockGuard(m_mutex, pProxy);
UNUSED_VALUE(proxyUnlockGuard);
pIndex->ForEachInIntervalAndScale(f, beg, end, scale);
}
}
}
void Add(string const & file)
{
threads::MutexGuard mutexGuard(m_mutex);
UNUSED_VALUE(mutexGuard);
for (size_t i = 0; i < m_indexes.size(); ++i)
if (m_indexes[i]->IsMyData(file))
return;
m_indexes.push_back(new IndexProxy(file));
UpdateIndexes();
}
void Remove(string const & file)
{
threads::MutexGuard mutexGuard(m_mutex);
UNUSED_VALUE(mutexGuard);
for (size_t i = 0; i < m_indexes.size(); ++i)
{
if (m_indexes[i]->IsMyData(file))
m_indexes[i]->m_action = IndexProxy::INDEX_REMOVE;
}
UpdateIndexes();
}
// Remove all indexes.
void Clean()
{
threads::MutexGuard mutexGuard(m_mutex);
UNUSED_VALUE(mutexGuard);
for (size_t i = 0; i < m_indexes.size(); ++i)
m_indexes[i]->m_action = IndexProxy::INDEX_REMOVE;
UpdateIndexes();
}
// Close all indexes.
void ClearCaches()
{
threads::MutexGuard mutexGuard(m_mutex);
UNUSED_VALUE(mutexGuard);
for (size_t i = 0; i < m_indexes.size(); ++i)
m_indexes[i]->m_action = IndexProxy::INDEX_CLOSE;
UpdateIndexes();
covering::IntervalsT intervals;
intervals.push_back(covering::IntervalsT::value_type(
0, static_cast<int64_t>((1ULL << 63) - 1)));
ForEachInIntervals(f, intervals, m2::RectD::GetInfiniteRect(), scale);
}
private:
// Updates m_index[i]. Returns true, if index wasn't removed.
bool UpdateIndex(size_t i) const
{
if (!m_indexes[i]->IsUnlocked())
return true;
if (m_indexes[i]->m_action == IndexProxy::INDEX_REMOVE)
{
delete m_indexes[i];
m_indexes.erase(m_indexes.begin() + i);
return false;
}
if (m_indexes[i]->m_action == IndexProxy::INDEX_CLOSE)
m_indexes[i]->CloseIfUnlocked();
return true;
}
// Updates all indexes.
void UpdateIndexes() const
{
for (size_t i = 0; i < m_indexes.size(); )
if (UpdateIndex(i))
++i;
}
class IndexProxy
{
public:
explicit IndexProxy(string const & file)
: m_action(INDEX_DO_NOTHING), m_file(file), m_lockCount(0),
m_queriesSkipped(0)
{
m_factory.Load(FilesContainerR(GetPlatform().GetReader(m_file)));
feature::DataHeader const & h = m_factory.GetHeader();
m_rect = h.GetBounds();
m_scaleRange = h.GetScaleRange();
}
IndexT * Lock(int scale, m2::RectD const & occlusionRect)
{
IndexT * & p = GetThreadIndex();
if ((m_scaleRange.first <= scale && scale <= m_scaleRange.second) &&
m_rect.IsIntersect(occlusionRect))
{
Open(p);
m_queriesSkipped = 0;
++m_lockCount;
return p;
}
else
{
if (p)
{
if (++m_queriesSkipped > 8)
Close(p);
}
return NULL;
}
}
void Unlock()
{
ASSERT_GREATER(m_lockCount, 0, ());
ASSERT(GetThreadIndex(), ());
if (m_lockCount > 0)
--m_lockCount;
}
bool IsUnlocked() const
{
return m_lockCount == 0;
}
bool IsMyData(string const & file) const
{
return Reader::IsEqual(m_file, file);
}
bool IsWorldData() const
{
return m_scaleRange.first <= 1;
}
search::SearchInfo * GetSearchInfo() const
{
return new search::SearchInfo(FilesContainerR(GetPlatform().GetReader(m_file)),
m_factory.GetHeader());
}
void CloseIfUnlocked()
{
if (IsUnlocked())
Close();
}
~IndexProxy()
{
ASSERT_EQUAL(m_lockCount, 0, ());
Close();
}
enum IndexAction
{
INDEX_DO_NOTHING = 0,
INDEX_CLOSE = 1,
INDEX_REMOVE = 2
};
IndexProxy * Clone() const
{
IndexProxy * pRes = new IndexProxy(m_file);
return pRes;
}
volatile IndexAction m_action;
private:
void Open(IndexT * & p)
{
if (p == 0)
{
p = new IndexT(FilesContainerR(GetPlatform().GetReader(m_file)), m_factory);
}
}
void Close(IndexT * & p)
{
if (p)
{
// LOG(LINFO, (m_Path));
delete p;
p = NULL;
m_queriesSkipped = 0;
}
}
void Close()
{
Close(GetThreadIndex());
}
IndexT * & GetThreadIndex()
{
return m_indexes[threads::GetCurrentThreadID()];
}
string m_file;
IndexFactory m_factory;
m2::RectD m_rect;
pair<int, int> m_scaleRange;
map<threads::ThreadID, IndexT *> m_indexes;
uint16_t volatile m_lockCount;
uint8_t volatile m_queriesSkipped;
};
// Helper class that unlocks given IndexProxy in destructor.
class ProxyUnlockGuard
{
threads::Mutex & m_mutex;
IndexProxy * m_pProxy;
public:
ProxyUnlockGuard(threads::Mutex & mutex, IndexProxy * pProxy)
: m_mutex(mutex), m_pProxy(pProxy)
{
}
~ProxyUnlockGuard()
{
threads::MutexGuard mutexGuard(m_mutex);
UNUSED_VALUE(mutexGuard);
m_pProxy->Unlock();
if (m_pProxy->m_action == IndexProxy::INDEX_CLOSE)
m_pProxy->CloseIfUnlocked();
}
};
mutable vector<IndexProxy *> m_indexes;
mutable threads::Mutex m_mutex;
};
template <class FeatureVectorT, class BaseT> class OffsetToFeatureAdapter : public BaseT
{
public:
OffsetToFeatureAdapter(FilesContainerR const & cont, IndexFactory & factory)
: BaseT(cont.GetReader(INDEX_FILE_TAG), factory),
m_FeatureVector(cont, factory.GetHeader())
{
}
template <typename F>
void ForEachInIntervalAndScale(F & f, int64_t beg, int64_t end, uint32_t scale) const
class ReadFeatureFunctor
{
OffsetToFeatureReplacer<F> offsetToFeatureReplacer(m_FeatureVector, f);
BaseT::ForEachInIntervalAndScale(offsetToFeatureReplacer, beg, end, scale);
}
private:
FeatureVectorT m_FeatureVector;
template <typename F>
class OffsetToFeatureReplacer
{
FeatureVectorT const & m_V;
FeaturesVector const & m_V;
F & m_F;
unordered_set<uint32_t> & m_offsets;
public:
OffsetToFeatureReplacer(FeatureVectorT const & v, F & f) : m_V(v), m_F(f) {}
ReadFeatureFunctor(FeaturesVector const & v, F & f, unordered_set<uint32_t> & offsets)
: m_V(v), m_F(f), m_offsets(offsets) {}
void operator() (uint32_t offset) const
{
FeatureType feature;
m_V.Get(offset, feature);
m_F(feature);
if (m_offsets.insert(offset).second)
{
FeatureType feature;
m_V.Get(offset, feature);
m_F(feature);
}
}
};
};
template <class BaseT> class UniqueOffsetAdapter : public BaseT
{
public:
template <typename T1>
explicit UniqueOffsetAdapter(T1 const & t1) : BaseT(t1) {}
template <typename T1, typename T2>
UniqueOffsetAdapter(T1 const & t1, T2 & t2) : BaseT(t1, t2) {}
template <typename F>
void ForEachInIntervalAndScale(F & f, int64_t beg, int64_t end, uint32_t scale) const
void ForEachInIntervals(F & f, covering::IntervalsT const & intervals,
m2::RectD const & rect, uint32_t scale) const
{
unordered_set<uint32_t> offsets;
UniqueOffsetFunctorAdapter<F> uniqueOffsetFunctorAdapter(offsets, f);
BaseT::ForEachInIntervalAndScale(uniqueOffsetFunctorAdapter, beg, end, scale);
vector<MwmInfo> mwm;
GetMwmInfo(mwm);
for (MwmId id = 0; id < mwm.size(); ++id)
{
if ((mwm[id].m_minScale <= scale && scale <= mwm[id].m_maxScale) &&
rect.IsIntersect(mwm[id].m_limitRect))
{
MwmLock lock(*this, id);
FilesContainerR * pContainer = lock.GetFileContainer();
if (pContainer)
{
IndexFactory factory;
factory.Load(*pContainer);
FeaturesVector fv(*pContainer, factory.GetHeader());
ScaleIndex<ModelReaderPtr> index(pContainer->GetReader(INDEX_FILE_TAG), factory);
unordered_set<uint32_t> offsets;
ReadFeatureFunctor<F> f1(fv, f, offsets);
for (size_t i = 0; i < intervals.size(); ++i)
{
index.ForEachInIntervalAndScale(f1, intervals[i].first, intervals[i].second, scale);
}
}
}
}
}
private:
template <typename F>
struct UniqueOffsetFunctorAdapter
{
UniqueOffsetFunctorAdapter(unordered_set<uint32_t> & offsets, F & f)
: m_Offsets(offsets), m_F(f) {}
void operator() (uint32_t offset) const
{
if (m_Offsets.insert(offset).second)
m_F(offset);
}
unordered_set<uint32_t> & m_Offsets;
F & m_F;
};
};
template <typename ReaderT>
struct Index
{
typedef IndexForEachAdapter<
MultiIndexAdapter<
OffsetToFeatureAdapter<FeaturesVector,
UniqueOffsetAdapter<
ScaleIndex<ReaderT>
>
>
>
> Type;
void FillInMwmInfo(string const & path, MwmInfo & info);
};

2
indexer/indexer.pro
View file

@ -37,6 +37,7 @@ SOURCES += \
feature_loader.cpp \
search_delimiters.cpp \
mwm_set.cpp \
index.cpp
HEADERS += \
feature.hpp \
@ -84,3 +85,4 @@ HEADERS += \
mwm_set.hpp \

2
indexer/indexer_tests/index_builder_test.cpp
View file

@ -60,7 +60,7 @@ UNIT_TEST(BuildIndexTest)
{
// Check that index actually works.
Index<ModelReaderPtr>::Type index;
Index index;
index.Add(fileName);
// Make sure that index is actually parsed.

2
indexer/indexer_tests/index_test.cpp
View file

@ -9,7 +9,7 @@
UNIT_TEST(IndexParseTest)
{
Index<ModelReaderPtr>::Type index;
Index index;
index.Add("minsk-pass" DATA_FILE_EXTENSION);
// Make sure that index is actually parsed.

10
indexer/mwm_set.cpp
View file

@ -16,7 +16,7 @@ struct MwmIdIsEqualTo
};
} // unnamed namespace
MwmSet::MwmLock::MwmLock(MwmSet & mwmSet, MwmId mwmId)
MwmSet::MwmLock::MwmLock(MwmSet const & mwmSet, MwmId mwmId)
: m_mwmSet(mwmSet), m_id(mwmId), m_pFileContainer(mwmSet.LockContainer(mwmId))
{
LOG(LINFO, ("MwmLock::MwmLock()", m_id));
@ -35,7 +35,7 @@ FilesContainerR * MwmSet::MwmLock::GetFileContainer() const
}
MwmSet::MwmSet(function<void (const string &, MwmInfo &)> const & fnGetMwmInfo,
MwmSet::MwmSet(function<void (string const &, MwmInfo &)> const & fnGetMwmInfo,
function<FilesContainerR * (string const &)> const & fnCreateContainer,
size_t cacheSize)
: m_cacheSize(cacheSize), m_fnGetMwmInfo(fnGetMwmInfo), m_fnCreateContainer(fnCreateContainer)
@ -135,7 +135,7 @@ void MwmSet::Remove(string const & fileName)
}
}
void MwmSet::GetMwmInfo(vector<MwmInfo> & info)
void MwmSet::GetMwmInfo(vector<MwmInfo> & info) const
{
threads::MutexGuard mutexGuard(m_lock);
UNUSED_VALUE(mutexGuard);
@ -146,7 +146,7 @@ void MwmSet::GetMwmInfo(vector<MwmInfo> & info)
info = m_info;
}
FilesContainerR * MwmSet::LockContainer(MwmId id)
FilesContainerR * MwmSet::LockContainer(MwmId id) const
{
threads::MutexGuard mutexGuard(m_lock);
UNUSED_VALUE(mutexGuard);
@ -176,7 +176,7 @@ FilesContainerR * MwmSet::LockContainer(MwmId id)
return m_fnCreateContainer(m_name[id]);
}
void MwmSet::UnlockContainer(MwmId id, FilesContainerR * pContainer)
void MwmSet::UnlockContainer(MwmId id, FilesContainerR * pContainer) const
{
threads::MutexGuard mutexGuard(m_lock);
UNUSED_VALUE(mutexGuard);

20
indexer/mwm_set.hpp
View file

@ -11,7 +11,7 @@ class FilesContainerR;
// Information about stored mwm.
struct MwmInfo
{
m2::RectU32 m_limitRect; // Limit rect of mwm.
m2::RectD m_limitRect; // Limit rect of mwm.
uint8_t m_minScale; // Min zoom level of mwm.
uint8_t m_maxScale; // Max zoom level of mwm.
@ -40,12 +40,12 @@ public:
class MwmLock
{
public:
MwmLock(MwmSet & mwmSet, MwmId mwmId);
MwmLock(MwmSet const & mwmSet, MwmId mwmId);
~MwmLock();
FilesContainerR * GetFileContainer() const;
private:
MwmSet & m_mwmSet;
MwmSet const & m_mwmSet;
MwmId m_id;
FilesContainerR * m_pFileContainer;
};
@ -57,7 +57,7 @@ public:
void Remove(string const & fileName);
// Get ids of all mwms. Some of them may be marked to remove.
void GetMwmInfo(vector<MwmInfo> & info);
void GetMwmInfo(vector<MwmInfo> & info) const;
// Clear caches.
void ClearCache();
@ -72,8 +72,8 @@ private:
// Update given MwmInfo.
inline static void UpdateMwmInfo(MwmInfo & info);
FilesContainerR * LockContainer(MwmId id);
void UnlockContainer(MwmId id, FilesContainerR * pContainer);
FilesContainerR * LockContainer(MwmId id) const;
void UnlockContainer(MwmId id, FilesContainerR * pContainer) const;
// Find first removed mwm or add a new one.
MwmId GetFreeId();
@ -84,11 +84,11 @@ private:
// Do the cleaning for [beg, end) without acquiring the mutex.
void ClearCacheImpl(CacheType::iterator beg, CacheType::iterator end);
vector<MwmInfo> m_info;
vector<string> m_name;
CacheType m_cache;
mutable vector<MwmInfo> m_info;
mutable vector<string> m_name;
mutable CacheType m_cache;
size_t m_cacheSize;
function<void (string const &, MwmInfo &)> const m_fnGetMwmInfo;
function<FilesContainerR * (string const &)> const m_fnCreateContainer;
threads::Mutex m_lock;
mutable threads::Mutex m_lock;
};

4
map/feature_vec_model.cpp
View file

@ -58,12 +58,12 @@ void FeaturesFetcher::RemoveMap(string const & fName)
void FeaturesFetcher::Clean()
{
m_rect.MakeEmpty();
m_multiIndex.Clean();
// TODO: m_multiIndex.Clear(); - is it needed?
}
void FeaturesFetcher::ClearCaches()
{
m_multiIndex.ClearCaches();
m_multiIndex.ClearCache();
}
m2::RectD FeaturesFetcher::GetWorldRect() const

2
map/feature_vec_model.hpp
View file

@ -28,7 +28,7 @@ namespace model
private:
m2::RectD m_rect;
typedef Index<ReaderT>::Type index_t;
typedef Index index_t;
index_t m_multiIndex;

2
search/engine.hpp
View file

@ -28,7 +28,7 @@ class Result;
class Engine
{
public:
typedef Index<ModelReaderPtr>::Type IndexType;
typedef Index IndexType;
/// Doesn't take ownership of @pIndex and @pTrieIterator. Modifies @categories.
Engine(IndexType const * pIndex,

4
search/query.cpp
View file

@ -138,7 +138,7 @@ Query::Query(string const & query, m2::RectD const & viewport, IndexType const *
m_pCategories(pCategories),
m_pTrieRoot(pTrieRoot),
m_pFeatures(pFeatures),
m_pIndex(pIndex ? new IndexType(*pIndex) : NULL),
m_pIndex(/*pIndex ? new IndexType(*pIndex) : */NULL),
m_resultsRemaining(10),
m_pEngine(pEngine), m_bTerminate(false)
{
@ -255,7 +255,7 @@ void Query::Search(function<void (Result const &)> const & f)
{
FeatureProcessor featureProcessor(*this);
/// @todo Tune depth scale search (1 is no enough)
m_pIndex->ForEachInRect(featureProcessor, m_viewport, min(scales::GetUpperScale(), scale + 7));
// m_pIndex->ForEachInRect(featureProcessor, m_viewport, min(scales::GetUpperScale(), scale + 7));
}
catch (FeatureProcessor::StopException &)
{