pastk/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

[assessment-tool] Save irrelevant results too.

Browse source
This commit is contained in:
Yuri Gorshenin 2017-07-14 17:24:37 +03:00 • committed by Arsentiy Milchakov
parent c4476ff999
commit de06e9d59d
6 changed files with 62 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
search/search_quality/assessment_tool/context.cpp
View file

@ -53,22 +53,22 @@ search::Sample Context::MakeSample(search::FeatureLoader & loader) const
{
auto const & entry = nonFoundEntries[k++];
auto const deleted = entry.m_deleted;
auto const relevance = entry.m_curr;
if (!deleted && relevance != search::Sample::Result::Relevance::Irrelevant)
auto const & curr = entry.m_curr;
if (!deleted && !curr.m_unknown)
{
auto result = m_sample.m_results[i];
result.m_relevance = relevance;
result.m_relevance = curr.m_relevance;
outResults.push_back(result);
}
continue;
}
// No need to keep irrelevant results.
if (foundEntries[j].m_curr == search::Sample::Result::Relevance::Irrelevant)
if (foundEntries[j].m_curr.m_unknown)
continue;
auto result = m_sample.m_results[i];
result.m_relevance = foundEntries[j].m_curr;
result.m_relevance = foundEntries[j].m_curr.m_relevance;
outResults.push_back(move(result));
}
@ -83,7 +83,7 @@ search::Sample Context::MakeSample(search::FeatureLoader & loader) const
}
// No need to keep irrelevant results.
if (foundEntries[i].m_curr == search::Sample::Result::Relevance::Irrelevant)
if (foundEntries[i].m_curr.m_unknown)
continue;
auto const & result = m_foundResults[i];
@ -93,7 +93,7 @@ search::Sample Context::MakeSample(search::FeatureLoader & loader) const
FeatureType ft;
CHECK(loader.Load(result.GetFeatureID(), ft), ());
outResults.push_back(search::Sample::Result::Build(ft, foundEntries[i].m_curr));
outResults.push_back(search::Sample::Result::Build(ft, foundEntries[i].m_curr.m_relevance));
}
return outSample;

16
search/search_quality/assessment_tool/edits.cpp
View file

@ -13,7 +13,7 @@ bool Edits::Editor::Set(Relevance relevance)
return m_parent.SetRelevance(m_index, relevance);
}
Edits::Relevance Edits::Editor::Get() const
Edits::MaybeRelevance Edits::Editor::Get() const
{
return m_parent.Get(m_index).m_curr;
}
@ -38,7 +38,7 @@ void Edits::Apply()
});
}
void Edits::Reset(std::vector<Relevance> const & relevances)
void Edits::Reset(std::vector<MaybeRelevance> const & relevances)
{
WithObserver(Update::MakeAll(), [this, &relevances]() {
m_entries.resize(relevances.size());
@ -61,12 +61,14 @@ bool Edits::SetRelevance(size_t index, Relevance relevance)
auto & entry = m_entries[index];
if (entry.m_curr != entry.m_orig && relevance == entry.m_orig)
MaybeRelevance const r(relevance);
if (entry.m_curr != entry.m_orig && r == entry.m_orig)
--m_numEdits;
else if (entry.m_curr == entry.m_orig && relevance != entry.m_orig)
else if (entry.m_curr == entry.m_orig && r != entry.m_orig)
++m_numEdits;
entry.m_curr = relevance;
entry.m_curr = r;
return entry.m_curr != entry.m_orig;
});
}
@ -125,9 +127,9 @@ Edits::Entry const & Edits::GetEntry(size_t index) const
return m_entries[index];
}
std::vector<Edits::Relevance> Edits::GetRelevances() const
std::vector<Edits::MaybeRelevance> Edits::GetRelevances() const
{
std::vector<Relevance> relevances(m_entries.size());
std::vector<MaybeRelevance> relevances(m_entries.size());
for (size_t i = 0; i < m_entries.size(); ++i)
relevances[i] = m_entries[i].m_curr;
return relevances;

38
search/search_quality/assessment_tool/edits.hpp
View file

@ -15,6 +15,31 @@ class Edits
public:
using Relevance = search::Sample::Result::Relevance;
struct MaybeRelevance
{
MaybeRelevance() = default;
MaybeRelevance(search::Sample::Result::Relevance relevance)
: m_relevance(relevance), m_unknown(false)
{
}
bool operator==(MaybeRelevance const & rhs) const
{
if (m_unknown && rhs.m_unknown)
return true;
if (m_unknown != rhs.m_unknown)
return false;
ASSERT(!m_unknown, ());
ASSERT(!rhs.m_unknown, ());
return m_relevance == rhs.m_relevance;
}
bool operator!=(MaybeRelevance const & rhs) const { return !(*this == rhs); }
Relevance m_relevance = Relevance::Irrelevant;
bool m_unknown = true;
};
struct Entry
{
enum class Type
@ -24,13 +49,12 @@ public:
};
Entry() = default;
Entry(Relevance relevance, Type type)
: m_curr(relevance), m_orig(relevance), m_type(type)
Entry(MaybeRelevance relevance, Type type) : m_curr(relevance), m_orig(relevance), m_type(type)
{
}
Relevance m_curr = Relevance::Irrelevant;
Relevance m_orig = Relevance::Irrelevant;
MaybeRelevance m_curr = {};
MaybeRelevance m_orig = {};
bool m_deleted = false;
Type m_type = Type::Loaded;
};
@ -71,7 +95,7 @@ public:
// Sets relevance to |relevance|. Returns true iff |relevance|
// differs from the original one.
bool Set(Relevance relevance);
Relevance Get() const;
MaybeRelevance Get() const;
bool HasChanges() const;
Entry::Type GetType() const;
@ -83,7 +107,7 @@ public:
explicit Edits(OnUpdate onUpdate) : m_onUpdate(onUpdate) {}
void Apply();
void Reset(std::vector<Relevance> const & relevances);
void Reset(std::vector<MaybeRelevance> const & relevances);
// Sets relevance at |index| to |relevance|. Returns true iff
// |relevance| differs from the original one.
@ -102,7 +126,7 @@ public:
Entry & GetEntry(size_t index);
Entry const & GetEntry(size_t index) const;
size_t NumEntries() const { return m_entries.size(); }
std::vector<Relevance> GetRelevances() const;
std::vector<MaybeRelevance> GetRelevances() const;
Entry const & Get(size_t index) const;

12
search/search_quality/assessment_tool/main_model.cpp
View file

@ -141,7 +141,7 @@ void MainModel::OnSampleSelected(int index)
search::SearchParams params;
sample.FillSearchParams(params);
params.m_onResults = [this, index, sample, timestamp](search::Results const & results) {
vector<Relevance> relevances;
vector<Edits::MaybeRelevance> relevances;
vector<size_t> goldenMatching;
vector<size_t> actualMatching;
@ -153,14 +153,14 @@ void MainModel::OnSampleSelected(int index)
vector<search::Result> const actual(results.begin(), results.end());
matcher.Match(sample.m_results, actual, goldenMatching, actualMatching);
relevances.assign(actual.size(), Relevance::Irrelevant);
relevances.resize(actual.size());
for (size_t i = 0; i < goldenMatching.size(); ++i)
{
auto const j = goldenMatching[i];
if (j != search::Matcher::kInvalidId)
{
CHECK_LESS(j, relevances.size(), ());
relevances[j] = sample.m_results[i].m_relevance;
relevances[j] = Edits::MaybeRelevance(sample.m_results[i].m_relevance);
}
}
}
@ -312,7 +312,7 @@ void MainModel::OnUpdate(View::ResultType type, size_t sampleIndex, Edits::Updat
}
void MainModel::OnResults(uint64_t timestamp, size_t sampleIndex, search::Results const & results,
vector<Relevance> const & relevances,
vector<Edits::MaybeRelevance> const & relevances,
vector<size_t> const & goldenMatching,
vector<size_t> const & actualMatching)
{
@ -338,7 +338,7 @@ void MainModel::OnResults(uint64_t timestamp, size_t sampleIndex, search::Result
context.m_actualMatching = actualMatching;
{
vector<Relevance> relevances;
vector<Edits::MaybeRelevance> relevances;
auto & nonFound = context.m_nonFoundResults;
CHECK(nonFound.empty(), ());
@ -348,7 +348,7 @@ void MainModel::OnResults(uint64_t timestamp, size_t sampleIndex, search::Result
if (j != search::Matcher::kInvalidId)
continue;
nonFound.push_back(context.m_sample.m_results[i]);
relevances.push_back(nonFound.back().m_relevance);
relevances.emplace_back(nonFound.back().m_relevance);
}
context.m_nonFoundResultsEdits.Reset(relevances);
}

2
search/search_quality/assessment_tool/main_model.hpp
View file

@ -47,7 +47,7 @@ private:
void OnUpdate(View::ResultType type, size_t sampleIndex, Edits::Update const & update);
void OnResults(uint64_t timestamp, size_t sampleIndex, search::Results const & results,
std::vector<search::Sample::Result::Relevance> const & relevances,
std::vector<Edits::MaybeRelevance> const & relevances,
std::vector<size_t> const & goldenMatching,
std::vector<size_t> const & actualMatching);

12
search/search_quality/assessment_tool/result_view.cpp
View file

@ -71,11 +71,15 @@ void ResultView::SetEditor(Edits::Editor && editor)
m_relevant->setChecked(false);
m_vital->setChecked(false);
switch (m_editor->Get())
auto const & r = m_editor->Get();
if (!r.m_unknown)
{
case Relevance::Irrelevant: m_irrelevant->setChecked(true); break;
case Relevance::Relevant: m_relevant->setChecked(true); break;
case Relevance::Vital: m_vital->setChecked(true); break;
switch (r.m_relevance)
{
case Relevance::Irrelevant: m_irrelevant->setChecked(true); break;
case Relevance::Relevant: m_relevant->setChecked(true); break;
case Relevance::Vital: m_vital->setChecked(true); break;
}
}
setEnabled(true);