forked from organicmaps/organicmaps
Merge pull request #3943 from mgsergio/booking-area-hotels
[booking] area hotels
This commit is contained in:
Ilya Zverev
GitHub
commit
970abc4add
20 changed files with 646 additions and 413 deletions
|
|
@ -1,10 +1,12 @@
|
|||
#include "generator/booking_dataset.hpp"
|
||||
|
||||
#include "generator/booking_scoring.hpp"
|
||||
#include "generator/feature_builder.hpp"
|
||||
|
||||
#include "platform/local_country_file_utils.hpp"
|
||||
#include "platform/platform.hpp"
|
||||
|
||||
#include "indexer/classificator.hpp"
|
||||
#include "indexer/ftypes_matcher.hpp"
|
||||
|
||||
#include "geometry/distance_on_sphere.hpp"
|
||||
|
|
@ -14,18 +16,13 @@
|
|||
|
||||
#include "std/fstream.hpp"
|
||||
#include "std/iostream.hpp"
|
||||
|
||||
#include "std/sstream.hpp"
|
||||
|
||||
namespace generator
|
||||
{
|
||||
namespace
|
||||
{
|
||||
bool CheckForValues(string const & value)
|
||||
{
|
||||
auto const & tags = ftypes::IsHotelChecker::GetHotelTags();
|
||||
return find(tags.begin(), tags.end(), value) != tags.end();
|
||||
}
|
||||
|
||||
string EscapeTabs(string const & str)
|
||||
{
|
||||
stringstream ss;
|
||||
|
|
@ -124,20 +121,19 @@ BookingDataset::BookingDataset(istream & dataSource, string const & addressRefer
|
|||
LoadHotels(dataSource, addressReferencePath);
|
||||
}
|
||||
|
||||
bool BookingDataset::BookingFilter(OsmElement const & e) const
|
||||
size_t BookingDataset::GetMatchingHotelIndex(FeatureBuilder1 const & fb) const
|
||||
{
|
||||
return Filter(e, [&](OsmElement const & e)
|
||||
{
|
||||
return MatchWithBooking(e);
|
||||
});
|
||||
if (CanBeBooking(fb))
|
||||
return MatchWithBooking(fb);
|
||||
return kInvalidHotelIndex;
|
||||
}
|
||||
|
||||
bool BookingDataset::TourismFilter(OsmElement const & e) const
|
||||
bool BookingDataset::CanBeBooking(FeatureBuilder1 const & fb) const
|
||||
{
|
||||
return Filter(e, [&](OsmElement const & e)
|
||||
{
|
||||
return true;
|
||||
});
|
||||
if (fb.GetName(StringUtf8Multilang::kDefaultCode).empty())
|
||||
return false;
|
||||
|
||||
return ftypes::IsHotelChecker::Instance()(fb.GetTypes());
|
||||
}
|
||||
|
||||
BookingDataset::Hotel const & BookingDataset::GetHotel(size_t index) const
|
||||
|
|
@ -152,73 +148,80 @@ BookingDataset::Hotel & BookingDataset::GetHotel(size_t index)
|
|||
return m_hotels[index];
|
||||
}
|
||||
|
||||
vector<size_t> BookingDataset::GetNearestHotels(double lat, double lon, size_t limit,
|
||||
double maxDistance /* = 0.0 */) const
|
||||
vector<size_t> BookingDataset::GetNearestHotels(ms::LatLon const & latLon, size_t const limit,
|
||||
double const maxDistance /* = 0.0 */) const
|
||||
{
|
||||
namespace bgi = boost::geometry::index;
|
||||
|
||||
vector<size_t> indexes;
|
||||
for_each(bgi::qbegin(m_rtree, bgi::nearest(TPoint(lat, lon), limit)), bgi::qend(m_rtree),
|
||||
[&](TValue const & v)
|
||||
for_each(bgi::qbegin(m_rtree, bgi::nearest(TPoint(latLon.lat, latLon.lon), limit)),
|
||||
bgi::qend(m_rtree), [&](TValue const & v)
|
||||
{
|
||||
auto const & hotel = m_hotels[v.second];
|
||||
double const dist = ms::DistanceOnEarth(lat, lon, hotel.lat, hotel.lon);
|
||||
auto const & hotel = GetHotel(v.second);
|
||||
double const dist = ms::DistanceOnEarth(latLon.lat, latLon.lon, hotel.lat, hotel.lon);
|
||||
if (maxDistance != 0.0 && dist > maxDistance /* max distance in meters */)
|
||||
return;
|
||||
|
||||
indexes.emplace_back(v.second);
|
||||
});
|
||||
|
||||
return indexes;
|
||||
}
|
||||
|
||||
void BookingDataset::BuildFeatures(function<void(OsmElement *)> const & fn) const
|
||||
void BookingDataset::BuildHotel(size_t const hotelIndex,
|
||||
function<void(FeatureBuilder1 &)> const & fn) const
|
||||
{
|
||||
for (auto const & hotel : m_hotels)
|
||||
auto const & hotel = m_hotels[hotelIndex];
|
||||
|
||||
FeatureBuilder1 fb;
|
||||
FeatureParams params;
|
||||
|
||||
fb.SetCenter(MercatorBounds::FromLatLon(hotel.lat, hotel.lon));
|
||||
|
||||
auto & metadata = params.GetMetadata();
|
||||
metadata.Set(feature::Metadata::FMD_SPONSORED_ID, strings::to_string(hotel.id));
|
||||
metadata.Set(feature::Metadata::FMD_WEBSITE, hotel.descUrl);
|
||||
metadata.Set(feature::Metadata::FMD_RATING, strings::to_string(hotel.ratingUser));
|
||||
metadata.Set(feature::Metadata::FMD_STARS, strings::to_string(hotel.stars));
|
||||
metadata.Set(feature::Metadata::FMD_PRICE_RATE, strings::to_string(hotel.priceCategory));
|
||||
|
||||
// params.AddAddress(hotel.address);
|
||||
// TODO(mgsergio): addr:full ???
|
||||
|
||||
if (!hotel.street.empty())
|
||||
fb.AddStreet(hotel.street);
|
||||
|
||||
if (!hotel.houseNumber.empty())
|
||||
fb.AddHouseNumber(hotel.houseNumber);
|
||||
|
||||
params.AddName(StringUtf8Multilang::GetLangByCode(StringUtf8Multilang::kDefaultCode),
|
||||
hotel.name);
|
||||
if (!hotel.translations.empty())
|
||||
{
|
||||
OsmElement e;
|
||||
e.type = OsmElement::EntityType::Node;
|
||||
e.id = 1;
|
||||
|
||||
e.lat = hotel.lat;
|
||||
e.lon = hotel.lon;
|
||||
|
||||
e.AddTag("sponsored", "booking");
|
||||
e.AddTag("name", hotel.name);
|
||||
e.AddTag("ref:sponsored", strings::to_string(hotel.id));
|
||||
e.AddTag("website", hotel.descUrl);
|
||||
e.AddTag("rating:sponsored", strings::to_string(hotel.ratingUser));
|
||||
e.AddTag("stars", strings::to_string(hotel.stars));
|
||||
e.AddTag("price_rate", strings::to_string(hotel.priceCategory));
|
||||
e.AddTag("addr:full", hotel.address);
|
||||
|
||||
if (!hotel.translations.empty())
|
||||
// TODO(mgsergio): Move parsing to the hotel costruction stage.
|
||||
vector<string> parts;
|
||||
strings::ParseCSVRow(hotel.translations, '|', parts);
|
||||
CHECK_EQUAL(parts.size() % 3, 0, ("Invalid translation string:", hotel.translations));
|
||||
for (auto i = 0; i < parts.size(); i += 3)
|
||||
{
|
||||
vector<string> parts;
|
||||
strings::ParseCSVRow(hotel.translations, '|', parts);
|
||||
CHECK(parts.size() % 3 == 0, ());
|
||||
for (auto i = 0; i < parts.size(); i += 3)
|
||||
{
|
||||
e.AddTag("name:" + parts[i], parts[i + 1]);
|
||||
e.AddTag("addr:full:" + parts[i], parts[i + 2]);
|
||||
}
|
||||
auto const langCode = StringUtf8Multilang::GetLangIndex(parts[i]);
|
||||
params.AddName(StringUtf8Multilang::GetLangByCode(langCode), parts[i + 1]);
|
||||
// TODO(mgsergio): e.AddTag("addr:full:" + parts[i], parts[i + 2]);
|
||||
}
|
||||
}
|
||||
|
||||
if (!hotel.street.empty())
|
||||
e.AddTag("addr:street", hotel.street);
|
||||
|
||||
if (!hotel.houseNumber.empty())
|
||||
e.AddTag("addr:housenumber", hotel.houseNumber);
|
||||
|
||||
// Matching booking.com hotel types to OpenStreetMap values.
|
||||
// Booking types are listed in the closed API docs.
|
||||
switch (hotel.type)
|
||||
{
|
||||
auto const & clf = classif();
|
||||
params.AddType(clf.GetTypeByPath({"sponsored", "booking"}));
|
||||
// Matching booking.com hotel types to OpenStreetMap values.
|
||||
// Booking types are listed in the closed API docs.
|
||||
switch (hotel.type)
|
||||
{
|
||||
case 19:
|
||||
case 205: e.AddTag("tourism", "motel"); break;
|
||||
case 205: params.AddType(clf.GetTypeByPath({"tourism", "motel"})); break;
|
||||
|
||||
case 21:
|
||||
case 206:
|
||||
case 212: e.AddTag("tourism", "resort"); break;
|
||||
case 212: params.AddType(clf.GetTypeByPath({"tourism", "resort"})); break;
|
||||
|
||||
case 3:
|
||||
case 23:
|
||||
|
|
@ -231,7 +234,7 @@ void BookingDataset::BuildFeatures(function<void(OsmElement *)> const & fn) cons
|
|||
case 210:
|
||||
case 216:
|
||||
case 220:
|
||||
case 223: e.AddTag("tourism", "guest_house"); break;
|
||||
case 223: params.AddType(clf.GetTypeByPath({"tourism", "guest_house"})); break;
|
||||
|
||||
case 14:
|
||||
case 204:
|
||||
|
|
@ -239,29 +242,30 @@ void BookingDataset::BuildFeatures(function<void(OsmElement *)> const & fn) cons
|
|||
case 218:
|
||||
case 219:
|
||||
case 226:
|
||||
case 222: e.AddTag("tourism", "hotel"); break;
|
||||
case 222: params.AddType(clf.GetTypeByPath({"tourism", "hotel"})); break;
|
||||
|
||||
case 211:
|
||||
case 224:
|
||||
case 228: e.AddTag("tourism", "chalet"); break;
|
||||
case 228: params.AddType(clf.GetTypeByPath({"tourism", "chalet"})); break;
|
||||
|
||||
case 13:
|
||||
case 225:
|
||||
case 203: e.AddTag("tourism", "hostel"); break;
|
||||
case 203: params.AddType(clf.GetTypeByPath({"tourism", "hostel"})); break;
|
||||
|
||||
case 215:
|
||||
case 221:
|
||||
case 227:
|
||||
case 2:
|
||||
case 201: e.AddTag("tourism", "apartment"); break;
|
||||
case 201: params.AddType(clf.GetTypeByPath({"tourism", "apartment"})); break;
|
||||
|
||||
case 214: e.AddTag("tourism", "camp_site"); break;
|
||||
case 214: params.AddType(clf.GetTypeByPath({"tourism", "camp_site"})); break;
|
||||
|
||||
default: e.AddTag("tourism", "hotel"); break;
|
||||
}
|
||||
|
||||
fn(&e);
|
||||
default: params.AddType(clf.GetTypeByPath({"tourism", "hotel"})); break;
|
||||
}
|
||||
|
||||
fb.SetParams(params);
|
||||
|
||||
fn(fb);
|
||||
}
|
||||
|
||||
void BookingDataset::LoadHotels(istream & src, string const & addressReferencePath)
|
||||
|
|
@ -306,58 +310,23 @@ void BookingDataset::LoadHotels(istream & src, string const & addressReferencePa
|
|||
}
|
||||
}
|
||||
|
||||
bool BookingDataset::MatchWithBooking(OsmElement const & e) const
|
||||
size_t BookingDataset::MatchWithBooking(FeatureBuilder1 const & fb) const
|
||||
{
|
||||
string name;
|
||||
for (auto const & tag : e.Tags())
|
||||
{
|
||||
if (tag.key == "name")
|
||||
{
|
||||
name = tag.value;
|
||||
break;
|
||||
}
|
||||
}
|
||||
auto const name = fb.GetName(StringUtf8Multilang::kDefaultCode);
|
||||
|
||||
if (name.empty())
|
||||
return false;
|
||||
|
||||
// Find |kMaxSelectedElements| nearest values to a point.
|
||||
auto const bookingIndexes =
|
||||
GetNearestHotels(e.lat, e.lon, kMaxSelectedElements, kDistanceLimitInMeters);
|
||||
|
||||
bool matched = false;
|
||||
auto const bookingIndexes = GetNearestHotels(MercatorBounds::ToLatLon(fb.GetKeyPoint()),
|
||||
kMaxSelectedElements, kDistanceLimitInMeters);
|
||||
|
||||
for (size_t const j : bookingIndexes)
|
||||
{
|
||||
if (booking_scoring::Match(GetHotel(j), e).IsMatched())
|
||||
break;
|
||||
if (booking_scoring::Match(GetHotel(j), fb).IsMatched())
|
||||
return j;
|
||||
}
|
||||
|
||||
return matched;
|
||||
return kInvalidHotelIndex;
|
||||
}
|
||||
|
||||
bool BookingDataset::Filter(OsmElement const & e,
|
||||
function<bool(OsmElement const &)> const & fn) const
|
||||
{
|
||||
if (e.type != OsmElement::EntityType::Node)
|
||||
return false;
|
||||
|
||||
if (e.Tags().empty())
|
||||
return false;
|
||||
|
||||
bool matched = false;
|
||||
for (auto const & tag : e.Tags())
|
||||
{
|
||||
if (tag.key == "tourism" && CheckForValues(tag.value))
|
||||
{
|
||||
matched = fn(e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: Need to write file with dropped osm features.
|
||||
|
||||
return matched;
|
||||
}
|
||||
|
||||
} // namespace generator
|
||||
|
|
|
|||
|
|
@ -1,7 +1,5 @@
|
|||
#pragma once
|
||||
|
||||
#include "generator/osm_element.hpp"
|
||||
|
||||
#include "indexer/index.hpp"
|
||||
|
||||
#include "search/reverse_geocoder.hpp"
|
||||
|
|
@ -12,15 +10,19 @@
|
|||
#include "boost/geometry/index/rtree.hpp"
|
||||
|
||||
#include "std/function.hpp"
|
||||
#include "std/limits.hpp"
|
||||
#include "std/string.hpp"
|
||||
|
||||
class FeatureBuilder1;
|
||||
|
||||
namespace generator
|
||||
{
|
||||
class BookingDataset
|
||||
{
|
||||
public:
|
||||
double static constexpr kDistanceLimitInMeters = 150;
|
||||
size_t static constexpr kMaxSelectedElements = 3;
|
||||
static double constexpr kDistanceLimitInMeters = 150;
|
||||
static size_t constexpr kMaxSelectedElements = 3;
|
||||
static size_t constexpr kInvalidHotelIndex = numeric_limits<size_t>::max();
|
||||
|
||||
struct Hotel
|
||||
{
|
||||
|
|
@ -77,33 +79,34 @@ public:
|
|||
explicit BookingDataset(string const & dataPath, string const & addressReferencePath = string());
|
||||
explicit BookingDataset(istream & dataSource, string const & addressReferencePath = string());
|
||||
|
||||
bool BookingFilter(OsmElement const & e) const;
|
||||
bool TourismFilter(OsmElement const & e) const;
|
||||
/// @return an index of a matched hotel or kInvalidHotelIndex on failure.
|
||||
size_t GetMatchingHotelIndex(FeatureBuilder1 const & fb) const;
|
||||
/// @return true if |fb| is a hotel with a name.
|
||||
bool CanBeBooking(FeatureBuilder1 const & fb) const;
|
||||
|
||||
inline size_t Size() const { return m_hotels.size(); }
|
||||
Hotel const & GetHotel(size_t index) const;
|
||||
Hotel & GetHotel(size_t index);
|
||||
vector<size_t> GetNearestHotels(double lat, double lon, size_t limit,
|
||||
vector<size_t> GetNearestHotels(ms::LatLon const & latLon, size_t limit,
|
||||
double maxDistance = 0.0) const;
|
||||
bool MatchByName(string const & osmName, vector<size_t> const & bookingIndexes) const;
|
||||
|
||||
void BuildFeatures(function<void(OsmElement *)> const & fn) const;
|
||||
void BuildHotel(size_t hotelIndex, function<void(FeatureBuilder1 &)> const & fn) const;
|
||||
|
||||
protected:
|
||||
vector<Hotel> m_hotels;
|
||||
|
||||
// create the rtree using default constructor
|
||||
using TPoint = boost::geometry::model::point<float, 2, boost::geometry::cs::cartesian>;
|
||||
using TBox = boost::geometry::model::box<TPoint>;
|
||||
using TValue = pair<TBox, size_t>;
|
||||
|
||||
// Create the rtree using default constructor.
|
||||
boost::geometry::index::rtree<TValue, boost::geometry::index::quadratic<16>> m_rtree;
|
||||
|
||||
void LoadHotels(istream & path, string const & addressReferencePath);
|
||||
bool MatchWithBooking(OsmElement const & e) const;
|
||||
bool Filter(OsmElement const & e, function<bool(OsmElement const &)> const & fn) const;
|
||||
/// @return an index of a matched hotel or numeric_limits<size_t>::max() on failure.
|
||||
size_t MatchWithBooking(FeatureBuilder1 const & e) const;
|
||||
};
|
||||
|
||||
ostream & operator<<(ostream & s, BookingDataset::Hotel const & h);
|
||||
|
||||
} // namespace generator
|
||||
|
|
|
|||
|
|
@ -1,18 +1,22 @@
|
|||
#include "generator/booking_dataset.hpp"
|
||||
#include "generator/booking_scoring.hpp"
|
||||
#include "generator/feature_builder.hpp"
|
||||
#include "generator/osm_source.hpp"
|
||||
|
||||
#include "indexer/classificator_loader.hpp"
|
||||
|
||||
#include "geometry/distance_on_sphere.hpp"
|
||||
|
||||
#include "coding/file_name_utils.hpp"
|
||||
|
||||
#include "std/fstream.hpp"
|
||||
#include "std/iostream.hpp"
|
||||
#include "std/numeric.hpp"
|
||||
#include "std/random.hpp"
|
||||
#include "std/unique_ptr.hpp"
|
||||
|
||||
#include "3party/gflags/src/gflags/gflags.h"
|
||||
|
||||
DEFINE_bool(generate_classif, false, "Generate classificator.");
|
||||
|
||||
DEFINE_string(osm_file_name, "", "Input .o5m file");
|
||||
DEFINE_string(booking_data, "", "Path to booking data in .tsv format");
|
||||
DEFINE_string(sample_data, "", "Sample output path");
|
||||
|
|
@ -21,18 +25,155 @@ DEFINE_uint64(seed, minstd_rand::default_seed, "Seed for random shuffle");
|
|||
|
||||
using namespace generator;
|
||||
|
||||
ostream & operator<<(ostream & s, OsmElement const & e)
|
||||
namespace
|
||||
{
|
||||
for (auto const & tag : e.Tags())
|
||||
string PrintBuilder(FeatureBuilder1 const & fb)
|
||||
{
|
||||
ostringstream s;
|
||||
|
||||
s << "Name: " << fb.GetName(StringUtf8Multilang::kDefaultCode) << '\t';
|
||||
|
||||
auto const params = fb.GetParams();
|
||||
auto const street = params.GetStreet();
|
||||
auto const house = params.house.Get();
|
||||
|
||||
string address = street;
|
||||
if (!house.empty())
|
||||
{
|
||||
auto t = tag;
|
||||
replace(t.key.begin(), t.key.end(), '\n', ' ');
|
||||
replace(t.value.begin(), t.value.end(), '\n', ' ');
|
||||
s << t.key << "=" << t.value << "\t";
|
||||
if (!street.empty())
|
||||
address += ", ";
|
||||
address += house;
|
||||
}
|
||||
return s;
|
||||
|
||||
if (!address.empty())
|
||||
s << "Address: " << address << '\t';
|
||||
|
||||
auto const center = MercatorBounds::ToLatLon(fb.GetKeyPoint());
|
||||
s << "lat: " << center.lat << " lon: " << center.lon << '\t';
|
||||
|
||||
if (fb.GetGeomType() == feature::GEOM_POINT)
|
||||
s << "GeomType: GEOM_POINT";
|
||||
else if (fb.GetGeomType() == feature::GEOM_AREA)
|
||||
s << "GeomType: GEOM_AREA";
|
||||
else
|
||||
CHECK(false, ());
|
||||
|
||||
return s.str();
|
||||
}
|
||||
|
||||
struct Emitter : public EmitterBase
|
||||
{
|
||||
Emitter(feature::GenerateInfo const & info)
|
||||
: m_bookingDataset(info.m_bookingDatafileName)
|
||||
{
|
||||
LOG_SHORT(LINFO, (m_bookingDataset.Size(), "hotels are loaded from Booking."));
|
||||
LOG_SHORT(LINFO, ("OSM data:", FLAGS_osm_file_name));
|
||||
}
|
||||
|
||||
void operator()(FeatureBuilder1 & fb) override
|
||||
{
|
||||
if (m_bookingDataset.CanBeBooking(fb))
|
||||
m_features.emplace_back(fb);
|
||||
}
|
||||
|
||||
void GetNames(vector<string> & names) const override
|
||||
{
|
||||
names.clear();
|
||||
}
|
||||
|
||||
bool Finish() override
|
||||
{
|
||||
LOG_SHORT(LINFO, ("Num of tourism elements:", m_features.size()));
|
||||
vector<size_t> elementIndexes(m_features.size());
|
||||
iota(elementIndexes.begin(), elementIndexes.end(), 0);
|
||||
|
||||
shuffle(elementIndexes.begin(), elementIndexes.end(), minstd_rand(FLAGS_seed));
|
||||
if (FLAGS_selection_size < elementIndexes.size())
|
||||
elementIndexes.resize(FLAGS_selection_size);
|
||||
|
||||
stringstream outStream;
|
||||
|
||||
for (size_t i : elementIndexes)
|
||||
{
|
||||
auto const & fb = m_features[i];
|
||||
auto const bookingIndexes = m_bookingDataset.GetNearestHotels(
|
||||
MercatorBounds::ToLatLon(fb.GetKeyPoint()),
|
||||
BookingDataset::kMaxSelectedElements,
|
||||
BookingDataset::kDistanceLimitInMeters);
|
||||
|
||||
for (size_t const j : bookingIndexes)
|
||||
{
|
||||
auto const & hotel = m_bookingDataset.GetHotel(j);
|
||||
auto const score = booking_scoring::Match(hotel, fb);
|
||||
|
||||
double const distanceMeters = ms::DistanceOnEarth(center.lat, center.lon,
|
||||
hotel.lat, hotel.lon);
|
||||
auto const matched = score.IsMatched();
|
||||
|
||||
outStream << "# ------------------------------------------" << fixed << setprecision(6)
|
||||
<< endl;
|
||||
outStream << (matched ? 'y' : 'n') << " \t" << i << "\t " << j
|
||||
<< "\tdistance: " << distanceMeters
|
||||
<< "\tdistance score: " << score.m_linearNormDistanceScore
|
||||
<< "\tname score: " << score.m_nameSimilarityScore
|
||||
<< "\tresult score: " << score.GetMatchingScore()
|
||||
<< endl;
|
||||
outStream << "# " << PrintBuilder(fb) << endl;
|
||||
outStream << "# " << hotel << endl;
|
||||
outStream << "# URL: https://www.openstreetmap.org/?mlat=" << hotel.lat
|
||||
<< "&mlon=" << hotel.lon << "#map=18/" << hotel.lat << "/" << hotel.lon << endl;
|
||||
}
|
||||
if (!bookingIndexes.empty())
|
||||
outStream << endl << endl;
|
||||
}
|
||||
|
||||
if (FLAGS_sample_data.empty())
|
||||
{
|
||||
cout << outStream.str();
|
||||
}
|
||||
else
|
||||
{
|
||||
ofstream file(FLAGS_sample_data);
|
||||
if (file.is_open())
|
||||
{
|
||||
file << outStream.str();
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG(LERROR, ("Can't output into", FLAGS_sample_data));
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
BookingDataset m_bookingDataset;
|
||||
vector<FeatureBuilder1> m_features;
|
||||
};
|
||||
|
||||
unique_ptr<Emitter> GetEmitter(feature::GenerateInfo const & info)
|
||||
{
|
||||
LOG_SHORT(LINFO, ("Booking data:", FLAGS_booking_data));
|
||||
return make_unique<Emitter>(info);
|
||||
}
|
||||
|
||||
feature::GenerateInfo GetGenerateInfo()
|
||||
{
|
||||
feature::GenerateInfo info;
|
||||
info.m_bookingDatafileName = FLAGS_booking_data;
|
||||
info.m_osmFileName = FLAGS_osm_file_name;
|
||||
info.SetNodeStorageType("map");
|
||||
info.SetOsmFileType("o5m");
|
||||
|
||||
info.m_intermediateDir = my::GetDirectory(FLAGS_sample_data);
|
||||
|
||||
// ...
|
||||
|
||||
return info;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
int main(int argc, char * argv[])
|
||||
{
|
||||
google::SetUsageMessage(
|
||||
|
|
@ -40,74 +181,11 @@ int main(int argc, char * argv[])
|
|||
" data and index files in several passes.");
|
||||
google::ParseCommandLineFlags(&argc, &argv, true);
|
||||
|
||||
LOG_SHORT(LINFO, ("Booking data:", FLAGS_booking_data));
|
||||
classificator::Load();
|
||||
|
||||
BookingDataset bookingDataset(FLAGS_booking_data);
|
||||
|
||||
vector<OsmElement> elements;
|
||||
LOG_SHORT(LINFO, ("OSM data:", FLAGS_osm_file_name));
|
||||
{
|
||||
SourceReader reader =
|
||||
FLAGS_osm_file_name.empty() ? SourceReader() : SourceReader(FLAGS_osm_file_name);
|
||||
ProcessOsmElementsFromO5M(reader, [&](OsmElement * e)
|
||||
{
|
||||
if (bookingDataset.TourismFilter(*e))
|
||||
elements.emplace_back(*e);
|
||||
});
|
||||
}
|
||||
LOG_SHORT(LINFO, ("Num of tourism elements:", elements.size()));
|
||||
|
||||
vector<size_t> elementIndexes(elements.size());
|
||||
iota(elementIndexes.begin(), elementIndexes.end(), 0);
|
||||
|
||||
shuffle(elementIndexes.begin(), elementIndexes.end(), minstd_rand(FLAGS_seed));
|
||||
if (FLAGS_selection_size < elementIndexes.size())
|
||||
elementIndexes.resize(FLAGS_selection_size);
|
||||
|
||||
stringstream outStream;
|
||||
|
||||
for (size_t i : elementIndexes)
|
||||
{
|
||||
OsmElement const & e = elements[i];
|
||||
auto const bookingIndexes = bookingDataset.GetNearestHotels(
|
||||
e.lat, e.lon, BookingDataset::kMaxSelectedElements, BookingDataset::kDistanceLimitInMeters);
|
||||
for (size_t const j : bookingIndexes)
|
||||
{
|
||||
auto const & hotel = bookingDataset.GetHotel(j);
|
||||
auto const score = booking_scoring::Match(hotel, e);
|
||||
|
||||
double const distanceMeters = ms::DistanceOnEarth(e.lat, e.lon, hotel.lat, hotel.lon);
|
||||
bool matched = score.IsMatched();
|
||||
|
||||
outStream << "# ------------------------------------------" << fixed << setprecision(6)
|
||||
<< endl;
|
||||
outStream << (matched ? 'y' : 'n') << " \t" << i << "\t " << j
|
||||
<< "\tdistance: " << distanceMeters
|
||||
<< "\tdistance score: " << score.m_linearNormDistanceScore
|
||||
<< "\tname score: " << score.m_nameSimilarityScore
|
||||
<< "\tresult score: " << score.GetMatchingScore()
|
||||
<< endl;
|
||||
outStream << "# " << e << endl;
|
||||
outStream << "# " << hotel << endl;
|
||||
outStream << "# URL: https://www.openstreetmap.org/?mlat=" << hotel.lat
|
||||
<< "&mlon=" << hotel.lon << "#map=18/" << hotel.lat << "/" << hotel.lon << endl;
|
||||
}
|
||||
if (!bookingIndexes.empty())
|
||||
outStream << endl << endl;
|
||||
}
|
||||
|
||||
if (FLAGS_sample_data.empty())
|
||||
{
|
||||
cout << outStream.str();
|
||||
}
|
||||
else
|
||||
{
|
||||
ofstream file(FLAGS_sample_data);
|
||||
if (file.is_open())
|
||||
file << outStream.str();
|
||||
else
|
||||
LOG(LERROR, ("Can't output into", FLAGS_sample_data));
|
||||
}
|
||||
auto info = GetGenerateInfo();
|
||||
GenerateIntermediateData(info);
|
||||
GenerateFeatures(info, GetEmitter);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
|
|
|||
38
generator/booking_quality_check/booking_quality_check.pro
Normal file
38
generator/booking_quality_check/booking_quality_check.pro
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
# Base functions tests.
|
||||
|
||||
TARGET = booking_quality_check
|
||||
CONFIG += console warn_on
|
||||
CONFIG -= app_bundle
|
||||
TEMPLATE = app
|
||||
|
||||
ROOT_DIR = ../..
|
||||
DEPENDENCIES = \
|
||||
generator \
|
||||
search \
|
||||
routing \
|
||||
indexer \
|
||||
geometry \
|
||||
editor \
|
||||
platform \
|
||||
coding \
|
||||
base \
|
||||
tomcrypt \
|
||||
jansson \
|
||||
pugixml \
|
||||
stats_client \
|
||||
opening_hours \
|
||||
gflags \
|
||||
oauthcpp \
|
||||
expat \
|
||||
protobuf \
|
||||
|
||||
include($$ROOT_DIR/common.pri)
|
||||
|
||||
INCLUDEPATH *= $$ROOT_DIR/3party/gflags/src
|
||||
|
||||
QT *= core
|
||||
|
||||
SOURCES += \
|
||||
booking_quality_check.cpp \
|
||||
|
||||
HEADERS +=
|
||||
|
|
@ -1,6 +1,7 @@
|
|||
#include "generator/booking_scoring.hpp"
|
||||
|
||||
#include "generator/booking_dataset.hpp"
|
||||
#include "generator/feature_builder.hpp"
|
||||
|
||||
#include "indexer/search_delimiters.hpp"
|
||||
#include "indexer/search_string_utils.hpp"
|
||||
|
|
@ -20,7 +21,7 @@ namespace booking_scoring
|
|||
namespace
|
||||
{
|
||||
// Calculated with tools/python/booking_hotels_quality.py.
|
||||
double constexpr kOptimalThreshold = 0.317324;
|
||||
double constexpr kOptimalThreshold = 0.304875;
|
||||
|
||||
template <typename T, typename U>
|
||||
struct decay_equiv :
|
||||
|
|
@ -125,15 +126,17 @@ bool BookingMatchScore::IsMatched() const
|
|||
return GetMatchingScore() > kOptimalThreshold;
|
||||
}
|
||||
|
||||
BookingMatchScore Match(BookingDataset::Hotel const & h, OsmElement const & e)
|
||||
BookingMatchScore Match(BookingDataset::Hotel const & h, FeatureBuilder1 const & fb)
|
||||
{
|
||||
BookingMatchScore score;
|
||||
|
||||
auto const distance = ms::DistanceOnEarth(e.lat, e.lon, h.lat, h.lon);
|
||||
auto const fbCenter = MercatorBounds::ToLatLon(fb.GetKeyPoint());
|
||||
auto const distance = ms::DistanceOnEarth(fbCenter.lat, fbCenter.lon, h.lat, h.lon);
|
||||
score.m_linearNormDistanceScore = GetLinearNormDistanceScore(distance);
|
||||
|
||||
// TODO(mgsergio): Check all translations and use the best one.
|
||||
score.m_nameSimilarityScore = GetNameSimilarityScore(h.name, e.GetTag("name"));
|
||||
score.m_nameSimilarityScore =
|
||||
GetNameSimilarityScore(h.name, fb.GetName(StringUtf8Multilang::kDefaultCode));
|
||||
|
||||
return score;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
#include "generator/booking_dataset.hpp"
|
||||
|
||||
struct OsmElement;
|
||||
class FeatureBuilder1;
|
||||
|
||||
namespace generator
|
||||
{
|
||||
|
|
@ -17,6 +17,6 @@ struct BookingMatchScore
|
|||
double m_nameSimilarityScore{};
|
||||
};
|
||||
|
||||
BookingMatchScore Match(BookingDataset::Hotel const & h, OsmElement const & e);
|
||||
BookingMatchScore Match(BookingDataset::Hotel const & h, FeatureBuilder1 const & fb);
|
||||
} // namespace booking_scoring
|
||||
} // namespace generator
|
||||
|
|
|
|||
|
|
@ -469,6 +469,25 @@ osm::Id FeatureBuilder1::GetLastOsmId() const
|
|||
return m_osmIds.back();
|
||||
}
|
||||
|
||||
osm::Id FeatureBuilder1::GetMostGenericOsmId() const
|
||||
{
|
||||
ASSERT(!m_osmIds.empty(), ());
|
||||
auto result = m_osmIds.front();
|
||||
for (auto const & id : m_osmIds)
|
||||
{
|
||||
if (id.IsRelation())
|
||||
{
|
||||
result = id;
|
||||
break;
|
||||
}
|
||||
else if (result.IsNode() && id.IsWay())
|
||||
{
|
||||
result = id;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
bool FeatureBuilder1::HasOsmId(osm::Id const & id) const
|
||||
{
|
||||
for (auto const & cid : m_osmIds)
|
||||
|
|
|
|||
|
|
@ -161,12 +161,15 @@ public:
|
|||
|
||||
inline FeatureParams const & GetParams() const { return m_params; }
|
||||
|
||||
/// @name For OSM debugging, store original OSM id
|
||||
/// @name For OSM debugging and osm objects replacement, store original OSM id
|
||||
//@{
|
||||
void AddOsmId(osm::Id id);
|
||||
void SetOsmId(osm::Id id);
|
||||
osm::Id GetFirstOsmId() const;
|
||||
osm::Id GetLastOsmId() const;
|
||||
/// @returns an id of the most general element: node's one if there is no area or relation,
|
||||
/// area's one if there is no relation, and relation id otherwise.
|
||||
osm::Id GetMostGenericOsmId() const;
|
||||
bool HasOsmId(osm::Id const & id) const;
|
||||
string GetOsmIdsString() const;
|
||||
//@}
|
||||
|
|
|
|||
|
|
@ -11,7 +11,6 @@
|
|||
|
||||
namespace feature
|
||||
{
|
||||
|
||||
struct GenerateInfo
|
||||
{
|
||||
enum class NodeStorageType
|
||||
|
|
@ -27,7 +26,6 @@ struct GenerateInfo
|
|||
O5M
|
||||
};
|
||||
|
||||
|
||||
// Directory for .mwm.tmp files.
|
||||
string m_tmpDir;
|
||||
// Directory for result .mwm files.
|
||||
|
|
@ -41,7 +39,7 @@ struct GenerateInfo
|
|||
NodeStorageType m_nodeStorageType;
|
||||
OsmSourceType m_osmFileType;
|
||||
string m_osmFileName;
|
||||
|
||||
|
||||
string m_bookingDatafileName;
|
||||
string m_bookingReferenceDir;
|
||||
|
||||
|
|
@ -86,18 +84,20 @@ struct GenerateInfo
|
|||
{
|
||||
return my::JoinFoldersToPath(m_tmpDir, fileName + ext);
|
||||
}
|
||||
|
||||
string GetTargetFileName(string const & fileName, char const * ext = DATA_FILE_EXTENSION) const
|
||||
{
|
||||
return my::JoinFoldersToPath(m_targetDir, fileName + ext);
|
||||
}
|
||||
|
||||
string GetIntermediateFileName(string const & fileName, char const * ext = DATA_FILE_EXTENSION) const
|
||||
{
|
||||
return my::JoinFoldersToPath(m_intermediateDir, fileName + ext);
|
||||
}
|
||||
|
||||
string GetAddressesFileName() const
|
||||
{
|
||||
return ((m_genAddresses && !m_fileName.empty()) ? GetTargetFileName(m_fileName, ADDR_FILE_EXTENSION) : string());
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace feature
|
||||
} // namespace feature
|
||||
|
|
|
|||
|
|
@ -122,6 +122,23 @@ UNIT_TEST(FBuilder_Waterfall)
|
|||
TEST_EQUAL(fb2.GetTypesCount(), 1, ());
|
||||
}
|
||||
|
||||
UNIT_TEST(FBbuilder_GetMostGeneralOsmId)
|
||||
{
|
||||
FeatureBuilder1 fb;
|
||||
|
||||
fb.AddOsmId(osm::Id::Node(1));
|
||||
TEST_EQUAL(fb.GetMostGenericOsmId(), osm::Id::Node(1), ());
|
||||
|
||||
fb.AddOsmId(osm::Id::Node(2));
|
||||
fb.AddOsmId(osm::Id::Way(1));
|
||||
TEST_EQUAL(fb.GetMostGenericOsmId(), osm::Id::Way(1), ());
|
||||
|
||||
fb.AddOsmId(osm::Id::Node(3));
|
||||
fb.AddOsmId(osm::Id::Way(2));
|
||||
fb.AddOsmId(osm::Id::Relation(1));
|
||||
TEST_EQUAL(fb.GetMostGenericOsmId(), osm::Id::Relation(1), ());
|
||||
}
|
||||
|
||||
UNIT_TEST(FVisibility_RemoveNoDrawableTypes)
|
||||
{
|
||||
classificator::Load();
|
||||
|
|
|
|||
|
|
@ -223,7 +223,7 @@ namespace ftype
|
|||
CachedTypes()
|
||||
{
|
||||
Classificator const & c = classif();
|
||||
|
||||
|
||||
for (auto const & e : (StringIL[]) { {"entrance"}, {"highway"} })
|
||||
m_types.push_back(c.GetTypeByPath(e));
|
||||
|
||||
|
|
@ -648,6 +648,7 @@ namespace ftype
|
|||
{ "addr:housename", "*", [¶ms](string & k, string & v) { params.AddHouseName(v); k.clear(); v.clear(); }},
|
||||
{ "addr:street", "*", [¶ms](string & k, string & v) { params.AddStreet(v); k.clear(); v.clear(); }},
|
||||
//{ "addr:streetnumber", "*", [¶ms](string & k, string & v) { params.AddStreet(v); k.clear(); v.clear(); }},
|
||||
// This line was first introduced by vng and was never used uncommented.
|
||||
//{ "addr:full", "*", [¶ms](string & k, string & v) { params.AddAddress(v); k.clear(); v.clear(); }},
|
||||
|
||||
// addr:postcode must be passed to the metadata processor.
|
||||
|
|
|
|||
|
|
@ -38,11 +38,21 @@ uint64_t Id::OsmId() const
|
|||
return m_encodedId & RESET;
|
||||
}
|
||||
|
||||
bool Id::IsNode() const
|
||||
{
|
||||
return ((m_encodedId & NODE) == NODE);
|
||||
}
|
||||
|
||||
bool Id::IsWay() const
|
||||
{
|
||||
return ((m_encodedId & WAY) == WAY);
|
||||
}
|
||||
|
||||
bool Id::IsRelation() const
|
||||
{
|
||||
return ((m_encodedId & RELATION) == RELATION);
|
||||
}
|
||||
|
||||
string Id::Type() const
|
||||
{
|
||||
if ((m_encodedId & RELATION) == RELATION)
|
||||
|
|
|
|||
|
|
@ -21,7 +21,9 @@ public:
|
|||
static Id Relation(uint64_t osmId);
|
||||
|
||||
uint64_t OsmId() const;
|
||||
bool IsNode() const;
|
||||
bool IsWay() const;
|
||||
bool IsRelation() const;
|
||||
|
||||
/// For debug output
|
||||
string Type() const;
|
||||
|
|
|
|||
|
|
@ -18,6 +18,9 @@
|
|||
#include "platform/platform.hpp"
|
||||
|
||||
#include "geometry/mercator.hpp"
|
||||
#include "geometry/tree4d.hpp"
|
||||
|
||||
#include "base/stl_helpers.hpp"
|
||||
|
||||
#include "coding/parse_xml.hpp"
|
||||
|
||||
|
|
@ -183,7 +186,82 @@ public:
|
|||
}
|
||||
};
|
||||
|
||||
class MainFeaturesEmitter
|
||||
/// Used to make a "good" node for a highway graph with OSRM for low zooms.
|
||||
class Place
|
||||
{
|
||||
public:
|
||||
Place(FeatureBuilder1 const & ft, uint32_t type) : m_ft(ft), m_pt(ft.GetKeyPoint()), m_type(type)
|
||||
{
|
||||
using namespace ftypes;
|
||||
|
||||
switch (IsLocalityChecker::Instance().GetType(m_type))
|
||||
{
|
||||
case COUNTRY: m_thresholdM = 300000.0; break;
|
||||
case STATE: m_thresholdM = 100000.0; break;
|
||||
case CITY: m_thresholdM = 30000.0; break;
|
||||
case TOWN: m_thresholdM = 20000.0; break;
|
||||
case VILLAGE: m_thresholdM = 10000.0; break;
|
||||
default: m_thresholdM = 10000.0; break;
|
||||
}
|
||||
}
|
||||
|
||||
FeatureBuilder1 const & GetFeature() const { return m_ft; }
|
||||
|
||||
m2::RectD GetLimitRect() const
|
||||
{
|
||||
return MercatorBounds::RectByCenterXYAndSizeInMeters(m_pt, m_thresholdM);
|
||||
}
|
||||
|
||||
bool IsEqual(Place const & r) const
|
||||
{
|
||||
return (AreTypesEqual(m_type, r.m_type) &&
|
||||
m_ft.GetName() == r.m_ft.GetName() &&
|
||||
(IsPoint() || r.IsPoint()) &&
|
||||
MercatorBounds::DistanceOnEarth(m_pt, r.m_pt) < m_thresholdM);
|
||||
}
|
||||
|
||||
/// Check whether we need to replace place @r with place @this.
|
||||
bool IsBetterThan(Place const & r) const
|
||||
{
|
||||
// Check ranks.
|
||||
uint8_t const r1 = m_ft.GetRank();
|
||||
uint8_t const r2 = r.m_ft.GetRank();
|
||||
if (r1 != r2)
|
||||
return r1 > r2;
|
||||
|
||||
// Check types length.
|
||||
// ("place-city-capital-2" is better than "place-city").
|
||||
uint8_t const l1 = ftype::GetLevel(m_type);
|
||||
uint8_t const l2 = ftype::GetLevel(r.m_type);
|
||||
if (l1 != l2)
|
||||
return l1 > l2;
|
||||
|
||||
// Assume that area places has better priority than point places at the very end ...
|
||||
/// @todo It was usefull when place=XXX type has any area fill style.
|
||||
/// Need to review priority logic here (leave the native osm label).
|
||||
return !IsPoint();
|
||||
}
|
||||
|
||||
private:
|
||||
bool IsPoint() const { return (m_ft.GetGeomType() == feature::GEOM_POINT); }
|
||||
|
||||
static bool AreTypesEqual(uint32_t t1, uint32_t t2)
|
||||
{
|
||||
// Use 2-arity places comparison for filtering.
|
||||
// ("place-city-capital-2" is equal to "place-city")
|
||||
ftype::TruncValue(t1, 2);
|
||||
ftype::TruncValue(t2, 2);
|
||||
return (t1 == t2);
|
||||
}
|
||||
|
||||
FeatureBuilder1 m_ft;
|
||||
m2::PointD m_pt;
|
||||
uint32_t m_type;
|
||||
double m_thresholdM;
|
||||
|
||||
};
|
||||
|
||||
class MainFeaturesEmitter : public EmitterBase
|
||||
{
|
||||
using TWorldGenerator = WorldMapGenerator<feature::FeaturesCollector>;
|
||||
using TCountriesGenerator = CountryMapGenerator<feature::Polygonizer<feature::FeaturesCollector>>;
|
||||
|
|
@ -193,9 +271,18 @@ class MainFeaturesEmitter
|
|||
unique_ptr<CoastlineFeaturesGenerator> m_coasts;
|
||||
unique_ptr<feature::FeaturesCollector> m_coastsHolder;
|
||||
|
||||
string const m_skippedElementsPath;
|
||||
ostringstream m_skippedElements;
|
||||
|
||||
string m_srcCoastsFile;
|
||||
bool m_failOnCoasts;
|
||||
|
||||
generator::BookingDataset m_bookingDataset;
|
||||
|
||||
/// Used to prepare a list of cities to serve as a list of nodes
|
||||
/// for building a highway graph with OSRM for low zooms.
|
||||
m4::Tree<Place> m_places;
|
||||
|
||||
enum TypeIndex
|
||||
{
|
||||
NATURAL_COASTLINE,
|
||||
|
|
@ -211,7 +298,9 @@ class MainFeaturesEmitter
|
|||
|
||||
public:
|
||||
MainFeaturesEmitter(feature::GenerateInfo const & info)
|
||||
: m_failOnCoasts(info.m_failOnCoasts)
|
||||
: m_skippedElementsPath(info.GetIntermediateFileName("skipped_elements", ".lst"))
|
||||
, m_failOnCoasts(info.m_failOnCoasts)
|
||||
, m_bookingDataset(info.m_bookingDatafileName, info.m_bookingReferenceDir)
|
||||
{
|
||||
Classificator const & c = classif();
|
||||
|
||||
|
|
@ -248,47 +337,69 @@ public:
|
|||
m_world.reset(new TWorldGenerator(info));
|
||||
}
|
||||
|
||||
void operator()(FeatureBuilder1 fb)
|
||||
void operator()(FeatureBuilder1 & fb) override
|
||||
{
|
||||
uint32_t const coastType = Type(NATURAL_COASTLINE);
|
||||
bool const hasCoast = fb.HasType(coastType);
|
||||
static uint32_t const placeType = classif().GetTypeByPath({"place"});
|
||||
uint32_t const type = fb.GetParams().FindType(placeType, 1);
|
||||
|
||||
if (m_coasts)
|
||||
auto hotelIndex = generator::BookingDataset::kInvalidHotelIndex;
|
||||
|
||||
if (type != ftype::GetEmptyValue() && !fb.GetName().empty())
|
||||
{
|
||||
if (hasCoast)
|
||||
m_places.ReplaceEqualInRect(
|
||||
Place(fb, type),
|
||||
[](Place const & p1, Place const & p2) { return p1.IsEqual(p2); },
|
||||
[](Place const & p1, Place const & p2) { return p1.IsBetterThan(p2); });
|
||||
}
|
||||
else if ((hotelIndex = m_bookingDataset.GetMatchingHotelIndex(fb)) !=
|
||||
generator::BookingDataset::kInvalidHotelIndex)
|
||||
{
|
||||
m_skippedElements << DebugPrint(fb.GetMostGenericOsmId()) << endl;
|
||||
|
||||
// Turn a hotel into a simple building.
|
||||
if (fb.GetGeomType() == feature::GEOM_AREA)
|
||||
{
|
||||
CHECK(fb.GetGeomType() != feature::GEOM_POINT, ());
|
||||
// leave only coastline type
|
||||
fb.SetType(coastType);
|
||||
(*m_coasts)(fb);
|
||||
// Remove all information about a hotel.
|
||||
auto params = fb.GetParams();
|
||||
params.ClearName();
|
||||
auto & meta = params.GetMetadata();
|
||||
meta.Drop(feature::Metadata::EType::FMD_STARS);
|
||||
meta.Drop(feature::Metadata::EType::FMD_WEBSITE);
|
||||
meta.Drop(feature::Metadata::EType::FMD_PHONE_NUMBER);
|
||||
|
||||
auto const & c = classif();
|
||||
auto const tourism = c.GetTypeByPath({"tourism"});
|
||||
my::EraseIf(params.m_Types, [&c, tourism](uint32_t type)
|
||||
{
|
||||
ftype::TruncValue(type, 1);
|
||||
return type == tourism;
|
||||
});
|
||||
fb.SetParams(params);
|
||||
|
||||
Emit(fb);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (hasCoast)
|
||||
else
|
||||
{
|
||||
fb.PopExactType(Type(NATURAL_LAND));
|
||||
fb.PopExactType(coastType);
|
||||
Emit(fb);
|
||||
}
|
||||
else if ((fb.HasType(Type(PLACE_ISLAND)) || fb.HasType(Type(PLACE_ISLET))) &&
|
||||
fb.GetGeomType() == feature::GEOM_AREA)
|
||||
{
|
||||
fb.AddType(Type(NATURAL_LAND));
|
||||
}
|
||||
|
||||
if (!fb.RemoveInvalidTypes())
|
||||
return;
|
||||
|
||||
if (m_world)
|
||||
(*m_world)(fb);
|
||||
|
||||
if (m_countries)
|
||||
(*m_countries)(fb);
|
||||
}
|
||||
|
||||
/// @return false if coasts are not merged and FLAG_fail_on_coasts is set
|
||||
bool Finish()
|
||||
bool Finish() override
|
||||
{
|
||||
DumpSkippedElements();
|
||||
|
||||
// Emit all booking objecs to the map.
|
||||
for (size_t hotelIndex = 0; hotelIndex < m_bookingDataset.Size(); ++hotelIndex)
|
||||
m_bookingDataset.BuildHotel(hotelIndex, [this](FeatureBuilder1 & fb) { Emit(fb); });
|
||||
|
||||
m_places.ForEach([this](Place const & p)
|
||||
{
|
||||
// m_places are no longer used after this point.
|
||||
Emit(const_cast<FeatureBuilder1 &>(p.GetFeature()));
|
||||
});
|
||||
|
||||
if (m_world)
|
||||
m_world->DoMerge();
|
||||
|
||||
|
|
@ -340,16 +451,82 @@ public:
|
|||
return true;
|
||||
}
|
||||
|
||||
inline void GetNames(vector<string> & names) const
|
||||
void GetNames(vector<string> & names) const override
|
||||
{
|
||||
if (m_countries)
|
||||
names = m_countries->Parent().Names();
|
||||
else
|
||||
names.clear();
|
||||
}
|
||||
|
||||
private:
|
||||
void Emit(FeatureBuilder1 & fb)
|
||||
{
|
||||
uint32_t const coastType = Type(NATURAL_COASTLINE);
|
||||
bool const hasCoast = fb.HasType(coastType);
|
||||
|
||||
if (m_coasts)
|
||||
{
|
||||
if (hasCoast)
|
||||
{
|
||||
CHECK(fb.GetGeomType() != feature::GEOM_POINT, ());
|
||||
// leave only coastline type
|
||||
fb.SetType(coastType);
|
||||
(*m_coasts)(fb);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (hasCoast)
|
||||
{
|
||||
fb.PopExactType(Type(NATURAL_LAND));
|
||||
fb.PopExactType(coastType);
|
||||
}
|
||||
else if ((fb.HasType(Type(PLACE_ISLAND)) || fb.HasType(Type(PLACE_ISLET))) &&
|
||||
fb.GetGeomType() == feature::GEOM_AREA)
|
||||
{
|
||||
fb.AddType(Type(NATURAL_LAND));
|
||||
}
|
||||
|
||||
if (!fb.RemoveInvalidTypes())
|
||||
return;
|
||||
|
||||
if (m_world)
|
||||
(*m_world)(fb);
|
||||
|
||||
if (m_countries)
|
||||
(*m_countries)(fb);
|
||||
}
|
||||
|
||||
void DumpSkippedElements()
|
||||
{
|
||||
auto const skippedElements = m_skippedElements.str();
|
||||
|
||||
if (skippedElements.empty())
|
||||
{
|
||||
LOG(LINFO, ("No osm object was skipped."));
|
||||
return;
|
||||
}
|
||||
|
||||
ofstream file(m_skippedElementsPath, ios_base::app);
|
||||
if (file.is_open())
|
||||
{
|
||||
file << m_skippedElements.str();
|
||||
LOG(LINFO, ("Saving skipped elements to", m_skippedElementsPath, "done."));
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG(LERROR, ("Can't output into", m_skippedElementsPath));
|
||||
}
|
||||
}
|
||||
};
|
||||
} // anonymous namespace
|
||||
|
||||
unique_ptr<EmitterBase> MakeMainFeatureEmitter(feature::GenerateInfo const & info)
|
||||
{
|
||||
LOG(LINFO, ("Processing booking data from", info.m_bookingDatafileName, "done."));
|
||||
return make_unique<MainFeaturesEmitter>(info);
|
||||
}
|
||||
|
||||
template <typename TElement, typename TCache>
|
||||
void AddElementToCache(TCache & cache, TElement const & em)
|
||||
|
|
@ -496,7 +673,7 @@ void ProcessOsmElementsFromO5M(SourceReader & stream, function<void(OsmElement *
|
|||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
template <class TNodesHolder>
|
||||
bool GenerateFeaturesImpl(feature::GenerateInfo & info)
|
||||
bool GenerateFeaturesImpl(feature::GenerateInfo & info, EmitterBase & emitter)
|
||||
{
|
||||
try
|
||||
{
|
||||
|
|
@ -506,9 +683,9 @@ bool GenerateFeaturesImpl(feature::GenerateInfo & info)
|
|||
TDataCache cache(nodes, info);
|
||||
cache.LoadIndex();
|
||||
|
||||
MainFeaturesEmitter bucketer(info);
|
||||
OsmToFeatureTranslator<MainFeaturesEmitter, TDataCache> parser(
|
||||
bucketer, cache, info.m_makeCoasts ? classif().GetCoastType() : 0,
|
||||
// TODO(mgsergio): Get rid of EmitterBase template parameter.
|
||||
OsmToFeatureTranslator<EmitterBase, TDataCache> parser(
|
||||
emitter, cache, info.m_makeCoasts ? classif().GetCoastType() : 0,
|
||||
info.GetAddressesFileName());
|
||||
|
||||
TagAdmixer tagAdmixer(info.GetIntermediateFileName("ways", ".csv"),
|
||||
|
|
@ -516,25 +693,13 @@ bool GenerateFeaturesImpl(feature::GenerateInfo & info)
|
|||
TagReplacer tagReplacer(GetPlatform().ResourcesDir() + REPLACED_TAGS_FILE);
|
||||
OsmTagMixer osmTagMixer(GetPlatform().ResourcesDir() + MIXED_TAGS_FILE);
|
||||
|
||||
// If info.m_bookingDatafileName is empty then no data will be loaded.
|
||||
generator::BookingDataset bookingDataset(info.m_bookingDatafileName,
|
||||
info.m_bookingReferenceDir);
|
||||
|
||||
stringstream skippedElements;
|
||||
|
||||
// Here we can add new tags to element!!!
|
||||
// Here we can add new tags to the elements!
|
||||
auto const fn = [&](OsmElement * e)
|
||||
{
|
||||
tagReplacer(e);
|
||||
tagAdmixer(e);
|
||||
osmTagMixer(e);
|
||||
|
||||
if (bookingDataset.BookingFilter(*e))
|
||||
{
|
||||
skippedElements << e->id << endl;
|
||||
return;
|
||||
}
|
||||
|
||||
parser.EmitElement(e);
|
||||
};
|
||||
|
||||
|
|
@ -551,30 +716,11 @@ bool GenerateFeaturesImpl(feature::GenerateInfo & info)
|
|||
|
||||
LOG(LINFO, ("Processing", info.m_osmFileName, "done."));
|
||||
|
||||
if (!info.m_bookingDatafileName.empty())
|
||||
{
|
||||
bookingDataset.BuildFeatures([&](OsmElement * e) { parser.EmitElement(e); });
|
||||
LOG(LINFO, ("Processing booking data from", info.m_bookingDatafileName, "done."));
|
||||
string skippedElementsPath = info.GetIntermediateFileName("skipped_elements", ".lst");
|
||||
ofstream file(skippedElementsPath);
|
||||
if (file.is_open())
|
||||
{
|
||||
file << skippedElements.str();
|
||||
LOG(LINFO, ("Saving skipped elements to", skippedElementsPath, "done."));
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG(LERROR, ("Can't output into", skippedElementsPath));
|
||||
}
|
||||
}
|
||||
|
||||
parser.Finish();
|
||||
|
||||
// Stop if coasts are not merged and FLAG_fail_on_coasts is set
|
||||
if (!bucketer.Finish())
|
||||
if (!emitter.Finish())
|
||||
return false;
|
||||
|
||||
bucketer.GetNames(info.m_bucketNames);
|
||||
emitter.GetNames(info.m_bucketNames);
|
||||
}
|
||||
catch (Reader::Exception const & ex)
|
||||
{
|
||||
|
|
@ -619,16 +765,17 @@ bool GenerateIntermediateDataImpl(feature::GenerateInfo & info)
|
|||
return true;
|
||||
}
|
||||
|
||||
bool GenerateFeatures(feature::GenerateInfo & info)
|
||||
bool GenerateFeatures(feature::GenerateInfo & info, EmitterFactory factory)
|
||||
{
|
||||
auto emitter = factory(info);
|
||||
switch (info.m_nodeStorageType)
|
||||
{
|
||||
case feature::GenerateInfo::NodeStorageType::File:
|
||||
return GenerateFeaturesImpl<cache::RawFilePointStorage<cache::EMode::Read>>(info);
|
||||
return GenerateFeaturesImpl<cache::RawFilePointStorage<cache::EMode::Read>>(info, *emitter);
|
||||
case feature::GenerateInfo::NodeStorageType::Index:
|
||||
return GenerateFeaturesImpl<cache::MapFilePointStorage<cache::EMode::Read>>(info);
|
||||
return GenerateFeaturesImpl<cache::MapFilePointStorage<cache::EMode::Read>>(info, *emitter);
|
||||
case feature::GenerateInfo::NodeStorageType::Memory:
|
||||
return GenerateFeaturesImpl<cache::RawMemPointStorage<cache::EMode::Read>>(info);
|
||||
return GenerateFeaturesImpl<cache::RawMemPointStorage<cache::EMode::Read>>(info, *emitter);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,11 +30,31 @@ public:
|
|||
uint64_t Read(char * buffer, uint64_t bufferSize);
|
||||
};
|
||||
|
||||
class FeatureBuilder1;
|
||||
|
||||
// Emitter is used in OsmElemen to FeatureBuilder translation process.
|
||||
class EmitterBase
|
||||
{
|
||||
public:
|
||||
virtual ~EmitterBase() = default;
|
||||
|
||||
bool GenerateFeatures(feature::GenerateInfo & info);
|
||||
/// This method is used by OsmTranslator to pass |fb| to Emitter for further processing.
|
||||
virtual void operator()(FeatureBuilder1 & fb) = 0;
|
||||
/// Finish is used in GenerateFeatureImpl to make whatever work is needed after
|
||||
/// all OmsElements are processed.
|
||||
virtual bool Finish() { return true; }
|
||||
/// Sets buckets (mwm names).
|
||||
// TODO(syershov): Make this topic clear.
|
||||
virtual void GetNames(vector<string> & names) const = 0;
|
||||
};
|
||||
|
||||
unique_ptr<EmitterBase> MakeMainFeatureEmitter(feature::GenerateInfo const & info);
|
||||
|
||||
using EmitterFactory = function<unique_ptr<EmitterBase>(feature::GenerateInfo const &)>;
|
||||
|
||||
bool GenerateFeatures(feature::GenerateInfo & info,
|
||||
EmitterFactory factory = MakeMainFeatureEmitter);
|
||||
bool GenerateIntermediateData(feature::GenerateInfo & info);
|
||||
|
||||
void ProcessOsmElementsFromO5M(SourceReader & stream, function<void(OsmElement *)> processor);
|
||||
void ProcessOsmElementsFromXML(SourceReader & stream, function<void(OsmElement *)> processor);
|
||||
|
||||
|
|
|
|||
|
|
@ -9,8 +9,6 @@
|
|||
#include "indexer/feature_visibility.hpp"
|
||||
#include "indexer/ftypes_matcher.hpp"
|
||||
|
||||
#include "geometry/tree4d.hpp"
|
||||
|
||||
#include "coding/file_writer.hpp"
|
||||
|
||||
#include "base/cache.hpp"
|
||||
|
|
@ -24,77 +22,6 @@
|
|||
|
||||
namespace
|
||||
{
|
||||
class Place
|
||||
{
|
||||
FeatureBuilder1 m_ft;
|
||||
m2::PointD m_pt;
|
||||
uint32_t m_type;
|
||||
double m_thresholdM;
|
||||
|
||||
bool IsPoint() const { return (m_ft.GetGeomType() == feature::GEOM_POINT); }
|
||||
static bool IsEqualTypes(uint32_t t1, uint32_t t2)
|
||||
{
|
||||
// Use 2-arity places comparison for filtering.
|
||||
// ("place-city-capital-2" is equal to "place-city")
|
||||
ftype::TruncValue(t1, 2);
|
||||
ftype::TruncValue(t2, 2);
|
||||
return (t1 == t2);
|
||||
}
|
||||
|
||||
public:
|
||||
Place(FeatureBuilder1 const & ft, uint32_t type) : m_ft(ft), m_pt(ft.GetKeyPoint()), m_type(type)
|
||||
{
|
||||
using namespace ftypes;
|
||||
|
||||
switch (IsLocalityChecker::Instance().GetType(m_type))
|
||||
{
|
||||
case COUNTRY: m_thresholdM = 300000.0; break;
|
||||
case STATE: m_thresholdM = 100000.0; break;
|
||||
case CITY: m_thresholdM = 30000.0; break;
|
||||
case TOWN: m_thresholdM = 20000.0; break;
|
||||
case VILLAGE: m_thresholdM = 10000.0; break;
|
||||
default: m_thresholdM = 10000.0; break;
|
||||
}
|
||||
}
|
||||
|
||||
FeatureBuilder1 const & GetFeature() const { return m_ft; }
|
||||
|
||||
m2::RectD GetLimitRect() const
|
||||
{
|
||||
return MercatorBounds::RectByCenterXYAndSizeInMeters(m_pt, m_thresholdM);
|
||||
}
|
||||
|
||||
bool IsEqual(Place const & r) const
|
||||
{
|
||||
return (IsEqualTypes(m_type, r.m_type) &&
|
||||
m_ft.GetName() == r.m_ft.GetName() &&
|
||||
(IsPoint() || r.IsPoint()) &&
|
||||
MercatorBounds::DistanceOnEarth(m_pt, r.m_pt) < m_thresholdM);
|
||||
}
|
||||
|
||||
/// Check whether we need to replace place @r with place @this.
|
||||
bool IsBetterThan(Place const & r) const
|
||||
{
|
||||
// Check ranks.
|
||||
uint8_t const r1 = m_ft.GetRank();
|
||||
uint8_t const r2 = r.m_ft.GetRank();
|
||||
if (r1 != r2)
|
||||
return (r2 < r1);
|
||||
|
||||
// Check types length.
|
||||
// ("place-city-capital-2" is better than "place-city").
|
||||
uint8_t const l1 = ftype::GetLevel(m_type);
|
||||
uint8_t const l2 = ftype::GetLevel(r.m_type);
|
||||
if (l1 != l2)
|
||||
return (l2 < l1);
|
||||
|
||||
// Assume that area places has better priority than point places at the very end ...
|
||||
/// @todo It was usefull when place=XXX type has any area fill style.
|
||||
/// Need to review priority logic here (leave the native osm label).
|
||||
return !IsPoint();
|
||||
}
|
||||
};
|
||||
|
||||
/// Generated features should include parent relation tags to make
|
||||
/// full types matching and storing any additional info.
|
||||
class RelationTagsBase
|
||||
|
|
@ -243,7 +170,6 @@ protected:
|
|||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
/// @param TEmitter Feature accumulating policy
|
||||
|
|
@ -255,7 +181,7 @@ class OsmToFeatureTranslator
|
|||
TCache & m_holder;
|
||||
uint32_t m_coastType;
|
||||
unique_ptr<FileWriter> m_addrWriter;
|
||||
m4::Tree<Place> m_places;
|
||||
|
||||
RelationTagsNode m_nodeRelations;
|
||||
RelationTagsWay m_wayRelations;
|
||||
|
||||
|
|
@ -334,32 +260,25 @@ class OsmToFeatureTranslator
|
|||
return params.IsValid();
|
||||
}
|
||||
|
||||
void EmitFeatureBase(FeatureBuilder1 & ft, FeatureParams const & params)
|
||||
void EmitFeatureBase(FeatureBuilder1 & ft, FeatureParams const & params) const
|
||||
{
|
||||
ft.SetParams(params);
|
||||
if (ft.PreSerialize())
|
||||
{
|
||||
string addr;
|
||||
if (m_addrWriter && ftypes::IsBuildingChecker::Instance()(params.m_Types) && ft.FormatFullAddress(addr))
|
||||
m_addrWriter->Write(addr.c_str(), addr.size());
|
||||
|
||||
static uint32_t const placeType = classif().GetTypeByPath({"place"});
|
||||
uint32_t const type = params.FindType(placeType, 1);
|
||||
|
||||
if (type != ftype::GetEmptyValue() && !ft.GetName().empty())
|
||||
if (m_addrWriter && ftypes::IsBuildingChecker::Instance()(params.m_Types) &&
|
||||
ft.FormatFullAddress(addr))
|
||||
{
|
||||
m_places.ReplaceEqualInRect(Place(ft, type),
|
||||
[](Place const & p1, Place const & p2) { return p1.IsEqual(p2); },
|
||||
[](Place const & p1, Place const & p2) { return p1.IsBetterThan(p2); });
|
||||
m_addrWriter->Write(addr.c_str(), addr.size());
|
||||
}
|
||||
else
|
||||
m_emitter(ft);
|
||||
|
||||
m_emitter(ft);
|
||||
}
|
||||
}
|
||||
|
||||
/// @param[in] params Pass by value because it can be modified.
|
||||
//@{
|
||||
void EmitPoint(m2::PointD const & pt, FeatureParams params, osm::Id id)
|
||||
void EmitPoint(m2::PointD const & pt, FeatureParams params, osm::Id id) const
|
||||
{
|
||||
if (feature::RemoveNoDrawableTypes(params.m_Types, feature::GEOM_POINT))
|
||||
{
|
||||
|
|
@ -370,7 +289,7 @@ class OsmToFeatureTranslator
|
|||
}
|
||||
}
|
||||
|
||||
void EmitLine(FeatureBuilder1 & ft, FeatureParams params, bool isCoastLine)
|
||||
void EmitLine(FeatureBuilder1 & ft, FeatureParams params, bool isCoastLine) const
|
||||
{
|
||||
if (isCoastLine || feature::RemoveNoDrawableTypes(params.m_Types, feature::GEOM_LINE))
|
||||
{
|
||||
|
|
@ -380,7 +299,7 @@ class OsmToFeatureTranslator
|
|||
}
|
||||
|
||||
template <class MakeFnT>
|
||||
void EmitArea(FeatureBuilder1 & ft, FeatureParams params, MakeFnT makeFn)
|
||||
void EmitArea(FeatureBuilder1 & ft, FeatureParams params, MakeFnT makeFn) const
|
||||
{
|
||||
using namespace feature;
|
||||
|
||||
|
|
@ -564,19 +483,11 @@ public:
|
|||
}
|
||||
|
||||
public:
|
||||
OsmToFeatureTranslator(TEmitter & emitter, TCache & holder,
|
||||
uint32_t coastType, string const & addrFilePath)
|
||||
OsmToFeatureTranslator(TEmitter & emitter, TCache & holder, uint32_t coastType,
|
||||
string const & addrFilePath = {})
|
||||
: m_emitter(emitter), m_holder(holder), m_coastType(coastType)
|
||||
{
|
||||
if (!addrFilePath.empty())
|
||||
m_addrWriter.reset(new FileWriter(addrFilePath));
|
||||
}
|
||||
|
||||
void Finish()
|
||||
{
|
||||
m_places.ForEach([this] (Place const & p)
|
||||
{
|
||||
m_emitter(p.GetFeature());
|
||||
});
|
||||
}
|
||||
};
|
||||
|
|
|
|||
|
|
@ -36,34 +36,34 @@ void ReadCommon(unique_ptr<Reader> classificator,
|
|||
c.ReadTypesMapping(s);
|
||||
}
|
||||
}
|
||||
}
|
||||
} // namespace
|
||||
|
||||
namespace classificator
|
||||
{
|
||||
void Load()
|
||||
void Load()
|
||||
{
|
||||
LOG(LDEBUG, ("Reading of classificator started"));
|
||||
|
||||
Platform & p = GetPlatform();
|
||||
|
||||
MapStyle const originMapStyle = GetStyleReader().GetCurrentStyle();
|
||||
|
||||
for (size_t i = 0; i < MapStyleCount; ++i)
|
||||
{
|
||||
LOG(LDEBUG, ("Reading of classificator started"));
|
||||
|
||||
Platform & p = GetPlatform();
|
||||
|
||||
MapStyle const originMapStyle = GetStyleReader().GetCurrentStyle();
|
||||
|
||||
for (size_t i = 0; i < MapStyleCount; ++i)
|
||||
MapStyle const mapStyle = static_cast<MapStyle>(i);
|
||||
// Read the merged style only if it was requested.
|
||||
if (mapStyle != MapStyleMerged || originMapStyle == MapStyleMerged)
|
||||
{
|
||||
MapStyle const mapStyle = static_cast<MapStyle>(i);
|
||||
// Read the merged style only if it was requested.
|
||||
if (mapStyle != MapStyleMerged || originMapStyle == MapStyleMerged)
|
||||
{
|
||||
GetStyleReader().SetCurrentStyle(mapStyle);
|
||||
ReadCommon(p.GetReader("classificator.txt"),
|
||||
p.GetReader("types.txt"));
|
||||
GetStyleReader().SetCurrentStyle(mapStyle);
|
||||
ReadCommon(p.GetReader("classificator.txt"),
|
||||
p.GetReader("types.txt"));
|
||||
|
||||
drule::LoadRules();
|
||||
}
|
||||
drule::LoadRules();
|
||||
}
|
||||
|
||||
GetStyleReader().SetCurrentStyle(originMapStyle);
|
||||
|
||||
LOG(LDEBUG, ("Reading of classificator finished"));
|
||||
}
|
||||
|
||||
GetStyleReader().SetCurrentStyle(originMapStyle);
|
||||
|
||||
LOG(LDEBUG, ("Reading of classificator finished"));
|
||||
}
|
||||
} // namespace classificator
|
||||
|
|
|
|||
|
|
@ -239,6 +239,11 @@ bool IsDummyName(string const & s)
|
|||
// FeatureParams implementation
|
||||
/////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
void FeatureParams::ClearName()
|
||||
{
|
||||
name.Clear();
|
||||
}
|
||||
|
||||
bool FeatureParams::AddName(string const & lang, string const & s)
|
||||
{
|
||||
if (IsDummyName(s))
|
||||
|
|
|
|||
|
|
@ -222,6 +222,8 @@ public:
|
|||
|
||||
FeatureParams() : m_geomType(0xFF), m_reverseGeometry(false) {}
|
||||
|
||||
void ClearName();
|
||||
|
||||
bool AddName(string const & lang, string const & s);
|
||||
bool AddHouseName(string const & s);
|
||||
bool AddHouseNumber(string houseNumber);
|
||||
|
|
|
|||
5
omim.pro
5
omim.pro
|
|
@ -41,6 +41,11 @@ SUBDIRS = 3party base coding geometry editor indexer routing search
|
|||
CONFIG(gtool):!CONFIG(no-tests) {
|
||||
SUBDIRS *= map
|
||||
|
||||
# Booking quality check
|
||||
booking_quality_check.subdir = generator/booking_quality_check
|
||||
booking_quality_check.depends = $$SUBDIRS
|
||||
SUBDIRS *= booking_quality_check
|
||||
|
||||
routing_integration_tests.subdir = routing/routing_integration_tests
|
||||
routing_integration_tests.depends = $$SUBDIRS
|
||||
routing_consistency_tests.subdir = routing/routing_consistency_tests
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue