diff --git a/generator/CMakeLists.txt b/generator/CMakeLists.txt index 8a314b726a..a1cac29990 100644 --- a/generator/CMakeLists.txt +++ b/generator/CMakeLists.txt @@ -75,9 +75,9 @@ set(SRC search_index_builder.hpp sponsored_dataset.hpp sponsored_dataset_inl.hpp + sponsored_object_storage.hpp sponsored_scoring.cpp sponsored_scoring.hpp - sponsored_storage.hpp srtm_parser.cpp srtm_parser.hpp statistics.cpp diff --git a/generator/booking_dataset.cpp b/generator/booking_dataset.cpp index a590776e78..7cb0a65dad 100644 --- a/generator/booking_dataset.cpp +++ b/generator/booking_dataset.cpp @@ -196,11 +196,11 @@ BookingDataset::ObjectId BookingDataset::FindMatchingObjectIdImpl(FeatureBuilder // Find |kMaxSelectedElements| nearest values to a point. auto const bookingIndexes = - GetStorage().GetNearestObjects(MercatorBounds::ToLatLon(fb.GetKeyPoint())); + m_storage.GetNearestObjects(MercatorBounds::ToLatLon(fb.GetKeyPoint())); for (auto const j : bookingIndexes) { - if (sponsored_scoring::Match(GetStorage().GetObjectById(j), fb).IsMatched()) + if (sponsored_scoring::Match(m_storage.GetObjectById(j), fb).IsMatched()) return j; } diff --git a/generator/generate_info.hpp b/generator/generate_info.hpp index 4724cb7c65..0de215dee9 100644 --- a/generator/generate_info.hpp +++ b/generator/generate_info.hpp @@ -45,7 +45,6 @@ struct GenerateInfo std::string m_opentableDatafileName; std::string m_opentableReferenceDir; std::string m_viatorDatafileName; - std::string m_viatorReferenceDir; uint32_t m_versionDate = 0; diff --git a/generator/generator.pro b/generator/generator.pro index dc1e62f9d7..24373cc60d 100644 --- a/generator/generator.pro +++ b/generator/generator.pro @@ -94,8 +94,8 @@ HEADERS += \ search_index_builder.hpp \ sponsored_dataset.hpp \ sponsored_dataset_inl.hpp \ + sponsored_object_storage.hpp \ sponsored_scoring.hpp \ - sponsored_storage.hpp \ srtm_parser.hpp \ statistics.hpp \ tag_admixer.hpp \ diff --git a/generator/generator_tool/generator_tool.cpp b/generator/generator_tool/generator_tool.cpp index a5329a6f0c..8e6599fed2 100644 --- a/generator/generator_tool/generator_tool.cpp +++ b/generator/generator_tool/generator_tool.cpp @@ -144,7 +144,6 @@ int main(int argc, char ** argv) genInfo.m_opentableDatafileName = FLAGS_opentable_data; genInfo.m_opentableReferenceDir = FLAGS_opentable_reference_path; genInfo.m_viatorDatafileName = FLAGS_viator_data; - genInfo.m_viatorReferenceDir = FLAGS_viator_reference_path; genInfo.m_versionDate = static_cast(FLAGS_planet_version); diff --git a/generator/opentable_dataset.cpp b/generator/opentable_dataset.cpp index d880d29c7a..fff18b1816 100644 --- a/generator/opentable_dataset.cpp +++ b/generator/opentable_dataset.cpp @@ -54,7 +54,7 @@ void OpentableDataset::PreprocessMatchedOsmObject(ObjectId const matchedObjId, F { FeatureParams params = fb.GetParams(); - auto restaurant = GetStorage().GetObjectById(matchedObjId); + auto const & restaurant = m_storage.GetObjectById(matchedObjId); auto & metadata = params.GetMetadata(); metadata.Set(feature::Metadata::FMD_SPONSORED_ID, strings::to_string(restaurant.m_id.Get())); @@ -81,11 +81,11 @@ OpentableDataset::ObjectId OpentableDataset::FindMatchingObjectIdImpl(FeatureBui return Object::InvalidObjectId(); // Find |kMaxSelectedElements| nearest values to a point. - auto const nearbyIds = GetStorage().GetNearestObjects(MercatorBounds::ToLatLon(fb.GetKeyPoint())); + auto const nearbyIds = m_storage.GetNearestObjects(MercatorBounds::ToLatLon(fb.GetKeyPoint())); for (auto const objId : nearbyIds) { - if (sponsored_scoring::Match(GetStorage().GetObjectById(objId), fb).IsMatched()) + if (sponsored_scoring::Match(m_storage.GetObjectById(objId), fb).IsMatched()) return objId; } diff --git a/generator/osm_source.cpp b/generator/osm_source.cpp index e569ae50b8..90c67357c8 100644 --- a/generator/osm_source.cpp +++ b/generator/osm_source.cpp @@ -309,7 +309,7 @@ public: , m_failOnCoasts(info.m_failOnCoasts) , m_bookingDataset(info.m_bookingDatafileName, info.m_bookingReferenceDir) , m_opentableDataset(info.m_opentableDatafileName, info.m_opentableReferenceDir) - , m_viatorDataset(info.m_viatorDatafileName, info.m_viatorReferenceDir) + , m_viatorDataset(info.m_viatorDatafileName) { Classificator const & c = classif(); diff --git a/generator/sponsored_dataset.hpp b/generator/sponsored_dataset.hpp index edd7dc772c..701bf2666c 100644 --- a/generator/sponsored_dataset.hpp +++ b/generator/sponsored_dataset.hpp @@ -1,6 +1,6 @@ #pragma once -#include "generator/sponsored_storage.hpp" +#include "generator/sponsored_object_storage.hpp" #include "base/newtype.hpp" @@ -24,8 +24,6 @@ public: explicit SponsoredDataset(std::string const & dataPath, std::string const & addressReferencePath = std::string()); - explicit SponsoredDataset(std::istream & dataSource, - std::string const & addressReferencePath = std::string()); /// @return true if |fb| satisfies some necessary conditions to match one or serveral /// objects from dataset. @@ -41,8 +39,6 @@ public: private: void InitStorage(); - SponsoredStorage const & GetStorage() const; - SponsoredStorage & GetStorage(); void BuildObject(Object const & object, std::function const & fn) const; @@ -50,7 +46,7 @@ private: /// @return an id of a matched object or kInvalidObjectId on failure. ObjectId FindMatchingObjectIdImpl(FeatureBuilder1 const & fb) const; - SponsoredStorage m_storage; + SponsoredObjectStorage m_storage; }; } // namespace generator diff --git a/generator/sponsored_dataset_inl.hpp b/generator/sponsored_dataset_inl.hpp index c9a8dd612e..f829853772 100644 --- a/generator/sponsored_dataset_inl.hpp +++ b/generator/sponsored_dataset_inl.hpp @@ -45,24 +45,15 @@ SponsoredDataset::SponsoredDataset(std::string const & dataPath : m_storage(kDistanceLimitInMeters, kMaxSelectedElements) { InitStorage(); - GetStorage().LoadData(dataPath, addressReferencePath); -} - -template -SponsoredDataset::SponsoredDataset(std::istream & dataSource, - std::string const & addressReferencePath) - : m_storage(kDistanceLimitInMeters, kMaxSelectedElements) -{ - InitStorage(); - GetStorage().LoadData(dataSource, addressReferencePath); + m_storage.LoadData(dataPath, addressReferencePath); } template void SponsoredDataset::InitStorage() { - using Container = typename SponsoredStorage::ObjectsContainer; + using Container = typename SponsoredObjectStorage::ObjectsContainer; - m_storage.SetFillObject([](Container & objects) { + m_storage.SetFillObjects([](Container & objects) { AddressMatcher addressMatcher; size_t matchedCount = 0; @@ -78,27 +69,15 @@ void SponsoredDataset::InitStorage() ++matchedCount; } - LOG(LINFO, ("Num of objects:", objects.size(), "matched:", matchedCount, "empty addresses:", - emptyCount)); + LOG(LINFO, ("Num of objects:", objects.size(), "matched:", matchedCount, + "empty addresses:", emptyCount)); }); } -template -SponsoredStorage const & SponsoredDataset::GetStorage() const -{ - return m_storage; -} - -template -SponsoredStorage & SponsoredDataset::GetStorage() -{ - return m_storage; -} - template void SponsoredDataset::BuildOsmObjects(function const & fn) const { - for (auto const & item : GetStorage().GetObjects()) + for (auto const & item : m_storage.GetObjects()) BuildObject(item.second, fn); } diff --git a/generator/sponsored_storage.hpp b/generator/sponsored_object_storage.hpp similarity index 93% rename from generator/sponsored_storage.hpp rename to generator/sponsored_object_storage.hpp index ad51174ff1..50d0eb3568 100644 --- a/generator/sponsored_storage.hpp +++ b/generator/sponsored_object_storage.hpp @@ -21,18 +21,16 @@ namespace generator { template -class SponsoredStorage +class SponsoredObjectStorage { public: using ObjectId = typename Object::ObjectId; using ObjectsContainer = std::map; using FillObject = std::function; - SponsoredStorage(double distanceLimitMeters, size_t maxSelectedElements, - FillObject const & fn = {}) + SponsoredObjectStorage(double distanceLimitMeters, size_t maxSelectedElements) : m_distanceLimitMeters(distanceLimitMeters) , m_maxSelectedElements(maxSelectedElements) - , m_fillObject(fn) { } @@ -56,7 +54,7 @@ public: return m_objects.size(); } - void SetFillObject(FillObject const & fn) + void SetFillObjects(FillObject const & fn) { m_fillObject = fn; } @@ -95,7 +93,7 @@ public: std::string const backupPath = platform.WritableDir(); // MWMs can be loaded only from a writebledir or from a resourcedir, - // changig resourcedir can lead to probles with classificator, so + // changig resourcedir can lead to problems with classificator, so // we change writebledir. platform.SetWritableDirForTests(addressReferencePath); diff --git a/generator/viator_dataset.cpp b/generator/viator_dataset.cpp index 64264fd874..96230a71de 100644 --- a/generator/viator_dataset.cpp +++ b/generator/viator_dataset.cpp @@ -36,7 +36,7 @@ ViatorCity::ViatorCity(std::string const & src) vector rec; strings::ParseCSVRow(src, '\t', rec); CHECK_EQUAL(rec.size(), FieldsCount(), - ("Error parsing viator cities line:", boost::replace_all_copy(src, "\t", "\\t"))); + ("Error parsing viator cities, line:", boost::replace_all_copy(src, "\t", "\\t"))); CHECK(strings::to_uint(rec[FieldIndex(TsvFields::Id)], m_id.Get()), ()); CHECK(strings::to_double(rec[FieldIndex(TsvFields::Latitude)], m_latLon.lat), ()); @@ -53,13 +53,13 @@ ostream & operator<<(ostream & s, ViatorCity const & h) } // ViatorDataset ---------------------------------------------------------------------------------- -ViatorDataset::ViatorDataset(std::string const & dataPath, std::string const & addressReferencePath) +ViatorDataset::ViatorDataset(std::string const & dataPath) : m_storage(3000.0 /* distanceLimitMeters */, 3 /* maxSelectedElements */) { LoadIndex(m_index); m_cityFinder = make_unique(m_index); - m_storage.LoadData(dataPath, addressReferencePath); + m_storage.LoadData(dataPath, ""); } ViatorCity::ObjectId ViatorDataset::FindMatchingObjectId(FeatureBuilder1 const & fb) const @@ -97,7 +97,7 @@ void ViatorDataset::PreprocessMatchedOsmObject(ViatorCity::ObjectId const matche { FeatureParams params = fb.GetParams(); - auto city = m_storage.GetObjectById(matchedObjId); + auto const & city = m_storage.GetObjectById(matchedObjId); auto & metadata = params.GetMetadata(); metadata.Set(feature::Metadata::FMD_SPONSORED_ID, strings::to_string(city.m_id.Get())); diff --git a/generator/viator_dataset.hpp b/generator/viator_dataset.hpp index d5af5980d8..fa5d760392 100644 --- a/generator/viator_dataset.hpp +++ b/generator/viator_dataset.hpp @@ -1,6 +1,6 @@ #pragma once -#include "generator/sponsored_storage.hpp" +#include "generator/sponsored_object_storage.hpp" #include "search/city_finder.hpp" @@ -14,6 +14,8 @@ #include #include +class FeatureBuilder1; + namespace generator { struct ViatorCity @@ -39,7 +41,7 @@ NEWTYPE_SIMPLE_OUTPUT(ViatorCity::ObjectId); class ViatorDataset { public: - ViatorDataset(std::string const & dataPath, std::string const & addressReferencePath); + ViatorDataset(std::string const & dataPath); ViatorCity::ObjectId FindMatchingObjectId(FeatureBuilder1 const & fb) const; @@ -47,7 +49,7 @@ public: function const fn) const; private: - SponsoredStorage m_storage; + SponsoredObjectStorage m_storage; Index m_index; std::unique_ptr m_cityFinder; }; diff --git a/tools/python/viator_cities.py b/tools/python/viator_cities.py index 36c4423b11..c6818f1cef 100644 --- a/tools/python/viator_cities.py +++ b/tools/python/viator_cities.py @@ -15,8 +15,7 @@ class ViatorApi(object): def get_locations(self): url = 'http://viatorapi.viator.com/service/taxonomy/locations?apiKey=' + self.apikey - request = urllib2.Request(url) - stream = urllib2.urlopen(request) + stream = urllib2.urlopen(url) payload = stream.read() locations = json.loads(payload) return locations @@ -31,8 +30,9 @@ def check_errors(locations): def save_cities(locations, output_file_name): with open(output_file_name, 'w') as output_file: for l in locations['data']: - if l['destinationType'] == 'CITY' and l['destinationId'] and \ - l['destinationName'] and l['latitude'] and l['longitude']: + is_object_supported = (l['destinationType'] == 'CITY' and l['destinationId'] and + l['destinationName'] and l['latitude'] and l['longitude']) + if is_object_supported: city = '\t'.join([ str(l['destinationId']), l['destinationName'], diff --git a/xcode/generator/generator.xcodeproj/project.pbxproj b/xcode/generator/generator.xcodeproj/project.pbxproj index e0e6023dd6..d8cdc64109 100644 --- a/xcode/generator/generator.xcodeproj/project.pbxproj +++ b/xcode/generator/generator.xcodeproj/project.pbxproj @@ -28,11 +28,11 @@ 3D51BC571D5E512500F1FA8D /* region_meta.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 3D51BC4F1D5E512500F1FA8D /* region_meta.hpp */; }; 3D51BC581D5E512500F1FA8D /* srtm_parser.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 3D51BC501D5E512500F1FA8D /* srtm_parser.cpp */; }; 3D51BC591D5E512500F1FA8D /* srtm_parser.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 3D51BC511D5E512500F1FA8D /* srtm_parser.hpp */; }; - 3DFEBF7C1EF2D58900317D5C /* sponsored_storage.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 3DFEBF771EF2D58900317D5C /* sponsored_storage.hpp */; }; 3DFEBF7D1EF2D58900317D5C /* utils.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 3DFEBF781EF2D58900317D5C /* utils.cpp */; }; 3DFEBF7E1EF2D58900317D5C /* utils.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 3DFEBF791EF2D58900317D5C /* utils.hpp */; }; 3DFEBF7F1EF2D58900317D5C /* viator_dataset.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 3DFEBF7A1EF2D58900317D5C /* viator_dataset.cpp */; }; 3DFEBF801EF2D58900317D5C /* viator_dataset.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 3DFEBF7B1EF2D58900317D5C /* viator_dataset.hpp */; }; + 3DFEBF821EF423FB00317D5C /* sponsored_object_storage.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 3DFEBF811EF423FB00317D5C /* sponsored_object_storage.hpp */; }; 670B84BC1A8CDB0000CE4492 /* osm_source.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 670B84BA1A8CDB0000CE4492 /* osm_source.cpp */; }; 670B84BD1A8CDB0000CE4492 /* osm_source.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 670B84BB1A8CDB0000CE4492 /* osm_source.hpp */; }; 6726C1D51A4AFEF4005EEA39 /* osm2meta.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 6726C1D31A4AFEF4005EEA39 /* osm2meta.cpp */; }; @@ -124,11 +124,11 @@ 3D51BC4F1D5E512500F1FA8D /* region_meta.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = region_meta.hpp; sourceTree = ""; }; 3D51BC501D5E512500F1FA8D /* srtm_parser.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = srtm_parser.cpp; sourceTree = ""; }; 3D51BC511D5E512500F1FA8D /* srtm_parser.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = srtm_parser.hpp; sourceTree = ""; }; - 3DFEBF771EF2D58900317D5C /* sponsored_storage.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = sponsored_storage.hpp; sourceTree = ""; }; 3DFEBF781EF2D58900317D5C /* utils.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = utils.cpp; sourceTree = ""; }; 3DFEBF791EF2D58900317D5C /* utils.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = utils.hpp; sourceTree = ""; }; 3DFEBF7A1EF2D58900317D5C /* viator_dataset.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = viator_dataset.cpp; sourceTree = ""; }; 3DFEBF7B1EF2D58900317D5C /* viator_dataset.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = viator_dataset.hpp; sourceTree = ""; }; + 3DFEBF811EF423FB00317D5C /* sponsored_object_storage.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = sponsored_object_storage.hpp; sourceTree = ""; }; 670B84BA1A8CDB0000CE4492 /* osm_source.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = osm_source.cpp; sourceTree = ""; }; 670B84BB1A8CDB0000CE4492 /* osm_source.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = osm_source.hpp; sourceTree = ""; }; 6726C1D31A4AFEF4005EEA39 /* osm2meta.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = osm2meta.cpp; sourceTree = ""; }; @@ -241,7 +241,7 @@ 6753401D1A3F2A1B00A0A8C3 /* generator */ = { isa = PBXGroup; children = ( - 3DFEBF771EF2D58900317D5C /* sponsored_storage.hpp */, + 3DFEBF811EF423FB00317D5C /* sponsored_object_storage.hpp */, 3DFEBF781EF2D58900317D5C /* utils.cpp */, 3DFEBF791EF2D58900317D5C /* utils.hpp */, 3DFEBF7A1EF2D58900317D5C /* viator_dataset.cpp */, @@ -379,7 +379,6 @@ 34F5588B1DBF4C9600A4FC11 /* sponsored_dataset.hpp in Headers */, 6753405F1A3F2A7400A0A8C3 /* borders_loader.hpp in Headers */, 675340801A3F2A7400A0A8C3 /* polygonizer.hpp in Headers */, - 3DFEBF7C1EF2D58900317D5C /* sponsored_storage.hpp in Headers */, 0C5FEC711DDE19E50017688C /* routing_index_generator.hpp in Headers */, 67C79BB01E2CEEAB00C40034 /* restriction_collector.hpp in Headers */, 3DFEBF801EF2D58900317D5C /* viator_dataset.hpp in Headers */, @@ -388,6 +387,7 @@ 677E2A181CAACC5F001DC42A /* towns_dumper.hpp in Headers */, 675340821A3F2A7400A0A8C3 /* routing_generator.hpp in Headers */, 675340841A3F2A7400A0A8C3 /* statistics.hpp in Headers */, + 3DFEBF821EF423FB00317D5C /* sponsored_object_storage.hpp in Headers */, 6753406D1A3F2A7400A0A8C3 /* feature_generator.hpp in Headers */, 675340681A3F2A7400A0A8C3 /* dumper.hpp in Headers */, 675340791A3F2A7400A0A8C3 /* osm_translator.hpp in Headers */,