organicmaps/geocore
13
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity Actions

Clean unused parts from generator

Browse source
This commit is contained in:
l.fedorov 2019-10-17 16:42:05 +00:00 • committed by cc-engineering
parent 941c12ee66
commit ffcbd5ccdb
33 changed files with 0 additions and 2182 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
generator/CMakeLists.txt
View file

@ -5,13 +5,7 @@ set(
affiliation.cpp
affiliation.hpp
boost_helpers.hpp
check_model.cpp
check_model.hpp
collection_base.hpp
collector_addresses.cpp
collector_addresses.hpp
collector_city_area.cpp
collector_city_area.hpp
collector_collection.cpp
collector_collection.hpp
collector_interface.hpp
@ -40,11 +34,7 @@ set(
features_processing_helpers.hpp
filter_collection.cpp
filter_collection.hpp
filter_elements.cpp
filter_elements.hpp
filter_interface.hpp
filter_planet.cpp
filter_planet.hpp
filter_world.cpp
filter_world.hpp
final_processor_intermediate_mwm.cpp
@ -69,10 +59,6 @@ set(
key_value_storage.hpp
locality_sorter.cpp
locality_sorter.hpp
metalines_builder.cpp
metalines_builder.hpp
node_mixer.cpp
node_mixer.hpp
osm2meta.cpp
osm2meta.hpp
osm2type.cpp
@ -84,21 +70,16 @@ set(
osm_o5m_source.hpp
osm_source.cpp
osm_xml_source.hpp
place.cpp
place.hpp
place_node.hpp
processor_factory.hpp
processor_interface.hpp
processor_noop.hpp
processor_simple.cpp
processor_simple.hpp
promo_catalog_cities.hpp
raw_generator.cpp
raw_generator.hpp
raw_generator_writer.cpp
raw_generator_writer.hpp
region_meta.cpp
region_meta.hpp
regions/admin_suburbs_marker.cpp
regions/admin_suburbs_marker.hpp
regions/collector_region_info.cpp
@ -131,8 +112,6 @@ set(
relation_tags.hpp
relation_tags_enricher.cpp
relation_tags_enricher.hpp
routing_helpers.cpp
routing_helpers.hpp
statistics.cpp
statistics.hpp
streets/street_geometry.cpp

45
generator/check_model.cpp
View file

@ -1,45 +0,0 @@
#include "generator/check_model.hpp"
#include "indexer/features_vector.hpp"
#include "indexer/classificator.hpp"
#include "indexer/feature_visibility.hpp"
#include "base/logging.hpp"
#include <vector>
using namespace feature;
namespace check_model
{
void ReadFeatures(std::string const & fName)
{
Classificator const & c = classif();
FeaturesVectorTest(fName).GetVector().ForEach([&](FeatureType & ft, uint32_t) {
TypesHolder types(ft);
std::vector<uint32_t> vTypes;
for (uint32_t t : types)
{
CHECK_EQUAL(c.GetTypeForIndex(c.GetIndexForType(t)), t, ());
vTypes.push_back(t);
}
sort(vTypes.begin(), vTypes.end());
CHECK(unique(vTypes.begin(), vTypes.end()) == vTypes.end(), ());
m2::RectD const r = ft.GetLimitRect(FeatureType::BEST_GEOMETRY);
CHECK(r.IsValid(), ());
GeomType const type = ft.GetGeomType();
if (type == GeomType::Line)
CHECK_GREATER(ft.GetPointsCount(), 1, ());
IsDrawableLike(vTypes, ft.GetGeomType());
});
LOG(LINFO, ("OK"));
}
}

7
generator/check_model.hpp
View file

@ -1,7 +0,0 @@
#pragma once
#include <string>
namespace check_model
{
void ReadFeatures(std::string const & fName);
}

62
generator/collector_addresses.cpp
View file

@ -1,62 +0,0 @@
#include "generator/collector_addresses.hpp"
#include "generator/feature_builder.hpp"
#include "generator/intermediate_data.hpp"
#include "indexer/ftypes_matcher.hpp"
#include "platform/platform.hpp"
#include "coding/internal/file_data.hpp"
#include "base/assert.hpp"
using namespace feature;
namespace generator
{
CollectorAddresses::CollectorAddresses(std::string const & filename)
: CollectorInterface(filename)
{
m_stream.exceptions(std::fstream::failbit | std::fstream::badbit);
m_stream.open(GetTmpFilename());
}
std::shared_ptr<CollectorInterface>
CollectorAddresses::Clone(std::shared_ptr<cache::IntermediateDataReader> const &) const
{
return std::make_shared<CollectorAddresses>(GetFilename());
}
void CollectorAddresses::CollectFeature(feature::FeatureBuilder const & feature, OsmElement const &)
{
std::string addr;
auto const & checker = ftypes::IsBuildingChecker::Instance();
if (checker(feature.GetTypes()) && feature.FormatFullAddress(addr))
m_stream << addr << "\n";
}
void CollectorAddresses::Finish()
{
if (m_stream.is_open())
m_stream.close();
}
void CollectorAddresses::Save()
{
CHECK(!m_stream.is_open(), ("Finish() has not been called."));
if (Platform::IsFileExistsByFullPath(GetTmpFilename()))
CHECK(base::CopyFileX(GetTmpFilename(), GetFilename()), ());
}
void CollectorAddresses::Merge(CollectorInterface const & collector)
{
collector.MergeInto(*this);
}
void CollectorAddresses::MergeInto(CollectorAddresses & collector) const
{
CHECK(!m_stream.is_open() || !collector.m_stream.is_open(), ("Finish() has not been called."));
base::AppendFileToFile(GetTmpFilename(), collector.GetTmpFilename());
}
} // namespace generator

35
generator/collector_addresses.hpp
View file

@ -1,35 +0,0 @@
#pragma once
#include "generator/collector_interface.hpp"
#include <fstream>
#include <memory>
#include <string>
namespace generator
{
namespace cache
{
class IntermediateDataReader;
}
// The class CollectorAddresses is responsible for the collecting addresses to the file.
class CollectorAddresses : public CollectorInterface
{
public:
explicit CollectorAddresses(std::string const & filename);
// CollectorInterface overrides:
std::shared_ptr<CollectorInterface>
Clone(std::shared_ptr<cache::IntermediateDataReader> const & = {}) const override;
void CollectFeature(feature::FeatureBuilder const & feature, OsmElement const &) override;
void Finish() override;
void Save() override;
void Merge(CollectorInterface const & collector) override;
void MergeInto(CollectorAddresses & collector) const override;
private:
std::ofstream m_stream;
};
} // namespace generator

62
generator/collector_city_area.cpp
View file

@ -1,62 +0,0 @@
#include "generator/collector_city_area.hpp"
#include "generator/intermediate_data.hpp"
#include "indexer/ftypes_matcher.hpp"
#include "platform/platform.hpp"
#include "coding/internal/file_data.hpp"
#include "base/assert.hpp"
#include <iterator>
using namespace feature;
using namespace feature::serialization_policy;
namespace generator
{
CityAreaCollector::CityAreaCollector(std::string const & filename)
: CollectorInterface(filename),
m_writer(std::make_unique<FeatureBuilderWriter<MaxAccuracy>>(GetTmpFilename())) {}
std::shared_ptr<CollectorInterface>
CityAreaCollector::Clone(std::shared_ptr<cache::IntermediateDataReader> const &) const
{
return std::make_shared<CityAreaCollector>(GetFilename());
}
void CityAreaCollector::CollectFeature(FeatureBuilder const & feature, OsmElement const &)
{
if (!(feature.IsArea() && ftypes::IsCityTownOrVillage(feature.GetTypes())))
return;
auto copy = feature;
if (copy.PreSerialize())
m_writer->Write(copy);
}
void CityAreaCollector::Finish()
{
m_writer.reset({});
}
void CityAreaCollector::Save()
{
CHECK(!m_writer, ("Finish() has not been called."));
if (Platform::IsFileExistsByFullPath(GetTmpFilename()))
CHECK(base::CopyFileX(GetTmpFilename(), GetFilename()), ());
}
void CityAreaCollector::Merge(generator::CollectorInterface const & collector)
{
collector.MergeInto(*this);
}
void CityAreaCollector::MergeInto(CityAreaCollector & collector) const
{
CHECK(!m_writer || !collector.m_writer, ("Finish() has not been called."));
base::AppendFileToFile(GetTmpFilename(), collector.GetTmpFilename());
}
} // namespace generator

34
generator/collector_city_area.hpp
View file

@ -1,34 +0,0 @@
#pragma once
#include "generator/collector_interface.hpp"
#include "generator/feature_builder.hpp"
#include <memory>
namespace generator
{
namespace cache
{
class IntermediateDataReader;
} // namespace cache
class CityAreaCollector : public CollectorInterface
{
public:
explicit CityAreaCollector(std::string const & filename);
// CollectorInterface overrides:
std::shared_ptr<CollectorInterface>
Clone(std::shared_ptr<cache::IntermediateDataReader> const & = {}) const override;
void CollectFeature(feature::FeatureBuilder const & feature, OsmElement const &) override;
void Finish() override;
void Save() override;
void Merge(generator::CollectorInterface const & collector) override;
void MergeInto(CityAreaCollector & collector) const override;
private:
std::unique_ptr<feature::FeatureBuilderWriter<feature::serialization_policy::MaxAccuracy>> m_writer;
};
} // namespace generator

39
generator/emitter_coastline.hpp
View file

@ -1,39 +0,0 @@
#pragma once
#include "generator/emitter_interface.hpp"
#include <memory>
#include <string>
#include <vector>
class CoastlineFeaturesGenerator;
namespace feature
{
struct GenerateInfo;
class FeatureBuilder;
} // namespace feature
namespace generator
{
class PlaceProcessor;
class CountryMapper;
class LayerBase;
// This class is implementation of EmitterInterface for coastlines.
class EmitterCoastline : public EmitterInterface
{
public:
explicit EmitterCoastline(feature::GenerateInfo const & info);
// EmitterInterface overrides:
void Process(feature::FeatureBuilder & feature) override;
bool Finish() override;
void GetNames(std::vector<std::string> & names) const override;
private:
std::shared_ptr<PlaceProcessor> m_placeProcessor;
std::shared_ptr<CoastlineFeaturesGenerator> m_generator;
std::shared_ptr<LayerBase> m_processingChain;
std::string m_coastlineGeomFilename;
std::string m_coastlineRawGeomFilename;
};
} // namespace generator

66
generator/emitter_country.cpp
View file

@ -1,66 +0,0 @@
#include "generator/emitter_country.hpp"
#include "generator/feature_builder.hpp"
#include "generator/feature_processing_layers.hpp"
#include "generator/generate_info.hpp"
#include "generator/place_processor.hpp"
#include "base/logging.hpp"
#include <fstream>
#include "defines.hpp"
using namespace feature;
namespace generator
{
EmitterCountry::EmitterCountry(feature::GenerateInfo const & info)
: m_placeProcessor(std::make_shared<PlaceProcessor>(info.m_boundariesTable))
, m_countryMapper(std::make_shared<CountryMapper>(info))
, m_skippedListFilename(info.GetIntermediateFileName("skipped_elements", ".lst"))
{
m_processingChain = std::make_shared<RepresentationLayer>(m_placeProcessor);
m_processingChain->Add(std::make_shared<PrepareFeatureLayer>());
m_processingChain->Add(std::make_shared<PromoCatalogLayer>(info.m_promoCatalogCitiesFilename));
m_processingChain->Add(std::make_shared<PlaceLayer>(m_placeProcessor));
m_processingChain->Add(std::make_shared<BookingLayer>(info.m_bookingDataFilename, m_countryMapper));
m_processingChain->Add(std::make_shared<OpentableLayer>(info.m_opentableDataFilename, m_countryMapper));
m_processingChain->Add(std::make_shared<CountryMapperLayer>(m_countryMapper));
if (info.m_emitCoasts)
{
auto const geomFilename = info.GetIntermediateFileName(WORLD_COASTS_FILE_NAME, ".geom");
auto const worldCoastsFilename = info.GetTmpFileName(WORLD_COASTS_FILE_NAME);
m_processingChain->Add(std::make_shared<EmitCoastsLayer>(worldCoastsFilename, geomFilename, m_countryMapper));
}
}
void EmitterCountry::Process(FeatureBuilder & feature)
{
m_processingChain->Handle(feature);
}
bool EmitterCountry::Finish()
{
for (auto & feature : m_placeProcessor->GetFeatures())
m_countryMapper->RemoveInvalidTypesAndMap(feature);
WriteDump();
return true;
}
void EmitterCountry::GetNames(std::vector<std::string> & names) const
{
names = m_countryMapper->GetNames();
}
void EmitterCountry::WriteDump()
{
std::ofstream file;
file.exceptions(std::ios::failbit | std::ios::badbit);
file.open(m_skippedListFilename);
file << m_processingChain->GetAsStringRecursive();
LOG(LINFO, ("Skipped elements were saved to", m_skippedListFilename));
}
} // namespace generator

39
generator/emitter_country.hpp
View file

@ -1,39 +0,0 @@
#pragma once
#include "generator/emitter_interface.hpp"
#include <memory>
#include <string>
#include <vector>
namespace feature
{
class FeatureBuilder;
struct GenerateInfo;
} // namespace feature
namespace generator
{
class PlaceProcessor;
class CountryMapper;
class LayerBase;
// This class is the implementation of EmitterInterface for countries.
class EmitterCountry : public EmitterInterface
{
public:
explicit EmitterCountry(feature::GenerateInfo const & info);
// EmitterInterface overrides:
void Process(feature::FeatureBuilder & feature) override;
bool Finish() override;
void GetNames(std::vector<std::string> & names) const override;
private:
void WriteDump();
std::shared_ptr<PlaceProcessor> m_placeProcessor;
std::shared_ptr<CountryMapper> m_countryMapper;
std::string m_skippedListFilename;
std::shared_ptr<LayerBase> m_processingChain;
};
} // namespace generator

44
generator/emitter_world.cpp
View file

@ -1,44 +0,0 @@
#include "generator/emitter_world.hpp"
#include "generator/cities_boundaries_builder.hpp"
#include "generator/feature_builder.hpp"
#include "generator/feature_processing_layers.hpp"
#include "generator/generate_info.hpp"
#include "generator/place_processor.hpp"
#include "defines.hpp"
using namespace feature;
namespace generator
{
EmitterWorld::EmitterWorld(feature::GenerateInfo const & info)
: m_placeProcessor(std::make_shared<PlaceProcessor>(
std::make_shared<generator::OsmIdToBoundariesTable>()))
, m_worldMapper(std::make_shared<WorldMapper>(
info.GetTmpFileName(WORLD_FILE_NAME),
info.GetIntermediateFileName(WORLD_COASTS_FILE_NAME, RAW_GEOM_FILE_EXTENSION),
info.m_popularPlacesFilename))
{
m_processingChain = std::make_shared<RepresentationLayer>(m_placeProcessor);
m_processingChain->Add(std::make_shared<PrepareFeatureLayer>());
m_processingChain->Add(std::make_shared<PromoCatalogLayer>(info.m_promoCatalogCitiesFilename));
m_processingChain->Add(std::make_shared<PlaceLayer>(m_placeProcessor));
m_processingChain->Add(std::make_shared<WorldAreaLayer>(m_worldMapper));
}
void EmitterWorld::Process(FeatureBuilder & feature)
{
m_processingChain->Handle(feature);
}
bool EmitterWorld::Finish()
{
for (auto & feature : m_placeProcessor->GetFeatures())
m_worldMapper->RemoveInvalidTypesAndMap(feature);
return true;
}
void EmitterWorld::GetNames(vector<string> &) const {}
} // namespace generator

37
generator/emitter_world.hpp
View file

@ -1,37 +0,0 @@
#pragma once
#include "generator/emitter_interface.hpp"
#include <memory>
#include <string>
#include <vector>
namespace feature
{
class FeatureBuilder;
struct GenerateInfo;
} // namespace feature
namespace generator
{
class PlaceProcessor;
class WorldMapper;
class LayerBase;
// This class is implementation of EmitterInterface for the world.
class EmitterWorld : public EmitterInterface
{
public:
explicit EmitterWorld(feature::GenerateInfo const & info);
// EmitterInterface overrides:
void Process(feature::FeatureBuilder & feature) override;
bool Finish() override;
void GetNames(std::vector<std::string> & names) const override;
private:
std::shared_ptr<PlaceProcessor> m_placeProcessor;
std::shared_ptr<WorldMapper> m_worldMapper;
std::shared_ptr<LayerBase> m_processingChain;
};
} // namespace generator

1
generator/feature_processing_layers.hpp
View file

@ -6,7 +6,6 @@
#include "generator/features_processing_helpers.hpp"
#include "generator/filter_world.hpp"
#include "generator/processor_interface.hpp"
#include "generator/promo_catalog_cities.hpp"
#include "generator/world_map_generator.hpp"
#include <memory>

214
generator/filter_elements.cpp
View file

@ -1,214 +0,0 @@
#include "generator/filter_elements.hpp"
#include "base/logging.hpp"
#include <algorithm>
#include <fstream>
#include <iterator>
namespace
{
template <typename T>
class Set
{
public:
bool Add(T const & key)
{
if (Contains(key))
return false;
m_vec.push_back(key);
return true;
}
bool Contains(T const & key) const
{
return std::find(std::begin(m_vec), std::end(m_vec), key) != std::end(m_vec);
}
private:
std::vector<T> m_vec;
};
} // namespace
namespace generator
{
bool FilterData::IsMatch(Tags const & elementTags, Tags const & tags)
{
auto const fn = [&](OsmElement::Tag const & t)
{
auto const pred = [&](OsmElement::Tag const & tag) { return tag.m_key == t.m_key; };
auto const it = std::find_if(std::begin(elementTags), std::end(elementTags), pred);
return it == std::end(elementTags) ? false : t.m_value == "*" || it->m_value == t.m_value;
};
return std::all_of(std::begin(tags), std::end(tags), fn);
}
void FilterData::AddSkippedId(uint64_t id)
{
m_skippedIds.insert(id);
}
void FilterData::AddSkippedTags(Tags const & tags)
{
m_rulesStorage.push_back(tags);
for (auto const & t : tags)
m_skippedTags.emplace(t.m_key, m_rulesStorage.back());
}
bool FilterData::NeedSkipWithId(uint64_t id) const
{
return m_skippedIds.find(id) != std::end(m_skippedIds);
}
bool FilterData::NeedSkipWithTags(Tags const & tags) const
{
Set<Tags const *> s;
for (auto const & tag : tags)
{
auto const t = m_skippedTags.equal_range(tag.m_key);
for (auto it = t.first; it != t.second; ++it)
{
Tags const & t = it->second;
if (s.Add(&t) && IsMatch(tags, t))
return true;
}
}
return false;
}
bool FilterElements::ParseSection(json_t * json, FilterData & fdata)
{
if (!json_is_object(json))
return false;
char const * key = nullptr;
json_t * value = nullptr;
json_object_foreach(json, key, value)
{
if (std::strcmp("ids", key) == 0 && !ParseIds(value, fdata))
return false;
else if (std::strcmp("tags", key) == 0 && !ParseTags(value, fdata))
return false;
return true;
}
return true;
}
bool FilterElements::ParseIds(json_t * json, FilterData & fdata)
{
if (!json_is_array(json))
return false;
size_t const sz = json_array_size(json);
for (size_t i = 0; i < sz; ++i)
{
auto const * o = json_array_get(json, i);
if (!json_is_integer(o))
return false;
auto const val = json_integer_value(o);
if (val < 0)
return false;
fdata.AddSkippedId(static_cast<uint64_t>(val));
}
return true;
}
bool FilterElements::ParseTags(json_t * json, FilterData & fdata)
{
if (!json_is_array(json))
return false;
size_t const sz = json_array_size(json);
for (size_t i = 0; i < sz; ++i)
{
auto * o = json_array_get(json, i);
if (!json_is_object(o))
return false;
char const * key = nullptr;
json_t * value = nullptr;
FilterData::Tags tags;
json_object_foreach(o, key, value)
{
if (!json_is_string(value))
return false;
auto const val = json_string_value(value);
tags.emplace_back(key, val);
}
if (!tags.empty())
fdata.AddSkippedTags(tags);
}
return true;
}
FilterElements::FilterElements(std::string const & filename)
: m_filename(filename)
{
std::ifstream stream(m_filename);
std::string str((std::istreambuf_iterator<char>(stream)), std::istreambuf_iterator<char>());
if (!ParseString(str))
LOG(LERROR, ("Cannot parse file", m_filename));
}
std::shared_ptr<FilterInterface> FilterElements::Clone() const
{
return std::make_shared<FilterElements>(m_filename);
}
bool FilterElements::IsAccepted(OsmElement const & element)
{
return !NeedSkip(element);
}
bool FilterElements::NeedSkip(OsmElement const & element) const
{
switch (element.m_type)
{
case OsmElement::EntityType::Node: return NeedSkip(element, m_nodes);
case OsmElement::EntityType::Way: return NeedSkip(element, m_ways);
case OsmElement::EntityType::Relation: return NeedSkip(element, m_relations);
default: return false;
}
}
bool FilterElements::NeedSkip(OsmElement const & element, FilterData const & fdata) const
{
return fdata.NeedSkipWithId(element.m_id) || fdata.NeedSkipWithTags(element.Tags());
}
bool FilterElements::ParseString(std::string const & str)
{
base::Json json(str);
if (!json_is_object(json.get()))
return false;
char const * key = nullptr;
json_t * value = nullptr;
json_object_foreach(json.get(), key, value)
{
bool res = true;
if (std::strcmp("node", key) == 0)
res = ParseSection(value, m_nodes);
else if (std::strcmp("way", key) == 0)
res = ParseSection(value, m_ways);
else if (std::strcmp("relation", key) == 0)
res = ParseSection(value, m_relations);
if (!res)
return false;
}
return true;
}
} // namespace generator

99
generator/filter_elements.hpp
View file

@ -1,99 +0,0 @@
// This file contains the FilterData and FilterElements classes.
// With the help of them, a mechanism is implemented to skip elements from processing according to
// the rules from the configuration file.
// The configuration file is in json format.
// File example:
// {
// "node": {
// "ids": [
// 1435,
// 436
// ],
// "tags": [
// {
// "place": "city"
// },
// {
// "place": "town",
// "capital": "*"
// }
// ]
// },
// "way": {},
// "relation": {}
// }
// This means that we will skip node element if one of the following is true:
// 1. its id equals 1435
// 2. its id equals 436
// 3. if it contains {place:city} in tags
// 4. if it contains {place:city} and {capital:*} in tags.
// '*' - means any value.
// Record format for way and relation is the same as for node.
// This implementation does not support processing of multiple values
// (https://wiki.openstreetmap.org/wiki/Multiple_values).
#pragma once
#include "generator/filter_interface.hpp"
#include "generator/osm_element.hpp"
#include <cstdint>
#include <functional>
#include <list>
#include <memory>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <vector>
#include "3party/jansson/myjansson.hpp"
namespace generator
{
// This is a helper class for the FilterElements class.
// It works with identifiers and tags at a low level.
class FilterData
{
public:
using Tags = std::vector<OsmElement::Tag>;
void AddSkippedId(uint64_t id);
void AddSkippedTags(Tags const & tags);
bool NeedSkipWithId(uint64_t id) const;
bool NeedSkipWithTags(Tags const & tags) const;
private:
static bool IsMatch(Tags const & elementTags, Tags const & tags);
std::unordered_set<uint64_t> m_skippedIds;
std::unordered_multimap<std::string, std::reference_wrapper<Tags const>> m_skippedTags;
std::list<Tags> m_rulesStorage;
};
// This is the main class that implements the element skipping mechanism.
class FilterElements : public FilterInterface
{
public:
explicit FilterElements(const std::string & filename);
// FilterInterface overrides:
std::shared_ptr<FilterInterface> Clone() const override;
bool IsAccepted(OsmElement const & element) override;
bool NeedSkip(OsmElement const & element) const;
private:
static bool ParseSection(json_t * json, FilterData & fdata);
static bool ParseIds(json_t * json, FilterData & fdata);
static bool ParseTags(json_t * json, FilterData & fdata);
bool NeedSkip(OsmElement const & element, FilterData const & fdata) const;
bool ParseString(std::string const & str);
std::string m_filename;
FilterData m_nodes;
FilterData m_ways;
FilterData m_relations;
};
} // namespace generator

38
generator/filter_planet.cpp
View file

@ -1,38 +0,0 @@
#include "generator/filter_planet.hpp"
#include "generator/feature_builder.hpp"
#include "generator/osm_element.hpp"
#include "indexer/classificator.hpp"
#include "indexer/feature_visibility.hpp"
#include "indexer/ftypes_matcher.hpp"
#include "base/assert.hpp"
#include <algorithm>
using namespace feature;
namespace generator
{
std::shared_ptr<FilterInterface> FilterPlanet::Clone() const
{
return std::make_shared<FilterPlanet>();
}
bool FilterPlanet::IsAccepted(OsmElement const & element)
{
if (element.IsRelation())
return element.HasAnyTag({{"type", "multipolygon"}, {"type", "boundary"}});
if (element.IsNode())
return !element.m_tags.empty();
return true;
}
bool FilterPlanet::IsAccepted(FeatureBuilder const & feature)
{
auto const & params = feature.GetParams();
return params.IsValid();
}
} // namespace generator

16
generator/filter_planet.hpp
View file

@ -1,16 +0,0 @@
#pragma once
#include "generator/filter_interface.hpp"
namespace generator
{
class FilterPlanet : public FilterInterface
{
public:
// FilterInterface overrides:
std::shared_ptr<FilterInterface> Clone() const override;
bool IsAccepted(OsmElement const & element) override;
bool IsAccepted(feature::FeatureBuilder const & feature) override;
};
} // namespace generator

4
generator/generator_tests/CMakeLists.txt
View file

@ -3,18 +3,14 @@ project(generator_tests)
set(
SRC
coasts_test.cpp
collector_city_area_tests.cpp
common.cpp
common.hpp
feature_builder_test.cpp
feature_merger_test.cpp
filter_elements_tests.cpp
geo_objects_tests.cpp
intermediate_data_test.cpp
merge_collectors_tests.cpp
metadata_parser_test.cpp
metalines_tests.cpp
node_mixer_test.cpp
osm2meta_test.cpp
osm_o5m_source_test.cpp
osm_type_test.cpp

71
generator/generator_tests/collector_city_area_tests.cpp
View file

@ -1,71 +0,0 @@
#include "testing/testing.hpp"
#include "generator/collector_city_area.hpp"
#include "generator/feature_builder.hpp"
#include "generator/generator_tests/common.hpp"
#include "generator/osm2type.hpp"
#include "generator/osm_element.hpp"
#include "indexer/classificator_loader.hpp"
#include "platform/platform.hpp"
#include "geometry/point2d.hpp"
#include "base/geo_object_id.hpp"
#include "base/scope_guard.hpp"
#include <algorithm>
#include <memory>
#include <vector>
using namespace generator_tests;
namespace
{
feature::FeatureBuilder MakeFbForTest(OsmElement element)
{
feature::FeatureBuilder result;
ftype::GetNameAndType(&element, result.GetParams());
std::vector<m2::PointD> polygon = {{0, 0}, {0, 2}, {2, 2}, {2, 0}, {0, 0}};
result.SetOsmId(base::MakeOsmRelation(element.m_id));
result.AddPolygon(polygon);
result.SetArea();
return result;
}
bool HasRelationWithId(std::vector<feature::FeatureBuilder> const & fbs, uint64_t id) {
return std::find_if(std::begin(fbs), std::end(fbs), [&](auto const & fb) {
return fb.GetMostGenericOsmId() == base::MakeOsmRelation(id);
}) != std::end(fbs);
};
auto const o1 = MakeOsmElement(1 /* id */, {{"place", "city"}} /* tags */, OsmElement::EntityType::Relation);
auto const o2 = MakeOsmElement(2 /* id */, {{"place", "town"}} /* tags */, OsmElement::EntityType::Relation);
auto const o3 = MakeOsmElement(3 /* id */, {{"place", "village"}} /* tags */, OsmElement::EntityType::Relation);
auto const o4 = MakeOsmElement(4 /* id */, {{"place", "country"}} /* tags */, OsmElement::EntityType::Relation);
} // namespace
UNIT_TEST(CollectorCityArea_Merge)
{
classificator::Load();
auto const filename = generator_tests::GetFileName();
SCOPE_GUARD(_, std::bind(Platform::RemoveFileIfExists, std::cref(filename)));
auto c1 = std::make_shared<generator::CityAreaCollector>(filename);
auto c2 = c1->Clone();
c1->CollectFeature(MakeFbForTest(o1), o1);
c2->CollectFeature(MakeFbForTest(o2), o2);
c1->CollectFeature(MakeFbForTest(o3), o3);
c2->CollectFeature(MakeFbForTest(o4), o4);
c1->Finish();
c2->Finish();
c1->Merge(*c2);
c1->Save();
auto const fbs = feature::ReadAllDatRawFormat<feature::serialization_policy::MaxAccuracy>(filename);
TEST_EQUAL(fbs.size(), 3, ());
TEST(HasRelationWithId(fbs, 1 /* id */), ());
TEST(HasRelationWithId(fbs, 2 /* id */), ());
TEST(HasRelationWithId(fbs, 3 /* id */), ());
}

118
generator/generator_tests/filter_elements_tests.cpp
View file

@ -1,118 +0,0 @@
#include "testing/testing.hpp"
#include "generator/filter_elements.hpp"
#include "generator/generator_tests/common.hpp"
#include "generator/osm_element.hpp"
#include "platform/platform_tests_support/scoped_file.hpp"
using namespace generator_tests;
using namespace generator;
using namespace platform::tests_support;
namespace
{
auto const kOsmElementEmpty = MakeOsmElement(0, {}, OsmElement::EntityType::Node);
auto const kOsmElementCity = MakeOsmElement(1, {{"place", "city"},
{"admin_level", "6"}},
OsmElement::EntityType::Node);
auto const kOsmElementCountry = MakeOsmElement(2, {{"admin_level", "2"},
{"ISO3166-1:alpha2", "RU"},
{"ISO3166-1:alpha3", "RUS"},
{"ISO3166-1:numeric", "643"}},
OsmElement::EntityType::Relation);
} // namespace
UNIT_TEST(FilterData_Ids)
{
FilterData fd;
fd.AddSkippedId(kOsmElementEmpty.m_id);
TEST(fd.NeedSkipWithId(kOsmElementEmpty.m_id), ());
TEST(!fd.NeedSkipWithId(kOsmElementCity.m_id), ());
}
UNIT_TEST(FilterData_Tags)
{
FilterData fd;
FilterData::Tags tags{
{"admin_level", "2"}
};
fd.AddSkippedTags(tags);
TEST(fd.NeedSkipWithTags(kOsmElementCountry.Tags()), ());
TEST(!fd.NeedSkipWithTags(kOsmElementCity.Tags()), ());
}
UNIT_TEST(FilterElements_Case1)
{
ScopedFile sf("tmp.txt",
R"(
{
"node": {
"ids": [1]
},
"relation": {
"tags": [{
"ISO3166-1:alpha2": "*"
}]
}
}
)");
FilterElements fe(sf.GetFullPath());
TEST(fe.NeedSkip(kOsmElementCity), ());
TEST(!fe.NeedSkip(kOsmElementEmpty), ());
TEST(fe.NeedSkip(kOsmElementCountry), ());
}
UNIT_TEST(FilterElements_Case2)
{
ScopedFile sf("tmp.txt",
R"(
{
"node": {
"ids": [0, 1]
}
}
)");
FilterElements fe(sf.GetFullPath());
TEST(fe.NeedSkip(kOsmElementCity), ());
TEST(!fe.NeedSkip(kOsmElementCountry), ());
TEST(fe.NeedSkip(kOsmElementEmpty), ());
}
UNIT_TEST(FilterElements_Case3)
{
ScopedFile sf("tmp.txt",
R"(
{
"node": {
"tags": [{
"admin_level": "*"
}]
},
"relation": {
"tags": [{
"admin_level": "*"
}]
}
}
)");
FilterElements fe(sf.GetFullPath());
TEST(fe.NeedSkip(kOsmElementCity), ());
TEST(fe.NeedSkip(kOsmElementCountry), ());
TEST(!fe.NeedSkip(kOsmElementEmpty), ());
}
UNIT_TEST(FilterElements_Case4)
{
ScopedFile sf("tmp.txt",
R"(
{
}
)");
FilterElements fe(sf.GetFullPath());
TEST(!fe.NeedSkip(kOsmElementCity), ());
TEST(!fe.NeedSkip(kOsmElementCountry), ());
TEST(!fe.NeedSkip(kOsmElementEmpty), ());
}

199
generator/generator_tests/metalines_tests.cpp
View file

@ -1,199 +0,0 @@
#include "testing/testing.hpp"
#include "generator/generator_tests/common.hpp"
#include "generator/metalines_builder.hpp"
#include "generator/osm2type.hpp"
#include "generator/osm_element.hpp"
#include "indexer/classificator_loader.hpp"
#include "platform/platform.hpp"
#include "coding/read_write_utils.hpp"
#include "base/scope_guard.hpp"
#include <cstddef>
#include <cstdint>
#include <functional>
#include <memory>
#include <string>
#include <vector>
using namespace feature;
namespace
{
OsmElement MakeHighway(uint64_t id, std::string const & name, std::vector<uint64_t> const & nodes,
bool isOneway = false)
{
OsmElement element;
element.m_id = id;
element.m_type = OsmElement::EntityType::Way;
element.AddTag("ref", "");
element.AddTag("name", name);
element.AddTag("highway", "primary");
if (isOneway)
element.AddTag("oneway", "yes");
element.m_nodes = nodes;
return element;
}
feature::FeatureBuilder MakeFbForTest(OsmElement element)
{
feature::FeatureBuilder result;
ftype::GetNameAndType(&element, result.GetParams());
result.SetLinear();
return result;
}
size_t MakeKey(OsmElement const & element)
{
auto const name = element.GetTag("name");
auto const ref = element.GetTag("ref");
return std::hash<std::string>{}(name + '\0' + ref);
}
LineStringMerger::InputData MakeInputData(std::vector<OsmElement> const & elements)
{
LineStringMerger::InputData inputData;
for (auto const & element : elements)
inputData.emplace(MakeKey(element), std::make_shared<LineString>(element));
return inputData;
};
bool IsEqual(LineStringMerger::LinePtr const & lineString, std::vector<int32_t> const & ways)
{
auto const & w = lineString->GetWays();
return w == ways;
}
auto const w1 = MakeHighway(1/* id */, "w" /* name */, {1, 2, 3} /* nodes */);
auto const w2 = MakeHighway(2/* id */, "w" /* name */, {3, 4, 5} /* nodes */);
auto const w3 = MakeHighway(3/* id */, "w" /* name */, {5, 6, 7} /* nodes */);
auto const w4 = MakeHighway(4/* id */, "w" /* name */, {7, 8, 9} /* nodes */);
auto const w5 = MakeHighway(5/* id */, "w" /* name */, {9, 10, 11} /* nodes */);
auto const wo6 = MakeHighway(6/* id */, "w" /* name */, {13, 12, 3} /* nodes */, true /* isOneway */);
auto const wo7 = MakeHighway(7/* id */, "w" /* name */, {15, 14, 13} /* nodes */, true /* isOneway */);
auto const wo8 = MakeHighway(8/* id */, "w" /* name */, {17, 16, 15} /* nodes */, true /* isOneway */);
auto const b1 = MakeHighway(1/* id */, "b" /* name */, {1, 2, 3} /* nodes */);
auto const b2 = MakeHighway(2/* id */, "b" /* name */, {3, 4, 5} /* nodes */);
} // namespace
UNIT_TEST(MetalinesTest_Case0)
{
auto const inputData = MakeInputData({w1});
auto outputData = LineStringMerger::Merge(inputData);
TEST_EQUAL(outputData.size(), 0 /* unique names roads count */, ());
outputData = LineStringMerger::Merge({});
TEST_EQUAL(outputData.size(), 0 /* unique names roads count */, ());
}
UNIT_TEST(MetalinesTest_Case1)
{
auto const inputData = MakeInputData({w1, w2});
auto const outputData = LineStringMerger::Merge(inputData);
auto const key = MakeKey(w1);
TEST_EQUAL(outputData.size(), 1 /* unique names roads count */, ());
TEST_EQUAL(outputData.at(key)[0]->GetWays().size(), 2 /* merged way size */, ());
TEST(IsEqual(outputData.at(key)[0], {1, 2}) /* merged way */, ());
}
UNIT_TEST(MetalinesTest_Case2)
{
auto const inputData = MakeInputData({w1, w3, w2});
auto const outputData = LineStringMerger::Merge(inputData);
auto const key = MakeKey(w1);
TEST_EQUAL(outputData.size(), 1 /* unique names roads count */, ());
TEST_EQUAL(outputData.at(key)[0]->GetWays().size(), 3 /* merged way size */, ());
TEST(IsEqual(outputData.at(key)[0], {1, 2, 3}) /* merged way */, ());
}
UNIT_TEST(MetalinesTest_Case3)
{
auto const inputData = MakeInputData({w1, w4, w2, w5,});
auto const outputData = LineStringMerger::Merge(inputData);
auto const key = MakeKey(w1);
TEST_EQUAL(outputData.size(), 1 /* unique names roads count */, ());
TEST_EQUAL(outputData.at(key).size(), 2 /* ways count */, ());
TEST_EQUAL(outputData.at(key)[0]->GetWays().size(), 2 /* merged way size */, ());
TEST(IsEqual(outputData.at(key)[0], {1, 2}) /* merged way */, ());
TEST_EQUAL(outputData.at(key)[1]->GetWays().size(), 2 /* merged way size */, ());
TEST(IsEqual(outputData.at(key)[1], {4, 5}) /* merged way */, ());
}
UNIT_TEST(MetalinesTest_Case4)
{
auto const inputData = MakeInputData({w1, wo6,});
auto const outputData = LineStringMerger::Merge(inputData);
auto const key = MakeKey(w1);
TEST_EQUAL(outputData.size(), 1 /* unique names roads count */, ());
TEST_EQUAL(outputData.at(key).size(), 1 /* ways count */, ());
TEST(IsEqual(outputData.at(key)[0], {6, -1}) /* merged way */, ());
}
UNIT_TEST(MetalinesTest_Case5)
{
auto const inputData = MakeInputData({w1, w2, wo6,});
auto const outputData = LineStringMerger::Merge(inputData);
auto const key = MakeKey(w1);
TEST_EQUAL(outputData.size(), 1 /* unique names roads count */, ());
TEST_EQUAL(outputData.at(key).size(), 1 /* ways count */, ());
TEST(IsEqual(outputData.at(key)[0], {1, 2}) /* merged way */, ());
}
UNIT_TEST(MetalinesTest_Case6)
{
auto const inputData = MakeInputData({w1, b1, w2, b2,});
auto const outputData = LineStringMerger::Merge(inputData);
auto const keyW = MakeKey(w1);
auto const keyB = MakeKey(b1);
TEST_EQUAL(outputData.size(), 2 /* unique names roads count */, ());
TEST_EQUAL(outputData.at(keyW).size(), 1 /* ways count */, ());
TEST_EQUAL(outputData.at(keyB).size(), 1 /* ways count */, ());
}
UNIT_TEST(MetalinesTest_MetalinesBuilderMarge)
{
classificator::Load();
auto const filename = generator_tests::GetFileName();
SCOPE_GUARD(_, std::bind(Platform::RemoveFileIfExists, std::cref(filename)));
auto c1 = std::make_shared<MetalinesBuilder>(filename);
auto c2 = c1->Clone();
c1->CollectFeature(MakeFbForTest(w1), w1);
c2->CollectFeature(MakeFbForTest(w2), w2);
c1->CollectFeature(MakeFbForTest(w5), w5);
c2->CollectFeature(MakeFbForTest(w4), w4);
c1->Finish();
c2->Finish();
c1->Merge(*c2);
c1->Save();
FileReader reader(filename);
ReaderSource<FileReader> src(reader);
std::set<std::vector<int32_t>> s;
while (src.Size() > 0)
{
std::vector<int32_t> ways;
rw::ReadVectorOfPOD(src, ways);
s.emplace(std::move(ways));
}
TEST_EQUAL(s.size(), 2, ());
TEST_EQUAL(s.count({1, 2}), 1, ());
TEST_EQUAL(s.count({4, 5}), 1, ());
}

57
generator/generator_tests/node_mixer_test.cpp
View file

@ -1,57 +0,0 @@
#include "testing/testing.hpp"
#include "generator/node_mixer.hpp"
#include "generator/osm_element.hpp"
#include <sstream>
UNIT_TEST(NodeMixerTests)
{
std::istringstream stream1("");
generator::MixFakeNodes(stream1, [](OsmElement &) {
TEST(false, ("Returned an object for an empty input stream."));
});
std::istringstream stream2("shop=gift\nname=Shop\n");
generator::MixFakeNodes(stream2, [](OsmElement &) {
TEST(false, ("Returned an object for a source without coordinates."));
});
std::istringstream stream3("lat=4.0\nlon=-4.1\n");
generator::MixFakeNodes(stream3, [](OsmElement &) {
TEST(false, ("Returned an object for a source without tags."));
});
std::istringstream stream4("lat=10.0\nlon=-4.8\nshop=gift\nname=Shop");
int count4 = 0;
generator::MixFakeNodes(stream4, [&](OsmElement & p) {
count4++;
TEST_EQUAL(p.m_type, OsmElement::EntityType::Node, ());
TEST_EQUAL(p.m_lat, 10.0, ());
TEST_EQUAL(p.m_lon, -4.8, ());
TEST_EQUAL(p.Tags().size(), 2, ());
TEST_EQUAL(p.GetTag("name"), "Shop", ());
});
TEST_EQUAL(count4, 1, ());
std::istringstream stream5("lat=10.0\nlon=-4.8\nid=1\nname=First\n\nid=2\nlat=60\nlon=1\nname=Second\n\n\n");
int count5 = 0;
generator::MixFakeNodes(stream5, [&](OsmElement & p) {
count5++;
TEST_EQUAL(p.m_type, OsmElement::EntityType::Node, ());
TEST_EQUAL(p.Tags().size(), 2, ());
std::string id = p.GetTag("id");
TEST(!id.empty(), ("No id tag when every object has it."));
TEST_EQUAL(p.GetTag("name"), id == "1" ? "First" : "Second", ());
});
TEST_EQUAL(count5, 2, ());
std::istringstream stream6("lat=0\nlon=-4.8\nshop=mall");
int count6 = 0;
generator::MixFakeNodes(stream6, [&](OsmElement & p) {
count6++;
TEST_EQUAL(p.m_lat, 0.0, ());
TEST_EQUAL(p.m_lon, -4.8, ());
});
TEST_EQUAL(count6, 1, ());
}

63
generator/maxspeeds_collector.hpp
View file

@ -1,63 +0,0 @@
#pragma once
#include "generator/collector_interface.hpp"
#include "generator/feature_builder.hpp"
#include "generator/osm_element.hpp"
#include <sstream>
#include <memory>
#include <string>
namespace generator
{
namespace cache
{
class IntermediateDataReader;
} // namespace cache
/// \brief Collects all maxspeed tags value and saves them to a csv file.
/// Every line describes maxspeed, maxspeed:forward and maxspeed:backward
/// tags of features. The format of the lines is described below.
class MaxspeedsCollector : public CollectorInterface
{
public:
/// \param filePath path to csv file.
explicit MaxspeedsCollector(std::string const & filename);
// CollectorInterface overrides:
std::shared_ptr<CollectorInterface>
Clone(std::shared_ptr<cache::IntermediateDataReader> const & = {}) const override;
void CollectFeature(feature::FeatureBuilder const &, OsmElement const & p) override;
void Finish() override;
void Save() override;
void Merge(CollectorInterface const & collector) override;
void MergeInto(MaxspeedsCollector & collector) const override;
private:
// |m_data| contains strings with maxspeed tags value for corresponding features in one of the
// following formats
// 1. osm id,units kmh or mph,maxspeed value
// 2. osm id,units kmh or mph,maxspeed:forward value
// 3. osm id,units kmh or mph,maxspeed:forward value,maxspeed:backward value
// There are possible examples of strings contained in the list |m_data|:
// 2343313,Metric,60
// 13243214,Imperial,60
// 3243245345,Metric,60,80
// 134243,Imperial,30,50
// 45432423,Metric,60,65534
// 53445423,Metric,60,65533
//
// Note 1. 65534 means kNoneMaxSpeed and 65533 means kWalkMaxSpeed. They are constants for
// maxspeed tag value "none" and "walk" correspondingly.
// Note 2. Saying osm id means osm id of features with OsmElement::EntityType::Way type without
// any prefixes. It's done so to simplify the debugging process. This way it's very easy knowing
// osm id to see the feature on the web.
// Note 3. A string is saved in |m_data| only if it's valid and parsed successfully
// with ParseMaxspeedTag() function. That means all macro like RU:urban or GE:rural
// are converted to an appropriate speed value and macro "none" and "walk" are converted
// to |kNoneMaxSpeed| and |kWalkMaxSpeed|.
std::ofstream m_stream;
};
} // namespace generator

273
generator/metalines_builder.cpp
View file

@ -1,273 +0,0 @@
#include "generator/metalines_builder.hpp"
#include "generator/intermediate_data.hpp"
#include "generator/routing_helpers.hpp"
#include "indexer/classificator.hpp"
#include "coding/file_container.hpp"
#include "coding/read_write_utils.hpp"
#include "coding/varint.hpp"
#include "coding/write_to_sink.hpp"
#include "base/checked_cast.hpp"
#include "base/logging.hpp"
#include "defines.hpp"
#include <cstdint>
#include <limits>
#include <map>
namespace
{
uint8_t constexpr kMetaLinesSectionVersion = 1;
} // namespace
namespace feature
{
LineString::LineString(OsmElement const & way)
{
std::string const oneway = way.GetTag("oneway");
m_oneway = !oneway.empty() && oneway != "no";
int32_t const wayId = base::checked_cast<int32_t>(way.m_id);
m_ways.push_back(oneway == "-1" ? -wayId : wayId);
CHECK_GREATER_OR_EQUAL(way.Nodes().size(), 2, ());
m_start = way.Nodes().front();
m_end = way.Nodes().back();
}
void LineString::Reverse()
{
ASSERT(!m_oneway, ("Trying to reverse a one-way road."));
std::swap(m_start, m_end);
std::reverse(m_ways.begin(), m_ways.end());
for (auto & p : m_ways)
p = -p;
}
bool LineString::Add(LineString & line)
{
if (m_start == line.m_start || m_end == line.m_end)
{
if (!line.m_oneway)
line.Reverse();
else if (!m_oneway)
Reverse();
else
return false;
}
if (m_end == line.m_start)
{
m_ways.insert(m_ways.end(), line.m_ways.begin(), line.m_ways.end());
m_end = line.m_end;
m_oneway = m_oneway || line.m_oneway;
}
else if (m_start == line.m_end)
{
m_ways.insert(m_ways.begin(), line.m_ways.begin(), line.m_ways.end());
m_start = line.m_start;
m_oneway = m_oneway || line.m_oneway;
}
else
{
return false;
}
return true;
}
// static
LineStringMerger::OutputData LineStringMerger::Merge(InputData const & data)
{
InputData mergedLines;
auto const intermediateData = OrderData(data);
for (auto & p : intermediateData)
{
Buffer buffer;
for (auto & lineString : p.second)
TryMerge(lineString, buffer);
std::unordered_set<LinePtr> uniqLineStrings;
for (auto const & pb : buffer)
{
auto const & ways = pb.second->GetWays();
if (uniqLineStrings.emplace(pb.second).second && ways.size() > 1)
mergedLines.emplace(p.first, pb.second);
}
}
return OrderData(mergedLines);
}
// static
bool LineStringMerger::TryMerge(LinePtr const & lineString, Buffer & buffer)
{
bool merged = false;
while (TryMergeOne(lineString, buffer))
merged = true;
buffer.emplace(lineString->GetStart(), lineString);
buffer.emplace(lineString->GetEnd(), lineString);
return merged;
}
// static
bool LineStringMerger::TryMergeOne(LinePtr const & lineString, Buffer & buffer)
{
auto static const kUndef = std::numeric_limits<int>::max();
uint64_t index = kUndef;
if (buffer.count(lineString->GetStart()) != 0)
index = lineString->GetStart();
else if (buffer.count(lineString->GetEnd()) != 0)
index = lineString->GetEnd();
if (index != kUndef)
{
auto bufferedLineString = buffer[index];
buffer.erase(bufferedLineString->GetStart());
buffer.erase(bufferedLineString->GetEnd());
if (!lineString->Add(*bufferedLineString))
{
buffer.emplace(bufferedLineString->GetStart(), bufferedLineString);
buffer.emplace(bufferedLineString->GetEnd(), bufferedLineString);
buffer.emplace(lineString->GetStart(), lineString);
buffer.emplace(lineString->GetEnd(), lineString);
return false;
}
}
return index != kUndef;
}
// static
LineStringMerger::OutputData LineStringMerger::OrderData(InputData const & data)
{
OutputData intermediateData;
for (auto const & p : data)
intermediateData[p.first].emplace_back(p.second);
for (auto & p : intermediateData)
{
auto & lineStrings = intermediateData[p.first];
std::sort(std::begin(lineStrings), std::end(lineStrings), [](auto const & l, auto const & r) {
auto const & lways = l->GetWays();
auto const & rways = r->GetWays();
return lways.size() == rways.size() ? lways.front() < rways.front() : lways.size() > rways.size();
});
}
return intermediateData;
}
// MetalinesBuilder --------------------------------------------------------------------------------
MetalinesBuilder::MetalinesBuilder(std::string const & filename)
: generator::CollectorInterface(filename) {}
std::shared_ptr<generator::CollectorInterface>
MetalinesBuilder::Clone(std::shared_ptr<generator::cache::IntermediateDataReader> const &) const
{
return std::make_shared<MetalinesBuilder>(GetFilename());
}
void MetalinesBuilder::CollectFeature(FeatureBuilder const & feature, OsmElement const & element)
{
if (!feature.IsLine())
return;
static auto const highwayType = classif().GetTypeByPath({"highway"});
if (!feature.HasType(highwayType, 1 /* level */) || element.Nodes().front() == element.Nodes().back())
return;
auto const & params = feature.GetParams();
auto const name = feature.GetName();
if (name.empty() && params.ref.empty())
return;
auto const key = std::hash<std::string>{}(name + '\0' + params.ref);
m_data.emplace(key, std::make_shared<LineString>(element));
}
void MetalinesBuilder::Save()
{
FileWriter writer(GetFilename());
uint32_t countLines = 0;
uint32_t countWays = 0;
auto const mergedData = LineStringMerger::Merge(m_data);
for (auto const & p : mergedData)
{
for (auto const & lineString : p.second)
{
auto const & ways = lineString->GetWays();
rw::WriteVectorOfPOD(writer, ways);
countWays += ways.size();
++countLines;
}
}
LOG_SHORT(LINFO, ("Wrote", countLines, "metalines [with", countWays ,
"ways] with OSM IDs for the entire planet to", GetFilename()));
}
void MetalinesBuilder::Merge(generator::CollectorInterface const & collector)
{
collector.MergeInto(*this);
}
void MetalinesBuilder::MergeInto(MetalinesBuilder & collector) const
{
collector.m_data.insert(std::begin(m_data), std::end(m_data));
}
// Functions --------------------------------------------------------------------------------
bool WriteMetalinesSection(std::string const & mwmPath, std::string const & metalinesPath,
std::string const & osmIdsToFeatureIdsPath)
{
std::map<base::GeoObjectId, uint32_t> osmIdToFeatureId;
if (!routing::ParseRoadsOsmIdToFeatureIdMapping(osmIdsToFeatureIdsPath, osmIdToFeatureId))
return false;
FileReader reader(metalinesPath);
ReaderSource<FileReader> src(reader);
std::vector<uint8_t> buffer;
MemWriter<std::vector<uint8_t>> memWriter(buffer);
uint32_t count = 0;
while (src.Size() > 0)
{
std::vector<int32_t> featureIds;
std::vector<int32_t> ways;
rw::ReadVectorOfPOD(src, ways);
for (auto const wayId : ways)
{
// We get a negative wayId when a feature direction should be reversed.
auto fid = osmIdToFeatureId.find(base::MakeOsmWay(std::abs(wayId)));
if (fid == osmIdToFeatureId.cend())
break;
// Keeping the sign for the feature direction.
int32_t const featureId = static_cast<int32_t>(fid->second);
featureIds.push_back(wayId > 0 ? featureId : -featureId);
}
if (featureIds.size() > 1)
{
// Write vector to buffer if we have got at least two segments.
WriteVarUint(memWriter, featureIds.size());
for (auto const & fid : featureIds)
WriteVarInt(memWriter, fid);
++count;
}
}
// Write buffer to section.
FilesContainerW cont(mwmPath, FileWriter::OP_WRITE_EXISTING);
auto writer = cont.GetWriter(METALINES_FILE_TAG);
WriteToSink(writer, kMetaLinesSectionVersion);
WriteVarUint(writer, count);
writer->Write(buffer.data(), buffer.size());
LOG(LINFO, ("Finished writing metalines section, found", count, "metalines."));
return true;
}
} // namespace feature

88
generator/metalines_builder.hpp
View file

@ -1,88 +0,0 @@
#pragma once
#include "generator/collector_interface.hpp"
#include "generator/feature_builder.hpp"
#include "generator/osm_element.hpp"
#include <cstdlib>
#include <cstdint>
#include <map>
#include <memory>
#include <string>
#include <unordered_map>
#include <vector>
namespace generator
{
namespace cache
{
class IntermediateDataReader;
} // namespace cache
} // namespace generator
namespace feature
{
// A string of connected ways.
class LineString
{
public:
using Ways = std::vector<int32_t>;
explicit LineString(OsmElement const & way);
bool Add(LineString & line);
void Reverse();
Ways const & GetWays() const { return m_ways; }
uint64_t GetStart() const { return m_start; }
uint64_t GetEnd() const { return m_end; }
private:
uint64_t m_start;
uint64_t m_end;
bool m_oneway;
Ways m_ways;
};
class LineStringMerger
{
public:
using LinePtr = std::shared_ptr<LineString>;
using InputData = std::unordered_multimap<size_t, LinePtr>;
using OutputData = std::map<size_t, std::vector<LinePtr>>;
static OutputData Merge(InputData const & data);
private:
using Buffer = std::unordered_map<uint64_t, LinePtr>;
static bool TryMerge(LinePtr const & lineString, Buffer & buffer);
static bool TryMergeOne(LinePtr const & lineString, Buffer & buffer);
static OutputData OrderData(InputData const & data);
};
// Merges road segments with similar name and ref values into groups called metalines.
class MetalinesBuilder : public generator::CollectorInterface
{
public:
explicit MetalinesBuilder(std::string const & filename);
// CollectorInterface overrides:
std::shared_ptr<CollectorInterface>
Clone(std::shared_ptr<generator::cache::IntermediateDataReader> const & = {}) const override;
/// Add a highway segment to the collection of metalines.
void CollectFeature(FeatureBuilder const & feature, OsmElement const & element) override;
void Save() override;
void Merge(generator::CollectorInterface const & collector) override;
void MergeInto(MetalinesBuilder & collector) const override;
private:
std::unordered_multimap<size_t, std::shared_ptr<LineString>> m_data;
};
// Read an intermediate file from MetalinesBuilder and convert it to an mwm section.
bool WriteMetalinesSection(std::string const & mwmPath, std::string const & metalinesPath,
std::string const & osmIdsToFeatureIdsPath);
}

78
generator/node_mixer.cpp
View file

@ -1,78 +0,0 @@
#include "generator/node_mixer.hpp"
#include "base/logging.hpp"
#include "base/string_utils.hpp"
using namespace std;
namespace generator
{
void MixFakeNodes(istream & stream, function<void(OsmElement &)> processor)
{
if (stream.fail())
return;
// Max node id on 12.02.2018 times hundred — good enough until ~2030.
uint64_t constexpr baseNodeId = 5396734321 * 100;
uint8_t constexpr kCFLat = 1;
uint8_t constexpr kCFLon = 2;
uint8_t constexpr kCFTags = 4;
uint8_t constexpr kCFAll = kCFLat | kCFLon | kCFTags;
uint64_t count = 0;
uint8_t completionFlag = 0;
OsmElement p;
p.m_id = baseNodeId;
p.m_type = OsmElement::EntityType::Node;
string line;
while (getline(stream, line))
{
if (line.empty())
{
if (completionFlag == kCFAll)
{
processor(p);
count++;
p.Clear();
p.m_id = baseNodeId + count;
p.m_type = OsmElement::EntityType::Node;
completionFlag = 0;
}
continue;
}
auto const eqPos = line.find('=');
if (eqPos != string::npos)
{
string key = line.substr(0, eqPos);
string value = line.substr(eqPos + 1);
strings::Trim(key);
strings::Trim(value);
if (key == "lat")
{
if (strings::to_double(value, p.m_lat))
completionFlag |= kCFLat;
}
else if (key == "lon")
{
if (strings::to_double(value, p.m_lon))
completionFlag |= kCFLon;
}
else
{
p.AddTag(key, value);
completionFlag |= kCFTags;
}
}
}
if (completionFlag == kCFAll)
{
processor(p);
count++;
}
LOG(LINFO, ("Added", count, "fake nodes."));
}
} // namespace generator

18
generator/node_mixer.hpp
View file

@ -1,18 +0,0 @@
#pragma once
#include "generator/osm_element.hpp"
#include <fstream>
#include <functional>
#include <string>
namespace generator
{
void MixFakeNodes(std::istream & stream, std::function<void(OsmElement &)> processor);
inline void MixFakeNodes(std::string const filePath, std::function<void(OsmElement &)> processor)
{
std::ifstream stream(filePath);
MixFakeNodes(stream, processor);
}
} // namespace generator

75
generator/place.cpp
View file

@ -1,75 +0,0 @@
#include "generator/place.hpp"
#include "indexer/classificator.hpp"
#include "indexer/ftypes_matcher.hpp"
#include "geometry/mercator.hpp"
using namespace feature;
namespace generator
{
Place::Place(FeatureBuilder const & ft, uint32_t type, bool saveParams) :
m_ft(ft),
m_pt(ft.GetKeyPoint()),
m_type(type)
{
using namespace ftypes;
if (!saveParams)
m_ft.SetParams({}/* params */);
switch (IsLocalityChecker::Instance().GetType(m_type))
{
case COUNTRY: m_thresholdM = 300000.0; break;
case STATE: m_thresholdM = 100000.0; break;
case CITY: m_thresholdM = 30000.0; break;
case TOWN: m_thresholdM = 20000.0; break;
case VILLAGE: m_thresholdM = 10000.0; break;
default: m_thresholdM = 10000.0; break;
}
}
m2::RectD Place::GetLimitRect() const
{
return MercatorBounds::RectByCenterXYAndSizeInMeters(m_pt, m_thresholdM);
}
bool Place::IsEqual(Place const & r) const
{
return (AreTypesEqual(m_type, r.m_type) &&
m_ft.GetName() == r.m_ft.GetName() &&
(IsPoint() || r.IsPoint()) &&
MercatorBounds::DistanceOnEarth(m_pt, r.m_pt) < m_thresholdM);
}
bool Place::IsBetterThan(Place const & r) const
{
// Check ranks.
uint8_t const r1 = m_ft.GetRank();
uint8_t const r2 = r.m_ft.GetRank();
if (r1 != r2)
return r1 > r2;
// Check types length.
// ("place-city-capital-2" is better than "place-city").
uint8_t const l1 = ftype::GetLevel(m_type);
uint8_t const l2 = ftype::GetLevel(r.m_type);
if (l1 != l2)
return l1 > l2;
// Assume that area places has better priority than point places at the very end ...
/// @todo It was usefull when place=XXX type has any area fill style.
/// Need to review priority logic here (leave the native osm label).
return !IsPoint() && r.IsPoint();
}
bool Place::AreTypesEqual(uint32_t t1, uint32_t t2)
{
// Use 2-arity places comparison for filtering.
// ("place-city-capital-2" is equal to "place-city")
ftype::TruncValue(t1, 2);
ftype::TruncValue(t2, 2);
return (t1 == t2);
}
} // namespace generator

33
generator/place.hpp
View file

@ -1,33 +0,0 @@
#pragma once
#include "generator/feature_builder.hpp"
#include "geometry/point2d.hpp"
#include "geometry/rect2d.hpp"
#include <cstdint>
namespace generator
{
/// Used to make a "good" node for a highway graph with OSRM for low zooms.
class Place
{
public:
Place(feature::FeatureBuilder const & ft, uint32_t type, bool saveParams = true);
feature::FeatureBuilder const & GetFeature() const { return m_ft; }
m2::RectD GetLimitRect() const;
bool IsEqual(Place const & r) const;
/// Check whether we need to replace place @r with place @this.
bool IsBetterThan(Place const & r) const;
private:
bool IsPoint() const { return (m_ft.GetGeomType() == feature::GeomType::Point); }
static bool AreTypesEqual(uint32_t t1, uint32_t t2);
feature::FeatureBuilder m_ft;
m2::PointD m_pt;
uint32_t m_type;
double m_thresholdM;
};
} // namespace generator

44
generator/promo_catalog_cities.hpp
View file

@ -1,44 +0,0 @@
#pragma once
#include "coding/file_reader.hpp"
#include "base/geo_object_id.hpp"
#include "base/logging.hpp"
#include <string>
#include <unordered_set>
#include "3party/jansson/myjansson.hpp"
namespace promo
{
using Cities = std::unordered_set<base::GeoObjectId>;
inline Cities LoadCities(std::string const & filename)
{
if (filename.empty())
return {};
std::string src;
FileReader reader(filename);
reader.ReadAsString(src);
Cities result;
base::Json root(src.c_str());
auto const dataArray = json_object_get(root.get(), "data");
auto const size = json_array_size(dataArray);
result.reserve(size);
for (size_t i = 0; i < size; ++i)
{
int64_t id = 0;
auto const obj = json_array_get(dataArray, i);
FromJSONObject(obj, "osmid", id);
result.emplace(static_cast<uint64_t>(id));
}
return result;
}
} // namespace promo

125
generator/region_meta.cpp
View file

@ -1,125 +0,0 @@
#include "region_meta.hpp"
#include "coding/reader.hpp"
#include "platform/platform.hpp"
#include "3party/jansson/myjansson.hpp"
using namespace std;
namespace
{
int8_t ParseHolidayReference(std::string const & ref)
{
if (ref == "easter")
return feature::RegionData::PHReference::PH_EASTER;
if (ref == "orthodox easter")
return feature::RegionData::PHReference::PH_ORTHODOX_EASTER;
if (ref == "victoriaDay")
return feature::RegionData::PHReference::PH_VICTORIA_DAY;
if (ref == "canadaDay")
return feature::RegionData::PHReference::PH_CANADA_DAY;
return 0;
}
} // namespace
namespace feature
{
bool ReadRegionDataImpl(std::string const & countryName, RegionData & data)
{
try
{
auto reader = GetPlatform().GetReader(COUNTRIES_META_FILE);
std::string buffer;
reader->ReadAsString(buffer);
base::Json root(buffer.data());
json_t const * jsonData = nullptr;
FromJSONObjectOptionalField(root.get(), countryName, jsonData);
if (!jsonData)
return false;
vector<std::string> languages;
FromJSONObjectOptionalField(jsonData, "languages", languages);
if (!languages.empty())
data.SetLanguages(languages);
std::string driving;
FromJSONObjectOptionalField(jsonData, "driving", driving);
if (driving == "l" || driving == "r")
data.Set(RegionData::Type::RD_DRIVING, driving);
std::string timezone;
FromJSONObjectOptionalField(jsonData, "timezone", timezone);
if (!timezone.empty())
data.Set(RegionData::Type::RD_TIMEZONE, timezone);
bool allow_housenames;
FromJSONObjectOptionalField(jsonData, "housenames", allow_housenames);
if (allow_housenames)
data.Set(RegionData::Type::RD_ALLOW_HOUSENAMES, "y");
// Public holidays: an array of arrays of [string/number, number].
// See https://github.com/opening-hours/opening_hours.js/blob/master/docs/holidays.md
vector<json_t *> holidays;
FromJSONObjectOptionalField(jsonData, "holidays", holidays);
for (json_t * holiday : holidays)
{
if (!json_is_array(holiday) || json_array_size(holiday) != 2)
MYTHROW(base::Json::Exception, ("Holiday must be an array of two elements in", countryName));
json_t * reference = json_array_get(holiday, 0);
int8_t refId = 0;
if (json_is_integer(reference))
{
refId = json_integer_value(reference);
}
else if (json_is_string(reference))
{
refId = ParseHolidayReference(std::string(json_string_value(reference)));
}
else
{
MYTHROW(base::Json::Exception,
("Holiday month reference should be either a std::string or a number in", countryName));
}
if (refId <= 0)
MYTHROW(base::Json::Exception, ("Incorrect month reference in", countryName));
if (!json_is_integer(json_array_get(holiday, 1)))
MYTHROW(base::Json::Exception, ("Holiday day offset should be a number in", countryName));
data.AddPublicHoliday(refId, json_integer_value(json_array_get(holiday, 1)));
}
// TODO(@zverik): Implement formats reading when decided on data types.
return true;
}
catch (Reader::Exception const & e)
{
LOG(LWARNING, ("Error reading", COUNTRIES_META_FILE, ":", e.Msg()));
}
catch (base::Json::Exception const & e)
{
LOG(LERROR, ("Error parsing JSON in", COUNTRIES_META_FILE, ":", e.Msg()));
}
return false;
}
bool ReadRegionData(std::string const & countryName, RegionData & data)
{
// When there is a match for a complete countryName, simply relay the call.
if (ReadRegionDataImpl(countryName, data))
return true;
// If not, cut parts of a country name from the tail. E.g. "Russia_Moscow" -> "Russia".
auto p = countryName.find_last_of('_');
while (p != std::string::npos)
{
if (ReadRegionDataImpl(countryName.substr(0, p), data))
return true;
p = p > 0 ? countryName.find_last_of('_', p - 1) : std::string::npos;
}
return false;
}
} // namespace feature

10
generator/region_meta.hpp
View file

@ -1,10 +0,0 @@
#pragma once
#include "generate_info.hpp"
#include "indexer/feature_meta.hpp"
namespace feature
{
bool ReadRegionData(std::string const & countryName, RegionData & data);
} // namespace feature

67
generator/routing_helpers.cpp
View file

@ -1,67 +0,0 @@
#include "generator/routing_helpers.hpp"
#include "generator/utils.hpp"
#include "base/logging.hpp"
using std::map;
using std::string;
namespace
{
template <class ToDo>
bool ForEachRoadFromFile(string const & filename, ToDo && toDo)
{
return generator::ForEachOsmId2FeatureId(
filename, [&](base::GeoObjectId const & osmId, uint32_t const featureId) {
if (osmId.GetType() == base::GeoObjectId::Type::ObsoleteOsmWay)
toDo(featureId, osmId);
});
}
} // namespace
namespace routing
{
void AddFeatureId(base::GeoObjectId osmId, uint32_t featureId,
map<base::GeoObjectId, uint32_t> & osmIdToFeatureId)
{
// Failing to insert here usually means that two features were created
// from one osm id, for example an area and its boundary.
osmIdToFeatureId.emplace(osmId, featureId);
}
bool ParseRoadsOsmIdToFeatureIdMapping(string const & osmIdsToFeatureIdPath,
map<base::GeoObjectId, uint32_t> & osmIdToFeatureId)
{
return ForEachRoadFromFile(osmIdsToFeatureIdPath,
[&](uint32_t featureId, base::GeoObjectId osmId) {
AddFeatureId(osmId, featureId, osmIdToFeatureId);
});
}
bool ParseRoadsFeatureIdToOsmIdMapping(string const & osmIdsToFeatureIdPath,
map<uint32_t, base::GeoObjectId> & featureIdToOsmId)
{
featureIdToOsmId.clear();
bool idsAreOk = true;
bool const readSuccess = ForEachRoadFromFile(
osmIdsToFeatureIdPath, [&](uint32_t featureId, base::GeoObjectId const & osmId) {
auto const emplaced = featureIdToOsmId.emplace(featureId, osmId);
if (emplaced.second)
return;
idsAreOk = false;
LOG(LERROR, ("Feature id", featureId, "is included in two osm ids:", emplaced.first->second,
osmId));
});
if (readSuccess && idsAreOk)
return true;
LOG(LERROR, ("Can't load osm id mapping from", osmIdsToFeatureIdPath));
featureIdToOsmId.clear();
return false;
}
} // namespace routing