AndrewShkrob/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

Added osm country borders generation and used them when building planet

Browse source
This commit is contained in:
Alex Zolotarev 2011-04-01 03:09:07 +01:00 • committed by Alex Zolotarev
parent 06c612827e
commit 4ee5ce1013
14 changed files with 909 additions and 105 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

159
generator/borders_generator.cpp Normal file
View file

@ -0,0 +1,159 @@
#include "borders_generator.hpp"
#include "osm_xml_parser.hpp"
#include "../base/std_serialization.hpp"
#include "../base/logging.hpp"
#include "../indexer/mercator.hpp"
#include "../coding/file_reader.hpp"
#include "../coding/file_writer.hpp"
#include "../coding/streams_sink.hpp"
#include "../coding/parse_xml.hpp"
#include "../std/list.hpp"
#include "../std/vector.hpp"
namespace osm
{
class BordersCreator
{
vector<m2::RegionD> & m_borders;
OsmRawData const & m_data;
m2::RegionD m_currentRegion;
public:
BordersCreator(vector<m2::RegionD> & outBorders, OsmRawData const & data)
: m_borders(outBorders), m_data(data)
{
}
void CreateFromWays(OsmWays const & ways)
{
for (OsmWays::const_iterator it = ways.begin(); it != ways.end(); ++it)
{
// clear region
m_currentRegion = m2::RegionD();
it->ForEachPoint(*this);
CHECK(m_currentRegion.IsValid(), ("Invalid region"));
m_borders.push_back(m_currentRegion);
}
}
void operator()(OsmId nodeId)
{
try
{
OsmNode node = m_data.NodeById(nodeId);
m_currentRegion.AddPoint(m2::PointD(MercatorBounds::LonToX(node.m_lon),
MercatorBounds::LatToY(node.m_lat)));
}
catch (OsmRawData::OsmInvalidIdException const & e)
{
LOG(LWARNING, (e.what()));
}
}
};
void GenerateBordersFromOsm(string const & osmFile, string const & outFile)
{
OsmRawData osmData;
{
FileReader file(osmFile);
ReaderSource<FileReader> source(file);
OsmXmlParser parser(osmData);
CHECK(ParseXML(source, parser), ("Invalid XML"));
}
// find country borders relation
OsmIds relationIds = osmData.RelationsByKey("ISO3166-1");
CHECK(!relationIds.empty(), ("No relation with key 'ISO3166-1' found"));
CHECK_EQUAL(relationIds.size(), 1, ("More than one relation with key 'ISO3166-1' found"));
OsmRelation countryRelation = osmData.RelationById(relationIds[0]);
// get country name
string countryName;
if (!countryRelation.TagValueByKey("name:en", countryName))
countryRelation.TagValueByKey("name", countryName);
LOG(LINFO, ("Processing boundaries for country", countryName));
// find border ways
OsmIds wayIdsOuterRole = countryRelation.MembersByTypeAndRole("way", "outer");
OsmIds wayIdsNoRole = countryRelation.MembersByTypeAndRole("way", "");
// collect all ways
list<OsmWay> ways;
for(size_t i = 0; i < wayIdsOuterRole.size(); ++i)
ways.push_back(osmData.WayById(wayIdsOuterRole[i]));
for(size_t i = 0; i < wayIdsNoRole.size(); ++i)
ways.push_back(osmData.WayById(wayIdsNoRole[i]));
CHECK(!ways.empty(), ("No any ways in country border"));
// merge all collected ways
OsmWays mergedWays;
do
{
OsmId lastMergedWayId = -1; // for debugging
OsmWay merged = ways.front();
size_t lastSize = ways.size();
ways.pop_front();
// repeat until we merge everything
while (lastSize > ways.size())
{
lastSize = ways.size();
for (list<OsmWay>::iterator it = ways.begin(); it != ways.end(); ++it)
{
if (merged.MergeWith(*it))
{
lastMergedWayId = it->Id();
ways.erase(it);
break;
}
}
}
if (!merged.IsClosed())
{
LOG(LERROR, ("Way merge unsuccessful as borders are not closed. Last merged id:",
lastMergedWayId == -1 ? merged.Id() : lastMergedWayId));
}
else
{
LOG(LINFO, ("Successfully merged boundaries with points count:", merged.PointsCount()));
mergedWays.push_back(merged);
}
} while (!ways.empty());
CHECK(!mergedWays.empty(), ("No borders were generated for country:", countryName));
// save generated borders
vector<m2::RegionD> borders;
BordersCreator doCreateBorders(borders, osmData);
doCreateBorders.CreateFromWays(mergedWays);
CHECK_EQUAL(mergedWays.size(), borders.size(), ("Can't generate borders from ways"));
FileWriter writer(outFile);
stream::SinkWriterStream<Writer> stream(writer);
stream << borders;
LOG(LINFO, ("Saved", borders.size(), "border(s) for", countryName));
}
bool LoadBorders(string const & borderFile, vector<m2::RegionD> & outBorders)
{
try
{
FileReader file(borderFile);
ReaderSource<FileReader> source(file);
stream::SinkReaderStream<ReaderSource<FileReader> > stream(source);
stream >> outBorders;
CHECK(!outBorders.empty(), ("No borders were loaded from", borderFile));
}
catch (FileReader::OpenException const &)
{
return false;
}
return true;
}
}

14
generator/borders_generator.hpp Normal file
View file

@ -0,0 +1,14 @@
#pragma once
#include "../geometry/region2d.hpp"
#include "../geometry/point2d.hpp"
#include "../std/vector.hpp"
#include "../std/string.hpp"
namespace osm
{
void GenerateBordersFromOsm(string const & osmFile, string const & outFile);
/// @return false if borderFile can't be opened
bool LoadBorders(string const & borderFile, vector<m2::RegionD> & outBorders);
}

80
generator/borders_loader.cpp Normal file
View file

@ -0,0 +1,80 @@
#include "borders_loader.hpp"
#include "borders_generator.hpp"
#include "../base/logging.hpp"
#include "../base/string_utils.hpp"
#include "../std/fstream.hpp"
#include "../std/vector.hpp"
#define BORDERS_DIR "borders/"
#define BORDERS_EXTENSION ".borders"
#define POLYGONS_FILE "polygons.lst"
namespace borders
{
class PolygonLoader
{
string m_baseDir;
// @TODO not used
int m_level;
CountryPolygons & m_polygons;
m2::RectD & m_rect;
public:
// @TODO level is not used
PolygonLoader(string const & basePolygonsDir, int level, CountryPolygons & polygons, m2::RectD & rect)
: m_baseDir(basePolygonsDir), m_level(level), m_polygons(polygons), m_rect(rect)
{
}
void operator()(string const & name)
{
if (m_polygons.m_name.empty())
m_polygons.m_name = name;
vector<m2::RegionD> coutryBorders;
if (osm::LoadBorders(m_baseDir + BORDERS_DIR + name + BORDERS_EXTENSION, coutryBorders))
{
for (size_t i = 0; i < coutryBorders.size(); ++i)
{
m2::RectD const rect(coutryBorders[i].GetRect());
m_rect.Add(rect);
m_polygons.m_regions.Add(coutryBorders[i], rect);
}
}
}
};
bool LoadCountriesList(string const & baseDir, CountriesContainerT & countries,
int simplifyCountriesLevel)
{
if (simplifyCountriesLevel > 0)
{
LOG_SHORT(LINFO, ("Simplificator level for country polygons:", simplifyCountriesLevel));
}
countries.Clear();
ifstream stream((baseDir + POLYGONS_FILE).c_str());
string line;
LOG(LINFO, ("Loading countries."));
while (stream.good())
{
std::getline(stream, line);
if (line.empty())
continue;
CountryPolygons country;
m2::RectD rect;
PolygonLoader loader(baseDir, simplifyCountriesLevel, country, rect);
utils::TokenizeString(line, "|", loader);
if (!country.m_regions.IsEmpty())
countries.Add(country, rect);
}
LOG(LINFO, ("Countries loaded:", countries.GetSize()));
return !countries.IsEmpty();
}
}

27
generator/borders_loader.hpp Normal file
View file

@ -0,0 +1,27 @@
#pragma once
#include "../geometry/region2d.hpp"
#include "../geometry/tree4d.hpp"
#include "../std/string.hpp"
namespace borders
{
typedef m2::RegionD Region;
typedef m4::Tree<Region> RegionsContainerT;
struct CountryPolygons
{
CountryPolygons(string const & name = "") : m_name(name), m_index(-1) {}
RegionsContainerT m_regions;
string m_name;
mutable int m_index;
};
typedef m4::Tree<CountryPolygons> CountriesContainerT;
/// @param[in] simplifyCountriesLevel if positive, used as a level for simplificator
bool LoadCountriesList(string const & baseDir, CountriesContainerT & countries,
int simplifyCountriesLevel = -1);
}

8
generator/generator.pro
View file

@ -20,9 +20,11 @@ SOURCES += \
update_generator.cpp \
grid_generator.cpp \
statistics.cpp \
kml_parser.cpp \
osm2type.cpp \
classif_routine.cpp \
borders_generator.cpp \
osm_xml_parser.cpp \
borders_loader.cpp \
HEADERS += \
feature_merger.hpp \
@ -37,8 +39,10 @@ HEADERS += \
update_generator.hpp \
grid_generator.hpp \
statistics.hpp \
kml_parser.hpp \
polygonizer.hpp \
world_map_generator.hpp \
osm2type.hpp \
classif_routine.hpp \
borders_generator.hpp \
osm_xml_parser.hpp \
borders_loader.hpp \

1
generator/generator_tests/generator_tests.pro
View file

@ -21,3 +21,4 @@ SOURCES += \
../../testing/testingmain.cpp \
../../indexer/indexer_tests/feature_routine.cpp \
feature_bucketer_test.cpp \
osm_parser_test.cpp \

201
generator/generator_tests/osm_parser_test.cpp Normal file
View file

@ -0,0 +1,201 @@
#include "../../testing/testing.hpp"
#include "../osm_xml_parser.hpp"
#include "../borders_generator.hpp"
#include "../../coding/reader.hpp"
#include "../../coding/parse_xml.hpp"
#include "../../coding/file_reader.hpp"
#include "../../coding/file_writer.hpp"
using namespace osm;
static char const gOsmXml[] =
"<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
"<osm version=\"0.6\" generator=\"OpenStreetMap server\">"
" <node id=\"1\" lat=\"0.0\" lon=\"0.0\" visible=\"true\"/>"
" <node id=\"2\" lat=\"20.0\" lon=\"0.0\"/>"
" <node id=\"3\" lat=\"20.0\" lon=\"20.0\"/>"
" <node id=\"4\" lat=\"0.0\" lon=\"20.0\"/>"
" <node id=\"7\" lat=\"5.0\" lon=\"10.0\"/>"
" <node id=\"8\" lat=\"10.0\" lon=\"10.0\"/>"
" <node id=\"9\" lat=\"10.0\" lon=\"15.0\"/>"
" <node id=\"10\" lat=\"5.555555\" lon=\"4.222222\">"
" <tag k=\"name\" v=\"Andorra la Vella\"/>"
" <tag k=\"capital\" v=\"yes\"/>"
" </node>"
" <node id=\"11\" lat=\"105.0\" lon=\"110.0\"/>"
" <node id=\"12\" lat=\"110.0\" lon=\"110.0\"/>"
" <node id=\"13\" lat=\"110.0\" lon=\"115.0\"/>"
" <way id=\"100\" visible=\"true\">"
" <nd ref=\"1\"/>"
" <nd ref=\"2\"/>"
" <nd ref=\"3\"/>"
" <tag k=\"boundary\" v=\"administrative\"/>"
" <tag k=\"admin_level\" v=\"2\"/>"
" </way>"
" <way id=\"101\" visible=\"true\">"
" <nd ref=\"1\"/>"
" <nd ref=\"4\"/>"
" <nd ref=\"3\"/>"
" <tag k=\"boundary\" v=\"administrative\"/>"
" <tag k=\"admin_level\" v=\"2\"/>"
" </way>"
" <way id=\"102\" visible=\"true\">"
" <nd ref=\"11\"/>"
" <nd ref=\"12\"/>"
" <nd ref=\"13\"/>"
" <nd ref=\"11\"/>"
" <tag k=\"boundary\" v=\"administrative\"/>"
" <tag k=\"admin_level\" v=\"2\"/>"
" </way>"
" <way id=\"121\" visible=\"true\">"
" <nd ref=\"7\"/>"
" <nd ref=\"8\"/>"
" <nd ref=\"9\"/>"
" <nd ref=\"7\"/>"
" <tag k=\"boundary\" v=\"administrative\"/>"
" <tag k=\"admin_level\" v=\"2\"/>"
" </way>"
" <relation id=\"444\" visible=\"true\">"
" <member type=\"way\" ref=\"121\" role=\"outer\"/>"
" <tag k=\"admin_level\" v=\"4\"/>"
" <tag k=\"boundary\" v=\"administrative\"/>"
" <tag k=\"name\" v=\"Some Region\"/>"
" <tag k=\"type\" v=\"boundary\"/>"
" </relation>"
" <relation id=\"555\" visible=\"true\">"
" <member type=\"node\" ref=\"10\" role=\"capital\"/>"
" <member type=\"way\" ref=\"100\" role=\"\"/>"
" <member type=\"way\" ref=\"101\" role=\"outer\"/>"
" <member type=\"way\" ref=\"102\" role=\"\"/>"
" <member type=\"way\" ref=\"121\" role=\"inner\"/>"
" <member type=\"relation\" ref=\"444\" role=\"subarea\"/>"
" <tag k=\"admin_level\" v=\"2\"/>"
" <tag k=\"boundary\" v=\"administrative\"/>"
" <tag k=\"ISO3166-1\" v=\"ad\"/>"
" <tag k=\"name\" v=\"Andorra\"/>"
" <tag k=\"name:en\" v=\"Andorra\"/>"
" <tag k=\"type\" v=\"boundary\"/>"
" </relation>"
"</osm>";
#define TEST_EXCEPTION(exception, expression) do { \
bool gotException = false; \
try { expression; } \
catch (exception const &) { gotException = true; } \
TEST(gotException, ("Exception should be thrown:", #exception)); \
} while(0)
struct PointsTester
{
list<OsmId> m_controlPoints;
PointsTester()
{
m_controlPoints.push_back(1);
m_controlPoints.push_back(2);
m_controlPoints.push_back(3);
m_controlPoints.push_back(4);
m_controlPoints.push_back(1);
}
void operator()(OsmId const & ptId)
{
TEST(!m_controlPoints.empty(), ());
TEST_EQUAL(m_controlPoints.front(), ptId, ());
m_controlPoints.pop_front();
}
bool IsOk() const
{
return m_controlPoints.empty();
}
};
UNIT_TEST(OsmRawData_SmokeTest)
{
OsmRawData osmData;
{
// -1 to avoid finishing zero at the end of the string
MemReader xmlBlock(gOsmXml, ARRAY_SIZE(gOsmXml) - 1);
ReaderSource<MemReader> source(xmlBlock);
OsmXmlParser parser(osmData);
TEST(ParseXML(source, parser), ("Invalid XML"));
}
string outTagValue;
TEST_EXCEPTION(OsmRawData::OsmInvalidIdException, OsmNode node = osmData.NodeById(98764));
OsmNode node = osmData.NodeById(9);
TEST_EQUAL(node.m_lat, 10.0, ());
TEST_EQUAL(node.m_lon, 15.0, ());
TEST_EXCEPTION(OsmRawData::OsmInvalidIdException, OsmWay way = osmData.WayById(635794));
OsmWay way = osmData.WayById(100);
TEST_EQUAL(way.PointsCount(), 3, ());
TEST(!way.TagValueByKey("invalid_tag", outTagValue), ());
TEST(way.TagValueByKey("boundary", outTagValue), ());
TEST_EQUAL(outTagValue, "administrative", ());
TEST(way.TagValueByKey("admin_level", outTagValue), ());
TEST_EQUAL(outTagValue, "2", ());
OsmWay way2 = osmData.WayById(101);
TEST(way.MergeWith(way2), ());
TEST_EQUAL(way.PointsCount(), 5, ());
PointsTester tester;
way.ForEachPoint(tester);
TEST(tester.IsOk(), ());
TEST(way.IsClosed(), ());
TEST_EXCEPTION(OsmRawData::OsmInvalidIdException, OsmRelation relation = osmData.RelationById(64342));
OsmRelation rel1 = osmData.RelationById(444);
TEST(rel1.TagValueByKey("admin_level", outTagValue), ());
TEST_EQUAL(outTagValue, "4", ());
OsmIds relations = osmData.RelationsByKey("invalid_tag_key");
TEST(relations.empty(), ());
relations = osmData.RelationsByKey("ISO3166-1");
TEST_EQUAL(relations.size(), 1, ());
TEST_EQUAL(relations[0], 555, ());
OsmRelation rel2 = osmData.RelationById(relations[0]);
OsmIds members = rel2.MembersByTypeAndRole("way", "");
TEST_EQUAL(members.size(), 2, ());
TEST_EQUAL(members[0], 100, ());
members = rel2.MembersByTypeAndRole("way", "outer");
TEST_EQUAL(members.size(), 1, ());
TEST_EQUAL(members[0], 101, ());
members = rel2.MembersByTypeAndRole("relation", "invalid_role");
TEST(members.empty(), ());
relations.clear();
relations = osmData.RelationsByTag(OsmTag("boundary_invalid", "administrative"));
TEST(relations.empty(), ());
relations = osmData.RelationsByTag(OsmTag("boundary", "administrative"));
TEST_EQUAL(relations.size(), 2, ());
}
UNIT_TEST(BordersGenerator)
{
string const inputFile("BordersTestInputFile");
string const outputFile("BordersTestOutputFile");
{ // create input file
FileWriter f(inputFile);
f.Write(gOsmXml, ARRAY_SIZE(gOsmXml) - 1); // -1 to skip last zero
}
GenerateBordersFromOsm(inputFile, outputFile);
vector<m2::RegionD> borders;
LoadBorders(outputFile, borders);
TEST_EQUAL(borders.size(), 2, ());
FileWriter::DeleteFileX(inputFile);
FileWriter::DeleteFileX(outputFile);
}

15
generator/generator_tool/generator_tool.cpp
View file

@ -6,6 +6,7 @@
#include "../grid_generator.hpp"
#include "../statistics.hpp"
#include "../classif_routine.hpp"
#include "../borders_generator.hpp"
#include "../../indexer/features_vector.hpp"
#include "../../indexer/index_builder.hpp"
@ -51,6 +52,8 @@ DEFINE_int32(generate_world_scale, -1, "If specified, features for zoomlevels [0
DEFINE_bool(split_by_polygons, false, "Use kml shape files to split planet by regions and countries");
DEFINE_int32(simplify_countries_level, -1, "If positive, simplifies country polygons. Recommended values [10..15]");
DEFINE_bool(merge_coastlines, false, "If defined, tries to merge coastlines when renerating World file");
DEFINE_bool(generate_borders, false,
"Create binary country .borders file for osm xml file given in 'output' parameter");
string AddSlashIfNeeded(string const & str)
{
@ -193,5 +196,17 @@ int main(int argc, char ** argv)
update::GenerateFilesList(path);
}
if (FLAGS_generate_borders)
{
if (!FLAGS_output.empty())
{
osm::GenerateBordersFromOsm(path + FLAGS_output + ".osm", path + FLAGS_output + ".borders");
}
else
{
LOG(LINFO, ("Please specify osm country borders file in 'output' command line parameter."));
}
}
return 0;
}

64
generator/kml_parser.cpp
View file

@ -218,7 +218,7 @@ namespace kml
}
}
bool LoadPolygonsFromKml(string const & kmlFile, PolygonsContainerT & country, int level)
bool LoadPolygons(string const & kmlFile, PolygonsContainerT & country, int level)
{
KmlParser parser(country, level);
try
@ -233,66 +233,4 @@ namespace kml
}
return false;
}
class PolygonLoader
{
string m_baseDir;
int m_level;
CountryPolygons & m_polygons;
m2::RectD & m_rect;
public:
PolygonLoader(string const & basePolygonsDir, int level, CountryPolygons & polygons, m2::RectD & rect)
: m_baseDir(basePolygonsDir), m_level(level), m_polygons(polygons), m_rect(rect)
{
}
void operator()(string const & name)
{
if (m_polygons.m_name.empty())
m_polygons.m_name = name;
PolygonsContainerT current;
if (LoadPolygonsFromKml(m_baseDir + BORDERS_DIR + name + BORDERS_EXTENSION, current, m_level))
{
for (size_t i = 0; i < current.size(); ++i)
{
m2::RectD const rect(current[i].GetRect());
m_rect.Add(rect);
m_polygons.m_regions.Add(current[i], rect);
}
}
}
};
bool LoadCountriesList(string const & baseDir, CountriesContainerT & countries,
int simplifyCountriesLevel)
{
if (simplifyCountriesLevel > 0)
{
LOG_SHORT(LINFO, ("Simplificator level for country polygons:", simplifyCountriesLevel));
}
countries.Clear();
ifstream stream((baseDir + POLYGONS_FILE).c_str());
string line;
LOG(LINFO, ("Loading countries."));
while (stream.good())
{
std::getline(stream, line);
if (line.empty())
continue;
CountryPolygons country;
m2::RectD rect;
PolygonLoader loader(baseDir, simplifyCountriesLevel, country, rect);
utils::TokenizeString(line, "|", loader);
if (!country.m_regions.IsEmpty())
countries.Add(country, rect);
}
LOG(LINFO, ("Countries loaded:", countries.GetSize()));
return !countries.IsEmpty();
}
}

22
generator/kml_parser.hpp
View file

@ -1,28 +1,10 @@
#pragma once
#include "../geometry/region2d.hpp"
#include "../geometry/tree4d.hpp"
#include "../std/string.hpp"
class PolygonsContainerT;
namespace kml
{
typedef m2::RegionD Region;
typedef m4::Tree<Region> RegionsContainerT;
struct CountryPolygons
{
CountryPolygons(string const & name = "") : m_name(name), m_index(-1) {}
RegionsContainerT m_regions;
string m_name;
mutable int m_index;
};
typedef m4::Tree<CountryPolygons> CountriesContainerT;
/// @param[in] simplifyCountriesLevel if positive, used as a level for simplificator
bool LoadCountriesList(string const & baseDir, CountriesContainerT & countries,
int simplifyCountriesLevel = -1);
bool LoadPolygons(string const & kmlFile, PolygonsContainerT & country, int level);
}

244
generator/osm_xml_parser.cpp Normal file
View file

@ -0,0 +1,244 @@
#include "osm_xml_parser.hpp"
#include "../base/assert.hpp"
#include "../base/string_utils.hpp"
namespace osm
{
OsmIdAndTagHolder::OsmIdAndTagHolder(OsmId id, OsmTags const & tags)
: m_id(id), m_tags(tags)
{
}
bool OsmIdAndTagHolder::TagValueByKey(string const & key, string & outValue) const
{
for (OsmTags::const_iterator it = m_tags.begin(); it != m_tags.end(); ++it)
{
if (it->first == key)
{
outValue = it->second;
return true;
}
}
return false;
}
/////////////////////////////////////////////////////////
OsmNode::OsmNode(OsmId id, OsmTags const & tags, double lat, double lon)
: OsmIdAndTagHolder(id, tags), m_lat(lat), m_lon(lon)
{
}
/////////////////////////////////////////////////////////
OsmWay::OsmWay(OsmId id, OsmTags const & tags, OsmIds const & pointIds)
: OsmIdAndTagHolder(id, tags), m_points(pointIds)
{
CHECK_GREATER_OR_EQUAL(m_points.size(), 2, ("Can't construct a way with less than 2 points", id));
}
size_t OsmWay::PointsCount() const
{
return m_points.size();
}
bool OsmWay::IsClosed() const
{
return m_points.front() == m_points.back();
}
bool OsmWay::MergeWith(OsmWay const & way)
{
size_t const oldSize = m_points.size();
// first, try to connect our end with their start
if (m_points.back() == way.m_points.front())
m_points.insert(m_points.end(), ++way.m_points.begin(), way.m_points.end());
// our end and their end
else if (m_points.back() == way.m_points.back())
m_points.insert(m_points.end(), ++way.m_points.rbegin(), way.m_points.rend());
// our start and their end
else if (m_points.front() == way.m_points.back())
m_points.insert(m_points.begin(), way.m_points.begin(), --way.m_points.end());
// our start and their start
else if (m_points.front() == way.m_points.front())
m_points.insert(m_points.begin(), way.m_points.rbegin(), --way.m_points.rend());
return m_points.size() > oldSize;
}
///////////////////////////////////////////////////////////////
OsmRelation::OsmRelation(OsmId id, OsmTags const & tags, RelationMembers const & members)
: OsmIdAndTagHolder(id, tags), m_members(members)
{
CHECK(!m_members.empty(), ("Can't construct a relation without members", id));
}
OsmIds OsmRelation::MembersByTypeAndRole(string const & type, string const & role) const
{
OsmIds result;
for (RelationMembers::const_iterator it = m_members.begin(); it != m_members.end(); ++it)
if (it->m_type == type && it->m_role == role)
result.push_back(it->m_ref);
return result;
}
///////////////////////////////////////////////////////////////
void OsmRawData::AddNode(OsmId id, OsmTags const & tags, double lat, double lon)
{
m_nodes.insert(nodes_type::value_type(id, OsmNode(id, tags, lat, lon)));
}
void OsmRawData::AddWay(OsmId id, OsmTags const & tags, OsmIds const & nodeIds)
{
m_ways.insert(ways_type::value_type(id, OsmWay(id, tags, nodeIds)));
}
void OsmRawData::AddRelation(OsmId id, OsmTags const & tags, RelationMembers const & members)
{
m_relations.insert(relations_type::value_type(id, OsmRelation(id, tags, members)));
}
OsmNode OsmRawData::NodeById(OsmId id) const throw (OsmInvalidIdException)
{
nodes_type::const_iterator found = m_nodes.find(id);
if (found == m_nodes.end())
MYTHROW( OsmInvalidIdException, (id, "node not found") );
return found->second;
}
OsmWay OsmRawData::WayById(OsmId id) const throw (OsmInvalidIdException)
{
ways_type::const_iterator found = m_ways.find(id);
if (found == m_ways.end())
MYTHROW( OsmInvalidIdException, (id, "way not found") );
return found->second;
}
OsmRelation OsmRawData::RelationById(OsmId id) const throw (OsmInvalidIdException)
{
relations_type::const_iterator found = m_relations.find(id);
if (found == m_relations.end())
MYTHROW( OsmInvalidIdException, (id, "relation not found") );
return found->second;
}
OsmIds OsmRawData::RelationsByKey(string const & key) const
{
OsmIds result;
string value;
for (relations_type::const_iterator it = m_relations.begin(); it != m_relations.end(); ++it)
{
if (it->second.TagValueByKey(key, value))
result.push_back(it->first);
}
return result;
}
OsmIds OsmRawData::RelationsByTag(OsmTag const & tag) const
{
OsmIds result;
string value;
for (relations_type::const_iterator it = m_relations.begin(); it != m_relations.end(); ++it)
{
if (it->second.TagValueByKey(tag.first, value) && value == tag.second)
result.push_back(it->first);
}
return result;
}
/////////////////////////////////////////////////////////
OsmXmlParser::OsmXmlParser(OsmRawData & outData)
: m_osmRawData(outData)
{
}
bool OsmXmlParser::Push(string const & element)
{
m_xmlTags.push_back(element);
return true;
}
void OsmXmlParser::Pop(string const & element)
{
if (element == "node")
{
m_osmRawData.AddNode(m_id, m_tags, m_lat, m_lon);
m_tags.clear();
}
else if (element == "nd")
{
m_nds.push_back(m_ref);
}
else if (element == "way")
{
m_osmRawData.AddWay(m_id, m_tags, m_nds);
m_nds.clear();
m_tags.clear();
}
else if (element == "tag")
{
m_tags.push_back(OsmTag(m_k, m_v));
}
else if (element == "member")
{
m_members.push_back(m_member);
}
else if (element == "relation")
{
m_osmRawData.AddRelation(m_id, m_tags, m_members);
m_members.clear();
m_tags.clear();
}
m_xmlTags.pop_back();
}
void OsmXmlParser::AddAttr(string const & attr, string const & value)
{
string const & elem = m_xmlTags.back();
if (attr == "id" && (elem == "node" || elem == "way" || elem == "relation"))
{
uint64_t numVal;
CHECK(utils::to_uint64(value, numVal), ());
m_id = static_cast<int64_t>(numVal);
}
else if (attr == "lat" && elem == "node")
{
CHECK(utils::to_double(value, m_lat), ());
}
else if (attr == "lon" && elem == "node")
{
CHECK(utils::to_double(value, m_lon), ());
}
else if (attr == "ref")
{
uint64_t numVal;
CHECK(utils::to_uint64(value, numVal), ());
if (elem == "nd")
m_ref = static_cast<int64_t>(numVal);
else if (elem == "member")
m_member.m_ref = numVal;
}
else if (attr == "k" && elem == "tag")
{
m_k = value;
}
else if (attr == "v" && elem == "tag")
{
m_v = value;
}
else if (attr == "type" && elem == "member")
{
m_member.m_type = value;
}
else if (attr == "role" && elem == "member")
{
m_member.m_role = value;
}
}
}

138
generator/osm_xml_parser.hpp Normal file
View file

@ -0,0 +1,138 @@
#pragma once
#include "../base/exception.hpp"
#include "../geometry/point2d.hpp"
#include "../std/vector.hpp"
#include "../std/map.hpp"
#include "../std/set.hpp"
#include "../std/string.hpp"
namespace osm
{
typedef int64_t OsmId;
typedef vector<OsmId> OsmIds;
typedef pair<string, string> OsmTag;
typedef vector<OsmTag> OsmTags;
struct RelationMember
{
OsmId m_ref;
string m_type;
string m_role;
};
typedef vector<RelationMember> RelationMembers;
class OsmIdAndTagHolder
{
OsmId m_id;
OsmTags m_tags;
public:
OsmIdAndTagHolder(OsmId id, OsmTags const & tags);
OsmId Id() const { return m_id; }
bool TagValueByKey(string const & key, string & outValue) const;
template <class TFunctor> void ForEachTag(TFunctor & functor) const
{
for (OsmTags::const_iterator it = m_tags.begin(); it != m_tags.end(); ++it)
functor(*it);
}
};
class OsmNode : public OsmIdAndTagHolder
{
public:
double m_lat;
double m_lon;
OsmNode(OsmId id, OsmTags const & tags, double lat, double lon);
};
class OsmWay : public OsmIdAndTagHolder
{
OsmIds m_points;
public:
OsmWay(OsmId id, OsmTags const & tags, OsmIds const & pointIds);
size_t PointsCount() const;
/// checks if first and last points are equal
bool IsClosed() const;
/// Merges ways if they have one common point
/// @warning do not use it where merged way direction is important! (coastlines for example)
bool MergeWith(OsmWay const & way);
template <class TFunctor> void ForEachPoint(TFunctor & functor) const
{
for (typename OsmIds::const_iterator it = m_points.begin(); it != m_points.end(); ++it)
functor(*it);
}
};
typedef vector<OsmWay> OsmWays;
class OsmRelation : public OsmIdAndTagHolder
{
RelationMembers m_members;
public:
OsmRelation(OsmId id, OsmTags const & tags, RelationMembers const & members);
OsmIds MembersByTypeAndRole(string const & type, string const & role) const;
};
class OsmRawData
{
typedef map<OsmId, OsmNode> nodes_type;
nodes_type m_nodes;
typedef map<OsmId, OsmWay> ways_type;
ways_type m_ways;
typedef map<OsmId, OsmRelation> relations_type;
relations_type m_relations;
public:
DECLARE_EXCEPTION(OsmInvalidIdException, RootException);
void AddNode(OsmId id, OsmTags const & tags, double lat, double lon);
void AddWay(OsmId id, OsmTags const & tags, OsmIds const & nodeIds);
void AddRelation(OsmId id, OsmTags const & tags, RelationMembers const & members);
OsmNode NodeById(OsmId id) const throw (OsmInvalidIdException);
OsmWay WayById(OsmId id) const throw (OsmInvalidIdException);
OsmRelation RelationById(OsmId id) const throw (OsmInvalidIdException);
OsmIds RelationsByKey(string const & key) const;
OsmIds RelationsByTag(OsmTag const & tag) const;
};
class OsmXmlParser
{
vector<string> m_xmlTags;
OsmRawData & m_osmRawData;
OsmId m_id;
double m_lon;
double m_lat;
/// <tag k="..." v="..."/>
string m_k;
string m_v;
OsmTags m_tags;
/// <nd ref="..." />
OsmId m_ref;
OsmIds m_nds;
RelationMember m_member;
RelationMembers m_members;
public:
OsmXmlParser(OsmRawData & outData);
bool Push(string const & element);
void Pop(string const & element);
void AddAttr(string const & attr, string const & value);
void CharData(string const &) {}
};
}

24
generator/polygonizer.hpp
View file

@ -1,5 +1,5 @@
#pragma once
#include "kml_parser.hpp"
#include "borders_loader.hpp"
#include "world_map_generator.hpp"
#include "../indexer/feature.hpp"
@ -42,7 +42,7 @@ namespace feature
LOG(LINFO, ("Polygonizer thread pool threads:", m_ThreadPool.maxThreadCount()));
#endif
CHECK(kml::LoadCountriesList(info.datFilePrefix, m_countries, info.simplifyCountriesLevel),
CHECK(borders::LoadCountriesList(info.datFilePrefix, m_countries, info.simplifyCountriesLevel),
("Error loading country polygons files"));
//LOG_SHORT(LINFO, ("Loaded polygons count for regions:"));
@ -59,10 +59,10 @@ namespace feature
struct PointChecker
{
kml::RegionsContainerT const & m_regions;
borders::RegionsContainerT const & m_regions;
bool m_belongs;
PointChecker(kml::RegionsContainerT const & regions)
PointChecker(borders::RegionsContainerT const & regions)
: m_regions(regions), m_belongs(false) {}
bool operator()(m2::PointD const & pt)
@ -71,7 +71,7 @@ namespace feature
return !m_belongs;
}
void operator() (kml::Region const & rgn, kml::Region::value_type const & point)
void operator() (borders::Region const & rgn, borders::Region::value_type const & point)
{
if (!m_belongs)
m_belongs = rgn.Contains(point);
@ -80,12 +80,12 @@ namespace feature
class InsertCountriesPtr
{
typedef buffer_vector<kml::CountryPolygons const *, 32> vec_type;
typedef buffer_vector<borders::CountryPolygons const *, 32> vec_type;
vec_type & m_vec;
public:
InsertCountriesPtr(vec_type & vec) : m_vec(vec) {}
void operator() (kml::CountryPolygons const & c)
void operator() (borders::CountryPolygons const & c)
{
m_vec.push_back(&c);
}
@ -95,7 +95,7 @@ namespace feature
{
m_worldMap(fb);
buffer_vector<kml::CountryPolygons const *, 32> vec;
buffer_vector<borders::CountryPolygons const *, 32> vec;
m_countries.ForEachInRect(fb.GetLimitRect(), InsertCountriesPtr(vec));
switch (vec.size())
@ -125,7 +125,7 @@ namespace feature
#endif
}
void EmitFeature(kml::CountryPolygons const * country, FeatureBuilder1 const & fb)
void EmitFeature(borders::CountryPolygons const * country, FeatureBuilder1 const & fb)
{
#if PARALLEL_POLYGONIZER
QMutexLocker mutexLocker(&m_EmitFeatureMutex);
@ -151,7 +151,7 @@ namespace feature
vector<FeatureOutT*> m_Buckets;
vector<string> m_Names;
kml::CountriesContainerT m_countries;
borders::CountriesContainerT m_countries;
WorldMapGenerator<FeatureOutT> m_worldMap;
#if PARALLEL_POLYGONIZER
@ -169,7 +169,7 @@ namespace feature
{
public:
PolygonizerTask(Polygonizer * pPolygonizer,
buffer_vector<kml::CountryPolygons const *, 32> const & countries,
buffer_vector<borders::CountryPolygons const *, 32> const & countries,
FeatureBuilder1 const & fb)
: m_pPolygonizer(pPolygonizer), m_Countries(countries), m_FB(fb) {}
@ -196,7 +196,7 @@ namespace feature
private:
Polygonizer * m_pPolygonizer;
buffer_vector<kml::CountryPolygons const *, 32> m_Countries;
buffer_vector<borders::CountryPolygons const *, 32> m_Countries;
FeatureBuilder1 m_FB;
};
};

17
tools/unix/polygons.sh
View file

@ -10,6 +10,7 @@ set -e -u -x
# global params
LIGHT_NODES=false
PROCESSORS=4
SIMPLIFY=-1
# displays usage and exits
function Usage {
@ -46,13 +47,13 @@ BUCKETING_LEVEL=$2
#fi
# check if we have QT in PATH
if [ ! `which qmake` ]; then
echo 'You should add your qmake binary into the PATH. This can be done in 2 ways:'
echo ' 1. Set it temporarily by executing: export PATH=/c/qt/your_qt_dir/bin:$PATH'
echo ' 2. Set it permanently by adding export... string above to your ~/.bashrc'
echo 'Hint: for second solution you can type from git bash console: notepad ~/.bashrc'
exit 0
fi
#if [ ! `which qmake` ]; then
# echo 'You should add your qmake binary into the PATH. This can be done in 2 ways:'
# echo ' 1. Set it temporarily by executing: export PATH=/c/qt/your_qt_dir/bin:$PATH'
# echo ' 2. Set it permanently by adding export... string above to your ~/.bashrc'
# echo 'Hint: for second solution you can type from git bash console: notepad ~/.bashrc'
# exit 0
#fi
# determine script path
MY_PATH=`dirname $0`
@ -115,7 +116,7 @@ fi
# 2nd pass - not paralleled
$PV $OSM_BZ2 | bzip2 -d | $GENERATOR_TOOL --intermediate_data_path=$TMPDIR \
--use_light_nodes=$LIGHT_NODES --split_by_polygons -simplify_countries_level=10 \
--use_light_nodes=$LIGHT_NODES --split_by_polygons --simplify_countries_level=$SIMPLIFY \
--generate_features --generate_world_scale=6 --merge_coastlines=true \
--data_path=$DATA_PATH