RedAuburn/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

Merge pull request #3804 from Zverik/mwm_extra

Browse source 
[generator] Add region info section to mwms
This commit is contained in:
mpimenov 2016-07-19 13:38:36 +03:00 • committed by GitHub
commit 751e6ea7a9
19 changed files with 1155 additions and 41 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
3party/jansson/myjansson.cpp
View file

@ -81,4 +81,32 @@ void FromJSONObjectOptionalField(json_t * root, string const & field, json_int_t
MYTHROW(my::Json::Exception, ("The field", field, "must contain a json number."));
result = json_integer_value(val);
}
void FromJSONObjectOptionalField(json_t * root, string const & field, bool & result, bool def)
{
if (!json_is_object(root))
MYTHROW(my::Json::Exception, ("Bad json object when parsing", field));
json_t * val = json_object_get(root, field.c_str());
if (!val)
{
result = def;
return;
}
if (!json_is_boolean(val))
MYTHROW(my::Json::Exception, ("The field", field, "must contain a boolean value."));
result = json_is_true(val);
}
void FromJSONObjectOptionalField(json_t * root, string const & field, json_t *& result)
{
json_t * obj = json_object_get(root, field.c_str());
if (!obj)
{
result = nullptr;
return;
}
if (!json_is_object(obj))
MYTHROW(my::Json::Exception, ("The field", field, "must contain a json object."));
FromJSON(obj, result);
}
} // namespace my

2
3party/jansson/myjansson.hpp
View file

@ -52,6 +52,8 @@ void FromJSONObject(json_t * root, string const & field, vector<T> & result)
void FromJSONObjectOptionalField(json_t * root, string const & field, string & result);
void FromJSONObjectOptionalField(json_t * root, string const & field, json_int_t & result);
void FromJSONObjectOptionalField(json_t * root, string const & field, bool & result, bool def = false);
void FromJSONObjectOptionalField(json_t * root, string const & field, json_t *& result);
template <typename T>
void FromJSONObjectOptionalField(json_t * root, string const & field, vector<T> & result)

819
data/countries_meta.txt Normal file
View file

@ -0,0 +1,819 @@
{
"Abkhazia": {
"languages": ["ab", "ru"]
},
"Afghanistan": {
"languages": ["ps"]
},
"Albania": {
"languages": ["sq"]
},
"Algeria": {
"languages": ["ar"]
},
"Andorra": {
"languages": ["ca"]
},
"Angola": {
"languages": ["pt"]
},
"Anguilla": {
"driving": "l",
"languages": ["en"]
},
"Antigua and Barbuda": {
"driving": "l",
"languages": ["en"]
},
"Barbados": {
"driving": "l",
"languages": ["en"]
},
"Dominica": {
"driving": "l",
"languages": ["en"]
},
"Grenada": {
"driving": "l",
"languages": ["en"]
},
"Montserrat": {
"languages": ["en"]
},
"Saint Barthelemy": {
"languages": ["fr"]
},
"Saint Kitts and Nevis": {
"driving": "l",
"languages": ["en"]
},
"Saint Lucia": {
"driving": "l",
"languages": ["en"]
},
"Saint Vincent and the Grenadines": {
"driving": "l",
"languages": ["en"]
},
"British Virgin Islands": {
"driving": "l"
},
"Turks and Caicos Islands": {
"driving": "l"
},
"Trinidad and Tobago": {
"driving": "l",
"languages": ["en"]
},
"United States Virgin Islands": {
"driving": "l",
"languages": ["en"]
},
"Argentina": {
"languages": ["es"]
},
"Armenia": {
"languages": ["hy"]
},
"Austria": {
"languages": ["de"]
},
"Australia": {
"driving": "l",
"languages": ["en"]
},
"Azerbaijan": {
"languages": ["az"]
},
"Bahrain": {
"languages": ["ar"]
},
"Bangladesh": {
"driving": "l",
"languages": ["bn"]
},
"Belarus": {
"languages": ["ru"]
},
"Belgium": {
"languages": ["fr", "de", "nl"]
},
"Belgium_Limburg": {
"languages": ["nl"]
},
"Belize": {
"languages": ["en"]
},
"Benin": {
"languages": ["fr"]
},
"Bermuda": {
"driving": "l",
"languages": ["en"]
},
"Bhutan": {
"driving": "l",
"languages": ["dz"]
},
"Bolivia": {
"languages": ["es", "ay", "qu", "gn"]
},
"Bosnia and Herzegovina": {
"languages": ["sr"]
},
"Bosnia and Herzegovina_Entity Federation of Bosnia and Herzegovina": {
"languages": ["hr", "sr", "bs"]
},
"Botswana": {
"driving": "l",
"languages": ["en", "tn"]
},
"Brazil": {
"languages": ["pt"]
},
"Brunei": {
"driving": "l",
"languages": ["ms"]
},
"Bulgaria": {
"languages": ["bg"]
},
"Burkina Faso": {
"languages": ["fr"]
},
"Burundi": {
"languages": ["fr", "rn"]
},
"Cambodia": {
"languages": ["km"]
},
"Cameroon": {
"languages": ["fr", "en"]
},
"Canada": {
"languages": ["fr", "en"]
},
"Canada_Quebec": {
"languages": ["fr"]
},
"Cape Verde": {
"languages": ["pt"]
},
"Cayman Islands": {
"driving": "l",
"languages": ["en"]
},
"Central African Republic": {
"languages": ["fr", "sg"]
},
"Chad": {
"languages": ["fr", "ar"]
},
"Colombia": {
"languages": ["es"]
},
"Comoros": {
"languages": ["fr", "ar"]
},
"Congo-Brazzaville": {
"languages": ["fr"]
},
"Congo-Kinshasa": {
"languages": ["fr"]
},
"Cook Islands": {
"driving": "l",
"languages": ["en"]
},
"Costa Rica": {
"languages": ["es"]
},
"Croatia": {
"languages": ["hr"]
},
"Cuba": {
"languages": ["es"]
},
"Cyprus": {
"driving": "l",
"languages": ["tr"]
},
"Czech": {
"languages": ["cs"]
},
"Cote dIvoire": {
"languages": ["fr"]
},
"Denmark": {
"languages": ["da"]
},
"Djibouti": {
"languages": ["fr", "ar"]
},
"Dominican Republic": {
"languages": ["es"]
},
"East Timor": {
"driving": "l",
"languages": ["pt"]
},
"Campo de Hielo Sur": {
"languages": ["es"]
},
"Chile": {
"languages": ["es"]
},
"Ecuador": {
"languages": ["es"]
},
"Egypt": {
"languages": ["ar"]
},
"El Salvador": {
"languages": ["es"]
},
"Eritrea": {
"languages": ["en", "ar", "ti"]
},
"Estonia": {
"languages": ["et"]
},
"Ethiopia": {
"languages": ["am"]
},
"Faroe Islands": {
"languages": ["da", "fo"]
},
"Falkland Islands": {
"driving": "l"
}
"Fiji": {
"driving": "l",
"languages": ["en", "fj"]
},
"Finland": {
"languages": ["fi", "sv"]
},
"France": {
"languages": ["fr"]
},
"French Polynesia": {
"languages": ["fr"]
},
"Wallis and Futuna": {
"languages": ["fr"]
},
"Gabon": {
"languages": ["fr"]
},
"Georgia": {
"languages": ["ka"]
},
"Germany": {
"languages": ["de"]
},
"Ghana": {
"languages": ["en"]
},
"Gibraltar": {
"languages": ["en"]
},
"Greece": {
"languages": ["el"]
},
"Greenland": {
"languages": ["kl"]
},
"Guatemala": {
"languages": ["es"]
},
"Guernsey": {
"driving": "l",
"languages": ["fr", "en"]
},
"Guinea": {
"languages": ["fr"]
},
"Guinea-Bissau": {
"languages": ["pt"]
},
"Guyana": {
"driving": "l",
"languages": ["en"]
},
"Haiti": {
"languages": ["fr", "ht"]
},
"Honduras": {
"languages": ["es"]
},
"Hungary": {
"languages": ["hu"]
},
"Iceland": {
"languages": ["is"]
},
"India": {
"driving": "l",
"languages": ["hi", "mr", "en", "gu", "ta", "te", "bn", "as", "kn", "ml", "pa"]
},
"India_Kerala": {
"driving": "l",
"languages": ["en", "ml"]
},
"India_Rajasthan": {
"driving": "l",
"languages": ["hi"]
},
"India_Tamil Nadu": {
"driving": "l",
"languages": ["ta"]
},
"India_Haryana": {
"driving": "l",
"languages": ["hi"]
},
"India_Maharashtra": {
"driving": "l",
"languages": ["mr"]
},
"India_Delhi": {
"driving": "l",
"languages": ["hi", "ur", "en", "pa"]
},
"India_Uttar Pradesh": {
"driving": "l",
"languages": ["hi", "ur"]
},
"India_Chhattisgarh": {
"driving": "l",
"languages": ["hi"]
},
"India_Punjab": {
"driving": "l",
"languages": ["pa"]
},
"Indonesia": {
"driving": "l",
"languages": ["id"]
},
"Iran": {
"languages": ["fa"]
},
"Iraq": {
"languages": ["ar", "ku"]
},
"Isle of Man": {
"driving": "l",
"languages": ["en", "gv"]
},
"Jerusalem": {
"languages": ["he"]
},
"Israel": {
"languages": ["he"]
},
"Italy": {
"languages": ["it"]
},
"Jamaica": {
"driving": "l",
"languages": ["en"]
},
"Japan": {
"driving": "l",
"languages": ["ja"]
},
"Jersey": {
"driving": "l",
"languages": ["fr", "en"]
},
"Jordan": {
"languages": ["ar"]
},
"Kazakhstan": {
"languages": ["kk"]
},
"Kenya": {
"driving": "l",
"languages": ["en", "sw"]
},
"Kingdom of Lesotho": {
"driving": "l",
"languages": ["en", "st"]
},
"Kiribati": {
"driving": "l",
"languages": ["en"]
},
"Kuwait": {
"languages": ["ar"]
},
"Kyrgyzstan": {
"languages": ["ru", "ky"]
},
"Laos": {
"languages": ["lo"]
},
"Latvia": {
"languages": ["lv"]
},
"Lebanon": {
"languages": ["ar"]
},
"Liberia": {
"languages": ["en"]
},
"Libya": {
"languages": ["ar"]
},
"Liechtenstein": {
"languages": ["de"]
},
"Lithuania": {
"languages": ["lt"]
},
"Luxembourg": {
"languages": ["fr"]
},
"Macedonia": {
"languages": ["mk"]
},
"Malawi": {
"driving": "l",
"languages": ["en", "ny"]
},
"Malaysia": {
"driving": "l",
"languages": ["en"]
},
"Maldives": {
"driving": "l",
"languages": ["dv"]
},
"Mali": {
"languages": ["fr"]
},
"Malta": {
"driving": "l",
"languages": ["en", "mt"]
},
"Mauritius": {
"driving": "l"
},
"Marshall Islands": {
"languages": ["en", "mh"]
},
"Mexico": {
"languages": ["es"]
},
"Moldova": {
"languages": ["ro"]
},
"Monaco": {
"languages": ["fr"]
},
"Mongolia": {
"languages": ["mn"]
},
"Morocco": {
"languages": ["ar"]
},
"Mozambique": {
"driving": "l",
"languages": ["pt"]
},
"Myanmar": {
"languages": ["my"]
},
"Namibia": {
"driving": "l",
"languages": ["en"]
},
"Nauru": {
"driving": "l",
"languages": ["na"]
},
"Nepal": {
"driving": "l",
"languages": ["ne"]
},
"Nicaragua": {
"languages": ["es"]
},
"Niger": {
"languages": ["fr"]
},
"Nigeria": {
"languages": ["en"]
},
"Niue": {
"driving": "l",
"languages": ["en"]
},
"North Korea": {
"languages": ["ko"]
},
"Norway": {
"languages": ["no"]
},
"Oman": {
"languages": ["ar"]
},
"Pakistan": {
"driving": "l",
"languages": ["ur", "en"]
},
"Pitcairn Islands": {
"driving": "l"
},
"Saint Helena Ascension and Tristan da Cunha": {
"driving": "l"
},
"Palau": {
"languages": ["en"]
},
"Papua New Guinea": {
"driving": "l",
"languages": ["en", "ho"]
},
"Paraguay": {
"languages": ["es", "gn"]
},
"China": {
"languages": ["zh"]
},
"Taiwan": {
"languages": ["zh"]
},
"Peru": {
"languages": ["es"]
},
"Philippines": {
"languages": ["en"]
},
"Poland": {
"languages": ["pl"]
},
"Portugal": {
"languages": ["pt"]
},
"Qatar": {
"languages": ["ar"]
},
"Republic of Kosovo": {
"languages": ["sq", "sr"]
},
"Romania": {
"languages": ["ro"]
},
"Crimea": {
"languages": ["ru"]
},
"Russia": {
"languages": ["ru"]
},
"Russia_Moscow": {
"languages": ["ru"],
"timezone": "Europe/Moscow",
"driving": "r",
"housenames": false
},
"Russia_Bashkortostan": {
"languages": ["ru", "ba"]
},
"Russia_Chechen Republic": {
"languages": ["ru", "ce"]
},
"Russia_Chuvashia": {
"languages": ["ru", "cv"]
},
"Russia_Komi Republic": {
"languages": ["ru", "kv"]
},
"Russia_North Ossetia-Alania": {
"languages": ["ru", "os"]
},
"Russia_Republic of Dagestan": {
"languages": ["ru", "az", "av", "ce"]
},
"Russia_Tatarstan": {
"languages": ["ru", "tt"]
},
"Rwanda": {
"languages": ["fr", "en", "rw"]
},
"Samoa": {
"driving": "l",
"languages": ["en", "sm"]
},
"San Marino": {
"languages": ["it"]
},
"Saudi Arabia": {
"languages": ["ar"]
},
"Senegal": {
"languages": ["fr"]
},
"Serbia": {
"languages": ["sr"]
},
"Seychelles": {
"driving": "l",
"languages": ["fr", "en"]
},
"Sierra Leone": {
"languages": ["en"]
},
"Singapore": {
"driving": "l",
"languages": ["en", "ta", "ms"]
},
"Slovakia": {
"languages": ["sk"]
},
"Slovenia": {
"languages": ["sl"]
},
"Solomon Islands": {
"driving": "l",
"languages": ["en"]
},
"Somalia": {
"languages": ["so", "ar"]
},
"South Africa": {
"driving": "l",
"languages": ["en", "zu", "xh", "af", "ve", "ss", "tn", "ts", "st", "nr"]
},
"South Africa_Western Cape": {
"languages": ["en", "xh", "af"]
},
"South Georgia and the South Sandwich Islands": {
"languages": ["en"]
},
"South Ossetia": {
"languages": ["ru", "ka", "os"]
},
"South Sudan": {
"languages": ["en"]
},
"Spain": {
"languages": ["es"]
},
"Spain_Balearic Islands": {
"languages": ["es", "ca"]
},
"Spain_Catalonia": {
"languages": ["es", "ca"]
},
"Spain_Comunidad Foral de Navarra": {
"languages": ["es", "eu"]
},
"Spain_Valencian Community": {
"languages": ["es", "ca"]
},
"Palestine": {
"languages": ["ar"]
},
"Sri Lanka": {
"driving": "l",
"languages": ["ta", "si"]
},
"Sudan": {
"languages": ["en", "ar"]
},
"Suriname": {
"driving": "l",
"languages": ["nl"]
},
"Swaziland": {
"driving": "l",
"languages": ["en", "ss"]
},
"Sweden": {
"languages": ["fi", "yi", "sv"]
},
"Switzerland": {
"languages": ["fr", "de", "it", "rm"]
},
"Syria": {
"languages": ["ar"]
},
"Sao Tome and Principe": {
"languages": ["pt"]
},
"Tajikistan": {
"languages": ["ru", "tg"]
},
"Tanzania": {
"driving": "l",
"languages": ["en", "sw"]
},
"Thailand": {
"driving": "l",
"languages": ["th"]
},
"The Bahamas": {
"driving": "l",
"languages": ["en"]
},
"The Gambia": {
"languages": ["en"]
},
"Netherlands": {
"languages": ["nl"]
},
"Netherlands_Friesland": {
"languages": ["nl", "fy"]
},
"Togo": {
"languages": ["fr"]
},
"Tonga": {
"driving": "l",
"languages": ["en", "to"]
},
"Tunisia": {
"languages": ["ar"]
},
"Turkey": {
"languages": ["tr"]
},
"Turkmenistan": {
"languages": ["tk"]
},
"Tuvalu": {
"driving": "l",
"languages": ["en"]
},
"Uganda": {
"driving": "l",
"languages": ["en", "sw"]
},
"Ukraine": {
"languages": ["uk"]
},
"United Arab Emirates": {
"languages": ["ar"]
},
"UK": {
"driving": "l",
"languages": ["en"]
},
"UK_Wales": {
"driving": "l",
"languages": ["en", "cy"]
},
"Ireland": {
"driving": "l",
"languages": ["en", "ga"]
},
"US": {
"languages": ["en"]
},
"US_Guam": {
"languages": ["en", "ch"]
},
"US_Puerto Rico": {
"languages": ["es", "en"]
},
"Uruguay": {
"languages": ["es"]
},
"Uzbekistan": {
"languages": ["uz", "ru"]
},
"Vanuatu": {
"languages": ["fr", "en", "bi"]
},
"Venezuela": {
"languages": ["es"]
},
"Vietnam": {
"languages": ["vi"]
},
"Yemen": {
"languages": ["ar"]
},
"Zambia": {
"driving": "l",
"languages": ["en"]
},
"Zimbabwe": {
"driving": "l",
"languages": ["en", "sn", "nd"]
},
"Tokelau": {
"driving": "l",
"languages": ["en", "mi"]
},
"New Zealand": {
"driving": "l",
"languages": ["en", "mi"]
},
"South Korea": {
"languages": ["ko"]
}
}

2
defines.hpp
View file

@ -27,6 +27,7 @@
#define METADATA_INDEX_FILE_TAG "metaidx"
#define FEATURE_OFFSETS_FILE_TAG "offs"
#define RANKS_FILE_TAG "ranks"
#define REGION_INFO_FILE_TAG "rgninfo"
// Temporary addresses section that is used in search index generation.
#define SEARCH_TOKENS_FILE_TAG "addrtags"
@ -51,6 +52,7 @@
#define CELL2FEATURE_TMP_EXT ".c2f.tmp"
#define COUNTRIES_FILE "countries.txt"
#define COUNTRIES_META_FILE "countries_meta.txt"
#define COUNTRIES_OBSOLETE_FILE "countries_obsolete.txt"
#define WORLD_FILE_NAME "World"

51
generator/feature_sorter.cpp
View file

@ -1,8 +1,9 @@
#include "generator/feature_sorter.hpp"
#include "generator/feature_generator.hpp"
#include "generator/feature_builder.hpp"
#include "generator/tesselator.hpp"
#include "generator/feature_generator.hpp"
#include "generator/gen_mwm_info.hpp"
#include "generator/region_meta.hpp"
#include "generator/tesselator.hpp"
#include "defines.hpp"
@ -111,14 +112,16 @@ namespace feature
vector<pair<uint32_t, uint32_t>> m_metadataIndex;
DataHeader m_header;
RegionData m_regionData;
uint32_t m_versionDate;
gen::OsmID2FeatureID m_osm2ft;
public:
FeaturesCollector2(string const & fName, DataHeader const & header, uint32_t versionDate)
FeaturesCollector2(string const & fName, DataHeader const & header,
RegionData const & regionData, uint32_t versionDate)
: FeaturesCollector(fName + DATA_FILE_TAG), m_writer(fName),
m_header(header), m_versionDate(versionDate)
m_header(header), m_regionData(regionData), m_versionDate(versionDate)
{
for (size_t i = 0; i < m_header.GetScalesCount(); ++i)
{
@ -147,6 +150,12 @@ namespace feature
m_header.Save(w);
}
// write region info
{
FileWriter w = m_writer.GetWriter(REGION_INFO_FILE_TAG);
m_regionData.Serialize(w);
}
// assume like we close files
Flush();
@ -543,22 +552,6 @@ namespace feature
return static_cast<FeatureBuilder2 &>(fb);
}
class DoStoreLanguages
{
DataHeader & m_header;
public:
DoStoreLanguages(DataHeader & header) : m_header(header) {}
void operator() (string const & s)
{
int8_t const i = StringUtf8Multilang::GetLangIndex(s);
if (i > 0)
{
// 0 index is always 'default'
m_header.AddLanguage(i);
}
}
};
bool GenerateFinalFeatures(feature::GenerateInfo const & info, string const & name, int mapType)
{
string const srcFilePath = info.GetTmpFileName(name);
@ -596,23 +589,15 @@ namespace feature
// type
header.SetType(static_cast<DataHeader::MapType>(mapType));
// languages
try
{
FileReader reader(info.m_targetDir + "metainfo/" + name + ".meta");
string buffer;
reader.ReadAsString(buffer);
strings::Tokenize(buffer, "|", DoStoreLanguages(header));
}
catch (Reader::Exception const &)
{
LOG(LWARNING, ("No language file for country:", name));
}
// region data
RegionData regionData;
if (!ReadRegionData(name, regionData))
LOG(LWARNING, ("No extra data for country:", name));
// Transform features from raw format to optimized format.
try
{
FeaturesCollector2 collector(datFilePath, header, info.m_versionDate);
FeaturesCollector2 collector(datFilePath, header, regionData, info.m_versionDate);
for (size_t i = 0; i < midPoints.m_vec.size(); ++i)
{

5
generator/generator.pro
View file

@ -9,7 +9,8 @@ ROOT_DIR = ..
include($$ROOT_DIR/common.pri)
INCLUDEPATH *= $$ROOT_DIR/3party/gflags/src \
$$ROOT_DIR/3party/osrm/osrm-backend/include
$$ROOT_DIR/3party/osrm/osrm-backend/include \
$$ROOT_DIR/3party/jansson/src
QT *= core
@ -30,6 +31,7 @@ SOURCES += \
osm_element.cpp \
osm_id.cpp \
osm_source.cpp \
region_meta.cpp \
routing_generator.cpp \
search_index_builder.cpp \
srtm_parser.cpp \
@ -63,6 +65,7 @@ HEADERS += \
osm_translator.hpp \
osm_xml_source.hpp \
polygonizer.hpp \
region_meta.hpp \
routing_generator.hpp \
search_index_builder.hpp \
srtm_parser.hpp \

123
generator/region_meta.cpp Normal file
View file

@ -0,0 +1,123 @@
#include "region_meta.hpp"
#include "coding/reader.hpp"
#include "platform/platform.hpp"
#include "3party/jansson/myjansson.hpp"
namespace
{
int8_t ParseHolidayReference(string const & ref)
{
if (ref == "easter")
return feature::RegionData::PHReference::PH_EASTER;
if (ref == "orthodox easter")
return feature::RegionData::PHReference::PH_ORTHODOX_EASTER;
if (ref == "victoriaDay")
return feature::RegionData::PHReference::PH_VICTORIA_DAY;
if (ref == "canadaDay")
return feature::RegionData::PHReference::PH_CANADA_DAY;
return 0;
}
} // namespace
namespace feature
{
bool ReadRegionDataImpl(string const & countryName, RegionData & data)
{
try
{
auto reader = GetPlatform().GetReader(COUNTRIES_META_FILE);
string buffer;
reader->ReadAsString(buffer);
my::Json root(buffer.data());
json_t * jsonData = nullptr;
my::FromJSONObjectOptionalField(root.get(), countryName, jsonData);
if (!jsonData)
return false;
vector<string> languages;
my::FromJSONObjectOptionalField(jsonData, "languages", languages);
if (!languages.empty())
data.SetLanguages(languages);
string driving;
my::FromJSONObjectOptionalField(jsonData, "driving", driving);
if (driving == "l" || driving == "r")
data.Set(RegionData::Type::RD_DRIVING, driving);
string timezone;
my::FromJSONObjectOptionalField(jsonData, "timezone", timezone);
if (!timezone.empty())
data.Set(RegionData::Type::RD_TIMEZONE, timezone);
bool allow_housenames;
my::FromJSONObjectOptionalField(jsonData, "housenames", allow_housenames, false);
if (allow_housenames)
data.Set(RegionData::Type::RD_ALLOW_HOUSENAMES, "y");
// Public holidays: an array of arrays of [string/number, number].
// See https://github.com/opening-hours/opening_hours.js/blob/master/docs/holidays.md
vector<json_t *> holidays;
my::FromJSONObjectOptionalField(jsonData, "holidays", holidays);
for (json_t * holiday : holidays)
{
if (!json_is_array(holiday) || json_array_size(holiday) != 2)
MYTHROW(my::Json::Exception, ("Holiday must be an array of two elements in", countryName));
json_t * reference = json_array_get(holiday, 0);
int8_t refId = 0;
if (json_is_integer(reference))
{
refId = json_integer_value(reference);
}
else if (json_is_string(reference))
{
refId = ParseHolidayReference(string(json_string_value(reference)));
}
else
{
MYTHROW(my::Json::Exception,
("Holiday month reference should be either a string or a number in", countryName));
}
if (refId <= 0)
MYTHROW(my::Json::Exception, ("Incorrect month reference in", countryName));
if (!json_is_integer(json_array_get(holiday, 1)))
MYTHROW(my::Json::Exception, ("Holiday day offset should be a number in", countryName));
data.AddPublicHoliday(refId, json_integer_value(json_array_get(holiday, 1)));
}
// TODO(@zverik): Implement formats reading when decided on data types.
return true;
}
catch (Reader::Exception const & e)
{
LOG(LWARNING, ("Error reading", COUNTRIES_META_FILE, ":", e.Msg()));
}
catch (my::Json::Exception const & e)
{
LOG(LERROR, ("Error parsing JSON in", COUNTRIES_META_FILE, ":", e.Msg()));
}
return false;
}
bool ReadRegionData(string const & countryName, RegionData & data)
{
// When there is a match for a complete countryName, simply relay the call.
if (ReadRegionDataImpl(countryName, data))
return true;
// If not, cut parts of a country name from the tail. E.g. "Russia_Moscow" -> "Russia".
auto p = countryName.find_last_of('_');
while (p != string::npos)
{
if (ReadRegionDataImpl(countryName.substr(0, p), data))
return true;
p = p > 0 ? countryName.find_last_of('_', p - 1) : string::npos;
}
return false;
}
} // namespace feature

10
generator/region_meta.hpp Normal file
View file

@ -0,0 +1,10 @@
#pragma once
#include "generate_info.hpp"
#include "indexer/feature_meta.hpp"
namespace feature
{
bool ReadRegionData(string const & countryName, RegionData & data);
} // namespace feature

2
generator/search_index_builder.cpp
View file

@ -385,7 +385,7 @@ bool BuildSearchIndexFromDataFile(string const & filename, bool forceRebuild)
{
Platform & platform = GetPlatform();
FilesContainerR readContainer(platform.GetReader(filename));
FilesContainerR readContainer(platform.GetReader(filename, "f"));
if (readContainer.IsExist(SEARCH_INDEX_FILE_TAG) && !forceRebuild)
return true;

10
indexer/data_factory.cpp
View file

@ -2,11 +2,21 @@
#include "indexer/interval_index.hpp"
#include "indexer/old/interval_index_101.hpp"
#include "coding/file_container.hpp"
#include "defines.hpp"
void IndexFactory::Load(FilesContainerR const & cont)
{
ReadVersion(cont, m_version);
m_header.Load(cont);
if (cont.IsExist(REGION_INFO_FILE_TAG))
{
ReaderSource<FilesContainerR::TReader> src(cont.GetReader(REGION_INFO_FILE_TAG));
m_regionData.Deserialize(src);
}
}
IntervalIndexIFace * IndexFactory::CreateIndex(ModelReaderPtr reader) const

3
indexer/data_factory.hpp
View file

@ -1,5 +1,6 @@
#pragma once
#include "indexer/data_header.hpp"
#include "indexer/feature_meta.hpp"
#include "platform/mwm_version.hpp"
@ -11,12 +12,14 @@ class IndexFactory
{
version::MwmVersion m_version;
feature::DataHeader m_header;
feature::RegionData m_regionData;
public:
void Load(FilesContainerR const & cont);
inline version::MwmVersion const & GetMwmVersion() const { return m_version; }
inline feature::DataHeader const & GetHeader() const { return m_header; }
inline feature::RegionData const & GetRegionData() const { return m_regionData; }
IntervalIndexIFace * CreateIndex(ModelReaderPtr reader) const;
};

44
indexer/feature_meta.cpp
View file

@ -99,6 +99,50 @@ bool Metadata::TypeFromString(string const & k, Metadata::EType & outType)
return true;
}
void RegionData::SetLanguages(vector<string> const & codes)
{
string value;
for (string const & code : codes)
{
int8_t const lang = StringUtf8Multilang::GetLangIndex(code);
if (lang != StringUtf8Multilang::kUnsupportedLanguageCode)
value.push_back(lang);
}
MetadataBase::Set(RegionData::Type::RD_LANGUAGES, value);
}
void RegionData::GetLanguages(vector<int8_t> & langs) const
{
for (auto const lang : Get(RegionData::Type::RD_LANGUAGES))
langs.push_back(lang);
}
bool RegionData::HasLanguage(int8_t const lang) const
{
for (auto const lng : Get(RegionData::Type::RD_LANGUAGES))
{
if (lng == lang)
return true;
}
return false;
}
bool RegionData::IsSingleLanguage(int8_t const lang) const
{
string const value = Get(RegionData::Type::RD_LANGUAGES);
if (value.size() != 1)
return false;
return value.front() == lang;
}
void RegionData::AddPublicHoliday(int8_t month, int8_t offset)
{
string value = Get(RegionData::Type::RD_PUBLIC_HOLIDAYS);
value.push_back(month);
value.push_back(offset);
Set(RegionData::Type::RD_PUBLIC_HOLIDAYS, value);
}
} // namespace feature
// Warning: exact osm tag keys should be returned for valid enum values.

46
indexer/feature_meta.hpp
View file

@ -132,13 +132,8 @@ public:
/// Used to normalize tags like "contact:phone" and "phone" to a common metadata enum value.
static bool TypeFromString(string const & osmTagKey, feature::Metadata::EType & outType);
void Set(EType type, string const & value)
{
MetadataBase::Set(type, value);
}
void Set(EType type, string const & value) { MetadataBase::Set(type, value); }
void Drop(EType type) { Set(type, string()); }
string GetWikiURL() const;
// TODO: Commented code below is now longer neded, but I leave it here
@ -186,6 +181,45 @@ public:
MetadataBase::Set(type, s);
}
};
class RegionData : public MetadataBase
{
public:
enum Type : int8_t
{
RD_LANGUAGES, // list of written languages
RD_DRIVING, // left- or right-hand driving (letter 'l' or 'r')
RD_TIMEZONE, // UTC timezone offset, floating signed number of hours: -3, 4.5
RD_ADDRESS_FORMAT, // address format, re: mapzen
RD_PHONE_FORMAT, // list of strings in "+N NNN NN-NN-NN" format
RD_POSTCODE_FORMAT, // list of strings in "AAA ANN" format
RD_PUBLIC_HOLIDAYS, // fixed PH dates
RD_ALLOW_HOUSENAMES // 'y' if housenames are commonly used
};
// Special values for month references in public holiday definitions.
enum PHReference : int8_t
{
PH_EASTER = 20,
PH_ORTHODOX_EASTER = 21,
PH_VICTORIA_DAY = 22,
PH_CANADA_DAY = 23
};
void Set(Type type, string const & s)
{
CHECK_NOT_EQUAL(type, Type::RD_LANGUAGES, ("Please use RegionData::SetLanguages method"));
MetadataBase::Set(type, s);
}
void SetLanguages(vector<string> const & codes);
void GetLanguages(vector<int8_t> & langs) const;
bool HasLanguage(int8_t const lang) const;
bool IsSingleLanguage(int8_t const lang) const;
void AddPublicHoliday(int8_t month, int8_t offset);
// No public holidays getters until we know what to do with these.
};
} // namespace feature
// Prints types in osm-friendly format.

3
indexer/index.cpp
View file

@ -58,6 +58,9 @@ unique_ptr<MwmInfo> Index::CreateInfo(platform::LocalCountryFile const & localFi
info->m_minScale = static_cast<uint8_t>(scaleR.first);
info->m_maxScale = static_cast<uint8_t>(scaleR.second);
info->m_version = value.GetMwmVersion();
// Copying to drop the const qualifier.
feature::RegionData regionData(value.GetRegionData());
info->m_data = regionData;
return unique_ptr<MwmInfo>(move(info));
}

1
indexer/index.hpp
View file

@ -51,6 +51,7 @@ public:
void SetTable(MwmInfoEx & info);
inline feature::DataHeader const & GetHeader() const { return m_factory.GetHeader(); }
inline feature::RegionData const & GetRegionData() const { return m_factory.GetRegionData(); }
inline version::MwmVersion const & GetMwmVersion() const { return m_factory.GetMwmVersion(); }
inline string const & GetCountryFileName() const { return m_file.GetCountryFile().GetName(); }
};

23
indexer/indexer_tests/feature_metadata_test.cpp
View file

@ -100,3 +100,26 @@ UNIT_TEST(Feature_Metadata_GetWikipedia)
TEST_EQUAL(m.GetWikiURL(), "https://en.wikipedia.org/wiki/Article", ());
#endif
}
UNIT_TEST(Feature_Metadata_RegionData_Languages)
{
{
feature::RegionData rd;
vector<string> const langs = {"ru", "en", "et"};
rd.SetLanguages(langs);
TEST(rd.HasLanguage(StringUtf8Multilang::GetLangIndex("ru")), ());
TEST(rd.HasLanguage(StringUtf8Multilang::GetLangIndex("en")), ());
TEST(rd.HasLanguage(StringUtf8Multilang::GetLangIndex("et")), ());
TEST(!rd.HasLanguage(StringUtf8Multilang::GetLangIndex("es")), ());
TEST(!rd.IsSingleLanguage(StringUtf8Multilang::GetLangIndex("ru")), ());
}
{
feature::RegionData rd;
vector<string> const langs = {"et"};
rd.SetLanguages(langs);
TEST(rd.HasLanguage(StringUtf8Multilang::GetLangIndex("et")), ());
TEST(rd.IsSingleLanguage(StringUtf8Multilang::GetLangIndex("et")), ());
TEST(!rd.HasLanguage(StringUtf8Multilang::GetLangIndex("en")), ());
TEST(!rd.IsSingleLanguage(StringUtf8Multilang::GetLangIndex("en")), ());
}
}

6
indexer/mwm_set.hpp
View file

@ -8,6 +8,8 @@
#include "base/macros.hpp"
#include "indexer/feature_meta.hpp"
#include "std/atomic.hpp"
#include "std/deque.hpp"
#include "std/map.hpp"
@ -66,6 +68,8 @@ public:
MwmTypeT GetType() const;
inline feature::RegionData const & GetRegionData() const { return m_data; }
/// Returns the lock counter value for test needs.
uint8_t GetNumRefs() const { return m_numRefs; }
@ -77,6 +81,8 @@ protected:
return result;
}
feature::RegionData m_data;
platform::LocalCountryFile m_file; ///< Path to the mwm file.
atomic<Status> m_status; ///< Current country status.
uint32_t m_numRefs; ///< Number of active handles.

1
tools/python/mwm/dump_mwm.py
View file

@ -17,6 +17,7 @@ for tv in tvv:
v = mwm.read_version()
print('Format: {0}, version: {1}'.format(v['fmt'], v['version'].strftime('%Y-%m-%d %H:%M')))
print('Header: {0}'.format(mwm.read_header()))
print('Region Info: {0}'.format(mwm.read_region_info()))
print('Metadata count: {0}'.format(len(mwm.read_metadata())))
cross = mwm.read_crossmwm()

17
tools/python/mwm/mwm.py
View file

@ -31,6 +31,8 @@ class MWM:
"denomination", "building_levels", "test_id", "ref:sponsored", "price_rate",
"rating", "fuel", "routes"]
regiondata = ["languages", "driving", "timezone", "addr_fmt", "phone_fmt", "postcode_fmt", "holidays", "housenames"]
def __init__(self, f):
self.f = f
self.coord_size = None
@ -112,6 +114,21 @@ class MWM:
# COMPLEX READERS
def read_region_info(self):
if not self.has_tag('rgninfo'):
return {}
fields = {}
self.seek_tag('rgninfo')
sz = self.read_varuint()
if sz:
for i in range(sz):
t = self.read_varuint()
t = self.regiondata[t] if t < len(self.regiondata) else str(t)
fields[t] = self.read_string()
if t == 'languages':
fields[t] = [self.languages[ord(x)] for x in fields[t]]
return fields
def read_metadata(self):
"""Reads 'meta' and 'metaidx' sections."""
if not self.has_tag('metaidx'):