pastk/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

[generator] Added geo_objects kv only for houses with addresses.

Browse source
This commit is contained in:
Maksim Andrianov 2018-10-04 21:00:43 +03:00 • committed by mpimenov
parent fc192e0db3
commit 222ba6e9c2
5 changed files with 253 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
generator/CMakeLists.txt
View file

@ -58,6 +58,8 @@ set(SRC
gen_mwm_info.hpp
generate_info.hpp
geometry_holder.hpp
geo_objects/geo_objects.cpp
geo_objects/geo_objects.hpp
holes.cpp
holes.hpp
intermediate_data.cpp

19
generator/generator_tool/generator_tool.cpp
View file

@ -11,6 +11,7 @@
#include "generator/feature_generator.hpp"
#include "generator/feature_sorter.hpp"
#include "generator/generate_info.hpp"
#include "generator/geo_objects/geo_objects.hpp"
#include "generator/locality_sorter.hpp"
#include "generator/metalines_builder.hpp"
#include "generator/osm_source.hpp"
@ -168,6 +169,13 @@ DEFINE_bool(generate_addresses_file, false, "Generate .addr file (for '--output'
DEFINE_bool(generate_traffic_keys, false,
"Generate keys for the traffic map (road segment -> speed group).");
// Generating geo objects key-value.
DEFINE_string(regions_index, "", "Input regions index file.");
DEFINE_string(regions_key_value, "", "Input regions key-value file.");
DEFINE_string(geo_objects_features, "", "Input geo_objects tmp.mwm file.");
DEFINE_string(ids_without_address, "", "Output file with objects ids without adresses.");
DEFINE_string(geo_objects_key_value, "", "Output geo objects key-value file.");
// Common.
DEFINE_bool(verbose, false, "Provide more detailed output.");
@ -249,7 +257,8 @@ int main(int argc, char ** argv)
FLAGS_dump_feature_names != "" || FLAGS_check_mwm || FLAGS_srtm_path != "" ||
FLAGS_make_routing_index || FLAGS_make_cross_mwm || FLAGS_make_transit_cross_mwm ||
FLAGS_make_city_roads || FLAGS_generate_traffic_keys || FLAGS_transit_path != "" ||
FLAGS_ugc_data != "" || FLAGS_popular_places_data != "" || FLAGS_generate_geo_objects_features)
FLAGS_ugc_data != "" || FLAGS_popular_places_data != "" || FLAGS_generate_geo_objects_features ||
FLAGS_geo_objects_key_value != "")
{
classificator::Load();
classif().SortClassificator();
@ -317,6 +326,14 @@ int main(int argc, char ** argv)
}
}
if (!FLAGS_geo_objects_key_value.empty())
{
if (!geo_objects::GenerateGeoObjects(FLAGS_regions_index, FLAGS_regions_key_value,
FLAGS_geo_objects_features, FLAGS_ids_without_address,
FLAGS_geo_objects_key_value, FLAGS_verbose))
return -1;
}
if (genInfo.m_bucketNames.empty() && !FLAGS_output.empty())
genInfo.m_bucketNames.push_back(FLAGS_output);

215
generator/geo_objects/geo_objects.cpp Normal file
View file

@ -0,0 +1,215 @@
#include "generator/geo_objects/geo_objects.hpp"
#include "generator/feature_builder.hpp"
#include "generator/regions/region_base.hpp"
#include "indexer/locality_index.hpp"
#include "coding/file_container.hpp"
#include "coding/mmap_reader.hpp"
#include "base/geo_object_id.hpp"
#include "base/logging.hpp"
#include <cstdint>
#include <fstream>
#include <map>
#include "3party/jansson/myjansson.hpp"
namespace
{
using KeyValue = std::map<uint64_t, base::Json>;
using IndexReader = ReaderPtr<Reader>;
template <typename Index>
typename Index::Type ReadIndex(std::string const & pathIndx)
{
FilesContainerR cont(pathIndx);
auto const offsetSize = cont.GetAbsoluteOffsetAndSize(Index::kFileTag);
MmapReader reader(pathIndx);
typename Index::ReaderType subReader(reader.CreateSubReader(offsetSize.first, offsetSize.second));
typename Index::Type index(subReader);
return index;
}
std::vector<FeatureBuilder1> ReadTmpMwm(std::string const & pathInGeoObjectsTmpMwm)
{
std::vector<FeatureBuilder1> geoObjects;
auto const toDo = [&geoObjects](FeatureBuilder1 & fb, uint64_t /* currPos */)
{
geoObjects.emplace_back(std::move(fb));
};
feature::ForEachFromDatRawFormat(pathInGeoObjectsTmpMwm, toDo);
return geoObjects;
}
KeyValue ReadRegionsKv(std::string const & pathInRegionsKv)
{
KeyValue regionKv;
std::ifstream stream(pathInRegionsKv);
std::string line;
while (std::getline(stream, line))
{
auto const pos = line.find(" ");
if (pos == std::string::npos)
{
LOG(LWARNING, ("Can not find separator."));
continue;
}
int64_t tmpId;
if (!strings::to_int64(line.substr(0, pos), tmpId))
{
LOG(LWARNING, ("Can not parse id."));
continue;
}
base::Json json;
try
{
json = base::Json(line.substr(pos + 1, line.size()));
if (!json.get())
continue;
}
catch (...)
{
LOG(LWARNING, ("Can not create base::Json."));
continue;
}
regionKv.emplace(static_cast<uint64_t>(tmpId), json);
}
return regionKv;
}
template<typename Index>
std::vector<base::GeoObjectId> SearchObjectsInIndex(FeatureBuilder1 const & fb, Index const & index)
{
std::vector<base::GeoObjectId> ids;
auto const fn = [&ids] (const base::GeoObjectId & osmid) { ids.emplace_back(osmid); };
auto const center = fb.GetLimitRect().Center();
auto const rect = MercatorBounds::RectByCenterXYAndSizeInMeters(center, 0 /* meters */);
index.ForEachInRect(fn, rect);
return ids;
}
int GetRankFromValue(const base::Json & json)
{
int tempRank;
auto properties = json_object_get(json.get(), "properties");
FromJSONObject(properties, "rank", tempRank);
return tempRank;
}
base::Json GetDeepestRegion(std::vector<base::GeoObjectId> const & ids, KeyValue const & regionKv)
{
base::Json deepest;
int deepestRank = 0;
for (auto const & id : ids)
{
base::Json temp;
auto const it = regionKv.find(id.GetEncodedId());
if (it == std::end(regionKv))
{
LOG(LWARNING, ("Not found id in region key-value:", id));
continue;
}
temp = it->second;
if (!json_is_object(temp.get()))
continue;
if (!deepest.get())
{
deepestRank = GetRankFromValue(temp);
deepest = temp;
}
else
{
int tempRank = GetRankFromValue(temp);
if (tempRank > deepestRank)
{
deepest = temp;
deepestRank = tempRank;
}
}
}
return deepest;
}
base::Json AddAddress(FeatureBuilder1 const & fb, base::Json const regionJson)
{
base::Json result = regionJson.GetDeepCopy();
int const kHouseRank = 30;
ToJSONObject(*result.get(), "rank", kHouseRank);
auto properties = json_object_get(result.get(), "properties");
auto address = json_object_get(properties, "address");
ToJSONObject(*address, "house", fb.GetParams().house.Get());
auto const street = fb.GetParams().GetStreet();
if (!street.empty())
ToJSONObject(*address, "street", street);
// auto localies = json_object_get(result.get(), "localies");
// auto en = json_object_get(result.get(), "en");
// todo(maksimandrianov): Add en locales.
return result;
}
std::unique_ptr<char, JSONFreeDeleter>
MakeGeoObjectValue(FeatureBuilder1 const & fb,
typename indexer::RegionsIndex<IndexReader>::Type const & regionIndex,
KeyValue const & regionKv)
{
auto const ids = SearchObjectsInIndex(fb, regionIndex);
auto const json = GetDeepestRegion(ids, regionKv);
auto const jsonWithAddress = AddAddress(fb, json);
auto const cstr = json_dumps(jsonWithAddress.get(), JSON_COMPACT);
std::unique_ptr<char, JSONFreeDeleter> buffer(cstr);
return buffer;
}
bool GenerateGeoObjects(typename indexer::RegionsIndex<IndexReader>::Type const & regionIndex,
KeyValue const & regionKv,
std::vector<FeatureBuilder1> const & geoObjects,
std::ostream & streamIdsWithoutAddress,
std::ostream & streamGeoObjectsKv, bool verbose)
{
for (auto const & fb : geoObjects)
{
if (fb.GetParams().house.IsEmpty())
continue;
const auto value = MakeGeoObjectValue(fb, regionIndex, regionKv);
streamGeoObjectsKv << static_cast<int64_t>(fb.GetMostGenericOsmId().GetEncodedId()) << " "
<< value.get() << "\n";
}
return true;
}
}
namespace generator
{
namespace geo_objects
{
bool GenerateGeoObjects(std::string const & pathInRegionsIndx,
std::string const & pathInRegionsKv,
std::string const & pathInGeoObjectsTmpMwm,
std::string const & pathOutIdsWithoutAddress,
std::string const & pathOutGeoObjectsKv, bool verbose)
{
auto const index = ReadIndex<indexer::RegionsIndex<IndexReader>>(pathInRegionsIndx);
auto const regionsKv = ReadRegionsKv(pathInRegionsKv);
auto geoObjects = ReadTmpMwm(pathInGeoObjectsTmpMwm);
std::ofstream streamIdsWithoutAddress(pathOutIdsWithoutAddress);
std::ofstream streamGeoObjectsKv(pathOutGeoObjectsKv);
return ::GenerateGeoObjects(index, regionsKv, geoObjects, streamIdsWithoutAddress,
streamGeoObjectsKv, verbose);
}
} // namespace geo_objects
} // namespace generator

15
generator/geo_objects/geo_objects.hpp Normal file
View file

@ -0,0 +1,15 @@
#pragma once
#include <string>
namespace generator
{
namespace geo_objects
{
bool GenerateGeoObjects(std::string const & pathInRegionsIndx,
std::string const & pathInRegionsKv,
std::string const & pathInGeoObjectsTmpMwm,
std::string const & pathOutIdsWithoutAddress,
std::string const & pathOutGeoObjectsKv, bool verbose);
} // namespace geo_objects
} // namespace generator

6
indexer/indexer_tests/locality_index_test.cpp
View file

@ -83,7 +83,7 @@ UNIT_TEST(BuildLocalityIndexTest)
BuildGeoObjectsIndex(objects, writer, "tmp");
MemReader reader(localityIndex.data(), localityIndex.size());
indexer::GeoObjectsIndex<MemReader> index(reader);
indexer::GeoObjectsIndex<MemReader>::Type index(reader);
TEST_EQUAL(GetIds(index, m2::RectD{-0.5, -0.5, 0.5, 0.5}), (Ids{1}), ());
TEST_EQUAL(GetIds(index, m2::RectD{0.5, -0.5, 1.5, 1.5}), (Ids{2, 3}), ());
@ -104,7 +104,7 @@ UNIT_TEST(LocalityIndexRankTest)
BuildGeoObjectsIndex(objects, writer, "tmp");
MemReader reader(localityIndex.data(), localityIndex.size());
indexer::GeoObjectsIndex<MemReader> index(reader);
indexer::GeoObjectsIndex<MemReader>::Type index(reader);
TEST_EQUAL(GetRankedIds(index, m2::PointD{1, 0} /* center */, m2::PointD{4, 0} /* border */,
4 /* topSize */),
(vector<uint64_t>{1, 2, 3, 4}), ());
@ -136,7 +136,7 @@ UNIT_TEST(LocalityIndexTopSizeTest)
BuildGeoObjectsIndex(objects, writer, "tmp");
MemReader reader(localityIndex.data(), localityIndex.size());
indexer::GeoObjectsIndex<MemReader> index(reader);
indexer::GeoObjectsIndex<MemReader>::Type index(reader);
TEST_EQUAL(GetRankedIds(index, m2::PointD{1, 0} /* center */, m2::PointD{0, 0} /* border */,
4 /* topSize */)
.size(),