kirylkaveryn/organicmaps
1
Fork 0
forked from organicmaps/organicmaps-tmp
Code Pull requests Activity

Review fixes.

Browse source
This commit is contained in:
Maksim Andrianov 2018-10-08 12:54:52 +03:00 • committed by mpimenov
parent 222ba6e9c2
commit cbc5a80d09
4 changed files with 60 additions and 50 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
generator/generator_tool/generator_tool.cpp
View file

@ -172,8 +172,8 @@ DEFINE_bool(generate_traffic_keys, false,
// Generating geo objects key-value.
DEFINE_string(regions_index, "", "Input regions index file.");
DEFINE_string(regions_key_value, "", "Input regions key-value file.");
DEFINE_string(geo_objects_features, "", "Input geo_objects tmp.mwm file.");
DEFINE_string(ids_without_address, "", "Output file with objects ids without adresses.");
DEFINE_string(geo_objects_features, "", "Input tmp.mwm file with geo objects.");
DEFINE_string(ids_without_addresses, "", "Output file with objects ids without addresses.");
DEFINE_string(geo_objects_key_value, "", "Output geo objects key-value file.");
// Common.
@ -329,7 +329,7 @@ int main(int argc, char ** argv)
if (!FLAGS_geo_objects_key_value.empty())
{
if (!geo_objects::GenerateGeoObjects(FLAGS_regions_index, FLAGS_regions_key_value,
FLAGS_geo_objects_features, FLAGS_ids_without_address,
FLAGS_geo_objects_features, FLAGS_ids_without_addresses,
FLAGS_geo_objects_key_value, FLAGS_verbose))
return -1;
}

70
generator/geo_objects/geo_objects.cpp
View file

@ -5,7 +5,6 @@
#include "indexer/locality_index.hpp"
#include "coding/file_container.hpp"
#include "coding/mmap_reader.hpp"
#include "base/geo_object_id.hpp"
@ -22,89 +21,77 @@ namespace
using KeyValue = std::map<uint64_t, base::Json>;
using IndexReader = ReaderPtr<Reader>;
template <typename Index>
typename Index::Type ReadIndex(std::string const & pathIndx)
{
FilesContainerR cont(pathIndx);
auto const offsetSize = cont.GetAbsoluteOffsetAndSize(Index::kFileTag);
MmapReader reader(pathIndx);
typename Index::ReaderType subReader(reader.CreateSubReader(offsetSize.first, offsetSize.second));
typename Index::Type index(subReader);
return index;
}
std::vector<FeatureBuilder1> ReadTmpMwm(std::string const & pathInGeoObjectsTmpMwm)
std::vector<FeatureBuilder1> ReadTmpMwm(std::string const & pathToGeoObjectsTmpMwm)
{
std::vector<FeatureBuilder1> geoObjects;
auto const toDo = [&geoObjects](FeatureBuilder1 & fb, uint64_t /* currPos */)
{
geoObjects.emplace_back(std::move(fb));
};
feature::ForEachFromDatRawFormat(pathInGeoObjectsTmpMwm, toDo);
feature::ForEachFromDatRawFormat(pathToGeoObjectsTmpMwm, toDo);
return geoObjects;
}
KeyValue ReadRegionsKv(std::string const & pathInRegionsKv)
KeyValue ReadRegionsKv(std::string const & pathToRegionsKv)
{
KeyValue regionKv;
std::ifstream stream(pathInRegionsKv);
std::ifstream stream(pathToRegionsKv);
std::string line;
while (std::getline(stream, line))
{
auto const pos = line.find(" ");
if (pos == std::string::npos)
{
LOG(LWARNING, ("Can not find separator."));
LOG(LWARNING, ("Cannot find separator."));
continue;
}
int64_t tmpId;
if (!strings::to_int64(line.substr(0, pos), tmpId))
int64_t id;
if (!strings::to_int64(line.substr(0, pos), id))
{
LOG(LWARNING, ("Can not parse id."));
LOG(LWARNING, ("Cannot parse id."));
continue;
}
base::Json json;
try
{
json = base::Json(line.substr(pos + 1, line.size()));
json = base::Json(line.substr(pos + 1));
if (!json.get())
continue;
}
catch (...)
catch (base::Json::Exception const &)
{
LOG(LWARNING, ("Can not create base::Json."));
LOG(LWARNING, ("Cannot create base::Json."));
continue;
}
regionKv.emplace(static_cast<uint64_t>(tmpId), json);
regionKv.emplace(static_cast<uint64_t>(id), json);
}
return regionKv;
}
template<typename Index>
template <typename Index>
std::vector<base::GeoObjectId> SearchObjectsInIndex(FeatureBuilder1 const & fb, Index const & index)
{
std::vector<base::GeoObjectId> ids;
auto const fn = [&ids] (const base::GeoObjectId & osmid) { ids.emplace_back(osmid); };
auto const fn = [&ids] (base::GeoObjectId const & osmid) { ids.emplace_back(osmid); };
auto const center = fb.GetLimitRect().Center();
auto const rect = MercatorBounds::RectByCenterXYAndSizeInMeters(center, 0 /* meters */);
index.ForEachInRect(fn, rect);
return ids;
}
int GetRankFromValue(const base::Json & json)
int GetRankFromValue(base::Json const & json)
{
int tempRank;
int rank;
auto properties = json_object_get(json.get(), "properties");
FromJSONObject(properties, "rank", tempRank);
return tempRank;
FromJSONObject(properties, "rank", rank);
return rank;
}
base::Json GetDeepestRegion(std::vector<base::GeoObjectId> const & ids, KeyValue const & regionKv)
{
base::Json deepest;
int deepestRank = 0;
for (auto const & id : ids)
@ -113,13 +100,16 @@ base::Json GetDeepestRegion(std::vector<base::GeoObjectId> const & ids, KeyValue
auto const it = regionKv.find(id.GetEncodedId());
if (it == std::end(regionKv))
{
LOG(LWARNING, ("Not found id in region key-value:", id));
LOG(LWARNING, ("Id not found in region key-value:", id));
continue;
}
temp = it->second;
if (!json_is_object(temp.get()))
{
LOG(LWARNING, ("Value is not a json object in region key-value:", id));
continue;
}
if (!deepest.get())
{
@ -129,7 +119,7 @@ base::Json GetDeepestRegion(std::vector<base::GeoObjectId> const & ids, KeyValue
else
{
int tempRank = GetRankFromValue(temp);
if (tempRank > deepestRank)
if (deepestRank < tempRank)
{
deepest = temp;
deepestRank = tempRank;
@ -140,7 +130,7 @@ base::Json GetDeepestRegion(std::vector<base::GeoObjectId> const & ids, KeyValue
return deepest;
}
base::Json AddAddress(FeatureBuilder1 const & fb, base::Json const regionJson)
base::Json AddAddress(FeatureBuilder1 const & fb, base::Json const & regionJson)
{
base::Json result = regionJson.GetDeepCopy();
int const kHouseRank = 30;
@ -153,7 +143,7 @@ base::Json AddAddress(FeatureBuilder1 const & fb, base::Json const regionJson)
if (!street.empty())
ToJSONObject(*address, "street", street);
// auto localies = json_object_get(result.get(), "localies");
// auto locales = json_object_get(result.get(), "locales");
// auto en = json_object_get(result.get(), "en");
// todo(maksimandrianov): Add en locales.
return result;
@ -161,7 +151,7 @@ base::Json AddAddress(FeatureBuilder1 const & fb, base::Json const regionJson)
std::unique_ptr<char, JSONFreeDeleter>
MakeGeoObjectValue(FeatureBuilder1 const & fb,
typename indexer::RegionsIndex<IndexReader>::Type const & regionIndex,
typename indexer::RegionsIndexBox<IndexReader>::IndexType const & regionIndex,
KeyValue const & regionKv)
{
auto const ids = SearchObjectsInIndex(fb, regionIndex);
@ -173,7 +163,7 @@ MakeGeoObjectValue(FeatureBuilder1 const & fb,
return buffer;
}
bool GenerateGeoObjects(typename indexer::RegionsIndex<IndexReader>::Type const & regionIndex,
bool GenerateGeoObjects(typename indexer::RegionsIndexBox<IndexReader>::IndexType const & regionIndex,
KeyValue const & regionKv,
std::vector<FeatureBuilder1> const & geoObjects,
std::ostream & streamIdsWithoutAddress,
@ -184,14 +174,14 @@ bool GenerateGeoObjects(typename indexer::RegionsIndex<IndexReader>::Type const
if (fb.GetParams().house.IsEmpty())
continue;
const auto value = MakeGeoObjectValue(fb, regionIndex, regionKv);
auto const value = MakeGeoObjectValue(fb, regionIndex, regionKv);
streamGeoObjectsKv << static_cast<int64_t>(fb.GetMostGenericOsmId().GetEncodedId()) << " "
<< value.get() << "\n";
}
return true;
}
}
} // namespace
namespace generator
{
@ -203,7 +193,7 @@ bool GenerateGeoObjects(std::string const & pathInRegionsIndx,
std::string const & pathOutIdsWithoutAddress,
std::string const & pathOutGeoObjectsKv, bool verbose)
{
auto const index = ReadIndex<indexer::RegionsIndex<IndexReader>>(pathInRegionsIndx);
auto const index = indexer::ReadIndex<indexer::RegionsIndexBox<IndexReader>, MmapReader>(pathInRegionsIndx);
auto const regionsKv = ReadRegionsKv(pathInRegionsKv);
auto geoObjects = ReadTmpMwm(pathInGeoObjectsTmpMwm);
std::ofstream streamIdsWithoutAddress(pathOutIdsWithoutAddress);

6
indexer/indexer_tests/locality_index_test.cpp
View file

@ -83,7 +83,7 @@ UNIT_TEST(BuildLocalityIndexTest)
BuildGeoObjectsIndex(objects, writer, "tmp");
MemReader reader(localityIndex.data(), localityIndex.size());
indexer::GeoObjectsIndex<MemReader>::Type index(reader);
indexer::GeoObjectsIndex<MemReader> index(reader);
TEST_EQUAL(GetIds(index, m2::RectD{-0.5, -0.5, 0.5, 0.5}), (Ids{1}), ());
TEST_EQUAL(GetIds(index, m2::RectD{0.5, -0.5, 1.5, 1.5}), (Ids{2, 3}), ());
@ -104,7 +104,7 @@ UNIT_TEST(LocalityIndexRankTest)
BuildGeoObjectsIndex(objects, writer, "tmp");
MemReader reader(localityIndex.data(), localityIndex.size());
indexer::GeoObjectsIndex<MemReader>::Type index(reader);
indexer::GeoObjectsIndex<MemReader> index(reader);
TEST_EQUAL(GetRankedIds(index, m2::PointD{1, 0} /* center */, m2::PointD{4, 0} /* border */,
4 /* topSize */),
(vector<uint64_t>{1, 2, 3, 4}), ());
@ -136,7 +136,7 @@ UNIT_TEST(LocalityIndexTopSizeTest)
BuildGeoObjectsIndex(objects, writer, "tmp");
MemReader reader(localityIndex.data(), localityIndex.size());
indexer::GeoObjectsIndex<MemReader>::Type index(reader);
indexer::GeoObjectsIndex<MemReader> index(reader);
TEST_EQUAL(GetRankedIds(index, m2::PointD{1, 0} /* center */, m2::PointD{0, 0} /* border */,
4 /* topSize */)
.size(),

28
indexer/locality_index.hpp
View file

@ -6,6 +6,8 @@
#include "indexer/locality_object.hpp"
#include "indexer/scales.hpp"
#include "coding/file_container.hpp"
#include "geometry/rect2d.hpp"
#include "base/geo_object_id.hpp"
@ -14,6 +16,7 @@
#include <functional>
#include <memory>
#include <set>
#include <string>
#include "defines.hpp"
@ -77,20 +80,37 @@ private:
};
template <typename Reader>
struct GeoObjectsIndex
using GeoObjectsIndex = LocalityIndex<Reader, kGeoObjectsDepthLevels>;
template <typename Reader>
using RegionsIndex = LocalityIndex<Reader, kRegionsDepthLevels>;
template <typename Reader>
struct GeoObjectsIndexBox
{
static constexpr const char * kFileTag = GEO_OBJECTS_INDEX_FILE_TAG;
using ReaderType = Reader;
using Type = LocalityIndex<ReaderType, kGeoObjectsDepthLevels>;
using IndexType = GeoObjectsIndex<ReaderType>;
};
template <typename Reader>
struct RegionsIndex
struct RegionsIndexBox
{
static constexpr const char * kFileTag = REGIONS_INDEX_FILE_TAG;
using ReaderType = Reader;
using Type = LocalityIndex<ReaderType, kRegionsDepthLevels>;
using IndexType = RegionsIndex<ReaderType>;
};
template <typename IndexBox, typename Reader>
typename IndexBox::IndexType ReadIndex(std::string const & pathIndx)
{
FilesContainerR cont(pathIndx);
auto const offsetSize = cont.GetAbsoluteOffsetAndSize(IndexBox::kFileTag);
Reader reader(pathIndx);
typename IndexBox::ReaderType subReader(reader.CreateSubReader(offsetSize.first, offsetSize.second));
typename IndexBox::IndexType index(subReader);
return index;
}
} // namespace indexer