diff --git a/generator/generator_tool/generator_tool.cpp b/generator/generator_tool/generator_tool.cpp index 1d01fc3ed1..de9e42611d 100644 --- a/generator/generator_tool/generator_tool.cpp +++ b/generator/generator_tool/generator_tool.cpp @@ -598,26 +598,23 @@ MAIN_WITH_ERROR_HANDLING([](int argc, char ** argv) string const dataFile = base::JoinPath(path, FLAGS_output + DATA_FILE_EXTENSION); - if (FLAGS_calc_statistics) + if (FLAGS_calc_statistics || FLAGS_type_statistics) { - LOG(LINFO, ("Calculating statistics for", dataFile)); - auto file = OfstreamWithExceptions(genInfo.GetIntermediateFileName(FLAGS_output, STATS_EXTENSION)); - stats::FileContainerStatistic(file, dataFile); - stats::MapInfo info; - stats::CalcStatistic(dataFile, info); - stats::PrintStatistic(file, info); - } - - if (FLAGS_type_statistics) - { - LOG(LINFO, ("Calculating type statistics for", dataFile)); - - stats::MapInfo info; - stats::CalcStatistic(dataFile, info); - auto file = OfstreamWithExceptions(genInfo.GetIntermediateFileName(FLAGS_output, STATS_EXTENSION)); - stats::PrintTypeStatistic(file, info); + stats::CalcStatistics(dataFile, info); + + if (FLAGS_calc_statistics) + { + LOG(LINFO, ("Calculating statistics for", dataFile)); + stats::FileContainerStatistics(file, dataFile); + stats::PrintStatistics(file, info); + } + else + { + LOG(LINFO, ("Calculating type statistics for", dataFile)); + stats::PrintTypeStatistics(file, info); + } } if (FLAGS_dump_types) diff --git a/generator/statistics.cpp b/generator/statistics.cpp index 530d089e3e..fd2f5c1b69 100644 --- a/generator/statistics.cpp +++ b/generator/statistics.cpp @@ -17,14 +17,15 @@ using namespace std; namespace stats { - void FileContainerStatistic(std::ostream & os, string const & fPath) + void FileContainerStatistics(std::ostream & os, string const & fPath) { try { + os << "File section sizes" << endl; FilesContainerR cont(fPath); cont.ForEachTag([&] (FilesContainerR::Tag const & tag) { - os << std::setw(10) << tag << " : " << cont.GetReader(tag).Size() << '\n'; + os << std::setw(18) << tag << " : " << cont.GetReader(tag).Size() << endl; }); } catch (Reader::Exception const & ex) @@ -34,7 +35,7 @@ namespace stats } // 0.001 deg² ≈ 12.392 km² * cos(lat) - double arrAreas[] = { 10, 20, 50, 100, 200, 500, 1000, 360*360*12400 }; + double arrAreas[] = { 10, 20, 50, 100, 200, 500, 1000, 5000, 360*360*12400 }; size_t GetAreaIndex(double s) { @@ -55,19 +56,20 @@ namespace stats { f.ParseBeforeStatistic(); - FeatureType::InnerGeomStat const innerStats = f.GetInnerStatistic(); + FeatureType::InnerGeomStat const innerStats = f.GetInnerStats(); m_info.m_inner[0].Add(innerStats.m_points); m_info.m_inner[1].Add(innerStats.m_strips); m_info.m_inner[2].Add(innerStats.m_size); - // get geometry size for the best geometry - FeatureType::GeomStat const geom = f.GetGeometrySize(FeatureType::BEST_GEOMETRY); - FeatureType::GeomStat const trg = f.GetTrianglesSize(FeatureType::BEST_GEOMETRY); + // Get size stats and load the best geometry. + FeatureType::GeomStat const geom = f.GetOuterGeometrySize(); + FeatureType::GeomStat const trg = f.GetOuterTrianglesSize(); - m_info.m_byPointsCount[CountType(geom.m_count)].Add(geom.m_size); - m_info.m_byTrgCount[CountType(trg.m_count / 3)].Add(trg.m_size); + m_info.m_byPointsCount[CountType(geom.m_count)].Add(innerStats.m_points + geom.m_size); + m_info.m_byTrgCount[CountType(trg.m_count)].Add(innerStats.m_strips + trg.m_size); + // Header size (incl. inner geometry) + outer geometry size. uint32_t const allSize = innerStats.m_size + geom.m_size + trg.m_size; double len = 0.0; @@ -103,27 +105,33 @@ namespace stats m_info.m_byClassifType[ClassifType(type)].Add(allSize, len, area, hasName); }); - m_info.m_byAreaSize[AreaType(GetAreaIndex(area))].Add(trg.m_size, len, area, hasName); + m_info.m_byAreaSize[AreaType(GetAreaIndex(area))].Add(allSize, len, area, hasName); } }; - void CalcStatistic(std::string const & fPath, MapInfo & info) + void CalcStatistics(std::string const & fPath, MapInfo & info) { AccumulateStatistic doProcess(info); feature::ForEachFeature(fPath, doProcess); } - void PrintInfo(std::ostream & os, std::string const & prefix, GeneralInfo const & info, bool measurements) + void PrintInfo(std::ostream & os, std::string const & prefix, + GeneralInfo const & info, uint8_t prefixWidth = 1, + bool names = false, bool measurements = false) { - os << prefix << ": size = " << info.m_size << "; count = " << info.m_count; + os << std::setw(prefixWidth) << prefix + << ": size = " << std::setw(9) << info.m_size + << "; features = " << std::setw(8) << info.m_count; if (measurements) { - os << "; length = " << static_cast(info.m_length) - << " m; area = " << static_cast(info.m_area) << " m²"; + os << "; length = " << std::setw(10) << static_cast(info.m_length) + << " m; area = " << std::setw(10) << static_cast(info.m_area) << " m²"; } + if (names) + os << "; w/names = " << std::setw(8) << info.m_names; - os << "; names = " << info.m_names << '\n'; + os << endl; } std::string GetKey(GeomType type) @@ -143,7 +151,7 @@ namespace stats std::string GetKey(ClassifType t) { - return classif().GetFullObjectName(t.m_val); + return classif().GetReadableObjectName(t.m_val); } std::string GetKey(AreaType t) @@ -152,9 +160,10 @@ namespace stats } template - void PrintTop(std::ostream & os, char const * prefix, TSet const & theSet) + void PrintTop(std::ostream & os, char const * prefix, TSet const & theSet, + uint8_t prefixWidth = 5, bool names = false) { - os << prefix << endl; + os << endl << prefix << endl; vector> vec(theSet.begin(), theSet.end()); @@ -163,8 +172,8 @@ namespace stats size_t const count = min(static_cast(10), vec.size()); for (size_t i = 0; i < count; ++i) { - os << i << ". "; - PrintInfo(os, GetKey(vec[i].first), vec[i].second, false); + os << std::setw(2) << i << ". "; + PrintInfo(os, GetKey(vec[i].first), vec[i].second, prefixWidth, names); } } @@ -186,24 +195,28 @@ namespace stats } }; - void PrintStatistic(std::ostream & os, MapInfo & info) + void PrintStatistics(std::ostream & os, MapInfo & info) { - PrintInfo(os, "DAT header", info.m_inner[2], false); - PrintInfo(os, "Points header", info.m_inner[0], false); - PrintInfo(os, "Strips header", info.m_inner[1], false); + PrintInfo(os, "\nFeature headers", info.m_inner[2]); + PrintInfo(os, " incl. inner points", info.m_inner[0]); + PrintInfo(os, " incl. inner triangles (strips)", info.m_inner[1]); - PrintTop(os, "Top SIZE by Geometry Type", info.m_byGeomType); - PrintTop(os, "Top SIZE by Classificator Type", info.m_byClassifType); + PrintTop(os, "Top SIZE by Geometry Type", info.m_byGeomType, 5, true); + PrintTop(os, "Top SIZE by Classificator Type\n" + "(a single feature's size may be included in several types)", + info.m_byClassifType, 30, true); PrintTop(os, "Top SIZE by Points Count", info.m_byPointsCount); PrintTop(os, "Top SIZE by Triangles Count", info.m_byTrgCount); - PrintTop(os, "Top SIZE by Area", info.m_byAreaSize); + PrintTop(os, "Top SIZE by Area", info.m_byAreaSize, 5, true); } - void PrintTypeStatistic(std::ostream & os, MapInfo & info) + void PrintTypeStatistics(std::ostream & os, MapInfo & info) { + os << "NOTE: a single feature can contain several types and thus its size can be included in several type lines." + << endl << endl; for (auto it = info.m_byClassifType.begin(); it != info.m_byClassifType.end(); ++it) { - PrintInfo(os, GetKey(it->first).c_str(), it->second, true); + PrintInfo(os, GetKey(it->first).c_str(), it->second, 30, true, true); } } } diff --git a/generator/statistics.hpp b/generator/statistics.hpp index ab9b569f16..0486cf2eb8 100644 --- a/generator/statistics.hpp +++ b/generator/statistics.hpp @@ -55,9 +55,9 @@ namespace stats GeneralInfo m_inner[3]; }; - void FileContainerStatistic(std::ostream & os, std::string const & fPath); + void FileContainerStatistics(std::ostream & os, std::string const & fPath); - void CalcStatistic(std::string const & fPath, MapInfo & info); - void PrintStatistic(std::ostream & os, MapInfo & info); - void PrintTypeStatistic(std::ostream & os, MapInfo & info); + void CalcStatistics(std::string const & fPath, MapInfo & info); + void PrintStatistics(std::ostream & os, MapInfo & info); + void PrintTypeStatistics(std::ostream & os, MapInfo & info); } diff --git a/indexer/feature.cpp b/indexer/feature.cpp index 4806387070..26e0966b16 100644 --- a/indexer/feature.cpp +++ b/indexer/feature.cpp @@ -374,6 +374,7 @@ void FeatureType::ParseHeader2() auto const * start = src.PtrUint8(); src = ArrayByteSource(serial::LoadInnerPath(start, ptsCount, cp, m_points)); + // TODO: here and further m_innerStats is needed for stats calculation in generator_tool only m_innerStats.m_points = static_cast(src.PtrUint8() - start); } else @@ -397,6 +398,7 @@ void FeatureType::ParseHeader2() ReadOffsets(*m_loadInfo, src, trgMask, m_offsets.m_trg); } } + // Size of the whole header incl. inner geometry / triangles. m_innerStats.m_size = CalcOffset(src, m_data.data()); m_parsed.m_header2 = true; } @@ -475,9 +477,53 @@ uint32_t FeatureType::ParseGeometry(int scale) } m_parsed.m_points = true; } + // TODO: return value is needed for stats calculation in generator_tool only return sz; } +FeatureType::GeomStat FeatureType::GetOuterGeometrySize() +{ + uint32_t sz = 0; + + CHECK(m_loadInfo, ()); + + auto const headerGeomType = static_cast(Header(m_data) & HEADER_MASK_GEOMTYPE); + if (headerGeomType == HeaderGeomType::Line) + { + size_t const count = m_points.size(); + if (count < 2) + { + ASSERT_EQUAL(count, 1, ()); + + FeatureType::Points points; + int const n = m_loadInfo->GetScalesCount(); + for (int ind = 0; ind < n; ++ind) + { + if (m_offsets.m_pts[ind] != kInvalidOffset) + { + points.clear(); + points.emplace_back(m_points.front()); + + ReaderSource src(m_loadInfo->GetGeometryReader(ind)); + src.Skip(m_offsets.m_pts[ind]); + + serial::GeometryCodingParams cp = m_loadInfo->GetGeometryCodingParams(ind); + cp.SetBasePoint(points[0]); + serial::LoadOuterPath(src, cp, points); + + sz += static_cast(src.Pos() - m_offsets.m_pts[ind]); + } + } + // Retain best geometry. + m_points.swap(points); + } + CalcRect(m_points, m_limitRect); + } + m_parsed.m_points = true; + + return GeomStat(sz, m_points.size()); +} + uint32_t FeatureType::ParseTriangles(int scale) { uint32_t sz = 0; @@ -506,9 +552,44 @@ uint32_t FeatureType::ParseTriangles(int scale) } m_parsed.m_triangles = true; } + // TODO: return value is needed for stats calculation in generator_tool only return sz; } +FeatureType::GeomStat FeatureType::GetOuterTrianglesSize() +{ + uint32_t sz = 0; + + CHECK(m_loadInfo, ()); + + auto const headerGeomType = static_cast(Header(m_data) & HEADER_MASK_GEOMTYPE); + if (headerGeomType == HeaderGeomType::Area) + { + if (m_triangles.empty()) + { + int const n = m_loadInfo->GetScalesCount(); + for (int ind = 0; ind < n; ++ind) + { + if (m_offsets.m_trg[ind] != kInvalidOffset) + { + m_triangles.clear(); + + ReaderSource src(m_loadInfo->GetTrianglesReader(ind)); + src.Skip(m_offsets.m_trg[ind]); + serial::LoadOuterTriangles(src, m_loadInfo->GetGeometryCodingParams(ind), m_triangles); + + sz += static_cast(src.Pos() - m_offsets.m_trg[ind]); + } + } + // The best geometry is retained in m_triangles. + } + CalcRect(m_triangles, m_limitRect); + } + m_parsed.m_triangles = true; + + return GeomStat(sz, m_triangles.size() / 3); +} + void FeatureType::ParseMetadata() { if (m_parsed.m_metadata) @@ -670,24 +751,6 @@ void FeatureType::ParseGeometryAndTriangles(int scale) ParseTriangles(scale); } -FeatureType::GeomStat FeatureType::GetGeometrySize(int scale) -{ - uint32_t sz = ParseGeometry(scale); - if (sz == 0 && !m_points.empty()) - sz = m_innerStats.m_points; - - return GeomStat(sz, m_points.size()); -} - -FeatureType::GeomStat FeatureType::GetTrianglesSize(int scale) -{ - uint32_t sz = ParseTriangles(scale); - if (sz == 0 && !m_triangles.empty()) - sz = m_innerStats.m_strips; - - return GeomStat(sz, m_triangles.size()); -} - std::pair FeatureType::GetPreferredNames() { feature::NameParamsOut out; diff --git a/indexer/feature.hpp b/indexer/feature.hpp index a0af9b4b20..2e064a7641 100644 --- a/indexer/feature.hpp +++ b/indexer/feature.hpp @@ -166,14 +166,9 @@ public: struct InnerGeomStat { uint32_t m_points = 0, m_strips = 0, m_size = 0; - - void MakeZero() - { - m_points = m_strips = m_size = 0; - } }; - InnerGeomStat GetInnerStatistic() const { return m_innerStats; } + InnerGeomStat GetInnerStats() const { return m_innerStats; } struct GeomStat { @@ -182,8 +177,10 @@ public: GeomStat(uint32_t sz, size_t count) : m_size(sz), m_count(static_cast(count)) {} }; - GeomStat GetGeometrySize(int scale); - GeomStat GetTrianglesSize(int scale); + // Returns total outer geometry/triangles size for all geo levels and + // number of points/triangles in the best one. Loads the best geometry. + GeomStat GetOuterGeometrySize(); + GeomStat GetOuterTrianglesSize(); //@} private: