diff --git a/coding/zip_reader.cpp b/coding/zip_reader.cpp index 32de85bcbf..427048a903 100644 --- a/coding/zip_reader.cpp +++ b/coding/zip_reader.cpp @@ -3,13 +3,42 @@ #include "base/scope_guard.hpp" #include "base/logging.hpp" -#include "coding/file_writer.hpp" #include "coding/constants.hpp" #include "std/bind.hpp" #include "3party/minizip/unzip.h" +namespace +{ +class UnzipFileDelegate : public ZipFileReader::Delegate +{ +public: + UnzipFileDelegate(string const & path) + : m_file(make_unique(path)), m_path(path), m_completed(false) + { + } + + ~UnzipFileDelegate() override + { + if (!m_completed) + { + m_file.reset(); + FileWriter::DeleteFileX(m_path); + } + } + + // ZipFileReader::Delegate overrides: + void OnBlockUnzipped(size_t size, char const * data) override { m_file->Write(data, size); } + + void OnCompleted() override { m_completed = true; } + +private: + unique_ptr m_file; + string const m_path; + bool m_completed; +}; +} // namespace ZipFileReader::ZipFileReader(string const & container, string const & file, uint32_t logPageSize, uint32_t logPageCount) @@ -73,8 +102,9 @@ bool ZipFileReader::IsZip(string const & zipContainer) return true; } +// static void ZipFileReader::UnzipFile(string const & zipContainer, string const & fileInZip, - string const & outFilePath, ProgressFn progressFn) + Delegate & delegate) { unzFile zip = unzOpen64(zipContainer.c_str()); if (!zip) @@ -92,28 +122,28 @@ void ZipFileReader::UnzipFile(string const & zipContainer, string const & fileIn if (UNZ_OK != unzGetCurrentFileInfo64(zip, &fileInfo, NULL, 0, NULL, 0, NULL, 0)) MYTHROW(LocateZipException, ("Can't get uncompressed file size inside zip", fileInZip)); - // First outFile should be closed, then FileWriter::DeleteFileX is called, - // so make correct order of guards. - MY_SCOPE_GUARD(outFileGuard, bind(&FileWriter::DeleteFileX, cref(outFilePath))); - FileWriter outFile(outFilePath); - - uint64_t pos = 0; char buf[ZIP_FILE_BUFFER_SIZE]; - while (true) + int readBytes = 0; + + delegate.OnStarted(); + do { - int const readBytes = unzReadCurrentFile(zip, buf, ZIP_FILE_BUFFER_SIZE); - if (readBytes > 0) - outFile.Write(buf, static_cast(readBytes)); - else if (readBytes < 0) - MYTHROW(InvalidZipException, ("Error", readBytes, "while unzipping", fileInZip, "from", zipContainer)); - else - break; + readBytes = unzReadCurrentFile(zip, buf, ZIP_FILE_BUFFER_SIZE); + if (readBytes < 0) + { + MYTHROW(InvalidZipException, + ("Error", readBytes, "while unzipping", fileInZip, "from", zipContainer)); + } - pos += readBytes; - - if (progressFn) - progressFn(fileInfo.uncompressed_size, pos); - } - - outFileGuard.release(); + delegate.OnBlockUnzipped(static_cast(readBytes), buf); + } while (readBytes != 0); + delegate.OnCompleted(); +} + +// static +void ZipFileReader::UnzipFile(string const & zipContainer, string const & fileInZip, + string const & outPath) +{ + UnzipFileDelegate delegate(outPath); + UnzipFile(zipContainer, fileInZip, delegate); } diff --git a/coding/zip_reader.hpp b/coding/zip_reader.hpp index b518b8d9e3..bea0496ae5 100644 --- a/coding/zip_reader.hpp +++ b/coding/zip_reader.hpp @@ -1,6 +1,7 @@ #pragma once #include "coding/file_reader.hpp" +#include "coding/file_writer.hpp" #include "base/exception.hpp" @@ -13,6 +14,17 @@ private: uint64_t m_uncompressedFileSize; public: + struct Delegate + { + virtual ~Delegate() = default; + + // When |size| is zero, end of file is reached. + virtual void OnBlockUnzipped(size_t size, char const * data) = 0; + + virtual void OnStarted() {} + virtual void OnCompleted() {} + }; + typedef function ProgressFn; /// Contains file name inside zip and it's uncompressed size typedef vector > FileListT; @@ -29,8 +41,9 @@ public: uint64_t UncompressedSize() const { return m_uncompressedFileSize; } /// @warning Can also throw Writer::OpenException and Writer::WriteException + static void UnzipFile(string const & zipContainer, string const & fileInZip, Delegate & delegate); static void UnzipFile(string const & zipContainer, string const & fileInZip, - string const & outFilePath, ProgressFn progressFn = ProgressFn()); + string const & outPath); static void FilesList(string const & zipContainer, FileListT & filesList); diff --git a/generator/generator.pro b/generator/generator.pro index d2d76c211e..42725c1953 100644 --- a/generator/generator.pro +++ b/generator/generator.pro @@ -30,6 +30,7 @@ SOURCES += \ osm_source.cpp \ routing_generator.cpp \ search_index_builder.cpp \ + srtm_parser.cpp \ statistics.cpp \ tesselator.cpp \ towns_dumper.cpp \ @@ -60,6 +61,7 @@ HEADERS += \ polygonizer.hpp \ routing_generator.hpp \ search_index_builder.hpp \ + srtm_parser.hpp \ statistics.hpp \ tag_admixer.hpp \ tesselator.hpp \ diff --git a/generator/generator_tests/generator_tests.pro b/generator/generator_tests/generator_tests.pro index 072e71a601..c55a55bf38 100644 --- a/generator/generator_tests/generator_tests.pro +++ b/generator/generator_tests/generator_tests.pro @@ -28,6 +28,7 @@ SOURCES += \ osm_type_test.cpp \ source_data.cpp \ source_to_element_test.cpp \ + srtm_parser_test.cpp \ tag_admixer_test.cpp \ tesselator_test.cpp \ triangles_tree_coding_test.cpp \ diff --git a/generator/generator_tests/srtm_parser_test.cpp b/generator/generator_tests/srtm_parser_test.cpp new file mode 100644 index 0000000000..79ef60431e --- /dev/null +++ b/generator/generator_tests/srtm_parser_test.cpp @@ -0,0 +1,28 @@ +#include "testing/testing.hpp" + +#include "generator/srtm_parser.hpp" + +using namespace generator; + +namespace +{ +inline string GetBase(ms::LatLon const & coord) { return SrtmTile::GetBase(coord); } + +UNIT_TEST(FilenameTests) +{ + auto name = GetBase({56.4566, 37.3467}); + TEST_EQUAL(name, "N56E037", ()); + + name = GetBase({34.077433, -118.304569}); + TEST_EQUAL(name, "N34W119", ()); + + name = GetBase({0.1, 0.1}); + TEST_EQUAL(name, "N00E000", ()); + + name = GetBase({-35.35, -12.1}); + TEST_EQUAL(name, "S36W013", ()); + + name = GetBase({-34.622358, -58.383654}); + TEST_EQUAL(name, "S35W059", ()); +} +} // namespace diff --git a/generator/srtm_coverage_checker/srtm_coverage_checker.cpp b/generator/srtm_coverage_checker/srtm_coverage_checker.cpp new file mode 100644 index 0000000000..3df3a27eea --- /dev/null +++ b/generator/srtm_coverage_checker/srtm_coverage_checker.cpp @@ -0,0 +1,116 @@ +#include "generator/srtm_parser.hpp" + +#include "map/feature_vec_model.hpp" + +#include "routing/routing_integration_tests/routing_test_tools.hpp" + +#include "coding/file_name_utils.hpp" + +#include "platform/country_file.hpp" +#include "platform/local_country_file_utils.hpp" + +#include "base/logging.hpp" + +#include "std/algorithm.hpp" + +#include "3party/gflags/src/gflags/gflags.h" + +DEFINE_string(srtm_path, "", "Path to directory with SRTM files"); +DEFINE_string(mwm_path, "", "Path to mwm files (writable dir)"); + +int main(int argc, char * argv[]) +{ + google::SetUsageMessage("SRTM coverage checker."); + google::ParseCommandLineFlags(&argc, &argv, true); + + Platform & platform = GetPlatform(); + if (!FLAGS_mwm_path.empty()) + platform.SetWritableDirForTests(FLAGS_mwm_path); + + if (FLAGS_srtm_path.empty()) + { + LOG(LERROR, ("SRTM files directory is not specified.")); + return -1; + } + + LOG(LINFO, ("writable dir =", platform.WritableDir())); + LOG(LINFO, ("srtm dir =", FLAGS_srtm_path)); + + vector localFiles; + platform::FindAllLocalMapsAndCleanup(numeric_limits::max() /* latestVersion */, + localFiles); + + auto fetcher = integration::CreateFeaturesFetcher(localFiles); + generator::SrtmTileManager manager(FLAGS_srtm_path); + + for (auto & file : localFiles) + { + file.SyncWithDisk(); + if (file.GetFiles() != MapOptions::MapWithCarRouting) + { + LOG(LINFO, ("Warning! Routing file not found for:", file.GetCountryName())); + continue; + } + + FilesMappingContainer container(file.GetPath(MapOptions::CarRouting)); + if (!container.IsExist(ROUTING_FTSEG_FILE_TAG)) + { + LOG(LINFO, ("Warning! Mwm file has not routing ftseg section:", file.GetCountryName())); + continue; + } + + routing::TDataFacade dataFacade; + dataFacade.Load(container); + + OsrmFtSegMapping segMapping; + segMapping.Load(container, file); + segMapping.Map(container); + + size_t all = 0; + size_t good = 0; + + for (size_t i = 0; i < dataFacade.GetNumberOfNodes(); ++i) + { + buffer_vector buffer; + segMapping.ForEachFtSeg(i, MakeBackInsertFunctor(buffer)); + + vector path; + for (size_t k = 0; k < buffer.size(); ++k) + { + auto const & segment = buffer[k]; + if (!segment.IsValid()) + continue; + // Load data from drive. + FeatureType ft; + Index::FeaturesLoaderGuard loader( + fetcher->GetIndex(), fetcher->GetIndex().GetMwmIdByCountryFile(file.GetCountryFile())); + loader.GetFeatureByIndex(segment.m_fid, ft); + ft.ParseGeometry(FeatureType::BEST_GEOMETRY); + + // Get points in proper direction. + auto const startIdx = segment.m_pointStart; + auto const endIdx = segment.m_pointEnd; + for (auto idx = min(startIdx, endIdx); idx <= max(startIdx, endIdx); ++idx) + path.push_back(ft.GetPoint(idx)); + + all += path.size(); + for (auto const & point : path) + { + auto const height = manager.GetHeight(MercatorBounds::ToLatLon(point)); + if (height != generator::SrtmTile::kInvalidHeight) + good++; + } + } + } + + auto const bad = all - good; + auto const percent = all == 0 ? 0.0 : bad * 100.0 / all; + if (percent > 10.0) + { + LOG(LINFO, ("Huge error rate in:", file.GetCountryName(), "good:", good, "bad:", bad, "all:", + all, "%:", percent)); + } + } + + return 0; +} diff --git a/generator/srtm_coverage_checker/srtm_coverage_checker.pro b/generator/srtm_coverage_checker/srtm_coverage_checker.pro new file mode 100644 index 0000000000..6f62e71f15 --- /dev/null +++ b/generator/srtm_coverage_checker/srtm_coverage_checker.pro @@ -0,0 +1,25 @@ +# SRTM coverage checker tool. +# Checks coverage of car roads by SRTM data. + +TARGET = srtm_coverage_tool +CONFIG += console warn_on +CONFIG -= app_bundle +TEMPLATE = app + +ROOT_DIR = ../.. +DEPENDENCIES = generator map routing search storage indexer platform editor geometry coding base \ + osrm jansson protobuf tomcrypt succinct stats_client pugixml minizip gflags + +include($$ROOT_DIR/common.pri) + +INCLUDEPATH *= $$ROOT_DIR/3party/gflags/src + +QT *= core + +macx-* { + LIBS *= "-framework IOKit" "-framework SystemConfiguration" +} + +SOURCES += \ + ../../routing/routing_integration_tests/routing_test_tools.cpp \ + srtm_coverage_checker.cpp \ diff --git a/generator/srtm_parser.cpp b/generator/srtm_parser.cpp new file mode 100644 index 0000000000..97939570c5 --- /dev/null +++ b/generator/srtm_parser.cpp @@ -0,0 +1,172 @@ +#include "generator/srtm_parser.hpp" + +#include "coding/endianness.hpp" +#include "coding/zip_reader.hpp" + +#include "base/logging.hpp" + +#include "std/iomanip.hpp" +#include "std/sstream.hpp" + +namespace generator +{ +namespace +{ +size_t constexpr kArcSecondsInDegree = 60 * 60; +size_t constexpr kSrtmTileSize = (kArcSecondsInDegree + 1) * (kArcSecondsInDegree + 1) * 2; + +struct UnzipMemDelegate : public ZipFileReader::Delegate +{ + UnzipMemDelegate(string & buffer) : m_buffer(buffer), m_completed(false) {} + + // ZipFileReader::Delegate overrides: + void OnBlockUnzipped(size_t size, char const * data) override { m_buffer.append(data, size); } + + void OnStarted() override + { + m_buffer.clear(); + m_completed = false; + } + + void OnCompleted() override { m_completed = true; } + + string & m_buffer; + bool m_completed; +}; +} // namespace + +// SrtmTile ---------------------------------------------------------------------------------------- +SrtmTile::SrtmTile() +{ + Invalidate(); +} + +SrtmTile::SrtmTile(SrtmTile && rhs) : m_data(move(rhs.m_data)), m_valid(rhs.m_valid) +{ + rhs.Invalidate(); +} + +void SrtmTile::Init(string const & dir, ms::LatLon const & coord) +{ + Invalidate(); + + string const base = GetBase(coord); + string const cont = dir + base + ".SRTMGL1.hgt.zip"; + string file = base + ".hgt"; + + UnzipMemDelegate delegate(m_data); + try + { + ZipFileReader::UnzipFile(cont, file, delegate); + } + catch (ZipFileReader::LocateZipException const & e) + { + // Sometimes packed file has different name. See N39E051 measure. + file = base + ".SRTMGL1.hgt"; + + ZipFileReader::UnzipFile(cont, file, delegate); + } + + if (!delegate.m_completed) + { + LOG(LWARNING, ("Can't decompress SRTM file:", cont)); + Invalidate(); + return; + } + + if (m_data.size() != kSrtmTileSize) + { + LOG(LWARNING, ("Bad decompressed SRTM file size:", cont, m_data.size())); + Invalidate(); + return; + } + + m_valid = true; +} + +SrtmTile::THeight SrtmTile::GetHeight(ms::LatLon const & coord) +{ + if (!IsValid()) + return kInvalidHeight; + + double ln = coord.lon - static_cast(coord.lon); + if (ln < 0) + ln += 1; + double lt = coord.lat - static_cast(coord.lat); + if (lt < 0) + lt += 1; + lt = 1 - lt; // from North to South + + size_t const row = kArcSecondsInDegree * lt; + size_t const col = kArcSecondsInDegree * ln; + + size_t const ix = row * (kArcSecondsInDegree + 1) + col; + + if (ix >= Size()) + return kInvalidHeight; + return ReverseByteOrder(Data()[ix]); +} + +string SrtmTile::GetBase(ms::LatLon coord) +{ + ostringstream ss; + if (coord.lat < 0) + { + ss << "S"; + coord.lat *= -1; + coord.lat += 1; + } + else + { + ss << "N"; + } + ss << setw(2) << setfill('0') << static_cast(coord.lat); + + if (coord.lon < 0) + { + ss << "W"; + coord.lon *= -1; + coord.lon += 1; + } + else + { + ss << "E"; + } + ss << setw(3) << static_cast(coord.lon); + return ss.str(); +} + +void SrtmTile::Invalidate() +{ + m_data.clear(); + m_data.shrink_to_fit(); + m_valid = false; +} + +// SrtmTileManager --------------------------------------------------------------------------------- +SrtmTileManager::SrtmTileManager(string const & dir) : m_dir(dir) {} + +SrtmTile::THeight SrtmTileManager::GetHeight(ms::LatLon const & coord) +{ + string const base = SrtmTile::GetBase(coord); + auto it = m_tiles.find(base); + if (it == m_tiles.end()) + { + SrtmTile tile; + try + { + tile.Init(m_dir, coord); + } + catch (RootException const & e) + { + LOG(LWARNING, ("Can't init SRTM tile:", base, "reason:", e.Msg())); + } + + // It's OK to store even invalid tiles and return invalid height + // for them later. + it = m_tiles.emplace(base, move(tile)).first; + } + + return it->second.GetHeight(coord); +} +} // namespace generator diff --git a/generator/srtm_parser.hpp b/generator/srtm_parser.hpp new file mode 100644 index 0000000000..74bc14b755 --- /dev/null +++ b/generator/srtm_parser.hpp @@ -0,0 +1,58 @@ +#pragma once + +#include "geometry/latlon.hpp" + +#include "base/macros.hpp" + +#include "std/cstdint.hpp" +#include "std/string.hpp" +#include "std/unordered_map.hpp" + +namespace generator +{ +class SrtmTile +{ +public: + using THeight = int16_t; + + static THeight constexpr kInvalidHeight = -32768; + + SrtmTile(); + SrtmTile(SrtmTile && rhs); + + void Init(string const & dir, ms::LatLon const & coord); + + inline bool IsValid() const { return m_valid; } + + // Returns height in meters at |coord|, or kInvalidHeight if is not initialized. + THeight GetHeight(ms::LatLon const & coord); + + static string GetBase(ms::LatLon coord); + +private: + inline THeight const * Data() const { return reinterpret_cast(m_data.data()); }; + + inline size_t Size() const { return m_data.size() / sizeof(THeight); } + + void Invalidate(); + + string m_data; + bool m_valid; + + DISALLOW_COPY(SrtmTile); +}; + +class SrtmTileManager +{ +public: + SrtmTileManager(string const & dir); + + SrtmTile::THeight GetHeight(ms::LatLon const & coord); + +private: + string m_dir; + unordered_map m_tiles; + + DISALLOW_COPY(SrtmTileManager); +}; +} // namespace generator diff --git a/omim.pro b/omim.pro index 3a4a0ea76a..a6d6a159db 100644 --- a/omim.pro +++ b/omim.pro @@ -37,7 +37,9 @@ SUBDIRS = 3party base coding geometry editor indexer routing search routing_integration_tests.depends = $$SUBDIRS routing_consistency_tests.subdir = routing/routing_consistency_tests routing_consistency_tests.depends = $$SUBDIRS - SUBDIRS *= routing_integration_tests routing_consistency_tests + srtm_coverage_checker.subdir = generator/srtm_coverage_checker + srtm_coverage_checker.depends = $$SUBDIRS routing + SUBDIRS *= routing_integration_tests routing_consistency_tests srtm_coverage_checker } CONFIG(desktop) { @@ -168,6 +170,10 @@ SUBDIRS = 3party base coding geometry editor indexer routing search routing_consistency_tests.depends = $$MapDepLibs routing SUBDIRS *= routing_consistency_tests + srtm_coverage_checker.subdir = generator/srtm_coverage_checker + srtm_coverage_checker.depends = $$MapDepLibs routing + SUBDIRS *= srtm_coverage_checker + # TODO(AlexZ): Move pedestrian tests into routing dir. pedestrian_routing_tests.depends = $$MapDepLibs routing SUBDIRS *= pedestrian_routing_tests