diff --git a/coding/zip_reader.cpp b/coding/zip_reader.cpp index 102bdd77a4..427048a903 100644 --- a/coding/zip_reader.cpp +++ b/coding/zip_reader.cpp @@ -3,13 +3,42 @@ #include "base/scope_guard.hpp" #include "base/logging.hpp" -#include "coding/file_writer.hpp" #include "coding/constants.hpp" #include "std/bind.hpp" #include "3party/minizip/unzip.h" +namespace +{ +class UnzipFileDelegate : public ZipFileReader::Delegate +{ +public: + UnzipFileDelegate(string const & path) + : m_file(make_unique(path)), m_path(path), m_completed(false) + { + } + + ~UnzipFileDelegate() override + { + if (!m_completed) + { + m_file.reset(); + FileWriter::DeleteFileX(m_path); + } + } + + // ZipFileReader::Delegate overrides: + void OnBlockUnzipped(size_t size, char const * data) override { m_file->Write(data, size); } + + void OnCompleted() override { m_completed = true; } + +private: + unique_ptr m_file; + string const m_path; + bool m_completed; +}; +} // namespace ZipFileReader::ZipFileReader(string const & container, string const & file, uint32_t logPageSize, uint32_t logPageCount) @@ -73,8 +102,9 @@ bool ZipFileReader::IsZip(string const & zipContainer) return true; } +// static void ZipFileReader::UnzipFile(string const & zipContainer, string const & fileInZip, - string const & outFilePath, ProgressFn progressFn) + Delegate & delegate) { unzFile zip = unzOpen64(zipContainer.c_str()); if (!zip) @@ -92,54 +122,28 @@ void ZipFileReader::UnzipFile(string const & zipContainer, string const & fileIn if (UNZ_OK != unzGetCurrentFileInfo64(zip, &fileInfo, NULL, 0, NULL, 0, NULL, 0)) MYTHROW(LocateZipException, ("Can't get uncompressed file size inside zip", fileInZip)); - // First outFile should be closed, then FileWriter::DeleteFileX is called, - // so make correct order of guards. - MY_SCOPE_GUARD(outFileGuard, bind(&FileWriter::DeleteFileX, cref(outFilePath))); - FileWriter outFile(outFilePath); - - uint64_t pos = 0; char buf[ZIP_FILE_BUFFER_SIZE]; - while (true) + int readBytes = 0; + + delegate.OnStarted(); + do { - int const readBytes = unzReadCurrentFile(zip, buf, ZIP_FILE_BUFFER_SIZE); - if (readBytes > 0) - outFile.Write(buf, static_cast(readBytes)); - else if (readBytes < 0) - MYTHROW(InvalidZipException, ("Error", readBytes, "while unzipping", fileInZip, "from", zipContainer)); - else - break; + readBytes = unzReadCurrentFile(zip, buf, ZIP_FILE_BUFFER_SIZE); + if (readBytes < 0) + { + MYTHROW(InvalidZipException, + ("Error", readBytes, "while unzipping", fileInZip, "from", zipContainer)); + } - pos += readBytes; - - if (progressFn) - progressFn(fileInfo.uncompressed_size, pos); - } - - outFileGuard.release(); + delegate.OnBlockUnzipped(static_cast(readBytes), buf); + } while (readBytes != 0); + delegate.OnCompleted(); } -void ZipFileReader::UnzipFileToMemory(string const & cont, string const & file, - vector & data) +// static +void ZipFileReader::UnzipFile(string const & zipContainer, string const & fileInZip, + string const & outPath) { - unzFile zip = unzOpen64(cont.c_str()); - if (!zip) - MYTHROW(OpenZipException, ("Can't get zip file handle:", cont)); - MY_SCOPE_GUARD(zipCloser, bind(&unzClose, zip)); - - if (UNZ_OK != unzLocateFile(zip, file.c_str(), 1 /* case sensitivity */)) - MYTHROW(LocateZipException, ("Can't locate file inside zip container:", file)); - if (UNZ_OK != unzOpenCurrentFile(zip)) - MYTHROW(LocateZipException, ("Can't open file inside zip container:", file)); - MY_SCOPE_GUARD(currentFileCloser, bind(&unzCloseCurrentFile, zip)); - - unz_file_info64 info; - if (UNZ_OK != unzGetCurrentFileInfo64(zip, &info, NULL, 0, NULL, 0, NULL, 0)) - MYTHROW(LocateZipException, ("Can't get uncompressed file size inside zip:", file)); - - size_t const size = info.uncompressed_size; - data.resize(size); - - int const bytesRead = unzReadCurrentFile(zip, data.data(), size); - if (bytesRead < 0) - MYTHROW(InvalidZipException, ("Error:", bytesRead, "while unzipping", file, "in", cont)); + UnzipFileDelegate delegate(outPath); + UnzipFile(zipContainer, fileInZip, delegate); } diff --git a/coding/zip_reader.hpp b/coding/zip_reader.hpp index 7970cff087..bea0496ae5 100644 --- a/coding/zip_reader.hpp +++ b/coding/zip_reader.hpp @@ -1,6 +1,7 @@ #pragma once #include "coding/file_reader.hpp" +#include "coding/file_writer.hpp" #include "base/exception.hpp" @@ -13,6 +14,17 @@ private: uint64_t m_uncompressedFileSize; public: + struct Delegate + { + virtual ~Delegate() = default; + + // When |size| is zero, end of file is reached. + virtual void OnBlockUnzipped(size_t size, char const * data) = 0; + + virtual void OnStarted() {} + virtual void OnCompleted() {} + }; + typedef function ProgressFn; /// Contains file name inside zip and it's uncompressed size typedef vector > FileListT; @@ -29,13 +41,9 @@ public: uint64_t UncompressedSize() const { return m_uncompressedFileSize; } /// @warning Can also throw Writer::OpenException and Writer::WriteException + static void UnzipFile(string const & zipContainer, string const & fileInZip, Delegate & delegate); static void UnzipFile(string const & zipContainer, string const & fileInZip, - string const & outFilePath, ProgressFn progressFn = ProgressFn()); - - /// Unzips |file| in |cont| to |buffer|. - /// - /// @warning Can throw OpenZipException and LocateZipException. - static void UnzipFileToMemory(string const & cont, string const & file, vector & data); + string const & outPath); static void FilesList(string const & zipContainer, FileListT & filesList); diff --git a/generator/generator_tests/srtm_parser_test.cpp b/generator/generator_tests/srtm_parser_test.cpp index 4420ef91a7..79ef60431e 100644 --- a/generator/generator_tests/srtm_parser_test.cpp +++ b/generator/generator_tests/srtm_parser_test.cpp @@ -6,7 +6,7 @@ using namespace generator; namespace { -string GetBase(ms::LatLon const & coord) { return SrtmFile::GetBase(coord); } +inline string GetBase(ms::LatLon const & coord) { return SrtmTile::GetBase(coord); } UNIT_TEST(FilenameTests) { diff --git a/generator/srtm_source_tests/srtm_source_tests.cpp b/generator/srtm_coverage_checker/srtm_coverage_checker.cpp similarity index 61% rename from generator/srtm_source_tests/srtm_source_tests.cpp rename to generator/srtm_coverage_checker/srtm_coverage_checker.cpp index 5895b24bc5..3df3a27eea 100644 --- a/generator/srtm_source_tests/srtm_source_tests.cpp +++ b/generator/srtm_coverage_checker/srtm_coverage_checker.cpp @@ -1,5 +1,3 @@ -#include "testing/testing.hpp" - #include "generator/srtm_parser.hpp" #include "map/feature_vec_model.hpp" @@ -11,17 +9,39 @@ #include "platform/country_file.hpp" #include "platform/local_country_file_utils.hpp" -namespace -{ -#define SRTM_SOURCES_PATH "srtm/e4ftl01.cr.usgs.gov/SRTM/SRTMGL1.003/2000.02.11/" +#include "base/logging.hpp" -UNIT_TEST(SrtmCoverageChecker) +#include "std/algorithm.hpp" + +#include "3party/gflags/src/gflags/gflags.h" + +DEFINE_string(srtm_path, "", "Path to directory with SRTM files"); +DEFINE_string(mwm_path, "", "Path to mwm files (writable dir)"); + +int main(int argc, char * argv[]) { + google::SetUsageMessage("SRTM coverage checker."); + google::ParseCommandLineFlags(&argc, &argv, true); + + Platform & platform = GetPlatform(); + if (!FLAGS_mwm_path.empty()) + platform.SetWritableDirForTests(FLAGS_mwm_path); + + if (FLAGS_srtm_path.empty()) + { + LOG(LERROR, ("SRTM files directory is not specified.")); + return -1; + } + + LOG(LINFO, ("writable dir =", platform.WritableDir())); + LOG(LINFO, ("srtm dir =", FLAGS_srtm_path)); + vector localFiles; platform::FindAllLocalMapsAndCleanup(numeric_limits::max() /* latestVersion */, localFiles); - shared_ptr fetcher = integration::CreateFeaturesFetcher(localFiles); + auto fetcher = integration::CreateFeaturesFetcher(localFiles); + generator::SrtmTileManager manager(FLAGS_srtm_path); for (auto & file : localFiles) { @@ -46,10 +66,6 @@ UNIT_TEST(SrtmCoverageChecker) segMapping.Load(container, file); segMapping.Map(container); - Platform & platform = GetPlatform(); - generator::SrtmFileManager manager( - my::JoinFoldersToPath(platform.WritableDir(), SRTM_SOURCES_PATH)); - size_t all = 0; size_t good = 0; @@ -57,8 +73,8 @@ UNIT_TEST(SrtmCoverageChecker) { buffer_vector buffer; segMapping.ForEachFtSeg(i, MakeBackInsertFunctor(buffer)); - vector path; + vector path; for (size_t k = 0; k < buffer.size(); ++k) { auto const & segment = buffer[k]; @@ -72,35 +88,29 @@ UNIT_TEST(SrtmCoverageChecker) ft.ParseGeometry(FeatureType::BEST_GEOMETRY); // Get points in proper direction. - auto startIdx = segment.m_pointStart; - auto endIdx = segment.m_pointEnd; - if (startIdx < endIdx) - { - for (auto idx = startIdx; idx <= endIdx; ++idx) - path.push_back(ft.GetPoint(idx)); - } - else - { - // I use big signed type because endIdx can be 0. - for (int64_t idx = startIdx; idx >= static_cast(endIdx); --idx) - path.push_back(ft.GetPoint(idx)); - } + auto const startIdx = segment.m_pointStart; + auto const endIdx = segment.m_pointEnd; + for (auto idx = min(startIdx, endIdx); idx <= max(startIdx, endIdx); ++idx) + path.push_back(ft.GetPoint(idx)); + + all += path.size(); for (auto const & point : path) { auto const height = manager.GetHeight(MercatorBounds::ToLatLon(point)); - all++; - if (height != generator::SrtmFile::kInvalidHeight) + if (height != generator::SrtmTile::kInvalidHeight) good++; } } } - auto const delta = all - good; - auto const percent = delta * 100.0 / all; + + auto const bad = all - good; + auto const percent = all == 0 ? 0.0 : bad * 100.0 / all; if (percent > 10.0) { - LOG(LINFO, ("Huge error rate in:", file.GetCountryName(), "Good:", good, "All:", all, - "Delta:", delta, "%:", percent)); + LOG(LINFO, ("Huge error rate in:", file.GetCountryName(), "good:", good, "bad:", bad, "all:", + all, "%:", percent)); } } + + return 0; } -} // namespace diff --git a/generator/srtm_coverage_checker/srtm_coverage_checker.pro b/generator/srtm_coverage_checker/srtm_coverage_checker.pro new file mode 100644 index 0000000000..6f62e71f15 --- /dev/null +++ b/generator/srtm_coverage_checker/srtm_coverage_checker.pro @@ -0,0 +1,25 @@ +# SRTM coverage checker tool. +# Checks coverage of car roads by SRTM data. + +TARGET = srtm_coverage_tool +CONFIG += console warn_on +CONFIG -= app_bundle +TEMPLATE = app + +ROOT_DIR = ../.. +DEPENDENCIES = generator map routing search storage indexer platform editor geometry coding base \ + osrm jansson protobuf tomcrypt succinct stats_client pugixml minizip gflags + +include($$ROOT_DIR/common.pri) + +INCLUDEPATH *= $$ROOT_DIR/3party/gflags/src + +QT *= core + +macx-* { + LIBS *= "-framework IOKit" "-framework SystemConfiguration" +} + +SOURCES += \ + ../../routing/routing_integration_tests/routing_test_tools.cpp \ + srtm_coverage_checker.cpp \ diff --git a/generator/srtm_parser.cpp b/generator/srtm_parser.cpp index 6630c335de..97939570c5 100644 --- a/generator/srtm_parser.cpp +++ b/generator/srtm_parser.cpp @@ -10,18 +10,43 @@ namespace generator { -// SrtmFile ---------------------------------------------------------------------------------------- -SrtmFile::SrtmFile() +namespace +{ +size_t constexpr kArcSecondsInDegree = 60 * 60; +size_t constexpr kSrtmTileSize = (kArcSecondsInDegree + 1) * (kArcSecondsInDegree + 1) * 2; + +struct UnzipMemDelegate : public ZipFileReader::Delegate +{ + UnzipMemDelegate(string & buffer) : m_buffer(buffer), m_completed(false) {} + + // ZipFileReader::Delegate overrides: + void OnBlockUnzipped(size_t size, char const * data) override { m_buffer.append(data, size); } + + void OnStarted() override + { + m_buffer.clear(); + m_completed = false; + } + + void OnCompleted() override { m_completed = true; } + + string & m_buffer; + bool m_completed; +}; +} // namespace + +// SrtmTile ---------------------------------------------------------------------------------------- +SrtmTile::SrtmTile() { Invalidate(); } -SrtmFile::SrtmFile(SrtmFile && rhs) : m_data(move(rhs.m_data)), m_valid(rhs.m_valid) +SrtmTile::SrtmTile(SrtmTile && rhs) : m_data(move(rhs.m_data)), m_valid(rhs.m_valid) { rhs.Invalidate(); } -void SrtmFile::Init(string const & dir, ms::LatLon const & coord) +void SrtmTile::Init(string const & dir, ms::LatLon const & coord) { Invalidate(); @@ -29,21 +54,37 @@ void SrtmFile::Init(string const & dir, ms::LatLon const & coord) string const cont = dir + base + ".SRTMGL1.hgt.zip"; string file = base + ".hgt"; + UnzipMemDelegate delegate(m_data); try { - ZipFileReader::UnzipFileToMemory(cont, file, m_data); + ZipFileReader::UnzipFile(cont, file, delegate); } - catch (ZipFileReader::LocateZipException e) + catch (ZipFileReader::LocateZipException const & e) { // Sometimes packed file has different name. See N39E051 measure. file = base + ".SRTMGL1.hgt"; - ZipFileReader::UnzipFileToMemory(cont, file, m_data); + + ZipFileReader::UnzipFile(cont, file, delegate); + } + + if (!delegate.m_completed) + { + LOG(LWARNING, ("Can't decompress SRTM file:", cont)); + Invalidate(); + return; + } + + if (m_data.size() != kSrtmTileSize) + { + LOG(LWARNING, ("Bad decompressed SRTM file size:", cont, m_data.size())); + Invalidate(); + return; } m_valid = true; } -SrtmFile::THeight SrtmFile::GetHeight(ms::LatLon const & coord) +SrtmTile::THeight SrtmTile::GetHeight(ms::LatLon const & coord) { if (!IsValid()) return kInvalidHeight; @@ -56,17 +97,17 @@ SrtmFile::THeight SrtmFile::GetHeight(ms::LatLon const & coord) lt += 1; lt = 1 - lt; // from North to South - size_t const row = 3600 * lt; - size_t const col = 3600 * ln; + size_t const row = kArcSecondsInDegree * lt; + size_t const col = kArcSecondsInDegree * ln; - size_t const ix = row * 3601 + col; + size_t const ix = row * (kArcSecondsInDegree + 1) + col; if (ix >= Size()) return kInvalidHeight; return ReverseByteOrder(Data()[ix]); } -string SrtmFile::GetBase(ms::LatLon coord) +string SrtmTile::GetBase(ms::LatLon coord) { ostringstream ss; if (coord.lat < 0) @@ -95,37 +136,37 @@ string SrtmFile::GetBase(ms::LatLon coord) return ss.str(); } -void SrtmFile::Invalidate() +void SrtmTile::Invalidate() { m_data.clear(); m_data.shrink_to_fit(); m_valid = false; } -// SrtmFileManager --------------------------------------------------------------------------------- -SrtmFileManager::SrtmFileManager(string const & dir) : m_dir(dir) {} +// SrtmTileManager --------------------------------------------------------------------------------- +SrtmTileManager::SrtmTileManager(string const & dir) : m_dir(dir) {} -SrtmFile::THeight SrtmFileManager::GetHeight(ms::LatLon const & coord) +SrtmTile::THeight SrtmTileManager::GetHeight(ms::LatLon const & coord) { - string const base = SrtmFile::GetBase(coord); - auto it = m_storage.find(base); - if (it == m_storage.end()) + string const base = SrtmTile::GetBase(coord); + auto it = m_tiles.find(base); + if (it == m_tiles.end()) { - SrtmFile file; + SrtmTile tile; try { - file.Init(m_dir, coord); + tile.Init(m_dir, coord); } catch (RootException const & e) { - LOG(LERROR, ("Can't init SRTM file:", base, "reason:", e.Msg())); + LOG(LWARNING, ("Can't init SRTM tile:", base, "reason:", e.Msg())); } - // It's OK to store even invalid files and return invalid height + // It's OK to store even invalid tiles and return invalid height // for them later. - m_storage.emplace(base, move(file)); + it = m_tiles.emplace(base, move(tile)).first; } - return m_storage[base].GetHeight(coord); + return it->second.GetHeight(coord); } } // namespace generator diff --git a/generator/srtm_parser.hpp b/generator/srtm_parser.hpp index bf521ad988..74bc14b755 100644 --- a/generator/srtm_parser.hpp +++ b/generator/srtm_parser.hpp @@ -7,19 +7,18 @@ #include "std/cstdint.hpp" #include "std/string.hpp" #include "std/unordered_map.hpp" -#include "std/vector.hpp" namespace generator { -class SrtmFile +class SrtmTile { public: using THeight = int16_t; static THeight constexpr kInvalidHeight = -32768; - SrtmFile(); - SrtmFile(SrtmFile && rhs); + SrtmTile(); + SrtmTile(SrtmTile && rhs); void Init(string const & dir, ms::LatLon const & coord); @@ -37,23 +36,23 @@ private: void Invalidate(); - vector m_data; + string m_data; bool m_valid; - DISALLOW_COPY(SrtmFile); + DISALLOW_COPY(SrtmTile); }; -class SrtmFileManager +class SrtmTileManager { public: - SrtmFileManager(string const & dir); + SrtmTileManager(string const & dir); - SrtmFile::THeight GetHeight(ms::LatLon const & coord); + SrtmTile::THeight GetHeight(ms::LatLon const & coord); private: string m_dir; - unordered_map m_storage; + unordered_map m_tiles; - DISALLOW_COPY(SrtmFileManager); + DISALLOW_COPY(SrtmTileManager); }; } // namespace generator diff --git a/generator/srtm_source_tests/srtm_source_tests.pro b/generator/srtm_source_tests/srtm_source_tests.pro deleted file mode 100644 index 00d9b7d7c0..0000000000 --- a/generator/srtm_source_tests/srtm_source_tests.pro +++ /dev/null @@ -1,22 +0,0 @@ -# This subproject implements digital elevation model source test. -# This tests are launched on the raw SRTM dataset. - -TARGET = srtm_source_tests -CONFIG += console warn_on -CONFIG -= app_bundle -TEMPLATE = app - -ROOT_DIR = ../.. -DEPENDENCIES = generator map routing search storage indexer platform editor geometry coding base \ - osrm jansson protobuf tomcrypt succinct stats_client pugixml minizip - -macx-*: LIBS *= "-framework IOKit" "-framework SystemConfiguration" - -include($$ROOT_DIR/common.pri) - -QT *= core - -SOURCES += \ - ../../testing/testingmain.cpp \ - ../../routing/routing_integration_tests/routing_test_tools.cpp \ - srtm_source_tests.cpp \ diff --git a/omim.pro b/omim.pro index fed9f9abb8..a6d6a159db 100644 --- a/omim.pro +++ b/omim.pro @@ -37,9 +37,9 @@ SUBDIRS = 3party base coding geometry editor indexer routing search routing_integration_tests.depends = $$SUBDIRS routing_consistency_tests.subdir = routing/routing_consistency_tests routing_consistency_tests.depends = $$SUBDIRS - srtm_source_tests.subdir = generator/srtm_source_tests - srtm_source_tests.depends = $$SUBDIRS routing - SUBDIRS *= routing_integration_tests routing_consistency_tests srtm_source_tests + srtm_coverage_checker.subdir = generator/srtm_coverage_checker + srtm_coverage_checker.depends = $$SUBDIRS routing + SUBDIRS *= routing_integration_tests routing_consistency_tests srtm_coverage_checker } CONFIG(desktop) { @@ -170,9 +170,9 @@ SUBDIRS = 3party base coding geometry editor indexer routing search routing_consistency_tests.depends = $$MapDepLibs routing SUBDIRS *= routing_consistency_tests - srtm_source_tests.subdir = generator/srtm_source_tests - srtm_source_tests.depends = $$MapDepLibs routing - SUBDIRS *= srtm_source_tests + srtm_coverage_checker.subdir = generator/srtm_coverage_checker + srtm_coverage_checker.depends = $$MapDepLibs routing + SUBDIRS *= srtm_coverage_checker # TODO(AlexZ): Move pedestrian tests into routing dir. pedestrian_routing_tests.depends = $$MapDepLibs routing