From 9375997212876b739a713dd49caf99821be8466d Mon Sep 17 00:00:00 2001 From: Lev Dragunov Date: Fri, 22 Apr 2016 13:25:12 +0300 Subject: [PATCH 1/3] Srtm reader with tests. --- coding/zip_reader.cpp | 28 +++++ coding/zip_reader.hpp | 3 + generator/generator.pro | 2 + generator/generator_tests/generator_tests.pro | 1 + .../generator_tests/srtm_parser_test.cpp | 22 ++++ generator/srtm_parser.cpp | 62 +++++++++++ generator/srtm_parser.hpp | 65 +++++++++++ .../srtm_source_tests/srtm_source_tests.cpp | 102 ++++++++++++++++++ .../srtm_source_tests/srtm_source_tests.pro | 22 ++++ omim.pro | 8 +- 10 files changed, 314 insertions(+), 1 deletion(-) create mode 100644 generator/generator_tests/srtm_parser_test.cpp create mode 100644 generator/srtm_parser.cpp create mode 100644 generator/srtm_parser.hpp create mode 100644 generator/srtm_source_tests/srtm_source_tests.cpp create mode 100644 generator/srtm_source_tests/srtm_source_tests.pro diff --git a/coding/zip_reader.cpp b/coding/zip_reader.cpp index 32de85bcbf..4c37d66090 100644 --- a/coding/zip_reader.cpp +++ b/coding/zip_reader.cpp @@ -117,3 +117,31 @@ void ZipFileReader::UnzipFile(string const & zipContainer, string const & fileIn outFileGuard.release(); } + +char* ZipFileReader::UnzipFileToMemory(string const & zipContainer, string const & fileInZip) +{ + unzFile zip = unzOpen64(zipContainer.c_str()); + if (!zip) + MYTHROW(OpenZipException, ("Can't get zip file handle", zipContainer)); + MY_SCOPE_GUARD(zipGuard, bind(&unzClose, zip)); + + if (UNZ_OK != unzLocateFile(zip, fileInZip.c_str(), 1)) + MYTHROW(LocateZipException, ("Can't locate file inside zip", fileInZip)); + + if (UNZ_OK != unzOpenCurrentFile(zip)) + MYTHROW(LocateZipException, ("Can't open file inside zip", fileInZip)); + MY_SCOPE_GUARD(currentFileGuard, bind(&unzCloseCurrentFile, zip)); + + unz_file_info64 fileInfo; + if (UNZ_OK != unzGetCurrentFileInfo64(zip, &fileInfo, NULL, 0, NULL, 0, NULL, 0)) + MYTHROW(LocateZipException, ("Can't get uncompressed file size inside zip", fileInZip)); + size_t const uncompressedFileSize = fileInfo.uncompressed_size; + + char* buf= new char[uncompressedFileSize]; + + int const readBytes = unzReadCurrentFile(zip, buf, uncompressedFileSize); + if (readBytes < 0) + MYTHROW(InvalidZipException, ("Error", readBytes, "while unzipping", fileInZip, "from", zipContainer)); + + return buf; +} diff --git a/coding/zip_reader.hpp b/coding/zip_reader.hpp index b518b8d9e3..6c36693df4 100644 --- a/coding/zip_reader.hpp +++ b/coding/zip_reader.hpp @@ -32,6 +32,9 @@ public: static void UnzipFile(string const & zipContainer, string const & fileInZip, string const & outFilePath, ProgressFn progressFn = ProgressFn()); + /// @warning Counsumer must manually free result + static char* UnzipFileToMemory(string const & zipContainer, string const & fileInZip); + static void FilesList(string const & zipContainer, FileListT & filesList); /// Quick version without exceptions diff --git a/generator/generator.pro b/generator/generator.pro index d2d76c211e..42725c1953 100644 --- a/generator/generator.pro +++ b/generator/generator.pro @@ -30,6 +30,7 @@ SOURCES += \ osm_source.cpp \ routing_generator.cpp \ search_index_builder.cpp \ + srtm_parser.cpp \ statistics.cpp \ tesselator.cpp \ towns_dumper.cpp \ @@ -60,6 +61,7 @@ HEADERS += \ polygonizer.hpp \ routing_generator.hpp \ search_index_builder.hpp \ + srtm_parser.hpp \ statistics.hpp \ tag_admixer.hpp \ tesselator.hpp \ diff --git a/generator/generator_tests/generator_tests.pro b/generator/generator_tests/generator_tests.pro index 072e71a601..c55a55bf38 100644 --- a/generator/generator_tests/generator_tests.pro +++ b/generator/generator_tests/generator_tests.pro @@ -28,6 +28,7 @@ SOURCES += \ osm_type_test.cpp \ source_data.cpp \ source_to_element_test.cpp \ + srtm_parser_test.cpp \ tag_admixer_test.cpp \ tesselator_test.cpp \ triangles_tree_coding_test.cpp \ diff --git a/generator/generator_tests/srtm_parser_test.cpp b/generator/generator_tests/srtm_parser_test.cpp new file mode 100644 index 0000000000..7c72db3681 --- /dev/null +++ b/generator/generator_tests/srtm_parser_test.cpp @@ -0,0 +1,22 @@ +#include "generator/srtm_parser.hpp" + +#include "testing/testing.hpp" + +#include "base/logging.hpp" + +namespace +{ +UNIT_TEST(FilenameTests) +{ + auto name = GetSRTMBase({56.4566, 37.3467}); + TEST_EQUAL(name, "N56E037", ()); + name = GetSRTMBase({34.077433, -118.304569}); + TEST_EQUAL(name, "N34W119", ()); + name = GetSRTMBase({0.1, 0.1}); + TEST_EQUAL(name, "N00E000", ()); + name = GetSRTMBase({-35.35, -12.1}); + TEST_EQUAL(name, "S36W013", ()); + name = GetSRTMBase({-34.622358, -58.383654}); + TEST_EQUAL(name, "S35W059", ()); +} +} diff --git a/generator/srtm_parser.cpp b/generator/srtm_parser.cpp new file mode 100644 index 0000000000..ff88a35512 --- /dev/null +++ b/generator/srtm_parser.cpp @@ -0,0 +1,62 @@ +#include "srtm_parser.hpp" + +#include "coding/zip_reader.hpp" + +#include "std/iomanip.hpp" + +string GetSRTMBase(ms::LatLon coord) +{ + stringstream ss; + if (coord.lat < 0) + { + ss << "S"; + coord.lat *= -1; + coord.lat += 1; + } + else + { + ss << "N"; + } + ss << setw(2) << setfill('0') << int(coord.lat); + + if (coord.lon < 0) + { + ss << "W"; + coord.lon *= -1; + coord.lon += 1; + } + else + { + ss << "E"; + } + ss << setw(3) << int(coord.lon); + return ss.str(); +} + +void SrtmFile::Init(string dir, ms::LatLon coord) +{ + string base = GetSRTMBase(coord); + string path = dir + base + ".SRTMGL1.hgt.zip"; + string file = base + ".hgt"; + try + { + m_data = reinterpret_cast(ZipFileReader::UnzipFileToMemory(path, file)); + } + catch (ZipFileReader::OpenZipException e) + { + m_data = nullptr; + } + catch (ZipFileReader::LocateZipException e) + { + try + { + // Sometimes packed file has different name. See N39E051 measure. + file = base + ".SRTMGL1.hgt"; + m_data = reinterpret_cast(ZipFileReader::UnzipFileToMemory(path, file)); + } + catch (int) + { + m_data = nullptr; + } + } +} diff --git a/generator/srtm_parser.hpp b/generator/srtm_parser.hpp new file mode 100644 index 0000000000..d01b33f153 --- /dev/null +++ b/generator/srtm_parser.hpp @@ -0,0 +1,65 @@ +#pragma once + +#include "geometry/latlon.hpp" + +#include "coding/endianness.hpp" + +#include "std/map.hpp" +#include "std/string.hpp" + +using TSRTMHeighType = int16_t; + +int16_t constexpr kInvalidSRTMHeight = -32768; + +string GetSRTMBase(ms::LatLon coord); + +class SrtmFile +{ +public: + void Init(string dir, ms::LatLon coord); + + TSRTMHeighType getHeight(ms::LatLon coord) + { + if (m_data == nullptr) + return kInvalidSRTMHeight; + double ln = coord.lon - int(coord.lon); + if (ln < 0) + ln += 1; + double lt = coord.lat - int(coord.lat); + if (lt < 0) + lt += 1; + lt = 1 - lt; // From North to South + + size_t const row = 3600 * lt; + size_t const column = 3600 * ln; + return ReverseByteOrder(m_data[row * 3601 + column]); + // 3600 and 3601 because 3601 is owerlap column. + } + + ~SrtmFile() + { + if (m_data) + delete[] m_data; + } + +private: + TSRTMHeighType * m_data; +}; + +class SrtmFileManager +{ +public: + SrtmFileManager(string dir) : m_dir(dir) {} + TSRTMHeighType GetCoordHeight(ms::LatLon coord) + { + string base = GetSRTMBase(coord); + auto it = m_storage.find(base); + if (it == m_storage.end()) + m_storage[base].Init(m_dir, coord); + return m_storage[base].getHeight(coord); + } + +private: + string m_dir; + map m_storage; +}; diff --git a/generator/srtm_source_tests/srtm_source_tests.cpp b/generator/srtm_source_tests/srtm_source_tests.cpp new file mode 100644 index 0000000000..8c09903c2c --- /dev/null +++ b/generator/srtm_source_tests/srtm_source_tests.cpp @@ -0,0 +1,102 @@ +#include "testing/testing.hpp" + +#include "generator/srtm_parser.hpp" + +#include "map/feature_vec_model.hpp" + +#include "routing/routing_integration_tests/routing_test_tools.hpp" + +#include "platform/country_file.hpp" +#include "platform/local_country_file_utils.hpp" + +#include "coding/file_name_utils.hpp" + +namespace +{ +#define SRTM_SOURCES_PATH "srtm/e4ftl01.cr.usgs.gov/SRTM/SRTMGL1.003/2000.02.11/" + +UNIT_TEST(SRTMCoverageChecker) +{ + vector localFiles; + platform::FindAllLocalMapsAndCleanup(numeric_limits::max() /* latestVersion */, + localFiles); + + shared_ptr fetcher = integration::CreateFeaturesFetcher(localFiles); + + for (auto & file : localFiles) + { + file.SyncWithDisk(); + if (file.GetFiles() != MapOptions::MapWithCarRouting) + { + LOG(LINFO, ("Warning! Routing file not found for:", file.GetCountryName())); + continue; + } + + FilesMappingContainer container(file.GetPath(MapOptions::CarRouting)); + if (!container.IsExist(ROUTING_FTSEG_FILE_TAG)) + { + LOG(LINFO, ("Warning! Mwm file has not routing ftseg section:", file.GetCountryName())); + continue; + } + + routing::TDataFacade dataFacade; + dataFacade.Load(container); + + OsrmFtSegMapping segMapping; + segMapping.Load(container, file); + segMapping.Map(container); + + Platform & platform = GetPlatform(); + SrtmFileManager manager(my::JoinFoldersToPath(platform.WritableDir(), SRTM_SOURCES_PATH)); + size_t all = 0; + size_t good = 0; + + for (size_t i = 0; i < dataFacade.GetNumberOfNodes(); ++i) + { + buffer_vector buffer; + segMapping.ForEachFtSeg(i, MakeBackInsertFunctor(buffer)); + vector path; + + for (size_t k = 0; k < buffer.size(); ++k) + { + auto const & segment = buffer[k]; + if (!segment.IsValid()) + continue; + // Load data from drive. + FeatureType ft; + Index::FeaturesLoaderGuard loader( + fetcher->GetIndex(), fetcher->GetIndex().GetMwmIdByCountryFile(file.GetCountryFile())); + loader.GetFeatureByIndex(segment.m_fid, ft); + ft.ParseGeometry(FeatureType::BEST_GEOMETRY); + + // Get points in proper direction. + auto startIdx = segment.m_pointStart; + auto endIdx = segment.m_pointEnd; + if (startIdx < endIdx) + { + for (auto idx = startIdx; idx <= endIdx; ++idx) + path.push_back(ft.GetPoint(idx)); + } + else + { + // I use big signed type because endIdx can be 0. + for (int64_t idx = startIdx; idx >= static_cast(endIdx); --idx) + path.push_back(ft.GetPoint(idx)); + } + for (auto point : path) + { + auto height = manager.GetCoordHeight(MercatorBounds::ToLatLon(point)); + all++; + if (height != kInvalidSRTMHeight) + good++; + } + } + } + auto const delta = all - good; + auto const percent = delta * 100.0 / all; + if (percent > 10.0) + LOG(LINFO, ("Huge error rate in:", file.GetCountryName(), "Good:", good, "All:", all, + "Delta:", delta, "%:", percent)); + } +} +} // namespace diff --git a/generator/srtm_source_tests/srtm_source_tests.pro b/generator/srtm_source_tests/srtm_source_tests.pro new file mode 100644 index 0000000000..4d0ff89d59 --- /dev/null +++ b/generator/srtm_source_tests/srtm_source_tests.pro @@ -0,0 +1,22 @@ +# This subproject implements digital elevation model source test. +# This tests are launched on the raw SRTM dataset. + +TARGET = srtm_source_tests +CONFIG += console warn_on +CONFIG -= app_bundle +TEMPLATE = app + +ROOT_DIR = ../.. +DEPENDENCIES = map routing search storage indexer platform editor geometry coding base \ + osrm jansson protobuf tomcrypt succinct stats_client pugixml generator minizip + +macx-*: LIBS *= "-framework IOKit" "-framework SystemConfiguration" + +include($$ROOT_DIR/common.pri) + +QT *= core + +SOURCES += \ + ../../testing/testingmain.cpp \ + ../../routing/routing_integration_tests/routing_test_tools.cpp \ + srtm_source_tests.cpp \ diff --git a/omim.pro b/omim.pro index 3a4a0ea76a..fed9f9abb8 100644 --- a/omim.pro +++ b/omim.pro @@ -37,7 +37,9 @@ SUBDIRS = 3party base coding geometry editor indexer routing search routing_integration_tests.depends = $$SUBDIRS routing_consistency_tests.subdir = routing/routing_consistency_tests routing_consistency_tests.depends = $$SUBDIRS - SUBDIRS *= routing_integration_tests routing_consistency_tests + srtm_source_tests.subdir = generator/srtm_source_tests + srtm_source_tests.depends = $$SUBDIRS routing + SUBDIRS *= routing_integration_tests routing_consistency_tests srtm_source_tests } CONFIG(desktop) { @@ -168,6 +170,10 @@ SUBDIRS = 3party base coding geometry editor indexer routing search routing_consistency_tests.depends = $$MapDepLibs routing SUBDIRS *= routing_consistency_tests + srtm_source_tests.subdir = generator/srtm_source_tests + srtm_source_tests.depends = $$MapDepLibs routing + SUBDIRS *= srtm_source_tests + # TODO(AlexZ): Move pedestrian tests into routing dir. pedestrian_routing_tests.depends = $$MapDepLibs routing SUBDIRS *= pedestrian_routing_tests From 18287be1dbd25e98d3ac59e13a86687c5e697605 Mon Sep 17 00:00:00 2001 From: Yuri Gorshenin Date: Tue, 10 May 2016 15:24:46 +0300 Subject: [PATCH 2/3] [coding] Fixes to SRTM parser CL. --- coding/zip_reader.cpp | 36 +++--- coding/zip_reader.hpp | 6 +- .../generator_tests/srtm_parser_test.cpp | 24 ++-- generator/srtm_parser.cpp | 117 ++++++++++++++---- generator/srtm_parser.hpp | 76 ++++++------ .../srtm_source_tests/srtm_source_tests.cpp | 18 +-- .../srtm_source_tests/srtm_source_tests.pro | 4 +- 7 files changed, 177 insertions(+), 104 deletions(-) diff --git a/coding/zip_reader.cpp b/coding/zip_reader.cpp index 4c37d66090..102bdd77a4 100644 --- a/coding/zip_reader.cpp +++ b/coding/zip_reader.cpp @@ -118,30 +118,28 @@ void ZipFileReader::UnzipFile(string const & zipContainer, string const & fileIn outFileGuard.release(); } -char* ZipFileReader::UnzipFileToMemory(string const & zipContainer, string const & fileInZip) +void ZipFileReader::UnzipFileToMemory(string const & cont, string const & file, + vector & data) { - unzFile zip = unzOpen64(zipContainer.c_str()); + unzFile zip = unzOpen64(cont.c_str()); if (!zip) - MYTHROW(OpenZipException, ("Can't get zip file handle", zipContainer)); - MY_SCOPE_GUARD(zipGuard, bind(&unzClose, zip)); - - if (UNZ_OK != unzLocateFile(zip, fileInZip.c_str(), 1)) - MYTHROW(LocateZipException, ("Can't locate file inside zip", fileInZip)); + MYTHROW(OpenZipException, ("Can't get zip file handle:", cont)); + MY_SCOPE_GUARD(zipCloser, bind(&unzClose, zip)); + if (UNZ_OK != unzLocateFile(zip, file.c_str(), 1 /* case sensitivity */)) + MYTHROW(LocateZipException, ("Can't locate file inside zip container:", file)); if (UNZ_OK != unzOpenCurrentFile(zip)) - MYTHROW(LocateZipException, ("Can't open file inside zip", fileInZip)); - MY_SCOPE_GUARD(currentFileGuard, bind(&unzCloseCurrentFile, zip)); + MYTHROW(LocateZipException, ("Can't open file inside zip container:", file)); + MY_SCOPE_GUARD(currentFileCloser, bind(&unzCloseCurrentFile, zip)); - unz_file_info64 fileInfo; - if (UNZ_OK != unzGetCurrentFileInfo64(zip, &fileInfo, NULL, 0, NULL, 0, NULL, 0)) - MYTHROW(LocateZipException, ("Can't get uncompressed file size inside zip", fileInZip)); - size_t const uncompressedFileSize = fileInfo.uncompressed_size; + unz_file_info64 info; + if (UNZ_OK != unzGetCurrentFileInfo64(zip, &info, NULL, 0, NULL, 0, NULL, 0)) + MYTHROW(LocateZipException, ("Can't get uncompressed file size inside zip:", file)); - char* buf= new char[uncompressedFileSize]; + size_t const size = info.uncompressed_size; + data.resize(size); - int const readBytes = unzReadCurrentFile(zip, buf, uncompressedFileSize); - if (readBytes < 0) - MYTHROW(InvalidZipException, ("Error", readBytes, "while unzipping", fileInZip, "from", zipContainer)); - - return buf; + int const bytesRead = unzReadCurrentFile(zip, data.data(), size); + if (bytesRead < 0) + MYTHROW(InvalidZipException, ("Error:", bytesRead, "while unzipping", file, "in", cont)); } diff --git a/coding/zip_reader.hpp b/coding/zip_reader.hpp index 6c36693df4..7970cff087 100644 --- a/coding/zip_reader.hpp +++ b/coding/zip_reader.hpp @@ -32,8 +32,10 @@ public: static void UnzipFile(string const & zipContainer, string const & fileInZip, string const & outFilePath, ProgressFn progressFn = ProgressFn()); - /// @warning Counsumer must manually free result - static char* UnzipFileToMemory(string const & zipContainer, string const & fileInZip); + /// Unzips |file| in |cont| to |buffer|. + /// + /// @warning Can throw OpenZipException and LocateZipException. + static void UnzipFileToMemory(string const & cont, string const & file, vector & data); static void FilesList(string const & zipContainer, FileListT & filesList); diff --git a/generator/generator_tests/srtm_parser_test.cpp b/generator/generator_tests/srtm_parser_test.cpp index 7c72db3681..4420ef91a7 100644 --- a/generator/generator_tests/srtm_parser_test.cpp +++ b/generator/generator_tests/srtm_parser_test.cpp @@ -1,22 +1,28 @@ -#include "generator/srtm_parser.hpp" - #include "testing/testing.hpp" -#include "base/logging.hpp" +#include "generator/srtm_parser.hpp" + +using namespace generator; namespace { +string GetBase(ms::LatLon const & coord) { return SrtmFile::GetBase(coord); } + UNIT_TEST(FilenameTests) { - auto name = GetSRTMBase({56.4566, 37.3467}); + auto name = GetBase({56.4566, 37.3467}); TEST_EQUAL(name, "N56E037", ()); - name = GetSRTMBase({34.077433, -118.304569}); + + name = GetBase({34.077433, -118.304569}); TEST_EQUAL(name, "N34W119", ()); - name = GetSRTMBase({0.1, 0.1}); + + name = GetBase({0.1, 0.1}); TEST_EQUAL(name, "N00E000", ()); - name = GetSRTMBase({-35.35, -12.1}); + + name = GetBase({-35.35, -12.1}); TEST_EQUAL(name, "S36W013", ()); - name = GetSRTMBase({-34.622358, -58.383654}); + + name = GetBase({-34.622358, -58.383654}); TEST_EQUAL(name, "S35W059", ()); } -} +} // namespace diff --git a/generator/srtm_parser.cpp b/generator/srtm_parser.cpp index ff88a35512..6630c335de 100644 --- a/generator/srtm_parser.cpp +++ b/generator/srtm_parser.cpp @@ -1,12 +1,74 @@ -#include "srtm_parser.hpp" +#include "generator/srtm_parser.hpp" +#include "coding/endianness.hpp" #include "coding/zip_reader.hpp" -#include "std/iomanip.hpp" +#include "base/logging.hpp" -string GetSRTMBase(ms::LatLon coord) +#include "std/iomanip.hpp" +#include "std/sstream.hpp" + +namespace generator { - stringstream ss; +// SrtmFile ---------------------------------------------------------------------------------------- +SrtmFile::SrtmFile() +{ + Invalidate(); +} + +SrtmFile::SrtmFile(SrtmFile && rhs) : m_data(move(rhs.m_data)), m_valid(rhs.m_valid) +{ + rhs.Invalidate(); +} + +void SrtmFile::Init(string const & dir, ms::LatLon const & coord) +{ + Invalidate(); + + string const base = GetBase(coord); + string const cont = dir + base + ".SRTMGL1.hgt.zip"; + string file = base + ".hgt"; + + try + { + ZipFileReader::UnzipFileToMemory(cont, file, m_data); + } + catch (ZipFileReader::LocateZipException e) + { + // Sometimes packed file has different name. See N39E051 measure. + file = base + ".SRTMGL1.hgt"; + ZipFileReader::UnzipFileToMemory(cont, file, m_data); + } + + m_valid = true; +} + +SrtmFile::THeight SrtmFile::GetHeight(ms::LatLon const & coord) +{ + if (!IsValid()) + return kInvalidHeight; + + double ln = coord.lon - static_cast(coord.lon); + if (ln < 0) + ln += 1; + double lt = coord.lat - static_cast(coord.lat); + if (lt < 0) + lt += 1; + lt = 1 - lt; // from North to South + + size_t const row = 3600 * lt; + size_t const col = 3600 * ln; + + size_t const ix = row * 3601 + col; + + if (ix >= Size()) + return kInvalidHeight; + return ReverseByteOrder(Data()[ix]); +} + +string SrtmFile::GetBase(ms::LatLon coord) +{ + ostringstream ss; if (coord.lat < 0) { ss << "S"; @@ -17,7 +79,7 @@ string GetSRTMBase(ms::LatLon coord) { ss << "N"; } - ss << setw(2) << setfill('0') << int(coord.lat); + ss << setw(2) << setfill('0') << static_cast(coord.lat); if (coord.lon < 0) { @@ -29,34 +91,41 @@ string GetSRTMBase(ms::LatLon coord) { ss << "E"; } - ss << setw(3) << int(coord.lon); + ss << setw(3) << static_cast(coord.lon); return ss.str(); } -void SrtmFile::Init(string dir, ms::LatLon coord) +void SrtmFile::Invalidate() { - string base = GetSRTMBase(coord); - string path = dir + base + ".SRTMGL1.hgt.zip"; - string file = base + ".hgt"; - try - { - m_data = reinterpret_cast(ZipFileReader::UnzipFileToMemory(path, file)); - } - catch (ZipFileReader::OpenZipException e) - { - m_data = nullptr; - } - catch (ZipFileReader::LocateZipException e) + m_data.clear(); + m_data.shrink_to_fit(); + m_valid = false; +} + +// SrtmFileManager --------------------------------------------------------------------------------- +SrtmFileManager::SrtmFileManager(string const & dir) : m_dir(dir) {} + +SrtmFile::THeight SrtmFileManager::GetHeight(ms::LatLon const & coord) +{ + string const base = SrtmFile::GetBase(coord); + auto it = m_storage.find(base); + if (it == m_storage.end()) { + SrtmFile file; try { - // Sometimes packed file has different name. See N39E051 measure. - file = base + ".SRTMGL1.hgt"; - m_data = reinterpret_cast(ZipFileReader::UnzipFileToMemory(path, file)); + file.Init(m_dir, coord); } - catch (int) + catch (RootException const & e) { - m_data = nullptr; + LOG(LERROR, ("Can't init SRTM file:", base, "reason:", e.Msg())); } + + // It's OK to store even invalid files and return invalid height + // for them later. + m_storage.emplace(base, move(file)); } + + return m_storage[base].GetHeight(coord); } +} // namespace generator diff --git a/generator/srtm_parser.hpp b/generator/srtm_parser.hpp index d01b33f153..bf521ad988 100644 --- a/generator/srtm_parser.hpp +++ b/generator/srtm_parser.hpp @@ -2,64 +2,58 @@ #include "geometry/latlon.hpp" -#include "coding/endianness.hpp" +#include "base/macros.hpp" -#include "std/map.hpp" +#include "std/cstdint.hpp" #include "std/string.hpp" +#include "std/unordered_map.hpp" +#include "std/vector.hpp" -using TSRTMHeighType = int16_t; - -int16_t constexpr kInvalidSRTMHeight = -32768; - -string GetSRTMBase(ms::LatLon coord); - +namespace generator +{ class SrtmFile { public: - void Init(string dir, ms::LatLon coord); + using THeight = int16_t; - TSRTMHeighType getHeight(ms::LatLon coord) - { - if (m_data == nullptr) - return kInvalidSRTMHeight; - double ln = coord.lon - int(coord.lon); - if (ln < 0) - ln += 1; - double lt = coord.lat - int(coord.lat); - if (lt < 0) - lt += 1; - lt = 1 - lt; // From North to South + static THeight constexpr kInvalidHeight = -32768; - size_t const row = 3600 * lt; - size_t const column = 3600 * ln; - return ReverseByteOrder(m_data[row * 3601 + column]); - // 3600 and 3601 because 3601 is owerlap column. - } + SrtmFile(); + SrtmFile(SrtmFile && rhs); - ~SrtmFile() - { - if (m_data) - delete[] m_data; - } + void Init(string const & dir, ms::LatLon const & coord); + + inline bool IsValid() const { return m_valid; } + + // Returns height in meters at |coord|, or kInvalidHeight if is not initialized. + THeight GetHeight(ms::LatLon const & coord); + + static string GetBase(ms::LatLon coord); private: - TSRTMHeighType * m_data; + inline THeight const * Data() const { return reinterpret_cast(m_data.data()); }; + + inline size_t Size() const { return m_data.size() / sizeof(THeight); } + + void Invalidate(); + + vector m_data; + bool m_valid; + + DISALLOW_COPY(SrtmFile); }; class SrtmFileManager { public: - SrtmFileManager(string dir) : m_dir(dir) {} - TSRTMHeighType GetCoordHeight(ms::LatLon coord) - { - string base = GetSRTMBase(coord); - auto it = m_storage.find(base); - if (it == m_storage.end()) - m_storage[base].Init(m_dir, coord); - return m_storage[base].getHeight(coord); - } + SrtmFileManager(string const & dir); + + SrtmFile::THeight GetHeight(ms::LatLon const & coord); private: string m_dir; - map m_storage; + unordered_map m_storage; + + DISALLOW_COPY(SrtmFileManager); }; +} // namespace generator diff --git a/generator/srtm_source_tests/srtm_source_tests.cpp b/generator/srtm_source_tests/srtm_source_tests.cpp index 8c09903c2c..5895b24bc5 100644 --- a/generator/srtm_source_tests/srtm_source_tests.cpp +++ b/generator/srtm_source_tests/srtm_source_tests.cpp @@ -6,16 +6,16 @@ #include "routing/routing_integration_tests/routing_test_tools.hpp" +#include "coding/file_name_utils.hpp" + #include "platform/country_file.hpp" #include "platform/local_country_file_utils.hpp" -#include "coding/file_name_utils.hpp" - namespace { #define SRTM_SOURCES_PATH "srtm/e4ftl01.cr.usgs.gov/SRTM/SRTMGL1.003/2000.02.11/" -UNIT_TEST(SRTMCoverageChecker) +UNIT_TEST(SrtmCoverageChecker) { vector localFiles; platform::FindAllLocalMapsAndCleanup(numeric_limits::max() /* latestVersion */, @@ -47,7 +47,9 @@ UNIT_TEST(SRTMCoverageChecker) segMapping.Map(container); Platform & platform = GetPlatform(); - SrtmFileManager manager(my::JoinFoldersToPath(platform.WritableDir(), SRTM_SOURCES_PATH)); + generator::SrtmFileManager manager( + my::JoinFoldersToPath(platform.WritableDir(), SRTM_SOURCES_PATH)); + size_t all = 0; size_t good = 0; @@ -83,11 +85,11 @@ UNIT_TEST(SRTMCoverageChecker) for (int64_t idx = startIdx; idx >= static_cast(endIdx); --idx) path.push_back(ft.GetPoint(idx)); } - for (auto point : path) + for (auto const & point : path) { - auto height = manager.GetCoordHeight(MercatorBounds::ToLatLon(point)); + auto const height = manager.GetHeight(MercatorBounds::ToLatLon(point)); all++; - if (height != kInvalidSRTMHeight) + if (height != generator::SrtmFile::kInvalidHeight) good++; } } @@ -95,8 +97,10 @@ UNIT_TEST(SRTMCoverageChecker) auto const delta = all - good; auto const percent = delta * 100.0 / all; if (percent > 10.0) + { LOG(LINFO, ("Huge error rate in:", file.GetCountryName(), "Good:", good, "All:", all, "Delta:", delta, "%:", percent)); + } } } } // namespace diff --git a/generator/srtm_source_tests/srtm_source_tests.pro b/generator/srtm_source_tests/srtm_source_tests.pro index 4d0ff89d59..00d9b7d7c0 100644 --- a/generator/srtm_source_tests/srtm_source_tests.pro +++ b/generator/srtm_source_tests/srtm_source_tests.pro @@ -7,8 +7,8 @@ CONFIG -= app_bundle TEMPLATE = app ROOT_DIR = ../.. -DEPENDENCIES = map routing search storage indexer platform editor geometry coding base \ - osrm jansson protobuf tomcrypt succinct stats_client pugixml generator minizip +DEPENDENCIES = generator map routing search storage indexer platform editor geometry coding base \ + osrm jansson protobuf tomcrypt succinct stats_client pugixml minizip macx-*: LIBS *= "-framework IOKit" "-framework SystemConfiguration" From c9086ce741cf93eecada71a0f8e411476d66f1ff Mon Sep 17 00:00:00 2001 From: Yuri Gorshenin Date: Tue, 10 May 2016 15:44:09 +0300 Subject: [PATCH 3/3] Review fixes. --- coding/zip_reader.cpp | 96 ++++++++++--------- coding/zip_reader.hpp | 20 ++-- .../generator_tests/srtm_parser_test.cpp | 2 +- .../srtm_coverage_checker.cpp} | 74 +++++++------- .../srtm_coverage_checker.pro | 25 +++++ generator/srtm_parser.cpp | 91 +++++++++++++----- generator/srtm_parser.hpp | 21 ++-- .../srtm_source_tests/srtm_source_tests.pro | 22 ----- omim.pro | 12 +-- 9 files changed, 214 insertions(+), 149 deletions(-) rename generator/{srtm_source_tests/srtm_source_tests.cpp => srtm_coverage_checker/srtm_coverage_checker.cpp} (61%) create mode 100644 generator/srtm_coverage_checker/srtm_coverage_checker.pro delete mode 100644 generator/srtm_source_tests/srtm_source_tests.pro diff --git a/coding/zip_reader.cpp b/coding/zip_reader.cpp index 102bdd77a4..427048a903 100644 --- a/coding/zip_reader.cpp +++ b/coding/zip_reader.cpp @@ -3,13 +3,42 @@ #include "base/scope_guard.hpp" #include "base/logging.hpp" -#include "coding/file_writer.hpp" #include "coding/constants.hpp" #include "std/bind.hpp" #include "3party/minizip/unzip.h" +namespace +{ +class UnzipFileDelegate : public ZipFileReader::Delegate +{ +public: + UnzipFileDelegate(string const & path) + : m_file(make_unique(path)), m_path(path), m_completed(false) + { + } + + ~UnzipFileDelegate() override + { + if (!m_completed) + { + m_file.reset(); + FileWriter::DeleteFileX(m_path); + } + } + + // ZipFileReader::Delegate overrides: + void OnBlockUnzipped(size_t size, char const * data) override { m_file->Write(data, size); } + + void OnCompleted() override { m_completed = true; } + +private: + unique_ptr m_file; + string const m_path; + bool m_completed; +}; +} // namespace ZipFileReader::ZipFileReader(string const & container, string const & file, uint32_t logPageSize, uint32_t logPageCount) @@ -73,8 +102,9 @@ bool ZipFileReader::IsZip(string const & zipContainer) return true; } +// static void ZipFileReader::UnzipFile(string const & zipContainer, string const & fileInZip, - string const & outFilePath, ProgressFn progressFn) + Delegate & delegate) { unzFile zip = unzOpen64(zipContainer.c_str()); if (!zip) @@ -92,54 +122,28 @@ void ZipFileReader::UnzipFile(string const & zipContainer, string const & fileIn if (UNZ_OK != unzGetCurrentFileInfo64(zip, &fileInfo, NULL, 0, NULL, 0, NULL, 0)) MYTHROW(LocateZipException, ("Can't get uncompressed file size inside zip", fileInZip)); - // First outFile should be closed, then FileWriter::DeleteFileX is called, - // so make correct order of guards. - MY_SCOPE_GUARD(outFileGuard, bind(&FileWriter::DeleteFileX, cref(outFilePath))); - FileWriter outFile(outFilePath); - - uint64_t pos = 0; char buf[ZIP_FILE_BUFFER_SIZE]; - while (true) + int readBytes = 0; + + delegate.OnStarted(); + do { - int const readBytes = unzReadCurrentFile(zip, buf, ZIP_FILE_BUFFER_SIZE); - if (readBytes > 0) - outFile.Write(buf, static_cast(readBytes)); - else if (readBytes < 0) - MYTHROW(InvalidZipException, ("Error", readBytes, "while unzipping", fileInZip, "from", zipContainer)); - else - break; + readBytes = unzReadCurrentFile(zip, buf, ZIP_FILE_BUFFER_SIZE); + if (readBytes < 0) + { + MYTHROW(InvalidZipException, + ("Error", readBytes, "while unzipping", fileInZip, "from", zipContainer)); + } - pos += readBytes; - - if (progressFn) - progressFn(fileInfo.uncompressed_size, pos); - } - - outFileGuard.release(); + delegate.OnBlockUnzipped(static_cast(readBytes), buf); + } while (readBytes != 0); + delegate.OnCompleted(); } -void ZipFileReader::UnzipFileToMemory(string const & cont, string const & file, - vector & data) +// static +void ZipFileReader::UnzipFile(string const & zipContainer, string const & fileInZip, + string const & outPath) { - unzFile zip = unzOpen64(cont.c_str()); - if (!zip) - MYTHROW(OpenZipException, ("Can't get zip file handle:", cont)); - MY_SCOPE_GUARD(zipCloser, bind(&unzClose, zip)); - - if (UNZ_OK != unzLocateFile(zip, file.c_str(), 1 /* case sensitivity */)) - MYTHROW(LocateZipException, ("Can't locate file inside zip container:", file)); - if (UNZ_OK != unzOpenCurrentFile(zip)) - MYTHROW(LocateZipException, ("Can't open file inside zip container:", file)); - MY_SCOPE_GUARD(currentFileCloser, bind(&unzCloseCurrentFile, zip)); - - unz_file_info64 info; - if (UNZ_OK != unzGetCurrentFileInfo64(zip, &info, NULL, 0, NULL, 0, NULL, 0)) - MYTHROW(LocateZipException, ("Can't get uncompressed file size inside zip:", file)); - - size_t const size = info.uncompressed_size; - data.resize(size); - - int const bytesRead = unzReadCurrentFile(zip, data.data(), size); - if (bytesRead < 0) - MYTHROW(InvalidZipException, ("Error:", bytesRead, "while unzipping", file, "in", cont)); + UnzipFileDelegate delegate(outPath); + UnzipFile(zipContainer, fileInZip, delegate); } diff --git a/coding/zip_reader.hpp b/coding/zip_reader.hpp index 7970cff087..bea0496ae5 100644 --- a/coding/zip_reader.hpp +++ b/coding/zip_reader.hpp @@ -1,6 +1,7 @@ #pragma once #include "coding/file_reader.hpp" +#include "coding/file_writer.hpp" #include "base/exception.hpp" @@ -13,6 +14,17 @@ private: uint64_t m_uncompressedFileSize; public: + struct Delegate + { + virtual ~Delegate() = default; + + // When |size| is zero, end of file is reached. + virtual void OnBlockUnzipped(size_t size, char const * data) = 0; + + virtual void OnStarted() {} + virtual void OnCompleted() {} + }; + typedef function ProgressFn; /// Contains file name inside zip and it's uncompressed size typedef vector > FileListT; @@ -29,13 +41,9 @@ public: uint64_t UncompressedSize() const { return m_uncompressedFileSize; } /// @warning Can also throw Writer::OpenException and Writer::WriteException + static void UnzipFile(string const & zipContainer, string const & fileInZip, Delegate & delegate); static void UnzipFile(string const & zipContainer, string const & fileInZip, - string const & outFilePath, ProgressFn progressFn = ProgressFn()); - - /// Unzips |file| in |cont| to |buffer|. - /// - /// @warning Can throw OpenZipException and LocateZipException. - static void UnzipFileToMemory(string const & cont, string const & file, vector & data); + string const & outPath); static void FilesList(string const & zipContainer, FileListT & filesList); diff --git a/generator/generator_tests/srtm_parser_test.cpp b/generator/generator_tests/srtm_parser_test.cpp index 4420ef91a7..79ef60431e 100644 --- a/generator/generator_tests/srtm_parser_test.cpp +++ b/generator/generator_tests/srtm_parser_test.cpp @@ -6,7 +6,7 @@ using namespace generator; namespace { -string GetBase(ms::LatLon const & coord) { return SrtmFile::GetBase(coord); } +inline string GetBase(ms::LatLon const & coord) { return SrtmTile::GetBase(coord); } UNIT_TEST(FilenameTests) { diff --git a/generator/srtm_source_tests/srtm_source_tests.cpp b/generator/srtm_coverage_checker/srtm_coverage_checker.cpp similarity index 61% rename from generator/srtm_source_tests/srtm_source_tests.cpp rename to generator/srtm_coverage_checker/srtm_coverage_checker.cpp index 5895b24bc5..3df3a27eea 100644 --- a/generator/srtm_source_tests/srtm_source_tests.cpp +++ b/generator/srtm_coverage_checker/srtm_coverage_checker.cpp @@ -1,5 +1,3 @@ -#include "testing/testing.hpp" - #include "generator/srtm_parser.hpp" #include "map/feature_vec_model.hpp" @@ -11,17 +9,39 @@ #include "platform/country_file.hpp" #include "platform/local_country_file_utils.hpp" -namespace -{ -#define SRTM_SOURCES_PATH "srtm/e4ftl01.cr.usgs.gov/SRTM/SRTMGL1.003/2000.02.11/" +#include "base/logging.hpp" -UNIT_TEST(SrtmCoverageChecker) +#include "std/algorithm.hpp" + +#include "3party/gflags/src/gflags/gflags.h" + +DEFINE_string(srtm_path, "", "Path to directory with SRTM files"); +DEFINE_string(mwm_path, "", "Path to mwm files (writable dir)"); + +int main(int argc, char * argv[]) { + google::SetUsageMessage("SRTM coverage checker."); + google::ParseCommandLineFlags(&argc, &argv, true); + + Platform & platform = GetPlatform(); + if (!FLAGS_mwm_path.empty()) + platform.SetWritableDirForTests(FLAGS_mwm_path); + + if (FLAGS_srtm_path.empty()) + { + LOG(LERROR, ("SRTM files directory is not specified.")); + return -1; + } + + LOG(LINFO, ("writable dir =", platform.WritableDir())); + LOG(LINFO, ("srtm dir =", FLAGS_srtm_path)); + vector localFiles; platform::FindAllLocalMapsAndCleanup(numeric_limits::max() /* latestVersion */, localFiles); - shared_ptr fetcher = integration::CreateFeaturesFetcher(localFiles); + auto fetcher = integration::CreateFeaturesFetcher(localFiles); + generator::SrtmTileManager manager(FLAGS_srtm_path); for (auto & file : localFiles) { @@ -46,10 +66,6 @@ UNIT_TEST(SrtmCoverageChecker) segMapping.Load(container, file); segMapping.Map(container); - Platform & platform = GetPlatform(); - generator::SrtmFileManager manager( - my::JoinFoldersToPath(platform.WritableDir(), SRTM_SOURCES_PATH)); - size_t all = 0; size_t good = 0; @@ -57,8 +73,8 @@ UNIT_TEST(SrtmCoverageChecker) { buffer_vector buffer; segMapping.ForEachFtSeg(i, MakeBackInsertFunctor(buffer)); - vector path; + vector path; for (size_t k = 0; k < buffer.size(); ++k) { auto const & segment = buffer[k]; @@ -72,35 +88,29 @@ UNIT_TEST(SrtmCoverageChecker) ft.ParseGeometry(FeatureType::BEST_GEOMETRY); // Get points in proper direction. - auto startIdx = segment.m_pointStart; - auto endIdx = segment.m_pointEnd; - if (startIdx < endIdx) - { - for (auto idx = startIdx; idx <= endIdx; ++idx) - path.push_back(ft.GetPoint(idx)); - } - else - { - // I use big signed type because endIdx can be 0. - for (int64_t idx = startIdx; idx >= static_cast(endIdx); --idx) - path.push_back(ft.GetPoint(idx)); - } + auto const startIdx = segment.m_pointStart; + auto const endIdx = segment.m_pointEnd; + for (auto idx = min(startIdx, endIdx); idx <= max(startIdx, endIdx); ++idx) + path.push_back(ft.GetPoint(idx)); + + all += path.size(); for (auto const & point : path) { auto const height = manager.GetHeight(MercatorBounds::ToLatLon(point)); - all++; - if (height != generator::SrtmFile::kInvalidHeight) + if (height != generator::SrtmTile::kInvalidHeight) good++; } } } - auto const delta = all - good; - auto const percent = delta * 100.0 / all; + + auto const bad = all - good; + auto const percent = all == 0 ? 0.0 : bad * 100.0 / all; if (percent > 10.0) { - LOG(LINFO, ("Huge error rate in:", file.GetCountryName(), "Good:", good, "All:", all, - "Delta:", delta, "%:", percent)); + LOG(LINFO, ("Huge error rate in:", file.GetCountryName(), "good:", good, "bad:", bad, "all:", + all, "%:", percent)); } } + + return 0; } -} // namespace diff --git a/generator/srtm_coverage_checker/srtm_coverage_checker.pro b/generator/srtm_coverage_checker/srtm_coverage_checker.pro new file mode 100644 index 0000000000..6f62e71f15 --- /dev/null +++ b/generator/srtm_coverage_checker/srtm_coverage_checker.pro @@ -0,0 +1,25 @@ +# SRTM coverage checker tool. +# Checks coverage of car roads by SRTM data. + +TARGET = srtm_coverage_tool +CONFIG += console warn_on +CONFIG -= app_bundle +TEMPLATE = app + +ROOT_DIR = ../.. +DEPENDENCIES = generator map routing search storage indexer platform editor geometry coding base \ + osrm jansson protobuf tomcrypt succinct stats_client pugixml minizip gflags + +include($$ROOT_DIR/common.pri) + +INCLUDEPATH *= $$ROOT_DIR/3party/gflags/src + +QT *= core + +macx-* { + LIBS *= "-framework IOKit" "-framework SystemConfiguration" +} + +SOURCES += \ + ../../routing/routing_integration_tests/routing_test_tools.cpp \ + srtm_coverage_checker.cpp \ diff --git a/generator/srtm_parser.cpp b/generator/srtm_parser.cpp index 6630c335de..97939570c5 100644 --- a/generator/srtm_parser.cpp +++ b/generator/srtm_parser.cpp @@ -10,18 +10,43 @@ namespace generator { -// SrtmFile ---------------------------------------------------------------------------------------- -SrtmFile::SrtmFile() +namespace +{ +size_t constexpr kArcSecondsInDegree = 60 * 60; +size_t constexpr kSrtmTileSize = (kArcSecondsInDegree + 1) * (kArcSecondsInDegree + 1) * 2; + +struct UnzipMemDelegate : public ZipFileReader::Delegate +{ + UnzipMemDelegate(string & buffer) : m_buffer(buffer), m_completed(false) {} + + // ZipFileReader::Delegate overrides: + void OnBlockUnzipped(size_t size, char const * data) override { m_buffer.append(data, size); } + + void OnStarted() override + { + m_buffer.clear(); + m_completed = false; + } + + void OnCompleted() override { m_completed = true; } + + string & m_buffer; + bool m_completed; +}; +} // namespace + +// SrtmTile ---------------------------------------------------------------------------------------- +SrtmTile::SrtmTile() { Invalidate(); } -SrtmFile::SrtmFile(SrtmFile && rhs) : m_data(move(rhs.m_data)), m_valid(rhs.m_valid) +SrtmTile::SrtmTile(SrtmTile && rhs) : m_data(move(rhs.m_data)), m_valid(rhs.m_valid) { rhs.Invalidate(); } -void SrtmFile::Init(string const & dir, ms::LatLon const & coord) +void SrtmTile::Init(string const & dir, ms::LatLon const & coord) { Invalidate(); @@ -29,21 +54,37 @@ void SrtmFile::Init(string const & dir, ms::LatLon const & coord) string const cont = dir + base + ".SRTMGL1.hgt.zip"; string file = base + ".hgt"; + UnzipMemDelegate delegate(m_data); try { - ZipFileReader::UnzipFileToMemory(cont, file, m_data); + ZipFileReader::UnzipFile(cont, file, delegate); } - catch (ZipFileReader::LocateZipException e) + catch (ZipFileReader::LocateZipException const & e) { // Sometimes packed file has different name. See N39E051 measure. file = base + ".SRTMGL1.hgt"; - ZipFileReader::UnzipFileToMemory(cont, file, m_data); + + ZipFileReader::UnzipFile(cont, file, delegate); + } + + if (!delegate.m_completed) + { + LOG(LWARNING, ("Can't decompress SRTM file:", cont)); + Invalidate(); + return; + } + + if (m_data.size() != kSrtmTileSize) + { + LOG(LWARNING, ("Bad decompressed SRTM file size:", cont, m_data.size())); + Invalidate(); + return; } m_valid = true; } -SrtmFile::THeight SrtmFile::GetHeight(ms::LatLon const & coord) +SrtmTile::THeight SrtmTile::GetHeight(ms::LatLon const & coord) { if (!IsValid()) return kInvalidHeight; @@ -56,17 +97,17 @@ SrtmFile::THeight SrtmFile::GetHeight(ms::LatLon const & coord) lt += 1; lt = 1 - lt; // from North to South - size_t const row = 3600 * lt; - size_t const col = 3600 * ln; + size_t const row = kArcSecondsInDegree * lt; + size_t const col = kArcSecondsInDegree * ln; - size_t const ix = row * 3601 + col; + size_t const ix = row * (kArcSecondsInDegree + 1) + col; if (ix >= Size()) return kInvalidHeight; return ReverseByteOrder(Data()[ix]); } -string SrtmFile::GetBase(ms::LatLon coord) +string SrtmTile::GetBase(ms::LatLon coord) { ostringstream ss; if (coord.lat < 0) @@ -95,37 +136,37 @@ string SrtmFile::GetBase(ms::LatLon coord) return ss.str(); } -void SrtmFile::Invalidate() +void SrtmTile::Invalidate() { m_data.clear(); m_data.shrink_to_fit(); m_valid = false; } -// SrtmFileManager --------------------------------------------------------------------------------- -SrtmFileManager::SrtmFileManager(string const & dir) : m_dir(dir) {} +// SrtmTileManager --------------------------------------------------------------------------------- +SrtmTileManager::SrtmTileManager(string const & dir) : m_dir(dir) {} -SrtmFile::THeight SrtmFileManager::GetHeight(ms::LatLon const & coord) +SrtmTile::THeight SrtmTileManager::GetHeight(ms::LatLon const & coord) { - string const base = SrtmFile::GetBase(coord); - auto it = m_storage.find(base); - if (it == m_storage.end()) + string const base = SrtmTile::GetBase(coord); + auto it = m_tiles.find(base); + if (it == m_tiles.end()) { - SrtmFile file; + SrtmTile tile; try { - file.Init(m_dir, coord); + tile.Init(m_dir, coord); } catch (RootException const & e) { - LOG(LERROR, ("Can't init SRTM file:", base, "reason:", e.Msg())); + LOG(LWARNING, ("Can't init SRTM tile:", base, "reason:", e.Msg())); } - // It's OK to store even invalid files and return invalid height + // It's OK to store even invalid tiles and return invalid height // for them later. - m_storage.emplace(base, move(file)); + it = m_tiles.emplace(base, move(tile)).first; } - return m_storage[base].GetHeight(coord); + return it->second.GetHeight(coord); } } // namespace generator diff --git a/generator/srtm_parser.hpp b/generator/srtm_parser.hpp index bf521ad988..74bc14b755 100644 --- a/generator/srtm_parser.hpp +++ b/generator/srtm_parser.hpp @@ -7,19 +7,18 @@ #include "std/cstdint.hpp" #include "std/string.hpp" #include "std/unordered_map.hpp" -#include "std/vector.hpp" namespace generator { -class SrtmFile +class SrtmTile { public: using THeight = int16_t; static THeight constexpr kInvalidHeight = -32768; - SrtmFile(); - SrtmFile(SrtmFile && rhs); + SrtmTile(); + SrtmTile(SrtmTile && rhs); void Init(string const & dir, ms::LatLon const & coord); @@ -37,23 +36,23 @@ private: void Invalidate(); - vector m_data; + string m_data; bool m_valid; - DISALLOW_COPY(SrtmFile); + DISALLOW_COPY(SrtmTile); }; -class SrtmFileManager +class SrtmTileManager { public: - SrtmFileManager(string const & dir); + SrtmTileManager(string const & dir); - SrtmFile::THeight GetHeight(ms::LatLon const & coord); + SrtmTile::THeight GetHeight(ms::LatLon const & coord); private: string m_dir; - unordered_map m_storage; + unordered_map m_tiles; - DISALLOW_COPY(SrtmFileManager); + DISALLOW_COPY(SrtmTileManager); }; } // namespace generator diff --git a/generator/srtm_source_tests/srtm_source_tests.pro b/generator/srtm_source_tests/srtm_source_tests.pro deleted file mode 100644 index 00d9b7d7c0..0000000000 --- a/generator/srtm_source_tests/srtm_source_tests.pro +++ /dev/null @@ -1,22 +0,0 @@ -# This subproject implements digital elevation model source test. -# This tests are launched on the raw SRTM dataset. - -TARGET = srtm_source_tests -CONFIG += console warn_on -CONFIG -= app_bundle -TEMPLATE = app - -ROOT_DIR = ../.. -DEPENDENCIES = generator map routing search storage indexer platform editor geometry coding base \ - osrm jansson protobuf tomcrypt succinct stats_client pugixml minizip - -macx-*: LIBS *= "-framework IOKit" "-framework SystemConfiguration" - -include($$ROOT_DIR/common.pri) - -QT *= core - -SOURCES += \ - ../../testing/testingmain.cpp \ - ../../routing/routing_integration_tests/routing_test_tools.cpp \ - srtm_source_tests.cpp \ diff --git a/omim.pro b/omim.pro index fed9f9abb8..a6d6a159db 100644 --- a/omim.pro +++ b/omim.pro @@ -37,9 +37,9 @@ SUBDIRS = 3party base coding geometry editor indexer routing search routing_integration_tests.depends = $$SUBDIRS routing_consistency_tests.subdir = routing/routing_consistency_tests routing_consistency_tests.depends = $$SUBDIRS - srtm_source_tests.subdir = generator/srtm_source_tests - srtm_source_tests.depends = $$SUBDIRS routing - SUBDIRS *= routing_integration_tests routing_consistency_tests srtm_source_tests + srtm_coverage_checker.subdir = generator/srtm_coverage_checker + srtm_coverage_checker.depends = $$SUBDIRS routing + SUBDIRS *= routing_integration_tests routing_consistency_tests srtm_coverage_checker } CONFIG(desktop) { @@ -170,9 +170,9 @@ SUBDIRS = 3party base coding geometry editor indexer routing search routing_consistency_tests.depends = $$MapDepLibs routing SUBDIRS *= routing_consistency_tests - srtm_source_tests.subdir = generator/srtm_source_tests - srtm_source_tests.depends = $$MapDepLibs routing - SUBDIRS *= srtm_source_tests + srtm_coverage_checker.subdir = generator/srtm_coverage_checker + srtm_coverage_checker.depends = $$MapDepLibs routing + SUBDIRS *= srtm_coverage_checker # TODO(AlexZ): Move pedestrian tests into routing dir. pedestrian_routing_tests.depends = $$MapDepLibs routing