gpesquero/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

[coding] Fixes to SRTM parser CL.

Browse source
This commit is contained in:
Yuri Gorshenin 2016-05-10 15:24:46 +03:00
parent 9375997212
commit 18287be1db
7 changed files with 177 additions and 104 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
coding/zip_reader.cpp
View file

@ -118,30 +118,28 @@ void ZipFileReader::UnzipFile(string const & zipContainer, string const & fileIn
outFileGuard.release();
}
char* ZipFileReader::UnzipFileToMemory(string const & zipContainer, string const & fileInZip)
void ZipFileReader::UnzipFileToMemory(string const & cont, string const & file,
vector<char> & data)
{
unzFile zip = unzOpen64(zipContainer.c_str());
unzFile zip = unzOpen64(cont.c_str());
if (!zip)
MYTHROW(OpenZipException, ("Can't get zip file handle", zipContainer));
MY_SCOPE_GUARD(zipGuard, bind(&unzClose, zip));
if (UNZ_OK != unzLocateFile(zip, fileInZip.c_str(), 1))
MYTHROW(LocateZipException, ("Can't locate file inside zip", fileInZip));
MYTHROW(OpenZipException, ("Can't get zip file handle:", cont));
MY_SCOPE_GUARD(zipCloser, bind(&unzClose, zip));
if (UNZ_OK != unzLocateFile(zip, file.c_str(), 1 /* case sensitivity */))
MYTHROW(LocateZipException, ("Can't locate file inside zip container:", file));
if (UNZ_OK != unzOpenCurrentFile(zip))
MYTHROW(LocateZipException, ("Can't open file inside zip", fileInZip));
MY_SCOPE_GUARD(currentFileGuard, bind(&unzCloseCurrentFile, zip));
MYTHROW(LocateZipException, ("Can't open file inside zip container:", file));
MY_SCOPE_GUARD(currentFileCloser, bind(&unzCloseCurrentFile, zip));
unz_file_info64 fileInfo;
if (UNZ_OK != unzGetCurrentFileInfo64(zip, &fileInfo, NULL, 0, NULL, 0, NULL, 0))
MYTHROW(LocateZipException, ("Can't get uncompressed file size inside zip", fileInZip));
size_t const uncompressedFileSize = fileInfo.uncompressed_size;
unz_file_info64 info;
if (UNZ_OK != unzGetCurrentFileInfo64(zip, &info, NULL, 0, NULL, 0, NULL, 0))
MYTHROW(LocateZipException, ("Can't get uncompressed file size inside zip:", file));
char* buf= new char[uncompressedFileSize];
size_t const size = info.uncompressed_size;
data.resize(size);
int const readBytes = unzReadCurrentFile(zip, buf, uncompressedFileSize);
if (readBytes < 0)
MYTHROW(InvalidZipException, ("Error", readBytes, "while unzipping", fileInZip, "from", zipContainer));
return buf;
int const bytesRead = unzReadCurrentFile(zip, data.data(), size);
if (bytesRead < 0)
MYTHROW(InvalidZipException, ("Error:", bytesRead, "while unzipping", file, "in", cont));
}

6
coding/zip_reader.hpp
View file

@ -32,8 +32,10 @@ public:
static void UnzipFile(string const & zipContainer, string const & fileInZip,
string const & outFilePath, ProgressFn progressFn = ProgressFn());
/// @warning Counsumer must manually free result
static char* UnzipFileToMemory(string const & zipContainer, string const & fileInZip);
/// Unzips |file| in |cont| to |buffer|.
///
/// @warning Can throw OpenZipException and LocateZipException.
static void UnzipFileToMemory(string const & cont, string const & file, vector<char> & data);
static void FilesList(string const & zipContainer, FileListT & filesList);

24
generator/generator_tests/srtm_parser_test.cpp
View file

@ -1,22 +1,28 @@
#include "generator/srtm_parser.hpp"
#include "testing/testing.hpp"
#include "base/logging.hpp"
#include "generator/srtm_parser.hpp"
using namespace generator;
namespace
{
string GetBase(ms::LatLon const & coord) { return SrtmFile::GetBase(coord); }
UNIT_TEST(FilenameTests)
{
auto name = GetSRTMBase({56.4566, 37.3467});
auto name = GetBase({56.4566, 37.3467});
TEST_EQUAL(name, "N56E037", ());
name = GetSRTMBase({34.077433, -118.304569});
name = GetBase({34.077433, -118.304569});
TEST_EQUAL(name, "N34W119", ());
name = GetSRTMBase({0.1, 0.1});
name = GetBase({0.1, 0.1});
TEST_EQUAL(name, "N00E000", ());
name = GetSRTMBase({-35.35, -12.1});
name = GetBase({-35.35, -12.1});
TEST_EQUAL(name, "S36W013", ());
name = GetSRTMBase({-34.622358, -58.383654});
name = GetBase({-34.622358, -58.383654});
TEST_EQUAL(name, "S35W059", ());
}
}
} // namespace

117
generator/srtm_parser.cpp
View file

@ -1,12 +1,74 @@
#include "srtm_parser.hpp"
#include "generator/srtm_parser.hpp"
#include "coding/endianness.hpp"
#include "coding/zip_reader.hpp"
#include "std/iomanip.hpp"
#include "base/logging.hpp"
string GetSRTMBase(ms::LatLon coord)
#include "std/iomanip.hpp"
#include "std/sstream.hpp"
namespace generator
{
stringstream ss;
// SrtmFile ----------------------------------------------------------------------------------------
SrtmFile::SrtmFile()
{
Invalidate();
}
SrtmFile::SrtmFile(SrtmFile && rhs) : m_data(move(rhs.m_data)), m_valid(rhs.m_valid)
{
rhs.Invalidate();
}
void SrtmFile::Init(string const & dir, ms::LatLon const & coord)
{
Invalidate();
string const base = GetBase(coord);
string const cont = dir + base + ".SRTMGL1.hgt.zip";
string file = base + ".hgt";
try
{
ZipFileReader::UnzipFileToMemory(cont, file, m_data);
}
catch (ZipFileReader::LocateZipException e)
{
// Sometimes packed file has different name. See N39E051 measure.
file = base + ".SRTMGL1.hgt";
ZipFileReader::UnzipFileToMemory(cont, file, m_data);
}
m_valid = true;
}
SrtmFile::THeight SrtmFile::GetHeight(ms::LatLon const & coord)
{
if (!IsValid())
return kInvalidHeight;
double ln = coord.lon - static_cast<int>(coord.lon);
if (ln < 0)
ln += 1;
double lt = coord.lat - static_cast<int>(coord.lat);
if (lt < 0)
lt += 1;
lt = 1 - lt; // from North to South
size_t const row = 3600 * lt;
size_t const col = 3600 * ln;
size_t const ix = row * 3601 + col;
if (ix >= Size())
return kInvalidHeight;
return ReverseByteOrder(Data()[ix]);
}
string SrtmFile::GetBase(ms::LatLon coord)
{
ostringstream ss;
if (coord.lat < 0)
{
ss << "S";
@ -17,7 +79,7 @@ string GetSRTMBase(ms::LatLon coord)
{
ss << "N";
}
ss << setw(2) << setfill('0') << int(coord.lat);
ss << setw(2) << setfill('0') << static_cast<int>(coord.lat);
if (coord.lon < 0)
{
@ -29,34 +91,41 @@ string GetSRTMBase(ms::LatLon coord)
{
ss << "E";
}
ss << setw(3) << int(coord.lon);
ss << setw(3) << static_cast<int>(coord.lon);
return ss.str();
}
void SrtmFile::Init(string dir, ms::LatLon coord)
void SrtmFile::Invalidate()
{
string base = GetSRTMBase(coord);
string path = dir + base + ".SRTMGL1.hgt.zip";
string file = base + ".hgt";
try
{
m_data = reinterpret_cast<TSRTMHeighType *>(ZipFileReader::UnzipFileToMemory(path, file));
}
catch (ZipFileReader::OpenZipException e)
{
m_data = nullptr;
}
catch (ZipFileReader::LocateZipException e)
m_data.clear();
m_data.shrink_to_fit();
m_valid = false;
}
// SrtmFileManager ---------------------------------------------------------------------------------
SrtmFileManager::SrtmFileManager(string const & dir) : m_dir(dir) {}
SrtmFile::THeight SrtmFileManager::GetHeight(ms::LatLon const & coord)
{
string const base = SrtmFile::GetBase(coord);
auto it = m_storage.find(base);
if (it == m_storage.end())
{
SrtmFile file;
try
{
// Sometimes packed file has different name. See N39E051 measure.
file = base + ".SRTMGL1.hgt";
m_data = reinterpret_cast<TSRTMHeighType *>(ZipFileReader::UnzipFileToMemory(path, file));
file.Init(m_dir, coord);
}
catch (int)
catch (RootException const & e)
{
m_data = nullptr;
LOG(LERROR, ("Can't init SRTM file:", base, "reason:", e.Msg()));
}
// It's OK to store even invalid files and return invalid height
// for them later.
m_storage.emplace(base, move(file));
}
return m_storage[base].GetHeight(coord);
}
} // namespace generator

76
generator/srtm_parser.hpp
View file

@ -2,64 +2,58 @@
#include "geometry/latlon.hpp"
#include "coding/endianness.hpp"
#include "base/macros.hpp"
#include "std/map.hpp"
#include "std/cstdint.hpp"
#include "std/string.hpp"
#include "std/unordered_map.hpp"
#include "std/vector.hpp"
using TSRTMHeighType = int16_t;
int16_t constexpr kInvalidSRTMHeight = -32768;
string GetSRTMBase(ms::LatLon coord);
namespace generator
{
class SrtmFile
{
public:
void Init(string dir, ms::LatLon coord);
using THeight = int16_t;
TSRTMHeighType getHeight(ms::LatLon coord)
{
if (m_data == nullptr)
return kInvalidSRTMHeight;
double ln = coord.lon - int(coord.lon);
if (ln < 0)
ln += 1;
double lt = coord.lat - int(coord.lat);
if (lt < 0)
lt += 1;
lt = 1 - lt; // From North to South
static THeight constexpr kInvalidHeight = -32768;
size_t const row = 3600 * lt;
size_t const column = 3600 * ln;
return ReverseByteOrder(m_data[row * 3601 + column]);
// 3600 and 3601 because 3601 is owerlap column.
}
SrtmFile();
SrtmFile(SrtmFile && rhs);
~SrtmFile()
{
if (m_data)
delete[] m_data;
}
void Init(string const & dir, ms::LatLon const & coord);
inline bool IsValid() const { return m_valid; }
// Returns height in meters at |coord|, or kInvalidHeight if is not initialized.
THeight GetHeight(ms::LatLon const & coord);
static string GetBase(ms::LatLon coord);
private:
TSRTMHeighType * m_data;
inline THeight const * Data() const { return reinterpret_cast<THeight const *>(m_data.data()); };
inline size_t Size() const { return m_data.size() / sizeof(THeight); }
void Invalidate();
vector<char> m_data;
bool m_valid;
DISALLOW_COPY(SrtmFile);
};
class SrtmFileManager
{
public:
SrtmFileManager(string dir) : m_dir(dir) {}
TSRTMHeighType GetCoordHeight(ms::LatLon coord)
{
string base = GetSRTMBase(coord);
auto it = m_storage.find(base);
if (it == m_storage.end())
m_storage[base].Init(m_dir, coord);
return m_storage[base].getHeight(coord);
}
SrtmFileManager(string const & dir);
SrtmFile::THeight GetHeight(ms::LatLon const & coord);
private:
string m_dir;
map<string, SrtmFile> m_storage;
unordered_map<string, SrtmFile> m_storage;
DISALLOW_COPY(SrtmFileManager);
};
} // namespace generator

18
generator/srtm_source_tests/srtm_source_tests.cpp
View file

@ -6,16 +6,16 @@
#include "routing/routing_integration_tests/routing_test_tools.hpp"
#include "coding/file_name_utils.hpp"
#include "platform/country_file.hpp"
#include "platform/local_country_file_utils.hpp"
#include "coding/file_name_utils.hpp"
namespace
{
#define SRTM_SOURCES_PATH "srtm/e4ftl01.cr.usgs.gov/SRTM/SRTMGL1.003/2000.02.11/"
UNIT_TEST(SRTMCoverageChecker)
UNIT_TEST(SrtmCoverageChecker)
{
vector<platform::LocalCountryFile> localFiles;
platform::FindAllLocalMapsAndCleanup(numeric_limits<int64_t>::max() /* latestVersion */,
@ -47,7 +47,9 @@ UNIT_TEST(SRTMCoverageChecker)
segMapping.Map(container);
Platform & platform = GetPlatform();
SrtmFileManager manager(my::JoinFoldersToPath(platform.WritableDir(), SRTM_SOURCES_PATH));
generator::SrtmFileManager manager(
my::JoinFoldersToPath(platform.WritableDir(), SRTM_SOURCES_PATH));
size_t all = 0;
size_t good = 0;
@ -83,11 +85,11 @@ UNIT_TEST(SRTMCoverageChecker)
for (int64_t idx = startIdx; idx >= static_cast<int64_t>(endIdx); --idx)
path.push_back(ft.GetPoint(idx));
}
for (auto point : path)
for (auto const & point : path)
{
auto height = manager.GetCoordHeight(MercatorBounds::ToLatLon(point));
auto const height = manager.GetHeight(MercatorBounds::ToLatLon(point));
all++;
if (height != kInvalidSRTMHeight)
if (height != generator::SrtmFile::kInvalidHeight)
good++;
}
}
@ -95,8 +97,10 @@ UNIT_TEST(SRTMCoverageChecker)
auto const delta = all - good;
auto const percent = delta * 100.0 / all;
if (percent > 10.0)
{
LOG(LINFO, ("Huge error rate in:", file.GetCountryName(), "Good:", good, "All:", all,
"Delta:", delta, "%:", percent));
}
}
}
} // namespace

4
generator/srtm_source_tests/srtm_source_tests.pro
View file

@ -7,8 +7,8 @@ CONFIG -= app_bundle
TEMPLATE = app
ROOT_DIR = ../..
DEPENDENCIES = map routing search storage indexer platform editor geometry coding base \
osrm jansson protobuf tomcrypt succinct stats_client pugixml generator minizip
DEPENDENCIES = generator map routing search storage indexer platform editor geometry coding base \
osrm jansson protobuf tomcrypt succinct stats_client pugixml minizip
macx-*: LIBS *= "-framework IOKit" "-framework SystemConfiguration"