RedAuburn/organicmaps
1
Fork 0
forked from organicmaps/organicmaps
Code Pull requests Activity

Merge pull request #3149 from ygorshenin/srtm-parser

Browse source 
Srtm parser
This commit is contained in:
mpimenov 2016-05-11 19:29:01 +04:00
commit 2a1a0ee05b
10 changed files with 476 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

76
coding/zip_reader.cpp
View file

@ -3,13 +3,42 @@
#include "base/scope_guard.hpp"
#include "base/logging.hpp"
#include "coding/file_writer.hpp"
#include "coding/constants.hpp"
#include "std/bind.hpp"
#include "3party/minizip/unzip.h"
namespace
{
class UnzipFileDelegate : public ZipFileReader::Delegate
{
public:
UnzipFileDelegate(string const & path)
: m_file(make_unique<FileWriter>(path)), m_path(path), m_completed(false)
{
}
~UnzipFileDelegate() override
{
if (!m_completed)
{
m_file.reset();
FileWriter::DeleteFileX(m_path);
}
}
// ZipFileReader::Delegate overrides:
void OnBlockUnzipped(size_t size, char const * data) override { m_file->Write(data, size); }
void OnCompleted() override { m_completed = true; }
private:
unique_ptr<FileWriter> m_file;
string const m_path;
bool m_completed;
};
} // namespace
ZipFileReader::ZipFileReader(string const & container, string const & file,
uint32_t logPageSize, uint32_t logPageCount)
@ -73,8 +102,9 @@ bool ZipFileReader::IsZip(string const & zipContainer)
return true;
}
// static
void ZipFileReader::UnzipFile(string const & zipContainer, string const & fileInZip,
string const & outFilePath, ProgressFn progressFn)
Delegate & delegate)
{
unzFile zip = unzOpen64(zipContainer.c_str());
if (!zip)
@ -92,28 +122,28 @@ void ZipFileReader::UnzipFile(string const & zipContainer, string const & fileIn
if (UNZ_OK != unzGetCurrentFileInfo64(zip, &fileInfo, NULL, 0, NULL, 0, NULL, 0))
MYTHROW(LocateZipException, ("Can't get uncompressed file size inside zip", fileInZip));
// First outFile should be closed, then FileWriter::DeleteFileX is called,
// so make correct order of guards.
MY_SCOPE_GUARD(outFileGuard, bind(&FileWriter::DeleteFileX, cref(outFilePath)));
FileWriter outFile(outFilePath);
uint64_t pos = 0;
char buf[ZIP_FILE_BUFFER_SIZE];
while (true)
int readBytes = 0;
delegate.OnStarted();
do
{
int const readBytes = unzReadCurrentFile(zip, buf, ZIP_FILE_BUFFER_SIZE);
if (readBytes > 0)
outFile.Write(buf, static_cast<size_t>(readBytes));
else if (readBytes < 0)
MYTHROW(InvalidZipException, ("Error", readBytes, "while unzipping", fileInZip, "from", zipContainer));
else
break;
readBytes = unzReadCurrentFile(zip, buf, ZIP_FILE_BUFFER_SIZE);
if (readBytes < 0)
{
MYTHROW(InvalidZipException,
("Error", readBytes, "while unzipping", fileInZip, "from", zipContainer));
}
pos += readBytes;
if (progressFn)
progressFn(fileInfo.uncompressed_size, pos);
}
outFileGuard.release();
delegate.OnBlockUnzipped(static_cast<size_t>(readBytes), buf);
} while (readBytes != 0);
delegate.OnCompleted();
}
// static
void ZipFileReader::UnzipFile(string const & zipContainer, string const & fileInZip,
string const & outPath)
{
UnzipFileDelegate delegate(outPath);
UnzipFile(zipContainer, fileInZip, delegate);
}

15
coding/zip_reader.hpp
View file

@ -1,6 +1,7 @@
#pragma once
#include "coding/file_reader.hpp"
#include "coding/file_writer.hpp"
#include "base/exception.hpp"
@ -13,6 +14,17 @@ private:
uint64_t m_uncompressedFileSize;
public:
struct Delegate
{
virtual ~Delegate() = default;
// When |size| is zero, end of file is reached.
virtual void OnBlockUnzipped(size_t size, char const * data) = 0;
virtual void OnStarted() {}
virtual void OnCompleted() {}
};
typedef function<void(uint64_t, uint64_t)> ProgressFn;
/// Contains file name inside zip and it's uncompressed size
typedef vector<pair<string, uint32_t> > FileListT;
@ -29,8 +41,9 @@ public:
uint64_t UncompressedSize() const { return m_uncompressedFileSize; }
/// @warning Can also throw Writer::OpenException and Writer::WriteException
static void UnzipFile(string const & zipContainer, string const & fileInZip, Delegate & delegate);
static void UnzipFile(string const & zipContainer, string const & fileInZip,
string const & outFilePath, ProgressFn progressFn = ProgressFn());
string const & outPath);
static void FilesList(string const & zipContainer, FileListT & filesList);

2
generator/generator.pro
View file

@ -30,6 +30,7 @@ SOURCES += \
osm_source.cpp \
routing_generator.cpp \
search_index_builder.cpp \
srtm_parser.cpp \
statistics.cpp \
tesselator.cpp \
towns_dumper.cpp \
@ -60,6 +61,7 @@ HEADERS += \
polygonizer.hpp \
routing_generator.hpp \
search_index_builder.hpp \
srtm_parser.hpp \
statistics.hpp \
tag_admixer.hpp \
tesselator.hpp \

1
generator/generator_tests/generator_tests.pro
View file

@ -28,6 +28,7 @@ SOURCES += \
osm_type_test.cpp \
source_data.cpp \
source_to_element_test.cpp \
srtm_parser_test.cpp \
tag_admixer_test.cpp \
tesselator_test.cpp \
triangles_tree_coding_test.cpp \

28
generator/generator_tests/srtm_parser_test.cpp Normal file
View file

@ -0,0 +1,28 @@
#include "testing/testing.hpp"
#include "generator/srtm_parser.hpp"
using namespace generator;
namespace
{
inline string GetBase(ms::LatLon const & coord) { return SrtmTile::GetBase(coord); }
UNIT_TEST(FilenameTests)
{
auto name = GetBase({56.4566, 37.3467});
TEST_EQUAL(name, "N56E037", ());
name = GetBase({34.077433, -118.304569});
TEST_EQUAL(name, "N34W119", ());
name = GetBase({0.1, 0.1});
TEST_EQUAL(name, "N00E000", ());
name = GetBase({-35.35, -12.1});
TEST_EQUAL(name, "S36W013", ());
name = GetBase({-34.622358, -58.383654});
TEST_EQUAL(name, "S35W059", ());
}
} // namespace

116
generator/srtm_coverage_checker/srtm_coverage_checker.cpp Normal file
View file

@ -0,0 +1,116 @@
#include "generator/srtm_parser.hpp"
#include "map/feature_vec_model.hpp"
#include "routing/routing_integration_tests/routing_test_tools.hpp"
#include "coding/file_name_utils.hpp"
#include "platform/country_file.hpp"
#include "platform/local_country_file_utils.hpp"
#include "base/logging.hpp"
#include "std/algorithm.hpp"
#include "3party/gflags/src/gflags/gflags.h"
DEFINE_string(srtm_path, "", "Path to directory with SRTM files");
DEFINE_string(mwm_path, "", "Path to mwm files (writable dir)");
int main(int argc, char * argv[])
{
google::SetUsageMessage("SRTM coverage checker.");
google::ParseCommandLineFlags(&argc, &argv, true);
Platform & platform = GetPlatform();
if (!FLAGS_mwm_path.empty())
platform.SetWritableDirForTests(FLAGS_mwm_path);
if (FLAGS_srtm_path.empty())
{
LOG(LERROR, ("SRTM files directory is not specified."));
return -1;
}
LOG(LINFO, ("writable dir =", platform.WritableDir()));
LOG(LINFO, ("srtm dir =", FLAGS_srtm_path));
vector<platform::LocalCountryFile> localFiles;
platform::FindAllLocalMapsAndCleanup(numeric_limits<int64_t>::max() /* latestVersion */,
localFiles);
auto fetcher = integration::CreateFeaturesFetcher(localFiles);
generator::SrtmTileManager manager(FLAGS_srtm_path);
for (auto & file : localFiles)
{
file.SyncWithDisk();
if (file.GetFiles() != MapOptions::MapWithCarRouting)
{
LOG(LINFO, ("Warning! Routing file not found for:", file.GetCountryName()));
continue;
}
FilesMappingContainer container(file.GetPath(MapOptions::CarRouting));
if (!container.IsExist(ROUTING_FTSEG_FILE_TAG))
{
LOG(LINFO, ("Warning! Mwm file has not routing ftseg section:", file.GetCountryName()));
continue;
}
routing::TDataFacade dataFacade;
dataFacade.Load(container);
OsrmFtSegMapping segMapping;
segMapping.Load(container, file);
segMapping.Map(container);
size_t all = 0;
size_t good = 0;
for (size_t i = 0; i < dataFacade.GetNumberOfNodes(); ++i)
{
buffer_vector<OsrmMappingTypes::FtSeg, 8> buffer;
segMapping.ForEachFtSeg(i, MakeBackInsertFunctor(buffer));
vector<m2::PointD> path;
for (size_t k = 0; k < buffer.size(); ++k)
{
auto const & segment = buffer[k];
if (!segment.IsValid())
continue;
// Load data from drive.
FeatureType ft;
Index::FeaturesLoaderGuard loader(
fetcher->GetIndex(), fetcher->GetIndex().GetMwmIdByCountryFile(file.GetCountryFile()));
loader.GetFeatureByIndex(segment.m_fid, ft);
ft.ParseGeometry(FeatureType::BEST_GEOMETRY);
// Get points in proper direction.
auto const startIdx = segment.m_pointStart;
auto const endIdx = segment.m_pointEnd;
for (auto idx = min(startIdx, endIdx); idx <= max(startIdx, endIdx); ++idx)
path.push_back(ft.GetPoint(idx));
all += path.size();
for (auto const & point : path)
{
auto const height = manager.GetHeight(MercatorBounds::ToLatLon(point));
if (height != generator::SrtmTile::kInvalidHeight)
good++;
}
}
}
auto const bad = all - good;
auto const percent = all == 0 ? 0.0 : bad * 100.0 / all;
if (percent > 10.0)
{
LOG(LINFO, ("Huge error rate in:", file.GetCountryName(), "good:", good, "bad:", bad, "all:",
all, "%:", percent));
}
}
return 0;
}

25
generator/srtm_coverage_checker/srtm_coverage_checker.pro Normal file
View file

@ -0,0 +1,25 @@
# SRTM coverage checker tool.
# Checks coverage of car roads by SRTM data.
TARGET = srtm_coverage_tool
CONFIG += console warn_on
CONFIG -= app_bundle
TEMPLATE = app
ROOT_DIR = ../..
DEPENDENCIES = generator map routing search storage indexer platform editor geometry coding base \
osrm jansson protobuf tomcrypt succinct stats_client pugixml minizip gflags
include($$ROOT_DIR/common.pri)
INCLUDEPATH *= $$ROOT_DIR/3party/gflags/src
QT *= core
macx-* {
LIBS *= "-framework IOKit" "-framework SystemConfiguration"
}
SOURCES += \
../../routing/routing_integration_tests/routing_test_tools.cpp \
srtm_coverage_checker.cpp \

172
generator/srtm_parser.cpp Normal file
View file

@ -0,0 +1,172 @@
#include "generator/srtm_parser.hpp"
#include "coding/endianness.hpp"
#include "coding/zip_reader.hpp"
#include "base/logging.hpp"
#include "std/iomanip.hpp"
#include "std/sstream.hpp"
namespace generator
{
namespace
{
size_t constexpr kArcSecondsInDegree = 60 * 60;
size_t constexpr kSrtmTileSize = (kArcSecondsInDegree + 1) * (kArcSecondsInDegree + 1) * 2;
struct UnzipMemDelegate : public ZipFileReader::Delegate
{
UnzipMemDelegate(string & buffer) : m_buffer(buffer), m_completed(false) {}
// ZipFileReader::Delegate overrides:
void OnBlockUnzipped(size_t size, char const * data) override { m_buffer.append(data, size); }
void OnStarted() override
{
m_buffer.clear();
m_completed = false;
}
void OnCompleted() override { m_completed = true; }
string & m_buffer;
bool m_completed;
};
} // namespace
// SrtmTile ----------------------------------------------------------------------------------------
SrtmTile::SrtmTile()
{
Invalidate();
}
SrtmTile::SrtmTile(SrtmTile && rhs) : m_data(move(rhs.m_data)), m_valid(rhs.m_valid)
{
rhs.Invalidate();
}
void SrtmTile::Init(string const & dir, ms::LatLon const & coord)
{
Invalidate();
string const base = GetBase(coord);
string const cont = dir + base + ".SRTMGL1.hgt.zip";
string file = base + ".hgt";
UnzipMemDelegate delegate(m_data);
try
{
ZipFileReader::UnzipFile(cont, file, delegate);
}
catch (ZipFileReader::LocateZipException const & e)
{
// Sometimes packed file has different name. See N39E051 measure.
file = base + ".SRTMGL1.hgt";
ZipFileReader::UnzipFile(cont, file, delegate);
}
if (!delegate.m_completed)
{
LOG(LWARNING, ("Can't decompress SRTM file:", cont));
Invalidate();
return;
}
if (m_data.size() != kSrtmTileSize)
{
LOG(LWARNING, ("Bad decompressed SRTM file size:", cont, m_data.size()));
Invalidate();
return;
}
m_valid = true;
}
SrtmTile::THeight SrtmTile::GetHeight(ms::LatLon const & coord)
{
if (!IsValid())
return kInvalidHeight;
double ln = coord.lon - static_cast<int>(coord.lon);
if (ln < 0)
ln += 1;
double lt = coord.lat - static_cast<int>(coord.lat);
if (lt < 0)
lt += 1;
lt = 1 - lt; // from North to South
size_t const row = kArcSecondsInDegree * lt;
size_t const col = kArcSecondsInDegree * ln;
size_t const ix = row * (kArcSecondsInDegree + 1) + col;
if (ix >= Size())
return kInvalidHeight;
return ReverseByteOrder(Data()[ix]);
}
string SrtmTile::GetBase(ms::LatLon coord)
{
ostringstream ss;
if (coord.lat < 0)
{
ss << "S";
coord.lat *= -1;
coord.lat += 1;
}
else
{
ss << "N";
}
ss << setw(2) << setfill('0') << static_cast<int>(coord.lat);
if (coord.lon < 0)
{
ss << "W";
coord.lon *= -1;
coord.lon += 1;
}
else
{
ss << "E";
}
ss << setw(3) << static_cast<int>(coord.lon);
return ss.str();
}
void SrtmTile::Invalidate()
{
m_data.clear();
m_data.shrink_to_fit();
m_valid = false;
}
// SrtmTileManager ---------------------------------------------------------------------------------
SrtmTileManager::SrtmTileManager(string const & dir) : m_dir(dir) {}
SrtmTile::THeight SrtmTileManager::GetHeight(ms::LatLon const & coord)
{
string const base = SrtmTile::GetBase(coord);
auto it = m_tiles.find(base);
if (it == m_tiles.end())
{
SrtmTile tile;
try
{
tile.Init(m_dir, coord);
}
catch (RootException const & e)
{
LOG(LWARNING, ("Can't init SRTM tile:", base, "reason:", e.Msg()));
}
// It's OK to store even invalid tiles and return invalid height
// for them later.
it = m_tiles.emplace(base, move(tile)).first;
}
return it->second.GetHeight(coord);
}
} // namespace generator

58
generator/srtm_parser.hpp Normal file
View file

@ -0,0 +1,58 @@
#pragma once
#include "geometry/latlon.hpp"
#include "base/macros.hpp"
#include "std/cstdint.hpp"
#include "std/string.hpp"
#include "std/unordered_map.hpp"
namespace generator
{
class SrtmTile
{
public:
using THeight = int16_t;
static THeight constexpr kInvalidHeight = -32768;
SrtmTile();
SrtmTile(SrtmTile && rhs);
void Init(string const & dir, ms::LatLon const & coord);
inline bool IsValid() const { return m_valid; }
// Returns height in meters at |coord|, or kInvalidHeight if is not initialized.
THeight GetHeight(ms::LatLon const & coord);
static string GetBase(ms::LatLon coord);
private:
inline THeight const * Data() const { return reinterpret_cast<THeight const *>(m_data.data()); };
inline size_t Size() const { return m_data.size() / sizeof(THeight); }
void Invalidate();
string m_data;
bool m_valid;
DISALLOW_COPY(SrtmTile);
};
class SrtmTileManager
{
public:
SrtmTileManager(string const & dir);
SrtmTile::THeight GetHeight(ms::LatLon const & coord);
private:
string m_dir;
unordered_map<string, SrtmTile> m_tiles;
DISALLOW_COPY(SrtmTileManager);
};
} // namespace generator

8
omim.pro
View file

@ -37,7 +37,9 @@ SUBDIRS = 3party base coding geometry editor indexer routing search
routing_integration_tests.depends = $$SUBDIRS
routing_consistency_tests.subdir = routing/routing_consistency_tests
routing_consistency_tests.depends = $$SUBDIRS
SUBDIRS *= routing_integration_tests routing_consistency_tests
srtm_coverage_checker.subdir = generator/srtm_coverage_checker
srtm_coverage_checker.depends = $$SUBDIRS routing
SUBDIRS *= routing_integration_tests routing_consistency_tests srtm_coverage_checker
}
CONFIG(desktop) {
@ -168,6 +170,10 @@ SUBDIRS = 3party base coding geometry editor indexer routing search
routing_consistency_tests.depends = $$MapDepLibs routing
SUBDIRS *= routing_consistency_tests
srtm_coverage_checker.subdir = generator/srtm_coverage_checker
srtm_coverage_checker.depends = $$MapDepLibs routing
SUBDIRS *= srtm_coverage_checker
# TODO(AlexZ): Move pedestrian tests into routing dir.
pedestrian_routing_tests.depends = $$MapDepLibs routing
SUBDIRS *= pedestrian_routing_tests