From 3363aba81c28935c23de60f5874ce11c77796868 Mon Sep 17 00:00:00 2001 From: Sergey Yershov Date: Fri, 13 Feb 2015 21:22:46 +0300 Subject: [PATCH] Added generator_tool option for set osm data type [xml or o5m] --- 3party/3party.pro | 3 +- 3party/o5mreader/o5mreader.c | 516 ++++++++++++++++ 3party/o5mreader/o5mreader.h | 93 +++ 3party/o5mreader/o5mreader.pro | 17 + coding/compressed_bit_vector.cpp | 1 + coding/streams_sink.hpp | 4 +- coding/write_to_sink.hpp | 2 +- generator/data_generator.cpp | 107 ---- generator/data_generator.hpp | 8 - generator/feature_generator.cpp | 328 +--------- generator/feature_generator.hpp | 4 - generator/generator.pro | 2 +- generator/generator_tool/generator_tool.cpp | 13 +- generator/generator_tool/generator_tool.pro | 4 +- generator/osm_element.hpp | 1 + generator/osm_source.cpp | 647 ++++++++++++++++++++ generator/osm_source.hpp | 11 + generator/point_storage.hpp | 257 ++------ generator/xml_element.cpp | 66 +- generator/xml_element.hpp | 9 +- std/type_traits.hpp | 20 +- tools/unix/planet.sh | 12 +- tools/unix/split_planet_by_polygons.sh | 6 +- 23 files changed, 1402 insertions(+), 729 deletions(-) create mode 100644 3party/o5mreader/o5mreader.c create mode 100644 3party/o5mreader/o5mreader.h create mode 100644 3party/o5mreader/o5mreader.pro delete mode 100644 generator/data_generator.cpp delete mode 100644 generator/data_generator.hpp create mode 100644 generator/osm_source.cpp create mode 100644 generator/osm_source.hpp diff --git a/3party/3party.pro b/3party/3party.pro index c84d19a984..cb2a4a42de 100644 --- a/3party/3party.pro +++ b/3party/3party.pro @@ -2,7 +2,8 @@ TEMPLATE = subdirs -SUBDIRS = freetype fribidi zlib bzip2 jansson tomcrypt protobuf osrm expat +SUBDIRS = freetype fribidi zlib bzip2 jansson tomcrypt protobuf osrm expat \ + o5mreader !iphone*:!tizen*:!android* { SUBDIRS += gflags \ diff --git a/3party/o5mreader/o5mreader.c b/3party/o5mreader/o5mreader.c new file mode 100644 index 0000000000..b3c214bf43 --- /dev/null +++ b/3party/o5mreader/o5mreader.c @@ -0,0 +1,516 @@ +#include "o5mreader.h" +#include +#include +#include +#include +#include + + +#define STR_PAIR_TABLE_SIZE 15000 +#define STR_PAIR_STRING_SIZE 256 + +void o5mreader_setError(O5mreader *pReader,int code, const char* message); +void o5mreader_setNoError(O5mreader *pReader); + +O5mreaderRet o5mreader_readUInt(O5mreader *pReader, uint64_t *ret) { + uint8_t b; + uint8_t i = 0; + *ret = 0LL; + + do { + if ( fread(&b,1,1,pReader->f) == 0 ) { + o5mreader_setError(pReader, + O5MREADER_ERR_CODE_UNEXPECTED_END_OF_FILE, + NULL + ); + return O5MREADER_RET_ERR; + } + *ret |= (long long)(b & 0x7f) << (i++ * 7); + } while ( b & 0x80 ); + + o5mreader_setNoError(pReader); + + return O5MREADER_RET_OK; +} + +O5mreaderRet o5mreader_readInt(O5mreader *pReader, uint64_t *ret) { + if ( o5mreader_readUInt(pReader, ret) == O5MREADER_RET_ERR ) + return O5MREADER_RET_ERR; + *ret = *ret & 1 + ? -(int64_t)(*ret >> 1) - 1 + : (int64_t)(*ret >> 1); + return O5MREADER_RET_OK; +} + + +O5mreaderRet o5mreader_readStrPair(O5mreader *pReader, char **tagpair, int single) { + static char buffer[1024]; + char* pBuf; + static uint64_t pointer = 0; + int length; + char byte; + uint64_t key; + int i; + + if ( o5mreader_readUInt(pReader,&key) == O5MREADER_RET_ERR ) { + return O5MREADER_RET_ERR; + } + + if ( key ) { + *tagpair = pReader->strPairTable[(pointer+15000-key)%15000]; + return key; + } + else { + pBuf = buffer; + for ( i=0; i<(single?1:2); i++ ) { + do { + if ( fread(pBuf,1,1,pReader->f) == 0 ) { + o5mreader_setError(pReader, + O5MREADER_ERR_CODE_UNEXPECTED_END_OF_FILE, + NULL + ); + return O5MREADER_RET_ERR; + } + } while ( *(pBuf++) ); + } + + length = strlen(buffer) + (single ? 1 : strlen(buffer+strlen(buffer) +1) + 2); + + if ( length <= 252 ) { + *tagpair = pReader->strPairTable[(pointer+15000)%15000]; + memcpy(pReader->strPairTable[((pointer++)+15000)%15000],buffer,length); + } + else { + *tagpair = buffer; + } + + } + + return O5MREADER_RET_OK; +} + +O5mreaderRet o5mreader_reset(O5mreader *pReader) { + pReader->nodeId = pReader->wayId = pReader->wayNodeId = pReader->relId = pReader->nodeRefId = pReader->wayRefId = pReader->relRefId = 0; + pReader->lon = pReader->lat = 0; + pReader->offset = 0; + pReader->canIterateTags = pReader->canIterateNds = pReader->canIterateRefs = 0; + return O5MREADER_RET_OK; +} + +O5mreaderRet o5mreader_open(O5mreader **ppReader,FILE* f) { + uint8_t byte; + int i; + *ppReader = malloc(sizeof(O5mreader)); + if ( !(*ppReader) ) { + return O5MREADER_RET_ERR; + } + (*ppReader)->errMsg = NULL; + (*ppReader)->f = f; + if ( fread(&byte,1,1,(*ppReader)->f) == 0 ) { + o5mreader_setError(*ppReader, + O5MREADER_ERR_CODE_UNEXPECTED_END_OF_FILE, + NULL + ); + return O5MREADER_RET_ERR; + } + if ( byte != O5MREADER_DS_RESET ) { + o5mreader_setError(*ppReader, + O5MREADER_ERR_CODE_FILE_HAS_WRONG_START, + NULL + ); + return O5MREADER_RET_ERR; + } + + o5mreader_reset(*ppReader); + + (*ppReader)->strPairTable = malloc(STR_PAIR_TABLE_SIZE*sizeof(char*)); + if ( (*ppReader)->strPairTable == 0 ) { + o5mreader_setError(*ppReader, + O5MREADER_ERR_CODE_MEMORY_ERROR, + NULL + ); + return O5MREADER_RET_ERR; + } + for ( i = 0; i < STR_PAIR_TABLE_SIZE; ++i ) { + (*ppReader)->strPairTable[i] = malloc(sizeof(char)*STR_PAIR_STRING_SIZE); + if ( (*ppReader)->strPairTable[i] == 0 ) { + o5mreader_setError(*ppReader, + O5MREADER_ERR_CODE_MEMORY_ERROR, + NULL + ); + return O5MREADER_RET_ERR; + } + } + + o5mreader_setNoError(*ppReader); + return O5MREADER_RET_OK; +} + +void o5mreader_close(O5mreader *pReader) { + int i; + if ( pReader ) { + if ( pReader->strPairTable ) { + for ( i = 0; i < STR_PAIR_TABLE_SIZE; ++i ) + if ( pReader->strPairTable[i] ) + free(pReader->strPairTable[i]); + free(pReader->strPairTable); + } + o5mreader_setNoError(pReader); + free(pReader); + } +} + +const char* o5mreader_strerror(int errCode) { + switch ( errCode ) { + case O5MREADER_ERR_CODE_FILE_HAS_WRONG_START: + return "'0xFF' isn't first byte of file."; + case O5MREADER_ERR_CODE_MEMORY_ERROR: + return "Memory error."; + case O5MREADER_ERR_CODE_UNEXPECTED_END_OF_FILE: + return "Unexpected end of file."; + case O5MREADER_ERR_CODE_CAN_NOT_ITERATE_TAGS_HERE: + return "Tags iteration is not allowed here."; + case O5MREADER_ERR_CODE_CAN_NOT_ITERATE_NDS_HERE: + return "Nodes iteration is not allowed here."; + case O5MREADER_ERR_CODE_CAN_NOT_ITERATE_REFS_HERE: + return "References iteration is not allowed here."; + default: + return "Unknown error code"; + } +} + +void o5mreader_setError(O5mreader *pReader,int code, const char* message) { + pReader->errCode = code; + if ( pReader->errMsg ) { + free(pReader->errMsg); + } + if ( message ) { + pReader->errMsg = malloc(strlen(message)+1); + strcpy(pReader->errMsg,message); + } +} + +void o5mreader_setNoError(O5mreader *pReader) { + pReader->errCode = O5MREADER_ERR_CODE_OK; + if ( pReader->errMsg ) { + free(pReader->errMsg); + } + pReader->errMsg = NULL; +} + +O5mreaderIterateRet o5mreader_iterateDataSet(O5mreader *pReader, O5mreaderDataset* ds) { + for (;;) { + if ( pReader->offset ) { + if ( o5mreader_skipTags(pReader) == O5MREADER_ITERATE_RET_ERR ) + return O5MREADER_ITERATE_RET_ERR; + + fseek( + pReader->f, + (pReader->current - ftell(pReader->f)) + pReader->offset, + SEEK_CUR + ); + + pReader->offset = 0; + } + + if ( fread(&(ds->type),1,1,pReader->f) == 0 ) { + o5mreader_setError(pReader, + O5MREADER_ERR_CODE_UNEXPECTED_END_OF_FILE, + NULL + ); + return O5MREADER_ITERATE_RET_ERR; + } + + if ( O5MREADER_DS_END == ds->type ) + return O5MREADER_ITERATE_RET_DONE; + + if ( O5MREADER_DS_RESET == ds->type ) { + o5mreader_reset(pReader); + } + else if ( 0xf0 == ds->type ) {} + else { + if ( o5mreader_readUInt(pReader,&pReader->offset) == O5MREADER_RET_ERR ) { + return O5MREADER_ITERATE_RET_ERR; + } + pReader->current = ftell(pReader->f); + + switch ( ds->type ) { + case O5MREADER_DS_NODE: + return o5mreader_readNode(pReader, ds); + case O5MREADER_DS_WAY: + return o5mreader_readWay(pReader, ds); + case O5MREADER_DS_REL: + return o5mreader_readRel(pReader, ds); + /* + case O5MREADER_DS_BBOX: + case O5MREADER_DS_TSTAMP: + case O5MREADER_DS_HEADER: + case O5MREADER_DS_SYNC: + case O5MREADER_DS_JUMP: + default: + break; + */ + } + } + } + + +} + +int o5mreader_thereAreNoMoreData(O5mreader *pReader) { + return (int)((pReader->current - ftell(pReader->f)) + pReader->offset) <= 0; +} + +O5mreaderIterateRet o5mreader_readVersion(O5mreader *pReader, O5mreaderDataset* ds) { + uint64_t tmp; + if ( o5mreader_readUInt(pReader,&tmp) == O5MREADER_ITERATE_RET_ERR ) { + return O5MREADER_ITERATE_RET_ERR; + } + ds->version = tmp; + if ( tmp ) { + if ( o5mreader_readUInt(pReader,&tmp) == O5MREADER_ITERATE_RET_ERR ) { + return O5MREADER_ITERATE_RET_ERR; + } + + if ( o5mreader_readInt(pReader,&tmp) == O5MREADER_ITERATE_RET_ERR ) { + return O5MREADER_ITERATE_RET_ERR; + } + + if ( o5mreader_thereAreNoMoreData(pReader) ) + return O5MREADER_ITERATE_RET_DONE; + + if ( o5mreader_readStrPair(pReader,&pReader->tagPair,0) == O5MREADER_ITERATE_RET_ERR ) { + return O5MREADER_ITERATE_RET_ERR; + } + } + + if ( o5mreader_thereAreNoMoreData(pReader) ) + return O5MREADER_ITERATE_RET_DONE; + + return O5MREADER_ITERATE_RET_NEXT; +} + +O5mreaderIterateRet o5mreader_iterateTags(O5mreader *pReader, char** pKey, char** pVal) { + if ( pReader->canIterateRefs ) { + if ( o5mreader_skipRefs(pReader) == O5MREADER_ITERATE_RET_ERR ) + return O5MREADER_ITERATE_RET_ERR; + } + if ( pReader->canIterateNds ) { + if ( o5mreader_skipNds(pReader) == O5MREADER_ITERATE_RET_ERR ) + return O5MREADER_ITERATE_RET_ERR; + } + if ( !pReader->canIterateTags ) { + o5mreader_setError(pReader, + O5MREADER_ERR_CODE_CAN_NOT_ITERATE_TAGS_HERE, + NULL + ); + return O5MREADER_ITERATE_RET_ERR; + } + if ( o5mreader_thereAreNoMoreData(pReader) ) { + pReader->canIterateTags = 0; + return O5MREADER_ITERATE_RET_DONE; + } + + if ( o5mreader_readStrPair(pReader,&pReader->tagPair,0) == O5MREADER_RET_ERR ) { + return O5MREADER_ITERATE_RET_ERR; + } + if ( pKey ) + *pKey = pReader->tagPair; + if ( pVal ) + *pVal = pReader->tagPair + strlen(pReader->tagPair) + 1; + + return O5MREADER_ITERATE_RET_NEXT; +} + +O5mreaderIterateRet o5mreader_skipTags(O5mreader *pReader) { + int ret; + if ( pReader->canIterateTags ) { + while ( O5MREADER_ITERATE_RET_NEXT == (ret = o5mreader_iterateTags(pReader, NULL, NULL)) ); + } + + return ret; +} + +O5mreaderIterateRet o5mreader_readNode(O5mreader *pReader, O5mreaderDataset* ds) { + uint64_t nodeId; + int64_t lon,lat; + if ( o5mreader_readInt(pReader,&nodeId) == O5MREADER_RET_ERR ) + return O5MREADER_ITERATE_RET_ERR; + + pReader->canIterateRefs = 0; + pReader->canIterateNds = 0; + pReader->canIterateTags = 1; + + pReader->nodeId += nodeId; + ds->id = pReader->nodeId; + + + if ( o5mreader_readVersion(pReader, ds) == O5MREADER_ITERATE_RET_DONE ) { + ds->isEmpty = 1; + return O5MREADER_ITERATE_RET_NEXT; + } + else + ds->isEmpty = 0; + + if ( o5mreader_thereAreNoMoreData(pReader) ) { + return O5MREADER_ITERATE_RET_NEXT; + } + + if ( o5mreader_readInt(pReader,&lon) == O5MREADER_RET_ERR ) + return O5MREADER_ITERATE_RET_ERR; + pReader->lon += (int32_t)lon; + + if ( o5mreader_readInt(pReader,&lat) == O5MREADER_RET_ERR ) { + return O5MREADER_ITERATE_RET_ERR; + } + pReader->lat += (int32_t)lat; + + ds->lon = pReader->lon; + ds->lat = pReader->lat; + + return O5MREADER_ITERATE_RET_NEXT; +} + +O5mreaderIterateRet o5mreader_iterateNds(O5mreader *pReader, uint64_t *nodeId) { + int64_t wayNodeId; + + if ( !pReader->canIterateNds ) { + o5mreader_setError(pReader, + O5MREADER_ERR_CODE_CAN_NOT_ITERATE_NDS_HERE, + NULL + ); + return O5MREADER_ITERATE_RET_ERR; + } + if ( ftell(pReader->f) >= pReader->offsetNd ) { + pReader->canIterateNds = 0; + pReader->canIterateTags = 1; + pReader->canIterateRefs = 0; + return O5MREADER_ITERATE_RET_DONE; + } + + if ( o5mreader_readInt(pReader,&wayNodeId) == O5MREADER_RET_ERR ) + return O5MREADER_ITERATE_RET_ERR; + + pReader->wayNodeId += wayNodeId; + + if ( nodeId ) + *nodeId = pReader->wayNodeId; + + return O5MREADER_ITERATE_RET_NEXT; +} + +O5mreaderIterateRet o5mreader_skipNds(O5mreader *pReader) { + uint8_t ret; + while ( pReader->canIterateNds && + O5MREADER_ITERATE_RET_NEXT == (ret = o5mreader_iterateNds(pReader, NULL)) ); + return ret; +} + +O5mreaderIterateRet o5mreader_readWay(O5mreader *pReader, O5mreaderDataset* ds) { + int64_t wayId; + if ( o5mreader_readInt(pReader,&wayId) == O5MREADER_RET_ERR) + return O5MREADER_ITERATE_RET_ERR; + + pReader->wayId += wayId; + ds->id = pReader->wayId; + if ( o5mreader_readVersion(pReader, ds) == O5MREADER_ITERATE_RET_DONE ) { + ds->isEmpty = 1; + return O5MREADER_ITERATE_RET_NEXT; + } + else + ds->isEmpty = 0; + if ( o5mreader_readUInt(pReader,&pReader->offsetNd) == O5MREADER_RET_ERR ) { + return O5MREADER_ITERATE_RET_ERR; + } + pReader->offsetNd += ftell(pReader->f); + pReader->canIterateRefs = 0; + pReader->canIterateNds = 1; + pReader->canIterateTags = 0; + return O5MREADER_ITERATE_RET_NEXT; +} + +O5mreaderIterateRet o5mreader_iterateRefs(O5mreader *pReader, uint64_t *refId, uint8_t *type, char** pRole) { + int64_t relRefId; + + if ( !pReader->canIterateRefs ) { + o5mreader_setError(pReader, + O5MREADER_ERR_CODE_CAN_NOT_ITERATE_REFS_HERE, + NULL + ); + return O5MREADER_ITERATE_RET_ERR; + } + if ( ftell(pReader->f) >= pReader->offsetRf ) { + pReader->canIterateNds = 0; + pReader->canIterateTags = 1; + pReader->canIterateRefs = 0; + return O5MREADER_ITERATE_RET_DONE; + } + + if ( o5mreader_readInt(pReader, &relRefId) == O5MREADER_RET_ERR ) + return O5MREADER_ITERATE_RET_ERR; + + + //fread(_,1,1,pReader->f); + + if ( o5mreader_readStrPair(pReader, &pReader->tagPair,1) == O5MREADER_RET_ERR ) { + return O5MREADER_ITERATE_RET_ERR; + } + + switch( pReader->tagPair[0] ) { + case '0': + if ( type ) + *type = O5MREADER_DS_NODE; + pReader->nodeRefId += relRefId; + if ( refId ) + *refId = pReader->nodeRefId; + break; + case '1': + if ( type ) + *type = O5MREADER_DS_WAY; + pReader->wayRefId += relRefId; + if ( refId ) + *refId = pReader->wayRefId; + break; + case '2': + if ( type ) + *type = O5MREADER_DS_REL; + pReader->relRefId += relRefId; + if ( refId ) + *refId = pReader->relRefId; + break; + } + + if ( pRole ) { + *pRole = pReader->tagPair + 1; + } + + return O5MREADER_ITERATE_RET_NEXT; +} + +O5mreaderIterateRet o5mreader_skipRefs(O5mreader *pReader) { + uint8_t ret; + while ( pReader->canIterateRefs && + O5MREADER_ITERATE_RET_NEXT == (ret = o5mreader_iterateRefs(pReader, NULL, NULL, NULL)) ); + return ret; +} + +O5mreaderIterateRet o5mreader_readRel(O5mreader *pReader, O5mreaderDataset* ds) { + int64_t relId; + if ( o5mreader_readInt(pReader,&relId) == O5MREADER_RET_ERR ) + return O5MREADER_ITERATE_RET_ERR; + pReader->relId += relId; + ds->id = pReader->relId; + if ( o5mreader_readVersion(pReader,ds) == O5MREADER_ITERATE_RET_DONE ) { + ds->isEmpty = 1; + return O5MREADER_ITERATE_RET_NEXT; + } + else + ds->isEmpty = 0; + o5mreader_readUInt(pReader,&pReader->offsetRf); + pReader->offsetRf += ftell(pReader->f); + + pReader->canIterateRefs = 1; + pReader->canIterateNds = 0; + pReader->canIterateTags = 0; + return O5MREADER_ITERATE_RET_NEXT; +} diff --git a/3party/o5mreader/o5mreader.h b/3party/o5mreader/o5mreader.h new file mode 100644 index 0000000000..08ce5c3a6e --- /dev/null +++ b/3party/o5mreader/o5mreader.h @@ -0,0 +1,93 @@ +#ifndef __O5MREADER__H__ +#define __O5MREADER__H__ + +#include +#include + + +#define O5MREADER_RET_OK 1 +#define O5MREADER_RET_ERR 0 + +#define O5MREADER_ITERATE_RET_DONE 0 +#define O5MREADER_ITERATE_RET_ERR -1 +#define O5MREADER_ITERATE_RET_NEXT 1 + +#define O5MREADER_DS_END 0xfe +#define O5MREADER_DS_NODE 0x10 +#define O5MREADER_DS_WAY 0x11 +#define O5MREADER_DS_REL 0x12 +#define O5MREADER_DS_BBOX 0xdb +#define O5MREADER_DS_TSTAMP 0xdc +#define O5MREADER_DS_HEADER 0xe0 +#define O5MREADER_DS_SYNC 0xee +#define O5MREADER_DS_JUMP 0xef +#define O5MREADER_DS_RESET 0xff + + +#define O5MREADER_ERR_CODE_OK 0 +#define O5MREADER_ERR_CODE_FILE_HAS_WRONG_START 1 +#define O5MREADER_ERR_CODE_UNEXPECTED_END_OF_FILE 2 +#define O5MREADER_ERR_CODE_MEMORY_ERROR 3 +#define O5MREADER_ERR_CODE_CAN_NOT_ITERATE_TAGS_HERE 4 +#define O5MREADER_ERR_CODE_CAN_NOT_ITERATE_NDS_HERE 5 +#define O5MREADER_ERR_CODE_CAN_NOT_ITERATE_REFS_HERE 6 + +typedef int O5mreaderRet; +typedef int O5mreaderIterateRet; + +typedef struct { + int errCode; + char* errMsg; + FILE *f; + uint64_t offset; + uint64_t offsetNd; + uint64_t offsetRf; + uint64_t current; + char* tagPair; + int64_t nodeId; + int64_t wayId; + int64_t wayNodeId; + int64_t relId; + int64_t nodeRefId; + int64_t wayRefId; + int64_t relRefId; + int32_t lon; + int32_t lat; + uint8_t canIterateTags; + uint8_t canIterateNds; + uint8_t canIterateRefs; + char** strPairTable; +} O5mreader; + +typedef struct { + uint8_t type; + uint64_t id; + uint32_t version; + uint8_t isEmpty; + int32_t lon; + int32_t lat; +} O5mreaderDataset; + +#if defined (__cplusplus) +extern "C" { +#endif + +O5mreaderRet o5mreader_open(O5mreader **ppReader,FILE* f); + +void o5mreader_close(O5mreader *pReader); + +const char* o5mreader_strerror(int errCode); + +O5mreaderIterateRet o5mreader_iterateDataSet(O5mreader *pReader, O5mreaderDataset* ds); + +O5mreaderIterateRet o5mreader_iterateTags(O5mreader *pReader, char** pKey, char** pVal); + +O5mreaderIterateRet o5mreader_iterateNds(O5mreader *pReader, uint64_t *nodeId); + +O5mreaderIterateRet o5mreader_iterateRefs(O5mreader *pReader, uint64_t *refId, uint8_t *type, char** pRole); + +#if defined (__cplusplus) +} +#endif + +#endif //__O5MREADER__H__ diff --git a/3party/o5mreader/o5mreader.pro b/3party/o5mreader/o5mreader.pro new file mode 100644 index 0000000000..638021b7a3 --- /dev/null +++ b/3party/o5mreader/o5mreader.pro @@ -0,0 +1,17 @@ +#------------------------------------------------- +# +# Project created by QtCreator 2015-02-13T20:23:55 +# +#------------------------------------------------- + +ROOT_DIR = ../.. + +TARGET = o5mreader +TEMPLATE = lib +CONFIG += staticlib + +include($$ROOT_DIR/common.pri) + +SOURCES += o5mreader.c + +HEADERS += o5mreader.h diff --git a/coding/compressed_bit_vector.cpp b/coding/compressed_bit_vector.cpp index 6573f573ed..beb9304c30 100644 --- a/coding/compressed_bit_vector.cpp +++ b/coding/compressed_bit_vector.cpp @@ -8,6 +8,7 @@ #include "../base/assert.hpp" #include "../base/bits.hpp" +#include "../std/cmath.hpp" namespace { vector SerialFreqsToDistrTable(Reader & reader, uint64_t & decodeOffset, uint64_t cnt) diff --git a/coding/streams_sink.hpp b/coding/streams_sink.hpp index b80669560d..bc3bd13cd7 100644 --- a/coding/streams_sink.hpp +++ b/coding/streams_sink.hpp @@ -17,7 +17,7 @@ namespace stream SinkReaderStream(TReader & reader) : m_reader(reader) {} template - typename enable_if, SinkReaderStream &>::type + typename enable_if::value, SinkReaderStream &>::type operator >> (T & t) { t = ReadPrimitiveFromSource(m_reader); @@ -53,7 +53,7 @@ namespace stream SinkWriterStream(TWriter & writer) : m_writer(writer) {} template - typename enable_if, SinkWriterStream &>::type + typename enable_if::value, SinkWriterStream &>::type operator << (T const & t) { WriteToSink(m_writer, t); diff --git a/coding/write_to_sink.hpp b/coding/write_to_sink.hpp index 163a54cad8..318c38e21e 100644 --- a/coding/write_to_sink.hpp +++ b/coding/write_to_sink.hpp @@ -6,7 +6,7 @@ template -typename enable_if, void>::type WriteToSink(TSink & sink, T const & v) +typename enable_if::value, void>::type WriteToSink(TSink & sink, T const & v) { T const t = SwapIfBigEndian(v); sink.Write(&t, sizeof(T)); diff --git a/generator/data_generator.cpp b/generator/data_generator.cpp deleted file mode 100644 index 39702af888..0000000000 --- a/generator/data_generator.cpp +++ /dev/null @@ -1,107 +0,0 @@ -#include "data_generator.hpp" -#include "data_cache_file.hpp" -#include "first_pass_parser.hpp" -#include "osm_decl.hpp" - -#include "../base/std_serialization.hpp" -#include "../base/logging.hpp" - -#include "../std/bind.hpp" -#include "point_storage.hpp" - -namespace data -{ - -template -class FileHolder : public cache::BaseFileHolder -{ - typedef cache::BaseFileHolder base_type; - - typedef typename base_type::user_id_t user_id_t; - - template - void add_id2rel_vector(TMap & rMap, user_id_t relid, TVec const & v) - { - for (size_t i = 0; i < v.size(); ++i) - rMap.write(v[i].first, relid); - } - -public: - FileHolder(TNodesHolder & nodes, string const & dir) : base_type(nodes, dir) {} - - void AddNode(uint64_t id, double lat, double lng) - { - this->m_nodes.AddPoint(id, lat, lng); - } - - void AddWay(user_id_t id, WayElement const & e) - { - this->m_ways.Write(id, e); - } - - void AddRelation(user_id_t id, RelationElement const & e) - { - const string relationType = e.GetType(); - if (relationType == "multipolygon" || relationType == "route" || relationType == "boundary") - { - this->m_relations.Write(id, e); - - add_id2rel_vector(this->m_nodes2rel, id, e.nodes); - add_id2rel_vector(this->m_ways2rel, id, e.ways); - } - } - - void SaveIndex() - { - this->m_ways.SaveOffsets(); - this->m_relations.SaveOffsets(); - - this->m_nodes2rel.flush_to_file(); - this->m_ways2rel.flush_to_file(); - } -}; - -template -bool GenerateImpl(string const & dir, string const & osmFileName = string()) -{ - try - { - TNodesHolder nodes(dir + NODES_FILE); - typedef FileHolder holder_t; - holder_t holder(nodes, dir); - - FirstPassParser parser(holder); - if (osmFileName.empty()) - ParseXMLFromStdIn(parser); - else - ParseXMLFromFile(parser, osmFileName); - - LOG(LINFO, ("Added points count = ", nodes.GetCount())); - - holder.SaveIndex(); - } - catch (Writer::Exception const & e) - { - LOG(LERROR, ("Error with file ", e.what())); - return false; - } - - return true; -} - -bool GenerateToFile(string const & dir, string const & nodeStorage, string const & osmFileName) -{ - if (nodeStorage == "raw") - return GenerateImpl>(dir, osmFileName); - else if (nodeStorage == "map") - return GenerateImpl>(dir, osmFileName); - else if (nodeStorage == "sqlite") - return GenerateImpl>(dir, osmFileName); - else if (nodeStorage == "mem") - return GenerateImpl>(dir, osmFileName); - else - CHECK(nodeStorage.empty(), ("Incorrect node_storage type:", nodeStorage)); - return false; -} - -} diff --git a/generator/data_generator.hpp b/generator/data_generator.hpp deleted file mode 100644 index 424308a8a6..0000000000 --- a/generator/data_generator.hpp +++ /dev/null @@ -1,8 +0,0 @@ -#pragma once - -#include "../std/string.hpp" - -namespace data -{ - bool GenerateToFile(string const & dir, string const & nodeStorage, string const & osmFileName); -} diff --git a/generator/feature_generator.cpp b/generator/feature_generator.cpp index c4c616d129..f74d68192b 100644 --- a/generator/feature_generator.cpp +++ b/generator/feature_generator.cpp @@ -1,18 +1,13 @@ #include "feature_generator.hpp" #include "data_cache_file.hpp" #include "osm_element.hpp" -#include "polygonizer.hpp" + #include "osm_decl.hpp" #include "generate_info.hpp" -#include "coastlines_generator.hpp" -#include "world_map_generator.hpp" - -#include "../defines.hpp" #include "../indexer/data_header.hpp" #include "../indexer/mercator.hpp" #include "../indexer/cell_id.hpp" -#include "../indexer/classificator.hpp" #include "../coding/varint.hpp" @@ -24,103 +19,11 @@ #include "../std/unordered_map.hpp" #include "../std/target_os.hpp" -#include "point_storage.hpp" - -namespace feature -{ - -template -class FileHolder : public cache::BaseFileHolder -{ - typedef cache::DataFileReader reader_t; - typedef cache::BaseFileHolder base_type; - - typedef typename base_type::offset_map_t offset_map_t; - - typedef typename base_type::user_id_t user_id_t; - - template struct process_base - { - protected: - reader_t & m_reader; - ToDo & m_toDo; - public: - process_base(reader_t & reader, ToDo & toDo) : m_reader(reader), m_toDo(toDo) {} - - bool operator() (uint64_t id) - { - TElement e; - if (m_reader.Read(id, e)) - return m_toDo(id, e); - return false; - } - }; - - template struct process_relation : public process_base - { - typedef process_base base_type; - public: - process_relation(reader_t & reader, ToDo & toDo) : base_type(reader, toDo) {} - }; - - template struct process_relation_cached : public process_relation - { - typedef process_relation base_type; - - public: - process_relation_cached(reader_t & rels, ToDo & toDo) - : base_type(rels, toDo) {} - - bool operator() (uint64_t id) - { - return this->m_toDo(id, this->m_reader); - } - }; - -public: - FileHolder(TNodesHolder & holder, string const & dir) : base_type(holder, dir) {} - - bool GetNode(uint64_t id, double & lat, double & lng) - { - return this->m_nodes.GetPoint(id, lat, lng); - } - - bool GetWay(user_id_t id, WayElement & e) - { - return this->m_ways.Read(id, e); - } - - template void ForEachRelationByWay(user_id_t id, ToDo & toDo) - { - process_relation processor(this->m_relations, toDo); - this->m_ways2rel.for_each_ret(id, processor); - } - - template void ForEachRelationByNodeCached(user_id_t id, ToDo & toDo) - { - process_relation_cached processor(this->m_relations, toDo); - this->m_nodes2rel.for_each_ret(id, processor); - } - - template void ForEachRelationByWayCached(user_id_t id, ToDo & toDo) - { - process_relation_cached processor(this->m_relations, toDo); - this->m_ways2rel.for_each_ret(id, processor); - } - - void LoadIndex() - { - this->m_ways.LoadOffsets(); - this->m_relations.LoadOffsets(); - - this->m_nodes2rel.read_to_memory(); - this->m_ways2rel.read_to_memory(); - } -}; /////////////////////////////////////////////////////////////////////////////////////////////////// // FeaturesCollector implementation /////////////////////////////////////////////////////////////////////////////////////////////////// +namespace feature { FeaturesCollector::FeaturesCollector(string const & fName) : m_datFile(fName) @@ -213,232 +116,5 @@ void FeaturesCollector::operator() (FeatureBuilder1 const & fb) (void)WriteFeatureBase(bytes, fb); } -/////////////////////////////////////////////////////////////////////////////////////////////////// -// Generate functions implementations. -/////////////////////////////////////////////////////////////////////////////////////////////////// - -namespace -{ - -class MainFeaturesEmitter -{ - typedef WorldMapGenerator WorldGenerator; - typedef CountryMapGenerator > CountriesGenerator; - unique_ptr m_countries; - unique_ptr m_world; - - unique_ptr m_coasts; - unique_ptr m_coastsHolder; - - string m_srcCoastsFile; - - template class CombinedEmitter - { - T1 * m_p1; - T2 * m_p2; - public: - CombinedEmitter(T1 * p1, T2 * p2) : m_p1(p1), m_p2(p2) {} - void operator() (FeatureBuilder1 const & fb, uint64_t) - { - if (m_p1) (*m_p1)(fb); - if (m_p2) (*m_p2)(fb); - } - }; - - enum TypeIndex - { - NATURAL_COASTLINE, - NATURAL_LAND, - PLACE_ISLAND, - PLACE_ISLET, - - TYPES_COUNT - }; - uint32_t m_types[TYPES_COUNT]; - - inline uint32_t Type(TypeIndex i) const { return m_types[i]; } - -public: - MainFeaturesEmitter(GenerateInfo const & info) - { - Classificator const & c = classif(); - - char const * arr[][2] = { - { "natural", "coastline" }, - { "natural", "land" }, - { "place", "island" }, - { "place", "islet" } - }; - STATIC_ASSERT(ARRAY_SIZE(arr) == TYPES_COUNT); - - for (size_t i = 0; i < ARRAY_SIZE(arr); ++i) - m_types[i] = c.GetTypeByPath(vector(arr[i], arr[i] + 2)); - - m_srcCoastsFile = info.m_tmpDir + WORLD_COASTS_FILE_NAME + info.m_datFileSuffix; - - CHECK(!info.m_makeCoasts || !info.m_createWorld, - ("We can't do make_coasts and generate_world at the same time")); - - if (!info.m_makeCoasts) - { - m_countries.reset(new CountriesGenerator(info)); - - if (info.m_emitCoasts) - { - m_coastsHolder.reset(new FeaturesCollector( - info.m_datFilePrefix + WORLD_COASTS_FILE_NAME + info.m_datFileSuffix)); - } - } - else - { - // 4-10 - level range for cells - // 20000 - max points count per feature - m_coasts.reset(new CoastlineFeaturesGenerator(Type(NATURAL_COASTLINE), 4, 10, 20000)); - - m_coastsHolder.reset(new FeaturesCollector(m_srcCoastsFile)); - } - - if (info.m_createWorld) - m_world.reset(new WorldGenerator(info)); - } - - void operator() (FeatureBuilder1 fb) - { - uint32_t const coastType = Type(NATURAL_COASTLINE); - bool const hasCoast = fb.HasType(coastType); - - if (m_coasts) - { - if (hasCoast) - { - CHECK ( fb.GetGeomType() != feature::GEOM_POINT, () ); - - // leave only coastline type - fb.SetType(coastType); - (*m_coasts)(fb); - } - } - else - { - if (hasCoast) - { - fb.PopExactType(Type(NATURAL_LAND)); - fb.PopExactType(coastType); - } - else if ((fb.HasType(Type(PLACE_ISLAND)) || fb.HasType(Type(PLACE_ISLET))) && - fb.GetGeomType() == feature::GEOM_AREA) - { - fb.AddType(Type(NATURAL_LAND)); - } - - if (fb.RemoveInvalidTypes()) - { - if (m_world) - (*m_world)(fb); - - if (m_countries) - (*m_countries)(fb); - } - } - } - - /// @return false if coasts are not merged and FLAG_fail_on_coasts is set - bool Finish() - { - if (m_world) - m_world->DoMerge(); - - if (m_coasts) - { - // Check and stop if some coasts were not merged - if (!m_coasts->Finish()) - return false; - - size_t const count = m_coasts->GetCellsCount(); - LOG(LINFO, ("Generating coastline polygons", count)); - - for (size_t i = 0; i < count; ++i) - { - vector vecFb; - m_coasts->GetFeatures(i, vecFb); - - for (size_t j = 0; j < vecFb.size(); ++j) - (*m_coastsHolder)(vecFb[j]); - } - } - else if (m_coastsHolder) - { - CombinedEmitter - emitter(m_coastsHolder.get(), m_countries.get()); - feature::ForEachFromDatRawFormat(m_srcCoastsFile, emitter); - } - - return true; - } - - inline void GetNames(vector & names) const - { - if (m_countries) - names = m_countries->Parent().Names(); - else - names.clear(); - } -}; - -} - -template -bool GenerateImpl(GenerateInfo & info, string const & osmFileName = string()) -{ - try - { - NodesHolderT nodes(info.m_tmpDir + NODES_FILE); - - typedef FileHolder HolderT; - HolderT holder(nodes, info.m_tmpDir); - holder.LoadIndex(); - - MainFeaturesEmitter bucketer(info); - SecondPassParser parser( - bucketer, holder, - info.m_makeCoasts ? classif().GetCoastType() : 0, - info.m_addressFile); - - if (osmFileName.empty()) - ParseXMLFromStdIn(parser); - else - ParseXMLFromFile(parser, osmFileName); - - parser.Finish(); - - // Stop if coasts are not merged and FLAG_fail_on_coasts is set - if (!bucketer.Finish()) - return false; - - bucketer.GetNames(info.m_bucketNames); - } - catch (Reader::Exception const & e) - { - LOG(LERROR, ("Error with file ", e.what())); - return false; - } - - return true; -} - -bool GenerateFeatures(GenerateInfo & info, string const & nodeStorage, string const & osmFileName) -{ - if (nodeStorage == "raw") - return GenerateImpl>(info, osmFileName); - else if (nodeStorage == "map") - return GenerateImpl>(info, osmFileName); - else if (nodeStorage == "sqlite") - return GenerateImpl>(info, osmFileName); - else if (nodeStorage == "mem") - return GenerateImpl>(info, osmFileName); - else - CHECK(nodeStorage.empty(), ("Incorrect node_storage type:", nodeStorage)); - return false; -} } diff --git a/generator/feature_generator.hpp b/generator/feature_generator.hpp index c0323940f3..8ff2f66956 100644 --- a/generator/feature_generator.hpp +++ b/generator/feature_generator.hpp @@ -12,10 +12,6 @@ class FeatureBuilder1; namespace feature { - struct GenerateInfo; - - bool GenerateFeatures(GenerateInfo & info, string const & nodeStorage, string const & osmFileName); - // Writes features to dat file. class FeaturesCollector { diff --git a/generator/generator.pro b/generator/generator.pro index 79e1a12343..711f292870 100644 --- a/generator/generator.pro +++ b/generator/generator.pro @@ -16,7 +16,7 @@ QT *= core SOURCES += \ feature_merger.cpp \ xml_element.cpp \ - data_generator.cpp \ + osm_source.cpp \ feature_generator.cpp \ feature_sorter.cpp \ update_generator.cpp \ diff --git a/generator/generator_tool/generator_tool.cpp b/generator/generator_tool/generator_tool.cpp index 330375d818..820023f7e3 100644 --- a/generator/generator_tool/generator_tool.cpp +++ b/generator/generator_tool/generator_tool.cpp @@ -1,4 +1,3 @@ -#include "../data_generator.hpp" #include "../feature_generator.hpp" #include "../feature_sorter.hpp" #include "../update_generator.hpp" @@ -10,6 +9,7 @@ #include "../generate_info.hpp" #include "../check_model.hpp" #include "../routing_generator.hpp" +#include "../osm_source.hpp" #include "../../indexer/drawing_rules.hpp" #include "../../indexer/classificator_loader.hpp" @@ -35,7 +35,7 @@ DEFINE_bool(generate_update, false, DEFINE_bool(generate_classif, false, "Generate classificator."); -DEFINE_bool(preprocess_xml, false, "1st pass - create nodes/ways/relations data"); +DEFINE_bool(preprocess, false, "1st pass - create nodes/ways/relations data"); DEFINE_bool(make_coasts, false, "create intermediate file with coasts data"); DEFINE_bool(emit_coasts, false, "push coasts features from intermediate file to out files/countries"); @@ -44,7 +44,7 @@ DEFINE_bool(generate_geometry, false, "3rd pass - split and simplify geometry an DEFINE_bool(generate_index, false, "4rd pass - generate index"); DEFINE_bool(generate_search_index, false, "5th pass - generate search index"); DEFINE_bool(calc_statistics, false, "Calculate feature statistics for specified mwm bucket files"); -DEFINE_string(node_storage, "raw", "Type of storage for intermediate points representation. Available: raw, map, sqlite, mem"); +DEFINE_string(node_storage, "map", "Type of storage for intermediate points representation. Available: raw, map, mem"); DEFINE_string(data_path, "", "Working directory, 'path_to_exe/../../data' if empty."); DEFINE_string(output, "", "File name for process (without 'mwm' ext)."); DEFINE_string(intermediate_data_path, "", "Path to stored nodes, ways, relations."); @@ -65,6 +65,7 @@ DEFINE_string(address_file_name, "", "Output file name for storing full addresse DEFINE_string(export_poly_path, "", "Output dir for osm .poly files created from .borders (countires are read from polygons.lst)"); DEFINE_string(osrm_file_name, "", "Input osrm file to generate routing info"); DEFINE_string(osm_file_name, "", "Input osm area file"); +DEFINE_string(osm_file_type, "xml", "Input osm area file type [xml, o5m]"); DEFINE_string(user_resource_path, "", "User defined resource path for classificator.txt and etc."); @@ -101,10 +102,10 @@ int main(int argc, char ** argv) FLAGS_data_path.empty() ? pl.WritableDir() : AddSlashIfNeeded(FLAGS_data_path); // Generating intermediate files - if (FLAGS_preprocess_xml) + if (FLAGS_preprocess) { LOG(LINFO, ("Generating intermediate data ....")); - if (!data::GenerateToFile(FLAGS_intermediate_data_path, FLAGS_node_storage, FLAGS_osm_file_name)) + if (!GenerateIntermediateData(FLAGS_intermediate_data_path, FLAGS_node_storage, FLAGS_osm_file_type, FLAGS_osm_file_name)) return -1; } @@ -148,7 +149,7 @@ int main(int argc, char ** argv) if (!FLAGS_address_file_name.empty()) genInfo.m_addressFile = path + FLAGS_address_file_name; - if (!feature::GenerateFeatures(genInfo, FLAGS_node_storage, FLAGS_osm_file_name)) + if (!GenerateFeatures(genInfo, FLAGS_node_storage, FLAGS_osm_file_type, FLAGS_osm_file_name)) return -1; // without --spit_by_polygons, we have empty name country as result - assign it diff --git a/generator/generator_tool/generator_tool.pro b/generator/generator_tool/generator_tool.pro index 9735eea9c7..6ac03d0285 100644 --- a/generator/generator_tool/generator_tool.pro +++ b/generator/generator_tool/generator_tool.pro @@ -2,7 +2,7 @@ ROOT_DIR = ../.. DEPENDENCIES = generator routing storage indexer platform geometry coding base \ - osrm gflags expat sgitess jansson protobuf tomcrypt + osrm gflags expat sgitess jansson protobuf tomcrypt o5mreader include($$ROOT_DIR/common.pri) @@ -18,8 +18,6 @@ QT *= core win32: LIBS *= -lShell32 macx-*: LIBS *= "-framework Foundation" -LIBS *= -lsqlite3 - SOURCES += \ generator_tool.cpp \ diff --git a/generator/osm_element.hpp b/generator/osm_element.hpp index c4eea48969..4ed82bc531 100644 --- a/generator/osm_element.hpp +++ b/generator/osm_element.hpp @@ -382,6 +382,7 @@ class SecondPassParser : public BaseOSMParser } //@} +public: /// The main entry point for parsing process. virtual void EmitElement(XMLElement * p) { diff --git a/generator/osm_source.cpp b/generator/osm_source.cpp new file mode 100644 index 0000000000..825773e995 --- /dev/null +++ b/generator/osm_source.cpp @@ -0,0 +1,647 @@ +#include "osm_source.hpp" + +#include "osm_decl.hpp" +#include "data_cache_file.hpp" + +#include "coastlines_generator.hpp" +#include "world_map_generator.hpp" +#include "feature_generator.hpp" +#include "polygonizer.hpp" + +#include "point_storage.hpp" + +#include "xml_element.hpp" + +#include "first_pass_parser.hpp" +#include "osm_element.hpp" + +#include "../defines.hpp" +#include "../indexer/mercator.hpp" +#include "../indexer/classificator.hpp" + +#include "../coding/parse_xml.hpp" + +#include "../3party/o5mreader/o5mreader.h" + + +#define DECODE_O5M_COORD(coord) (static_cast(coord) / 1E+7) + +namespace +{ + class SourceReader + { + string const &m_filename; + FILE * m_file; + + public: + SourceReader(string const & filename) : m_filename(filename) + { + if (m_filename.empty()) + { + LOG(LINFO, ("Read OSM data from stdin...")); + m_file = freopen(NULL, "rb", stdin); + } + else + { + LOG(LINFO, ("Read OSM data from", filename)); + m_file = fopen(filename.c_str(), "rb"); + } + } + + ~SourceReader() + { + if (!m_filename.empty()) + fclose(m_file); + } + + inline FILE * Handle() { return m_file; } + + uint64_t Read(char * buffer, uint64_t bufferSize) + { + return fread(buffer, sizeof(char), bufferSize, m_file); + } + }; +} + + +namespace feature +{ + + template + class FileHolder : public cache::BaseFileHolder + { + typedef cache::DataFileReader reader_t; + typedef cache::BaseFileHolder base_type; + + typedef typename base_type::offset_map_t offset_map_t; + + typedef typename base_type::user_id_t user_id_t; + + template struct process_base + { + protected: + reader_t & m_reader; + ToDo & m_toDo; + public: + process_base(reader_t & reader, ToDo & toDo) : m_reader(reader), m_toDo(toDo) {} + + bool operator() (uint64_t id) + { + TElement e; + if (m_reader.Read(id, e)) + return m_toDo(id, e); + return false; + } + }; + + template struct process_relation : public process_base + { + typedef process_base base_type; + public: + process_relation(reader_t & reader, ToDo & toDo) : base_type(reader, toDo) {} + }; + + template struct process_relation_cached : public process_relation + { + typedef process_relation base_type; + + public: + process_relation_cached(reader_t & rels, ToDo & toDo) + : base_type(rels, toDo) {} + + bool operator() (uint64_t id) + { + return this->m_toDo(id, this->m_reader); + } + }; + + public: + FileHolder(TNodesHolder & holder, string const & dir) : base_type(holder, dir) {} + + bool GetNode(uint64_t id, double & lat, double & lng) + { + return this->m_nodes.GetPoint(id, lat, lng); + } + + bool GetWay(user_id_t id, WayElement & e) + { + return this->m_ways.Read(id, e); + } + + template void ForEachRelationByWay(user_id_t id, ToDo & toDo) + { + process_relation processor(this->m_relations, toDo); + this->m_ways2rel.for_each_ret(id, processor); + } + + template void ForEachRelationByNodeCached(user_id_t id, ToDo & toDo) + { + process_relation_cached processor(this->m_relations, toDo); + this->m_nodes2rel.for_each_ret(id, processor); + } + + template void ForEachRelationByWayCached(user_id_t id, ToDo & toDo) + { + process_relation_cached processor(this->m_relations, toDo); + this->m_ways2rel.for_each_ret(id, processor); + } + + void LoadIndex() + { + this->m_ways.LoadOffsets(); + this->m_relations.LoadOffsets(); + + this->m_nodes2rel.read_to_memory(); + this->m_ways2rel.read_to_memory(); + } + }; +} // namespace feature + +namespace data +{ + + template + class FileHolder : public cache::BaseFileHolder + { + typedef cache::BaseFileHolder base_type; + + typedef typename base_type::user_id_t user_id_t; + + template + void add_id2rel_vector(TMap & rMap, user_id_t relid, TVec const & v) + { + for (size_t i = 0; i < v.size(); ++i) + rMap.write(v[i].first, relid); + } + + public: + FileHolder(TNodesHolder & nodes, string const & dir) : base_type(nodes, dir) {} + + void AddNode(uint64_t id, double lat, double lng) + { + this->m_nodes.AddPoint(id, lat, lng); + } + + void AddWay(user_id_t id, WayElement const & e) + { + this->m_ways.Write(id, e); + } + + void AddRelation(user_id_t id, RelationElement const & e) + { + const string relationType = e.GetType(); + if (relationType == "multipolygon" || relationType == "route" || relationType == "boundary") + { + this->m_relations.Write(id, e); + + add_id2rel_vector(this->m_nodes2rel, id, e.nodes); + add_id2rel_vector(this->m_ways2rel, id, e.ways); + } + } + + void SaveIndex() + { + this->m_ways.SaveOffsets(); + this->m_relations.SaveOffsets(); + + this->m_nodes2rel.flush_to_file(); + this->m_ways2rel.flush_to_file(); + } + }; +} // namespace data + +namespace +{ + + class MainFeaturesEmitter + { + typedef WorldMapGenerator WorldGenerator; + typedef CountryMapGenerator > CountriesGenerator; + unique_ptr m_countries; + unique_ptr m_world; + + unique_ptr m_coasts; + unique_ptr m_coastsHolder; + + string m_srcCoastsFile; + + template class CombinedEmitter + { + T1 * m_p1; + T2 * m_p2; + public: + CombinedEmitter(T1 * p1, T2 * p2) : m_p1(p1), m_p2(p2) {} + void operator() (FeatureBuilder1 const & fb, uint64_t) + { + if (m_p1) (*m_p1)(fb); + if (m_p2) (*m_p2)(fb); + } + }; + + enum TypeIndex + { + NATURAL_COASTLINE, + NATURAL_LAND, + PLACE_ISLAND, + PLACE_ISLET, + + TYPES_COUNT + }; + uint32_t m_types[TYPES_COUNT]; + + inline uint32_t Type(TypeIndex i) const { return m_types[i]; } + + public: + MainFeaturesEmitter(feature::GenerateInfo const & info) + { + Classificator const & c = classif(); + + char const * arr[][2] = { + { "natural", "coastline" }, + { "natural", "land" }, + { "place", "island" }, + { "place", "islet" } + }; + STATIC_ASSERT(ARRAY_SIZE(arr) == TYPES_COUNT); + + for (size_t i = 0; i < ARRAY_SIZE(arr); ++i) + m_types[i] = c.GetTypeByPath(vector(arr[i], arr[i] + 2)); + + m_srcCoastsFile = info.m_tmpDir + WORLD_COASTS_FILE_NAME + info.m_datFileSuffix; + + CHECK(!info.m_makeCoasts || !info.m_createWorld, + ("We can't do make_coasts and generate_world at the same time")); + + if (!info.m_makeCoasts) + { + m_countries.reset(new CountriesGenerator(info)); + + if (info.m_emitCoasts) + { + m_coastsHolder.reset(new feature::FeaturesCollector( + info.m_datFilePrefix + WORLD_COASTS_FILE_NAME + info.m_datFileSuffix)); + } + } + else + { + // 4-10 - level range for cells + // 20000 - max points count per feature + m_coasts.reset(new CoastlineFeaturesGenerator(Type(NATURAL_COASTLINE), 4, 10, 20000)); + + m_coastsHolder.reset(new feature::FeaturesCollector(m_srcCoastsFile)); + } + + if (info.m_createWorld) + m_world.reset(new WorldGenerator(info)); + } + + void operator() (FeatureBuilder1 fb) + { + uint32_t const coastType = Type(NATURAL_COASTLINE); + bool const hasCoast = fb.HasType(coastType); + + if (m_coasts) + { + if (hasCoast) + { + CHECK ( fb.GetGeomType() != feature::GEOM_POINT, () ); + + // leave only coastline type + fb.SetType(coastType); + (*m_coasts)(fb); + } + } + else + { + if (hasCoast) + { + fb.PopExactType(Type(NATURAL_LAND)); + fb.PopExactType(coastType); + } + else if ((fb.HasType(Type(PLACE_ISLAND)) || fb.HasType(Type(PLACE_ISLET))) && + fb.GetGeomType() == feature::GEOM_AREA) + { + fb.AddType(Type(NATURAL_LAND)); + } + + if (fb.RemoveInvalidTypes()) + { + if (m_world) + (*m_world)(fb); + + if (m_countries) + (*m_countries)(fb); + } + } + } + + /// @return false if coasts are not merged and FLAG_fail_on_coasts is set + bool Finish() + { + if (m_world) + m_world->DoMerge(); + + if (m_coasts) + { + // Check and stop if some coasts were not merged + if (!m_coasts->Finish()) + return false; + + size_t const count = m_coasts->GetCellsCount(); + LOG(LINFO, ("Generating coastline polygons", count)); + + for (size_t i = 0; i < count; ++i) + { + vector vecFb; + m_coasts->GetFeatures(i, vecFb); + + for (size_t j = 0; j < vecFb.size(); ++j) + (*m_coastsHolder)(vecFb[j]); + } + } + else if (m_coastsHolder) + { + CombinedEmitter + emitter(m_coastsHolder.get(), m_countries.get()); + feature::ForEachFromDatRawFormat(m_srcCoastsFile, emitter); + } + return true; + } + + inline void GetNames(vector & names) const + { + if (m_countries) + names = m_countries->Parent().Names(); + else + names.clear(); + } + }; +} // namespace anonymous + + +template +void BuildIntermediateDataFromO5M(SourceReader & stream, HolderT & holder) +{ + O5mreader * reader; + O5mreaderRet rc; + + if ((rc = o5mreader_open(&reader, stream.Handle())) != O5MREADER_RET_OK) + { + LOG(LCRITICAL, ("O5M Open error:", o5mreader_strerror(rc))); + exit(1); + } + + O5mreaderDataset ds; + O5mreaderIterateRet ret; + while ((ret = o5mreader_iterateDataSet(reader, &ds)) == O5MREADER_ITERATE_RET_NEXT) + { + O5mreaderIterateRet ret2; + switch (ds.type) + { + case O5MREADER_DS_NODE: + { + // Could do something with ds.id, ds.lon, ds.lat here, lon and lat are ints in 1E+7 * degree units + // convert to mercator + double const lat = MercatorBounds::LatToY(DECODE_O5M_COORD(ds.lat)); + double const lng = MercatorBounds::LonToX(DECODE_O5M_COORD(ds.lon)); + holder.AddNode(ds.id, lat, lng); + } break; + + case O5MREADER_DS_WAY: + { + // store way + WayElement way(ds.id); + uint64_t nodeId = 0; + while ((ret2 = o5mreader_iterateNds(reader, &nodeId)) == O5MREADER_ITERATE_RET_NEXT) + way.nodes.push_back(nodeId); + + if (way.IsValid()) + holder.AddWay(ds.id, way); + } break; + + case O5MREADER_DS_REL: + { + // store relation + RelationElement relation; + uint64_t refId = 0; + uint8_t type = 0; + char * role = nullptr; + while ((ret2 = o5mreader_iterateRefs(reader, &refId, &type, &role)) == O5MREADER_ITERATE_RET_NEXT) + { + // Could do something with refId (way or node or rel id depends on type), type and role + if (type == O5MREADER_DS_NODE) + relation.nodes.emplace_back(make_pair(refId, string(role))); + else if (type == O5MREADER_DS_WAY) + relation.ways.emplace_back(make_pair(refId, string(role))); + // we just ignore type == "relation" + } + + char * key = nullptr; + char * val = nullptr; + while ((ret2 = o5mreader_iterateTags(reader, &key, &val)) == O5MREADER_ITERATE_RET_NEXT) + relation.tags.emplace(make_pair(string(key), string(val))); + + if (relation.IsValid()) + holder.AddRelation(ds.id, relation); + } break; + } + } +} + +void BuildFeaturesFromO5M(SourceReader & stream, BaseOSMParser & parser) +{ + O5mreader * reader; + O5mreaderRet rc; + + if ((rc = o5mreader_open(&reader, stream.Handle())) != O5MREADER_RET_OK) + { + LOG(LCRITICAL, ("O5M Open error:", o5mreader_strerror(rc))); + exit(1); + } + + O5mreaderDataset ds; + O5mreaderIterateRet ret; + while ((ret = o5mreader_iterateDataSet(reader, &ds)) == O5MREADER_ITERATE_RET_NEXT) + { + XMLElement p; + p.id = ds.id; + + O5mreaderIterateRet ret2; + char * key = nullptr; + char * val = nullptr; + switch (ds.type) + { + case O5MREADER_DS_NODE: + { + p.tagKey = XMLElement::ET_NODE; + p.lat = DECODE_O5M_COORD(ds.lat); + p.lng = DECODE_O5M_COORD(ds.lon); + } break; + + case O5MREADER_DS_WAY: + { + p.tagKey = XMLElement::ET_WAY; + uint64_t nodeId = 0; + while ((ret2 = o5mreader_iterateNds(reader, &nodeId)) == O5MREADER_ITERATE_RET_NEXT) + p.AddND(nodeId); + } break; + + case O5MREADER_DS_REL: + { + p.tagKey = XMLElement::ET_RELATION; + uint64_t refId = 0; + uint8_t type = 0; + char * role = nullptr; + while ((ret2 = o5mreader_iterateRefs(reader, &refId, &type, &role)) == O5MREADER_ITERATE_RET_NEXT) + { + string strType; + switch (type) + { + case O5MREADER_DS_NODE: + strType = "node"; + break; + case O5MREADER_DS_WAY: + strType = "way"; + break; + case O5MREADER_DS_REL: + strType = "relation"; + break; + + default: + break; + } + p.AddMEMBER(refId, strType, role); + } + } break; + } + + while ((ret2 = o5mreader_iterateTags(reader, &key, &val)) == O5MREADER_ITERATE_RET_NEXT) + p.AddKV(key, val); + parser.EmitElement(&p); + } +} + + +/////////////////////////////////////////////////////////////////////////////////////////////////// +// Generate functions implementations. +/////////////////////////////////////////////////////////////////////////////////////////////////// + + +template +bool GenerateFeaturesImpl(feature::GenerateInfo & info, string const &osmFileType, string const & osmFileName = string()) +{ + try + { + NodesHolderT nodes(info.m_tmpDir + NODES_FILE); + + typedef feature::FileHolder HolderT; + HolderT holder(nodes, info.m_tmpDir); + holder.LoadIndex(); + + MainFeaturesEmitter bucketer(info); + SecondPassParser parser( + bucketer, holder, + info.m_makeCoasts ? classif().GetCoastType() : 0, + info.m_addressFile); + + SourceReader reader(osmFileName); + + if (osmFileType == "xml") + { + ParseXMLSequence(reader, parser); + } + else if (osmFileType == "o5m") + { + BuildFeaturesFromO5M(reader, parser); + } + else + { + LOG(LERROR, ("Unknown source type:", osmFileType)); + return false; + } + + parser.Finish(); + + // Stop if coasts are not merged and FLAG_fail_on_coasts is set + if (!bucketer.Finish()) + return false; + + bucketer.GetNames(info.m_bucketNames); + } + catch (Reader::Exception const & e) + { + LOG(LERROR, ("Error with file ", e.what())); + return false; + } + + return true; +} + +template +bool GenerateIntermediateDataImpl(string const & dir, string const &osmFileType, string const & osmFileName = string()) +{ + try + { + TNodesHolder nodes(dir + NODES_FILE); + typedef data::FileHolder HolderT; + HolderT holder(nodes, dir); + + SourceReader reader(osmFileName); + + LOG(LINFO, ("Data sorce format:", osmFileType)); + + if (osmFileType == "xml") + { + FirstPassParser parser(holder); + ParseXMLSequence(reader, parser); + } + else if (osmFileType == "o5m") + { + BuildIntermediateDataFromO5M(reader, holder); + } + else + { + LOG(LERROR, ("Unknown source type:", osmFileType)); + return false; + } + + LOG(LINFO, ("Added points count = ", nodes.GetCount())); + + holder.SaveIndex(); + } + catch (Writer::Exception const & e) + { + LOG(LERROR, ("Error with file ", e.what())); + return false; + } + return true; +} + + +bool GenerateFeatures(feature::GenerateInfo & info, string const & nodeStorage, string const &osmFileType, string const & osmFileName) +{ + if (nodeStorage == "raw") + return GenerateFeaturesImpl>(info, osmFileType, osmFileName); + else if (nodeStorage == "map") + return GenerateFeaturesImpl>(info, osmFileType, osmFileName); + else if (nodeStorage == "mem") + return GenerateFeaturesImpl>(info, osmFileType, osmFileName); + else + CHECK(nodeStorage.empty(), ("Incorrect node_storage type:", nodeStorage)); + return false; +} + + +bool GenerateIntermediateData(string const & dir, string const & nodeStorage, string const &osmFileType, string const & osmFileName) +{ + if (nodeStorage == "raw") + return GenerateIntermediateDataImpl>(dir, osmFileType, osmFileName); + else if (nodeStorage == "map") + return GenerateIntermediateDataImpl>(dir, osmFileType, osmFileName); + else if (nodeStorage == "mem") + return GenerateIntermediateDataImpl>(dir, osmFileType, osmFileName); + else + CHECK(nodeStorage.empty(), ("Incorrect node_storage type:", nodeStorage)); + return false; +} + diff --git a/generator/osm_source.hpp b/generator/osm_source.hpp new file mode 100644 index 0000000000..6a2d9f297e --- /dev/null +++ b/generator/osm_source.hpp @@ -0,0 +1,11 @@ +#pragma once + +#include "generate_info.hpp" + +#include "../std/string.hpp" + + +bool GenerateFeatures(feature::GenerateInfo & info, string const & nodeStorage, string const &osmFileType, string const & osmFileName); +bool GenerateIntermediateData(string const & dir, string const & nodeStorage, string const &osmFileType, string const & osmFileName); + + diff --git a/generator/point_storage.hpp b/generator/point_storage.hpp index a3929d08d3..79d9a3afb5 100644 --- a/generator/point_storage.hpp +++ b/generator/point_storage.hpp @@ -1,8 +1,9 @@ #pragma once #include "../coding/mmap_reader.hpp" -#include +#include "../std/iostream.hpp" +#include "../std/type_traits.hpp" /// Used to store all world nodes inside temporary index file. /// To find node by id, just calculate offset inside index file: @@ -12,14 +13,14 @@ struct LatLon double lat; double lon; }; -STATIC_ASSERT(sizeof(LatLon) == 16); +static_assert(sizeof(LatLon) == 16, "Invalid structure size"); struct ShortLatLon { int32_t lat; int32_t lon; }; -STATIC_ASSERT(sizeof(ShortLatLon) == 8); +static_assert(sizeof(ShortLatLon) == 8, "Invalid structure size"); struct LatLonPos { @@ -27,7 +28,7 @@ struct LatLonPos double lat; double lon; }; -STATIC_ASSERT(sizeof(LatLonPos) == 24); +static_assert(sizeof(LatLonPos) == 24, "Invalid structure size"); struct ShortLatLonPos { @@ -35,7 +36,7 @@ struct ShortLatLonPos int32_t lat; int32_t lon; }; -STATIC_ASSERT(sizeof(ShortLatLonPos) == 16); +static_assert(sizeof(ShortLatLonPos) == 16, "Invalid structure size"); class BasePointStorage @@ -53,7 +54,7 @@ protected: public: enum EStorageMode {MODE_READ = false, MODE_WRITE = true}; - BasePointStorage(string const & name, size_t factor) + BasePointStorage(string const & name, size_t factor = 1000) { m_progress.Begin(name, factor); } @@ -61,155 +62,22 @@ public: uint64_t GetCount() const { return m_progress.GetCount(); } }; -template < BasePointStorage::EStorageMode ModeT > -class SQLitePointStorage : public BasePointStorage -{ - sqlite3 *m_db; - sqlite3_stmt *m_prepared_statement; - -public: - SQLitePointStorage(string const & name) : BasePointStorage(name, 1000) - { - if( sqlite3_open((name+".sqlite").c_str(), &m_db) != SQLITE_OK ){ - fprintf(stderr, "Can't open database: %s\n", sqlite3_errmsg(m_db)); - sqlite3_close(m_db); - exit(1); - } - InitStorage(EnableIf()); - } - - ~SQLitePointStorage() - { - DoneStorage(EnableIf()); - sqlite3_finalize(m_prepared_statement); - sqlite3_close(m_db); - } - - void InitStorage(EnableIf) - { - string create_table("drop table if exists points; drop index if exists points_idx; create table points(id integer PRIMARY KEY, ll blob) WITHOUT ROWID;"); - if( sqlite3_exec(m_db, create_table.c_str(), NULL, NULL, NULL ) != SQLITE_OK ){ - fprintf(stderr, "Can't open database: %s\n", sqlite3_errmsg(m_db)); - sqlite3_close(m_db); - exit(1); - } - - char* errorMessage; - sqlite3_exec(m_db, "PRAGMA synchronous=OFF", NULL, NULL, &errorMessage); - sqlite3_exec(m_db, "PRAGMA count_changes=OFF", NULL, NULL, &errorMessage); - sqlite3_exec(m_db, "PRAGMA journal_mode=MEMORY", NULL, NULL, &errorMessage); - sqlite3_exec(m_db, "PRAGMA temp_store=MEMORY", NULL, NULL, &errorMessage); - - string insert("insert into points(id, ll) values(?,?);"); - if( sqlite3_prepare_v2(m_db, insert.c_str(), -1, &m_prepared_statement, NULL) != SQLITE_OK ){ - fprintf(stderr, "Can't open database: %s\n", sqlite3_errmsg(m_db)); - sqlite3_close(m_db); - exit(1); - } - if( sqlite3_exec(m_db, "BEGIN TRANSACTION", NULL, NULL, NULL ) != SQLITE_OK ){ - fprintf(stderr, "Can't start transaction: %s\n", sqlite3_errmsg(m_db)); - sqlite3_close(m_db); - exit(1); - } - } - - void InitStorage(EnableIf) - { - string select("select ll from points where id=?;"); - if( sqlite3_prepare_v2(m_db, select.c_str(), -1, &m_prepared_statement, NULL) != SQLITE_OK ){ - fprintf(stderr, "failed sqlite3_prepare_v2: %s\n", sqlite3_errmsg(m_db)); - sqlite3_close(m_db); - exit(1); - } - } - - void DoneStorage(EnableIf) - { - if( sqlite3_exec(m_db, "COMMIT TRANSACTION", NULL, NULL, NULL ) != SQLITE_OK ){ - fprintf(stderr, "Can't end transaction: %s\n", sqlite3_errmsg(m_db)); - sqlite3_close(m_db); - exit(1); - } - if( sqlite3_exec(m_db, "create unique index points_idx on points(id);", NULL, NULL, NULL ) != SQLITE_OK ){ - fprintf(stderr, "Can't end transaction: %s\n", sqlite3_errmsg(m_db)); - sqlite3_close(m_db); - exit(1); - } - } - - void DoneStorage(EnableIf) {} - - void AddPoint(uint64_t id, double lat, double lng) - { - LatLon ll = {lat, lng}; - - if (sqlite3_bind_int64(m_prepared_statement, 1, id) != SQLITE_OK) { - cerr << "bind1 failed: " << sqlite3_errmsg(m_db) << endl; - exit(1); - } - if (sqlite3_bind_blob(m_prepared_statement, 2, &ll, sizeof(ll), SQLITE_STATIC) != SQLITE_OK) { - cerr << "bind2 failed: " << sqlite3_errmsg(m_db) << endl; - exit(1); - } - - if (sqlite3_step(m_prepared_statement) != SQLITE_DONE) { - cerr << "execution failed: " << sqlite3_errmsg(m_db) << endl; - exit(1); - } - - sqlite3_reset(m_prepared_statement); - m_progress.Inc(); - } - - bool GetPoint(uint64_t id, double & lat, double & lng) const - { - bool found = true; - if (sqlite3_bind_int64(m_prepared_statement, 1, id) != SQLITE_OK) { - cerr << "bind1 failed: " << sqlite3_errmsg(m_db) << endl; - exit(1); - } - int rc = sqlite3_step(m_prepared_statement); - if (rc == SQLITE_DONE) - { - found = false; - } - else if (rc == SQLITE_ROW) - { - const void * data; - data = sqlite3_column_blob (m_prepared_statement, 0); - LatLon const &ll = *((LatLon const *)data); - lat = ll.lat; - lng = ll.lon; - } - else - { - cerr << "execution failed: " << sqlite3_errmsg(m_db) << endl; - exit(1); - } - - sqlite3_reset(m_prepared_statement); - return found; - } - -}; - template < BasePointStorage::EStorageMode ModeT > class RawFilePointStorage : public BasePointStorage { - #ifdef OMIM_OS_WINDOWS typedef FileReader FileReaderT; #else typedef MmapReader FileReaderT; #endif - typename std::conditional::type m_file; + typename conditional::type m_file; public: - RawFilePointStorage(string const & name) : BasePointStorage(name, 1000), m_file(name) {} + RawFilePointStorage(string const & name) : BasePointStorage(name), m_file(name) {} template - typename std::enable_if::type AddPoint(uint64_t id, double lat, double lng) + typename enable_if::type AddPoint(uint64_t id, double lat, double lng) { LatLon ll; ll.lat = lat; @@ -221,16 +89,10 @@ public: } template - typename std::enable_if::type GetPoint(uint64_t id, double & lat, double & lng) const + typename enable_if::type GetPoint(uint64_t id, double & lat, double & lng) const { - // I think, it's not good idea to write this ugly code. - // memcpy isn't to slow for that. - //#ifdef OMIM_OS_WINDOWS LatLon ll; m_file.Read(id * sizeof(ll), &ll, sizeof(ll)); - //#else - // LatLon const & ll = *reinterpret_cast(m_file.Data() + id * sizeof(ll)); - //#endif // assume that valid coordinate is not (0, 0) if (ll.lat != 0.0 || ll.lon != 0.0) @@ -251,25 +113,24 @@ public: template < BasePointStorage::EStorageMode ModeT > class RawFileShortPointStorage : public BasePointStorage { - #ifdef OMIM_OS_WINDOWS typedef FileReader FileReaderT; #else typedef MmapReader FileReaderT; #endif - typename std::conditional::type m_file; + typename conditional::type m_file; - double const m_precision = 10000000; + constexpr static double const kValueOrder = 1E+7; public: - RawFileShortPointStorage(string const & name) : BasePointStorage(name, 1000), m_file(name) {} + RawFileShortPointStorage(string const & name) : BasePointStorage(name), m_file(name) {} template - typename std::enable_if::type AddPoint(uint64_t id, double lat, double lng) + typename enable_if::type AddPoint(uint64_t id, double lat, double lng) { - int64_t const lat64 = lat * m_precision; - int64_t const lng64 = lng * m_precision; + int64_t const lat64 = lat * kValueOrder; + int64_t const lng64 = lng * kValueOrder; ShortLatLon ll; ll.lat = static_cast(lat64); @@ -284,22 +145,16 @@ public: } template - typename std::enable_if::type GetPoint(uint64_t id, double & lat, double & lng) const + typename enable_if::type GetPoint(uint64_t id, double & lat, double & lng) const { - // I think, it's not good idea to write this ugly code. - // memcpy isn't to slow for that. - //#ifdef OMIM_OS_WINDOWS ShortLatLon ll; m_file.Read(id * sizeof(ll), &ll, sizeof(ll)); - //#else - // LatLon const & ll = *reinterpret_cast(m_file.Data() + id * sizeof(ll)); - //#endif // assume that valid coordinate is not (0, 0) if (ll.lat != 0.0 || ll.lon != 0.0) { - lat = static_cast(ll.lat) / m_precision; - lng = static_cast(ll.lon) / m_precision; + lat = static_cast(ll.lat) / kValueOrder; + lng = static_cast(ll.lon) / kValueOrder; return true; } else @@ -314,15 +169,17 @@ public: template < BasePointStorage::EStorageMode ModeT > class RawMemShortPointStorage : public BasePointStorage { - typename std::conditional::type m_file; + typename conditional::type m_file; - double const m_precision = 10000000; + constexpr static double const kValueOrder = 1E+7; - vector m_data; + typedef ShortLatLon LonLatT; + + vector m_data; public: RawMemShortPointStorage(string const & name) - : BasePointStorage(name, 1000) + : BasePointStorage(name) , m_file(name) , m_data((size_t)0xFFFFFFFF) { @@ -338,21 +195,21 @@ public: void InitStorage(EnableIf) { - m_file.Read(0, m_data.data(), m_data.size()); + m_file.Read(0, m_data.data(), m_data.size() * sizeof(LonLatT)); } void DoneStorage(EnableIf) { - m_file.Write(m_data.data(), m_data.size()); + m_file.Write(m_data.data(), m_data.size() * sizeof(LonLatT)); } void DoneStorage(EnableIf) {} template - typename std::enable_if::type AddPoint(uint64_t id, double lat, double lng) + typename enable_if::type AddPoint(uint64_t id, double lat, double lng) { - int64_t const lat64 = lat * m_precision; - int64_t const lng64 = lng * m_precision; + int64_t const lat64 = lat * kValueOrder; + int64_t const lng64 = lng * kValueOrder; ShortLatLon & ll = m_data[id]; ll.lat = static_cast(lat64); @@ -364,14 +221,14 @@ public: } template - typename std::enable_if::type GetPoint(uint64_t id, double & lat, double & lng) const + typename enable_if::type GetPoint(uint64_t id, double & lat, double & lng) const { ShortLatLon const & ll = m_data[id]; // assume that valid coordinate is not (0, 0) if (ll.lat != 0.0 || ll.lon != 0.0) { - lat = static_cast(ll.lat) / m_precision; - lng = static_cast(ll.lon) / m_precision; + lat = static_cast(ll.lat) / kValueOrder; + lng = static_cast(ll.lon) / kValueOrder; return true; } else @@ -382,13 +239,10 @@ public: } }; - template < BasePointStorage::EStorageMode ModeT > class MapFilePointStorage : public BasePointStorage { - typename std::conditional::type m_file; - - + typename conditional::type m_file; typedef unordered_map > ContainerT; ContainerT m_map; @@ -412,7 +266,7 @@ public: LatLonPos ll; m_file.Read(pos, &ll, sizeof(ll)); - (void)m_map.insert(make_pair(ll.pos, make_pair(ll.lat, ll.lon))); + m_map.emplace(make_pair(ll.pos, make_pair(ll.lat, ll.lon))); pos += sizeof(ll); } @@ -434,27 +288,22 @@ public: bool GetPoint(uint64_t id, double & lat, double & lng) const { auto i = m_map.find(id); - if (i != m_map.end()) - { - lat = i->second.first; - lng = i->second.second; - return true; - } - return false; + if (i == m_map.end()) + return false; + lat = i->second.first; + lng = i->second.second; + return true; } - }; template < BasePointStorage::EStorageMode ModeT > class MapFileShortPointStorage : public BasePointStorage { - typename std::conditional::type m_file; - - + typename conditional::type m_file; typedef unordered_map > ContainerT; ContainerT m_map; - double const m_precision = 10000000; + constexpr static double const kValueOrder = 1E+7; public: MapFileShortPointStorage(string const & name) : BasePointStorage(name, 10000), m_file(name+".short") @@ -476,7 +325,7 @@ public: ShortLatLonPos ll; m_file.Read(pos, &ll, sizeof(ll)); - (void)m_map.insert(make_pair(ll.pos, make_pair(ll.lat, ll.lon))); + m_map.emplace(make_pair(ll.pos, make_pair(ll.lat, ll.lon))); pos += sizeof(ll); } @@ -487,10 +336,8 @@ public: void AddPoint(uint64_t id, double lat, double lng) { - int64_t const lat64 = lat * m_precision; - int64_t const lng64 = lng * m_precision; - - + int64_t const lat64 = lat * kValueOrder; + int64_t const lng64 = lng * kValueOrder; ShortLatLonPos ll; ll.pos = id; @@ -506,13 +353,11 @@ public: bool GetPoint(uint64_t id, double & lat, double & lng) const { auto i = m_map.find(id); - if (i != m_map.end()) - { - lat = static_cast(i->second.first) / m_precision; - lng = static_cast(i->second.second) / m_precision; - return true; - } - return false; + if (i == m_map.end()) + return false; + lat = static_cast(i->second.first) / kValueOrder; + lng = static_cast(i->second.second) / kValueOrder; + return true; } }; diff --git a/generator/xml_element.cpp b/generator/xml_element.cpp index 6c792544eb..7ff351f333 100644 --- a/generator/xml_element.cpp +++ b/generator/xml_element.cpp @@ -18,6 +18,29 @@ void XMLElement::AddKV(string const & k, string const & v) e.parent = this; } +void XMLElement::AddND(uint64_t ref) +{ + childs.push_back(XMLElement()); + XMLElement & e = childs.back(); + + e.tagKey = ET_ND; + e.ref = ref; + e.parent = this; +} + +void XMLElement::AddMEMBER(uint64_t ref, string const & type, string const & role) +{ + childs.push_back(XMLElement()); + XMLElement & e = childs.back(); + + e.tagKey = ET_MEMBER; + e.ref = ref; + e.type = type; + e.role = role; + e.parent = this; +} + + void BaseOSMParser::AddAttr(string const & key, string const & value) { if (m_current) @@ -101,46 +124,3 @@ void BaseOSMParser::Pop(string const &) (*m_current) = XMLElement(); } } - -namespace -{ - struct StdinReader - { - uint64_t Read(char * buffer, uint64_t bufferSize) - { - return fread(buffer, sizeof(char), bufferSize, stdin); - } - }; - - struct FileReader - { - FILE * m_file; - - FileReader(string const & filename) - { - m_file = fopen(filename.c_str(), "rb"); - } - - ~FileReader() - { - fclose(m_file); - } - - uint64_t Read(char * buffer, uint64_t bufferSize) - { - return fread(buffer, sizeof(char), bufferSize, m_file); - } - }; -} - -void ParseXMLFromStdIn(BaseOSMParser & parser) -{ - StdinReader reader; - (void)ParseXMLSequence(reader, parser); -} - -void ParseXMLFromFile(BaseOSMParser & parser, string const & osmFileName) -{ - FileReader reader(osmFileName); - (void)ParseXMLSequence(reader, parser); -} diff --git a/generator/xml_element.hpp b/generator/xml_element.hpp index cd0ade8991..d4b22fe4cc 100644 --- a/generator/xml_element.hpp +++ b/generator/xml_element.hpp @@ -34,6 +34,9 @@ struct XMLElement vector childs; void AddKV(string const & k, string const & v); + void AddND(uint64_t ref); + void AddMEMBER(uint64_t ref, string const & type, string const & role); + }; class BaseOSMParser @@ -53,10 +56,8 @@ public: void Pop(string const &); void CharData(string const &) {} + virtual void EmitElement(XMLElement * p) = 0; + protected: bool MatchTag(string const & tagName, XMLElement::ETag & tagKey); - virtual void EmitElement(XMLElement * p) = 0; }; - -void ParseXMLFromStdIn(BaseOSMParser & parser); -void ParseXMLFromFile(BaseOSMParser & parser, string const & osmFileName); diff --git a/std/type_traits.hpp b/std/type_traits.hpp index 769bbf22a6..659b6ba275 100644 --- a/std/type_traits.hpp +++ b/std/type_traits.hpp @@ -14,18 +14,18 @@ #endif #endif -#include -#include +#include -using boost::enable_if; +using std::enable_if; +using std::conditional; -using boost::is_same; -using boost::make_signed; -using boost::make_unsigned; -using boost::is_signed; -using boost::is_unsigned; -using boost::is_floating_point; -using boost::is_integral; +using std::is_same; +using std::make_signed; +using std::make_unsigned; +using std::is_signed; +using std::is_unsigned; +using std::is_floating_point; +using std::is_integral; #ifdef DEBUG_NEW #define new DEBUG_NEW diff --git a/tools/unix/planet.sh b/tools/unix/planet.sh index 879e39fa75..c9fe2b55e3 100755 --- a/tools/unix/planet.sh +++ b/tools/unix/planet.sh @@ -108,11 +108,11 @@ function merge_coasts() { log "TIMEMARK" "Filter coastlines done" # Preprocess coastlines to separate intermediate directory log "TIMEMARK" "Generate coastlines intermediate" - $CONVERT_TOOL $COASTS_FILE | $GENERATOR_TOOL -intermediate_data_path=$INTCOASTSDIR -node_storage=map -preprocess_xml + $GENERATOR_TOOL -intermediate_data_path=$INTCOASTSDIR -node_storage=map -preprocess --osm_file_type=o5m --osm_file_name=$COASTS_FILE log "TIMEMARK" "Generate coastlines intermediate done" # Generate temporary coastlines file in the coasts intermediate dir log "TIMEMARK" "Generate coastlines" - $CONVERT_TOOL $COASTS_FILE | $GENERATOR_TOOL -intermediate_data_path=$INTCOASTSDIR -node_storage=map -make_coasts -fail_on_coasts=$FAIL_ON_COASTS + $GENERATOR_TOOL -intermediate_data_path=$INTCOASTSDIR -node_storage=map -make_coasts -fail_on_coasts=$FAIL_ON_COASTS --osm_file_type=o5m --osm_file_name=$COASTS_FILE log "TIMEMARK" "Generate coastlines done" } @@ -163,20 +163,20 @@ fi # make a working copy of generated coastlines file cp $INTCOASTSDIR/WorldCoasts.mwm.tmp $INTDIR/WorldCoasts.mwm.tmp -NODE_STORAGE=raw +NODE_STORAGE=mem if [[ $1 == "--generate" || $1 == "--full" ]]; then log "TIMEMARK" "Generate intermediate data" # 1st pass, run in parallel - preprocess whole planet to speed up generation if all coastlines are correct - $CONVERT_TOOL $PLANET_FILE | $GENERATOR_TOOL -intermediate_data_path=$INTDIR -node_storage=$NODE_STORAGE -preprocess_xml + $GENERATOR_TOOL -intermediate_data_path=$INTDIR -node_storage=$NODE_STORAGE -preprocess --osm_file_type=o5m --osm_file_name=$PLANET_FILE log "TIMEMARK" "Generate intermediate data done" fi if [[ $1 == "--generate" || $1 == "--continue" || $1 == "--full" ]]; then # 2nd pass - paralleled in the code log "TIMEMARK" "Generate features" - $CONVERT_TOOL $PLANET_FILE | $GENERATOR_TOOL -intermediate_data_path=$INTDIR \ - -node_storage=$NODE_STORAGE -split_by_polygons \ + $GENERATOR_TOOL -intermediate_data_path=$INTDIR \ + --osm_file_type=o5m --osm_file_name=$PLANET_FILE -node_storage=$NODE_STORAGE -split_by_polygons \ -generate_features -generate_world \ -data_path=$DATA_PATH -emit_coasts log "TIMEMARK" "Generate features done" diff --git a/tools/unix/split_planet_by_polygons.sh b/tools/unix/split_planet_by_polygons.sh index 8f78f13338..b9bafaebda 100755 --- a/tools/unix/split_planet_by_polygons.sh +++ b/tools/unix/split_planet_by_polygons.sh @@ -21,4 +21,8 @@ mkdir $OUT_DIR || true $GENERATOR_TOOL -export_poly_path $POLY_FILES_PATH -ls $POLY_FILES_PATH | parallel -t -v "$OSMCONVERT_TOOL $PLANET_FILE --hash-memory=2000 -B=$POLY_FILES_PATH/{} --complex-ways --out-pbf -o=$OUT_DIR/{.}.pbf" +EXT=.poly +NUM_INSTANCES=8 + +COUNTRY_LIST=${COUNTRY_LIST-$(ls -1 $POLY_FILES_PATH/*$EXT | xargs -d "\n" basename -s $EXT)} +echo "$COUNTRY_LIST" | xargs -d "\n" -P $NUM_INSTANCES -I % $OSMCONVERT_TOOL $PLANET_FILE --hash-memory=2000 -B=$POLY_FILES_PATH/%.poly --complex-ways --out-pbf -o=$OUT_DIR/%.pbf &>~/split_planet_osmconvert.log