forked from organicmaps/organicmaps-tmp
Added generator_tool option for set osm data type [xml or o5m]
This commit is contained in:
Sergey Yershov
Alex Zolotarev
parent
351a5deb33
commit
3363aba81c
23 changed files with 1402 additions and 729 deletions
|
|
@ -2,7 +2,8 @@
|
|||
|
||||
TEMPLATE = subdirs
|
||||
|
||||
SUBDIRS = freetype fribidi zlib bzip2 jansson tomcrypt protobuf osrm expat
|
||||
SUBDIRS = freetype fribidi zlib bzip2 jansson tomcrypt protobuf osrm expat \
|
||||
o5mreader
|
||||
|
||||
!iphone*:!tizen*:!android* {
|
||||
SUBDIRS += gflags \
|
||||
|
|
|
|||
516
3party/o5mreader/o5mreader.c
Normal file
516
3party/o5mreader/o5mreader.c
Normal file
|
|
@ -0,0 +1,516 @@
|
|||
#include "o5mreader.h"
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <errno.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
|
||||
#define STR_PAIR_TABLE_SIZE 15000
|
||||
#define STR_PAIR_STRING_SIZE 256
|
||||
|
||||
void o5mreader_setError(O5mreader *pReader,int code, const char* message);
|
||||
void o5mreader_setNoError(O5mreader *pReader);
|
||||
|
||||
O5mreaderRet o5mreader_readUInt(O5mreader *pReader, uint64_t *ret) {
|
||||
uint8_t b;
|
||||
uint8_t i = 0;
|
||||
*ret = 0LL;
|
||||
|
||||
do {
|
||||
if ( fread(&b,1,1,pReader->f) == 0 ) {
|
||||
o5mreader_setError(pReader,
|
||||
O5MREADER_ERR_CODE_UNEXPECTED_END_OF_FILE,
|
||||
NULL
|
||||
);
|
||||
return O5MREADER_RET_ERR;
|
||||
}
|
||||
*ret |= (long long)(b & 0x7f) << (i++ * 7);
|
||||
} while ( b & 0x80 );
|
||||
|
||||
o5mreader_setNoError(pReader);
|
||||
|
||||
return O5MREADER_RET_OK;
|
||||
}
|
||||
|
||||
O5mreaderRet o5mreader_readInt(O5mreader *pReader, uint64_t *ret) {
|
||||
if ( o5mreader_readUInt(pReader, ret) == O5MREADER_RET_ERR )
|
||||
return O5MREADER_RET_ERR;
|
||||
*ret = *ret & 1
|
||||
? -(int64_t)(*ret >> 1) - 1
|
||||
: (int64_t)(*ret >> 1);
|
||||
return O5MREADER_RET_OK;
|
||||
}
|
||||
|
||||
|
||||
O5mreaderRet o5mreader_readStrPair(O5mreader *pReader, char **tagpair, int single) {
|
||||
static char buffer[1024];
|
||||
char* pBuf;
|
||||
static uint64_t pointer = 0;
|
||||
int length;
|
||||
char byte;
|
||||
uint64_t key;
|
||||
int i;
|
||||
|
||||
if ( o5mreader_readUInt(pReader,&key) == O5MREADER_RET_ERR ) {
|
||||
return O5MREADER_RET_ERR;
|
||||
}
|
||||
|
||||
if ( key ) {
|
||||
*tagpair = pReader->strPairTable[(pointer+15000-key)%15000];
|
||||
return key;
|
||||
}
|
||||
else {
|
||||
pBuf = buffer;
|
||||
for ( i=0; i<(single?1:2); i++ ) {
|
||||
do {
|
||||
if ( fread(pBuf,1,1,pReader->f) == 0 ) {
|
||||
o5mreader_setError(pReader,
|
||||
O5MREADER_ERR_CODE_UNEXPECTED_END_OF_FILE,
|
||||
NULL
|
||||
);
|
||||
return O5MREADER_RET_ERR;
|
||||
}
|
||||
} while ( *(pBuf++) );
|
||||
}
|
||||
|
||||
length = strlen(buffer) + (single ? 1 : strlen(buffer+strlen(buffer) +1) + 2);
|
||||
|
||||
if ( length <= 252 ) {
|
||||
*tagpair = pReader->strPairTable[(pointer+15000)%15000];
|
||||
memcpy(pReader->strPairTable[((pointer++)+15000)%15000],buffer,length);
|
||||
}
|
||||
else {
|
||||
*tagpair = buffer;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return O5MREADER_RET_OK;
|
||||
}
|
||||
|
||||
O5mreaderRet o5mreader_reset(O5mreader *pReader) {
|
||||
pReader->nodeId = pReader->wayId = pReader->wayNodeId = pReader->relId = pReader->nodeRefId = pReader->wayRefId = pReader->relRefId = 0;
|
||||
pReader->lon = pReader->lat = 0;
|
||||
pReader->offset = 0;
|
||||
pReader->canIterateTags = pReader->canIterateNds = pReader->canIterateRefs = 0;
|
||||
return O5MREADER_RET_OK;
|
||||
}
|
||||
|
||||
O5mreaderRet o5mreader_open(O5mreader **ppReader,FILE* f) {
|
||||
uint8_t byte;
|
||||
int i;
|
||||
*ppReader = malloc(sizeof(O5mreader));
|
||||
if ( !(*ppReader) ) {
|
||||
return O5MREADER_RET_ERR;
|
||||
}
|
||||
(*ppReader)->errMsg = NULL;
|
||||
(*ppReader)->f = f;
|
||||
if ( fread(&byte,1,1,(*ppReader)->f) == 0 ) {
|
||||
o5mreader_setError(*ppReader,
|
||||
O5MREADER_ERR_CODE_UNEXPECTED_END_OF_FILE,
|
||||
NULL
|
||||
);
|
||||
return O5MREADER_RET_ERR;
|
||||
}
|
||||
if ( byte != O5MREADER_DS_RESET ) {
|
||||
o5mreader_setError(*ppReader,
|
||||
O5MREADER_ERR_CODE_FILE_HAS_WRONG_START,
|
||||
NULL
|
||||
);
|
||||
return O5MREADER_RET_ERR;
|
||||
}
|
||||
|
||||
o5mreader_reset(*ppReader);
|
||||
|
||||
(*ppReader)->strPairTable = malloc(STR_PAIR_TABLE_SIZE*sizeof(char*));
|
||||
if ( (*ppReader)->strPairTable == 0 ) {
|
||||
o5mreader_setError(*ppReader,
|
||||
O5MREADER_ERR_CODE_MEMORY_ERROR,
|
||||
NULL
|
||||
);
|
||||
return O5MREADER_RET_ERR;
|
||||
}
|
||||
for ( i = 0; i < STR_PAIR_TABLE_SIZE; ++i ) {
|
||||
(*ppReader)->strPairTable[i] = malloc(sizeof(char)*STR_PAIR_STRING_SIZE);
|
||||
if ( (*ppReader)->strPairTable[i] == 0 ) {
|
||||
o5mreader_setError(*ppReader,
|
||||
O5MREADER_ERR_CODE_MEMORY_ERROR,
|
||||
NULL
|
||||
);
|
||||
return O5MREADER_RET_ERR;
|
||||
}
|
||||
}
|
||||
|
||||
o5mreader_setNoError(*ppReader);
|
||||
return O5MREADER_RET_OK;
|
||||
}
|
||||
|
||||
void o5mreader_close(O5mreader *pReader) {
|
||||
int i;
|
||||
if ( pReader ) {
|
||||
if ( pReader->strPairTable ) {
|
||||
for ( i = 0; i < STR_PAIR_TABLE_SIZE; ++i )
|
||||
if ( pReader->strPairTable[i] )
|
||||
free(pReader->strPairTable[i]);
|
||||
free(pReader->strPairTable);
|
||||
}
|
||||
o5mreader_setNoError(pReader);
|
||||
free(pReader);
|
||||
}
|
||||
}
|
||||
|
||||
const char* o5mreader_strerror(int errCode) {
|
||||
switch ( errCode ) {
|
||||
case O5MREADER_ERR_CODE_FILE_HAS_WRONG_START:
|
||||
return "'0xFF' isn't first byte of file.";
|
||||
case O5MREADER_ERR_CODE_MEMORY_ERROR:
|
||||
return "Memory error.";
|
||||
case O5MREADER_ERR_CODE_UNEXPECTED_END_OF_FILE:
|
||||
return "Unexpected end of file.";
|
||||
case O5MREADER_ERR_CODE_CAN_NOT_ITERATE_TAGS_HERE:
|
||||
return "Tags iteration is not allowed here.";
|
||||
case O5MREADER_ERR_CODE_CAN_NOT_ITERATE_NDS_HERE:
|
||||
return "Nodes iteration is not allowed here.";
|
||||
case O5MREADER_ERR_CODE_CAN_NOT_ITERATE_REFS_HERE:
|
||||
return "References iteration is not allowed here.";
|
||||
default:
|
||||
return "Unknown error code";
|
||||
}
|
||||
}
|
||||
|
||||
void o5mreader_setError(O5mreader *pReader,int code, const char* message) {
|
||||
pReader->errCode = code;
|
||||
if ( pReader->errMsg ) {
|
||||
free(pReader->errMsg);
|
||||
}
|
||||
if ( message ) {
|
||||
pReader->errMsg = malloc(strlen(message)+1);
|
||||
strcpy(pReader->errMsg,message);
|
||||
}
|
||||
}
|
||||
|
||||
void o5mreader_setNoError(O5mreader *pReader) {
|
||||
pReader->errCode = O5MREADER_ERR_CODE_OK;
|
||||
if ( pReader->errMsg ) {
|
||||
free(pReader->errMsg);
|
||||
}
|
||||
pReader->errMsg = NULL;
|
||||
}
|
||||
|
||||
O5mreaderIterateRet o5mreader_iterateDataSet(O5mreader *pReader, O5mreaderDataset* ds) {
|
||||
for (;;) {
|
||||
if ( pReader->offset ) {
|
||||
if ( o5mreader_skipTags(pReader) == O5MREADER_ITERATE_RET_ERR )
|
||||
return O5MREADER_ITERATE_RET_ERR;
|
||||
|
||||
fseek(
|
||||
pReader->f,
|
||||
(pReader->current - ftell(pReader->f)) + pReader->offset,
|
||||
SEEK_CUR
|
||||
);
|
||||
|
||||
pReader->offset = 0;
|
||||
}
|
||||
|
||||
if ( fread(&(ds->type),1,1,pReader->f) == 0 ) {
|
||||
o5mreader_setError(pReader,
|
||||
O5MREADER_ERR_CODE_UNEXPECTED_END_OF_FILE,
|
||||
NULL
|
||||
);
|
||||
return O5MREADER_ITERATE_RET_ERR;
|
||||
}
|
||||
|
||||
if ( O5MREADER_DS_END == ds->type )
|
||||
return O5MREADER_ITERATE_RET_DONE;
|
||||
|
||||
if ( O5MREADER_DS_RESET == ds->type ) {
|
||||
o5mreader_reset(pReader);
|
||||
}
|
||||
else if ( 0xf0 == ds->type ) {}
|
||||
else {
|
||||
if ( o5mreader_readUInt(pReader,&pReader->offset) == O5MREADER_RET_ERR ) {
|
||||
return O5MREADER_ITERATE_RET_ERR;
|
||||
}
|
||||
pReader->current = ftell(pReader->f);
|
||||
|
||||
switch ( ds->type ) {
|
||||
case O5MREADER_DS_NODE:
|
||||
return o5mreader_readNode(pReader, ds);
|
||||
case O5MREADER_DS_WAY:
|
||||
return o5mreader_readWay(pReader, ds);
|
||||
case O5MREADER_DS_REL:
|
||||
return o5mreader_readRel(pReader, ds);
|
||||
/*
|
||||
case O5MREADER_DS_BBOX:
|
||||
case O5MREADER_DS_TSTAMP:
|
||||
case O5MREADER_DS_HEADER:
|
||||
case O5MREADER_DS_SYNC:
|
||||
case O5MREADER_DS_JUMP:
|
||||
default:
|
||||
break;
|
||||
*/
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
int o5mreader_thereAreNoMoreData(O5mreader *pReader) {
|
||||
return (int)((pReader->current - ftell(pReader->f)) + pReader->offset) <= 0;
|
||||
}
|
||||
|
||||
O5mreaderIterateRet o5mreader_readVersion(O5mreader *pReader, O5mreaderDataset* ds) {
|
||||
uint64_t tmp;
|
||||
if ( o5mreader_readUInt(pReader,&tmp) == O5MREADER_ITERATE_RET_ERR ) {
|
||||
return O5MREADER_ITERATE_RET_ERR;
|
||||
}
|
||||
ds->version = tmp;
|
||||
if ( tmp ) {
|
||||
if ( o5mreader_readUInt(pReader,&tmp) == O5MREADER_ITERATE_RET_ERR ) {
|
||||
return O5MREADER_ITERATE_RET_ERR;
|
||||
}
|
||||
|
||||
if ( o5mreader_readInt(pReader,&tmp) == O5MREADER_ITERATE_RET_ERR ) {
|
||||
return O5MREADER_ITERATE_RET_ERR;
|
||||
}
|
||||
|
||||
if ( o5mreader_thereAreNoMoreData(pReader) )
|
||||
return O5MREADER_ITERATE_RET_DONE;
|
||||
|
||||
if ( o5mreader_readStrPair(pReader,&pReader->tagPair,0) == O5MREADER_ITERATE_RET_ERR ) {
|
||||
return O5MREADER_ITERATE_RET_ERR;
|
||||
}
|
||||
}
|
||||
|
||||
if ( o5mreader_thereAreNoMoreData(pReader) )
|
||||
return O5MREADER_ITERATE_RET_DONE;
|
||||
|
||||
return O5MREADER_ITERATE_RET_NEXT;
|
||||
}
|
||||
|
||||
O5mreaderIterateRet o5mreader_iterateTags(O5mreader *pReader, char** pKey, char** pVal) {
|
||||
if ( pReader->canIterateRefs ) {
|
||||
if ( o5mreader_skipRefs(pReader) == O5MREADER_ITERATE_RET_ERR )
|
||||
return O5MREADER_ITERATE_RET_ERR;
|
||||
}
|
||||
if ( pReader->canIterateNds ) {
|
||||
if ( o5mreader_skipNds(pReader) == O5MREADER_ITERATE_RET_ERR )
|
||||
return O5MREADER_ITERATE_RET_ERR;
|
||||
}
|
||||
if ( !pReader->canIterateTags ) {
|
||||
o5mreader_setError(pReader,
|
||||
O5MREADER_ERR_CODE_CAN_NOT_ITERATE_TAGS_HERE,
|
||||
NULL
|
||||
);
|
||||
return O5MREADER_ITERATE_RET_ERR;
|
||||
}
|
||||
if ( o5mreader_thereAreNoMoreData(pReader) ) {
|
||||
pReader->canIterateTags = 0;
|
||||
return O5MREADER_ITERATE_RET_DONE;
|
||||
}
|
||||
|
||||
if ( o5mreader_readStrPair(pReader,&pReader->tagPair,0) == O5MREADER_RET_ERR ) {
|
||||
return O5MREADER_ITERATE_RET_ERR;
|
||||
}
|
||||
if ( pKey )
|
||||
*pKey = pReader->tagPair;
|
||||
if ( pVal )
|
||||
*pVal = pReader->tagPair + strlen(pReader->tagPair) + 1;
|
||||
|
||||
return O5MREADER_ITERATE_RET_NEXT;
|
||||
}
|
||||
|
||||
O5mreaderIterateRet o5mreader_skipTags(O5mreader *pReader) {
|
||||
int ret;
|
||||
if ( pReader->canIterateTags ) {
|
||||
while ( O5MREADER_ITERATE_RET_NEXT == (ret = o5mreader_iterateTags(pReader, NULL, NULL)) );
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
O5mreaderIterateRet o5mreader_readNode(O5mreader *pReader, O5mreaderDataset* ds) {
|
||||
uint64_t nodeId;
|
||||
int64_t lon,lat;
|
||||
if ( o5mreader_readInt(pReader,&nodeId) == O5MREADER_RET_ERR )
|
||||
return O5MREADER_ITERATE_RET_ERR;
|
||||
|
||||
pReader->canIterateRefs = 0;
|
||||
pReader->canIterateNds = 0;
|
||||
pReader->canIterateTags = 1;
|
||||
|
||||
pReader->nodeId += nodeId;
|
||||
ds->id = pReader->nodeId;
|
||||
|
||||
|
||||
if ( o5mreader_readVersion(pReader, ds) == O5MREADER_ITERATE_RET_DONE ) {
|
||||
ds->isEmpty = 1;
|
||||
return O5MREADER_ITERATE_RET_NEXT;
|
||||
}
|
||||
else
|
||||
ds->isEmpty = 0;
|
||||
|
||||
if ( o5mreader_thereAreNoMoreData(pReader) ) {
|
||||
return O5MREADER_ITERATE_RET_NEXT;
|
||||
}
|
||||
|
||||
if ( o5mreader_readInt(pReader,&lon) == O5MREADER_RET_ERR )
|
||||
return O5MREADER_ITERATE_RET_ERR;
|
||||
pReader->lon += (int32_t)lon;
|
||||
|
||||
if ( o5mreader_readInt(pReader,&lat) == O5MREADER_RET_ERR ) {
|
||||
return O5MREADER_ITERATE_RET_ERR;
|
||||
}
|
||||
pReader->lat += (int32_t)lat;
|
||||
|
||||
ds->lon = pReader->lon;
|
||||
ds->lat = pReader->lat;
|
||||
|
||||
return O5MREADER_ITERATE_RET_NEXT;
|
||||
}
|
||||
|
||||
O5mreaderIterateRet o5mreader_iterateNds(O5mreader *pReader, uint64_t *nodeId) {
|
||||
int64_t wayNodeId;
|
||||
|
||||
if ( !pReader->canIterateNds ) {
|
||||
o5mreader_setError(pReader,
|
||||
O5MREADER_ERR_CODE_CAN_NOT_ITERATE_NDS_HERE,
|
||||
NULL
|
||||
);
|
||||
return O5MREADER_ITERATE_RET_ERR;
|
||||
}
|
||||
if ( ftell(pReader->f) >= pReader->offsetNd ) {
|
||||
pReader->canIterateNds = 0;
|
||||
pReader->canIterateTags = 1;
|
||||
pReader->canIterateRefs = 0;
|
||||
return O5MREADER_ITERATE_RET_DONE;
|
||||
}
|
||||
|
||||
if ( o5mreader_readInt(pReader,&wayNodeId) == O5MREADER_RET_ERR )
|
||||
return O5MREADER_ITERATE_RET_ERR;
|
||||
|
||||
pReader->wayNodeId += wayNodeId;
|
||||
|
||||
if ( nodeId )
|
||||
*nodeId = pReader->wayNodeId;
|
||||
|
||||
return O5MREADER_ITERATE_RET_NEXT;
|
||||
}
|
||||
|
||||
O5mreaderIterateRet o5mreader_skipNds(O5mreader *pReader) {
|
||||
uint8_t ret;
|
||||
while ( pReader->canIterateNds &&
|
||||
O5MREADER_ITERATE_RET_NEXT == (ret = o5mreader_iterateNds(pReader, NULL)) );
|
||||
return ret;
|
||||
}
|
||||
|
||||
O5mreaderIterateRet o5mreader_readWay(O5mreader *pReader, O5mreaderDataset* ds) {
|
||||
int64_t wayId;
|
||||
if ( o5mreader_readInt(pReader,&wayId) == O5MREADER_RET_ERR)
|
||||
return O5MREADER_ITERATE_RET_ERR;
|
||||
|
||||
pReader->wayId += wayId;
|
||||
ds->id = pReader->wayId;
|
||||
if ( o5mreader_readVersion(pReader, ds) == O5MREADER_ITERATE_RET_DONE ) {
|
||||
ds->isEmpty = 1;
|
||||
return O5MREADER_ITERATE_RET_NEXT;
|
||||
}
|
||||
else
|
||||
ds->isEmpty = 0;
|
||||
if ( o5mreader_readUInt(pReader,&pReader->offsetNd) == O5MREADER_RET_ERR ) {
|
||||
return O5MREADER_ITERATE_RET_ERR;
|
||||
}
|
||||
pReader->offsetNd += ftell(pReader->f);
|
||||
pReader->canIterateRefs = 0;
|
||||
pReader->canIterateNds = 1;
|
||||
pReader->canIterateTags = 0;
|
||||
return O5MREADER_ITERATE_RET_NEXT;
|
||||
}
|
||||
|
||||
O5mreaderIterateRet o5mreader_iterateRefs(O5mreader *pReader, uint64_t *refId, uint8_t *type, char** pRole) {
|
||||
int64_t relRefId;
|
||||
|
||||
if ( !pReader->canIterateRefs ) {
|
||||
o5mreader_setError(pReader,
|
||||
O5MREADER_ERR_CODE_CAN_NOT_ITERATE_REFS_HERE,
|
||||
NULL
|
||||
);
|
||||
return O5MREADER_ITERATE_RET_ERR;
|
||||
}
|
||||
if ( ftell(pReader->f) >= pReader->offsetRf ) {
|
||||
pReader->canIterateNds = 0;
|
||||
pReader->canIterateTags = 1;
|
||||
pReader->canIterateRefs = 0;
|
||||
return O5MREADER_ITERATE_RET_DONE;
|
||||
}
|
||||
|
||||
if ( o5mreader_readInt(pReader, &relRefId) == O5MREADER_RET_ERR )
|
||||
return O5MREADER_ITERATE_RET_ERR;
|
||||
|
||||
|
||||
//fread(_,1,1,pReader->f);
|
||||
|
||||
if ( o5mreader_readStrPair(pReader, &pReader->tagPair,1) == O5MREADER_RET_ERR ) {
|
||||
return O5MREADER_ITERATE_RET_ERR;
|
||||
}
|
||||
|
||||
switch( pReader->tagPair[0] ) {
|
||||
case '0':
|
||||
if ( type )
|
||||
*type = O5MREADER_DS_NODE;
|
||||
pReader->nodeRefId += relRefId;
|
||||
if ( refId )
|
||||
*refId = pReader->nodeRefId;
|
||||
break;
|
||||
case '1':
|
||||
if ( type )
|
||||
*type = O5MREADER_DS_WAY;
|
||||
pReader->wayRefId += relRefId;
|
||||
if ( refId )
|
||||
*refId = pReader->wayRefId;
|
||||
break;
|
||||
case '2':
|
||||
if ( type )
|
||||
*type = O5MREADER_DS_REL;
|
||||
pReader->relRefId += relRefId;
|
||||
if ( refId )
|
||||
*refId = pReader->relRefId;
|
||||
break;
|
||||
}
|
||||
|
||||
if ( pRole ) {
|
||||
*pRole = pReader->tagPair + 1;
|
||||
}
|
||||
|
||||
return O5MREADER_ITERATE_RET_NEXT;
|
||||
}
|
||||
|
||||
O5mreaderIterateRet o5mreader_skipRefs(O5mreader *pReader) {
|
||||
uint8_t ret;
|
||||
while ( pReader->canIterateRefs &&
|
||||
O5MREADER_ITERATE_RET_NEXT == (ret = o5mreader_iterateRefs(pReader, NULL, NULL, NULL)) );
|
||||
return ret;
|
||||
}
|
||||
|
||||
O5mreaderIterateRet o5mreader_readRel(O5mreader *pReader, O5mreaderDataset* ds) {
|
||||
int64_t relId;
|
||||
if ( o5mreader_readInt(pReader,&relId) == O5MREADER_RET_ERR )
|
||||
return O5MREADER_ITERATE_RET_ERR;
|
||||
pReader->relId += relId;
|
||||
ds->id = pReader->relId;
|
||||
if ( o5mreader_readVersion(pReader,ds) == O5MREADER_ITERATE_RET_DONE ) {
|
||||
ds->isEmpty = 1;
|
||||
return O5MREADER_ITERATE_RET_NEXT;
|
||||
}
|
||||
else
|
||||
ds->isEmpty = 0;
|
||||
o5mreader_readUInt(pReader,&pReader->offsetRf);
|
||||
pReader->offsetRf += ftell(pReader->f);
|
||||
|
||||
pReader->canIterateRefs = 1;
|
||||
pReader->canIterateNds = 0;
|
||||
pReader->canIterateTags = 0;
|
||||
return O5MREADER_ITERATE_RET_NEXT;
|
||||
}
|
||||
93
3party/o5mreader/o5mreader.h
Normal file
93
3party/o5mreader/o5mreader.h
Normal file
|
|
@ -0,0 +1,93 @@
|
|||
#ifndef __O5MREADER__H__
|
||||
#define __O5MREADER__H__
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
|
||||
|
||||
#define O5MREADER_RET_OK 1
|
||||
#define O5MREADER_RET_ERR 0
|
||||
|
||||
#define O5MREADER_ITERATE_RET_DONE 0
|
||||
#define O5MREADER_ITERATE_RET_ERR -1
|
||||
#define O5MREADER_ITERATE_RET_NEXT 1
|
||||
|
||||
#define O5MREADER_DS_END 0xfe
|
||||
#define O5MREADER_DS_NODE 0x10
|
||||
#define O5MREADER_DS_WAY 0x11
|
||||
#define O5MREADER_DS_REL 0x12
|
||||
#define O5MREADER_DS_BBOX 0xdb
|
||||
#define O5MREADER_DS_TSTAMP 0xdc
|
||||
#define O5MREADER_DS_HEADER 0xe0
|
||||
#define O5MREADER_DS_SYNC 0xee
|
||||
#define O5MREADER_DS_JUMP 0xef
|
||||
#define O5MREADER_DS_RESET 0xff
|
||||
|
||||
|
||||
#define O5MREADER_ERR_CODE_OK 0
|
||||
#define O5MREADER_ERR_CODE_FILE_HAS_WRONG_START 1
|
||||
#define O5MREADER_ERR_CODE_UNEXPECTED_END_OF_FILE 2
|
||||
#define O5MREADER_ERR_CODE_MEMORY_ERROR 3
|
||||
#define O5MREADER_ERR_CODE_CAN_NOT_ITERATE_TAGS_HERE 4
|
||||
#define O5MREADER_ERR_CODE_CAN_NOT_ITERATE_NDS_HERE 5
|
||||
#define O5MREADER_ERR_CODE_CAN_NOT_ITERATE_REFS_HERE 6
|
||||
|
||||
typedef int O5mreaderRet;
|
||||
typedef int O5mreaderIterateRet;
|
||||
|
||||
typedef struct {
|
||||
int errCode;
|
||||
char* errMsg;
|
||||
FILE *f;
|
||||
uint64_t offset;
|
||||
uint64_t offsetNd;
|
||||
uint64_t offsetRf;
|
||||
uint64_t current;
|
||||
char* tagPair;
|
||||
int64_t nodeId;
|
||||
int64_t wayId;
|
||||
int64_t wayNodeId;
|
||||
int64_t relId;
|
||||
int64_t nodeRefId;
|
||||
int64_t wayRefId;
|
||||
int64_t relRefId;
|
||||
int32_t lon;
|
||||
int32_t lat;
|
||||
uint8_t canIterateTags;
|
||||
uint8_t canIterateNds;
|
||||
uint8_t canIterateRefs;
|
||||
char** strPairTable;
|
||||
} O5mreader;
|
||||
|
||||
typedef struct {
|
||||
uint8_t type;
|
||||
uint64_t id;
|
||||
uint32_t version;
|
||||
uint8_t isEmpty;
|
||||
int32_t lon;
|
||||
int32_t lat;
|
||||
} O5mreaderDataset;
|
||||
|
||||
#if defined (__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
O5mreaderRet o5mreader_open(O5mreader **ppReader,FILE* f);
|
||||
|
||||
void o5mreader_close(O5mreader *pReader);
|
||||
|
||||
const char* o5mreader_strerror(int errCode);
|
||||
|
||||
O5mreaderIterateRet o5mreader_iterateDataSet(O5mreader *pReader, O5mreaderDataset* ds);
|
||||
|
||||
O5mreaderIterateRet o5mreader_iterateTags(O5mreader *pReader, char** pKey, char** pVal);
|
||||
|
||||
O5mreaderIterateRet o5mreader_iterateNds(O5mreader *pReader, uint64_t *nodeId);
|
||||
|
||||
O5mreaderIterateRet o5mreader_iterateRefs(O5mreader *pReader, uint64_t *refId, uint8_t *type, char** pRole);
|
||||
|
||||
#if defined (__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif //__O5MREADER__H__
|
||||
17
3party/o5mreader/o5mreader.pro
Normal file
17
3party/o5mreader/o5mreader.pro
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
#-------------------------------------------------
|
||||
#
|
||||
# Project created by QtCreator 2015-02-13T20:23:55
|
||||
#
|
||||
#-------------------------------------------------
|
||||
|
||||
ROOT_DIR = ../..
|
||||
|
||||
TARGET = o5mreader
|
||||
TEMPLATE = lib
|
||||
CONFIG += staticlib
|
||||
|
||||
include($$ROOT_DIR/common.pri)
|
||||
|
||||
SOURCES += o5mreader.c
|
||||
|
||||
HEADERS += o5mreader.h
|
||||
|
|
@ -8,6 +8,7 @@
|
|||
|
||||
#include "../base/assert.hpp"
|
||||
#include "../base/bits.hpp"
|
||||
#include "../std/cmath.hpp"
|
||||
|
||||
namespace {
|
||||
vector<uint32_t> SerialFreqsToDistrTable(Reader & reader, uint64_t & decodeOffset, uint64_t cnt)
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ namespace stream
|
|||
SinkReaderStream(TReader & reader) : m_reader(reader) {}
|
||||
|
||||
template <typename T>
|
||||
typename enable_if<is_integral<T>, SinkReaderStream &>::type
|
||||
typename enable_if<is_integral<T>::value, SinkReaderStream &>::type
|
||||
operator >> (T & t)
|
||||
{
|
||||
t = ReadPrimitiveFromSource<T>(m_reader);
|
||||
|
|
@ -53,7 +53,7 @@ namespace stream
|
|||
SinkWriterStream(TWriter & writer) : m_writer(writer) {}
|
||||
|
||||
template <typename T>
|
||||
typename enable_if<is_integral<T>, SinkWriterStream &>::type
|
||||
typename enable_if<is_integral<T>::value, SinkWriterStream &>::type
|
||||
operator << (T const & t)
|
||||
{
|
||||
WriteToSink(m_writer, t);
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
|
||||
|
||||
template <class TSink, typename T>
|
||||
typename enable_if<is_integral<T>, void>::type WriteToSink(TSink & sink, T const & v)
|
||||
typename enable_if<is_integral<T>::value, void>::type WriteToSink(TSink & sink, T const & v)
|
||||
{
|
||||
T const t = SwapIfBigEndian(v);
|
||||
sink.Write(&t, sizeof(T));
|
||||
|
|
|
|||
|
|
@ -1,107 +0,0 @@
|
|||
#include "data_generator.hpp"
|
||||
#include "data_cache_file.hpp"
|
||||
#include "first_pass_parser.hpp"
|
||||
#include "osm_decl.hpp"
|
||||
|
||||
#include "../base/std_serialization.hpp"
|
||||
#include "../base/logging.hpp"
|
||||
|
||||
#include "../std/bind.hpp"
|
||||
#include "point_storage.hpp"
|
||||
|
||||
namespace data
|
||||
{
|
||||
|
||||
template <class TNodesHolder>
|
||||
class FileHolder : public cache::BaseFileHolder<TNodesHolder, cache::DataFileWriter, FileWriter>
|
||||
{
|
||||
typedef cache::BaseFileHolder<TNodesHolder, cache::DataFileWriter, FileWriter> base_type;
|
||||
|
||||
typedef typename base_type::user_id_t user_id_t;
|
||||
|
||||
template <class TMap, class TVec>
|
||||
void add_id2rel_vector(TMap & rMap, user_id_t relid, TVec const & v)
|
||||
{
|
||||
for (size_t i = 0; i < v.size(); ++i)
|
||||
rMap.write(v[i].first, relid);
|
||||
}
|
||||
|
||||
public:
|
||||
FileHolder(TNodesHolder & nodes, string const & dir) : base_type(nodes, dir) {}
|
||||
|
||||
void AddNode(uint64_t id, double lat, double lng)
|
||||
{
|
||||
this->m_nodes.AddPoint(id, lat, lng);
|
||||
}
|
||||
|
||||
void AddWay(user_id_t id, WayElement const & e)
|
||||
{
|
||||
this->m_ways.Write(id, e);
|
||||
}
|
||||
|
||||
void AddRelation(user_id_t id, RelationElement const & e)
|
||||
{
|
||||
const string relationType = e.GetType();
|
||||
if (relationType == "multipolygon" || relationType == "route" || relationType == "boundary")
|
||||
{
|
||||
this->m_relations.Write(id, e);
|
||||
|
||||
add_id2rel_vector(this->m_nodes2rel, id, e.nodes);
|
||||
add_id2rel_vector(this->m_ways2rel, id, e.ways);
|
||||
}
|
||||
}
|
||||
|
||||
void SaveIndex()
|
||||
{
|
||||
this->m_ways.SaveOffsets();
|
||||
this->m_relations.SaveOffsets();
|
||||
|
||||
this->m_nodes2rel.flush_to_file();
|
||||
this->m_ways2rel.flush_to_file();
|
||||
}
|
||||
};
|
||||
|
||||
template <class TNodesHolder>
|
||||
bool GenerateImpl(string const & dir, string const & osmFileName = string())
|
||||
{
|
||||
try
|
||||
{
|
||||
TNodesHolder nodes(dir + NODES_FILE);
|
||||
typedef FileHolder<TNodesHolder> holder_t;
|
||||
holder_t holder(nodes, dir);
|
||||
|
||||
FirstPassParser<holder_t> parser(holder);
|
||||
if (osmFileName.empty())
|
||||
ParseXMLFromStdIn(parser);
|
||||
else
|
||||
ParseXMLFromFile(parser, osmFileName);
|
||||
|
||||
LOG(LINFO, ("Added points count = ", nodes.GetCount()));
|
||||
|
||||
holder.SaveIndex();
|
||||
}
|
||||
catch (Writer::Exception const & e)
|
||||
{
|
||||
LOG(LERROR, ("Error with file ", e.what()));
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool GenerateToFile(string const & dir, string const & nodeStorage, string const & osmFileName)
|
||||
{
|
||||
if (nodeStorage == "raw")
|
||||
return GenerateImpl<RawFileShortPointStorage<BasePointStorage::MODE_WRITE>>(dir, osmFileName);
|
||||
else if (nodeStorage == "map")
|
||||
return GenerateImpl<MapFileShortPointStorage<BasePointStorage::MODE_WRITE>>(dir, osmFileName);
|
||||
else if (nodeStorage == "sqlite")
|
||||
return GenerateImpl<SQLitePointStorage<BasePointStorage::MODE_WRITE>>(dir, osmFileName);
|
||||
else if (nodeStorage == "mem")
|
||||
return GenerateImpl<RawMemShortPointStorage<BasePointStorage::MODE_WRITE>>(dir, osmFileName);
|
||||
else
|
||||
CHECK(nodeStorage.empty(), ("Incorrect node_storage type:", nodeStorage));
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
#pragma once
|
||||
|
||||
#include "../std/string.hpp"
|
||||
|
||||
namespace data
|
||||
{
|
||||
bool GenerateToFile(string const & dir, string const & nodeStorage, string const & osmFileName);
|
||||
}
|
||||
|
|
@ -1,18 +1,13 @@
|
|||
#include "feature_generator.hpp"
|
||||
#include "data_cache_file.hpp"
|
||||
#include "osm_element.hpp"
|
||||
#include "polygonizer.hpp"
|
||||
|
||||
#include "osm_decl.hpp"
|
||||
#include "generate_info.hpp"
|
||||
#include "coastlines_generator.hpp"
|
||||
#include "world_map_generator.hpp"
|
||||
|
||||
#include "../defines.hpp"
|
||||
|
||||
#include "../indexer/data_header.hpp"
|
||||
#include "../indexer/mercator.hpp"
|
||||
#include "../indexer/cell_id.hpp"
|
||||
#include "../indexer/classificator.hpp"
|
||||
|
||||
#include "../coding/varint.hpp"
|
||||
|
||||
|
|
@ -24,103 +19,11 @@
|
|||
#include "../std/unordered_map.hpp"
|
||||
#include "../std/target_os.hpp"
|
||||
|
||||
#include "point_storage.hpp"
|
||||
|
||||
namespace feature
|
||||
{
|
||||
|
||||
template <class TNodesHolder>
|
||||
class FileHolder : public cache::BaseFileHolder<TNodesHolder, cache::DataFileReader, FileReader>
|
||||
{
|
||||
typedef cache::DataFileReader reader_t;
|
||||
typedef cache::BaseFileHolder<TNodesHolder, reader_t, FileReader> base_type;
|
||||
|
||||
typedef typename base_type::offset_map_t offset_map_t;
|
||||
|
||||
typedef typename base_type::user_id_t user_id_t;
|
||||
|
||||
template <class TElement, class ToDo> struct process_base
|
||||
{
|
||||
protected:
|
||||
reader_t & m_reader;
|
||||
ToDo & m_toDo;
|
||||
public:
|
||||
process_base(reader_t & reader, ToDo & toDo) : m_reader(reader), m_toDo(toDo) {}
|
||||
|
||||
bool operator() (uint64_t id)
|
||||
{
|
||||
TElement e;
|
||||
if (m_reader.Read(id, e))
|
||||
return m_toDo(id, e);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
template <class ToDo> struct process_relation : public process_base<RelationElement, ToDo>
|
||||
{
|
||||
typedef process_base<RelationElement, ToDo> base_type;
|
||||
public:
|
||||
process_relation(reader_t & reader, ToDo & toDo) : base_type(reader, toDo) {}
|
||||
};
|
||||
|
||||
template <class ToDo> struct process_relation_cached : public process_relation<ToDo>
|
||||
{
|
||||
typedef process_relation<ToDo> base_type;
|
||||
|
||||
public:
|
||||
process_relation_cached(reader_t & rels, ToDo & toDo)
|
||||
: base_type(rels, toDo) {}
|
||||
|
||||
bool operator() (uint64_t id)
|
||||
{
|
||||
return this->m_toDo(id, this->m_reader);
|
||||
}
|
||||
};
|
||||
|
||||
public:
|
||||
FileHolder(TNodesHolder & holder, string const & dir) : base_type(holder, dir) {}
|
||||
|
||||
bool GetNode(uint64_t id, double & lat, double & lng)
|
||||
{
|
||||
return this->m_nodes.GetPoint(id, lat, lng);
|
||||
}
|
||||
|
||||
bool GetWay(user_id_t id, WayElement & e)
|
||||
{
|
||||
return this->m_ways.Read(id, e);
|
||||
}
|
||||
|
||||
template <class ToDo> void ForEachRelationByWay(user_id_t id, ToDo & toDo)
|
||||
{
|
||||
process_relation<ToDo> processor(this->m_relations, toDo);
|
||||
this->m_ways2rel.for_each_ret(id, processor);
|
||||
}
|
||||
|
||||
template <class ToDo> void ForEachRelationByNodeCached(user_id_t id, ToDo & toDo)
|
||||
{
|
||||
process_relation_cached<ToDo> processor(this->m_relations, toDo);
|
||||
this->m_nodes2rel.for_each_ret(id, processor);
|
||||
}
|
||||
|
||||
template <class ToDo> void ForEachRelationByWayCached(user_id_t id, ToDo & toDo)
|
||||
{
|
||||
process_relation_cached<ToDo> processor(this->m_relations, toDo);
|
||||
this->m_ways2rel.for_each_ret(id, processor);
|
||||
}
|
||||
|
||||
void LoadIndex()
|
||||
{
|
||||
this->m_ways.LoadOffsets();
|
||||
this->m_relations.LoadOffsets();
|
||||
|
||||
this->m_nodes2rel.read_to_memory();
|
||||
this->m_ways2rel.read_to_memory();
|
||||
}
|
||||
};
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// FeaturesCollector implementation
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
namespace feature {
|
||||
|
||||
FeaturesCollector::FeaturesCollector(string const & fName)
|
||||
: m_datFile(fName)
|
||||
|
|
@ -213,232 +116,5 @@ void FeaturesCollector::operator() (FeatureBuilder1 const & fb)
|
|||
(void)WriteFeatureBase(bytes, fb);
|
||||
}
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Generate functions implementations.
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
class MainFeaturesEmitter
|
||||
{
|
||||
typedef WorldMapGenerator<FeaturesCollector> WorldGenerator;
|
||||
typedef CountryMapGenerator<Polygonizer<FeaturesCollector> > CountriesGenerator;
|
||||
unique_ptr<CountriesGenerator> m_countries;
|
||||
unique_ptr<WorldGenerator> m_world;
|
||||
|
||||
unique_ptr<CoastlineFeaturesGenerator> m_coasts;
|
||||
unique_ptr<FeaturesCollector> m_coastsHolder;
|
||||
|
||||
string m_srcCoastsFile;
|
||||
|
||||
template <class T1, class T2> class CombinedEmitter
|
||||
{
|
||||
T1 * m_p1;
|
||||
T2 * m_p2;
|
||||
public:
|
||||
CombinedEmitter(T1 * p1, T2 * p2) : m_p1(p1), m_p2(p2) {}
|
||||
void operator() (FeatureBuilder1 const & fb, uint64_t)
|
||||
{
|
||||
if (m_p1) (*m_p1)(fb);
|
||||
if (m_p2) (*m_p2)(fb);
|
||||
}
|
||||
};
|
||||
|
||||
enum TypeIndex
|
||||
{
|
||||
NATURAL_COASTLINE,
|
||||
NATURAL_LAND,
|
||||
PLACE_ISLAND,
|
||||
PLACE_ISLET,
|
||||
|
||||
TYPES_COUNT
|
||||
};
|
||||
uint32_t m_types[TYPES_COUNT];
|
||||
|
||||
inline uint32_t Type(TypeIndex i) const { return m_types[i]; }
|
||||
|
||||
public:
|
||||
MainFeaturesEmitter(GenerateInfo const & info)
|
||||
{
|
||||
Classificator const & c = classif();
|
||||
|
||||
char const * arr[][2] = {
|
||||
{ "natural", "coastline" },
|
||||
{ "natural", "land" },
|
||||
{ "place", "island" },
|
||||
{ "place", "islet" }
|
||||
};
|
||||
STATIC_ASSERT(ARRAY_SIZE(arr) == TYPES_COUNT);
|
||||
|
||||
for (size_t i = 0; i < ARRAY_SIZE(arr); ++i)
|
||||
m_types[i] = c.GetTypeByPath(vector<string>(arr[i], arr[i] + 2));
|
||||
|
||||
m_srcCoastsFile = info.m_tmpDir + WORLD_COASTS_FILE_NAME + info.m_datFileSuffix;
|
||||
|
||||
CHECK(!info.m_makeCoasts || !info.m_createWorld,
|
||||
("We can't do make_coasts and generate_world at the same time"));
|
||||
|
||||
if (!info.m_makeCoasts)
|
||||
{
|
||||
m_countries.reset(new CountriesGenerator(info));
|
||||
|
||||
if (info.m_emitCoasts)
|
||||
{
|
||||
m_coastsHolder.reset(new FeaturesCollector(
|
||||
info.m_datFilePrefix + WORLD_COASTS_FILE_NAME + info.m_datFileSuffix));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// 4-10 - level range for cells
|
||||
// 20000 - max points count per feature
|
||||
m_coasts.reset(new CoastlineFeaturesGenerator(Type(NATURAL_COASTLINE), 4, 10, 20000));
|
||||
|
||||
m_coastsHolder.reset(new FeaturesCollector(m_srcCoastsFile));
|
||||
}
|
||||
|
||||
if (info.m_createWorld)
|
||||
m_world.reset(new WorldGenerator(info));
|
||||
}
|
||||
|
||||
void operator() (FeatureBuilder1 fb)
|
||||
{
|
||||
uint32_t const coastType = Type(NATURAL_COASTLINE);
|
||||
bool const hasCoast = fb.HasType(coastType);
|
||||
|
||||
if (m_coasts)
|
||||
{
|
||||
if (hasCoast)
|
||||
{
|
||||
CHECK ( fb.GetGeomType() != feature::GEOM_POINT, () );
|
||||
|
||||
// leave only coastline type
|
||||
fb.SetType(coastType);
|
||||
(*m_coasts)(fb);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (hasCoast)
|
||||
{
|
||||
fb.PopExactType(Type(NATURAL_LAND));
|
||||
fb.PopExactType(coastType);
|
||||
}
|
||||
else if ((fb.HasType(Type(PLACE_ISLAND)) || fb.HasType(Type(PLACE_ISLET))) &&
|
||||
fb.GetGeomType() == feature::GEOM_AREA)
|
||||
{
|
||||
fb.AddType(Type(NATURAL_LAND));
|
||||
}
|
||||
|
||||
if (fb.RemoveInvalidTypes())
|
||||
{
|
||||
if (m_world)
|
||||
(*m_world)(fb);
|
||||
|
||||
if (m_countries)
|
||||
(*m_countries)(fb);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// @return false if coasts are not merged and FLAG_fail_on_coasts is set
|
||||
bool Finish()
|
||||
{
|
||||
if (m_world)
|
||||
m_world->DoMerge();
|
||||
|
||||
if (m_coasts)
|
||||
{
|
||||
// Check and stop if some coasts were not merged
|
||||
if (!m_coasts->Finish())
|
||||
return false;
|
||||
|
||||
size_t const count = m_coasts->GetCellsCount();
|
||||
LOG(LINFO, ("Generating coastline polygons", count));
|
||||
|
||||
for (size_t i = 0; i < count; ++i)
|
||||
{
|
||||
vector<FeatureBuilder1> vecFb;
|
||||
m_coasts->GetFeatures(i, vecFb);
|
||||
|
||||
for (size_t j = 0; j < vecFb.size(); ++j)
|
||||
(*m_coastsHolder)(vecFb[j]);
|
||||
}
|
||||
}
|
||||
else if (m_coastsHolder)
|
||||
{
|
||||
CombinedEmitter<FeaturesCollector, CountriesGenerator>
|
||||
emitter(m_coastsHolder.get(), m_countries.get());
|
||||
feature::ForEachFromDatRawFormat(m_srcCoastsFile, emitter);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
inline void GetNames(vector<string> & names) const
|
||||
{
|
||||
if (m_countries)
|
||||
names = m_countries->Parent().Names();
|
||||
else
|
||||
names.clear();
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
template <class NodesHolderT>
|
||||
bool GenerateImpl(GenerateInfo & info, string const & osmFileName = string())
|
||||
{
|
||||
try
|
||||
{
|
||||
NodesHolderT nodes(info.m_tmpDir + NODES_FILE);
|
||||
|
||||
typedef FileHolder<NodesHolderT> HolderT;
|
||||
HolderT holder(nodes, info.m_tmpDir);
|
||||
holder.LoadIndex();
|
||||
|
||||
MainFeaturesEmitter bucketer(info);
|
||||
SecondPassParser<MainFeaturesEmitter, HolderT> parser(
|
||||
bucketer, holder,
|
||||
info.m_makeCoasts ? classif().GetCoastType() : 0,
|
||||
info.m_addressFile);
|
||||
|
||||
if (osmFileName.empty())
|
||||
ParseXMLFromStdIn(parser);
|
||||
else
|
||||
ParseXMLFromFile(parser, osmFileName);
|
||||
|
||||
parser.Finish();
|
||||
|
||||
// Stop if coasts are not merged and FLAG_fail_on_coasts is set
|
||||
if (!bucketer.Finish())
|
||||
return false;
|
||||
|
||||
bucketer.GetNames(info.m_bucketNames);
|
||||
}
|
||||
catch (Reader::Exception const & e)
|
||||
{
|
||||
LOG(LERROR, ("Error with file ", e.what()));
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool GenerateFeatures(GenerateInfo & info, string const & nodeStorage, string const & osmFileName)
|
||||
{
|
||||
if (nodeStorage == "raw")
|
||||
return GenerateImpl<RawFileShortPointStorage<BasePointStorage::MODE_READ>>(info, osmFileName);
|
||||
else if (nodeStorage == "map")
|
||||
return GenerateImpl<MapFileShortPointStorage<BasePointStorage::MODE_READ>>(info, osmFileName);
|
||||
else if (nodeStorage == "sqlite")
|
||||
return GenerateImpl<SQLitePointStorage<BasePointStorage::MODE_READ>>(info, osmFileName);
|
||||
else if (nodeStorage == "mem")
|
||||
return GenerateImpl<RawMemShortPointStorage<BasePointStorage::MODE_READ>>(info, osmFileName);
|
||||
else
|
||||
CHECK(nodeStorage.empty(), ("Incorrect node_storage type:", nodeStorage));
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,10 +12,6 @@ class FeatureBuilder1;
|
|||
|
||||
namespace feature
|
||||
{
|
||||
struct GenerateInfo;
|
||||
|
||||
bool GenerateFeatures(GenerateInfo & info, string const & nodeStorage, string const & osmFileName);
|
||||
|
||||
// Writes features to dat file.
|
||||
class FeaturesCollector
|
||||
{
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ QT *= core
|
|||
SOURCES += \
|
||||
feature_merger.cpp \
|
||||
xml_element.cpp \
|
||||
data_generator.cpp \
|
||||
osm_source.cpp \
|
||||
feature_generator.cpp \
|
||||
feature_sorter.cpp \
|
||||
update_generator.cpp \
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
#include "../data_generator.hpp"
|
||||
#include "../feature_generator.hpp"
|
||||
#include "../feature_sorter.hpp"
|
||||
#include "../update_generator.hpp"
|
||||
|
|
@ -10,6 +9,7 @@
|
|||
#include "../generate_info.hpp"
|
||||
#include "../check_model.hpp"
|
||||
#include "../routing_generator.hpp"
|
||||
#include "../osm_source.hpp"
|
||||
|
||||
#include "../../indexer/drawing_rules.hpp"
|
||||
#include "../../indexer/classificator_loader.hpp"
|
||||
|
|
@ -35,7 +35,7 @@ DEFINE_bool(generate_update, false,
|
|||
|
||||
DEFINE_bool(generate_classif, false, "Generate classificator.");
|
||||
|
||||
DEFINE_bool(preprocess_xml, false, "1st pass - create nodes/ways/relations data");
|
||||
DEFINE_bool(preprocess, false, "1st pass - create nodes/ways/relations data");
|
||||
DEFINE_bool(make_coasts, false, "create intermediate file with coasts data");
|
||||
DEFINE_bool(emit_coasts, false, "push coasts features from intermediate file to out files/countries");
|
||||
|
||||
|
|
@ -44,7 +44,7 @@ DEFINE_bool(generate_geometry, false, "3rd pass - split and simplify geometry an
|
|||
DEFINE_bool(generate_index, false, "4rd pass - generate index");
|
||||
DEFINE_bool(generate_search_index, false, "5th pass - generate search index");
|
||||
DEFINE_bool(calc_statistics, false, "Calculate feature statistics for specified mwm bucket files");
|
||||
DEFINE_string(node_storage, "raw", "Type of storage for intermediate points representation. Available: raw, map, sqlite, mem");
|
||||
DEFINE_string(node_storage, "map", "Type of storage for intermediate points representation. Available: raw, map, mem");
|
||||
DEFINE_string(data_path, "", "Working directory, 'path_to_exe/../../data' if empty.");
|
||||
DEFINE_string(output, "", "File name for process (without 'mwm' ext).");
|
||||
DEFINE_string(intermediate_data_path, "", "Path to stored nodes, ways, relations.");
|
||||
|
|
@ -65,6 +65,7 @@ DEFINE_string(address_file_name, "", "Output file name for storing full addresse
|
|||
DEFINE_string(export_poly_path, "", "Output dir for osm .poly files created from .borders (countires are read from polygons.lst)");
|
||||
DEFINE_string(osrm_file_name, "", "Input osrm file to generate routing info");
|
||||
DEFINE_string(osm_file_name, "", "Input osm area file");
|
||||
DEFINE_string(osm_file_type, "xml", "Input osm area file type [xml, o5m]");
|
||||
DEFINE_string(user_resource_path, "", "User defined resource path for classificator.txt and etc.");
|
||||
|
||||
|
||||
|
|
@ -101,10 +102,10 @@ int main(int argc, char ** argv)
|
|||
FLAGS_data_path.empty() ? pl.WritableDir() : AddSlashIfNeeded(FLAGS_data_path);
|
||||
|
||||
// Generating intermediate files
|
||||
if (FLAGS_preprocess_xml)
|
||||
if (FLAGS_preprocess)
|
||||
{
|
||||
LOG(LINFO, ("Generating intermediate data ...."));
|
||||
if (!data::GenerateToFile(FLAGS_intermediate_data_path, FLAGS_node_storage, FLAGS_osm_file_name))
|
||||
if (!GenerateIntermediateData(FLAGS_intermediate_data_path, FLAGS_node_storage, FLAGS_osm_file_type, FLAGS_osm_file_name))
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
|
@ -148,7 +149,7 @@ int main(int argc, char ** argv)
|
|||
if (!FLAGS_address_file_name.empty())
|
||||
genInfo.m_addressFile = path + FLAGS_address_file_name;
|
||||
|
||||
if (!feature::GenerateFeatures(genInfo, FLAGS_node_storage, FLAGS_osm_file_name))
|
||||
if (!GenerateFeatures(genInfo, FLAGS_node_storage, FLAGS_osm_file_type, FLAGS_osm_file_name))
|
||||
return -1;
|
||||
|
||||
// without --spit_by_polygons, we have empty name country as result - assign it
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
ROOT_DIR = ../..
|
||||
DEPENDENCIES = generator routing storage indexer platform geometry coding base \
|
||||
osrm gflags expat sgitess jansson protobuf tomcrypt
|
||||
osrm gflags expat sgitess jansson protobuf tomcrypt o5mreader
|
||||
|
||||
include($$ROOT_DIR/common.pri)
|
||||
|
||||
|
|
@ -18,8 +18,6 @@ QT *= core
|
|||
win32: LIBS *= -lShell32
|
||||
macx-*: LIBS *= "-framework Foundation"
|
||||
|
||||
LIBS *= -lsqlite3
|
||||
|
||||
SOURCES += \
|
||||
generator_tool.cpp \
|
||||
|
||||
|
|
|
|||
|
|
@ -382,6 +382,7 @@ class SecondPassParser : public BaseOSMParser
|
|||
}
|
||||
//@}
|
||||
|
||||
public:
|
||||
/// The main entry point for parsing process.
|
||||
virtual void EmitElement(XMLElement * p)
|
||||
{
|
||||
|
|
|
|||
647
generator/osm_source.cpp
Normal file
647
generator/osm_source.cpp
Normal file
|
|
@ -0,0 +1,647 @@
|
|||
#include "osm_source.hpp"
|
||||
|
||||
#include "osm_decl.hpp"
|
||||
#include "data_cache_file.hpp"
|
||||
|
||||
#include "coastlines_generator.hpp"
|
||||
#include "world_map_generator.hpp"
|
||||
#include "feature_generator.hpp"
|
||||
#include "polygonizer.hpp"
|
||||
|
||||
#include "point_storage.hpp"
|
||||
|
||||
#include "xml_element.hpp"
|
||||
|
||||
#include "first_pass_parser.hpp"
|
||||
#include "osm_element.hpp"
|
||||
|
||||
#include "../defines.hpp"
|
||||
#include "../indexer/mercator.hpp"
|
||||
#include "../indexer/classificator.hpp"
|
||||
|
||||
#include "../coding/parse_xml.hpp"
|
||||
|
||||
#include "../3party/o5mreader/o5mreader.h"
|
||||
|
||||
|
||||
#define DECODE_O5M_COORD(coord) (static_cast<double>(coord) / 1E+7)
|
||||
|
||||
namespace
|
||||
{
|
||||
class SourceReader
|
||||
{
|
||||
string const &m_filename;
|
||||
FILE * m_file;
|
||||
|
||||
public:
|
||||
SourceReader(string const & filename) : m_filename(filename)
|
||||
{
|
||||
if (m_filename.empty())
|
||||
{
|
||||
LOG(LINFO, ("Read OSM data from stdin..."));
|
||||
m_file = freopen(NULL, "rb", stdin);
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG(LINFO, ("Read OSM data from", filename));
|
||||
m_file = fopen(filename.c_str(), "rb");
|
||||
}
|
||||
}
|
||||
|
||||
~SourceReader()
|
||||
{
|
||||
if (!m_filename.empty())
|
||||
fclose(m_file);
|
||||
}
|
||||
|
||||
inline FILE * Handle() { return m_file; }
|
||||
|
||||
uint64_t Read(char * buffer, uint64_t bufferSize)
|
||||
{
|
||||
return fread(buffer, sizeof(char), bufferSize, m_file);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
namespace feature
|
||||
{
|
||||
|
||||
template <class TNodesHolder>
|
||||
class FileHolder : public cache::BaseFileHolder<TNodesHolder, cache::DataFileReader, FileReader>
|
||||
{
|
||||
typedef cache::DataFileReader reader_t;
|
||||
typedef cache::BaseFileHolder<TNodesHolder, reader_t, FileReader> base_type;
|
||||
|
||||
typedef typename base_type::offset_map_t offset_map_t;
|
||||
|
||||
typedef typename base_type::user_id_t user_id_t;
|
||||
|
||||
template <class TElement, class ToDo> struct process_base
|
||||
{
|
||||
protected:
|
||||
reader_t & m_reader;
|
||||
ToDo & m_toDo;
|
||||
public:
|
||||
process_base(reader_t & reader, ToDo & toDo) : m_reader(reader), m_toDo(toDo) {}
|
||||
|
||||
bool operator() (uint64_t id)
|
||||
{
|
||||
TElement e;
|
||||
if (m_reader.Read(id, e))
|
||||
return m_toDo(id, e);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
template <class ToDo> struct process_relation : public process_base<RelationElement, ToDo>
|
||||
{
|
||||
typedef process_base<RelationElement, ToDo> base_type;
|
||||
public:
|
||||
process_relation(reader_t & reader, ToDo & toDo) : base_type(reader, toDo) {}
|
||||
};
|
||||
|
||||
template <class ToDo> struct process_relation_cached : public process_relation<ToDo>
|
||||
{
|
||||
typedef process_relation<ToDo> base_type;
|
||||
|
||||
public:
|
||||
process_relation_cached(reader_t & rels, ToDo & toDo)
|
||||
: base_type(rels, toDo) {}
|
||||
|
||||
bool operator() (uint64_t id)
|
||||
{
|
||||
return this->m_toDo(id, this->m_reader);
|
||||
}
|
||||
};
|
||||
|
||||
public:
|
||||
FileHolder(TNodesHolder & holder, string const & dir) : base_type(holder, dir) {}
|
||||
|
||||
bool GetNode(uint64_t id, double & lat, double & lng)
|
||||
{
|
||||
return this->m_nodes.GetPoint(id, lat, lng);
|
||||
}
|
||||
|
||||
bool GetWay(user_id_t id, WayElement & e)
|
||||
{
|
||||
return this->m_ways.Read(id, e);
|
||||
}
|
||||
|
||||
template <class ToDo> void ForEachRelationByWay(user_id_t id, ToDo & toDo)
|
||||
{
|
||||
process_relation<ToDo> processor(this->m_relations, toDo);
|
||||
this->m_ways2rel.for_each_ret(id, processor);
|
||||
}
|
||||
|
||||
template <class ToDo> void ForEachRelationByNodeCached(user_id_t id, ToDo & toDo)
|
||||
{
|
||||
process_relation_cached<ToDo> processor(this->m_relations, toDo);
|
||||
this->m_nodes2rel.for_each_ret(id, processor);
|
||||
}
|
||||
|
||||
template <class ToDo> void ForEachRelationByWayCached(user_id_t id, ToDo & toDo)
|
||||
{
|
||||
process_relation_cached<ToDo> processor(this->m_relations, toDo);
|
||||
this->m_ways2rel.for_each_ret(id, processor);
|
||||
}
|
||||
|
||||
void LoadIndex()
|
||||
{
|
||||
this->m_ways.LoadOffsets();
|
||||
this->m_relations.LoadOffsets();
|
||||
|
||||
this->m_nodes2rel.read_to_memory();
|
||||
this->m_ways2rel.read_to_memory();
|
||||
}
|
||||
};
|
||||
} // namespace feature
|
||||
|
||||
namespace data
|
||||
{
|
||||
|
||||
template <class TNodesHolder>
|
||||
class FileHolder : public cache::BaseFileHolder<TNodesHolder, cache::DataFileWriter, FileWriter>
|
||||
{
|
||||
typedef cache::BaseFileHolder<TNodesHolder, cache::DataFileWriter, FileWriter> base_type;
|
||||
|
||||
typedef typename base_type::user_id_t user_id_t;
|
||||
|
||||
template <class TMap, class TVec>
|
||||
void add_id2rel_vector(TMap & rMap, user_id_t relid, TVec const & v)
|
||||
{
|
||||
for (size_t i = 0; i < v.size(); ++i)
|
||||
rMap.write(v[i].first, relid);
|
||||
}
|
||||
|
||||
public:
|
||||
FileHolder(TNodesHolder & nodes, string const & dir) : base_type(nodes, dir) {}
|
||||
|
||||
void AddNode(uint64_t id, double lat, double lng)
|
||||
{
|
||||
this->m_nodes.AddPoint(id, lat, lng);
|
||||
}
|
||||
|
||||
void AddWay(user_id_t id, WayElement const & e)
|
||||
{
|
||||
this->m_ways.Write(id, e);
|
||||
}
|
||||
|
||||
void AddRelation(user_id_t id, RelationElement const & e)
|
||||
{
|
||||
const string relationType = e.GetType();
|
||||
if (relationType == "multipolygon" || relationType == "route" || relationType == "boundary")
|
||||
{
|
||||
this->m_relations.Write(id, e);
|
||||
|
||||
add_id2rel_vector(this->m_nodes2rel, id, e.nodes);
|
||||
add_id2rel_vector(this->m_ways2rel, id, e.ways);
|
||||
}
|
||||
}
|
||||
|
||||
void SaveIndex()
|
||||
{
|
||||
this->m_ways.SaveOffsets();
|
||||
this->m_relations.SaveOffsets();
|
||||
|
||||
this->m_nodes2rel.flush_to_file();
|
||||
this->m_ways2rel.flush_to_file();
|
||||
}
|
||||
};
|
||||
} // namespace data
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
class MainFeaturesEmitter
|
||||
{
|
||||
typedef WorldMapGenerator<feature::FeaturesCollector> WorldGenerator;
|
||||
typedef CountryMapGenerator<feature::Polygonizer<feature::FeaturesCollector> > CountriesGenerator;
|
||||
unique_ptr<CountriesGenerator> m_countries;
|
||||
unique_ptr<WorldGenerator> m_world;
|
||||
|
||||
unique_ptr<CoastlineFeaturesGenerator> m_coasts;
|
||||
unique_ptr<feature::FeaturesCollector> m_coastsHolder;
|
||||
|
||||
string m_srcCoastsFile;
|
||||
|
||||
template <class T1, class T2> class CombinedEmitter
|
||||
{
|
||||
T1 * m_p1;
|
||||
T2 * m_p2;
|
||||
public:
|
||||
CombinedEmitter(T1 * p1, T2 * p2) : m_p1(p1), m_p2(p2) {}
|
||||
void operator() (FeatureBuilder1 const & fb, uint64_t)
|
||||
{
|
||||
if (m_p1) (*m_p1)(fb);
|
||||
if (m_p2) (*m_p2)(fb);
|
||||
}
|
||||
};
|
||||
|
||||
enum TypeIndex
|
||||
{
|
||||
NATURAL_COASTLINE,
|
||||
NATURAL_LAND,
|
||||
PLACE_ISLAND,
|
||||
PLACE_ISLET,
|
||||
|
||||
TYPES_COUNT
|
||||
};
|
||||
uint32_t m_types[TYPES_COUNT];
|
||||
|
||||
inline uint32_t Type(TypeIndex i) const { return m_types[i]; }
|
||||
|
||||
public:
|
||||
MainFeaturesEmitter(feature::GenerateInfo const & info)
|
||||
{
|
||||
Classificator const & c = classif();
|
||||
|
||||
char const * arr[][2] = {
|
||||
{ "natural", "coastline" },
|
||||
{ "natural", "land" },
|
||||
{ "place", "island" },
|
||||
{ "place", "islet" }
|
||||
};
|
||||
STATIC_ASSERT(ARRAY_SIZE(arr) == TYPES_COUNT);
|
||||
|
||||
for (size_t i = 0; i < ARRAY_SIZE(arr); ++i)
|
||||
m_types[i] = c.GetTypeByPath(vector<string>(arr[i], arr[i] + 2));
|
||||
|
||||
m_srcCoastsFile = info.m_tmpDir + WORLD_COASTS_FILE_NAME + info.m_datFileSuffix;
|
||||
|
||||
CHECK(!info.m_makeCoasts || !info.m_createWorld,
|
||||
("We can't do make_coasts and generate_world at the same time"));
|
||||
|
||||
if (!info.m_makeCoasts)
|
||||
{
|
||||
m_countries.reset(new CountriesGenerator(info));
|
||||
|
||||
if (info.m_emitCoasts)
|
||||
{
|
||||
m_coastsHolder.reset(new feature::FeaturesCollector(
|
||||
info.m_datFilePrefix + WORLD_COASTS_FILE_NAME + info.m_datFileSuffix));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// 4-10 - level range for cells
|
||||
// 20000 - max points count per feature
|
||||
m_coasts.reset(new CoastlineFeaturesGenerator(Type(NATURAL_COASTLINE), 4, 10, 20000));
|
||||
|
||||
m_coastsHolder.reset(new feature::FeaturesCollector(m_srcCoastsFile));
|
||||
}
|
||||
|
||||
if (info.m_createWorld)
|
||||
m_world.reset(new WorldGenerator(info));
|
||||
}
|
||||
|
||||
void operator() (FeatureBuilder1 fb)
|
||||
{
|
||||
uint32_t const coastType = Type(NATURAL_COASTLINE);
|
||||
bool const hasCoast = fb.HasType(coastType);
|
||||
|
||||
if (m_coasts)
|
||||
{
|
||||
if (hasCoast)
|
||||
{
|
||||
CHECK ( fb.GetGeomType() != feature::GEOM_POINT, () );
|
||||
|
||||
// leave only coastline type
|
||||
fb.SetType(coastType);
|
||||
(*m_coasts)(fb);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (hasCoast)
|
||||
{
|
||||
fb.PopExactType(Type(NATURAL_LAND));
|
||||
fb.PopExactType(coastType);
|
||||
}
|
||||
else if ((fb.HasType(Type(PLACE_ISLAND)) || fb.HasType(Type(PLACE_ISLET))) &&
|
||||
fb.GetGeomType() == feature::GEOM_AREA)
|
||||
{
|
||||
fb.AddType(Type(NATURAL_LAND));
|
||||
}
|
||||
|
||||
if (fb.RemoveInvalidTypes())
|
||||
{
|
||||
if (m_world)
|
||||
(*m_world)(fb);
|
||||
|
||||
if (m_countries)
|
||||
(*m_countries)(fb);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// @return false if coasts are not merged and FLAG_fail_on_coasts is set
|
||||
bool Finish()
|
||||
{
|
||||
if (m_world)
|
||||
m_world->DoMerge();
|
||||
|
||||
if (m_coasts)
|
||||
{
|
||||
// Check and stop if some coasts were not merged
|
||||
if (!m_coasts->Finish())
|
||||
return false;
|
||||
|
||||
size_t const count = m_coasts->GetCellsCount();
|
||||
LOG(LINFO, ("Generating coastline polygons", count));
|
||||
|
||||
for (size_t i = 0; i < count; ++i)
|
||||
{
|
||||
vector<FeatureBuilder1> vecFb;
|
||||
m_coasts->GetFeatures(i, vecFb);
|
||||
|
||||
for (size_t j = 0; j < vecFb.size(); ++j)
|
||||
(*m_coastsHolder)(vecFb[j]);
|
||||
}
|
||||
}
|
||||
else if (m_coastsHolder)
|
||||
{
|
||||
CombinedEmitter<feature::FeaturesCollector, CountriesGenerator>
|
||||
emitter(m_coastsHolder.get(), m_countries.get());
|
||||
feature::ForEachFromDatRawFormat(m_srcCoastsFile, emitter);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
inline void GetNames(vector<string> & names) const
|
||||
{
|
||||
if (m_countries)
|
||||
names = m_countries->Parent().Names();
|
||||
else
|
||||
names.clear();
|
||||
}
|
||||
};
|
||||
} // namespace anonymous
|
||||
|
||||
|
||||
template <typename HolderT>
|
||||
void BuildIntermediateDataFromO5M(SourceReader & stream, HolderT & holder)
|
||||
{
|
||||
O5mreader * reader;
|
||||
O5mreaderRet rc;
|
||||
|
||||
if ((rc = o5mreader_open(&reader, stream.Handle())) != O5MREADER_RET_OK)
|
||||
{
|
||||
LOG(LCRITICAL, ("O5M Open error:", o5mreader_strerror(rc)));
|
||||
exit(1);
|
||||
}
|
||||
|
||||
O5mreaderDataset ds;
|
||||
O5mreaderIterateRet ret;
|
||||
while ((ret = o5mreader_iterateDataSet(reader, &ds)) == O5MREADER_ITERATE_RET_NEXT)
|
||||
{
|
||||
O5mreaderIterateRet ret2;
|
||||
switch (ds.type)
|
||||
{
|
||||
case O5MREADER_DS_NODE:
|
||||
{
|
||||
// Could do something with ds.id, ds.lon, ds.lat here, lon and lat are ints in 1E+7 * degree units
|
||||
// convert to mercator
|
||||
double const lat = MercatorBounds::LatToY(DECODE_O5M_COORD(ds.lat));
|
||||
double const lng = MercatorBounds::LonToX(DECODE_O5M_COORD(ds.lon));
|
||||
holder.AddNode(ds.id, lat, lng);
|
||||
} break;
|
||||
|
||||
case O5MREADER_DS_WAY:
|
||||
{
|
||||
// store way
|
||||
WayElement way(ds.id);
|
||||
uint64_t nodeId = 0;
|
||||
while ((ret2 = o5mreader_iterateNds(reader, &nodeId)) == O5MREADER_ITERATE_RET_NEXT)
|
||||
way.nodes.push_back(nodeId);
|
||||
|
||||
if (way.IsValid())
|
||||
holder.AddWay(ds.id, way);
|
||||
} break;
|
||||
|
||||
case O5MREADER_DS_REL:
|
||||
{
|
||||
// store relation
|
||||
RelationElement relation;
|
||||
uint64_t refId = 0;
|
||||
uint8_t type = 0;
|
||||
char * role = nullptr;
|
||||
while ((ret2 = o5mreader_iterateRefs(reader, &refId, &type, &role)) == O5MREADER_ITERATE_RET_NEXT)
|
||||
{
|
||||
// Could do something with refId (way or node or rel id depends on type), type and role
|
||||
if (type == O5MREADER_DS_NODE)
|
||||
relation.nodes.emplace_back(make_pair(refId, string(role)));
|
||||
else if (type == O5MREADER_DS_WAY)
|
||||
relation.ways.emplace_back(make_pair(refId, string(role)));
|
||||
// we just ignore type == "relation"
|
||||
}
|
||||
|
||||
char * key = nullptr;
|
||||
char * val = nullptr;
|
||||
while ((ret2 = o5mreader_iterateTags(reader, &key, &val)) == O5MREADER_ITERATE_RET_NEXT)
|
||||
relation.tags.emplace(make_pair(string(key), string(val)));
|
||||
|
||||
if (relation.IsValid())
|
||||
holder.AddRelation(ds.id, relation);
|
||||
} break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void BuildFeaturesFromO5M(SourceReader & stream, BaseOSMParser & parser)
|
||||
{
|
||||
O5mreader * reader;
|
||||
O5mreaderRet rc;
|
||||
|
||||
if ((rc = o5mreader_open(&reader, stream.Handle())) != O5MREADER_RET_OK)
|
||||
{
|
||||
LOG(LCRITICAL, ("O5M Open error:", o5mreader_strerror(rc)));
|
||||
exit(1);
|
||||
}
|
||||
|
||||
O5mreaderDataset ds;
|
||||
O5mreaderIterateRet ret;
|
||||
while ((ret = o5mreader_iterateDataSet(reader, &ds)) == O5MREADER_ITERATE_RET_NEXT)
|
||||
{
|
||||
XMLElement p;
|
||||
p.id = ds.id;
|
||||
|
||||
O5mreaderIterateRet ret2;
|
||||
char * key = nullptr;
|
||||
char * val = nullptr;
|
||||
switch (ds.type)
|
||||
{
|
||||
case O5MREADER_DS_NODE:
|
||||
{
|
||||
p.tagKey = XMLElement::ET_NODE;
|
||||
p.lat = DECODE_O5M_COORD(ds.lat);
|
||||
p.lng = DECODE_O5M_COORD(ds.lon);
|
||||
} break;
|
||||
|
||||
case O5MREADER_DS_WAY:
|
||||
{
|
||||
p.tagKey = XMLElement::ET_WAY;
|
||||
uint64_t nodeId = 0;
|
||||
while ((ret2 = o5mreader_iterateNds(reader, &nodeId)) == O5MREADER_ITERATE_RET_NEXT)
|
||||
p.AddND(nodeId);
|
||||
} break;
|
||||
|
||||
case O5MREADER_DS_REL:
|
||||
{
|
||||
p.tagKey = XMLElement::ET_RELATION;
|
||||
uint64_t refId = 0;
|
||||
uint8_t type = 0;
|
||||
char * role = nullptr;
|
||||
while ((ret2 = o5mreader_iterateRefs(reader, &refId, &type, &role)) == O5MREADER_ITERATE_RET_NEXT)
|
||||
{
|
||||
string strType;
|
||||
switch (type)
|
||||
{
|
||||
case O5MREADER_DS_NODE:
|
||||
strType = "node";
|
||||
break;
|
||||
case O5MREADER_DS_WAY:
|
||||
strType = "way";
|
||||
break;
|
||||
case O5MREADER_DS_REL:
|
||||
strType = "relation";
|
||||
break;
|
||||
|
||||
default:
|
||||
break;
|
||||
}
|
||||
p.AddMEMBER(refId, strType, role);
|
||||
}
|
||||
} break;
|
||||
}
|
||||
|
||||
while ((ret2 = o5mreader_iterateTags(reader, &key, &val)) == O5MREADER_ITERATE_RET_NEXT)
|
||||
p.AddKV(key, val);
|
||||
parser.EmitElement(&p);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
// Generate functions implementations.
|
||||
///////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
|
||||
template <class NodesHolderT>
|
||||
bool GenerateFeaturesImpl(feature::GenerateInfo & info, string const &osmFileType, string const & osmFileName = string())
|
||||
{
|
||||
try
|
||||
{
|
||||
NodesHolderT nodes(info.m_tmpDir + NODES_FILE);
|
||||
|
||||
typedef feature::FileHolder<NodesHolderT> HolderT;
|
||||
HolderT holder(nodes, info.m_tmpDir);
|
||||
holder.LoadIndex();
|
||||
|
||||
MainFeaturesEmitter bucketer(info);
|
||||
SecondPassParser<MainFeaturesEmitter, HolderT> parser(
|
||||
bucketer, holder,
|
||||
info.m_makeCoasts ? classif().GetCoastType() : 0,
|
||||
info.m_addressFile);
|
||||
|
||||
SourceReader reader(osmFileName);
|
||||
|
||||
if (osmFileType == "xml")
|
||||
{
|
||||
ParseXMLSequence(reader, parser);
|
||||
}
|
||||
else if (osmFileType == "o5m")
|
||||
{
|
||||
BuildFeaturesFromO5M(reader, parser);
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG(LERROR, ("Unknown source type:", osmFileType));
|
||||
return false;
|
||||
}
|
||||
|
||||
parser.Finish();
|
||||
|
||||
// Stop if coasts are not merged and FLAG_fail_on_coasts is set
|
||||
if (!bucketer.Finish())
|
||||
return false;
|
||||
|
||||
bucketer.GetNames(info.m_bucketNames);
|
||||
}
|
||||
catch (Reader::Exception const & e)
|
||||
{
|
||||
LOG(LERROR, ("Error with file ", e.what()));
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
template <class TNodesHolder>
|
||||
bool GenerateIntermediateDataImpl(string const & dir, string const &osmFileType, string const & osmFileName = string())
|
||||
{
|
||||
try
|
||||
{
|
||||
TNodesHolder nodes(dir + NODES_FILE);
|
||||
typedef data::FileHolder<TNodesHolder> HolderT;
|
||||
HolderT holder(nodes, dir);
|
||||
|
||||
SourceReader reader(osmFileName);
|
||||
|
||||
LOG(LINFO, ("Data sorce format:", osmFileType));
|
||||
|
||||
if (osmFileType == "xml")
|
||||
{
|
||||
FirstPassParser<HolderT> parser(holder);
|
||||
ParseXMLSequence(reader, parser);
|
||||
}
|
||||
else if (osmFileType == "o5m")
|
||||
{
|
||||
BuildIntermediateDataFromO5M(reader, holder);
|
||||
}
|
||||
else
|
||||
{
|
||||
LOG(LERROR, ("Unknown source type:", osmFileType));
|
||||
return false;
|
||||
}
|
||||
|
||||
LOG(LINFO, ("Added points count = ", nodes.GetCount()));
|
||||
|
||||
holder.SaveIndex();
|
||||
}
|
||||
catch (Writer::Exception const & e)
|
||||
{
|
||||
LOG(LERROR, ("Error with file ", e.what()));
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
bool GenerateFeatures(feature::GenerateInfo & info, string const & nodeStorage, string const &osmFileType, string const & osmFileName)
|
||||
{
|
||||
if (nodeStorage == "raw")
|
||||
return GenerateFeaturesImpl<RawFileShortPointStorage<BasePointStorage::MODE_READ>>(info, osmFileType, osmFileName);
|
||||
else if (nodeStorage == "map")
|
||||
return GenerateFeaturesImpl<MapFileShortPointStorage<BasePointStorage::MODE_READ>>(info, osmFileType, osmFileName);
|
||||
else if (nodeStorage == "mem")
|
||||
return GenerateFeaturesImpl<RawMemShortPointStorage<BasePointStorage::MODE_READ>>(info, osmFileType, osmFileName);
|
||||
else
|
||||
CHECK(nodeStorage.empty(), ("Incorrect node_storage type:", nodeStorage));
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
bool GenerateIntermediateData(string const & dir, string const & nodeStorage, string const &osmFileType, string const & osmFileName)
|
||||
{
|
||||
if (nodeStorage == "raw")
|
||||
return GenerateIntermediateDataImpl<RawFileShortPointStorage<BasePointStorage::MODE_WRITE>>(dir, osmFileType, osmFileName);
|
||||
else if (nodeStorage == "map")
|
||||
return GenerateIntermediateDataImpl<MapFileShortPointStorage<BasePointStorage::MODE_WRITE>>(dir, osmFileType, osmFileName);
|
||||
else if (nodeStorage == "mem")
|
||||
return GenerateIntermediateDataImpl<RawMemShortPointStorage<BasePointStorage::MODE_WRITE>>(dir, osmFileType, osmFileName);
|
||||
else
|
||||
CHECK(nodeStorage.empty(), ("Incorrect node_storage type:", nodeStorage));
|
||||
return false;
|
||||
}
|
||||
|
||||
11
generator/osm_source.hpp
Normal file
11
generator/osm_source.hpp
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
#pragma once
|
||||
|
||||
#include "generate_info.hpp"
|
||||
|
||||
#include "../std/string.hpp"
|
||||
|
||||
|
||||
bool GenerateFeatures(feature::GenerateInfo & info, string const & nodeStorage, string const &osmFileType, string const & osmFileName);
|
||||
bool GenerateIntermediateData(string const & dir, string const & nodeStorage, string const &osmFileType, string const & osmFileName);
|
||||
|
||||
|
||||
|
|
@ -1,8 +1,9 @@
|
|||
#pragma once
|
||||
|
||||
#include "../coding/mmap_reader.hpp"
|
||||
#include <sqlite3.h>
|
||||
|
||||
#include "../std/iostream.hpp"
|
||||
#include "../std/type_traits.hpp"
|
||||
|
||||
/// Used to store all world nodes inside temporary index file.
|
||||
/// To find node by id, just calculate offset inside index file:
|
||||
|
|
@ -12,14 +13,14 @@ struct LatLon
|
|||
double lat;
|
||||
double lon;
|
||||
};
|
||||
STATIC_ASSERT(sizeof(LatLon) == 16);
|
||||
static_assert(sizeof(LatLon) == 16, "Invalid structure size");
|
||||
|
||||
struct ShortLatLon
|
||||
{
|
||||
int32_t lat;
|
||||
int32_t lon;
|
||||
};
|
||||
STATIC_ASSERT(sizeof(ShortLatLon) == 8);
|
||||
static_assert(sizeof(ShortLatLon) == 8, "Invalid structure size");
|
||||
|
||||
struct LatLonPos
|
||||
{
|
||||
|
|
@ -27,7 +28,7 @@ struct LatLonPos
|
|||
double lat;
|
||||
double lon;
|
||||
};
|
||||
STATIC_ASSERT(sizeof(LatLonPos) == 24);
|
||||
static_assert(sizeof(LatLonPos) == 24, "Invalid structure size");
|
||||
|
||||
struct ShortLatLonPos
|
||||
{
|
||||
|
|
@ -35,7 +36,7 @@ struct ShortLatLonPos
|
|||
int32_t lat;
|
||||
int32_t lon;
|
||||
};
|
||||
STATIC_ASSERT(sizeof(ShortLatLonPos) == 16);
|
||||
static_assert(sizeof(ShortLatLonPos) == 16, "Invalid structure size");
|
||||
|
||||
|
||||
class BasePointStorage
|
||||
|
|
@ -53,7 +54,7 @@ protected:
|
|||
public:
|
||||
enum EStorageMode {MODE_READ = false, MODE_WRITE = true};
|
||||
|
||||
BasePointStorage(string const & name, size_t factor)
|
||||
BasePointStorage(string const & name, size_t factor = 1000)
|
||||
{
|
||||
m_progress.Begin(name, factor);
|
||||
}
|
||||
|
|
@ -61,155 +62,22 @@ public:
|
|||
uint64_t GetCount() const { return m_progress.GetCount(); }
|
||||
};
|
||||
|
||||
template < BasePointStorage::EStorageMode ModeT >
|
||||
class SQLitePointStorage : public BasePointStorage
|
||||
{
|
||||
sqlite3 *m_db;
|
||||
sqlite3_stmt *m_prepared_statement;
|
||||
|
||||
public:
|
||||
SQLitePointStorage(string const & name) : BasePointStorage(name, 1000)
|
||||
{
|
||||
if( sqlite3_open((name+".sqlite").c_str(), &m_db) != SQLITE_OK ){
|
||||
fprintf(stderr, "Can't open database: %s\n", sqlite3_errmsg(m_db));
|
||||
sqlite3_close(m_db);
|
||||
exit(1);
|
||||
}
|
||||
InitStorage(EnableIf<ModeT>());
|
||||
}
|
||||
|
||||
~SQLitePointStorage()
|
||||
{
|
||||
DoneStorage(EnableIf<ModeT>());
|
||||
sqlite3_finalize(m_prepared_statement);
|
||||
sqlite3_close(m_db);
|
||||
}
|
||||
|
||||
void InitStorage(EnableIf<MODE_WRITE>)
|
||||
{
|
||||
string create_table("drop table if exists points; drop index if exists points_idx; create table points(id integer PRIMARY KEY, ll blob) WITHOUT ROWID;");
|
||||
if( sqlite3_exec(m_db, create_table.c_str(), NULL, NULL, NULL ) != SQLITE_OK ){
|
||||
fprintf(stderr, "Can't open database: %s\n", sqlite3_errmsg(m_db));
|
||||
sqlite3_close(m_db);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
char* errorMessage;
|
||||
sqlite3_exec(m_db, "PRAGMA synchronous=OFF", NULL, NULL, &errorMessage);
|
||||
sqlite3_exec(m_db, "PRAGMA count_changes=OFF", NULL, NULL, &errorMessage);
|
||||
sqlite3_exec(m_db, "PRAGMA journal_mode=MEMORY", NULL, NULL, &errorMessage);
|
||||
sqlite3_exec(m_db, "PRAGMA temp_store=MEMORY", NULL, NULL, &errorMessage);
|
||||
|
||||
string insert("insert into points(id, ll) values(?,?);");
|
||||
if( sqlite3_prepare_v2(m_db, insert.c_str(), -1, &m_prepared_statement, NULL) != SQLITE_OK ){
|
||||
fprintf(stderr, "Can't open database: %s\n", sqlite3_errmsg(m_db));
|
||||
sqlite3_close(m_db);
|
||||
exit(1);
|
||||
}
|
||||
if( sqlite3_exec(m_db, "BEGIN TRANSACTION", NULL, NULL, NULL ) != SQLITE_OK ){
|
||||
fprintf(stderr, "Can't start transaction: %s\n", sqlite3_errmsg(m_db));
|
||||
sqlite3_close(m_db);
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
void InitStorage(EnableIf<MODE_READ>)
|
||||
{
|
||||
string select("select ll from points where id=?;");
|
||||
if( sqlite3_prepare_v2(m_db, select.c_str(), -1, &m_prepared_statement, NULL) != SQLITE_OK ){
|
||||
fprintf(stderr, "failed sqlite3_prepare_v2: %s\n", sqlite3_errmsg(m_db));
|
||||
sqlite3_close(m_db);
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
void DoneStorage(EnableIf<MODE_WRITE>)
|
||||
{
|
||||
if( sqlite3_exec(m_db, "COMMIT TRANSACTION", NULL, NULL, NULL ) != SQLITE_OK ){
|
||||
fprintf(stderr, "Can't end transaction: %s\n", sqlite3_errmsg(m_db));
|
||||
sqlite3_close(m_db);
|
||||
exit(1);
|
||||
}
|
||||
if( sqlite3_exec(m_db, "create unique index points_idx on points(id);", NULL, NULL, NULL ) != SQLITE_OK ){
|
||||
fprintf(stderr, "Can't end transaction: %s\n", sqlite3_errmsg(m_db));
|
||||
sqlite3_close(m_db);
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
void DoneStorage(EnableIf<MODE_READ>) {}
|
||||
|
||||
void AddPoint(uint64_t id, double lat, double lng)
|
||||
{
|
||||
LatLon ll = {lat, lng};
|
||||
|
||||
if (sqlite3_bind_int64(m_prepared_statement, 1, id) != SQLITE_OK) {
|
||||
cerr << "bind1 failed: " << sqlite3_errmsg(m_db) << endl;
|
||||
exit(1);
|
||||
}
|
||||
if (sqlite3_bind_blob(m_prepared_statement, 2, &ll, sizeof(ll), SQLITE_STATIC) != SQLITE_OK) {
|
||||
cerr << "bind2 failed: " << sqlite3_errmsg(m_db) << endl;
|
||||
exit(1);
|
||||
}
|
||||
|
||||
if (sqlite3_step(m_prepared_statement) != SQLITE_DONE) {
|
||||
cerr << "execution failed: " << sqlite3_errmsg(m_db) << endl;
|
||||
exit(1);
|
||||
}
|
||||
|
||||
sqlite3_reset(m_prepared_statement);
|
||||
m_progress.Inc();
|
||||
}
|
||||
|
||||
bool GetPoint(uint64_t id, double & lat, double & lng) const
|
||||
{
|
||||
bool found = true;
|
||||
if (sqlite3_bind_int64(m_prepared_statement, 1, id) != SQLITE_OK) {
|
||||
cerr << "bind1 failed: " << sqlite3_errmsg(m_db) << endl;
|
||||
exit(1);
|
||||
}
|
||||
int rc = sqlite3_step(m_prepared_statement);
|
||||
if (rc == SQLITE_DONE)
|
||||
{
|
||||
found = false;
|
||||
}
|
||||
else if (rc == SQLITE_ROW)
|
||||
{
|
||||
const void * data;
|
||||
data = sqlite3_column_blob (m_prepared_statement, 0);
|
||||
LatLon const &ll = *((LatLon const *)data);
|
||||
lat = ll.lat;
|
||||
lng = ll.lon;
|
||||
}
|
||||
else
|
||||
{
|
||||
cerr << "execution failed: " << sqlite3_errmsg(m_db) << endl;
|
||||
exit(1);
|
||||
}
|
||||
|
||||
sqlite3_reset(m_prepared_statement);
|
||||
return found;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
template < BasePointStorage::EStorageMode ModeT >
|
||||
class RawFilePointStorage : public BasePointStorage
|
||||
{
|
||||
|
||||
#ifdef OMIM_OS_WINDOWS
|
||||
typedef FileReader FileReaderT;
|
||||
#else
|
||||
typedef MmapReader FileReaderT;
|
||||
#endif
|
||||
|
||||
typename std::conditional<ModeT, FileWriter, FileReaderT>::type m_file;
|
||||
typename conditional<ModeT, FileWriter, FileReaderT>::type m_file;
|
||||
|
||||
public:
|
||||
RawFilePointStorage(string const & name) : BasePointStorage(name, 1000), m_file(name) {}
|
||||
RawFilePointStorage(string const & name) : BasePointStorage(name), m_file(name) {}
|
||||
|
||||
template <bool T = (ModeT == BasePointStorage::MODE_WRITE)>
|
||||
typename std::enable_if<T, void>::type AddPoint(uint64_t id, double lat, double lng)
|
||||
typename enable_if<T, void>::type AddPoint(uint64_t id, double lat, double lng)
|
||||
{
|
||||
LatLon ll;
|
||||
ll.lat = lat;
|
||||
|
|
@ -221,16 +89,10 @@ public:
|
|||
}
|
||||
|
||||
template <bool T = (ModeT == BasePointStorage::MODE_READ)>
|
||||
typename std::enable_if<T, bool>::type GetPoint(uint64_t id, double & lat, double & lng) const
|
||||
typename enable_if<T, bool>::type GetPoint(uint64_t id, double & lat, double & lng) const
|
||||
{
|
||||
// I think, it's not good idea to write this ugly code.
|
||||
// memcpy isn't to slow for that.
|
||||
//#ifdef OMIM_OS_WINDOWS
|
||||
LatLon ll;
|
||||
m_file.Read(id * sizeof(ll), &ll, sizeof(ll));
|
||||
//#else
|
||||
// LatLon const & ll = *reinterpret_cast<LatLon const *>(m_file.Data() + id * sizeof(ll));
|
||||
//#endif
|
||||
|
||||
// assume that valid coordinate is not (0, 0)
|
||||
if (ll.lat != 0.0 || ll.lon != 0.0)
|
||||
|
|
@ -251,25 +113,24 @@ public:
|
|||
template < BasePointStorage::EStorageMode ModeT >
|
||||
class RawFileShortPointStorage : public BasePointStorage
|
||||
{
|
||||
|
||||
#ifdef OMIM_OS_WINDOWS
|
||||
typedef FileReader FileReaderT;
|
||||
#else
|
||||
typedef MmapReader FileReaderT;
|
||||
#endif
|
||||
|
||||
typename std::conditional<ModeT, FileWriter, FileReaderT>::type m_file;
|
||||
typename conditional<ModeT, FileWriter, FileReaderT>::type m_file;
|
||||
|
||||
double const m_precision = 10000000;
|
||||
constexpr static double const kValueOrder = 1E+7;
|
||||
|
||||
public:
|
||||
RawFileShortPointStorage(string const & name) : BasePointStorage(name, 1000), m_file(name) {}
|
||||
RawFileShortPointStorage(string const & name) : BasePointStorage(name), m_file(name) {}
|
||||
|
||||
template <bool T = (ModeT == BasePointStorage::MODE_WRITE)>
|
||||
typename std::enable_if<T, void>::type AddPoint(uint64_t id, double lat, double lng)
|
||||
typename enable_if<T, void>::type AddPoint(uint64_t id, double lat, double lng)
|
||||
{
|
||||
int64_t const lat64 = lat * m_precision;
|
||||
int64_t const lng64 = lng * m_precision;
|
||||
int64_t const lat64 = lat * kValueOrder;
|
||||
int64_t const lng64 = lng * kValueOrder;
|
||||
|
||||
ShortLatLon ll;
|
||||
ll.lat = static_cast<int32_t>(lat64);
|
||||
|
|
@ -284,22 +145,16 @@ public:
|
|||
}
|
||||
|
||||
template <bool T = (ModeT == BasePointStorage::MODE_READ)>
|
||||
typename std::enable_if<T, bool>::type GetPoint(uint64_t id, double & lat, double & lng) const
|
||||
typename enable_if<T, bool>::type GetPoint(uint64_t id, double & lat, double & lng) const
|
||||
{
|
||||
// I think, it's not good idea to write this ugly code.
|
||||
// memcpy isn't to slow for that.
|
||||
//#ifdef OMIM_OS_WINDOWS
|
||||
ShortLatLon ll;
|
||||
m_file.Read(id * sizeof(ll), &ll, sizeof(ll));
|
||||
//#else
|
||||
// LatLon const & ll = *reinterpret_cast<LatLon const *>(m_file.Data() + id * sizeof(ll));
|
||||
//#endif
|
||||
|
||||
// assume that valid coordinate is not (0, 0)
|
||||
if (ll.lat != 0.0 || ll.lon != 0.0)
|
||||
{
|
||||
lat = static_cast<double>(ll.lat) / m_precision;
|
||||
lng = static_cast<double>(ll.lon) / m_precision;
|
||||
lat = static_cast<double>(ll.lat) / kValueOrder;
|
||||
lng = static_cast<double>(ll.lon) / kValueOrder;
|
||||
return true;
|
||||
}
|
||||
else
|
||||
|
|
@ -314,15 +169,17 @@ public:
|
|||
template < BasePointStorage::EStorageMode ModeT >
|
||||
class RawMemShortPointStorage : public BasePointStorage
|
||||
{
|
||||
typename std::conditional<ModeT, FileWriter, FileReader>::type m_file;
|
||||
typename conditional<ModeT, FileWriter, FileReader>::type m_file;
|
||||
|
||||
double const m_precision = 10000000;
|
||||
constexpr static double const kValueOrder = 1E+7;
|
||||
|
||||
vector<ShortLatLon> m_data;
|
||||
typedef ShortLatLon LonLatT;
|
||||
|
||||
vector<LonLatT> m_data;
|
||||
|
||||
public:
|
||||
RawMemShortPointStorage(string const & name)
|
||||
: BasePointStorage(name, 1000)
|
||||
: BasePointStorage(name)
|
||||
, m_file(name)
|
||||
, m_data((size_t)0xFFFFFFFF)
|
||||
{
|
||||
|
|
@ -338,21 +195,21 @@ public:
|
|||
|
||||
void InitStorage(EnableIf<MODE_READ>)
|
||||
{
|
||||
m_file.Read(0, m_data.data(), m_data.size());
|
||||
m_file.Read(0, m_data.data(), m_data.size() * sizeof(LonLatT));
|
||||
}
|
||||
|
||||
void DoneStorage(EnableIf<MODE_WRITE>)
|
||||
{
|
||||
m_file.Write(m_data.data(), m_data.size());
|
||||
m_file.Write(m_data.data(), m_data.size() * sizeof(LonLatT));
|
||||
}
|
||||
|
||||
void DoneStorage(EnableIf<MODE_READ>) {}
|
||||
|
||||
template <bool T = (ModeT == BasePointStorage::MODE_WRITE)>
|
||||
typename std::enable_if<T, void>::type AddPoint(uint64_t id, double lat, double lng)
|
||||
typename enable_if<T, void>::type AddPoint(uint64_t id, double lat, double lng)
|
||||
{
|
||||
int64_t const lat64 = lat * m_precision;
|
||||
int64_t const lng64 = lng * m_precision;
|
||||
int64_t const lat64 = lat * kValueOrder;
|
||||
int64_t const lng64 = lng * kValueOrder;
|
||||
|
||||
ShortLatLon & ll = m_data[id];
|
||||
ll.lat = static_cast<int32_t>(lat64);
|
||||
|
|
@ -364,14 +221,14 @@ public:
|
|||
}
|
||||
|
||||
template <bool T = (ModeT == BasePointStorage::MODE_READ)>
|
||||
typename std::enable_if<T, bool>::type GetPoint(uint64_t id, double & lat, double & lng) const
|
||||
typename enable_if<T, bool>::type GetPoint(uint64_t id, double & lat, double & lng) const
|
||||
{
|
||||
ShortLatLon const & ll = m_data[id];
|
||||
// assume that valid coordinate is not (0, 0)
|
||||
if (ll.lat != 0.0 || ll.lon != 0.0)
|
||||
{
|
||||
lat = static_cast<double>(ll.lat) / m_precision;
|
||||
lng = static_cast<double>(ll.lon) / m_precision;
|
||||
lat = static_cast<double>(ll.lat) / kValueOrder;
|
||||
lng = static_cast<double>(ll.lon) / kValueOrder;
|
||||
return true;
|
||||
}
|
||||
else
|
||||
|
|
@ -382,13 +239,10 @@ public:
|
|||
}
|
||||
};
|
||||
|
||||
|
||||
template < BasePointStorage::EStorageMode ModeT >
|
||||
class MapFilePointStorage : public BasePointStorage
|
||||
{
|
||||
typename std::conditional<ModeT, FileWriter, FileReader>::type m_file;
|
||||
|
||||
|
||||
typename conditional<ModeT, FileWriter, FileReader>::type m_file;
|
||||
typedef unordered_map<uint64_t, pair<double, double> > ContainerT;
|
||||
ContainerT m_map;
|
||||
|
||||
|
|
@ -412,7 +266,7 @@ public:
|
|||
LatLonPos ll;
|
||||
m_file.Read(pos, &ll, sizeof(ll));
|
||||
|
||||
(void)m_map.insert(make_pair(ll.pos, make_pair(ll.lat, ll.lon)));
|
||||
m_map.emplace(make_pair(ll.pos, make_pair(ll.lat, ll.lon)));
|
||||
|
||||
pos += sizeof(ll);
|
||||
}
|
||||
|
|
@ -434,27 +288,22 @@ public:
|
|||
bool GetPoint(uint64_t id, double & lat, double & lng) const
|
||||
{
|
||||
auto i = m_map.find(id);
|
||||
if (i != m_map.end())
|
||||
{
|
||||
lat = i->second.first;
|
||||
lng = i->second.second;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
if (i == m_map.end())
|
||||
return false;
|
||||
lat = i->second.first;
|
||||
lng = i->second.second;
|
||||
return true;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
template < BasePointStorage::EStorageMode ModeT >
|
||||
class MapFileShortPointStorage : public BasePointStorage
|
||||
{
|
||||
typename std::conditional<ModeT, FileWriter, FileReader>::type m_file;
|
||||
|
||||
|
||||
typename conditional<ModeT, FileWriter, FileReader>::type m_file;
|
||||
typedef unordered_map<uint64_t, pair<int32_t, int32_t> > ContainerT;
|
||||
ContainerT m_map;
|
||||
|
||||
double const m_precision = 10000000;
|
||||
constexpr static double const kValueOrder = 1E+7;
|
||||
|
||||
public:
|
||||
MapFileShortPointStorage(string const & name) : BasePointStorage(name, 10000), m_file(name+".short")
|
||||
|
|
@ -476,7 +325,7 @@ public:
|
|||
ShortLatLonPos ll;
|
||||
m_file.Read(pos, &ll, sizeof(ll));
|
||||
|
||||
(void)m_map.insert(make_pair(ll.pos, make_pair(ll.lat, ll.lon)));
|
||||
m_map.emplace(make_pair(ll.pos, make_pair(ll.lat, ll.lon)));
|
||||
|
||||
pos += sizeof(ll);
|
||||
}
|
||||
|
|
@ -487,10 +336,8 @@ public:
|
|||
void AddPoint(uint64_t id, double lat, double lng)
|
||||
{
|
||||
|
||||
int64_t const lat64 = lat * m_precision;
|
||||
int64_t const lng64 = lng * m_precision;
|
||||
|
||||
|
||||
int64_t const lat64 = lat * kValueOrder;
|
||||
int64_t const lng64 = lng * kValueOrder;
|
||||
|
||||
ShortLatLonPos ll;
|
||||
ll.pos = id;
|
||||
|
|
@ -506,13 +353,11 @@ public:
|
|||
bool GetPoint(uint64_t id, double & lat, double & lng) const
|
||||
{
|
||||
auto i = m_map.find(id);
|
||||
if (i != m_map.end())
|
||||
{
|
||||
lat = static_cast<double>(i->second.first) / m_precision;
|
||||
lng = static_cast<double>(i->second.second) / m_precision;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
if (i == m_map.end())
|
||||
return false;
|
||||
lat = static_cast<double>(i->second.first) / kValueOrder;
|
||||
lng = static_cast<double>(i->second.second) / kValueOrder;
|
||||
return true;
|
||||
}
|
||||
|
||||
};
|
||||
|
|
|
|||
|
|
@ -18,6 +18,29 @@ void XMLElement::AddKV(string const & k, string const & v)
|
|||
e.parent = this;
|
||||
}
|
||||
|
||||
void XMLElement::AddND(uint64_t ref)
|
||||
{
|
||||
childs.push_back(XMLElement());
|
||||
XMLElement & e = childs.back();
|
||||
|
||||
e.tagKey = ET_ND;
|
||||
e.ref = ref;
|
||||
e.parent = this;
|
||||
}
|
||||
|
||||
void XMLElement::AddMEMBER(uint64_t ref, string const & type, string const & role)
|
||||
{
|
||||
childs.push_back(XMLElement());
|
||||
XMLElement & e = childs.back();
|
||||
|
||||
e.tagKey = ET_MEMBER;
|
||||
e.ref = ref;
|
||||
e.type = type;
|
||||
e.role = role;
|
||||
e.parent = this;
|
||||
}
|
||||
|
||||
|
||||
void BaseOSMParser::AddAttr(string const & key, string const & value)
|
||||
{
|
||||
if (m_current)
|
||||
|
|
@ -101,46 +124,3 @@ void BaseOSMParser::Pop(string const &)
|
|||
(*m_current) = XMLElement();
|
||||
}
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
struct StdinReader
|
||||
{
|
||||
uint64_t Read(char * buffer, uint64_t bufferSize)
|
||||
{
|
||||
return fread(buffer, sizeof(char), bufferSize, stdin);
|
||||
}
|
||||
};
|
||||
|
||||
struct FileReader
|
||||
{
|
||||
FILE * m_file;
|
||||
|
||||
FileReader(string const & filename)
|
||||
{
|
||||
m_file = fopen(filename.c_str(), "rb");
|
||||
}
|
||||
|
||||
~FileReader()
|
||||
{
|
||||
fclose(m_file);
|
||||
}
|
||||
|
||||
uint64_t Read(char * buffer, uint64_t bufferSize)
|
||||
{
|
||||
return fread(buffer, sizeof(char), bufferSize, m_file);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
void ParseXMLFromStdIn(BaseOSMParser & parser)
|
||||
{
|
||||
StdinReader reader;
|
||||
(void)ParseXMLSequence(reader, parser);
|
||||
}
|
||||
|
||||
void ParseXMLFromFile(BaseOSMParser & parser, string const & osmFileName)
|
||||
{
|
||||
FileReader reader(osmFileName);
|
||||
(void)ParseXMLSequence(reader, parser);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,6 +34,9 @@ struct XMLElement
|
|||
vector<XMLElement> childs;
|
||||
|
||||
void AddKV(string const & k, string const & v);
|
||||
void AddND(uint64_t ref);
|
||||
void AddMEMBER(uint64_t ref, string const & type, string const & role);
|
||||
|
||||
};
|
||||
|
||||
class BaseOSMParser
|
||||
|
|
@ -53,10 +56,8 @@ public:
|
|||
void Pop(string const &);
|
||||
void CharData(string const &) {}
|
||||
|
||||
virtual void EmitElement(XMLElement * p) = 0;
|
||||
|
||||
protected:
|
||||
bool MatchTag(string const & tagName, XMLElement::ETag & tagKey);
|
||||
virtual void EmitElement(XMLElement * p) = 0;
|
||||
};
|
||||
|
||||
void ParseXMLFromStdIn(BaseOSMParser & parser);
|
||||
void ParseXMLFromFile(BaseOSMParser & parser, string const & osmFileName);
|
||||
|
|
|
|||
|
|
@ -14,18 +14,18 @@
|
|||
#endif
|
||||
#endif
|
||||
|
||||
#include <boost/type_traits.hpp>
|
||||
#include <boost/utility/enable_if.hpp>
|
||||
#include <type_traits>
|
||||
|
||||
using boost::enable_if;
|
||||
using std::enable_if;
|
||||
using std::conditional;
|
||||
|
||||
using boost::is_same;
|
||||
using boost::make_signed;
|
||||
using boost::make_unsigned;
|
||||
using boost::is_signed;
|
||||
using boost::is_unsigned;
|
||||
using boost::is_floating_point;
|
||||
using boost::is_integral;
|
||||
using std::is_same;
|
||||
using std::make_signed;
|
||||
using std::make_unsigned;
|
||||
using std::is_signed;
|
||||
using std::is_unsigned;
|
||||
using std::is_floating_point;
|
||||
using std::is_integral;
|
||||
|
||||
#ifdef DEBUG_NEW
|
||||
#define new DEBUG_NEW
|
||||
|
|
|
|||
|
|
@ -108,11 +108,11 @@ function merge_coasts() {
|
|||
log "TIMEMARK" "Filter coastlines done"
|
||||
# Preprocess coastlines to separate intermediate directory
|
||||
log "TIMEMARK" "Generate coastlines intermediate"
|
||||
$CONVERT_TOOL $COASTS_FILE | $GENERATOR_TOOL -intermediate_data_path=$INTCOASTSDIR -node_storage=map -preprocess_xml
|
||||
$GENERATOR_TOOL -intermediate_data_path=$INTCOASTSDIR -node_storage=map -preprocess --osm_file_type=o5m --osm_file_name=$COASTS_FILE
|
||||
log "TIMEMARK" "Generate coastlines intermediate done"
|
||||
# Generate temporary coastlines file in the coasts intermediate dir
|
||||
log "TIMEMARK" "Generate coastlines"
|
||||
$CONVERT_TOOL $COASTS_FILE | $GENERATOR_TOOL -intermediate_data_path=$INTCOASTSDIR -node_storage=map -make_coasts -fail_on_coasts=$FAIL_ON_COASTS
|
||||
$GENERATOR_TOOL -intermediate_data_path=$INTCOASTSDIR -node_storage=map -make_coasts -fail_on_coasts=$FAIL_ON_COASTS --osm_file_type=o5m --osm_file_name=$COASTS_FILE
|
||||
log "TIMEMARK" "Generate coastlines done"
|
||||
}
|
||||
|
||||
|
|
@ -163,20 +163,20 @@ fi
|
|||
# make a working copy of generated coastlines file
|
||||
cp $INTCOASTSDIR/WorldCoasts.mwm.tmp $INTDIR/WorldCoasts.mwm.tmp
|
||||
|
||||
NODE_STORAGE=raw
|
||||
NODE_STORAGE=mem
|
||||
|
||||
if [[ $1 == "--generate" || $1 == "--full" ]]; then
|
||||
log "TIMEMARK" "Generate intermediate data"
|
||||
# 1st pass, run in parallel - preprocess whole planet to speed up generation if all coastlines are correct
|
||||
$CONVERT_TOOL $PLANET_FILE | $GENERATOR_TOOL -intermediate_data_path=$INTDIR -node_storage=$NODE_STORAGE -preprocess_xml
|
||||
$GENERATOR_TOOL -intermediate_data_path=$INTDIR -node_storage=$NODE_STORAGE -preprocess --osm_file_type=o5m --osm_file_name=$PLANET_FILE
|
||||
log "TIMEMARK" "Generate intermediate data done"
|
||||
fi
|
||||
|
||||
if [[ $1 == "--generate" || $1 == "--continue" || $1 == "--full" ]]; then
|
||||
# 2nd pass - paralleled in the code
|
||||
log "TIMEMARK" "Generate features"
|
||||
$CONVERT_TOOL $PLANET_FILE | $GENERATOR_TOOL -intermediate_data_path=$INTDIR \
|
||||
-node_storage=$NODE_STORAGE -split_by_polygons \
|
||||
$GENERATOR_TOOL -intermediate_data_path=$INTDIR \
|
||||
--osm_file_type=o5m --osm_file_name=$PLANET_FILE -node_storage=$NODE_STORAGE -split_by_polygons \
|
||||
-generate_features -generate_world \
|
||||
-data_path=$DATA_PATH -emit_coasts
|
||||
log "TIMEMARK" "Generate features done"
|
||||
|
|
|
|||
|
|
@ -21,4 +21,8 @@ mkdir $OUT_DIR || true
|
|||
|
||||
$GENERATOR_TOOL -export_poly_path $POLY_FILES_PATH
|
||||
|
||||
ls $POLY_FILES_PATH | parallel -t -v "$OSMCONVERT_TOOL $PLANET_FILE --hash-memory=2000 -B=$POLY_FILES_PATH/{} --complex-ways --out-pbf -o=$OUT_DIR/{.}.pbf"
|
||||
EXT=.poly
|
||||
NUM_INSTANCES=8
|
||||
|
||||
COUNTRY_LIST=${COUNTRY_LIST-$(ls -1 $POLY_FILES_PATH/*$EXT | xargs -d "\n" basename -s $EXT)}
|
||||
echo "$COUNTRY_LIST" | xargs -d "\n" -P $NUM_INSTANCES -I % $OSMCONVERT_TOOL $PLANET_FILE --hash-memory=2000 -B=$POLY_FILES_PATH/%.poly --complex-ways --out-pbf -o=$OUT_DIR/%.pbf &>~/split_planet_osmconvert.log
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue