organicmaps/osmctools
13
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity Actions

Updated tools to the most recent versions

Browse source
This commit is contained in:
Alexander Borsuk 2020-12-24 20:33:27 +01:00
parent 189f70dec1
commit 922d13544d
3 changed files with 82 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

68
osmconvert.c
View file

@ -1,10 +1,10 @@
// osmconvert 2017-06-18 21:30 // osmconvert 2020-03-31 14:20
#define VERSION "0.8.8" #define VERSION "0.8.11"
// //
// compile this file: // compile this file:
// gcc osmconvert.c -lz -O3 -o osmconvert // gcc osmconvert.c -lz -O3 -o osmconvert
// //
// (c) 2011..2017 Markus Weber, Nuernberg // (c) 2011..2020 Markus Weber, Nuernberg
// Richard Russo contributed the initiative to --add-bbox-tags option // Richard Russo contributed the initiative to --add-bbox-tags option
// //
// This program is free software; you can redistribute it and/or // This program is free software; you can redistribute it and/or
@ -387,15 +387,15 @@ const char* helptext=
"Tuning\n" "Tuning\n"
"\n" "\n"
"To speed-up the process, the program uses some main memory for a\n" "To speed-up the process, the program uses some main memory for a\n"
"hash table. By default, it uses 900 MB for storing a flag for every\n" "hash table. By default, it uses 1800 MB for storing a flag for every\n"
"possible node, 90 for the way flags, and 10 relation flags.\n" "possible node, 180 for the way flags, and 20 relation flags.\n"
"Every byte holds the flags for 8 ID numbers, i.e., in 900 MB the\n" "Every byte holds the flags for 8 ID numbers, i.e., in 1800 MB the\n"
"program can store 7200 million flags. As there are less than 3200\n" "program can store 14400 million flags. As there are less than 7400\n"
"million IDs for nodes at present (Oct 2014), 400 MB would suffice.\n" "million IDs for nodes at present (Mar 2020), 925 MB would suffice.\n"
"So, for example, you can decrease the hash sizes to e.g. 400, 50 and\n" "So, for example, you can decrease the hash sizes to e.g. 1000, 120\n"
"2 MB using this option:\n" "and 4 MB using this option:\n"
"\n" "\n"
" --hash-memory=400-50-2\n" " --hash-memory=1000-120-4\n"
"\n" "\n"
"But keep in mind that the OSM database is continuously expanding. For\n" "But keep in mind that the OSM database is continuously expanding. For\n"
"this reason the program-own default value is higher than shown in the\n" "this reason the program-own default value is higher than shown in the\n"
@ -404,10 +404,10 @@ const char* helptext=
"amount of memory as a sum, and the program will divide it by itself.\n" "amount of memory as a sum, and the program will divide it by itself.\n"
"For example:\n" "For example:\n"
"\n" "\n"
" --hash-memory=1500\n" " --hash-memory=3000\n"
"\n" "\n"
"These 1500 MB will be split in three parts: 1350 for nodes, 135 for\n" "These 3000 MB will be split in three parts: 2700 for nodes, 270 for\n"
"ways, and 15 for relations.\n" "ways, and 30 for relations.\n"
"\n" "\n"
"Because we are taking hashes, it is not necessary to provide all the\n" "Because we are taking hashes, it is not necessary to provide all the\n"
"suggested memory; the program will operate with less hash memory too.\n" "suggested memory; the program will operate with less hash memory too.\n"
@ -456,7 +456,7 @@ const char* helptext=
"loss. Do not use the program in productive or commercial systems.\n" "loss. Do not use the program in productive or commercial systems.\n"
"\n" "\n"
"There is NO WARRANTY, to the extent permitted by law.\n" "There is NO WARRANTY, to the extent permitted by law.\n"
"Please send any bug reports to markus.weber@gmx.com\n\n"; "Please send any bug reports to marqqs@gmx.eu\n\n";
#define _FILE_OFFSET_BITS 64 #define _FILE_OFFSET_BITS 64
#include <zlib.h> #include <zlib.h>
@ -1621,7 +1621,7 @@ static int hash_ini(int n,int w,int r) {
return 0; // ignore the call of this procedure return 0; // ignore the call of this procedure
// check parameters and store the values // check parameters and store the values
#define D(x,o) if(x<1) x= 1; else if(x>4000) x= 4000; \ #define D(x,o) if(x<1) x= 1; else if(x>4000) x= 4000; \
hash__max[o]= x*(1024*1024); hash__max[o]= x*(1024u*1024u);
D(n,0u) D(w,1u) D(r,2u) D(n,0u) D(w,1u) D(r,2u)
#undef D #undef D
// allocate memory for each hash table // allocate memory for each hash table
@ -11273,10 +11273,10 @@ return 23;
keyp= key; valp= val; keyp= key; valp= val;
while(keyp<keye) { // for all key/val pairs of this object while(keyp<keye) { // for all key/val pairs of this object
if(strcmp(*keyp,"type")==0) { if(strcmp(*keyp,"type")==0) {
if((global_completemp && if(global_completemp &&
strcmp(*valp,"multipolygon")==0) || strcmp(*valp,"multipolygon")==0 ||
(global_completeboundaries && global_completeboundaries &&
strcmp(*valp,"boundary")==0)) strcmp(*valp,"boundary")==0)
ismp= true; ismp= true;
break; break;
} }
@ -11523,8 +11523,8 @@ return 26;
keyp= key; valp= val; keyp= key; valp= val;
while(keyp<keye) { // for all key/val pairs of this object while(keyp<keye) { // for all key/val pairs of this object
if(modi_CHECK(otype,*keyp,*valp)) { if(modi_CHECK(otype,*keyp,*valp)) {
if(modi_check_add) wo_node_keyval(*keyp++,*valp++); if(modi_check_add) wo_node_keyval(*keyp,*valp);
else keyp++; valp++; keyp++; valp++;
wo_node_keyval(modi_check_key,modi_check_val); wo_node_keyval(modi_check_key,modi_check_val);
} }
else else
@ -11666,8 +11666,8 @@ return 26;
while(keyp<keye) { while(keyp<keye) {
// for all key/val pairs of this object // for all key/val pairs of this object
if(modi_CHECK(otype,*keyp,*valp)) { if(modi_CHECK(otype,*keyp,*valp)) {
if(modi_check_add) wo_node_keyval(*keyp++,*valp++); if(modi_check_add) wo_node_keyval(*keyp,*valp);
else keyp++; valp++; keyp++; valp++;
wo_node_keyval(modi_check_key,modi_check_val); wo_node_keyval(modi_check_key,modi_check_val);
} }
else else
@ -11691,8 +11691,8 @@ return 26;
while(keyp<keye) { while(keyp<keye) {
// for all key/val pairs of this object // for all key/val pairs of this object
if(modi_CHECK(otype,*keyp,*valp)) { if(modi_CHECK(otype,*keyp,*valp)) {
if(modi_check_add) wo_wayrel_keyval(*keyp++,*valp++); if(modi_check_add) wo_wayrel_keyval(*keyp,*valp);
else keyp++; valp++; keyp++; valp++;
wo_wayrel_keyval(modi_check_key,modi_check_val); wo_wayrel_keyval(modi_check_key,modi_check_val);
} }
else else
@ -11714,8 +11714,8 @@ return 26;
while(keyp<keye) { while(keyp<keye) {
// for all key/val pairs of this object // for all key/val pairs of this object
if(modi_CHECK(otype,*keyp,*valp)) { if(modi_CHECK(otype,*keyp,*valp)) {
if(modi_check_add) wo_wayrel_keyval(*keyp++,*valp++); if(modi_check_add) wo_wayrel_keyval(*keyp,*valp);
else keyp++; valp++; keyp++; valp++;
wo_wayrel_keyval(modi_check_key,modi_check_val); wo_wayrel_keyval(modi_check_key,modi_check_val);
} }
else else
@ -11838,8 +11838,8 @@ return 26;
while(keyp<keye) { while(keyp<keye) {
// for all key/val pairs of this object // for all key/val pairs of this object
if(modi_CHECK(otype,*keyp,*valp)) { if(modi_CHECK(otype,*keyp,*valp)) {
if(modi_check_add) wo_node_keyval(*keyp++,*valp++); if(modi_check_add) wo_node_keyval(*keyp,*valp);
else keyp++; valp++; keyp++; valp++;
wo_node_keyval(modi_check_key,modi_check_val); wo_node_keyval(modi_check_key,modi_check_val);
} }
else else
@ -11897,8 +11897,8 @@ return 26;
while(keyp<keye) { while(keyp<keye) {
// for all key/val pairs of this object // for all key/val pairs of this object
if(modi_CHECK(otype,*keyp,*valp)) { if(modi_CHECK(otype,*keyp,*valp)) {
if(modi_check_add) wo_wayrel_keyval(*keyp++,*valp++); if(modi_check_add) wo_wayrel_keyval(*keyp,*valp);
else keyp++; valp++; keyp++; valp++;
wo_wayrel_keyval(modi_check_key,modi_check_val); wo_wayrel_keyval(modi_check_key,modi_check_val);
} }
else else
@ -13145,7 +13145,9 @@ return 3;
"-b=, -B=, --drop-brokenrefs must not be combined with --diff"); "-b=, -B=, --drop-brokenrefs must not be combined with --diff");
return 6; return 6;
} }
if(h_n==0) h_n= 1000; // use standard value if not set otherwise if(h_n==0) { // use standard values if not set otherwise
h_n= 1800; h_w= 180; h_r= 20;
}
if(h_w==0 && h_r==0) { if(h_w==0 && h_r==0) {
// user chose simple form for hash memory value // user chose simple form for hash memory value
// take the one given value as reference and determine the // take the one given value as reference and determine the

57
osmfilter.c
View file

@ -1,10 +1,10 @@
// osmfilter 2017-09-22 14:00 // osmfilter 2020-03-31 14:50
#define VERSION "1.4.3" #define VERSION "1.4.6"
// //
// compile this file: // compile this file:
// gcc osmfilter.c -O3 -o osmfilter // gcc osmfilter.c -O3 -o osmfilter
// //
// (c) 2011..2017 Markus Weber, Nuernberg // (c) 2011..2020 Markus Weber, Nuernberg
// //
// This program is free software; you can redistribute it and/or // This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU Affero General Public License // modify it under the terms of the GNU Affero General Public License
@ -56,6 +56,7 @@ const char* shorthelptext=
"-h display this parameter overview\n" "-h display this parameter overview\n"
"--help display a more detailed help\n" "--help display a more detailed help\n"
"--ignore-dependencies ignore dependencies between OSM objects\n" "--ignore-dependencies ignore dependencies between OSM objects\n"
"--raw-comparison do not compare following data numerically\n"
"--out-key= write statistics (for the key, if supplied)\n" "--out-key= write statistics (for the key, if supplied)\n"
"--out-count= same as before, but sorted by occurrence\n" "--out-count= same as before, but sorted by occurrence\n"
"--out-osm write output in .osm format (default)\n" "--out-osm write output in .osm format (default)\n"
@ -204,6 +205,12 @@ const char* helptext=
" included relations. If you activate this option, all these\n" " included relations. If you activate this option, all these\n"
" dependencies between OSM objects will be ignored.\n" " dependencies between OSM objects will be ignored.\n"
"\n" "\n"
"--raw-comparison\n"
" By default, values are compared numerically if they start\n"
" with a digit. Use this option to prevent this behaviour.\n"
" Please note that this option will not apply to filter\n"
" expressions which have already been entered left to it.\n"
"\n"
"--out-key=KEYNAME\n" "--out-key=KEYNAME\n"
" The output will contain no regular OSM data but only\n" " The output will contain no regular OSM data but only\n"
" statistics: a list of all used keys is assembled. Left to\n" " statistics: a list of all used keys is assembled. Left to\n"
@ -324,15 +331,15 @@ const char* helptext=
"Tuning\n" "Tuning\n"
"\n" "\n"
"To speed-up the process, the program uses some main memory for a\n" "To speed-up the process, the program uses some main memory for a\n"
"hash table. By default, it uses 900 MB for storing a flag for every\n" "hash table. By default, it uses 1800 MB for storing a flag for every\n"
"possible node, 90 for the way flags, and 10 relation flags.\n" "possible node, 180 for the way flags, and 20 relation flags.\n"
"Every byte holds the flags for 8 ID numbers, i.e., in 900 MB the\n" "Every byte holds the flags for 8 ID numbers, i.e., in 1800 MB the\n"
"program can store 7200 million flags. As there are less than 3200\n" "program can store 14400 million flags. As there are less than 7400\n"
"million IDs for nodes at present (Oct 2014), 400 MB would suffice.\n" "million IDs for nodes at present (Mar 2020), 925 MB would suffice.\n"
"So, for example, you can decrease the hash sizes to e.g. 400, 50 and\n" "So, for example, you can decrease the hash sizes to e.g. 1000, 120\n"
"2 MB (for relations, 2 flags are needed each) using this option:\n" "and 4 MB using this option:\n"
"\n" "\n"
" --hash-memory=400-50-2\n" " --hash-memory=1000-120-4\n"
"\n" "\n"
"But keep in mind that the OSM database is continuously expanding. For\n" "But keep in mind that the OSM database is continuously expanding. For\n"
"this reason the program-own default value is higher than shown in the\n" "this reason the program-own default value is higher than shown in the\n"
@ -341,10 +348,10 @@ const char* helptext=
"amount of memory as a sum, and the program will divide it by itself.\n" "amount of memory as a sum, and the program will divide it by itself.\n"
"For example:\n" "For example:\n"
"\n" "\n"
" --hash-memory=1500\n" " --hash-memory=3000\n"
"\n" "\n"
"These 1500 MB will be split in three parts: 1350 for nodes, 135 for\n" "These 3000 MB will be split in three parts: 2700 for nodes, 270 for\n"
"ways, and 15 for relations.\n" "ways, and 30 for relations.\n"
"\n" "\n"
"Because we are taking hashes, it is not necessary to provide all the\n" "Because we are taking hashes, it is not necessary to provide all the\n"
"suggested memory; the program will operate with less hash memory too.\n" "suggested memory; the program will operate with less hash memory too.\n"
@ -370,7 +377,7 @@ const char* helptext=
"1000, the length of each key or val is limited to 100.\n" "1000, the length of each key or val is limited to 100.\n"
"\n" "\n"
"There is NO WARRANTY, to the extent permitted by law.\n" "There is NO WARRANTY, to the extent permitted by law.\n"
"Please send any bug reports to markus.weber@gmx.com\n\n"; "Please send any bug reports to marqqs@gmx.eu\n\n";
#define _FILE_OFFSET_BITS 64 #define _FILE_OFFSET_BITS 64
#include <inttypes.h> #include <inttypes.h>
@ -479,6 +486,8 @@ static char global_tempfilename[350]= "osmfilter_tempfile";
static bool global_recursive= false; // recursive processing necessary static bool global_recursive= false; // recursive processing necessary
static bool global_ignoredependencies= false; static bool global_ignoredependencies= false;
// user wants interobject dependencies to be ignored // user wants interobject dependencies to be ignored
static bool global_rawcomparison= false;
// user wants data to be compared on raw basis, not by numerical value
#define PERR(f) { static int msgn= 3; if(--msgn>=0) \ #define PERR(f) { static int msgn= 3; if(--msgn>=0) \
fprintf(stderr,"osmfilter Error: " f "\n"); } fprintf(stderr,"osmfilter Error: " f "\n"); }
// print error message // print error message
@ -892,7 +901,7 @@ static int hash_ini(int n,int w,int r) {
return 0; // ignore the call of this procedure return 0; // ignore the call of this procedure
// check parameters and store the values // check parameters and store the values
#define D(x,o) if(x<1) x= 1; else if(x>4000) x= 4000; \ #define D(x,o) if(x<1) x= 1; else if(x>4000) x= 4000; \
hash__max[o]= x*(1024*1024); hash__max[o]= x*(1024u*1024u);
D(n,0) D(w,1) D(r,2) D(r,3) D(n,0) D(w,1) D(r,2) D(r,3)
#undef D #undef D
// allocate memory for each hash table // allocate memory for each hash table
@ -2413,7 +2422,7 @@ static inline void fil_cpy(char *dest, const char *src,
v= src; v= src;
if(*v=='-') v++; // jump over sign if(*v=='-') v++; // jump over sign
if(isdig(*v)) // numeric value if(!global_rawcomparison && isdig(*v)) // numeric value
op+= 8; op+= 8;
dest[0]= op; dest[0]= op;
fil__stresccpy(dest+1,src,len); // store this value fil__stresccpy(dest+1,src,len); // store this value
@ -3499,7 +3508,7 @@ static inline void modi_cpy(char *dest, const char *src,
v= src; v= src;
if(*v=='-') v++; // jump over sign if(*v=='-') v++; // jump over sign
if(isdig(*v)) // numeric value if(!global_rawcomparison && isdig(*v)) // numeric value
op+= 8; op+= 8;
dest[0]= op; dest[0]= op;
modi__stresccpy(dest+1,src,len); // store this value modi__stresccpy(dest+1,src,len); // store this value
@ -6758,8 +6767,14 @@ return 0;
continue; // take next parameter continue; // take next parameter
} }
if(strzcmp(a,"--ignore-dep")==0) { if(strzcmp(a,"--ignore-dep")==0) {
// user does interobject dependencies to be ignored // user wants interobject dependencies to be ignored
global_ignoredependencies= true; global_ignoredependencies= true;
continue; // take next parameter
}
if(strzcmp(a,"--raw-comparison")==0) {
// user wants data to be compared on raw basis,
// not by numerical value
global_rawcomparison= true;
continue; // take next parameter continue; // take next parameter
} }
if(strcmp(argv[0],"--in-josm")==0) { if(strcmp(argv[0],"--in-josm")==0) {
@ -6967,7 +6982,9 @@ return 3;
if(global_recursive) { if(global_recursive) {
int r; int r;
if(h_n==0) h_n= 1000; // use standard value if not set otherwise if(h_n==0) { // use standard values if not set otherwise
h_n= 1800; h_w= 180; h_r= 20;
}
if(h_w==0 && h_r==0) { if(h_w==0 && h_r==0) {
// user chose simple form for hash memory value // user chose simple form for hash memory value
// take the one given value as reference and determine the // take the one given value as reference and determine the

18
osmupdate.c
View file

@ -1,10 +1,10 @@
// osmupdate 2017-02-26 16:40 // osmupdate 2018-05-27 12:00
#define VERSION "0.4.4" #define VERSION "0.4.5"
// //
// compile this file: // compile this file:
// gcc osmupdate.c -o osmupdate // gcc osmupdate.c -o osmupdate
// //
// (c) 2011..2017 Markus Weber, Nuernberg // (c) 2011..2018 Markus Weber, Nuernberg
// //
// This program is free software; you can redistribute it and/or // This program is free software; you can redistribute it and/or
// modify it under the terms of the GNU Affero General Public License // modify it under the terms of the GNU Affero General Public License
@ -111,9 +111,9 @@ const char* helptext=
" changefiles in one run. This ability increases merging speed.\n" " changefiles in one run. This ability increases merging speed.\n"
" Unfortunately, every changefile consumes about 200 MB of main\n" " Unfortunately, every changefile consumes about 200 MB of main\n"
" memory while being processed. For this reason, the number of\n" " memory while being processed. For this reason, the number of\n"
" parallely processable changefiles is limited.\n" " parallelly processable changefiles is limited.\n"
" Use this commandline argument to determine the maximum number\n" " Use this commandline argument to determine the maximum number\n"
" of parallely processed changefiles. The default value is 7.\n" " of parallelly processed changefiles. The default value is 7.\n"
"\n" "\n"
"-t=TEMPPATH\n" "-t=TEMPPATH\n"
"--tempfiles=TEMPPATH\n" "--tempfiles=TEMPPATH\n"
@ -165,7 +165,7 @@ const char* helptext=
"loss. Do not use the program in productive or commercial systems.\n" "loss. Do not use the program in productive or commercial systems.\n"
"\n" "\n"
"There is NO WARRANTY, to the extent permitted by law.\n" "There is NO WARRANTY, to the extent permitted by law.\n"
"Please send any bug reports to markus.weber@gmx.com\n\n"; "Please send any bug reports to marqqs@gmx.eu\n\n";
#define _FILE_OFFSET_BITS 64 #define _FILE_OFFSET_BITS 64
#include <inttypes.h> #include <inttypes.h>
@ -511,7 +511,7 @@ static char global_osmconvert_arguments[2000]= "";
// general command line arguments for osmconvert; // general command line arguments for osmconvert;
#define max_number_of_changefiles_in_cache 100 #define max_number_of_changefiles_in_cache 100
static int global_max_merge= 7; static int global_max_merge= 7;
// maximum number of parallely processed changefiles // maximum number of parallelly processed changefiles
static const char* global_gzip_parameters= ""; static const char* global_gzip_parameters= "";
// parameters for gzip compression // parameters for gzip compression
static char global_base_url[400]= static char global_base_url[400]=
@ -1199,7 +1199,7 @@ return 0;
continue; // take next parameter continue; // take next parameter
} }
if(strzcmp(a,"--max-merge=")==0) { if(strzcmp(a,"--max-merge=")==0) {
// maximum number of parallely processed changefiles // maximum number of parallelly processed changefiles
global_max_merge= strtouint32(a+12); global_max_merge= strtouint32(a+12);
if(global_max_merge<2) { if(global_max_merge<2) {
global_max_merge= 2; global_max_merge= 2;
@ -1280,6 +1280,8 @@ return 0;
return 1; return 1;
} }
if(strcmp(a,"--complete-ways")==0 || if(strcmp(a,"--complete-ways")==0 ||
strcmp(a,"--complete-multipolygons")==0 ||
strcmp(a,"--complete-boundaries")==0 ||
strcmp(a,"--complex-ways")==0 || strcmp(a,"--complex-ways")==0 ||
strcmp(a,"--drop-brokenrefs")==0 || strcmp(a,"--drop-brokenrefs")==0 ||
strcmp(a,"--drop-broken-refs")==0) { strcmp(a,"--drop-broken-refs")==0) {