organicmaps/name-suggestion-index
13
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity Actions

In trees schema, name tag patterns do not need to be Arrays

Browse source
This commit is contained in:
Bryan Housel 2021-03-15 11:36:34 -04:00
parent 760d040eb0
commit 21f5b4e838
13 changed files with 301 additions and 331 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
config/trees.json
View file

@ -5,20 +5,16 @@
"mainTag": "brand:wikidata",
"sourceTags": ["brand", "name"],
"nameTags": {
"primary": ["^(name|name:\\w+)$"],
"alternate": [
"^(brand|brand:\\w+|operator|operator:\\w+|\\w+_name|\\w+_name:\\w+)$"
]
"primary": "^(name|name:\\w+)$",
"alternate": "^(brand|brand:\\w+|operator|operator:\\w+|\\w+_name|\\w+_name:\\w+)$"
}
},
"flags": {
"emoji": "🚩",
"mainTag": "flag:wikidata",
"nameTags": {
"primary": ["^(flag:name|flag:name:\\w+)$"],
"alternate": [
"^(country|country:\\w+|flag|flag:\\w+|subject|subject:\\w+)$"
]
"primary": "^(flag:name|flag:name:\\w+)$",
"alternate": "^(country|country:\\w+|flag|flag:\\w+|subject|subject:\\w+)$"
}
},
"operators": {
@ -26,8 +22,8 @@
"mainTag": "operator:wikidata",
"sourceTags": ["operator"],
"nameTags": {
"primary": ["^(name|name:\\w+|operator|operator:\\w+)$"],
"alternate": ["^(brand|brand:\\w+|\\w+_name|\\w+_name:\\w+)$"]
"primary": "^(name|name:\\w+|operator|operator:\\w+)$",
"alternate": "^(brand|brand:\\w+|\\w+_name|\\w+_name:\\w+)$"
}
},
"transit": {
@ -35,10 +31,8 @@
"mainTag": "network:wikidata",
"sourceTags": ["network"],
"nameTags": {
"primary": ["^network$"],
"alternate": [
"^(operator|operator:\\w+|network:\\w+|\\w+_name|\\w+_name:\\w+)$"
]
"primary": "^network$",
"alternate": "^(operator|operator:\\w+|network:\\w+|\\w+_name|\\w+_name:\\w+)$"
}
}
}

29
dist/index.js vendored
View file

@ -350,7 +350,7 @@
genericWords: genericWords
};
var trees = {brands:{emoji:"🍔",mainTag:"brand:wikidata",sourceTags:["brand","name"],nameTags:{primary:["^(name|name:\\w+)$"],alternate:["^(brand|brand:\\w+|operator|operator:\\w+|\\w+_name|\\w+_name:\\w+)$"]}},flags:{emoji:"🚩",mainTag:"flag:wikidata",nameTags:{primary:["^(flag:name|flag:name:\\w+)$"],alternate:["^(country|country:\\w+|flag|flag:\\w+|subject|subject:\\w+)$"]}},operators:{emoji:"💼",mainTag:"operator:wikidata",sourceTags:["operator"],nameTags:{primary:["^(name|name:\\w+|operator|operator:\\w+)$"],alternate:["^(brand|brand:\\w+|\\w+_name|\\w+_name:\\w+)$"]}},transit:{emoji:"🚇",mainTag:"network:wikidata",sourceTags:["network"],nameTags:{primary:["^network$"],alternate:["^(operator|operator:\\w+|network:\\w+|\\w+_name|\\w+_name:\\w+)$"]}}};
var trees = {brands:{emoji:"🍔",mainTag:"brand:wikidata",sourceTags:["brand","name"],nameTags:{primary:"^(name|name:\\w+)$",alternate:"^(brand|brand:\\w+|operator|operator:\\w+|\\w+_name|\\w+_name:\\w+)$"}},flags:{emoji:"🚩",mainTag:"flag:wikidata",nameTags:{primary:"^(flag:name|flag:name:\\w+)$",alternate:"^(country|country:\\w+|flag|flag:\\w+|subject|subject:\\w+)$"}},operators:{emoji:"💼",mainTag:"operator:wikidata",sourceTags:["operator"],nameTags:{primary:"^(name|name:\\w+|operator|operator:\\w+)$",alternate:"^(brand|brand:\\w+|\\w+_name|\\w+_name:\\w+)$"}},transit:{emoji:"🚇",mainTag:"network:wikidata",sourceTags:["network"],nameTags:{primary:"^network$",alternate:"^(operator|operator:\\w+|network:\\w+|\\w+_name|\\w+_name:\\w+)$"}}};
var require$$2 = {
trees: trees
};
@ -1350,16 +1350,16 @@
// Primary name patterns, match tags to take first
// e.g. `name`, `name:ru`
const primaryNames = tree.nameTags.primary.map(s => new RegExp(s, 'i'));
const primaryName = new RegExp(tree.nameTags.primary, 'i');
// Alternate name patterns, match tags to consider after primary
// e.g. `alt_name`, `short_name`, `brand`, `brand:ru`, etc..
const alternateNames = tree.nameTags.alternate.map(s => new RegExp(s, 'i'));
const alternateName = new RegExp(tree.nameTags.alternate, 'i');
// There are a few exceptions to the name matching regexes.
// Usually a tag suffix contains a language code like `name:en`, `name:ru`
// but we want to exclude things like `operator:type`, `name:etymology`, etc..
const notNames = /:(colou?r|type|forward|backward|left|right|etymology|pronunciation|wikipedia)$/i;
const notName = /:(colou?r|type|forward|backward|left|right|etymology|pronunciation|wikipedia)$/i;
// For certain categories we do not want to match generic KV pairs like `building/yes` or `amenity/yes`
const skipGenericKV = skipGenericKVMatches(t, k, v);
@ -1383,8 +1383,7 @@
});
});
// Add each item to the matchIndex
// For each item, insert all [key, value, name] combinations into the match index
items.forEach(item => {
if (!item.id) return;
@ -1406,19 +1405,17 @@
.concat(Array.from(genericKV)); // #3454 - match some generic tags
}
Object.keys(item.tags).forEach(osmkey => { // Check all tags for "names"
// Index all the namelike tag values
Object.keys(item.tags).forEach(osmkey => {
if (notName.test(osmkey)) return; // osmkey is not a namelike tag, skip
const osmvalue = item.tags[osmkey];
primaryNames.forEach(regex => {
if (!regex.test(osmkey) || notNames.test(osmkey)) return; // osmkey is not a namelike tag, skip
if (excludeRegexes.some(regex => regex.test(osmvalue))) return; // osmvalue is excluded
kvTags.forEach(kv => insertName('primary', kv, simplify(osmvalue), item.id));
});
if (!osmvalue || excludeRegexes.some(regex => regex.test(osmvalue))) return; // osmvalue missing or excluded
alternateNames.forEach(regex => {
if (!regex.test(osmkey) || notNames.test(osmkey)) return; // osmkey is not a namelike tag, skip
if (excludeRegexes.some(regex => regex.test(osmvalue))) return; // osmvalue is excluded
if (primaryName.test(osmkey)) {
kvTags.forEach(kv => insertName('primary', kv, simplify(osmvalue), item.id));
} else if (alternateName.test(osmkey)) {
kvTags.forEach(kv => insertName('alternate', kv, simplify(osmvalue), item.id));
});
}
});
// Index `matchNames` after indexing all other names..

2
dist/nsi.json vendored
View file

@ -1,7 +1,7 @@
{
"_meta": {
"version": "5.0.0-pre",
"generated": "2021-03-15T14:53:27.410Z",
"generated": "2021-03-15T15:35:41.277Z",
"url": "https://raw.githubusercontent.com/osmlab/name-suggestion-index/main/dist/nsi.json",
"hash": "0075800d3d0a9cda32961e2561cc02c9"
},

2
dist/nsi.min.json vendored
View file

File diff suppressed because one or more lines are too long

2
dist/presets/nsi-id-presets.json vendored
View file

@ -1,7 +1,7 @@
{
"_meta": {
"version": "5.0.0-pre",
"generated": "2021-03-15T14:53:27.932Z",
"generated": "2021-03-15T15:35:41.804Z",
"url": "https://raw.githubusercontent.com/osmlab/name-suggestion-index/main/dist/presets/nsi-id-presets.json",
"hash": "54a5bfb3cbf6ee97c21c1e88c8aaae79"
},

2
dist/presets/nsi-id-presets.min.json vendored
View file

File diff suppressed because one or more lines are too long

26
dist/trees.json vendored
View file

@ -1,9 +1,9 @@
{
"_meta": {
"version": "5.0.0-pre",
"generated": "2021-02-24T15:04:34.982Z",
"generated": "2021-03-15T15:25:09.624Z",
"url": "https://raw.githubusercontent.com/osmlab/name-suggestion-index/main/dist/trees.json",
"hash": "ecb3f726478118c7cb1e3e3d1a1416ea"
"hash": "645795426bb78b0dd268283696ac3ea2"
},
"trees": {
"brands": {
@ -11,20 +11,16 @@
"mainTag": "brand:wikidata",
"sourceTags": ["brand", "name"],
"nameTags": {
"primary": ["^(name|name:\\w+)$"],
"alternate": [
"^(brand|brand:\\w+|operator|operator:\\w+|\\w+_name|\\w+_name:\\w+)$"
]
"primary": "^(name|name:\\w+)$",
"alternate": "^(brand|brand:\\w+|operator|operator:\\w+|\\w+_name|\\w+_name:\\w+)$"
}
},
"flags": {
"emoji": "🚩",
"mainTag": "flag:wikidata",
"nameTags": {
"primary": ["^(flag:name|flag:name:\\w+)$"],
"alternate": [
"^(country|country:\\w+|flag|flag:\\w+|subject|subject:\\w+)$"
]
"primary": "^(flag:name|flag:name:\\w+)$",
"alternate": "^(country|country:\\w+|flag|flag:\\w+|subject|subject:\\w+)$"
}
},
"operators": {
@ -32,8 +28,8 @@
"mainTag": "operator:wikidata",
"sourceTags": ["operator"],
"nameTags": {
"primary": ["^(name|name:\\w+|operator|operator:\\w+)$"],
"alternate": ["^(brand|brand:\\w+|\\w+_name|\\w+_name:\\w+)$"]
"primary": "^(name|name:\\w+|operator|operator:\\w+)$",
"alternate": "^(brand|brand:\\w+|\\w+_name|\\w+_name:\\w+)$"
}
},
"transit": {
@ -41,10 +37,8 @@
"mainTag": "network:wikidata",
"sourceTags": ["network"],
"nameTags": {
"primary": ["^network$"],
"alternate": [
"^(operator|operator:\\w+|network:\\w+|\\w+_name|\\w+_name:\\w+)$"
]
"primary": "^network$",
"alternate": "^(operator|operator:\\w+|network:\\w+|\\w+_name|\\w+_name:\\w+)$"
}
}
}

2
dist/trees.min.json vendored
View file

@ -1 +1 @@
{"_meta":{"version":"5.0.0-pre","generated":"2021-02-24T15:04:34.982Z","url":"https://raw.githubusercontent.com/osmlab/name-suggestion-index/main/dist/trees.json","hash":"ecb3f726478118c7cb1e3e3d1a1416ea"},"trees":{"brands":{"emoji":"🍔","mainTag":"brand:wikidata","sourceTags":["brand","name"],"nameTags":{"primary":["^(name|name:\\w+)$"],"alternate":["^(brand|brand:\\w+|operator|operator:\\w+|\\w+_name|\\w+_name:\\w+)$"]}},"flags":{"emoji":"🚩","mainTag":"flag:wikidata","nameTags":{"primary":["^(flag:name|flag:name:\\w+)$"],"alternate":["^(country|country:\\w+|flag|flag:\\w+|subject|subject:\\w+)$"]}},"operators":{"emoji":"💼","mainTag":"operator:wikidata","sourceTags":["operator"],"nameTags":{"primary":["^(name|name:\\w+|operator|operator:\\w+)$"],"alternate":["^(brand|brand:\\w+|\\w+_name|\\w+_name:\\w+)$"]}},"transit":{"emoji":"🚇","mainTag":"network:wikidata","sourceTags":["network"],"nameTags":{"primary":["^network$"],"alternate":["^(operator|operator:\\w+|network:\\w+|\\w+_name|\\w+_name:\\w+)$"]}}}}
{"_meta":{"version":"5.0.0-pre","generated":"2021-03-15T15:25:09.624Z","url":"https://raw.githubusercontent.com/osmlab/name-suggestion-index/main/dist/trees.json","hash":"645795426bb78b0dd268283696ac3ea2"},"trees":{"brands":{"emoji":"🍔","mainTag":"brand:wikidata","sourceTags":["brand","name"],"nameTags":{"primary":"^(name|name:\\w+)$","alternate":"^(brand|brand:\\w+|operator|operator:\\w+|\\w+_name|\\w+_name:\\w+)$"}},"flags":{"emoji":"🚩","mainTag":"flag:wikidata","nameTags":{"primary":"^(flag:name|flag:name:\\w+)$","alternate":"^(country|country:\\w+|flag|flag:\\w+|subject|subject:\\w+)$"}},"operators":{"emoji":"💼","mainTag":"operator:wikidata","sourceTags":["operator"],"nameTags":{"primary":"^(name|name:\\w+|operator|operator:\\w+)$","alternate":"^(brand|brand:\\w+|\\w+_name|\\w+_name:\\w+)$"}},"transit":{"emoji":"🚇","mainTag":"network:wikidata","sourceTags":["network"],"nameTags":{"primary":"^network$","alternate":"^(operator|operator:\\w+|network:\\w+|\\w+_name|\\w+_name:\\w+)$"}}}}

476
docs/sitemap.xml
View file

File diff suppressed because it is too large Load diff

27
lib/matcher.js
View file

@ -132,16 +132,16 @@ module.exports = () => {
// Primary name patterns, match tags to take first
// e.g. `name`, `name:ru`
const primaryNames = tree.nameTags.primary.map(s => new RegExp(s, 'i'));
const primaryName = new RegExp(tree.nameTags.primary, 'i');
// Alternate name patterns, match tags to consider after primary
// e.g. `alt_name`, `short_name`, `brand`, `brand:ru`, etc..
const alternateNames = tree.nameTags.alternate.map(s => new RegExp(s, 'i'));
const alternateName = new RegExp(tree.nameTags.alternate, 'i');
// There are a few exceptions to the name matching regexes.
// Usually a tag suffix contains a language code like `name:en`, `name:ru`
// but we want to exclude things like `operator:type`, `name:etymology`, etc..
const notNames = /:(colou?r|type|forward|backward|left|right|etymology|pronunciation|wikipedia)$/i;
const notName = /:(colou?r|type|forward|backward|left|right|etymology|pronunciation|wikipedia)$/i;
// For certain categories we do not want to match generic KV pairs like `building/yes` or `amenity/yes`
const skipGenericKV = skipGenericKVMatches(t, k, v);
@ -165,8 +165,7 @@ module.exports = () => {
});
});
// Add each item to the matchIndex
// For each item, insert all [key, value, name] combinations into the match index
items.forEach(item => {
if (!item.id) return;
@ -188,19 +187,17 @@ module.exports = () => {
.concat(Array.from(genericKV)); // #3454 - match some generic tags
}
Object.keys(item.tags).forEach(osmkey => { // Check all tags for "names"
// Index all the namelike tag values
Object.keys(item.tags).forEach(osmkey => {
if (notName.test(osmkey)) return; // osmkey is not a namelike tag, skip
const osmvalue = item.tags[osmkey];
primaryNames.forEach(regex => {
if (!regex.test(osmkey) || notNames.test(osmkey)) return; // osmkey is not a namelike tag, skip
if (excludeRegexes.some(regex => regex.test(osmvalue))) return; // osmvalue is excluded
kvTags.forEach(kv => insertName('primary', kv, simplify(osmvalue), item.id));
});
if (!osmvalue || excludeRegexes.some(regex => regex.test(osmvalue))) return; // osmvalue missing or excluded
alternateNames.forEach(regex => {
if (!regex.test(osmkey) || notNames.test(osmkey)) return; // osmkey is not a namelike tag, skip
if (excludeRegexes.some(regex => regex.test(osmvalue))) return; // osmvalue is excluded
if (primaryName.test(osmkey)) {
kvTags.forEach(kv => insertName('primary', kv, simplify(osmvalue), item.id));
} else if (alternateName.test(osmkey)) {
kvTags.forEach(kv => insertName('alternate', kv, simplify(osmvalue), item.id));
});
}
});
// Index `matchNames` after indexing all other names..

16
schema/trees.json
View file

@ -47,21 +47,13 @@
"properties": {
"primary": {
"description": "(required) Patterns for matching primary name tags (e.g. 'name')",
"type": "array",
"uniqueItems": true,
"items": {
"type": "string",
"format": "regex"
}
"type": "string",
"format": "regex"
},
"alternate": {
"description": "(required) Patterns for matching alternate name tags (e.g. 'short_name', 'alt_name')",
"type": "array",
"uniqueItems": true,
"items": {
"type": "string",
"format": "regex"
}
"type": "string",
"format": "regex"
}
}
}

22
scripts/build_dist.js
View file

@ -154,15 +154,14 @@ function buildIDPresets() {
const wdTag = tree.mainTag;
// Primary/alternate names may be used as preset search terms
const primaryNames = tree.nameTags.primary.map(s => new RegExp(s, 'i'));
const alternateNames = tree.nameTags.alternate.map(s => new RegExp(s, 'i'));
const primaryName = new RegExp(tree.nameTags.primary, 'i');
const alternateName = new RegExp(tree.nameTags.alternate, 'i');
// There are a few exceptions to the name matching regexes.
// Usually a tag suffix contains a language code like `name:en`, `name:ru`
// but we want to exclude things like `operator:type`, `name:etymology`, etc..
// NOTE: here we intentionally exclude `:wikidata`, in `matcher.js` we do not.
const notNames = /:(colour|type|left|right|etymology|pronunciation|wikipedia|wikidata)$/i;
const notName = /:(colour|type|left|right|etymology|pronunciation|wikipedia|wikidata)$/i;
items.forEach(item => {
const tags = item.tags;
@ -210,16 +209,13 @@ function buildIDPresets() {
// Gather search terms - include all primary/alternate names and matchNames
// (There is similar code in lib/matcher.js)
let terms = new Set(item.matchNames || []);
Object.keys(tags).forEach(osmkey => { // Check all tags for alternate names
primaryNames.forEach(regex => {
if (osmkey === 'name') return; // exclude `name` tag, as iD prioritizes it above `preset.terms` already
if (!regex.test(osmkey) || notNames.test(osmkey)) return; // osmkey is not a namelike tag, skip
Object.keys(tags).forEach(osmkey => {
if (osmkey === 'name') return; // exclude `name` tag, as iD prioritizes it above `preset.terms` already
if (notName.test(osmkey)) return; // osmkey is not a namelike tag, skip
if (primaryName.test(osmkey) || alternateName.test(osmkey)) {
terms.add(tags[osmkey].toLowerCase());
});
alternateNames.forEach(regex => {
if (!regex.test(osmkey) || notNames.test(osmkey)) return; // osmkey is not a namelike tag, skip
terms.add(tags[osmkey].toLowerCase());
});
}
});
// generate our target preset

4
scripts/build_index.js
View file

@ -72,8 +72,8 @@ function loadConfig() {
// check regexes
if (which === 'trees') {
Object.values(data.trees).forEach(tree => {
tree.nameTags.primary.forEach(pattern => checkRegex(file, pattern));
tree.nameTags.alternate.forEach(pattern => checkRegex(file, pattern));
checkRegex(file, tree.nameTags.primary);
checkRegex(file, tree.nameTags.alternate);
});
} else if (which === 'genericWords') {