organicmaps/name-suggestion-index
13
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity Actions

Enable --experimental-json-modules so we can import json files

Browse source 
https://nodejs.org/docs/latest-v14.x/api/esm.html#esm_experimental_json_modules
This was something that we did a lot of with require()
This commit is contained in:
Bryan Housel 2021-06-22 15:17:54 -04:00
parent 8c7037eede
commit b21ac2eb66
16 changed files with 88 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.npmrc Normal file
View file

@ -0,0 +1 @@
node-options="--no-warnings --experimental-json-modules"

15
lib/file_tree.js
View file

@ -1,21 +1,22 @@
// External
import colors from 'colors/safe.js';
import fs from 'fs-extra';
import glob from 'glob';
import JSON5 from 'json5';
import localeCompare from 'locale-compare';
import stringify from '@aitodotai/json-stringify-pretty-compact';
const withLocale = localeCompare('en-US');
// Internal
import { idgen } from './idgen.js';
import { sortObject } from './sort_object.js';
import { validate } from './validate.js';
const withLocale = localeCompare('en-US');
// JSON
import treesJSON from '../config/trees.json';
const trees = treesJSON.trees;
import categoriesSchemaJSON from '../schema/categories.json';
// metadata about the trees
const trees = JSON5.parse(fs.readFileSync('./config/trees.json', 'utf8')).trees;
// validate the files as we read them
const categoriesSchema = JSON5.parse(fs.readFileSync('./schema/categories.json', 'utf8'));
// The code in here
// - validates data on read, generating any missing data
@ -66,7 +67,7 @@ read: (cache, loco) => {
}
// check JSON schema
validate(file, input, categoriesSchema);
validate(file, input, categoriesSchemaJSON);
let seenkv = {};

3
lib/idgen.js
View file

@ -1,4 +1,7 @@
// External
import crypto from 'node:crypto';
// Internal
import { simplify } from './simplify.js';

19
lib/matcher.js
View file

@ -1,14 +1,17 @@
// External
import whichPolygon from 'which-polygon';
// Internal
import { simplify } from './simplify.js';
// This will not work in the browser :(
// We may be able to switch to `import`, but:
// - for node, with node with --experimental-json-modules https://stackoverflow.com/a/59758333/7620
// - for browser, both esbuild and rollup should be able to bundle a .json import
import fs from 'fs';
const matchGroups = JSON.parse(fs.readFileSync('./config/matchGroups.json', 'utf8')).matchGroups;
const genericWords = JSON.parse(fs.readFileSync('./config/genericWords.json', 'utf8')).genericWords;
const trees = JSON.parse(fs.readFileSync('./config/trees.json', 'utf8')).trees;
// JSON
import matchGroupsJSON from '../config/matchGroups.json';
import genericWordsJSON from '../config/genericWords.json';
import treesJSON from '../config/trees.json';
const matchGroups = matchGroupsJSON.matchGroups;
const genericWords = genericWordsJSON.genericWords;
const trees = treesJSON.trees;
export function Matcher() {

1
lib/simplify.js
View file

@ -1,3 +1,4 @@
// External
import diacritics from 'diacritics';
// remove spaces, punctuation, diacritics

1
lib/sort_object.js
View file

@ -1,3 +1,4 @@
// External
import localeCompare from 'locale-compare';
const withLocale = localeCompare('en-US');

1
lib/stemmer.js
View file

@ -1,3 +1,4 @@
// Internal
import { simplify } from './simplify.js';
// Removes noise from the name so that we can compare

1
lib/validate.js
View file

@ -1,3 +1,4 @@
// External
import colors from 'colors/safe.js';
import jsonschema from 'jsonschema';

5
lib/write_file_with_meta.js
View file

@ -1,8 +1,11 @@
// External
import crypto from 'node:crypto';
import fs from 'node:fs';
import JSON5 from 'json5';
const packageJSON = JSON.parse(fs.readFileSync('./package.json', 'utf8'));
// JSON
import packageJSON from '../package.json';
const URLRoot = 'https://raw.githubusercontent.com/osmlab/name-suggestion-index/main';
//

11
scripts/build_features.js
View file

@ -1,3 +1,4 @@
// External
import colors from 'colors/safe.js';
import fs from 'node:fs';
import geojsonArea from '@mapbox/geojson-area';
@ -10,14 +11,16 @@ import jsonschema from 'jsonschema';
import path from 'node:path';
import stringify from '@aitodotai/json-stringify-pretty-compact';
// Internal
import { writeFileWithMeta } from '../lib/write_file_with_meta.js';
const geojsonSchema = JSON.parse(fs.readFileSync('./schema/geojson.json', 'utf8'));
const featureSchema = JSON.parse(fs.readFileSync('./schema/feature.json', 'utf8'));
// JSON
import geojsonSchemaJSON from '../schema/geojson.json';
import featureSchemaJSON from '../schema/feature.json';
const Validator = jsonschema.Validator;
let v = new Validator();
v.addSchema(geojsonSchema, 'http://json.schemastore.org/geojson.json');
v.addSchema(geojsonSchemaJSON, 'http://json.schemastore.org/geojson.json');
console.log(colors.blue('-'.repeat(70)));
@ -124,7 +127,7 @@ function collectFeatures() {
feature = obj;
validateFile(file, feature, featureSchema);
validateFile(file, feature, featureSchemaJSON);
prettifyFile(file, feature, contents);
if (files[id]) {

14
scripts/build_index.js
View file

@ -1,3 +1,4 @@
// External
import colors from 'colors/safe.js';
import fs from 'node:fs';
import JSON5 from 'json5';
@ -6,7 +7,9 @@ import LocationConflation from '@ideditor/location-conflation';
import safeRegex from 'safe-regex';
import shell from 'shelljs';
import stringify from '@aitodotai/json-stringify-pretty-compact';
const withLocale = localeCompare('en-US');
// Internal
import { fileTree } from '../lib/file_tree.js';
import { idgen } from '../lib/idgen.js';
import { Matcher } from '../lib/matcher.js';
@ -15,14 +18,13 @@ import { sortObject } from '../lib/sort_object.js';
import { stemmer } from '../lib/stemmer.js';
import { validate } from '../lib/validate.js';
const withLocale = localeCompare('en-US');
// metadata about the trees
const trees = JSON5.parse(fs.readFileSync('./config/trees.json', 'utf8')).trees;
// JSON
import treesJSON from '../config/trees.json';
const trees = treesJSON.trees;
// We use LocationConflation for validating and processing the locationSets
const featureCollection = JSON5.parse(fs.readFileSync('./dist/featureCollection.json', 'utf8'));
const loco = new LocationConflation(featureCollection);
import featureCollectionJSON from '../dist/featureCollection.json';
const loco = new LocationConflation(featureCollectionJSON);
console.log(colors.blue('-'.repeat(70)));
console.log(colors.blue('🗂 Build index'));

16
scripts/build_wikidata.js
View file

@ -1,3 +1,4 @@
// External
import colors from 'colors/safe.js';
import fs from 'node:fs';
import crypto from 'node:crypto';
@ -12,18 +13,21 @@ import stringify from '@aitodotai/json-stringify-pretty-compact';
import Twitter from 'Twitter';
import wikibase from 'wikibase-sdk';
import wikibaseEdit from 'wikibase-edit';
const withLocale = localeCompare('en-US');
// Internal
import { sortObject } from '../lib/sort_object.js';
import { fileTree } from '../lib/file_tree.js';
import { writeFileWithMeta } from '../lib/write_file_with_meta.js';
const withLocale = localeCompare('en-US');
const project = JSON5.parse(fs.readFileSync('./package.json', 'utf8'));
const trees = JSON5.parse(fs.readFileSync('./config/trees.json', 'utf8')).trees;
// JSON
import packageJSON from '../package.json';
import treesJSON from '../config/trees.json';
const trees = treesJSON.trees;
// We use LocationConflation for validating and processing the locationSets
const featureCollection = JSON.parse(fs.readFileSync('./dist/featureCollection.json', 'utf8'));
const loco = new LocationConflation(featureCollection);
import featureCollectionJSON from '../dist/featureCollection.json';
const loco = new LocationConflation(featureCollectionJSON);
const wbk = wikibase({
instance: 'https://www.wikidata.org',
@ -106,7 +110,7 @@ if (_secrets && _secrets.wikibase) {
instance: 'https://www.wikidata.org',
credentials: _secrets.wikibase,
summary: 'Updated name-suggestion-index related claims, see https://nsi.guide for project details.',
userAgent: `${project.name}/${project.version} (${project.homepage})`,
userAgent: `${packageJSON.name}/${packageJSON.version} (${packageJSON.homepage})`,
});
}

12
scripts/check_wikiTags.js
View file

@ -1,21 +1,23 @@
// External
import clearConsole from 'clear';
import colors from 'colors/safe.js';
import fetch from 'node-fetch';
import fs from 'node:fs';
import LocationConflation from '@ideditor/location-conflation';
import wikibase from 'wikibase-sdk';
// Internal
import { fileTree } from '../lib/file_tree.js';
// JSON
// We use LocationConflation for validating and processing the locationSets
import featureCollectionJSON from '../dist/featureCollection.json';
const loco = new LocationConflation(featureCollectionJSON);
const wbk = wikibase({
instance: 'https://www.wikidata.org',
sparqlEndpoint: 'https://query.wikidata.org/sparql'
});
// We use LocationConflation for validating and processing the locationSets
const featureCollection = JSON.parse(fs.readFileSync('./dist/featureCollection.json', 'utf8'));
const loco = new LocationConflation(featureCollection);
let _cache = {};
fileTree.read(_cache, loco);

2
scripts/collect_all.js
View file

@ -9,12 +9,14 @@
//
// Please see README.md for more info
// External
import colors from 'colors/safe.js';
import fs from 'node:fs';
import osmium from 'osmium';
import shell from 'shelljs';
import stringify from '@aitodotai/json-stringify-pretty-compact';
// Internal
import { sortObject } from '../lib/sort_object.js';
if (process.argv.length < 3) {

30
scripts/dist_files.js
View file

@ -1,3 +1,4 @@
// External
import colors from 'colors/safe.js';
import fs from 'node:fs';
import glob from 'glob';
@ -8,24 +9,29 @@ import shell from 'shelljs';
import stringify from '@aitodotai/json-stringify-pretty-compact';
import xmlbuilder2 from 'xmlbuilder2';
const withLocale = localeCompare('en-US');
// Internal
import { fileTree } from '../lib/file_tree.js';
import { sortObject } from '../lib/sort_object.js';
import { writeFileWithMeta } from '../lib/write_file_with_meta.js';
const withLocale = localeCompare('en-US');
// JSON
import dissolvedJSON from '../dist/dissolved.json';
import packageJSON from '../package.json';
import treesJSON from '../config/trees.json';
import wikidataJSON from '../dist/wikidata.json';
// JSON imports
const dissolved = JSON5.parse(fs.readFileSync('./dist/dissolved.json', 'utf8')).dissolved;
const packageJSON = JSON5.parse(fs.readFileSync('./package.json', 'utf8'));
const trees = JSON5.parse(fs.readFileSync('./config/trees.json', 'utf8')).trees;
const wikidata = JSON5.parse(fs.readFileSync('./dist/wikidata.json', 'utf8')).wikidata;
const dissolved = dissolvedJSON.dissolved;
const trees = treesJSON.trees;
const wikidata = wikidataJSON.wikidata;
// iD's presets which we will build on
const sourcePresets = JSON5.parse(fs.readFileSync('./node_modules/@openstreetmap/id-tagging-schema/dist/presets.json', 'utf8'));
import presetsJSON from '@openstreetmap/id-tagging-schema/dist/presets.json';
// We use LocationConflation for validating and processing the locationSets
const featureCollection = JSON.parse(fs.readFileSync('./dist/featureCollection.json', 'utf8'));
const loco = new LocationConflation(featureCollection);
import featureCollectionJSON from '../dist/featureCollection.json';
const loco = new LocationConflation(featureCollectionJSON);
let _cache = {};
fileTree.read(_cache, loco);
@ -203,7 +209,7 @@ function buildIDPresets() {
if (val === 'parcel_pickup;parcel_mail_in') { // this one is just special
presetID = `${presetPath}/parcel_pickup_dropoff`;
preset = sourcePresets[presetID];
preset = presetsJSON[presetID];
if (preset) return; // it matched
}
@ -211,7 +217,7 @@ function buildIDPresets() {
let vals = val.split(';');
for (let i = 0; i < vals.length; i++) {
presetID = presetPath + '/' + vals[i].trim();
preset = sourcePresets[presetID];
preset = presetsJSON[presetID];
if (preset) return; // it matched
}
});
@ -219,7 +225,7 @@ function buildIDPresets() {
// fallback to `key/value`
if (!preset) {
presetID = presetPath;
preset = sourcePresets[presetID];
preset = presetsJSON[presetID];
}
// still no match?

8
scripts/dist_version.js
View file

@ -1,7 +1,9 @@
// External
import colors from 'colors/safe.js';
import fs from 'node:fs';
const packageJSON = JSON.parse(fs.readFileSync('./package.json', 'utf8'));
// JSON
import packageJSON from '../package.json';
// YYYYMMDD
const now = new Date();
@ -13,7 +15,7 @@ const oldVersion = packageJSON.version;
const newVersion = oldVersion.replace(/(\d){8}/, `${yyyy}${mm}${dd}`);
if (newVersion !== oldVersion) {
console.log('🎉 ' + colors.green('Bumping package version to ') + colors.green.bold(newVersion));
console.log('🎉 ' + colors.green('Bumping package version to ') + colors.green.bold(`v${newVersion}`));
const output = Object.assign(packageJSON, { version: newVersion });
fs.writeFileSync('package.json', JSON.stringify(output, null, 2));
fs.writeFileSync('package.json', JSON.stringify(output, null, 2) + '\n');
}