From fd67ef394d540fca9fc15d1c39dc39e3f10e4d26 Mon Sep 17 00:00:00 2001
From: Roman Tsisyk <roman@tsisyk.com>
Date: Sun, 5 Jan 2025 19:03:48 +0000
Subject: [PATCH] [github] GitHub Action to update Wikipedia

Signed-off-by: Roman Tsisyk <roman@tsisyk.com>
---
 .github/workflows/generator-wikipedia.yaml | 61 ++++++++++++++++++++++
 1 file changed, 61 insertions(+)
 create mode 100644 .github/workflows/generator-wikipedia.yaml

diff --git a/.github/workflows/generator-wikipedia.yaml b/.github/workflows/generator-wikipedia.yaml
new file mode 100644
index 0000000000..9716292ff7
--- /dev/null
+++ b/.github/workflows/generator-wikipedia.yaml
@@ -0,0 +1,61 @@
+#
+# Update Wikipedia data on the generator host.
+#
+name: Generator / Wikipedia
+on:
+  push:
+    branches:
+      - generator-actions
+  workflow_dispatch:
+
+jobs:
+  update-wikipedia:
+    name: Update Wikipedia
+    runs-on: [self-hosted, planet]
+    environment: planet
+    concurrency:
+      group: planet
+      cancel-in-progress: false
+    timeout-minutes: 1440 # 24 hours
+    steps:
+      - name: Sanity checks
+        run: |
+          [[ -n "${{ vars.PLANET_PBF }}" && -f "${{ vars.PLANET_PBF }}" ]] && \
+          [[ -n "${{ vars.PLANET_WIKIPEDIA_DUMPS }}" && "${{ vars.PLANET_WIKIPEDIA_DUMPS }}" != *$'\n'* ]] && \
+          [[ -n "${{ vars.PLANET_WIKIPEDIA_DESCRIPTIONS }}" && "${{ vars.PLANET_WIKIPEDIA_DESCRIPTIONS }}" != *$'\n'* ]] && \
+          [[ -n "${{ vars.PLANET_WIKIPEDIA_BUILD }}" && "${{ vars.PLANET_WIKIPEDIA_BUILD }}" != *$'\n'* ]] && \
+          echo 'GitHub Variables are OK'
+
+      - name: Checkout wikiparser sources
+        uses: actions/checkout@v4
+        with:
+          repository: 'organicmaps/wikiparser'
+          path: wikiparser
+
+      - name: Build wikiparser
+        working-directory: wikiparser
+        run: |
+          cargo build --release
+
+      - name: Download new Wikipedia dumps
+        if: false # temporary disable because 20250101+ dumps are incompatible.
+        working-directory: wikiparser
+        run: |
+          mkdir -p "${{ vars.PLANET_WIKIPEDIA_DUMPS }}"
+          ./download.sh -D "${{ vars.PLANET_WIKIPEDIA_DUMPS }}"
+          ls -la "${{ vars.PLANET_WIKIPEDIA_DUMPS }}/latest"
+
+      - name: Generate Wikipedia descriptions
+        working-directory: wikiparser
+        run: |
+          mkdir -p "${{ vars.PLANET_WIKIPEDIA_DESCRIPTIONS }}" "${{ vars.PLANET_WIKIPEDIA_BUILD }}"
+          rm -rf "${{ vars.PLANET_WIKIPEDIA_BUILD }}"/* || :
+          sed -i 's|OUTPUT_DIR=.*|OUTPUT_DIR="${{ vars.PLANET_WIKIPEDIA_DESCRIPTIONS }}"|' run.sh
+          ./run.sh "${{ vars.PLANET_WIKIPEDIA_BUILD }}" "${{ vars.PLANET_PBF }}" "${{ vars.PLANET_WIKIPEDIA_DUMPS }}/latest/"*.json.tar.gz
+
+      - name: Upload tsv files
+        uses: actions/upload-artifact@v4
+        with:
+          name: wikipedia-osm-tags
+          path: '${{ vars.PLANET_WIKIPEDIA_BUILD }}/osm_tags*.tsv'
+          if-no-files-found: error