Use virtual environments

.github/workflows/python-app.yml

@@ -23,17 +23,25 @@ jobs:
       uses: actions/setup-python@v3
       with:
         python-version: "3.11"
-    - name: Install dependencies
+    - name: Install dependencies for linters
       run: |
         python -m pip install --upgrade pip
-        pip install flake8==6.0.0 black==23.1.0 shapely==2.0.1
-        pip install -r subways/requirements.txt
+        pip install flake8==6.0.0 black==23.1.0
     - name: Lint with flake8
       run: |
         flake8
     - name: Check with black
       run: |
         black --check --line-length 79 .
-    - name: Test with unittest
+    - name: Test subways with unittest
       run: |
-        python -m unittest discover tests
+        export PYTHONPATH=$(pwd)
+        pip freeze | xargs pip uninstall -y
+        pip install -r subways/requirements.txt
+        python -m unittest discover subways
+    - name: Test tools with unittest
+      run: |
+        export PYTHONPATH=$(pwd)
+        pip freeze | xargs pip uninstall -y
+        pip install -r tools/make_poly/requirements.txt
+        python -m unittest discover tools/make_poly

README.md

@@ -59,9 +59,11 @@ if you allow the `scripts/process_subway.py` to fetch data from Overpass API. He
     git clone https://github.com/alexey-zakharenkov/subways.git subways_validator
     cd subways_validator
    ```
-3. Install python dependencies
+3. Configure python environment, e.g.
    ```bash
-   pip install -r subways/requirements.txt
+   python3 -m venv scripts/.venv
+   source scripts/.venv/bin/activate
+   pip install scripts/requirements.txt
    ```
 4. Execute
     ```bash

scripts/process_subways.sh

@@ -63,6 +63,22 @@ EOF
 fi
 
 
+function activate_venv_at_path() {
+  path=$1
+
+  if [ ! -d "$path/".venv ]; then
+    "${PYTHON:-python3.11}" -m venv "$path"/.venv
+  fi
+
+  source "$path"/.venv/bin/activate
+
+  if [ -f "$path"/requirements.txt ]; then
+    pip install --upgrade pip
+    pip install -r "$path"/requirements.txt
+  fi
+}
+
+
 function check_osmctools() {
   OSMCTOOLS="${OSMCTOOLS:-$HOME/osmctools}"
   if [ ! -f "$OSMCTOOLS/osmupdate" ]; then
@@ -91,39 +107,39 @@ function check_poly() {
 
       if [ -z "${POLY-}" -o ! -f "${POLY-}" ]; then
         POLY=${POLY:-$(mktemp "$TMPDIR/all-metro.XXXXXXXX.poly")}
-        if [ -n "$("$PYTHON" -c "import shapely" 2>&1)" ]; then
-          "$PYTHON" -m pip install shapely==2.0.1
-        fi
-        "$PYTHON" "$SUBWAYS_REPO_PATH"/tools/make_poly/make_all_metro_poly.py \
+        activate_venv_at_path "$SUBWAYS_REPO_PATH/tools/make_poly"
+        python "$SUBWAYS_REPO_PATH"/tools/make_poly/make_all_metro_poly.py \
             ${CITIES_INFO_URL:+--cities-info-url "$CITIES_INFO_URL"} > "$POLY"
+        deactivate
       fi
     fi
     POLY_CHECKED=1
   fi
 }
 
-
-PYTHON=${PYTHON:-python3}
-# This will fail if there is no python
-"$PYTHON" --version > /dev/null
-
 # "readlink -f" echoes canonicalized absolute path to a file/directory
 SUBWAYS_REPO_PATH="$(readlink -f $(dirname "$0")/..)"
 
 if [ ! -f "$SUBWAYS_REPO_PATH/scripts/process_subways.py" ]; then
-  echo "Please clone the subways repo to $SUBWAYS_PATH"
+  echo "Please clone the subways repo to $SUBWAYS_REPO_PATH"
   exit 2
 fi
 
-TMPDIR="${TMPDIR:-$SUBWAYS_REPO_PATH}"
-mkdir -p "$TMPDIR"
+# Contains 'subways' dir and is required by the main validator python script
+# as well as by some tools
+export PYTHONPATH="$SUBWAYS_REPO_PATH"
 
 # Downloading the latest version of the subways script
 if [ -n "${GIT_PULL-}" ]; then (
-  cd "$SUBWAYS_PATH"
+  pushd "$SUBWAYS_REPO_PATH"
   git pull origin master
+  popd
 ) fi
 
+
+TMPDIR="${TMPDIR:-"$SUBWAYS_REPO_PATH"}"
+mkdir -p "$TMPDIR"
+
 if [ -z "${FILTERED_DATA-}" ]; then
   FILTERED_DATA="$TMPDIR/subways.osm"
   NEED_TO_REMOVE_FILTERED_DATA=1
@@ -244,7 +260,9 @@ if [ -n "${DUMP-}" ]; then
 fi
 
 VALIDATION="$TMPDIR/validation.json"
-"$PYTHON" "$SUBWAYS_REPO_PATH/scripts/process_subways.py" ${QUIET:+-q} \
+
+activate_venv_at_path "$SUBWAYS_REPO_PATH/scripts"
+python "$SUBWAYS_REPO_PATH/scripts/process_subways.py" ${QUIET:+-q} \
     -x "$FILTERED_DATA" -l "$VALIDATION" \
     ${CITIES_INFO_URL:+--cities-info-url "$CITIES_INFO_URL"} \
     ${MAPSME:+--output-mapsme "$MAPSME"} \
@@ -256,6 +274,8 @@ VALIDATION="$TMPDIR/validation.json"
     ${ELEMENTS_CACHE:+-i "$ELEMENTS_CACHE"} \
     ${CITY_CACHE:+--cache "$CITY_CACHE"} \
     ${RECOVERY_PATH:+-r "$RECOVERY_PATH"}
+deactivate
+
 
 if [ -n "${NEED_TO_REMOVE_FILTERED_DATA-}" ]; then
   rm "$FILTERED_DATA"
@@ -270,9 +290,12 @@ fi
 
 mkdir -p $HTML_DIR
 rm -f "$HTML_DIR"/*.html
-"$PYTHON" "$SUBWAYS_REPO_PATH/tools/v2h/validation_to_html.py" \
+
+activate_venv_at_path "$SUBWAYS_REPO_PATH/tools/v2h"
+python "$SUBWAYS_REPO_PATH/tools/v2h/validation_to_html.py" \
     ${CITIES_INFO_URL:+--cities-info-url "$CITIES_INFO_URL"} \
     "$VALIDATION" "$HTML_DIR"
+deactivate
 
 # Uploading files to the server
 

scripts/requirements.txt

@@ -0,0 +1 @@
+-r ../subways/requirements.txt

subways/tests/README.md

@@ -1,13 +1,10 @@
-To perform tests manually, run this command from the top directory
+To perform tests, run this command from the top directory
 of the repository:
 
 ```bash
-python -m unittest discover tests
+export PYTHONPATH=$(pwd)
+[ -d "subways/tests/.venv" ] || python3 -m venv subways/tests/.venv
+source subways/tests/.venv/bin/activate
+pip install -r subways/requirements.txt
+python -m unittest discover subways
 ```
-
-or simply
-
-```bash
-python -m unittest
-```
-

tools/make_poly/requirements.txt

@@ -0,0 +1 @@
+shapely==2.0.1