fill-coords script

.gitignore

@@ -2,3 +2,4 @@
 mapsme-state.txt
 mapsme-changes.db
 mapsme-process.log
+venv/

server/db.py

@@ -0,0 +1,28 @@
+import os.path, sys
+import peewee
+
+path = os.path.dirname(sys.argv[0]) if len(sys.argv) < 2 else sys.argv[1]
+database = peewee.SqliteDatabase(os.path.join(path, 'mapsme-changes.db'))
+
+class Change(peewee.Model):
+  """A model for the change. Just a single table."""
+  changeset = peewee.IntegerField()
+  user = peewee.CharField(max_length=250, index=True)
+  version = peewee.CharField(max_length=250)
+  timestamp = peewee.DateTimeField(index=True)
+  action = peewee.FixedCharField(max_length=1) # c=created, d=deleted, m=modified, a=anomaly, n=note
+  obj_type = peewee.FixedCharField(max_length=1, null=True)
+  obj_id = peewee.IntegerField(null=True)
+  main_tag = peewee.CharField(max_length=100, null=True)
+  address = peewee.BooleanField(default=False)
+  changes = peewee.TextField()
+
+  class Meta:
+    database = database
+    db_table = 'mapsme_change'
+
+class Seen(peewee.Model):
+  """A model for a storage of processed objects."""
+  obj = peewee.TextField(index=True)
+  class Meta:
+    database = database

server/fill-coords.py

@@ -1,32 +1,11 @@
-#!/usr/bin/python
+#!/usr/bin/env python
 # Adds coordinates to modified objects in the database
-import sys, os, urllib2, re, gzip
-import peewee, json
+import sys, urllib2, json
+from db import *
 from lxml import etree
-from StringIO import StringIO
-from datetime import datetime
 
-path = os.path.dirname(sys.argv[0]) if len(sys.argv) < 2 else sys.argv[1]
-database = peewee.SqliteDatabase(os.path.join(path, 'mapsme-changes.db'))
 API_ENDPOINT = 'https://api.openstreetmap.org/api/0.6'
 
-class Change(peewee.Model):
-  """A model for the change. Just a single table."""
-  changeset = peewee.IntegerField()
-  user = peewee.CharField(max_length=250, index=True)
-  version = peewee.CharField(max_length=250)
-  timestamp = peewee.DateTimeField(index=True)
-  action = peewee.FixedCharField(max_length=1) # c=created, d=deleted, m=modified, a=anomaly, n=note
-  obj_type = peewee.FixedCharField(max_length=1, null=True)
-  obj_id = peewee.IntegerField(null=True)
-  main_tag = peewee.CharField(max_length=100, null=True)
-  address = peewee.BooleanField(default=False)
-  changes = peewee.TextField()
-
-  class Meta:
-    database = database
-    db_table = 'mapsme_change'
-
 def find_coord(obj_type, obj_id):
   if obj_type not in ('n', 'w'):
     return None
@@ -47,9 +26,11 @@ def find_coord(obj_type, obj_id):
 
 if __name__ == '__main__':
   database.connect()
-  q = Change.select().where(Change.changes.contains('[null,') & Change.action == 'm')
+  q = Change.select().where(Change.changes.startswith('[null,') & (Change.action == 'm')).limit(10)
   for row in q:
     changes = json.loads(row.changes)
     if not changes[0]:
-      changes[0] = find_coord(row.obj_type, row_obj_id)
+      coord = find_coord(row.obj_type, row.obj_id)
+      changes[0] = (coord, coord)
+      row.changes = changes
       row.save()

server/mapsme-process.py

@@ -1,41 +1,16 @@
-#!/usr/bin/python
-import sys, os, urllib2, re, gzip
-import peewee, json
+#!/usr/bin/env python
+import sys, os, urllib2, re, gzip, json
+from db import *
 from lxml import etree
 from StringIO import StringIO
 from datetime import datetime
 
-path = os.path.dirname(sys.argv[0]) if len(sys.argv) < 2 else sys.argv[1]
-database = peewee.SqliteDatabase(os.path.join(path, 'mapsme-changes.db'))
 STATE_FILENAME = os.path.join(path, 'mapsme-state.txt')
 REPLICATION_BASE_URL = 'http://planet.openstreetmap.org/replication/changesets'
 API_ENDPOINT = 'https://api.openstreetmap.org/api/0.6'
 MAIN_TAGS = ('amenity', 'shop', 'tourism', 'historic', 'craft', 'office', 'emergency', 'barrier', 'highway', 'leisure', 'waterway', 'entrance', 'building')
 INTERESTING_TAGS = list(MAIN_TAGS) + ['name']
 
-class Change(peewee.Model):
-  """A model for the change. Just a single table."""
-  changeset = peewee.IntegerField()
-  user = peewee.CharField(max_length=250, index=True)
-  version = peewee.CharField(max_length=250)
-  timestamp = peewee.DateTimeField(index=True)
-  action = peewee.FixedCharField(max_length=1) # c=created, d=deleted, m=modified, a=anomaly, n=note
-  obj_type = peewee.FixedCharField(max_length=1, null=True)
-  obj_id = peewee.IntegerField(null=True)
-  main_tag = peewee.CharField(max_length=100, null=True)
-  address = peewee.BooleanField(default=False)
-  changes = peewee.TextField()
-
-  class Meta:
-    database = database
-    db_table = 'mapsme_change'
-
-class Seen(peewee.Model):
-  """A model for a storage of processed objects."""
-  obj = peewee.TextField(index=True)
-  class Meta:
-    database = database
-
 def download_last_state():
   """Downloads last changeset replication sequence number from the planet website."""
   state = urllib2.urlopen(REPLICATION_BASE_URL + '/state.yaml').read()