Rename the main table, add user ranking

server/db.py

@@ -1,29 +1,40 @@
 import os.path, sys
-import peewee
+from peewee import *
 
 #path = os.path.dirname(sys.argv[0]) if len(sys.argv) < 2 else sys.argv[1]
 path = os.path.dirname(sys.argv[0])
-database = peewee.SqliteDatabase(os.path.join(path, 'mapsme-changes.db'))
+database = SqliteDatabase(os.path.join(path, 'mapsme-changes.db'))
 
-class Change(peewee.Model):
+class Change(Model):
   """A model for the change. Just a single table."""
-  changeset = peewee.IntegerField()
-  user = peewee.CharField(max_length=250, index=True)
-  version = peewee.CharField(max_length=250)
-  timestamp = peewee.DateTimeField(index=True)
-  action = peewee.FixedCharField(max_length=1) # c=created, d=deleted, m=modified, a=anomaly, n=note
-  obj_type = peewee.FixedCharField(max_length=1, null=True)
-  obj_id = peewee.IntegerField(null=True)
-  main_tag = peewee.CharField(max_length=100, null=True)
-  address = peewee.BooleanField(default=False)
-  changes = peewee.TextField()
+  changeset = IntegerField()
+  user = CharField(max_length=250, index=True)
+  version = CharField(max_length=250)
+  timestamp = DateTimeField(index=True)
+  action = FixedCharField(max_length=1, index=True) # c=created, d=deleted, m=modified, a=anomaly, n=note
+  obj_type = FixedCharField(max_length=1, null=True)
+  obj_id = IntegerField(null=True)
+  main_tag = CharField(max_length=100, null=True)
+  address = BooleanField(default=False)
+  processed = IntegerField(null=True) # number of hours between modifying and an external fix of the object
+  changes = TextField()
 
   class Meta:
     database = database
-    db_table = 'mapsme_change'
 
-class Seen(peewee.Model):
+class Seen(Model):
   """A model for a storage of processed objects."""
-  obj = peewee.TextField(index=True)
+  obj = TextField(index=True)
+
+  class Meta:
+    database = database
+
+class User(Model):
+  """A model for user stats."""
+  user = CharField(max_length=250, unique=True)
+  edits = IntegerField()
+  rank = IntegerField(default=0)
+  joined = DateField()
+
   class Meta:
     database = database

server/mapsme-process.py

@@ -3,7 +3,7 @@ import sys, os, urllib2, re, gzip, json
 from db import *
 from lxml import etree
 from StringIO import StringIO
-from datetime import datetime
+from datetime import datetime, date
 
 STATE_FILENAME = os.path.join(path, 'mapsme-state.txt')
 REPLICATION_BASE_URL = 'http://planet.openstreetmap.org/replication/changesets'
@@ -89,6 +89,18 @@ def record_object(obj1):
   seen.obj = obj_signature(obj1)
   seen.save()
 
+def record_user_edit(name):
+  """Update user edits count."""
+  try:
+    user = User.get(User.user == name)
+  except User.DoesNotExist:
+    user = User()
+    user.user = name
+    user.edits = 0
+    user.joined = date.today()
+  user.edits += 1
+  user.save()
+
 def create_change(changeset, obj):
   """Creates a Change object, ready to be populated with changes."""
   # Find the main tag
@@ -181,6 +193,7 @@ def record_obj_diff(changeset, obj, prev, anomalies):
   if ch is not None:
     ch.save()
     record_object(obj)
+    record_user_edit(changeset['user'])
 
 def record_changeset_diff(changeset):
   """Received changeset data dict, downloads individual object changes and store changes to a database."""
@@ -209,6 +222,20 @@ def record_changeset_diff(changeset):
     ch.changes = json.dumps(anomalies)
     ch.save()
 
+def update_user_ranks():
+  """Updates rank fields for users, so they are sorted in the descending order. O(n^2) complexity!"""
+  with database.atomic():
+    query = User.select().order_by(-User.edits, User.joined)
+    rank = count = 1
+    last_edits = -1
+    for user in query:
+      if user.edits != last_edits:
+        rank = count
+        last_edits = user.edits
+      user.rank = rank
+      user.save()
+      count += 1
+
 if __name__ == '__main__':
   try:
     cur_state = download_last_state()
@@ -221,7 +248,7 @@ if __name__ == '__main__':
     state = cur_state - 1
 
   database.connect()
-  database.create_tables([Change, Seen], safe=True)
+  database.create_tables([Change, Seen, User], safe=True)
 
   for i in range(state + 1, cur_state + 1):
     print i
@@ -234,3 +261,4 @@ if __name__ == '__main__':
       print 'Failed to download and process replication {0}: {1}'.format(i, e)
       raise e
     write_last_state(i)
+  update_user_ranks()

www/mmwatch.py

@@ -18,11 +18,12 @@ class Change(peewee.Model):
   user = peewee.CharField(max_length=250, index=True)
   version = peewee.CharField(max_length=250)
   timestamp = peewee.DateTimeField(index=True)
-  action = peewee.FixedCharField(max_length=1) # c, d, m, a
+  action = peewee.FixedCharField(max_length=1, index=True) # c, d, m, a, a
   obj_type = peewee.FixedCharField(max_length=1, null=True)
   obj_id = peewee.IntegerField(null=True)
   main_tag = peewee.CharField(max_length=100, null=True)
   address = peewee.BooleanField(default=False)
+  processed = IntegerField(null=True)
   changes = peewee.TextField()
 
   def explain_action(self):
@@ -58,7 +59,6 @@ class Change(peewee.Model):
 
   class Meta:
     database = database
-    db_table = 'mapsme_change'
 
 @app.before_request
 def before_request():