Spread the full import to fewer months at a time.

3 files changed, 44 insertions, 41 deletions

diff --git a/meetingtools/apps/content/tasks.py b/meetingtools/apps/content/tasks.py
index eef0e58..ad426ef 100644
--- a/meetingtools/apps/content/tasks.py
+++ b/meetingtools/apps/content/tasks.py
@@ -93,7 +93,7 @@ def get_domain_content(domain_tag):
 #@periodic_task(run_every=crontab(hour="1", minute="0", day_of_week="*"))
 def timed_full_import():
     years = [2009, 2010, 2011, 2012, 2013, 2014]
-    months = [(1, 7), (8, 12)]  # Ugly hack as June does not have 31 days
+    months = [(1, 3), (4, 7), (8, 10), (9, 12)]
     for acc in ACCluster.objects.all():
         nr = 0
         for year in years:
diff --git a/meetingtools/apps/room/tasks.py b/meetingtools/apps/room/tasks.py
index 4fc0c74..518669b 100644
--- a/meetingtools/apps/room/tasks.py
+++ b/meetingtools/apps/room/tasks.py
@@ -314,25 +314,26 @@ def clean_old_rooms():
                 send_message.apply_async([room.creator,"You have an unused meetingroom at %s" % acc.name ,"Do you still need %s (%s)?" % (room.name,room.permalink())])
 
 
-
 #@periodic_task(run_every=crontab(hour="3", minute="0", day_of_week="*"))
 def timed_full_import():
     years = [2009, 2010, 2011, 2012, 2013, 2014]
+    months = [(1, 3), (4, 7), (8, 10), (9, 12)]
     for acc in ACCluster.objects.all():
         for year in years:
-            begin = datetime(year=year, month=1, day=1)
-            end = datetime(year=year, month=12, day=31)
-            with ac_api_client(acc) as api:
-                r = api.request('report-bulk-objects', {'filter-type': 'meeting',
-                                                        'filter-gte-date-modified': begin.isoformat(),
-                                                        'filter-lte-date-modified': end.isoformat()})
-                nr = 0
-                ne = 0
-                for row in r.et.xpath("//row"):
-                    try:
-                        _import_one_room(acc, api, row)
-                        nr += 1
-                    except Exception, ex:
-                        logging.error(ex)
-                        ne += 1
-                logging.info("%s: Imported %d rooms and got %d errors" % (acc, nr, ne))
\ No newline at end of file
+            for month in months:
+                begin = datetime(year=year, month=month[0], day=1)
+                end = datetime(year=year, month=month[1], day=31)
+                with ac_api_client(acc) as api:
+                    r = api.request('report-bulk-objects', {'filter-type': 'meeting',
+                                                            'filter-gte-date-modified': begin.isoformat(),
+                                                            'filter-lte-date-modified': end.isoformat()})
+                    nr = 0
+                    ne = 0
+                    for row in r.et.xpath("//row"):
+                        try:
+                            _import_one_room(acc, api, row)
+                            nr += 1
+                        except Exception, ex:
+                            logging.error(ex)
+                            ne += 1
+                    logging.info("%s: Imported %d rooms and got %d errors" % (acc, nr, ne))
\ No newline at end of file
diff --git a/meetingtools/apps/stats/tasks.py b/meetingtools/apps/stats/tasks.py
index 77e7513..acbf76e 100644
--- a/meetingtools/apps/stats/tasks.py
+++ b/meetingtools/apps/stats/tasks.py
@@ -55,28 +55,30 @@ def _hourly_import():
 #@periodic_task(run_every=crontab(hour="5", minute="0", day_of_week="*"))
 def timed_full_import():
     years = [2009, 2010, 2011, 2012, 2013, 2014]
+    months = [(1, 3), (4, 7), (8, 10), (9, 12)]
     for acc in ACCluster.objects.all():
         for year in years:
-            begin = datetime(year=year, month=1, day=1)
-            end = datetime(year=year, month=12, day=31)
-            with ac_api_client(acc) as api:
-                p = {'sort': 'asc', 'sort1': 'date-created', 'filter-type': 'meeting',
-                     'filter-gte-date-created': begin.isoformat(), 'filter-lte-date-created': end.isoformat()}
-                r = api.request('report-bulk-consolidated-transactions', p, False)
-                nr = 0
-                ne = 0
-                for tx in r.et.findall(".//row"):
-                    try:
-                        tx = UserMeetingTransaction.create(acc, tx)
-                        if tx:
-                            rooms = Room.objects.filter(sco=tx.sco)
-                            if len(rooms) == 1:
-                                room = rooms[0]
-                                if room.lastvisited is None or room.lastvisited < tx.date_closed:
-                                    room.lastvisited = tx.date_created
-                                    room.save()
-                            nr += 1
-                    except Exception, ex:
-                        logging.error(ex)
-                        ne += 1
-                logging.info("%s: Imported %d transactions with %d errors" % (acc, nr, ne))
\ No newline at end of file
+            for month in months:
+                begin = datetime(year=year, month=month[0], day=1)
+                end = datetime(year=year, month=month[1], day=31)
+                with ac_api_client(acc) as api:
+                    p = {'sort': 'asc', 'sort1': 'date-created', 'filter-type': 'meeting',
+                         'filter-gte-date-created': begin.isoformat(), 'filter-lte-date-created': end.isoformat()}
+                    r = api.request('report-bulk-consolidated-transactions', p, False)
+                    nr = 0
+                    ne = 0
+                    for tx in r.et.findall(".//row"):
+                        try:
+                            tx = UserMeetingTransaction.create(acc, tx)
+                            if tx:
+                                rooms = Room.objects.filter(sco=tx.sco)
+                                if len(rooms) == 1:
+                                    room = rooms[0]
+                                    if room.lastvisited is None or room.lastvisited < tx.date_closed:
+                                        room.lastvisited = tx.date_created
+                                        room.save()
+                                nr += 1
+                        except Exception, ex:
+                            logging.error(ex)
+                            ne += 1
+                    logging.info("%s: Imported %d transactions with %d errors" % (acc, nr, ne))
\ No newline at end of file