stats in DB

14 files changed, 237 insertions, 12 deletions

diff --git a/meetingtools/apps/room/views.py b/meetingtools/apps/room/views.py
index 92c9f8d..c19a242 100644
--- a/meetingtools/apps/room/views.py
+++ b/meetingtools/apps/room/views.py
@@ -319,7 +319,7 @@ def list_rooms(request,username=None):
             
     rooms = []
     if user:
-        rooms = Room.objects.filter(creator=user).order_by('name').all()
+        rooms = Room.objects.filter(creator=user).order_by('name').all().prefetch_related("creator","sco","folder_sco","source_sco","deleted_sco")
     
     return respond_to(request,
                       {'text/html':'apps/room/list.html'},
@@ -461,7 +461,7 @@ def _room2dict(request,room):
 # should not require login
 def list_by_tag(request,tn):
     tags = tn.split('+')
-    rooms = TaggedItem.objects.get_by_model(Room, tags).order_by('name').all()
+    rooms = TaggedItem.objects.get_by_model(Room, tags).order_by('name').all().prefetch_related("creator","sco","folder_sco","source_sco","deleted_sco")
     title = 'Rooms tagged with %s' % " and ".join(tags)
     return respond_to(request,
                       {'text/html':'apps/room/list.html',
@@ -547,7 +547,7 @@ def room_recordings(request,room):
              'url': room.sco.acc.make_url(ar.urlpath),
              'dl': room.sco.acc.make_url(ar.urlpath),
              'date_created': ar.timecreated,
-             'date_modified': ar.lastupdated} for ar in room.archives.all()
+             'date_modified': ar.lastupdated} for ar in room.archives.all().prefetch_related("creator","sco","folder_sco","source_sco","deleted_sco")
         ]
 @login_required
 def recordings(request,rid):
diff --git a/meetingtools/apps/sco/models.py b/meetingtools/apps/sco/models.py
index 325baae..3607579 100644
--- a/meetingtools/apps/sco/models.py
+++ b/meetingtools/apps/sco/models.py
@@ -1,6 +1,7 @@
 """
 Abstract sco objects and utility methods
 """
+import logging
 
 __author__ = 'leifj'
 
@@ -53,7 +54,7 @@ def get_sco(acc,sco_id):
     if sco is None:
         sco,created = ACObject.objects.get_or_create(acc=acc,sco_id=sco_id)
         assert sco is not None
-        cache.set(key,sco)
+        cache.set(key,sco,30)
     return sco
 
 def get_sco_shortcuts(acc,shortcut_id):
diff --git a/meetingtools/apps/stats/admin.py b/meetingtools/apps/stats/admin.py
new file mode 100644
index 0000000..f9ae86e
--- /dev/null
+++ b/meetingtools/apps/stats/admin.py
@@ -0,0 +1,10 @@
+'''
+Created on Jan 31, 2011
+
+@author: leifj
+'''
+
+from django.contrib import admin
+from meetingtools.apps.stats.models import UserMeetingTransaction
+
+admin.site.register(UserMeetingTransaction)
\ No newline at end of file
diff --git a/meetingtools/apps/stats/forms.py b/meetingtools/apps/stats/forms.py
index 9fa5225..fbe9d5b 100644
--- a/meetingtools/apps/stats/forms.py
+++ b/meetingtools/apps/stats/forms.py
@@ -3,9 +3,15 @@ Created on Jan 16, 2012
 
 @author: leifj
 """
+from django.contrib.auth.models import User
+from django.forms import ModelChoiceField
 from django.forms.forms import Form
-from django.forms.fields import DateTimeField
+from django.forms.fields import DateTimeField, CharField
+from meetingtools.apps.sco.models import ACObject
 
 class StatCaledarForm(Form):
+    tags = CharField(required=False)
+    user = ModelChoiceField(User.objects,required=False)
+    sco = ModelChoiceField(ACObject.objects,required=False)
     begin = DateTimeField(required=False)
     end = DateTimeField(required=False)
\ No newline at end of file
diff --git a/meetingtools/apps/stats/management/__init__.py b/meetingtools/apps/stats/management/__init__.py
new file mode 100644
index 0000000..3929ed7
--- /dev/null
+++ b/meetingtools/apps/stats/management/__init__.py
@@ -0,0 +1 @@
+__author__ = 'leifj'
diff --git a/meetingtools/apps/stats/management/commands/__init__.py b/meetingtools/apps/stats/management/commands/__init__.py
new file mode 100644
index 0000000..3929ed7
--- /dev/null
+++ b/meetingtools/apps/stats/management/commands/__init__.py
@@ -0,0 +1 @@
+__author__ = 'leifj'
diff --git a/meetingtools/apps/stats/management/commands/import_sessions.py b/meetingtools/apps/stats/management/commands/import_sessions.py
new file mode 100644
index 0000000..d98b081
--- /dev/null
+++ b/meetingtools/apps/stats/management/commands/import_sessions.py
@@ -0,0 +1,20 @@
+from optparse import make_option
+from django.core.management import BaseCommand
+from meetingtools.apps.stats.tasks import import_acc_sessions
+from meetingtools.apps.cluster.models import ACCluster
+
+__author__ = 'leifj'
+
+class Command(BaseCommand):
+
+    option_list = BaseCommand.option_list + (
+        make_option('--since',
+            type='int',
+            dest='since',
+            default=0,
+            help='Import all sessions <since> seconds ago'),
+        )
+
+    def handle(self, *args, **options):
+        for acc in ACCluster.objects.all():
+            import_acc_sessions(acc,since=options['since'])
\ No newline at end of file
diff --git a/meetingtools/apps/stats/models.py b/meetingtools/apps/stats/models.py
new file mode 100644
index 0000000..495bd65
--- /dev/null
+++ b/meetingtools/apps/stats/models.py
@@ -0,0 +1,66 @@
+import logging
+from django.contrib.auth.models import User
+from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned
+from iso8601 import iso8601
+import tagging
+from tagging.models import Tag
+from meetingtools.apps.sco.models import ACObject, get_sco
+
+__author__ = 'leifj'
+
+from django.db import models
+from django.db.models import fields, ForeignKey, DateTimeField, IntegerField
+import lxml.etree as etree
+
+class UserMeetingTransaction(models.Model):
+    sco = ForeignKey(ACObject)
+    user = ForeignKey(User)
+    txid = IntegerField()
+    date_created = DateTimeField()
+    date_closed = DateTimeField()
+
+    def seconds(self):
+        delta = self.date_closed - self.date_created
+        return delta.total_seconds()
+
+    def __unicode__(self):
+        return "(%d) %d seconds in %s" % (self.txid,self.seconds(),self.sco)
+
+    @staticmethod
+    def create(acc,row):
+        txid = int(row.get('transaction-id'))
+        sco_id = int(row.get('sco-id'))
+
+        status = row.findtext("status")
+        if not status or status != "completed":
+            logging.debug("Ignoring transaction %s" % etree.tostring(row))
+            return
+
+        txo = None
+        try:
+            txo = UserMeetingTransaction.objects.get(sco__acc=acc,txid=txid)
+        except MultipleObjectsReturned,ex:
+            logging.error(ex)
+        except ObjectDoesNotExist:
+            login = row.findtext("login")
+            if not login:
+                raise ValueError("No user for transaction %d" % txid)
+            user,created = User.objects.get_or_create(username=login)
+            date_created=iso8601.parse_date(row.findtext("date-created"))
+            #date_created = date_created.replace(tzinfo=None)
+            date_closed=iso8601.parse_date(row.findtext("date-closed"))
+            #date_close = date_closed.replace(tzinfo=None)
+            txo = UserMeetingTransaction.objects.create(sco=get_sco(acc,sco_id),
+                                                        txid=txid,
+                                                        user=user,
+                                                        date_created=date_created,
+                                                        date_closed=date_closed)
+        tags = []
+        for group in txo.user.groups.all():
+            tags.append("group:%s" % group.name)
+
+        (local,domain) = txo.user.username.split("@")
+        tags.append("domain:%s" % domain)
+        Tag.objects.update_tags(txo, ' '.join(tags))
+
+tagging.register(UserMeetingTransaction)
\ No newline at end of file
diff --git a/meetingtools/apps/stats/tasks.py b/meetingtools/apps/stats/tasks.py
new file mode 100644
index 0000000..fb4b5c7
--- /dev/null
+++ b/meetingtools/apps/stats/tasks.py
@@ -0,0 +1,33 @@
+import logging
+from celery.schedules import crontab
+from celery.task import periodic_task
+from meetingtools.ac import ac_api_client
+from meetingtools.apps.cluster.models import ACCluster
+from datetime import datetime,timedelta
+from meetingtools.apps.stats.models import UserMeetingTransaction
+
+__author__ = 'leifj'
+
+def import_acc_sessions(acc,since=0):
+    with ac_api_client(acc) as api:
+        p = {'sort': 'asc','sort1': 'date-created','filter-type': 'meeting'}
+
+        begin = None
+        if since > 0:
+            begin = datetime.now()-timedelta(seconds=since)
+            begin = begin.replace(microsecond=0)
+
+        if begin is not None:
+            p['filter-gte-date-created'] = begin.isoformat()
+
+        r = api.request('report-bulk-consolidated-transactions',p,True)
+        for tx in r.et.findall(".//row"):
+            try:
+                UserMeetingTransaction.create(acc,tx)
+            except Exception,ex:
+                logging.error(ex)
+
+@periodic_task(run_every=crontab(hour="*", minute="*", day_of_week="*"))
+def import_sessions(since=3700):
+    for acc in ACCluster.objects.all():
+        import_acc_sessions(acc,since)
\ No newline at end of file
diff --git a/meetingtools/apps/stats/views.py b/meetingtools/apps/stats/views.py
index 252b827..60628a2 100644
--- a/meetingtools/apps/stats/views.py
+++ b/meetingtools/apps/stats/views.py
@@ -3,10 +3,12 @@ Created on Jan 16, 2012
 
 @author: leifj
 """
+import logging
 
 from django.contrib.auth.decorators import login_required
 from django.contrib.humanize.templatetags.humanize import naturalday
 from django.http import HttpResponseForbidden, HttpResponseBadRequest
+from meetingtools.apps.stats.models import UserMeetingTransaction
 from meetingtools.ac import ac_api_client
 from iso8601 import iso8601
 from time import mktime
@@ -19,6 +21,9 @@ def _iso2datesimple(iso):
     (date,rest) = iso.split("T")
     return date
 
+def _dt2ts(dt):
+    return mktime(dt.timetuple())*1000
+
 def _iso2ts(iso):
     return mktime(iso8601.parse_date(iso).timetuple())*1000
 
@@ -126,6 +131,7 @@ def user_minutes_api(request,username=None):
                 #logging.debug("midnight: %d (%d)" % (ts_date_ts,ts_closed))
                 ms = (ts_closed - ts_date_ts)
                 #logging.debug("nms: %d" % ms)
+
                 
         if curdate is not None and ms > 0:
             series.append([_date_ts(curdate),int(ms/60000)])
@@ -133,6 +139,85 @@ def user_minutes_api(request,username=None):
         return json_response({'data': sorted(series,key=lambda x: x[0]), 'rooms': len(rc.keys()), 'minutes': int(t_ms/60000)},request)
 
 @login_required
+def tagged_minutes_api(request):
+    form = StatCaledarForm(request.GET) # convenient way to parse dates
+    if not form.is_valid():
+        return HttpResponseBadRequest()
+
+    tags = filter(lambda x: bool(x), form.cleaned_data['tags'].strip().split("+"))
+    sco = form.cleaned_data['sco']
+    begin = form.cleaned_data['begin']
+    end = form.cleaned_data['end']
+    user = form.cleaned_data['user']
+
+    qs = UserMeetingTransaction.objects
+
+    if user:
+        qs = qs.filter(user=user)
+    if sco:
+        qs = qs.filter(sco=sco)
+    if begin:
+        qs = qs.filter(date_created__gt=begin)
+    if end:
+        qs = qs.filter(date_closed__lt=end)
+
+    if len(tags) > 0:
+        qs = UserMeetingTransaction.tagged.with_all(tags,qs)
+
+    series = []
+    d_created  = None
+    d_closed = None
+    ms = 0
+    curdate = None
+    t_ms = 0
+    rc = {}
+    uc = {}
+
+    for tx in qs.all().prefetch_related("sco").prefetch_related("user"):
+        rc[tx.sco.id] = True
+        uc[tx.user.username] = True
+        ts_created = _dt2ts(tx.date_created)
+        ts_closed = _dt2ts(tx.date_closed)
+
+        d1 = tx.date_created
+        if d_created is None:
+            d_created = d1
+
+        d2 = tx.date_closed
+        if d_closed is None:
+            d_closed = d2
+
+        if curdate is None:
+            curdate = d1
+
+        if curdate != d1:
+            series.append([_dt2ts(curdate),int(ms/60)])
+            ms = 0
+            curdate = d1
+
+        if d1 == d2: #same date
+            diff = (ts_closed - ts_created)
+            ms += diff
+            t_ms += diff
+        else: # meeting spanned midnight
+            ts_date_ts = _dt2ts(d2)
+            ms += ts_date_ts - ts_created
+            series.append([_dt2ts(d1),int(ms/60)])
+            t_ms += ms
+            curdate = d2
+            ms = (ts_closed - ts_date_ts)
+
+    if curdate is not None and ms > 0:
+        series.append([_date_ts(curdate),int(ms/60)])
+
+    return json_response({'data': sorted(series,key=lambda x: x[0]),
+                          'rooms': len(rc.keys()),
+                          'begin': naturalday(begin),
+                          'end': naturalday(end),
+                          'users': len(uc.keys()),
+                          'minutes': int(t_ms/60)},request)
+
+@login_required
 def domain_minutes_api(request,domain):
     with ac_api_client(request) as api:
         p = {'sort': 'asc','sort1': 'date-created','filter-type': 'meeting'}
diff --git a/meetingtools/context_processors.py b/meetingtools/context_processors.py
index f284867..f20b3fb 100644
--- a/meetingtools/context_processors.py
+++ b/meetingtools/context_processors.py
@@ -22,4 +22,7 @@ def theme(request):
     return _w(ctx)
 
 def misc_urls(request):
-    return {'LOGIN_URL': settings.LOGIN_URL,'BASE_URL':base_url(request)}
\ No newline at end of file
+    return {'LOGIN_URL': settings.LOGIN_URL,'BASE_URL':base_url(request)}
+
+def request(request):
+    return {'request': request}
\ No newline at end of file
diff --git a/meetingtools/multiresponse.py b/meetingtools/multiresponse.py
index 42139b5..1510760 100644
--- a/meetingtools/multiresponse.py
+++ b/meetingtools/multiresponse.py
@@ -30,7 +30,7 @@ def make_response_dict(request,d=dict()):
     if request.user.is_authenticated():
         d['user'] = request.user
 
-    ctx = RequestContext(request,d,[context_processors.theme,context_processors.misc_urls])
+    ctx = RequestContext(request,d,[context_processors.theme,context_processors.misc_urls,context_processors.request])
     print repr(ctx['theme'])
     return ctx
 
diff --git a/meetingtools/settings.py b/meetingtools/settings.py
index dac6890..0bb0ca1 100644
--- a/meetingtools/settings.py
+++ b/meetingtools/settings.py
@@ -56,6 +56,7 @@ USE_I18N = True
 # If you set this to False, Django will not format dates, numbers and
 # calendars according to the current locale
 USE_L10N = True
+USE_TZ = True
 
 STATIC_ROOT = "%s/static" % BASE_DIR
 STATIC_URL = "/static/"
@@ -149,18 +150,15 @@ djcelery.setup_loader()
 NOREPLY = "no-reply@sunet.se"
 AUTH_PROFILE_MODULE = "userprofile.UserProfile"
 
-from django.conf import settings
-from saml2 import BINDING_HTTP_POST, BINDING_HTTP_REDIRECT
-
 LOGIN_URL = '/saml2/sp/login/'
 LOGIN_REDIRECT_URL = "/rooms"
 
-AUTHENTICATION_BACKENDS += ('asgard.saml.Saml2Backend',)
-
 AUTO_REMOTE_SUPERUSERS = ['leifj@nordu.net']
 
 try:
     from asgard.loader import *
     DEBUG=True
+
+    AUTHENTICATION_BACKENDS += ('asgard.saml.Saml2Backend',)
 except ImportError,ex:
     print ex
diff --git a/meetingtools/urls.py b/meetingtools/urls.py
index 7e39f5c..cc0c3fb 100644
--- a/meetingtools/urls.py
+++ b/meetingtools/urls.py
@@ -55,6 +55,7 @@ urlpatterns = patterns('',
     (r'^widget/?\+?(.*)$','meetingtools.apps.room.views.widget'),
     # Uncomment the admin/doc line below to enable admin documentation:
     # (r'^admin/doc/', include('django.contrib.admindocs.urls')),
+    (r'^api/stats/?','meetingtools.apps.stats.views.tagged_minutes_api'),
     (r'^api/stats/user/(.*)$','meetingtools.apps.stats.views.user_minutes_api'),
     (r'^api/stats/domain/(.+)$','meetingtools.apps.stats.views.domain_minutes_api'),
     (r'^api/stats/room/(\d+)$','meetingtools.apps.stats.views.room_minutes_api'),