analytics - Django saving the whole request for statistics, whats available?

Question

I want to save everything that can be used for statistics, such as referrer, os, browser etc. What is available and what's the best way to store it?

This is only important for 1 application (1 page) in the project, the other pages some standard analytics product will be used such as google analytics.

I had a look at django-tracking, but it seems this is overkill as I only want to use it on 1 view. The ideal situation would be, passing the whole request object to a TaskQue and do the processing later. So the user is redirected first and the analytics processing will be done behind the scenes.

Answer 1

We use some simple middleware.. below is an excerpt. You can modify it to use directly within a view.

class WebRequest(models.Model):
    time = models.DateTimeField(auto_now_add=True)
    host = models.CharField(max_length=1000)
    path = models.CharField(max_length=1000)
    method = models.CharField(max_length=50)
    uri = models.CharField(max_length=2000)
    status_code = models.IntegerField()
    user_agent = models.CharField(max_length=1000,blank=True,null=True)
    remote_addr = models.IPAddressField()
    remote_addr_fwd = models.IPAddressField(blank=True,null=True)
    meta = models.TextField()
    cookies = models.TextField(blank=True,null=True)
    get = models.TextField(blank=True,null=True)
    post = models.TextField(blank=True,null=True)
    raw_post = models.TextField(blank=True,null=True)
    is_secure = models.BooleanField()
    is_ajax = models.BooleanField()
    user = models.ForeignKey(User,blank=True,null=True)

def dumps(value):
    return json.dumps(value,default=lambda o:None)

class WebRequestMiddleware(object):
    def process_view(self, request, view_func, view_args, view_kwargs):
        setattr(request,'hide_post',view_kwargs.pop('hide_post',False))


    def process_response(self, request, response):

        if request.path.endswith('/favicon.ico'):
            return response

        if type(response) == HttpResponsePermanentRedirect and settings.APPEND_SLASH:
            new_location = response.get('location',None)
            content_length = response.get('content-length',None)

            if new_location and content_length is '0':
                new_parsed = urlparse(new_location)

                old = (('http','https')[request.is_secure()], request.get_host(), '{0}/'.format(request.path), request.META['QUERY_STRING'])
                new = (new_parsed.scheme, new_parsed.netloc, new_parsed.path, new_parsed.query)

                if old == new:
                    #dont log - it's just adding a /
                    return response
        try:
            self.save(request, response)
        except Exception as e:
            print >> sys.stderr, "Error saving request log", e

        return response

    def save(self, request, response):
        if hasattr(request, 'user'):
            user = request.user if type(request.user) == User else None
        else:
            user = None

        meta = request.META.copy()
        meta.pop('QUERY_STRING',None)
        meta.pop('HTTP_COOKIE',None)
        remote_addr_fwd = None

        if 'HTTP_X_FORWARDED_FOR' in meta:
            remote_addr_fwd = meta['HTTP_X_FORWARDED_FOR'].split(",")[0].strip()
            if remote_addr_fwd == meta['HTTP_X_FORWARDED_FOR']:
                meta.pop('HTTP_X_FORWARDED_FOR')

        post = None
        uri = request.build_absolute_uri()
        if request.POST and uri != '/login/':
            post = dumps(request.POST)

        models.WebRequest(
            host = request.get_host(),
            path = request.path,
            method = request.method,
            uri = request.build_absolute_uri(),
            status_code = response.status_code,
            user_agent = meta.pop('HTTP_USER_AGENT',None),
            remote_addr = meta.pop('REMOTE_ADDR',None),
            remote_addr_fwd = remote_addr_fwd,
            meta = None if not meta else dumps(meta),
            cookies = None if not request.COOKIES else dumps(request.COOKIES),
            get = None if not request.GET else dumps(request.GET),
            post = None if (not request.POST or getattr(request,'hide_post') == True) else dumps(request.POST),
            raw_post = None if getattr(request,'hide_post') else request.raw_post_data,
            is_secure = request.is_secure(),
            is_ajax = request.is_ajax(),
            user = user
        ).save()