Django 1.11 or 1.8 Admin App/Model Reordering

Django apps are registered via installed_apps in settings and models are registered via admin.py. And once registered they show up alphabetically on the admin panel. To reorder them we have to either tinker with the admin html files or add template tags. Here is a middleware that avoids all of that and helps you label admin apps, models and also order them the way you want.

Steps to follow:

Let us assume the following django directory structure:

myproject/
    manage.py
    myproject/
        __init__.py
        urls.py
        wsgi.py
        settings/
            __init__.py
            base.py
            dev.py
            prod.py
    myapp1/
        __init__.py
        models.py
        views.py
        urls.py
        templates/
           …
        static/
           …
        tests/
    myapp2/
        __init__.py
        models.py
        views.py
        urls.py
        templates/
           …
        static/
            …
        tests/

 

1- Create a mymiddleware.py file within any of you apps (say myapp1)

2- Add the following content to the mymiddleware.py file

from copy import deepcopy
from django.conf import settings
from django.contrib import admin
from django.core.exceptions import ImproperlyConfigured
from django.core.urlresolvers import resolve
'''
For Django 1.11 use this init function and class instead
from django.utils.deprecation import MiddlewareMixin
class AdminReorder(MiddlewareMixin):
    def __init__(self, get_response):
        self.get_response = get_response
'''
class AdminReorder(object):

    def init_config(self, request, app_list):
        self.request = request
        self.app_list = app_list

        self.config = getattr(settings, 'ADMIN_REORDER', None)
        if not self.config:
            # ADMIN_REORDER settings is not defined.
            raise ImproperlyConfigured('ADMIN_REORDER config is not defined.')

        if not isinstance(self.config, (tuple, list)):
            raise ImproperlyConfigured(
                'ADMIN_REORDER config parameter must be tuple or list. '
                'Got {config}'.format(config=self.config))

        admin_index = admin.site.index(request)
        try:
            # try to get all installed models
            app_list = admin_index.context_data['app_list']
        except KeyError:
            # use app_list from context if this fails
            pass

        # Flatten all models from apps
        self.models_list = []
        for app in app_list:
            for model in app['models']:
                model['model_name'] = self.get_model_name(
                    app['app_label'], model['object_name'])
                self.models_list.append(model)

    def get_app_list(self):
        ordered_app_list = []
        for app_config in self.config:
            app = self.make_app(app_config)
            if app:
                ordered_app_list.append(app)
        return ordered_app_list

    def make_app(self, app_config):
        if not isinstance(app_config, (dict, str)):
            raise TypeError('ADMIN_REORDER list item must be '
                            'dict or string. Got %s' % repr(app_config))

        if isinstance(app_config, str):
            # Keep original label and models
            return self.find_app(app_config)
        else:
            return self.process_app(app_config)

    def find_app(self, app_label):
        for app in self.app_list:
            if app['app_label'] == app_label:
                return app

    def get_model_name(self, app_name, model_name):
        if '.' not in model_name:
            model_name = '%s.%s' % (app_name, model_name)
        return model_name

    def process_app(self, app_config):
        if 'app' not in app_config:
            raise NameError('ADMIN_REORDER list item must define '
                            'a "app" name. Got %s' % repr(app_config))

        app = self.find_app(app_config['app'])
        if app:
            app = deepcopy(app)
            # Rename app
            if 'label' in app_config:
                app['name'] = app_config['label']

            # Process app models
            if 'models' in app_config:
                models_config = app_config.get('models')
                models = self.process_models(models_config)
                if models:
                    app['models'] = models
            return app

    def process_models(self, models_config):
        if not isinstance(models_config, (dict, list, tuple)):
            raise TypeError("models" config for ADMIN_REORDER list '
                            'item must be dict or list/tuple. '
                            'Got %s' % repr(models_config))

        ordered_models_list = []
        for model_config in models_config:
            model = None
            if isinstance(model_config, dict):
                model = self.process_model(model_config)
            else:
                model = self.find_model(model_config)

            if model:
                ordered_models_list.append(model)

        return ordered_models_list

    def find_model(self, model_name):
        for model in self.models_list:
            if model['model_name'] == model_name:
                return model

    def process_model(self, model_config):
        # Process model defined as { model: 'model', 'label': 'label' }
        for key in ('model', 'label', ):
            if key not in model_config:
                return
        model = self.find_model(model_config['model'])
        if model:
            model['name'] = model_config['label']
            return model

    def process_template_response(self, request, response):
        url = resolve(request.path)
        if not url.app_name == 'admin' and \
                url.url_name not in ['index', 'app_list']:
            # current view is not a django admin index
            # or app_list view, bail out!
            return response

        try:
            app_list = response.context_data['app_list']
        except KeyError:
            # there is no app_list! nothing to reorder
            return response

        self.init_config(request, app_list)
        ordered_app_list = self.get_app_list()
        response.context_data['app_list'] = ordered_app_list
        return response

3- Next add the following to the myproject/settings/base.py

MIDDLEWARE_CLASSES = (
…
'myapp1.mymiddleware.AdminReorder',
)

3- Now lets reorder and rename your models and apps.

Let say within myapp1 we have 3 models myapp1model1, myapp1model2, myapp1model3 and within myapp2 we have 2 models myapp2model1, myapp2model2

So within base.py again lets add this:

ADMIN_REORDER = (
{'app': 'myapp1', 'label': 'My Application 1',
'models': ({'model':"myapp1.myapp1model3",'label':'My App1 Model3'},
{'model':"myapp1.myapp1model2",'label': 'MyModel2'},
myapp1.myapp1model1)},
{'app': 'myapp2', 'label': 'My Application 2',
'models': ("myapp2.myapp2model2", "myapp2.myapp2model1")
)

So that will reorder you apps. Hope that helps!

Django Admin Custom Download Button

Django Admin interface provides action buttons to work with multiple records. But in order to create a custom button in django admin per row/record we might need some help of JS. The aim here is to create a Download button for each row which when clicked will download a field of that record and save it to a csv file.

Let’s say our models is as follows:

class OurModel(models.Model):
    items = JSONField(null=True, blank=True)
    name = models.CharField(max_length=200)

The items is JSON field with the following data:
{1:’abc’,2:’def’}

In order to display our Download button we will need the help of admin.py and a .js file. On admin.py we define the admin class:

class OurModelAdmin(admin.ModelAdmin):
    list_display = ('item', 'name', 'download_content')
    class Media:
        js = ('admin/js/ourmodel.js',)
    def download_content(self, obj):
        return '<a href="#">Download</a>'
    download_content.allow_tags = True
    download_content.short_description = "Download Content File"

And within ourmodel.js file we define how the download button works:

var $ = django.jQuery;
$(document).ready(function() {
    $('.field-download_content').click('a', function() {
        var id = $(this).parent().children('.action-checkbox').children('.action-select').val().toString();
        var items = JSON.parse($(this).parent().children('.field-item').html());
        var keys = Object.keys(items);
        var A = [['id', 'string']];
        for (var j = 1; j < keys.length; ++j) {
            A.push([keys[j], '"'+items[keys[j]]+'"', '']);
        }
        var csvRows = [];
        for (var i = 0, l = A.length; i < l; ++i) {
            csvRows.push(A[i].join(','));
        }
        var csvString = csvRows.join("%0A");
        var a = document.createElement('a');
        a.href = 'data:text/csv;charset=utf-8,' + csvString;
        a.target = '_blank';
        a.download = 'ourfile_'+id+'.csv';
        document.body.appendChild(a);
        a.click();
    });
});

ManytoMany fields in Django Model with specific order

Django Models has limited option when it comes to ManyToMany fields. Say you have a Image model with a records: image1, image2 .. upto image7. And you want to create a Gallery model as a list of images that has a specific order image4, image7, image2. Then we need to have image4, 7 and 2 inserted into Image model in the same order. Else the multi selection widget will not allow us to select in random order and hence the gallery will always have objects in the same order as has been inserted into Image model.

To solve this issue Django provides a through argument that allows to create another model to govern the many-to-many relationship between your Gallery and Image model.  Say we call Order Model and then our model class will look like this:

class Gallery(models.Model):
    name = models.CharField(max_length=200, null=False, blank=False, unique=True)
    image = models.ForeignKey('Image',through='Order')
class Image(models.Model):
    name = models.CharField(max_length=200, null=False, blank=False, unique=True)
    image = models.ImageField(upload_to='folder-path/static/img')
class Order(models.Model):
    image = models.ForeignKey('Image')
    gallery = models.ForeignKey('Gallery')

The problem with this method is, it is quite crude way to implement such a thing as everytime you create a gallery with 10 Image records you have to enter 10 records to Order model to say in what order they should be pushed into Gallery. A much cleaner way of doing this is by using a external javascript which will fake a list of Image dropdowns in a ordered pattern and allow the user to enter the images in the order needed and then saving it to json object and sending it to backend to save as a field. Following is a simple implementation of this:

Django models would look like this:

class Gallery(models.Model):
    name = models.CharField(max_length=200, null=False, blank=False, unique=True)
    size = models.IntegerField(null=False, blank=False, default=1,
                               help_text='Enter the number of images in the gallery. Default Min=1')
    #this field will be duplicated to create a fake ordering for user
    image = models.ForeignKey('Image')
    #this will be a hidden field to maintain our ordering
    image_order = JSONField()

class Image(models.Model):
    name = models.CharField(max_length=200, null=False, blank=False, unique=True)
    image = models.ImageField(upload_to='folder-path/static/img')
    url = models.CharField(max_length=3000, null=True, blank=True,
                           help_text='Path that will render this image')

The admin.py for this model would define the js file and save method as follows:

class GalleryAdmin(admin.ModelAdmin):
    class Media:
        js = ('admin/js/admin_custom_gallery.js',)
    @staticmethod
    def save_model(request, obj, form, change):
        if type(obj.image_order) is not str:
            obj.image_order = json.dumps(obj.image_order)
        obj.save()
class ImageAdmin(admin.ModelAdmin):
    @staticmethod
    def save_model(request, obj, form, change):
        obj.url = "img/" + str(obj.image)
        obj.save()

And the javascript file would look like this:

var $ = django.jQuery;
$(document).ready(function(){
    //Set the image order json field in admin to display none
    $('.image_order').css('display','none');
    var size = $('#id_size').val();

    /*Count maintains the count of dropdowns and index
    is the position. The json object is the input to the field
    that maintains the list of images and their order when
    saving gallery object
     */
    var count = 1;
    var index = 0;
    var jsonobject = {}

    /*
    We use the on function to check if there is a change in
    the size of gallery item. If the user changes it to a new
    value greater than current value, we add dropdowns else
    we delete them. Also we account for the case where the user
    deletes the size to blank before entering a new value and
    we ignore any changes for this case
    */
    $('#id_size').on('input', function() {
        if ($('#id_size').val()!=''){
            size = $('#id_size').val();
            //when element deleted remove dropdown and delete from jsonobject list
            if ($('#id_size').val() &lt; count ) {                 while (count &gt; $('#id_size').val()){
                    if ($('#id_size').val() == 0){
                        alert("Gallery size cannot be zero.");
                        $('#id_size').val(Object.keys(jsonobject).length);
                        break;
                    }
                    $('#id_image_'+index).remove();
                    delete jsonobject[index];
                    $('#id_image_order').text(JSON.stringify(jsonobject));
                    index--;
                    count--;
                }
            }

            //when element added add a new dropdown with blank value
            else{
                while (count &lt; $('#id_size').val()){                     if (index == 0){                         $('#id_image').clone().attr('id', 'id_image_'+count).insertAfter('#id_image');                         $("#id_image_"+count).val("");                     }                     else{                         $('#id_image_'+index).clone().attr('id', 'id_image_'+count).insertAfter('#id_image_'+index);                         $("#id_image_"+count).val("");                     }                     index++;                     count++;                 }             }             //when blank dropdown value changes update jsonobject list             $('[id^="id_image"]').change(function() {                 var id = $(this).attr('id').substring($(this).attr('id').lastIndexOf("_") + 1, $(this).attr('id').length);                 if (id=="image"){                     jsonobject[0] = $(this).val();                 }                 else {                     jsonobject[id] = $(this).val();                 }                 $('#id_image_order').text(JSON.stringify(jsonobject));             });         }     });     //if gallery object already exist we loop through images and create pre-populated dropdowns     if ($('#id_image_order').length &gt; 0) {
        var imageorder = JSON.parse($('#id_image_order').val().
            replace(/\\/g, '').replace(/"{/g, '{').replace(/}"/g, '}'));
        //first create dropdowns
        while (count &lt; $('#id_size').val()){
            if (index == 0){
                $('#id_image').clone().attr('id', 'id_image_'+count).insertAfter('#id_image');
            }
            else{
                $('#id_image_'+index).clone().attr('id', 'id_image_'+count).insertAfter('#id_image_'+index);
            }
            index++;
            count++;
        }
        //populate dropdowns with selected values
        $.each(imageorder, function(k, v) {
            if (k==0){
                $('#id_image option[value='+v+']').attr("selected","selected");
            }
            else{
                $('#id_image_'+k+' option[value='+v+']').attr("selected","selected");
            }
        });
        jsonobject = imageorder;
    }
    //when existing dropdown value changes update jsonobject list
    $('[id^="id_image"]').change(function() {
        var id = $(this).attr('id').substring($(this).attr('id').lastIndexOf("_") + 1, $(this).attr('id').length);
        if (id=="image"){
            jsonobject[0] = $(this).val();
        }
        else {
            jsonobject[id] = $(this).val();
        }
        $('#id_image_order').text(JSON.stringify(jsonobject));
    });
});

Last but not the least when retrieving values in views, this can be done using:

for gal in Gallery.objects.all():
    image_order = json.loads(gal.image_order)
    imagelist = []
    for key in range(0, len(image_order)):
        imagelist.append(Image.objects.get(id=int(image_order[str(key)])))
    gallery[gal.name] = imagelist

This gives a simple example of how to use javascript/jquery to implement admin functionality in Django.

Using Ajax Calls in Django Framework

Django is a high level Python based web framework. In order to create a working project in Django, the basic components includes a views which is a python file with the backend logic, an html template that would render all static components and a urls file that links these two python and html files. But what if we want to add some dynamic content to our html page or update the page on fly. We don’t want to refresh it each time we receive a new input from user or process it. For this purpose we use Ajax which uses XMLHTTPRequest to update content of the existing html page without refreshing it. Lets take a scenario where Ajax is used within Django framework to dynamically update the contents of the page. For example, say our html template loads the first time with a div that has a form to enter number of records to retrieve from database. Based on the user input, we display the contents using ajax and not reloading the entire page.

The initial base.html would render all static content and include all necessary jquery and css files and this would be inherited within our specific html template say myreport.html. This myreport.html would look something like this.

# extends 'base.html'
{% block content %}
<!-- This block is loaded initially to ask for user input -->
<div class="row-fluid">
	<div class="span12">
		<div class="widget-box">
			<div class="widget-title">
				<span class="icon">
					<i class="icon-list-alt"></i>
				</span>
				<h5>Input Values from User</h5>
			</div>	
            <div class="widget-content">
                Enter Number Of Records: <input type='text' name='numrecs' id='num_recs'>
                <button type="submit" value="View Report" onclick="send_request()>View Report</button>
            </div>
		</div>
    </div>
</div>
<!-- This block is loaded only after ajax call retrieves the data from table -->
<div class="row-fluid" id='example_table' style="display:none;">
	<div class="span12">
		<div class="widget-box">
			<div class="widget-title">
				<span class="icon">
					<i class="icon-list-alt"></i>
				</span>
				<h5>Table Name</h5>
			</div>	
            <div class="widget-content"></div>
			<div id="placeholder"></div>
    	</div>
	</div>
</div>
{% endblock content %}	

{% block script %}
<script id="source" language="javascript" type="text/javascript">
//This sends the user input on number of recs to views
function delete_request()
{
    params = {'records':$("#num_recs").val()};
    var serializedData = jQuery.param( params );
	$('#info_form').remove()
	$.ajax({
	  type: "POST",
	  url: "{{ lib.url('json_function') }}",
	  data: serializedData,
	  success: function(return_val){show_table(return_val[0]);},
	  dataType: 'JSON'
	});
}
//renders the report
function show_table(data)
{			
	var str="";
	var i=0;
	var j=0;
	if(data.hasOwnProperty('report'))
	{
		str="<table class='table' id='example_table'><thead><tr>";
		for(i=1;i<data.headers.length;i++)
			str=str+"<th>"+data.headers[i]+"</th>";
		str=str+"</tr></thead>";
		str=str+"<tbody><tr>";
		for(i=0;i<data.report.length;i++)
		{
			var row=jQuery.parseJSON(data.report[i]);
			for(j=1;j<data.fields.length;j++)
			{
				field=data.fields[j];
				str=str+"<td>"+row[field]+"</td>";
			}
		}
		str=str+"</tr></tbody>";
		str=str+"</table>";
		$('#placeholder').html(str);
		$('#example_table').css('display','block');
	}
	//table initialization
	var oTable = $('#example_table').dataTable( {
	    "bJQueryUI": true,
	    "bRetrieve": true,
	    "sDom": '<"F"lfT>t<"F"p>',
	    "iDisplayLength": 10,
	    "sPaginationType": "full_numbers",
	    "aaSorting": [[ 0, "desc" ]],
	    "oTableTools": {
			"sSwfPath": "/static/plugins/TableTools/media/swf/copy_csv_xls.swf",
			"aButtons": [
				"print",
				{
					"sExtends":    "collection",
					"sButtonText": 'Export <span class="caret" />',
					"aButtons":    [ "csv", "xls", "pdf" ]
				}
			]
		}
	} );  
}
</script>
{% endblock script %}

Next within our views.py we will create the function to render this html template and the function that will handle the ajax requests.

'''This page returns nothing except rendering the static form for user input'''
@render('myreport.html')
def myreport(request):
    return {
    }
'''This handles the ajax request'''
def json_function(request):
	data={}
	jdata=[]
	report = []
	records_to_retrieve = 0
	url = request.REQUEST
	fields = ['fieldA','fieldB','fieldC','fieldD']
	headers = ['A','B','C','D']
	if 'info_request' in url and 'records' in url:
		records_to_retrieve = int(url['records'])
		temp = DatabaseModel.objects.all().values(*fields)[:records_to_retrieve]
	try:
		records = CapacityDemand.objects.get(id = int(url['demand_id']))
		for row in records:
			report.append(json.dumps(row)) 
	data['report']=report
	data['headers']=headers
	data['fields']=fields
	jdata.append(data)
	response = json.dumps(jdata)
	return HttpResponse(response, mimetype="text/plain")

And in the end we link these two files in the urls.py file as follows:

    url(r'^myreport/$', 'appname.views.myreport', name='myreport'),
    url(r'^json_function/$', 'appname.views.json_function', name='json_function'),

Multilevel Drilldown in Jquery Datatables

Jquery datatables is powerful plugin to create html tables with numerous features like sorting, pagination, editing, deleting etc. For more information refer the following link.
https://datatables.net/
Implementing single level drilldown on this plugin is quite straightforwards. But when it comes to multilevel drilldown it becomes more complex to make it work with paginations and multiple levels of drilldown within it. The normal jquery bind and live methods some times fail to work as expected. So I have used ‘on’ instead of the bind/live. To understand the usage of bind vs live vs on, please refer http://www.elijahmanor.com/differences-between-jquery-bind-vs-live-vs-delegate-vs-on/. We also need to pass the element position while creating the drilldown tables. This will avoid the drilldown bug where in, it will not confuse between the element of two different level when opening and closing them.
So here is a working code of jquery datatable multilevel drilldown.

Service-Now data pull – python vs perl

Service-now is a IT service management software that hosts a wide ranging list of reports from different modules of IT services that are organized into tables. ServiceNow publishes its underlying table structures and associated data that can be pulled via SOAP query. This can be achieved through any scripting language. Although, python is my default goto scripting language, the soap client in python are not very well maintained. The most pythonic of those is the SUDS which does not seem to work well with service now api. While it is pretty slick and easy to get the basic data output using python/suds, as you move to more complex queries python fails. This is a known issue.

https://fedorahosted.org/suds/ticket/330
http://lists.fedoraproject.org/pipermail/suds/2011-October/001526.html
http://stackoverflow.com/questions/11850275/parameters-with-leading-underscores-in-python-suds

The __limit and encoded query does not seem to have the desired effect on the output as SUDS never passes these. So after few trials I ended up using Perl for the data pull and calling Perl from my Django framework. This worked like a charm. So here goes a snippet that can be used to get multiple records from an incident with max limit 10000 records and pulling single records from incident.

#!/usr/bin/perl -w
use SOAP::Lite;

# basic auth
sub SOAP::Transport::HTTP::Client::get_basic_credentials {
return 'user' => 'password';
}

# specify the endpoint to connect. This is the first incident that pulls say two values field1, field2 and field3 is used as a filter for records.
my $soap1 = SOAP::Lite -> proxy('https://instance.service-now.com/incident1.do?displayvalue=all&SOAP');
my $method1 = SOAP::Data->name("getRecords")-> attr({xmlns => "http://www.service-now.com/"});
my @params1 = ( SOAP::Data->name(field3=> "Your Value") );
push(@params1, SOAP::Data->name(__limit => "10000") );
my $som1 = $soap1->call($method1 => @params1);
my @serverData1 = @{$som1->body->{getRecordsResponse}->{getRecordsResult}};
foreach my $serverRec (@serverData1) {
	$f1 = $serverRec->{field1};
	$f2 = $serverRec->{field2};
	print $f1,',',$f2,"\n";
}
#incident 2 match  field4 and then retrieve field5 and 6 for single record
my $soap2 = SOAP::Lite -> proxy('https://instance.service-now.com/incident2.do?displayvalue=all&SOAP');
my @params2 = ( SOAP::Data->name(field4 => "Your Value") );
push(@params2, SOAP::Data->name(__limit => "1") );
$f5 = $soap2->call($method1 => @params2)->body->{getRecordsResponse}->{getRecordsResult}->{field5} || "None";
$f6 = $soap2->call($method1 => @params2)->body->{getRecordsResponse}->{getRecordsResult}->{field6} || "None";
print  $f5,',',$f6,"\n";

And then from python function we can easily call this perl script:

import subprocess
ret = subprocess.call(["perl","/path/name/only/servicenow.pl"])