Python/Django Storage Backend Notes

Inspired by:
https://django-storages.readthedocs.org/en/latest/backends/amazon-S3.html#storage

Interacting directly with storage backend
from django.conf import settings
from django.core.files.storage import get_storage_class
STORAGE_CLASS_STRING = getattr(settings, "MY_STORAGE_CLASS",  \
        settings.DEFAULT_FILE_STORAGE)
sc = get_storage_class(STORAGE_CLASS_STRING)
s = sc()
s.url("jjj-test/JOE.jpg")
# '/media/jjj-test/JOE.jpg'
s.path("jjj-test/JOE.jpg")
# u'/home/jjasinski/Sites/mysite/htdocs/media/jjj-test/JOE.jpg'
s.exists("jjj-test/JOE.jpg")
# False
f = s.open("jjj-test/JOE.jpg", 'w')
f.write("joe test")
f.close()
s.exists("jjj-test/JOE.jpg")
# True
s.delete("jjj-test/JOE.jpg")
s.exists("jjj-test/JOE.jpg")
# False

Interacting with a Model's storage
from django.core.files.base import ContentFile
obj = MyModel()
obj.photo.save('django_test.txt', ContentFile('content'))
obj.photo.size
obj.photo.read()
obj.delete()

Python Email Send

Simple email test from python

import smtplib
from email.mime.text import MIMEText
f = "no-reply@example.com"
t = "joe@example.com"
s = smtplib.SMTP('localhost')
msg = MIMEText("this is a test")
msg["Subject"] = "This is a test - ignore"
msg["To"] = t
msg["From"] = f
s.sendmail(f, [t], msg.as_string())

Python Tokenlib

Make an expiring token using a known secret.

Source:
https://github.com/mozilla-services/tokenlib

Make a token, hashed with a secret, and storing an IP, that expires in 12 seconds
>>> t = tokenlib.make_token({"ip":"1.1.1.1."}, secret="1234567890", timeout=12)

Parse a token before expiration time is up
>>> tokenlib.parse_token(t, secret="1234567890")
{u'ip': u'1.1.1.1.', u'expires': 1428098130.01954, u'salt': u'fbd08e'}

Parse a token after expiration time is up
>>> tokenlib.parse_token(t, secret="1234567890")
Traceback (most recent call last):
  File "", line 1, in
  File "tokenlib/__init__.py", line 186, in parse_token
    return TokenManager(**kwds).parse_token(token, now=now)
  File "/tokenlib/__init__.py", line 143, in parse_token
    raise errors.ExpiredTokenError()
tokenlib.errors.ExpiredTokenError: token has expired

Simple Python Http Server

Serves up files in the current directory.

Python 3
python3 -m http.server
or
python3 -m http.server 8080

Python 2
python -m SimpleHTTPServer
or 
python -m SimpleHTTPServer 8080

Source:
http://www.reddit.com/r/Python/comments/2ysd91/what_are_some_nifty_python_snippets_that_you_have/

Python: lxml insert element before and after given element

Python: lxml insert element before and after given element

Based on: http://stackoverflow.com/questions/7474972/python-lxml-append-element-after-another-element

from lxml import etree

xml = etree.parse('old_xml.xml')

count = 1

for pp in xml.xpath('/response/items/item'):

   prod_id = etree.Element('new_element')  # this element will be inserted

   prod_id.text = "%s" % count

   nav = pp.find('name')  # we will insert before/after this element

   div = nav.getparent()

   pp.insert(pp.index(nav), prod_id) # insert before 'name'

   pp.insert(pp.index(nav) + 1, prod_id) # insert after 'name'

   count += 1

out = open('new_xml.xml','w')

out.write(etree.tostring(xml, pretty_print=True))

out.close()

Python: Detect Charset and convert to Unicode


Converting data to Unicode via chardet 
http://stackoverflow.com/questions/2686709/encoding-in-python-with-lxml-complex-solution

import chardet
from lxml import html

f = open('my.xml')

content = f.read()

f.close()

encoding = chardet.detect(content)['encoding']
if encoding != 'utf-8':
    content = content.decode(encoding, 'replace').encode('utf-8')

Converting Data to Unicode via UnicodeDammit 

http://lxml.de/elementsoup.html

from BeautifulSoup import UnicodeDammit

converted = UnicodeDammit(content)

   if not converted.unicode: 

    raise UnicodeDecodeError( 

       "Failed to detect encoding, tried [%s]", 

        ', '.join(converted.triedEncodings))  

        # print converted.originalEncoding

         return converted.unicode

Credit goes to Ian Bicking and others on the lxml team

SQLAlchemy Notes

Summarized From 
http://www.sqlalchemy.org/docs/core/tutorial.html


This covers the basics of the Non-orm usage of SQLAlchemy

# Define connection engine
from sqlalchemy import create_engine
engine = create_engine('sqlite:///:memory:', echo=True)

# Create some database tables 
from sqlalchemy import Table, Column, Integer, String, MetaData, ForeignKey
metadata = MetaData()
users = Table('users', metadata,
    Column('id', Integer, primary_key=True),
    Column('name', String),
    Column('fullname', String),
)

addresses = Table('addresses', metadata,
  Column('id', Integer, primary_key=True),
  Column('user_id', None, ForeignKey('users.id')),
  Column('email_address', String, nullable=False)
)

metadata.create_all(engine)

# Insert into the tables
ins = users.insert()
ins.values(name='jack', fullname='Jack the Pumpkin king')

conn = engine.connect()
result = conn.execute(ins)
result.inserted_primary_key

# Multiple insert into the tables
conn.execute(addresses.insert(), [
   {'user_id': 1, 'email_address' : 'jack@yahoo.com'},
   {'user_id': 1, 'email_address' : 'jack@msn.com'},
   {'user_id': 2, 'email_address' : 'www@www.org'},
   {'user_id': 2, 'email_address' : 'wendy@aol.com'},
])

result = engine.execute(users.insert(), name='fred', fullname="Fred Flintstone")

# Select from the tables
from sqlalchemy.sql import select

# Select all fields in user table
s = select([users])
result = conn.execute(s)
list(result)

# Select name and fullname columns from user table
s = select([users.c.name, users.c.fullname])

# Join two user and address tables
#  Cartesian product 
list(conn.execute(select([users, addresses])))
# Join on id columns
list(conn.execute(select([users, addresses], users.c.id==addresses.c.user_id)))

# create conditions for the where clause
from sqlalchemy.sql import and_, or_, not_
print and_(users.c.name.like('j%'), users.c.id==addresses.c.user_id, )


s = select([(users.c.fullname + "--" + addresses.c.email_address).label('title')])
print conn.execute(s).fetchall()

print users.join(addresses)
print users.join(addresses, addresses.c.email_address.like(users.c.name + '%'))


s = select([users.c.fullname], from_obj=[
   users.join(addresses, addresses.c.email_address.like(users.c.name + '%'))
   ])
print conn.execute(s).fetchall()

# Delete records based on a condition
conn.execute(users.delete().where(users.c.name > 'm'))

Python Process Info

Pipe the output of one process into the input of another via python:
import subprocess
p1 = subprocess.Popen(['echo','joe'], shell=False, stdout=subprocess.PIPE)
p2 = subprocess.Popen(['cat','-n'],
    stdin=p1.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
p2.stdout.read()

# Or 
out,err=p2.communicate()

Pipe standard input to a program
import subprocess
p1 = subprocess.Popen(['node', 'index.js'], shell=False,
    stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p1.communicate('standard input here')
p1.returncode

Convenience method for splitting apart shell commands for use in the above:
>>> import shlex
>>> shlex.split('ps -ef')
['ps', '-ef']

Whoosh Python Search Notes

Baisc Whoosh Search Walkthrough 


# Import required 
import os
from whoosh.index import create_in
from whoosh.fields import *

# Create a "Schema" object to store text to search
# You can create custom search fields such as "title", "domain", "path"
# and "content". 
schema = Schema(title=TEXT(stored=True), domain=ID(stored=True),
                              path=ID(stored=True), content=TEXT)

# Check to see if a search index directory exists; if not, create one
if not os.path.exists("search_test"):
    os.mkdir("search_test")

# Create a search index in the search index directory
ix = create_in("search_test", schema, indexname="testindex")
writer = ix.writer()

# Index some documents
writer.add_document(title=u"Document 1", domain=u"www.mysearch.com",
                         path=u"/mywebsite", content=u'Article')
writer.add_document(title=u"Document 2", domain=u"www.mysearch.com",
                         path=u"/mywebsite/2/article/1", content=u'Document a')
writer.commit()

# Running search queries
from whoosh.qparser import QueryParser
import whoosh.index as index
# Check to see if the given search index exists
if index.exists_in("search_test", indexname="testindex"):
    # open the search index
    ix = index.open_dir("search_test", indexname="testindex")
    # generate a search query based on some string.
    # Here is creates a query looking for "Document" in the "content"
    # search index field. 
    query = QueryParser("content", ix.schema).parse(u'*Document*')
    with ix.searcher() as searcher:  
        # do the actual search
        results = searcher.search(query)
        results

# Running search queries
from whoosh.qparser import QueryParser
import whoosh.index as index
if index.exists_in("search_test", indexname="testindex"):
    ix = index.open_dir("search_test", indexname="testindex"))
    query = QueryParser("domain", ix.schema).parse(u'www.mydomain.com')
    with ix.searcher() as searcher:
        results = searcher.search(query)
        results

# Running search queries
from whoosh.qparser import QueryParser, MultifieldParser
import whoosh.index as index
from whoosh.query import *
if index.exists_in("search_test", indexname="testindex"):
    ix = index.open_dir("search_test")
    query = MultifieldParser(["domain","title","content"], schema=ix.schema).parse(u"Document OR Article")
    with ix.searcher() as searcher:
        results = searcher.search(query)
        results

Cool Python Modules

String Similarity

google-diff-match-patch - The Diff Match and Patch libraries offer robust algorithms to perform the operations required for synchronizing plain text.

http://code.google.com/p/google-diff-match-patch/wiki/API

pylevenshtein - Levenshtein (edit) distance, and edit operations, string similarity...

http://code.google.com/p/pylevenshtein/

difflib - classes and functions for comparing sequences

http://docs.python.org/library/difflib.html

Database Tools

uuid - Python built-in module for creating unique ids
http://docs.python.org/library/uuid.html

django-command-extensions: UUIDField -  uuid field distributed with django-comamnd-extensions.
http://code.google.com/p/django-command-extensions/

django-uuidfield - stand-alone Django UUIDField implementation.
https://github.com/dcramer/django-uuidfield

System Tools

Imp - get python internals
http://docs.python.org/library/imp.html

Django Tools

AutoSlugify - automate slug field
http://packages.python.org/django-autoslug/

Pretty Print XML with Python

Pretty Print XML

import xml.dom.minidom

# Read XML from File

xml = xml.dom.minidom.parse(xml_filename) 

# Read XML from String

xml = xml.dom.minidom.parseString(xml_string)

xml.toprettyxml()

http://stackoverflow.com/questions/749796/pretty-printing-xml-in-python

Python Date Operations

Get current time:
>>> import datetime
>>> datetime.date.today()
datetime.date(2011, 1, 12)
>>> datetime.datetime.now()
datetime.datetime(2011, 1, 12, 20, 36, 21, 430985)

Adding time and timedelta:
>>> datetime.timedelta(hours=7) + datetime.datetime(2011,1,1)
datetime.datetime(2011, 1, 1, 7, 0)
>>> datetime.timedelta(days=50) + datetime.date(2011,1,1)
datetime.date(2011, 2, 20)

Subtracting dates:
>>> datetime.date(2011,1,2) - datetime.date(2011,1,1)
datetime.timedelta(1)

Convert date and time into datetime:
>>> import datetime
>>> d = datetime.date(2010,01,05)
>>> t = datetime.time(10,25)

>>>datetime.datetime.combine(d,t)

datetime.datetime(2010, 1, 5, 10, 25)

Convert String to datetime:

>>> datetime.datetime.strptime("2011-01-05","%Y-%m-%d")

datetime.datetime(2011, 1, 5, 0, 0)

Convert datetime to individual components:

 >>> dt = datetime.datetime(2011,01,05,12,00)

>>> (dt.year, dt.month, dt.day, dt.hour, dt.minute)

(2011, 1, 5, 12, 0)

Convert datetime to date:

 >>> dt = datetime.datetime(2011,01,05,12,00)

>>> dt.date()

datetime.date(2011, 1, 5)

Add one day to a date:
>>> dt = datetime.datetime(2011,01,05,12,00)
>>> delta = datetime.timedelta(days=1)

>>> dt + delta

datetime.datetime(2011, 1, 6, 12, 0)

Number of days in a month:
>>> import calendar
>>> calendar.monthrange(2011,1)
(5, 31)

Timezone
from pytz import timezone as tz
localtz = tz('US/Central')
#dt_aware = localtz.localize(dt_unaware)
dt_unaware.replace(tzinfo=localtz)

more to come...

http://stackoverflow.com/questions/1937622/convert-date-to-datetime-in-python

http://www.linuxtopia.org/online_books/programming_books/python_programming/python_ch32s03.html

Pretty Print JSON with python

Pretty Print JSON
echo '{"foo": "lorem", "bar": "ipsum"}' | python -mjson.tool

Dump Django data as pretty JSON
./manage.py dumpdata mymodule | python -mjson.tool



Prety Print JSON from Python Shell
import json
json_data = [1,2,3,{'a','b',c'}]
json.dumps(json_data, sort_keys=True, indent=4)


http://stackoverflow.com/questions/352098/how-to-pretty-print-json-script
http://docs.python.org/library/json.html

Django List Installed Tables and Models

Get Database connection
# Django 1.2
from django.db import connections

connection = connections['default']

# Django 1.1.1<=

from django.db import connection

# Get Database tables

tables = connection.introspection.table_names()

# Get Database models

modelclasses = connection.introspection.installed_models(tables)

# Get Django model name and app name
for model in list(modelclasses):
    print model._meta.app_label, model._meta.module_name

# A better way to get Django models
from django.db.models.loading import get_models, get_app
get_app('app_label')  # returns an app class given a label
get_models()  # all models 
get_models(application)  # models belonging to app class

# Get a model from a string
from django.db import models
models.get_model(*"myapp.mymodel".split('.')) # returns model class

# Get all Python modules

import sys
sys.modules

http://www.slideshare.net/ubernostrum/django-in-depth?src=related_normal&rel=3017822

Use alternate Mirror for Python pypi installs (when pypi goes down)

Specify alternate mirror from the command line:
pip install -i http://d.pypi.python.org/simple $PACKAGE

Specify alternate mirror in ~/.pip/pip.conf:
[global]
index-url = http://d.pypi.python.org/simple

More mirrors:
http://b.pypi.python.org/
http://c.pypi.python.org/
http://d.pypi.python.org/
http://e.pypi.python.org/
http://f.pypi.python.org/

Lists of mirrors:
http://pypi.python.org/mirrors/
http://www.pypi-mirrors.org/


Source:
http://jacobian.org/writing/when-pypi-goes-down/

Django Management Commands

Django syncdb without prompts
python manage.py syncdb --noinput

http://docs.djangoproject.com/en/dev/ref/django-admin/

Dynamic Fieldsets in Django Admin

Problem: the fieldsets option for the admin interface (classes defined in admin.py) requires that you explicitly list every single field in the model in the fieldset list that you pass to it.  This gets tedious if you add a field to a model, as you also have to add it to this admin.py.  Not very DRY.

Solution:
In models.py
from django.db import models
class My(models.Model):
    a = models.CharField(max_length=10)
    b = models.CharField(max_length=10)
    c = models.CharField(max_length=10)
    d = models.CharField(max_length=10)
    e = models.CharField(max_length=10)
    f = models.CharField(max_length=10)
    g = models.CharField(max_length=10)
    h = models.CharField(max_length=10)

In admin.py
from django.contrib import admin
from django.forms.models import fields_for_model
from f import models
class MyAdmin(admin.ModelAdmin):
    def __init__(self, *args, **kwargs):
        super(MyAdmin, self).__init__(*args, **kwargs)
        all_fields = set(fields_for_model(models.My))
        fieldset1_fields = ('e', 'f',)
        fieldset2_fields = ('g', 'h',)
        fieldset_fields = set(fieldset1_fields) | set(fieldset2_fields)
        rest_fields = list(all_fields - fieldset_fields)
        self.fieldsets = (
            (None, {
                'fields': rest_fields
            }),
            ('Fieldset 1', {
                'classes': ('collapse',),
                'fields': fieldset1_fields
            }),
            ('Fieldset 2', {
                'classes': ('collapse',),
                'fields': fieldset2_fields
            }),
        )
    models = models.My
admin.site.register(models.My, MyAdmin)


Thanks for the help lorochka85

Drill through a Login with Python

import urllib2
import urllib
import BeautifulSoup

# build opener with HTTPCookieProcessor
o = urllib2.build_opener( urllib2.HTTPCookieProcessor() )
urllib2.install_opener( o )

# assuming the site expects 'user' and 'pass' as query params
p = urllib.urlencode( { 'username': 'myusername', 'password': 'mypassword' } )

# perform login with params
f = o.open( 'https://www.mysite.com/login', p )
data = f.read()
f.close()

# second request should automatically pass back any
# cookies received during login... thanks to the HTTPCookieProcessor
f = o.open( 'http://www.mysite.com/home/' )
data = f.read()
f.close()

soup = BeautifulSoup.BeautifulSoup(data)

http://www.nomadjourney.com/2009/03/automatic-site-login-using-python-urllib2/

Python String format

Python Format dates into strings
from datetime import date
d1 = date(2010,01,23)
d2 = date(2010,01,23)
"%s - %s" % (d1.strftime("%m / %d / %Y"), d1.strftime("%m / %d / %Y"))

Python Unicode Convert

Convert weird unicode string characters to string:
import unicodedata
unicodedata.normalize('NFKD',unicode_string).encode('ascii','ignore')

Django module for helping convert things to ascii strings:
from django.utils.encoding import smart_str
smart_str('weird string goes here')

Another hack for removing unicode errors:
unicode('my weird string', errors='ignore')


http://docs.python.org/howto/unicode.html#the-unicode-type