Dulwich Command Notes

# Load Git repo into object

from dulwich.repo import Repo

repo = Repo("/home/jjasinsk/Sites/env/proj/gitdjango/")

# Find sha hash of Git repo head

repo.head()

# 'e80b3ada92d2a02ce353c83683a6533e25240a1a'

# Find the Commit object of the repo head

repo[repo.head()]

# Commit e80b3ada92d2a02ce353c83683a6533e25240a1a

# Find all the repo tags

repo.refs.subkeys('refs/tags')

# set(['v0.0.1'])

# Find all the repo branches

repo.refs.subkeys('refs/heads')

# set(['master', 'develop'])

# Find all the repo refs and their sha hashes

repo.get_refs()

# {'HEAD': 'e80b3ada92d2a02ce353c83683a6533e25240a1a', 

   'refs/heads/develop': 'b53fe83f58f293decae39a70106a5ae78886ba5e', 

   'refs/tags/v0.0.1': '5554750c44f8dc17ab9dc5926129cf409d06d0bb', 

   'refs/heads/master': 'e80b3ada92d2a02ce353c83683a6533e25240a1a'}

# Find all the commits tracing back to head Commit 

repo.revision_history(repo.head())

# [Commit fcc1e1eb645f6731c4a8701b5facf852738850a9, 

    Commit 39eaa48156e3df2fd2cbe9922e33787cbafdad90, 

    Commit e80b3ada92d2a02ce353c83683a6533e25240a1a,

   ...

TAG OBJECTS

# Get the Tag object by looking up it's ref path
repo['refs/tags/v0.0.1']

# Tag 5554750c44f8dc17ab9dc5926129cf409d06d0b;

# Find Tag by sha hash

repo.get_object('5554750c44f8dc17ab9dc5926129cf409d06d0bb')

# Tag 5554750c44f8dc17ab9dc5926129cf409d06d0bb

repo.tag("5554750c44f8dc17ab9dc5926129cf409d06d0bb")

# Tag 5554750c44f8dc17ab9dc5926129cf409d06d0bb

repo['5554750c44f8dc17ab9dc5926129cf409d06d0bb']

# Tag 5554750c44f8dc17ab9dc5926129cf409d06d0bb

# Find information about a tag

t = repo['5554750c44f8dc17ab9dc5926129cf409d06d0bb']

# Find tagger 
t.tagger

# 'Joe Jasinski joe.jasinski@example.com'

# Find tag date

from datetime import datetime

datetime.utcfromtimestamp(t.tag_time)

datetime.datetime(2012, 3, 30, 6, 58, 53)

# Print tag contents

print t.as_pretty_string()

# ...

# Find Tag sha hash (id)

t.id

# '5554750c44f8dc17ab9dc5926129cf409d06d0bb'

   OR
s = t.sha()

s.hexdigest()

# '5554750c44f8dc17ab9dc5926129cf409d06d0bb'

# Find Tag message

t.message

# 'test tag\n'

# Find Tag Type

t.get_type()

# 4

COMMIT OBJECTS

# Find Commit object from ref spec
repo['refs/heads/master']

# Commit e80b3ada92d2a02ce353c83683a6533e25240a1a

# Look up commit several different ways

repo.get_object('b53fe83f58f293decae39a70106a5ae78886ba5e')

# Commit b53fe83f58f293decae39a70106a5ae78886ba5e

repo.commit("b53fe83f58f293decae39a70106a5ae78886ba5e")

# Commit b53fe83f58f293decae39a70106a5ae78886ba5e

repo['b53fe83f58f293decae39a70106a5ae78886ba5e']

# Commit b53fe83f58f293decae39a70106a5ae78886ba5e

# Find Commit hash 

repo.ref('refs/heads/master')

# 'e80b3ada92d2a02ce353c83683a6533e25240a1a'

# Find commit author 

repo['refs/heads/master'].author

# 'Joe Jasinski joe.jasinski@example.com'

# Find committer 

repo['refs/heads/master'].committer

# 'Joe Jasinski joe.jasinski@example.com'

# Find Commit message

repo['refs/heads/master'].message

# 'updated style\n'

# Find commit time

from datetime import datetime

datetime.utcfromtimestamp(repo['refs/heads/master'].commit_time)

# Find Trees associated with commit

repo['b53fe83f58f293decae39a70106a5ae78886ba5e'].tree

'b3437c8329fc3f6306936c7ca1986cefc1629862'

repo[c.tree].items()

[TreeEntry(path='.gitignore', mode=33188, sha='13609a506289eba22b61df5eab0f89b7aede323d'), 

TreeEntry(path='example', mode=16384, sha='f8e95712570c4316868ee9af32088324e8d3e498'), 

TreeEntry(path='gitdjango', mode=16384, sha='32b164fac85e5b40dd8d4265a0a8df765bf57300'), 

TreeEntry(path='manage.py', mode=33261, sha='2605e3768e2bab92ef0550d452036472e9a6a210'), 

...

 repo[repo[c.tree].items()[0].sha]

TREE OBJECTS

# Find Tree associated with hash 
repo.get_object('b3437c8329fc3f6306936c7ca1986cefc1629862')

# Tree b3437c8329fc3f6306936c7ca1986cefc1629862

# Find all items under a Tree

repo[b3437c8329fc3f6306936c7ca1986cefc1629862].items()

BLOB OBJECTS

# Find Blob associated with hash 

repo.get_blob("2605e3768e2bab92ef0550d452036472e9a6a210")

# Blob 2605e3768e2bab92ef0550d452036472e9a6a210

repo['2605e3768e2bab92ef0550d452036472e9a6a210']

# Blob 2605e3768e2bab92ef0550d452036472e9a6a210

MISC

# Iterate through all objects under a gree
for entry in repo.object_store.iter_tree_contents(repo['refs/heads/master'].tree):

    print entry

# TreeEntry(path='.gitignore', mode=33188, sha='13609a506289eba22b61df5eab0f89b7aede323d')

# TreeEntry(path='example/__init__.py', mode=33188, sha='e69de29bb2d1d6434b8b29ae775ad8c2e48c5391')

# TreeEntry(path='example/__init__.pyc', mode=33188, sha='ced988bb5f3f22f4f1956bf852f106b23eea5c09')

# ...

# Find a the hash of an object by specifying a filesystem path

from dulwich.object_store import tree_lookup_path  

(mode, hash) = tree_lookup_path(repo.get_object, repo['refs/heads/master'].tree, 'gitdjango/models.py')

# Find an object filehandle relative to inside the .git directory

repo.get_named_file('../requirements.pip')

# open file '/home/jjasinsk/Sites/env/proj/gitdjango/.git/../requirements.pip', mode 'rb' at 0x101cdf938

f.read()

# 'Django==1.4\nGitPython==0.3.2.RC1\nasync==0.6.1\ngitdb==0.5.4\ninclude-server==3.1-toolwhip.1\nsmmap==0.8.2\nwsgiref==0.1.2\n'

Monitoring Connections

Find IPs that have the most connections
netstat -plan|grep :80|awk {'print $5'}|cut -d: -f 1|sort|uniq -c|sort -nk 1
http://www.webhostingtalk.com/showthread.php?t=673467

Find Listening Connections (open ports)
netstat -an | grep -i listening

Other Netstat Commands:

netstat -ntl
netstat -nputw
netstat -a -n -p
netstat -tulpn
netstat -tunapl

http://unix.stackexchange.com/questions/56453/how-can-i-monitor-all-outgoing-requests-connections-from-my-machine

http://www.webhostingtalk.com/showthread.php?t=991911

Basic Vagrant Usage

Doing a 'vagrant box add boxname.box' creates a .vagrant.d/ directory in your home directory. This directory stores base boxes installed. Doing a 'vagrant init' in some other directory creates a Vagrantfile inside of that directory which specifies which base box to use and provides any other provisioning information. 

List vagrant base boxes installed:
vagrant box list
# returns list of boxnames and type
mybox (virtualbox)

Add a new vagrant base box named "my_box" based on box image:
vagrant box add my_box http://files.vagrantup.com/lucid32.box

Remove a base box:
vagrant box remove my_box type
# for a virtualbox use virtualbox as the type

Location of vagrant base boxes:
cd ~/.vagrant.d/

Create a directory where customizations off the of the base box should take place.  then run the following: 
cd ~/Sites/VM/test/
vagrant init
vagrant init lucid32  # as an alternative to explicitly indicate which base box to use

Example Vagrant file in ~/Sites/VM/test/Vagrantfile

Vagrant::Config.run do |config|

     config.vm.box = "lucid32"  # specify which base box to use

end

Run a virtual machine.  Builds a virtual machine using box and Vagrantfile and runs it.  You must be inside the given vagrant directory (~/Sites/VM/test) in order for this to work.

vagrant up

Destroy a virtual machine. Destroy's the virtual machine:
vagrant destroy

SSH into a virtual machine:
vagrant ssh

Running a packaged virtual machine
vagrant box add my_box /path/to/the/package.box
vagrant init my_box
vagrant up

Suspend a virtual machine (save state to disk):
vagrant suspend

Resume a suspended virtual machine:
vagrant resume

Some Customizations to the default config (Pre-vagrant 1.1): 
Vagrant::Config.run do |config|
    # this is basically a way to programmatically invoke the
    # 'VBoxManage modifyvm' command from within Vagrant 

    # NOTE: this requires a fairly new version of vagrant 
    # to use this syntax. 
    config.vm.customize ["modifyvm", :id, "--memory", 1024]

    # By default vagrant boots boxes headless.  This enables the 
    # Virtualbox GUI
    config.vm.boot_mode = :gui

    # Path to ssh private key used to log into box with 
    # vagrant ssh. If it's a relative path, it will be relative 
    # to the dir holding the Vagrantfile

    config.ssh.private_key_path = 'vagrant_id_dsa'

    # Creates a shared folder on host called myshared in the 
    # same directory as the Vagrantfile.  On the guest, it 
    # mounts the shared folder as /my/shared.  The :create 
    # setting creates the directory on the host if it doesn't
    # exist.   

    config.vm.share_folder 'my-shared', '/my/shared/', 'myshared', {:create, true}

end

Some Customizations to the default config (Post-vagrant 1.1): 

Vagrant.configure("2") do |config|
   # setup port forwarding 
  config.vm.network :forwarded_port, guest: 8000, host: 18000, auto_correct: true
  config.vm.network :forwarded_port, guest: 5432, host: 15432, auto_correct: true

  config.vm.provider :virtualbox do |vb|
      # Don't boot with headless mode
      vb.gui = true

      # Use VBoxManage to customize the VM. For example to change memory:
      vb.customize ["modifyvm", :id, "--memory", "1024"]

     # Create a shared folder   (host path, guest path)
     config.vm.synced_folder "../data", "/vagrant_data"
  end


Turn on Debugging
export VAGRANT_LOG=debug


List of vagrant base boxes:
http://vagrantbox.es/

Get SSH configuration for machines:

vagrant ssh-config
Host swarm_manager
  HostName 127.0.0.1
  User vagrant
  Port 2222
  UserKnownHostsFile /dev/null
  StrictHostKeyChecking no
  PasswordAuthentication no
  IdentityFile /Users/jjasinski/Sites/test_docker_swarm/.vagrant/machines/swarm_manager/virtualbox/private_key
  IdentitiesOnly yes
  LogLevel FATAL

Host swarm_node1
  HostName 127.0.0.1
  User vagrant
  Port 2200
  UserKnownHostsFile /dev/null
  StrictHostKeyChecking no
  PasswordAuthentication no
  IdentityFile /Users/jjasinski/Sites/test_docker_swarm/.vagrant/machines/swarm_node1/virtualbox/private_key
  IdentitiesOnly yes
  LogLevel FATAL

Email Web Applications 

http://postmarkapp.com/
http://cloudmailin.com/
http://mailchimp.com
http://myemma.com/

Finding table sizes in postgres

Finding table sizes in postgres

Taken from: http://wiki.postgresql.org/wiki/Disk_Usage

SELECT nspname || '.' || relname AS "relation",
    pg_size_pretty(pg_total_relation_size(C.oid)) AS "total_size"
  FROM pg_class C
  LEFT JOIN pg_namespace N ON (N.oid = C.relnamespace)
  WHERE nspname NOT IN ('pg_catalog', 'information_schema')
    AND C.relkind <> 'i'
    AND nspname !~ '^pg_toast'
  ORDER BY pg_total_relation_size(C.oid) DESC
  LIMIT 20;

Copy File through Compressed Tar

Copy and Compress File through an ssh connection
tar jcvf - myfile.txt | ssh myhost.com "cat > myfile.tar.bz2"

Django Session Introspection

Get Data Stored in a Session
from django.contrib.sessions import models
s = models.Session.objects.get(session_key='05adfa345a54525cad912')
s.get_decoded()

ImageMatik


Convert all TIFs in a directory to JPEGs
mogrify -format jpg -quality 50 *.tiff

http://linuxuser32.wordpress.com/2007/06/16/batch-image-convert-scale-thumbnail-jpegs-pdf/

Python: lxml insert element before and after given element

Python: lxml insert element before and after given element

Based on: http://stackoverflow.com/questions/7474972/python-lxml-append-element-after-another-element

from lxml import etree

xml = etree.parse('old_xml.xml')

count = 1

for pp in xml.xpath('/response/items/item'):

   prod_id = etree.Element('new_element')  # this element will be inserted

   prod_id.text = "%s" % count

   nav = pp.find('name')  # we will insert before/after this element

   div = nav.getparent()

   pp.insert(pp.index(nav), prod_id) # insert before 'name'

   pp.insert(pp.index(nav) + 1, prod_id) # insert after 'name'

   count += 1

out = open('new_xml.xml','w')

out.write(etree.tostring(xml, pretty_print=True))

out.close()

Python: Detect Charset and convert to Unicode


Converting data to Unicode via chardet 
http://stackoverflow.com/questions/2686709/encoding-in-python-with-lxml-complex-solution

import chardet
from lxml import html

f = open('my.xml')

content = f.read()

f.close()

encoding = chardet.detect(content)['encoding']
if encoding != 'utf-8':
    content = content.decode(encoding, 'replace').encode('utf-8')

Converting Data to Unicode via UnicodeDammit 

http://lxml.de/elementsoup.html

from BeautifulSoup import UnicodeDammit

converted = UnicodeDammit(content)

   if not converted.unicode: 

    raise UnicodeDecodeError( 

       "Failed to detect encoding, tried [%s]", 

        ', '.join(converted.triedEncodings))  

        # print converted.originalEncoding

         return converted.unicode

Credit goes to Ian Bicking and others on the lxml team

Using lxml xpath to get elements with a default namespace

xml_string="""

<a xmlns="http://www.w3.org/1999/xhtml">

  <b>
    <r>
      <Value>string</Value>

    </r>

    <r>

      <Value>string</Value>

    </r>

  </b>

  <e>

    <string>string1</string>

    <string>string2</string>

  </e>

</a>

"""

from lxml import etree

xml = etree.fromstring(xml_string)

xml.xpath('/w3:a/w3:b/w3:r/w3:Value/text()', namespaces={'w3':'http://www.w3.org/1999/xhtml'})

SQLAlchemy Notes

Summarized From 
http://www.sqlalchemy.org/docs/core/tutorial.html


This covers the basics of the Non-orm usage of SQLAlchemy

# Define connection engine
from sqlalchemy import create_engine
engine = create_engine('sqlite:///:memory:', echo=True)

# Create some database tables 
from sqlalchemy import Table, Column, Integer, String, MetaData, ForeignKey
metadata = MetaData()
users = Table('users', metadata,
    Column('id', Integer, primary_key=True),
    Column('name', String),
    Column('fullname', String),
)

addresses = Table('addresses', metadata,
  Column('id', Integer, primary_key=True),
  Column('user_id', None, ForeignKey('users.id')),
  Column('email_address', String, nullable=False)
)

metadata.create_all(engine)

# Insert into the tables
ins = users.insert()
ins.values(name='jack', fullname='Jack the Pumpkin king')

conn = engine.connect()
result = conn.execute(ins)
result.inserted_primary_key

# Multiple insert into the tables
conn.execute(addresses.insert(), [
   {'user_id': 1, 'email_address' : 'jack@yahoo.com'},
   {'user_id': 1, 'email_address' : 'jack@msn.com'},
   {'user_id': 2, 'email_address' : 'www@www.org'},
   {'user_id': 2, 'email_address' : 'wendy@aol.com'},
])

result = engine.execute(users.insert(), name='fred', fullname="Fred Flintstone")

# Select from the tables
from sqlalchemy.sql import select

# Select all fields in user table
s = select([users])
result = conn.execute(s)
list(result)

# Select name and fullname columns from user table
s = select([users.c.name, users.c.fullname])

# Join two user and address tables
#  Cartesian product 
list(conn.execute(select([users, addresses])))
# Join on id columns
list(conn.execute(select([users, addresses], users.c.id==addresses.c.user_id)))

# create conditions for the where clause
from sqlalchemy.sql import and_, or_, not_
print and_(users.c.name.like('j%'), users.c.id==addresses.c.user_id, )


s = select([(users.c.fullname + "--" + addresses.c.email_address).label('title')])
print conn.execute(s).fetchall()

print users.join(addresses)
print users.join(addresses, addresses.c.email_address.like(users.c.name + '%'))


s = select([users.c.fullname], from_obj=[
   users.join(addresses, addresses.c.email_address.like(users.c.name + '%'))
   ])
print conn.execute(s).fetchall()

# Delete records based on a condition
conn.execute(users.delete().where(users.c.name > 'm'))

Django: Quickly testing an authenticated ajax

Make a authenticated post with the django test client and mark simulate an ajax post:
from django.test import Client

c = Client()

c.login(username='jjasinski', password='xxxx')

print c.post('/user/posts/1/',{},HTTP_X_REQUESTED_WITH='XMLHttpRequest')

Self-signed SSL Certificate

Generate self-signed certificate
# Generate Encrypted RSA Private Key with passphrase
openssl genrsa -des3 -out myssl.key 2048

# Generate Unencrypted RSA Private Key 
openssl genrsa -out myssl.key 2048

# Remove Passphrase from key

mv myssl.key myssl.key.org
openssl rsa -in myssl.key.org -out myssl.key

# Generate Certificate Signing Request with an existing Private Key

openssl req -sha256 -new -key myssl.key -out myssl.csr

# Create SSL certificate
openssl x509 -req -days 365 -in myssl.csr -signkey myssl.key -out myssl.crt

Generating a key and csr in one command

# for providing to your certificate provider

openssl req -sha256 -new -newkey rsa:2048 -nodes -keyout yourdomain.key -out yourdomain.csr

   req = create PKCS#10 X.509 Certificate Signing Request
   -sha256 = adds support for SHA-2

Decode PEM encoded SSL certificate meta information
openssl x509 -in certificate.crt -text -noout
OR
openssl x509 -noout -text -modulus -in https://example.com.crt

Decode DER encoded SSL certificate meta information
openssl x509 -in certificate.crt -inform der -text -noout

Decode SSL key meta information
openssl rsa -noout -text -modulus -in https://example.com.key

Convert PEM encoded SSL certificate to DER encoded:
openssl x509 -in certificate.crt -outform der -out certificate.der

Convert DER encoded SSL certificate to PEM encoded:
openssl x509 -in certfile.crt -inform der -outform pem -out certificate.pem


Note: in the meta info, the Modulus should be the same for the key and cert.


Sources
http://articles.slicehost.com/2007/12/19/ubuntu-gutsy-self-signed-ssl-certificates-and-nginx
http://support.godaddy.com/help/article/3601/generating-a-certificate-signing-request-nginx
http://www.sslshopper.com/certificate-decoder.html
https://www.digitalocean.com/community/tutorials/openssl-essentials-working-with-ssl-certificates-private-keys-and-csrs
https://support.ssl.com/Knowledgebase/Article/View/19/0/der-vs-crt-vs-cer-vs-pem-certificates-and-how-to-convert-them

Using Rsync to copy file Structure

Simple rsync copy on local filesystem:
rsync -r -t -v -l source/ dest

rsync over ssh:
rsync -r -t -v -l username@source.com:/path/to/src/ /path/to/dest

rsync copy shortcut to archive:
rsync -avz username@source.com:/path/to/src/ /path/to/dest
rsync -avz --progress username@source.com:/path/to/src/ /path/to/dest

rsync copy archive without caring about owner and group:
rsync -rlptD  username@source.com:/path/to/src/ /path/to/dest
rsync -a --no-o --no-g username@source.com:/path/to/src/ /path/to/dest

Common Flags:
 -a = archive copy.  Equivalent to flags -rlptgoD
 -r = recursive copy
 -t = save timestamp
 -v = verbose copy
 -vv = more verbose
 -u = don't copy over existing files newer than source
 -l = preserve symlinks
 -p = preserve premissions
 -g = preserve group
 -o = preserve owner
 -D = preserve devices (superuser only)
 -z = compress transfer
 --progress = show copy progress
--delete = delete stuff on the receiving side that doesn't exist on the sending side

http://www.cyberciti.biz/tips/linux-use-rsync-transfer-mirror-files-directories.html
http://en.wikipedia.org/wiki/Rsync
http://serverfault.com/questions/364709/how-to-keep-rsync-from-chowning-transfered-files

Git INFO Common Commands

Configure Git client options:
# show config options
git config --list # show all
git config --global --list # show only global
git config --local --list # show only local

# --global option impacts options at a global level
# --local option impacts options at a local level (.git/config)
# set username/email associated with client
git config --global user.email "joe@example.com"
git config --global user.name "Joe Jaz"

# add color syntax highlighting
git config --global color.status auto
git config --global color.branch auto

# set default pager and editor
git config --global core.editor vim # or set the GIT_EDITOR environment variable
git config --global core.pager less # or set the GIT_PAGER environment variable

# display an option value
git config group.value
git config user.name

Start a new repository and import:
# In the root folder to add to version control:
git init
git init --bare  # creates a new repository (with no working copy)
git add . # add everything in directory to repo
git commit -m 'initial commit'

Making commits:
# commit specific files
git commit file1 file2 -m 'comment'

# commit all 'added' files
git commit -a -m 'comment'

# change a commit message
git commit --amend -m 'New Message'

Create new repository by cloning existing one:
git clone --bare . path_or_url_to_git_dir.git

Setting the remote repository (to push and pull from):
git remote add short_name path_or_url_to_git_dir.git
git remote add origin git://github.com/imagescape/iscape-authlog.git

Show remote repositories available:
git remote
git remote -v # show the url
git remote show origin  # show detail information about origin

Remove a remote repository: 
git remove repo_name

Rename a remote repository: 
git rename old_repo_name new_repo_name

Push and Pull from remote repository:
git push origin # push recent commits to remote repository called "origin"
git pull origin # pull updates from origin

Revert changes in working copy:
git clean -n
git clean --dry-run

Show file differences:
git diff   # show differences between working copy and any files staged for commit
git diff --staged   # compares staged changes with last commit
git diff --cached  # same as above for git older than 1.6.1

git difftool -t [meld|kdiff3|vimdiff|etc.] [diff criteria]
git difftool -t meld --staged
git difftool --tool=meld --staged
git difftool origin/master  --dir-diff # use difftool to diff entire dir tree

git diff -U999999999 master.. -- file.txt # shows the full file.txt, with diff markup, in the master/branch diff results

git diff --name-status brancha..branchb # shows list of changed files

Show File/s History:
git log filename # Show single file versions

# Show number of lines changed in file/s
git log -stat filename # single file
git log -stat # multiple files

# show log summaries on one line
git log --pretty=oneline # Pre-defined format
git log --pretty=format:"%h - %cD (%an) %s" # custom format (see below)
git log --pretty=format:"%h %s" --graph
git log --graph --full-history --all --color \

           --pretty=format:"%x1b[31m%h%x09%x1b[32m%d%x1b[0m%x20%s"
    Pre-Defined Pretty formats:
    oneline
    short
    medium
    full
    fuller
    email
    raw
    format:string # where string is

        Format Strings
        %H: commit hash
        %h: abbreviated commit hash
        %T: tree hash
        %t: abbreviated tree hash
        %P: parent hashes
        %p: abbreviated parent hashes
        %an: author name
        %aN: author name (respecting .mailmap, see git-shortlog(1) or git-blame(1))
        %ae: author email
        %aE: author email (respecting .mailmap, see git-shortlog(1) or git-blame(1))
        %ad: author date (format respects --date= option)
        %aD: author date, RFC2822 style
        %ar: author date, relative
        %at: author date, UNIX timestamp
        %ai: author date, ISO 8601 format
        %cn: committer name
        %cN: committer name (respecting .mailmap, see git-shortlog(1) or git-blame(1))
        %ce: committer email
        %cE: committer email (respecting .mailmap, see git-shortlog(1) or git-blame(1))
        %cd: committer date
        %cD: committer date, RFC2822 style
        %cr: committer date, relative
        %ct: committer date, UNIX timestamp
        %ci: committer date, ISO 8601 format
        %d: ref names, like the --decorate option of git-log(1)
        %e: encoding
        %s: subject
        %f: sanitized subject line, suitable for a filename
        %b: body
        %B: raw body (unwrapped subject and body)
        %N: commit notes
        %gD: reflog selector, e.g., refs/stash@{1}
        %gd: shortened reflog selector, e.g., stash@{1}
        %gs: reflog subject
        %Cred: switch color to red
        %Cgreen: switch color to green
        %Cblue: switch color to blue
        %Creset: reset color
        %C(…): color specification, as described in color.branch.* config option
        %m: left, right or boundary mark
        %n: newline
        %%: a raw %
        %x00: print a byte from a hex code
        %w([[,[,]]]): switch line wrapping, like the -w option of git-shortlog(1).

Show git branches available:
git branch

Show all branches including remote branches:
git branch -a

Create new branch:
git branch new_branch

Change branches:
git checkout new_branch

Delete branch:
git branch -d new_branch # delete only if all changes are merged
git branch -D new_branch  # delete even if there are unmerged changes

Merge branch into current copy:
git merge --no-ff branch_to_merge_in

Resolve a Merge:
1) Edit file to resolve
2) git add conflict_file
3) git commit

Using Mergetool
git mergetool
git mergetool --tool=meld
git mergetool -t meld

Abort a merge conflict before committing:
git merge --abort

Undo a Merge:  # restores state to pre-merge
git reset --heard HEAD

Undo a local commit: 
git reset --soft HEAD^

Find the commit where two branches diverge:
git merge-base branch1 branch2

    http://stackoverflow.com/questions/1549146/find-common-ancestor-of-two-branches

Find what branches were merged into a given branch:
git branch --merged master  # local branches only
git branch -a --merged master   # local and remote branches

Find what branches were NOT merged into a given branch:
git branch --no-merged master  # local branches only
git branch -a --no-merged master   # local and remote branches

Find what branches are merged into develop, but not master:
comm -12 <(sort <(git branch -a --no-merged origin/master)) <(sort <(git branch -a --merged origin/develop))

    https://stackoverflow.com/questions/8071079/git-list-branches-merged-into-a-branch-but-not-into-another

Copy all branches from origin to new_remote: 
git push new_remote refs/remotes/origin/*:refs/heads/*

Get number of commits per author
apt-get install git-extras

git summary -n

References:
http://jonas.nitro.dk/git/quick-reference.html
http://www.vogella.de/articles/Git/article.html#git_stagingindex
http://book.git-scm.com/3_basic_branching_and_merging.html
http://stackoverflow.com/questions/1838873/visualizing-branch-topology-in-git
http://stackoverflow.com/questions/226976/how-can-i-know-in-git-if-a-branch-has-been-already-merged-into-master
http://stackoverflow.com/questions/7818927/git-push-branch-from-one-remote-to-another

Python Process Info

Pipe the output of one process into the input of another via python:
import subprocess
p1 = subprocess.Popen(['echo','joe'], shell=False, stdout=subprocess.PIPE)
p2 = subprocess.Popen(['cat','-n'],
    stdin=p1.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
p2.stdout.read()

# Or 
out,err=p2.communicate()

Pipe standard input to a program
import subprocess
p1 = subprocess.Popen(['node', 'index.js'], shell=False,
    stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p1.communicate('standard input here')
p1.returncode

Convenience method for splitting apart shell commands for use in the above:
>>> import shlex
>>> shlex.split('ps -ef')
['ps', '-ef']

Common Open ID Urls

Taken From http://stackoverflow.com/questions/1116743/where-can-i-find-a-list-of-openid-provider-urls

Google https://www.google.com/accounts/o8/id
Yahoo https://me.yahoo.com
Flickr http://www.flickr.com/username
AOL http://openid.aol.com/username
Blogspot https://www.blogspot.com/
LiveJournal http://username.livejournal.com/
Wordpress https://username.wordpress.com/
VerisignLabs https://pip.verisignlabs.com/
MyOpenID https://www.myopenid.com/
MyVidoop https://myvidoop.com/
ClaimID https://claimid.com/username
Technorati https://technorati.com/people/technorati/username/

SVN List Codes

Codes Displayed when doing an "svn stat"
U = Working file was updated
G = Changes on the repository were automatically merged into the working copy
M = Working Copy is modified
C = This file conflicts with the version in the repo
? = This file is not under version control
! = This file is under version control but is missing or incomplete
A = This file will be added to version control after commit
A+ = This file will be moved after commit
D = This file will be deleted after commit
I = This item is being ignored (due to the svn:ignore porperty)
L = This file is locked
X =  Item is unversioned, but used by an externals definition
~ = Messed up item (file when it should be dir). resolve by moving and doing "svn up"

http://knaddison.com/technology/svn-status-code-cheat-sheet

Whoosh Python Search Notes

Baisc Whoosh Search Walkthrough 


# Import required 
import os
from whoosh.index import create_in
from whoosh.fields import *

# Create a "Schema" object to store text to search
# You can create custom search fields such as "title", "domain", "path"
# and "content". 
schema = Schema(title=TEXT(stored=True), domain=ID(stored=True),
                              path=ID(stored=True), content=TEXT)

# Check to see if a search index directory exists; if not, create one
if not os.path.exists("search_test"):
    os.mkdir("search_test")

# Create a search index in the search index directory
ix = create_in("search_test", schema, indexname="testindex")
writer = ix.writer()

# Index some documents
writer.add_document(title=u"Document 1", domain=u"www.mysearch.com",
                         path=u"/mywebsite", content=u'Article')
writer.add_document(title=u"Document 2", domain=u"www.mysearch.com",
                         path=u"/mywebsite/2/article/1", content=u'Document a')
writer.commit()

# Running search queries
from whoosh.qparser import QueryParser
import whoosh.index as index
# Check to see if the given search index exists
if index.exists_in("search_test", indexname="testindex"):
    # open the search index
    ix = index.open_dir("search_test", indexname="testindex")
    # generate a search query based on some string.
    # Here is creates a query looking for "Document" in the "content"
    # search index field. 
    query = QueryParser("content", ix.schema).parse(u'*Document*')
    with ix.searcher() as searcher:  
        # do the actual search
        results = searcher.search(query)
        results

# Running search queries
from whoosh.qparser import QueryParser
import whoosh.index as index
if index.exists_in("search_test", indexname="testindex"):
    ix = index.open_dir("search_test", indexname="testindex"))
    query = QueryParser("domain", ix.schema).parse(u'www.mydomain.com')
    with ix.searcher() as searcher:
        results = searcher.search(query)
        results

# Running search queries
from whoosh.qparser import QueryParser, MultifieldParser
import whoosh.index as index
from whoosh.query import *
if index.exists_in("search_test", indexname="testindex"):
    ix = index.open_dir("search_test")
    query = MultifieldParser(["domain","title","content"], schema=ix.schema).parse(u"Document OR Article")
    with ix.searcher() as searcher:
        results = searcher.search(query)
        results

Different way to alter a Postgres table

Alter Postgres Tables via metadata 
(Thanks Louis R.) 

select atttypmod from pg_attribute where attrelid = 'TABLE1'::regclass and attname = 'title';
# then
update pg_attribute set atttypmod = 200+4 where attrelid = 'TABLE1'::regclass and attname = 'title';

http://sniptools.com/databases/resize-a-column-in-a-postgresql-table-without-changing-data

Quick Setup Instructions for SVN Compile and Local Install

Steps to make a custom build of SVN and place it in non-system directory.

Make a place to store the new copy:
mkdir -p /home/myuser/customsvn/src/
cd /home/myuser/customsvn/src/

Download from Apache Subversion; place in local install src directory:
ls -la /home/myuser/customsvn/src/
subversion-1.6.xx.tar.bz2
subversion-deps-1.6.xx.tar.bz2

Untar subversion then subversion-deps:
tar -jxvf subversion-1.6.xx.tar.bz2
tar -jxvf subversion-deps-1.6.xx.tar.bz2

Configure Command (Configure without Apache stuff):
./configure --prefix=/home/myuser/customsvn/bin/svn/ --with-apxs=no --disable-mod-activation --with-serf=no --with-berkeley-db=no  | tee configure.jjj.log
make | tee make.jjj.log
make install | tee make_install.jjj.log

SVN Merging For Version 1.4-

Merging for svn 1.4- is a bit different than for newer versions.  
This is what I have found that works for merging a trunk to a branch:
svn merge \
  svn+ssh://svn.example.com/repo/myproject/branches/newbranch@HEAD \
  svn+ssh://svn.example.com/repo/myproject/trunk@HEAD \
  /sites/myproject/src/myproject-workingcopy


*NOTE: seems weird that the trunk is the argument after the branch, but my trunk changes weren't being merged in when I did it the other way.

It seems that with svn version pre 1.4, you must specify the version where your branch was last copied or merged from.  If your branch was created at version 234, then the following command would merge it. 
svn merge --dry-run \
  svn+ssh://svn.example.com/export/myproject/branches/newbranchh@234 \
  svn+ssh://svn.example.com/export/myproject/trunk/@HEAD .


http://www.svnforum.org/threads/38394-1.4-merge

Django Content Type Quick Reference

from django.contrib.contenttypes.models import ContentType

# Get the content type for a given model
ct = ContentType.objects.get(app_label="myapp", model="mymodel")
# or
ct = ContentType.objects.get_for_model(MyModel)

# Get the class associated with the content type
model_cls = ct.model_class()

# Filter the queryset as normal              
queryset = model_cls.objects.filter()

# Get a specific object instance given a content type
ct.get_object_for_this_type(username='Guido')

http://docs.djangoproject.com/en/dev/ref/contrib/contenttypes/

Cool Python Modules

String Similarity

google-diff-match-patch - The Diff Match and Patch libraries offer robust algorithms to perform the operations required for synchronizing plain text.

http://code.google.com/p/google-diff-match-patch/wiki/API

pylevenshtein - Levenshtein (edit) distance, and edit operations, string similarity...

http://code.google.com/p/pylevenshtein/

difflib - classes and functions for comparing sequences

http://docs.python.org/library/difflib.html

Database Tools

uuid - Python built-in module for creating unique ids
http://docs.python.org/library/uuid.html

django-command-extensions: UUIDField -  uuid field distributed with django-comamnd-extensions.
http://code.google.com/p/django-command-extensions/

django-uuidfield - stand-alone Django UUIDField implementation.
https://github.com/dcramer/django-uuidfield

System Tools

Imp - get python internals
http://docs.python.org/library/imp.html

Django Tools

AutoSlugify - automate slug field
http://packages.python.org/django-autoslug/

Postgresql Sequences

Create a new sequence:
CREATE SEQUENCE mytable_myid_seq;

Add the sequence as the incrementor for a table column:
ALTER TABLE mytable
ALTER COLUMN myid
SET DEFAULT NEXTVAL('mytable_myid_seq');

Update existing rows in table with the sequence:
UPDATE mytable
SET myid = NEXTVAL('mytable_myid_seq');

Display the sequence information:
Select * from mytable_myid_seq;

Update a sequence to be one greater than a table's id column:
select setval('mytable_myid_seq', (select max(id) + 1 from mytable));
http://railspikes.com/2009/3/6/duplicate-key-violates-unique-constraint (thanks Louis)
http://pointbeing.net/weblog/2008/03/mysql-versus-postgresql-adding-an-auto-increment-column-to-a-table.html

Django News/Code At www.djangocurrent.com

I've created a more formal blog called www.djangocurrent.com, focused around Django.  I'll continue to post code snippets and information on this site, but will additionally post more formal write-ups at djangocurrent.com.  Check it out!

Pretty Print XML with Python

Pretty Print XML

import xml.dom.minidom

# Read XML from File

xml = xml.dom.minidom.parse(xml_filename) 

# Read XML from String

xml = xml.dom.minidom.parseString(xml_string)

xml.toprettyxml()

http://stackoverflow.com/questions/749796/pretty-printing-xml-in-python

Django Chain QuerySet

Based off of:
http://zmsmith.com/2010/04/using-custom-django-querysets/

Find Open Ports

Find open ports on local Linux machine:
sudo netstat -lnptu

http://www.techrepublic.com/blog/security/list-open-ports-and-listening-services/443

Django Testing Views

Setting up the example:

--- views.py -------------------------------------
# define a view for testing
def index(request, template_name=''):
    context = {'things':MyThing.objects.all(),}
    return render_to_response(
           template_name,
           context, context_instance=RequestContext(request))

--- urls.py ---------------------------------------
# define a url for testing
urlpatterns = patterns('',
...
    url('^$', 'core.views.index',
        {'template_name': 'someindex.html'},
         name="index"),
)

Testing from a Dango shell:
import django
from django.test.client import Client
django.test.utils.setup_test_environment() # needed to see response context
client = Client()
response = client.get('/')
response.context[-1]['things']

# we now have access to the response context and can use this

# test code in a unittest tests.py file.  (not yet shown)

Testing in a test.py file:
class SimpleTest(TestCase):

    def test_view_list(self):
        resp = self.client.get(reverse('some_named_url'))
        self.assertEqual(resp.status_code, 200)
        self.assertTrue('somevar' in resp.context[-1])

http://ericholscher.com/blog/2008/nov/11/practical-examples-test-django/

Favorite Plugins

Firefox Plugins for Web Development
- ColorZilla # color picker
- Firebug # CSS / HTML / JS debugger
- Firecookie # cookie info for firebug
- JSONView # Pretty JSON viewer
- RESTClient # GET and POST to restful interfaces
- Selenium IDE # Create tests you can replay in the browser
- Tamper Data # Allow one to tamper with headers and POST vars the client sends
- View Source Chart # Pretty HTML viewer
- Web Developer # Web developer tools
- GMarks # google bookmars sync
- LIVE HTTP headers # view http headers
- SQLite Manager # manage SQLite db files
- YSlow # Yahoo performance tool
- SearchStatus # find website rankings

Thunderbird Plugins for Google Sync
- Lightening # Calendar plugin
- google_contacts # sync google contacts
- CalDev sync google calendar # http://www.google.com/support/calendar/bin/answer.py?hl=en&answer=99358

Eclipse Plugins
- Pydev # python web development
- Subclipse # subversion for eclipse
- Egit # git plugin for eclipse
- m2eclipse # maven plugin for Java package management
- IndentGuide  # nice guidelines for code indenting http://atlanto.github.io/IndentGuide/

Python Date Operations

Get current time:
>>> import datetime
>>> datetime.date.today()
datetime.date(2011, 1, 12)
>>> datetime.datetime.now()
datetime.datetime(2011, 1, 12, 20, 36, 21, 430985)

Adding time and timedelta:
>>> datetime.timedelta(hours=7) + datetime.datetime(2011,1,1)
datetime.datetime(2011, 1, 1, 7, 0)
>>> datetime.timedelta(days=50) + datetime.date(2011,1,1)
datetime.date(2011, 2, 20)

Subtracting dates:
>>> datetime.date(2011,1,2) - datetime.date(2011,1,1)
datetime.timedelta(1)

Convert date and time into datetime:
>>> import datetime
>>> d = datetime.date(2010,01,05)
>>> t = datetime.time(10,25)

>>>datetime.datetime.combine(d,t)

datetime.datetime(2010, 1, 5, 10, 25)

Convert String to datetime:

>>> datetime.datetime.strptime("2011-01-05","%Y-%m-%d")

datetime.datetime(2011, 1, 5, 0, 0)

Convert datetime to individual components:

 >>> dt = datetime.datetime(2011,01,05,12,00)

>>> (dt.year, dt.month, dt.day, dt.hour, dt.minute)

(2011, 1, 5, 12, 0)

Convert datetime to date:

 >>> dt = datetime.datetime(2011,01,05,12,00)

>>> dt.date()

datetime.date(2011, 1, 5)

Add one day to a date:
>>> dt = datetime.datetime(2011,01,05,12,00)
>>> delta = datetime.timedelta(days=1)

>>> dt + delta

datetime.datetime(2011, 1, 6, 12, 0)

Number of days in a month:
>>> import calendar
>>> calendar.monthrange(2011,1)
(5, 31)

Timezone
from pytz import timezone as tz
localtz = tz('US/Central')
#dt_aware = localtz.localize(dt_unaware)
dt_unaware.replace(tzinfo=localtz)

more to come...

http://stackoverflow.com/questions/1937622/convert-date-to-datetime-in-python

http://www.linuxtopia.org/online_books/programming_books/python_programming/python_ch32s03.html

Pretty Print JSON with python

Pretty Print JSON
echo '{"foo": "lorem", "bar": "ipsum"}' | python -mjson.tool

Dump Django data as pretty JSON
./manage.py dumpdata mymodule | python -mjson.tool



Prety Print JSON from Python Shell
import json
json_data = [1,2,3,{'a','b',c'}]
json.dumps(json_data, sort_keys=True, indent=4)


http://stackoverflow.com/questions/352098/how-to-pretty-print-json-script
http://docs.python.org/library/json.html

Django: Random Image/File Field name

By default, Django image/file fields place uploaded files in the MEDIA_ROOT directory, under a subdirectory given by the "upload_to" model field parameter.  However, the name of uploaded file is the same name as the original filename uploaded by the client.  The filename can be changed by creating a custom storage manager. 

---- myapp/storage.py
import os, itertools, random, string
from django.core.files.storage import FileSystemStorage

def rand_key(size):
    return "".join([random.choice(string.letters + string.digits)
                          for i in range(size)])

class RandomFileSystemStorage(FileSystemStorage):

    def get_valid_name(self, name):
        file_root, file_ext = os.path.splitext(name)
        return "%s%s" % (rand_key(32), file_ext)

    def get_available_name(self, name):
        dir_name, file_name = os.path.split(name)
        file_root, file_ext = os.path.splitext(file_name)
        count = itertools.count(1)
        while self.exists(name):
            name = os.path.join(dir_name, "%s_%s%s" % (
                             rand_key(32),  count.next(), file_ext))
        return name

---- myapp/models.py

from django.db import models

from myapp.storage import RandomFileSystemStorage

class Photo(models.Model):

    image = models.ImageField(upload_to='media',
                  storage=RandomFileSystemStorage())

Setup Django RabbitMQ and Celery

Rabbit Message Queue is a separate server that remotely executes tasks given to it. 
Celery is a Python client program that sends tasks to the RabbitMQ.  
Django-celery is a Django wrapper for Celery that makes it 
work with Django more nicely.  Tasks are executed in Django 
view code or wherever.  The tasks can be defined in the Django app, 
are sent to a Celery client daemon executed by ./manage.py and
Celery serializes the task and sends it to RabbitMQ for processing.
RabbitMQ notifies Celery when it is done with each task. 


1) Install deps
UBUNTU 10.04 NOTE
sudo aptitude install python-virtualenv  #(recommended but not required)
sudo aptitude install rabbitmq-server

2) Create rabbitmq user and vhost.  These settings must be set again if you change your server hostname.  Or you can set the NODENAME to rabbit@localhost in the rabbitmq configuration. 
sudo rabbitmqctl add_user my_rabbit_user mypasswd
sudo rabbitmqctl add_vhost my_vhost
sudo rabbitmqctl set_permissions -p my_vhost my_rabbit_user ".*" ".*" ".*"

3) Setup python environment
virtualenv --no-site-packages testrabbitmq  #(create a virtualenv, not required)
cd testrabbitmq
. ./bin/activate # (activate virtualenv, not required)
pip install django
pip install celery
pip install django-celery

4) Setup Django project
django-admin.py startproject testc  # create a test project
cd testc
python ./manage.py startapp cel  # create a test app

5) Create a test model
Edit cel/models.py:
from django.db import models
class MyModel(models.Model):
    field1 = models.CharField(max_length=12)
    field2 = models.CharField(max_length=12)

6) Create some test tasks:
Edit cel/tasks.py
from celery.decorators import task
@task
def add(x, y)
    return x + y

from cel import models
@task
def addmodel(x, y):
    record = models.MyModel.objects.create(field1=x, field2=y)
    record.save()
    return record

from cel import models

@task(ignore_result=True)  # Celery will ignore results sent back to it

def addmodel2(x, y):

    record = models.MyModel.objects.create(field1=x, field2=y)

    record.save()

    return record

7) configure django settings
Edit settings.py


# set django-celery autoloader 
import djcelery
djcelery.setup_loader()

...

# Set database settings

...

# set information to connect to rabbitmq (broker) 
BROKER_HOST = "127.0.0.1"
BROKER_PORT = 5672
BROKER_VHOST = "/my_vhost"
BROKER_USER = "my_rabbit_user"
BROKER_PASSWORD = "1234"

# add to installed apps
INSTALLED_APPS = (
    ...
    'djcelery',  # django-celery
    'cel',
)

8) Syncdb
python ./manage syncdb

9) Restart the rabbitmq server (optional)
UBUNTU 10.04 NOTE - Ubuntu starts the rabbitmq 
server by default and installs an init script 
(/etc/init.d/rabbitmq-server start|stop).  For testing, 
let's stop the server and restart manually run it in the 
foreground to see more output. 
sudo /etc/init.d/rabbitmq-server stop
sudo rabbitmq-server start
# will display lots of output and say broker is running.  
# The terminal will wait.


10) Start the celery client process
In another terminal (if using virtualenv, be sure to activate it), 
in the testrabbitmq Django project, execute the following:

python ./manage.py celeryd -l info

# this will hang the terminal and set the Celery (the message client) waiting. 

11) Send a message
In another terminal (if using virtualenv, be sure to activate it), 
in the testrabbitmq Django project, execute the following:
python ./manage.py shell
>>> from cel import tasks
>>> result = tasks.add.delay(1, 2)
>>> result.ready()  # waits until task is done
True
>>> result.state
u'SUCCESS'
>>> result.successful()
True
>>> result = tasks.add.delay(1 + 2) # will cause an error
>>> result.successful()
False
>>> result = tasks.addmodel.delay('a','b')
>>> result.successful()
True
>>> for i in range(0,1000):  # stresstest
....            result = tasks.addmodel.delay('a','b')

# alternate syntax (more flexible - can pass args)
>>> result = tasks.addmodel.apply_asyc(args=['c','d'])
>>> result.successful()

# execute at a given time

>>> from datetime import datetime, timedelta

>>> result = tasks.addmodel.apply_asyc(args=['c','d'],
                             eta=datetime.now() + timedelta(minutes=1))

>>> result.successful()

# execute after a given number of seconds

>>> result = tasks.addmodel.apply_asyc(args=['c','d'],

                             countdown=3)

>>> result.successful()

# alternate syntax  (can queue functions you don't own)
>>> from celery.execute import send_task
>>> tasks.addmodel.name
'cel.tasks.addmodel'
>>> result = send_task("cel.tasks.addmodel", args=['e', 'f'])
>>> result.get()

>>> from djcelery.models import TaskMeta
>>> TaskMeta.objects.count()  # this table contains all results meta info 
                                                  # for tasks not defined with ignore_result=True

APPENDIX - rabbitmq commands

sudo rabbitmqctl stop  # stop the rabbitmq
sudo rabbitmqctl list_users # list the available users
sudo rabbitmqctl list_vhosts # list the available vhosts
sudo rabbitmqctl list_queues
Listing queues ...
celery 544   # by default, lists the queues in the server 
                   # and how many messages in them
sudo rabbitmqctl list_queues [options]
    common list_queues options 
         name = name of queue
         durable = queue survives server restarts
         pid = erlang process id  
         messages_ready = ready to be delivered to clients
         messages = queue depth
         memory = bytes of mem consumed by erlang process

# NOTES, for any changes to the code, be sure to restart the celery client (./manage.py celeryd)


UPDATE: I did a talk about this at Chicago Djangonaughts.  See here: 
https://groups.google.com/forum/?fromgroups#!topic/django-chicago/iJr9nStrM-U


http://www.turnkeylinux.org/blog/django-celery-rabbitmq
http://ask.github.com/celery/userguide/executing.html

Tell Browser not to cache forms on back

This works in IE and Firefox but not Chrome or Opera


<meta http-equiv="cache-control" content="no-cache"> <!-- tells browser not to cache -->
<meta http-equiv="cache-control" content="no-store">
<meta http-equiv="expires" content="0"> <!-- says that the cache expires 'now' -->
<meta http-equiv="pragma" content="no-cache"> <!-- says not to use cached stuff, if there is any -->

Django List Installed Tables and Models

Get Database connection
# Django 1.2
from django.db import connections

connection = connections['default']

# Django 1.1.1<=

from django.db import connection

# Get Database tables

tables = connection.introspection.table_names()

# Get Database models

modelclasses = connection.introspection.installed_models(tables)

# Get Django model name and app name
for model in list(modelclasses):
    print model._meta.app_label, model._meta.module_name

# A better way to get Django models
from django.db.models.loading import get_models, get_app
get_app('app_label')  # returns an app class given a label
get_models()  # all models 
get_models(application)  # models belonging to app class

# Get a model from a string
from django.db import models
models.get_model(*"myapp.mymodel".split('.')) # returns model class

# Get all Python modules

import sys
sys.modules

http://www.slideshare.net/ubernostrum/django-in-depth?src=related_normal&rel=3017822

Use alternate Mirror for Python pypi installs (when pypi goes down)

Specify alternate mirror from the command line:
pip install -i http://d.pypi.python.org/simple $PACKAGE

Specify alternate mirror in ~/.pip/pip.conf:
[global]
index-url = http://d.pypi.python.org/simple

More mirrors:
http://b.pypi.python.org/
http://c.pypi.python.org/
http://d.pypi.python.org/
http://e.pypi.python.org/
http://f.pypi.python.org/

Lists of mirrors:
http://pypi.python.org/mirrors/
http://www.pypi-mirrors.org/


Source:
http://jacobian.org/writing/when-pypi-goes-down/

Query Django Admin Log

Show Django Admin Log entries


select u.username, u.first_name, u.last_name, l.action_time, c.name, c.app_label,
       l.object_id as "Object Modified", l.object_repr, l.action_flag, l.change_message
from (auth_user u inner join django_admin_log l on u.id = l.user_id )
      inner join django_content_type c on c.id = l.content_type_id
where l.object_id = '9'  /* filter by object id if desired */
       and l.action_time > to_date('2010-10-04 01:00','YYYY-MM-DD HH:MI')  /* filter by date range */
order by l.action_time;

Change Default Linux Shell

Needs sudo privileges: to change the default shell:

chsh

http://forums.devshed.com/unix-help-35/how-to-change-default-shell-52749.html

Host a Test email server for Django header debugging

Set the following in settings.py
EMAIL_HOST="localhost"
EMAIL_PORT="1025"

Run the following to host the test webserver

python -m smtpd -n -c DebuggingServer localhost:1025

http://docs.djangoproject.com/en/dev/topics/email/ 

Django Management Commands

Django syncdb without prompts
python manage.py syncdb --noinput

http://docs.djangoproject.com/en/dev/ref/django-admin/

Bash customize autocomplete

http://www.linuxjournal.com/content/more-using-bash-complete-command

Dynamic Fieldsets in Django Admin

Problem: the fieldsets option for the admin interface (classes defined in admin.py) requires that you explicitly list every single field in the model in the fieldset list that you pass to it.  This gets tedious if you add a field to a model, as you also have to add it to this admin.py.  Not very DRY.

Solution:
In models.py
from django.db import models
class My(models.Model):
    a = models.CharField(max_length=10)
    b = models.CharField(max_length=10)
    c = models.CharField(max_length=10)
    d = models.CharField(max_length=10)
    e = models.CharField(max_length=10)
    f = models.CharField(max_length=10)
    g = models.CharField(max_length=10)
    h = models.CharField(max_length=10)

In admin.py
from django.contrib import admin
from django.forms.models import fields_for_model
from f import models
class MyAdmin(admin.ModelAdmin):
    def __init__(self, *args, **kwargs):
        super(MyAdmin, self).__init__(*args, **kwargs)
        all_fields = set(fields_for_model(models.My))
        fieldset1_fields = ('e', 'f',)
        fieldset2_fields = ('g', 'h',)
        fieldset_fields = set(fieldset1_fields) | set(fieldset2_fields)
        rest_fields = list(all_fields - fieldset_fields)
        self.fieldsets = (
            (None, {
                'fields': rest_fields
            }),
            ('Fieldset 1', {
                'classes': ('collapse',),
                'fields': fieldset1_fields
            }),
            ('Fieldset 2', {
                'classes': ('collapse',),
                'fields': fieldset2_fields
            }),
        )
    models = models.My
admin.site.register(models.My, MyAdmin)


Thanks for the help lorochka85

Drill through a Login with Python

import urllib2
import urllib
import BeautifulSoup

# build opener with HTTPCookieProcessor
o = urllib2.build_opener( urllib2.HTTPCookieProcessor() )
urllib2.install_opener( o )

# assuming the site expects 'user' and 'pass' as query params
p = urllib.urlencode( { 'username': 'myusername', 'password': 'mypassword' } )

# perform login with params
f = o.open( 'https://www.mysite.com/login', p )
data = f.read()
f.close()

# second request should automatically pass back any
# cookies received during login... thanks to the HTTPCookieProcessor
f = o.open( 'http://www.mysite.com/home/' )
data = f.read()
f.close()

soup = BeautifulSoup.BeautifulSoup(data)

http://www.nomadjourney.com/2009/03/automatic-site-login-using-python-urllib2/

Python String format

Python Format dates into strings
from datetime import date
d1 = date(2010,01,23)
d2 = date(2010,01,23)
"%s - %s" % (d1.strftime("%m / %d / %Y"), d1.strftime("%m / %d / %Y"))

Python Unicode Convert

Convert weird unicode string characters to string:
import unicodedata
unicodedata.normalize('NFKD',unicode_string).encode('ascii','ignore')

Django module for helping convert things to ascii strings:
from django.utils.encoding import smart_str
smart_str('weird string goes here')

Another hack for removing unicode errors:
unicode('my weird string', errors='ignore')


http://docs.python.org/howto/unicode.html#the-unicode-type