Thursday, October 27, 2011

urllib2 POST progress

Innen.

It is possible but you need to do a few things:

  • Fake out the urllib2 subsystem into passing a file handle down to httplib by attaching a __len__attribute which makes len(data) return the correct size, used to populate the Content-Length header.
  • Override the read() method on your file handle: as httplib calls read() your callback will be invoked, letting you calculate the percentage and update your progress bar.

This could work with any file-like object, but I've wrapped file to show how it could work with a really large file streamed from disk:

import os, urllib2
from cStringIO import StringIO

class Progress(object):
def __init__(self):
self._seen = 0.0

def update(self, total, size, name):
self._seen += size
pct
= (self._seen / total) * 100.0
print '%s progress: %.2f' % (name, pct)

class file_with_callback(file):
def __init__(self, path, mode, callback, *args):
file
.__init__(self, path, mode)
self.seek(0, os.SEEK_END)
self._total = self.tell()
self.seek(0)
self._callback = callback
self._args = args

def __len__(self):
return self._total

def read(self, size):
data
= file.read(self, size)
self._callback(self._total, len(data), *self._args)
return data

path
= 'large_file.txt'
progress
= Progress()
stream
= file_with_callback(path, 'rb', progress.update, path)
req
= urllib2.Request(url, stream)
res
= urllib2.urlopen(req)

Output:

large_file.txt progress: 0.68
large_file
.txt progress: 1.36
large_file
.txt progress: 2.04
large_file
.txt progress: 2.72
large_file
.txt progress: 3.40
...
large_file
.txt progress: 99.20
large_file
.txt progress: 99.87
large_file
.txt progress: 100.00

No comments: