#! /usr/bin/python # A very simple external downloader # Copyright 2011-2012 Zdenek Pavlas # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. # # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the # Free Software Foundation, Inc., # 59 Temple Place, Suite 330, # Boston, MA 02111-1307 USA import time, os, errno, sys from urlgrabber.grabber import \ _readlines, URLGrabberOptions, _loads, \ PyCurlFileObject, URLGrabError def write(fmt, *arg): try: os.write(1, fmt % arg) except OSError, e: if e.args[0] != errno.EPIPE: raise sys.exit(1) class ProxyProgress: def start(self, *d1, **d2): self.next_update = 0 def update(self, _amount_read): t = time.time() if t < self.next_update: return self.next_update = t + 0.31 write('%d %d\n', self._id, _amount_read) def main(): import signal signal.signal(signal.SIGINT, lambda n, f: sys.exit(1)) cnt = 0 while True: lines = _readlines(0) if not lines: break for line in lines: cnt += 1 opts = URLGrabberOptions() opts._id = cnt for k in line.split(' '): k, v = k.split('=', 1) setattr(opts, k, _loads(v)) if opts.progress_obj: opts.progress_obj = ProxyProgress() opts.progress_obj._id = cnt dlsz = dltm = 0 try: fo = PyCurlFileObject(opts.url, opts.filename, opts) fo._do_grab() fo.fo.close() size = fo._amount_read if fo._tm_last: dlsz = fo._tm_last[0] - fo._tm_first[0] dltm = fo._tm_last[1] - fo._tm_first[1] ug_err = 'OK' except URLGrabError, e: size = 0 ug_err = '%d %d %s' % (e.errno, getattr(e, 'code', 0), e.strerror) write('%d %d %d %.3f %s\n', opts._id, size, dlsz, dltm, ug_err) if __name__ == '__main__': main()