Writing an FTP Client - Multiple Downloads and Multipart Downloads

For simultaneous multipart download, you have three problems to solve:

1) How to download several parts at the same time
2) How to start each part at a different point in the file
3) How to stop each part at the right point in the file.

1) Is easy: use threads.
2) Is also easy: ftplib's retrbinary() method takes a `rest` argument telling where to start in the file
3) Isn't directly supported by ftplib, so you have to arrange to stop each part once enough bytes have been downloaded.

Here's the working code:

import ftplib
from threading import Thread

NUM_PARTS = 4    # Number of parts to read at the same time

class Done(Exception):
  pass

def open_ftp(server, directory):
  """Opens an FTP connection."""
  ftp = ftplib.FTP(server)
  ftp.login()
  ftp.set_pasv(False)
  ftp.cwd(directory)
  return ftp

def go(server, directory, filename):
  # Work out how big the file is
  ftp = open_ftp(server, directory)
  filesize = ftp.size(filename)
  print filename, "is", filesize, "bytes in size"
  ftp.quit()

  # Create some Downloaders to do the work, and set them off.
  chunk_size = filesize/NUM_PARTS
  last_chunk_size = filesize - chunk_size*(NUM_PARTS-1)   # To account for rounding errors
  downloaders = []
  for i in range(NUM_PARTS):
    if i == NUM_PARTS-1:
      this_chunk_size = last_chunk_size
    else:
      this_chunk_size = chunk_size
    downloaders.append(Downloader(server, directory, filename, chunk_size*i, this_chunk_size))

  # Wait for all the downloaders to finish.
  for downloader in downloaders:
    downloader.thread.join()

  # Join together the pieces and write them to a local file.
  whole_file = ''.join([d.data for d in downloaders])
  open(filename, 'wb').write(whole_file)
  print "Written", filename

class Downloader:
  """One of these downloads a part of a file in a separate thread."""

  thread_number = 0

  def __init__(self, server, directory, filename, part_start, part_size):
    self.filename = filename
    self.part_start = part_start
    self.part_size = part_size
    Downloader.thread_number += 1
    self.thread_number = Downloader.thread_number
    self.data = ''
    self.ftp = open_ftp(server, directory)
    self.thread = Thread(target=self.receive_thread)
    self.thread.start()

  def receive_thread(self):
    """In a separate thread, do the download."""
    try:
      self.ftp.retrbinary('RETR '+self.filename, self.on_data, 10000, self.part_start)
    except Done:
      pass

  def on_data(self, data):
    """This is called by ftplib with each piece of data as it arrives."""
    print "Thread", self.thread_number, "received", len(data), "bytes"
    self.data = self.data + data
    if len(self.data) >= self.part_size:
      # We have enough data, so break out.  Chop off any over-read data.
      self.data = self.data[:self.part_size]
      raise Done

go('ftp.microsoft.com', 'deskapps', 'readme.txt')