subscribe
Tags:
 
2012
May
April
March
February
January
2011
December
November
October
September
August
July
June
May
April
March
February
January
2010
December
November
October
September
August
July
June
May
April
March
February
January
2009
December
November
October
September
August
July
June
May
April
March
February
January
2008
December
November
October
September
August
June
May
April
2011-03-26

I was recently playing around with node a command line javascript interpreter based on the V8 javascript engine and wanted/needed to implement a file downloader class that properly handles HTTP 302 Redirects.

After a bit of yelling and screaming, I had a working implementation
Enter the Javascript

var path = require('path');
var url = require('url');
var http = require('http');
var fs = require('fs');
var write_file;
//what global variable do we have?
var complete = false;
var content_length = 0;
var downloaded_bytes = 0;

//create the downloader 'class'
var Downloader = function() {

  //we will need to be able to set the remote file to download
  this.set_remote_file = function(file) {
    remote_file = file;
    local_file = path.basename( remote_file );
  }
  
  //we want to set the local file to write to
  this.set_local_file = function(file) {
    local_file = file;
  }
//run this fukker!
  this.run = function() {
    //start the download
    this.download( remote_file, local_file, 0 );
  }

  this.download = function(remote, local, num) {
    console.log( remote );
    if ( num > 10 ) {
      console.log( 'Too many redirects' );
    }
    //remember who we are
    var self = this;
    //set some default values  
    var redirect = false;
    var new_remote = null;
    var write_to_file = false;
    var write_file_ready = false;
    //parse the url of the remote file
    var u = url.parse(remote);
    //set the options for the 'get' from the remote file
    var opts = {
      host: u.hostname,
      port: u.port,
      path: u.pathname
    };
    //get the file
    var request = http.get(opts, function(response ) {
      switch(response.statusCode) {
        case 200:
          //this is good
          //what is the content length?
          content_length = response.headers['content-length'];
          break;
        case 302:
          new_remote = response.headers.location;
          self.download(new_remote, local_file, num+1 );
          return;
          break;
        case 404:
          console.log("File Not Found");
        default:
          //what the hell is default in this situation? 404?
          request.abort();
          return;
      }
      response.on('data', function(chunk) {
        //are we supposed to be writing to file?
        if(!write_file_ready) {
          //set up the write file
          write_file = fs.createWriteStream(local_file);
          write_file_ready = true;
        }
        write_file.write(chunk);
        downloaded_bytes+=chunk.length;
        percent = parseInt( (downloaded_bytes/content_length)*100 );
        console.log( percent );
      });
      response.on('end', function() {
        complete = true;
        write_file.end();
      });
    });
    request.on('error', function(e) {
      console.log("Got error: " + e.message);
    });
  }
}
exports.Downloader = Downloader;

To use this, one would need to create a script that utilizes the class. For example, suppose we wanted to download an episode of Rathole Radio we would need to write something along the lines of the following:

var Downloader = require('./Downloader');
var dl = new Downloader.Downloader();
dl.set_remote_file('http://traffic.libsyn.com/ratholeradio/RR048_20_03_11.ogg');
dl.run();

Awesome 'Possum! Now quit reading and go do something nifty.

file downloader in Node.js that handles redirects from jezra.net
Comments
2011-03-27 jrobb:
nifty indeed!
2012-01-08 BenNG:
Thanks for the article :)
I was wondering something : the variable local in " this.download = function(remote, local, num) {...}" is never used right ?
2012-01-08 jezra:
@BenNG, haha! it certainly appears that way. The lines that actually write to the local file should probably be changed to use 'local' instead of 'local_file', or 'local' should be removed from the method.

It really comes down to scope preference.
Name:
not required
Email:
not required (will not be displayed)
Website:
not required (will link your name to your site)
Comment:
required
Please do not post HTML code or bbcode unless you want it to show up as code in your post. (or if you are a blog spammer, in which case, you probably aren't reading this anyway).
Prove you are human by solving a math problem! I'm sorry, but due to an increase of blog spam, I've had to implement a CAPTCHA.
Problem:
3 plus 9
Answer:
required