2012-02-18

While hacking away on MuttonChop it became rather obvious that I would need to be able to download files from the internet in order to listen to serialized audio and video recording broadcasts.

Hmmmm, audio + broadcast, I'll call it an audcast. Yea yea, I know, most people call it a podcast, but I have no idea where "pod" comes from so I'll call it as I see it.

Actually, I don't really need to download the files because MuttonChop uses Gstreamer for media playback and Gstreamer is fully capable of streaming media over HTTP. However, since I don't hate the organizations that are serving the files, I'll try to limit the amount of bandwidth that MuttonChop consumes.

Alright, enough yammering, let's get to the code
Enter the Vala

/* 
 * copyright 2012 Jezra Lickter
 * Licensed GPLv3
 
 */
using GLib;

public class ParsedURL {
  public string scheme;
  public string host;
  public int port;
  public string request;
  public ParsedURL(string url) {
    request "";
    //parse that shit!
    string[] bits url.split("://");
    scheme bits[0];
    if (scheme.down() == "http" )
      port 80;
    if (scheme.down() == "ftp" )
      port 21;
    bits bits[1].split("/");
    string hp bits[0];
    string[] hpbits hp.split(":");
    host hpbits[0];
    if (hpbits.length ){
      port int.parse(hpbits[1]);
    }
    if(bits.length 1) {
      for (int i=i<bits.lengthi++) {
        request+="/"+bits[i];
      }
    else {
      request "/";
    }
    stdout.printf(@"$scheme $host $port $request\n");
  }
}

public class Header {
  Regex key_value_regex;
  public string status;
  public string Date;
  public string Server;
  public string Location;
  public uint64 Content_Length;
  public string Content_Type;
  public string code;
  public Header() {
    try {
      key_value_regex new Regex("(?P<key>.*):\\s+(?P<value>.*)");
    catch (RegexError err) {
      stderr.printf(err.message+"\n");
    }
  }
  public void add_valuestring line ){
    if(status==null) {
      status line;
      var line_bits status.split(" ");
      code line_bits[1];
    else {
      MatchInfo match_info;
      string key,val
      ifkey_value_regex.match(line,GLib.RegexMatchFlags.ANCHOREDout match_info) ) {
        key match_info.fetch_named("key");
        val match_info.fetch_named("value");
        switch (key) {
          case "Date" :
            Date val;
            break;
          case "Server":
            Server val;
            break;
          case "Location":
            Location val;
            break;
          case "Content-Length":
            Content_Length uint64.parse(val);
            break;
          case "Content-Type":
            Content_Type val;
            break;
          default:
            break;
        }
      }
    }
  }
}

public class Downloader Object {
  private uint64 length;
  private uint64 bytes_read;
  private int dl_length;
  private int throttler;
  private bool download_complete;
  private bool downloader_finished;
  private string url;
  private bool kill_download;
  private string local_file_path;
  private bool borked;
  public Downloaderstring urlstringlocal_file=nullintthrottler=0) { 
    borked false;
    length 0;
    dl_length 0;
    this.throttler throttler;
    download_complete false;
    downloader_finished false;
    this.url url;
    kill_downloadfalse;
    if (local_file != null) {
      local_file_path local_file;
    else {
      local_file_path Path.get_basename(url);
    
    downloadurl );
  }
  
  public bool is_borked() {
    return borked;
  }
  
  void download(string remoteintredirects 10) {
    ParsedURL pu new ParsedURL(remote);
    /* http://live.gnome.org/Vala/GIONetworkingSample */
    try {
      // Resolve hostname to IP address
      var resolver Resolver.get_default ();
      var addresses resolver.lookup_by_name (pu.hostnull);
      var address addresses.nth_data (0);

      // Connect
      var client new SocketClient ();
      var conn client.connect (new InetSocketAddress (address, (uint16)pu.port));

      // Send HTTP GET request
      var message "GET %s HTTP/1.1\r\nHost: %s\r\n\r\n".printf(pu.requestpu.host);
      conn.output_stream.write (message.data);

      // Receive response
      var response new DataInputStream (conn.input_stream);
      size_t length=0;
      string line;
      Header header new Header();
      do {
        line response.read_line(out length);
        header.add_value(line);
      }while(length );  
      //check and handle redirects 
      if(header.code == "301" || header.code=="302") {
        //how many redirects is this?
        if redirects<=) {
          stderr.printf("Error: Too many Redirects\n");
        else {
          //download from the redirect location
          downloadheader.Locationredirects--);
          return;
        }
      }
      //where is this thing going?
      string partial_file local_file_path+".partial";
      var file File.new_for_path (partial_file);
      // delete file if it already exists
      if (file.query_exists ()) {
          file.delete ();
      }
      var dos new DataOutputStream (file.create (FileCreateFlags.REPLACE_DESTINATION));
      //that's it for the headers
      bytes_read 0;
      uchar[] bytes;
      while ( !conn.is_closed () && bytes_read header.Content_Length ) {
        //clear the bytes array
        bytes = {};
        bytes += response.read_byte();
        bytes_read++;
        //write the byte
        dos.write (bytes);
        //TODO: sleep the thread if throttling
      }
      //close the output stream
      dos.close();
      //move the partial file to the finished file
      FileUtils.rename(partial_filelocal_file_path);
      
    catch (Error e) {
        stderr.printf ("%s\n"e.message);
        return ;
    }
  }
}
public static void mainstring[] args) {
  string url "";
  string local_file "";
  bool die false;
  Downloader d;
  switchargs.length ) {
    case :
      local_file args[2];
      url args[1];
      new Downloader(urllocal_file);
      break;
    case :
      url args[1];
      new Downloader(url);
      break;
    default:
      stdout.printf("--USAGE--\n");
      stdout.printf("%s REMOTE_URL [LOCAL_FILENAME]\n"args[0]);
      die true;
      break;
  
  if die ) {
    return;
  }
}

The code is also available in the hoof at http://hoof.jezra.net/snip/nU. Save the code as "downloader.vala" and compile with

valac --pkg gio-2.0 downloader.vala

Because this code is destined to be part of a larger project, there will be parts of the code that currently aren't being used, but will be necessary in the future... and I'm too lazy to remove the unused bits for this posting.

Similarly, the code hasn't been tested with FTP. Oh well, it does what I need it to do, and that is the important thing.

Comments
2013-03-22 Nasser:
i am new in linux and php, i am .net programmer and i am learning vala and php, thanks for this code, hove i can use this code in a sample application? or is its download jobs, restartable?
2013-03-22 jezra:
Hello Nasser,
This is experimental code that was the basis for the downloader in MuttonChop.

I'm going to suggest downloading the MuttonChop code https://code.launchpad.net/muttonchop and looking at the downloader application in the 'test_apps' directory.
2018-01-25 Manish Jain:
Hello Mr. Jezra,

I am writing a book on modern Unix (FreeBSD / Linux), which has a dedicated section for Vala programming. I would like to include your File downloader code as a code snippet (duly credited to you and linked to this URL).

Would that be okay with you ? I would be grateful if you could send me an email mentioning your approval.

Thanks
Manish Jain
Name:
not required
Email:
not required (will not be displayed)
Website:
not required (will link your name to your site)
Comment:
required
Please do not post HTML code or bbcode unless you want it to show up as code in your post. (or if you are a blog spammer, in which case, you probably aren't reading this anyway).
Prove you are human by solving a math problem! I'm sorry, but due to an increase of blog spam, I've had to implement a CAPTCHA.
Problem:
9 minus 7
Answer:
required
subscribe
 
2019
2016
2015
2014
2013
2012
2011
2010
December
November
October
September
August
July
June
May
April
March
February
January
2009
December
November
October
September
August
July
June
May
April
March
February
January
2008