fetch-audio-mw

#! /usr/bin/perl

use warnings;
use strict;

use HTML::Parser;
use LWP::UserAgent;
use URI::URL;

my $URL = "http://www.merriam-webster.com/dictionary";

my $POPWIN = qr/
  popWin
  \s*    \(      \s*
  ["']   (.+?)   ["']
  \s*    \)
/ix;

my @words = qw(
  a      all   an     and   are     as      at     be      but   by
  can    do    each   for   from    had     have   he      his   how
  I      if    in     is    it      not     of     on      one   or
  said   she   that   the   their   there   they   this    to    use
  was    we    were   what  when    which   with   words   you   your
);

$| = 1;

my %audio;

my $ua = LWP::UserAgent->new;

use Data::Dumper;
$Data::Dumper::Indent = 1;

my $p = HTML::Parser->new(api_version => 3);

WORD:
foreach my $word (qw/ you your /) { # (@words) {
  my $audio;
  my @lookup;
  my $store;
  my @todo = ("$URL/$word");

  my $onclick = sub {
    my($tag,$attr,$p) = @_;

    return unless $tag eq "a" && ($attr->{class} || "") =~ /lookup|audio/;

    if ($attr->{class} && $attr->{class} eq "lookup") {
      push @lookup => $attr->{href};
    }
    elsif ($attr->{onclick} && $attr->{onclick} =~ /$POPWIN/) {
      if (index($1, lc "=$word") == length($1) - length($word) - 1) {
        $audio = $1;
        $p->eof;
      }
    }
  };

  my $embed = sub {
    my($tag,$attr,$p) = @_;

    return unless $tag eq "embed" && $attr->{src};

    $store = $attr->{src};
    $p->eof;
  };

  $p->eof;
  $p->handler(start => $onclick, "tagname,attr,self");

  while (@todo) {
    my $url = shift @todo;
print "$0: fetching $url...\n";

    my $res = $ua->request(
      HTTP::Request->new(GET => $url),
      sub { $p->parse($_[0]) }
    );

    unless ($res->is_success) {
      warn "$0: word '$word':\n" . $res->content;
      next;
    }

    my $base = $res->base;

    if (defined $store) {
      my $path = "words/$word";
      $path .= ".$1" if $store =~ /^.*\.(.+)$/;

      print "$0: storing $path...\n";

      my $abs = absurl($store, $base);
      my $res = $ua->request(HTTP::Request->new(GET => $abs), $path);

      warn "$0: word '$word':\n" . $res->content
        unless $res->is_success;
   
      next WORD;
    }
    elsif (defined $audio) {
      push @todo => absurl($audio, $base);
      $p->handler(start => $embed, "tagname,attr,self");
      next;
    }
    else {
      push @todo => map absurl($_,$base) => @lookup;
    }
  }
}

exit;

sub absurl {
  my($url,$base) = @_;
  url($url,$base)->abs;
}