Board logo

标题: [原创教程] Mojo::UserAgent 批量获取 nes 游戏资源 [打印本页]

作者: 523066680    时间: 2019-1-15 14:40     标题: Mojo::UserAgent 批量获取 nes 游戏资源

本帖最后由 523066680 于 2019-1-15 14:46 编辑

获取midi(背景音乐)、nes、图片、作弊码、以及PDF(通常是手册)
运行环境:Strawberry Perl v5.26
路径设置:our $wdir = "F:/temp/nesgames";
  1. =info
  2.     Mojo::UserAgent 批量获取 nes 游戏资源
  3.     523066680/vicyang
  4.     2018-12
  5. =cut
  6. use File::Slurp;
  7. use File::Path;
  8. use File::Basename;
  9. use Mojo::UserAgent;
  10. use Mojo::DOM;
  11. use Try::Tiny;
  12. STDOUT->autoflush(1);
  13. our $wdir = "F:/temp/nesgames";
  14. our $main = "http://www.nesfiles.com";
  15. our $games = "http://www.nesfiles.com/Games";
  16. mkpath $wdir unless -e $wdir;
  17. chdir $wdir;
  18. our $ua = Mojo::UserAgent->new();
  19. our @headers = ( "User-Agent" => "Firefox/63.0" );
  20. get_games_list($games);
  21. sub get_games_list
  22. {
  23.     our ($ua, $main, @heaee);
  24.     my ($link) = @_;
  25.     my $res = $ua->get( $link, \@headers )->res;
  26.     my $dom = $res->dom;
  27.     for my $e ( $dom->find(".nesfilesTable a")->each )
  28.     {
  29.         #printf "%s %s\n", $e->attr("href"), $e->text;
  30.         get_files( $main .$e->attr("href"), $e->text );
  31.     }
  32. }
  33. sub get_files
  34. {
  35.     our ($main, $ua, @headers);
  36.     my ($link, $name) = @_;
  37.     my $title = basename($link);
  38.     my ($res, $dom);
  39.     my $fname = "${title}.html";
  40.     if ( -e $fname ) {
  41.         my $html = read_file($fname);
  42.         $dom = Mojo::DOM->new( $html );
  43.     } else {
  44.         $res = $ua->get( $link, \@headers )->res;
  45.         $dom = $res->dom;
  46.         write_file( $fname, {binmode=>":raw"}, $res->body );
  47.     }
  48.     # 获取资源明细,略过 Ebay 相关的条目
  49.     mkdir $title unless -e $title;
  50.     my ($head, $list, $res2);
  51.     for my $section ($dom->find(".GameSection")->each)
  52.     {
  53.         $head = $section->at("header")->text;
  54.         last if $head=~/Ebay$/i;
  55.         # 秘籍/代码
  56.         if ($head=~/Codes/i) {
  57.             write_file( $title ."/Codes_Cheats.txt", $section->all_text );
  58.             next;
  59.         }
  60.         if ($head=~/Screenshots/i) {
  61.             # 如果是屏幕截图
  62.             $list = $section->find("img")->map(attr=>"src");
  63.         } else {
  64.             # 其他情况获取 href
  65.             $list = $section->find("a")->map(attr=>"href");
  66.         }
  67.         printf "%s\n", $head;
  68.         for my $href ( $list->each )
  69.         {
  70.             printf "%s\n", $href;
  71.             $fname = $title ."/". basename($href);
  72.             
  73.             next if -e $fname;         # 跳过已经存在的文件
  74.             $res2 = try_to_get( "${main}$href" );
  75.             next unless defined $res2; # 如果获取失败
  76.             write_file( $fname, {binmode=>":raw"}, $res2->body);
  77.         }
  78.     }
  79. }
  80. sub try_to_get
  81. {
  82.     our ($ua, @headers);
  83.     my ($link) = @_;
  84.     my $res;
  85.     my $times = 0;
  86.     while (1)
  87.     {
  88.         try { $res = $ua->get( $link )->result; }
  89.         catch { printf "Error %s, retry: %d\n", $_, $times; };
  90.         $times++;
  91.         last if (defined $res and $res->is_success);
  92.         return undef if ( $times > 5 );
  93.     }
  94.     return $res;
  95. }
复制代码





欢迎光临 批处理之家 (http://www.bathome.net/) Powered by Discuz! 7.2