HTTP Clients and Web Scraping

Scraping etiquette: respect robots.txt (WWW::RobotRules), rate-limit, identify your bot, and check the terms of service. Use Firefox::Marionette or WWW::Mechanize::Chrome for JavaScript-heavy sites.

use v5.36;
use HTTP::Tiny;                                    # core: simple and dependency-free
my $http = HTTP::Tiny->new(timeout => 10, agent => 'MyBot/1.0');
my $res = $http->get('https://httpbin.org/get');
die "Failed: $res->{status} $res->{reason}\n" unless $res->{success};
say $res->{content};
my $post = $http->post_form('https://httpbin.org/post', { q => 'perl' });
my $json_res = $http->request('POST', 'https://httpbin.org/post', {
    headers => { 'Content-Type' => 'application/json' },
    content => '{"name":"Ana"}',
});

use Mojo::UserAgent;                               # full-featured, async-capable
my $ua = Mojo::UserAgent->new(max_redirects => 3);
my $tx = $ua->get('https://www.perl.org');
say $tx->result->dom->at('title')->text;
say for $tx->result->dom->find('a[href]')->map(attr => 'href')->each;   # all link URLs
my $data = $ua->get('https://api.github.com/repos/Perl/perl5' => { Accept => 'application/json' })->result->json;
say "Stars: $data->{stargazers_count}";

use LWP::UserAgent;                                # the classic library
my $lwp = LWP::UserAgent->new(timeout => 10);
my $r = $lwp->get('https://example.com');
say $r->decoded_content if $r->is_succes
…