Http::response("User-agent: *\nDisallow: /admin\n"),
'https://shop.example.com/outdoor' => Http::response('Outdoor gear'),
]);
$fetcher = new SameHostPageFetcher(new ProductPageScraper, 'https://shop.example.com', 40);
$page = $fetcher->fetch('https://shop.example.com/outdoor');
$this->assertNull($page['skipped']);
$this->assertSame('Outdoor gear', $page['text']);
$this->assertSame(38, $page['remaining']);
}
public function test_it_extracts_same_host_links_from_the_page(): void
{
Http::preventStrayRequests();
Http::fake([
'https://shop.example.com/robots.txt' => Http::response("User-agent: *\nDisallow: /admin\n"),
'https://shop.example.com/outdoor' => Http::response(
''
),
]);
$fetcher = new SameHostPageFetcher(new ProductPageScraper, 'https://shop.example.com');
$page = $fetcher->fetch('https://shop.example.com/outdoor');
$this->assertSame([
['name' => 'Women', 'url' => 'https://shop.example.com/women'],
], $page['links']);
}
public function test_it_still_fetches_the_submitted_url_when_robots_disallows_all_paths(): void
{
Http::preventStrayRequests();
Http::fake([
'https://shop.example.com/robots.txt' => Http::response("User-agent: *\nDisallow: /\n"),
'https://shop.example.com/us/store/index.jsp' => Http::response(''),
]);
$fetcher = new SameHostPageFetcher(
new ProductPageScraper,
'https://shop.example.com/us/store/index.jsp',
);
$page = $fetcher->fetch('https://shop.example.com/us/store/index.jsp');
$this->assertNull($page['skipped']);
$this->assertSame([
['name' => 'Women', 'url' => 'https://shop.example.com/women'],
], $page['links']);
}
public function test_a_catch_all_robots_disallow_does_not_block_category_pages(): void
{
Http::preventStrayRequests();
Http::fake([
'https://shop.example.com/robots.txt' => Http::response("User-agent: *\nDisallow: /\nDisallow: /admin\n"),
'https://shop.example.com/women' => Http::response('Dresses'),
]);
$fetcher = new SameHostPageFetcher(new ProductPageScraper, 'https://shop.example.com/us/store/index.jsp');
$page = $fetcher->fetch('https://shop.example.com/women');
$this->assertNull($page['skipped']);
$this->assertSame([
['name' => 'Dresses', 'url' => 'https://shop.example.com/women/dresses'],
], $page['links']);
}
public function test_it_skips_other_hosts(): void
{
Http::preventStrayRequests();
Http::fake([
'https://shop.example.com/robots.txt' => Http::response('User-agent: *\nDisallow:\n'),
]);
$fetcher = new SameHostPageFetcher(new ProductPageScraper, 'https://shop.example.com');
$page = $fetcher->fetch('https://evil.example.com/product');
$this->assertSame('host', $page['skipped']);
$this->assertSame('', $page['text']);
}
public function test_it_skips_robots_disallowed_paths(): void
{
Http::preventStrayRequests();
Http::fake([
'https://shop.example.com/robots.txt' => Http::response("User-agent: *\nDisallow: /admin\n"),
'https://shop.example.com/admin/secret' => Http::response('secret'),
]);
$fetcher = new SameHostPageFetcher(new ProductPageScraper, 'https://shop.example.com');
$page = $fetcher->fetch('https://shop.example.com/admin/secret');
$this->assertSame('robots', $page['skipped']);
Http::assertNotSent(fn ($request) => $request->url() === 'https://shop.example.com/admin/secret');
}
public function test_it_stops_when_the_fetch_budget_is_exhausted(): void
{
Http::preventStrayRequests();
Http::fake([
'https://shop.example.com/robots.txt' => Http::response("User-agent: *\nDisallow:\n"),
'https://shop.example.com/one' => Http::response('one'),
'https://shop.example.com/two' => Http::response('two'),
]);
$fetcher = new SameHostPageFetcher(new ProductPageScraper, 'https://shop.example.com', 2);
$first = $fetcher->fetch('https://shop.example.com/one');
$second = $fetcher->fetch('https://shop.example.com/two');
$this->assertNull($first['skipped']);
$this->assertSame('budget', $second['skipped']);
$this->assertSame(0, $second['remaining']);
}
public function test_it_skips_pages_that_time_out(): void
{
Http::preventStrayRequests();
Http::fake([
'https://shop.example.com/robots.txt' => Http::response("User-agent: *\nDisallow:\n"),
'https://shop.example.com/tents' => fn () => throw new ConnectionException('cURL error 28: Operation timed out'),
]);
$fetcher = new SameHostPageFetcher(new ProductPageScraper, 'https://shop.example.com');
$page = $fetcher->fetch('https://shop.example.com/tents');
$this->assertSame('timeout', $page['skipped']);
$this->assertSame('', $page['text']);
$this->assertSame([], $page['links']);
}
}