Http::response("User-agent: *\nDisallow:\n"),
'https://shop.example.com' => Http::response(''),
'https://shop.example.com/sitemap.xml' => Http::response(''),
]);
StructuredAnonymousAgent::fake([
['categories' => [
['name' => 'Outdoor', 'url' => 'https://shop.example.com/outdoor'],
['name' => 'Other store', 'url' => 'https://amazon.example.com/outdoor'],
]],
]);
$categories = (new CatalogWebsiteResearch(new ProductPageScraper, new NullListingScraper))->categories('https://shop.example.com');
$this->assertSame([
['name' => 'Outdoor', 'url' => 'https://shop.example.com/outdoor'],
['name' => 'Other store', 'url' => ''],
], $categories);
}
public function test_it_keeps_at_most_six_same_host_product_urls(): void
{
Http::preventStrayRequests();
Http::fake([
'https://shop.example.com/robots.txt' => Http::response("User-agent: *\nDisallow:\n"),
'https://shop.example.com/tents' => Http::response($this->listingHtmlWithProducts(8)),
...$this->productPageFakes(8),
]);
$products = [];
for ($i = 1; $i <= 8; $i++) {
$products[] = [
'name' => "Tent {$i}",
'url' => "https://shop.example.com/p/{$i}",
'popularity_note' => 'Top rated',
];
}
StructuredAnonymousAgent::fake([
['products' => $products],
]);
$picked = (new CatalogWebsiteResearch(new ProductPageScraper, new NullListingScraper))->topProducts(
'https://shop.example.com',
'Outdoor',
'Tents',
['https://shop.example.com/tents'],
);
$this->assertCount(6, $picked);
$this->assertSame('https://shop.example.com/p/1', $picked[0]['url']);
$this->assertSame('https://shop.example.com/p/6', $picked[5]['url']);
}
public function test_it_drops_product_urls_that_are_missing_or_have_no_image(): void
{
Http::preventStrayRequests();
Http::fake([
'https://shop.example.com/robots.txt' => Http::response("User-agent: *\nDisallow:\n"),
'https://shop.example.com/tents' => Http::response($this->listingHtmlWithProducts(8)),
'https://shop.example.com/p/1' => Http::response($this->productHtml()),
'https://shop.example.com/p/2' => Http::response('Not found', 404),
'https://shop.example.com/p/3' => Http::response('No image'),
'https://shop.example.com/p/4' => Http::response($this->productHtml()),
'https://shop.example.com/p/5' => Http::response($this->productHtml()),
'https://shop.example.com/p/6' => Http::response($this->productHtml()),
'https://shop.example.com/p/7' => Http::response($this->productHtml()),
'https://shop.example.com/p/8' => Http::response($this->productHtml()),
]);
StructuredAnonymousAgent::fake([
['products' => $this->productRows(8)],
]);
$picked = (new CatalogWebsiteResearch(new ProductPageScraper, new NullListingScraper))->topProducts(
'https://shop.example.com',
'Outdoor',
'Tents',
['https://shop.example.com/tents'],
);
$this->assertSame([
'https://shop.example.com/p/1',
'https://shop.example.com/p/4',
'https://shop.example.com/p/5',
'https://shop.example.com/p/6',
'https://shop.example.com/p/7',
'https://shop.example.com/p/8',
], array_column($picked, 'url'));
}
public function test_it_falls_back_to_extracted_nav_links_when_the_model_returns_no_categories(): void
{
Http::preventStrayRequests();
Http::fake([
'https://shop.example.com/robots.txt' => Http::response("User-agent: *\nDisallow:\n"),
'https://shop.example.com' => Http::response(
''
),
'https://shop.example.com/sitemap.xml' => Http::response(''),
]);
StructuredAnonymousAgent::fake([
['categories' => []],
]);
$categories = (new CatalogWebsiteResearch(new ProductPageScraper, new NullListingScraper))->categories('https://shop.example.com');
$this->assertSame([
['name' => 'Women', 'url' => 'https://shop.example.com/women'],
['name' => 'Men', 'url' => 'https://shop.example.com/men'],
], $categories);
}
public function test_it_resolves_relative_category_urls(): void
{
Http::preventStrayRequests();
Http::fake([
'https://shop.example.com/robots.txt' => Http::response("User-agent: *\nDisallow:\n"),
'https://shop.example.com' => Http::response(''),
'https://shop.example.com/sitemap.xml' => Http::response(''),
]);
StructuredAnonymousAgent::fake([
['categories' => [
['name' => 'Outdoor', 'url' => '/outdoor'],
]],
]);
$categories = (new CatalogWebsiteResearch(new ProductPageScraper, new NullListingScraper))->categories('https://shop.example.com');
$this->assertSame([
['name' => 'Outdoor', 'url' => 'https://shop.example.com/outdoor'],
], $categories);
}
public function test_it_logs_category_subcategory_and_product_url_fetches(): void
{
Http::preventStrayRequests();
Http::fake([
'https://shop.example.com/robots.txt' => Http::response("User-agent: *\nDisallow:\n"),
'https://shop.example.com' => Http::response(''),
'https://shop.example.com/sitemap.xml' => Http::response(''),
'https://shop.example.com/outdoor' => Http::response(
'TentsTent 1'
),
'https://shop.example.com/p/1' => Http::response($this->productHtml()),
]);
StructuredAnonymousAgent::fake([
['categories' => [
['name' => 'Outdoor', 'url' => 'https://shop.example.com/outdoor'],
]],
['subcategories' => [
['name' => 'Tents', 'url' => 'https://shop.example.com/outdoor/tents'],
]],
['products' => [
['name' => 'Tent 1', 'url' => 'https://shop.example.com/p/1', 'popularity_note' => 'Top rated'],
]],
]);
$logged = [];
Log::listen(function (MessageLogged $entry) use (&$logged): void {
$logged[] = $entry;
});
$research = new CatalogWebsiteResearch(new ProductPageScraper, new NullListingScraper);
$research->categories('https://shop.example.com');
$research->subcategories('https://shop.example.com', 'Outdoor', 'https://shop.example.com/outdoor');
$research->topProducts(
'https://shop.example.com',
'Outdoor',
'Tents',
['https://shop.example.com/outdoor'],
);
$messages = array_map(fn (MessageLogged $entry): string => $entry->message, $logged);
$this->assertContains('Catalog category URL fetch', $messages);
$this->assertContains('Catalog subcategory URL fetch', $messages);
$this->assertContains('Catalog listing URL fetch', $messages);
$this->assertContains('Catalog product URL fetch', $messages);
$categoryFetch = $this->firstLog($logged, 'Catalog category URL fetch');
$this->assertSame('https://shop.example.com', $categoryFetch->context['home']['url']);
$this->assertNull($categoryFetch->context['home']['skipped']);
$subcategoryFetch = $this->firstLog($logged, 'Catalog subcategory URL fetch');
$this->assertSame('https://shop.example.com/outdoor', $subcategoryFetch->context['page']['url']);
$this->assertSame('Outdoor', $subcategoryFetch->context['category']);
$listingFetch = $this->firstLog($logged, 'Catalog listing URL fetch');
$this->assertSame(['https://shop.example.com/outdoor'], array_column($listingFetch->context['pages'], 'url'));
$productFetch = $this->firstLog($logged, 'Catalog product URL fetch');
$this->assertSame('https://shop.example.com/p/1', $productFetch->context['url']);
$this->assertTrue($productFetch->context['live']);
$this->assertTrue($productFetch->context['has_image']);
}
public function test_it_skips_subcategories_when_the_category_page_is_already_a_product_listing(): void
{
Http::preventStrayRequests();
Http::fake([
'https://shop.example.com/robots.txt' => Http::response("User-agent: *\nDisallow:\n"),
'https://shop.example.com/fresh-seedlings' => Http::response(
''
.'View Alphonso Mango Tree'
.'View Chinese Money Plant'
),
]);
StructuredAnonymousAgent::fake([
['subcategories' => []],
]);
$subcategories = (new CatalogWebsiteResearch(new ProductPageScraper, new NullListingScraper))->subcategories(
'https://shop.example.com',
'Fresh Seedlings',
'https://shop.example.com/fresh-seedlings',
);
$this->assertSame([], $subcategories);
}
public function test_it_drops_product_and_account_links_returned_as_subcategories(): void
{
Http::preventStrayRequests();
Http::fake([
'https://shop.example.com/robots.txt' => Http::response("User-agent: *\nDisallow:\n"),
'https://shop.example.com/fresh-seedlings' => Http::response(
'Membership'
.'View Alphonso Mango Tree'
.'Indoor seedlings'
),
]);
StructuredAnonymousAgent::fake([
['subcategories' => [
['name' => 'Membership', 'url' => 'https://shop.example.com/membership'],
['name' => 'View Alphonso Mango Tree', 'url' => 'https://shop.example.com/products/alphonso-mango-tree'],
['name' => 'Indoor seedlings', 'url' => 'https://shop.example.com/fresh-seedlings/indoor'],
]],
]);
$subcategories = (new CatalogWebsiteResearch(new ProductPageScraper, new NullListingScraper))->subcategories(
'https://shop.example.com',
'Fresh Seedlings',
'https://shop.example.com/fresh-seedlings',
);
$this->assertSame([
['name' => 'Indoor seedlings', 'url' => 'https://shop.example.com/fresh-seedlings/indoor'],
], $subcategories);
}
public function test_it_returns_no_products_when_the_listing_page_times_out(): void
{
Http::preventStrayRequests();
Http::fake([
'https://shop.example.com/robots.txt' => Http::response("User-agent: *\nDisallow:\n"),
'https://shop.example.com/tents' => fn () => throw new ConnectionException('cURL error 28: Operation timed out'),
...$this->productPageFakes(6),
]);
StructuredAnonymousAgent::fake([
['products' => $this->productRows(6)],
]);
$picked = (new CatalogWebsiteResearch(new ProductPageScraper, new NullListingScraper))->topProducts(
'https://shop.example.com',
'Outdoor',
'Tents',
['https://shop.example.com/tents'],
);
$this->assertSame([], $picked);
}
public function test_it_fills_from_extracted_links_when_the_listing_url_has_no_query_and_the_model_returns_too_few(): void
{
Http::preventStrayRequests();
Http::fake([
'https://shop.example.com/robots.txt' => Http::response("User-agent: *\nDisallow:\n"),
'https://shop.example.com/collections/all' => Http::response($this->listingHtmlWithProducts(8)),
...$this->productPageFakes(8),
]);
StructuredAnonymousAgent::fake([
['products' => $this->productRows(4)],
]);
$picked = (new CatalogWebsiteResearch(new ProductPageScraper, new NullListingScraper))->topProducts(
'https://shop.example.com',
'Shop Kits',
'',
['https://shop.example.com/collections/all'],
);
$this->assertSame([
'https://shop.example.com/p/1',
'https://shop.example.com/p/2',
'https://shop.example.com/p/3',
'https://shop.example.com/p/4',
'https://shop.example.com/p/5',
'https://shop.example.com/p/6',
], array_column($picked, 'url'));
}
public function test_it_drops_listing_products_that_do_not_match_the_listing_query(): void
{
Http::preventStrayRequests();
Http::fake([
'https://shop.example.com/robots.txt' => Http::response("User-agent: *\nDisallow:\n"),
'https://shop.example.com/collections/all?filter=flowering-plants' => Http::response(
''
.'Monstera Deliciosa'
.'Snake Plant'
.'Hibiscus Flowering Plant'
.'Jasmine Flowering Plant'
.'Rose Flowering Plant'
.'Marigold Flowering Plant'
.'Bougainvillea Flowering Plant'
.''
),
'https://shop.example.com/products/monstera-deliciosa' => Http::response($this->productHtml()),
'https://shop.example.com/products/snake-plant' => Http::response($this->productHtml()),
'https://shop.example.com/products/hibiscus-flowering-plant' => Http::response($this->productHtml()),
'https://shop.example.com/products/jasmine-flowering-plant' => Http::response($this->productHtml()),
'https://shop.example.com/products/rose-flowering-plant' => Http::response($this->productHtml()),
'https://shop.example.com/products/marigold-flowering-plant' => Http::response($this->productHtml()),
'https://shop.example.com/products/bougainvillea-flowering-plant' => Http::response($this->productHtml()),
]);
StructuredAnonymousAgent::fake([
['products' => [
['name' => 'Monstera Deliciosa', 'url' => 'https://shop.example.com/products/monstera-deliciosa', 'popularity_note' => 'Bestseller'],
['name' => 'Snake Plant', 'url' => 'https://shop.example.com/products/snake-plant', 'popularity_note' => 'Bestseller'],
['name' => 'Hibiscus Flowering Plant', 'url' => 'https://shop.example.com/products/hibiscus-flowering-plant', 'popularity_note' => 'Top rated'],
['name' => 'Jasmine Flowering Plant', 'url' => 'https://shop.example.com/products/jasmine-flowering-plant', 'popularity_note' => 'Top rated'],
['name' => 'Rose Flowering Plant', 'url' => 'https://shop.example.com/products/rose-flowering-plant', 'popularity_note' => 'Top rated'],
['name' => 'Marigold Flowering Plant', 'url' => 'https://shop.example.com/products/marigold-flowering-plant', 'popularity_note' => 'Top rated'],
['name' => 'Bougainvillea Flowering Plant', 'url' => 'https://shop.example.com/products/bougainvillea-flowering-plant', 'popularity_note' => 'Top rated'],
]],
]);
$picked = (new CatalogWebsiteResearch(new ProductPageScraper, new NullListingScraper))->topProducts(
'https://shop.example.com',
'',
'',
['https://shop.example.com/collections/all?filter=flowering-plants'],
);
$urls = array_column($picked, 'url');
$this->assertSame([
'https://shop.example.com/products/hibiscus-flowering-plant',
'https://shop.example.com/products/jasmine-flowering-plant',
'https://shop.example.com/products/rose-flowering-plant',
'https://shop.example.com/products/marigold-flowering-plant',
'https://shop.example.com/products/bougainvillea-flowering-plant',
], $urls);
$this->assertNotContains('https://shop.example.com/products/monstera-deliciosa', $urls);
$this->assertNotContains('https://shop.example.com/products/snake-plant', $urls);
}
public function test_it_keeps_mixed_listing_products_when_the_listing_url_has_no_filter(): void
{
Http::preventStrayRequests();
Http::fake([
'https://shop.example.com/robots.txt' => Http::response("User-agent: *\nDisallow:\n"),
'https://shop.example.com/collections/all' => Http::response(
''
.'Monstera Deliciosa'
.'Hibiscus Flowering Plant'
.'Snake Plant'
.'Rose Flowering Plant'
.'Golden Pothos'
.''
),
'https://shop.example.com/products/monstera-deliciosa' => Http::response($this->productHtml()),
'https://shop.example.com/products/snake-plant' => Http::response($this->productHtml()),
'https://shop.example.com/products/hibiscus-flowering-plant' => Http::response($this->productHtml()),
'https://shop.example.com/products/rose-flowering-plant' => Http::response($this->productHtml()),
'https://shop.example.com/products/golden-pothos' => Http::response($this->productHtml()),
]);
StructuredAnonymousAgent::fake([
['products' => [
['name' => 'Monstera Deliciosa', 'url' => 'https://shop.example.com/products/monstera-deliciosa', 'popularity_note' => 'Bestseller'],
['name' => 'Hibiscus Flowering Plant', 'url' => 'https://shop.example.com/products/hibiscus-flowering-plant', 'popularity_note' => 'Top rated'],
['name' => 'Snake Plant', 'url' => 'https://shop.example.com/products/snake-plant', 'popularity_note' => 'Bestseller'],
['name' => 'Rose Flowering Plant', 'url' => 'https://shop.example.com/products/rose-flowering-plant', 'popularity_note' => 'Top rated'],
['name' => 'Golden Pothos', 'url' => 'https://shop.example.com/products/golden-pothos', 'popularity_note' => 'Bestseller'],
]],
]);
$picked = (new CatalogWebsiteResearch(new ProductPageScraper, new NullListingScraper))->topProducts(
'https://shop.example.com',
'',
'',
['https://shop.example.com/collections/all'],
);
$this->assertSame([
'https://shop.example.com/products/monstera-deliciosa',
'https://shop.example.com/products/hibiscus-flowering-plant',
'https://shop.example.com/products/snake-plant',
'https://shop.example.com/products/rose-flowering-plant',
'https://shop.example.com/products/golden-pothos',
], array_column($picked, 'url'));
}
public function test_it_uses_puppeteer_when_enabled_and_enough_products_are_scraped(): void
{
config(['scraping.puppeteer.enabled' => true]);
Http::preventStrayRequests();
$listingScraper = $this->createMock(ListingScraper::class);
$listingScraper->expects($this->once())
->method('scrape')
->with('https://shop.example.com/tents')
->willReturn(new ScrapedListingResult(
url: 'https://shop.example.com/tents',
products: $this->scrapedListingProducts(6),
error: null,
source: 'puppeteer',
));
$picked = (new CatalogWebsiteResearch(new ProductPageScraper, $listingScraper))->topProducts(
'https://shop.example.com',
'Outdoor',
'Tents',
['https://shop.example.com/tents'],
);
$this->assertCount(6, $picked);
$this->assertSame('puppeteer', $picked[0]['discovery_source']);
$this->assertSame('Tent 6', $picked[0]['name']);
$this->assertSame('5 stars, 6 reviews', $picked[0]['popularity_note']);
$this->assertSame('https://shop.example.com/p/6', $picked[0]['url']);
}
public function test_it_falls_back_to_http_when_puppeteer_is_disabled(): void
{
config(['scraping.puppeteer.enabled' => false]);
Http::preventStrayRequests();
Http::fake([
'https://shop.example.com/robots.txt' => Http::response("User-agent: *\nDisallow:\n"),
'https://shop.example.com/tents' => Http::response($this->listingHtmlWithProducts(8)),
...$this->productPageFakes(8),
]);
StructuredAnonymousAgent::fake([
['products' => $this->productRows(8)],
]);
$listingScraper = $this->createMock(ListingScraper::class);
$listingScraper->expects($this->never())->method('scrape');
$picked = (new CatalogWebsiteResearch(new ProductPageScraper, $listingScraper))->topProducts(
'https://shop.example.com',
'Outdoor',
'Tents',
['https://shop.example.com/tents'],
);
$this->assertCount(6, $picked);
$this->assertSame('https://shop.example.com/p/1', $picked[0]['url']);
}
public function test_it_falls_back_to_http_when_puppeteer_returns_too_few_products(): void
{
config(['scraping.puppeteer.enabled' => true]);
Http::preventStrayRequests();
Http::fake([
'https://shop.example.com/robots.txt' => Http::response("User-agent: *\nDisallow:\n"),
'https://shop.example.com/tents' => Http::response($this->listingHtmlWithProducts(8)),
...$this->productPageFakes(8),
]);
StructuredAnonymousAgent::fake([
['products' => $this->productRows(8)],
]);
$listingScraper = $this->createMock(ListingScraper::class);
$listingScraper->expects($this->once())
->method('scrape')
->willReturn(new ScrapedListingResult(
url: 'https://shop.example.com/tents',
products: $this->scrapedListingProducts(2),
error: null,
source: 'puppeteer',
));
$picked = (new CatalogWebsiteResearch(new ProductPageScraper, $listingScraper))->topProducts(
'https://shop.example.com',
'Outdoor',
'Tents',
['https://shop.example.com/tents'],
);
$this->assertCount(6, $picked);
$this->assertSame('https://shop.example.com/p/1', $picked[0]['url']);
$this->assertArrayNotHasKey('discovery_source', $picked[0]);
}
/**
* @param list $logged
*/
private function firstLog(array $logged, string $message): MessageLogged
{
foreach ($logged as $entry) {
if ($entry->message === $message) {
return $entry;
}
}
$this->fail("Missing log message [{$message}].");
}
private function listingHtmlWithProducts(int $count): string
{
$anchors = '';
for ($i = 1; $i <= $count; $i++) {
$anchors .= "Tent {$i}";
}
return "{$anchors}";
}
/**
* @return array
*/
private function productPageFakes(int $count): array
{
$fakes = [];
for ($i = 1; $i <= $count; $i++) {
$fakes["https://shop.example.com/p/{$i}"] = Http::response($this->productHtml());
}
return $fakes;
}
/**
* @return list
*/
private function productRows(int $count): array
{
$rows = [];
for ($i = 1; $i <= $count; $i++) {
$rows[] = [
'name' => "Tent {$i}",
'url' => "https://shop.example.com/p/{$i}",
'popularity_note' => 'Top rated',
];
}
return $rows;
}
private function productHtml(): string
{
return 'Tent';
}
/**
* @return list
*/
private function scrapedListingProducts(int $count): array
{
$products = [];
for ($i = 1; $i <= $count; $i++) {
$products[] = new ScrapedListingProduct(
name: "Tent {$i}",
url: "https://shop.example.com/p/{$i}",
price: '$'.(10 * $i).'.00',
rating: (string) min(5, $i),
reviewCount: (string) $i,
imageUrl: "https://cdn.example.com/tent-{$i}.jpg",
);
}
return $products;
}
}