Http::response('Not found', 404),
]);
$html = (new ProductPageScraper)->fetchLiveHtml('https://shop.example.com/p/missing');
$this->assertNull($html);
}
public function test_it_returns_live_html_when_the_page_exists(): void
{
Http::preventStrayRequests();
Http::fake([
'https://shop.example.com/p/1' => Http::response('Tent'),
]);
$html = (new ProductPageScraper)->fetchLiveHtml('https://shop.example.com/p/1');
$this->assertSame('Tent', $html);
}
public function test_it_returns_empty_html_when_the_listing_request_times_out(): void
{
Http::preventStrayRequests();
Http::fake([
'https://shop.example.com/tents' => fn () => throw new ConnectionException('cURL error 28: Operation timed out'),
]);
$html = (new ProductPageScraper)->fetchHtml(
'https://shop.example.com/tents',
ProductPageScraper::LISTING_TIMEOUT_SECONDS,
);
$this->assertSame('', $html);
$this->assertTrue(
(new ProductPageScraper)->tryFetchHtml(
'https://shop.example.com/tents',
ProductPageScraper::LISTING_TIMEOUT_SECONDS,
)['timed_out'],
);
}
public function test_it_returns_null_live_html_when_the_product_request_times_out(): void
{
Http::preventStrayRequests();
Http::fake([
'https://shop.example.com/p/1' => fn () => throw new ConnectionException('cURL error 28: Operation timed out'),
]);
$html = (new ProductPageScraper)->fetchLiveHtml('https://shop.example.com/p/1');
$this->assertNull($html);
}
public function test_it_uses_json_ld_content_url_instead_of_image_object_type(): void
{
$jsonLd = json_encode([
'@type' => 'Product',
'name' => 'Magellan Desk',
'image' => [
'@type' => 'ImageObject',
'contentUrl' => 'https://cdn.officedepot.com/desks/magellan.jpg',
],
]);
$html = ''
.'';
$image = (new ProductPageScraper)->primaryImageUrl(
$html,
'https://www.officedepot.com/a/products/956652/Realspace-Magellan-Performance-71W-L-Shape',
);
$this->assertSame('https://cdn.officedepot.com/desks/magellan.jpg', $image);
}
public function test_page_text_keeps_structured_copy_when_visible_samples_are_mostly_whitespace(): void
{
$jsonLd = json_encode([
'@type' => 'Product',
'name' => 'Whalen Marilla Desk',
'offers' => ['price' => '99.99', 'priceCurrency' => 'USD'],
]);
$html = 'Whalen Desk'
.str_repeat(' ', 1_100_000)
.''
.'';
$text = (new ProductPageScraper)->pageText($html);
$this->assertStringContainsString('Whalen Marilla Desk', $text);
$this->assertStringContainsString('99.99', $text);
}
public function test_it_uses_json_ld_product_image_when_it_appears_after_the_image_scan_limit(): void
{
$gallery = 'https://media.officedepot.com/images/f_auto,q_auto,e_sharpen,h_450/products/2146434/2146434_o01_whalen_marilla_48w_pedestal_desk_071723/2146434_o01_whalen_marilla_48w_pedestal_desk_071723.jpg';
$jsonLd = json_encode([
'@type' => 'Product',
'name' => 'Whalen Marilla Desk',
'image' => [
'@type' => 'ImageObject',
'contentUrl' => $gallery,
],
]);
$lexmarkLogo = 'https://media.officedepot.com/image/upload/v1614112807/assets/lexmark_logo.jpg';
$padding = str_repeat(' ', 1_100_000);
$html = ''
.''
.'
'
.''
.$padding
.''
.'';
$image = (new ProductPageScraper)->primaryImageUrl(
$html,
'https://www.officedepot.com/a/products/2146434/Whalen-Marilla-48W-Pedestal-Desk-Driftwood/',
);
$this->assertSame($gallery, $image);
}
public function test_it_uses_json_ld_url_on_an_image_object(): void
{
$jsonLd = json_encode([
'@type' => 'Product',
'image' => [
'@type' => 'ImageObject',
'url' => 'https://cdn.officedepot.com/desks/koru.jpg',
],
]);
$image = (new ProductPageScraper)->primaryImageUrl(
'',
'https://www.officedepot.com/a/products/123/Koru',
);
$this->assertSame('https://cdn.officedepot.com/desks/koru.jpg', $image);
}
public function test_prefers_gallery_image_containing_the_product_id_over_a_promo_tile(): void
{
$promo = 'https://media.officedepot.com/image/upload/v1614112582/assets/350x191_hp.png';
$gallery = 'https://media.officedepot.com/images/f_auto,q_auto,e_sharpen,h_450/products/810838/810838_o01_office_depot_brand_file_folders_112519/810838';
$html = ''
.''
.'
'
.'
'
.'';
$image = (new ProductPageScraper)->primaryImageUrl(
$html,
'https://www.officedepot.com/a/products/810838/Office-Depot-Brand-File-Folders-13/',
);
$this->assertSame($gallery, $image);
}
public function test_falls_back_to_open_graph_image_when_it_is_the_only_product_photo(): void
{
$html = 'Watering can';
$image = (new ProductPageScraper)->primaryImageUrl(
$html,
'https://example.com/can',
);
$this->assertSame('https://cdn.example.com/can.png', $image);
}
public function test_prefers_image_matching_the_product_slug_when_the_page_has_no_numeric_id(): void
{
$html = ''
.''
.'
'
.'';
$image = (new ProductPageScraper)->primaryImageUrl(
$html,
'https://shop.example.com/products/manila-file-folders',
);
$this->assertSame('https://cdn.example.com/cdn/shop/products/manila-file-folders.jpg', $image);
}
public function test_uses_the_largest_srcset_candidate_on_an_img_tag(): void
{
$html = '
';
$image = (new ProductPageScraper)->primaryImageUrl(
$html,
'https://shop.example.com/products/canvas-tent',
);
$this->assertSame('https://cdn.example.com/p/large.jpg', $image);
}
public function test_uses_a_zoom_image_attribute_instead_of_a_thumbnail_src(): void
{
$html = '
';
$image = (new ProductPageScraper)->primaryImageUrl(
$html,
'https://shop.example.com/products/canvas-tent',
);
$this->assertSame('https://cdn.example.com/products/canvas-tent.jpg', $image);
}
public function test_prefers_the_gallery_photo_over_a_header_help_icon(): void
{
$icon = 'https://media.officedepot.com/image/upload/v1698167931/content/od/header/Question.svg';
$gallery = 'https://media.officedepot.com/images/f_auto,q_auto,e_sharpen,h_450/products/9420365/9420365_o01/9420365_o01';
$html = ''
.'
'
.''
.'';
$image = (new ProductPageScraper)->primaryImageUrl(
$html,
'https://www.officedepot.com/a/products/9420365/Crayola-Colors-Of-The-World-Coloring-Book/',
);
$this->assertSame($gallery, $image);
}
public function test_does_not_use_a_header_help_icon_as_the_product_photo(): void
{
$html = '
';
$image = (new ProductPageScraper)->primaryImageUrl(
$html,
'https://www.officedepot.com/a/products/9420365/Crayola-Colors-Of-The-World-Coloring-Book/',
);
$this->assertNull($image);
}
public function test_keeps_a_product_svg_in_the_gallery(): void
{
$html = '
';
$image = (new ProductPageScraper)->primaryImageUrl(
$html,
'https://shop.example.com/products/9420365/diagram',
);
$this->assertSame('https://cdn.example.com/products/9420365/diagram.svg', $image);
}
public function test_uses_the_largest_srcset_candidate_when_urls_contain_commas(): void
{
$small = 'https://media.officedepot.com/images/f_auto,q_auto,e_sharpen,w_100/products/9420365/9420365_o01/9420365_o01';
$large = 'https://media.officedepot.com/images/f_auto,q_auto,e_sharpen,w_800/products/9420365/9420365_o01/9420365_o01';
$html = '
';
$image = (new ProductPageScraper)->primaryImageUrl(
$html,
'https://www.officedepot.com/a/products/9420365/Crayola-Colors-Of-The-World-Coloring-Book/',
);
$this->assertSame($large, $image);
}
public function test_falls_back_to_src_when_srcset_commas_would_truncate_the_url(): void
{
$gallery = 'https://media.officedepot.com/images/f_auto,q_auto,e_sharpen,h_450/products/9420365/9420365_o01/9420365_o01';
$html = '';
$image = (new ProductPageScraper)->primaryImageUrl(
$html,
'https://www.officedepot.com/a/products/9420365/Crayola-Colors-Of-The-World-Coloring-Book/',
);
$this->assertSame($gallery, $image);
}
public function test_page_text_extracts_structured_product_data_from_large_html_without_script_noise(): void
{
$jsonLd = json_encode([
'@type' => 'Product',
'name' => 'Memory Safe Chair',
'description' => 'Top-rated folding chair for offices.',
'offers' => ['price' => '49.99', 'priceCurrency' => 'USD'],
'aggregateRating' => ['ratingValue' => '4.8', 'reviewCount' => '1250'],
]);
$html = ''
.'Memory Safe Chair | Office Depot'
.''
.''
.''
.''
.'Top Rated Folding Chairs
'
.str_repeat(' filler ', 200_000)
.'';
$text = (new ProductPageScraper)->pageText($html);
$this->assertStringContainsString('Memory Safe Chair', $text);
$this->assertStringContainsString('Top-rated folding chair for offices.', $text);
$this->assertStringContainsString('Top Rated Folding Chairs', $text);
$this->assertStringContainsString('49.99', $text);
$this->assertStringContainsString('4.8', $text);
$this->assertStringNotContainsString('window.bundle=', $text);
}
public function test_page_text_keeps_early_product_copy_on_very_large_pages(): void
{
$marker = 'Top Rated Folding Chairs';
$html = 'Large page'
.''.$marker.'
'
.str_repeat(' filler ', 300_000)
.'';
$text = (new ProductPageScraper)->pageText($html);
$this->assertStringContainsString($marker, $text);
$this->assertStringNotContainsString(str_repeat('filler', 20), $text);
}
public function test_product_offer_from_html_reads_json_ld_prices(): void
{
$html = '';
$offer = (new ProductPageScraper)->productOfferFromHtml($html);
$this->assertSame('$49.99', $offer['price']);
$this->assertSame('$79.99', $offer['list_price']);
}
public function test_product_ratings_from_html_reads_json_ld_and_bazaarvoice_labels(): void
{
$jsonLd = json_encode([
'@type' => 'Product',
'aggregateRating' => [
'ratingValue' => '4.8',
'reviewCount' => '1250',
],
]);
$ratings = (new ProductPageScraper)->productRatingsFromHtml(
'',
);
$this->assertSame('4.8', $ratings['rating']);
$this->assertSame('1250', $ratings['review_count']);
$ratings = (new ProductPageScraper)->productRatingsFromHtml(
'',
);
$this->assertSame('3.6', $ratings['rating']);
$this->assertSame('14', $ratings['review_count']);
}
public function test_listing_image_matches_product_requires_product_id_in_image_path(): void
{
$scraper = new ProductPageScraper;
$url = 'https://www.officedepot.com/a/products/254901/Realspace-Metal-Folding-Chair-Black/';
$this->assertTrue($scraper->listingImageMatchesProduct(
'https://media.officedepot.com/images/t_large,f_auto/products/254901/1.jpg',
$url,
));
$this->assertFalse($scraper->listingImageMatchesProduct(
'https://media.officedepot.com/image/upload/v1753716258/CREATIVE/CREATIVE_2025/SITE/GLOBAL/promo.jpg',
$url,
));
}
public function test_listing_image_matches_kohls_product_id_in_media_url(): void
{
$scraper = new ProductPageScraper;
$url = 'https://www.kohls.com/product/prd-6554861/adidas-vl-court-30-womens-shoes.jsp';
$this->assertTrue($scraper->listingImageMatchesProduct(
'https://media.kohlsimg.com/is/image/kohls/6554861_Brown?wid=342&hei=342&op_sharpen=1',
$url,
));
$this->assertSame('6554861', $scraper->productIdFromUrl($url));
}
public function test_listing_image_matches_marshalls_product_id_in_media_url(): void
{
$scraper = new ProductPageScraper;
$url = 'https://www.marshalls.com/us/store/jump/product/women-women-clothing/new-arrivals/Superlow-Loose-Jeans/4000510735?colorId=NS19643052';
$this->assertSame('4000510735', $scraper->productIdFromUrl($url));
$this->assertTrue($scraper->listingImageMatchesProduct(
'https://img.marshalls.com/marshalls?set=DisplayName[f2],prd[4000510735_NS19643052],ag[no]&call=url[file:tjxrPRD2_RB.chain]',
$url,
));
}
public function test_listing_image_matches_amazon_media_urls(): void
{
$scraper = new ProductPageScraper;
$url = 'https://www.amazon.com/dp/B0C1234567/ref=zg_bs_appliances_1';
$this->assertSame('B0C1234567', $scraper->productIdFromUrl($url));
$this->assertTrue($scraper->listingImageMatchesProduct(
'https://m.media-amazon.com/images/I/71abc._AC_UL320_.jpg',
$url,
));
}
}
';
$image = (new ProductPageScraper)->primaryImageUrl(
$html,
'https://www.officedepot.com/a/products/9420365/Crayola-Colors-Of-The-World-Coloring-Book/',
);
$this->assertSame($gallery, $image);
}
public function test_page_text_extracts_structured_product_data_from_large_html_without_script_noise(): void
{
$jsonLd = json_encode([
'@type' => 'Product',
'name' => 'Memory Safe Chair',
'description' => 'Top-rated folding chair for offices.',
'offers' => ['price' => '49.99', 'priceCurrency' => 'USD'],
'aggregateRating' => ['ratingValue' => '4.8', 'reviewCount' => '1250'],
]);
$html = ''
.'