Scrapy Spider returns only first product for every page

I uses scrapy to scrap products from amazon but It only returns first product for each page and go to next page what wrong with my code

import scrapy
from ..items import AmazonItem


class AmazonSpSpider(scrapy.Spider):
    name = "amazon_sp"
    allowed_domains = ["www.amazon.com"]
    start_urls = ["https://www.amazon.com/s?k=laptop"]
    page_number = 1

    def parse(self, response):
        items = response.css('h2 a.a-text-normal::attr(href)').getall()
        print(items)
        for result in items:
            item_link = "https://www.amazon.com/" + result
            print("Hi I am in loop")
            yield scrapy.Request(url=item_link, callback=self.parse_item)

        next_page = f"https://www.amazon.com/s?k=laptop&page={AmazonSpSpider.page_number}"
        print('I am not in loop')
        AmazonSpSpider.page_number += 1
        yield response.follow(url=next_page, callback=self.parse)

    def parse_item(self, response):
        item = AmazonItem()
        item['title'] = response.css('#productTitle::text').get()
        item['image_link'] = response.css('imgTagWrapperId img::attr(src)').get()
        item['price'] = response.css('.a-price span::text').get()
        item['brand'] = response.css('.po-brand .po-break-word::text').get()
        item['model'] = response.css('.po-model_name .po-break-word::text').get()
        item['rating'] = response.css('#acrPopover .a-color-base::text').get()
        yield item



And screenshot of my output
enter image description hereenter image description here

Leave a Comment