I uses scrapy to scrap products from amazon but It only returns first product for each page and go to next page what wrong with my code
import scrapy
from ..items import AmazonItem
class AmazonSpSpider(scrapy.Spider):
name = "amazon_sp"
allowed_domains = ["www.amazon.com"]
start_urls = ["https://www.amazon.com/s?k=laptop"]
page_number = 1
def parse(self, response):
items = response.css('h2 a.a-text-normal::attr(href)').getall()
print(items)
for result in items:
item_link = "https://www.amazon.com/" + result
print("Hi I am in loop")
yield scrapy.Request(url=item_link, callback=self.parse_item)
next_page = f"https://www.amazon.com/s?k=laptop&page={AmazonSpSpider.page_number}"
print('I am not in loop')
AmazonSpSpider.page_number += 1
yield response.follow(url=next_page, callback=self.parse)
def parse_item(self, response):
item = AmazonItem()
item['title'] = response.css('#productTitle::text').get()
item['image_link'] = response.css('imgTagWrapperId img::attr(src)').get()
item['price'] = response.css('.a-price span::text').get()
item['brand'] = response.css('.po-brand .po-break-word::text').get()
item['model'] = response.css('.po-model_name .po-break-word::text').get()
item['rating'] = response.css('#acrPopover .a-color-base::text').get()
yield item