I am extracing the products data from different paging but it get the first page data not different page, even i added waits structure is completed loaded but still not getting the other page data.
please help me out to solve
import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException, TimeoutException, StaleElementReferenceException, WebDriverException
import xlwings as xw
# Create a webdriver instance
driver = webdriver.Chrome()
try:
# Navigate to the website
url="https://www.vinted.fr/"
driver.get(url)
# Access the Excel file
wb = xw.Book('D:/Salman_Projects/EbuyScraping/EbayExcel.xlsm')
sheet = wb.sheets['Sheet1']
keywords = sheet.range('L5').value
# Input keywords into the search box
search_input = WebDriverWait(driver, 10).until(
EC.presence_of_element_located((By.ID, 'search_text'))
)
search_input.send_keys(keywords + Keys.RETURN)
while True:
# Wait for the dynamic content to be loaded
parent_locator = (By.CLASS_NAME, 'feed-grid')
try:
parent_element = WebDriverWait(driver, 20).until(
EC.visibility_of_element_located(parent_locator)
)
# Extract product information
product_items = parent_element.find_elements(By.CLASS_NAME, 'new-item-box__container')
for product_item in product_items:
owner_name = product_item.find_element(By.CLASS_NAME, 'web_ui__Text__caption').text
image_url = product_item.find_element(By.CSS_SELECTOR, '[data-testid$="--image--img"]').get_attribute('src')
title = product_item.find_element(By.CLASS_NAME, 'web_ui__Text__subtitle').text
try:
price_element = product_item.find_element(By.CLASS_NAME, 'web_ui__Text__price')
price = price_element.text
except NoSuchElementException:
price = "Price not available"
# Print the extracted data for each product
print(f"Owner: {owner_name}, Image URL: {image_url}, Title: {title}, Price: {price}")
# Check if the "Next" button is disabled
try:
next_button = WebDriverWait(driver, 10).until(
EC.element_to_be_clickable((By.CLASS_NAME, 'web_ui__Pagination__next'))
)
next_button_classes = next_button.get_attribute('class')
if 'web_ui__Pagination__next web_ui__Pagination__is-disabled' in next_button_classes:
break # Break the loop if the "Next" button is disabled
except NoSuchElementException:
break # Break the loop if the "Next" button is not found
except TimeoutException:
print("Timeout waiting for the 'Next' button to be clickable.")
break
except StaleElementReferenceException:
print("Stale element reference while checking 'Next' button.")
break
except WebDriverException as e:
print(f"WebDriverException: {e}")
break
# Click the "Next" button and wait for the new set of products to load
next_button.click()
# Additional wait for the new set of products to load
WebDriverWait(driver, 20).until(
EC.staleness_of(parent_element)
)
WebDriverWait(driver, 20).until(
EC.visibility_of_element_located(parent_locator)
)
except TimeoutException as e:
print(f"Error: {e}")
print("Page load timed out.")
except Exception as e:
print(f"An unexpected error occurred: {e}")
finally:
# Close the browser
driver.quit()
this .
I want to extract the products data from different pages but it getting just one page data.