Web Scrapping using Selenium with Python Dynamically website

I am extracing the products data from different paging but it get the first page data not different page, even i added waits structure is completed loaded but still not getting the other page data.

please help me out to solve

import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException, TimeoutException, StaleElementReferenceException, WebDriverException

import xlwings as xw

# Create a webdriver instance
driver = webdriver.Chrome()

try:
    # Navigate to the website
    url="https://www.vinted.fr/"
    driver.get(url)

    # Access the Excel file
    wb = xw.Book('D:/Salman_Projects/EbuyScraping/EbayExcel.xlsm')
    sheet = wb.sheets['Sheet1']
    keywords = sheet.range('L5').value

    # Input keywords into the search box
    search_input = WebDriverWait(driver, 10).until(
        EC.presence_of_element_located((By.ID, 'search_text'))
    )
    search_input.send_keys(keywords + Keys.RETURN)


    while True:
        # Wait for the dynamic content to be loaded
        parent_locator = (By.CLASS_NAME, 'feed-grid')
        try:
            parent_element = WebDriverWait(driver, 20).until(
                EC.visibility_of_element_located(parent_locator)
            )

            # Extract product information
            product_items = parent_element.find_elements(By.CLASS_NAME, 'new-item-box__container')

            for product_item in product_items:
                owner_name = product_item.find_element(By.CLASS_NAME, 'web_ui__Text__caption').text
                image_url = product_item.find_element(By.CSS_SELECTOR, '[data-testid$="--image--img"]').get_attribute('src')
                title = product_item.find_element(By.CLASS_NAME, 'web_ui__Text__subtitle').text

                try:
                    price_element = product_item.find_element(By.CLASS_NAME, 'web_ui__Text__price')
                    price = price_element.text
                except NoSuchElementException:
                    price = "Price not available"
                # Print the extracted data for each product
                print(f"Owner: {owner_name}, Image URL: {image_url}, Title: {title}, Price: {price}")
            # Check if the "Next" button is disabled
            try:
                next_button = WebDriverWait(driver, 10).until(
                    EC.element_to_be_clickable((By.CLASS_NAME, 'web_ui__Pagination__next'))
                )
                next_button_classes = next_button.get_attribute('class')
                if 'web_ui__Pagination__next web_ui__Pagination__is-disabled' in next_button_classes:
                    break  # Break the loop if the "Next" button is disabled
            except NoSuchElementException:
                break  # Break the loop if the "Next" button is not found
            except TimeoutException:
                print("Timeout waiting for the 'Next' button to be clickable.")
                break
            except StaleElementReferenceException:
                print("Stale element reference while checking 'Next' button.")
                break
            except WebDriverException as e:
                print(f"WebDriverException: {e}")
                break



            # Click the "Next" button and wait for the new set of products to load
            next_button.click()

            # Additional wait for the new set of products to load
            WebDriverWait(driver, 20).until(
                EC.staleness_of(parent_element)
            )
            WebDriverWait(driver, 20).until(
                EC.visibility_of_element_located(parent_locator)
            )


        except TimeoutException as e:
            print(f"Error: {e}")
            print("Page load timed out.")

except Exception as e:
    print(f"An unexpected error occurred: {e}")

finally:
    # Close the browser
    driver.quit()

this .

I want to extract the products data from different pages but it getting just one page data.

Leave a Comment