I am trying to build a script that will send me an email about upcoming races and marathons for the following month on the first of the current month (for testing I have it set to the current day). My script works on my Pycharm IDE, but doesn’t work with Pythonanywhere – python anywhere is not able to find the elements that I want to interact with, (‘First Error’ printed every time) and sometimes it crashes-It’s not loading the page properly. Does anyone know what I’m doing wrong?
Here is the code:
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.wait import WebDriverWait
from smtplib import SMTP
from datetime import datetime
import requests
import os
import time
from bs4 import BeautifulSoup
EMAIL = '[email protected]'
PASSWORD = os.environ.get('PASSWORD')
print(PASSWORD)
response = requests.get('https://www.runguides.com/')
response.raise_for_status()
# print(response.text)
soup = BeautifulSoup(response.text, 'html.parser')
link_extension = soup.select_one(selector="#footer > div.wrapper.footer-bottom > div > div.col-12.hide-on-tablet > div > ul > li:nth-child(66) > a").get('href')
print(link_extension)
chrome_options = webdriver.ChromeOptions()
#I tried to add these to pythonanywhere to fix a different problem
#---------------------------------------
# chrome_options.add_argument("--no-sandbox")
# chrome_options.add_argument("--headless")
# chrome_options.add_argument("--disable-gpu")
# chrome_options.add_argument('--disable-dev-shm-usage')
#------------------------------------------
chrome_options.add_experimental_option('detach',True)
main_driver = webdriver.Chrome(options=chrome_options)
if (datetime.now().day == 31):
main_driver.get(f'https://www.runguides.com{link_extension}')
main_driver.maximize_window()
try:
WebDriverWait(main_driver, 20).until(expected_conditions.element_to_be_clickable((By.XPATH,'/html/body/div[2]/div[3]/div/div[6]/div[4]/md-menu[1]/button')))
except TimeoutException:
print("First ERR")
else:
print('Success')
main_driver.find_element(By.XPATH, value="//*[@id="main-listing-container"]/div[4]/md-menu[1]/button").click()
try:
WebDriverWait(main_driver, 120).until(expected_conditions.presence_of_element_located((By.CSS_SELECTOR,'md-menu-content md-menu-item .md-button .ng-binding')))
except TimeoutException:
print("ERROR")
else:
months_elements = main_driver.find_elements(By.CSS_SELECTOR,
value="md-menu-content md-menu-item .md-button .ng-binding")
print(months_elements)
names = [item.text for item in months_elements]
print(names)
try:
month_names = months_elements[names.index('January'):]
except ValueError:
time.sleep(5)
names = [item.text for item in months_elements]
month_names = months_elements[names.index('January'):]
print(month_names)
month_names[datetime.now().month].click()
try:
WebDriverWait(main_driver, 120).until(expected_conditions.presence_of_element_located(
(By.CSS_SELECTOR, 'tbody tr')))
except TimeoutException:
print('Error')
else:
print('Success')
try:
all_race_dates = [race_date.text for race_date in main_driver.find_elements(By.CSS_SELECTOR, value="tbody .listing-container .listing-date .clickable") if 'Ontario' not in race_date.text]
print(all_race_dates)
all_race_names = [race_names.text for race_names in main_driver.find_elements(By.CSS_SELECTOR, value="tbody .listing-container .listing-title .clickable")]
print(all_race_names)
email_string = ''
for i in range(len(all_race_names)):
email_string +=f'{all_race_names[i]} on {all_race_dates[i]} \n'
if email_string != '':
with SMTP('smtp.gmail.com') as connection:
connection.starttls()
connection.login(user=EMAIL, password=PASSWORD)
connection.sendmail(from_addr=EMAIL, to_addrs=EMAIL, msg=f'Subject: Races next Month!!\n\n{email_string}')
except:
print('Password Error')
main_driver.quit()
#Obviously if it works, there should be a "Password Error" because that is an environment variable
#I've tried looking at documentation, adding the arguments I did to get the driver to not crash, and I tried adding print statemnts in various places to debug`
You need those options that you have commented out. You also need to make sure that you are closing and quitting the browser no matter what happens in your code. As you have it now, any exception in the main part of your code will skip the browser quit and leave the browser running. Enough running browsers and you will not be able to start new ones. See https://help.pythonanywhere.com/pages/selenium/