I dont get errors from the script below but this script returns no data. I am trying to get all the games for each of the weeks which start in table 4 in the html. When I enter the xpath commands in the scrapy shell I get data but once I put in the parse definition I dont get anything in return.
import scrapy
class NFLOddsSpider(scrapy.Spider):
name="NFLOdds"
allowed_domains = ['www.sportsoddshistory.com']
start_urls = ['http://www.sportsoddshistory.com/']
def parse(self, response):
for row in response.xpath('//table[@class="soh1"]//tbody/tr'):
day = row.xpath('td[1]//text()').extract_first()
date = row.xpath('td[2]//text()').extract_first()
time = row.xpath('td[3]//text()').extract_first()
AtFav = row.xpath('td[4]//text()').extract_first()
favorite = row.xpath('td[5]//text()').extract_first()
score = row.xpath('td[6]//text()').extract_first()
spread = row.xpath('td[7]//text()').extract_first()
AtDog = row.xpath('td[8]//text()').extract_first()
underdog = row.xpath('td[9]//text()').extract_first()
OvUn = row.xpath('td[10]//text()').extract_first()
notes = row.xpath('td[11]//text()').extract_first()
week = row.xpath('//*[@id="content"]/div/table[4]/tbody/tr/td/h3').extract_first()
oddsTable = {
'day': day,
'date': date,
'time': time,
'AtFav': AtFav,
'favorite': favorite,
'score': score,
'spread': spread,
'AtDog': AtDog,
'underdog': underdog,
'OvUn': OvUn,
'notes': notes,
'week' : week
}
yield oddsTable
I don’t see any
table[@class="soh1"]
in the page. Either your xpath is wrong or you’re trying to scrape the wrong page.