RTV Failing #3

Stat-Boy · 2024-01-03T13:39:27Z

Stat-Boy
Jan 3, 2024

Hi @Caprice63, I too have been learning Python and am still very much a novice. I found your fork of rpscrape very useful, particularly as it pulled in the odds from RTV... For some reason that has stopped working and I can't figure out why, do you know what is causing the error?

Thanks

Andy

Caprice63 · 2024-01-03T15:09:12Z

Caprice63
Jan 3, 2024
Maintainer

Hi Andy,
They changed the site to Java.
This is what I use now. I can't remember where I copied most of it from but it seems to work.
All the best,
Chris

def get_ratings():

odds.scrape(odds_date = '/tomorrow')

odds.scrape(odds_date = '')

`import os
import requests
from bs4 import BeautifulSoup
import csv
from selenium import webdriver
from selenium.webdriver.edge.options import Options
#import logging
#logging.basicConfig(level=logging.DEBUG)

from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

def scrape(odds_date):

url = 'https://www.racingtv.com/racecards/runners'

url = 'https://www.racingtv.com/racecards/runners/tomorrow'

url = 'https://www.racingtv.com/racecards/runners' + odds_date

# Specify the path to the Microsoft Edge WebDriver executable
edge_driver_path = r'C:\DRIVERS\msedgedriver.exe'

# Create EdgeOptions and set the path to the WebDriver executable
edge_options = Options()
edge_options.binary_location = edge_driver_path

driver = webdriver.Edge()

# Navigate to the URL
driver.get(url)

select_y()

# Wait for the page to load (adjust the sleep time if needed)

driver.implicitly_wait(500)

# Get the page source after JavaScript execution
page_source = driver.page_source

# Parse the page source with BeautifulSoup
soup = BeautifulSoup(page_source, 'html.parser')
    

# Find all div elements with the specified class and an <a> element with the desired href pattern
div_elements = soup.find_all('div', class_='css-175oi2r r-5fd7l6 r-5kkj8d r-18u37iz r-hrzydr r-1a1dyw r-1sp51qo r-13qz1uu')

filtered_divs = [div for div in div_elements if div.find('a', href=lambda x: x and x.startswith('/profiles/horse/'))]

# Extract and save specific information to a CSV file
csv_file_path = r'C:\Users\chris\Documents\UKHR\Python\Racecards\RacecardOdds.csv'

with open(csv_file_path, 'w', newline='', encoding='utf-8') as csv_file:
    csv_writer = csv.writer(csv_file)
    csv_writer.writerow(['Horse Name', 'Odds'])

    # Extract and print specific information from each filtered div
    for div in filtered_divs:
        # Extract horse name
        horse_name_element = div.find('a', href=lambda x: x and x.startswith('/profiles/horse/'))
        horse_name = horse_name_element.find('div', class_='css-1rynq56').get_text(strip=True)

        # Extract odds if available
        odds_element = div.find('div', class_='css-1rynq56', style='font-family: TitilliumWeb_600SemiBold; color: rgb(29, 28, 57); font-size: 16px;')
        odds = odds_element.get_text(strip=True) if odds_element else 'N/A'

        # If odds_element is not present, try to get odds from the alternative element
        if odds == 'N/A':
            alt_odds_element = div.find('div', class_='css-1rynq56', string=lambda x: x and 'SP:' in x)
            alt_odds = alt_odds_element.get_text(strip=True).split('SP: ')[1] if alt_odds_element else 'N/A'
            odds = alt_odds

        # Write the data to the CSV file
        csv_writer.writerow([horse_name, odds])

print(f'Data has been saved to: {csv_file_path}')        

return

def select_y():
# import datetime
# import os
# import shutil

os.system('cls')
print()
print()

user_input = input("Enter 'Y' when RTV webpage loaded.   ")

# You can use a loop to wait until the user enters 'Y'
while user_input.upper() != 'Y':
    user_input = input("Please enter 'Y' to continue: ")

return

def scrape1():

url = 'https://www.racingtv.com/racecards/runners'

url = 'https://www.racingtv.com/racecards/runners/tomorrow'

# Specify the path to the Microsoft Edge WebDriver executable
#edge_driver_path = 'C:\DRIVERS/msedgedriver.exe'  # Replace with the actual path  C:\DRIVERS
edge_driver_path = r'C:\DRIVERS\msedgedriver.exe'


# # configure webdriver
# edge_options = Options()
# edge_options.headless = True  # hide GUI
# edge_options.add_argument("--window-size=1920,1080")  # set window size to native GUI size
# edge_options.add_argument("start-maximized")  # ensure window is full-screen


# Create EdgeOptions and set the path to the WebDriver executable
edge_options = Options()
edge_options.binary_location = edge_driver_path

# Create a Microsoft Edge WebDriver instance

driver = webdriver.Edge(options=edge_options)

driver = webdriver.Edge()

# Navigate to the URL
driver.get(url)

# Wait for the page to load (adjust the sleep time if needed)
driver.implicitly_wait(500)

# Get the page source after JavaScript execution
page_source = driver.page_source

# Close the web driver

driver.quit()

# Parse the page source with BeautifulSoup
soup = BeautifulSoup(page_source, 'html.parser')
    

# Find all div elements with the specified class and an <a> element with the desired href pattern
div_elements = soup.find_all('div', class_='css-175oi2r r-5fd7l6 r-5kkj8d r-18u37iz r-hrzydr r-1a1dyw r-1sp51qo r-13qz1uu')



filtered_divs = [div for div in div_elements if div.find('a', href=lambda x: x and x.startswith('/profiles/horse/'))]

# Extract and save specific information to a CSV file

csv_file_path = r'C:\Users\chris\Documents\UKHR\Python\Racecards\RacecardOdds.csv'

csv_file_path = r'C:\Users\chris\Documents\UKHR\Python\Racecards\RacecardOdds_Tom.csv'

with open(csv_file_path, 'w', newline='', encoding='utf-8') as csv_file:
    csv_writer = csv.writer(csv_file)
    csv_writer.writerow(['Horse Name', 'Odds'])

    # Extract and print specific information from each filtered div
    for div in filtered_divs:
        # Extract horse name
        horse_name_element = div.find('a', href=lambda x: x and x.startswith('/profiles/horse/'))
        horse_name = horse_name_element.find('div', class_='css-1rynq56').get_text(strip=True)

        # Extract odds if available
        odds_element = div.find('div', class_='css-1rynq56', style='font-family: TitilliumWeb_600SemiBold; color: rgb(29, 28, 57); font-size: 16px;')
        odds = odds_element.get_text(strip=True) if odds_element else 'N/A'

        # If odds_element is not present, try to get odds from the alternative element
        if odds == 'N/A':
            alt_odds_element = div.find('div', class_='css-1rynq56', string=lambda x: x and 'SP:' in x)
            alt_odds = alt_odds_element.get_text(strip=True).split('SP: ')[1] if alt_odds_element else 'N/A'
            odds = alt_odds

        # # Extract additional information
        # additional_info = div.find('div', class_='css-175oi2r', style='min-width: 230px; flex-direction: row;')
        # additional_info_text = additional_info.get_text(strip=True)

        # # Extract information before the first ':' in additional_info_text
        # info_before_colon = additional_info_text.split(':')[0].strip()


        # Write the data to the CSV file
        csv_writer.writerow([horse_name, odds])


print(f'Data has been saved to: {csv_file_path}')

  #  print(f"{horse_name}, {odds}") # {info_before_colon}, {odds}")

    
# Extract and print specific information from each filtered div
for div in filtered_divs:
    # Extract horse name
    horse_name_element = div.find('a', href=lambda x: x and x.startswith('/profiles/horse/'))
    horse_name = horse_name_element.find('div', class_='css-1rynq56').get_text(strip=True)

        
    # Extract odds if available
    odds_element = div.find('div', class_='css-1rynq56', style='font-family: TitilliumWeb_600SemiBold; color: rgb(29, 28, 57); font-size: 16px;')
    odds = odds_element.get_text(strip=True) if odds_element else 'N/A'

    # Extract additional information
    additional_info = div.find('div', class_='css-175oi2r', style='min-width: 230px; flex-direction: row;')
    additional_info_text = additional_info.get_text(strip=True)

    # Extract information before the first ':' in additional_info_text
    info_before_colon = additional_info_text.split(':')[0].strip()

    print(f"{horse_name}, {info_before_colon}, {odds}")


    




# Find all div elements with the specified class and an <a> element with the desired href pattern
filtered_divs = [div for div in div_elements if div.find('a', href=lambda x: x and x.startswith('/profiles/horse/'))]

# Extract and print text content from each filtered div
for div in filtered_divs:
    text_content = div.get_text(strip=True)
    print(text_content)




# Extract and print horse names and odds from each filtered div
for div in div_elements:
    horse_name_element = div.find('a', href=lambda x: x and x.startswith('/profiles/horse/'))
    odds_element = div.find('a', href=lambda x: x and x.startswith('/racecards/'))

odds_element = div.find('div', class_='css-1rynq56', style='font-family: TitilliumWeb_600SemiBold; color: rgb(29, 28, 57); font-size: 16px;')

    if horse_name_element and odds_element:
        horse_name = horse_name_element.find('div', class_='css-1rynq56').get_text(strip=True)
        odds = odds_element.get_text(strip=True)
        print(f"Horse Name: {horse_name}, Odds: {odds}")


        
# Extract and print horse names from each filtered div
for div in div_elements:
    horse_name_element = div.find('a', href=lambda x: x and x.startswith('/profiles/horse/'))
    if horse_name_element:
        horse_name = horse_name_element.find('div', class_='css-1rynq56').get_text(strip=True)
        print(horse_name)
        
            
# Find all div elements with the specified class
div_elements = soup.find_all('div', class_='css-175oi2r r-5fd7l6 r-5kkj8d r-18u37iz r-hrzydr r-1a1dyw r-1sp51qo r-13qz1uu')

# Extract and print text content from each div
for div in div_elements:
    text_content = div.get_text(strip=True)
    print(text_content)



# # Find all text elements on the page
# text_elements = soup.find_all(text=True)

# # Print the text content
# for text in text_elements:
#     print(text.strip())


# Close the web driver
driver.quit()





race_cnt = 0
    
headersrtv =   {
        'accept':'*/*',
        'user-agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36',
        }


url = 'https://www.racingtv.com/racecards/runners'
response = requests.get(url)


if response.status_code == 200:
    soup = BeautifulSoup(response.text, 'html.parser')

    # Find all text elements on the page
    text_elements = soup.find_all(text=True)

    # Print the text content
    for text in text_elements:
        print(text.strip())
else:
    print(f"Failed to retrieve the webpage. Status code: {response.status_code}")




if response.status_code == 200:
    soup = BeautifulSoup(response.text, 'html.parser')

    # Find the first div with the specified class
    first_div = soup.find('div', class_='css-175oi2r r-5fd7l6 r-5kkj8d r-18u37iz r-hrzydr r-1a1dyw r-1sp51qo r-13qz1uu')

    if first_div:
        # Find the nested 'a' element
        a_element = first_div.find('a', class_='css-175oi2r')

        if a_element:
            # Extract the horse name from the 'a' element
            horse_name = a_element.find('div', class_='css-1rynq56').get_text(strip=True)

            print(f"Horse Name: {horse_name}")
        else:
            print("No 'a' element found within the specified div.")
    else:
        print("No div with the specified class found.")
else:
    print(f"Failed to retrieve the webpage. Status code: {response.status_code}")




if response.status_code == 200:
    soup = BeautifulSoup(response.text, 'html.parser')

    with open('race_data.csv', 'w', newline='', encoding='utf-8') as csvfile:
        fieldnames = ['horse', 'odds']
        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
        writer.writeheader()

        divs = soup.find_all('div', class_='css-175oi2r r-5fd7l6 r-5kkj8d r-18u37iz r-hrzydr r-1a1dyw r-1sp51qo r-13qz1uu')
        for div in divs:
            horse_element = div.find('div', class_='css-1rynq56', style='font-family: TitilliumWeb_600SemiBold; color: rgb(225, 8, 126); font-size: 16px;')
            odds_element = div.find('div', class_='css-1rynq56', style='font-family: TitilliumWeb_600SemiBold; color: rgb(29, 28, 57); font-size: 16px;')

            if horse_element and odds_element:
                horse = horse_element.get_text(strip=True)
                odds = odds_element.get_text(strip=True)
                writer.writerow({'horse': horse, 'odds': odds})
else:
    print(f"Failed to retrieve the webpage. Status code: {response.status_code}")






# url = f'https://www.racingtv.com/racecards/runners'  #https://www.racingtv.com/racecards/runners
# res = requests.get(url, headers = headersrtv)
# # The HTML code you provided
# html_code = '<div class="css-175oi2r r-5fd7l6 r-5kkj8d r-18u37iz r-hrzydr r-1a1dyw r-1sp51qo r-13qz1uu"><div class="css-175oi2r" style="flex: 1 1 0%; justify-content: center;"><div class="css-175oi2r" style="flex-direction: row;"><a href="/profiles/horse/absolute-queen" role="link" class="css-175oi2r" style="flex-shrink: 1;"><div dir="auto" class="css-1rynq56" style="font-family: TitilliumWeb_600SemiBold; color: rgb(225, 8, 126); font-size: 16px;">Absolute Queen</div></a></div><div class="css-175oi2r" style="flex-direction: row;"><div class="css-175oi2r" style="min-width: 230px; flex-direction: row;"><a href="/profiles/jockey/billy-loughnane" role="link" class="css-175oi2r" style="flex-direction: row; flex-shrink: 1;"><div dir="auto" class="css-1rynq56" style="font-family: TitilliumWeb_400Regular; color: rgb(225, 8, 126); font-size: 16px;">J: </div><div dir="auto" class="css-1rynq56" style="font-family: TitilliumWeb_400Regular; color: rgb(29, 28, 57); font-size: 16px;">Billy Loughnane</div></a><div class="css-175oi2r" style="flex: 1 1 0%; width: 100%;"></div></div><div class="css-175oi2r r-1niwhzg" style="height: 5px;"></div><div class="css-175oi2r" style="flex: 1 1 0%; flex-direction: row;"><a href="/profiles/trainer/george-boughey" role="link" class="css-175oi2r" style="flex-direction: row; flex-shrink: 1;"><div dir="auto" class="css-1rynq56" style="font-family: TitilliumWeb_400Regular; color: rgb(225, 8, 126); font-size: 16px;">T: </div><div dir="auto" class="css-1rynq56" style="font-family: TitilliumWeb_400Regular; color: rgb(29, 28, 57); font-size: 16px;">George Boughey</div></a><div class="css-175oi2r" style="flex: 1 1 0%; width: 100%;"></div></div></div></div><div class="css-175oi2r" style="flex-direction: row; align-items: center; justify-content: flex-end;"><a href="/racecards/2023-12-06/kempton-park/2030" role="link" class="css-175oi2r"><div dir="auto" class="css-1rynq56" style="font-family: TitilliumWeb_600SemiBold; color: rgb(225, 8, 126); font-size: 16px;">20:30 Kempton Park</div></a><div class="css-175oi2r r-1niwhzg" style="width: 20px;"></div><div class="css-175oi2r" style="min-width: 100px; align-items: flex-end;"><div tabindex="0" class="css-175oi2r r-1loqt21 r-1otgn73 r-1kihuf0 r-6t5ypu r-kicko2 r-18u37iz r-17s6mgv r-1ye3whp r-ytfskt" style="width: 32px; height: 32px;"><div class="css-175oi2r r-1xgcvxh r-6t5ypu r-kicko2" style="width: 32px; height: 32px;"><img loading="lazy" src="https://api.racingtv.com/onspace/media/assets/eyJfcmFpbHMiOnsibWVzc2FnZSI6Ild6RTBNaXd4WFE9PSIsImV4cCI6bnVsbCwicHVyIjoib25zcGFjZV9tZWRpYV9hc3NldF9pZCJ9fQ==--e72a6b31b3f1c885c81790e95552837fd087e2dd8452f0657ccc6182b90ee29d/bookie-square-betfair" style="border-top-left-radius: 4px; border-bottom-left-radius: 4px; object-fit: contain; width: 100%; height: 100%;"></div><div class="css-175oi2r r-1awozwy r-1dpl46z r-82t2op r-1phboty r-notknq r-rs99b7 r-18u37iz r-1pi2tsx r-1777fci r-gxnn5r" style="width: 48px;"><div dir="auto" class="css-1rynq56" style="font-family: TitilliumWeb_600SemiBold; color: rgb(29, 28, 57); font-size: 16px;">11/2</div></div></div></div></div></div>'

# # Parse the HTML code using BeautifulSoup
# soup = BeautifulSoup(html_code, 'html.parser')

# Find the element with the class that contains the odds
odds_element = soup.find('div', class_='css-175oi2r r-1awozwy r-1dpl46z r-82t2op r-1phboty r-notknq r-rs99b7 r-18u37iz r-1pi2tsx r-1777fci r-gxnn5r')

# Extract the text from the found element
odds_text = odds_element.get_text(strip=True)

print(odds_text)



url = f'https://www.racingtv.com/racecards/runners'  #https://www.racingtv.com/racecards/runners
res = requests.get(url, headers = headersrtv)
soup = BeautifulSoup(res.text, 'html.parser')
meetings = soup.select('div.css-175oi2r.r-1awozwy.r-1dpl46z.r-82t2op.r-1phboty.r-notknq.r-rs99b7.r-18u37iz.r-1pi2tsx.r-1777fci.r-gxnn5r')

meetings1 = [a['href'] for a in soup.select('div.css-175oi2r.r-1awozwy.r-1dpl46z.r-82t2op.r-1phboty.r-notknq.r-rs99b7.r-18u37iz.r-1pi2tsx.r-1777fci.r-gxnn5r')]
course_num = len(meetings1)
if race_cnt == 0:
    course_num = len(meetings1)
else:
    course_num = race_cnt

cnt01 = 0
n = 0


df = pd.DataFrame(meetings1)

savepath = (r'C:\Users\chris\Documents\UKHR\Python\Racecards')
flat_meetings = os.path.join(savepath, 'rtv_urls'+'.csv')

df.to_csv(flat_meetings, index=False)

return

def main():

add_prize_totals()

scrape()

if name == 'main':
main()`

0 replies

Caprice63 · 2024-01-03T15:11:44Z

Caprice63
Jan 3, 2024
Maintainer

PS. you have to wait for the RTV page to load and then enter "Y" to proceed. I couldn't work out how to use python to test when the page loaded so I had to do it manually.

1 reply

Stat-Boy Jan 5, 2024
Author

Thanks a million, I haven't got around to trying it yet but will do soon.

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

RTV Failing #3

Uh oh!

{{title}}

Uh oh!

Replies: 2 comments 1 reply

Uh oh!

{{title}}

Uh oh!

Uh oh!

{{title}}

Uh oh!

Uh oh!

{{title}}

Uh oh!

Select a reply

Uh oh!

RTV Failing #3

Uh oh!

Stat-Boy Jan 3, 2024

Replies: 2 comments · 1 reply

Uh oh!

Caprice63 Jan 3, 2024 Maintainer

odds.scrape(odds_date = '')

url = 'https://www.racingtv.com/racecards/runners'

url = 'https://www.racingtv.com/racecards/runners/tomorrow'

driver.implicitly_wait(500)

url = 'https://www.racingtv.com/racecards/runners'

driver = webdriver.Edge(options=edge_options)

driver.quit()

csv_file_path = r'C:\Users\chris\Documents\UKHR\Python\Racecards\RacecardOdds.csv'

odds_element = div.find('div', class_='css-1rynq56', style='font-family: TitilliumWeb_600SemiBold; color: rgb(29, 28, 57); font-size: 16px;')

add_prize_totals()

Uh oh!

Caprice63 Jan 3, 2024 Maintainer

Uh oh!

Stat-Boy Jan 5, 2024 Author

Stat-Boy
Jan 3, 2024

Replies: 2 comments 1 reply

Caprice63
Jan 3, 2024
Maintainer

Caprice63
Jan 3, 2024
Maintainer

Stat-Boy Jan 5, 2024
Author