webscraper/scraper.py at master · LHolmberg/webscraper · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
from selenium import webdriver
from bs4 import BeautifulSoup
import json

#Variable setup
SCHOOL = "school"
USERNAME = "username"
PASSWORD = "password"
MAIN_URL = "https://sms.schoolsoft.se/{}/sso".format(SCHOOL)
###############################################################

#Logging in to the website
driver = webdriver.PhantomJS()
driver.get(MAIN_URL)

btn = driver.find_elements_by_xpath("//*[@id='username']")[0]
btn.send_keys(USERNAME)
btn = driver.find_elements_by_xpath("//*[@id='password']")[0]
btn.send_keys(PASSWORD)
btn = driver.find_elements_by_xpath("//*[@class='form__button form__button--primary']")[0]
btn.click()

#Fetch schedule
driver.get("https://sms.schoolsoft.se/{}/jsp/student/right_student_schedule.jsp?menu=schedule".format(SCHOOL))
html = driver.page_source
schedule = BeautifulSoup(html)

theSchedule = []

for a in schedule.find_all("a", {"class": "schedule"}):
    info = a.find("span")
    i = info.get_text()
    theSchedule.append(i)

#Fetch messages
driver.get("https://sms.schoolsoft.se/{}/jsp/student/right_student_message.jsp?menu=message".format(SCHOOL))
html = driver.page_source
msg = BeautifulSoup(html)

messages = []
senders = []

for a in msg.find_all("div", {"class": "accordion_inner_left"}):
    info = a.find("span")
    i = info.get_text()
    messages.append(i)

for a in msg.find_all("div", {"class": "accordion_inner_right"}):
    info = a.find("span")
    i = info.get_text()
    senders.append(i)

#Parse to a json file
data = open('a.json').read()
count = data.count('message')
count2 = data.count('schema')

n = 0
i = 0
if messages.__len__() > count:
    with open('a.json', 'w') as f:
        while i < messages.__len__():
            if theSchedule.__len__() > count2 and n < theSchedule.__len__():
                json.dump({
                'schema' : theSchedule[n]},
                f, sort_keys=True, indent=4,
                ensure_ascii=False)
            n += 1
            if messages.__len__() > count:
                json.dump({
                'sender' : senders[i],
                'message' : messages[i]},
                f, sort_keys=True, indent=4,
                ensure_ascii=False)
            i += 1