BeautifulSoup/parser.py at main · Razorlance/BeautifulSoup · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
import requests
import json
from tqdm import tqdm
from bs4 import BeautifulSoup

habr_link = "https://career.habr.com/vacancies?type=all&page="

with open('vacancies.json', 'r', encoding="utf8") as f:
    vacancies_data = json.load(f)


def main():
    for page in range(1, 11):
        vacancies = requests.get(habr_link + str(page))
        sp = BeautifulSoup(vacancies.content, "html.parser")
        all_cards = sp.find_all("div", class_="vacancy-card__info")
        for card in tqdm(all_cards, "parsing cards..."):
            company = card.find("div", class_="vacancy-card__company-title")
            vacancy_name = card.find("div", class_="vacancy-card__title")
            vacancy_additions = card.find("div", class_="vacancy-card__meta")
            vacancy_skills = card.find("div", class_="vacancy-card__skills")
            link = card.find("div", class_="vacancy-card__title")
            vacancy_link = "https://career.habr.com/" + link.a["href"]
            vacancy_page = requests.get(vacancy_link)
            vacancy_content = BeautifulSoup(vacancy_page.content, "html.parser")
            vacancy_description = vacancy_content.find("div", class_="style-ugc").extract()

            vacancies_data["list"][link.a["href"]] = {}
            vacancies_data["list"][link.a["href"]]["company_name"] = company.a.string
            vacancies_data["list"][link.a["href"]]["vacancy_name"] = vacancy_name.a.string
            vacancies_data["list"][link.a["href"]]["vacancy_additions"] = vacancy_additions.get_text()
            vacancies_data["list"][link.a["href"]]["vacancy_skills"] = vacancy_skills.get_text()
            vacancies_data["list"][link.a["href"]]["vacancy_description"] = vacancy_description.get_text()

    with open('vacancies.json', 'w', encoding="utf8") as f:
        json.dump(vacancies_data, f)


if __name__ == '__main__':
    main()