add homework5

hh_parse/items.py

@@ -0,0 +1,27 @@
+# Define here the models for your scraped items
+#
+# See documentation in:
+# https://docs.scrapy.org/en/latest/topics/items.html
+
+import scrapy
+
+
+class VacancyItem(scrapy.Item):
+    # define the fields for your item here like:
+    # name = scrapy.Field()
+    url = scrapy.Field()
+    title = scrapy.Field()
+    salary = scrapy.Field()
+    description = scrapy.Field()
+    skills = scrapy.Field()
+    company_url = scrapy.Field()
+
+
+class CompanyItem(scrapy.Item):
+    # define the fields for your item here like:
+    # name = scrapy.Field()
+    url = scrapy.Field()
+    title = scrapy.Field()
+    site = scrapy.Field()
+    area_of_activity = scrapy.Field()
+    description = scrapy.Field()

hh_parse/loaders.py

@@ -0,0 +1,44 @@
+# from scrapy import Selector
+from scrapy.loader import ItemLoader
+from itemloaders.processors import TakeFirst, MapCompose, Join
+from urllib.parse import urljoin
+
+
+def clear_xa0(itm):
+    return itm.replace("\xa0", " ")
+
+
+def space_delete_descr(descr):
+    if descr == ' ':
+        return ''
+    else:
+        descr += ' '
+        return descr
+
+
+def get_url(itm):
+    return urljoin("https://hh.ru/", itm)
+
+
+class VacancyLoader(ItemLoader):
+    default_item_class = dict
+    url_out = TakeFirst()
+    title_in = MapCompose(clear_xa0, space_delete_descr)
+    title_out = Join('')
+    salary_in = MapCompose(clear_xa0)
+    salary_out = Join('')
+    description_in = MapCompose(clear_xa0, space_delete_descr)
+    description_out = Join('')
+    skills_in = MapCompose(clear_xa0)
+    company_url_in = MapCompose(get_url)
+    company_url_out = TakeFirst()
+
+
+class CompanyLoader(ItemLoader):
+    default_item_class = dict
+    url_out = TakeFirst()
+    title_in = MapCompose(clear_xa0, space_delete_descr)
+    title_out = Join('')
+    site_out = TakeFirst()
+    description_in = MapCompose(clear_xa0, space_delete_descr)
+    description_out = Join('')

hh_parse/middlewares.py

@@ -0,0 +1,103 @@
+# Define here the models for your spider middleware
+#
+# See documentation in:
+# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+
+from scrapy import signals
+
+# useful for handling different item types with a single interface
+from itemadapter import is_item, ItemAdapter
+
+
+class HhParseSpiderMiddleware:
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the spider middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_spider_input(self, response, spider):
+        # Called for each response that goes through the spider
+        # middleware and into the spider.
+
+        # Should return None or raise an exception.
+        return None
+
+    def process_spider_output(self, response, result, spider):
+        # Called with the results returned from the Spider, after
+        # it has processed the response.
+
+        # Must return an iterable of Request, or item objects.
+        for i in result:
+            yield i
+
+    def process_spider_exception(self, response, exception, spider):
+        # Called when a spider or process_spider_input() method
+        # (from other spider middleware) raises an exception.
+
+        # Should return either None or an iterable of Request or item objects.
+        pass
+
+    def process_start_requests(self, start_requests, spider):
+        # Called with the start requests of the spider, and works
+        # similarly to the process_spider_output() method, except
+        # that it doesn’t have a response associated.
+
+        # Must return only requests (not items).
+        for r in start_requests:
+            yield r
+
+    def spider_opened(self, spider):
+        spider.logger.info('Spider opened: %s' % spider.name)
+
+
+class HhParseDownloaderMiddleware:
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the downloader middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_request(self, request, spider):
+        # Called for each request that goes through the downloader
+        # middleware.
+
+        # Must either:
+        # - return None: continue processing this request
+        # - or return a Response object
+        # - or return a Request object
+        # - or raise IgnoreRequest: process_exception() methods of
+        #   installed downloader middleware will be called
+        return None
+
+    def process_response(self, request, response, spider):
+        # Called with the response returned from the downloader.
+
+        # Must either;
+        # - return a Response object
+        # - return a Request object
+        # - or raise IgnoreRequest
+        return response
+
+    def process_exception(self, request, exception, spider):
+        # Called when a download handler or a process_request()
+        # (from other downloader middleware) raises an exception.
+
+        # Must either:
+        # - return None: continue processing this exception
+        # - return a Response object: stops process_exception() chain
+        # - return a Request object: stops process_exception() chain
+        pass
+
+    def spider_opened(self, spider):
+        spider.logger.info('Spider opened: %s' % spider.name)

hh_parse/pipelines.py

@@ -0,0 +1,20 @@
+# Define your item pipelines here
+#
+# Don't forget to add your pipeline to the ITEM_PIPELINES setting
+# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
+
+
+# useful for handling different item types with a single interface
+from itemadapter import ItemAdapter
+from .settings import BOT_NAME
+from pymongo import MongoClient
+
+
+class HhParsePipeline:
+    def __init__(self):
+        client = MongoClient()
+        self.db = client[BOT_NAME]
+
+    def process_item(self, item, spider):
+        self.db[spider.name + type(item).__name__].insert_one(ItemAdapter(item).asdict())
+        return item

hh_parse/settings.py

@@ -0,0 +1,90 @@
+# Scrapy settings for hh_parse project
+#
+# For simplicity, this file contains only settings considered important or
+# commonly used. You can find more settings consulting the documentation:
+#
+#     https://docs.scrapy.org/en/latest/topics/settings.html
+#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
+#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+
+BOT_NAME = 'hh_parse'
+
+SPIDER_MODULES = ['hh_parse.spiders']
+NEWSPIDER_MODULE = 'hh_parse.spiders'
+
+LOG_ENABLE = True
+LOG_LEVEL = "DEBUG"
+
+# Crawl responsibly by identifying yourself (and your website) on the user-agent
+USER_AGENT = "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0"
+
+# Obey robots.txt rules
+ROBOTSTXT_OBEY = False
+
+# Configure maximum concurrent requests performed by Scrapy (default: 16)
+#CONCURRENT_REQUESTS = 32
+
+# Configure a delay for requests for the same website (default: 0)
+# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
+# See also autothrottle settings and docs
+DOWNLOAD_DELAY = 1.2
+# The download delay setting will honor only one of:
+#CONCURRENT_REQUESTS_PER_DOMAIN = 16
+#CONCURRENT_REQUESTS_PER_IP = 16
+
+# Disable cookies (enabled by default)
+COOKIES_ENABLED = True
+
+# Disable Telnet Console (enabled by default)
+TELNETCONSOLE_ENABLED = False
+
+# Override the default request headers:
+DEFAULT_REQUEST_HEADERS = {
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Accept-Language": "ru-RU,ru;q=0.8,en-US;q=0.5,en;q=0.3",
+}
+
+# Enable or disable spider middlewares
+# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+#SPIDER_MIDDLEWARES = {
+#    'hh_parse.middlewares.HhParseSpiderMiddleware': 543,
+#}
+
+# Enable or disable downloader middlewares
+# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
+#DOWNLOADER_MIDDLEWARES = {
+#    'hh_parse.middlewares.HhParseDownloaderMiddleware': 543,
+#}
+
+# Enable or disable extensions
+# See https://docs.scrapy.org/en/latest/topics/extensions.html
+#EXTENSIONS = {
+#    'scrapy.extensions.telnet.TelnetConsole': None,
+#}
+
+# Configure item pipelines
+# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
+ITEM_PIPELINES = {
+   'hh_parse.pipelines.HhParsePipeline': 300,
+}
+
+# Enable and configure the AutoThrottle extension (disabled by default)
+# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
+#AUTOTHROTTLE_ENABLED = True
+# The initial download delay
+#AUTOTHROTTLE_START_DELAY = 5
+# The maximum download delay to be set in case of high latencies
+#AUTOTHROTTLE_MAX_DELAY = 60
+# The average number of requests Scrapy should be sending in parallel to
+# each remote server
+#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
+# Enable showing throttling stats for every response received:
+#AUTOTHROTTLE_DEBUG = False
+
+# Enable and configure HTTP caching (disabled by default)
+# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
+#HTTPCACHE_ENABLED = True
+#HTTPCACHE_EXPIRATION_SECS = 0
+#HTTPCACHE_DIR = 'httpcache'
+#HTTPCACHE_IGNORE_HTTP_CODES = []
+#HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'

hh_parse/spiders/__init__.py

@@ -0,0 +1,4 @@
+# This package will contain the spiders of your Scrapy project
+#
+# Please refer to the documentation for information on how to create and manage
+# your spiders.

hh_parse/spiders/headhunter.py

@@ -0,0 +1,66 @@
+import scrapy
+from ..loaders import VacancyLoader, CompanyLoader
+from ..items import VacancyItem, CompanyItem
+
+
+class HeadhunterSpider(scrapy.Spider):
+    name = 'headhunter'
+    allowed_domains = ['hh.ru']
+    start_urls = ['https://hh.ru/search/vacancy?schedule=remote&L_profession_id=0&area=113']
+
+    _xpath_selectors = {
+        "vacancy_pagination": "//div[@data-qa='pager-block']//a[@data-qa='pager-page']/@href",
+        "vacancy_url": "//div[@class='vacancy-serp']//a[@data-qa='vacancy-serp__vacancy-title']/@href",
+        "company_url": "//div[@data-qa='vacancy-company']//a[@data-qa='vacancy-company-name']/@href",
+        "company_vacancies": '//div[@class="employer-sidebar-block"]'
+                             '/a[@data-qa="employer-page__employer-vacancies-link"]/@href',
+    }
+
+    _xpath_data_vacancy_selectors = {
+        "title": '//div[@class="vacancy-title"]/h1[@data-qa="vacancy-title"]/text()',
+        "salary": '//div[@class="vacancy-title"]//p[@class="vacancy-salary"]/span/text()',
+        "description": '//div[@data-qa="vacancy-description"]//text()',
+        "skills": '//div[@class="vacancy-section"]//div[@data-qa="bloko-tag bloko-tag_inline skills-element"]//text()',
+        "company_url": "//div[@data-qa='vacancy-company']//a[@data-qa='vacancy-company-name']/@href",
+    }
+
+    _xpath_data_company_selectors = {
+        "title": '//div[@class="company-header"]//span[@data-qa="company-header-title-name"]/text()',
+        "site": '//div[@class="employer-sidebar-content"]/a[@data-qa="sidebar-company-site"]/@href',
+        "area_of_activity": '//div[@class="employer-sidebar-block"]/p/text()',
+        "description": '//div[@data-qa="company-description-text"]/div[@class="g-user-content"]//text()',
+    }
+
+    def _get_follow(self, response, selector_str, callback):
+        for itm in response.xpath(selector_str):
+            yield response.follow(itm, callback=callback)
+
+    def parse(self, response, *args, **kwargs):
+        yield from self._get_follow(
+            response, self._xpath_selectors["vacancy_pagination"], self.parse
+        )
+        yield from self._get_follow(
+            response, self._xpath_selectors["vacancy_url"], self.vacancy_parse
+        )
+
+    def vacancy_parse(self, response):
+        item = VacancyItem()
+        vacancy_loader = VacancyLoader(response=response, item=item)
+        vacancy_loader.add_value("url", response.url)
+        for key, xpath in self._xpath_data_vacancy_selectors.items():
+            vacancy_loader.add_xpath(key, xpath)
+        yield from self._get_follow(
+            response, self._xpath_selectors["company_url"], self.company_parse
+        )
+        yield vacancy_loader.load_item()
+
+    def company_parse(self, response):
+        item = CompanyItem()
+        company_loader = CompanyLoader(response=response, item=item)
+        company_loader.add_value("url", response.url)
+        for key, xpath in self._xpath_data_company_selectors.items():
+            company_loader.add_xpath(key, xpath)
+        yield from self._get_follow(
+            response, self._xpath_selectors["company_vacancies"], self.parse
+        )
+        yield company_loader.load_item()

main.py

@@ -0,0 +1,11 @@
+from scrapy.crawler import CrawlerProcess
+from scrapy.settings import Settings
+from hh_parse.spiders.headhunter import HeadhunterSpider
+
+
+if __name__ == "__main__":
+    crawler_settings = Settings()
+    crawler_settings.setmodule("hh_parse.settings")
+    crawler_process = CrawlerProcess(settings=crawler_settings)
+    crawler_process.crawl(HeadhunterSpider)
+    crawler_process.start()

scrapy.cfg

@@ -0,0 +1,11 @@
+# Automatically created by: scrapy startproject
+#
+# For more information about the [deploy] section see:
+# https://scrapyd.readthedocs.io/en/latest/deploy.html
+
+[settings]
+default = hh_parse.settings
+
+[deploy]
+#url = http://localhost:6800/
+project = hh_parse