SE_Web/DocumentParser.py at master · ashish0707/SE_Web · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
# import os
# from bs4 import BeautifulSoup
#
#
# class Parser:
#
#     def __init__(self):
#         pass
#
#     def parse_document(self, file_name):
#
#         soup = BeautifulSoup(open(file_name, 'r'), 'html.parser')
#         body = soup.find("div", class_="mw-body-content")
#         mainPara = body.find('div', {"id":"mw-content-text"})
#         print mainPara.text
#         return mainPara.text
#
#     def get_documents(self, directory_name):
#         return os.listdir(directory_name)
#
#
#
#
#
import os
from bs4 import BeautifulSoup


class Parser:

    def __init__(self):
        pass

    def parse_document(self, file_name):

        soup = BeautifulSoup(open(file_name, 'r'), 'html.parser')

        for script in soup (['script']):
            script.extract()
        body = soup.find("div", class_="mw-body-content")
        array_of_para = body.find_all('div', {"id": "mw-content-text"})
        temp = ""
        for value in array_of_para:
            temp += value.text
        return temp


    def get_documents(self, directory_name):
        return os.listdir(directory_name)