Web-Scraper/app.py at master · reyha/Web-Scraper · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
from flask import Flask, render_template, request, redirect, url_for
from bs4 import BeautifulSoup as BS
import requests
from flask_pymongo import PyMongo
import urllib.parse
from scraping import scrape
from searchurl import search_url
from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.triggers.combining import OrTrigger
from apscheduler.triggers.cron import CronTrigger

app = Flask(__name__)

# DB config
app.config['MONGO_DBNAME'] = 'connect_to_mongo'
app.config['MONGO_URI'] = 'mongodb://{username}:' + urllib.parse.quote({password}) + '{db_url}/connect_to_mongo'
mongo = PyMongo(app)

@app.route('/', methods = ['GET','POST'])
def main():
    if request.method == 'POST':
        company_name = request.form ['cname']
        frequency = request.form ['freq']
        start_time = request.form ['stime']
        end_time = request.form ['etime']
        company_url = search_url(company_name)

        if company_url:
            c_url = requests.get(company_url)
            soup = BS(c_url.text, "html.parser")

            # Returns bse and nse contents if present
            b,n = scrape(soup)

            # Adding info to Database
            bse_db = mongo.db.bse
            nse_db = mongo.db.nse

            if b:
                bse_entry = bse_db.insert({'BSE Date': b['bse_date'], 'BSE Time': b['bse_time'], \
                            'BSE Current Price': b['bse_current_price'], 'BSE absolute price': b['bse_abs_price'],\
                            'BSE percentage': b['bse_per'], 'BSE Volume': b['bse_volume'], \
                            'BSE Prev close': b['bse_prev_close'], 'BSE Open price': b['bse_open_price'],\
                            'BSE bid price': b['bse_bid_price'], 'BSE offer price': b['bse_offer_price']})
            if n:
                nse_entry = nse_db.insert({'NSE Date': n['nse_date'], 'NSE Time':n['nse_time'], \
                           'NSE Current Price': n['nse_current_price'], 'NSE absolute price': n['nse_abs_price'], \
                           'NSE percentage': n['nse_per'], 'NSE Volume': n['nse_volume'], \
                           'NSE Prev close': n['nse_prev_close'], 'NSE Open price': n['nse_open_price'], \
                           'NSE bid price': n['nse_bid_price'], 'NSE offer price': n['nse_offer_price']})

            # Job scheduling
            if frequency and start_time and end_time:

                # Check to ensure start time is before end time
                if start_time < end_time:
                    trigger = OrTrigger([ CronTrigger(hour= start_time+ '-' + end_time, minute=frequency) ])
                    scheduler.add_job(main, trigger)
                else:
                    error = "End time should be after start time"
                    return render_template('index.html', error = error)

            if bse_db or nse_db:
                return redirect(url_for('info'))

        else:
            error = "Sorry! Company not found."
            return render_template('index.html', error = error)

    return render_template('index.html')

@app.route("/info")
def info():
     bse_entries = mongo.db.bse.find()
     nse_entries = mongo.db.nse.find()
     if bse_entries or nse_entries:
         return render_template('scraped_data.html', bse_entries=bse_entries, nse_entries=nse_entries)
     return render_template('index.html')

if __name__ == "__main__":

    # Initializing scheduler
    scheduler = BackgroundScheduler()
    scheduler.start()

    app.run(debug=True)