parse.py

import re
import sys

def parse_stock(stock_name, year):
    # input AAPL 2017, open Tickers/AAPL/AAPL10k2017
    sections = ['1A']
    filings = ['_10Q_','_10K_']
    for section in sections:
        for filing in filings :
            f = open(stock_name + '/'+ stock_name + filing + year + '.txt', 'r')
            # write to Tickers/AAPL/AAPL10k1A2017
            g = open(stock_name + '/'+ stock_name + filing + section + '_' + year + '.txt', 'w')

            # get string of 10K/10Q html
            s = f.read()
            # remove newlines
            s = re.sub('\n', ' ', s)
            # remove   special chars
            s = re.sub(r' ', ' ', s)
            # find all Item tags, replace with >!!!!! for easy location later
            s = re.sub(r'(?s)(?i)(?m)> +Item|>Item|^Item', '>!!!!!', s)
            # remove other special chars
            s = re.sub(r'&[#a-zA-Z0-9]*;', ' ', s)
            # remove HTML divs/tables/other elements
            s = re.sub(r'<[a-zA-Z0-9=:;\"\'-\.\s_#?%/]*>', ' ', s)
            # remove unnecessary whitespace
            s = re.sub(r'\s+', ' ', s)
            # break into sections, by !!!!!
            t = s.split('!!!!!')

        # find sections and write to new file
        for u in t:
            if (u[: 3] == ' ' + section):
                g.write(u)

def parse_stock_multiple_years(stock_name, years):
    for year in years:
        try:
            parse_stock(stock_name, year)
        except:
            pass

if __name__ == "__main__":
    parse_stock(sys.argv[1], sys.argv[2])