PinterestScraper/CSVHelper.py at master · Ethiventure/PinterestScraper · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
# Cullen, Riley
# CSVHelper.py
# Created on 5/30/20

# TODO update documentation, add CSV helper methods from PinterestScraper

import os, glob
import pandas as pd

def CreateMasterCSV(root, filename):
    rootContents = __GetSubDirectories(root)
    dataList = []
    for dir in rootContents:
        data = pd.read_csv(glob.glob(os.path.join(dir, "*.csv"))[0])
        dataList.append(data)

    if (DoesCSVExist(root, filename)):
        print("removed!")
        RemoveCSV(root, filename)

    masterCSV = pd.concat(dataList, axis=0, ignore_index=True)
    masterCSV.to_csv(root + "/" + filename, index=True)


def __GetSubDirectories(root):
    rootContents = os.listdir(root + "/.")
    results = []
    for elem in rootContents:
        elem = root + '/' + elem
        if (os.path.isdir(os.path.join(os.path.abspath("."), elem))):
            results.append(elem)
    return results

def DoesCSVExist(root, filename):
    if (os.path.isfile(root + "/" + filename)):
        return True
    return False

def RemoveCSV(root, filename):
    os.remove(root + "/" + filename)