From 5bf24629b76c32adb1b22c2373655d44ecae6d39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JDaniel=20Jim=C3=A9nez=20Paniagua?= Date: Tue, 21 Jun 2016 23:13:29 +0200 Subject: [PATCH 1/5] Added arg parse to control these vars from commandline: debug output, logfile location, path to pictures to upload, extension to mark files as uploaded and the path to the config file --- flickup.py | 132 +++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 98 insertions(+), 34 deletions(-) diff --git a/flickup.py b/flickup.py index 4737214..4ac268c 100644 --- a/flickup.py +++ b/flickup.py @@ -8,69 +8,133 @@ import datetime import sys import configparser +import argparse -import smtplib, os +import smtplib +import os from email.mime.multipart import MIMEMultipart from email.mime.base import MIMEBase from email.mime.text import MIMEText from email.utils import COMMASPACE, formatdate from email import encoders -config_file = "/etc/flickrup.conf" -config = configparser.ConfigParser() -try: - config.read(config_file) - api_key = config.get("main","api_key") - api_secret = config.get("main","api_secret") - api_token = config.get("main","api_token") - email_from = config.get("main","email_from") - email_to = config.get("main","email_to") -except: - print("Missing "+config_file) - sys.exit(0) +# Default vars +default_args_config = '/etc/flickrup.conf' +default_log_file = 'flickrup.log' +default_uploaded_extension = 'done' + +# Initializing argparser +parser = argparse.ArgumentParser(description='Flickrup: Flicker uploader.') + +# Enable debug +parser.add_argument('-d', + '--debug', + help='Debug output. Default: Disabled', + action='store_true' + ) + +# Path to config file +parser.add_argument('-c', + '--config', + help='Path to config file. Defaul: {}'.format( + default_args_config), + required=False, + default=default_args_config + ) + +# Path to dir with pictures +parser.add_argument('-p', + '--pictures', + help='Path to dir with pictures to upload', + required=False, + default='/etc/flickrup.conf' + ) + +# Path to log file +parser.add_argument('-l', + '--log-file', + help='Path to log file. Default: {}'.format( + default_log_file), + required=False, + default=default_log_file + ) + +# Extension to mark files as uploaded +parser.add_argument('-e', + '--uploaded-extension', + help='Extension to mark files as uploaded. Default: {}'.format( + default_uploaded_extension), + required=False, + default=default_uploaded_extension + ) + + +args = parser.parse_args() + +# Print resume +if args.debug: + print("Running with these config options:") + for arg in vars(args): + print("\t- {}: {}".format(arg, getattr(args, arg))) -done = ".done" -sdcard = sys.argv[1] + "/" +config = configparser.ConfigParser() +try: + config.read(args.config) + api_key = config.get("main", "api_key") + api_secret = config.get("main", "api_secret") + api_token = config.get("main", "api_token") + email_from = config.get("main", "email_from") + email_to = config.get("main", "email_to") +except: + print("Missing " + args.config) + sys.exit(0) -logfile = sdcard + "upload-log.txt" -includes = ['*.jpg'] # for files only / case insensitive -excludes = ["*"+done,logfile] # for dirs and files / case insensitive +includes = ['*.jpg'] # for files only / case insensitive +# for dirs and files / case insensitive +excludes = ["*." + args.uploaded_extension, args.log_file] flickr = flickrapi.FlickrAPI(api_key, api_secret, api_token) -def send_mail( send_from, send_to, subject, text, files=[], server="localhost", port=25, username='', password='', isTls=True): + +def send_mail(send_from, send_to, subject, text, files=[], server="localhost", port=25, username='', password='', isTls=True): msg = MIMEMultipart() msg['From'] = send_from msg['To'] = COMMASPACE.join(send_to) - msg['Date'] = formatdate(localtime = True) + msg['Date'] = formatdate(localtime=True) msg['Subject'] = subject - msg.attach( MIMEText(text) ) + msg.attach(MIMEText(text)) for f in files: part = MIMEBase('application', "octet-stream") - part.set_payload( open(f,"rb").read() ) + part.set_payload(open(f, "rb").read()) encoders.encode_base64(part) - part.add_header('Content-Disposition', 'attachment; filename="{0}"'.format(os.path.basename(f))) + part.add_header('Content-Disposition', + 'attachment; filename="{0}"'.format(os.path.basename(f))) msg.attach(part) smtp = smtplib.SMTP(server, port) - if isTls: smtp.starttls() - if len(username)>0 : smtp.login(username,password) + if isTls: + smtp.starttls() + if len(username) > 0: + smtp.login(username, password) smtp.sendmail(send_from, send_to, msg.as_string()) smtp.quit() -def logit( text ): - log.write(str(datetime.datetime.utcnow())+ " "+text+"\n") + +def logit(text): + log.write(str(datetime.datetime.utcnow()) + " " + text + "\n") + def cleanexit(): log.close() - send_mail( email_from, [email_to], "Photo Upload", "Photo upload log", [logfile]) + send_mail(email_from, [email_to], "Photo Upload", + "Photo upload log", [args.log_file]) sys.exit() @@ -83,7 +147,7 @@ def cleanexit(): # Open a browser at the authentication URL. Do this however # you want, as long as the user visits that URL. authorize_url = flickr.auth_url(perms='write') - print("Open the following URL on your web browser and copy the code to " + config_file) + print("Open the following URL on your web browser and copy the code to " + args.config) print(authorize_url) # Get the verifier code from the user. Do this however you @@ -99,17 +163,17 @@ def cleanexit(): excludes = r'|'.join([fnmatch.translate(x) for x in excludes]) or r'$.' if len(sys.argv) < 2: - print("Usage: "+sys.argv[0]+" path") + print("Usage: " + sys.argv[0] + " path") sys.exit(0) try: - log = open(logfile, 'a') + log = open(args.log_file, 'a') except: - print("Can't open log file for writing ("+logfile+")") + print("Can't open log file for writing (" + args.log_file + ")") sys.exit() logit("New run") -for root, dirs, files in os.walk(sdcard): +for root, dirs, files in os.walk(args.pictures + '/'): # exclude dirs dirs[:] = [os.path.join(root, d) for d in dirs] @@ -125,7 +189,7 @@ def cleanexit(): try: resp = flickr.upload(filename=fname, tags='rpiup', is_public=0) pre, ext = os.path.splitext(fname) - os.rename(fname, pre + done) + os.rename(fname, pre + '.' + args.uploaded_extension) logit(fname + " : " + resp.find('photoid').text) except: logit("Error on " + fname) From da47411505cc8c964f44b5a0d283bfb8c14ac76e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JDaniel=20Jim=C3=A9nez=20Paniagua?= Date: Wed, 22 Jun 2016 22:12:01 +0200 Subject: [PATCH 2/5] Added function to touch files --- flickup.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/flickup.py b/flickup.py index 4ac268c..69a9971 100644 --- a/flickup.py +++ b/flickup.py @@ -63,8 +63,8 @@ # Extension to mark files as uploaded parser.add_argument('-e', '--uploaded-extension', - help='Extension to mark files as uploaded. Default: {}'.format( - default_uploaded_extension), + help='Extension to mark files as uploaded. Default: {}' + .format(default_uploaded_extension), required=False, default=default_uploaded_extension ) @@ -99,6 +99,12 @@ flickr = flickrapi.FlickrAPI(api_key, api_secret, api_token) +def touch(file_name): + with open(file_name, 'w+'): + pass + return True + + def send_mail(send_from, send_to, subject, text, files=[], server="localhost", port=25, username='', password='', isTls=True): msg = MIMEMultipart() msg['From'] = send_from From 77462c3108d9c70bbf261dfc88127880c78a8b2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JDaniel=20Jim=C3=A9nez=20Paniagua?= Date: Wed, 22 Jun 2016 22:12:49 +0200 Subject: [PATCH 3/5] Added .gitignore and ignored .log files --- .gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..397b4a7 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.log From 4d8e035d90946b50dbe514d31432d5cc293452c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JDaniel=20Jim=C3=A9nez=20Paniagua?= Date: Wed, 22 Jun 2016 23:17:15 +0200 Subject: [PATCH 4/5] Control uploaded images creating files with prefix. Default '.', to create hidden empty files --- .gitignore | 2 ++ flickup.conf | 4 ++-- flickup.py | 37 ++++++++++++++++++++++--------------- 3 files changed, 26 insertions(+), 17 deletions(-) mode change 100644 => 100755 flickup.py diff --git a/.gitignore b/.gitignore index 397b4a7..72cfa5a 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ *.log +miflickup.conf +imgs diff --git a/flickup.conf b/flickup.conf index 18bcaa1..a04c7e7 100644 --- a/flickup.conf +++ b/flickup.conf @@ -1,6 +1,6 @@ [main] -api_key: 5c590366f1af8afdb7190ae6d82a1388 -api_secret: acf7934b3c2a5cd2 +api_key: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAA +api_secret: BBBBBBBBBBBB api_token: 0000000 email_from: from@null.com email_to: to@example.com diff --git a/flickup.py b/flickup.py old mode 100644 new mode 100755 index 69a9971..297b84f --- a/flickup.py +++ b/flickup.py @@ -9,6 +9,7 @@ import sys import configparser import argparse +import ntpath import smtplib import os @@ -22,7 +23,7 @@ # Default vars default_args_config = '/etc/flickrup.conf' default_log_file = 'flickrup.log' -default_uploaded_extension = 'done' +default_uploaded_prefix = '.' # Initializing argparser parser = argparse.ArgumentParser(description='Flickrup: Flicker uploader.') @@ -60,13 +61,13 @@ default=default_log_file ) -# Extension to mark files as uploaded -parser.add_argument('-e', - '--uploaded-extension', - help='Extension to mark files as uploaded. Default: {}' - .format(default_uploaded_extension), +# prefix for files created to mark uploaded files +parser.add_argument('-u', + '--uploaded-prefix', + help='File prefix to prevent upload files twice. Default: {}' + .format(default_uploaded_prefix), required=False, - default=default_uploaded_extension + default=default_uploaded_prefix ) @@ -94,7 +95,7 @@ includes = ['*.jpg'] # for files only / case insensitive # for dirs and files / case insensitive -excludes = ["*." + args.uploaded_extension, args.log_file] +excludes = [args.log_file] flickr = flickrapi.FlickrAPI(api_key, api_secret, api_token) @@ -192,13 +193,19 @@ def cleanexit(): files = [f for f in files if re.match(includes, f.lower())] for fname in files: - try: - resp = flickr.upload(filename=fname, tags='rpiup', is_public=0) - pre, ext = os.path.splitext(fname) - os.rename(fname, pre + '.' + args.uploaded_extension) - logit(fname + " : " + resp.find('photoid').text) - except: - logit("Error on " + fname) + control_file = ntpath.dirname(fname) + \ + '/' + \ + args.uploaded_prefix + \ + ntpath.basename(fname) + + if not os.path.isfile(control_file): + try: + resp = flickr.upload(filename=fname, tags='rpiup', is_public=0) + pre, ext = os.path.splitext(fname) + touch(control_file) + logit(fname + " : " + resp.find('photoid').text) + except: + logit("Error on " + fname) logit("End run") cleanexit() From 7689ea9e441b80a5ab6325fa2cb684838b50f92d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JDaniel=20Jim=C3=A9nez=20Paniagua?= Date: Wed, 22 Jun 2016 23:37:52 +0200 Subject: [PATCH 5/5] Added progress bar to print info during upload process --- flickup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flickup.py b/flickup.py index 297b84f..f7a6d0c 100755 --- a/flickup.py +++ b/flickup.py @@ -11,6 +11,7 @@ import argparse import ntpath + import smtplib import os from email.mime.multipart import MIMEMultipart @@ -18,6 +19,7 @@ from email.mime.text import MIMEText from email.utils import COMMASPACE, formatdate from email import encoders +from tqdm import tqdm # Default vars @@ -192,7 +194,7 @@ def cleanexit(): files = [f for f in files if not re.match(excludes, f.lower())] files = [f for f in files if re.match(includes, f.lower())] - for fname in files: + for fname in tqdm(files): control_file = ntpath.dirname(fname) + \ '/' + \ args.uploaded_prefix + \