From 56d4e40b06f0cdf2a2a1cf2d7e18313d824a315f Mon Sep 17 00:00:00 2001 From: harlo Date: Thu, 30 Apr 2015 14:19:51 -0400 Subject: [PATCH 1/7] error handling for git-annex (being phased out) --- Models/uv_annex_watcher.py | 78 ++++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 36 deletions(-) diff --git a/Models/uv_annex_watcher.py b/Models/uv_annex_watcher.py index e58b5a7..a01974a 100644 --- a/Models/uv_annex_watcher.py +++ b/Models/uv_annex_watcher.py @@ -97,30 +97,33 @@ def get_new_hash(self, file): return new_hash def uploadToAnnex(self, netcat_stub): + use_git_annex = False this_dir = os.getcwd() os.chdir(ANNEX_DIR) if type(netcat_stub['file']) in [str, unicode]: - if GIT_ANNEX is None: + if GIT_ANNEX is not None: + use_git_annex = True if DEBUG: - print "GIT ANNEX NOT ATTACHED TO INSTANCE." - - return None - - with settings(warn_only=True): - # has this stub been uploaded? - is_absorbed = local("%s metadata \"%s\" --json --get=uv_uploaded" % ( - GIT_ANNEX, netcat_stub['save_as']), capture=True) - - if DEBUG: print "%s absorbed? (uv_uploaded = %s type = %s)" % ( - netcat_stub['save_as'], is_absorbed, type(is_absorbed)) - - if is_absorbed == "" or "False": - is_absorbed = False - elif is_absorbed == "True": - is_absorbed = True - else: - is_absorbed = False + print "GIT ANNEX ATTACHED TO INSTANCE." + + if use_git_annex: + with settings(warn_only=True): + # has this stub been uploaded? + is_absorbed = local("%s metadata \"%s\" --json --get=uv_uploaded" % ( + GIT_ANNEX, netcat_stub['save_as']), capture=True) + + if DEBUG: print "%s absorbed? (uv_uploaded = %s type = %s)" % ( + netcat_stub['save_as'], is_absorbed, type(is_absorbed)) + + if is_absorbed == "" or "False": + is_absorbed = False + elif is_absorbed == "True": + is_absorbed = True + else: + is_absorbed = False + else: + is_absorbed = False else: is_absorbed = False @@ -162,7 +165,7 @@ def uploadToAnnex(self, netcat_stub): with settings(warn_only=True): local("mv \"%s\" %s" % (netcat_stub['file'], new_file)) - if GIT_ANNEX is not None: + if use_git_annex: local("%s metadata %s --json --set=uv_file_alias=\"%s\"" % (GIT_ANNEX, new_file, netcat_stub['save_as'])) netcat_stub['file'] = new_file @@ -174,7 +177,7 @@ def uploadToAnnex(self, netcat_stub): # look up to see if this file is already in the annex with settings(warn_only=True): - if type(netcat_stub['file']) in [str, unicode] and GIT_ANNEX is not None: + if type(netcat_stub['file']) in [str, unicode] and use_git_annex: local("%s add %s" % (GIT_ANNEX, netcat_stub['save_as'])) p = UnveillanceFabricProcess(netcat, netcat_stub) @@ -195,7 +198,7 @@ def uploadToAnnex(self, netcat_stub): print "ERROR:" print p.error - if type(netcat_stub['file']) in [str, unicode] and GIT_ANNEX is not None: + if type(netcat_stub['file']) in [str, unicode] and use_git_annex: local("%s metadata \"%s\" --json --set=uv_uploaded=%s" % ( GIT_ANNEX, netcat_stub['save_as'], str(success_tag))) @@ -205,11 +208,13 @@ def uploadToAnnex(self, netcat_stub): return { 'uploaded' : success_tag, '_id' : new_hash } def on_created(self, event): - if GIT_ANNEX is None: + use_git_annex = False + + if GIT_ANNEX is not None: if DEBUG: - print "GIT ANNEX NOT ATTACHED TO INSTANCE." + print "GIT ANNEX ATTACHED TO INSTANCE." - return + use_git_annex = True if event.event_type != "created": return @@ -230,19 +235,20 @@ def on_created(self, event): filename = event.src_path.split("/")[-1] never_upload = False - with settings(warn_only=True): - # has this stub been uploaded? - never_upload = local("%s metadata \"%s\" --json --get=uv_never_upload" % ( - GIT_ANNEX, filename), capture=True) + if use_git_annex: + with settings(warn_only=True): + # has this stub been uploaded? + never_upload = local("%s metadata \"%s\" --json --get=uv_never_upload" % ( + GIT_ANNEX, filename), capture=True) - if DEBUG: - print "%s valid? (uv_never_upload = %s type = %s)" % ( \ - filename, never_upload, type(never_upload)) + if DEBUG: + print "%s valid? (uv_never_upload = %s type = %s)" % ( \ + filename, never_upload, type(never_upload)) - if never_upload == "True": - never_upload = True - elif never_upload == "": - never_upload = False + if never_upload == "True": + never_upload = True + elif never_upload == "": + never_upload = False print "NEVER UPLOAD? %s" % never_upload if never_upload: From ba9474747bbe998cde2f76c626daf0bb7bc9ae63 Mon Sep 17 00:00:00 2001 From: harlo Date: Sat, 2 May 2015 02:05:42 -0400 Subject: [PATCH 2/7] picky --- vars.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vars.py b/vars.py index e5ab13e..32bcd5a 100644 --- a/vars.py +++ b/vars.py @@ -1,5 +1,6 @@ -from lib.Core.vars import * +import os from collections import namedtuple +from lib.Core.vars import * PostBatchRequestStub = namedtuple("PostBatchRequestStub", "files uri") class PostBatchStub(object): @@ -11,6 +12,7 @@ def __init__(self, files, uri): FILE_NON_OVERWRITES = [] IMPORTER_SOURCES = ["file_added"] +APP_DIR = os.path.join(os.path.abspath(os.path.join(__file__, os.pardir, os.pardir, os.pardir)), "app") USER_CREDENTIAL_PACK = { "username" : "", From 43781af383531835f394cf837461172e83172026 Mon Sep 17 00:00:00 2001 From: harlo Date: Wed, 6 May 2015 20:00:57 +0000 Subject: [PATCH 3/7] pin to latest core --- lib/Core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Core b/lib/Core index 190f08d..4a6b354 160000 --- a/lib/Core +++ b/lib/Core @@ -1 +1 @@ -Subproject commit 190f08d7ec012f581f18c64dbe3a998b6a8e4417 +Subproject commit 4a6b354c2ce34d44cfc394bbfca24df1403ec5dc From 89699e09e0a08daba200157ecf56e0999ab58cb9 Mon Sep 17 00:00:00 2001 From: harlo Date: Fri, 8 May 2015 08:33:04 -0400 Subject: [PATCH 4/7] file handler fixes --- unveillance_frontend.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/unveillance_frontend.py b/unveillance_frontend.py index 14f0e08..c4c5f9a 100644 --- a/unveillance_frontend.py +++ b/unveillance_frontend.py @@ -51,6 +51,7 @@ def __init__(self): self.on_loads_by_status = [[] for i in range(4)] self.restricted_routes_by_status = [[] for i in range(4)] self.restricted_mime_types_by_status = [[] for i in range(4)] + self.restrict_source_files = {} self.on_loads = {} self.get_page_load_extras = {} @@ -142,6 +143,21 @@ def get(self, uri): class FileHandler(tornado.web.RequestHandler): @tornado.web.asynchronous def get(self, file): + if 'restrict' in self.application.restrict_source_files.keys(): + + if 'exempt' not in self.application.restrict_source_files.keys() or \ + file not in self.application.restrict_source_files['exempt']: + + srx = r'^/files/\.data/.*' + if not re.match(srx, self.request.uri): + if self.application.do_get_status(self) in self.application.restrict_source_files['restrict']: + res = Result() + res.result = 403 + + self.set_status(res.result) + self.finish(res.emit()) + return + url = "%s%s" % (buildServerURL(), self.request.uri) if DEBUG: print url @@ -149,12 +165,6 @@ def get(self, file): r = requests.get(url, verify=False) self.set_header("Content-Type", r.headers['content-type']) - - ''' - self.set_header("Content-Disposition", - 'attachment; filename="%s"' % self.request.uri.split('/')[-1]) - ''' - self.finish(r.content) class TaskHandler(tornado.web.RequestHandler): @@ -205,7 +215,8 @@ def get(self, route): if hasattr(self.application, "WEB_TITLE"): web_title = self.application.WEB_TITLE - else: web_title = WEB_TITLE + else: + web_title = WEB_TITLE if route is None: if hasattr(self.application, "INDEX_HEADER"): From 256cb5f7e0ce007dd36b046f5d36f47c704967e5 Mon Sep 17 00:00:00 2001 From: jonny Date: Mon, 6 Jul 2015 16:58:18 -0400 Subject: [PATCH 5/7] removed Harlo's Core submodule --- .gitmodules | 3 --- lib/Core | 1 - 2 files changed, 4 deletions(-) delete mode 160000 lib/Core diff --git a/.gitmodules b/.gitmodules index 63dda78..e69de29 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "lib/Core"] - path = lib/Core - url = git@github.com:harlo/UnveillanceCore.git diff --git a/lib/Core b/lib/Core deleted file mode 160000 index 190f08d..0000000 --- a/lib/Core +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 190f08d7ec012f581f18c64dbe3a998b6a8e4417 From fc8ed51f8b82db7cd1997c78d5130a6fb30f28e1 Mon Sep 17 00:00:00 2001 From: jonny Date: Mon, 6 Jul 2015 17:23:16 -0400 Subject: [PATCH 6/7] linked to Core submodule at https://github.com/unveillance/UnveillanceCore --- .gitmodules | 3 +++ lib/Core | 1 + 2 files changed, 4 insertions(+) create mode 160000 lib/Core diff --git a/.gitmodules b/.gitmodules index e69de29..e733c9e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "lib/Core"] + path = lib/Core + url = https://github.com/unveillance/UnveillanceCore diff --git a/lib/Core b/lib/Core new file mode 160000 index 0000000..190f08d --- /dev/null +++ b/lib/Core @@ -0,0 +1 @@ +Subproject commit 190f08d7ec012f581f18c64dbe3a998b6a8e4417 From f99e48b747047d23990441c5164d6418bb5a64a8 Mon Sep 17 00:00:00 2001 From: jonny Date: Tue, 15 Sep 2015 22:40:07 -0400 Subject: [PATCH 7/7] better handling of default css --- web/css/reset.css | 51 +++++++++++++++++++++++++++++++++++++++++++++++ web/index.html | 3 ++- web/module.html | 3 ++- 3 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 web/css/reset.css diff --git a/web/css/reset.css b/web/css/reset.css new file mode 100644 index 0000000..b9d745f --- /dev/null +++ b/web/css/reset.css @@ -0,0 +1,51 @@ +/*from http://meyerweb.com/eric/thoughts/2008/01/15/resetting-again/ */ + +html, body, div, span, applet, object, iframe, +h1, h2, h3, h4, h5, h6, p, blockquote, pre, +a, abbr, acronym, address, big, cite, code, +del, dfn, em, font, img, ins, kbd, q, s, samp, +small, strike, strong, sub, sup, tt, var, +b, u, i, center, +dl, dt, dd, ol, ul, li, +fieldset, form, label, legend, +table, caption, tbody, tfoot, thead, tr, th, td, hr { + margin: 0; + padding: 0; + border: 0; + outline: 0; + font-size: 100%; + vertical-align: baseline; + background: transparent; + font-weight:normal; +} +strong, b { + font-weight:bold; +} +body { + line-height: 1; +} +ol, ul { + list-style: none; +} +blockquote, q { + quotes: none; +} + +/* remember to define focus styles! */ +:focus { + outline: 0; +} + +/* remember to highlight inserts somehow! */ +ins { + text-decoration: none; +} +del { + text-decoration: line-through; +} + +/* tables still need 'cellspacing="0"' in the markup */ +table { + border-collapse: collapse; + border-spacing: 0; +} diff --git a/web/index.html b/web/index.html index 170373c..a3e3b7c 100644 --- a/web/index.html +++ b/web/index.html @@ -10,10 +10,11 @@ - ${ on_loader } + + ${ on_loader } diff --git a/web/module.html b/web/module.html index 5523fb5..33eb8e5 100755 --- a/web/module.html +++ b/web/module.html @@ -10,10 +10,11 @@ - ${ on_loader } + + ${ on_loader }