From 16d6933c42089f012f6e1ec75a4de2e9abc4cb9c Mon Sep 17 00:00:00 2001 From: George Hunt Date: Thu, 30 Jun 2016 18:45:53 -0700 Subject: [PATCH 01/27] start work on making a data chunk --- roles/upstream/files/mkchunk | 41 ++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100755 roles/upstream/files/mkchunk diff --git a/roles/upstream/files/mkchunk b/roles/upstream/files/mkchunk new file mode 100755 index 00000000..3c06ec1a --- /dev/null +++ b/roles/upstream/files/mkchunk @@ -0,0 +1,41 @@ +#!/bin/bash -x +# generate a zip file with data bound for upstream +# +# define the upstream location in filesystem tree +UPENV=/library/upstream +mkdir -p $UPENV + +# create places for current staging, and historic bundles +mkdir -p $UPENV/staging +mkdir -p $UPENV/history + +# get the UUID for this device +UUID=`cat /etc/xsce/uuid` +UUID_SNIP=${UUID:0:4} + +# create a sequence number +if [ ! -f $UPENV/seq ]; then + SEQ="0000" +else + SEQ=`cat $UPENV/seq` +fi +# increment and store next +NEXT=`printf "%05d" $((SEQ + 1))` +echo $NEXT > $UPENV/seq +YYMMDD=`date +%y%m%d` +echo $UUID_SNIP-$SEQ-$YYMMDD + +# put interesting data into staging +cp /etc/xsce/uuid $UPENV/staging +uptime -p > $UPENV/staging/uptime +vnstat > $UPENV/staging/vnstat +grep -e admin /var/log/httpd/access* > $UPENV/staging/access.logs + + + + + + + + +# vim: background=dark et ts=3 From 202a294aca5ee914fbdc0774ac95a7052e3ec3f0 Mon Sep 17 00:00:00 2001 From: George Hunt Date: Fri, 1 Jul 2016 16:48:53 -0700 Subject: [PATCH 02/27] add python to generate csv --- roles/upstream/files/mkchunk | 8 ++++-- roles/upstream/files/sift.py | 53 ++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 3 deletions(-) create mode 100755 roles/upstream/files/sift.py diff --git a/roles/upstream/files/mkchunk b/roles/upstream/files/mkchunk index 3c06ec1a..9270f104 100755 --- a/roles/upstream/files/mkchunk +++ b/roles/upstream/files/mkchunk @@ -9,20 +9,22 @@ mkdir -p $UPENV mkdir -p $UPENV/staging mkdir -p $UPENV/history +source ./upstream_functions + # get the UUID for this device UUID=`cat /etc/xsce/uuid` UUID_SNIP=${UUID:0:4} +YYMMDD=`date +%y%m%d` # create a sequence number if [ ! -f $UPENV/seq ]; then SEQ="0000" else - SEQ=`cat $UPENV/seq` + SEQ=`cat $UPENV/seq | cut -d" " -f1` fi # increment and store next NEXT=`printf "%05d" $((SEQ + 1))` -echo $NEXT > $UPENV/seq -YYMMDD=`date +%y%m%d` +echo "$NEXT $YYMMDD" > $UPENV/seq echo $UUID_SNIP-$SEQ-$YYMMDD # put interesting data into staging diff --git a/roles/upstream/files/sift.py b/roles/upstream/files/sift.py new file mode 100755 index 00000000..83e6edac --- /dev/null +++ b/roles/upstream/files/sift.py @@ -0,0 +1,53 @@ +#!/bin/env python +# read apache logs, sifting for records we want to save +import apache_log_parser +import sys +from os import path +import os +import datetime +from pprint import pprint +import glob +import json + +LOC='/library/upstream' + +# fetch the dictionary of previous downloads if it exists +if path.isfile(path.join(LOC,"data","downloads")): + strm = open(path.join(LOC,"data","downloads"),"r") + downloads = json.load(strm) +else: downloads = {} +added = 0 + +line_parser = apache_log_parser.make_parser("%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"") + +# traverse the apache logs +for fn in glob.glob('/var/log/httpd/access*'): + for line in open(fn, 'r'): + if line.find('admin'): + line_dict = line_parser(line) + if line_dict['request_method'] != "GET": continue + if line_dict['status'] != "200": continue + print("%s,%s" % (line_dict['time_received_tz_isoformat'], + line_dict['request_url'],)) + # put the data in the dictionary + key = line_dict['time_received_tz_isoformat'] + \ + line_dict['request_url'] + dt = line_dict['time_received_tz_datetimeobj'] + if not key in downloads: + downloads[key] = {"time": line_dict['time_received_tz_isoformat'], + "url": line_dict['request_url'], + "week": dt.isocalendar()[1] } + added += 1 + else: + continue +# now store away the accumulated data + +with open(path.join(LOC,"data","downloads"),"w") as outfile: + json.dump(downloads, outfile) + +# now create the final csv file +outfile = open(path.join(LOC,"staging","downloads.csv"),'w') + +for key in sorted(downloads): + outfile.write("%s,%s,%s\n" % (downloads[key]["time"],\ + downloads[key]["url"], downloads[key]["week"],)) From 552b05e84373973168223bb683163913daab40b7 Mon Sep 17 00:00:00 2001 From: George Hunt Date: Sat, 2 Jul 2016 12:43:29 -0700 Subject: [PATCH 03/27] lots of progress --- roles/upstream/files/harvest.py | 66 ++++++++++++++++++++++++++++++ roles/upstream/files/mkchunk | 29 ++++++++++--- roles/upstream/files/sift.py | 9 +++- roles/upstream/files/upstream.conf | 14 +++++++ roles/upstream/files/upstream.wsgi | 27 ++++++++++++ 5 files changed, 138 insertions(+), 7 deletions(-) create mode 100755 roles/upstream/files/harvest.py create mode 100644 roles/upstream/files/upstream.conf create mode 100755 roles/upstream/files/upstream.wsgi diff --git a/roles/upstream/files/harvest.py b/roles/upstream/files/harvest.py new file mode 100755 index 00000000..6564c01d --- /dev/null +++ b/roles/upstream/files/harvest.py @@ -0,0 +1,66 @@ +#!/bin/env python +# fetch email from xscenet@gmail.com, unzip it into raw_data +import glob +import imaplib +import email +import os +import zipfile + +upenv = "/library/upstream" +detach_dir = os.path.join(upenv,"html","zips") +raw_dir = os.path.join(upenv,"html","raw_data") +m = imaplib.IMAP4_SSL('imap.gmail.com') +m.login('xscenet@gmail.com', 'immi96?Bronx') + +""" +mail.select("inbox") # connect to inbox. +result, data = mail.search(None, "ALL") + +ids = data[0] # data is a list. +id_list = ids.split() # ids is a space separated string +latest_email_id = id_list[-1] # get the latest + +result, data = mail.fetch(latest_email_id, "(RFC822)") # fetch the email body (RFC822) for the given ID + +raw_email = data[0][1] # here's the body, which is raw text of the whole email +# including headers and alternate payloads +""" +m.select("[Gmail]/All Mail") + +resp, items = m.search(None, "(ALL)") +items = items[0].split() + +for emailid in items: + resp, data = m.fetch(emailid, "(RFC822)") + email_body = data[0][1] + mail = email.message_from_string(email_body) + temp = m.store(emailid,'+FLAGS', '\\Seen') + m.expunge() + + if mail.get_content_maintype() != 'multipart': + continue + +# print "["+mail["From"]+"] :" + mail["Subject"] + + for part in mail.walk(): + if part.get_content_maintype() == 'multipart': + continue + if part.get('Content-Disposition') is None: + continue + + filename = part.get_filename() + att_path = os.path.join(detach_dir, filename) + + if not os.path.isfile(att_path) : + print("writing: ",att_path) + fp = open(att_path, 'wb') + fp.write(part.get_payload(decode=True)) + fp.close() + +# go through the zip files, and expand them if not already expanded +for zf in glob.glob(detach_dir+"/*"): + # get the directory name we want in raw directory + raw_base = os.path.join(raw_dir,os.path.basename(zf)[ :-4]) + if not os.path.isdir(raw_base): + with zipfile.ZipFile(zf,"r") as cpzip: + cpzip.extractall(raw_dir) diff --git a/roles/upstream/files/mkchunk b/roles/upstream/files/mkchunk index 9270f104..c9c0a48b 100755 --- a/roles/upstream/files/mkchunk +++ b/roles/upstream/files/mkchunk @@ -8,33 +8,52 @@ mkdir -p $UPENV # create places for current staging, and historic bundles mkdir -p $UPENV/staging mkdir -p $UPENV/history +mkdir -p $UPENV/data +mkdir -p $UPENV/html/zips +mkdir -p $UPENV/html/raw_data -source ./upstream_functions # get the UUID for this device UUID=`cat /etc/xsce/uuid` UUID_SNIP=${UUID:0:4} -YYMMDD=`date +%y%m%d` +LAST_RUN= # create a sequence number if [ ! -f $UPENV/seq ]; then SEQ="0000" else SEQ=`cat $UPENV/seq | cut -d" " -f1` + LAST_RUN=`cat $UPENV/seq | cut -d" " -f2` fi # increment and store next NEXT=`printf "%05d" $((SEQ + 1))` +YYMMDD=`date +%y%m%d` +# there may be many training situations where this run repeatedly +# it's not an error, but we don't need a lot of empty bundles +if [ $YYMMDD == $LAST_RUN ]; then + echo "Data bundle is up to date" + exit 0 +fi + echo "$NEXT $YYMMDD" > $UPENV/seq -echo $UUID_SNIP-$SEQ-$YYMMDD +ZIPNAME=$UUID_SNIP-$SEQ-$YYMMDD # put interesting data into staging cp /etc/xsce/uuid $UPENV/staging uptime -p > $UPENV/staging/uptime vnstat > $UPENV/staging/vnstat -grep -e admin /var/log/httpd/access* > $UPENV/staging/access.logs - +cp /var/log/httpd/access* $UPENV/staging +# sift through the apache logs and generate the CSV +$UPENV/sift.py +# now create the zip file +mv $UPENV/staging $UPENV/$ZIPNAME +/bin/zip -r $UPENV/history/$ZIPNAME.zip ./$ZIPNAME +if [ $? -eq 0 ]; then + rm -rf $UPENV/$ZIPNAME + mkdir -p $UPENV/staging +fi diff --git a/roles/upstream/files/sift.py b/roles/upstream/files/sift.py index 83e6edac..b0d629dd 100755 --- a/roles/upstream/files/sift.py +++ b/roles/upstream/files/sift.py @@ -18,6 +18,10 @@ else: downloads = {} added = 0 +# get the UUID of this machine +with open("/etc/xsce/uuid", "r") as infile: + uuid=infile.read() + line_parser = apache_log_parser.make_parser("%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"") # traverse the apache logs @@ -49,5 +53,6 @@ outfile = open(path.join(LOC,"staging","downloads.csv"),'w') for key in sorted(downloads): - outfile.write("%s,%s,%s\n" % (downloads[key]["time"],\ - downloads[key]["url"], downloads[key]["week"],)) + outfile.write("%s,%s,%s,%s\n" % (downloads[key]["time"],\ + downloads[key]["week"],\ + downloads[key]["url"], uuid.rstrip(), )) diff --git a/roles/upstream/files/upstream.conf b/roles/upstream/files/upstream.conf new file mode 100644 index 00000000..eb7e4656 --- /dev/null +++ b/roles/upstream/files/upstream.conf @@ -0,0 +1,14 @@ +#WikiHealth Apache2 configuration file + +XSendFile on +XSendFilePath /library/content + +WSGIScriptAlias /analytics /library/upstream/upstream.wsgi + +Alias /communityanalytics /library/upstream/html + + + Options +Indexes + IndexOptions FancyIndexing + require all granted + diff --git a/roles/upstream/files/upstream.wsgi b/roles/upstream/files/upstream.wsgi new file mode 100755 index 00000000..31964ba7 --- /dev/null +++ b/roles/upstream/files/upstream.wsgi @@ -0,0 +1,27 @@ +#!/usr/bin/python + +import sys +import os +import glob +from wsgiref.simple_server import make_server + +def application(environ, start_response): + upenv = "/library/upstream" + block_size = 1024 + downloads_available = sorted(glob.glob(os.path.join(upenv,"history/*"))) + last = downloads_available[-1] + fd = open(last,"r") + status = '200 OK' + headers = [('Content-type', 'application/zip'), + ('Content-Disposition','attachment; filename='+os.path.basename(last))] + start_response(status, headers) + if 'wsgi.file_wrapper' in environ: + return environ['wsgi.file_wrapper'](fd, block_size) + else: + return iter(lambda: fd.read(block_size), '') + + +if __name__ == '__main__': + httpd = make_server('', 8051, application) + print "Serving on port 8051..." + httpd.serve_forever() From c3327be4bd67d0a5592ea4ffbaa537d06ca7a782 Mon Sep 17 00:00:00 2001 From: George Hunt Date: Sun, 3 Jul 2016 00:39:00 +0000 Subject: [PATCH 04/27] fixes for first demo --- roles/upstream/files/harvest.py | 67 +++++++++++++++++++++++------- roles/upstream/files/sift.py | 4 +- roles/upstream/files/upstream.conf | 1 + 3 files changed, 55 insertions(+), 17 deletions(-) diff --git a/roles/upstream/files/harvest.py b/roles/upstream/files/harvest.py index 6564c01d..38c5bbc9 100755 --- a/roles/upstream/files/harvest.py +++ b/roles/upstream/files/harvest.py @@ -5,26 +5,48 @@ import email import os import zipfile +import json upenv = "/library/upstream" detach_dir = os.path.join(upenv,"html","zips") raw_dir = os.path.join(upenv,"html","raw_data") m = imaplib.IMAP4_SSL('imap.gmail.com') m.login('xscenet@gmail.com', 'immi96?Bronx') +# declare location of dictionry with all our download data +# -- the key for dictionary is datetime+download_url +download_data = os.path.join(upenv,"downloads.json") + +def merge_data(filename=None): + if filename == None: + print("no filename in merge_data") + return + # fetch the dictionary of previous downloads if it exists + if os.path.isfile(download_data): + with open(download_data,"r") as strm: + downloads = json.load(strm) + else: downloads = {} + added = 0 + + for line in open(filename, 'r'): + data_chunks = line.split(',') + # put the data in the dictionary + key = data_chunks[0] + data_chunks[2] + if not key in downloads: + downloads[key] = {"time": data_chunks[0], + "week": data_chunks[1], + "url": data_chunks[2], + "uuid": data_chunks[3], + } + added += 1 + else: + continue + print("added records to data store: %s" % added) + + # now store away the accumulated data + with open(os.path.join(download_data),"w") as outfile: + json.dump(downloads, outfile) + -""" -mail.select("inbox") # connect to inbox. -result, data = mail.search(None, "ALL") - -ids = data[0] # data is a list. -id_list = ids.split() # ids is a space separated string -latest_email_id = id_list[-1] # get the latest - -result, data = mail.fetch(latest_email_id, "(RFC822)") # fetch the email body (RFC822) for the given ID - -raw_email = data[0][1] # here's the body, which is raw text of the whole email -# including headers and alternate payloads -""" m.select("[Gmail]/All Mail") resp, items = m.search(None, "(ALL)") @@ -62,5 +84,20 @@ # get the directory name we want in raw directory raw_base = os.path.join(raw_dir,os.path.basename(zf)[ :-4]) if not os.path.isdir(raw_base): - with zipfile.ZipFile(zf,"r") as cpzip: - cpzip.extractall(raw_dir) + with zipfile.ZipFile(zf,"r") as cpzip: + cpzip.extractall(raw_dir) + # now merge the data in the downloads.csv with our data_store at + merge_data(os.path.join(raw_base,"downloads_csv")) + + +# regenerate the publicly visible merged data from all reporters +with open(download_data,"r") as strm: + downloads = json.load(strm) +with open(os.path.join(upenv,"html","downloads.csv.txt"),"w") as outfile: + for dl in sorted(downloads.keys()): + + outfile.write("%s,%s,%s,%s\n" % (downloads[dl]["time"],\ + downloads[dl]["week"],\ + downloads[dl]["url"],\ + downloads[dl]["uuid"],\ + )) diff --git a/roles/upstream/files/sift.py b/roles/upstream/files/sift.py index b0d629dd..2c3adbaa 100755 --- a/roles/upstream/files/sift.py +++ b/roles/upstream/files/sift.py @@ -50,9 +50,9 @@ json.dump(downloads, outfile) # now create the final csv file -outfile = open(path.join(LOC,"staging","downloads.csv"),'w') +outfile = open(path.join(LOC,"staging","downloads_csv"),'w') for key in sorted(downloads): - outfile.write("%s,%s,%s,%s\n" % (downloads[key]["time"],\ + outfile.write("%s,%s,%s,%s,\n" % (downloads[key]["time"],\ downloads[key]["week"],\ downloads[key]["url"], uuid.rstrip(), )) diff --git a/roles/upstream/files/upstream.conf b/roles/upstream/files/upstream.conf index eb7e4656..d7a4022e 100644 --- a/roles/upstream/files/upstream.conf +++ b/roles/upstream/files/upstream.conf @@ -6,6 +6,7 @@ XSendFilePath /library/content WSGIScriptAlias /analytics /library/upstream/upstream.wsgi Alias /communityanalytics /library/upstream/html +Alias /analytics /library/upstream/html Options +Indexes From 235818278aef8bf0e2596b5dad326a110c6ae8f0 Mon Sep 17 00:00:00 2001 From: George Hunt Date: Sat, 2 Jul 2016 21:07:48 -0700 Subject: [PATCH 05/27] remove the typos in ansible upstream --- roles/8-mgmt-tools/meta/main.yml | 1 + roles/upstream/{files => templates}/harvest.py | 0 roles/upstream/{files => templates}/mkchunk | 0 roles/upstream/{files => templates}/sift.py | 0 roles/upstream/{files => templates}/upstream.conf | 0 roles/upstream/{files => templates}/upstream.wsgi | 0 6 files changed, 1 insertion(+) rename roles/upstream/{files => templates}/harvest.py (100%) rename roles/upstream/{files => templates}/mkchunk (100%) rename roles/upstream/{files => templates}/sift.py (100%) rename roles/upstream/{files => templates}/upstream.conf (100%) rename roles/upstream/{files => templates}/upstream.wsgi (100%) diff --git a/roles/8-mgmt-tools/meta/main.yml b/roles/8-mgmt-tools/meta/main.yml index 05c4bc31..6cc7435b 100644 --- a/roles/8-mgmt-tools/meta/main.yml +++ b/roles/8-mgmt-tools/meta/main.yml @@ -8,3 +8,4 @@ dependencies: - { role: phpmyadmin, tags: ['services','phpmyadmin','tools'], when: phpmyadmin_install } - { role: awstats, tags: ['services','awstats','tools'], when: awstats_install } - { role: teamviewer, tags: ['services','teamviewer','tools'], when: teamviewer_install } + - { role: upstream, tags: ['upstream'], when: upstream_install } diff --git a/roles/upstream/files/harvest.py b/roles/upstream/templates/harvest.py similarity index 100% rename from roles/upstream/files/harvest.py rename to roles/upstream/templates/harvest.py diff --git a/roles/upstream/files/mkchunk b/roles/upstream/templates/mkchunk similarity index 100% rename from roles/upstream/files/mkchunk rename to roles/upstream/templates/mkchunk diff --git a/roles/upstream/files/sift.py b/roles/upstream/templates/sift.py similarity index 100% rename from roles/upstream/files/sift.py rename to roles/upstream/templates/sift.py diff --git a/roles/upstream/files/upstream.conf b/roles/upstream/templates/upstream.conf similarity index 100% rename from roles/upstream/files/upstream.conf rename to roles/upstream/templates/upstream.conf diff --git a/roles/upstream/files/upstream.wsgi b/roles/upstream/templates/upstream.wsgi similarity index 100% rename from roles/upstream/files/upstream.wsgi rename to roles/upstream/templates/upstream.wsgi From bf8908e93fdbbf863fabd28b3dc75ba430672fdc Mon Sep 17 00:00:00 2001 From: George Hunt Date: Sat, 2 Jul 2016 22:05:46 -0700 Subject: [PATCH 06/27] give apache permissions --- roles/upstream/defaults/main.yml | 2 + roles/upstream/tasks/main.yml | 52 ++++++++++++++++++++++++++ roles/upstream/templates/upstream.wsgi | 10 +++++ 3 files changed, 64 insertions(+) create mode 100644 roles/upstream/defaults/main.yml create mode 100644 roles/upstream/tasks/main.yml diff --git a/roles/upstream/defaults/main.yml b/roles/upstream/defaults/main.yml new file mode 100644 index 00000000..cb795605 --- /dev/null +++ b/roles/upstream/defaults/main.yml @@ -0,0 +1,2 @@ +upstream_install: False +upstream_enabled: False diff --git a/roles/upstream/tasks/main.yml b/roles/upstream/tasks/main.yml new file mode 100644 index 00000000..9079de0f --- /dev/null +++ b/roles/upstream/tasks/main.yml @@ -0,0 +1,52 @@ +- name: Make folder and set permission + file: path={{ item }} + owner=apache + group=root + mode=0755 + state=directory + with_items: + - "{{ content_base }}/upstream/html/raw_data" + - "{{ content_base }}/upstream/html/zips" + - "{{ content_base }}/upstream/history" + - "{{ content_base }}/upstream/staging" + - "{{ content_base }}/upstream/data" + +- name: Put files where they belong + template: src={{ item.src }} + dest={{ item.dest }} + owner=root + group={{ item.group }} + mode={{ item.mode }} + with_items: + - { src: 'upstream.conf', dest: '/etc/httpd/conf.d/', group: "root" , mode: '0644' } + - { src: 'upstream.wsgi', dest: '{{ content_base }}/upstream', group: "root" , mode: '0644' } + - { src: 'sift.py', dest: '{{ content_base }}/upstream', group: "apache" , mode: '0755' } + - { src: 'harvest.py', dest: '{{ content_base }}/upstream', group: "apache" , mode: '0755' } + - { src: 'mkchunk', dest: '{{ content_base }}/upstream', group: "apache" , mode: '0755' } + when: upstream_install == True + +- name: enable the downloading of data chunk + template: src='upstream.conf' + mode=0644 + dest=/etc/httpd/conf.d/ + when: upstream_enabled + +- name: remove config file to disable + file: path=/etc/httpd/conf.d/upstream.conf + state=absent + when: not upstream_enabled == True + +- name: add upstream to service list + ini_file: dest='{{ service_filelist }}' + section=upstream + option='{{ item.option }}' + value='{{ item.value }}' + with_items: + - option: name + value: Upstream + - option: description + value: '"Upstream is method of communicating usage information from the XSCE server to a centralized data collection location, It uses a smart phone to download a zipped directory of information while offline, and then later, when connected to the internet, emails that package,as an attachment,upstream"' + - option: enabled + value: "{{ upstream_enabled }}" + - option: installed + value: "{{ upstream_install }}" diff --git a/roles/upstream/templates/upstream.wsgi b/roles/upstream/templates/upstream.wsgi index 31964ba7..2a9ef2ba 100755 --- a/roles/upstream/templates/upstream.wsgi +++ b/roles/upstream/templates/upstream.wsgi @@ -2,12 +2,22 @@ import sys import os +import subprocess import glob from wsgiref.simple_server import make_server def application(environ, start_response): upenv = "/library/upstream" block_size = 1024 + + # execute the bash script which generates the zip file + rc = subprocess.call(upenv + "/mkchunk") + if rc != 0: + status = '500 Internal Server Error' + headers = [('Content-type', 'html'), + start_response(status, headers) + return () + downloads_available = sorted(glob.glob(os.path.join(upenv,"history/*"))) last = downloads_available[-1] fd = open(last,"r") From 9868b723b35e3c23ec38ad3b2f1e2ec38044c5d4 Mon Sep 17 00:00:00 2001 From: George Hunt Date: Sun, 3 Jul 2016 10:33:02 -0700 Subject: [PATCH 07/27] refinements --- roles/upstream/templates/cp2git | 5 +++++ roles/upstream/templates/mkchunk | 4 +++- roles/upstream/templates/sift.py | 7 +++++-- roles/upstream/templates/upstream.wsgi | 9 +++++++-- 4 files changed, 20 insertions(+), 5 deletions(-) create mode 100755 roles/upstream/templates/cp2git diff --git a/roles/upstream/templates/cp2git b/roles/upstream/templates/cp2git new file mode 100755 index 00000000..ac517d90 --- /dev/null +++ b/roles/upstream/templates/cp2git @@ -0,0 +1,5 @@ +#!/bin/bash -x +# copy scripts to the upstream role in xsce git +for f in `find /library/upstream -maxdepth 1 -type f`; do + cp $f /opt/schoolserver/xsce/roles/upstream/templates/ +done diff --git a/roles/upstream/templates/mkchunk b/roles/upstream/templates/mkchunk index c9c0a48b..f04a709c 100755 --- a/roles/upstream/templates/mkchunk +++ b/roles/upstream/templates/mkchunk @@ -1,5 +1,6 @@ #!/bin/bash -x # generate a zip file with data bound for upstream + # # define the upstream location in filesystem tree UPENV=/library/upstream @@ -49,7 +50,8 @@ $UPENV/sift.py # now create the zip file mv $UPENV/staging $UPENV/$ZIPNAME -/bin/zip -r $UPENV/history/$ZIPNAME.zip ./$ZIPNAME +sleep 1 +/bin/zip -r $UPENV/history/$ZIPNAME.zip $UPENV/$ZIPNAME if [ $? -eq 0 ]; then rm -rf $UPENV/$ZIPNAME mkdir -p $UPENV/staging diff --git a/roles/upstream/templates/sift.py b/roles/upstream/templates/sift.py index 2c3adbaa..01952000 100755 --- a/roles/upstream/templates/sift.py +++ b/roles/upstream/templates/sift.py @@ -1,5 +1,6 @@ #!/bin/env python # read apache logs, sifting for records we want to save + import apache_log_parser import sys from os import path @@ -27,12 +28,14 @@ # traverse the apache logs for fn in glob.glob('/var/log/httpd/access*'): for line in open(fn, 'r'): + + # this find selects apache log records of interest if line.find('admin'): + line_dict = line_parser(line) + # discard unsuccessful GETs or redirects if line_dict['request_method'] != "GET": continue if line_dict['status'] != "200": continue - print("%s,%s" % (line_dict['time_received_tz_isoformat'], - line_dict['request_url'],)) # put the data in the dictionary key = line_dict['time_received_tz_isoformat'] + \ line_dict['request_url'] diff --git a/roles/upstream/templates/upstream.wsgi b/roles/upstream/templates/upstream.wsgi index 2a9ef2ba..9eef9a34 100755 --- a/roles/upstream/templates/upstream.wsgi +++ b/roles/upstream/templates/upstream.wsgi @@ -14,11 +14,16 @@ def application(environ, start_response): rc = subprocess.call(upenv + "/mkchunk") if rc != 0: status = '500 Internal Server Error' - headers = [('Content-type', 'html'), + headers = [('Content-type', 'html')] start_response(status, headers) - return () + return() downloads_available = sorted(glob.glob(os.path.join(upenv,"history/*"))) + if len(downloads_available) == 0: + status = '500 Internal Server Error' + headers = [('Content-type', 'html')] + start_response(status, headers) + return() last = downloads_available[-1] fd = open(last,"r") status = '200 OK' From 80c742b7e5894eaf9c8efffe306b696784661854 Mon Sep 17 00:00:00 2001 From: georgejhunt Date: Sun, 3 Jul 2016 11:49:35 -0700 Subject: [PATCH 08/27] Create README.md --- roles/upstream/README.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 roles/upstream/README.md diff --git a/roles/upstream/README.md b/roles/upstream/README.md new file mode 100644 index 00000000..ec7e6360 --- /dev/null +++ b/roles/upstream/README.md @@ -0,0 +1,32 @@ +# Note for Upstream data path +**Objective** +* To provide feedback to Wikimedia about how many times a health related Android wiki reader has been downloaded from the XSCE servers onto a smart phone. + +**Notes to myself** + +This was a need that surfaced at the wikimedia conference in late June 2016. After a discussion on the XSCE weekly call, I cobbled together pieces of the larger solution, which evolved from that discussion. The thrust of the discussion was: + + * Health workers either have, or could be provided with, smart phones, which themselves have internet connectivity at least part of the time. + * So XSCE servers could create on the demand from a smart phone browser, a zip file which included the usage information which was wanted by creators of the health wikipedia information. + * Email is part of the normal usage of almost every smart phone, and is ideally suited to intermittent internet connectivity. + * There will need to be a cloud based presence, which gathers and consolidates the email from individual deployments. + +**The Initial Solution** + +A combination of scripts in bash and python create the zipped smartphone download. Perhaps a weak link in this strategy is that it relies on a human being to learn how to download from the XSCE server, and how to attach that downloaded zip file to an email, and send it off (to xscenet@gmail.com). + + * Upstream.wsgi -- A browser request to a url on the XSCE server (http://wikihealth.lan/data) stimulates this python program. This in turn calls a bash script which generates the zipped information package. The completed zip file is returned to the browser's request. + * mkchunk -- Is a bash script which generates the zipped requested information. In doing its job, it calls another python script (sift.py), which parses the apache log files, and selects the web server downloads that are of interest + * sift.py -- Takes advantage of apache log parsing functions, to provide flexible and easy access to the desired information. + * harvest.py -- Runs in the cloud, and uses python imap libraries to access the gmail mail server, and fetch the uploaded email, and their attached zip files. It unzips and consolidates the information, and makes it available at http://xscenet/analytics/downloads_csv.txt + +**How to install Upstream on XSCE - Release-6.1** + * Follow the normal install instructions at https://github.com/XSCE/xsce/wiki/XSCE-Installation. + * Then add the "upstream" software: +``` + cd /opt/schoolserver/xsce + git remote add ghunt https://github.com/georgejhunt/xsce + git checkout -b upstream + git pull ghunt upstream + ./runtags upstream +``` From a1e60f9bdda301bb968986bed8b184965681d29a Mon Sep 17 00:00:00 2001 From: georgejhunt Date: Sun, 3 Jul 2016 11:55:46 -0700 Subject: [PATCH 09/27] Update README.md --- roles/upstream/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/upstream/README.md b/roles/upstream/README.md index ec7e6360..38995fd3 100644 --- a/roles/upstream/README.md +++ b/roles/upstream/README.md @@ -18,7 +18,7 @@ A combination of scripts in bash and python create the zipped smartphone downloa * Upstream.wsgi -- A browser request to a url on the XSCE server (http://wikihealth.lan/data) stimulates this python program. This in turn calls a bash script which generates the zipped information package. The completed zip file is returned to the browser's request. * mkchunk -- Is a bash script which generates the zipped requested information. In doing its job, it calls another python script (sift.py), which parses the apache log files, and selects the web server downloads that are of interest * sift.py -- Takes advantage of apache log parsing functions, to provide flexible and easy access to the desired information. - * harvest.py -- Runs in the cloud, and uses python imap libraries to access the gmail mail server, and fetch the uploaded email, and their attached zip files. It unzips and consolidates the information, and makes it available at http://xscenet/analytics/downloads_csv.txt + * harvest.py -- Runs in the cloud, and uses python imap libraries to access the gmail mail server, and fetch the uploaded email, and their attached zip files. It unzips and consolidates the information, and makes it available at http://xscenet.net/analytics/downloads_csv.txt **How to install Upstream on XSCE - Release-6.1** * Follow the normal install instructions at https://github.com/XSCE/xsce/wiki/XSCE-Installation. From 23bae63028411cf45ccfa68ce2106da04798aaee Mon Sep 17 00:00:00 2001 From: georgejhunt Date: Sun, 3 Jul 2016 12:07:00 -0700 Subject: [PATCH 10/27] Update README.md --- roles/upstream/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/upstream/README.md b/roles/upstream/README.md index 38995fd3..71946dfd 100644 --- a/roles/upstream/README.md +++ b/roles/upstream/README.md @@ -18,7 +18,7 @@ A combination of scripts in bash and python create the zipped smartphone downloa * Upstream.wsgi -- A browser request to a url on the XSCE server (http://wikihealth.lan/data) stimulates this python program. This in turn calls a bash script which generates the zipped information package. The completed zip file is returned to the browser's request. * mkchunk -- Is a bash script which generates the zipped requested information. In doing its job, it calls another python script (sift.py), which parses the apache log files, and selects the web server downloads that are of interest * sift.py -- Takes advantage of apache log parsing functions, to provide flexible and easy access to the desired information. - * harvest.py -- Runs in the cloud, and uses python imap libraries to access the gmail mail server, and fetch the uploaded email, and their attached zip files. It unzips and consolidates the information, and makes it available at http://xscenet.net/analytics/downloads_csv.txt + * harvest.py -- Runs in the cloud, and uses python imap libraries to access the gmail mail server, and fetch the uploaded email, and their attached zip files. It unzips and consolidates the information, and makes it available at http://xscenet.net/analytics **How to install Upstream on XSCE - Release-6.1** * Follow the normal install instructions at https://github.com/XSCE/xsce/wiki/XSCE-Installation. From 76d56ef8363f4130b8bfd8caf2168cba76017861 Mon Sep 17 00:00:00 2001 From: georgejhunt Date: Sun, 3 Jul 2016 15:09:50 -0700 Subject: [PATCH 11/27] Update README.md --- roles/upstream/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/upstream/README.md b/roles/upstream/README.md index 71946dfd..a3a91c70 100644 --- a/roles/upstream/README.md +++ b/roles/upstream/README.md @@ -24,6 +24,8 @@ A combination of scripts in bash and python create the zipped smartphone downloa * Follow the normal install instructions at https://github.com/XSCE/xsce/wiki/XSCE-Installation. * Then add the "upstream" software: ``` + echo "upstream_install: True" >> /opt/schoolserver/xsce/vars/local_vars.yml + echo "upstream_enabledl: True" >> /opt/schoolserver/xsce/vars/local_vars.yml cd /opt/schoolserver/xsce git remote add ghunt https://github.com/georgejhunt/xsce git checkout -b upstream From d83b559a6fa99e5f0fc5bbb743d9b964db137620 Mon Sep 17 00:00:00 2001 From: georgejhunt Date: Sun, 3 Jul 2016 15:10:23 -0700 Subject: [PATCH 12/27] Update README.md --- roles/upstream/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/upstream/README.md b/roles/upstream/README.md index a3a91c70..bfba4512 100644 --- a/roles/upstream/README.md +++ b/roles/upstream/README.md @@ -25,7 +25,7 @@ A combination of scripts in bash and python create the zipped smartphone downloa * Then add the "upstream" software: ``` echo "upstream_install: True" >> /opt/schoolserver/xsce/vars/local_vars.yml - echo "upstream_enabledl: True" >> /opt/schoolserver/xsce/vars/local_vars.yml + echo "upstream_enabled: True" >> /opt/schoolserver/xsce/vars/local_vars.yml cd /opt/schoolserver/xsce git remote add ghunt https://github.com/georgejhunt/xsce git checkout -b upstream From 4e40d1fedb79ba9850cd58bab105a6a63e11e8ff Mon Sep 17 00:00:00 2001 From: George Hunt Date: Sun, 3 Jul 2016 20:11:53 -0700 Subject: [PATCH 13/27] add source email to the zips dir --- roles/upstream/templates/harvest.py | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/roles/upstream/templates/harvest.py b/roles/upstream/templates/harvest.py index 38c5bbc9..08b45c67 100755 --- a/roles/upstream/templates/harvest.py +++ b/roles/upstream/templates/harvest.py @@ -6,9 +6,10 @@ import os import zipfile import json +from time import sleep upenv = "/library/upstream" -detach_dir = os.path.join(upenv,"html","zips") +zips_dir = os.path.join(upenv,"html","zips") raw_dir = os.path.join(upenv,"html","raw_data") m = imaplib.IMAP4_SSL('imap.gmail.com') m.login('xscenet@gmail.com', 'immi96?Bronx') @@ -71,7 +72,12 @@ def merge_data(filename=None): continue filename = part.get_filename() - att_path = os.path.join(detach_dir, filename) + original_zip_dir = filename[:-4] + # put insert the from into the zip file name + s=mail["From"] + sender = s[s.find("<")+1:s.find(">")] + filename = filename[:-4] + "-" + sender + '.zip' + att_path = os.path.join(zips_dir, filename) if not os.path.isfile(att_path) : print("writing: ",att_path) @@ -80,14 +86,26 @@ def merge_data(filename=None): fp.close() # go through the zip files, and expand them if not already expanded -for zf in glob.glob(detach_dir+"/*"): +for zf in glob.glob(zips_dir+"/*"): # get the directory name we want in raw directory - raw_base = os.path.join(raw_dir,os.path.basename(zf)[ :-4]) + raw_base = os.path.join(raw_dir,original_zip_dir) if not os.path.isdir(raw_base): with zipfile.ZipFile(zf,"r") as cpzip: cpzip.extractall(raw_dir) # now merge the data in the downloads.csv with our data_store at - merge_data(os.path.join(raw_base,"downloads_csv")) + csv = os.path.join(raw_base,"downloads_csv") + breakout = 0 + while True: + if os.path.isfile(csv): + break + sleep(.2) + breakout += 1 + if breakout > 10: + break + if breakout > 10: + print "failed to find %s" % csv + raise + merge_data(csv) # regenerate the publicly visible merged data from all reporters From a797feef750e3c3bc9c18880de99fa3c2145806d Mon Sep 17 00:00:00 2001 From: George Hunt Date: Mon, 4 Jul 2016 15:13:12 -0700 Subject: [PATCH 14/27] use content_base for /library --- roles/upstream/templates/harvest.py | 2 +- roles/upstream/templates/mkchunk | 12 ++---------- roles/upstream/templates/sift.py | 2 +- roles/upstream/templates/upstream.conf | 10 +++++----- roles/upstream/templates/upstream.wsgi | 2 +- 5 files changed, 10 insertions(+), 18 deletions(-) diff --git a/roles/upstream/templates/harvest.py b/roles/upstream/templates/harvest.py index 08b45c67..f6218c1f 100755 --- a/roles/upstream/templates/harvest.py +++ b/roles/upstream/templates/harvest.py @@ -8,7 +8,7 @@ import json from time import sleep -upenv = "/library/upstream" +upenv = "{{ content_base }}/upstream" zips_dir = os.path.join(upenv,"html","zips") raw_dir = os.path.join(upenv,"html","raw_data") m = imaplib.IMAP4_SSL('imap.gmail.com') diff --git a/roles/upstream/templates/mkchunk b/roles/upstream/templates/mkchunk index f04a709c..dd577b2d 100755 --- a/roles/upstream/templates/mkchunk +++ b/roles/upstream/templates/mkchunk @@ -3,16 +3,7 @@ # # define the upstream location in filesystem tree -UPENV=/library/upstream -mkdir -p $UPENV - -# create places for current staging, and historic bundles -mkdir -p $UPENV/staging -mkdir -p $UPENV/history -mkdir -p $UPENV/data -mkdir -p $UPENV/html/zips -mkdir -p $UPENV/html/raw_data - +UPENV={{ content_base}}/upstream # get the UUID for this device UUID=`cat /etc/xsce/uuid` @@ -28,6 +19,7 @@ else fi # increment and store next NEXT=`printf "%05d" $((SEQ + 1))` + YYMMDD=`date +%y%m%d` # there may be many training situations where this run repeatedly # it's not an error, but we don't need a lot of empty bundles diff --git a/roles/upstream/templates/sift.py b/roles/upstream/templates/sift.py index 01952000..d7ad17f1 100755 --- a/roles/upstream/templates/sift.py +++ b/roles/upstream/templates/sift.py @@ -10,7 +10,7 @@ import glob import json -LOC='/library/upstream' +LOC='{{ content_base }}/upstream' # fetch the dictionary of previous downloads if it exists if path.isfile(path.join(LOC,"data","downloads")): diff --git a/roles/upstream/templates/upstream.conf b/roles/upstream/templates/upstream.conf index d7a4022e..912ede47 100644 --- a/roles/upstream/templates/upstream.conf +++ b/roles/upstream/templates/upstream.conf @@ -1,14 +1,14 @@ #WikiHealth Apache2 configuration file XSendFile on -XSendFilePath /library/content +XSendFilePath {{ content_base }}/content -WSGIScriptAlias /analytics /library/upstream/upstream.wsgi +WSGIScriptAlias /analytics {{ content_base }}/upstream/upstream.wsgi -Alias /communityanalytics /library/upstream/html -Alias /analytics /library/upstream/html +Alias /communityanalytics {{ content_base }}/upstream/html +Alias /analytics {{ content_base }}/upstream/html - + Options +Indexes IndexOptions FancyIndexing require all granted diff --git a/roles/upstream/templates/upstream.wsgi b/roles/upstream/templates/upstream.wsgi index 9eef9a34..e916c6ae 100755 --- a/roles/upstream/templates/upstream.wsgi +++ b/roles/upstream/templates/upstream.wsgi @@ -7,7 +7,7 @@ import glob from wsgiref.simple_server import make_server def application(environ, start_response): - upenv = "/library/upstream" + upenv = "{{ content_base }}/upstream" block_size = 1024 # execute the bash script which generates the zip file From 3f3203e5baea81eec4d8028ec80e48e5deae230a Mon Sep 17 00:00:00 2001 From: georgejhunt Date: Thu, 7 Jul 2016 08:09:10 -0700 Subject: [PATCH 15/27] Update README.md --- roles/upstream/README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/roles/upstream/README.md b/roles/upstream/README.md index bfba4512..ce4ffbe3 100644 --- a/roles/upstream/README.md +++ b/roles/upstream/README.md @@ -32,3 +32,20 @@ A combination of scripts in bash and python create the zipped smartphone downloa git pull ghunt upstream ./runtags upstream ``` + +**Assumptions:** + +* The number of downloaded APK's will be less than 10000 on a per station basis. -- Until proven to be a problem, each upload will contain a full data set -- the zip deflate is very good at substituting a token for repeating text chunks. (estimated record size = 30 which implies zip file size of 300K) +* Logrotate on the servers will make apache logs disappear. So a json representation of a python dictionary will be used to accumulate and preserve data +* The apache logs will be searched for ".apk", and GET, and status= 200 success, and added to the data set if found +* Date-time + downloaded URL will the the dictionary key. +* All available apache logs will be scanned, every time a zip file is to be generated, and each record "of interest" will be checked against the dictionary. +* output records in csv have the following fields: date-time, week number, URL,UUID of server. ( week permits quick/dirty trend charts) +* Initially, during the debug phase, the apache logs (full) will be included in the zip file. +* There is nothing propritary about the uploaded data, and it does not need to be encrypted, or the download protected with password. +*Currently, the xscenet@gmail.com account is accessed via a password, which is visible at github. I need to generate a ssl public/private key pair, and put it in place for the cloud to use. +* The email address of the data collector is known at the time of harvesting email from the gmail server. It is capturesd and presented somewhere in the cloud presentation. +* The zip file can easily contain additional information -- at this point it includes + * uptime + * vnstat + * apache logs From e5a4e4c10fd0251f43886c1783e484a02dd9ed5b Mon Sep 17 00:00:00 2001 From: George Hunt Date: Fri, 8 Jul 2016 12:56:50 -0700 Subject: [PATCH 16/27] move password to a file -- make it readable by apache, but not world --- roles/upstream/templates/harvest.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/roles/upstream/templates/harvest.py b/roles/upstream/templates/harvest.py index f6218c1f..21eb47ae 100755 --- a/roles/upstream/templates/harvest.py +++ b/roles/upstream/templates/harvest.py @@ -8,11 +8,14 @@ import json from time import sleep +# go get the password for interaction with xscenet@gmail.com +with open("/root/.xscenet_gmail",'r') as passwd: + credential = passwd.read() upenv = "{{ content_base }}/upstream" zips_dir = os.path.join(upenv,"html","zips") raw_dir = os.path.join(upenv,"html","raw_data") m = imaplib.IMAP4_SSL('imap.gmail.com') -m.login('xscenet@gmail.com', 'immi96?Bronx') +m.login('xscenet@gmail.com', credential) # declare location of dictionry with all our download data # -- the key for dictionary is datetime+download_url download_data = os.path.join(upenv,"downloads.json") From 1e7e9dc75681dcc95ad5006879284fa14305a397 Mon Sep 17 00:00:00 2001 From: George Hunt Date: Fri, 8 Jul 2016 13:15:36 -0700 Subject: [PATCH 17/27] cosmetic changes to upstream description --- roles/upstream/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/upstream/tasks/main.yml b/roles/upstream/tasks/main.yml index 9079de0f..09d9fe7d 100644 --- a/roles/upstream/tasks/main.yml +++ b/roles/upstream/tasks/main.yml @@ -45,7 +45,7 @@ - option: name value: Upstream - option: description - value: '"Upstream is method of communicating usage information from the XSCE server to a centralized data collection location, It uses a smart phone to download a zipped directory of information while offline, and then later, when connected to the internet, emails that package,as an attachment,upstream"' + value: '"Upstream is a method of communicating usage information from the XSCE server to a centralized data collection location. It uses a smart phone to download a zipped directory of information while offline, and then later, when connected to the internet, emails that package, as an attachment, upstream"' - option: enabled value: "{{ upstream_enabled }}" - option: installed From b9a76d81b3a8315d29a28c0a077bb2d515ee4d51 Mon Sep 17 00:00:00 2001 From: georgejhunt Date: Fri, 8 Jul 2016 13:21:58 -0700 Subject: [PATCH 18/27] Update README.md --- roles/upstream/README.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/roles/upstream/README.md b/roles/upstream/README.md index ce4ffbe3..97f71c8a 100644 --- a/roles/upstream/README.md +++ b/roles/upstream/README.md @@ -17,19 +17,16 @@ A combination of scripts in bash and python create the zipped smartphone downloa * Upstream.wsgi -- A browser request to a url on the XSCE server (http://wikihealth.lan/data) stimulates this python program. This in turn calls a bash script which generates the zipped information package. The completed zip file is returned to the browser's request. * mkchunk -- Is a bash script which generates the zipped requested information. In doing its job, it calls another python script (sift.py), which parses the apache log files, and selects the web server downloads that are of interest - * sift.py -- Takes advantage of apache log parsing functions, to provide flexible and easy access to the desired information. + * sift.py -- Takes advantage of apache log parsing functions, to provide flexible and easy access to the desired information. The logs will be searched for any downloads of files in /library/content/wikimed. * harvest.py -- Runs in the cloud, and uses python imap libraries to access the gmail mail server, and fetch the uploaded email, and their attached zip files. It unzips and consolidates the information, and makes it available at http://xscenet.net/analytics **How to install Upstream on XSCE - Release-6.1** * Follow the normal install instructions at https://github.com/XSCE/xsce/wiki/XSCE-Installation. - * Then add the "upstream" software: + * Then install and enable the "upstream" software: ``` echo "upstream_install: True" >> /opt/schoolserver/xsce/vars/local_vars.yml echo "upstream_enabled: True" >> /opt/schoolserver/xsce/vars/local_vars.yml cd /opt/schoolserver/xsce - git remote add ghunt https://github.com/georgejhunt/xsce - git checkout -b upstream - git pull ghunt upstream ./runtags upstream ``` From 21684a0e168852776f810f08e07c35eb59b51e6d Mon Sep 17 00:00:00 2001 From: George Hunt Date: Sat, 9 Jul 2016 20:54:13 -0700 Subject: [PATCH 19/27] use our own special token in log, and systemd to start and stop --- roles/upstream/defaults/main.yml | 1 + roles/upstream/tasks/main.yml | 17 +++++ roles/upstream/templates/acpower.service | 11 +++ roles/upstream/templates/analyze-power | 97 ++++++++++++++++++++++++ roles/upstream/templates/heartbeat | 8 ++ 5 files changed, 134 insertions(+) create mode 100644 roles/upstream/templates/acpower.service create mode 100755 roles/upstream/templates/analyze-power create mode 100755 roles/upstream/templates/heartbeat diff --git a/roles/upstream/defaults/main.yml b/roles/upstream/defaults/main.yml index cb795605..4dfceee3 100644 --- a/roles/upstream/defaults/main.yml +++ b/roles/upstream/defaults/main.yml @@ -1,2 +1,3 @@ upstream_install: False upstream_enabled: False +acpower_enabled: False diff --git a/roles/upstream/tasks/main.yml b/roles/upstream/tasks/main.yml index 09d9fe7d..20055863 100644 --- a/roles/upstream/tasks/main.yml +++ b/roles/upstream/tasks/main.yml @@ -23,6 +23,9 @@ - { src: 'sift.py', dest: '{{ content_base }}/upstream', group: "apache" , mode: '0755' } - { src: 'harvest.py', dest: '{{ content_base }}/upstream', group: "apache" , mode: '0755' } - { src: 'mkchunk', dest: '{{ content_base }}/upstream', group: "apache" , mode: '0755' } + - { src: 'acpower.service', dest: '/etc/systemd/system/', group: "root" , mode: '0644' } + - { src: 'heartbeat', dest: '/usr/bin/', group: "root" , mode: '0755' } + - { src: 'analyze-power', dest: '/usr/bin/', group: "root" , mode: '0755' } when: upstream_install == True - name: enable the downloading of data chunk @@ -36,6 +39,18 @@ state=absent when: not upstream_enabled == True +- name: Start periodic logging of acpower + service: name=acpower + state=started + enabled=yes + when: acpower_enabled == True + +- name: Stop periodic logging of acpower + service: name=acpower + state=stop + enabled=no + when: acpower_enabled != True + - name: add upstream to service list ini_file: dest='{{ service_filelist }}' section=upstream @@ -50,3 +65,5 @@ value: "{{ upstream_enabled }}" - option: installed value: "{{ upstream_install }}" + - option: acpower_enabled + value: "{{ acpower_enabled }}" diff --git a/roles/upstream/templates/acpower.service b/roles/upstream/templates/acpower.service new file mode 100644 index 00000000..92103fb3 --- /dev/null +++ b/roles/upstream/templates/acpower.service @@ -0,0 +1,11 @@ +[Unit] +Description=Puts heartbeat, startup, shutdown records into log +After=syslog.target + +[Service] +PIDFile=/var/run/acpower.pid +ExecStart=/usr/xs-heartbeat +ExecStop=/usr/bin/kill -SIGINT $MAINPID + +[Install] +WantedBy=multi-user.target diff --git a/roles/upstream/templates/analyze-power b/roles/upstream/templates/analyze-power new file mode 100755 index 00000000..57ed68f2 --- /dev/null +++ b/roles/upstream/templates/analyze-power @@ -0,0 +1,97 @@ +#!/bin/env python +# read apache logs, sifting for records we want to save + +import sys +from os import path +import os +import datetime +from pprint import pprint +import glob +import json +from acrecord import Tools, tzlocal, tzutc + +tz = tzlocal() +tzu = tzutc() +LOC='/library/upstream' + +# fetch the dictionary of previous downloads if it exists +if path.isfile(path.join(LOC,"data","downloads")): + strm = open(path.join(LOC,"data","downloads"),"r") + downloads = json.load(strm) +else: downloads = {} +added = 0 + +# get the UUID of this machine +with open("/etc/xsce/uuid", "r") as infile: + uuid=infile.read() + +# get the datetime tools +tools = Tools() + +filedata = [] +# traverse the apache logs and get the date in first record +# The following will need to change for Fedora +for fn in glob.glob('/var/log/messages*'): + for line in open(fn, 'r'): + datestr = line[0:15] + try: + dt= datetime.datetime.strptime(datestr,"%b %d %H:%M:%S") + except: + continue + dt = dt.replace(tzinfo=tz) + epoch = tools.tstamp(dt) + filedata.append( (epoch,fn) ) + break + +pprint(filedata) +# traverse the apache logs +started = False +for ts,fn in sorted(filedata): + for line in open(fn, 'r'): + + datestr = line[0:15] + try: + dt= datetime.datetime.strptime(datestr,"%b %d %H:%M:%S") + except: + continue + dt = dt.replace(tzinfo=tz) + epoch = tools.tstamp(dt) + if not started: + last_heartbeat = epoch + last_powerdown_epoch = epoch + started = True + # look for a start up record in the log + nibbles = line.split() + #pprint(nibbles) + if nibbles[4] != 'root:': continue + if nibbles[5] == 'xsce_startup': + pass + print epoch, "startup. Downtime:" epoch - last_heartbeat,line[0:14] + """ + if last_heartbeat == last_powerdown_epoch: + print 'normal shutdown' + else: + print 'power interruption shutdown' + """ + elif nibbles[5] == "xsce_tick": + print epoch,'tick',line[0:14] + last_heatbeat = epoch + elif nibbles[5] == 'xsce_shutdown' : + print epoch, 'Normal system shutdown',line[0:14] + last_heatbeat = epoch + last_powerdown_epoch = epoch + +# now store away the accumulated data + +with open(path.join(LOC,"data","downloads"),"w") as outfile: + json.dump(downloads, outfile) + +# now create the final csv file +outfile = open(path.join(LOC,"staging","downloads_csv"),'w') + +for key in sorted(downloads): + outfile.write("%s,%s,%s,%s,\n" % (downloads[key]["time"],\ + downloads[key]["week"],\ + downloads[key]["url"], uuid.rstrip(), )) + +# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 background=light diff --git a/roles/upstream/templates/heartbeat b/roles/upstream/templates/heartbeat new file mode 100755 index 00000000..02e4fbe5 --- /dev/null +++ b/roles/upstream/templates/heartbeat @@ -0,0 +1,8 @@ +#!/bin/bash -x +# this makes an entry in the log every 5 minutes +logger "xsce_startup +trap "logger xsce_shutdown" -sigint +while [ 0 ]; do + logger "xsce_tick" + sleep 300 +done From 70088627a8d36b795840072142343350830e524c Mon Sep 17 00:00:00 2001 From: George Hunt Date: Sat, 9 Jul 2016 21:41:19 -0700 Subject: [PATCH 20/27] try to get the daemon working --- roles/upstream/tasks/main.yml | 1 + roles/upstream/templates/acpower.service | 4 +- roles/upstream/templates/heartbeat | 4 +- roles/upstream/templates/xsce-acpower-init | 54 ++++++++++++++++++++++ 4 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 roles/upstream/templates/xsce-acpower-init diff --git a/roles/upstream/tasks/main.yml b/roles/upstream/tasks/main.yml index 20055863..4f7786bb 100644 --- a/roles/upstream/tasks/main.yml +++ b/roles/upstream/tasks/main.yml @@ -26,6 +26,7 @@ - { src: 'acpower.service', dest: '/etc/systemd/system/', group: "root" , mode: '0644' } - { src: 'heartbeat', dest: '/usr/bin/', group: "root" , mode: '0755' } - { src: 'analyze-power', dest: '/usr/bin/', group: "root" , mode: '0755' } + - { src: 'xsce-acpower-init', dest: '/usr/libexec/', group: "root" , mode: '0755' } when: upstream_install == True - name: enable the downloading of data chunk diff --git a/roles/upstream/templates/acpower.service b/roles/upstream/templates/acpower.service index 92103fb3..2b9d848f 100644 --- a/roles/upstream/templates/acpower.service +++ b/roles/upstream/templates/acpower.service @@ -4,8 +4,8 @@ After=syslog.target [Service] PIDFile=/var/run/acpower.pid -ExecStart=/usr/xs-heartbeat -ExecStop=/usr/bin/kill -SIGINT $MAINPID +ExecStart=/usr/libexec/xsce-acpower-init start +ExecStop=/usr/bin/kill -s SIGINT $MAINPID [Install] WantedBy=multi-user.target diff --git a/roles/upstream/templates/heartbeat b/roles/upstream/templates/heartbeat index 02e4fbe5..6d472f3e 100755 --- a/roles/upstream/templates/heartbeat +++ b/roles/upstream/templates/heartbeat @@ -1,8 +1,8 @@ #!/bin/bash -x # this makes an entry in the log every 5 minutes -logger "xsce_startup +logger xsce_startup trap "logger xsce_shutdown" -sigint while [ 0 ]; do - logger "xsce_tick" + logger xsce_tick sleep 300 done diff --git a/roles/upstream/templates/xsce-acpower-init b/roles/upstream/templates/xsce-acpower-init new file mode 100644 index 00000000..8811f7ea --- /dev/null +++ b/roles/upstream/templates/xsce-acpower-init @@ -0,0 +1,54 @@ +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Library General Public +# License along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA +# 02111-1307, USA. + +# xsce-cmdsrv This is the script that starts up the +# Command Server on the XSCE School server +# +# Source function library +. /etc/rc.d/init.d/functions + +PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin +PID_FILE="/var/run/acpower.pid" +#OPTS="$PID_FILE " +OPTS="--daemon " +prog=heartbeat +SERVER=/library/upstream/heartbeat +RETVAL=0 + +start() { + # Start daemons. + echo -n "Starting $prog: " + daemon --pidfile=${PID_FILE} $SERVER $OPTS + RETVAL=$? + return $RETVAL +} + +stop() { + # Stop is handled by /usr/bin/xsce-cmdsrv-ctl STOP + RETVAL=1 + return $RETVAL +} + +# See how we were called. +case "$1" in + start) + start + ;; + stop) + stop + ;; + + status) + status xsce-cmdsrv $PID_FILE + RETVAL=$? + ;; + *) + echo $"Usage: $0 {start|status}" + exit 1 +esac + +exit $RETVAL From e4eaa73fa809a8ae6c6943cad80e3001d3426f40 Mon Sep 17 00:00:00 2001 From: George Hunt Date: Sun, 10 Jul 2016 11:05:25 -0700 Subject: [PATCH 21/27] do not print ticks --- roles/upstream/tasks/main.yml | 5 +- roles/upstream/templates/acrecord.py | 603 +++++++++++++++++++++ roles/upstream/templates/analyze-power | 10 +- roles/upstream/templates/heartbeat | 7 +- roles/upstream/templates/xsce-acpower-init | 2 +- 5 files changed, 619 insertions(+), 8 deletions(-) create mode 100644 roles/upstream/templates/acrecord.py mode change 100644 => 100755 roles/upstream/templates/xsce-acpower-init diff --git a/roles/upstream/tasks/main.yml b/roles/upstream/tasks/main.yml index 4f7786bb..a65343bd 100644 --- a/roles/upstream/tasks/main.yml +++ b/roles/upstream/tasks/main.yml @@ -24,9 +24,10 @@ - { src: 'harvest.py', dest: '{{ content_base }}/upstream', group: "apache" , mode: '0755' } - { src: 'mkchunk', dest: '{{ content_base }}/upstream', group: "apache" , mode: '0755' } - { src: 'acpower.service', dest: '/etc/systemd/system/', group: "root" , mode: '0644' } - - { src: 'heartbeat', dest: '/usr/bin/', group: "root" , mode: '0755' } - - { src: 'analyze-power', dest: '/usr/bin/', group: "root" , mode: '0755' } + - { src: 'heartbeat', dest: '{{ content_base }}/upstream', group: "root" , mode: '0755' } + - { src: 'analyze-power', dest: '{{ content_base }}/upstream', group: "root" , mode: '0755' } - { src: 'xsce-acpower-init', dest: '/usr/libexec/', group: "root" , mode: '0755' } + - { src: 'acrecord.py', dest: '{{ content_base }}/upstream', group: "root" , mode: '0755' } when: upstream_install == True - name: enable the downloading of data chunk diff --git a/roles/upstream/templates/acrecord.py b/roles/upstream/templates/acrecord.py new file mode 100644 index 00000000..eac0e53f --- /dev/null +++ b/roles/upstream/templates/acrecord.py @@ -0,0 +1,603 @@ +#!/usr/bin/env python +# Copyright 2012 George Hunt -- georgejhunt@gmail.com +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +# reminder to myself: +# +# Time Zones are really difficult. datetime.now() returns local time, and +# tstampto string also generates a tz offset string. Perhaps simplist is to +# do allmy processing in UTC. + +import time +from subprocess import Popen, PIPE +import datetime +import os, sys +import logging +import json +import glob +from gettext import gettext as _ + +VERSION = "0.2" + +#WORK_DIR="/home/olpc" +WORK_DIR="." + +DATA_FILE = os.path.expanduser("~/.acpower") +# data_dict is global config file initialized in is_exist_data_file - used throughout +data_dict = {} + +"""the following class is stolen from dateutil -- becuse dateutil needs to be installed online and we're trying to make an offline install """ +ZERO = datetime.timedelta(0) +EPOCHORDINAL = datetime.datetime.utcfromtimestamp(0).toordinal() +GRAPH = True + +class PowerChunk(): + def __init__(self,pdstart,pdlength): + self.startsec = pdstart + self.lensec = pdlength + +class tzlocal(datetime.tzinfo): + global tz_offset + + _std_offset = datetime.timedelta(seconds=-time.timezone) + if time.daylight: + _dst_offset = datetime.timedelta(seconds=-time.altzone) + else: + _dst_offset = _std_offset + tz_offset = _std_offset.total_seconds() + + def utcoffset(self, dt): + if self._isdst(dt): + return self._dst_offset + else: + return self._std_offset + + def dst(self, dt): + if self._isdst(dt): + return self._dst_offset-self._std_offset + else: + return ZERO + + def tzname(self, dt): + return time.tzname[self._isdst(dt)] + + def _isdst(self, dt): + # We can't use mktime here. It is unstable when deciding if + # the hour near to a change is DST or not. + # + # timestamp = time.mktime((dt.year, dt.month, dt.day, dt.hour, + # dt.minute, dt.second, dt.weekday(), 0, -1)) + # return time.localtime(timestamp).tm_isdst + # + # The code above yields the following result: + # + #>>> import tz, datetime + #>>> t = tz.tzlocal() + #>>> datetime.datetime(2003,2,15,23,tzinfo=t).tzname() + #'BRDT' + #>>> datetime.datetime(2003,2,16,0,tzinfo=t).tzname() + #'BRST' + #>>> datetime.datetime(2003,2,15,23,tzinfo=t).tzname() + #'BRST' + #>>> datetime.datetime(2003,2,15,22,tzinfo=t).tzname() + #'BRDT' + #>>> datetime.datetime(2003,2,15,23,tzinfo=t).tzname() + #'BRDT' + # + # Here is a more stable implementation: + # + timestamp = ((dt.toordinal() - EPOCHORDINAL) * 86400 + + dt.hour * 3600 + + dt.minute * 60 + + dt.second) + return time.localtime(timestamp+time.timezone).tm_isdst + + def __eq__(self, other): + if not isinstance(other, tzlocal): + return False + return (self._std_offset == other._std_offset and + self._dst_offset == other._dst_offset) + return True + + def __ne__(self, other): + return not self.__eq__(other) + + def __repr__(self): + return "%s()" % self.__class__.__name__ + + __reduce__ = object.__reduce__ + +class tzutc(datetime.tzinfo): + + def utcoffset(self, dt): + return ZERO + + def dst(self, dt): + return ZERO + + def tzname(self, dt): + return "UTC" + + def __eq__(self, other): + return (isinstance(other, tzutc) or + (isinstance(other, tzoffset) and other._offset == ZERO)) + + def __ne__(self, other): + return not self.__eq__(other) + + def __repr__(self): + return "%s()" % self.__class__.__name__ + + __reduce__ = object.__reduce__ + +class AcException(): + def __init__(self, msg): + print(msg) + sys.exit(1) +# get a global instance of the tzlocal class +tz = tzlocal() +tzu = tzutc() +UTC2LOCAL = datetime.datetime.now(tz) - datetime.datetime.now(tzu) +UTC2LOCALSECONDS = UTC2LOCAL.total_seconds() + +class Tools: + def __init__(self): + global tz_offset + pass + + def cli(self, cmd): + """send cmd line to shell, rtn (text,error code)""" + p1 = Popen(cmd,stdout=PIPE, shell=True) + output = p1.communicate() + if p1.returncode != 0 : + print('error returned from shell command: %s was %s'%(cmd,output[0])) + return output[0],p1.returncode + + def is_exist_data_file(self): + #get the tmp data file + global data_dict + if (len(data_dict)> 0): + return True + try: + fd = file(DATA_FILE,'r') + data_str = fd.read() + data_dict = json.loads(data_str) + fd.close() + return True + except IOError: + return False + + def get_data_dict(self): + global data_dict + if self.is_exist_data_file(): + return data_dict + return None + + def put_data_file(self): + """ writes the data_dict """ + try: + fd = file(DATA_FILE,'w') + data_str = json.dumps(data_dict) + fd.write(data_str) + fd.close() + except IOError,e: + logging.exception("failed to write data file. error:%s"% (e,)) + raise AcException("Datafile write error") + + def print_data_file(self): + """ Prints the data_dict """ + try: + fd = file(DATA_FILE,'r') + data_str = json.dumps(data_dict) + fd.close() + print("Contents of persistent data record:") + for k in data_dict: + print(" ",k,data_dict[k]) + except IOError,e: + logging.exception("failed to write data file. error:%s"% (e,)) + raise AcException("Datafile write error") + + def get_summary_filename(self): + """ returns the filename of current summary file or "" if it doesn't exist """ + fn = os.path.join(SUMMARY_PREFIX,SUMMARY_CURRENT) + if (os.path.isfile(fn)): + try: + fd = open(fn,"r") + fname = fd.read() + except : + cmd = "rm -f %s"%fn + result,status = self.cli(cmd) + return "" + return fname + return "" + + + def get_datetime(self, datestr): + """ translate ymdhms string into datetime """ + dt = datetime.datetime.strptime(datestr, "%Y/%m/%d-%H:%M:%S-%Z") + if datestr.find("GMT"): + tzaware = dt.replace(tzinfo=tzu) + else: + tzaware = dt.replace(tzinfo=tz) + return tzaware + + def tstamp(self, dtime): + '''return a UNIX style seconds since 1970 for datetime input''' + epoch = datetime.datetime(1970, 1, 1,tzinfo=tzu) + newdtime = dtime.astimezone(tzu) + since_epoch_delta = newdtime - epoch + return since_epoch_delta.total_seconds() + + def get_utc_tmtamp_from_local_string(self,instr): + localdt = self.get_datetime(instr) + return self.tstamp(localdt) + tzoffset + + def parse_date(self,datestr): + try: + unawaredt = datetime.datetime.strptime(datestr, "%m/%d/%Y") + tzaware = unawaredt.replace(tzinfo=tz) + return self.tstamp(tzaware) # + tzoffset + except Exception as e: + print("returned error:%s. Error in date [%s]. Expected in format mm/dd/yyyy"% (e,datestr,)) + return 0L + + def str2tstamp(self, thestr): + '''return a UNIX style seconds since 1970 for string input''' + dtime = datetime.datetime.strptime(thestr.strip(), "%Y/%m/%d-%H:%M:%S-%Z") + awaredt = dtime.replace(tzinfo=tz) + newdtime = awaredt.astimezone(tz) + epoch = datetime.datetime(1970, 1, 1,tzinfo=tzu) + since_epoch_delta = newdtime - epoch + return since_epoch_delta.total_seconds() + + def tstamp_now(self): + """ return seconds since 1970 """ + return self.tstamp(datetime.datetime.now(tz)) + + def format_datetime(self, dt): + """ return ymdhms string """ + return datetime.datetime.strftime(dt, "%Y/%m/%d-%H:%M:%S-%z") + + def dhm_from_seconds(self,s): + """ translate seconds into days, hour, minutes """ + #print s + days, remainder = divmod(s, 86400) + hours, remainder = divmod(remainder, 3600) + minutes, remainder = divmod(remainder, 60) + return (days, hours, minutes) + + def ts2str(self,ts): + """ change a time stamp into a string expressed in local time zone""" + dttime = datetime.datetime.fromtimestamp(ts) + return self.format_datetime(dttime) + + def ts2date(self,ts): + """ change a time stamp into a string expressed in local time zone""" + dttime = datetime.datetime.fromtimestamp(ts) + return datetime.datetime.strftime(dttime, "%Y/%m/%d") + +class ShowPowerHistory(Tools): + def __init__(self): + global tz_offset + pass + + def set_start(self, start_str, debug): + self.is_exist_data_file() + start = self.parse_date(start_str) + if start <> 0: + data_dict["start"] = start + if not data_dict.has_key("end"): + data_dict["end"] = self.parse_date("12/31/2030") + self.put_data_file() + if debug: + self.print_data_file() + sys.exit(0) + else: + print("start date not recognized") + sys.exit(0) + + + def set_end(self, end_str, debug): + self.is_exist_data_file() + end = self.parse_date(end_str) + if end <> 0: + data_dict['end'] = end + self.put_data_file() + if debug: + self.print_data_file() + sys.exit(0) + else: + print("end date not recognized") + sys.exit(0) + + def output_summary(self, data, args): + # online is a dictoionary with key=start_time_stamp, value on time_seconds + debug = args.verbose + MATRIX = args.daily + if args.start: + self.set_start(args.start, args.verbose) + if args.end: + self.set_end(args.end, args.verbose) + # online is a dictionary. key=utc_timestamp when power came on, value=seconds ontime + online = {} + gap_start = None + if len(data) == 0: + print("No data for this period") + sys.exit(0) + first_ts = data[0][0] + first_str = self.ts2date(first_ts) + last_ts = data[len(data)-1][0] + last_str = self.ts2date(last_ts) + print("\n SUMMARY OF AC POWER DURING PERIOD: %s to %s" % (first_str, last_str,)) + print(" (Data ignored outside time period between %s and %s.)\n" % (self.ts2date(data_dict["start"]), self.ts2date(data_dict["end"]),)) + first = data[0][0] + power_state = None + power_start = first + for index in range(len(data)): + if debug: + print(data[index][0], data[index][7],data[index][8], self.ts2str(data[index][0])) + if data[index][7].find('ac-online-event') != -1 or \ + data[index][7].find('startup') != -1: + if not power_state: + power_start = data[index][0] + power_state = True + elif data[index][7].find('ac-offline-event') != -1: + online[power_start] = data[index][0] - power_start + power_state = None + elif power_state and data[index][7].find('shutdown') != -1: + online[power_start] = data[index][0] - power_start + power_state = None + for k in sorted(online): + if debug: + print(self.ts2str(k), "minutes:",(online[k])/60) + total_seconds = last_ts - first_ts + (days, hours, minutes) = self.dhm_from_seconds(total_seconds) + print "length of log %s days, %s hours, %s minutes" % (days, hours, minutes) + number_of_gaps = len(online) - 1 + print "number of power outages: %s" % number_of_gaps + # mysum is total power online seconds + mysum = 0L + power_list = [] + gap_length_list = [] + first = False + if len(online) > 0: + for key in sorted(online): + mysum += online[key] + power_list.append( (key,online[key]) ) + if first: + gap_length_list.append( (last_key + online[last_key],key -last_key + online[last_key]) ) + first = True + last_key = key + # compute the average length of outage + average_seconds = (total_seconds - mysum) / float(number_of_gaps) + (days, hours, minutes) = self.dhm_from_seconds(average_seconds) + print "average length of outage: %s days %s hours %s minutes" % \ + (days, hours,minutes) + gap_list = sorted(gap_length_list, key=lambda x:x[1]) + ts_list = sorted(power_list) + last_seconds = power_list[len(power_list)-1][0] + power_list[len(power_list)-1][1] + if debug: + print("power_list - utc_start, power on seconds") + for item, value in power_list: + print item, value + shortest_gap = gap_list[0][1] + if debug: + print("sorted gap list:") + for i in range(len(gap_list)): + print(gap_list[i][0],gap_list[i][1]) + if shortest_gap < 60: + print "shortest outage: %s seconds " % (shortest_gap) + else: + (days, hours, minutes) = self.dhm_from_seconds(shortest_gap) + print "shortest outage: %s days %s hours %s minutes " % \ + (days, hours, minutes,) + longest_gap = gap_list[len(gap_list)-1][1] + (days, hours, minutes) = self.dhm_from_seconds(longest_gap) + print "longest outage: %s days %s hours %s minutes " % \ + (days,hours, minutes,) + average_per_day = (float(mysum)/total_seconds) * 24 + print("Average power within 24 hours:%2.2f hours"%average_per_day) + + print "\n\nDISTRUBUTION OF POWER OVER THE DAY" + buckets = [] + + #divide up the total time into 15 minute chunks and distribute + # X's across the 96 columns of a day for each chunk that has power + + # first get the offset of the first entry from midnight + firstdt = datetime.datetime.fromtimestamp(first_ts, tz) + first_midnight_local = firstdt - datetime.timedelta(\ + hours=firstdt.hour,minutes=firstdt.minute, seconds=firstdt.second) + # lets do all the time in seconds since 1970 (tstamp) and in UTC + midnight_str = self.format_datetime(first_midnight_local) + if debug: + print("midnight should be:%s"%midnight_str) + first_midnight_seconds = self.tstamp(first_midnight_local) + last_seconds = last_ts + current_bucket_seconds = first_midnight_seconds + current_power_state = False # we backtracked from the time when the monitor was enabled + key_index = 0 + power_on_seconds = first_ts + power_off_seconds = power_on_seconds + power_list[0][1] + seconds_in_day = 24.0 * 60 * 60 + seconds_in_current_day = 1000 + bucket_size = 60 * 15.0 + + if debug: + for k,v in ts_list: + print("key:%s, string:%s, value:%s"%(k,self.ts2str(k), v,)) + print("Before loop begins: on:%s, off:%s,bucket_seconds:%s"%(\ + self.ts2str(power_on_seconds),self.ts2str(power_off_seconds),\ + self.ts2str(current_bucket_seconds),)) + buckets = [] + for j in range(96): + buckets.append(0) + if MATRIX: + current_day_ts = first_ts + current_day_str = self.ts2date(current_day_ts) + print("One line per day. Current day: %s"%current_day_str) + while current_bucket_seconds < last_seconds: + for index in range(len(ts_list)): + if ts_list[index][0] > current_bucket_seconds: + break + if ts_list[index][0] + ts_list[index][1] > current_bucket_seconds: + current_power_state = True + else: + current_power_state = False + if MATRIX: + if current_power_state: + sys.stdout.write("X") + else: + sys.stdout.write(" ") + if GRAPH: + bucket_index = int(seconds_in_current_day / bucket_size) + if current_power_state: + buckets[bucket_index] += 1 + + + current_bucket_seconds += bucket_size + seconds_in_current_day = (current_bucket_seconds - first_midnight_seconds) % seconds_in_day + if seconds_in_current_day < 10 and MATRIX: + print + if (current_bucket_seconds - first_midnight_seconds)%864000 >= 863100 and MATRIX: + current_day_str = self.ts2date(current_bucket_seconds) + print("\n%s 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23"%current_day_str) + + if GRAPH: +# find the max of the buckets + print("\nBar Graph") + bucket_max = max(buckets) + if debug: + print("bucket_max:%s"%bucket_max) + for row in range(bucket_max - 1,-1,-1): + for i in range(96): + if row + 1 - buckets[i] <= 0: + sys.stdout.write("X") + else: + sys.stdout.write(" ") + print + print "\n0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23" + + if args.powersegments: + print("\nINDIVIDUAL POWER PERIODS:") + for item, value in power_list: + localts = item + localstr = self.ts2str(localts) + (days, hours, minutes) = self.dhm_from_seconds(value) + print "%s %s days %s hours and %s minutes" % \ + (localstr, days, hours, minutes, ) + def output_state(self): + if self.isenabled(): + state = "ENABLED" + else: + state = "DISABLED" + print("") + print("AC Power Monitor is currently %s"%state) + +class RawData(Tools): + def __init__(self): + global data_dict + datafile_exists = False + try: + fd = file(DATA_FILE,'r') + data_str = fd.read() + data_dict = json.loads(data_str) + fd.close() + datafile_exists = True + except IOError,e: + logging.exception("failed to write data file. error:%s"% (e,)) + #raise AcException("Datafile read error in RawData") + if datafile_exists: + keylist = sorted(data_dict.keys()) + print "Current data file:" + for item in keylist: + print item, data_dict[item] + print("Battery state percent:%s"%(self.get_battery_percent(),)) + + global summary_dict + name = self.get_summary_filename() + if (len(name)>0): + try: + fsummary = file(name,'r') + data_str = fsummary.read() + summary_dict = json.loads(data_str) + except IOError: + raise AcException("Summaary file read error in init of RawData") + keylist = sorted(summary_dict.keys()) + print "Summary file:" + for item in keylist: + print item, summary_dict[item] + +""" +if __name__ == "__main__": + tls = Tools() + try: + tzfile = open("%s/timezone"%WORK_DIR, "r") + timezone = tzfile.read() + tzfile.close() + except IOError: + Print("could not set timezone") + exit(1) + print("Timezone is set to %s"%timezone) + local_ts = tls.tstamp(datetime.datetime.now(tz)) #returns local time + tzoffset = time.time()-local_ts #time returns UTC + matrix = False + print("tz offset in seconds:%s"%tzoffset) + mystrnow = tls.format_datetime(datetime.datetime.now(tz)) + print("the string version of now:%s"%mystrnow) + mydt = tls.get_datetime(mystrnow) + myts = tls.tstamp(mydt) + print("the timestamp is %s"%myts) + print("The UTC timestame is %s"%time.time()) + print("returned by get_utc_from_string:%s"%tls.get_utc_tstamp_from_local_string(mystrnow)) + mynewstr = tls.ts2str(myts) + print("and the final string is %s"%mynewstr) + print("utc2local:%s"%UTC2LOCALSECONDS) + dif = time.time()-myts + print("the difference between local ts and utc is %s"%dif) + print("the corrected version o string is %s"%tls.ts2str(myts)) + if len(sys.argv) == 1: + pi = ShowPowerHistory() + elif (len(sys.argv )== 2): + # if coming from cron, the check for an action to do + if sys.argv[1] == '--timeout': + print("environment value of TZ:%s"%os.environ["TZ"]) + pa = CollectData() + # dump the data in understandable form + if sys.argv[1] == '--debug': + debug = True + matrix = True + pa = RawData() + pi = ShowPowerHistory() + matrix = False + debug = False + if sys.argv[1] == '--delete': + tools = Tools() + tools.disable() + tools.delete() + if sys.argv[1] == '--enable': + tools = Tools() + tools.enable() + if sys.argv[1] == '--disable': + tools = Tools() + tools.disable() + sys.exit(0) + + # pop up the GUI + #Gtk.main() + sys.exit(0) +""" +# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 diff --git a/roles/upstream/templates/analyze-power b/roles/upstream/templates/analyze-power index 57ed68f2..b62b8469 100755 --- a/roles/upstream/templates/analyze-power +++ b/roles/upstream/templates/analyze-power @@ -59,14 +59,15 @@ for ts,fn in sorted(filedata): if not started: last_heartbeat = epoch last_powerdown_epoch = epoch + last_powerup = epoch started = True # look for a start up record in the log nibbles = line.split() #pprint(nibbles) if nibbles[4] != 'root:': continue if nibbles[5] == 'xsce_startup': - pass - print epoch, "startup. Downtime:" epoch - last_heartbeat,line[0:14] + last_powerup = epoch + print epoch, "startup. Downtime:", epoch - last_heartbeat,line[0:14] """ if last_heartbeat == last_powerdown_epoch: print 'normal shutdown' @@ -74,10 +75,11 @@ for ts,fn in sorted(filedata): print 'power interruption shutdown' """ elif nibbles[5] == "xsce_tick": - print epoch,'tick',line[0:14] + #print epoch,'tick',line[0:14] last_heatbeat = epoch elif nibbles[5] == 'xsce_shutdown' : - print epoch, 'Normal system shutdown',line[0:14] + + print epoch, 'Normal system shutdown',line[0:14],"Powered seconds: ",epoch - last_poweerup last_heatbeat = epoch last_powerdown_epoch = epoch diff --git a/roles/upstream/templates/heartbeat b/roles/upstream/templates/heartbeat index 6d472f3e..8f7a64a9 100755 --- a/roles/upstream/templates/heartbeat +++ b/roles/upstream/templates/heartbeat @@ -1,7 +1,12 @@ #!/bin/bash -x # this makes an entry in the log every 5 minutes +cd / +echo $$ > /var/run/acpower.pid logger xsce_startup -trap "logger xsce_shutdown" -sigint +trap record_shutdown -s sigint +record_shutdown { + logger xsce_shutdown +} while [ 0 ]; do logger xsce_tick sleep 300 diff --git a/roles/upstream/templates/xsce-acpower-init b/roles/upstream/templates/xsce-acpower-init old mode 100644 new mode 100755 index 8811f7ea..5d2d4b02 --- a/roles/upstream/templates/xsce-acpower-init +++ b/roles/upstream/templates/xsce-acpower-init @@ -22,7 +22,7 @@ RETVAL=0 start() { # Start daemons. echo -n "Starting $prog: " - daemon --pidfile=${PID_FILE} $SERVER $OPTS + nohup $SERVER $OPTS & RETVAL=$? return $RETVAL } From 8eaf266f697f72864790a84b773e3be016b1947f Mon Sep 17 00:00:00 2001 From: George Hunt Date: Sun, 10 Jul 2016 11:06:32 -0700 Subject: [PATCH 22/27] change upstream => reports in roles --- roles/{upstream => reports}/README.md | 0 roles/{upstream => reports}/defaults/main.yml | 0 roles/{upstream => reports}/tasks/main.yml | 0 roles/{upstream => reports}/templates/acpower.service | 0 roles/{upstream => reports}/templates/acrecord.py | 0 roles/{upstream => reports}/templates/analyze-power | 0 roles/{upstream => reports}/templates/cp2git | 0 roles/{upstream => reports}/templates/harvest.py | 0 roles/{upstream => reports}/templates/heartbeat | 0 roles/{upstream => reports}/templates/mkchunk | 0 roles/{upstream => reports}/templates/sift.py | 0 roles/{upstream => reports}/templates/upstream.conf | 0 roles/{upstream => reports}/templates/upstream.wsgi | 0 roles/{upstream => reports}/templates/xsce-acpower-init | 0 14 files changed, 0 insertions(+), 0 deletions(-) rename roles/{upstream => reports}/README.md (100%) rename roles/{upstream => reports}/defaults/main.yml (100%) rename roles/{upstream => reports}/tasks/main.yml (100%) rename roles/{upstream => reports}/templates/acpower.service (100%) rename roles/{upstream => reports}/templates/acrecord.py (100%) rename roles/{upstream => reports}/templates/analyze-power (100%) rename roles/{upstream => reports}/templates/cp2git (100%) rename roles/{upstream => reports}/templates/harvest.py (100%) rename roles/{upstream => reports}/templates/heartbeat (100%) rename roles/{upstream => reports}/templates/mkchunk (100%) rename roles/{upstream => reports}/templates/sift.py (100%) rename roles/{upstream => reports}/templates/upstream.conf (100%) rename roles/{upstream => reports}/templates/upstream.wsgi (100%) rename roles/{upstream => reports}/templates/xsce-acpower-init (100%) diff --git a/roles/upstream/README.md b/roles/reports/README.md similarity index 100% rename from roles/upstream/README.md rename to roles/reports/README.md diff --git a/roles/upstream/defaults/main.yml b/roles/reports/defaults/main.yml similarity index 100% rename from roles/upstream/defaults/main.yml rename to roles/reports/defaults/main.yml diff --git a/roles/upstream/tasks/main.yml b/roles/reports/tasks/main.yml similarity index 100% rename from roles/upstream/tasks/main.yml rename to roles/reports/tasks/main.yml diff --git a/roles/upstream/templates/acpower.service b/roles/reports/templates/acpower.service similarity index 100% rename from roles/upstream/templates/acpower.service rename to roles/reports/templates/acpower.service diff --git a/roles/upstream/templates/acrecord.py b/roles/reports/templates/acrecord.py similarity index 100% rename from roles/upstream/templates/acrecord.py rename to roles/reports/templates/acrecord.py diff --git a/roles/upstream/templates/analyze-power b/roles/reports/templates/analyze-power similarity index 100% rename from roles/upstream/templates/analyze-power rename to roles/reports/templates/analyze-power diff --git a/roles/upstream/templates/cp2git b/roles/reports/templates/cp2git similarity index 100% rename from roles/upstream/templates/cp2git rename to roles/reports/templates/cp2git diff --git a/roles/upstream/templates/harvest.py b/roles/reports/templates/harvest.py similarity index 100% rename from roles/upstream/templates/harvest.py rename to roles/reports/templates/harvest.py diff --git a/roles/upstream/templates/heartbeat b/roles/reports/templates/heartbeat similarity index 100% rename from roles/upstream/templates/heartbeat rename to roles/reports/templates/heartbeat diff --git a/roles/upstream/templates/mkchunk b/roles/reports/templates/mkchunk similarity index 100% rename from roles/upstream/templates/mkchunk rename to roles/reports/templates/mkchunk diff --git a/roles/upstream/templates/sift.py b/roles/reports/templates/sift.py similarity index 100% rename from roles/upstream/templates/sift.py rename to roles/reports/templates/sift.py diff --git a/roles/upstream/templates/upstream.conf b/roles/reports/templates/upstream.conf similarity index 100% rename from roles/upstream/templates/upstream.conf rename to roles/reports/templates/upstream.conf diff --git a/roles/upstream/templates/upstream.wsgi b/roles/reports/templates/upstream.wsgi similarity index 100% rename from roles/upstream/templates/upstream.wsgi rename to roles/reports/templates/upstream.wsgi diff --git a/roles/upstream/templates/xsce-acpower-init b/roles/reports/templates/xsce-acpower-init similarity index 100% rename from roles/upstream/templates/xsce-acpower-init rename to roles/reports/templates/xsce-acpower-init From e87be7b1a0764b84607f68aa5dfa9f650e584c30 Mon Sep 17 00:00:00 2001 From: George Hunt Date: Sun, 10 Jul 2016 11:20:17 -0700 Subject: [PATCH 23/27] change upstream to reports in all --- roles/reports/README.md | 8 ++-- roles/reports/defaults/main.yml | 4 +- roles/reports/tasks/main.yml | 46 +++++++++---------- roles/reports/templates/analyze-power | 2 +- roles/reports/templates/cp2git | 6 +-- roles/reports/templates/harvest.py | 2 +- roles/reports/templates/mkchunk | 6 +-- roles/reports/templates/reports.conf | 15 ++++++ .../templates/{upstream.wsgi => reports.wsgi} | 2 +- roles/reports/templates/sift.py | 2 +- roles/reports/templates/upstream.conf | 15 ------ roles/reports/templates/xsce-acpower-init | 2 +- 12 files changed, 55 insertions(+), 55 deletions(-) create mode 100644 roles/reports/templates/reports.conf rename roles/reports/templates/{upstream.wsgi => reports.wsgi} (96%) delete mode 100644 roles/reports/templates/upstream.conf diff --git a/roles/reports/README.md b/roles/reports/README.md index 97f71c8a..92aec28f 100644 --- a/roles/reports/README.md +++ b/roles/reports/README.md @@ -22,12 +22,12 @@ A combination of scripts in bash and python create the zipped smartphone downloa **How to install Upstream on XSCE - Release-6.1** * Follow the normal install instructions at https://github.com/XSCE/xsce/wiki/XSCE-Installation. - * Then install and enable the "upstream" software: + * Then install and enable the "reports" software: ``` - echo "upstream_install: True" >> /opt/schoolserver/xsce/vars/local_vars.yml - echo "upstream_enabled: True" >> /opt/schoolserver/xsce/vars/local_vars.yml + echo "reports_install: True" >> /opt/schoolserver/xsce/vars/local_vars.yml + echo "reports_enabled: True" >> /opt/schoolserver/xsce/vars/local_vars.yml cd /opt/schoolserver/xsce - ./runtags upstream + ./runtags reports ``` **Assumptions:** diff --git a/roles/reports/defaults/main.yml b/roles/reports/defaults/main.yml index 4dfceee3..e771c5f3 100644 --- a/roles/reports/defaults/main.yml +++ b/roles/reports/defaults/main.yml @@ -1,3 +1,3 @@ -upstream_install: False -upstream_enabled: False +reports_install: False +reports_enabled: False acpower_enabled: False diff --git a/roles/reports/tasks/main.yml b/roles/reports/tasks/main.yml index a65343bd..83d130f6 100644 --- a/roles/reports/tasks/main.yml +++ b/roles/reports/tasks/main.yml @@ -5,11 +5,11 @@ mode=0755 state=directory with_items: - - "{{ content_base }}/upstream/html/raw_data" - - "{{ content_base }}/upstream/html/zips" - - "{{ content_base }}/upstream/history" - - "{{ content_base }}/upstream/staging" - - "{{ content_base }}/upstream/data" + - "{{ content_base }}/reports/html/raw_data" + - "{{ content_base }}/reports/html/zips" + - "{{ content_base }}/reports/history" + - "{{ content_base }}/reports/staging" + - "{{ content_base }}/reports/data" - name: Put files where they belong template: src={{ item.src }} @@ -18,28 +18,28 @@ group={{ item.group }} mode={{ item.mode }} with_items: - - { src: 'upstream.conf', dest: '/etc/httpd/conf.d/', group: "root" , mode: '0644' } - - { src: 'upstream.wsgi', dest: '{{ content_base }}/upstream', group: "root" , mode: '0644' } - - { src: 'sift.py', dest: '{{ content_base }}/upstream', group: "apache" , mode: '0755' } - - { src: 'harvest.py', dest: '{{ content_base }}/upstream', group: "apache" , mode: '0755' } - - { src: 'mkchunk', dest: '{{ content_base }}/upstream', group: "apache" , mode: '0755' } + - { src: 'reports.conf', dest: '/etc/httpd/conf.d/', group: "root" , mode: '0644' } + - { src: 'reports.wsgi', dest: '{{ content_base }}/reports', group: "root" , mode: '0644' } + - { src: 'sift.py', dest: '{{ content_base }}/reports', group: "apache" , mode: '0755' } + - { src: 'harvest.py', dest: '{{ content_base }}/reports', group: "apache" , mode: '0755' } + - { src: 'mkchunk', dest: '{{ content_base }}/reports', group: "apache" , mode: '0755' } - { src: 'acpower.service', dest: '/etc/systemd/system/', group: "root" , mode: '0644' } - - { src: 'heartbeat', dest: '{{ content_base }}/upstream', group: "root" , mode: '0755' } - - { src: 'analyze-power', dest: '{{ content_base }}/upstream', group: "root" , mode: '0755' } + - { src: 'heartbeat', dest: '{{ content_base }}/reports', group: "root" , mode: '0755' } + - { src: 'analyze-power', dest: '{{ content_base }}/reports', group: "root" , mode: '0755' } - { src: 'xsce-acpower-init', dest: '/usr/libexec/', group: "root" , mode: '0755' } - - { src: 'acrecord.py', dest: '{{ content_base }}/upstream', group: "root" , mode: '0755' } - when: upstream_install == True + - { src: 'acrecord.py', dest: '{{ content_base }}/reports', group: "root" , mode: '0755' } + when: reports_install == True - name: enable the downloading of data chunk - template: src='upstream.conf' + template: src='reports.conf' mode=0644 dest=/etc/httpd/conf.d/ - when: upstream_enabled + when: reports_enabled - name: remove config file to disable - file: path=/etc/httpd/conf.d/upstream.conf + file: path=/etc/httpd/conf.d/reports.conf state=absent - when: not upstream_enabled == True + when: not reports_enabled == True - name: Start periodic logging of acpower service: name=acpower @@ -53,19 +53,19 @@ enabled=no when: acpower_enabled != True -- name: add upstream to service list +- name: add reports to service list ini_file: dest='{{ service_filelist }}' - section=upstream + section=reports option='{{ item.option }}' value='{{ item.value }}' with_items: - option: name value: Upstream - option: description - value: '"Upstream is a method of communicating usage information from the XSCE server to a centralized data collection location. It uses a smart phone to download a zipped directory of information while offline, and then later, when connected to the internet, emails that package, as an attachment, upstream"' + value: '"Upstream is a method of communicating usage information from the XSCE server to a centralized data collection location. It uses a smart phone to download a zipped directory of information while offline, and then later, when connected to the internet, emails that package, as an attachment, reports"' - option: enabled - value: "{{ upstream_enabled }}" + value: "{{ reports_enabled }}" - option: installed - value: "{{ upstream_install }}" + value: "{{ reports_install }}" - option: acpower_enabled value: "{{ acpower_enabled }}" diff --git a/roles/reports/templates/analyze-power b/roles/reports/templates/analyze-power index b62b8469..471d8a19 100755 --- a/roles/reports/templates/analyze-power +++ b/roles/reports/templates/analyze-power @@ -12,7 +12,7 @@ from acrecord import Tools, tzlocal, tzutc tz = tzlocal() tzu = tzutc() -LOC='/library/upstream' +LOC='/library/reports' # fetch the dictionary of previous downloads if it exists if path.isfile(path.join(LOC,"data","downloads")): diff --git a/roles/reports/templates/cp2git b/roles/reports/templates/cp2git index ac517d90..3caf41f2 100755 --- a/roles/reports/templates/cp2git +++ b/roles/reports/templates/cp2git @@ -1,5 +1,5 @@ #!/bin/bash -x -# copy scripts to the upstream role in xsce git -for f in `find /library/upstream -maxdepth 1 -type f`; do - cp $f /opt/schoolserver/xsce/roles/upstream/templates/ +# copy scripts to the reports role in xsce git +for f in `find /library/reports -maxdepth 1 -type f`; do + cp $f /opt/schoolserver/xsce/roles/reports/templates/ done diff --git a/roles/reports/templates/harvest.py b/roles/reports/templates/harvest.py index 21eb47ae..8711eecd 100755 --- a/roles/reports/templates/harvest.py +++ b/roles/reports/templates/harvest.py @@ -11,7 +11,7 @@ # go get the password for interaction with xscenet@gmail.com with open("/root/.xscenet_gmail",'r') as passwd: credential = passwd.read() -upenv = "{{ content_base }}/upstream" +upenv = "{{ content_base }}/reports" zips_dir = os.path.join(upenv,"html","zips") raw_dir = os.path.join(upenv,"html","raw_data") m = imaplib.IMAP4_SSL('imap.gmail.com') diff --git a/roles/reports/templates/mkchunk b/roles/reports/templates/mkchunk index dd577b2d..964ab7f7 100755 --- a/roles/reports/templates/mkchunk +++ b/roles/reports/templates/mkchunk @@ -1,9 +1,9 @@ #!/bin/bash -x -# generate a zip file with data bound for upstream +# generate a zip file with data bound for reports # -# define the upstream location in filesystem tree -UPENV={{ content_base}}/upstream +# define the reports location in filesystem tree +UPENV={{ content_base}}/reports # get the UUID for this device UUID=`cat /etc/xsce/uuid` diff --git a/roles/reports/templates/reports.conf b/roles/reports/templates/reports.conf new file mode 100644 index 00000000..7f1063dd --- /dev/null +++ b/roles/reports/templates/reports.conf @@ -0,0 +1,15 @@ +#WikiHealth Apache2 configuration file + +XSendFile on +XSendFilePath {{ content_base }}/content + +WSGIScriptAlias /analytics {{ content_base }}/reports/reports.wsgi + +Alias /communityanalytics {{ content_base }}/reports/html +Alias /analytics {{ content_base }}/reports/html + + + Options +Indexes + IndexOptions FancyIndexing + require all granted + diff --git a/roles/reports/templates/upstream.wsgi b/roles/reports/templates/reports.wsgi similarity index 96% rename from roles/reports/templates/upstream.wsgi rename to roles/reports/templates/reports.wsgi index e916c6ae..087ac8dc 100755 --- a/roles/reports/templates/upstream.wsgi +++ b/roles/reports/templates/reports.wsgi @@ -7,7 +7,7 @@ import glob from wsgiref.simple_server import make_server def application(environ, start_response): - upenv = "{{ content_base }}/upstream" + upenv = "{{ content_base }}/reports" block_size = 1024 # execute the bash script which generates the zip file diff --git a/roles/reports/templates/sift.py b/roles/reports/templates/sift.py index d7ad17f1..a2e26545 100755 --- a/roles/reports/templates/sift.py +++ b/roles/reports/templates/sift.py @@ -10,7 +10,7 @@ import glob import json -LOC='{{ content_base }}/upstream' +LOC='{{ content_base }}/reports' # fetch the dictionary of previous downloads if it exists if path.isfile(path.join(LOC,"data","downloads")): diff --git a/roles/reports/templates/upstream.conf b/roles/reports/templates/upstream.conf deleted file mode 100644 index 912ede47..00000000 --- a/roles/reports/templates/upstream.conf +++ /dev/null @@ -1,15 +0,0 @@ -#WikiHealth Apache2 configuration file - -XSendFile on -XSendFilePath {{ content_base }}/content - -WSGIScriptAlias /analytics {{ content_base }}/upstream/upstream.wsgi - -Alias /communityanalytics {{ content_base }}/upstream/html -Alias /analytics {{ content_base }}/upstream/html - - - Options +Indexes - IndexOptions FancyIndexing - require all granted - diff --git a/roles/reports/templates/xsce-acpower-init b/roles/reports/templates/xsce-acpower-init index 5d2d4b02..0beb679e 100755 --- a/roles/reports/templates/xsce-acpower-init +++ b/roles/reports/templates/xsce-acpower-init @@ -16,7 +16,7 @@ PID_FILE="/var/run/acpower.pid" #OPTS="$PID_FILE " OPTS="--daemon " prog=heartbeat -SERVER=/library/upstream/heartbeat +SERVER=/library/reports/heartbeat RETVAL=0 start() { From fa8725770f371fd7f9fa553eeb0f4ded85eb5988 Mon Sep 17 00:00:00 2001 From: George Hunt Date: Sun, 10 Jul 2016 11:29:26 -0700 Subject: [PATCH 24/27] remember 8-mgmt-tools/meta --- roles/8-mgmt-tools/meta/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/8-mgmt-tools/meta/main.yml b/roles/8-mgmt-tools/meta/main.yml index 6cc7435b..e5a76aa9 100644 --- a/roles/8-mgmt-tools/meta/main.yml +++ b/roles/8-mgmt-tools/meta/main.yml @@ -8,4 +8,4 @@ dependencies: - { role: phpmyadmin, tags: ['services','phpmyadmin','tools'], when: phpmyadmin_install } - { role: awstats, tags: ['services','awstats','tools'], when: awstats_install } - { role: teamviewer, tags: ['services','teamviewer','tools'], when: teamviewer_install } - - { role: upstream, tags: ['upstream'], when: upstream_install } + - { role: reports, tags: ['reports'], when: reports_install } From b89cbcf476623f6c06467a6580dcb2931841d51e Mon Sep 17 00:00:00 2001 From: George Hunt Date: Sun, 10 Jul 2016 13:43:54 -0700 Subject: [PATCH 25/27] stop =>stopped --- roles/reports/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/reports/tasks/main.yml b/roles/reports/tasks/main.yml index 83d130f6..51246911 100644 --- a/roles/reports/tasks/main.yml +++ b/roles/reports/tasks/main.yml @@ -49,7 +49,7 @@ - name: Stop periodic logging of acpower service: name=acpower - state=stop + state=stopped enabled=no when: acpower_enabled != True From 68e269a20cc122a841165dd23f5c5fd583e6ea65 Mon Sep 17 00:00:00 2001 From: George Hunt Date: Sun, 10 Jul 2016 20:31:51 -0400 Subject: [PATCH 26/27] refinement, test, until it works -- acpower --- roles/reports/tasks/main.yml | 2 +- roles/reports/templates/acpower.service | 3 ++- roles/reports/templates/heartbeat | 5 +++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/roles/reports/tasks/main.yml b/roles/reports/tasks/main.yml index 83d130f6..51246911 100644 --- a/roles/reports/tasks/main.yml +++ b/roles/reports/tasks/main.yml @@ -49,7 +49,7 @@ - name: Stop periodic logging of acpower service: name=acpower - state=stop + state=stopped enabled=no when: acpower_enabled != True diff --git a/roles/reports/templates/acpower.service b/roles/reports/templates/acpower.service index 2b9d848f..fcd11e4f 100644 --- a/roles/reports/templates/acpower.service +++ b/roles/reports/templates/acpower.service @@ -5,7 +5,8 @@ After=syslog.target [Service] PIDFile=/var/run/acpower.pid ExecStart=/usr/libexec/xsce-acpower-init start -ExecStop=/usr/bin/kill -s SIGINT $MAINPID +# docs say systemd sents sigterm +#ExecStop=/usr/bin/kill -s SIGINT $MAINPID [Install] WantedBy=multi-user.target diff --git a/roles/reports/templates/heartbeat b/roles/reports/templates/heartbeat index 8f7a64a9..5a13e914 100755 --- a/roles/reports/templates/heartbeat +++ b/roles/reports/templates/heartbeat @@ -3,9 +3,10 @@ cd / echo $$ > /var/run/acpower.pid logger xsce_startup -trap record_shutdown -s sigint -record_shutdown { +trap record_shutdown -s SIGTERM +record_shutdown() { logger xsce_shutdown + exit 0 } while [ 0 ]; do logger xsce_tick From 6f2556de02468ea3afdaab74175ce5fc97bfcd62 Mon Sep 17 00:00:00 2001 From: George Hunt Date: Wed, 13 Jul 2016 15:48:51 -0700 Subject: [PATCH 27/27] fix timestamp, rem acrecord --- roles/reports/templates/acrecord.py | 603 -------------------------- roles/reports/templates/analyze-power | 55 +-- 2 files changed, 30 insertions(+), 628 deletions(-) delete mode 100644 roles/reports/templates/acrecord.py diff --git a/roles/reports/templates/acrecord.py b/roles/reports/templates/acrecord.py deleted file mode 100644 index eac0e53f..00000000 --- a/roles/reports/templates/acrecord.py +++ /dev/null @@ -1,603 +0,0 @@ -#!/usr/bin/env python -# Copyright 2012 George Hunt -- georgejhunt@gmail.com -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -# reminder to myself: -# -# Time Zones are really difficult. datetime.now() returns local time, and -# tstampto string also generates a tz offset string. Perhaps simplist is to -# do allmy processing in UTC. - -import time -from subprocess import Popen, PIPE -import datetime -import os, sys -import logging -import json -import glob -from gettext import gettext as _ - -VERSION = "0.2" - -#WORK_DIR="/home/olpc" -WORK_DIR="." - -DATA_FILE = os.path.expanduser("~/.acpower") -# data_dict is global config file initialized in is_exist_data_file - used throughout -data_dict = {} - -"""the following class is stolen from dateutil -- becuse dateutil needs to be installed online and we're trying to make an offline install """ -ZERO = datetime.timedelta(0) -EPOCHORDINAL = datetime.datetime.utcfromtimestamp(0).toordinal() -GRAPH = True - -class PowerChunk(): - def __init__(self,pdstart,pdlength): - self.startsec = pdstart - self.lensec = pdlength - -class tzlocal(datetime.tzinfo): - global tz_offset - - _std_offset = datetime.timedelta(seconds=-time.timezone) - if time.daylight: - _dst_offset = datetime.timedelta(seconds=-time.altzone) - else: - _dst_offset = _std_offset - tz_offset = _std_offset.total_seconds() - - def utcoffset(self, dt): - if self._isdst(dt): - return self._dst_offset - else: - return self._std_offset - - def dst(self, dt): - if self._isdst(dt): - return self._dst_offset-self._std_offset - else: - return ZERO - - def tzname(self, dt): - return time.tzname[self._isdst(dt)] - - def _isdst(self, dt): - # We can't use mktime here. It is unstable when deciding if - # the hour near to a change is DST or not. - # - # timestamp = time.mktime((dt.year, dt.month, dt.day, dt.hour, - # dt.minute, dt.second, dt.weekday(), 0, -1)) - # return time.localtime(timestamp).tm_isdst - # - # The code above yields the following result: - # - #>>> import tz, datetime - #>>> t = tz.tzlocal() - #>>> datetime.datetime(2003,2,15,23,tzinfo=t).tzname() - #'BRDT' - #>>> datetime.datetime(2003,2,16,0,tzinfo=t).tzname() - #'BRST' - #>>> datetime.datetime(2003,2,15,23,tzinfo=t).tzname() - #'BRST' - #>>> datetime.datetime(2003,2,15,22,tzinfo=t).tzname() - #'BRDT' - #>>> datetime.datetime(2003,2,15,23,tzinfo=t).tzname() - #'BRDT' - # - # Here is a more stable implementation: - # - timestamp = ((dt.toordinal() - EPOCHORDINAL) * 86400 - + dt.hour * 3600 - + dt.minute * 60 - + dt.second) - return time.localtime(timestamp+time.timezone).tm_isdst - - def __eq__(self, other): - if not isinstance(other, tzlocal): - return False - return (self._std_offset == other._std_offset and - self._dst_offset == other._dst_offset) - return True - - def __ne__(self, other): - return not self.__eq__(other) - - def __repr__(self): - return "%s()" % self.__class__.__name__ - - __reduce__ = object.__reduce__ - -class tzutc(datetime.tzinfo): - - def utcoffset(self, dt): - return ZERO - - def dst(self, dt): - return ZERO - - def tzname(self, dt): - return "UTC" - - def __eq__(self, other): - return (isinstance(other, tzutc) or - (isinstance(other, tzoffset) and other._offset == ZERO)) - - def __ne__(self, other): - return not self.__eq__(other) - - def __repr__(self): - return "%s()" % self.__class__.__name__ - - __reduce__ = object.__reduce__ - -class AcException(): - def __init__(self, msg): - print(msg) - sys.exit(1) -# get a global instance of the tzlocal class -tz = tzlocal() -tzu = tzutc() -UTC2LOCAL = datetime.datetime.now(tz) - datetime.datetime.now(tzu) -UTC2LOCALSECONDS = UTC2LOCAL.total_seconds() - -class Tools: - def __init__(self): - global tz_offset - pass - - def cli(self, cmd): - """send cmd line to shell, rtn (text,error code)""" - p1 = Popen(cmd,stdout=PIPE, shell=True) - output = p1.communicate() - if p1.returncode != 0 : - print('error returned from shell command: %s was %s'%(cmd,output[0])) - return output[0],p1.returncode - - def is_exist_data_file(self): - #get the tmp data file - global data_dict - if (len(data_dict)> 0): - return True - try: - fd = file(DATA_FILE,'r') - data_str = fd.read() - data_dict = json.loads(data_str) - fd.close() - return True - except IOError: - return False - - def get_data_dict(self): - global data_dict - if self.is_exist_data_file(): - return data_dict - return None - - def put_data_file(self): - """ writes the data_dict """ - try: - fd = file(DATA_FILE,'w') - data_str = json.dumps(data_dict) - fd.write(data_str) - fd.close() - except IOError,e: - logging.exception("failed to write data file. error:%s"% (e,)) - raise AcException("Datafile write error") - - def print_data_file(self): - """ Prints the data_dict """ - try: - fd = file(DATA_FILE,'r') - data_str = json.dumps(data_dict) - fd.close() - print("Contents of persistent data record:") - for k in data_dict: - print(" ",k,data_dict[k]) - except IOError,e: - logging.exception("failed to write data file. error:%s"% (e,)) - raise AcException("Datafile write error") - - def get_summary_filename(self): - """ returns the filename of current summary file or "" if it doesn't exist """ - fn = os.path.join(SUMMARY_PREFIX,SUMMARY_CURRENT) - if (os.path.isfile(fn)): - try: - fd = open(fn,"r") - fname = fd.read() - except : - cmd = "rm -f %s"%fn - result,status = self.cli(cmd) - return "" - return fname - return "" - - - def get_datetime(self, datestr): - """ translate ymdhms string into datetime """ - dt = datetime.datetime.strptime(datestr, "%Y/%m/%d-%H:%M:%S-%Z") - if datestr.find("GMT"): - tzaware = dt.replace(tzinfo=tzu) - else: - tzaware = dt.replace(tzinfo=tz) - return tzaware - - def tstamp(self, dtime): - '''return a UNIX style seconds since 1970 for datetime input''' - epoch = datetime.datetime(1970, 1, 1,tzinfo=tzu) - newdtime = dtime.astimezone(tzu) - since_epoch_delta = newdtime - epoch - return since_epoch_delta.total_seconds() - - def get_utc_tmtamp_from_local_string(self,instr): - localdt = self.get_datetime(instr) - return self.tstamp(localdt) + tzoffset - - def parse_date(self,datestr): - try: - unawaredt = datetime.datetime.strptime(datestr, "%m/%d/%Y") - tzaware = unawaredt.replace(tzinfo=tz) - return self.tstamp(tzaware) # + tzoffset - except Exception as e: - print("returned error:%s. Error in date [%s]. Expected in format mm/dd/yyyy"% (e,datestr,)) - return 0L - - def str2tstamp(self, thestr): - '''return a UNIX style seconds since 1970 for string input''' - dtime = datetime.datetime.strptime(thestr.strip(), "%Y/%m/%d-%H:%M:%S-%Z") - awaredt = dtime.replace(tzinfo=tz) - newdtime = awaredt.astimezone(tz) - epoch = datetime.datetime(1970, 1, 1,tzinfo=tzu) - since_epoch_delta = newdtime - epoch - return since_epoch_delta.total_seconds() - - def tstamp_now(self): - """ return seconds since 1970 """ - return self.tstamp(datetime.datetime.now(tz)) - - def format_datetime(self, dt): - """ return ymdhms string """ - return datetime.datetime.strftime(dt, "%Y/%m/%d-%H:%M:%S-%z") - - def dhm_from_seconds(self,s): - """ translate seconds into days, hour, minutes """ - #print s - days, remainder = divmod(s, 86400) - hours, remainder = divmod(remainder, 3600) - minutes, remainder = divmod(remainder, 60) - return (days, hours, minutes) - - def ts2str(self,ts): - """ change a time stamp into a string expressed in local time zone""" - dttime = datetime.datetime.fromtimestamp(ts) - return self.format_datetime(dttime) - - def ts2date(self,ts): - """ change a time stamp into a string expressed in local time zone""" - dttime = datetime.datetime.fromtimestamp(ts) - return datetime.datetime.strftime(dttime, "%Y/%m/%d") - -class ShowPowerHistory(Tools): - def __init__(self): - global tz_offset - pass - - def set_start(self, start_str, debug): - self.is_exist_data_file() - start = self.parse_date(start_str) - if start <> 0: - data_dict["start"] = start - if not data_dict.has_key("end"): - data_dict["end"] = self.parse_date("12/31/2030") - self.put_data_file() - if debug: - self.print_data_file() - sys.exit(0) - else: - print("start date not recognized") - sys.exit(0) - - - def set_end(self, end_str, debug): - self.is_exist_data_file() - end = self.parse_date(end_str) - if end <> 0: - data_dict['end'] = end - self.put_data_file() - if debug: - self.print_data_file() - sys.exit(0) - else: - print("end date not recognized") - sys.exit(0) - - def output_summary(self, data, args): - # online is a dictoionary with key=start_time_stamp, value on time_seconds - debug = args.verbose - MATRIX = args.daily - if args.start: - self.set_start(args.start, args.verbose) - if args.end: - self.set_end(args.end, args.verbose) - # online is a dictionary. key=utc_timestamp when power came on, value=seconds ontime - online = {} - gap_start = None - if len(data) == 0: - print("No data for this period") - sys.exit(0) - first_ts = data[0][0] - first_str = self.ts2date(first_ts) - last_ts = data[len(data)-1][0] - last_str = self.ts2date(last_ts) - print("\n SUMMARY OF AC POWER DURING PERIOD: %s to %s" % (first_str, last_str,)) - print(" (Data ignored outside time period between %s and %s.)\n" % (self.ts2date(data_dict["start"]), self.ts2date(data_dict["end"]),)) - first = data[0][0] - power_state = None - power_start = first - for index in range(len(data)): - if debug: - print(data[index][0], data[index][7],data[index][8], self.ts2str(data[index][0])) - if data[index][7].find('ac-online-event') != -1 or \ - data[index][7].find('startup') != -1: - if not power_state: - power_start = data[index][0] - power_state = True - elif data[index][7].find('ac-offline-event') != -1: - online[power_start] = data[index][0] - power_start - power_state = None - elif power_state and data[index][7].find('shutdown') != -1: - online[power_start] = data[index][0] - power_start - power_state = None - for k in sorted(online): - if debug: - print(self.ts2str(k), "minutes:",(online[k])/60) - total_seconds = last_ts - first_ts - (days, hours, minutes) = self.dhm_from_seconds(total_seconds) - print "length of log %s days, %s hours, %s minutes" % (days, hours, minutes) - number_of_gaps = len(online) - 1 - print "number of power outages: %s" % number_of_gaps - # mysum is total power online seconds - mysum = 0L - power_list = [] - gap_length_list = [] - first = False - if len(online) > 0: - for key in sorted(online): - mysum += online[key] - power_list.append( (key,online[key]) ) - if first: - gap_length_list.append( (last_key + online[last_key],key -last_key + online[last_key]) ) - first = True - last_key = key - # compute the average length of outage - average_seconds = (total_seconds - mysum) / float(number_of_gaps) - (days, hours, minutes) = self.dhm_from_seconds(average_seconds) - print "average length of outage: %s days %s hours %s minutes" % \ - (days, hours,minutes) - gap_list = sorted(gap_length_list, key=lambda x:x[1]) - ts_list = sorted(power_list) - last_seconds = power_list[len(power_list)-1][0] + power_list[len(power_list)-1][1] - if debug: - print("power_list - utc_start, power on seconds") - for item, value in power_list: - print item, value - shortest_gap = gap_list[0][1] - if debug: - print("sorted gap list:") - for i in range(len(gap_list)): - print(gap_list[i][0],gap_list[i][1]) - if shortest_gap < 60: - print "shortest outage: %s seconds " % (shortest_gap) - else: - (days, hours, minutes) = self.dhm_from_seconds(shortest_gap) - print "shortest outage: %s days %s hours %s minutes " % \ - (days, hours, minutes,) - longest_gap = gap_list[len(gap_list)-1][1] - (days, hours, minutes) = self.dhm_from_seconds(longest_gap) - print "longest outage: %s days %s hours %s minutes " % \ - (days,hours, minutes,) - average_per_day = (float(mysum)/total_seconds) * 24 - print("Average power within 24 hours:%2.2f hours"%average_per_day) - - print "\n\nDISTRUBUTION OF POWER OVER THE DAY" - buckets = [] - - #divide up the total time into 15 minute chunks and distribute - # X's across the 96 columns of a day for each chunk that has power - - # first get the offset of the first entry from midnight - firstdt = datetime.datetime.fromtimestamp(first_ts, tz) - first_midnight_local = firstdt - datetime.timedelta(\ - hours=firstdt.hour,minutes=firstdt.minute, seconds=firstdt.second) - # lets do all the time in seconds since 1970 (tstamp) and in UTC - midnight_str = self.format_datetime(first_midnight_local) - if debug: - print("midnight should be:%s"%midnight_str) - first_midnight_seconds = self.tstamp(first_midnight_local) - last_seconds = last_ts - current_bucket_seconds = first_midnight_seconds - current_power_state = False # we backtracked from the time when the monitor was enabled - key_index = 0 - power_on_seconds = first_ts - power_off_seconds = power_on_seconds + power_list[0][1] - seconds_in_day = 24.0 * 60 * 60 - seconds_in_current_day = 1000 - bucket_size = 60 * 15.0 - - if debug: - for k,v in ts_list: - print("key:%s, string:%s, value:%s"%(k,self.ts2str(k), v,)) - print("Before loop begins: on:%s, off:%s,bucket_seconds:%s"%(\ - self.ts2str(power_on_seconds),self.ts2str(power_off_seconds),\ - self.ts2str(current_bucket_seconds),)) - buckets = [] - for j in range(96): - buckets.append(0) - if MATRIX: - current_day_ts = first_ts - current_day_str = self.ts2date(current_day_ts) - print("One line per day. Current day: %s"%current_day_str) - while current_bucket_seconds < last_seconds: - for index in range(len(ts_list)): - if ts_list[index][0] > current_bucket_seconds: - break - if ts_list[index][0] + ts_list[index][1] > current_bucket_seconds: - current_power_state = True - else: - current_power_state = False - if MATRIX: - if current_power_state: - sys.stdout.write("X") - else: - sys.stdout.write(" ") - if GRAPH: - bucket_index = int(seconds_in_current_day / bucket_size) - if current_power_state: - buckets[bucket_index] += 1 - - - current_bucket_seconds += bucket_size - seconds_in_current_day = (current_bucket_seconds - first_midnight_seconds) % seconds_in_day - if seconds_in_current_day < 10 and MATRIX: - print - if (current_bucket_seconds - first_midnight_seconds)%864000 >= 863100 and MATRIX: - current_day_str = self.ts2date(current_bucket_seconds) - print("\n%s 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23"%current_day_str) - - if GRAPH: -# find the max of the buckets - print("\nBar Graph") - bucket_max = max(buckets) - if debug: - print("bucket_max:%s"%bucket_max) - for row in range(bucket_max - 1,-1,-1): - for i in range(96): - if row + 1 - buckets[i] <= 0: - sys.stdout.write("X") - else: - sys.stdout.write(" ") - print - print "\n0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23" - - if args.powersegments: - print("\nINDIVIDUAL POWER PERIODS:") - for item, value in power_list: - localts = item - localstr = self.ts2str(localts) - (days, hours, minutes) = self.dhm_from_seconds(value) - print "%s %s days %s hours and %s minutes" % \ - (localstr, days, hours, minutes, ) - def output_state(self): - if self.isenabled(): - state = "ENABLED" - else: - state = "DISABLED" - print("") - print("AC Power Monitor is currently %s"%state) - -class RawData(Tools): - def __init__(self): - global data_dict - datafile_exists = False - try: - fd = file(DATA_FILE,'r') - data_str = fd.read() - data_dict = json.loads(data_str) - fd.close() - datafile_exists = True - except IOError,e: - logging.exception("failed to write data file. error:%s"% (e,)) - #raise AcException("Datafile read error in RawData") - if datafile_exists: - keylist = sorted(data_dict.keys()) - print "Current data file:" - for item in keylist: - print item, data_dict[item] - print("Battery state percent:%s"%(self.get_battery_percent(),)) - - global summary_dict - name = self.get_summary_filename() - if (len(name)>0): - try: - fsummary = file(name,'r') - data_str = fsummary.read() - summary_dict = json.loads(data_str) - except IOError: - raise AcException("Summaary file read error in init of RawData") - keylist = sorted(summary_dict.keys()) - print "Summary file:" - for item in keylist: - print item, summary_dict[item] - -""" -if __name__ == "__main__": - tls = Tools() - try: - tzfile = open("%s/timezone"%WORK_DIR, "r") - timezone = tzfile.read() - tzfile.close() - except IOError: - Print("could not set timezone") - exit(1) - print("Timezone is set to %s"%timezone) - local_ts = tls.tstamp(datetime.datetime.now(tz)) #returns local time - tzoffset = time.time()-local_ts #time returns UTC - matrix = False - print("tz offset in seconds:%s"%tzoffset) - mystrnow = tls.format_datetime(datetime.datetime.now(tz)) - print("the string version of now:%s"%mystrnow) - mydt = tls.get_datetime(mystrnow) - myts = tls.tstamp(mydt) - print("the timestamp is %s"%myts) - print("The UTC timestame is %s"%time.time()) - print("returned by get_utc_from_string:%s"%tls.get_utc_tstamp_from_local_string(mystrnow)) - mynewstr = tls.ts2str(myts) - print("and the final string is %s"%mynewstr) - print("utc2local:%s"%UTC2LOCALSECONDS) - dif = time.time()-myts - print("the difference between local ts and utc is %s"%dif) - print("the corrected version o string is %s"%tls.ts2str(myts)) - if len(sys.argv) == 1: - pi = ShowPowerHistory() - elif (len(sys.argv )== 2): - # if coming from cron, the check for an action to do - if sys.argv[1] == '--timeout': - print("environment value of TZ:%s"%os.environ["TZ"]) - pa = CollectData() - # dump the data in understandable form - if sys.argv[1] == '--debug': - debug = True - matrix = True - pa = RawData() - pi = ShowPowerHistory() - matrix = False - debug = False - if sys.argv[1] == '--delete': - tools = Tools() - tools.disable() - tools.delete() - if sys.argv[1] == '--enable': - tools = Tools() - tools.enable() - if sys.argv[1] == '--disable': - tools = Tools() - tools.disable() - sys.exit(0) - - # pop up the GUI - #Gtk.main() - sys.exit(0) -""" -# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 diff --git a/roles/reports/templates/analyze-power b/roles/reports/templates/analyze-power index 471d8a19..e336e3ce 100755 --- a/roles/reports/templates/analyze-power +++ b/roles/reports/templates/analyze-power @@ -8,12 +8,18 @@ import datetime from pprint import pprint import glob import json -from acrecord import Tools, tzlocal, tzutc +from dateutil.tz import * +from dateutil.parser import * -tz = tzlocal() -tzu = tzutc() LOC='/library/reports' +def tstamp(dtime): + '''return a UNIX style seconds since 1970 for datetime input''' + epoch = datetime.datetime(1970, 1, 1,tzinfo=tzutc()) + newdtime = dtime.astimezone(tzutc()) + since_epoch_delta = newdtime - epoch + return since_epoch_delta.total_seconds() + # fetch the dictionary of previous downloads if it exists if path.isfile(path.join(LOC,"data","downloads")): strm = open(path.join(LOC,"data","downloads"),"r") @@ -25,9 +31,6 @@ added = 0 with open("/etc/xsce/uuid", "r") as infile: uuid=infile.read() -# get the datetime tools -tools = Tools() - filedata = [] # traverse the apache logs and get the date in first record # The following will need to change for Fedora @@ -35,15 +38,17 @@ for fn in glob.glob('/var/log/messages*'): for line in open(fn, 'r'): datestr = line[0:15] try: - dt= datetime.datetime.strptime(datestr,"%b %d %H:%M:%S") + dt= parse(datestr) except: continue - dt = dt.replace(tzinfo=tz) - epoch = tools.tstamp(dt) - filedata.append( (epoch,fn) ) + dt = dt.replace(tzinfo=tzlocal()) + timestamp = tstamp(dt) + filedata.append( (timestamp,fn) ) + #print( line[0:14], dt.strftime("%y%m%d")) break -pprint(filedata) + +#pprint(filedata) # traverse the apache logs started = False for ts,fn in sorted(filedata): @@ -51,37 +56,37 @@ for ts,fn in sorted(filedata): datestr = line[0:15] try: - dt= datetime.datetime.strptime(datestr,"%b %d %H:%M:%S") + dt= parse(datestr) except: continue - dt = dt.replace(tzinfo=tz) - epoch = tools.tstamp(dt) + dt = dt.replace(tzinfo=tzlocal()) + timestamp = tstamp(dt) if not started: - last_heartbeat = epoch - last_powerdown_epoch = epoch - last_powerup = epoch + last_heartbeat = timestamp + last_powerdown_timestamp = timestamp + last_powerup = timestamp started = True # look for a start up record in the log nibbles = line.split() #pprint(nibbles) if nibbles[4] != 'root:': continue if nibbles[5] == 'xsce_startup': - last_powerup = epoch - print epoch, "startup. Downtime:", epoch - last_heartbeat,line[0:14] + last_powerup = timestamp + print timestamp, "startup. Downtime:", timestamp - last_heartbeat,line[0:14] """ - if last_heartbeat == last_powerdown_epoch: + if last_heartbeat == last_powerdown_timestamp: print 'normal shutdown' else: print 'power interruption shutdown' """ elif nibbles[5] == "xsce_tick": - #print epoch,'tick',line[0:14] - last_heatbeat = epoch + #print timestamp,'tick',line[0:14] + last_heartbeat = timestamp elif nibbles[5] == 'xsce_shutdown' : - print epoch, 'Normal system shutdown',line[0:14],"Powered seconds: ",epoch - last_poweerup - last_heatbeat = epoch - last_powerdown_epoch = epoch + print timestamp, 'Normal system shutdown',line[0:14],"Powered seconds: ",timestamp - last_powerup + last_heatbeat = timestamp + last_powerdown_timestamp = timestamp # now store away the accumulated data