new account-actions table to track user activities

configurations/analytics.sql

@@ -0,0 +1,26 @@
+
+--
+-- table to get account_id, action, count(*)
+-- Notes:
+-- * default null_string is "\N" -- any Optional.absent() values should be replaced with this
+--   http://docs.aws.amazon.com/redshift/latest/dg/copy-parameters-data-conversion.html#copy-null-as
+-- * use interleaved sortkey which is better for ad-hoc queries.
+--
+
+CREATE TABLE prod_actions (
+  account_id BIGINT,
+  action VARCHAR(256) NOT NULL,      -- action that user is performing
+  result VARCHAR(256) DEFAULT NULL,  -- result of action if exist. e.g. pass/fail/value
+  ts TIMESTAMP WITHOUT TIME ZONE,    -- utc 
+  offset_millis INTEGER DEFAULT NULL -- local offset if available
+) DISTSTYLE KEY DISTKEY (account_id)
+INTERLEAVED SORTKEY (action, ts, account_id);
+
+GRANT ALL ON prod_actions to migrator;
+GRANT ALL ON prod_actions to group ops;
+
+CREATE TABLE dev_actions (like prod_actions);
+GRANT ALL ON dev_actions to migrator;
+GRANT ALL ON dev_actions to group ops;
+
+

configurations/dynamodb_snapshots.sql

@@ -220,3 +220,53 @@ ALTER TABLE prod_speech_timeline OWNER to migrator;
 ALTER TABLE prod_speech_results ADD COLUMN fw INTEGER DEFAULT 0;
 
 
+-- 2017-01-30
+CREATE TABLE key_store (
+  aes_key VARCHAR(256),
+  created_at TIMESTAMP WITHOUT TIME ZONE,
+  device_id VARCHAR(64),
+  hw_version INTEGER,
+  metadata VARCHAR(512),
+  note VARCHAR(512)
+) DISTSTYLE KEY DISTKEY (device_id)
+INTERLEAVED SORTKEY (device_id, metadata);
+
+GRANT ALL ON key_store TO group ops;
+GRANT ALL ON key_store to migrator;
+ALTER TABLE key_store OWNER to migrator;
+
+CREATE TABLE key_store_admin (
+  created_at TIMESTAMP WITHOUT TIME ZONE,
+  device_id VARCHAR(64),
+  hw_version INTEGER,
+  metadata VARCHAR(512),
+  note VARCHAR(512)
+) DISTSTYLE KEY DISTKEY (device_id)
+INTERLEAVED SORTKEY (device_id, metadata);
+
+ALTER TABLE key_store_admin OWNER to migrator;
+GRANT SELECT ON key_store_admin to admin_tool;
+
+
+CREATE TABLE pill_key_store (
+  aes_key VARCHAR(256),
+  created_at TIMESTAMP WITHOUT TIME ZONE,
+  device_id VARCHAR(64),
+  metadata VARCHAR(512),
+  note VARCHAR(512)
+) DISTSTYLE KEY DISTKEY (device_id)
+INTERLEAVED SORTKEY (device_id, metadata);
+
+GRANT ALL ON pill_key_store TO group ops;
+ALTER TABLE pill_key_store OWNER to migrator;
+
+CREATE TABLE prod_alarm (
+  account_id BIGINT,
+  alarm_templates VARCHAR(65535),
+  updated_at BIGINT
+) DISTSTYLE KEY DISTKEY (account_id)
+COMPOUND SORTKEY (account_id, updated_at);
+
+GRANT ALL ON prod_alarm TO group ops;
+ALTER TABLE prod_alarm OWNER to migrator;
+

scripts/get_speech_stats.py

@@ -0,0 +1,152 @@
+import os
+from datetime import datetime
+import json
+import arrow
+import psycopg2
+import psycopg2.extras
+import requests
+
+REDSHIT_HOSTNAME = 'sensors2.cy7n0vzxfedi.us-east-1.redshift.amazonaws.com'
+REDSHIT_USER = 'migrator'
+REDSHIT_DB = 'sensors1'
+
+SLACK_TOKEN = "T024FJP19/B30RWGBB9/KtT7y8rXKGhaH3yAduGuPgGh"
+SLACK_URL = "https://hooks.slack.com/services/%s" % SLACK_TOKEN
+HEADERS = {'content-type': 'application/json'}
+
+def get_db():
+    """Connects to the specific database."""
+    conn = psycopg2.connect(
+        database=REDSHIT_DB,
+        user=REDSHIT_USER,
+        password=os.getenv('REDSHIT_PASS'),
+        host=REDSHIT_HOSTNAME,
+        port=5439
+    )
+    return conn
+
+def post_slack(slack_string, title):
+    attachments = {
+        "fallback": "Voice command stats",
+        "fields": [
+            {
+                "title": title,
+                "value": slack_string
+            }
+        ]
+    }
+
+    payload = {
+        'channel': '#voice-stats',
+        'username': 'shout',
+        "attachments": [attachments]
+    }
+
+    requests.post(SLACK_URL, data=json.dumps(payload), headers=HEADERS)
+
+
+def main():
+    utc_now = arrow.get(datetime.utcnow())
+    week_ago = utc_now.replace(days=-7)
+    date_string = week_ago.format('YYYY-MM-DD')
+
+    # number of commands per day
+    query = """SELECT date_trunc('day', created_utc) AS day, COUNT(*) AS c
+        FROM prod_speech_results 
+        WHERE created_utc >= %s GROUP BY day ORDER BY day"""
+
+    stats = {}
+    days = []
+    db = get_db()
+    with db.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur:
+        cur.execute(query, (date_string,))
+        rows = cur.fetchall()
+        slack_string = ""
+        for row in rows:
+            # slack_string += "\n%s | %s" % (str(row['day'])[:10], row['c'])
+            days.append(row['day'])
+            stats.setdefault(row['day'], {'c':0, 'u': 0})
+            stats[row['day']]['c'] = row['c']
+
+        # if slack_string != "":
+        #     post_slack(slack_string, "Commands per day (testing)")
+
+
+    # number of distinct account-ids using speech
+    query = """SELECT date_trunc('day', ts) AS day,
+        COUNT(distinct account_id) AS c 
+        FROM prod_speech_timeline 
+        WHERE ts >= %s GROUP BY day ORDER BY day"""
+
+    with db.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur:
+        cur.execute(query, (date_string,))
+        rows = cur.fetchall()
+        slack_string = ""
+        for row in rows:
+            # slack_string += "\n%s | %s" % (str(row['day'])[:10], row['c'])
+            if row['day'] in stats:
+                stats[row['day']]['u'] = row['c']
+
+        # if slack_string != "":
+        #     post_slack(slack_string,
+        #             "Distinct accounts using per day (testing)")
+
+    if stats:
+        slack_string = ""
+        for day in days:
+            slack_string += "\n%s | %s | %s" % (
+                day, stats[day]['c'], stats[day]['u'])
+
+        if slack_string != "":
+            post_slack(slack_string, "Commands and distinct accounts per day")
+
+    cmd_query = """
+        SELECT date, fw,
+          ROUND((1.0*empty_text)/total, 2) as empty_text_perc,
+          ROUND((1.0*text_try_again)/total, 2) as text_try_again_perc,
+          ROUND((1.0*text_reject)/total, 2) as text_reject_perc,
+          ROUND((1.0*ok_cmds)/total, 2) as ok_commands,
+          total AS total_commands
+        FROM (
+            SELECT DATE_TRUNC('day', created_utc) AS date, fw,
+              SUM(CASE WHEN (text != '' AND cmd_result='TRY_AGAIN')  THEN 1 ELSE 0 END) AS text_try_again,
+              SUM(CASE WHEN (text != '' AND cmd_result='REJECTED')  THEN 1 ELSE 0 END) AS text_reject,
+              SUM(CASE WHEN (text = '')  THEN 1 ELSE 0 END) as empty_text,
+              SUM(CASE WHEN (cmd != '')  THEN 1 ELSE 0 END) as ok_cmds,
+              COUNT(*) AS total
+            FROM prod_speech_results WHERE created_utc >= current_date - INTERVAL '7 day'
+            GROUP BY date, fw ORDER BY date, fw
+        ) ORDER BY date, fw
+    """
+    with db.cursor(cursor_factory=psycopg2.extras.DictCursor) as cur:
+        cur.execute(cmd_query, (date_string,))
+        rows = cur.fetchall()
+        slack_title = "Date       | empty_text   | reject     | try_again  | ok_cmd | total "
+        num_data = 0
+        slack_data = {}
+        for row in rows:
+            fw_version = str(row['fw'])
+            if fw_version is None:
+                fw_version = 'null'
+            if fw_version in ['-1', '0', '1', 'None']:
+                continue
+            slack_data.setdefault(fw_version, [])
+            num_data += 1
+            slack_string = "\n%s | %s         | %s       | %s        | %s      | %s" % (
+                str(row['date'])[:10], 
+                row['empty_text_perc'],
+                row['text_reject_perc'],
+                row['text_try_again_perc'],
+                row['ok_commands'],
+                row['total_commands'])
+            slack_data[fw_version].append(slack_string)
+
+
+        for fw in sorted(slack_data):
+            slack_string = slack_title
+            for row in slack_data[fw]:
+                slack_string += row
+            post_slack(slack_string, "Commands breakdown by FW version %s" % (fw))
+
+if __name__ == "__main__":
+    main()

scripts/slackbot.py

@@ -0,0 +1,24 @@
+"""Slack"""
+import os
+import json
+import logging
+import requests
+
+SLACK_TOKEN = os.getenv('REDSHIFT_SLACK')
+SLACK_URL = "https://hooks.slack.com/services/%s" % (SLACK_TOKEN)
+HEADERS = {'content-type': 'application/json'}
+
+def post(table, date, text):
+    """Post a status message to ops"""
+    if not SLACK_TOKEN:
+        logging.error("Wrong url")
+        return
+
+    prefix = "*Copy %s %s:* " % (table, date)
+    payload = {'text': prefix + text,
+        'channel': '#ops', 'username': 'redshit'}
+    try:
+        requests.post(SLACK_URL, data=json.dumps(payload), headers=HEADERS)
+    except requests.exceptions.RequestException as err:
+        logging.error("Fail to send to Slack")
+        logging.error(err)

scripts/test_slack.py

@@ -0,0 +1,11 @@
+
+import requests
+import os
+import json
+
+data = {"text": "testing webhook", "channel": "#ops", "username": "redshit"}
+slack_token = os.getenv('REDSHIFT_SLACK')
+if slack_token:
+    slack_url = "https://hooks.slack.com/services/%s" % (slack_token)
+    headers={'content-type': 'application/json'}
+    r = requests.post(slack_url, json.dumps(data), headers=headers)