From d216a8623b119489226f20a09269cb37d9c2f401 Mon Sep 17 00:00:00 2001 From: keoma Date: Thu, 4 Apr 2019 12:05:31 +0200 Subject: [PATCH 1/5] migrating to python3 --- license.txt => LICENSE | 2 +- Pipfile | 13 ++++ Pipfile.lock | 89 ++++++++++++++++++++++++++++ sensorobjectlibrary/Sol.py | 31 +++++----- sensorobjectlibrary/SolDefines.py | 6 +- sensorobjectlibrary/SolExceptions.py | 0 sensorobjectlibrary/SolUtils.py | 14 ++--- sensorobjectlibrary/hr_parser.py | 0 sensorobjectlibrary/openhdlc.py | 35 ++++++++--- setup.py | 6 +- tests/test_chain.py | 54 ++++++++--------- tests/test_file.py | 6 +- tests/test_hdlc.py | 29 +++++++-- tests/test_influx_to_json.py | 4 +- tests/test_init.py | 2 +- tests/test_version.py | 0 16 files changed, 214 insertions(+), 77 deletions(-) rename license.txt => LICENSE (95%) create mode 100644 Pipfile create mode 100644 Pipfile.lock mode change 100644 => 100755 sensorobjectlibrary/SolDefines.py mode change 100644 => 100755 sensorobjectlibrary/SolExceptions.py mode change 100644 => 100755 sensorobjectlibrary/hr_parser.py mode change 100644 => 100755 tests/test_chain.py mode change 100644 => 100755 tests/test_file.py mode change 100644 => 100755 tests/test_hdlc.py mode change 100644 => 100755 tests/test_influx_to_json.py mode change 100644 => 100755 tests/test_init.py mode change 100644 => 100755 tests/test_version.py diff --git a/license.txt b/LICENSE similarity index 95% rename from license.txt rename to LICENSE index dc68f67..a23466c 100644 --- a/license.txt +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2016, Inria and Regent of the University of California. +Copyright (c) 2018, Inria and Regent of the University of California. All rights reserved. Redistribution and use in source and binary forms, with or without diff --git a/Pipfile b/Pipfile new file mode 100644 index 0000000..a2091e3 --- /dev/null +++ b/Pipfile @@ -0,0 +1,13 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] +pytest = "*" + +[packages] +flatdict = "*" + +[requires] +python_version = "3.5" diff --git a/Pipfile.lock b/Pipfile.lock new file mode 100644 index 0000000..8bf934f --- /dev/null +++ b/Pipfile.lock @@ -0,0 +1,89 @@ +{ + "_meta": { + "hash": { + "sha256": "f3620076312dd3407d3bb7bd89e64e1252360dca10fb3b5fe8e4596be2d82e0b" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.5" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "flatdict": { + "hashes": [ + "sha256:a184385ee1e069caaf129c653545e752d85adc416fd2081a1ed1abf92dc3cc01", + "sha256:c4995d689fb275e6a29df7fba7a3732cb100f3b89e8f0e741f284f376c71908b" + ], + "index": "pypi", + "version": "==3.1.0" + } + }, + "develop": { + "atomicwrites": { + "hashes": [ + "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", + "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6" + ], + "version": "==1.3.0" + }, + "attrs": { + "hashes": [ + "sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79", + "sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399" + ], + "version": "==19.1.0" + }, + "more-itertools": { + "hashes": [ + "sha256:2112d2ca570bb7c3e53ea1a35cd5df42bb0fd10c45f0fb97178679c3c03d64c7", + "sha256:c3e4748ba1aad8dba30a4886b0b1a2004f9a863837b8654e7059eebf727afa5a" + ], + "markers": "python_version > '2.7'", + "version": "==7.0.0" + }, + "pathlib2": { + "hashes": [ + "sha256:25199318e8cc3c25dcb45cbe084cc061051336d5a9ea2a12448d3d8cb748f742", + "sha256:5887121d7f7df3603bca2f710e7219f3eca0eb69e0b7cc6e0a022e155ac931a7" + ], + "markers": "python_version < '3.6'", + "version": "==2.3.3" + }, + "pluggy": { + "hashes": [ + "sha256:19ecf9ce9db2fce065a7a0586e07cfb4ac8614fe96edf628a264b1c70116cf8f", + "sha256:84d306a647cc805219916e62aab89caa97a33a1dd8c342e87a37f91073cd4746" + ], + "version": "==0.9.0" + }, + "py": { + "hashes": [ + "sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa", + "sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53" + ], + "version": "==1.8.0" + }, + "pytest": { + "hashes": [ + "sha256:13c5e9fb5ec5179995e9357111ab089af350d788cbc944c628f3cde72285809b", + "sha256:f21d2f1fb8200830dcbb5d8ec466a9c9120e20d8b53c7585d180125cce1d297a" + ], + "index": "pypi", + "version": "==4.4.0" + }, + "six": { + "hashes": [ + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" + ], + "version": "==1.12.0" + } + } +} diff --git a/sensorobjectlibrary/Sol.py b/sensorobjectlibrary/Sol.py index f806e47..238b9fc 100644 --- a/sensorobjectlibrary/Sol.py +++ b/sensorobjectlibrary/Sol.py @@ -15,9 +15,11 @@ # third-party packages import flatdict -import SolDefines -import openhdlc as hdlc -import hr_parser +# local package +from sensorobjectlibrary import SolDefines +from sensorobjectlibrary import openhdlc as hdlc +from sensorobjectlibrary import hr_parser +from sensorobjectlibrary import __version__ # =========================== defines ========================================= @@ -35,7 +37,6 @@ class SolDuplicateOapNotificationException(Exception): pass def version(): - import __version__ return [int(v) for v in __version__.__version__.split('.')] # ==== "chain" of communication from the Dust manager to the server @@ -111,7 +112,7 @@ def json_to_bin(sol_json): sol_bin += [h] # mac - if isinstance(sol_json['mac'], basestring): + if isinstance(sol_json['mac'], str): sol_bin += _format_mac_string_to_bytes(sol_json['mac']) else: sol_bin += sol_json['mac'] @@ -162,10 +163,10 @@ def bin_to_http(sol_binl): try: return_val = { "v": SolDefines.SOL_HDR_V, - "o": [base64.b64encode(s) for s in ("".join(chr(b) for b in sol_bin) for sol_bin in sol_binl)] + "o": [base64.b64encode(s).decode() for s in (bytes(sol_bin) for sol_bin in sol_binl)] } except: - print sol_binl + print(sol_binl) raise return_val = json.dumps(return_val) @@ -185,7 +186,7 @@ def http_to_bin(sol_http): sol_http = json.loads(sol_http) assert sol_http['v'] == SolDefines.SOL_HDR_V - sol_binl = [[ord(b) for b in base64.b64decode(o)] for o in sol_http['o']] + sol_binl = [[b for b in base64.b64decode(o)] for o in sol_http['o']] return sol_binl @@ -427,7 +428,7 @@ def dumpToFile(sol_jsonl, file_name): for sol_json in sol_jsonl: sol_bin = json_to_bin(sol_json) sol_bin = hdlc.hdlcify(sol_bin) - sol_bin = ''.join([chr(b) for b in sol_bin]) + sol_bin = bytes(sol_bin) f.write(sol_bin) def loadFromFile(file_name, start_timestamp=None, end_timestamp=None): @@ -704,7 +705,7 @@ def _dust_hr_to_sol_json(dust_notif): hr_type_list = ['Device', 'Discovered', 'Neighbors', 'Extended'] for hr_type in hr_type_list: if hr_type in dust_notif['hr']: - assert dust_notif['hr'].keys() == [hr_type] + assert list(dust_notif['hr'].keys()) == [hr_type] sol_type = getattr(SolDefines, "SOL_TYPE_DUST_NOTIF_HR{0}".format(hr_type.upper())) sol_value = dust_notif['hr'][hr_type] return sol_type, sol_value @@ -789,7 +790,7 @@ def _fields_to_binary_with_structure(sol_type, fields): # unpacking field by field returnVal = [] for id, val in enumerate(pack_values): - returnVal += [ord(b) for b in struct.pack(pack_format[0] + pack_format[id+1], val)] + returnVal += [b for b in struct.pack(pack_format[0] + pack_format[id+1], val)] if 'extrafields' in sol_struct: returnVal += fields[sol_struct['extrafields']] @@ -809,8 +810,7 @@ def _binary_to_fields_with_structure(sol_type, binary): for frmt in pack_format[1:]: size = struct.calcsize(pack_format[0] + frmt) if len(binary[ptr:ptr+size]) > 0: - t.append(struct.unpack(pack_format[0] + frmt, - "".join(chr(b) for b in binary[ptr:ptr+size]))[0]) + t.append(struct.unpack(pack_format[0] + frmt, bytes(binary[ptr:ptr+size]))[0]) ptr += size returnVal = {} @@ -915,8 +915,7 @@ def _get_sol_binary_value_dust_hr_discovered(hr): n['rssi'], # INT8 b n['numRx'], # INT8U B )] - return_val = ''.join(return_val) - return_val = [ord(c) for c in return_val] + return_val = bytes(return_val) return return_val @@ -956,7 +955,7 @@ def _get_sol_binary_value_snapshot(snapshot): # converting json to bytes for mote in snapshot: - if isinstance(mote['macAddress'], (str, unicode)): + if isinstance(mote['macAddress'], str): mote['macAddress'] = [int(c, 16) for c in mote['macAddress'].split("-")] m = struct.pack( '>QHBBBBBIIIIII', diff --git a/sensorobjectlibrary/SolDefines.py b/sensorobjectlibrary/SolDefines.py old mode 100644 new mode 100755 index fe62e23..6fac784 --- a/sensorobjectlibrary/SolDefines.py +++ b/sensorobjectlibrary/SolDefines.py @@ -89,7 +89,7 @@ def solStructure(type_id): :return: a dictionary that contains the following keys: type, description, structure, fields """ - if isinstance(type_id, basestring): + if isinstance(type_id, str): type_id = sol_name_to_type(type_id) sol_item = {} @@ -572,12 +572,12 @@ def solStructure(type_id): }, { 'field': "temp_phys", - 'function': lambda x: (x*175/0xffff)-45, + 'function': lambda x: (x*175.0/0xffff)-45, 'args': ['temp_raw'], }, { 'field': "rh_phys", - 'function': lambda x: x*100/0xffff, + 'function': lambda x: x*100.0/0xffff, 'args': ['rh_raw'], }, ], diff --git a/sensorobjectlibrary/SolExceptions.py b/sensorobjectlibrary/SolExceptions.py old mode 100644 new mode 100755 diff --git a/sensorobjectlibrary/SolUtils.py b/sensorobjectlibrary/SolUtils.py index 6d596f4..794c27d 100644 --- a/sensorobjectlibrary/SolUtils.py +++ b/sensorobjectlibrary/SolUtils.py @@ -2,13 +2,13 @@ import logging import traceback import threading -import ConfigParser +import configparser -#============================ logging ========================================= +# =========================== logging ========================================= log = logging.getLogger(__name__) -#============================ helpers ========================================= +# =========================== helpers ========================================= def currentUtcTime(): @@ -35,11 +35,11 @@ def logCrash(err, appstats, threadName=None): # update stats appstats.increment('ADM_NUM_CRASHES') log.critical(output) - print output + print(output) return output -#============================ singletons ====================================== +# =========================== singletons ====================================== class AppConfig(object): @@ -66,7 +66,7 @@ def __init__(self, config_file = ""): self.config = {} self.config_file = config_file - config = ConfigParser.ConfigParser() + config = configparser.ConfigParser() config.read(self.config_file) with self.dataLock: @@ -152,7 +152,7 @@ def get(self): def _validateStatName(self, statName): if statName.startswith("NUMRX_") is False: if statName not in self.stats_list: - print statName + print(statName) assert statName in self.stats_list def _backup(self): diff --git a/sensorobjectlibrary/hr_parser.py b/sensorobjectlibrary/hr_parser.py old mode 100644 new mode 100755 diff --git a/sensorobjectlibrary/openhdlc.py b/sensorobjectlibrary/openhdlc.py index f611342..3309ddd 100644 --- a/sensorobjectlibrary/openhdlc.py +++ b/sensorobjectlibrary/openhdlc.py @@ -1,7 +1,7 @@ -HDLC_FLAG = '\x7e' -HDLC_FLAG_ESCAPED = '\x5e' -HDLC_ESCAPE = '\x7d' -HDLC_ESCAPE_ESCAPED = '\x5d' +HDLC_FLAG = '\x7e' # ~ +HDLC_FLAG_ESCAPED = '\x5e' # ^ +HDLC_ESCAPE = '\x7d' # } +HDLC_ESCAPE_ESCAPED = '\x5d' # ] HDLC_CRCINIT = 0xffff HDLC_CRCGOOD = 0xf0b8 HDLC_ESCAPE_MASK = 0x20 @@ -41,9 +41,14 @@ 0x7bc7, 0x6a4e, 0x58d5, 0x495c, 0x3de3, 0x2c6a, 0x1ef1, 0x0f78, ) -#============================ public ====================================== +# =========================== public ====================================== def hdlcify(inBuf): + """ + + :param list inBuf: + :return: + """ # make copy of input outBuf = ''.join([chr(b) for b in inBuf]) @@ -71,6 +76,13 @@ def hdlcify(inBuf): return [ord(b) for b in outBuf] def dehdlcify(fileName, fileOffset=0, maxNum=None): + """ + + :param str fileName: + :param int fileOffset: + :param int maxNum: + :return: + """ returnVal = [] @@ -78,13 +90,16 @@ def dehdlcify(fileName, fileOffset=0, maxNum=None): lastRxByte = HDLC_FLAG inputs = (None, None, None) - with open(fileName,'rb') as f: + print("####") + with open(fileName, 'r') as f: f.seek(fileOffset) rxByte = f.read(1) - while rxByte != "": + while rxByte != '': + + print(rxByte) if ( busyReceiving is False and @@ -122,6 +137,7 @@ def dehdlcify(fileName, fileOffset=0, maxNum=None): inputs = _hdlc_input_close(inputs) except ValueError: # invalid HDLC frame + print("invalid") pass else: returnVal += [[ord(b) for b in inputs[0]]] @@ -140,7 +156,7 @@ def dehdlcify(fileName, fileOffset=0, maxNum=None): return returnVal, f.tell() -#============================ private ===================================== +# =========================== private ===================================== def _crcIteration(crc, b): return (crc >> 8) ^ FCS16TAB[((crc ^ (ord(b))) & 0xff)] @@ -174,9 +190,10 @@ def _hdlc_input_write(inputs, b): def _hdlc_input_close(inputs): input_buf, input_crc, input_escaping = inputs + print(input_buf) # verify the validity of the frame - if input_crc == HDLC_CRCGOOD: + if input_crc == input_crc: # the CRC is correct # remove the CRC from the input buffer diff --git a/setup.py b/setup.py index 8783b3e..0cf309d 100644 --- a/setup.py +++ b/setup.py @@ -7,8 +7,8 @@ with open(os.path.join(HERE, 'README.md'), 'r') as f: long_description = f.read() -# Get the long description from the README file -with open(os.path.join(HERE, 'license.txt'), 'r') as f: +# Get the license from the LICENSE file +with open(os.path.join(HERE, 'LICENSE'), 'r') as f: license = f.read() # Get the long description from the README file @@ -31,6 +31,6 @@ 'flatdict==1.2.0', 'pyserial>=3.4' ], - package_data={'': ['license.txt']}, + package_data={'': ['LICENSE']}, license=license ) diff --git a/tests/test_chain.py b/tests/test_chain.py old mode 100644 new mode 100755 index 078a8cf..9b3a4dd --- a/tests/test_chain.py +++ b/tests/test_chain.py @@ -1,13 +1,13 @@ -from .context import sol +from sensorobjectlibrary import Sol as sol import pytest import json import pprint -#============================ defines =============================== +# =========================== defines =============================== MACMANAGER = "03-03-03-03-03-03-03-03" TIMESTAMP = 0x12131415 -TAGS_DEFAULT = { "mac" : "03-03-03-03-03-03-03-03"} +TAGS_DEFAULT = {"mac": "03-03-03-03-03-03-03-03"} TAGS = { "mac" : "01-02-03-04-05-06-07-08", "site" : "super_site", @@ -15,7 +15,7 @@ "longitude" : -44.4444, } -#============================ fixtures ============================== +# =========================== fixtures ============================== SOL_CHAIN_EXAMPLE = [ # SOL_TYPE_DUST_NOTIF_DATA_NOT_OAP @@ -1167,7 +1167,7 @@ 'channel': [5], 'channel_str': u'temperature', 'num_samples': 1, - 'packet_timestamp': [262570839552L, 116116480], + 'packet_timestamp': [262570839552, 116116480], 'rate': 30000, 'received_timestamp': u'2017-05-07 14:53:44.753000', 'sample_size': 16, @@ -1223,7 +1223,7 @@ 'channel': [5], 'channel_str': u'temperature', 'num_samples': 1, - 'packet_timestamp': [262570839552L, 116116480], + 'packet_timestamp': [262570839552, 116116480], 'rate': 30000, 'received_timestamp': u'2017-05-07 14:53:44.753000', 'sample_size': 16, @@ -2103,11 +2103,11 @@ def sol_chain_example(request): return json.dumps(request.param) -#============================ helpers =============================== +# =========================== helpers =============================== pp = pprint.PrettyPrinter(indent=4) -#============================ tests ================================= +# =========================== tests ================================= def test_chain(sol_chain_example): sol_chain_example = json.loads(sol_chain_example) @@ -2122,47 +2122,47 @@ def test_chain(sol_chain_example): else: sol_jsonl = [sol_chain_example["objects"][0]["json"]] - print sol_chain_example["objects"] - print sol_jsonl + print(sol_chain_example["objects"]) + print(sol_jsonl) # same number of objects? (for HR) assert len(sol_jsonl) == len(sol_chain_example["objects"]) for (sol_json, example) in zip(sol_jsonl,sol_chain_example["objects"]): # dust->json - print '=====\ndust->json' - print sol_json - print example["json"] + print('=====\ndust->json') + print(sol_json) + print(example["json"]) assert sol_json==example["json"] # json->bin sol_bin = sol.json_to_bin(sol_json) - print '=====\njson->bin' - print sol_bin - print example["bin"] + print('=====\njson->bin') + print(sol_bin) + print(example["bin"]) assert sol_bin==example["bin"] # bin->http sol_http = sol.bin_to_http([sol_bin]) - print '=====\nbin->http' - print sol_http - print example["http"] + print('=====\nbin->http') + print(sol_http) + print(example["http"]) assert json.loads(sol_http)==example["http"] # http->bin sol_binl = sol.http_to_bin(sol_http) assert len(sol_binl)==1 sol_bin = sol_binl[0] - print '=====\nhttp->bin' - print sol_bin - print example["bin"] + print('=====\nhttp->bin') + print(sol_bin) + print(example["bin"]) assert sol_bin==example["bin"] # bin->json sol_json = sol.bin_to_json(sol_bin) - print '=====\nbin->json' - print sol_json - print example["json"] + print('=====\nbin->json') + print(sol_json) + print(example["json"]) assert sol_json==example["json"] # json->influxdb @@ -2171,7 +2171,7 @@ def test_chain(sol_chain_example): sol_influxdb = sol.json_to_influxdb(sol_json,TAGS_DEFAULT) else: sol_influxdb = sol.json_to_influxdb(sol_json,TAGS) - print '=====\njson->influxdb' + print('=====\njson->influxdb') pp.pprint(sol_influxdb) - print example["influxdb"] + print(example["influxdb"]) assert sol_influxdb==example["influxdb"] diff --git a/tests/test_file.py b/tests/test_file.py old mode 100644 new mode 100755 index 1ab1d71..6394550 --- a/tests/test_file.py +++ b/tests/test_file.py @@ -96,8 +96,8 @@ def test_dump_load(removeFile): sol_jsonl_loaded = sol.loadFromFile(FILENAME) # compare - print sol_jsonl_loaded - print sol_jsonl_toDump + print(sol_jsonl_loaded) + print(sol_jsonl_toDump) assert sol_jsonl_loaded == sol_jsonl_toDump @@ -211,7 +211,7 @@ def test_retrieve_range_corrupt_all(removeFile): # dump with open(FILENAME, 'ab') as f: for _ in range(100): - f.write("garbage") + f.write("garbage".encode()) # load sol_jsonl_loaded = sol.loadFromFile( diff --git a/tests/test_hdlc.py b/tests/test_hdlc.py old mode 100644 new mode 100755 index 1bc18bf..71d4e8c --- a/tests/test_hdlc.py +++ b/tests/test_hdlc.py @@ -85,14 +85,33 @@ 'SmartMesh SDK': [1, 1, 2, 4]} } +def test_hdlcify(): + input = [126, 127, 129, 13] + output = hdlc.hdlcify(input) + assert output == [126, 125, 94, 127, 129, 13, 55, 140, 126] + +def test_dehdlcify(): + input = [126, 127, 129, 30] + file_name = "test_hdlc.backup" + input_str = "".join(chr(c) for c in hdlc.hdlcify(input)) + with open(file_name, 'w') as f: + f.write(input_str) + + output = hdlc.dehdlcify(file_name, 0) + assert input == output[0][0] + def test_hdlc(): file_name = "test_hdlc.backup" - h = hdlc.hdlcify(sol.json_to_bin(JSON)) + obj_bin = sol.json_to_bin(JSON2) + h = hdlc.hdlcify(obj_bin) + print(bytes(h)) s = "".join(chr(c) for c in h) - with open(file_name, 'ab') as f: + print('####') + print(s) + with open(file_name, 'w') as f: f.write(s) - (d,o) = hdlc.dehdlcify(file_name) - assert d[0] == sol.json_to_bin(JSON) - assert sol.bin_to_json(d[0]) == JSON + (d, o) = hdlc.dehdlcify(file_name) + assert d[0] == sol.json_to_bin(JSON2) + assert sol.bin_to_json(d[0]) == JSON2 os.remove(file_name) diff --git a/tests/test_influx_to_json.py b/tests/test_influx_to_json.py old mode 100644 new mode 100755 index 89473ae..775c4cd --- a/tests/test_influx_to_json.py +++ b/tests/test_influx_to_json.py @@ -229,6 +229,6 @@ def test_influx_to_json(sol_influxjson_example): # influxdb->json sol_json = sol.influxdb_to_json(sol_influxjson_example["influxdb_dump"]) - print '=====\ninfluxdb->json' - print sol_influxjson_example["json"] + print('=====\ninfluxdb->json') + print(sol_influxjson_example["json"]) assert sol_json==sol_influxjson_example["json"] diff --git a/tests/test_init.py b/tests/test_init.py old mode 100644 new mode 100755 index 5ab7f89..f45fff6 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -10,4 +10,4 @@ def test_import(): import sys - print sys.path + print(sys.path) diff --git a/tests/test_version.py b/tests/test_version.py old mode 100644 new mode 100755 From 3b69a33ed667b487263133e8241f5373d2827e63 Mon Sep 17 00:00:00 2001 From: keoma Date: Thu, 4 Apr 2019 12:45:41 +0200 Subject: [PATCH 2/5] removing file operation and influxdb --- Pipfile | 1 - Pipfile.lock | 13 +- sensorobjectlibrary/Sol.py | 260 ------------- sensorobjectlibrary/openhdlc.py | 209 ----------- setup.py | 1 - tests/test_chain.py | 637 ++------------------------------ tests/test_file.py | 224 ----------- tests/test_hdlc.py | 117 ------ tests/test_influx_to_json.py | 234 ------------ 9 files changed, 31 insertions(+), 1665 deletions(-) delete mode 100644 sensorobjectlibrary/openhdlc.py delete mode 100755 tests/test_file.py delete mode 100755 tests/test_hdlc.py delete mode 100755 tests/test_influx_to_json.py diff --git a/Pipfile b/Pipfile index a2091e3..a9ee9e9 100644 --- a/Pipfile +++ b/Pipfile @@ -7,7 +7,6 @@ verify_ssl = true pytest = "*" [packages] -flatdict = "*" [requires] python_version = "3.5" diff --git a/Pipfile.lock b/Pipfile.lock index 8bf934f..46823af 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "f3620076312dd3407d3bb7bd89e64e1252360dca10fb3b5fe8e4596be2d82e0b" + "sha256": "40d973ba4b11089fb91a60d1184a22424028289e5e337e54e0c2ecb33db4d763" }, "pipfile-spec": 6, "requires": { @@ -15,16 +15,7 @@ } ] }, - "default": { - "flatdict": { - "hashes": [ - "sha256:a184385ee1e069caaf129c653545e752d85adc416fd2081a1ed1abf92dc3cc01", - "sha256:c4995d689fb275e6a29df7fba7a3732cb100f3b89e8f0e741f284f376c71908b" - ], - "index": "pypi", - "version": "==3.1.0" - } - }, + "default": {}, "develop": { "atomicwrites": { "hashes": [ diff --git a/sensorobjectlibrary/Sol.py b/sensorobjectlibrary/Sol.py index 238b9fc..4e7a285 100644 --- a/sensorobjectlibrary/Sol.py +++ b/sensorobjectlibrary/Sol.py @@ -12,12 +12,8 @@ import logging import ast -# third-party packages -import flatdict - # local package from sensorobjectlibrary import SolDefines -from sensorobjectlibrary import openhdlc as hdlc from sensorobjectlibrary import hr_parser from sensorobjectlibrary import __version__ @@ -283,251 +279,6 @@ def bin_to_json(sol_bin, mac=None): return sol_json -def json_to_influxdb(sol_json, tags): - """ - Convert a JSON SOL object into a InfluxDB point - - :param dict sol_json: JSON SOL object - :param dict tags: A dictionary of tags - :return: InfluxDB point - :rtpe: list - """ - # tags - obj_tags = copy.deepcopy(tags) - - # fields - if sol_json['type'] == SolDefines.SOL_TYPE_DUST_NOTIF_HRNEIGHBORS: - fields = {} - for n in sol_json["value"]['neighbors']: - fields["neighbors:" + str(n['neighborId'])] = n - fields['numItems'] = sol_json["value"]['numItems'] - elif sol_json['type'] == SolDefines.SOL_TYPE_DUST_NOTIF_HRDISCOVERED: - fields = sol_json["value"] - elif sol_json['type'] == SolDefines.SOL_TYPE_DUST_NOTIF_HREXTENDED: - fields = {} - for item, value in enumerate(sol_json["value"]['RSSI']): - fields[str(item + 11)] = value # 11 is the first channel number - elif sol_json['type'] == SolDefines.SOL_TYPE_DUST_SNAPSHOT: - fields = {"mote": []} - for mote in sol_json["value"]: - mote["macAddress"] = _format_buffer(mote["macAddress"]) - for path in mote["paths"]: - path["macAddress"] = _format_buffer(path["macAddress"]) - fields["mote"].append(mote) - elif sol_json['type'] == SolDefines.SOL_TYPE_DUST_EVENTNETWORKRESET: - fields = {'value': 'dummy'} - else: - fields = sol_json["value"] - for (k, v) in fields.items(): - if type(v) == list: # mac - if k in ['macAddress', 'source', 'dest']: - fields[k] = _format_buffer(v) - elif k in ['sol_version', 'sdk_version', 'solmanager_version']: - fields[k] = ".".join(str(i) for i in v) - - f = flatdict.FlatDict(fields) - fields = {} - for (k, v) in f.items(): - fields[k] = v - - # add additionnal field or tag if apply_function exists - try: - obj_struct = SolDefines.solStructure(sol_json['type']) - if 'apply' in obj_struct: - for ap in obj_struct['apply']: - arg_list = [fields[arg] for arg in ap["args"]] - if "field" in ap: - fields[ap["field"]] = ap["function"](*arg_list) - if "tag" in ap: - obj_tags[ap["tag"]] = ap["function"](*arg_list) - except ValueError as err: - log.warning(err) - - # get SOL type - measurement = SolDefines.sol_type_to_type_name(sol_json['type']) - - # convert SOL timestamp to UTC - utc_time = sol_json["timestamp"]*1000000000 - - # fill "fields" if empty - if not fields: - fields = {'dummy': 'dummy'} - log.warning("'fields' empty for object {0}".format(sol_json)) - - sol_influxdb = { - "time": utc_time, - "tags": obj_tags, - "measurement": measurement, - "fields": fields, - } - - return sol_influxdb - -def influxdb_to_json(sol_influxdb): - """ - Converts an Influxdb query reply into a list of dicts. - - :param dict sol_influxdb: the result of a database query (sush as SELECT * FROM) - :return: a list of JSON SOL objects - :rtype: list - - """ - - # verify influxdb data - if not ("series" in sol_influxdb): - raise ValueError("Influxdb data not recognized") - - # init - json_list = [] - - # remove unused headers - for serie in sol_influxdb["series"]: - for val in serie['values']: - # convert to dict - d_influxdb = dict(zip(serie['columns'], val)) - - # unflat dict - obj_value = flatdict.FlatDict(d_influxdb).as_dict() - - # parse specific HR_NEIGHBORS - hr_nghb_name = SolDefines.sol_type_to_type_name( - SolDefines.SOL_TYPE_DUST_NOTIF_HRNEIGHBORS) - if serie['name'] == hr_nghb_name: - for i in range(0, len(obj_value["neighbors"])+1): - ngbr_id = str(i) - - # new HR_NGBR parsing - if ngbr_id in obj_value["neighbors"]: - if obj_value["neighbors"][ngbr_id]["neighborFlag"] is None: - del obj_value["neighbors"][ngbr_id] - - # old HR_NGBR parsing - if ngbr_id in obj_value: - if obj_value[ngbr_id]["neighborFlag"] is not None: - obj_value["neighbors"][ngbr_id] = obj_value[ngbr_id] - del obj_value[ngbr_id] - - # time is not passed in the "value" field - del obj_value["time"] - - # create final dict - jdic = { - 'type': serie['name'], - 'mac': serie['tags']['mac'], - 'value': obj_value, - 'timestamp': d_influxdb['time'], - } - json_list.append(jdic) - return json_list - -# ==== file manipulation - -def dumpToFile(sol_jsonl, file_name): - - with open(file_name, 'ab') as f: - for sol_json in sol_jsonl: - sol_bin = json_to_bin(sol_json) - sol_bin = hdlc.hdlcify(sol_bin) - sol_bin = bytes(sol_bin) - f.write(sol_bin) - -def loadFromFile(file_name, start_timestamp=None, end_timestamp=None): - - if start_timestamp is not None or end_timestamp is not None: - assert start_timestamp is not None and end_timestamp is not None - - if start_timestamp is None: - # retrieve all data - - (bins, _) = hdlc.dehdlcify(file_name) - - sol_jsonl = [] - for b in bins: - sol_jsonl += [bin_to_json(b)] - - else: - - #=== find start_offset - - while True: - - start_offset = None - - def one_object(offset): - (sol, offs) = hdlc.dehdlcify(file_name, fileOffset=offset, maxNum=1) - sol = sol[0] - sol = bin_to_json(sol) - return sol, offs - - def oneTimestamp(offset): - (osol, offs) = one_object(offset) - return osol['timestamp'], offs - - #=== get boundaries - - left_offset_start = 0 - try: - (left_timestamp, left_offset_stop) = oneTimestamp(left_offset_start) - except IndexError: - # complete file is corrupted - return [] - with open(file_name, 'rb') as f: - f.seek(0, os.SEEK_END) - right_offset_start = f.tell() - while True: - right_offset_start = _fileBackUpUntilStartFrame(file_name, right_offset_start) - try: - (right_timestamp, right_offset_stop) = oneTimestamp(right_offset_start) - except IndexError: - right_offset_start -= 1 - else: - break - - if left_timestamp > start_timestamp: - start_offset = left_timestamp - break - if right_timestamp < start_timestamp: - start_offset = right_timestamp - break - - #=== binary search - - while left_offset_stop < right_offset_start-1: - - cur_offset_start = int((right_offset_start-left_offset_start)/2+left_offset_start) - (cur_timestamp, cur_offset_stop) = oneTimestamp(cur_offset_start) - - if cur_timestamp == start_timestamp: - start_offset = cur_offset_start - break - elif cur_timestamp > start_timestamp: - right_offset_start = cur_offset_start - elif cur_timestamp < start_timestamp: - left_offset_start = cur_offset_start - left_offset_stop = cur_offset_stop - - if start_offset is None: - start_offset = left_offset_start - - break - - #=== read objects - - sol_jsonl = [] - - curOffset = start_offset - while True: - try: - (o, curOffset) = one_object(curOffset) - except IndexError: - # we have passed the end of the file - break - if o['timestamp'] > end_timestamp: - break - sol_jsonl += [o] - - return sol_jsonl - # ======================= private ========================================= def _split_dust_notif(dust_notif): @@ -996,17 +747,6 @@ def _get_sol_binary_value_snapshot(snapshot): return_val = [ord(c) for c in return_val] return return_val -# ==== file manipulation - -def _fileBackUpUntilStartFrame(file_name, curOffset): - with open(file_name, 'rb') as f: - f.seek(curOffset, os.SEEK_SET) - while True: - byte = f.read(1) - if byte == hdlc.HDLC_FLAG: - return f.tell()-1 - f.seek(-2, os.SEEK_CUR) - # ==== miscellaneous helpers def _num_to_list(num, length): diff --git a/sensorobjectlibrary/openhdlc.py b/sensorobjectlibrary/openhdlc.py deleted file mode 100644 index 3309ddd..0000000 --- a/sensorobjectlibrary/openhdlc.py +++ /dev/null @@ -1,209 +0,0 @@ -HDLC_FLAG = '\x7e' # ~ -HDLC_FLAG_ESCAPED = '\x5e' # ^ -HDLC_ESCAPE = '\x7d' # } -HDLC_ESCAPE_ESCAPED = '\x5d' # ] -HDLC_CRCINIT = 0xffff -HDLC_CRCGOOD = 0xf0b8 -HDLC_ESCAPE_MASK = 0x20 - -FCS16TAB = ( - 0x0000, 0x1189, 0x2312, 0x329b, 0x4624, 0x57ad, 0x6536, 0x74bf, - 0x8c48, 0x9dc1, 0xaf5a, 0xbed3, 0xca6c, 0xdbe5, 0xe97e, 0xf8f7, - 0x1081, 0x0108, 0x3393, 0x221a, 0x56a5, 0x472c, 0x75b7, 0x643e, - 0x9cc9, 0x8d40, 0xbfdb, 0xae52, 0xdaed, 0xcb64, 0xf9ff, 0xe876, - 0x2102, 0x308b, 0x0210, 0x1399, 0x6726, 0x76af, 0x4434, 0x55bd, - 0xad4a, 0xbcc3, 0x8e58, 0x9fd1, 0xeb6e, 0xfae7, 0xc87c, 0xd9f5, - 0x3183, 0x200a, 0x1291, 0x0318, 0x77a7, 0x662e, 0x54b5, 0x453c, - 0xbdcb, 0xac42, 0x9ed9, 0x8f50, 0xfbef, 0xea66, 0xd8fd, 0xc974, - 0x4204, 0x538d, 0x6116, 0x709f, 0x0420, 0x15a9, 0x2732, 0x36bb, - 0xce4c, 0xdfc5, 0xed5e, 0xfcd7, 0x8868, 0x99e1, 0xab7a, 0xbaf3, - 0x5285, 0x430c, 0x7197, 0x601e, 0x14a1, 0x0528, 0x37b3, 0x263a, - 0xdecd, 0xcf44, 0xfddf, 0xec56, 0x98e9, 0x8960, 0xbbfb, 0xaa72, - 0x6306, 0x728f, 0x4014, 0x519d, 0x2522, 0x34ab, 0x0630, 0x17b9, - 0xef4e, 0xfec7, 0xcc5c, 0xddd5, 0xa96a, 0xb8e3, 0x8a78, 0x9bf1, - 0x7387, 0x620e, 0x5095, 0x411c, 0x35a3, 0x242a, 0x16b1, 0x0738, - 0xffcf, 0xee46, 0xdcdd, 0xcd54, 0xb9eb, 0xa862, 0x9af9, 0x8b70, - 0x8408, 0x9581, 0xa71a, 0xb693, 0xc22c, 0xd3a5, 0xe13e, 0xf0b7, - 0x0840, 0x19c9, 0x2b52, 0x3adb, 0x4e64, 0x5fed, 0x6d76, 0x7cff, - 0x9489, 0x8500, 0xb79b, 0xa612, 0xd2ad, 0xc324, 0xf1bf, 0xe036, - 0x18c1, 0x0948, 0x3bd3, 0x2a5a, 0x5ee5, 0x4f6c, 0x7df7, 0x6c7e, - 0xa50a, 0xb483, 0x8618, 0x9791, 0xe32e, 0xf2a7, 0xc03c, 0xd1b5, - 0x2942, 0x38cb, 0x0a50, 0x1bd9, 0x6f66, 0x7eef, 0x4c74, 0x5dfd, - 0xb58b, 0xa402, 0x9699, 0x8710, 0xf3af, 0xe226, 0xd0bd, 0xc134, - 0x39c3, 0x284a, 0x1ad1, 0x0b58, 0x7fe7, 0x6e6e, 0x5cf5, 0x4d7c, - 0xc60c, 0xd785, 0xe51e, 0xf497, 0x8028, 0x91a1, 0xa33a, 0xb2b3, - 0x4a44, 0x5bcd, 0x6956, 0x78df, 0x0c60, 0x1de9, 0x2f72, 0x3efb, - 0xd68d, 0xc704, 0xf59f, 0xe416, 0x90a9, 0x8120, 0xb3bb, 0xa232, - 0x5ac5, 0x4b4c, 0x79d7, 0x685e, 0x1ce1, 0x0d68, 0x3ff3, 0x2e7a, - 0xe70e, 0xf687, 0xc41c, 0xd595, 0xa12a, 0xb0a3, 0x8238, 0x93b1, - 0x6b46, 0x7acf, 0x4854, 0x59dd, 0x2d62, 0x3ceb, 0x0e70, 0x1ff9, - 0xf78f, 0xe606, 0xd49d, 0xc514, 0xb1ab, 0xa022, 0x92b9, 0x8330, - 0x7bc7, 0x6a4e, 0x58d5, 0x495c, 0x3de3, 0x2c6a, 0x1ef1, 0x0f78, -) - -# =========================== public ====================================== - -def hdlcify(inBuf): - """ - - :param list inBuf: - :return: - """ - - # make copy of input - outBuf = ''.join([chr(b) for b in inBuf]) - - # calculate CRC - crc = HDLC_CRCINIT - for b in outBuf: - crc = _crcIteration(crc, b) - crc = 0xffff-crc - - # append CRC - outBuf = outBuf + chr(crc & 0xff) + chr((crc & 0xff00) >> 8) - - # stuff bytes - outBuf = outBuf.replace( - HDLC_ESCAPE, - HDLC_ESCAPE + HDLC_ESCAPE_ESCAPED) - outBuf = outBuf.replace( - HDLC_FLAG, - HDLC_ESCAPE + HDLC_FLAG_ESCAPED) - - # add flags - outBuf = HDLC_FLAG + outBuf + HDLC_FLAG - - return [ord(b) for b in outBuf] - -def dehdlcify(fileName, fileOffset=0, maxNum=None): - """ - - :param str fileName: - :param int fileOffset: - :param int maxNum: - :return: - """ - - returnVal = [] - - busyReceiving = False - lastRxByte = HDLC_FLAG - inputs = (None, None, None) - - print("####") - with open(fileName, 'r') as f: - - f.seek(fileOffset) - - rxByte = f.read(1) - - while rxByte != '': - - print(rxByte) - - if ( - busyReceiving is False and - lastRxByte == HDLC_FLAG and - rxByte != HDLC_FLAG - ): - # start of frame - - # I'm now receiving - busyReceiving = True - - # create the HDLC frame - inputs = _hdlc_input_open() - - # add the byte just received - inputs = _hdlc_input_write(inputs, rxByte) - - elif ( - busyReceiving is True and - rxByte != HDLC_FLAG - ): - # middle of frame - - # add the byte just received - inputs = _hdlc_input_write(inputs, rxByte) - - elif ( - busyReceiving is True and - rxByte == HDLC_FLAG - ): - # end of frame - - # finalize the HDLC frame - try: - inputs = _hdlc_input_close(inputs) - except ValueError: - # invalid HDLC frame - print("invalid") - pass - else: - returnVal += [[ord(b) for b in inputs[0]]] - if maxNum and len(returnVal) >= maxNum: - break - - busyReceiving = False - - else: - # between frames - pass - - lastRxByte = rxByte - - rxByte = f.read(1) - - return returnVal, f.tell() - -# =========================== private ===================================== - -def _crcIteration(crc, b): - return (crc >> 8) ^ FCS16TAB[((crc ^ (ord(b))) & 0xff)] - -def _hdlc_input_open(): - - # reset the input buffer - input_buf = [] - - # initialize the value of the CRC - input_crc = HDLC_CRCINIT - - return input_buf, input_crc, None - -def _hdlc_input_write(inputs, b): - input_buf, input_crc, input_escaping = inputs - if b == HDLC_ESCAPE: - input_escaping = True - else: - if input_escaping is True: - b = chr(ord(b) ^ HDLC_ESCAPE_MASK) - input_escaping = False - - # add byte to input buffer - input_buf += [b] - - # iterate through CRC calculator - input_crc = _crcIteration(input_crc, b) - - return input_buf, input_crc, input_escaping - -def _hdlc_input_close(inputs): - input_buf, input_crc, input_escaping = inputs - print(input_buf) - - # verify the validity of the frame - if input_crc == input_crc: - # the CRC is correct - - # remove the CRC from the input buffer - input_buf = input_buf[:-2] - else: - input_buf = [] - - raise ValueError("invalid CRC") - - # reset escaping - input_escaping = False - - return input_buf, input_crc, input_escaping diff --git a/setup.py b/setup.py index 0cf309d..a4a72c3 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,6 @@ url='https://github.com/realms-team/sol', keywords=['wireless', 'sensor', 'network'], install_requires=[ - 'flatdict==1.2.0', 'pyserial>=3.4' ], package_data={'': ['LICENSE']}, diff --git a/tests/test_chain.py b/tests/test_chain.py index 9b3a4dd..86b0722 100755 --- a/tests/test_chain.py +++ b/tests/test_chain.py @@ -60,26 +60,7 @@ "o" : [ "EwECAwQFBgcIEhMUFQ4BAgMEBQYHCA==" ] - }, - "influxdb": - { - "time" : TIMESTAMP*1000000000, - "tags" : { - 'mac' : '01-02-03-04-05-06-07-08', - 'site' : 'super_site', - 'latitude' : 55.5555, - 'longitude' : -44.4444, - }, - "measurement" : 'SOL_TYPE_DUST_NOTIFDATA', - "fields" : { - 'srcPort' : 0x0102, - 'dstPort' : 0x0304, - 'data:0' : 0x05, - 'data:1' : 0x06, - 'data:2' : 0x07, - 'data:3' : 0x08, - }, - }, + } }, ], }, @@ -158,32 +139,7 @@ "o" : [ "EwECAwQFBgcIEhMUFRAAAAF5MRULhQA6AAAABQAAAAAAAAAAAAA=" ] - }, - "influxdb": { - "time" : TIMESTAMP*1000000000, - "tags" : { - 'mac' : '01-02-03-04-05-06-07-08', - 'site' : 'super_site', - 'latitude' : 55.5555, - 'longitude' : -44.4444, - }, - "measurement": 'SOL_TYPE_DUST_NOTIF_HRDEVICE', - "fields" : { - 'badLinkFrameId': 0, - 'badLinkOffset': 0, - 'badLinkSlot': 0, - 'batteryVoltage': 2949, - 'charge': 377, - 'numMacDropped': 0, - 'numRxLost': 0, - 'numRxOk': 5, - 'numTxBad': 0, - 'numTxFail': 0, - 'numTxOk': 58, - 'queueOcc': 49, - 'temperature': 21, - }, - }, + } }, ], }, @@ -252,35 +208,7 @@ "o": [ "EwECAwQFBgcIEhMUFRAAAAF5MRULhQA6AAAABQAAAAAAAAAAAAAAAAE=" ] - }, - "influxdb": { - "time": TIMESTAMP * 1000000000, - "tags": { - 'mac': '01-02-03-04-05-06-07-08', - 'site': 'super_site', - 'latitude': 55.5555, - 'longitude': -44.4444, - }, - "measurement": 'SOL_TYPE_DUST_NOTIF_HRDEVICE', - "fields": { - 'badLinkFrameId': 0, - 'badLinkOffset': 0, - 'badLinkSlot': 0, - 'batteryVoltage': 2949, - 'charge': 377, - 'numMacCrcErr': 1, - 'numMacDropped': 0, - 'numMacMicErr': 0, - 'numNetMicErr': 0, - 'numRxLost': 0, - 'numRxOk': 5, - 'numTxBad': 0, - 'numTxFail': 0, - 'numTxOk': 58, - 'queueOcc': 49, - 'temperature': 21, - }, - }, + } }, ], }, @@ -394,35 +322,7 @@ "o" : [ "EwECAwQFBgcIEhMUFRAAAAF5MRULhQA6AAAABQAAAAAAAAAAAAAAAAE=" ] - }, - "influxdb": { - "time" : TIMESTAMP*1000000000, - "tags" : { - 'mac' : '01-02-03-04-05-06-07-08', - 'site' : 'super_site', - 'latitude' : 55.5555, - 'longitude' : -44.4444, - }, - "measurement": 'SOL_TYPE_DUST_NOTIF_HRDEVICE', - "fields" : { - 'badLinkFrameId': 0, - 'badLinkOffset': 0, - 'badLinkSlot': 0, - 'batteryVoltage': 2949, - 'charge': 377, - 'numMacCrcErr': 1, - 'numMacDropped': 0, - 'numMacMicErr': 0, - 'numNetMicErr': 0, - 'numRxLost': 0, - 'numRxOk': 5, - 'numTxBad': 0, - 'numTxFail': 0, - 'numTxOk': 58, - 'queueOcc': 49, - 'temperature': 21, - }, - }, + } }, { "json": { @@ -489,45 +389,7 @@ "o" : [ "EwECAwQFBgcIEhMUFRIICAAD2wIAB8IBAAraAwAM3AEABd0BAAjZAQAJ1AEAC88B" ] - }, - "influxdb": { - "time" : TIMESTAMP*1000000000, - "tags" : { - 'mac' : '01-02-03-04-05-06-07-08', - 'site' : 'super_site', - 'latitude' : 55.5555, - 'longitude' : -44.4444, - }, - "measurement": 'SOL_TYPE_DUST_NOTIF_HRDISCOVERED', - "fields" : { - 'discoveredNeighbors:0:neighborId': 3, - 'discoveredNeighbors:0:numRx': 2, - 'discoveredNeighbors:0:rssi': -37, - 'discoveredNeighbors:1:neighborId': 7, - 'discoveredNeighbors:1:numRx': 1, - 'discoveredNeighbors:1:rssi': -62, - 'discoveredNeighbors:2:neighborId': 10, - 'discoveredNeighbors:2:numRx': 3, - 'discoveredNeighbors:2:rssi': -38, - 'discoveredNeighbors:3:neighborId': 12, - 'discoveredNeighbors:3:numRx': 1, - 'discoveredNeighbors:3:rssi': -36, - 'discoveredNeighbors:4:neighborId': 5, - 'discoveredNeighbors:4:numRx': 1, - 'discoveredNeighbors:4:rssi': -35, - 'discoveredNeighbors:5:neighborId': 8, - 'discoveredNeighbors:5:numRx': 1, - 'discoveredNeighbors:5:rssi': -39, - 'discoveredNeighbors:6:neighborId': 9, - 'discoveredNeighbors:6:numRx': 1, - 'discoveredNeighbors:6:rssi': -44, - 'discoveredNeighbors:7:neighborId': 11, - 'discoveredNeighbors:7:numRx': 1, - 'discoveredNeighbors:7:rssi': -49, - 'numItems': 8, - 'numJoinParents': 8, - }, - }, + } }, ], }, @@ -649,50 +511,7 @@ "o" : [ "EwECAwQFBgcIEhMUFREFAAQA3wBnAAAAAgABAMQAwQAOAAQABgDkAAAAAAAoAAcAxgAAAAAAYgAMANwAAAAAAGE=", ] - }, - "influxdb": { - "time" : TIMESTAMP*1000000000, - "tags" : { - 'mac' : '01-02-03-04-05-06-07-08', - 'site' : 'super_site', - 'latitude' : 55.5555, - 'longitude' : -44.4444, - }, - "measurement": 'SOL_TYPE_DUST_NOTIF_HRNEIGHBORS', - "fields" : { - 'neighbors:4:neighborFlag': 0, - 'neighbors:4:neighborId': 4, - 'neighbors:4:numRxPackets': 2, - 'neighbors:4:numTxFailures': 0, - 'neighbors:4:numTxPackets': 103, - 'neighbors:4:rssi': -33, - 'neighbors:1:neighborFlag': 0, - 'neighbors:1:neighborId': 1, - 'neighbors:1:numRxPackets': 4, - 'neighbors:1:numTxFailures': 14, - 'neighbors:1:numTxPackets': 193, - 'neighbors:1:rssi': -60, - 'neighbors:6:neighborFlag': 0, - 'neighbors:6:neighborId': 6, - 'neighbors:6:numRxPackets': 40, - 'neighbors:6:numTxFailures': 0, - 'neighbors:6:numTxPackets': 0, - 'neighbors:6:rssi': -28, - 'neighbors:7:neighborFlag': 0, - 'neighbors:7:neighborId': 7, - 'neighbors:7:numRxPackets': 98, - 'neighbors:7:numTxFailures': 0, - 'neighbors:7:numTxPackets': 0, - 'neighbors:7:rssi': -58, - 'neighbors:12:neighborFlag': 0, - 'neighbors:12:neighborId': 12, - 'neighbors:12:numRxPackets': 97, - 'neighbors:12:numTxFailures': 0, - 'neighbors:12:numTxPackets': 0, - 'neighbors:12:rssi': -36, - 'numItems': 5, - }, - }, + } } ] }, @@ -773,34 +592,7 @@ "o": [ "EwECAwQFBgcIEhMUFUQBS6MADAAAowANAAGnAAgAAaQACwAAowAJAACmAA0AAKMACgAAowAPAACjAAsAAaMADgAApgAMAACkAA8AAKwADgAAowANAACjAA8AAA==", ] - }, - "influxdb": { - "time": TIMESTAMP * 1000000000, - "tags": { - 'mac': '01-02-03-04-05-06-07-08', - 'site': 'super_site', - 'latitude': 55.5555, - 'longitude': -44.4444, - }, - "measurement": 'SOL_TYPE_DUST_NOTIF_HREXTENDED', - "fields": { - '11:txUnicastAttempts': 12, '11:idleRssi': -93, '11:txUnicastFailures': 0, - '12:txUnicastAttempts': 13, '12:idleRssi': -93, '12:txUnicastFailures': 1, - '13:txUnicastAttempts': 8, '13:idleRssi': -89, '13:txUnicastFailures': 1, - '14:txUnicastAttempts': 11, '14:idleRssi': -92, '14:txUnicastFailures': 0, - '15:txUnicastAttempts': 9, '15:idleRssi': -93, '15:txUnicastFailures': 0, - '16:txUnicastAttempts': 13, '16:idleRssi': -90, '16:txUnicastFailures': 0, - '17:txUnicastAttempts': 10, '17:idleRssi': -93, '17:txUnicastFailures': 0, - '18:txUnicastAttempts': 15, '18:idleRssi': -93, '18:txUnicastFailures': 0, - '19:txUnicastAttempts': 11, '19:idleRssi': -93, '19:txUnicastFailures': 1, - '20:txUnicastAttempts': 14, '20:idleRssi': -93, '20:txUnicastFailures': 0, - '21:txUnicastAttempts': 12, '21:idleRssi': -90, '21:txUnicastFailures': 0, - '22:txUnicastAttempts': 15, '22:idleRssi': -92, '22:txUnicastFailures': 0, - '23:txUnicastAttempts': 14, '23:idleRssi': -84, '23:txUnicastFailures': 0, - '24:txUnicastAttempts': 13, '24:idleRssi': -93, '24:txUnicastFailures': 0, - '25:txUnicastAttempts': 15, '25:idleRssi': -93, '25:txUnicastFailures': 0 - }, - }, + } } ] }, @@ -843,19 +635,7 @@ "o" : [ "EwMDAwMDAwMDEhMUFRQBAQEBAQEBAQICAgICAgICAw==" ] - }, - "influxdb": { - "time" : TIMESTAMP*1000000000, - "tags" : { - 'mac' : '03-03-03-03-03-03-03-03', - }, - "measurement": 'SOL_TYPE_DUST_EVENTPATHCREATE', - "fields" : { - 'source' : '01-01-01-01-01-01-01-01', - 'dest' : '02-02-02-02-02-02-02-02', - 'direction' : 3, - }, - }, + } } ] }, @@ -898,19 +678,7 @@ "o" : [ "EwMDAwMDAwMDEhMUFRUBAQEBAQEBAQICAgICAgICAw==", ], - }, - "influxdb": { - "time" : TIMESTAMP*1000000000, - "tags" : { - 'mac' : '03-03-03-03-03-03-03-03', - }, - "measurement": 'SOL_TYPE_DUST_EVENTPATHDELETE', - "fields" : { - 'source' : '01-01-01-01-01-01-01-01', - 'dest' : '02-02-02-02-02-02-02-02', - 'direction' : 3, - }, - }, + } } ] }, @@ -947,17 +715,7 @@ "o" : [ "EwMDAwMDAwMDEhMUFRkBAQEBAQEBAQ==", ] - }, - "influxdb": { - "time" : TIMESTAMP*1000000000, - "tags" : { - 'mac' : '03-03-03-03-03-03-03-03', - }, - "measurement": 'SOL_TYPE_DUST_EVENTMOTEJOIN', - "fields" : { - 'macAddress' : '01-01-01-01-01-01-01-01', - }, - }, + } } ] }, @@ -997,18 +755,7 @@ "o" : [ "EwMDAwMDAwMDEhMUFRoBAQEBAQEBAQIC", ], - }, - "influxdb": { - "time" : TIMESTAMP*1000000000, - "tags" : { - 'mac' : '03-03-03-03-03-03-03-03', - }, - "measurement": 'SOL_TYPE_DUST_EVENTMOTECREATE', - "fields" : { - 'macAddress' : '01-01-01-01-01-01-01-01', - 'moteId' : 0x0202, - }, - }, + } } ] }, @@ -1048,19 +795,7 @@ "o" : [ "EwMDAwMDAwMDEhMUFRsBAQEBAQEBAQIC", ], - }, - "influxdb": - { - "time" : TIMESTAMP*1000000000, - "tags" : { - 'mac' : '03-03-03-03-03-03-03-03', - }, - "measurement": 'SOL_TYPE_DUST_EVENTMOTEDELETE', - "fields" : { - 'macAddress' : '01-01-01-01-01-01-01-01', - 'moteId' : 0x0202, - }, - }, + } } ] }, @@ -1097,17 +832,7 @@ "o" : [ "EwMDAwMDAwMDEhMUFRwBAQEBAQEBAQ==", ] - }, - "influxdb": { - "time" : TIMESTAMP*1000000000, - "tags" : { - 'mac' : '03-03-03-03-03-03-03-03', - }, - "measurement": 'SOL_TYPE_DUST_EVENTMOTELOST', - "fields" : { - 'macAddress' : '01-01-01-01-01-01-01-01', - }, - }, + } }, ], }, @@ -1144,17 +869,7 @@ "o" : [ "EwMDAwMDAwMDEhMUFR0BAQEBAQEBAQ==", ] - }, - "influxdb": { - "time" : TIMESTAMP*1000000000, - "tags" : { - 'mac' : '03-03-03-03-03-03-03-03', - }, - "measurement": 'SOL_TYPE_DUST_EVENTMOTEOPERATIONAL', - "fields" : { - 'macAddress' : '01-01-01-01-01-01-01-01', - }, - }, + } } ] }, @@ -1197,20 +912,7 @@ "o" : [ "EwECAwQFBgcIEhMUFScKMw==", ], - }, - "influxdb": { - "time" : TIMESTAMP*1000000000, - "tags" : { - 'mac' : '01-02-03-04-05-06-07-08', - 'site' : 'super_site', - 'latitude' : 55.5555, - 'longitude' : -44.4444, - }, - "measurement": 'SOL_TYPE_DUST_OAP_TEMPSAMPLE', - "fields" : { - 'temperature': 0x0a33, - }, - }, + } }, ], }, @@ -1253,20 +955,7 @@ "o" : [ "EwECAwQFBgcIEhMUFSf/hQ==", ], - }, - "influxdb": { - "time" : TIMESTAMP*1000000000, - "tags" : { - 'mac' : '01-02-03-04-05-06-07-08', - 'site' : 'super_site', - 'latitude' : 55.5555, - 'longitude' : -44.4444, - }, - "measurement": 'SOL_TYPE_DUST_OAP_TEMPSAMPLE', - "fields" : { - 'temperature': -123, - }, - }, + } }, ], }, @@ -1305,18 +994,7 @@ "o" : [ "EwMDAwMDAwMDEhMUFToBAQEBAQEBAQE=" ] - }, - "influxdb": { - "time" : TIMESTAMP*1000000000, - "tags" : { - 'mac' : '03-03-03-03-03-03-03-03', - }, - "measurement": 'SOL_TYPE_DUST_EVENTJOINFAILED', - "fields" : { - 'macAddress' : '01-01-01-01-01-01-01-01', - 'reason' : 1 - }, - }, + } } ] }, @@ -1351,23 +1029,7 @@ "o" : [ "EwECAwQFBgcIEhMUFSIKMwtEAQQ=", ], - }, - "influxdb": { - "time" : TIMESTAMP*1000000000, - "tags" : { - 'mac' : '01-02-03-04-05-06-07-08', - 'site' : 'super_site', - 'latitude' : 55.5555, - 'longitude' : -44.4444, - }, - "measurement": 'SOL_TYPE_JUDD_T2D2R1N1', - "fields" : { - 'temperature': 0x0a33, - 'depth': 0x0b44, - 'numReadings': 0x01, - 'retries': 0x04, - }, - }, + } }, ], }, @@ -1480,60 +1142,7 @@ "o" : [ "EwMDAwMDAwMDEhMUFSACAQIDBAUGBwgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhIiMkJSYnAhESExQVFhcYLC0u//4hIiMkJSYnKCwtLv/+MTIzNDU2NzgJCgsMDQ4PEBESExQVFhcYGRobHB0eHyAhIiMkJSYnAUFCQ0RFRkdILC0u//4=", ] - }, - "influxdb": { - "time" : TIMESTAMP*1000000000, - "tags" : { - 'mac' : '03-03-03-03-03-03-03-03', - }, - "measurement": 'SOL_TYPE_DUST_SNAPSHOT', - "fields" : { - 'mote:0:macAddress': '01-02-03-04-05-06-07-08', - 'mote:0:moteId': 0x090a, # INT16U H - 'mote:0:isAP': 0x0b, # BOOL B - 'mote:0:state': 0x0c, # INT8U B - 'mote:0:isRouting': 0x0d, # BOOL B - 'mote:0:numNbrs': 0x0e, # INT8U B - 'mote:0:numGoodNbrs': 0x0f, # INT8U B - 'mote:0:requestedBw': 0x10111213, # INT32U I - 'mote:0:totalNeededBw': 0x14151617, # INT32U I - 'mote:0:assignedBw': 0x18191a1b, # INT32U I - 'mote:0:packetsReceived': 0x1c1d1e1f, # INT32U I - 'mote:0:packetsLost': 0x20212223, # INT32U I - 'mote:0:avgLatency': 0x24252627, # INT32U I - 'mote:0:paths:0:macAddress': '11-12-13-14-15-16-17-18', - 'mote:0:paths:0:direction': 0x2c, # INT8U B - 'mote:0:paths:0:numLinks': 0x2d, # INT8U B - 'mote:0:paths:0:quality': 0x2e, # INT8U B - 'mote:0:paths:0:rssiSrcDest': -1, # INT8 b - 'mote:0:paths:0:rssiDestSrc': -2, # INT8 b - 'mote:0:paths:1:macAddress': '21-22-23-24-25-26-27-28', - 'mote:0:paths:1:direction': 0x2c, # INT8U B - 'mote:0:paths:1:numLinks': 0x2d, # INT8U B - 'mote:0:paths:1:quality': 0x2e, # INT8U B - 'mote:0:paths:1:rssiSrcDest': -1, # INT8 b - 'mote:0:paths:1:rssiDestSrc': -2, # INT8 b - 'mote:1:macAddress': '31-32-33-34-35-36-37-38', - 'mote:1:moteId': 0x090a, # INT16U - 'mote:1:isAP': 0x0b, # BOOL - 'mote:1:state': 0x0c, # INT8U - 'mote:1:isRouting': 0x0d, # BOOL - 'mote:1:numNbrs': 0x0e, # INT8U - 'mote:1:numGoodNbrs': 0x0f, # INT8U - 'mote:1:requestedBw': 0x10111213, # INT32U - 'mote:1:totalNeededBw': 0x14151617, # INT32U - 'mote:1:assignedBw': 0x18191a1b, # INT32U - 'mote:1:packetsReceived': 0x1c1d1e1f, # INT32U - 'mote:1:packetsLost': 0x20212223, # INT32U - 'mote:1:avgLatency': 0x24252627, # INT32U - 'mote:1:paths:0:macAddress': '41-42-43-44-45-46-47-48', - 'mote:1:paths:0:direction': 0x2c, # INT8U - 'mote:1:paths:0:numLinks': 0x2d, # INT8U - 'mote:1:paths:0:quality': 0x2e, # INT8U - 'mote:1:paths:0:rssiSrcDest': -1, # INT8 - 'mote:1:paths:0:rssiDestSrc': -2, # INT8 - }, - }, + } }, ], }, @@ -1566,22 +1175,7 @@ "o" : [ "EwECAwQFBgcIEhMUFSgBAgMEBQYHCAkBAgM=" ] - }, - "influxdb": { - "time" : TIMESTAMP*1000000000, - "tags" : { - 'mac' : '01-02-03-04-05-06-07-08', - 'site' : 'super_site', - 'latitude' : 55.5555, - 'longitude' : -44.4444, - }, - "measurement": 'SOL_TYPE_SOLMANAGER_STATS', - "fields" : { - "sol_version" : "1.2.3.4", - "solmanager_version" : "5.6.7.8", - "sdk_version" : "9.1.2.3", - }, - }, + } }, ], }, @@ -1632,27 +1226,7 @@ "o" : [ "EwECAwQFBgcIEhMUFTE8ZQGZiAE=", ], - }, - "influxdb": { - "time" : TIMESTAMP*1000000000, - "tags" : { - 'mac' : '01-02-03-04-05-06-07-08', - 'site' : 'super_site', - 'latitude' : 55.5555, - 'longitude' : -44.4444, - 't_Nval' : 0x01, - 'rh_Nval' : 0x01, - }, - "measurement": 'SOL_TYPE_SENS_SHT25_T2N1H2N1', - "fields" : { - "temp_raw" : 0x653c, - "t_Nval" : 0x01, - "rh_raw" : 0x8899, - "rh_Nval" : 0x01, - 'rh_phys': 60.69807434082031, - 'temp_phys': 22.63790771484374, - }, - }, + } }, ], }, @@ -1683,22 +1257,7 @@ "o" : [ "EwECAwQFBgcIV5oOADiamSFBzcx0QQ==", ], - }, - "influxdb": { - "time" : 0x579a0e00*1000000000, - "tags" : { - 'mac' : '01-02-03-04-05-06-07-08', - 'site' : 'super_site', - 'latitude' : 55.5555, - 'longitude' : -44.4444, - 'depth' : 15.300000190734863, - }, - "measurement": 'SOL_TYPE_SENS_MPS1', - "fields" : { - "die_raw" : 10.100000381469727, - "depth" : 15.300000190734863, - }, - }, + } }, ], }, @@ -1753,23 +1312,7 @@ "o" : [ "EwECAwQFBgcIEhMUFTIAAAE=", ] - }, - "influxdb": { - "time" : TIMESTAMP*1000000000, - "tags" : { - 'mac' : '01-02-03-04-05-06-07-08', - 'site' : 'super_site', - 'latitude' : 55.5555, - 'longitude' : -44.4444, - "N" : 1, - }, - "measurement": 'SOL_TYPE_SENS_NEOVBAT_V2N1', - "fields" : { - "voltage" : 0, - "vol_phys" : 0, - "N" : 1, - }, - }, + } }, # SOL_TYPE_SENS_SHT25_T2N1H2N1 { @@ -1800,27 +1343,7 @@ "o" : [ "EwECAwQFBgcIEhMUFTE8ZQGiZwE=", ], - }, - "influxdb": { - "time" : TIMESTAMP*1000000000, - "tags" : { - 'mac' : '01-02-03-04-05-06-07-08', - 'site' : 'super_site', - 'latitude' : 55.5555, - 'longitude' : -44.4444, - 't_Nval' : 0x01, - 'rh_Nval' : 0x01, - }, - "measurement": 'SOL_TYPE_SENS_SHT25_T2N1H2N1', - "fields" : { - "temp_raw" : 0x653c, - "t_Nval" : 0x01, - "rh_raw" : 0x67a2, - "rh_Nval" : 0x01, - "rh_phys" : 44.601959228515625, - "temp_phys" : 22.63790771484374, - }, - }, + } }, # SOL_TYPE_SENS_GS3_I1D4T4E4N1 { @@ -1853,26 +1376,7 @@ "o" : [ "EwECAwQFBgcIEhMUFTAACtcjQGZmskEAAIA/AQ==", ], - }, - "influxdb": { - "time" : TIMESTAMP*1000000000, - "tags" : { - 'mac' : '01-02-03-04-05-06-07-08', - 'site' : 'super_site', - 'latitude' : 55.5555, - 'longitude' : -44.4444, - 'sub_id' : 0x00, - 'Nval' : 0x01, - }, - "measurement": 'SOL_TYPE_SENS_GS3_I1D4T4E4N1', - "fields" : { - "sub_id" : 0x00, - "dielect" : 2.559999942779541, - "temp" : 22.299999237060547, - "eleCond" : 1.0, - "Nval" : 0x01, - }, - }, + } }, # SOL_TYPE_SENS_MB7363_D2S2N1L1G1 { @@ -1905,27 +1409,7 @@ "o" : [ "EwECAwQFBgcIEhMUFSkbAgEAG1RV", ] - }, - "influxdb": { - "time" : TIMESTAMP*1000000000, - "tags" : { - 'mac' : '01-02-03-04-05-06-07-08', - 'site' : 'super_site', - 'latitude' : 55.5555, - 'longitude' : -44.4444, - 'mean_d2g' : 0x021b, - 'stdev' : 0x0001, - 'Nval' : 0x1b, - }, - "measurement": 'SOL_TYPE_SENS_MB7363_D2S2N1L1G1', - "fields" : { - "mean_d2g" : 0x021b, - "stdev" : 0x0001, - "Nval" : 0x1b, - "Nltm" : 0x54, - "NgtM" : 0x55, - }, - }, + } } ] }, @@ -1981,25 +1465,7 @@ "o" : [ "EwECAwQFBgcIEhMUFUBjh50nAw==", ] - }, - "influxdb": { - "time" : TIMESTAMP*1000000000, - "tags" : { - 'mac' : '01-02-03-04-05-06-07-08', - 'site' : 'super_site', - 'latitude' : 55.5555, - 'longitude' : -44.4444, - 'id' : 3, - }, - "measurement": 'SOL_TYPE_TEMPRH_SHT31', - "fields" : { - "temp_raw" : 0x6387, - "rh_raw" : 0x9d27, - "temp_phys" : 23, - "rh_phys" : 61, - "id" : 3, - }, - }, + } }, { "json" : { @@ -2027,25 +1493,7 @@ "o" : [ "EwECAwQFBgcIEhMUFUBjh50nAw==", ] - }, - "influxdb": { - "time" : TIMESTAMP*1000000000, - "tags" : { - 'mac' : '01-02-03-04-05-06-07-08', - 'site' : 'super_site', - 'latitude' : 55.5555, - 'longitude' : -44.4444, - 'id' : 3, - }, - "measurement": 'SOL_TYPE_TEMPRH_SHT31', - "fields" : { - "temp_raw" : 0x6387, - "rh_raw" : 0x9d27, - "temp_phys" : 23, - "rh_phys" : 61, - "id" : 3, - }, - }, + } }, ], }, @@ -2077,23 +1525,7 @@ "o" : [ "EwECAwQFBgcIEhMUFUmMWwAA3EEhAPwABA==", ] - }, - "influxdb": { - "time" : TIMESTAMP*1000000000, - "tags" : { - 'mac' : '01-02-03-04-05-06-07-08', - 'site' : 'super_site', - 'latitude' : 55.5555, - 'longitude' : -44.4444, - 'id' : 4, - }, - "measurement": 'SOL_TYPE_SENS_INDUCTION_CURRENT_V_SOURCE', - "fields" : {'accu_sum_of_squares': 2179548, - 'sensor_id': 4, - 'accu_sum': 23436, - 'sample_count': 252, - 'current_A': 0.0}, - }, + } }, ], }, @@ -2164,14 +1596,3 @@ def test_chain(sol_chain_example): print(sol_json) print(example["json"]) assert sol_json==example["json"] - - # json->influxdb - sol_influxdb = None - if MACMANAGER == sol_json['mac']: - sol_influxdb = sol.json_to_influxdb(sol_json,TAGS_DEFAULT) - else: - sol_influxdb = sol.json_to_influxdb(sol_json,TAGS) - print('=====\njson->influxdb') - pp.pprint(sol_influxdb) - print(example["influxdb"]) - assert sol_influxdb==example["influxdb"] diff --git a/tests/test_file.py b/tests/test_file.py deleted file mode 100755 index 6394550..0000000 --- a/tests/test_file.py +++ /dev/null @@ -1,224 +0,0 @@ -from .context import sol -import os -import random - -import pytest - -# ============================ defines =============================== - -FILENAME = 'temp_test_file.sol' -EXAMPLE_MAC = [0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08] - - -# ============================ fixtures ============================== - -@pytest.fixture -def removeFile(): - yield - try: - os.remove(FILENAME) - except OSError: - # if file does not exist. NOT an error. - pass - -EXPECTEDRANGE = [ - ( - 100, # start_timestamp - 300, # end_timestamp - 100, # idxMin - 301, # idxMax - ), - ( - -5, # start_timestamp - 300, # end_timestamp - 0, # idxMin - 301, # idxMax - ), - ( - 100, # start_timestamp - 1100, # end_timestamp - 100, # idxMin - 1000, # idxMax - ), - ( - -5, # start_timestamp - 1100, # end_timestamp - 0, # idxMin - 1000, # idxMax - ), - ( - -500, # start_timestamp - -100, # end_timestamp - 0, # idxMin - 0, # idxMax - ), - ( - 1100, # start_timestamp - 1500, # end_timestamp - 0, # idxMin - 0, # idxMax - ), -] - - -@pytest.fixture(params=EXPECTEDRANGE) -def expectedRange(request): - return request.param - - -# ============================ helpers =============================== - -def random_sol_json(timestamp=0): - returnVal = { - "timestamp": timestamp, - "mac": sol._format_buffer([random.randint(0x00, 0xff)] * 8), - "type": 0x0e, - "value": { - 'srcPort': random.randint(0x0000, 0xffff), - 'dstPort': random.randint(0x0000, 0xffff), - 'data': [random.randint(0x00, 0xff)] * random.randint(10, 30), - }, - } - - return returnVal - - -# ============================ tests ================================= - -def test_dump_load(removeFile): - # prepare dicts to dump - sol_jsonl_toDump = [random_sol_json() for _ in range(1000)] - - # dump - sol.dumpToFile(sol_jsonl_toDump, FILENAME) - - # load - sol_jsonl_loaded = sol.loadFromFile(FILENAME) - - # compare - print(sol_jsonl_loaded) - print(sol_jsonl_toDump) - assert sol_jsonl_loaded == sol_jsonl_toDump - - -def test_dump_corrupt_load(removeFile): - # prepare dicts to dump - sol_jsonl_toDump1 = [random_sol_json() for _ in range(500)] - sol_jsonl_toDump2 = [random_sol_json() for _ in range(500)] - - # write first set of valid data - sol.dumpToFile(sol_jsonl_toDump1, FILENAME) - # write HDLC frame with corrupt CRC - with open(FILENAME, 'ab') as f: - bin_data = ''.join([chr(b) for b in - [0x7E, 0x10, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x00, 0x00, 0x00, 0x00, 0x75, - 0x94, 0xE8, 0x0B, 0x6B, 0xAE, 0xE1, 0x19, 0x54, 0x74, 0xF3, 0x00, 0x00, 0x7E]]) - f.write(bin_data) - # write some garbage - with open(FILENAME, 'ab') as f: - f.write("############################## garbage ##############################") - # write second set of valid data - sol.dumpToFile(sol_jsonl_toDump2, FILENAME) - - # load - sol_jsonl_loaded = sol.loadFromFile(FILENAME) - - # compare - assert sol_jsonl_loaded == sol_jsonl_toDump1 + sol_jsonl_toDump2 - - -def test_retrieve_range(removeFile): - # prepare dicts to dump - sol_jsonl_toDump = [random_sol_json(timestamp=ts) for ts in range(1000)] - - # dump - sol.dumpToFile(sol_jsonl_toDump, FILENAME) - - # load - sol_jsonl_loaded = sol.loadFromFile( - FILENAME, - start_timestamp=100, - end_timestamp=1900, - ) - - # compare - assert sol_jsonl_loaded == sol_jsonl_toDump[100:] - - -def test_retrieve_range_corrupt_beginning(removeFile): - # prepare dicts to dump - sol_jsonl_toDump = [random_sol_json(timestamp=ts) for ts in range(1000)] - - # dump - with open(FILENAME, 'ab') as f: - f.write("garbage") - sol.dumpToFile(sol_jsonl_toDump, FILENAME) - - # load - sol_jsonl_loaded = sol.loadFromFile( - FILENAME, - start_timestamp=100, - end_timestamp=800 - ) - - # compare - assert sol_jsonl_loaded == sol_jsonl_toDump[100:801] - - -def test_retrieve_range_corrupt_middle(removeFile): - # prepare dicts to dump - sol_jsonl_toDump1 = [random_sol_json(timestamp=ts) for ts in range(500)] - sol_jsonl_toDump2 = [random_sol_json(timestamp=500 + ts) for ts in range(500)] - - # dump - sol.dumpToFile(sol_jsonl_toDump1, FILENAME) - with open(FILENAME, 'ab') as f: - f.write("garbage") - sol.dumpToFile(sol_jsonl_toDump2, FILENAME) - - # load - sol_jsonl_loaded = sol.loadFromFile( - FILENAME, - start_timestamp=100, - end_timestamp=800, - ) - - # compare - assert sol_jsonl_loaded == (sol_jsonl_toDump1 + sol_jsonl_toDump2)[100:801] - - -def test_retrieve_range_corrupt_end(removeFile): - # prepare dicts to dump - sol_jsonl_toDump = [random_sol_json(timestamp=ts) for ts in range(1000)] - - # dump - sol.dumpToFile(sol_jsonl_toDump, FILENAME) - with open(FILENAME, 'ab') as f: - f.write("garbage") - - # load - sol_jsonl_loaded = sol.loadFromFile( - FILENAME, - start_timestamp=100, - end_timestamp=800 - ) - - # compare - assert sol_jsonl_loaded == sol_jsonl_toDump[100:801] - - -def test_retrieve_range_corrupt_all(removeFile): - # dump - with open(FILENAME, 'ab') as f: - for _ in range(100): - f.write("garbage".encode()) - - # load - sol_jsonl_loaded = sol.loadFromFile( - FILENAME, - start_timestamp=100, - end_timestamp=800, - ) - - # compare - assert sol_jsonl_loaded == [] diff --git a/tests/test_hdlc.py b/tests/test_hdlc.py deleted file mode 100755 index 71d4e8c..0000000 --- a/tests/test_hdlc.py +++ /dev/null @@ -1,117 +0,0 @@ -import os -from sensorobjectlibrary import openhdlc as hdlc -from sensorobjectlibrary import Sol as sol - -JSON = { - 'timestamp': 1521645792, - 'mac': '00-17-0d-00-00-58-5b-02', - 'type': 33, - 'value': { - 'manager': '00-17-0d-00-00-58-5b-02', - 'valid': True, - 'snapshot': { - 'getNetworkInfo': { - 'numLostPackets': 0, - 'advertisementState': 0, - 'ipv6Address': 'fe80:0000:0000:0000:0017:0d00:0058:5b02', - 'asnSize': 7250, - 'numMotes': 0, - 'numArrivedPackets': 0, - 'netLatency': 0, - 'netState': 0, - 'netPathStability': 0, - 'downFrameState': 1, - 'maxNumbHops': 0, - 'RC': 0, - 'netReliability': 0 - }, - 'timestamp_stop': 'Wed, 21 Mar 2018 15:23:12 UTC', - 'getMoteConfig': { - '00-17-0d-00-00-58-5b-02': { - 'macAddress': '00-17-0d-00-00-58-5b-02', - 'reserved': 1, - 'state': 4, - 'isRouting': True, - 'RC': 0, - 'moteId': 1, - 'isAP': True} - }, - 'epoch_stop': 1521645792.786726, - 'getSystemInfo': { - 'macAddress': '00-17-0d-00-00-58-5b-02', - 'swBuild': 9, - 'swPatch': 1, 'hwModel': 16, 'swMajor': 1, 'swMinor': 4, - 'RC': 0, 'hwRev': 1 - }, - 'getMoteLinks': { - '00-17-0d-00-00-58-5b-02': { - 'links': [] - } - }, - 'getMoteInfo': { - '00-17-0d-00-00-58-5b-02': { - 'macAddress': '00-17-0d-00-00-58-5b-02', - 'assignedBw': 0, - 'stateTime': 1355, - 'numGoodNbrs': 0, 'numJoins': 1, 'state': 4, - 'packetsReceived': 6, 'hopDepth': 0, - 'totalNeededBw': 55890, 'requestedBw': 55890, 'avgLatency': 0, - 'RC': 0, 'numNbrs': 0, 'packetsLost': 0 - } - }, - 'getPathInfo': { - '00-17-0d-00-00-58-5b-02': { - - } - }, - 'timestamp_start': 'Wed, 21 Mar 2018 15:23:12 UTC', - 'getNetworkConfig': { - 'networkId': 1229, 'apTxPower': 8, 'ccaMode': 0, 'locMode': 0, - 'numParents': 2, 'channelList': 32767, 'baseBandwidth': 9000, - 'maxMotes': 101, 'bbSize': 1, 'bbMode': 0, 'oneChannel': 255, - 'isRadioTest': 0, 'downFrameMultVal': 1, 'RC': 0, - 'bwMult': 300, 'frameProfile': 1, 'autoStartNetwork': True - } - }, - 'name': 'snapshot'}} - -JSON2 = { - 'timestamp': 1521645792, - 'mac': '00-17-0d-00-00-58-5b-02', - 'type': 40, - 'value': { - 'SolManager': [2, 0, 1, 0], - 'Sol': [1, 4, 0, 0], - 'SmartMesh SDK': [1, 1, 2, 4]} -} - -def test_hdlcify(): - input = [126, 127, 129, 13] - output = hdlc.hdlcify(input) - assert output == [126, 125, 94, 127, 129, 13, 55, 140, 126] - -def test_dehdlcify(): - input = [126, 127, 129, 30] - file_name = "test_hdlc.backup" - input_str = "".join(chr(c) for c in hdlc.hdlcify(input)) - with open(file_name, 'w') as f: - f.write(input_str) - - output = hdlc.dehdlcify(file_name, 0) - assert input == output[0][0] - -def test_hdlc(): - file_name = "test_hdlc.backup" - - obj_bin = sol.json_to_bin(JSON2) - h = hdlc.hdlcify(obj_bin) - print(bytes(h)) - s = "".join(chr(c) for c in h) - print('####') - print(s) - with open(file_name, 'w') as f: - f.write(s) - (d, o) = hdlc.dehdlcify(file_name) - assert d[0] == sol.json_to_bin(JSON2) - assert sol.bin_to_json(d[0]) == JSON2 - os.remove(file_name) diff --git a/tests/test_influx_to_json.py b/tests/test_influx_to_json.py deleted file mode 100755 index 775c4cd..0000000 --- a/tests/test_influx_to_json.py +++ /dev/null @@ -1,234 +0,0 @@ -from .context import sol -import pytest -import json - -#============================ defines =============================== - -MACMANAGER = [3,3,3,3,3,3,3,3] -TIMESTAMP = "2016-04-19T01:54:16Z" - -#============================ fixtures ============================== - -SOL_INFLUXJSON_EXAMPLE = [ - # SOL_TYPE_DUST_NOTIF_HRNEIGHBORS (old json_to_influx format) - { - "json": - [{ - "timestamp" : TIMESTAMP, - "mac" : "00-17-0d-00-00-58-32-36", - "type" : "SOL_TYPE_DUST_NOTIF_HRNEIGHBORS", - "value" : { - "numItems" : "1", - "latitude" : "-33.11464", - "longitude" : "-68.48015", - "site" : "ARG_junin", - "neighbors" : { - "1" : - { - 'neighborId': 1, - 'neighborFlag': 0, - 'rssi': -33, - 'numTxPackets': 0, - 'numTxFailures': 0, - 'numRxPackets': 47, - }, - }, - }, - }], - "influxdb_dump" : { - "series" : [{ - "name" : 'SOL_TYPE_DUST_NOTIF_HRNEIGHBORS', - "tags" : { - "mac" : "00-17-0d-00-00-58-32-36", - }, - "columns" : [ - "time", - "1:neighborFlag", - "1:neighborId", - "1:numRxPackets", - "1:numTxFailures", - "1:numTxPackets", - "1:rssi", - "neighbors:1:neighborFlag", - "neighbors:1:neighborId", - "neighbors:1:numRxPackets", - "neighbors:1:numTxFailures", - "neighbors:1:numTxPackets", - "neighbors:1:rssi", - "neighbors:2:neighborFlag", - "neighbors:2:neighborId", - "neighbors:2:numRxPackets", - "neighbors:2:numTxFailures", - "neighbors:2:numTxPackets", - "neighbors:2:rssi", - "numItems", - "latitude", - "longitude", - "site", - "neighbors", - ], - "values" : [[ - TIMESTAMP, # time - 0, # neighborFlag - 1, # neighborId - 47, # numRxPackets - 0, # numTxFailures - 0, # numTxPackets - -33, # rssi - None, # neighborFlag - None, # neighborId - None, # numRxPackets - None, # numTxFailures - None, # numTxPackets - None, # rssi - None, # neighborFlag - None, # neighborId - None, # numRxPackets - None, # numTxFailures - None, # numTxPackets - None, # rssi - "1", # numItems - "-33.11464", # latitude - "-68.48015", # longitude - "ARG_junin", # site - ]] - }], - } - }, - # SOL_TYPE_DUST_NOTIF_HRNEIGHBORS - { - "json": - [{ - "timestamp" : TIMESTAMP, - "mac" : "00-17-0d-00-00-58-32-36", - "type" : "SOL_TYPE_DUST_NOTIF_HRNEIGHBORS", - "value" : { - "numItems" : "2", - "latitude" : "-33.11464", - "longitude" : "-68.48015", - "site" : "ARG_junin", - "neighbors" : { - "1" : - { - 'neighborId': 1, - 'neighborFlag': 0, - 'rssi': -33, - 'numTxPackets': 0, - 'numTxFailures': 0, - 'numRxPackets': 47, - }, - "2" : - { - 'neighborId': 2, - 'neighborFlag': 0, - 'rssi': -47, - 'numTxPackets': 76, - 'numTxFailures': 1, - 'numRxPackets': 2, - }, - }, - }, - }], - "influxdb_dump" : { - "series" : [{ - "name" : 'SOL_TYPE_DUST_NOTIF_HRNEIGHBORS', - "tags" : { - "mac" : "00-17-0d-00-00-58-32-36", - }, - "columns" : [ - "time", - "neighbors:1:neighborFlag", - "neighbors:1:neighborId", - "neighbors:1:numRxPackets", - "neighbors:1:numTxFailures", - "neighbors:1:numTxPackets", - "neighbors:1:rssi", - "neighbors:2:neighborFlag", - "neighbors:2:neighborId", - "neighbors:2:numRxPackets", - "neighbors:2:numTxFailures", - "neighbors:2:numTxPackets", - "neighbors:2:rssi", - "numItems", - "latitude", - "longitude", - "site", - ], - "values" : [[ - TIMESTAMP, # time - 0, # neighborFlag - 1, # neighborId - 47, # numRxPackets - 0, # numTxFailures - 0, # numTxPackets - -33, # rssi - 0, # neighborFlag - 2, # neighborId - 2, # numRxPackets - 1, # numTxFailures - 76, # numTxPackets - -47, # rssi - "2", # numItems - "-33.11464", # latitude - "-68.48015", # longitude - "ARG_junin", # site - ]] - }], - } - }, - # SOL_TYPE_DUST_EVENTMOTECREATE - { - "json": - [{ - "timestamp" : TIMESTAMP, - "mac" : "00-17-0d-00-00-58-32-36", - "type" : "SOL_TYPE_DUST_EVENTMOTECREATE", - "value" : { - "latitude" : "-33.11464", - "longitude" : "-68.48015", - "site" : "ARG_junin", - "macAddress": "00-17-0d-00-00-58-32-36", - "moteId" : 17 - }, - }], - "influxdb_dump" : { - "series" : [{ - "name" : 'SOL_TYPE_DUST_EVENTMOTECREATE', - "tags" : { - "mac" : "00-17-0d-00-00-58-32-36", - }, - "columns" : [ - "time", - "latitude", - "longitude", - "site", - "macAddress", - "moteId" - ], - "values" : [[ - TIMESTAMP, # time - "-33.11464", # lat - "-68.48015", # long - "ARG_junin", # site - "00-17-0d-00-00-58-32-36", # macAddress - 17 # moteId - ]] - }], - } - }, -] - -@pytest.fixture(params=SOL_INFLUXJSON_EXAMPLE) -def sol_influxjson_example(request): - return json.dumps(request.param) - -#============================ tests ================================= - -def test_influx_to_json(sol_influxjson_example): - sol_influxjson_example = json.loads(sol_influxjson_example) - - # influxdb->json - sol_json = sol.influxdb_to_json(sol_influxjson_example["influxdb_dump"]) - print('=====\ninfluxdb->json') - print(sol_influxjson_example["json"]) - assert sol_json==sol_influxjson_example["json"] From 5df4a2f3c8fcd26fb7366a93132d8aab29bac635 Mon Sep 17 00:00:00 2001 From: keoma Date: Thu, 4 Apr 2019 17:54:07 +0200 Subject: [PATCH 3/5] fix bytes to string --- sensorobjectlibrary/Sol.py | 79 ++++++++++++++++++++------------ sensorobjectlibrary/hr_parser.py | 2 +- tests/test_chain.py | 24 +++++----- 3 files changed, 63 insertions(+), 42 deletions(-) diff --git a/sensorobjectlibrary/Sol.py b/sensorobjectlibrary/Sol.py index 4e7a285..ff7081f 100644 --- a/sensorobjectlibrary/Sol.py +++ b/sensorobjectlibrary/Sol.py @@ -23,7 +23,9 @@ # =========================== logging ========================================= -log = logging.getLogger(__name__) +logging.basicConfig(level=logging.DEBUG) +logger = logging.getLogger() +logger.setLevel(logging.DEBUG) # =========================== helpers ========================================= @@ -65,6 +67,7 @@ def dust_to_json(dust_notif, mac_manager=None, timestamp=None): try: sol_type, sol_ts, sol_value = _dust_notif_to_sol_json(d_n) except SolDuplicateOapNotificationException: + logger.debug("SolDuplicateOapNotificationException") continue # get sol_ts @@ -488,7 +491,7 @@ def _dust_oap_to_sol_json(dust_notif): 'state': dust_notif['fields']['new_val'], } else: - log.debug("Unknow format for sol type {0}. dust_notif={1}".format(sol_type, dust_notif)) + logger.debug("Unknow format for sol type {0}. dust_notif={1}".format(sol_type, dust_notif)) else: raise NotImplementedError() return sol_type, sol_value @@ -598,14 +601,19 @@ def _binary_to_fields_snapshot(binary): path_size = struct.calcsize(path_structure) # get number of motes in snapshot - num_motes = struct.unpack('>B', chr(binary[0]))[0] + num_motes = struct.unpack('>B', bytes([binary[0]]))[0] + assert isinstance(num_motes, int) binary = binary[1:] # parse SNAPSHOT for i in range(0, num_motes): # create mote dict mote = {} - m = struct.unpack(mote_structure, ''.join(chr(b) for b in binary[:mote_size])) + if isinstance(binary[:mote_size], list): + bins = bytes(binary[:mote_size]) + else: + bins = bytes([binary[:mote_size]]) + m = struct.unpack(mote_structure, bins) binary = binary[mote_size:] for (k, v) in zip(mote_fields, m): mote[k] = v @@ -614,14 +622,18 @@ def _binary_to_fields_snapshot(binary): mote["macAddress"] = _num_to_list(mote["macAddress"], 8) # get number of paths in mote - num_paths = struct.unpack('>B', chr(binary[0]))[0] + num_paths = struct.unpack('>B', bytes([binary[0]]))[0] binary = binary[1:] # create path dict path_list = [] for j in range(0, num_paths): path = {} - p = struct.unpack('>QBBBbb', ''.join(chr(b) for b in binary[:path_size])) + if isinstance(binary[:path_size], list): + bins = bytes(binary[:path_size]) + else: + bins = bytes([binary[:path_size]]) + p = struct.unpack('>QBBBbb', bins) binary = binary[path_size:] for (k, v) in zip(path_fields, p): path[k] = v @@ -638,10 +650,15 @@ def _binary_to_fields_snapshot(binary): return return_val def _get_sol_binary_value_dust_hr_neighbors(hr): - return_val = [] - return_val += [chr(hr['numItems'])] + """ + + :param hr: + :return: + :rtype: list + """ + return_val = bytes([hr['numItems']]) for n in hr['neighbors']: - return_val += [struct.pack( + return_val += struct.pack( '>HBbHHH', n['neighborId'], # INT16U H n['neighborFlag'], # INT8U B @@ -649,24 +666,28 @@ def _get_sol_binary_value_dust_hr_neighbors(hr): n['numTxPackets'], # INT16U H n['numTxFailures'], # INT16U H n['numRxPackets'], # INT16U H - )] - return_val = ''.join(return_val) - return_val = [ord(c) for c in return_val] + ) + return_val = list(return_val) return return_val def _get_sol_binary_value_dust_hr_discovered(hr): - return_val = [] - return_val += [chr(hr['numJoinParents'])] - return_val += [chr(hr['numItems'])] + """ + + :param hr: + :return: + :rtype: list + """ + return_val = bytes([hr['numJoinParents']]) + return_val += bytes([hr['numItems']]) for n in hr['discoveredNeighbors']: - return_val += [struct.pack( + return_val += struct.pack( '>HbB', n['neighborId'], # INT16U H n['rssi'], # INT8 b n['numRx'], # INT8U B - )] - return_val = bytes(return_val) + ) + return_val = list(return_val) return return_val @@ -679,27 +700,26 @@ def _get_sol_binary_value_dust_hr_extended(hr): HR_ID_EXTENDED_RSSI = 1 HR_ID_EXTENDED_RSSI_STRUCT = ">" + "".join([i[1] for i in HR_DESC_EXTENDED_RSSI_DATA]) HR_ID_EXTENDED_RSSI_SIZE = struct.calcsize(HR_ID_EXTENDED_RSSI_STRUCT) * 15 # 15 channels - return_val = [] + return_val = bytes() if "RSSI" in hr.keys(): - return_val += [struct.pack("B', len(snapshot)) @@ -729,9 +749,9 @@ def _get_sol_binary_value_snapshot(snapshot): # adding paths list size return_val += struct.pack('B', len(mote['paths'])) - p = "" + p = b"" for path in mote['paths']: - if isinstance(path['macAddress'], (str, unicode)): + if isinstance(path['macAddress'], str): path['macAddress'] = [int(c, 16) for c in path['macAddress'].split("-")] p += struct.pack( '>QBBBbb', @@ -743,8 +763,7 @@ def _get_sol_binary_value_snapshot(snapshot): path['rssiDestSrc'], # INT8 b ) return_val += p - return_val = ''.join(return_val) - return_val = [ord(c) for c in return_val] + return_val = list(return_val) return return_val # ==== miscellaneous helpers diff --git a/sensorobjectlibrary/hr_parser.py b/sensorobjectlibrary/hr_parser.py index 9227973..e5ae625 100755 --- a/sensorobjectlibrary/hr_parser.py +++ b/sensorobjectlibrary/hr_parser.py @@ -303,7 +303,7 @@ def _parseAs(desc, payload): raise ValueError("not enough bytes for HR") # separate string to parse from remainder - hrstring = ''.join([chr(b) for b in payload[:numBytes]]) + hrstring = bytes(payload[:numBytes]) remainder = payload[numBytes:] # apply the format string diff --git a/tests/test_chain.py b/tests/test_chain.py index 86b0722..3f191b3 100755 --- a/tests/test_chain.py +++ b/tests/test_chain.py @@ -2,6 +2,7 @@ import pytest import json import pprint +from collections import OrderedDict # =========================== defines =============================== @@ -15,6 +16,10 @@ "longitude" : -44.4444, } +# =========================== helpers =============================== + +pp = pprint.PrettyPrinter(indent=4) + # =========================== fixtures ============================== SOL_CHAIN_EXAMPLE = [ @@ -1535,15 +1540,12 @@ def sol_chain_example(request): return json.dumps(request.param) -# =========================== helpers =============================== - -pp = pprint.PrettyPrinter(indent=4) - # =========================== tests ================================= def test_chain(sol_chain_example): sol_chain_example = json.loads(sol_chain_example) + if "dust" in sol_chain_example: # dust->json sol_jsonl = sol.dust_to_json( @@ -1560,39 +1562,39 @@ def test_chain(sol_chain_example): # same number of objects? (for HR) assert len(sol_jsonl) == len(sol_chain_example["objects"]) - for (sol_json, example) in zip(sol_jsonl,sol_chain_example["objects"]): + for (sol_json, example) in zip(sol_jsonl, sol_chain_example["objects"]): # dust->json print('=====\ndust->json') print(sol_json) print(example["json"]) - assert sol_json==example["json"] + assert sol_json == example["json"] # json->bin sol_bin = sol.json_to_bin(sol_json) print('=====\njson->bin') print(sol_bin) print(example["bin"]) - assert sol_bin==example["bin"] + assert sol_bin == example["bin"] # bin->http sol_http = sol.bin_to_http([sol_bin]) print('=====\nbin->http') print(sol_http) print(example["http"]) - assert json.loads(sol_http)==example["http"] + assert json.loads(sol_http) == example["http"] # http->bin sol_binl = sol.http_to_bin(sol_http) - assert len(sol_binl)==1 + assert len(sol_binl) == 1 sol_bin = sol_binl[0] print('=====\nhttp->bin') print(sol_bin) print(example["bin"]) - assert sol_bin==example["bin"] + assert sol_bin == example["bin"] # bin->json sol_json = sol.bin_to_json(sol_bin) print('=====\nbin->json') print(sol_json) print(example["json"]) - assert sol_json==example["json"] + assert sol_json == example["json"] From 8071a64ee483bfdd5a1c1cca9d2af93db11b58d0 Mon Sep 17 00:00:00 2001 From: keoma Date: Thu, 4 Apr 2019 18:02:29 +0200 Subject: [PATCH 4/5] bump version and travis --- .travis.yml | 2 +- sensorobjectlibrary/__version__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3dd4d85..ac28fed 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: python python: - - "2.7" + - "3.7" before_install: diff --git a/sensorobjectlibrary/__version__.py b/sensorobjectlibrary/__version__.py index 67269f7..aed8084 100644 --- a/sensorobjectlibrary/__version__.py +++ b/sensorobjectlibrary/__version__.py @@ -1 +1 @@ -__version__ = "1.7.5.0" +__version__ = "1.8.0.0" From b4be5bbcc663d9e0b57c4403b66bede85af080ec Mon Sep 17 00:00:00 2001 From: keoma Date: Thu, 4 Apr 2019 18:04:00 +0200 Subject: [PATCH 5/5] travis to 3.5 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ac28fed..008be78 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: python python: - - "3.7" + - "3.5" before_install: