diff --git a/.gitignore b/.gitignore index 0dc8398..f8c7889 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ .idea/ +.vscode/ __pycache__/ src/oscsim/modules/__pycache__/ +wosc/ \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 76806d0..f0b5cd2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,21 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.2.0] - 2021-10-19 +This release contains mainly 2 topics: +* **Direct Quantumleap loadtesting** - Starting with this release it is now possible send payload directly to Quantumleap. +* **Changes in Options** - This release has 2 new options. DIRECT-QL ehanges url and test load a bit and lc in number payload add linearily increasing integer value. Both require using insert-always option. + +### Added +- Option DIRECT-QL for protocols. +- Option -lc for number payload. + +### Deleted +- Nothing + +### Changed +- Internal creation of number payload when using protocol NGSI-V2 or DIRECT-QL in conjunction with insert always. + ## [1.1.2] - 2021-07-04 This release contains bugfixes only. diff --git a/README.md b/README.md index 6bca94c..948409e 100644 --- a/README.md +++ b/README.md @@ -32,11 +32,10 @@ optional arguments: This host-name will be prepended by "https://", if protocol is omitted and appended with "/v2/" (NGSI-V2), "/ngsi-ld/v1" (NGSI-LD) or "/v1.1/" (SensorThings) resp. depending on the server-type (see -p/--protocol). - -p {NGSI-V2,NGSI-LD,SensorThings-MQTT,SensorThings-HTTP}, --protocol {NGSI-V2,NGSI-LD,SensorThings-MQTT,SensorThings-HTTP} + -p {NGSI-V2,NGSI-LD,SensorThings-MQTT,SensorThings-HTTP,DIRECT-QL}, --protocol {NGSI-V2,NGSI-LD,SensorThings-MQTT,SensorThings-HTTP,DIRECT-QL} Define the type of server. [Default: NGSI-V2] - -i, --insert-always [Only NGSI-V2 and NGSI-LD!] If set, the contexts will always be inserted (via POST with - option 'upsert') instead of trying to update first (via PATCH) and insert (via POST), if - not existing (i.e. PATCH returns '404 Not Found'). + -i, --insert-always [Only NGSI-V2, DIRECT-QL(mandatory) and NGSI-LD!] If set, the contexts will + always be inserted (via POST with option 'upsert') instead of trying to update first (via PATCH) and insert (via POST), if not existing (i.e. PATCH returns '404 Not Found'). -a id, --datastream-id id [Only SensorThings!] If set, this Datastream-Id will be used for ALL Observations, instead of first searching for the Thing by it's name and the correct Datastream-Id @@ -62,13 +61,10 @@ optional arguments: If set, limits the frequency of the messages sent to the given number (per thread!). -l seconds, --limit-time seconds Only in conjunction with '-u/--unlimited': Stops after the given time in seconds. - -y name, --type name [Only NGSI-V2 and NGSI-LD!] If set, this type-name will be used in the payload. + -y name, --type name [Only NGSI-V2, DIRECT-QL(mandatory) and NGSI-LD!] If set, this type-name will be + used in the payload. -an name,type,number[,max-number], --attribute-number name,type,number[,max-number] - Define a number attribute used for the payload by 'name' (The name of the attribute, - e.g.: temperature), 'type' (One of i [integer] or f [floating point])and 'number' (The - value to be used). If 'max-number' is set, the number written will be randomly between - 'number' and 'max-number' (each including). Note: Multiple number attributes can be - defined by repeating -an. + Define a number attribute used for the payload by 'name' (The name of the attribute, e.g.: temperature), 'type' (One of i [integer] or lc [linear counter] or f [floating point])and 'number' (The value to be used). If 'max-number' is set, the number written will be randomly between 'number' and 'max-number' (each including). Note: Multiple number attributes can be defined by repeating -an. -as name value, --attribute-string name value Define a string attribute used for the payload by 'name' (The name of the attribute, e.g.: instruction) and 'value' (the actual string). Note: Multiple string attributes can diff --git a/src/oscsim/modules/arguments.py b/src/oscsim/modules/arguments.py index f4e2297..ec116a7 100644 --- a/src/oscsim/modules/arguments.py +++ b/src/oscsim/modules/arguments.py @@ -116,6 +116,7 @@ def _fill_text(self, text, width, indent): helper.PROTOCOL_NGSI_LD, helper.PROTOCOL_SENSOR_THINGS_MQTT, helper.PROTOCOL_SENSOR_THINGS_HTTP, + helper.PROTOCOL_DIRECT_QL, ], default=helper.PROTOCOL_NGSI_V2, dest="protocol", @@ -128,7 +129,7 @@ def _fill_text(self, text, width, indent): dest="insert_always", action="store_true", default=False, - help="[Only NGSI-V2 and NGSI-LD!] If set, the contexts will always be inserted " + help="[Only NGSI-V2, DIRECT-QL(mandatory) and NGSI-LD!] If set, the contexts will always be inserted " "(via POST with option 'upsert') instead " "of trying to update first (via PATCH) and insert (via POST), if not " "existing (i.e. PATCH returns '404 Not Found').", @@ -251,7 +252,7 @@ def _fill_text(self, text, width, indent): "--type", metavar="name", dest="type", - help="[Only NGSI-V2 and NGSI-LD!] If set, this type-name will be used in the payload.", + help="[Only NGSI-V2, DIRECT-QL(mandatory) and NGSI-LD!] If set, this type-name will be used in the payload.", ) parser.add_argument( @@ -264,7 +265,7 @@ def _fill_text(self, text, width, indent): help="Define a number attribute used for the payload by 'name' (The name " "of the " "attribute, e.g.: temperature), 'type' (One of i [integer] or " - "f [floating point])" + "lc [linear counter] or f [floating point])" "and 'number' (The value to be used). If 'max-number' is set, the " "number written will be" " randomly between 'number' and 'max-number' (each including). " @@ -436,7 +437,8 @@ def check_arguments(parser, args): "-an argument ['-an %s'] expects 3 or 4 comma-delimited " "parameters!" % ",".join(attribute_args) ) - if attribute_args[1] == "i": + if (attribute_args[1] == "i" + or attribute_args[1] == "lc"): t = 0 f = 0 try: @@ -486,7 +488,7 @@ def check_arguments(parser, args): ) else: parser.error( - 'Please check attribute "%s": type must be one of [i | f]!' + 'Please check attribute "%s": type must be one of [i | lc | f]!' % attribute_args[0] ) @@ -591,15 +593,17 @@ def check_arguments(parser, args): if args.insert_always and ( args.protocol != helper.PROTOCOL_NGSI_V2 and args.protocol != helper.PROTOCOL_NGSI_LD + and args.protocol != helper.PROTOCOL_DIRECT_QL ): parser.error( "Insert always scheme [-i/--insert-always] is only valid " - "for NGSI-V2 and NGSI-LD!" + "for NGSI-V2, DIRECT-QL and NGSI-LD!" ) # is there any payload? if ( args.protocol == helper.PROTOCOL_NGSI_V2 + or args.protocol == helper.PROTOCOL_DIRECT_QL or args.protocol != helper.PROTOCOL_NGSI_LD ): if ( diff --git a/src/oscsim/modules/dataload.py b/src/oscsim/modules/dataload.py new file mode 100644 index 0000000..41abf81 --- /dev/null +++ b/src/oscsim/modules/dataload.py @@ -0,0 +1,103 @@ +from abc import ABC, abstractmethod +from threading import Lock +import random + +class DataItem(ABC): + """ + Abstract representation of feedable datarow. + """ + + itemname: str + itemtype: str + + def getnameforitem(self) -> str: + return self.itemname + + @abstractmethod + def getdictionaryforitem(self) -> dict: + """ + Build dictionary for data item consisting of Name, Type and Value. + + :return: Dict. + """ + pass + + +class BaseNumberItem(DataItem): + minvalue = None + maxvalue = None + + def __init__(self, data:[str]): + self.itemname = data[0] + self.itemtype = "Number" + if data[1] == "i" or data[1] == "lc": + f = int(data[2]) + self.minvalue = f + if len(data) > 3: + self.maxvalue = int(data[3]) + + @abstractmethod + def getdictionaryforitem(self) -> dict: + """ + Build dictionary for data item consisting of Name, Type and Value. + + :return: Dict. + """ + pass + + +class LCIntegerNumberItem(BaseNumberItem): + + itemvalue = 0 + lock = None + + def __init__(self, data:[str], lock: Lock = None): + super(LCIntegerNumberItem, self).__init__(data) + self.lock = lock + with lock: + self.itemvalue = self.minvalue + if self.maxvalue is None: + self.maxvalue = 1000000000 # Unlimited magic number + + def getdictionaryforitem(self) -> dict: + attr = {} + attr["type"] = self.itemtype + with self.lock: + attr["value"] = self.itemvalue + if self.itemvalue < self.maxvalue: + self.itemvalue += 1 + return attr + + +class RandomIntegerNumberItem(BaseNumberItem): + + def __init__(self, data:[str]): + super(RandomIntegerNumberItem, self).__init__(data) + + def getdictionaryforitem(self) -> dict: + value = self.minvalue + if self.maxvalue is not None: + value = random.randint(self.minvalue, self.maxvalue) + + itemdict = dict() + itemdict["type"] = self.itemtype + itemdict["value"] = value + return itemdict + + +class RandomFloatNumberItem(BaseNumberItem): + + def __init__(self, data:[str]): + super(RandomFloatNumberItem, self).__init__(data) + self.minvalue = float(data[2]) + if len(data) > 3: + self.maxvalue = float(data[3]) + + def getdictionaryforitem(self) -> dict: + value = self.minvalue + if self.maxvalue is not None: + value = round(random.uniform(self.minvalue, self.maxvalue), 1) + itemdict = dict() + itemdict["type"] = self.itemtype + itemdict["value"] = value + return itemdict diff --git a/src/oscsim/modules/helper.py b/src/oscsim/modules/helper.py index 3062fcc..b232545 100644 --- a/src/oscsim/modules/helper.py +++ b/src/oscsim/modules/helper.py @@ -3,9 +3,10 @@ import json import random from datetime import datetime +from . import dataload # some "consts" -VERSION = "1.1.2" +VERSION = "1.2.0" CONTENT_TYPE = "Content-Type" APPLICATION_JSON = "application/json" # TODO: swagger says, content-type is "application/json;application/ld+json"! @@ -15,7 +16,7 @@ PROTOCOL_NGSI_LD = "NGSI-LD" PROTOCOL_SENSOR_THINGS_HTTP = "SensorThings-HTTP" PROTOCOL_SENSOR_THINGS_MQTT = "SensorThings-MQTT" - +PROTOCOL_DIRECT_QL = "DIRECT-QL" def get_version(): return VERSION @@ -143,7 +144,7 @@ def create_observation_payload(value, indent): return json.dumps(payload) -def create_payload_ngsi_v2(first_id, meta_data, args): +def create_payload_ngsi_v2(first_id, meta_data, args, number_payload = None): payload = dict() if meta_data: if first_id is not None: @@ -165,20 +166,9 @@ def create_payload_ngsi_v2(first_id, meta_data, args): } payload[date_time[0]] = attr - if args.numbers is not None: - for number in args.numbers: - attribute_args = number[0].split(",") - - attr = {} - - # type - if attribute_args[1] == "i" or attribute_args[1] == "f": - attr["type"] = "Number" - - # value - value = create_value_from_attribute_args(attribute_args) - attr["value"] = value - payload[attribute_args[0]] = attr + if number_payload is not None: + for number in number_payload: + payload[number.getnameforitem()] = number.getdictionaryforitem() if args.strings is not None: for string in args.strings: @@ -221,6 +211,11 @@ def create_payload_ngsi_v2(first_id, meta_data, args): attr["value"] = coord payload[attribute_args[0]] = attr + if args.protocol == PROTOCOL_DIRECT_QL: + data = [payload] + payload = dict() + payload["data"] = data + if args.indent > 0: return json.dumps(payload, indent=args.indent) else: diff --git a/src/oscsim/modules/ngsi.py b/src/oscsim/modules/ngsi.py index 780bfaa..c070564 100644 --- a/src/oscsim/modules/ngsi.py +++ b/src/oscsim/modules/ngsi.py @@ -5,6 +5,7 @@ from . import helper # some "consts" +QL_NOTIFY = "/v2/notify" V2_ENTITIES = "/v2/entities/" LD_ENTITIES = "/ngsi-ld/v1/entities/" OPTIONS_UPSERT = "?options=upsert" @@ -31,12 +32,15 @@ def do_delete(session, host, ngsi_id, headers, args): return None -def do_post(session, host, first_id, headers, upsert, args): +def do_post(session, host, first_id, headers, upsert, args, number_payload = None): if args.protocol == helper.PROTOCOL_NGSI_V2: url = host + V2_ENTITIES if upsert: url += OPTIONS_UPSERT - payload = helper.create_payload_ngsi_v2(first_id, True, args) + payload = helper.create_payload_ngsi_v2(first_id, True, args, number_payload) + elif args.protocol == helper.PROTOCOL_DIRECT_QL: + url = host + QL_NOTIFY + payload = helper.create_payload_ngsi_v2(first_id, True, args, number_payload) else: url = host + LD_ENTITIES if upsert: diff --git a/src/oscsim/modules/output.py b/src/oscsim/modules/output.py index 28f6e94..7e49879 100644 --- a/src/oscsim/modules/output.py +++ b/src/oscsim/modules/output.py @@ -74,7 +74,8 @@ def print_type_of_server(protocol): def print_schema(args): - if args.protocol == helper.PROTOCOL_NGSI_V2: + if (args.protocol == helper.PROTOCOL_NGSI_V2 + or args.protocol == helper.PROTOCOL_DIRECT_QL): if args.insert_always: print( "Note: POST-always schema is used to store contexts. " @@ -158,18 +159,19 @@ def print_id_used(args, msg_num): ) -def print_payload(args): - if args.protocol == helper.PROTOCOL_NGSI_V2: +def print_payload(args, number_payload): + if (args.protocol == helper.PROTOCOL_NGSI_V2 + or args.protocol == helper.PROTOCOL_DIRECT_QL): if args.insert_always: print( "The payload will look like:\n%s" - % helper.create_payload_ngsi_v2(args.first_id, True, args), + % helper.create_payload_ngsi_v2(args.first_id, True, args, number_payload), flush=True, ) else: print( "The payload will look like:\n%s" - % helper.create_payload_ngsi_v2(None, False, args), + % helper.create_payload_ngsi_v2(None, False, args, number_payload), flush=True, ) elif args.protocol == helper.PROTOCOL_NGSI_LD: diff --git a/src/oscsim/run.py b/src/oscsim/run.py index 12612d2..bc196bc 100755 --- a/src/oscsim/run.py +++ b/src/oscsim/run.py @@ -9,7 +9,7 @@ import requests -from .modules import arguments, helper, ngsi, output, sensor_things +from .modules import arguments, helper, ngsi, output, sensor_things, dataload # some globals start = datetime.now() @@ -22,7 +22,7 @@ send_threads = [] delete_thread = Thread() halt = False - +number_payload = [] # Array items are DataItem type def do_delete(session, lock, args, max_id_length): global errors, deleted, not_deleted, overall_messages @@ -128,7 +128,7 @@ def do_delete(session, lock, args, max_id_length): def do_send(mqtt_client, session, lock, args, offset, max_id_length): - global errors, unique_errors, overall_time, overall_messages + global errors, unique_errors, overall_time, overall_messages, number_payload first_id = args.first_id + offset host = helper.create_host_url(args.server) @@ -155,8 +155,12 @@ def do_send(mqtt_client, session, lock, args, offset, max_id_length): if ( args.protocol == helper.PROTOCOL_NGSI_V2 or args.protocol == helper.PROTOCOL_NGSI_LD + or args.protocol == helper.PROTOCOL_DIRECT_QL ): - if args.protocol == helper.PROTOCOL_NGSI_V2: + if ( + args.protocol == helper.PROTOCOL_NGSI_V2 + or args.protocol == helper.PROTOCOL_DIRECT_QL + ): headers = {helper.CONTENT_TYPE: helper.APPLICATION_JSON} else: headers = {helper.CONTENT_TYPE: helper.APPLICATION_JSON_LD} @@ -166,14 +170,18 @@ def do_send(mqtt_client, session, lock, args, offset, max_id_length): if args.insert_always: resp, payload = ngsi.do_post( - session, host, first_id, headers, True, args + session, host, first_id, headers, True, args, number_payload ) if resp is None: ms = 0 okay = False else: ms = int(resp.elapsed.total_seconds() * 1000) - okay = resp.status_code == 204 or resp.status_code == 201 + okay = ( + resp.status_code == 204 + or resp.status_code == 201 + or resp.status_code == 200 + ) else: resp, payload = ngsi.do_patch(session, host, first_id, headers, args) if resp is None: @@ -568,6 +576,20 @@ def handle_delete(args, session, lock, max_id_length): print("\nReady", flush=True) +def create_number_loads(args, lock): + if args.numbers is not None: + for number in args.numbers: + numberdata = None + attribute_args = number[0].split(",") + if attribute_args[1] == "i": + numberdata = dataload.RandomIntegerNumberItem(attribute_args) + elif attribute_args[1] == "lc": + numberdata = dataload.LCIntegerNumberItem(attribute_args, lock) + else: + numberdata = dataload.RandomFloatNumberItem(attribute_args) + + number_payload.append(numberdata) + def create_send_threads(args, mqtt_client, session, lock, max_id_length): print("Starting %i thread(s)" % args.num_threads, end="", flush=True) @@ -646,11 +668,14 @@ def stop_send_threads(): def handle_send(args, session, lock, msg_num, max_id_length): - global start, send_threads + global start, send_threads, number_payload # noinspection PyTypeChecker signal.signal(signal.SIGINT, signal_handler) + # Needs to create some loads so that they are shown correctly + create_number_loads(args, lock) + # print what will be done... output.print_server_used(False, args.server) output.print_type_of_server(args.protocol) @@ -660,7 +685,7 @@ def handle_send(args, session, lock, msg_num, max_id_length): else: output.print_data_stream_id_used(args.datastream_id) if args.dry_run: - output.print_payload(args) + output.print_payload(args, number_payload) output.print_will_send_messages(args, msg_num) if args.frequency is not None: output.print_frequency(args.frequency, args.num_threads > 1)