From 826de52bf8e376d0e0803fb2039ffabecd49ac84 Mon Sep 17 00:00:00 2001 From: fdrobnic <35068993+fdrobnic@users.noreply.github.com> Date: Wed, 24 Jan 2024 16:50:38 +0100 Subject: [PATCH 1/9] Update pysmartdatamodels.py Added the ngsi_ld_example_generator_str() function to be able to generate fake examples before the model is published. --- .../pysmartdatamodels/pysmartdatamodels.py | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/pysmartdatamodels/pysmartdatamodels/pysmartdatamodels.py b/pysmartdatamodels/pysmartdatamodels/pysmartdatamodels.py index 387c03c3a6..496365fae4 100644 --- a/pysmartdatamodels/pysmartdatamodels/pysmartdatamodels.py +++ b/pysmartdatamodels/pysmartdatamodels/pysmartdatamodels.py @@ -1120,6 +1120,26 @@ def ngsi_ld_example_generator(schema_url: str): payload = open_jsonref(schema_url) if payload == "": return False + return ngsi_ld_example_generator_str(payload, dataModel, subject) + +def ngsi_ld_example_generator_str(schema: str, dataModel: str, subject: str): + """It returns a fake normalized ngsi-ld format example based on the given json schema + Parameters: + schema: schema.json contents + dataModel: repo name + subject: model name + + Returns: + if the input parameter exists and the json schema is a valide json: + a fake normalized ngsi-ld format example stored in dictionary format + if there's any problem related to input parameter and json schema: + False + """ + + payload = json.loads(schema) + if payload == "": + return False + # print(payload["allOf"]) output = {} fullDict = {} @@ -1159,7 +1179,7 @@ def ngsi_ld_example_generator(schema_url: str): else: output = {**output, **{prop: parsedProperty}} # echo("output", output) - output["@context"] = [create_context(subject)] + output["@context"] = [create_context('dataModel.' + dataModel)] # print("======================") # print(json.dumps(output)) @@ -1504,4 +1524,4 @@ def generate_sql_schema(model_yaml: str) -> str: result = sql_data_types + "\n" + table_create_statement print(result) - return result \ No newline at end of file + return result From 8589c17578b781f6df8f7eb1cc3d9f568b78ca9c Mon Sep 17 00:00:00 2001 From: fdrobnic <35068993+fdrobnic@users.noreply.github.com> Date: Wed, 24 Jan 2024 17:50:45 +0100 Subject: [PATCH 2/9] Update README.md Added the ngsi_ld_example_generator_str() function to be able to generate fake examples before the model is published. --- pysmartdatamodels/README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pysmartdatamodels/README.md b/pysmartdatamodels/README.md index 487573e204..17e0b35a7a 100644 --- a/pysmartdatamodels/README.md +++ b/pysmartdatamodels/README.md @@ -356,6 +356,19 @@ print(sdm.update_broker(dataModel, subject, attribute, value, serverUrl=serverUr if there's any problem related to input parameter and json schema: False +Function ngsi_ld_example_generator_str(schema: str, dataModel: str, subject: str) + + It returns a fake normalized ngsi-ld format example based on the given json schema + Parameters: + schema: schema.json contents + dataModel: repo name + subject: model name + + Returns: + if the input parameter exists and the json schema is a valide json: + a fake normalized ngsi-ld format example stored in dictionary format + if there's any problem related to input parameter and json schema: + False 18- Return a fake key value ngsi-ld format example. Function ngsi_ld_keyvalue_example_generator(schemaUrl) From b229bb3f14cb00d4fb981b4b3f734f952187346f Mon Sep 17 00:00:00 2001 From: fdrobnic <35068993+fdrobnic@users.noreply.github.com> Date: Wed, 24 Jan 2024 19:22:53 +0100 Subject: [PATCH 3/9] Update pysmartdatamodels.py Added the ngsi_ld_example_generator_str() function to be able to generate fake examples before the model is published. --- pysmartdatamodels/pysmartdatamodels/pysmartdatamodels.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pysmartdatamodels/pysmartdatamodels/pysmartdatamodels.py b/pysmartdatamodels/pysmartdatamodels/pysmartdatamodels.py index 496365fae4..a9d3e74b5c 100644 --- a/pysmartdatamodels/pysmartdatamodels/pysmartdatamodels.py +++ b/pysmartdatamodels/pysmartdatamodels/pysmartdatamodels.py @@ -1144,6 +1144,7 @@ def ngsi_ld_example_generator_str(schema: str, dataModel: str, subject: str): output = {} fullDict = {} # echo("payload", payload) + fullDict['id'] = {} if "allOf" in payload: for index in range(len(payload["allOf"])): if "properties" in payload["allOf"][index]: From 8a116d8e20cbb8ec4686b995ca4025e3e064d639 Mon Sep 17 00:00:00 2001 From: fdrobnic <35068993+fdrobnic@users.noreply.github.com> Date: Wed, 8 May 2024 23:44:51 +0200 Subject: [PATCH 4/9] Update pysmartdatamodels.py --- .../pysmartdatamodels/pysmartdatamodels.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/pysmartdatamodels/pysmartdatamodels/pysmartdatamodels.py b/pysmartdatamodels/pysmartdatamodels/pysmartdatamodels.py index a9d3e74b5c..4e6147a08e 100644 --- a/pysmartdatamodels/pysmartdatamodels/pysmartdatamodels.py +++ b/pysmartdatamodels/pysmartdatamodels/pysmartdatamodels.py @@ -1136,7 +1136,21 @@ def ngsi_ld_example_generator_str(schema: str, dataModel: str, subject: str): False """ - payload = json.loads(schema) + if dataModel == "" or subject == "": + return False + + tz = pytz.timezone("Europe/Madrid") + + try: + payload = json.loads(schema) + except ValueError: + output["result"] = False + output["cause"] = "Schema parameter value is not a valid json" + output["time"] = str(datetime.datetime.now(tz=tz)) + # output["parameters"] = {"schema_url: ": schema_url} + print(json.dumps(output)) + sys.exit() + if payload == "": return False From 09b3c2691aa48781aa941dc9e22266ec7bf04118 Mon Sep 17 00:00:00 2001 From: fdrobnic <35068993+fdrobnic@users.noreply.github.com> Date: Thu, 9 May 2024 00:11:17 +0200 Subject: [PATCH 5/9] Update pysmartdatamodels.py --- pysmartdatamodels/pysmartdatamodels/pysmartdatamodels.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysmartdatamodels/pysmartdatamodels/pysmartdatamodels.py b/pysmartdatamodels/pysmartdatamodels/pysmartdatamodels.py index 4e6147a08e..ff86eead56 100644 --- a/pysmartdatamodels/pysmartdatamodels/pysmartdatamodels.py +++ b/pysmartdatamodels/pysmartdatamodels/pysmartdatamodels.py @@ -1139,6 +1139,7 @@ def ngsi_ld_example_generator_str(schema: str, dataModel: str, subject: str): if dataModel == "" or subject == "": return False + output = {} tz = pytz.timezone("Europe/Madrid") try: @@ -1155,7 +1156,6 @@ def ngsi_ld_example_generator_str(schema: str, dataModel: str, subject: str): return False # print(payload["allOf"]) - output = {} fullDict = {} # echo("payload", payload) fullDict['id'] = {} From c1cd47b5076cb8c7cdfe3dd91433d6549e1ec7cd Mon Sep 17 00:00:00 2001 From: fdrobnic <35068993+fdrobnic@users.noreply.github.com> Date: Sat, 14 Jun 2025 00:43:58 +0200 Subject: [PATCH 6/9] Update pysmartdatamodels.py --- .../pysmartdatamodels/pysmartdatamodels.py | 20 ++----------------- 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/pysmartdatamodels/pysmartdatamodels/pysmartdatamodels.py b/pysmartdatamodels/pysmartdatamodels/pysmartdatamodels.py index c6f90b247d..72fe53ea46 100644 --- a/pysmartdatamodels/pysmartdatamodels/pysmartdatamodels.py +++ b/pysmartdatamodels/pysmartdatamodels/pysmartdatamodels.py @@ -842,7 +842,6 @@ def ngsi_ld_example_generator(schema_url: str): payload = open_jsonref(schema_url) if payload == "": return False -<<<<<<< master return ngsi_ld_example_generator_str(payload, dataModel, subject) def ngsi_ld_example_generator_str(schema: str, dataModel: str, subject: str): @@ -862,8 +861,6 @@ def ngsi_ld_example_generator_str(schema: str, dataModel: str, subject: str): if dataModel == "" or subject == "": return False -======= ->>>>>>> master output = {} tz = pytz.timezone("Europe/Madrid") @@ -882,13 +879,9 @@ def ngsi_ld_example_generator_str(schema: str, dataModel: str, subject: str): # print(payload["allOf"]) fullDict = {} -<<<<<<< master - # echo("payload", payload) fullDict['id'] = {} -======= # Parse the "allOf", "anyOf", "oneOf" structure ->>>>>>> master if "allOf" in payload: for index in range(len(payload["allOf"])): if "properties" in payload["allOf"][index]: @@ -921,15 +914,9 @@ def ngsi_ld_example_generator_str(schema: str, dataModel: str, subject: str): output = {**output, **{prop: parsedProperty}} else: output = {**output, **{prop: parsedProperty}} -<<<<<<< master - # echo("output", output) - output["@context"] = [create_context('dataModel.' + dataModel)] - # print("======================") - # print(json.dumps(output)) -======= - output["@context"] = [create_context(subject)] ->>>>>>> master + output["@context"] = [create_context('dataModel.' + dataModel)] + # output["@context"] = [create_context(subject)] return output @@ -1287,8 +1274,6 @@ def generate_sql_schema(model_yaml: str) -> str: print(result) return result -<<<<<<< master -======= # 22 @@ -1678,4 +1663,3 @@ def validate_payload(datamodel, subject, payload): return [False, "datamodel/subject not found"] # result = validate_payload("WeatherObserved", "dataModel.Weather", "") # print(result) ->>>>>>> master From 5d4e1f20bdb2550a19ce32c67a010e2f534504c8 Mon Sep 17 00:00:00 2001 From: fdrobnic <35068993+fdrobnic@users.noreply.github.com> Date: Sat, 14 Jun 2025 00:51:54 +0200 Subject: [PATCH 7/9] Update README.md --- pysmartdatamodels/README.md | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/pysmartdatamodels/README.md b/pysmartdatamodels/README.md index 9f32107db2..431b7a7fb2 100644 --- a/pysmartdatamodels/README.md +++ b/pysmartdatamodels/README.md @@ -200,6 +200,10 @@ print(sdm.validate_dcat_ap_distribution_sdm(content_DCAT)) print("25:") print(sdm.subject_for_datamodel(dataModel)) +# Return a fake normalized ngsi-ld format example based on the given json schema (passed as a string) +print("26 : ") +print(sdm.ngsi_ld_example_generator_str(schema: str, dataModel: str, subject: str)) + ``` ## Functions available include: @@ -439,20 +443,6 @@ print(sdm.subject_for_datamodel(dataModel)) if there's any problem related to input parameter and json schema: False -Function ngsi_ld_example_generator_str(schema: str, dataModel: str, subject: str) - - It returns a fake normalized ngsi-ld format example based on the given json schema - Parameters: - schema: schema.json contents - dataModel: repo name - subject: model name - - Returns: - if the input parameter exists and the json schema is a valide json: - a fake normalized ngsi-ld format example stored in dictionary format - if there's any problem related to input parameter and json schema: - False - 18- Return a fake key value ngsi-ld format example. Function ngsi_ld_keyvalue_example_generator(schemaUrl) It returns a fake key value ngsi-ld format example based on the given json schema @@ -591,6 +581,20 @@ Function ngsi_ld_example_generator_str(schema: str, dataModel: str, subject: str Usually only one element in the array isa returned because there are few clashes in data model names False if no subject is found +26- Return a fake normalized ngsi-ld format example. Function ngsi_ld_example_generator_str(schema: str, dataModel: str, subject: str) + + It returns a fake normalized ngsi-ld format example based on the given json schema + Parameters: + schema: schema.json contents + dataModel: repo name + subject: model name + + Returns: + if the input parameter exists and the json schema is a valide json: + a fake normalized ngsi-ld format example stored in dictionary format + if there's any problem related to input parameter and json schema: + False + ## Pending features (glad to receive contributions to them) A.- Function to allow submission of improvements (i.e. missing recommended units or model) and comments to the different data models. Currently, you can do it searching for your data model here From bcc49eee9a73c25b50eae4357c555b7d2b96e43b Mon Sep 17 00:00:00 2001 From: fdrobnic <35068993+fdrobnic@users.noreply.github.com> Date: Sat, 14 Jun 2025 00:54:05 +0200 Subject: [PATCH 8/9] Update README.md --- pysmartdatamodels/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysmartdatamodels/README.md b/pysmartdatamodels/README.md index 431b7a7fb2..f161348a94 100644 --- a/pysmartdatamodels/README.md +++ b/pysmartdatamodels/README.md @@ -609,6 +609,6 @@ if you want to suggest other functions/ needs please let us know at info@smartda Special thanks to the following contributors: -- [fdrobnic](https://github.com/fdrobnic): Changes for porting to Windows +- [fdrobnic](https://github.com/fdrobnic): Changes for porting to Windows and a function for generating a fake example from a non-published schema - [Antonio Jara](https://twitter.com/Antonio_Jara): New function for inserting data into broker - [María José Bernal](mj.bernal@libelium.com): Necessary extension for function update_broker() to allow updating nonexistent attribute into broker From dd61bfe40cb4066432fe03bc87a639cb382415a9 Mon Sep 17 00:00:00 2001 From: fdrobnic <35068993+fdrobnic@users.noreply.github.com> Date: Sat, 14 Jun 2025 01:03:08 +0200 Subject: [PATCH 9/9] Update pysmartdatamodels.py --- pysmartdatamodels/pysmartdatamodels.py | 116 ++++++++++++++++--------- 1 file changed, 76 insertions(+), 40 deletions(-) diff --git a/pysmartdatamodels/pysmartdatamodels.py b/pysmartdatamodels/pysmartdatamodels.py index 2ce51e97e2..dd02f16297 100644 --- a/pysmartdatamodels/pysmartdatamodels.py +++ b/pysmartdatamodels/pysmartdatamodels.py @@ -843,46 +843,7 @@ def ngsi_ld_example_generator(schema_url: str): payload = open_jsonref(schema_url) if payload == "": return False - output = {} - fullDict = {} - - # Parse the "allOf", "anyOf", "oneOf" structure - if "allOf" in payload: - for index in range(len(payload["allOf"])): - if "properties" in payload["allOf"][index]: - fullDict = {**fullDict, **payload["allOf"][index]["properties"]} - else: - fullDict = {**fullDict, **payload["allOf"][index]} - elif "anyOf" in payload: - for index in range(len(payload["anyOf"])): - if "properties" in payload["anyOf"][index]: - fullDict = {**fullDict, **payload["anyOf"][index]["properties"]} - else: - fullDict = {**fullDict, **payload["anyOf"][index]} - elif "oneOf" in payload: - for index in range(len(payload["oneOf"])): - if "properties" in payload["oneOf"][index]: - fullDict = {**fullDict, **payload["oneOf"][index]["properties"]} - else: - fullDict = {**fullDict, **payload["oneOf"][index]} - else: - fullDict = payload["properties"].copy() - - for prop in fullDict: - - parsedProperty = parse_property2ngsild_example({prop: fullDict[prop]}, dataModel, 0) - - # id and type should be key-value format in ngsild format - if prop in ["id"]: - output = {**output, **parsedProperty} - elif prop in ["type"]: - output = {**output, **{prop: parsedProperty}} - else: - output = {**output, **{prop: parsedProperty}} - - output["@context"] = [create_context(subject)] - - return output + return ngsi_ld_example_generator_str(payload, dataModel, subject) # 18 @@ -1613,3 +1574,78 @@ def subject_for_datamodel(datamodel): else: return subjects + +# 26 +def ngsi_ld_example_generator_str(schema: str, dataModel: str, subject: str): + """It returns a fake normalized ngsi-ld format example based on the given json schema + Parameters: + schema: schema.json contents + dataModel: repo name + subject: model name + Returns: + if the input parameter exists and the json schema is a valide json: + a fake normalized ngsi-ld format example stored in dictionary format + if there's any problem related to input parameter and json schema: + False + """ + + if dataModel == "" or subject == "": + return False + + output = {} + tz = pytz.timezone("Europe/Madrid") + + try: + payload = json.loads(schema) + except ValueError: + output["result"] = False + output["cause"] = "Schema parameter value is not a valid json" + output["time"] = str(datetime.datetime.now(tz=tz)) + # output["parameters"] = {"schema_url: ": schema_url} + print(json.dumps(output)) + sys.exit() + + if payload == "": + return False + + fullDict = {} + fullDict['id'] = {} + + # Parse the "allOf", "anyOf", "oneOf" structure + if "allOf" in payload: + for index in range(len(payload["allOf"])): + if "properties" in payload["allOf"][index]: + fullDict = {**fullDict, **payload["allOf"][index]["properties"]} + else: + fullDict = {**fullDict, **payload["allOf"][index]} + elif "anyOf" in payload: + for index in range(len(payload["anyOf"])): + if "properties" in payload["anyOf"][index]: + fullDict = {**fullDict, **payload["anyOf"][index]["properties"]} + else: + fullDict = {**fullDict, **payload["anyOf"][index]} + elif "oneOf" in payload: + for index in range(len(payload["oneOf"])): + if "properties" in payload["oneOf"][index]: + fullDict = {**fullDict, **payload["oneOf"][index]["properties"]} + else: + fullDict = {**fullDict, **payload["oneOf"][index]} + else: + fullDict = payload["properties"].copy() + + for prop in fullDict: + + parsedProperty = parse_property2ngsild_example({prop: fullDict[prop]}, dataModel, 0) + + # id and type should be key-value format in ngsild format + if prop in ["id"]: + output = {**output, **parsedProperty} + elif prop in ["type"]: + output = {**output, **{prop: parsedProperty}} + else: + output = {**output, **{prop: parsedProperty}} + + output["@context"] = [create_context('dataModel.' + dataModel)] + # output["@context"] = [create_context(subject)] + + return output