diff --git a/.circleci/config.yml b/.circleci/config.yml index 11fa34f..caa9bcf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -88,4 +88,4 @@ workflows: - publish: requires: - build - - test + - test \ No newline at end of file diff --git a/.idea/hydra.xml b/.idea/hydra.xml deleted file mode 100644 index 5f791e0..0000000 --- a/.idea/hydra.xml +++ /dev/null @@ -1,9 +0,0 @@ - - - - - \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 9a37cdd..2fbf24d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,8 +17,10 @@ COPY . /harness-cli ENV PATH=/harness-cli/harness-cli/:$PATH RUN apt update && \ - apt-get install -y tzdata && \ - dpkg-reconfigure --frontend noninteractive tzdata + apt-get install -y tzdata curl unzip build-essential groff less nano jq vim && \ + dpkg-reconfigure --frontend noninteractive tzdata && \ + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ + unzip awscliv2.zip && ./aws/install RUN apt install -y python3 python3-pip curl openjdk-8-jdk && \ pip3 install pytz && \ @@ -30,4 +32,4 @@ RUN cd /harness-cli/diff-tool/ && make build && \ apt autoremove && \ rm -rf /harness-cli/python-sdk && \ rm -rf /var/apt/cache/* -CMD ["tail", "-f", "/dev/null"] \ No newline at end of file +CMD ["tail", "-f", "/dev/null"] diff --git a/README.md b/README.md index 9f9a3a9..630c70e 100644 --- a/README.md +++ b/README.md @@ -140,6 +140,10 @@ The "status" gives an "OK" if connections can be made, the rest of the info is c The Harness-CLI follows the same version numbers as the Harness Server. If you build from source there will be a git tag in the master branch for every release after 0.4.0-RC1. For containers the image tags also follow Harness naming. +## harness-cli version 1.0.0-SNAPSHOT + +Synced with harness-1.0.0-SNAPSHOT. No planned changes from 0.6.0. This version is in the "develop" branch of this repo. + ## harness-cli version 0.6.0 This is in sync with Harness 0.6.0 but should be backward compatible with all 0.5.x harness versions. diff --git a/examples/ur/expected-ur-results.txt b/examples/ur/expected-ur-results.txt index 4a769ef..79b2167 100644 --- a/examples/ur/expected-ur-results.txt +++ b/examples/ur/expected-ur-results.txt @@ -1,4 +1,3 @@ -Queries to: http://localhost:9090 +++++++++++++ User-based +++++++++++++ diff --git a/examples/ur/sample-mobile-device-ur-data-long-history.csv b/examples/ur/sample-mobile-device-ur-data-long-history.csv new file mode 100644 index 0000000..0bb072c --- /dev/null +++ b/examples/ur/sample-mobile-device-ur-data-long-history.csv @@ -0,0 +1,259 @@ +u1,purchase,iPhone XS +u1,purchase,iPhone XR +u1,purchase,iPhone 8 +u1,purchase,iPad Pro +U 2,purchase,Pixel Slate +U 2,purchase,Galaxy 8 +u-3,purchase,Surface Pro +u-4,purchase,iPhone XR +u-4,purchase,iPhone XR +u-4,purchase,iPhone XR +u-4,purchase,iPhone XR +u-4,purchase,iPhone 8 +u-4,purchase,Galaxy 8 +u1,view,Phones +u1,view,Phones +u1,view,Phones +u1,view,Phones +u1,view,Phones +u1,view,Phones +u1,view,Mobile-acc +U 2,view,Phones +U 2,view,Tablets +U 2,view,Mobile-acc +u-3,view,Mobile-acc +u-4,view,Phones +u-4,view,Tablets +u-4,view,Soap +u5,view,Soap +u1,category-pref,phones +u1,category-pref,phones +u1,category-pref,phones +u1,category-pref,tablets +U 2,category-pref,tablets +U 2,category-pref,phones +u-3,category-pref,tablets +u-4,category-pref,phones +u-4,category-pref,phones +u5,category-pref,toiletries +iPhone XS,$set,categories:Phones:Electronics:Apple +iPhone XR,$set,categories:Phones:Electronics:Apple +iPhone 8,$set,categories:Phones:Electronics:Apple +iPad Pro,$set,categories:Tablets:Electronics:Apple +Pixel Slate,$set,categories:Tablets:Electronics:Google +Galaxy 8,$set,categories:Phones:Electronics:Samsung +Surface Pro,$set,categories:Tablets:Electronics:Microsoft +iPhone 8,$set,countries:United States:Canada:Estados Unidos Mexicanos +iPad Pro,$set,countries:United States:Estados Unidos Mexicanos +Pixel Slate,$set,countries:United States:Canada +Galaxy 8,$set,countries:United States +Surface Pro,$set,countries:United States:Canada +iPhone XS,$set,categories:Cameras +iPhone XR,$set,categories:Cameras +iPhone 8,$set,categories:Cameras +iPad Pro,$set,categories:Computers +Pixel Slate,$set,categories:Computers +Galaxy 8,$set,categories:Cameras +Surface Pro,$set,categories:Computers +iPhone 8,$set,countries:Cuba +iPad Pro,$set,countries:Cuba +Pixel Slate,$set,countries:Cuba +Galaxy 8,$set,countries:Cuba +Surface Pro,$set,countries:Cuba +u1,purchase,iPhone XS +u1,purchase,iPhone XR +u1,purchase,iPhone 8 +u1,purchase,iPad Pro +U 2,purchase,Pixel Slate +U 2,purchase,Galaxy 8 +u-3,purchase,Surface Pro +u-4,purchase,iPhone XR +u-4,purchase,iPhone 8 +u-4,purchase,Galaxy 8 +u1,view,Phones +u1,view,Mobile-acc +U 2,view,Phones +U 2,view,Tablets +U 2,view,Mobile-acc +u-3,view,Mobile-acc +u-4,view,Phones +u-4,view,Tablets +u-4,view,Soap +u5,view,Soap +u1,purchase,Galaxy 8 +u1,purchase,Galaxy 8 +u1,purchase,Galaxy 8 +u1,purchase,Galaxy 8 +u1,view,Galaxy 8 +U 2,purchase,iPhone XR +U 2,purchase,iPhone XR +u1,view,Soap +u1,view,Soap +u-3,view,Mobile-acc +u-4,view,Mobile-acc +u-4,view,Mobile-acc +iPhone XS,$set,categories:Phones:Electronics:Apple +iPhone XR,$set,categories:Phones:Electronics:Apple +iPhone 8,$set,categories:Phones:Electronics:Apple +iPad Pro,$set,categories:Tablets:Electronics:Apple +Pixel Slate,$set,categories:Tablets:Electronics:Google +Galaxy 8,$set,categories:Phones:Electronics:Samsung +Surface Pro,$set,categories:Tablets:Electronics:Microsoft +iPhone 8,$set,countries:United States:Canada:Estados Unidos Mexicanos +iPad Pro,$set,countries:United States:Estados Unidos Mexicanos +Pixel Slate,$set,countries:United States:Canada +Galaxy 8,$set,countries:United States +Surface Pro,$set,countries:United States:Canada +u1,purchase,iPhone Case +u1,purchase, AirPods +U 2,purchase,USB-C +u-3,purchase,Sleeve +u-4,purchase,USB-C Ear Buds +u-4,purchase,USB-C +u1,view,Orders +u1,view,Past Orders +U 2,view,Orders +U 2,view,Accessories +u-3,view,Accessories +u-4,view,Orders +u-4,view,Phones +u-4,view,Clothing +u5,view,Clothing +u1,category-pref,Toys +u1,category-pref,Videos +u1,category-pref,Electronics +U 2,category-pref,Toys +U 2,category-pref,Electronics +u-3,category-pref,Videos +u-4,category-pref,Videos +u-4,category-pref,Games +u5,category-pref,Games +u1,purchase,item 1 +u1,purchase,item 2 +u1,purchase,item 3 +u1,purchase,item 4 +u1,purchase,item 5 +u1,purchase,item 6 +u1,purchase,item 7 +u1,purchase,item 8 +u1,purchase,item 9 +u1,purchase,item 10 +u1,purchase,item 11 +u1,purchase,item 12 +u1,purchase,item 13 +u1,purchase,item 14 +u1,purchase,item 15 +u1,purchase,item 16 +u1,purchase,item 17 +u1,purchase,item 18 +u1,purchase,item 19 +u1,purchase,item 20 +u1,purchase,item 21 +u1,purchase,item 22 +u1,purchase,item 23 +u1,purchase,item 24 +u1,purchase,item 25 +u1,purchase,item 26 +u1,purchase,item 27 +u1,purchase,item 28 +u1,purchase,item 29 +u1,purchase,item 30 +u1,purchase,item 31 +u1,purchase,item 32 +u1,purchase,item 33 +u1,purchase,item 34 +u1,purchase,item 35 +u1,purchase,item 36 +u1,purchase,item 37 +u1,purchase,item 38 +u1,purchase,item 39 +u1,purchase,item 40 +u1,purchase,item 41 +u1,purchase,item 42 +u1,purchase,item 43 +u1,purchase,item 44 +u1,purchase,item 45 +u1,purchase,item 46 +u1,purchase,item 47 +u1,purchase,item 48 +u1,purchase,item 49 +u1,purchase,item 50 +u1,purchase,item 51 +u1,purchase,item 52 +u1,purchase,item 53 +u1,purchase,item 54 +u1,purchase,item 55 +u1,purchase,item 56 +u1,purchase,item 57 +u1,purchase,item 58 +u1,purchase,item 59 +u1,purchase,item 60 +u1,purchase,item 61 +u1,purchase,item 62 +u1,purchase,item 63 +u1,purchase,item 64 +u1,purchase,item 65 +u1,purchase,item 66 +u1,purchase,item 67 +u1,purchase,item 68 +u1,purchase,item 69 +u1,purchase,item 70 +u1,purchase,item 71 +u1,purchase,item 72 +u1,purchase,item 73 +u1,purchase,item 74 +u1,purchase,item 75 +u1,purchase,item 76 +u1,purchase,item 77 +u1,purchase,item 78 +u1,purchase,item 79 +u1,purchase,item 80 +u1,purchase,item 81 +u1,purchase,item 82 +u1,purchase,item 83 +u1,purchase,item 84 +u1,purchase,item 85 +u1,purchase,item 86 +u1,purchase,item 87 +u1,purchase,item 88 +u1,purchase,item 89 +u1,purchase,item 90 +u1,purchase,item 91 +u1,purchase,item 92 +u1,purchase,item 93 +u1,purchase,item 94 +u1,purchase,item 95 +u1,purchase,item 96 +u1,purchase,item 97 +u1,purchase,item 98 +u1,purchase,item 99 +u1,purchase,item 100 +u1,purchase,item 101 +u1,purchase,item 102 +u1,purchase,item 103 +u1,purchase,item 104 +u1,purchase,item 105 +u1,purchase,item 106 +u1,purchase,item 107 +u1,purchase,item 108 +u1,purchase,item 109 +u1,purchase,item 110 +u1,purchase,item 111 +u1,purchase,item 112 +u1,purchase,item 113 +u1,purchase,item 114 +u1,purchase,item 115 +u1,purchase,item 116 +u1,purchase,item 117 +u1,purchase,item 118 +u1,purchase,item 119 +u1,purchase,item 120 +u1,purchase,item 121 +u1,purchase,item 122 +u1,purchase,item 123 +u1,purchase,item 124 +u1,purchase,item 125 +u1,purchase,item 126 +u1,purchase,item 127 +u1,purchase,item 128 +u1,purchase,item 129 diff --git a/examples/ur/simple-integration-test.sh b/examples/ur/simple-integration-test.sh index c913362..c17f837 100755 --- a/examples/ur/simple-integration-test.sh +++ b/examples/ur/simple-integration-test.sh @@ -121,6 +121,10 @@ else diff ${actual_query_results} ${expected_test_results} > ${diffs_and_errors_file} cat ${actual_query_results} | grep "error" >> ${diffs_and_errors_file_property_changes} + echo "Running Phase 3: User data" + python3 examples/ur/user-data.py --url ${host_url} --engineId test_ur --entityId u1 --compare-with ${user_events} >> ${diffs_and_errors_file_property_changes} + echo + if [ -s ${diffs_and_errors_file_property_changes} ] then echo "Input, train, query tests pass but realtime model updates test fails" diff --git a/examples/ur/test-ur-mobile-device-queries.sh b/examples/ur/test-ur-mobile-device-queries.sh index b827006..4ef3405 100755 --- a/examples/ur/test-ur-mobile-device-queries.sh +++ b/examples/ur/test-ur-mobile-device-queries.sh @@ -6,7 +6,7 @@ else export host_url=$1 fi -echo "Queries to: "${host_url} +#echo "Queries to: "${host_url} echo echo "+++++++++++++ User-based +++++++++++++" diff --git a/examples/ur/user-data.py b/examples/ur/user-data.py new file mode 100644 index 0000000..6fecedd --- /dev/null +++ b/examples/ur/user-data.py @@ -0,0 +1,58 @@ +import urllib.request +import json +import argparse +import csv +import time + + +def get_entity_data(url, engineId, entityId, offset, limit): + with urllib.request.urlopen(f"{url}/engines/{engineId}/entities/{entityId}?from={offset}&num={limit}") as b: + return json.loads(b.read().decode('utf-8')) + +def parseCsv(file): + with open(file) as csvfile: + reader = csv.DictReader(csvfile, fieldnames=['entityId', 'event', 'targetEntityId']) + result = [] + for row in reader: + if row['event'] != "$set": + for j in row.items(): + result.append({'entityId': j[1]}) + break + return result + +def filterByEntityId(entities, id): + result = [] + for i in entities: + if i['entityId'] == id: + result.append(i) + return result + +def compareEntities(actual, expected): + return len(actual) == len(expected) + +def deleteEntities(url, engineId, entityId): + req = urllib.request.Request(f"{url}/engines/{engineId}/entities/{entityId}", method="DELETE") + with urllib.request.urlopen(req) as b: + return + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--url', default="http://localhost:9090") + parser.add_argument('--engineId') + parser.add_argument('--entityId') + parser.add_argument('--offset', default=0) + parser.add_argument('--limit', default=1000) + parser.add_argument('--compare-with') + + args = parser.parse_args() + getPassed = compareEntities(get_entity_data(args.url, args.engineId, args.entityId, args.offset, args.limit), + filterByEntityId(parseCsv(args.compare_with), args.entityId)) + deleteEntities(args.url, args.engineId, args.entityId) + time.sleep(2) + deletePassed = not get_entity_data(args.url, args.engineId, args.entityId, args.offset, args.limit) + + if not getPassed: + print("Get user data test fails") + if not deletePassed: + print("Delete user data test fails") diff --git a/harness-cli/harness-cli-env b/harness-cli/harness-cli-env index a80f36a..9d17f31 100644 --- a/harness-cli/harness-cli-env +++ b/harness-cli/harness-cli-env @@ -1,7 +1,6 @@ # harness cli environment file (sourced by scripts) # Harness CLI config, should work as-is unless you are using SSL or connecting to a remote Harness Server - export HARNESS_SERVER_ADDRESS=${HARNESS_SERVER_ADDRESS:-localhost} export HARNESS_SERVER_PORT=${HARNESS_SERVER_PORT:-9090} diff --git a/harness-cli/harness-help b/harness-cli/harness-help index ba45f2d..fee635d 100755 --- a/harness-cli/harness-help +++ b/harness-cli/harness-help @@ -6,58 +6,54 @@ echo echo -e "${CYAN}Harness CLI Help${NC}" echo -echo -e " ${CYAN}harness-cli add ${NC}" +echo -e " ${CYAN}hctl add ${NC}" echo -e " Create a new engine instance and set it's configuration" echo "" -echo -e " ${CYAN}harness-cli update ${NC}" +echo -e " ${CYAN}hctl update ${NC}" echo -e " Resets the Engine instance parameters." echo "" -echo -e " ${CYAN}harness-cli import ${NC}" +echo -e " ${CYAN}hctl import ${NC}" echo -e " Imports events into the engine specified. The path can be to a directory or a single file." echo "" -echo -e " ${CYAN}harness-cli delete ${NC}" +echo -e " ${CYAN}hctl delete ${NC}" echo -e " Deletes engine and all data" echo "" -echo -e " ${CYAN}harness-cli train ${NC}" +echo -e " ${CYAN}hctl train ${NC}" echo -e " For batch trained Engines this creates a model from previously accumulated data." echo "" -echo -e " ${CYAN}harness-cli cancel ${NC}" echo -e " Cancels a train job. Import jobs cannot be canceled." echo "" -echo -e " ${CYAN}harness-cli status [[ engines | engines ]]${NC}" -echo -e " Prints a status message for the Harness CLI, all engines, or the engine specified." -echo "" - -echo -e " ${CYAN}harness-cli system${NC}" -echo -e " Prints a message about build version numbers and other CLI and Server info." +echo -e " ${CYAN}hctl status [[ engines | engines | system ]]${NC}" +echo -e " Prints a status message for the Harness CLI or the objects specified." echo "" echo echo -e "${CYAN}Harness Auth CLI Help${NC}" echo -echo -e " ${CYAN}harness-cli user-add [client | admin]${NC}" +echo -e " ${CYAN}hctl user-add [client | admin]${NC}" echo -e " Returns a new user-id their secret. Grants the role's permissions." echo "" -echo -e " ${CYAN}harness-cli user-delete ${NC}" +echo -e " ${CYAN}hctl user-delete ${NC}" echo -e " Returns the user-id for the deleted user and deletes all attached permissions." echo "" -echo -e " ${CYAN}harness-cli grant [client | admin]${NC}" +echo -e " ${CYAN}hctl grant [client | admin]${NC}" echo -e " Returns true if permissions are added to the user-id" echo "" -echo -e " ${CYAN}harness-cli revoke [client | admin]${NC}" +echo -e " ${CYAN}hctl revoke [client | admin]${NC}" echo -e " Revokes the user permissions for the engine specified, or removes admin permissions." echo "" -echo -e " ${CYAN}harness-cli status users []${NC}" +echo -e " ${CYAN}hctl status users []${NC}" echo -e " Prints the permissions for all users, or the user specified." echo "" diff --git a/python-sdk/harness/__init__.py b/python-sdk/harness/__init__.py index 1a3936f..ac4cb34 100644 --- a/python-sdk/harness/__init__.py +++ b/python-sdk/harness/__init__.py @@ -3,7 +3,7 @@ Python applications with ActionML's REST API for the Harness Server. """ -__version__ = "0.6.0" +__version__ = "1.0.0-SNAPSHOT" # import packages import re @@ -407,6 +407,39 @@ def async_delete(self, engine_id): def delete(self, engine_id): return self.async_delete(engine_id).get_response() + def delete_user_data(self, engine_id, user_id): + """Synchronously delete all user data from Harness Server.""" + return self.async_delete_user_data(engine_id, user_id).get_response() + + def async_delete_user_data(self, engine_id, user_id): + """Asynchronouly delete an event from Harness Server. + :param user_id: user id returned by the HarnessServer when creating the + event. + :returns: + AsyncRequest object. + """ + request = AsyncRequest("DELETE", "%s/%s/%s/%s" % (self.path, engine_id, "entities", user_id)) + request.set_response_handler(self._ok_response_handler) + self._connection.make_request(request) + return request + + def get_user_data(self, engine_id, user_id): + """Synchronously delete all user data from Harness Server.""" + return self.async_get_user_data(engine_id, user_id).get_response() + + def async_get_user_data(self, engine_id, user_id): + """Asynchronouly delete an event from Harness Server. + :param user_id: user id returned by the HarnessServer when creating the + event. + :returns: + AsyncRequest object. + """ + request = AsyncRequest("GET", "%s/%s/%s/%s" % (self.path, engine_id, "entities", user_id)) + print(request) + request.set_response_handler(self._ok_response_handler) + self._connection.make_request(request) + return request + class QueriesClient(BaseClient): """ diff --git a/python-sdk/setup.cfg b/python-sdk/setup.cfg index b88034e..08aedd7 100644 --- a/python-sdk/setup.cfg +++ b/python-sdk/setup.cfg @@ -1,2 +1,2 @@ [metadata] -description-file = README.md +description_file = README.md