From 39213235df4c4b8181de0e116c9ea261ce6212a1 Mon Sep 17 00:00:00 2001 From: Adrian Smijulj Date: Mon, 24 Aug 2020 10:06:25 +0200 Subject: [PATCH 01/26] wip: introduce DynamoDb driver --- packages/fields-storage-dynamodb/.babelrc | 14 + packages/fields-storage-dynamodb/CHANGELOG.md | 642 ++++++++++++++++++ packages/fields-storage-dynamodb/LICENSE | 21 + .../OPERATORSsrc/index.js | 3 + .../model/attributes/arrayAttribute.js | 20 + .../model/attributes/booleanAttribute.js | 35 + .../model/attributes/dateAttribute.js | 27 + .../OPERATORSsrc/model/attributes/index.js | 7 + .../model/attributes/modelAttribute.js | 18 + .../model/attributes/modelsAttribute.js | 17 + .../model/attributes/objectAttribute.js | 20 + .../OPERATORSsrc/model/index.js | 2 + .../model/mysqlAttributesContainer.js | 60 ++ .../OPERATORSsrc/model/mysqlModel.js | 11 + .../OPERATORSsrc/mysqlDriver.js | 325 +++++++++ .../OPERATORSsrc/operators/comparison/all.js | 22 + .../OPERATORSsrc/operators/comparison/eq.js | 58 ++ .../OPERATORSsrc/operators/comparison/gt.js | 13 + .../OPERATORSsrc/operators/comparison/gte.js | 13 + .../OPERATORSsrc/operators/comparison/in.js | 39 ++ .../comparison/jsonArrayFindValue.js | 14 + .../comparison/jsonArrayStrictEquality.js | 14 + .../OPERATORSsrc/operators/comparison/like.js | 18 + .../OPERATORSsrc/operators/comparison/lt.js | 13 + .../OPERATORSsrc/operators/comparison/lte.js | 13 + .../OPERATORSsrc/operators/comparison/ne.js | 17 + .../operators/comparison/search.js | 22 + .../OPERATORSsrc/operators/index.js | 37 + .../OPERATORSsrc/operators/logical/and.js | 40 ++ .../OPERATORSsrc/operators/logical/or.js | 40 ++ .../OPERATORSsrc/statements/delete.js | 17 + .../OPERATORSsrc/statements/index.js | 7 + .../OPERATORSsrc/statements/insert.js | 32 + .../OPERATORSsrc/statements/select.js | 23 + .../OPERATORSsrc/statements/statement.js | 111 +++ .../OPERATORSsrc/statements/update.js | 22 + packages/fields-storage-dynamodb/README.md | 16 + .../__tests__/delete.test.js | 63 ++ .../__tests__/findOne.test.js | 30 + .../__tests__/models/SimpleModel.js | 28 + .../__tests__/models/index.js | 1 + .../__tests__/models/useModels.js | 45 ++ .../__tests__/save.test.js | 63 ++ .../fields-storage-dynamodb/jest.config.js | 6 + packages/fields-storage-dynamodb/package.json | 33 + .../src/DynamoDbClient.js | 184 +++++ .../src/DynamoDbDriver.js | 70 ++ .../src/QueryGenerator.js | 42 ++ packages/fields-storage-dynamodb/src/index.js | 5 + .../src/operators/comparison/beginsWith.js | 18 + .../src/operators/comparison/between.js | 20 + .../src/operators/comparison/eq.js | 28 + .../src/operators/comparison/gt.js | 18 + .../src/operators/comparison/gte.js | 17 + .../src/operators/comparison/lt.js | 17 + .../src/operators/comparison/lte.js | 17 + .../src/operators/index.js | 18 + .../src/statements/KeyConditionExpression.js | 23 + 58 files changed, 2569 insertions(+) create mode 100644 packages/fields-storage-dynamodb/.babelrc create mode 100644 packages/fields-storage-dynamodb/CHANGELOG.md create mode 100644 packages/fields-storage-dynamodb/LICENSE create mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/index.js create mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/arrayAttribute.js create mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/booleanAttribute.js create mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/dateAttribute.js create mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/index.js create mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/modelAttribute.js create mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/modelsAttribute.js create mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/objectAttribute.js create mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/model/index.js create mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/model/mysqlAttributesContainer.js create mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/model/mysqlModel.js create mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/mysqlDriver.js create mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/all.js create mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/eq.js create mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/gt.js create mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/gte.js create mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/in.js create mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/jsonArrayFindValue.js create mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/jsonArrayStrictEquality.js create mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/like.js create mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/lt.js create mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/lte.js create mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/ne.js create mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/search.js create mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/operators/index.js create mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/operators/logical/and.js create mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/operators/logical/or.js create mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/statements/delete.js create mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/statements/index.js create mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/statements/insert.js create mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/statements/select.js create mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/statements/statement.js create mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/statements/update.js create mode 100644 packages/fields-storage-dynamodb/README.md create mode 100644 packages/fields-storage-dynamodb/__tests__/delete.test.js create mode 100644 packages/fields-storage-dynamodb/__tests__/findOne.test.js create mode 100644 packages/fields-storage-dynamodb/__tests__/models/SimpleModel.js create mode 100644 packages/fields-storage-dynamodb/__tests__/models/index.js create mode 100644 packages/fields-storage-dynamodb/__tests__/models/useModels.js create mode 100644 packages/fields-storage-dynamodb/__tests__/save.test.js create mode 100644 packages/fields-storage-dynamodb/jest.config.js create mode 100644 packages/fields-storage-dynamodb/package.json create mode 100644 packages/fields-storage-dynamodb/src/DynamoDbClient.js create mode 100644 packages/fields-storage-dynamodb/src/DynamoDbDriver.js create mode 100644 packages/fields-storage-dynamodb/src/QueryGenerator.js create mode 100644 packages/fields-storage-dynamodb/src/index.js create mode 100644 packages/fields-storage-dynamodb/src/operators/comparison/beginsWith.js create mode 100644 packages/fields-storage-dynamodb/src/operators/comparison/between.js create mode 100644 packages/fields-storage-dynamodb/src/operators/comparison/eq.js create mode 100644 packages/fields-storage-dynamodb/src/operators/comparison/gt.js create mode 100644 packages/fields-storage-dynamodb/src/operators/comparison/gte.js create mode 100644 packages/fields-storage-dynamodb/src/operators/comparison/lt.js create mode 100644 packages/fields-storage-dynamodb/src/operators/comparison/lte.js create mode 100644 packages/fields-storage-dynamodb/src/operators/index.js create mode 100644 packages/fields-storage-dynamodb/src/statements/KeyConditionExpression.js diff --git a/packages/fields-storage-dynamodb/.babelrc b/packages/fields-storage-dynamodb/.babelrc new file mode 100644 index 0000000..4767347 --- /dev/null +++ b/packages/fields-storage-dynamodb/.babelrc @@ -0,0 +1,14 @@ +{ + "presets": [ + ["@babel/preset-env", { + "targets": { + "node": "8.10" + } + }], + "@babel/preset-flow" + ], + "plugins": [ + ["@babel/plugin-proposal-object-rest-spread", {"useBuiltIns": true}], + ["@babel/plugin-transform-runtime"] + ] +} \ No newline at end of file diff --git a/packages/fields-storage-dynamodb/CHANGELOG.md b/packages/fields-storage-dynamodb/CHANGELOG.md new file mode 100644 index 0000000..3012ba7 --- /dev/null +++ b/packages/fields-storage-dynamodb/CHANGELOG.md @@ -0,0 +1,642 @@ +# Change Log + +All notable changes to this project will be documented in this file. +See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. + +## [2.0.1](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@2.0.1-next.0...@commodo/fields-storage-dynamoDb@2.0.1) (2020-07-29) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +## 2.0.1-next.0 (2020-07-28) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +# 2.0.0 (2020-06-10) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +# 2.0.0-next.28 (2020-06-10) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +# 2.0.0-next.27 (2020-06-04) + + +### Bug Fixes + +* use withStorageName, fallback to withName ([12f7ebe](https://github.com/webiny/commodo/commit/12f7ebe19f0c0301cf792295228be47896b1efae)) + + + + + +# 2.0.0-next.26 (2020-06-03) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +# 2.0.0-next.25 (2020-06-02) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +# 2.0.0-next.24 (2020-06-02) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +# 2.0.0-next.23 (2020-05-31) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +# 2.0.0-next.22 (2020-05-31) + + +### Bug Fixes + +* remove unused dependencies ([2baac91](https://github.com/webiny/commodo/commit/2baac9175a21cd05dc071efc54593cf6ca0e0648)) + + + + + +# 2.0.0-next.21 (2020-05-31) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +# 2.0.0-next.20 (2020-05-31) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +# 2.0.0-next.19 (2020-05-31) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +# 2.0.0-next.18 (2020-05-31) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +# 2.0.0-next.17 (2020-05-31) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +# 2.0.0-next.16 (2020-05-25) + + +### Bug Fixes + +* add missing "nedb-promises" dependency ([36a3172](https://github.com/webiny/commodo/commit/36a317222d49b860f09512fbc48f56a977a0245e)) + + + + + +# 2.0.0-next.15 (2020-05-25) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +# 2.0.0-next.14 (2020-05-22) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +# 2.0.0-next.13 (2020-05-22) + + +### Features + +* add support for custom id structure ([2a8aa49](https://github.com/webiny/commodo/commit/2a8aa49a6ddc8ff830d1771b662da3dd48bcfd1c)) + + + + + +# 2.0.0-next.12 (2020-05-22) + + +### Bug Fixes + +* add missing dependency ([d9fdb96](https://github.com/webiny/commodo/commit/d9fdb96d96358f50c280566d5da71fd5079cf212)) + + + + + +# 2.0.0-next.11 (2020-05-20) + + +### Bug Fixes + +* improve "isId" regex and fix tests ([7ddaaf5](https://github.com/webiny/commodo/commit/7ddaaf5e6062a1306f2c574bdd34f7cb073441f0)) + + + + + +# 2.0.0-next.10 (2020-05-19) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +# 2.0.0-next.9 (2020-05-14) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +# 2.0.0-next.8 (2020-05-14) + + +### Bug Fixes + +* update to work with the new driver interface ([588beb1](https://github.com/webiny/commodo/commit/588beb1b5c01e14b5eb49ed3cbb5aaa020c29724)) + + + + + +# 2.0.0-next.7 (2020-05-11) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +# 2.0.0-canary.6 (2020-05-10) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +# 2.0.0-canary.5 (2020-05-10) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +# 2.0.0-canary.4 (2020-05-10) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +# 2.0.0-canary.3 (2020-05-10) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +# 2.0.0-canary.2 (2020-05-10) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +# 2.0.0-canary.1 (2020-05-10) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +# 2.0.0-canary.0 (2020-05-10) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +# [2.0.0-next.0](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@1.0.2...@commodo/fields-storage-dynamoDb@2.0.0-next.0) (2020-05-05) + + +### Bug Fixes + +* do not pass sort/match into aggregation pipeline if not needed ([e619fc9](https://github.com/webiny/commodo/commit/e619fc9282845ea2d0f98779891b8703c853b7b9)) + + +### Features + +* remove page/perPage handling ([b845316](https://github.com/webiny/commodo/commit/b845316ef0c5670b32fba857cbf318dfe730cc5c)) +* use raw data instead of model instances in storage drivers ([7b9e15b](https://github.com/webiny/commodo/commit/7b9e15b6a4883c8d5f28269a3daf97aa2563098d)) + + +### BREAKING CHANGES + +* Storage drivers no longer accept a model instance. They now work with raw data passed from fields-storage layer. +* Handling of page/perPage was removed to make driver more generic. Developers who need this, need to handle parameters before calling the driver, and calculate proper limit/offset themselves. + + + + + +## [1.0.3](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@1.0.2...@commodo/fields-storage-dynamoDb@1.0.3) (2020-01-21) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +## [1.0.2](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@1.0.3...@commodo/fields-storage-dynamoDb@1.0.2) (2020-01-19) + + +### Bug Fixes + +* update package versions ([aa1831e](https://github.com/webiny/commodo/commit/aa1831e)) + + + + + +## [1.0.1](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@1.0.3...@commodo/fields-storage-dynamoDb@1.0.1) (2020-01-19) + + +### Bug Fixes + +* update package versions ([aa1831e](https://github.com/webiny/commodo/commit/aa1831e)) + + + + + +## [1.0.3](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@1.0.1...@commodo/fields-storage-dynamoDb@1.0.3) (2020-01-17) + + +### Bug Fixes + +* update versions ([e5b4c61](https://github.com/webiny/commodo/commit/e5b4c61)) +* update versions ([95852d7](https://github.com/webiny/commodo/commit/95852d7)) + + + + + +## [1.0.1](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@1.0.1...@commodo/fields-storage-dynamoDb@1.0.1) (2020-01-17) + + +### Bug Fixes + +* update versions ([95852d7](https://github.com/webiny/commodo/commit/95852d7)) + + + + + +## [1.0.1](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@0.4.0...@commodo/fields-storage-dynamoDb@1.0.1) (2020-01-17) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +# [0.4.0](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@0.4.0-next.0...@commodo/fields-storage-dynamoDb@0.4.0) (2020-01-10) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +# [0.4.0-next.0](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@0.3.0...@commodo/fields-storage-dynamoDb@0.4.0-next.0) (2020-01-07) + + +### Bug Fixes + +* check results received from aggregate ([d1a5313](https://github.com/webiny/commodo/commit/d1a5313)) + + +### Features + +* use $facet to count totalCount, allow old approach too (optional) ([2e81f19](https://github.com/webiny/commodo/commit/2e81f19)) + + + + + +# [0.3.0](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@0.2.15...@commodo/fields-storage-dynamoDb@0.3.0) (2019-12-17) + + +### Features + +* add option to skip totalCount query ([e257dab](https://github.com/webiny/commodo/commit/e257dab)) +* remove findByIds ([4c3e7df](https://github.com/webiny/commodo/commit/4c3e7df)) + + + + + +## [0.2.15](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@0.2.14...@commodo/fields-storage-dynamoDb@0.2.15) (2019-10-26) + + +### Bug Fixes + +* replace getAttribute with getField ([b0c3aec](https://github.com/webiny/commodo/commit/b0c3aec)) + + + + + +## [0.2.14](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@0.2.13...@commodo/fields-storage-dynamoDb@0.2.14) (2019-10-20) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +## [0.2.13](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@0.2.12...@commodo/fields-storage-dynamoDb@0.2.13) (2019-10-14) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +## [0.2.12](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@0.2.11...@commodo/fields-storage-dynamoDb@0.2.12) (2019-09-26) + + +### Bug Fixes + +* update dependency versions ([2a30863](https://github.com/webiny/commodo/commit/2a30863)) + + + + + +## [0.2.11](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@0.2.10...@commodo/fields-storage-dynamoDb@0.2.11) (2019-09-25) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +## [0.2.10](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@0.2.9...@commodo/fields-storage-dynamoDb@0.2.10) (2019-09-23) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +## [0.2.9](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@0.2.8...@commodo/fields-storage-dynamoDb@0.2.9) (2019-09-23) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +## [0.2.8](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@0.2.7...@commodo/fields-storage-dynamoDb@0.2.8) (2019-09-23) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +## [0.2.7](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@0.2.5...@commodo/fields-storage-dynamoDb@0.2.7) (2019-09-11) + + +### Bug Fixes + +* upgrade to work with new version of repropose ([0c4c983](https://github.com/webiny/commodo/commit/0c4c983)) + + + + + +## [0.2.5](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@0.2.4...@commodo/fields-storage-dynamoDb@0.2.5) (2019-05-24) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +## [0.2.4](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@0.2.3...@commodo/fields-storage-dynamoDb@0.2.4) (2019-05-23) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +## [0.2.3](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@0.2.2...@commodo/fields-storage-dynamoDb@0.2.3) (2019-05-09) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +## [0.2.2](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@0.2.1...@commodo/fields-storage-dynamoDb@0.2.2) (2019-05-07) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +## [0.2.1](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@0.2.0...@commodo/fields-storage-dynamoDb@0.2.1) (2019-05-05) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +# [0.2.0](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@0.1.2...@commodo/fields-storage-dynamoDb@0.2.0) (2019-04-28) + + +### Features + +* rename "object" field to "fields" ([3f17ceb](https://github.com/webiny/commodo/commit/3f17ceb)) + + + + + +## [0.1.2](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@0.1.1...@commodo/fields-storage-dynamoDb@0.1.2) (2019-04-28) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +## [0.1.1](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@0.1.0...@commodo/fields-storage-dynamoDb@0.1.1) (2019-04-28) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +# [0.1.0](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@0.0.9...@commodo/fields-storage-dynamoDb@0.1.0) (2019-04-27) + + +### Features + +* add "aggregate" function (specific to DynamoDb) ([466f4fa](https://github.com/webiny/commodo/commit/466f4fa)) + + + + + +## [0.0.9](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@0.0.8...@commodo/fields-storage-dynamoDb@0.0.9) (2019-04-24) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +## [0.0.8](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@0.0.7...@commodo/fields-storage-dynamoDb@0.0.8) (2019-04-24) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +## [0.0.7](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@0.0.6...@commodo/fields-storage-dynamoDb@0.0.7) (2019-04-24) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +## [0.0.6](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@0.0.5...@commodo/fields-storage-dynamoDb@0.0.6) (2019-04-24) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +## [0.0.5](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@0.0.4...@commodo/fields-storage-dynamoDb@0.0.5) (2019-04-24) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +## [0.0.4](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@0.0.3...@commodo/fields-storage-dynamoDb@0.0.4) (2019-04-23) + + +### Bug Fixes + +* add missing README.md files ([7228d9c](https://github.com/webiny/commodo/commit/7228d9c)) + + + + + +## [0.0.3](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@0.0.2...@commodo/fields-storage-dynamoDb@0.0.3) (2019-04-23) + + +### Bug Fixes + +* update keywords in package.json ([cc544ea](https://github.com/webiny/commodo/commit/cc544ea)) + + + + + +## [0.0.2](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@0.0.1...@commodo/fields-storage-dynamoDb@0.0.2) (2019-04-23) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +## [0.0.1](https://github.com/webiny/commodo/compare/@commodo/fields-storage-dynamoDb@0.0.0...@commodo/fields-storage-dynamoDb@0.0.1) (2019-04-23) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb + + + + + +# 0.0.0 (2019-04-23) + +**Note:** Version bump only for package @commodo/fields-storage-dynamoDb diff --git a/packages/fields-storage-dynamodb/LICENSE b/packages/fields-storage-dynamodb/LICENSE new file mode 100644 index 0000000..b32afdb --- /dev/null +++ b/packages/fields-storage-dynamodb/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 Adrian Smijulj + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/index.js b/packages/fields-storage-dynamodb/OPERATORSsrc/index.js new file mode 100644 index 0000000..ed2f1ee --- /dev/null +++ b/packages/fields-storage-dynamodb/OPERATORSsrc/index.js @@ -0,0 +1,3 @@ +// @flow +export { default as MySQLDriver } from "./mysqlDriver"; +export { default as operators } from "./operators"; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/arrayAttribute.js b/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/arrayAttribute.js new file mode 100644 index 0000000..b069dbf --- /dev/null +++ b/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/arrayAttribute.js @@ -0,0 +1,20 @@ +// @flow +import { ArrayAttribute as BaseArrayAttribute } from "webiny-model"; + +class ArrayAttribute extends BaseArrayAttribute { + setStorageValue(value: mixed) { + if (typeof value === "string") { + super.setStorageValue(JSON.parse(value)); + } else { + super.setStorageValue(value); + } + return this; + } + + async getStorageValue() { + const value = await BaseArrayAttribute.prototype.getStorageValue.call(this); + return value ? JSON.stringify(value) : value; + } +} + +export default ArrayAttribute; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/booleanAttribute.js b/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/booleanAttribute.js new file mode 100644 index 0000000..bfc17e8 --- /dev/null +++ b/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/booleanAttribute.js @@ -0,0 +1,35 @@ +// @flow +import { BooleanAttribute as BaseBooleanAttribute } from "webiny-model"; + +class BooleanAttribute extends BaseBooleanAttribute { + /** + * We must make sure a boolean value is sent, and not 0 or 1, which are stored in MySQL. + * @param value + */ + setStorageValue(value: mixed) { + if (value === 1) { + return super.setStorageValue(true); + } + + if (value === 0) { + return super.setStorageValue(false); + } + + return super.setStorageValue(value); + } + + async getStorageValue() { + const value = await BaseBooleanAttribute.prototype.getStorageValue.call(this); + if (value === true) { + return 1; + } + + if (value === false) { + return 0; + } + + return value; + } +} + +export default BooleanAttribute; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/dateAttribute.js b/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/dateAttribute.js new file mode 100644 index 0000000..292d6d5 --- /dev/null +++ b/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/dateAttribute.js @@ -0,0 +1,27 @@ +// @flow +import { DateAttribute as BaseDateAttribute } from "webiny-model"; +import fecha from "fecha"; + +class DateAttribute extends BaseDateAttribute { + setStorageValue(value: mixed) { + if (value === null) { + return super.setStorageValue(value); + } + + if (value instanceof Date) { + return super.setStorageValue(value); + } + + return super.setStorageValue(fecha.parse(value, "YYYY-MM-DD HH:mm:ss")); + } + + async getStorageValue() { + const value = await BaseDateAttribute.prototype.getStorageValue.call(this); + if (value instanceof Date) { + return fecha.format(value, "YYYY-MM-DD HH:mm:ss"); + } + return value; + } +} + +export default DateAttribute; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/index.js b/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/index.js new file mode 100644 index 0000000..153f953 --- /dev/null +++ b/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/index.js @@ -0,0 +1,7 @@ +// @flow +export { default as BooleanAttribute } from "./booleanAttribute"; +export { default as DateAttribute } from "./dateAttribute"; +export { default as ArrayAttribute } from "./arrayAttribute"; +export { default as ObjectAttribute } from "./objectAttribute"; +export { default as ModelAttribute } from "./modelAttribute"; +export { default as ModelsAttribute } from "./modelsAttribute"; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/modelAttribute.js b/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/modelAttribute.js new file mode 100644 index 0000000..2ca6e86 --- /dev/null +++ b/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/modelAttribute.js @@ -0,0 +1,18 @@ +// @flow +import { ModelAttribute as BaseModelAttribute } from "webiny-model"; + +class ModelAttribute extends BaseModelAttribute { + setStorageValue(value: mixed): this { + if (typeof value === "string") { + return super.setStorageValue(JSON.parse(value)); + } + return this; + } + + async getStorageValue() { + const value = await BaseModelAttribute.prototype.getStorageValue.call(this); + return value ? JSON.stringify(value) : value; + } +} + +export default ModelAttribute; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/modelsAttribute.js b/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/modelsAttribute.js new file mode 100644 index 0000000..2fdc1dd --- /dev/null +++ b/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/modelsAttribute.js @@ -0,0 +1,17 @@ +// @flow +import { ModelsAttribute as BaseModelsAttribute } from "webiny-model"; + +class ModelsAttribute extends BaseModelsAttribute { + setStorageValue(value: mixed): this { + if (typeof value === "string") { + super.setStorageValue(JSON.parse(value)); + } + return this; + } + + async getStorageValue(): Promise { + return JSON.stringify(await BaseModelsAttribute.prototype.getStorageValue.call(this)); + } +} + +export default ModelsAttribute; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/objectAttribute.js b/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/objectAttribute.js new file mode 100644 index 0000000..5298a8f --- /dev/null +++ b/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/objectAttribute.js @@ -0,0 +1,20 @@ +// @flow +import { ObjectAttribute as BaseObjectAttribute } from "webiny-model"; + +class ObjectAttribute extends BaseObjectAttribute { + setStorageValue(value: mixed) { + if (typeof value === "string") { + super.setStorageValue(JSON.parse(value)); + } else { + super.setStorageValue(value); + } + return this; + } + + async getStorageValue() { + const value = await BaseObjectAttribute.prototype.getStorageValue.call(this); + return value ? JSON.stringify(value) : value; + } +} + +export default ObjectAttribute; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/model/index.js b/packages/fields-storage-dynamodb/OPERATORSsrc/model/index.js new file mode 100644 index 0000000..498fbc8 --- /dev/null +++ b/packages/fields-storage-dynamodb/OPERATORSsrc/model/index.js @@ -0,0 +1,2 @@ +// @flow +export { default as MySQLModel } from "./mysqlModel"; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/model/mysqlAttributesContainer.js b/packages/fields-storage-dynamodb/OPERATORSsrc/model/mysqlAttributesContainer.js new file mode 100644 index 0000000..3381443 --- /dev/null +++ b/packages/fields-storage-dynamodb/OPERATORSsrc/model/mysqlAttributesContainer.js @@ -0,0 +1,60 @@ +// @flow +import type { Model } from "webiny-model"; +import type { + ModelAttribute as BaseModelAttribute, + ModelsAttribute as BaseModelsAttribute +} from "webiny-entity"; + +import { EntityAttributesContainer } from "webiny-entity"; +import { + ArrayAttribute, + BooleanAttribute, + DateAttribute, + ModelAttribute, + ModelsAttribute, + ObjectAttribute +} from "./attributes"; + +/** + * Contains basic attributes. If needed, this class can be extended to add additional attributes, + * and then be set as a new attributes container as the default one. + */ +class MySQLAttributesContainer extends EntityAttributesContainer { + boolean(): BooleanAttribute { + const model = this.getParentModel(); + model.setAttribute(this.name, new BooleanAttribute(this.name, this)); + return ((model.getAttribute(this.name): any): BooleanAttribute); + } + + date(): DateAttribute { + const model = this.getParentModel(); + model.setAttribute(this.name, new DateAttribute(this.name, this)); + return ((model.getAttribute(this.name): any): DateAttribute); + } + + array(): ArrayAttribute { + const model = this.getParentModel(); + model.setAttribute(this.name, new ArrayAttribute(this.name, this)); + return ((model.getAttribute(this.name): any): ArrayAttribute); + } + + object(): ObjectAttribute { + const model = this.getParentModel(); + model.setAttribute(this.name, new ObjectAttribute(this.name, this)); + return ((model.getAttribute(this.name): any): ObjectAttribute); + } + + model(model: Class): BaseModelAttribute & ModelAttribute { + const parent = this.getParentModel(); + parent.setAttribute(this.name, new ModelAttribute(this.name, this, model)); + return ((parent.getAttribute(this.name): any): BaseModelAttribute & ModelAttribute); + } + + models(model: Class): BaseModelsAttribute & ModelsAttribute { + const parent = this.getParentModel(); + parent.setAttribute(this.name, new ModelsAttribute(this.name, this, model)); + return ((parent.getAttribute(this.name): any): BaseModelsAttribute & ModelsAttribute); + } +} + +export default MySQLAttributesContainer; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/model/mysqlModel.js b/packages/fields-storage-dynamodb/OPERATORSsrc/model/mysqlModel.js new file mode 100644 index 0000000..9a273ae --- /dev/null +++ b/packages/fields-storage-dynamodb/OPERATORSsrc/model/mysqlModel.js @@ -0,0 +1,11 @@ +// @flow +import { EntityModel } from "webiny-entity"; +import MySQLAttributesContainer from "./mysqlAttributesContainer"; + +class MySQLModel extends EntityModel { + createAttributesContainer(): MySQLAttributesContainer { + return new MySQLAttributesContainer(this); + } +} + +export default MySQLModel; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/mysqlDriver.js b/packages/fields-storage-dynamodb/OPERATORSsrc/mysqlDriver.js new file mode 100644 index 0000000..3e795bf --- /dev/null +++ b/packages/fields-storage-dynamodb/OPERATORSsrc/mysqlDriver.js @@ -0,0 +1,325 @@ +// @flow +import _ from "lodash"; +import mdbid from "mdbid"; +import type { Connection, Pool } from "mysql"; +import { Entity, Driver, QueryResult } from "webiny-entity"; +import { MySQLConnection } from "webiny-mysql-connection"; +import { Attribute } from "webiny-model"; +import type { + EntitySaveParams, + EntityFindParams, + EntityDeleteParams, + EntityFindOneParams +} from "webiny-entity/types"; +import type { Operator } from "./../types"; + +import { Insert, Update, Delete, Select } from "./statements"; +import { MySQLModel } from "./model"; +import operators from "./operators"; + +declare type MySQLDriverOptions = { + connection: Connection | Pool, + model?: Class, + operators?: { [string]: Operator }, + tables?: { + prefix: string, + naming: ?Function + }, + autoIncrementIds?: boolean +}; + +class MySQLDriver extends Driver { + connection: MySQLConnection; + model: Class; + operators: { [string]: Operator }; + tables: { + prefix: string, + naming: ?Function + }; + autoIncrementIds: boolean; + constructor(options: MySQLDriverOptions) { + super(); + this.operators = { ...operators, ...(options.operators || {}) }; + this.connection = new MySQLConnection(options.connection); + this.model = options.model || MySQLModel; + + this.tables = { + prefix: "", + ...(options.tables || {}) + }; + this.autoIncrementIds = options.autoIncrementIds || false; + } + + setOperator(name: string, operator: Operator) { + this.operators[name] = operator; + return this; + } + + onEntityConstruct(entity: Entity) { + if (this.autoIncrementIds) { + entity + .attr("id") + .integer() + .setValidators((value, attribute) => + this.isId(attribute.getParentModel().getParentEntity(), value) + ); + } else { + entity + .attr("id") + .char() + .setValidators((value, attribute) => + this.isId(attribute.getParentModel().getParentEntity(), value) + ); + } + } + + getModelClass(): Class { + return this.model; + } + + // eslint-disable-next-line + async save(entity: Entity, options: EntitySaveParams & {}): Promise { + if (entity.isExisting()) { + const data = await entity.toStorage(); + if (_.isEmpty(data)) { + return new QueryResult(true); + } + + const sql = new Update( + { + operators: this.operators, + table: this.getTableName(entity), + data, + where: { id: entity.id }, + limit: 1 + }, + entity + ).generate(); + + await this.getConnection().query(sql); + return new QueryResult(true); + } + + if (!this.autoIncrementIds) { + entity.id = MySQLDriver.__generateID(); + } + + const data = await entity.toStorage(); + const sql = new Insert( + { + operators: this.operators, + data, + table: this.getTableName(entity) + }, + entity + ).generate(); + + try { + const results = await this.getConnection().query(sql); + if (this.autoIncrementIds) { + entity.id = results.insertId; + } + } catch (e) { + const idAttribute: Attribute = (entity.getAttribute("id"): any); + idAttribute.reset(); + throw e; + } + + return new QueryResult(true); + } + + // eslint-disable-next-line + async delete(entity: Entity, options: EntityDeleteParams & {}): Promise { + const sql = new Delete( + { + operators: this.operators, + table: this.getTableName(entity), + where: { id: entity.id }, + limit: 1 + }, + entity + ).generate(); + + await this.getConnection().query(sql); + return new QueryResult(true); + } + + async find( + entity: Entity | Class, + options: EntityFindParams & {} + ): Promise { + const clonedOptions = _.merge({}, options, { + operators: this.operators, + table: this.getTableName(entity), + operation: "select", + limit: 10, + offset: 0 + }); + + MySQLDriver.__preparePerPageOption(clonedOptions); + MySQLDriver.__preparePageOption(clonedOptions); + MySQLDriver.__prepareQueryOption(clonedOptions); + MySQLDriver.__prepareSearchOption(clonedOptions); + + clonedOptions.calculateFoundRows = true; + + const sql = new Select(clonedOptions, entity).generate(); + const results = await this.getConnection().query([sql, "SELECT FOUND_ROWS() as count"]); + + return new QueryResult(results[0], { totalCount: results[1][0].count }); + } + + async findOne( + entity: Entity | Class, + options: EntityFindOneParams & {} + ): Promise { + const clonedOptions = { + operators: this.operators, + table: this.getTableName(entity), + where: options.query, + search: options.search, + limit: 1 + }; + + MySQLDriver.__prepareQueryOption(clonedOptions); + MySQLDriver.__prepareSearchOption(clonedOptions); + + const sql = new Select(clonedOptions, entity).generate(); + + const results = await this.getConnection().query(sql); + return new QueryResult(results[0]); + } + + async count( + entity: Entity | Class, + options: EntityFindParams & {} + ): Promise { + const clonedOptions = _.merge( + {}, + options, + { + operators: this.operators, + table: this.getTableName(entity), + columns: ["COUNT(*) AS count"] + }, + entity + ); + + MySQLDriver.__prepareQueryOption(clonedOptions); + MySQLDriver.__prepareSearchOption(clonedOptions); + + const sql = new Select(clonedOptions, entity).generate(); + + const results = await this.getConnection().query(sql); + return new QueryResult(results[0].count); + } + + // eslint-disable-next-line + isId(entity: Entity | Class, value: mixed, options: ?Object): boolean { + if (this.autoIncrementIds) { + return typeof value === "number" && Number.isInteger(value) && value > 0; + } + + if (typeof value === "string") { + return value.match(new RegExp("^[0-9a-fA-F]{24}$")) !== null; + } + return false; + } + + getConnection(): MySQLConnection { + return this.connection; + } + + setTablePrefix(tablePrefix: string): this { + this.tables.prefix = tablePrefix; + return this; + } + + getTablePrefix(): string { + return this.tables.prefix; + } + + setTableNaming(tableNameValue: Function): this { + this.tables.naming = tableNameValue; + return this; + } + + getTableNaming(): ?Function { + return this.tables.naming; + } + + getTableName(entity: Entity | Class): string { + const params = { + classId: _.get(entity, "constructor.classId", _.get(entity, "classId")), + storageClassId: _.get( + entity, + "constructor.storageClassId", + _.get(entity, "storageClassId") + ), + tableName: _.get(entity, "constructor.tableName", _.get(entity, "tableName")) + }; + + const getTableName = this.getTableNaming(); + if (typeof getTableName === "function") { + return getTableName({ entity, ...params, driver: this }); + } + + if (params.tableName) { + return this.tables.prefix + params.tableName; + } + + return ( + this.tables.prefix + (params.storageClassId ? params.storageClassId : params.classId) + ); + } + + async test() { + await this.getConnection().test(); + return true; + } + + static __preparePerPageOption(clonedOptions: Object): void { + if ("perPage" in clonedOptions) { + clonedOptions.limit = clonedOptions.perPage; + delete clonedOptions.perPage; + } + } + + static __preparePageOption(clonedOptions: Object): void { + if ("page" in clonedOptions) { + clonedOptions.offset = clonedOptions.limit * (clonedOptions.page - 1); + delete clonedOptions.page; + } + } + + static __prepareQueryOption(clonedOptions: Object): void { + if (clonedOptions.query instanceof Object) { + clonedOptions.where = clonedOptions.query; + delete clonedOptions.query; + } + } + + static __prepareSearchOption(clonedOptions: Object): void { + // Here we handle search (if passed) - we transform received arguments into linked LIKE statements. + if (clonedOptions.search instanceof Object) { + const { query, operator, fields: columns } = clonedOptions.search; + const search = { $search: { operator, columns, query } }; + + if (clonedOptions.where instanceof Object) { + clonedOptions.where = { + $and: [search, clonedOptions.where] + }; + } else { + clonedOptions.where = search; + } + + delete clonedOptions.search; + } + } + + static __generateID() { + return mdbid(); + } +} + +export default MySQLDriver; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/all.js b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/all.js new file mode 100644 index 0000000..4ad3347 --- /dev/null +++ b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/all.js @@ -0,0 +1,22 @@ +// @flow +import _ from "lodash"; +import type { Operator } from "../../../types"; +import and from "./../logical/and"; + +const all: Operator = { + canProcess: ({ key, value }) => { + if (key.charAt(0) === "$") { + return false; + } + + return _.has(value, "$all"); + }, + process: ({ key, value, statement }) => { + const andValue = value["$all"].map(v => { + return { [key]: { $jsonArrayFindValue: v } }; + }); + return and.process({ key, value: andValue, statement }); + } +}; + +export default all; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/eq.js b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/eq.js new file mode 100644 index 0000000..0229612 --- /dev/null +++ b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/eq.js @@ -0,0 +1,58 @@ +// @flow +import _ from "lodash"; +import type { Operator } from "../../../types"; +import { ArrayAttribute } from "webiny-model"; +import jsonArrayStrictEquality from "./../comparison/jsonArrayStrictEquality"; +import jsonArrayFindValue from "./../comparison/jsonArrayFindValue"; + +const eq: Operator = { + canProcess: ({ key, value, statement }) => { + if (key.charAt(0) === "$") { + return false; + } + + if (_.has(value, "$eq")) { + return true; + } + + // Valid values are 1, '1', null, true, false. + if (_.isString(value) || _.isNumber(value) || [null, true, false].includes(value)) { + return true; + } + + const instance = + typeof statement.entity === "function" ? new statement.entity() : statement.entity; + const attribute = instance.getAttribute(key); + return attribute instanceof ArrayAttribute && Array.isArray(value); + }, + process: ({ key, value, statement }) => { + value = _.get(value, "$eq", value); + if (value === null) { + return "`" + key + "` IS NULL"; + } + + const instance = + typeof statement.entity === "function" ? new statement.entity() : statement.entity; + const attribute = instance.getAttribute(key); + if (attribute instanceof ArrayAttribute) { + // Match all values (strict array equality check) + if (Array.isArray(value)) { + return jsonArrayStrictEquality.process({ + key, + value: { $jsonArrayStrictEquality: value }, + statement + }); + } else { + return jsonArrayFindValue.process({ + key, + value: { $jsonArrayFindValue: value }, + statement + }); + } + } + + return "`" + key + "` = " + statement.escape(value); + } +}; + +export default eq; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/gt.js b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/gt.js new file mode 100644 index 0000000..a088a49 --- /dev/null +++ b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/gt.js @@ -0,0 +1,13 @@ +// @flow +import _ from "lodash"; +import type { Operator } from "../../../types"; + +const gt: Operator = { + canProcess: ({ value }) => { + return _.has(value, "$gt"); + }, + process: ({ key, value, statement }) => { + return "`" + key + "` > " + statement.escape(value["$gt"]); + } +}; +export default gt; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/gte.js b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/gte.js new file mode 100644 index 0000000..ceec0a5 --- /dev/null +++ b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/gte.js @@ -0,0 +1,13 @@ +// @flow +import _ from "lodash"; +import type { Operator } from "../../../types"; + +const gte: Operator = { + canProcess: ({ value }) => { + return _.has(value, "$gte"); + }, + process: ({ key, value, statement }) => { + return "`" + key + "` >= " + statement.escape(value["$gte"]); + } +}; +export default gte; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/in.js b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/in.js new file mode 100644 index 0000000..2a9689c --- /dev/null +++ b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/in.js @@ -0,0 +1,39 @@ +// @flow +import _ from "lodash"; +import type { Operator } from "../../../types"; +import { ArrayAttribute } from "webiny-model"; +import or from "../logical/or"; + +const inOperator: Operator = { + canProcess: ({ key, value, statement }) => { + if (key.charAt(0) === "$") { + return false; + } + + if (_.has(value, "$in")) { + return true; + } + + const instance = + typeof statement.entity === "function" ? new statement.entity() : statement.entity; + const attribute = instance.getAttribute(key); + + return Array.isArray(value) && !(attribute instanceof ArrayAttribute); + }, + process: ({ key, value, statement }) => { + value = _.get(value, "$in", value); + + const instance = + typeof statement.entity === "function" ? new statement.entity() : statement.entity; + const attribute = instance.getAttribute(key); + if (attribute instanceof ArrayAttribute) { + const andValue = value.map(v => { + return { [key]: { $jsonArrayFindValue: v } }; + }); + return or.process({ key, value: andValue, statement }); + } + + return "`" + key + "` IN(" + value.map(item => statement.escape(item)).join(", ") + ")"; + } +}; +export default inOperator; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/jsonArrayFindValue.js b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/jsonArrayFindValue.js new file mode 100644 index 0000000..74cd73a --- /dev/null +++ b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/jsonArrayFindValue.js @@ -0,0 +1,14 @@ +// @flow +import _ from "lodash"; +import type { Operator } from "../../../types"; + +const jsonArrayFindValue: Operator = { + canProcess: ({ value }) => { + return _.has(value, "$jsonArrayFindValue"); + }, + process: ({ key, value, statement }) => { + value = value["$jsonArrayFindValue"]; + return "JSON_SEARCH(`" + key + "`, 'one', " + statement.escape(value) + ") IS NOT NULL"; + } +}; +export default jsonArrayFindValue; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/jsonArrayStrictEquality.js b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/jsonArrayStrictEquality.js new file mode 100644 index 0000000..fb1cb14 --- /dev/null +++ b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/jsonArrayStrictEquality.js @@ -0,0 +1,14 @@ +// @flow +import _ from "lodash"; +import type { Operator } from "../../../types"; + +const jsonArrayStrictEquality: Operator = { + canProcess: ({ value }) => { + return _.has(value, "$jsonArrayStrictEquality"); + }, + process: ({ key, value, statement }) => { + value = value["$jsonArrayStrictEquality"]; + return "`" + key + "` = JSON_ARRAY(" + value.map(v => statement.escape(v)).join(", ") + ")"; + } +}; +export default jsonArrayStrictEquality; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/like.js b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/like.js new file mode 100644 index 0000000..5911066 --- /dev/null +++ b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/like.js @@ -0,0 +1,18 @@ +// @flow +import _ from "lodash"; +import type { Operator } from "../../../types"; + +const like: Operator = { + canProcess: ({ key, value }) => { + if (key.charAt(0) === "$") { + return false; + } + + return _.has(value, "$like"); + }, + process: ({ key, value, statement }) => { + return "`" + key + "` LIKE " + statement.escape(value["$like"]); + } +}; + +export default like; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/lt.js b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/lt.js new file mode 100644 index 0000000..ff10b80 --- /dev/null +++ b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/lt.js @@ -0,0 +1,13 @@ +// @flow +import _ from "lodash"; +import type { Operator } from "../../../types"; + +const lt: Operator = { + canProcess: ({ value }) => { + return _.has(value, "$lt"); + }, + process: ({ key, value, statement }) => { + return "`" + key + "` < " + statement.escape(value["$lt"]); + } +}; +export default lt; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/lte.js b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/lte.js new file mode 100644 index 0000000..6e807f4 --- /dev/null +++ b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/lte.js @@ -0,0 +1,13 @@ +// @flow +import _ from "lodash"; +import type { Operator } from "../../../types"; + +const lte: Operator = { + canProcess: ({ value }) => { + return _.has(value, "$lte"); + }, + process: ({ key, value, statement }) => { + return "`" + key + "` <= " + statement.escape(value["$lte"]); + } +}; +export default lte; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/ne.js b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/ne.js new file mode 100644 index 0000000..6902773 --- /dev/null +++ b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/ne.js @@ -0,0 +1,17 @@ +// @flow +import _ from "lodash"; +import type { Operator } from "../../../types"; + +const ne: Operator = { + canProcess: ({ value }) => { + return _.has(value, "$ne"); + }, + process: ({ key, value, statement }) => { + if (value["$ne"] === null) { + return "`" + key + "` IS NOT NULL"; + } + + return "`" + key + "` <> " + statement.escape(value["$ne"]); + } +}; +export default ne; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/search.js b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/search.js new file mode 100644 index 0000000..67a730c --- /dev/null +++ b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/search.js @@ -0,0 +1,22 @@ +// @flow +import type { Operator } from "../../../types"; +import or from "../logical/or"; +import and from "../logical/and"; + +const search: Operator = { + canProcess: ({ key }) => { + return key === "$search"; + }, + process: ({ value, statement }) => { + const columns = value.columns.map(columns => { + return { [columns]: { $like: "%" + value.query + "%" } }; + }); + + if (value.operator === "and") { + return and.process({ key: "$and", value: columns, statement }); + } + return or.process({ key: "$or", value: columns, statement }); + } +}; + +export default search; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/index.js b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/index.js new file mode 100644 index 0000000..1555144 --- /dev/null +++ b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/index.js @@ -0,0 +1,37 @@ +// @flow + +// Logical Operators (A-Z) +import $and from "./logical/and"; +import $or from "./logical/or"; + +// Comparison operators (A-Z) +import $all from "./comparison/all"; +import $eq from "./comparison/eq"; +import $gt from "./comparison/gt"; +import $gte from "./comparison/gte"; +import $in from "./comparison/in"; +import $jsonArrayFindValue from "./comparison/jsonArrayFindValue"; +import $jsonArrayStrictEquality from "./comparison/jsonArrayStrictEquality"; +import $like from "./comparison/like"; +import $lt from "./comparison/lt"; +import $lte from "./comparison/lte"; +import $ne from "./comparison/ne"; +import $search from "./comparison/search"; + +export default { + $or, + $and, + + $all, + $eq, + $gt, + $gte, + $in, + $jsonArrayFindValue, + $jsonArrayStrictEquality, + $like, + $lt, + $lte, + $ne, + $search +}; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/logical/and.js b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/logical/and.js new file mode 100644 index 0000000..cb4ed4e --- /dev/null +++ b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/logical/and.js @@ -0,0 +1,40 @@ +// @flow +import _ from "lodash"; +import type { Operator } from "../../../types"; + +const and: Operator = { + canProcess: ({ key }) => { + return key === "$and"; + }, + process: ({ value, statement }) => { + let output = ""; + switch (true) { + case _.isArray(value): + value.forEach(object => { + for (const [andKey, andValue] of Object.entries(object)) { + if (output === "") { + output = statement.process({ [andKey]: andValue }); + } else { + output += " AND " + statement.process({ [andKey]: andValue }); + } + } + }); + break; + case _.isPlainObject(value): + for (const [andKey, andValue] of Object.entries(value)) { + if (output === "") { + output = statement.process({ [andKey]: andValue }); + } else { + output += " AND " + statement.process({ [andKey]: andValue }); + } + } + break; + default: + throw Error("$and operator must receive an object or an array."); + } + + return "(" + output + ")"; + } +}; + +export default and; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/logical/or.js b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/logical/or.js new file mode 100644 index 0000000..501ccb4 --- /dev/null +++ b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/logical/or.js @@ -0,0 +1,40 @@ +// @flow +import _ from "lodash"; +import type { Operator } from "../../../types"; + +const or: Operator = { + canProcess: ({ key }) => { + return key === "$or"; + }, + process: ({ value, statement }) => { + let output = ""; + switch (true) { + case _.isArray(value): + value.forEach(object => { + for (const [orKey, orValue] of Object.entries(object)) { + if (output === "") { + output = statement.process({ [orKey]: orValue }); + } else { + output += " OR " + statement.process({ [orKey]: orValue }); + } + } + }); + break; + case _.isPlainObject(value): + for (const [orKey, orValue] of Object.entries(value)) { + if (output === "") { + output = statement.process({ [orKey]: orValue }); + } else { + output += " OR " + statement.process({ [orKey]: orValue }); + } + } + break; + default: + throw Error("$or operator must receive an object or an array."); + } + + return "(" + output + ")"; + } +}; + +export default or; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/statements/delete.js b/packages/fields-storage-dynamodb/OPERATORSsrc/statements/delete.js new file mode 100644 index 0000000..00e6f7a --- /dev/null +++ b/packages/fields-storage-dynamodb/OPERATORSsrc/statements/delete.js @@ -0,0 +1,17 @@ +// @flow +import Statement from "./statement"; + +class Delete extends Statement { + generate() { + const options = this.options; + let output = `DELETE FROM \`${options.table}\``; + output += this.getWhere(options); + output += this.getOrder(options); + output += this.getLimit(options); + output += this.getOffset(options); + + return output; + } +} + +export default Delete; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/statements/index.js b/packages/fields-storage-dynamodb/OPERATORSsrc/statements/index.js new file mode 100644 index 0000000..a0ebb41 --- /dev/null +++ b/packages/fields-storage-dynamodb/OPERATORSsrc/statements/index.js @@ -0,0 +1,7 @@ +// @flow +// Table data statements. +export { default as Statement } from "./statement"; +export { default as Insert } from "./insert"; +export { default as Select } from "./select"; +export { default as Update } from "./update"; +export { default as Delete } from "./delete"; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/statements/insert.js b/packages/fields-storage-dynamodb/OPERATORSsrc/statements/insert.js new file mode 100644 index 0000000..362f405 --- /dev/null +++ b/packages/fields-storage-dynamodb/OPERATORSsrc/statements/insert.js @@ -0,0 +1,32 @@ +// @flow +import Statement from "./statement"; +import _ from "lodash"; + +class Insert extends Statement { + generate() { + const options = this.options; + const columns = _.keys(options.data) + .map(c => "`" + c + "`") + .join(", "); + const insertValues = _.values(options.data) + .map(value => this.escape(value)) + .join(", "); + + if (!options.onDuplicateKeyUpdate) { + return `INSERT INTO \`${options.table}\` (${columns}) VALUES (${insertValues})`; + } + + const updateValues = []; + for (const [key, value] of Object.entries(options.data)) { + updateValues.push(key + " = " + this.escape(value)); + } + + return `INSERT INTO \`${ + options.table + }\` (${columns}) VALUES (${insertValues}) ON DUPLICATE KEY UPDATE ${updateValues.join( + ", " + )}`; + } +} + +export default Insert; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/statements/select.js b/packages/fields-storage-dynamodb/OPERATORSsrc/statements/select.js new file mode 100644 index 0000000..7aa1e03 --- /dev/null +++ b/packages/fields-storage-dynamodb/OPERATORSsrc/statements/select.js @@ -0,0 +1,23 @@ +// @flow +import Statement from "./statement"; + +class Select extends Statement { + generate() { + const options = this.options; + let output = `SELECT`; + if (options.calculateFoundRows) { + output += ` SQL_CALC_FOUND_ROWS`; + } + + output += this.getColumns(options); + output += ` FROM \`${options.table}\``; + output += this.getWhere(options); + output += this.getOrder(options); + output += this.getLimit(options); + output += this.getOffset(options); + + return output; + } +} + +export default Select; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/statements/statement.js b/packages/fields-storage-dynamodb/OPERATORSsrc/statements/statement.js new file mode 100644 index 0000000..fbe8a49 --- /dev/null +++ b/packages/fields-storage-dynamodb/OPERATORSsrc/statements/statement.js @@ -0,0 +1,111 @@ +// @flow +import SqlString from "sqlstring"; +import _ from "lodash"; +import type { Entity } from "webiny-entity"; +import type { Operator, Payload } from "../../types"; + +declare type StatementOptions = { + operators: { [string]: Operator }, + calculateFoundRows: boolean, + table: string, + data: Object, + limit?: number, + offset?: number, + sort?: string, + where?: Object, + columns?: Array, + onDuplicateKeyUpdate?: boolean +}; + +class Statement { + entity: Entity | Class; + options: StatementOptions; + constructor(options: Object = {}, entity: Entity | Class) { + this.options = options; + this.entity = entity; + } + + generate(): string { + return ""; + } + + getColumns(options: StatementOptions): string { + const columns = options.columns || []; + + if (_.isEmpty(columns)) { + return " *"; + } + + return " " + columns.join(", "); + } + + getWhere(options: StatementOptions): string { + if (_.isEmpty(options.where)) { + return ""; + } + + return " WHERE " + this.process({ $and: options.where }); + } + + getOrder(options: StatementOptions): string { + if (!(options.sort instanceof Object) || _.isEmpty(options.sort)) { + return ""; + } + + let query = []; + + for (let key in options.sort) { + query.push(`${key} ${options.sort[key] === 1 ? "ASC" : "DESC"}`); + } + + return " ORDER BY " + query.join(", "); + } + + getLimit(options: StatementOptions): string { + const limit = options.limit || 0; + + if (_.isNumber(limit) && limit > 0) { + return ` LIMIT ${limit}`; + } + return ""; + } + + getOffset(options: StatementOptions): string { + const offset = options.offset || 0; + + if (_.isNumber(offset) && offset > 0) { + return ` OFFSET ${offset}`; + } + return ""; + } + + escape(value: mixed) { + return SqlString.escape(value); + } + + /** + * Traverse the payload and apply operators to construct a valid MySQL statement + * @private + * @param {Object} payload + * @returns {string} SQL query + */ + process(payload: Payload): string { + let output = ""; + + outerLoop: for (const [key, value] of Object.entries(payload)) { + const operators: Array = Object.values(this.options.operators); + for (let i = 0; i < operators.length; i++) { + const operator = operators[i]; + if (operator.canProcess({ key, value, statement: this })) { + output += operator.process({ key, value, statement: this }); + continue outerLoop; + } + } + throw new Error(`Invalid operator {${key} : ${(value: any)}}.`); + } + + return output; + } +} + +export default Statement; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/statements/update.js b/packages/fields-storage-dynamodb/OPERATORSsrc/statements/update.js new file mode 100644 index 0000000..4128f9e --- /dev/null +++ b/packages/fields-storage-dynamodb/OPERATORSsrc/statements/update.js @@ -0,0 +1,22 @@ +// @flow +import Statement from "./statement"; + +class Update extends Statement { + generate() { + const values = [], + options = this.options; + for (const [key, value] of Object.entries(options.data)) { + values.push("`" + key + "` = " + this.escape(value)); + } + + let output = `UPDATE \`${options.table}\` SET ${values.join(", ")}`; + output += this.getWhere(options); + output += this.getOrder(options); + output += this.getLimit(options); + output += this.getOffset(options); + + return output; + } +} + +export default Update; diff --git a/packages/fields-storage-dynamodb/README.md b/packages/fields-storage-dynamodb/README.md new file mode 100644 index 0000000..94a3e4d --- /dev/null +++ b/packages/fields-storage-dynamodb/README.md @@ -0,0 +1,16 @@ +# @commodo/fields-storage-dynamoDb + +We're working hard to get all the docs in order. New articles will be added daily. + +In the meantime, take a look at our [Github repo](https://github.com/webiny/webiny-js), it contains tons of examples to get you on track. + +For API examples, take a look at the packages that have an `api-` prefix. Some good packages to study: + +- [@webiny/api-i18n](https://github.com/webiny/webiny-js/tree/master/packages/api-i18n) +- [@webiny/api-page-builder](https://github.com/webiny/webiny-js/tree/master/packages/api-page-builder) +- [@webiny/api-security](https://github.com/webiny/webiny-js/tree/master/packages/api-security) + +If you still can't find what you're looking for, please open an issue and we'll point you in the right direction. + +Thank you for your patience! + diff --git a/packages/fields-storage-dynamodb/__tests__/delete.test.js b/packages/fields-storage-dynamodb/__tests__/delete.test.js new file mode 100644 index 0000000..4646d84 --- /dev/null +++ b/packages/fields-storage-dynamodb/__tests__/delete.test.js @@ -0,0 +1,63 @@ +import { useModels } from "./models"; +import { getName } from "@commodo/name"; + +describe("delete test", function() { + const { models, getDocumentClient } = useModels(); + + it("should be able to perform create & update operations", async () => { + const { SimpleModel } = models; + const simpleModel = new SimpleModel(); + simpleModel.populate({ + pk: getName(SimpleModel), + sk: "something-1", + name: "Something-1", + enabled: true, + tags: ["one", "two", "three"], + age: 55 + }); + + await simpleModel.save(); + + let item = await getDocumentClient() + .get({ + TableName: "pk-sk", + Key: { pk: getName(SimpleModel), sk: "something-1" } + }) + .promise(); + + expect(item).toEqual({ + Item: { + sk: "something-1", + name: "Something-1", + pk: "SimpleModel", + slug: "something1", + enabled: true, + age: 55, + tags: ["one", "two", "three"] + } + }); + + simpleModel.name = "Something-1-edited"; + await simpleModel.save(); + + item = await getDocumentClient() + .get({ + TableName: "pk-sk", + Key: { pk: getName(SimpleModel), sk: "something-1" } + }) + .promise(); + + expect(item).toEqual({ + Item: { + sk: "something-1", + name: "Something-1-edited", + pk: "SimpleModel", + slug: "something1Edited", + enabled: true, + age: 55, + tags: ["one", "two", "three"] + } + }); + + }); +}); diff --git a/packages/fields-storage-dynamodb/__tests__/findOne.test.js b/packages/fields-storage-dynamodb/__tests__/findOne.test.js new file mode 100644 index 0000000..16edae7 --- /dev/null +++ b/packages/fields-storage-dynamodb/__tests__/findOne.test.js @@ -0,0 +1,30 @@ +import { useModels } from "./models"; + +describe("findOne test", function() { + const { models, getDocumentClient } = useModels(); + + it("should be able to use the findOne method", async () => { + const { SimpleModel } = models; + + for (let i = 0; i < 3; i++) { + await getDocumentClient() + .put({ + TableName: "pk-sk", + Item: { + pk: `find-one`, + sk: String(i), + name: `one-${i}`, + slug: `one1`, + enabled: true, + age: i * 10, + tags: [i] + } + }) + .promise(); + } + + const model1 = await SimpleModel.findOne({ + query: { pk: `find-one`, sk: String("0") } + }); + }); +}); diff --git a/packages/fields-storage-dynamodb/__tests__/models/SimpleModel.js b/packages/fields-storage-dynamodb/__tests__/models/SimpleModel.js new file mode 100644 index 0000000..ade56f4 --- /dev/null +++ b/packages/fields-storage-dynamodb/__tests__/models/SimpleModel.js @@ -0,0 +1,28 @@ +import { compose } from "ramda"; +import camelcase from "camelcase"; +import { withName } from "@commodo/name"; +import { withHooks } from "@commodo/hooks"; +import { withFields, string, boolean, number } from "@commodo/fields"; +import { withPrimaryKey } from "@commodo/fields-storage"; + +export default base => + compose( + withName("SimpleModel"), + withHooks({ + beforeSave() { + if (this.name) { + this.slug = camelcase(this.name); + } + } + }), + withPrimaryKey("pk", "sk"), + withFields({ + pk: string(), + sk: string(), + name: string(), + slug: string(), + enabled: boolean({ value: true }), + tags: string({ list: true }), + age: number() + }) + )(base()); diff --git a/packages/fields-storage-dynamodb/__tests__/models/index.js b/packages/fields-storage-dynamodb/__tests__/models/index.js new file mode 100644 index 0000000..6dd1ab6 --- /dev/null +++ b/packages/fields-storage-dynamodb/__tests__/models/index.js @@ -0,0 +1 @@ +export {default as useModels} from "./useModels"; diff --git a/packages/fields-storage-dynamodb/__tests__/models/useModels.js b/packages/fields-storage-dynamodb/__tests__/models/useModels.js new file mode 100644 index 0000000..52babd5 --- /dev/null +++ b/packages/fields-storage-dynamodb/__tests__/models/useModels.js @@ -0,0 +1,45 @@ +import { DocumentClient } from "aws-sdk/clients/dynamodb"; +import { withStorage } from "@commodo/fields-storage"; +import { DynamoDbDriver } from "@commodo/fields-storage-dynamodb"; +import { compose } from "ramda"; + +// Models. +import simpleModel from "./SimpleModel"; + +export default ({ init = true } = {}) => { + const self = { + models: {}, + documentClient: null, + beforeAll: () => { + self.documentClient = new DocumentClient({ + convertEmptyValues: true, + endpoint: "localhost:8000", + sslEnabled: false, + region: "local-env" + }); + + const base = () => + compose( + withStorage({ + driver: new DynamoDbDriver({ + documentClient: self.documentClient, + tableName: "pk-sk" + }) + }) + )(); + + Object.assign(self.models, { + SimpleModel: simpleModel(base) + }); + }, + getDocumentClient() { + return self.documentClient; + } + }; + + if (init !== false) { + beforeAll(self.beforeAll); + } + + return self; +}; diff --git a/packages/fields-storage-dynamodb/__tests__/save.test.js b/packages/fields-storage-dynamodb/__tests__/save.test.js new file mode 100644 index 0000000..420dae2 --- /dev/null +++ b/packages/fields-storage-dynamodb/__tests__/save.test.js @@ -0,0 +1,63 @@ +import { useModels } from "./models"; +import { getName } from "@commodo/name"; + +describe("save test", function() { + const { models, getDocumentClient } = useModels(); + + it("should be able to perform create & update operations", async () => { + const { SimpleModel } = models; + const simpleModel = new SimpleModel(); + simpleModel.populate({ + pk: getName(SimpleModel), + sk: "something-1", + name: "Something-1", + enabled: true, + tags: ["one", "two", "three"], + age: 55 + }); + + await simpleModel.save(); + + let item = await getDocumentClient() + .get({ + TableName: "pk-sk", + Key: { pk: getName(SimpleModel), sk: "something-1" } + }) + .promise(); + + expect(item).toEqual({ + Item: { + sk: "something-1", + name: "Something-1", + pk: "SimpleModel", + slug: "something1", + enabled: true, + age: 55, + tags: ["one", "two", "three"] + } + }); + + simpleModel.name = "Something-1-edited"; + await simpleModel.save(); + + item = await getDocumentClient() + .get({ + TableName: "pk-sk", + Key: { pk: getName(SimpleModel), sk: "something-1" } + }) + .promise(); + + expect(item).toEqual({ + Item: { + sk: "something-1", + name: "Something-1-edited", + pk: "SimpleModel", + slug: "something1Edited", + enabled: true, + age: 55, + tags: ["one", "two", "three"] + } + }); + + }); +}); diff --git a/packages/fields-storage-dynamodb/jest.config.js b/packages/fields-storage-dynamodb/jest.config.js new file mode 100644 index 0000000..c1cff1c --- /dev/null +++ b/packages/fields-storage-dynamodb/jest.config.js @@ -0,0 +1,6 @@ +const dynamoDbPreset = require("@shelf/jest-dynamodb/jest-preset"); +const base = require("../../jest.config.base"); + +module.exports = { + ...base({ path: __dirname }, [dynamoDbPreset]) +}; diff --git a/packages/fields-storage-dynamodb/package.json b/packages/fields-storage-dynamodb/package.json new file mode 100644 index 0000000..2f140c8 --- /dev/null +++ b/packages/fields-storage-dynamodb/package.json @@ -0,0 +1,33 @@ +{ + "name": "@commodo/fields-storage-dynamodb", + "version": "2.0.1", + "main": "index.js", + "repository": { + "type": "git", + "url": "https://github.com/webiny/commodo.git" + }, + "contributors": [ + "Adrian Smijulj " + ], + "license": "MIT", + "dependencies": { + "aws-sdk": "^2.729.0" + }, + "devDependencies": { + "@shelf/jest-dynamodb": "^1.7.0" + }, + "publishConfig": { + "access": "public" + }, + "keywords": [ + "commodo", + "composeable", + "models", + "storage", + "dynamoDb" + ], + "scripts": { + "build": "babel src --ignore src/__tests__ --out-dir dist --source-maps", + "watch": "yarn build --watch" + } +} diff --git a/packages/fields-storage-dynamodb/src/DynamoDbClient.js b/packages/fields-storage-dynamodb/src/DynamoDbClient.js new file mode 100644 index 0000000..9433462 --- /dev/null +++ b/packages/fields-storage-dynamodb/src/DynamoDbClient.js @@ -0,0 +1,184 @@ +import { DocumentClient } from "aws-sdk/clients/dynamodb"; +import QueryGenerator from "./QueryGenerator"; + +class DynamoDbClient { + constructor({ documentClient } = {}) { + this.client = documentClient || new DocumentClient(); + } + + async create(rawItems) { + const items = Array.isArray(rawItems) ? rawItems : [rawItems]; + const results = []; + for (let i = 0; i < items.length; i++) { + const item = items[i]; + results.push( + await this.client + .put({ + TableName: item.table, + Item: item.data + }) + .promise() + ); + } + + return results; + } + + async delete(rawItems) { + const items = Array.isArray(rawItems) ? rawItems : [rawItems]; + const results = []; + for (let i = 0; i < items.length; i++) { + let item = items[i]; + results.push( + await this.client + .delete({ + TableName: item.table, + Key: item.query + }) + .promise() + ); + } + + return results; + } + + async update(rawItems) { + const items = Array.isArray(rawItems) ? rawItems : [rawItems]; + + for (let i = 0; i < items.length; i++) { + const item = items[i]; + + const update = { + UpdateExpression: "SET ", + ExpressionAttributeNames: {}, + ExpressionAttributeValues: {} + }; + + const updateExpression = []; + for (const key in item.data) { + updateExpression.push(`#${key} = :${key}`); + update.ExpressionAttributeNames[`#${key}`] = key; + update.ExpressionAttributeValues[`:${key}`] = item.data[key]; + } + + update.UpdateExpression += updateExpression.join(", "); + + await this.client + .update({ + TableName: item.table, + Key: item.query, + ...update + }) + .promise(); + } + + return true; + } + + async findOne(rawItems) { + const items = Array.isArray(rawItems) ? rawItems : [rawItems]; + for (let i = 0; i < items.length; i++) { + let item = items[i]; + item.limit = 1; + } + return this.find(items); + } + + async find(rawItems) { + const items = Array.isArray(rawItems) ? rawItems : [rawItems]; + const results = []; + for (let i = 0; i < items.length; i++) { + let item = items[i]; + + const queryGenerator = new QueryGenerator(); + const queryParams = queryGenerator.generate({ query: item.query, keys: item.keys }); + + results.push( + await this.client() + .query({ + TableName: item.table, + Limit: item.limit, + IndexName: key.primary ? "Index" : key.name, + ...queryParams + }) + .promise() + ); + } + + return results; + } + + async count() { + throw new Error(`Cannot run "count" operation - not supported.`); + } + + setCollectionPrefix(collectionPrefix) { + this.collections.prefix = collectionPrefix; + return this; + } + + getCollectionPrefix() { + return this.collections.prefix; + } + + setCollectionNaming(collectionNameValue) { + this.collections.naming = collectionNameValue; + return this; + } + + getCollectionNaming() { + return this.collections.naming; + } + + getCollectionName(name) { + const getCollectionName = this.getCollectionNaming(); + if (typeof getCollectionName === "function") { + return getCollectionName({ name, driver: this }); + } + + return this.collections.prefix + name; + } + + static __prepareSearchOption(options) { + // Here we handle search (if passed) - we transform received arguments into linked LIKE statements. + if (options.search && options.search.query) { + const { query, operator, fields } = options.search; + + const searches = []; + fields.forEach(field => { + searches.push({ [field]: { $regex: `.*${query}.*`, $options: "i" } }); + }); + + const search = { + [operator === "and" ? "$and" : "$or"]: searches + }; + + if (options.query instanceof Object) { + options.query = { + $and: [search, options.query] + }; + } else { + options.query = search; + } + + delete options.search; + } + } + + static __prepareProjectFields(options) { + // Here we convert requested fields into a "project" parameter + if (options.fields) { + options.project = options.fields.reduce( + (acc, item) => { + acc[item] = 1; + return acc; + }, + { id: 1 } + ); + + delete options.fields; + } + } +} + +export default DynamoDbClient; diff --git a/packages/fields-storage-dynamodb/src/DynamoDbDriver.js b/packages/fields-storage-dynamodb/src/DynamoDbDriver.js new file mode 100644 index 0000000..f5c8206 --- /dev/null +++ b/packages/fields-storage-dynamodb/src/DynamoDbDriver.js @@ -0,0 +1,70 @@ +import DynamoDbClient from "./DynamoDbClient"; +import { getKeys } from "@commodo/fields-storage"; +import { DocumentClient } from "aws-sdk/clients/dynamodb"; + +class DynamoDbDriver { + constructor({ documentClient, tableName } = {}) { + this.documentClient = new DynamoDbClient({ + documentClient: documentClient || new DocumentClient() + }); + + this.tableName = tableName; + } + + getDocumentClient() { + return this.documentClient; + } + + async create({ model }) { + const table = this.getTableName(model); + const data = await model.toStorage(); + return this.getDocumentClient().create({ table, data }); + } + + async update({ model, primaryKey }) { + const table = this.getTableName(model); + const query = {}; + for (let i = 0; i < primaryKey.fields.length; i++) { + let field = primaryKey.fields[i]; + query[field.name] = model[field.name]; + } + + const data = await model.toStorage(); + + return this.getDocumentClient().update({ table, data, query }); + } + + async delete({ model, primaryKey }) { + const table = this.getTableName(model); + const query = {}; + for (let i = 0; i < primaryKey.fields.length; i++) { + let field = primaryKey.fields[i]; + query[field.name] = model[field.name]; + } + + return this.getDocumentClient().delete({ table, query }); + } + + async findOne({ model, args }) { + const table = this.getTableName(model); + return this.getDocumentClient().findOne({ ...args, table, keys: getKeys(model) }); + } + + async find({ model, args }) { + const table = this.getTableName(model); + return this.getDocumentClient().find({ ...args, table, keys: getKeys(model) }); + } + + async count({ model }) { + const table = this.getTableName(model); + + // Will throw an error - counts not supported in DynamoDb. + return this.getDocumentClient().count({ table }); + } + + getTableName(model) { + return this.tableName || model.getStorageName(); + } +} + +module.exports = DynamoDbDriver; diff --git a/packages/fields-storage-dynamodb/src/QueryGenerator.js b/packages/fields-storage-dynamodb/src/QueryGenerator.js new file mode 100644 index 0000000..771071f --- /dev/null +++ b/packages/fields-storage-dynamodb/src/QueryGenerator.js @@ -0,0 +1,42 @@ +import KeyConditionExpression from "./statements/KeyConditionExpression"; + +// const key = findQueryKey(item.query, item.keys); +// const keyConditionExpression = new KeyConditionExpression().process(item.query); + +class QueryGenerator { + generate({ query, keys }) { + // 1. Which key can we use in this query operation? + const key = this.findQueryKey(query, keys); + + // 2. Now that we know the key, let's separate the key attributes from the rest. + const keyAttributesValues = {}, + nonKeyAttributesValues = {}; + for (let queryKey in query) { + if (key.fields.find(item => item.name === queryKey)) { + keyAttributesValues[queryKey] = query[queryKey]; + } else { + nonKeyAttributesValues[queryKey] = query[queryKey]; + } + } + } + + findQueryKey(query = {}, keys = []) { + for (let i = 0; i < keys.length; i++) { + let key = keys[i]; + let hasAllFields = true; + for (let j = 0; j < key.fields.length; j++) { + let field = key.fields[j]; + if (!query[field.name]) { + hasAllFields = false; + break; + } + } + + if (hasAllFields) { + return key; + } + } + } +} + +export default QueryGenerator; diff --git a/packages/fields-storage-dynamodb/src/index.js b/packages/fields-storage-dynamodb/src/index.js new file mode 100644 index 0000000..0745b66 --- /dev/null +++ b/packages/fields-storage-dynamodb/src/index.js @@ -0,0 +1,5 @@ +// @flow +import { default as DynamoDbDriver } from "./DynamoDbDriver"; +import { default as DynamoDbClient } from "./DynamoDbClient"; + +export { DynamoDbDriver, DynamoDbClient }; diff --git a/packages/fields-storage-dynamodb/src/operators/comparison/beginsWith.js b/packages/fields-storage-dynamodb/src/operators/comparison/beginsWith.js new file mode 100644 index 0000000..8fdaf29 --- /dev/null +++ b/packages/fields-storage-dynamodb/src/operators/comparison/beginsWith.js @@ -0,0 +1,18 @@ +const beginsWith = { + canProcess: ({ value }) => { + return value && typeof value["$beginsWith"] !== "undefined"; + }, + process: ({ key, value }) => { + return { + statement: `begins_with (#${key}, :${key})`, + attributeNames: { + [`#${key}`]: key + }, + attributeValues: { + [`:${key}`]: value["$beginsWith"] + } + }; + } +}; + +module.exports = beginsWith; diff --git a/packages/fields-storage-dynamodb/src/operators/comparison/between.js b/packages/fields-storage-dynamodb/src/operators/comparison/between.js new file mode 100644 index 0000000..bd64ee5 --- /dev/null +++ b/packages/fields-storage-dynamodb/src/operators/comparison/between.js @@ -0,0 +1,20 @@ +const between = { + canProcess: ({ value }) => { + return value && typeof value["$between"] !== "undefined"; + }, + process: ({ key, value }) => { + return { + statement: `#${key} BETWEEN :${key}Gte AND :${key}Lte`, + attributeNames: { + [`#${key}`]: key, + }, + attributeValues: { + [`:${key}Gte`]: value[0], + [`:${key}Lte`]: value[1] + } + }; + + } +}; + +module.exports = between; diff --git a/packages/fields-storage-dynamodb/src/operators/comparison/eq.js b/packages/fields-storage-dynamodb/src/operators/comparison/eq.js new file mode 100644 index 0000000..87d545a --- /dev/null +++ b/packages/fields-storage-dynamodb/src/operators/comparison/eq.js @@ -0,0 +1,28 @@ +const validTypes = ["string", "boolean", "number"]; + +const eq = { + canProcess: ({ key, value }) => { + if (key && key.charAt(0) === "$") { + return false; + } + + if (value && typeof value["$eq"] !== "undefined") { + return true; + } + + return validTypes.includes(typeof value); + }, + process: ({ key, value }) => { + return { + expression: `#${key} = :${key}`, + attributeNames: { + [`#${key}`]: key + }, + attributeValues: { + [`:${key}`]: value + } + }; + } +}; + +module.exports = eq; diff --git a/packages/fields-storage-dynamodb/src/operators/comparison/gt.js b/packages/fields-storage-dynamodb/src/operators/comparison/gt.js new file mode 100644 index 0000000..82f7cf3 --- /dev/null +++ b/packages/fields-storage-dynamodb/src/operators/comparison/gt.js @@ -0,0 +1,18 @@ +const gt = { + canProcess: ({ value }) => { + return value && typeof value["$gt"] !== "undefined"; + }, + process: ({ key, value }) => { + return { + expression: `#${key} > :${key}`, + attributeNames: { + [`#${key}`]: key + }, + attributeValues: { + [`:${key}`]: value["$gt"] + } + }; + } +}; + +module.exports = gt; diff --git a/packages/fields-storage-dynamodb/src/operators/comparison/gte.js b/packages/fields-storage-dynamodb/src/operators/comparison/gte.js new file mode 100644 index 0000000..cd80f5b --- /dev/null +++ b/packages/fields-storage-dynamodb/src/operators/comparison/gte.js @@ -0,0 +1,17 @@ +const gte = { + canProcess: ({ value }) => { + return value && typeof value["$gte"] !== "undefined"; + }, + process: ({ key, value }) => { + return { + expression: `#${key} >= :${key}`, + attributeNames: { + [`#${key}`]: key + }, + attributeValues: { + [`:${key}`]: value["$gt"] + } + }; + } +}; +module.exports = gte; diff --git a/packages/fields-storage-dynamodb/src/operators/comparison/lt.js b/packages/fields-storage-dynamodb/src/operators/comparison/lt.js new file mode 100644 index 0000000..700ea14 --- /dev/null +++ b/packages/fields-storage-dynamodb/src/operators/comparison/lt.js @@ -0,0 +1,17 @@ +const lt = { + canProcess: ({ value }) => { + return value && typeof value["$lt"] !== "undefined"; + }, + process: ({ key, value }) => { + return { + expression: `#${key} < :${key}`, + attributeNames: { + [`#${key}`]: key + }, + attributeValues: { + [`:${key}`]: value["$gt"] + } + }; + } +}; +module.exports = lt; diff --git a/packages/fields-storage-dynamodb/src/operators/comparison/lte.js b/packages/fields-storage-dynamodb/src/operators/comparison/lte.js new file mode 100644 index 0000000..0fae983 --- /dev/null +++ b/packages/fields-storage-dynamodb/src/operators/comparison/lte.js @@ -0,0 +1,17 @@ +const lte = { + canProcess: ({ value }) => { + return value && typeof value["$lte"] !== "undefined"; + }, + process: ({ key, value }) => { + return { + expression: `#${key} <= :${key}`, + attributeNames: { + [`#${key}`]: key + }, + attributeValues: { + [`:${key}`]: value["$gt"] + } + }; + } +}; +module.exports = lte; diff --git a/packages/fields-storage-dynamodb/src/operators/index.js b/packages/fields-storage-dynamodb/src/operators/index.js new file mode 100644 index 0000000..1122640 --- /dev/null +++ b/packages/fields-storage-dynamodb/src/operators/index.js @@ -0,0 +1,18 @@ +// Comparison operators (A-Z) +const $beginsWith = require("./comparison/beginsWith"); +const $between = require("./comparison/between"); +const $gt = require("./comparison/gt"); +const $gte = require("./comparison/gte"); +const $lt = require("./comparison/lt"); +const $lte = require("./comparison/lte"); +const $eq = require("./comparison/eq"); + +module.exports = { + $beginsWith, + $between, + $eq, + $gt, + $gte, + $lt, + $lte +}; diff --git a/packages/fields-storage-dynamodb/src/statements/KeyConditionExpression.js b/packages/fields-storage-dynamodb/src/statements/KeyConditionExpression.js new file mode 100644 index 0000000..977609c --- /dev/null +++ b/packages/fields-storage-dynamodb/src/statements/KeyConditionExpression.js @@ -0,0 +1,23 @@ +const allOperators = require("./../operators"); + +class KeyConditionExpression { + process(payload) { + let output = []; + + outerLoop: for (const [key, value] of Object.entries(payload)) { + const operators = Object.values(allOperators); + for (let i = 0; i < operators.length; i++) { + const operator = operators[i]; + if (operator.canProcess({ key, value })) { + output.push(operator.process({ key, value })); + continue outerLoop; + } + } + throw new Error(`Invalid operator {${key} : ${value}}.`); + } + + return output; + } +} + +module.exports = KeyConditionExpression; From 3777ef22c61da752404142b9d6c8e0a3dd058fa4 Mon Sep 17 00:00:00 2001 From: Adrian Smijulj Date: Mon, 24 Aug 2020 10:06:55 +0200 Subject: [PATCH 02/26] feat: remove `id` and introduce keys --- packages/fields-storage/package.json | 3 +- packages/fields-storage/src/getKeys.js | 13 ++ packages/fields-storage/src/getPrimaryKey.js | 7 + packages/fields-storage/src/idGenerator.js | 5 - packages/fields-storage/src/index.js | 4 + packages/fields-storage/src/withKey.js | 28 +++ packages/fields-storage/src/withPrimaryKey.js | 19 ++ packages/fields-storage/src/withStorage.js | 191 ++++++++++-------- packages/fields-storage/src/withUniqueKey.js | 18 ++ 9 files changed, 198 insertions(+), 90 deletions(-) create mode 100644 packages/fields-storage/src/getKeys.js create mode 100644 packages/fields-storage/src/getPrimaryKey.js delete mode 100644 packages/fields-storage/src/idGenerator.js create mode 100644 packages/fields-storage/src/withKey.js create mode 100644 packages/fields-storage/src/withPrimaryKey.js create mode 100644 packages/fields-storage/src/withUniqueKey.js diff --git a/packages/fields-storage/package.json b/packages/fields-storage/package.json index af7aa3e..5f0d218 100644 --- a/packages/fields-storage/package.json +++ b/packages/fields-storage/package.json @@ -35,6 +35,7 @@ "storage" ], "scripts": { - "build": "babel src --ignore src/__tests__ --out-dir dist --source-maps" + "build": "babel src --ignore src/__tests__ --out-dir dist --source-maps", + "watch": "yarn build --watch" } } diff --git a/packages/fields-storage/src/getKeys.js b/packages/fields-storage/src/getKeys.js new file mode 100644 index 0000000..a2baaa3 --- /dev/null +++ b/packages/fields-storage/src/getKeys.js @@ -0,0 +1,13 @@ +const getPrimaryKey = model => { + if (Array.isArray(model.__storageKeys)) { + return model.__storageKeys; + } + + if (model.constructor && Array.isArray(model.constructor.__storageKeys)) { + return model.constructor.__storageKeys; + } + + return []; +}; + +export default getPrimaryKey; diff --git a/packages/fields-storage/src/getPrimaryKey.js b/packages/fields-storage/src/getPrimaryKey.js new file mode 100644 index 0000000..d34b15b --- /dev/null +++ b/packages/fields-storage/src/getPrimaryKey.js @@ -0,0 +1,7 @@ +import getKeys from "./getKeys"; + +const getPrimaryKey = model => { + return getKeys(model).find(item => item.primary); +}; + +export default getPrimaryKey; diff --git a/packages/fields-storage/src/idGenerator.js b/packages/fields-storage/src/idGenerator.js deleted file mode 100644 index a789aaf..0000000 --- a/packages/fields-storage/src/idGenerator.js +++ /dev/null @@ -1,5 +0,0 @@ -import mdbid from "mdbid"; - -export default { - generate: mdbid -} diff --git a/packages/fields-storage/src/index.js b/packages/fields-storage/src/index.js index cca2b51..3f88632 100644 --- a/packages/fields-storage/src/index.js +++ b/packages/fields-storage/src/index.js @@ -8,3 +8,7 @@ export { default as hasWithStorage } from "./hasWithStorage"; export { default as withStorageName } from "./withStorageName"; export { default as getStorageName } from "./getStorageName"; export { default as hasStorageName } from "./hasStorageName"; +export { default as withPrimaryKey } from "./withPrimaryKey"; +export { default as withKey } from "./withKey"; +export { default as withUniqueKey } from "./withUniqueKey"; +export { default as getKeys } from "./getKeys"; diff --git a/packages/fields-storage/src/withKey.js b/packages/fields-storage/src/withKey.js new file mode 100644 index 0000000..36b6d18 --- /dev/null +++ b/packages/fields-storage/src/withKey.js @@ -0,0 +1,28 @@ +import { withStaticProps } from "repropose"; + +const withKey = (...params) => { + let newKey; + if (typeof params[0] === "string") { + newKey = { + fields: params.map(item => ({ name: item })) + }; + } else { + newKey = params[0]; + } + + if (!newKey.name) { + newKey.name = newKey.fields.map(item => item.name).join("_"); + } + + return function(fn) { + if (!fn.__storageKeys) { + withStaticProps({ __storageKeys: [newKey] })(fn); + } else { + fn.__storageKeys.push(newKey); + } + + return fn; + }; +}; + +export default withKey; diff --git a/packages/fields-storage/src/withPrimaryKey.js b/packages/fields-storage/src/withPrimaryKey.js new file mode 100644 index 0000000..0e6bfb7 --- /dev/null +++ b/packages/fields-storage/src/withPrimaryKey.js @@ -0,0 +1,19 @@ +import withKey from "./withKey"; + +const withPrimaryKey = (...params) => { + let key; + if (typeof params[0] === "string") { + key = { + fields: params.map(item => ({ name: item })) + }; + } else { + key = params[0]; + } + + key.primary = true; + key.unique = true; + + return withKey(key); +}; + +export default withPrimaryKey; diff --git a/packages/fields-storage/src/withStorage.js b/packages/fields-storage/src/withStorage.js index c044ade..f139caa 100644 --- a/packages/fields-storage/src/withStorage.js +++ b/packages/fields-storage/src/withStorage.js @@ -1,6 +1,9 @@ // @flow import { getName as defaultGetName } from "@commodo/name"; import getStorageName from "./getStorageName"; +import getKeys from "./getKeys"; +import getPrimaryKey from "./getPrimaryKey"; +import findQueryKey from "./findQueryKey"; import { withStaticProps, withProps } from "repropose"; import cloneDeep from "lodash.clonedeep"; import { withHooks } from "@commodo/hooks"; @@ -10,7 +13,6 @@ import Collection from "./Collection"; import StoragePool from "./StoragePool"; import FieldsStorageAdapter from "./FieldsStorageAdapter"; import { decodeCursor, encodeCursor } from "./cursor"; -import idGenerator from "./idGenerator"; interface IStorageDriver {} type Configuration = { @@ -28,8 +30,6 @@ const defaults = { } }; -const generateId = () => idGenerator.generate(); - const hook = async (name, { options, model }) => { if (options.hooks[name] === false) { return; @@ -46,7 +46,7 @@ const registerSaveUpdateCreateHooks = async (prefix, { existing, model, options } }; -const getName = (instance) => { +const getName = instance => { return getStorageName(instance) || defaultGetName(instance); }; @@ -95,10 +95,10 @@ const withStorage = (configuration: Configuration) => { processing: false, fieldsStorageAdapter: new FieldsStorageAdapter() }, - generateId, - isId(value) { - return typeof value === "string" && !!value.match(/^[a-zA-Z0-9]*$/); - }, + // generateId, + // isId(value) { + // return typeof value === "string" && !!value.match(/^[a-zA-Z0-9]*$/); + // }, isExisting() { return this.__withStorage.existing; }, @@ -107,6 +107,13 @@ const withStorage = (configuration: Configuration) => { return this; }, async save(options: ?SaveParams): Promise { + const primaryKey = getPrimaryKey(this); + if (!primaryKey) { + throw Error( + `Cannot save "${this.getStorageName()}" model, no primary key defined.` + ); + } + options = { ...options, ...defaults.save }; if (this.__withStorage.processing) { @@ -136,27 +143,23 @@ const withStorage = (configuration: Configuration) => { }); if (this.isDirty()) { - if (!this.id) { - this.id = this.constructor.generateId(); - } + // if (!this.id) { + // this.id = this.constructor.generateId(); + // } if (existing) { - const { getId } = options; - await this.getStorageDriver().update([ - { - name: getName(this), - query: - typeof getId === "function" ? getId(this) : { id: this.id }, - data: await this.toStorage() - } - ]); + // const { getId } = options; + // await this.getStorageDriver().update(model[ + // { + // name: getName(this), + // query: + // typeof getId === "function" ? getId(this) : { id: this.id }, + // data: await this.toStorage() + // } + // ]); + await this.getStorageDriver().update({ model: this, primaryKey }); } else { - await this.getStorageDriver().create([ - { - name: getName(this), - data: await this.toStorage() - } - ]); + await this.getStorageDriver().create({ model: this, primaryKey }); } } @@ -172,7 +175,7 @@ const withStorage = (configuration: Configuration) => { this.constructor.getStoragePool().add(this); } catch (e) { if (!existing) { - this.getField("id").reset(); + // this.getField("id").reset(); } throw e; } finally { @@ -187,6 +190,13 @@ const withStorage = (configuration: Configuration) => { * @param options */ async delete(options: ?Object) { + const primaryKey = getPrimaryKey(this); + if (!primaryKey) { + throw Error( + `Cannot delete "${this.getStorageName()}" model, no primary key defined.` + ); + } + if (this.__withStorage.processing) { return; } @@ -226,6 +236,10 @@ const withStorage = (configuration: Configuration) => { return this.constructor.__withStorage.driver; }, + getStorageName() { + return getName(this); + }, + async populateFromStorage(data: Object) { await this.__withStorage.fieldsStorageAdapter.fromStorage({ data, @@ -276,10 +290,73 @@ const withStorage = (configuration: Configuration) => { getStorageDriver() { return this.__withStorage.driver; }, - isId(value) { - return typeof value === "string" && !!value.match(/^[0-9a-fA-F]{24}$/); + getStorageName() { + return getName(this); }, - generateId, + + /** + * Finds a single model matched by given ID. + * @param id + * @param options + */ + // async findById(id: mixed, options: ?Object): Promise { + // if (!id || !this.isId(id)) { + // return null; + // } + // + // const pooled = this.getStoragePool().get(this, id); + // if (pooled) { + // return pooled; + // } + // + // if (!options) { + // options = {}; + // } + // + // const newParams = { ...options, query: { id } }; + // return await this.findOne(newParams); + // }, + + /** + * Finds one model matched by given query parameters. + * @param rawArgs + */ + async findOne(rawArgs: ?Object): Promise> { + if (!rawArgs) { + rawArgs = {}; + } + + const args = cloneDeep(rawArgs); + // + // { + // name: getName(this), + // options: prepared + // } + // + const result = await this.getStorageDriver().findOne({ + model: this, + args + }); + + return result; + if (result) { + const pooled = this.getStoragePool().get(this, result.id); + if (pooled) { + return pooled; + } + + const model: $Subtype = new this(); + model.setExisting(); + await model.populateFromStorage(((result: any): Object)); + this.getStoragePool().add(model); + return model; + } + return null; + }, + // isId(value) { + // return typeof value === "string" && !!value.match(/^[0-9a-fA-F]{24}$/); + // }, + // generateId, async find(options: ?FindParams) { if (!options) { options = {}; @@ -429,60 +506,6 @@ const withStorage = (configuration: Configuration) => { return collection; }, - /** - * Finds a single model matched by given ID. - * @param id - * @param options - */ - async findById(id: mixed, options: ?Object): Promise { - if (!id || !this.isId(id)) { - return null; - } - - const pooled = this.getStoragePool().get(this, id); - if (pooled) { - return pooled; - } - - if (!options) { - options = {}; - } - - const newParams = { ...options, query: { id } }; - return await this.findOne(newParams); - }, - - /** - * Finds one model matched by given query parameters. - * @param options - */ - async findOne(options: ?Object): Promise> { - if (!options) { - options = {}; - } - - const prepared = { ...options }; - - const result = await this.getStorageDriver().findOne({ - name: getName(this), - options: prepared - }); - - if (result) { - const pooled = this.getStoragePool().get(this, result.id); - if (pooled) { - return pooled; - } - - const model: $Subtype = new this(); - model.setExisting(); - await model.populateFromStorage(((result: any): Object)); - this.getStoragePool().add(model); - return model; - } - return null; - }, - /** * Counts total number of models matched by given query parameters. * @param options diff --git a/packages/fields-storage/src/withUniqueKey.js b/packages/fields-storage/src/withUniqueKey.js new file mode 100644 index 0000000..b459eef --- /dev/null +++ b/packages/fields-storage/src/withUniqueKey.js @@ -0,0 +1,18 @@ +import withKey from "./withKey"; + +const withUniqueKey = (...params) => { + let key; + if (typeof params[0] === "string") { + key = { + fields: params.map(item => ({ name: item })) + }; + } else { + key = params[0]; + } + + key.unique = true; + + return withKey(key); +}; + +export default withUniqueKey; From 8f4d1e1baf7abaeaa368b0db8a98f92b8fa8bf61 Mon Sep 17 00:00:00 2001 From: Adrian Smijulj Date: Mon, 24 Aug 2020 10:07:11 +0200 Subject: [PATCH 03/26] chore: add missing watch command --- packages/fields/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/fields/package.json b/packages/fields/package.json index d538431..bbbab10 100644 --- a/packages/fields/package.json +++ b/packages/fields/package.json @@ -27,6 +27,7 @@ "fields" ], "scripts": { - "build": "babel src --ignore src/__tests__ --out-dir dist --source-maps" + "build": "babel src --ignore src/__tests__ --out-dir dist --source-maps", + "watch": "yarn build --watch" } } From c453ee4642e3779ce83cde6852147fdb1dc55048 Mon Sep 17 00:00:00 2001 From: Adrian Smijulj Date: Mon, 24 Aug 2020 10:07:43 +0200 Subject: [PATCH 04/26] chore: add config for `@shelf/jest-dynamodb` --- jest-dynamodb-config.js | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 jest-dynamodb-config.js diff --git a/jest-dynamodb-config.js b/jest-dynamodb-config.js new file mode 100644 index 0000000..ae32d3e --- /dev/null +++ b/jest-dynamodb-config.js @@ -0,0 +1,16 @@ +module.exports = { + tables: [ + { + TableName: `pk-sk`, + KeySchema: [ + { AttributeName: "pk", KeyType: "HASH" }, + { AttributeName: "sk", KeyType: "RANGE" } + ], + AttributeDefinitions: [ + { AttributeName: "pk", AttributeType: "S" }, + { AttributeName: "sk", AttributeType: "S" } + ], + ProvisionedThroughput: { ReadCapacityUnits: 1, WriteCapacityUnits: 1 } + } + ] +}; From 09d9be61a320b7c135738fc6fc58c4116adf4fe4 Mon Sep 17 00:00:00 2001 From: Adrian Smijulj Date: Sun, 30 Aug 2020 20:30:48 +0200 Subject: [PATCH 05/26] fix: if model not provided, return an empty array --- packages/fields-storage/src/getKeys.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/fields-storage/src/getKeys.js b/packages/fields-storage/src/getKeys.js index a2baaa3..45433ad 100644 --- a/packages/fields-storage/src/getKeys.js +++ b/packages/fields-storage/src/getKeys.js @@ -1,4 +1,8 @@ const getPrimaryKey = model => { + if (!model) { + return []; + } + if (Array.isArray(model.__storageKeys)) { return model.__storageKeys; } From 75b5371935606962ed46676df5b930ea1dcfa3f3 Mon Sep 17 00:00:00 2001 From: Adrian Smijulj Date: Sun, 30 Aug 2020 20:31:20 +0200 Subject: [PATCH 06/26] chore: remove unused imports --- packages/fields-storage/src/withStorage.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/fields-storage/src/withStorage.js b/packages/fields-storage/src/withStorage.js index f139caa..04b338c 100644 --- a/packages/fields-storage/src/withStorage.js +++ b/packages/fields-storage/src/withStorage.js @@ -1,9 +1,7 @@ // @flow import { getName as defaultGetName } from "@commodo/name"; import getStorageName from "./getStorageName"; -import getKeys from "./getKeys"; import getPrimaryKey from "./getPrimaryKey"; -import findQueryKey from "./findQueryKey"; import { withStaticProps, withProps } from "repropose"; import cloneDeep from "lodash.clonedeep"; import { withHooks } from "@commodo/hooks"; From 9f4efafaa4ec2ee9a696754590453af80f143969 Mon Sep 17 00:00:00 2001 From: Adrian Smijulj Date: Sun, 30 Aug 2020 20:31:46 +0200 Subject: [PATCH 07/26] wip: adjust sent driver args --- packages/fields-storage/src/withStorage.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/fields-storage/src/withStorage.js b/packages/fields-storage/src/withStorage.js index 04b338c..d3d212c 100644 --- a/packages/fields-storage/src/withStorage.js +++ b/packages/fields-storage/src/withStorage.js @@ -438,7 +438,8 @@ const withStorage = (configuration: Configuration) => { const params = { query, sort, limit: limit + 1, ...other }; let [results, meta] = await this.getStorageDriver().find({ name: getName(this), - options: params + args: params, + model: this }); // Have we reached the last record? @@ -454,11 +455,12 @@ const withStorage = (configuration: Configuration) => { let totalCount = null; if (countTotal) { totalCount = await this.getStorageDriver().count({ + model: this, name: getName(this), - options: { + args: { query: originalQuery, ...other - } + }, }); } From d6444536d89c094f43cd7c15cf28a6b8587f48af Mon Sep 17 00:00:00 2001 From: Adrian Smijulj Date: Sun, 6 Sep 2020 19:55:09 +0200 Subject: [PATCH 08/26] chore: remove redundant files --- .../OPERATORSsrc/index.js | 3 - .../model/attributes/arrayAttribute.js | 20 -- .../model/attributes/booleanAttribute.js | 35 -- .../model/attributes/dateAttribute.js | 27 -- .../OPERATORSsrc/model/attributes/index.js | 7 - .../model/attributes/modelAttribute.js | 18 - .../model/attributes/modelsAttribute.js | 17 - .../model/attributes/objectAttribute.js | 20 -- .../OPERATORSsrc/model/index.js | 2 - .../model/mysqlAttributesContainer.js | 60 ---- .../OPERATORSsrc/model/mysqlModel.js | 11 - .../OPERATORSsrc/mysqlDriver.js | 325 ------------------ .../OPERATORSsrc/operators/comparison/all.js | 22 -- .../OPERATORSsrc/operators/comparison/eq.js | 58 ---- .../OPERATORSsrc/operators/comparison/gt.js | 13 - .../OPERATORSsrc/operators/comparison/gte.js | 13 - .../OPERATORSsrc/operators/comparison/in.js | 39 --- .../comparison/jsonArrayFindValue.js | 14 - .../comparison/jsonArrayStrictEquality.js | 14 - .../OPERATORSsrc/operators/comparison/like.js | 18 - .../OPERATORSsrc/operators/comparison/lt.js | 13 - .../OPERATORSsrc/operators/comparison/lte.js | 13 - .../OPERATORSsrc/operators/comparison/ne.js | 17 - .../operators/comparison/search.js | 22 -- .../OPERATORSsrc/operators/index.js | 37 -- .../OPERATORSsrc/operators/logical/and.js | 40 --- .../OPERATORSsrc/operators/logical/or.js | 40 --- .../OPERATORSsrc/statements/delete.js | 17 - .../OPERATORSsrc/statements/index.js | 7 - .../OPERATORSsrc/statements/insert.js | 32 -- .../OPERATORSsrc/statements/select.js | 23 -- .../OPERATORSsrc/statements/statement.js | 111 ------ .../OPERATORSsrc/statements/update.js | 22 -- 33 files changed, 1130 deletions(-) delete mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/index.js delete mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/arrayAttribute.js delete mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/booleanAttribute.js delete mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/dateAttribute.js delete mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/index.js delete mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/modelAttribute.js delete mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/modelsAttribute.js delete mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/objectAttribute.js delete mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/model/index.js delete mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/model/mysqlAttributesContainer.js delete mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/model/mysqlModel.js delete mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/mysqlDriver.js delete mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/all.js delete mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/eq.js delete mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/gt.js delete mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/gte.js delete mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/in.js delete mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/jsonArrayFindValue.js delete mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/jsonArrayStrictEquality.js delete mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/like.js delete mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/lt.js delete mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/lte.js delete mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/ne.js delete mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/search.js delete mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/operators/index.js delete mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/operators/logical/and.js delete mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/operators/logical/or.js delete mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/statements/delete.js delete mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/statements/index.js delete mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/statements/insert.js delete mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/statements/select.js delete mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/statements/statement.js delete mode 100644 packages/fields-storage-dynamodb/OPERATORSsrc/statements/update.js diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/index.js b/packages/fields-storage-dynamodb/OPERATORSsrc/index.js deleted file mode 100644 index ed2f1ee..0000000 --- a/packages/fields-storage-dynamodb/OPERATORSsrc/index.js +++ /dev/null @@ -1,3 +0,0 @@ -// @flow -export { default as MySQLDriver } from "./mysqlDriver"; -export { default as operators } from "./operators"; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/arrayAttribute.js b/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/arrayAttribute.js deleted file mode 100644 index b069dbf..0000000 --- a/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/arrayAttribute.js +++ /dev/null @@ -1,20 +0,0 @@ -// @flow -import { ArrayAttribute as BaseArrayAttribute } from "webiny-model"; - -class ArrayAttribute extends BaseArrayAttribute { - setStorageValue(value: mixed) { - if (typeof value === "string") { - super.setStorageValue(JSON.parse(value)); - } else { - super.setStorageValue(value); - } - return this; - } - - async getStorageValue() { - const value = await BaseArrayAttribute.prototype.getStorageValue.call(this); - return value ? JSON.stringify(value) : value; - } -} - -export default ArrayAttribute; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/booleanAttribute.js b/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/booleanAttribute.js deleted file mode 100644 index bfc17e8..0000000 --- a/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/booleanAttribute.js +++ /dev/null @@ -1,35 +0,0 @@ -// @flow -import { BooleanAttribute as BaseBooleanAttribute } from "webiny-model"; - -class BooleanAttribute extends BaseBooleanAttribute { - /** - * We must make sure a boolean value is sent, and not 0 or 1, which are stored in MySQL. - * @param value - */ - setStorageValue(value: mixed) { - if (value === 1) { - return super.setStorageValue(true); - } - - if (value === 0) { - return super.setStorageValue(false); - } - - return super.setStorageValue(value); - } - - async getStorageValue() { - const value = await BaseBooleanAttribute.prototype.getStorageValue.call(this); - if (value === true) { - return 1; - } - - if (value === false) { - return 0; - } - - return value; - } -} - -export default BooleanAttribute; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/dateAttribute.js b/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/dateAttribute.js deleted file mode 100644 index 292d6d5..0000000 --- a/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/dateAttribute.js +++ /dev/null @@ -1,27 +0,0 @@ -// @flow -import { DateAttribute as BaseDateAttribute } from "webiny-model"; -import fecha from "fecha"; - -class DateAttribute extends BaseDateAttribute { - setStorageValue(value: mixed) { - if (value === null) { - return super.setStorageValue(value); - } - - if (value instanceof Date) { - return super.setStorageValue(value); - } - - return super.setStorageValue(fecha.parse(value, "YYYY-MM-DD HH:mm:ss")); - } - - async getStorageValue() { - const value = await BaseDateAttribute.prototype.getStorageValue.call(this); - if (value instanceof Date) { - return fecha.format(value, "YYYY-MM-DD HH:mm:ss"); - } - return value; - } -} - -export default DateAttribute; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/index.js b/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/index.js deleted file mode 100644 index 153f953..0000000 --- a/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/index.js +++ /dev/null @@ -1,7 +0,0 @@ -// @flow -export { default as BooleanAttribute } from "./booleanAttribute"; -export { default as DateAttribute } from "./dateAttribute"; -export { default as ArrayAttribute } from "./arrayAttribute"; -export { default as ObjectAttribute } from "./objectAttribute"; -export { default as ModelAttribute } from "./modelAttribute"; -export { default as ModelsAttribute } from "./modelsAttribute"; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/modelAttribute.js b/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/modelAttribute.js deleted file mode 100644 index 2ca6e86..0000000 --- a/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/modelAttribute.js +++ /dev/null @@ -1,18 +0,0 @@ -// @flow -import { ModelAttribute as BaseModelAttribute } from "webiny-model"; - -class ModelAttribute extends BaseModelAttribute { - setStorageValue(value: mixed): this { - if (typeof value === "string") { - return super.setStorageValue(JSON.parse(value)); - } - return this; - } - - async getStorageValue() { - const value = await BaseModelAttribute.prototype.getStorageValue.call(this); - return value ? JSON.stringify(value) : value; - } -} - -export default ModelAttribute; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/modelsAttribute.js b/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/modelsAttribute.js deleted file mode 100644 index 2fdc1dd..0000000 --- a/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/modelsAttribute.js +++ /dev/null @@ -1,17 +0,0 @@ -// @flow -import { ModelsAttribute as BaseModelsAttribute } from "webiny-model"; - -class ModelsAttribute extends BaseModelsAttribute { - setStorageValue(value: mixed): this { - if (typeof value === "string") { - super.setStorageValue(JSON.parse(value)); - } - return this; - } - - async getStorageValue(): Promise { - return JSON.stringify(await BaseModelsAttribute.prototype.getStorageValue.call(this)); - } -} - -export default ModelsAttribute; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/objectAttribute.js b/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/objectAttribute.js deleted file mode 100644 index 5298a8f..0000000 --- a/packages/fields-storage-dynamodb/OPERATORSsrc/model/attributes/objectAttribute.js +++ /dev/null @@ -1,20 +0,0 @@ -// @flow -import { ObjectAttribute as BaseObjectAttribute } from "webiny-model"; - -class ObjectAttribute extends BaseObjectAttribute { - setStorageValue(value: mixed) { - if (typeof value === "string") { - super.setStorageValue(JSON.parse(value)); - } else { - super.setStorageValue(value); - } - return this; - } - - async getStorageValue() { - const value = await BaseObjectAttribute.prototype.getStorageValue.call(this); - return value ? JSON.stringify(value) : value; - } -} - -export default ObjectAttribute; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/model/index.js b/packages/fields-storage-dynamodb/OPERATORSsrc/model/index.js deleted file mode 100644 index 498fbc8..0000000 --- a/packages/fields-storage-dynamodb/OPERATORSsrc/model/index.js +++ /dev/null @@ -1,2 +0,0 @@ -// @flow -export { default as MySQLModel } from "./mysqlModel"; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/model/mysqlAttributesContainer.js b/packages/fields-storage-dynamodb/OPERATORSsrc/model/mysqlAttributesContainer.js deleted file mode 100644 index 3381443..0000000 --- a/packages/fields-storage-dynamodb/OPERATORSsrc/model/mysqlAttributesContainer.js +++ /dev/null @@ -1,60 +0,0 @@ -// @flow -import type { Model } from "webiny-model"; -import type { - ModelAttribute as BaseModelAttribute, - ModelsAttribute as BaseModelsAttribute -} from "webiny-entity"; - -import { EntityAttributesContainer } from "webiny-entity"; -import { - ArrayAttribute, - BooleanAttribute, - DateAttribute, - ModelAttribute, - ModelsAttribute, - ObjectAttribute -} from "./attributes"; - -/** - * Contains basic attributes. If needed, this class can be extended to add additional attributes, - * and then be set as a new attributes container as the default one. - */ -class MySQLAttributesContainer extends EntityAttributesContainer { - boolean(): BooleanAttribute { - const model = this.getParentModel(); - model.setAttribute(this.name, new BooleanAttribute(this.name, this)); - return ((model.getAttribute(this.name): any): BooleanAttribute); - } - - date(): DateAttribute { - const model = this.getParentModel(); - model.setAttribute(this.name, new DateAttribute(this.name, this)); - return ((model.getAttribute(this.name): any): DateAttribute); - } - - array(): ArrayAttribute { - const model = this.getParentModel(); - model.setAttribute(this.name, new ArrayAttribute(this.name, this)); - return ((model.getAttribute(this.name): any): ArrayAttribute); - } - - object(): ObjectAttribute { - const model = this.getParentModel(); - model.setAttribute(this.name, new ObjectAttribute(this.name, this)); - return ((model.getAttribute(this.name): any): ObjectAttribute); - } - - model(model: Class): BaseModelAttribute & ModelAttribute { - const parent = this.getParentModel(); - parent.setAttribute(this.name, new ModelAttribute(this.name, this, model)); - return ((parent.getAttribute(this.name): any): BaseModelAttribute & ModelAttribute); - } - - models(model: Class): BaseModelsAttribute & ModelsAttribute { - const parent = this.getParentModel(); - parent.setAttribute(this.name, new ModelsAttribute(this.name, this, model)); - return ((parent.getAttribute(this.name): any): BaseModelsAttribute & ModelsAttribute); - } -} - -export default MySQLAttributesContainer; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/model/mysqlModel.js b/packages/fields-storage-dynamodb/OPERATORSsrc/model/mysqlModel.js deleted file mode 100644 index 9a273ae..0000000 --- a/packages/fields-storage-dynamodb/OPERATORSsrc/model/mysqlModel.js +++ /dev/null @@ -1,11 +0,0 @@ -// @flow -import { EntityModel } from "webiny-entity"; -import MySQLAttributesContainer from "./mysqlAttributesContainer"; - -class MySQLModel extends EntityModel { - createAttributesContainer(): MySQLAttributesContainer { - return new MySQLAttributesContainer(this); - } -} - -export default MySQLModel; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/mysqlDriver.js b/packages/fields-storage-dynamodb/OPERATORSsrc/mysqlDriver.js deleted file mode 100644 index 3e795bf..0000000 --- a/packages/fields-storage-dynamodb/OPERATORSsrc/mysqlDriver.js +++ /dev/null @@ -1,325 +0,0 @@ -// @flow -import _ from "lodash"; -import mdbid from "mdbid"; -import type { Connection, Pool } from "mysql"; -import { Entity, Driver, QueryResult } from "webiny-entity"; -import { MySQLConnection } from "webiny-mysql-connection"; -import { Attribute } from "webiny-model"; -import type { - EntitySaveParams, - EntityFindParams, - EntityDeleteParams, - EntityFindOneParams -} from "webiny-entity/types"; -import type { Operator } from "./../types"; - -import { Insert, Update, Delete, Select } from "./statements"; -import { MySQLModel } from "./model"; -import operators from "./operators"; - -declare type MySQLDriverOptions = { - connection: Connection | Pool, - model?: Class, - operators?: { [string]: Operator }, - tables?: { - prefix: string, - naming: ?Function - }, - autoIncrementIds?: boolean -}; - -class MySQLDriver extends Driver { - connection: MySQLConnection; - model: Class; - operators: { [string]: Operator }; - tables: { - prefix: string, - naming: ?Function - }; - autoIncrementIds: boolean; - constructor(options: MySQLDriverOptions) { - super(); - this.operators = { ...operators, ...(options.operators || {}) }; - this.connection = new MySQLConnection(options.connection); - this.model = options.model || MySQLModel; - - this.tables = { - prefix: "", - ...(options.tables || {}) - }; - this.autoIncrementIds = options.autoIncrementIds || false; - } - - setOperator(name: string, operator: Operator) { - this.operators[name] = operator; - return this; - } - - onEntityConstruct(entity: Entity) { - if (this.autoIncrementIds) { - entity - .attr("id") - .integer() - .setValidators((value, attribute) => - this.isId(attribute.getParentModel().getParentEntity(), value) - ); - } else { - entity - .attr("id") - .char() - .setValidators((value, attribute) => - this.isId(attribute.getParentModel().getParentEntity(), value) - ); - } - } - - getModelClass(): Class { - return this.model; - } - - // eslint-disable-next-line - async save(entity: Entity, options: EntitySaveParams & {}): Promise { - if (entity.isExisting()) { - const data = await entity.toStorage(); - if (_.isEmpty(data)) { - return new QueryResult(true); - } - - const sql = new Update( - { - operators: this.operators, - table: this.getTableName(entity), - data, - where: { id: entity.id }, - limit: 1 - }, - entity - ).generate(); - - await this.getConnection().query(sql); - return new QueryResult(true); - } - - if (!this.autoIncrementIds) { - entity.id = MySQLDriver.__generateID(); - } - - const data = await entity.toStorage(); - const sql = new Insert( - { - operators: this.operators, - data, - table: this.getTableName(entity) - }, - entity - ).generate(); - - try { - const results = await this.getConnection().query(sql); - if (this.autoIncrementIds) { - entity.id = results.insertId; - } - } catch (e) { - const idAttribute: Attribute = (entity.getAttribute("id"): any); - idAttribute.reset(); - throw e; - } - - return new QueryResult(true); - } - - // eslint-disable-next-line - async delete(entity: Entity, options: EntityDeleteParams & {}): Promise { - const sql = new Delete( - { - operators: this.operators, - table: this.getTableName(entity), - where: { id: entity.id }, - limit: 1 - }, - entity - ).generate(); - - await this.getConnection().query(sql); - return new QueryResult(true); - } - - async find( - entity: Entity | Class, - options: EntityFindParams & {} - ): Promise { - const clonedOptions = _.merge({}, options, { - operators: this.operators, - table: this.getTableName(entity), - operation: "select", - limit: 10, - offset: 0 - }); - - MySQLDriver.__preparePerPageOption(clonedOptions); - MySQLDriver.__preparePageOption(clonedOptions); - MySQLDriver.__prepareQueryOption(clonedOptions); - MySQLDriver.__prepareSearchOption(clonedOptions); - - clonedOptions.calculateFoundRows = true; - - const sql = new Select(clonedOptions, entity).generate(); - const results = await this.getConnection().query([sql, "SELECT FOUND_ROWS() as count"]); - - return new QueryResult(results[0], { totalCount: results[1][0].count }); - } - - async findOne( - entity: Entity | Class, - options: EntityFindOneParams & {} - ): Promise { - const clonedOptions = { - operators: this.operators, - table: this.getTableName(entity), - where: options.query, - search: options.search, - limit: 1 - }; - - MySQLDriver.__prepareQueryOption(clonedOptions); - MySQLDriver.__prepareSearchOption(clonedOptions); - - const sql = new Select(clonedOptions, entity).generate(); - - const results = await this.getConnection().query(sql); - return new QueryResult(results[0]); - } - - async count( - entity: Entity | Class, - options: EntityFindParams & {} - ): Promise { - const clonedOptions = _.merge( - {}, - options, - { - operators: this.operators, - table: this.getTableName(entity), - columns: ["COUNT(*) AS count"] - }, - entity - ); - - MySQLDriver.__prepareQueryOption(clonedOptions); - MySQLDriver.__prepareSearchOption(clonedOptions); - - const sql = new Select(clonedOptions, entity).generate(); - - const results = await this.getConnection().query(sql); - return new QueryResult(results[0].count); - } - - // eslint-disable-next-line - isId(entity: Entity | Class, value: mixed, options: ?Object): boolean { - if (this.autoIncrementIds) { - return typeof value === "number" && Number.isInteger(value) && value > 0; - } - - if (typeof value === "string") { - return value.match(new RegExp("^[0-9a-fA-F]{24}$")) !== null; - } - return false; - } - - getConnection(): MySQLConnection { - return this.connection; - } - - setTablePrefix(tablePrefix: string): this { - this.tables.prefix = tablePrefix; - return this; - } - - getTablePrefix(): string { - return this.tables.prefix; - } - - setTableNaming(tableNameValue: Function): this { - this.tables.naming = tableNameValue; - return this; - } - - getTableNaming(): ?Function { - return this.tables.naming; - } - - getTableName(entity: Entity | Class): string { - const params = { - classId: _.get(entity, "constructor.classId", _.get(entity, "classId")), - storageClassId: _.get( - entity, - "constructor.storageClassId", - _.get(entity, "storageClassId") - ), - tableName: _.get(entity, "constructor.tableName", _.get(entity, "tableName")) - }; - - const getTableName = this.getTableNaming(); - if (typeof getTableName === "function") { - return getTableName({ entity, ...params, driver: this }); - } - - if (params.tableName) { - return this.tables.prefix + params.tableName; - } - - return ( - this.tables.prefix + (params.storageClassId ? params.storageClassId : params.classId) - ); - } - - async test() { - await this.getConnection().test(); - return true; - } - - static __preparePerPageOption(clonedOptions: Object): void { - if ("perPage" in clonedOptions) { - clonedOptions.limit = clonedOptions.perPage; - delete clonedOptions.perPage; - } - } - - static __preparePageOption(clonedOptions: Object): void { - if ("page" in clonedOptions) { - clonedOptions.offset = clonedOptions.limit * (clonedOptions.page - 1); - delete clonedOptions.page; - } - } - - static __prepareQueryOption(clonedOptions: Object): void { - if (clonedOptions.query instanceof Object) { - clonedOptions.where = clonedOptions.query; - delete clonedOptions.query; - } - } - - static __prepareSearchOption(clonedOptions: Object): void { - // Here we handle search (if passed) - we transform received arguments into linked LIKE statements. - if (clonedOptions.search instanceof Object) { - const { query, operator, fields: columns } = clonedOptions.search; - const search = { $search: { operator, columns, query } }; - - if (clonedOptions.where instanceof Object) { - clonedOptions.where = { - $and: [search, clonedOptions.where] - }; - } else { - clonedOptions.where = search; - } - - delete clonedOptions.search; - } - } - - static __generateID() { - return mdbid(); - } -} - -export default MySQLDriver; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/all.js b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/all.js deleted file mode 100644 index 4ad3347..0000000 --- a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/all.js +++ /dev/null @@ -1,22 +0,0 @@ -// @flow -import _ from "lodash"; -import type { Operator } from "../../../types"; -import and from "./../logical/and"; - -const all: Operator = { - canProcess: ({ key, value }) => { - if (key.charAt(0) === "$") { - return false; - } - - return _.has(value, "$all"); - }, - process: ({ key, value, statement }) => { - const andValue = value["$all"].map(v => { - return { [key]: { $jsonArrayFindValue: v } }; - }); - return and.process({ key, value: andValue, statement }); - } -}; - -export default all; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/eq.js b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/eq.js deleted file mode 100644 index 0229612..0000000 --- a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/eq.js +++ /dev/null @@ -1,58 +0,0 @@ -// @flow -import _ from "lodash"; -import type { Operator } from "../../../types"; -import { ArrayAttribute } from "webiny-model"; -import jsonArrayStrictEquality from "./../comparison/jsonArrayStrictEquality"; -import jsonArrayFindValue from "./../comparison/jsonArrayFindValue"; - -const eq: Operator = { - canProcess: ({ key, value, statement }) => { - if (key.charAt(0) === "$") { - return false; - } - - if (_.has(value, "$eq")) { - return true; - } - - // Valid values are 1, '1', null, true, false. - if (_.isString(value) || _.isNumber(value) || [null, true, false].includes(value)) { - return true; - } - - const instance = - typeof statement.entity === "function" ? new statement.entity() : statement.entity; - const attribute = instance.getAttribute(key); - return attribute instanceof ArrayAttribute && Array.isArray(value); - }, - process: ({ key, value, statement }) => { - value = _.get(value, "$eq", value); - if (value === null) { - return "`" + key + "` IS NULL"; - } - - const instance = - typeof statement.entity === "function" ? new statement.entity() : statement.entity; - const attribute = instance.getAttribute(key); - if (attribute instanceof ArrayAttribute) { - // Match all values (strict array equality check) - if (Array.isArray(value)) { - return jsonArrayStrictEquality.process({ - key, - value: { $jsonArrayStrictEquality: value }, - statement - }); - } else { - return jsonArrayFindValue.process({ - key, - value: { $jsonArrayFindValue: value }, - statement - }); - } - } - - return "`" + key + "` = " + statement.escape(value); - } -}; - -export default eq; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/gt.js b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/gt.js deleted file mode 100644 index a088a49..0000000 --- a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/gt.js +++ /dev/null @@ -1,13 +0,0 @@ -// @flow -import _ from "lodash"; -import type { Operator } from "../../../types"; - -const gt: Operator = { - canProcess: ({ value }) => { - return _.has(value, "$gt"); - }, - process: ({ key, value, statement }) => { - return "`" + key + "` > " + statement.escape(value["$gt"]); - } -}; -export default gt; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/gte.js b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/gte.js deleted file mode 100644 index ceec0a5..0000000 --- a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/gte.js +++ /dev/null @@ -1,13 +0,0 @@ -// @flow -import _ from "lodash"; -import type { Operator } from "../../../types"; - -const gte: Operator = { - canProcess: ({ value }) => { - return _.has(value, "$gte"); - }, - process: ({ key, value, statement }) => { - return "`" + key + "` >= " + statement.escape(value["$gte"]); - } -}; -export default gte; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/in.js b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/in.js deleted file mode 100644 index 2a9689c..0000000 --- a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/in.js +++ /dev/null @@ -1,39 +0,0 @@ -// @flow -import _ from "lodash"; -import type { Operator } from "../../../types"; -import { ArrayAttribute } from "webiny-model"; -import or from "../logical/or"; - -const inOperator: Operator = { - canProcess: ({ key, value, statement }) => { - if (key.charAt(0) === "$") { - return false; - } - - if (_.has(value, "$in")) { - return true; - } - - const instance = - typeof statement.entity === "function" ? new statement.entity() : statement.entity; - const attribute = instance.getAttribute(key); - - return Array.isArray(value) && !(attribute instanceof ArrayAttribute); - }, - process: ({ key, value, statement }) => { - value = _.get(value, "$in", value); - - const instance = - typeof statement.entity === "function" ? new statement.entity() : statement.entity; - const attribute = instance.getAttribute(key); - if (attribute instanceof ArrayAttribute) { - const andValue = value.map(v => { - return { [key]: { $jsonArrayFindValue: v } }; - }); - return or.process({ key, value: andValue, statement }); - } - - return "`" + key + "` IN(" + value.map(item => statement.escape(item)).join(", ") + ")"; - } -}; -export default inOperator; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/jsonArrayFindValue.js b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/jsonArrayFindValue.js deleted file mode 100644 index 74cd73a..0000000 --- a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/jsonArrayFindValue.js +++ /dev/null @@ -1,14 +0,0 @@ -// @flow -import _ from "lodash"; -import type { Operator } from "../../../types"; - -const jsonArrayFindValue: Operator = { - canProcess: ({ value }) => { - return _.has(value, "$jsonArrayFindValue"); - }, - process: ({ key, value, statement }) => { - value = value["$jsonArrayFindValue"]; - return "JSON_SEARCH(`" + key + "`, 'one', " + statement.escape(value) + ") IS NOT NULL"; - } -}; -export default jsonArrayFindValue; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/jsonArrayStrictEquality.js b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/jsonArrayStrictEquality.js deleted file mode 100644 index fb1cb14..0000000 --- a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/jsonArrayStrictEquality.js +++ /dev/null @@ -1,14 +0,0 @@ -// @flow -import _ from "lodash"; -import type { Operator } from "../../../types"; - -const jsonArrayStrictEquality: Operator = { - canProcess: ({ value }) => { - return _.has(value, "$jsonArrayStrictEquality"); - }, - process: ({ key, value, statement }) => { - value = value["$jsonArrayStrictEquality"]; - return "`" + key + "` = JSON_ARRAY(" + value.map(v => statement.escape(v)).join(", ") + ")"; - } -}; -export default jsonArrayStrictEquality; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/like.js b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/like.js deleted file mode 100644 index 5911066..0000000 --- a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/like.js +++ /dev/null @@ -1,18 +0,0 @@ -// @flow -import _ from "lodash"; -import type { Operator } from "../../../types"; - -const like: Operator = { - canProcess: ({ key, value }) => { - if (key.charAt(0) === "$") { - return false; - } - - return _.has(value, "$like"); - }, - process: ({ key, value, statement }) => { - return "`" + key + "` LIKE " + statement.escape(value["$like"]); - } -}; - -export default like; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/lt.js b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/lt.js deleted file mode 100644 index ff10b80..0000000 --- a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/lt.js +++ /dev/null @@ -1,13 +0,0 @@ -// @flow -import _ from "lodash"; -import type { Operator } from "../../../types"; - -const lt: Operator = { - canProcess: ({ value }) => { - return _.has(value, "$lt"); - }, - process: ({ key, value, statement }) => { - return "`" + key + "` < " + statement.escape(value["$lt"]); - } -}; -export default lt; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/lte.js b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/lte.js deleted file mode 100644 index 6e807f4..0000000 --- a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/lte.js +++ /dev/null @@ -1,13 +0,0 @@ -// @flow -import _ from "lodash"; -import type { Operator } from "../../../types"; - -const lte: Operator = { - canProcess: ({ value }) => { - return _.has(value, "$lte"); - }, - process: ({ key, value, statement }) => { - return "`" + key + "` <= " + statement.escape(value["$lte"]); - } -}; -export default lte; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/ne.js b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/ne.js deleted file mode 100644 index 6902773..0000000 --- a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/ne.js +++ /dev/null @@ -1,17 +0,0 @@ -// @flow -import _ from "lodash"; -import type { Operator } from "../../../types"; - -const ne: Operator = { - canProcess: ({ value }) => { - return _.has(value, "$ne"); - }, - process: ({ key, value, statement }) => { - if (value["$ne"] === null) { - return "`" + key + "` IS NOT NULL"; - } - - return "`" + key + "` <> " + statement.escape(value["$ne"]); - } -}; -export default ne; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/search.js b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/search.js deleted file mode 100644 index 67a730c..0000000 --- a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/comparison/search.js +++ /dev/null @@ -1,22 +0,0 @@ -// @flow -import type { Operator } from "../../../types"; -import or from "../logical/or"; -import and from "../logical/and"; - -const search: Operator = { - canProcess: ({ key }) => { - return key === "$search"; - }, - process: ({ value, statement }) => { - const columns = value.columns.map(columns => { - return { [columns]: { $like: "%" + value.query + "%" } }; - }); - - if (value.operator === "and") { - return and.process({ key: "$and", value: columns, statement }); - } - return or.process({ key: "$or", value: columns, statement }); - } -}; - -export default search; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/index.js b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/index.js deleted file mode 100644 index 1555144..0000000 --- a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/index.js +++ /dev/null @@ -1,37 +0,0 @@ -// @flow - -// Logical Operators (A-Z) -import $and from "./logical/and"; -import $or from "./logical/or"; - -// Comparison operators (A-Z) -import $all from "./comparison/all"; -import $eq from "./comparison/eq"; -import $gt from "./comparison/gt"; -import $gte from "./comparison/gte"; -import $in from "./comparison/in"; -import $jsonArrayFindValue from "./comparison/jsonArrayFindValue"; -import $jsonArrayStrictEquality from "./comparison/jsonArrayStrictEquality"; -import $like from "./comparison/like"; -import $lt from "./comparison/lt"; -import $lte from "./comparison/lte"; -import $ne from "./comparison/ne"; -import $search from "./comparison/search"; - -export default { - $or, - $and, - - $all, - $eq, - $gt, - $gte, - $in, - $jsonArrayFindValue, - $jsonArrayStrictEquality, - $like, - $lt, - $lte, - $ne, - $search -}; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/logical/and.js b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/logical/and.js deleted file mode 100644 index cb4ed4e..0000000 --- a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/logical/and.js +++ /dev/null @@ -1,40 +0,0 @@ -// @flow -import _ from "lodash"; -import type { Operator } from "../../../types"; - -const and: Operator = { - canProcess: ({ key }) => { - return key === "$and"; - }, - process: ({ value, statement }) => { - let output = ""; - switch (true) { - case _.isArray(value): - value.forEach(object => { - for (const [andKey, andValue] of Object.entries(object)) { - if (output === "") { - output = statement.process({ [andKey]: andValue }); - } else { - output += " AND " + statement.process({ [andKey]: andValue }); - } - } - }); - break; - case _.isPlainObject(value): - for (const [andKey, andValue] of Object.entries(value)) { - if (output === "") { - output = statement.process({ [andKey]: andValue }); - } else { - output += " AND " + statement.process({ [andKey]: andValue }); - } - } - break; - default: - throw Error("$and operator must receive an object or an array."); - } - - return "(" + output + ")"; - } -}; - -export default and; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/logical/or.js b/packages/fields-storage-dynamodb/OPERATORSsrc/operators/logical/or.js deleted file mode 100644 index 501ccb4..0000000 --- a/packages/fields-storage-dynamodb/OPERATORSsrc/operators/logical/or.js +++ /dev/null @@ -1,40 +0,0 @@ -// @flow -import _ from "lodash"; -import type { Operator } from "../../../types"; - -const or: Operator = { - canProcess: ({ key }) => { - return key === "$or"; - }, - process: ({ value, statement }) => { - let output = ""; - switch (true) { - case _.isArray(value): - value.forEach(object => { - for (const [orKey, orValue] of Object.entries(object)) { - if (output === "") { - output = statement.process({ [orKey]: orValue }); - } else { - output += " OR " + statement.process({ [orKey]: orValue }); - } - } - }); - break; - case _.isPlainObject(value): - for (const [orKey, orValue] of Object.entries(value)) { - if (output === "") { - output = statement.process({ [orKey]: orValue }); - } else { - output += " OR " + statement.process({ [orKey]: orValue }); - } - } - break; - default: - throw Error("$or operator must receive an object or an array."); - } - - return "(" + output + ")"; - } -}; - -export default or; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/statements/delete.js b/packages/fields-storage-dynamodb/OPERATORSsrc/statements/delete.js deleted file mode 100644 index 00e6f7a..0000000 --- a/packages/fields-storage-dynamodb/OPERATORSsrc/statements/delete.js +++ /dev/null @@ -1,17 +0,0 @@ -// @flow -import Statement from "./statement"; - -class Delete extends Statement { - generate() { - const options = this.options; - let output = `DELETE FROM \`${options.table}\``; - output += this.getWhere(options); - output += this.getOrder(options); - output += this.getLimit(options); - output += this.getOffset(options); - - return output; - } -} - -export default Delete; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/statements/index.js b/packages/fields-storage-dynamodb/OPERATORSsrc/statements/index.js deleted file mode 100644 index a0ebb41..0000000 --- a/packages/fields-storage-dynamodb/OPERATORSsrc/statements/index.js +++ /dev/null @@ -1,7 +0,0 @@ -// @flow -// Table data statements. -export { default as Statement } from "./statement"; -export { default as Insert } from "./insert"; -export { default as Select } from "./select"; -export { default as Update } from "./update"; -export { default as Delete } from "./delete"; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/statements/insert.js b/packages/fields-storage-dynamodb/OPERATORSsrc/statements/insert.js deleted file mode 100644 index 362f405..0000000 --- a/packages/fields-storage-dynamodb/OPERATORSsrc/statements/insert.js +++ /dev/null @@ -1,32 +0,0 @@ -// @flow -import Statement from "./statement"; -import _ from "lodash"; - -class Insert extends Statement { - generate() { - const options = this.options; - const columns = _.keys(options.data) - .map(c => "`" + c + "`") - .join(", "); - const insertValues = _.values(options.data) - .map(value => this.escape(value)) - .join(", "); - - if (!options.onDuplicateKeyUpdate) { - return `INSERT INTO \`${options.table}\` (${columns}) VALUES (${insertValues})`; - } - - const updateValues = []; - for (const [key, value] of Object.entries(options.data)) { - updateValues.push(key + " = " + this.escape(value)); - } - - return `INSERT INTO \`${ - options.table - }\` (${columns}) VALUES (${insertValues}) ON DUPLICATE KEY UPDATE ${updateValues.join( - ", " - )}`; - } -} - -export default Insert; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/statements/select.js b/packages/fields-storage-dynamodb/OPERATORSsrc/statements/select.js deleted file mode 100644 index 7aa1e03..0000000 --- a/packages/fields-storage-dynamodb/OPERATORSsrc/statements/select.js +++ /dev/null @@ -1,23 +0,0 @@ -// @flow -import Statement from "./statement"; - -class Select extends Statement { - generate() { - const options = this.options; - let output = `SELECT`; - if (options.calculateFoundRows) { - output += ` SQL_CALC_FOUND_ROWS`; - } - - output += this.getColumns(options); - output += ` FROM \`${options.table}\``; - output += this.getWhere(options); - output += this.getOrder(options); - output += this.getLimit(options); - output += this.getOffset(options); - - return output; - } -} - -export default Select; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/statements/statement.js b/packages/fields-storage-dynamodb/OPERATORSsrc/statements/statement.js deleted file mode 100644 index fbe8a49..0000000 --- a/packages/fields-storage-dynamodb/OPERATORSsrc/statements/statement.js +++ /dev/null @@ -1,111 +0,0 @@ -// @flow -import SqlString from "sqlstring"; -import _ from "lodash"; -import type { Entity } from "webiny-entity"; -import type { Operator, Payload } from "../../types"; - -declare type StatementOptions = { - operators: { [string]: Operator }, - calculateFoundRows: boolean, - table: string, - data: Object, - limit?: number, - offset?: number, - sort?: string, - where?: Object, - columns?: Array, - onDuplicateKeyUpdate?: boolean -}; - -class Statement { - entity: Entity | Class; - options: StatementOptions; - constructor(options: Object = {}, entity: Entity | Class) { - this.options = options; - this.entity = entity; - } - - generate(): string { - return ""; - } - - getColumns(options: StatementOptions): string { - const columns = options.columns || []; - - if (_.isEmpty(columns)) { - return " *"; - } - - return " " + columns.join(", "); - } - - getWhere(options: StatementOptions): string { - if (_.isEmpty(options.where)) { - return ""; - } - - return " WHERE " + this.process({ $and: options.where }); - } - - getOrder(options: StatementOptions): string { - if (!(options.sort instanceof Object) || _.isEmpty(options.sort)) { - return ""; - } - - let query = []; - - for (let key in options.sort) { - query.push(`${key} ${options.sort[key] === 1 ? "ASC" : "DESC"}`); - } - - return " ORDER BY " + query.join(", "); - } - - getLimit(options: StatementOptions): string { - const limit = options.limit || 0; - - if (_.isNumber(limit) && limit > 0) { - return ` LIMIT ${limit}`; - } - return ""; - } - - getOffset(options: StatementOptions): string { - const offset = options.offset || 0; - - if (_.isNumber(offset) && offset > 0) { - return ` OFFSET ${offset}`; - } - return ""; - } - - escape(value: mixed) { - return SqlString.escape(value); - } - - /** - * Traverse the payload and apply operators to construct a valid MySQL statement - * @private - * @param {Object} payload - * @returns {string} SQL query - */ - process(payload: Payload): string { - let output = ""; - - outerLoop: for (const [key, value] of Object.entries(payload)) { - const operators: Array = Object.values(this.options.operators); - for (let i = 0; i < operators.length; i++) { - const operator = operators[i]; - if (operator.canProcess({ key, value, statement: this })) { - output += operator.process({ key, value, statement: this }); - continue outerLoop; - } - } - throw new Error(`Invalid operator {${key} : ${(value: any)}}.`); - } - - return output; - } -} - -export default Statement; diff --git a/packages/fields-storage-dynamodb/OPERATORSsrc/statements/update.js b/packages/fields-storage-dynamodb/OPERATORSsrc/statements/update.js deleted file mode 100644 index 4128f9e..0000000 --- a/packages/fields-storage-dynamodb/OPERATORSsrc/statements/update.js +++ /dev/null @@ -1,22 +0,0 @@ -// @flow -import Statement from "./statement"; - -class Update extends Statement { - generate() { - const values = [], - options = this.options; - for (const [key, value] of Object.entries(options.data)) { - values.push("`" + key + "` = " + this.escape(value)); - } - - let output = `UPDATE \`${options.table}\` SET ${values.join(", ")}`; - output += this.getWhere(options); - output += this.getOrder(options); - output += this.getLimit(options); - output += this.getOffset(options); - - return output; - } -} - -export default Update; From 8077bdc9e537a8a8148053c474a6ed1a02e8fc19 Mon Sep 17 00:00:00 2001 From: Adrian Smijulj Date: Sun, 6 Sep 2020 19:55:47 +0200 Subject: [PATCH 09/26] feat: add logical and comparison operators --- .../src/operators/comparison/beginsWith.js | 16 ++--- .../src/operators/comparison/between.js | 5 +- .../src/operators/comparison/eq.js | 16 ++--- .../src/operators/comparison/gt.js | 18 ++---- .../src/operators/comparison/gte.js | 17 ++--- .../src/operators/comparison/lt.js | 17 ++--- .../src/operators/comparison/lte.js | 17 ++--- .../src/operators/index.js | 16 ++--- .../src/operators/logical/and.js | 62 +++++++++++++++++++ .../src/operators/logical/or.js | 40 ++++++++++++ 10 files changed, 147 insertions(+), 77 deletions(-) create mode 100644 packages/fields-storage-dynamodb/src/operators/logical/and.js create mode 100644 packages/fields-storage-dynamodb/src/operators/logical/or.js diff --git a/packages/fields-storage-dynamodb/src/operators/comparison/beginsWith.js b/packages/fields-storage-dynamodb/src/operators/comparison/beginsWith.js index 8fdaf29..a846dec 100644 --- a/packages/fields-storage-dynamodb/src/operators/comparison/beginsWith.js +++ b/packages/fields-storage-dynamodb/src/operators/comparison/beginsWith.js @@ -2,17 +2,11 @@ const beginsWith = { canProcess: ({ value }) => { return value && typeof value["$beginsWith"] !== "undefined"; }, - process: ({ key, value }) => { - return { - statement: `begins_with (#${key}, :${key})`, - attributeNames: { - [`#${key}`]: key - }, - attributeValues: { - [`:${key}`]: value["$beginsWith"] - } - }; + process: ({ key, value, args }) => { + args.expression += `begins_with (#${key}, :${key})`; + args.attributeNames[`#${key}`] = key; + args.attributeValues[`:${key}`] = value["$beginsWith"]; } }; -module.exports = beginsWith; +export default beginsWith; diff --git a/packages/fields-storage-dynamodb/src/operators/comparison/between.js b/packages/fields-storage-dynamodb/src/operators/comparison/between.js index bd64ee5..0ed7f75 100644 --- a/packages/fields-storage-dynamodb/src/operators/comparison/between.js +++ b/packages/fields-storage-dynamodb/src/operators/comparison/between.js @@ -6,15 +6,14 @@ const between = { return { statement: `#${key} BETWEEN :${key}Gte AND :${key}Lte`, attributeNames: { - [`#${key}`]: key, + [`#${key}`]: key }, attributeValues: { [`:${key}Gte`]: value[0], [`:${key}Lte`]: value[1] } }; - } }; -module.exports = between; +export default between; diff --git a/packages/fields-storage-dynamodb/src/operators/comparison/eq.js b/packages/fields-storage-dynamodb/src/operators/comparison/eq.js index 87d545a..29f2168 100644 --- a/packages/fields-storage-dynamodb/src/operators/comparison/eq.js +++ b/packages/fields-storage-dynamodb/src/operators/comparison/eq.js @@ -12,17 +12,11 @@ const eq = { return validTypes.includes(typeof value); }, - process: ({ key, value }) => { - return { - expression: `#${key} = :${key}`, - attributeNames: { - [`#${key}`]: key - }, - attributeValues: { - [`:${key}`]: value - } - }; + process: ({ key, value, args }) => { + args.expression += `#${key} = :${key}`; + args.attributeNames[`#${key}`] = key; + args.attributeValues[`:${key}`] = value; } }; -module.exports = eq; +export default eq; diff --git a/packages/fields-storage-dynamodb/src/operators/comparison/gt.js b/packages/fields-storage-dynamodb/src/operators/comparison/gt.js index 82f7cf3..29ab05c 100644 --- a/packages/fields-storage-dynamodb/src/operators/comparison/gt.js +++ b/packages/fields-storage-dynamodb/src/operators/comparison/gt.js @@ -1,18 +1,12 @@ const gt = { canProcess: ({ value }) => { - return value && typeof value["$gt"] !== "undefined"; + return value && typeof value["$gt"] !== "undefined"; }, - process: ({ key, value }) => { - return { - expression: `#${key} > :${key}`, - attributeNames: { - [`#${key}`]: key - }, - attributeValues: { - [`:${key}`]: value["$gt"] - } - }; + process: ({ key, value, args }) => { + args.expression += `#${key} > :${key}`; + args.attributeNames[`#${key}`] = key; + args.attributeValues[`:${key}`] = value["$gt"]; } }; -module.exports = gt; +export default gt; diff --git a/packages/fields-storage-dynamodb/src/operators/comparison/gte.js b/packages/fields-storage-dynamodb/src/operators/comparison/gte.js index cd80f5b..6fc0384 100644 --- a/packages/fields-storage-dynamodb/src/operators/comparison/gte.js +++ b/packages/fields-storage-dynamodb/src/operators/comparison/gte.js @@ -2,16 +2,11 @@ const gte = { canProcess: ({ value }) => { return value && typeof value["$gte"] !== "undefined"; }, - process: ({ key, value }) => { - return { - expression: `#${key} >= :${key}`, - attributeNames: { - [`#${key}`]: key - }, - attributeValues: { - [`:${key}`]: value["$gt"] - } - }; + process: ({ key, value, args }) => { + args.expression += `#${key} >= :${key}`; + args.attributeNames[`#${key}`] = key; + args.attributeValues[`:${key}`] = value["$gte"] } }; -module.exports = gte; + +export default gte; diff --git a/packages/fields-storage-dynamodb/src/operators/comparison/lt.js b/packages/fields-storage-dynamodb/src/operators/comparison/lt.js index 700ea14..db91ce3 100644 --- a/packages/fields-storage-dynamodb/src/operators/comparison/lt.js +++ b/packages/fields-storage-dynamodb/src/operators/comparison/lt.js @@ -2,16 +2,11 @@ const lt = { canProcess: ({ value }) => { return value && typeof value["$lt"] !== "undefined"; }, - process: ({ key, value }) => { - return { - expression: `#${key} < :${key}`, - attributeNames: { - [`#${key}`]: key - }, - attributeValues: { - [`:${key}`]: value["$gt"] - } - }; + process: ({ key, value, args }) => { + args.expression += `#${key} < :${key}`; + args.attributeNames[`#${key}`] = key; + args.attributeValues[`:${key}`] = value["$lt"] } }; -module.exports = lt; + +export default lt; diff --git a/packages/fields-storage-dynamodb/src/operators/comparison/lte.js b/packages/fields-storage-dynamodb/src/operators/comparison/lte.js index 0fae983..711b01a 100644 --- a/packages/fields-storage-dynamodb/src/operators/comparison/lte.js +++ b/packages/fields-storage-dynamodb/src/operators/comparison/lte.js @@ -2,16 +2,11 @@ const lte = { canProcess: ({ value }) => { return value && typeof value["$lte"] !== "undefined"; }, - process: ({ key, value }) => { - return { - expression: `#${key} <= :${key}`, - attributeNames: { - [`#${key}`]: key - }, - attributeValues: { - [`:${key}`]: value["$gt"] - } - }; + process: ({ key, value, args }) => { + args.expression += `#${key} <= :${key}`; + args.attributeNames[`#${key}`] = key; + args.attributeValues[`:${key}`] = value["$lte"] } }; -module.exports = lte; + +export default lte; diff --git a/packages/fields-storage-dynamodb/src/operators/index.js b/packages/fields-storage-dynamodb/src/operators/index.js index 1122640..12f1305 100644 --- a/packages/fields-storage-dynamodb/src/operators/index.js +++ b/packages/fields-storage-dynamodb/src/operators/index.js @@ -1,13 +1,15 @@ // Comparison operators (A-Z) -const $beginsWith = require("./comparison/beginsWith"); -const $between = require("./comparison/between"); -const $gt = require("./comparison/gt"); -const $gte = require("./comparison/gte"); -const $lt = require("./comparison/lt"); -const $lte = require("./comparison/lte"); -const $eq = require("./comparison/eq"); +import $and from "./logical/and"; +import $beginsWith from "./comparison/beginsWith"; +import $between from "./comparison/between"; +import $gt from "./comparison/gt"; +import $gte from "./comparison/gte"; +import $lt from "./comparison/lt"; +import $lte from "./comparison/lte"; +import $eq from "./comparison/eq"; module.exports = { + $and, $beginsWith, $between, $eq, diff --git a/packages/fields-storage-dynamodb/src/operators/logical/and.js b/packages/fields-storage-dynamodb/src/operators/logical/and.js new file mode 100644 index 0000000..bc005a0 --- /dev/null +++ b/packages/fields-storage-dynamodb/src/operators/logical/and.js @@ -0,0 +1,62 @@ +const isObject = value => value && typeof value === "object"; + +const and = { + canProcess: ({ key }) => { + return key === "$and"; + }, + process: ({ value, args, processStatement }) => { + const andArgs = { + expression: "", + attributeNames: {}, + attributeValues: {} + }; + + switch (true) { + /* case Array.isArray(value): + value.forEach(object => { + for (const [andKey, andValue] of Object.entries(object)) { + if (andArgs.expression === "") { + processStatement({ + args: andArgs, + query: { [andKey]: andValue } + }); + } else { + andExpression += + " AND " + processStatement({ args: andArgs, query: { [andKey]: andValue } }); + + } + } + }); + break;*/ + case isObject(value): { + for (const [andKey, andValue] of Object.entries(value)) { + const currentArgs = { + expression: "", + attributeNames: {}, + attributeValues: {} + }; + + processStatement({ args: currentArgs, query: { [andKey]: andValue } }); + + Object.assign(andArgs.attributeNames, currentArgs.attributeNames); + Object.assign(andArgs.attributeValues, currentArgs.attributeValues); + + if (andArgs.expression === "") { + andArgs.expression = currentArgs.expression; + } else { + andArgs.expression += " and " + currentArgs.expression; + } + } + break; + } + default: + throw Error("$and operator must receive an object or an array."); + } + + args.expression += "(" + andArgs.expression + ")"; + Object.assign(args.attributeNames, andArgs.attributeNames); + Object.assign(args.attributeValues, andArgs.attributeValues); + } +}; + +export default and; diff --git a/packages/fields-storage-dynamodb/src/operators/logical/or.js b/packages/fields-storage-dynamodb/src/operators/logical/or.js new file mode 100644 index 0000000..501ccb4 --- /dev/null +++ b/packages/fields-storage-dynamodb/src/operators/logical/or.js @@ -0,0 +1,40 @@ +// @flow +import _ from "lodash"; +import type { Operator } from "../../../types"; + +const or: Operator = { + canProcess: ({ key }) => { + return key === "$or"; + }, + process: ({ value, statement }) => { + let output = ""; + switch (true) { + case _.isArray(value): + value.forEach(object => { + for (const [orKey, orValue] of Object.entries(object)) { + if (output === "") { + output = statement.process({ [orKey]: orValue }); + } else { + output += " OR " + statement.process({ [orKey]: orValue }); + } + } + }); + break; + case _.isPlainObject(value): + for (const [orKey, orValue] of Object.entries(value)) { + if (output === "") { + output = statement.process({ [orKey]: orValue }); + } else { + output += " OR " + statement.process({ [orKey]: orValue }); + } + } + break; + default: + throw Error("$or operator must receive an object or an array."); + } + + return "(" + output + ")"; + } +}; + +export default or; From a082eb415f2237e7bb1b5bc790141d65c9a3b3d7 Mon Sep 17 00:00:00 2001 From: Adrian Smijulj Date: Sun, 6 Sep 2020 19:56:22 +0200 Subject: [PATCH 10/26] wip: create DynamoDb driver --- .../src/BatchProcess.js | 66 +++++++ .../src/DynamoDbClient.js | 184 ------------------ .../src/DynamoDbDriver.js | 175 +++++++++++++---- .../src/QueryGenerator.js | 19 +- packages/fields-storage-dynamodb/src/index.js | 3 +- .../src/statements/KeyConditionExpression.js | 23 --- .../createKeyConditionExpressionArgs.js | 24 +++ .../src/statements/processStatement.js | 15 ++ 8 files changed, 254 insertions(+), 255 deletions(-) create mode 100644 packages/fields-storage-dynamodb/src/BatchProcess.js delete mode 100644 packages/fields-storage-dynamodb/src/DynamoDbClient.js delete mode 100644 packages/fields-storage-dynamodb/src/statements/KeyConditionExpression.js create mode 100644 packages/fields-storage-dynamodb/src/statements/createKeyConditionExpressionArgs.js create mode 100644 packages/fields-storage-dynamodb/src/statements/processStatement.js diff --git a/packages/fields-storage-dynamodb/src/BatchProcess.js b/packages/fields-storage-dynamodb/src/BatchProcess.js new file mode 100644 index 0000000..cbae420 --- /dev/null +++ b/packages/fields-storage-dynamodb/src/BatchProcess.js @@ -0,0 +1,66 @@ +class BatchProcess { + constructor(batch, documentClient) { + this.documentClient = documentClient; + this.batchProcess = batch; + + this.queryBuildResolveFunction = null; + this.queryBuilding = new Promise(resolve => { + this.queryBuildResolveFunction = resolve; + }); + + this.queryExecutionResolveFunction = null; + this.queryExecution = new Promise(resolve => { + this.queryExecutionResolveFunction = resolve; + }); + + this.operations = []; + this.results = []; + } + + addOperation(type, operation) { + this.operations.push([type, operation]); + return this.operations.length - 1; + } + + allOperationsAdded() { + return this.operations.length === this.batchProcess.operations.length; + } + + markAsReady() { + this.queryBuildResolveFunction(); + } + + async execute() { + const batchWriteParams = { + RequestItems: {} + }; + + for (let i = 0; i < this.operations.length; i++) { + let [type, params] = this.operations[i]; + + if (!batchWriteParams.RequestItems[params.TableName]) { + batchWriteParams.RequestItems[params.TableName] = []; + } + + batchWriteParams.RequestItems[params.TableName].push({ + [type]: { + Item: params.Item + } + }); + } + + this.results = await this.documentClient.batchWrite(batchWriteParams).promise(); + this.queryExecutionResolveFunction(); + return []; + } + + waitForOperationsAdded() { + return this.queryBuilding; + } + + waitForQueryExecution() { + return this.queryExecution; + } +} + +export default BatchProcess; diff --git a/packages/fields-storage-dynamodb/src/DynamoDbClient.js b/packages/fields-storage-dynamodb/src/DynamoDbClient.js deleted file mode 100644 index 9433462..0000000 --- a/packages/fields-storage-dynamodb/src/DynamoDbClient.js +++ /dev/null @@ -1,184 +0,0 @@ -import { DocumentClient } from "aws-sdk/clients/dynamodb"; -import QueryGenerator from "./QueryGenerator"; - -class DynamoDbClient { - constructor({ documentClient } = {}) { - this.client = documentClient || new DocumentClient(); - } - - async create(rawItems) { - const items = Array.isArray(rawItems) ? rawItems : [rawItems]; - const results = []; - for (let i = 0; i < items.length; i++) { - const item = items[i]; - results.push( - await this.client - .put({ - TableName: item.table, - Item: item.data - }) - .promise() - ); - } - - return results; - } - - async delete(rawItems) { - const items = Array.isArray(rawItems) ? rawItems : [rawItems]; - const results = []; - for (let i = 0; i < items.length; i++) { - let item = items[i]; - results.push( - await this.client - .delete({ - TableName: item.table, - Key: item.query - }) - .promise() - ); - } - - return results; - } - - async update(rawItems) { - const items = Array.isArray(rawItems) ? rawItems : [rawItems]; - - for (let i = 0; i < items.length; i++) { - const item = items[i]; - - const update = { - UpdateExpression: "SET ", - ExpressionAttributeNames: {}, - ExpressionAttributeValues: {} - }; - - const updateExpression = []; - for (const key in item.data) { - updateExpression.push(`#${key} = :${key}`); - update.ExpressionAttributeNames[`#${key}`] = key; - update.ExpressionAttributeValues[`:${key}`] = item.data[key]; - } - - update.UpdateExpression += updateExpression.join(", "); - - await this.client - .update({ - TableName: item.table, - Key: item.query, - ...update - }) - .promise(); - } - - return true; - } - - async findOne(rawItems) { - const items = Array.isArray(rawItems) ? rawItems : [rawItems]; - for (let i = 0; i < items.length; i++) { - let item = items[i]; - item.limit = 1; - } - return this.find(items); - } - - async find(rawItems) { - const items = Array.isArray(rawItems) ? rawItems : [rawItems]; - const results = []; - for (let i = 0; i < items.length; i++) { - let item = items[i]; - - const queryGenerator = new QueryGenerator(); - const queryParams = queryGenerator.generate({ query: item.query, keys: item.keys }); - - results.push( - await this.client() - .query({ - TableName: item.table, - Limit: item.limit, - IndexName: key.primary ? "Index" : key.name, - ...queryParams - }) - .promise() - ); - } - - return results; - } - - async count() { - throw new Error(`Cannot run "count" operation - not supported.`); - } - - setCollectionPrefix(collectionPrefix) { - this.collections.prefix = collectionPrefix; - return this; - } - - getCollectionPrefix() { - return this.collections.prefix; - } - - setCollectionNaming(collectionNameValue) { - this.collections.naming = collectionNameValue; - return this; - } - - getCollectionNaming() { - return this.collections.naming; - } - - getCollectionName(name) { - const getCollectionName = this.getCollectionNaming(); - if (typeof getCollectionName === "function") { - return getCollectionName({ name, driver: this }); - } - - return this.collections.prefix + name; - } - - static __prepareSearchOption(options) { - // Here we handle search (if passed) - we transform received arguments into linked LIKE statements. - if (options.search && options.search.query) { - const { query, operator, fields } = options.search; - - const searches = []; - fields.forEach(field => { - searches.push({ [field]: { $regex: `.*${query}.*`, $options: "i" } }); - }); - - const search = { - [operator === "and" ? "$and" : "$or"]: searches - }; - - if (options.query instanceof Object) { - options.query = { - $and: [search, options.query] - }; - } else { - options.query = search; - } - - delete options.search; - } - } - - static __prepareProjectFields(options) { - // Here we convert requested fields into a "project" parameter - if (options.fields) { - options.project = options.fields.reduce( - (acc, item) => { - acc[item] = 1; - return acc; - }, - { id: 1 } - ); - - delete options.fields; - } - } -} - -export default DynamoDbClient; diff --git a/packages/fields-storage-dynamodb/src/DynamoDbDriver.js b/packages/fields-storage-dynamodb/src/DynamoDbDriver.js index f5c8206..db2cee0 100644 --- a/packages/fields-storage-dynamodb/src/DynamoDbDriver.js +++ b/packages/fields-storage-dynamodb/src/DynamoDbDriver.js @@ -1,70 +1,163 @@ -import DynamoDbClient from "./DynamoDbClient"; -import { getKeys } from "@commodo/fields-storage"; import { DocumentClient } from "aws-sdk/clients/dynamodb"; +import BatchProcess from "./BatchProcess"; +import QueryGenerator from "./QueryGenerator"; class DynamoDbDriver { constructor({ documentClient, tableName } = {}) { - this.documentClient = new DynamoDbClient({ - documentClient: documentClient || new DocumentClient() - }); - + this.batchProcesses = {}; + this.documentClient = documentClient || new DocumentClient(); this.tableName = tableName; } - getDocumentClient() { + getClient() { return this.documentClient; } - async create({ model }) { - const table = this.getTableName(model); - const data = await model.toStorage(); - return this.getDocumentClient().create({ table, data }); - } + async create({ name, data, batch }) { + if (!batch) { + return await this.documentClient + .put({ TableName: this.tableName || name, Item: data }) + .promise(); + } + + const batchProcess = this.getBatchProcess(batch); + batchProcess.addOperation("PutRequest", { TableName: this.tableName || name, Item: data }); - async update({ model, primaryKey }) { - const table = this.getTableName(model); - const query = {}; - for (let i = 0; i < primaryKey.fields.length; i++) { - let field = primaryKey.fields[i]; - query[field.name] = model[field.name]; + if (batchProcess.allOperationsAdded()) { + batchProcess.execute(); + batchProcess.markAsReady(); + } else { + await batchProcess.waitForOperationsAdded(); } - const data = await model.toStorage(); + await batchProcess.waitForQueryExecution(); - return this.getDocumentClient().update({ table, data, query }); + return true; } - async delete({ model, primaryKey }) { - const table = this.getTableName(model); - const query = {}; - for (let i = 0; i < primaryKey.fields.length; i++) { - let field = primaryKey.fields[i]; - query[field.name] = model[field.name]; + async update({ query, data, name, batch, instance }) { + if (!batch) { + const update = { + UpdateExpression: "SET ", + ExpressionAttributeNames: {}, + ExpressionAttributeValues: {} + }; + + const updateExpression = []; + for (const key in data) { + updateExpression.push(`#${key} = :${key}`); + update.ExpressionAttributeNames[`#${key}`] = key; + update.ExpressionAttributeValues[`:${key}`] = data[key]; + } + + update.UpdateExpression += updateExpression.join(", "); + + return await this.documentClient + .update({ + TableName: this.tableName || name, + Key: query, + ...update + }) + .promise(); } - return this.getDocumentClient().delete({ table, query }); - } + const batchProcess = this.getBatchProcess(batch); - async findOne({ model, args }) { - const table = this.getTableName(model); - return this.getDocumentClient().findOne({ ...args, table, keys: getKeys(model) }); + // It would be nice if we could rely on the received data all the time, but that's not possible. Because + // "PutRequest" operations only insert or overwrite existing data (meaning => classic updates are NOT allowed), + // we must get complete model data, and use that in the operation. This is possible only if the update + // call is part of an model instance update (not part of SomeModel.save() call), where we can access the + // toStorage function. So, if that's the case, we'll call it with the skipDifferenceCheck flag enabled. + // Normally we wouldn't have to do all of this dancing, but it's how DynamoDB works, there's no way around it. + let Item = instance ? await instance.toStorage({ skipDifferenceCheck: true }) : data; + + batchProcess.addOperation("PutRequest", { + TableName: this.tableName || name, + Item + }); + + if (batchProcess.allOperationsAdded()) { + batchProcess.execute(); + batchProcess.markAsReady(); + } else { + await batchProcess.waitForOperationsAdded(); + } + + await batchProcess.waitForQueryExecution(); + + return true; } - async find({ model, args }) { - const table = this.getTableName(model); - return this.getDocumentClient().find({ ...args, table, keys: getKeys(model) }); + async delete({ query, name, batch }) { + if (!batch) { + return await this.documentClient + .delete({ + TableName: this.tableName || name, + Key: query + }) + .promise(); + } + + const batchProcess = this.getBatchProcess(batch); + batchProcess.addOperation("DeleteRequest", { + Key: query + }); + + if (batchProcess.allOperationsAdded()) { + batchProcess.execute(); + batchProcess.markAsReady(); + } else { + await batchProcess.waitForOperationsAdded(); + } + + await batchProcess.waitForQueryExecution(); + + return true; } - async count({ model }) { - const table = this.getTableName(model); + async find({ name, query, sort, limit, batch, keys }) { + if (!batch) { + const queryGenerator = new QueryGenerator(); + const queryParams = queryGenerator.generate({ + query, + keys, + sort, + limit, + table: this.tableName || name + }); + + const { Items } = await this.documentClient.query(queryParams).promise(); + return [Items] + } + + const batchProcess = this.getBatchProcess(batch); + batchProcess.addOperation("DeleteRequest", { + Key: query + }); + + if (batchProcess.allOperationsAdded()) { + batchProcess.execute(); + batchProcess.markAsReady(); + } else { + await batchProcess.waitForOperationsAdded(); + } + + await batchProcess.waitForQueryExecution(); - // Will throw an error - counts not supported in DynamoDb. - return this.getDocumentClient().count({ table }); + return true; } - getTableName(model) { - return this.tableName || model.getStorageName(); + async count() { + throw new Error(`Cannot run "count" operation - not supported.`); + } + + getBatchProcess(batch) { + if (!this.batchProcesses[batch.id]) { + this.batchProcesses[batch.id] = new BatchProcess(batch, this.documentClient); + } + + return this.batchProcesses[batch.id]; } } -module.exports = DynamoDbDriver; +export default DynamoDbDriver; diff --git a/packages/fields-storage-dynamodb/src/QueryGenerator.js b/packages/fields-storage-dynamodb/src/QueryGenerator.js index 771071f..2176658 100644 --- a/packages/fields-storage-dynamodb/src/QueryGenerator.js +++ b/packages/fields-storage-dynamodb/src/QueryGenerator.js @@ -1,13 +1,14 @@ -import KeyConditionExpression from "./statements/KeyConditionExpression"; - -// const key = findQueryKey(item.query, item.keys); -// const keyConditionExpression = new KeyConditionExpression().process(item.query); +import createKeyConditionExpressionArgs from "./statements/createKeyConditionExpressionArgs"; class QueryGenerator { - generate({ query, keys }) { + generate({ query, keys, sort, limit, table }) { // 1. Which key can we use in this query operation? const key = this.findQueryKey(query, keys); + if (!key) { + throw new Error("Cannot perform query - key not found."); + } + // 2. Now that we know the key, let's separate the key attributes from the rest. const keyAttributesValues = {}, nonKeyAttributesValues = {}; @@ -18,6 +19,14 @@ class QueryGenerator { nonKeyAttributesValues[queryKey] = query[queryKey]; } } + + const keyConditionExpression = createKeyConditionExpressionArgs({ + query: keyAttributesValues, + sort, + key + }); + + return { ...keyConditionExpression, TableName: table, Limit: limit }; } findQueryKey(query = {}, keys = []) { diff --git a/packages/fields-storage-dynamodb/src/index.js b/packages/fields-storage-dynamodb/src/index.js index 0745b66..15d78cf 100644 --- a/packages/fields-storage-dynamodb/src/index.js +++ b/packages/fields-storage-dynamodb/src/index.js @@ -1,5 +1,4 @@ // @flow import { default as DynamoDbDriver } from "./DynamoDbDriver"; -import { default as DynamoDbClient } from "./DynamoDbClient"; -export { DynamoDbDriver, DynamoDbClient }; +export { DynamoDbDriver }; diff --git a/packages/fields-storage-dynamodb/src/statements/KeyConditionExpression.js b/packages/fields-storage-dynamodb/src/statements/KeyConditionExpression.js deleted file mode 100644 index 977609c..0000000 --- a/packages/fields-storage-dynamodb/src/statements/KeyConditionExpression.js +++ /dev/null @@ -1,23 +0,0 @@ -const allOperators = require("./../operators"); - -class KeyConditionExpression { - process(payload) { - let output = []; - - outerLoop: for (const [key, value] of Object.entries(payload)) { - const operators = Object.values(allOperators); - for (let i = 0; i < operators.length; i++) { - const operator = operators[i]; - if (operator.canProcess({ key, value })) { - output.push(operator.process({ key, value })); - continue outerLoop; - } - } - throw new Error(`Invalid operator {${key} : ${value}}.`); - } - - return output; - } -} - -module.exports = KeyConditionExpression; diff --git a/packages/fields-storage-dynamodb/src/statements/createKeyConditionExpressionArgs.js b/packages/fields-storage-dynamodb/src/statements/createKeyConditionExpressionArgs.js new file mode 100644 index 0000000..36f6ec2 --- /dev/null +++ b/packages/fields-storage-dynamodb/src/statements/createKeyConditionExpressionArgs.js @@ -0,0 +1,24 @@ +import processStatement from "./processStatement"; + +export default ({ query, sort, key }) => { + const args = { + expression: "", + attributeNames: {}, + attributeValues: {} + }; + + processStatement({ args, query: { $and: query } }); + + const output = { + KeyConditionExpression: args.expression, + ExpressionAttributeNames: args.attributeNames, + ExpressionAttributeValues: args.attributeValues + }; + + const sortKey = key.fields && key.fields[1]; + if (sort && sort[sortKey.name] === -1) { + output.ScanIndexForward = true; + } + + return output; +}; diff --git a/packages/fields-storage-dynamodb/src/statements/processStatement.js b/packages/fields-storage-dynamodb/src/statements/processStatement.js new file mode 100644 index 0000000..10ffc4c --- /dev/null +++ b/packages/fields-storage-dynamodb/src/statements/processStatement.js @@ -0,0 +1,15 @@ +import allOperators from "./../operators"; + +export default function processStatement({ args, query }) { + outerLoop: for (const [key, value] of Object.entries(query)) { + const operators = Object.values(allOperators); + for (let i = 0; i < operators.length; i++) { + const operator = operators[i]; + if (operator.canProcess({ key, value, args })) { + operator.process({ key, value, args, processStatement }); + continue outerLoop; + } + } + throw new Error(`Invalid operator {${key} : ${(value: any)}}.`); + } +} From 56c9015539feca351711ddd3d2324d89f0a0ce02 Mon Sep 17 00:00:00 2001 From: Adrian Smijulj Date: Sun, 6 Sep 2020 19:56:53 +0200 Subject: [PATCH 11/26] test: add tests --- .../__tests__/delete.test.js | 17 +- .../__tests__/find.test.js | 116 ++++++++ .../__tests__/findOne.test.js | 100 ++++++- .../__tests__/models/SimpleModel.js | 99 ++++++- .../__tests__/queue.batch.test.js | 265 ++++++++++++++++++ 5 files changed, 568 insertions(+), 29 deletions(-) create mode 100644 packages/fields-storage-dynamodb/__tests__/find.test.js create mode 100644 packages/fields-storage-dynamodb/__tests__/queue.batch.test.js diff --git a/packages/fields-storage-dynamodb/__tests__/delete.test.js b/packages/fields-storage-dynamodb/__tests__/delete.test.js index 4646d84..cdf8ffe 100644 --- a/packages/fields-storage-dynamodb/__tests__/delete.test.js +++ b/packages/fields-storage-dynamodb/__tests__/delete.test.js @@ -1,10 +1,10 @@ import { useModels } from "./models"; import { getName } from "@commodo/name"; -describe("delete test", function() { +describe("delete test", () => { const { models, getDocumentClient } = useModels(); - it("should be able to perform create & update operations", async () => { + it("should be able to perform delete operation", async () => { const { SimpleModel } = models; const simpleModel = new SimpleModel(); simpleModel.populate({ @@ -37,8 +37,7 @@ describe("delete test", function() { } }); - simpleModel.name = "Something-1-edited"; - await simpleModel.save(); + await simpleModel.delete(); item = await getDocumentClient() .get({ @@ -48,16 +47,6 @@ describe("delete test", function() { .promise(); expect(item).toEqual({ - Item: { - sk: "something-1", - name: "Something-1-edited", - pk: "SimpleModel", - slug: "something1Edited", - enabled: true, - age: 55, - tags: ["one", "two", "three"] - } }); - }); }); diff --git a/packages/fields-storage-dynamodb/__tests__/find.test.js b/packages/fields-storage-dynamodb/__tests__/find.test.js new file mode 100644 index 0000000..e285c8a --- /dev/null +++ b/packages/fields-storage-dynamodb/__tests__/find.test.js @@ -0,0 +1,116 @@ +import { useModels } from "./models"; + +describe("find test", function() { + const { models, getDocumentClient } = useModels(); + + beforeAll(async () => { + for (let i = 0; i < 10; i++) { + await getDocumentClient() + .put({ + TableName: "pk-sk", + Item: { + pk: "SimpleModel", + sk: `something-${i}`, + name: `Something-${i}`, + enabled: true, + tags: [i], + age: i * 10 + } + }) + .promise(); + } + }); + + it("should be able to perform basic find queries", async () => { + const { SimpleModel } = models; + + const [something0Entry] = await SimpleModel.find({ + query: { pk: "SimpleModel", sk: "something-0" } + }); + + expect(something0Entry.pk).toBe("SimpleModel"); + expect(something0Entry.sk).toBe("something-0"); + expect(something0Entry.name).toBe("Something-0"); + expect(something0Entry.enabled).toBe(true); + expect(something0Entry.tags).toEqual([0]); + expect(something0Entry.age).toEqual(0); + + const [something4Entry] = await SimpleModel.find({ + query: { pk: "SimpleModel", sk: "something-4" } + }); + + expect(something4Entry.pk).toBe("SimpleModel"); + expect(something4Entry.sk).toBe("something-4"); + expect(something4Entry.name).toBe("Something-4"); + expect(something4Entry.enabled).toBe(true); + expect(something4Entry.tags).toEqual([4]); + expect(something4Entry.age).toEqual(40); + }); + + it("should be able to use basic comparison query operators", async () => { + const { SimpleModel } = models; + + // Should return two instances using the $gte operator. + let results = await SimpleModel.find({ + query: { pk: "SimpleModel", sk: { $gte: "something-8" } } + }); + + expect(results[0].pk).toBe("SimpleModel"); + expect(results[0].sk).toBe("something-8"); + expect(results[0].name).toBe("Something-8"); + + expect(results[1].pk).toBe("SimpleModel"); + expect(results[1].sk).toBe("something-9"); + expect(results[1].name).toBe("Something-9"); + + // Should return 9 instances (zero-indexed item not included in the result set). + results = await SimpleModel.find({ + query: { pk: "SimpleModel", sk: { $gt: "something-0" } } + }); + + expect(results.length).toBe(9); + + // Should return 10 instances (all items included in the result set). + results = await SimpleModel.find({ + query: { pk: "SimpleModel", sk: { $gte: "something-0" } } + }); + + expect(results.length).toBe(10); + }); + + it("should be able to use both ascending and descending ordering", async () => { + const { SimpleModel } = models; + + let results = await SimpleModel.find({ + query: { pk: "SimpleModel", sk: { $beginsWith: "something" } } + }); + + expect(results[0].sk).toBe("something-0"); + expect(results[9].sk).toBe("something-9"); + + // Let's test descending. + results = await SimpleModel.find({ + query: { pk: "SimpleModel", sk: { $beginsWith: "something" } }, + sort: { sk: -1 } + }); + + expect(results[0].sk).toBe("something-9"); + expect(results[9].sk).toBe("something-0"); + }); + + it("should be able to apply limits", async () => { + const { SimpleModel } = models; + + let results = await SimpleModel.find({ + limit: 3, + query: { pk: "SimpleModel", sk: { $beginsWith: "something" } } + }); + + expect(results[0].sk).toBe("something-0"); + expect(results[2].sk).toBe("something-2"); + }); + + it("should be able to paginate results", async () => { + throw new Error("TODO"); + }); +}); diff --git a/packages/fields-storage-dynamodb/__tests__/findOne.test.js b/packages/fields-storage-dynamodb/__tests__/findOne.test.js index 16edae7..c7d08ef 100644 --- a/packages/fields-storage-dynamodb/__tests__/findOne.test.js +++ b/packages/fields-storage-dynamodb/__tests__/findOne.test.js @@ -1,30 +1,104 @@ import { useModels } from "./models"; -describe("findOne test", function() { +describe("find test", function() { const { models, getDocumentClient } = useModels(); - it("should be able to use the findOne method", async () => { - const { SimpleModel } = models; - - for (let i = 0; i < 3; i++) { + beforeAll(async () => { + for (let i = 0; i < 10; i++) { await getDocumentClient() .put({ TableName: "pk-sk", Item: { - pk: `find-one`, - sk: String(i), - name: `one-${i}`, - slug: `one1`, + pk: "SimpleModel", + sk: `something-${i}`, + name: `Something-${i}`, enabled: true, - age: i * 10, - tags: [i] + tags: [i], + age: i * 10 } }) .promise(); } + }); + + it("should be able to perform basic findOne queries", async () => { + const { SimpleModel } = models; - const model1 = await SimpleModel.findOne({ - query: { pk: `find-one`, sk: String("0") } + const something0Entry = await SimpleModel.findOne({ + query: { pk: "SimpleModel", sk: "something-0" } }); + + expect(something0Entry.pk).toBe("SimpleModel"); + expect(something0Entry.sk).toBe("something-0"); + expect(something0Entry.name).toBe("Something-0"); + expect(something0Entry.enabled).toBe(true); + expect(something0Entry.tags).toEqual([0]); + expect(something0Entry.age).toEqual(0); + + const something4Entry = await SimpleModel.findOne({ + query: { pk: "SimpleModel", sk: "something-4" } + }); + + expect(something4Entry.pk).toBe("SimpleModel"); + expect(something4Entry.sk).toBe("something-4"); + expect(something4Entry.name).toBe("Something-4"); + expect(something4Entry.enabled).toBe(true); + expect(something4Entry.tags).toEqual([4]); + expect(something4Entry.age).toEqual(40); + }); + + it("should be able to use basic comparison query operators", async () => { + // Should return something-8 instance. + let result = await SimpleModel.findOne({ + query: { pk: "SimpleModel", sk: { $gte: "something-8" } } + }); + + expect(result.pk).toBe("SimpleModel"); + expect(result.sk).toBe("something-8"); + expect(result.name).toBe("Something-8"); + + result = await SimpleModel.findOne({ + query: { pk: "SimpleModel", sk: { $gt: "something-8" } } + }); + + expect(result.pk).toBe("SimpleModel"); + expect(result.sk).toBe("something-9"); + expect(result.name).toBe("Something-9"); + }); + + it("should be able to use both ascending and descending ordering", async () => { + const { SimpleModel } = models; + + let results = await SimpleModel.find({ + query: { pk: "SimpleModel", sk: { $beginsWith: "something" } } + }); + + expect(results[0].sk).toBe("something-0"); + expect(results[9].sk).toBe("something-9"); + + // Let's test descending. + results = await SimpleModel.find({ + query: { pk: "SimpleModel", sk: { $beginsWith: "something" } }, + sort: { sk: -1 } + }); + + expect(results[0].sk).toBe("something-9"); + expect(results[9].sk).toBe("something-0"); + }); + + it("should be able to apply limits", async () => { + const { SimpleModel } = models; + + let results = await SimpleModel.find({ + limit: 3, + query: { pk: "SimpleModel", sk: { $beginsWith: "something" } } + }); + + expect(results[0].sk).toBe("something-0"); + expect(results[2].sk).toBe("something-2"); + }); + + it("should be able to paginate results", async () => { + throw new Error("TODO"); }); }); diff --git a/packages/fields-storage-dynamodb/__tests__/models/SimpleModel.js b/packages/fields-storage-dynamodb/__tests__/models/SimpleModel.js index ade56f4..9aea95f 100644 --- a/packages/fields-storage-dynamodb/__tests__/models/SimpleModel.js +++ b/packages/fields-storage-dynamodb/__tests__/models/SimpleModel.js @@ -2,8 +2,8 @@ import { compose } from "ramda"; import camelcase from "camelcase"; import { withName } from "@commodo/name"; import { withHooks } from "@commodo/hooks"; -import { withFields, string, boolean, number } from "@commodo/fields"; -import { withPrimaryKey } from "@commodo/fields-storage"; +import { withFields, string, boolean, number, fields } from "@commodo/fields"; +import { withPrimaryKey, withUniqueKey } from "@commodo/fields-storage"; export default base => compose( @@ -26,3 +26,98 @@ export default base => age: number() }) )(base()); + +function somethingDoesntMatter() { + const PageAData = compose( + withName("PageAData"), + withFields({ + tags: string({ list: true }), + title: string() + }) + )(); + const PageCategoryData = compose( + withName("PageCategoryData"), + withFields({ + categoryId: string({ list: true }), + slug: string({ list: true }) + }) + )(); + // The main difference between the previous and DynamoDb approach is that models start NOT to represent + // a single entity, but an item collection (at least this is the technique I went with here). So in this case here, + // the base fields that all items in the collection are using are the PK, SK and GSI* fields, and the rest is + // always written to the "data" field, which can contain a couple of different data models. + const PageItem = compose( + withName("Pages"), + withPrimaryKey("PK", "SK"), + withUniqueKey("GSI1_PK", "GSI1_SK"), + withUniqueKey("GSI2_PK", "GSI2_SK"), + withUniqueKey("GSI3_PK", "GSI3_SK"), + withFields({ + PK: string(), + SK: string(), + GSI1_PK: string(), + GS1_SK: string(), + GSI2_PK: string(), + GS2_SK: string(), + GSI3_PK: string(), + GSI3_SK: string(), + // We use the same technique "ref" field has. We pass an array to the instanceOf property, and define + // the field in which we write the actual model used. We used this with "ref" fields in a couple of places. + // So, we'd just need to have this option here too. + data: fields({ + instanceOf: [PageAData, PageCategoryData], + refNameField: "dataModel" + }), + dataModel: string() // PageAData, PageCategoryData, PageTagData ... + }), + withIdentity(), + withHooks({ + async afterSave() { + // A check should be implemented here first, in order to determine which type of item we are saving. + // The operations below are only specific for saving A records. + // Once we save the A record, we can then save other items inside of a transaction or a batch operation. + // Whatever fits the particular needs better... + // const batch = new Batch(); + const transaction = new Transaction(); + const pageCategory = new PageItem(); + pageCategory.PK = this.id; + pageCategory.SK = `Category`; + pageCategory.GSI1_PK = `Page#${this.data.category}`; + pageCategory.GSI1_SK = `savedOn#${new Date()}`; + pageCategory.GSI3_PK = `Category#${this.data.category}`; + pageCategory.GSI3_SK = `${new Date()}`; + pageCategory.data = new PageCategoryData().populate({ + category: "1", + slug: "static" + }); + transaction.push(pageCategory.save); + for (let i = 0; i < this.data.tags.length; i++) { + let tag = this.data.tags[i]; + const pageTag = new PageItem().populate({ + PK: this.id, + SK: `Tag#${tag}`, + GSI3_PK: `Tag#${tag}`, + GSI3_SK: new Date() + }); + transaction.push(pageTag.save); + } + await transaction.send(); + // new Batch({}); + } + }) + )(base()); + // Creating a new page. Maybe some of these could be sent to beforeSave hook... but we'll have to be careful, so that other classes don't execute the same logic.. For example when saving a new `Page#2 / Tag#universe` entry, which will use the same PageItem class, we probably don't want to execute some of the `beforeSave` stuff. A good example is the stuff that's above in the `afterSave` hook, those shouldn't be there, because we only want to trigger those on the A record save. + const page = new PageItem(); + page.pk = "Page#1"; + page.sk = "A"; + page.GSI2_PK = "User#2"; + page.GSI2_SK = new Date(); + page.GSI3_PK = "Page"; + page.GSI3_SK = new Date(); + page.data = new PageAData().populate({ + category: "static", + title: "Welcome to Webiny!", + tags: ["science", "universe"] + }); + return PageItem; +} diff --git a/packages/fields-storage-dynamodb/__tests__/queue.batch.test.js b/packages/fields-storage-dynamodb/__tests__/queue.batch.test.js new file mode 100644 index 0000000..f3d9bec --- /dev/null +++ b/packages/fields-storage-dynamodb/__tests__/queue.batch.test.js @@ -0,0 +1,265 @@ +import { useModels } from "./models"; +import { Batch } from "@commodo/fields-storage"; + +describe("save test", function() { + const { models, getDocumentClient } = useModels(); + + it("should be able to batch save, create, and update calls", async () => { + const { SimpleModel } = models; + + const a = new SimpleModel().populate({ pk: "SimpleModel", sk: "a" }); + const b = new SimpleModel().populate({ pk: "SimpleModel", sk: "b" }); + const c = new SimpleModel().populate({ pk: "SimpleModel", sk: "c" }); + + const batchWriteSpy = jest.spyOn(SimpleModel.getStorageDriver().getClient(), "batchWrite"); + const putSpy = jest.spyOn(SimpleModel.getStorageDriver().getClient(), "put"); + const updateSpy = jest.spyOn(SimpleModel.getStorageDriver().getClient(), "update"); + + let batch = new Batch([a, "save"], [b, "save"], [c, "save"]); + await batch.execute(); + + expect(batchWriteSpy).toHaveBeenCalledTimes(1); + + let items = await getDocumentClient() + .query({ + TableName: "pk-sk", + KeyConditionExpression: "pk = :pk and sk >= :sk", + ExpressionAttributeValues: { + ":pk": "SimpleModel", + ":sk": "a" + } + }) + .promise(); + + expect(items).toEqual({ + Items: [ + { sk: "a", enabled: true, pk: "SimpleModel" }, + { sk: "b", enabled: true, pk: "SimpleModel" }, + { sk: "c", enabled: true, pk: "SimpleModel" } + ], + Count: 3, + ScannedCount: 3 + }); + + // Now, try to insert an item via the static "create" method. + batch = new Batch( + [SimpleModel, "create", { data: { pk: "SimpleModel", sk: "d" } }], + [SimpleModel, "create", { data: { pk: "SimpleModel", sk: "e" } }], + [SimpleModel, "create", { data: { pk: "SimpleModel", sk: "f" } }] + ); + + await batch.execute(); + + expect(batchWriteSpy).toHaveBeenCalledTimes(2); + + items = await getDocumentClient() + .query({ + TableName: "pk-sk", + KeyConditionExpression: "pk = :pk and sk >= :sk", + ExpressionAttributeValues: { + ":pk": "SimpleModel", + ":sk": "a" + } + }) + .promise(); + + expect(items).toEqual({ + Count: 6, + Items: [ + { + enabled: true, + pk: "SimpleModel", + sk: "a" + }, + { + enabled: true, + pk: "SimpleModel", + sk: "b" + }, + { + enabled: true, + pk: "SimpleModel", + sk: "c" + }, + { + pk: "SimpleModel", + sk: "d" + }, + { + pk: "SimpleModel", + sk: "e" + }, + { + pk: "SimpleModel", + sk: "f" + } + ], + ScannedCount: 6 + }); + + // Let's do updates now. First, update a, b, and c instances. + a.enabled = false; + b.enabled = false; + c.enabled = false; + + batch = new Batch([a, "save"], [b, "save"], [c, "save"]); + await batch.execute(); + expect(batchWriteSpy).toHaveBeenCalledTimes(3); + + items = await getDocumentClient() + .query({ + TableName: "pk-sk", + KeyConditionExpression: "pk = :pk and sk >= :sk", + ExpressionAttributeValues: { + ":pk": "SimpleModel", + ":sk": "a" + } + }) + .promise(); + + expect(items).toEqual({ + Count: 6, + Items: [ + { + age: null, + enabled: false, + name: null, + pk: "SimpleModel", + sk: "a", + slug: null, + tags: null + }, + { + age: null, + enabled: false, + name: null, + pk: "SimpleModel", + sk: "b", + slug: null, + tags: null + }, + { + age: null, + enabled: false, + name: null, + pk: "SimpleModel", + sk: "c", + slug: null, + tags: null + }, + { + pk: "SimpleModel", + sk: "d" + }, + { + pk: "SimpleModel", + sk: "e" + }, + { + pk: "SimpleModel", + sk: "f" + } + ], + ScannedCount: 6 + }); + + // Once we have that working, let's update d, e, and f items, which were inserted statically. + batch = new Batch( + [ + SimpleModel, + "update", + { + query: { pk: "SimpleModel", sk: "d" }, + data: { pk: "SimpleModel", sk: "d", enabled: false } + } + ], + [ + SimpleModel, + "update", + { + query: { pk: "SimpleModel", sk: "e" }, + data: { pk: "SimpleModel", sk: "e", enabled: false } + } + ], + [ + SimpleModel, + "update", + { + query: { pk: "SimpleModel", sk: "f" }, + data: { pk: "SimpleModel", sk: "f", enabled: false } + } + ] + ); + + await batch.execute(); + + expect(batchWriteSpy).toHaveBeenCalledTimes(4); + + items = await getDocumentClient() + .query({ + TableName: "pk-sk", + KeyConditionExpression: "pk = :pk and sk >= :sk", + ExpressionAttributeValues: { + ":pk": "SimpleModel", + ":sk": "a" + } + }) + .promise(); + + expect(items).toEqual({ + Count: 6, + Items: [ + { + age: null, + enabled: false, + name: null, + pk: "SimpleModel", + sk: "a", + slug: null, + tags: null + }, + { + age: null, + enabled: false, + name: null, + pk: "SimpleModel", + sk: "b", + slug: null, + tags: null + }, + { + age: null, + enabled: false, + name: null, + pk: "SimpleModel", + sk: "c", + slug: null, + tags: null + }, + { + pk: "SimpleModel", + sk: "d", + enabled: false + }, + { + pk: "SimpleModel", + sk: "e", + enabled: false + }, + { + pk: "SimpleModel", + sk: "f", + enabled: false + } + ], + ScannedCount: 6 + }); + + expect(putSpy).toHaveBeenCalledTimes(0); + expect(updateSpy).toHaveBeenCalledTimes(0); + + batchWriteSpy.mockRestore(); + putSpy.mockRestore(); + updateSpy.mockRestore(); + }); +}); From 70036e4e8f39b6c16158fa163bc644a162aeaaab Mon Sep 17 00:00:00 2001 From: Adrian Smijulj Date: Mon, 7 Sep 2020 09:11:18 +0200 Subject: [PATCH 12/26] chore: remove all-contributors --- package.json | 3 --- 1 file changed, 3 deletions(-) diff --git a/package.json b/package.json index 7362a82..c977233 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,6 @@ "@babel/preset-env": "^7.4.3", "@babel/preset-flow": "^7.0.0", "adio": "^1.1.1", - "all-contributors-cli": "^5.6.0", "babel-eslint": "^10.0.3", "babel-jest": "^24.7.1", "camelcase": "^5.2.0", @@ -45,8 +44,6 @@ "test:coverage": "yarn test:src --coverage", "test:coverage:coveralls": "npm run test:coverage && cat ./coverage/lcov.info | coveralls", "lint-staged": "lint-staged", - "contributors:add": "all-contributors add", - "contributors:generate": "all-contributors generate", "lerna:version": "lerna version --yes", "lerna:publish": "lerna publish --yes", "lerna:rm-dist": "lerna exec -- rm -rf dist", From 92c30f621c4301022ee0f3b151721a91da954820 Mon Sep 17 00:00:00 2001 From: Adrian Smijulj Date: Mon, 7 Sep 2020 11:55:04 +0200 Subject: [PATCH 13/26] test: add tests --- .../__tests__/delete.test.js | 15 ++- .../__tests__/find.test.js | 68 +++++++----- .../__tests__/findOne.test.js | 80 +++++++------- .../__tests__/models/SimpleModel.js | 103 ++---------------- .../__tests__/models/useModels.js | 8 +- .../__tests__/queue.batch.delete.test.js | 99 +++++++++++++++++ ...batch.test.js => queue.batch.save.test.js} | 78 ++++++------- .../__tests__/save.test.js | 13 +-- 8 files changed, 239 insertions(+), 225 deletions(-) create mode 100644 packages/fields-storage-dynamodb/__tests__/queue.batch.delete.test.js rename packages/fields-storage-dynamodb/__tests__/{queue.batch.test.js => queue.batch.save.test.js} (74%) diff --git a/packages/fields-storage-dynamodb/__tests__/delete.test.js b/packages/fields-storage-dynamodb/__tests__/delete.test.js index cdf8ffe..54923e9 100644 --- a/packages/fields-storage-dynamodb/__tests__/delete.test.js +++ b/packages/fields-storage-dynamodb/__tests__/delete.test.js @@ -1,14 +1,14 @@ import { useModels } from "./models"; -import { getName } from "@commodo/name"; describe("delete test", () => { - const { models, getDocumentClient } = useModels(); + const { models, getDocumentClient, id: pk } = useModels(); it("should be able to perform delete operation", async () => { const { SimpleModel } = models; + const simpleModel = new SimpleModel(); simpleModel.populate({ - pk: getName(SimpleModel), + pk, sk: "something-1", name: "Something-1", enabled: true, @@ -21,7 +21,7 @@ describe("delete test", () => { let item = await getDocumentClient() .get({ TableName: "pk-sk", - Key: { pk: getName(SimpleModel), sk: "something-1" } + Key: { pk, sk: "something-1" } }) .promise(); @@ -29,7 +29,7 @@ describe("delete test", () => { Item: { sk: "something-1", name: "Something-1", - pk: "SimpleModel", + pk, slug: "something1", enabled: true, age: 55, @@ -42,11 +42,10 @@ describe("delete test", () => { item = await getDocumentClient() .get({ TableName: "pk-sk", - Key: { pk: getName(SimpleModel), sk: "something-1" } + Key: { pk, sk: "something-1" } }) .promise(); - expect(item).toEqual({ - }); + expect(item).toEqual({}); }); }); diff --git a/packages/fields-storage-dynamodb/__tests__/find.test.js b/packages/fields-storage-dynamodb/__tests__/find.test.js index e285c8a..34f3cc2 100644 --- a/packages/fields-storage-dynamodb/__tests__/find.test.js +++ b/packages/fields-storage-dynamodb/__tests__/find.test.js @@ -1,16 +1,20 @@ import { useModels } from "./models"; describe("find test", function() { - const { models, getDocumentClient } = useModels(); + const { models, getDocumentClient, id: pk } = useModels(); - beforeAll(async () => { + beforeEach(async () => { for (let i = 0; i < 10; i++) { await getDocumentClient() .put({ TableName: "pk-sk", Item: { - pk: "SimpleModel", + pk, sk: `something-${i}`, + gsi1pk: `gsi1-${pk}`, + gsi1sk: `gsi1-something-${i}`, + gsi2pk: `gsi2-${pk}`, + gsi2sk: `gsi2-something-${i}`, name: `Something-${i}`, enabled: true, tags: [i], @@ -25,10 +29,10 @@ describe("find test", function() { const { SimpleModel } = models; const [something0Entry] = await SimpleModel.find({ - query: { pk: "SimpleModel", sk: "something-0" } + query: { pk, sk: "something-0" } }); - expect(something0Entry.pk).toBe("SimpleModel"); + expect(something0Entry.pk).toBe(pk); expect(something0Entry.sk).toBe("something-0"); expect(something0Entry.name).toBe("Something-0"); expect(something0Entry.enabled).toBe(true); @@ -36,10 +40,10 @@ describe("find test", function() { expect(something0Entry.age).toEqual(0); const [something4Entry] = await SimpleModel.find({ - query: { pk: "SimpleModel", sk: "something-4" } + query: { pk: pk, sk: "something-4" } }); - expect(something4Entry.pk).toBe("SimpleModel"); + expect(something4Entry.pk).toBe(pk); expect(something4Entry.sk).toBe("something-4"); expect(something4Entry.name).toBe("Something-4"); expect(something4Entry.enabled).toBe(true); @@ -52,65 +56,73 @@ describe("find test", function() { // Should return two instances using the $gte operator. let results = await SimpleModel.find({ - query: { pk: "SimpleModel", sk: { $gte: "something-8" } } + query: { pk: pk, sk: { $gte: "something-8" } } }); - expect(results[0].pk).toBe("SimpleModel"); + expect(results[0].pk).toBe(pk); expect(results[0].sk).toBe("something-8"); expect(results[0].name).toBe("Something-8"); - expect(results[1].pk).toBe("SimpleModel"); + expect(results[1].pk).toBe(pk); expect(results[1].sk).toBe("something-9"); expect(results[1].name).toBe("Something-9"); // Should return 9 instances (zero-indexed item not included in the result set). results = await SimpleModel.find({ - query: { pk: "SimpleModel", sk: { $gt: "something-0" } } + query: { pk: pk, sk: { $gt: "something-0" } } }); expect(results.length).toBe(9); // Should return 10 instances (all items included in the result set). results = await SimpleModel.find({ - query: { pk: "SimpleModel", sk: { $gte: "something-0" } } + query: { pk: pk, sk: { $gte: "something-0" } } }); expect(results.length).toBe(10); }); it("should be able to use both ascending and descending ordering", async () => { + // TODO + }); + + it("should be able to apply limits", async () => { const { SimpleModel } = models; let results = await SimpleModel.find({ - query: { pk: "SimpleModel", sk: { $beginsWith: "something" } } + limit: 3, + query: { pk: pk, sk: { $beginsWith: "something" } } }); expect(results[0].sk).toBe("something-0"); - expect(results[9].sk).toBe("something-9"); - - // Let's test descending. - results = await SimpleModel.find({ - query: { pk: "SimpleModel", sk: { $beginsWith: "something" } }, - sort: { sk: -1 } - }); + expect(results[2].sk).toBe("something-2"); + }); - expect(results[0].sk).toBe("something-9"); - expect(results[9].sk).toBe("something-0"); + it("should be able to paginate results", async () => { + // TODO: }); - it("should be able to apply limits", async () => { + it("should be able query GSIs", async () => { const { SimpleModel } = models; let results = await SimpleModel.find({ - limit: 3, - query: { pk: "SimpleModel", sk: { $beginsWith: "something" } } + query: { gsi1pk: "gsi1-SimpleModel", gsi1sk: { $beginsWith: "gsi1-something" } } }); + expect(results.length).toBe(10); + expect(results[0].sk).toBe("something-0"); expect(results[2].sk).toBe("something-2"); - }); + expect(results[9].sk).toBe("something-9"); - it("should be able to paginate results", async () => { - throw new Error("TODO"); + results = await SimpleModel.find({ + query: { gsi2pk: "gsi2-SimpleModel", gsi2sk: { $lte: "gsi2-something-3" } } + }); + + expect(results.length).toBe(4); + + expect(results[0].sk).toBe("something-0"); + expect(results[2].sk).toBe("something-2"); + expect(results[3].sk).toBe("something-3"); }); }); diff --git a/packages/fields-storage-dynamodb/__tests__/findOne.test.js b/packages/fields-storage-dynamodb/__tests__/findOne.test.js index c7d08ef..6bf6d05 100644 --- a/packages/fields-storage-dynamodb/__tests__/findOne.test.js +++ b/packages/fields-storage-dynamodb/__tests__/findOne.test.js @@ -1,7 +1,7 @@ import { useModels } from "./models"; describe("find test", function() { - const { models, getDocumentClient } = useModels(); + const { models, getDocumentClient, id: pk } = useModels(); beforeAll(async () => { for (let i = 0; i < 10; i++) { @@ -9,8 +9,12 @@ describe("find test", function() { .put({ TableName: "pk-sk", Item: { - pk: "SimpleModel", + pk: pk, sk: `something-${i}`, + gsi1pk: "gsi1-SimpleModel", + gsi1sk: `gsi1-something-${i}`, + gsi2pk: "gsi2-SimpleModel", + gsi2sk: `gsi2-something-${i}`, name: `Something-${i}`, enabled: true, tags: [i], @@ -25,10 +29,10 @@ describe("find test", function() { const { SimpleModel } = models; const something0Entry = await SimpleModel.findOne({ - query: { pk: "SimpleModel", sk: "something-0" } + query: { pk: pk, sk: "something-0" } }); - expect(something0Entry.pk).toBe("SimpleModel"); + expect(something0Entry.pk).toBe(pk); expect(something0Entry.sk).toBe("something-0"); expect(something0Entry.name).toBe("Something-0"); expect(something0Entry.enabled).toBe(true); @@ -36,10 +40,10 @@ describe("find test", function() { expect(something0Entry.age).toEqual(0); const something4Entry = await SimpleModel.findOne({ - query: { pk: "SimpleModel", sk: "something-4" } + query: { pk: pk, sk: "something-4" } }); - expect(something4Entry.pk).toBe("SimpleModel"); + expect(something4Entry.pk).toBe(pk); expect(something4Entry.sk).toBe("something-4"); expect(something4Entry.name).toBe("Something-4"); expect(something4Entry.enabled).toBe(true); @@ -48,57 +52,47 @@ describe("find test", function() { }); it("should be able to use basic comparison query operators", async () => { - // Should return something-8 instance. - let result = await SimpleModel.findOne({ - query: { pk: "SimpleModel", sk: { $gte: "something-8" } } - }); - - expect(result.pk).toBe("SimpleModel"); - expect(result.sk).toBe("something-8"); - expect(result.name).toBe("Something-8"); + const { SimpleModel } = models; - result = await SimpleModel.findOne({ - query: { pk: "SimpleModel", sk: { $gt: "something-8" } } + // Should return index 8 instance. + let somethingEntry = await SimpleModel.findOne({ + query: { pk: pk, sk: { $gte: "something-8" } } }); - expect(result.pk).toBe("SimpleModel"); - expect(result.sk).toBe("something-9"); - expect(result.name).toBe("Something-9"); - }); + expect(somethingEntry.pk).toBe(pk); + expect(somethingEntry.sk).toBe("something-8"); + expect(somethingEntry.name).toBe("Something-8"); - it("should be able to use both ascending and descending ordering", async () => { - const { SimpleModel } = models; + // Should return index 9 instance. - let results = await SimpleModel.find({ - query: { pk: "SimpleModel", sk: { $beginsWith: "something" } } + somethingEntry = await SimpleModel.findOne({ + query: { pk: pk, sk: { $gt: "something-8" } } }); - expect(results[0].sk).toBe("something-0"); - expect(results[9].sk).toBe("something-9"); - - // Let's test descending. - results = await SimpleModel.find({ - query: { pk: "SimpleModel", sk: { $beginsWith: "something" } }, - sort: { sk: -1 } - }); + expect(somethingEntry.pk).toBe(pk); + expect(somethingEntry.sk).toBe("something-9"); + expect(somethingEntry.name).toBe("Something-9"); + }); - expect(results[0].sk).toBe("something-9"); - expect(results[9].sk).toBe("something-0"); + it("should be able to use both ascending and descending ordering", async () => { + // TODO }); - it("should be able to apply limits", async () => { + it("should be able query GSIs", async () => { const { SimpleModel } = models; - let results = await SimpleModel.find({ - limit: 3, - query: { pk: "SimpleModel", sk: { $beginsWith: "something" } } + let result = await SimpleModel.findOne({ + query: { gsi1pk: "gsi1-SimpleModel", gsi1sk: "gsi1-something-3" } }); - expect(results[0].sk).toBe("something-0"); - expect(results[2].sk).toBe("something-2"); - }); + result.sk = 'something-3'; + result.gsi1sk = 'gsi1-something-3'; + + result = await SimpleModel.findOne({ + query: { gsi2pk: "gsi2-SimpleModel", gsi2sk: { $lt: "gsi2-something-3" } } + }); - it("should be able to paginate results", async () => { - throw new Error("TODO"); + result.sk = 'something-2'; + result.gsi1sk = 'gsi1-something-2'; }); }); diff --git a/packages/fields-storage-dynamodb/__tests__/models/SimpleModel.js b/packages/fields-storage-dynamodb/__tests__/models/SimpleModel.js index 9aea95f..8d67557 100644 --- a/packages/fields-storage-dynamodb/__tests__/models/SimpleModel.js +++ b/packages/fields-storage-dynamodb/__tests__/models/SimpleModel.js @@ -2,7 +2,7 @@ import { compose } from "ramda"; import camelcase from "camelcase"; import { withName } from "@commodo/name"; import { withHooks } from "@commodo/hooks"; -import { withFields, string, boolean, number, fields } from "@commodo/fields"; +import { withFields, string, boolean, number } from "@commodo/fields"; import { withPrimaryKey, withUniqueKey } from "@commodo/fields-storage"; export default base => @@ -16,9 +16,15 @@ export default base => } }), withPrimaryKey("pk", "sk"), + withUniqueKey("gsi1pk", "gsi1sk"), + withUniqueKey("gsi2pk", "gsi2sk"), withFields({ pk: string(), sk: string(), + gsi1pk: string(), + gsi1sk: string(), + gsi2pk: string(), + gsi2sk: string(), name: string(), slug: string(), enabled: boolean({ value: true }), @@ -26,98 +32,3 @@ export default base => age: number() }) )(base()); - -function somethingDoesntMatter() { - const PageAData = compose( - withName("PageAData"), - withFields({ - tags: string({ list: true }), - title: string() - }) - )(); - const PageCategoryData = compose( - withName("PageCategoryData"), - withFields({ - categoryId: string({ list: true }), - slug: string({ list: true }) - }) - )(); - // The main difference between the previous and DynamoDb approach is that models start NOT to represent - // a single entity, but an item collection (at least this is the technique I went with here). So in this case here, - // the base fields that all items in the collection are using are the PK, SK and GSI* fields, and the rest is - // always written to the "data" field, which can contain a couple of different data models. - const PageItem = compose( - withName("Pages"), - withPrimaryKey("PK", "SK"), - withUniqueKey("GSI1_PK", "GSI1_SK"), - withUniqueKey("GSI2_PK", "GSI2_SK"), - withUniqueKey("GSI3_PK", "GSI3_SK"), - withFields({ - PK: string(), - SK: string(), - GSI1_PK: string(), - GS1_SK: string(), - GSI2_PK: string(), - GS2_SK: string(), - GSI3_PK: string(), - GSI3_SK: string(), - // We use the same technique "ref" field has. We pass an array to the instanceOf property, and define - // the field in which we write the actual model used. We used this with "ref" fields in a couple of places. - // So, we'd just need to have this option here too. - data: fields({ - instanceOf: [PageAData, PageCategoryData], - refNameField: "dataModel" - }), - dataModel: string() // PageAData, PageCategoryData, PageTagData ... - }), - withIdentity(), - withHooks({ - async afterSave() { - // A check should be implemented here first, in order to determine which type of item we are saving. - // The operations below are only specific for saving A records. - // Once we save the A record, we can then save other items inside of a transaction or a batch operation. - // Whatever fits the particular needs better... - // const batch = new Batch(); - const transaction = new Transaction(); - const pageCategory = new PageItem(); - pageCategory.PK = this.id; - pageCategory.SK = `Category`; - pageCategory.GSI1_PK = `Page#${this.data.category}`; - pageCategory.GSI1_SK = `savedOn#${new Date()}`; - pageCategory.GSI3_PK = `Category#${this.data.category}`; - pageCategory.GSI3_SK = `${new Date()}`; - pageCategory.data = new PageCategoryData().populate({ - category: "1", - slug: "static" - }); - transaction.push(pageCategory.save); - for (let i = 0; i < this.data.tags.length; i++) { - let tag = this.data.tags[i]; - const pageTag = new PageItem().populate({ - PK: this.id, - SK: `Tag#${tag}`, - GSI3_PK: `Tag#${tag}`, - GSI3_SK: new Date() - }); - transaction.push(pageTag.save); - } - await transaction.send(); - // new Batch({}); - } - }) - )(base()); - // Creating a new page. Maybe some of these could be sent to beforeSave hook... but we'll have to be careful, so that other classes don't execute the same logic.. For example when saving a new `Page#2 / Tag#universe` entry, which will use the same PageItem class, we probably don't want to execute some of the `beforeSave` stuff. A good example is the stuff that's above in the `afterSave` hook, those shouldn't be there, because we only want to trigger those on the A record save. - const page = new PageItem(); - page.pk = "Page#1"; - page.sk = "A"; - page.GSI2_PK = "User#2"; - page.GSI2_SK = new Date(); - page.GSI3_PK = "Page"; - page.GSI3_SK = new Date(); - page.data = new PageAData().populate({ - category: "static", - title: "Welcome to Webiny!", - tags: ["science", "universe"] - }); - return PageItem; -} diff --git a/packages/fields-storage-dynamodb/__tests__/models/useModels.js b/packages/fields-storage-dynamodb/__tests__/models/useModels.js index 52babd5..24bf908 100644 --- a/packages/fields-storage-dynamodb/__tests__/models/useModels.js +++ b/packages/fields-storage-dynamodb/__tests__/models/useModels.js @@ -2,14 +2,16 @@ import { DocumentClient } from "aws-sdk/clients/dynamodb"; import { withStorage } from "@commodo/fields-storage"; import { DynamoDbDriver } from "@commodo/fields-storage-dynamodb"; import { compose } from "ramda"; +import uniqid from "uniqid"; // Models. import simpleModel from "./SimpleModel"; -export default ({ init = true } = {}) => { +export default () => { const self = { models: {}, documentClient: null, + id: uniqid(), beforeAll: () => { self.documentClient = new DocumentClient({ convertEmptyValues: true, @@ -37,9 +39,7 @@ export default ({ init = true } = {}) => { } }; - if (init !== false) { - beforeAll(self.beforeAll); - } + beforeAll(self.beforeAll); return self; }; diff --git a/packages/fields-storage-dynamodb/__tests__/queue.batch.delete.test.js b/packages/fields-storage-dynamodb/__tests__/queue.batch.delete.test.js new file mode 100644 index 0000000..3004589 --- /dev/null +++ b/packages/fields-storage-dynamodb/__tests__/queue.batch.delete.test.js @@ -0,0 +1,99 @@ +import { useModels } from "./models"; +import { Batch } from "@commodo/fields-storage"; + +describe("batch save test", function() { + const { models, getDocumentClient, id: pk } = useModels(); + + beforeAll(async () => { + for (let i = 0; i < 6; i++) { + await getDocumentClient() + .put({ + TableName: "pk-sk", + Item: { + pk: pk, + sk: `something-${i}` + } + }) + .promise(); + } + }); + + it("should be able to batch save, create, and update calls", async () => { + const { SimpleModel } = models; + + const a = new SimpleModel().populate({ pk: pk, sk: "something-0" }); + const b = new SimpleModel().populate({ pk: pk, sk: "something-1" }); + const c = new SimpleModel().populate({ pk: pk, sk: "something-2" }); + + const batchWriteSpy = jest.spyOn(SimpleModel.getStorageDriver().getClient(), "batchWrite"); + const deleteSpy = jest.spyOn(SimpleModel.getStorageDriver().getClient(), "delete"); + + let batch = new Batch([a, "delete"], [b, "delete"], [c, "delete"]); + await batch.execute(); + + expect(batchWriteSpy).toHaveBeenCalledTimes(1); + + let items = await getDocumentClient() + .query({ + TableName: "pk-sk", + KeyConditionExpression: "pk = :pk and sk >= :sk", + ExpressionAttributeValues: { + ":pk": pk, + ":sk": "a" + } + }) + .promise(); + + expect(items).toEqual({ + "Count": 3, + "Items": [ + { + "pk": pk, + "sk": "something-3" + }, + { + "pk": pk, + "sk": "something-4" + }, + { + "pk": pk, + "sk": "something-5" + } + ], + "ScannedCount": 3 + }); + + // Now, try to insert an item via the static "create" method. + batch = new Batch( + [SimpleModel, "delete", { query: { pk: pk, sk: "something-3" } }], + [SimpleModel, "delete", { query: { pk: pk, sk: "something-4" } }], + [SimpleModel, "delete", { query: { pk: pk, sk: "something-5" } }] + ); + + await batch.execute(); + + expect(batchWriteSpy).toHaveBeenCalledTimes(2); + + items = await getDocumentClient() + .query({ + TableName: "pk-sk", + KeyConditionExpression: "pk = :pk and sk >= :sk", + ExpressionAttributeValues: { + ":pk": pk, + ":sk": "a" + } + }) + .promise(); + + expect(items).toEqual({ + "Count": 0, + "Items": [], + "ScannedCount": 0 + }); + + expect(deleteSpy).toHaveBeenCalledTimes(0); + + batchWriteSpy.mockRestore(); + deleteSpy.mockRestore(); + }); +}); diff --git a/packages/fields-storage-dynamodb/__tests__/queue.batch.test.js b/packages/fields-storage-dynamodb/__tests__/queue.batch.save.test.js similarity index 74% rename from packages/fields-storage-dynamodb/__tests__/queue.batch.test.js rename to packages/fields-storage-dynamodb/__tests__/queue.batch.save.test.js index f3d9bec..bf351b8 100644 --- a/packages/fields-storage-dynamodb/__tests__/queue.batch.test.js +++ b/packages/fields-storage-dynamodb/__tests__/queue.batch.save.test.js @@ -1,15 +1,15 @@ import { useModels } from "./models"; import { Batch } from "@commodo/fields-storage"; -describe("save test", function() { - const { models, getDocumentClient } = useModels(); +describe("batch save test", function() { + const { models, getDocumentClient, id: pk } = useModels(); it("should be able to batch save, create, and update calls", async () => { const { SimpleModel } = models; - const a = new SimpleModel().populate({ pk: "SimpleModel", sk: "a" }); - const b = new SimpleModel().populate({ pk: "SimpleModel", sk: "b" }); - const c = new SimpleModel().populate({ pk: "SimpleModel", sk: "c" }); + const a = new SimpleModel().populate({ pk: pk, sk: "a" }); + const b = new SimpleModel().populate({ pk: pk, sk: "b" }); + const c = new SimpleModel().populate({ pk: pk, sk: "c" }); const batchWriteSpy = jest.spyOn(SimpleModel.getStorageDriver().getClient(), "batchWrite"); const putSpy = jest.spyOn(SimpleModel.getStorageDriver().getClient(), "put"); @@ -25,7 +25,7 @@ describe("save test", function() { TableName: "pk-sk", KeyConditionExpression: "pk = :pk and sk >= :sk", ExpressionAttributeValues: { - ":pk": "SimpleModel", + ":pk": pk, ":sk": "a" } }) @@ -33,9 +33,9 @@ describe("save test", function() { expect(items).toEqual({ Items: [ - { sk: "a", enabled: true, pk: "SimpleModel" }, - { sk: "b", enabled: true, pk: "SimpleModel" }, - { sk: "c", enabled: true, pk: "SimpleModel" } + { sk: "a", enabled: true, pk: pk }, + { sk: "b", enabled: true, pk: pk }, + { sk: "c", enabled: true, pk: pk } ], Count: 3, ScannedCount: 3 @@ -43,9 +43,9 @@ describe("save test", function() { // Now, try to insert an item via the static "create" method. batch = new Batch( - [SimpleModel, "create", { data: { pk: "SimpleModel", sk: "d" } }], - [SimpleModel, "create", { data: { pk: "SimpleModel", sk: "e" } }], - [SimpleModel, "create", { data: { pk: "SimpleModel", sk: "f" } }] + [SimpleModel, "create", { data: { pk: pk, sk: "d" } }], + [SimpleModel, "create", { data: { pk: pk, sk: "e" } }], + [SimpleModel, "create", { data: { pk: pk, sk: "f" } }] ); await batch.execute(); @@ -57,7 +57,7 @@ describe("save test", function() { TableName: "pk-sk", KeyConditionExpression: "pk = :pk and sk >= :sk", ExpressionAttributeValues: { - ":pk": "SimpleModel", + ":pk": pk, ":sk": "a" } }) @@ -68,29 +68,29 @@ describe("save test", function() { Items: [ { enabled: true, - pk: "SimpleModel", + pk: pk, sk: "a" }, { enabled: true, - pk: "SimpleModel", + pk: pk, sk: "b" }, { enabled: true, - pk: "SimpleModel", + pk: pk, sk: "c" }, { - pk: "SimpleModel", + pk: pk, sk: "d" }, { - pk: "SimpleModel", + pk: pk, sk: "e" }, { - pk: "SimpleModel", + pk: pk, sk: "f" } ], @@ -111,7 +111,7 @@ describe("save test", function() { TableName: "pk-sk", KeyConditionExpression: "pk = :pk and sk >= :sk", ExpressionAttributeValues: { - ":pk": "SimpleModel", + ":pk": pk, ":sk": "a" } }) @@ -124,7 +124,7 @@ describe("save test", function() { age: null, enabled: false, name: null, - pk: "SimpleModel", + pk: pk, sk: "a", slug: null, tags: null @@ -133,7 +133,7 @@ describe("save test", function() { age: null, enabled: false, name: null, - pk: "SimpleModel", + pk: pk, sk: "b", slug: null, tags: null @@ -142,21 +142,21 @@ describe("save test", function() { age: null, enabled: false, name: null, - pk: "SimpleModel", + pk: pk, sk: "c", slug: null, tags: null }, { - pk: "SimpleModel", + pk: pk, sk: "d" }, { - pk: "SimpleModel", + pk: pk, sk: "e" }, { - pk: "SimpleModel", + pk: pk, sk: "f" } ], @@ -169,24 +169,24 @@ describe("save test", function() { SimpleModel, "update", { - query: { pk: "SimpleModel", sk: "d" }, - data: { pk: "SimpleModel", sk: "d", enabled: false } + query: { pk: pk, sk: "d" }, + data: { pk: pk, sk: "d", enabled: false } } ], [ SimpleModel, "update", { - query: { pk: "SimpleModel", sk: "e" }, - data: { pk: "SimpleModel", sk: "e", enabled: false } + query: { pk: pk, sk: "e" }, + data: { pk: pk, sk: "e", enabled: false } } ], [ SimpleModel, "update", { - query: { pk: "SimpleModel", sk: "f" }, - data: { pk: "SimpleModel", sk: "f", enabled: false } + query: { pk: pk, sk: "f" }, + data: { pk: pk, sk: "f", enabled: false } } ] ); @@ -200,7 +200,7 @@ describe("save test", function() { TableName: "pk-sk", KeyConditionExpression: "pk = :pk and sk >= :sk", ExpressionAttributeValues: { - ":pk": "SimpleModel", + ":pk": pk, ":sk": "a" } }) @@ -213,7 +213,7 @@ describe("save test", function() { age: null, enabled: false, name: null, - pk: "SimpleModel", + pk: pk, sk: "a", slug: null, tags: null @@ -222,7 +222,7 @@ describe("save test", function() { age: null, enabled: false, name: null, - pk: "SimpleModel", + pk: pk, sk: "b", slug: null, tags: null @@ -231,23 +231,23 @@ describe("save test", function() { age: null, enabled: false, name: null, - pk: "SimpleModel", + pk: pk, sk: "c", slug: null, tags: null }, { - pk: "SimpleModel", + pk: pk, sk: "d", enabled: false }, { - pk: "SimpleModel", + pk: pk, sk: "e", enabled: false }, { - pk: "SimpleModel", + pk: pk, sk: "f", enabled: false } diff --git a/packages/fields-storage-dynamodb/__tests__/save.test.js b/packages/fields-storage-dynamodb/__tests__/save.test.js index 420dae2..06165cb 100644 --- a/packages/fields-storage-dynamodb/__tests__/save.test.js +++ b/packages/fields-storage-dynamodb/__tests__/save.test.js @@ -1,14 +1,13 @@ import { useModels } from "./models"; -import { getName } from "@commodo/name"; describe("save test", function() { - const { models, getDocumentClient } = useModels(); + const { models, getDocumentClient, id: pk } = useModels(); it("should be able to perform create & update operations", async () => { const { SimpleModel } = models; const simpleModel = new SimpleModel(); simpleModel.populate({ - pk: getName(SimpleModel), + pk, sk: "something-1", name: "Something-1", enabled: true, @@ -21,7 +20,7 @@ describe("save test", function() { let item = await getDocumentClient() .get({ TableName: "pk-sk", - Key: { pk: getName(SimpleModel), sk: "something-1" } + Key: { pk, sk: "something-1" } }) .promise(); @@ -29,7 +28,7 @@ describe("save test", function() { Item: { sk: "something-1", name: "Something-1", - pk: "SimpleModel", + pk: pk, slug: "something1", enabled: true, age: 55, @@ -43,7 +42,7 @@ describe("save test", function() { item = await getDocumentClient() .get({ TableName: "pk-sk", - Key: { pk: getName(SimpleModel), sk: "something-1" } + Key: { pk, sk: "something-1" } }) .promise(); @@ -51,7 +50,7 @@ describe("save test", function() { Item: { sk: "something-1", name: "Something-1-edited", - pk: "SimpleModel", + pk: pk, slug: "something1Edited", enabled: true, age: 55, From 4aba1ba9917d70c395bdd4cd5c437eadd3267cdc Mon Sep 17 00:00:00 2001 From: Adrian Smijulj Date: Mon, 7 Sep 2020 11:57:32 +0200 Subject: [PATCH 14/26] chore: create GSIs --- jest-dynamodb-config.js | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/jest-dynamodb-config.js b/jest-dynamodb-config.js index ae32d3e..9607496 100644 --- a/jest-dynamodb-config.js +++ b/jest-dynamodb-config.js @@ -8,9 +8,43 @@ module.exports = { ], AttributeDefinitions: [ { AttributeName: "pk", AttributeType: "S" }, - { AttributeName: "sk", AttributeType: "S" } + { AttributeName: "sk", AttributeType: "S" }, + { AttributeName: "gsi1pk", AttributeType: "S" }, + { AttributeName: "gsi1sk", AttributeType: "S" }, + { AttributeName: "gsi2pk", AttributeType: "S" }, + { AttributeName: "gsi2sk", AttributeType: "S" } ], - ProvisionedThroughput: { ReadCapacityUnits: 1, WriteCapacityUnits: 1 } + ProvisionedThroughput: { ReadCapacityUnits: 1, WriteCapacityUnits: 1 }, + GlobalSecondaryIndexes: [ + { + IndexName: "gsi1pk_gsi1sk", + KeySchema: [ + { AttributeName: "gsi1pk", KeyType: "HASH" }, + { AttributeName: "gsi1sk", KeyType: "RANGE" } + ], + Projection: { + ProjectionType: "ALL" + }, + ProvisionedThroughput: { + ReadCapacityUnits: 1, + WriteCapacityUnits: 1 + } + }, + { + IndexName: "gsi2pk_gsi2sk", + KeySchema: [ + { AttributeName: "gsi2pk", KeyType: "HASH" }, + { AttributeName: "gsi2sk", KeyType: "RANGE" } + ], + Projection: { + ProjectionType: "ALL" + }, + ProvisionedThroughput: { + ReadCapacityUnits: 1, + WriteCapacityUnits: 1 + } + } + ] } ] }; From d54b5d9315ca922aa85b201c6b263a46e8135f9a Mon Sep 17 00:00:00 2001 From: Adrian Smijulj Date: Mon, 7 Sep 2020 11:58:14 +0200 Subject: [PATCH 15/26] chore: update yarn.lock --- yarn.lock | 317 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 178 insertions(+), 139 deletions(-) diff --git a/yarn.lock b/yarn.lock index db39711..3e2ff51 100644 --- a/yarn.lock +++ b/yarn.lock @@ -800,6 +800,49 @@ exec-sh "^0.3.2" minimist "^1.2.0" +"@commodo/fields-storage-ref@1.1.0-next.2": + version "1.1.0-next.2" + resolved "https://registry.npmjs.org/@commodo/fields-storage-ref/-/fields-storage-ref-1.1.0-next.2.tgz#c67a944336a0cb312799d91802e297b877fc77e6" + integrity sha512-qQ0tuGbwkNLLRQm22aDgAQ/RRV06tZtJpKnb+jDi7mnY7yW1Crrp0/D+Rowo3Z6rgImlMrmRBbkCpfFs7FCFnA== + dependencies: + "@commodo/fields" "^1.1.0-next.2" + "@commodo/fields-storage" "^2.0.0-next.15" + "@commodo/hooks" "^1.1.0-next.2" + "@commodo/name" "^1.2.0-next.2" + repropose "^1.0.2" + +"@commodo/fields-storage@^2.0.0-next.15": + version "2.0.1" + resolved "https://registry.npmjs.org/@commodo/fields-storage/-/fields-storage-2.0.1.tgz#e985efb0e3069a8ea5ec2136e0c61f09bc58cf25" + integrity sha512-0b4ZoW1oT03wVFsVvKsczIJLnE4KuW1dHAM4bHBbpCDcMlmLg5o+zPrvbYo25Ow6B+/rEW+0j6xWAC01LFnNKA== + dependencies: + "@commodo/hooks" "^1.1.1" + "@commodo/name" "^1.2.1" + mdbid "^1.0.0" + repropose "^1.0.2" + +"@commodo/fields@^1.1.0-next.2": + version "1.1.1" + resolved "https://registry.npmjs.org/@commodo/fields/-/fields-1.1.1.tgz#aec8419a2d6b3f5173f5a98cb718cd35bdd0d580" + integrity sha512-l/qpXKbvWjDStwMUsbirhbkymai4yYnhTbT4WH/GNBMtIEaTlt3xmAPyYFdbhGwHaZ913VoJKFOSFmCbj+yXUA== + dependencies: + "@commodo/name" "^1.2.1" + repropose "^1.0.2" + +"@commodo/hooks@^1.1.0-next.2", "@commodo/hooks@^1.1.1": + version "1.1.1" + resolved "https://registry.npmjs.org/@commodo/hooks/-/hooks-1.1.1.tgz#e0cb5c0ba617536b20494709514d6af35393ac34" + integrity sha512-GYOuldt/KZGvQmJKyo4tDirk3nPiVGvARwVNgjsNOcArll00sEQDsWrdBNQ+E5UCLb/YbFqEqj9s8+JXA4FxXA== + dependencies: + repropose "^1.0.2" + +"@commodo/name@^1.2.0-next.2", "@commodo/name@^1.2.1": + version "1.2.1" + resolved "https://registry.npmjs.org/@commodo/name/-/name-1.2.1.tgz#811f1a6359409f283687bdddfdeefc15d730c47a" + integrity sha512-xO8ST3b8OvbsaiAkWKUoAA5tSmF3Sm31E0ufsBNBGdPXwmYtXiq4i+1fM+EbSs9StPLqADYK794dk5jczyUXxA== + dependencies: + repropose "^1.0.2" + "@evocateur/libnpmaccess@^3.1.2": version "3.1.2" resolved "https://registry.npmjs.org/@evocateur/libnpmaccess/-/libnpmaccess-3.1.2.tgz#ecf7f6ce6b004e9f942b098d92200be4a4b1c845" @@ -1829,6 +1872,15 @@ dependencies: any-observable "^0.3.0" +"@shelf/jest-dynamodb@^1.7.0": + version "1.7.0" + resolved "https://registry.npmjs.org/@shelf/jest-dynamodb/-/jest-dynamodb-1.7.0.tgz#94d6cc08984f2449db951a7ba8fcd1883f330c32" + integrity sha512-ASm/b43lXCgu8xmyIuth+iYp28JSVzn5GPuUq769AWveHMpuSUVESKSMpR0xrAlado4kxzqs0eupQOcuZamaeA== + dependencies: + cwd "0.10.0" + debug "4.1.1" + dynamodb-local "0.0.31" + "@shelf/jest-mongodb@^1.1.5": version "1.1.5" resolved "https://registry.npmjs.org/@shelf/jest-mongodb/-/jest-mongodb-1.1.5.tgz#50573c9cb8cf12f6ee93fb79ea7420a431138c9c" @@ -2218,20 +2270,6 @@ ajv@^6.10.2, ajv@^6.5.5, ajv@^6.9.1: json-schema-traverse "^0.4.1" uri-js "^4.2.2" -all-contributors-cli@^5.6.0: - version "5.11.0" - resolved "https://registry.npmjs.org/all-contributors-cli/-/all-contributors-cli-5.11.0.tgz#5fc022b8064ee09e4370dc5e9d92d59c7ee0bbf3" - integrity sha512-+xL38RoYh4caJVlxqGsr7jzV5pXLquIEa5AsfUi6/8joOT1m18AKK0qwt3xgOG/P/zOdcs/PBLjmm8NqXh5T+g== - dependencies: - "@babel/runtime" "^7.2.0" - async "^2.0.0-rc.1" - chalk "^2.3.0" - inquirer "^6.2.1" - lodash "^4.11.2" - pify "^4.0.1" - request "^2.72.0" - yargs "^12.0.5" - ansi-escapes@^3.0.0, ansi-escapes@^3.2.0: version "3.2.0" resolved "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.2.0.tgz#8780b98ff9dbf5638152d1f1fe5c1d7b4442976b" @@ -2436,13 +2474,6 @@ async@3.1.0: resolved "https://registry.npmjs.org/async/-/async-3.1.0.tgz#42b3b12ae1b74927b5217d8c0016baaf62463772" integrity sha512-4vx/aaY6j/j3Lw3fbCHNWP0pPaTCew3F6F3hYyl/tHs/ndmV1q7NW9T5yuJ2XAGwdQrP+6Wu20x06U4APo/iQQ== -async@^2.0.0-rc.1: - version "2.6.3" - resolved "https://registry.npmjs.org/async/-/async-2.6.3.tgz#d72625e2344a3656e3a3ad4fa749fa83299d82ff" - integrity sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg== - dependencies: - lodash "^4.17.14" - asynckit@^0.4.0: version "0.4.0" resolved "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" @@ -2463,6 +2494,21 @@ atob@^2.1.2: resolved "https://registry.npmjs.org/atob/-/atob-2.1.2.tgz#6d9517eb9e030d2436666651e86bd9f6f13533c9" integrity sha512-Wm6ukoaOGJi/73p/cl2GvLjTI5JM1k/O14isD73YML8StrH/7/lRFgmg8nICZgD3bZZvjwCGxtMOD3wWNAu8cg== +aws-sdk@^2.729.0: + version "2.738.0" + resolved "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.738.0.tgz#37e4b75ab1acf9bc01bea5d4f83ebb937842de6a" + integrity sha512-oO1odRT4DGssivoP6lHO3yB6I+5sU0uqpj6UJ0kS5wrHQ9J9EGrictLVKA9y6XhN0sNW+XPNLD9jMMY/A+gXWA== + dependencies: + buffer "4.9.2" + events "1.1.1" + ieee754 "1.1.13" + jmespath "0.15.0" + querystring "0.2.0" + sax "1.2.1" + url "0.10.3" + uuid "3.3.2" + xml2js "0.4.19" + aws-sign2@~0.7.0: version "0.7.0" resolved "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz#b46e890934a9591f2d2f6f86d7e6a9f1b3fe76a8" @@ -2721,6 +2767,15 @@ buffer-from@^1.0.0: resolved "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.1.tgz#32713bc028f75c02fdb710d7c7bcec1f2c6070ef" integrity sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A== +buffer@4.9.2: + version "4.9.2" + resolved "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz#230ead344002988644841ab0244af8c44bbe3ef8" + integrity sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg== + dependencies: + base64-js "^1.0.2" + ieee754 "^1.1.4" + isarray "^1.0.0" + buffer@^5.5.0: version "5.6.0" resolved "https://registry.npmjs.org/buffer/-/buffer-5.6.0.tgz#a31749dc7d81d84db08abf937b6b8c4033f62786" @@ -2887,7 +2942,7 @@ caseless@~0.12.0: resolved "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz#1b681c21ff84033c826543090689420d187151dc" integrity sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw= -chalk@2.4.2, chalk@^2.0.0, chalk@^2.0.1, chalk@^2.1.0, chalk@^2.3.0, chalk@^2.3.1, chalk@^2.4.1, chalk@^2.4.2: +chalk@2.4.2, chalk@^2.0.0, chalk@^2.0.1, chalk@^2.1.0, chalk@^2.3.1, chalk@^2.4.1, chalk@^2.4.2: version "2.4.2" resolved "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== @@ -2971,15 +3026,6 @@ cli-width@^2.0.0: resolved "https://registry.npmjs.org/cli-width/-/cli-width-2.2.1.tgz#b0433d0b4e9c847ef18868a4ef16fd5fc8271c48" integrity sha512-GRMWDxpOB6Dgk2E5Uo+3eEBvtOOlimMmpbFiKuLFnQzYDavtLFY3K5ona41jgN/WdRZtG7utuVSVTL4HbZHGkw== -cliui@^4.0.0: - version "4.1.0" - resolved "https://registry.npmjs.org/cliui/-/cliui-4.1.0.tgz#348422dbe82d800b3022eef4f6ac10bf2e4d1b49" - integrity sha512-4FG+RSG9DL7uEwRUZXZn3SS34DiDPfzP0VOiEwtUWlE+AR2EIg+hSyvrIgUUfhdgR/UkAeW2QHgeP+hWrXs7jQ== - dependencies: - string-width "^2.1.1" - strip-ansi "^4.0.0" - wrap-ansi "^2.0.0" - cliui@^5.0.0: version "5.0.0" resolved "https://registry.npmjs.org/cliui/-/cliui-5.0.0.tgz#deefcfdb2e800784aa34f46fa08e06851c7bbbc5" @@ -3434,7 +3480,7 @@ debug@3.1.0: dependencies: ms "2.0.0" -debug@4, debug@4.1.1, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1: +debug@4, debug@4.1.1, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1, debug@~4.1.0: version "4.1.1" resolved "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz#3b72260255109c6b589cee050f1d516139664791" integrity sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw== @@ -3654,6 +3700,16 @@ duplexify@^3.4.2, duplexify@^3.6.0: readable-stream "^2.0.0" stream-shift "^1.0.0" +dynamodb-local@0.0.31: + version "0.0.31" + resolved "https://registry.npmjs.org/dynamodb-local/-/dynamodb-local-0.0.31.tgz#ce6057aab0e1f8265c41658e5b12e679a0ce6ef6" + integrity sha512-5Ro47g989oLSC2JKFtPAWWSSicZOkqQrBGwHActqN6zdCZEy/eGvhq7Ki9Lgx/gNE+ksIF2wD6VNff3bjTI8ow== + dependencies: + debug "~4.1.0" + mkdirp "~0.5.0" + q "~1.4.1" + tar "~4.4.8" + ecc-jsbn@~0.1.1: version "0.1.2" resolved "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz#3a83a904e54353287874c564b7549386849a98c9" @@ -3919,6 +3975,11 @@ eventemitter3@^3.1.0: resolved "https://registry.npmjs.org/eventemitter3/-/eventemitter3-3.1.2.tgz#2d3d48f9c346698fce83a85d7d664e98535df6e7" integrity sha512-tvtQIeLVHjDkJYnzf2dgVMxfuSGJeM/7UCG17TT4EumTfNtF+0nebF/4zWOIkCreAbtNqhGEboB6BWrwqNaw4Q== +events@1.1.1: + version "1.1.1" + resolved "https://registry.npmjs.org/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924" + integrity sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ= + exec-sh@^0.3.2: version "0.3.4" resolved "https://registry.npmjs.org/exec-sh/-/exec-sh-0.3.4.tgz#3a018ceb526cc6f6df2bb504b2bfe8e3a4934ec5" @@ -4460,11 +4521,6 @@ gensync@^1.0.0-beta.1: resolved "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.1.tgz#58f4361ff987e5ff6e1e7a210827aa371eaac269" integrity sha512-r8EC6NO1sngH/zdD9fiRDLdcgnbayXah+mLgManTaIZJqEC1MZstmnox8KpnI2/fxQwrp5OpCOYWLp4rBl4Jcg== -get-caller-file@^1.0.1: - version "1.0.3" - resolved "https://registry.npmjs.org/get-caller-file/-/get-caller-file-1.0.3.tgz#f978fa4c90d1dfe7ff2d6beda2a515e713bdcf4a" - integrity sha512-3t6rVToeoZfYSGd8YoLFR2DJkiQrIiUrGcjvFX2mDw3bn6k2OtwHN0TNCLbBO+w8qTvimhDkv+LSscbJY1vE6w== - get-caller-file@^2.0.1: version "2.0.5" resolved "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e" @@ -4943,7 +4999,7 @@ iconv-lite@0.4.24, iconv-lite@^0.4.24, iconv-lite@~0.4.13: dependencies: safer-buffer ">= 2.1.2 < 3" -ieee754@^1.1.4: +ieee754@1.1.13, ieee754@^1.1.4: version "1.1.13" resolved "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz#ec168558e95aa181fd87d37f55c32bbcb6708b84" integrity sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg== @@ -5058,7 +5114,7 @@ init-package-json@^1.10.3: validate-npm-package-license "^3.0.1" validate-npm-package-name "^3.0.0" -inquirer@^6.2.0, inquirer@^6.2.1, inquirer@^6.2.2: +inquirer@^6.2.0, inquirer@^6.2.2: version "6.5.2" resolved "https://registry.npmjs.org/inquirer/-/inquirer-6.5.2.tgz#ad50942375d036d327ff528c08bd5fab089928ca" integrity sha512-cntlB5ghuB0iuO65Ovoi8ogLHiWGs/5yNrtUcKjFhSSiVeAIVpD7koaSU9RM8mpXw5YDi9RdYXGQMaOURB7ycQ== @@ -5084,11 +5140,6 @@ invariant@^2.2.2, invariant@^2.2.4: dependencies: loose-envify "^1.0.0" -invert-kv@^2.0.0: - version "2.0.0" - resolved "https://registry.npmjs.org/invert-kv/-/invert-kv-2.0.0.tgz#7393f5afa59ec9ff5f67a27620d11c226e3eec02" - integrity sha512-wPVv/y/QQ/Uiirj/vh3oP+1Ww+AWehmi1g5fFWGPF6IpCBCDVrhgHRMvrLfdYcwDh3QJbGXDW4JAuzxElLSqKA== - ip-regex@^2.1.0: version "2.1.0" resolved "https://registry.npmjs.org/ip-regex/-/ip-regex-2.1.0.tgz#fa78bf5d2e6913c911ce9f819ee5146bb6d844e9" @@ -5372,7 +5423,7 @@ isarray@0.0.1: resolved "https://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz#8a18acfca9a8f4177e09abfc6038939b05d1eedf" integrity sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8= -isarray@1.0.0, isarray@~1.0.0: +isarray@1.0.0, isarray@^1.0.0, isarray@~1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" integrity sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE= @@ -5803,6 +5854,11 @@ jest@^24.7.1: import-local "^2.0.0" jest-cli "^24.9.0" +jmespath@0.15.0: + version "0.15.0" + resolved "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217" + integrity sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc= + js-base64@2.5.1: version "2.5.1" resolved "https://registry.npmjs.org/js-base64/-/js-base64-2.5.1.tgz#1efa39ef2c5f7980bb1784ade4a8af2de3291121" @@ -6045,13 +6101,6 @@ kleur@^3.0.3: resolved "https://registry.npmjs.org/kleur/-/kleur-3.0.3.tgz#a79c9ecc86ee1ce3fa6206d1216c501f147fc07e" integrity sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w== -lcid@^2.0.0: - version "2.0.0" - resolved "https://registry.npmjs.org/lcid/-/lcid-2.0.0.tgz#6ef5d2df60e52f82eb228a4c373e8d1f397253cf" - integrity sha512-avPEb8P8EGnwXKClwsNUgryVjllcRqtMYa49NTsbQagYuT1DcXnl1915oxWjoyGrXR6zH/Y0Zc96xWsPcoDKeA== - dependencies: - invert-kv "^2.0.0" - lcov-parse@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/lcov-parse/-/lcov-parse-1.0.0.tgz#eb0d46b54111ebc561acb4c408ef9363bdc8f7e0" @@ -6356,7 +6405,7 @@ lodash@4.17.11: resolved "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz#b39ea6229ef607ecd89e2c8df12536891cac9b8d" integrity sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg== -lodash@4.17.15, lodash@^4.11.2, lodash@^4.17.11, lodash@^4.17.12, lodash@^4.17.13, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.2.1: +lodash@4.17.15, lodash@^4.17.11, lodash@^4.17.12, lodash@^4.17.13, lodash@^4.17.14, lodash@^4.17.15, lodash@^4.2.1: version "4.17.15" resolved "https://registry.npmjs.org/lodash/-/lodash-4.17.15.tgz#b447f6670a0455bbfeedd11392eff330ea097548" integrity sha512-8xOcRHvCjnocdS5cpwXQXVzmmh5e5+saE2QGoeQmbKmRS6J3VQppPOIt0MnmE+4xlZoumy0GPG0D0MVIQbNA1A== @@ -6479,13 +6528,6 @@ makeerror@1.0.x: dependencies: tmpl "1.0.x" -map-age-cleaner@^0.1.1: - version "0.1.3" - resolved "https://registry.npmjs.org/map-age-cleaner/-/map-age-cleaner-0.1.3.tgz#7d583a7306434c055fe474b0f45078e6e1b4b92a" - integrity sha512-bJzx6nMoP6PDLPBFmg7+xRKeFZvFboMrGlxmNj9ClvX53KrmvM5bXFXEWjbz4cz1AFn+jWJ9z/DJSz7hrs0w3w== - dependencies: - p-defer "^1.0.0" - map-cache@^0.2.2: version "0.2.2" resolved "https://registry.npmjs.org/map-cache/-/map-cache-0.2.2.tgz#c32abd0bd6525d9b051645bb4f26ac5dc98a0dbf" @@ -6547,15 +6589,6 @@ media-typer@0.3.0: resolved "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748" integrity sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g= -mem@^4.0.0: - version "4.3.0" - resolved "https://registry.npmjs.org/mem/-/mem-4.3.0.tgz#461af497bc4ae09608cdb2e60eefb69bff744178" - integrity sha512-qX2bG48pTqYRVmDB37rn/6PT7LcR8T7oAX3bf99u1Tt1nzxYfxkgqDwUwolPlXweM0XzBOBFzSx4kfp7KP1s/w== - dependencies: - map-age-cleaner "^0.1.1" - mimic-fn "^2.0.0" - p-is-promise "^2.0.0" - memory-pager@^1.0.2: version "1.5.0" resolved "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz#d8751655d22d384682741c972f2c3d6dfa3e66b5" @@ -6686,11 +6719,6 @@ mimic-fn@^1.0.0: resolved "https://registry.npmjs.org/mimic-fn/-/mimic-fn-1.2.0.tgz#820c86a39334640e99516928bd03fca88057d022" integrity sha512-jf84uxzwiuiIVKiOLpfYk7N46TSy8ubTonmneY9vrpHNAnp0QBt2BxWV9dO3/j+BoVAb+a5G6YDPW3M5HOdMWQ== -mimic-fn@^2.0.0: - version "2.1.0" - resolved "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b" - integrity sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg== - min-document@^2.19.0: version "2.19.0" resolved "https://registry.npmjs.org/min-document/-/min-document-2.19.0.tgz#7bd282e3f5842ed295bb748cdd9f1ffa2c824685" @@ -6799,7 +6827,7 @@ mkdirp@0.5.1: dependencies: minimist "0.0.8" -mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.1: +mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.0, mkdirp@~0.5.1: version "0.5.5" resolved "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz#d91cefd62d1436ca0f41620e251288d420099def" integrity sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ== @@ -6870,6 +6898,19 @@ mongodb@^3.5.4: optionalDependencies: saslprep "^1.0.0" +mongodb@^3.5.5: + version "3.6.0" + resolved "https://registry.npmjs.org/mongodb/-/mongodb-3.6.0.tgz#babd7172ec717e2ed3f85e079b3f1aa29dce4724" + integrity sha512-/XWWub1mHZVoqEsUppE0GV7u9kanLvHxho6EvBxQbShXTKYF9trhZC2NzbulRGeG7xMJHD8IOWRcdKx5LPjAjQ== + dependencies: + bl "^2.2.0" + bson "^1.1.4" + denque "^1.4.1" + require_optional "^1.0.1" + safe-buffer "^5.1.2" + optionalDependencies: + saslprep "^1.0.0" + move-concurrently@^1.0.1: version "1.0.1" resolved "https://registry.npmjs.org/move-concurrently/-/move-concurrently-1.0.1.tgz#be2c005fda32e0b29af1f05d7c4b33214c701f92" @@ -7322,15 +7363,6 @@ os-homedir@^1.0.0, os-homedir@^1.0.1: resolved "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz#ffbc4988336e0e833de0c168c7ef152121aa7fb3" integrity sha1-/7xJiDNuDoM94MFox+8VISGqf7M= -os-locale@^3.0.0: - version "3.1.0" - resolved "https://registry.npmjs.org/os-locale/-/os-locale-3.1.0.tgz#a802a6ee17f24c10483ab9935719cef4ed16bf1a" - integrity sha512-Z8l3R4wYWM40/52Z+S265okfFj8Kt2cC2MKY+xNi3kFs+XGI7WXu/I309QQQYbRW4ijiZ+yxs9pqEhJh0DqW3Q== - dependencies: - execa "^1.0.0" - lcid "^2.0.0" - mem "^4.0.0" - os-name@^3.1.0: version "3.1.0" resolved "https://registry.npmjs.org/os-name/-/os-name-3.1.0.tgz#dec19d966296e1cd62d701a5a66ee1ddeae70801" @@ -7352,11 +7384,6 @@ osenv@^0.1.4, osenv@^0.1.5: os-homedir "^1.0.0" os-tmpdir "^1.0.0" -p-defer@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/p-defer/-/p-defer-1.0.0.tgz#9f6eb182f6c9aa8cd743004a7d4f96b196b0fb0c" - integrity sha1-n26xgvbJqozXQwBKfU+WsZaw+ww= - p-each-series@^1.0.0: version "1.0.0" resolved "https://registry.npmjs.org/p-each-series/-/p-each-series-1.0.0.tgz#930f3d12dd1f50e7434457a22cd6f04ac6ad7f71" @@ -7369,11 +7396,6 @@ p-finally@^1.0.0: resolved "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz#3fbcfb15b899a44123b34b6dcc18b724336a2cae" integrity sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4= -p-is-promise@^2.0.0: - version "2.1.0" - resolved "https://registry.npmjs.org/p-is-promise/-/p-is-promise-2.1.0.tgz#918cebaea248a62cf7ffab8e3bca8c5f882fc42e" - integrity sha512-Y3W0wlRPK8ZMRbNq97l4M5otioeA5lm1z7bkNkxCka8HSPjR0xRWmpCmc9utiaLP9Jb1eD8BgeIxTW4AIF45Pg== - p-limit@^1.1.0: version "1.3.0" resolved "https://registry.npmjs.org/p-limit/-/p-limit-1.3.0.tgz#b86bd5f0c25690911c7590fcbfc2010d54b3ccb8" @@ -7839,6 +7861,11 @@ pumpify@^1.3.3: inherits "^2.0.3" pump "^2.0.0" +punycode@1.3.2: + version "1.3.2" + resolved "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d" + integrity sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0= + punycode@^1.4.1: version "1.4.1" resolved "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e" @@ -7854,6 +7881,11 @@ q@^1.5.1: resolved "https://registry.npmjs.org/q/-/q-1.5.1.tgz#7e32f75b41381291d04611f1bf14109ac00651d7" integrity sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc= +q@~1.4.1: + version "1.4.1" + resolved "https://registry.npmjs.org/q/-/q-1.4.1.tgz#55705bcd93c5f3673530c2c2cbc0c2b3addc286e" + integrity sha1-VXBbzZPF82c1MMLCy8DCs63cKG4= + qs@6.5.2, qs@~6.5.1, qs@~6.5.2: version "6.5.2" resolved "https://registry.npmjs.org/qs/-/qs-6.5.2.tgz#cb3ae806e8740444584ef154ce8ee98d403f3e36" @@ -7864,6 +7896,11 @@ qs@6.7.0: resolved "https://registry.npmjs.org/qs/-/qs-6.7.0.tgz#41dc1a015e3d581f1621776be31afb2876a9b1bc" integrity sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ== +querystring@0.2.0: + version "0.2.0" + resolved "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" + integrity sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA= + quick-lru@^1.0.0: version "1.1.0" resolved "https://registry.npmjs.org/quick-lru/-/quick-lru-1.1.0.tgz#4360b17c61136ad38078397ff11416e186dcfbb8" @@ -8255,7 +8292,7 @@ request@2.88.0: tunnel-agent "^0.6.0" uuid "^3.3.2" -request@^2.72.0, request@^2.87.0, request@^2.88.0, request@^2.88.2: +request@^2.87.0, request@^2.88.0, request@^2.88.2: version "2.88.2" resolved "https://registry.npmjs.org/request/-/request-2.88.2.tgz#d73c918731cb5a87da047e207234146f664d12b3" integrity sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw== @@ -8286,11 +8323,6 @@ require-directory@^2.1.1: resolved "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" integrity sha1-jGStX9MNqxyXbiNE/+f3kqam30I= -require-main-filename@^1.0.1: - version "1.0.1" - resolved "https://registry.npmjs.org/require-main-filename/-/require-main-filename-1.0.1.tgz#97f717b69d48784f5f526a6c5aa8ffdda055a4d1" - integrity sha1-l/cXtp1IeE9fUmpsWqj/3aBVpNE= - require-main-filename@^2.0.0: version "2.0.0" resolved "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz#d0b329ecc7cc0f61649f62215be69af54aa8989b" @@ -8374,6 +8406,11 @@ retry@^0.10.0: resolved "https://registry.npmjs.org/retry/-/retry-0.10.1.tgz#e76388d217992c252750241d3d3956fed98d8ff4" integrity sha1-52OI0heZLCUnUCQdPTlW/tmNj/Q= +rfdc@^1.1.4: + version "1.1.4" + resolved "https://registry.npmjs.org/rfdc/-/rfdc-1.1.4.tgz#ba72cc1367a0ccd9cf81a870b3b58bd3ad07f8c2" + integrity sha512-5C9HXdzK8EAqN7JDif30jqsBzavB7wLpaubisuQIGHWf2gUXSpzy6ArX/+Da8RjFpagWsCn+pIgxTMAmKw9Zug== + rimraf@2.6.3: version "2.6.3" resolved "https://registry.npmjs.org/rimraf/-/rimraf-2.6.3.tgz#b2d104fe0d8fb27cf9e0a1cda8262dd3833c6cab" @@ -8480,7 +8517,12 @@ saslprep@^1.0.0: dependencies: sparse-bitfield "^3.0.3" -sax@^1.2.4: +sax@1.2.1: + version "1.2.1" + resolved "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" + integrity sha1-e45lYZCyKOgaZq6nSEgNgozS03o= + +sax@>=0.6.0, sax@^1.2.4: version "1.2.4" resolved "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" integrity sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw== @@ -8648,7 +8690,7 @@ simple-git@^1.85.0: dependencies: debug "^4.0.1" -sinon@^7.2.7: +sinon@^7.2.7, sinon@^7.3.2: version "7.5.0" resolved "https://registry.npmjs.org/sinon/-/sinon-7.5.0.tgz#e9488ea466070ea908fd44a3d6478fd4923c67ec" integrity sha512-AoD0oJWerp0/rY9czP/D6hDTTUYGpObhZjMpd7Cl/A6+j0xBE+ayL/ldfggkBXUs0IkvIiM1ljM8+WkOc5k78Q== @@ -8931,7 +8973,7 @@ string-width@^1.0.1: is-fullwidth-code-point "^1.0.0" strip-ansi "^3.0.0" -"string-width@^1.0.2 || 2", string-width@^2.0.0, string-width@^2.1.0, string-width@^2.1.1: +"string-width@^1.0.2 || 2", string-width@^2.1.0, string-width@^2.1.1: version "2.1.1" resolved "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz#ab93f27a8dc13d28cac815c462143a6d9012ae9e" integrity sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw== @@ -9131,7 +9173,7 @@ tar-stream@^2.1.1: inherits "^2.0.3" readable-stream "^3.1.1" -tar@^4.4.10, tar@^4.4.12, tar@^4.4.8: +tar@^4.4.10, tar@^4.4.12, tar@^4.4.8, tar@~4.4.8: version "4.4.13" resolved "https://registry.npmjs.org/tar/-/tar-4.4.13.tgz#43b364bc52888d555298637b10d60790254ab525" integrity sha512-w2VwSrBoHa5BsSyH+KxEqeQBAllHhccyMFVHtGtdMpF4W7IRWfZjFiQceJPChOeTsSDVUpER2T8FA93pr0L+QA== @@ -9455,6 +9497,11 @@ union-value@^1.0.0: is-extendable "^0.1.1" set-value "^2.0.1" +uniqid@^5.2.0: + version "5.2.0" + resolved "https://registry.npmjs.org/uniqid/-/uniqid-5.2.0.tgz#0d0589a7e9ce07116848126764fbff0b68e74329" + integrity sha512-LH8zsvwJ/GL6YtNfSOmMCrI9piraAUjBfw2MCvleNE6a4pVKJwXjG2+HWhkVeFcSg+nmaPKbMrMOoxwQluZ1Mg== + unique-filename@^1.1.1: version "1.1.1" resolved "https://registry.npmjs.org/unique-filename/-/unique-filename-1.1.1.tgz#1d69769369ada0583103a1e6ae87681b56573230" @@ -9528,6 +9575,14 @@ urix@^0.1.0: resolved "https://registry.npmjs.org/urix/-/urix-0.1.0.tgz#da937f7a62e21fec1fd18d49b35c2935067a6c72" integrity sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI= +url@0.10.3: + version "0.10.3" + resolved "https://registry.npmjs.org/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64" + integrity sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ= + dependencies: + punycode "1.3.2" + querystring "0.2.0" + use@^3.1.0: version "3.1.1" resolved "https://registry.npmjs.org/use/-/use-3.1.1.tgz#d50c8cac79a19fbc20f2911f56eb973f4e10070f" @@ -9560,6 +9615,11 @@ utils-merge@1.0.1: resolved "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz#9f95710f50a267947b2ccc124741c1028427e713" integrity sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM= +uuid@3.3.2: + version "3.3.2" + resolved "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131" + integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA== + uuid@^3.0.1, uuid@^3.1.0, uuid@^3.3.2: version "3.4.0" resolved "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee" @@ -9770,14 +9830,6 @@ wordwrap@~0.0.2: resolved "https://registry.npmjs.org/wordwrap/-/wordwrap-0.0.3.tgz#a3d5da6cd5c0bc0008d37234bbaf1bed63059107" integrity sha1-o9XabNXAvAAI03I0u68b7WMFkQc= -wrap-ansi@^2.0.0: - version "2.1.0" - resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz#d8fc3d284dd05794fe84973caecdd1cf824fdd85" - integrity sha1-2Pw9KE3QV5T+hJc8rs3Rz4JP3YU= - dependencies: - string-width "^1.0.1" - strip-ansi "^3.0.1" - wrap-ansi@^3.0.1: version "3.0.1" resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-3.0.1.tgz#288a04d87eda5c286e060dfe8f135ce8d007f8ba" @@ -9874,6 +9926,19 @@ xml-name-validator@^3.0.0: resolved "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-3.0.0.tgz#6ae73e06de4d8c6e47f9fb181f78d648ad457c6a" integrity sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw== +xml2js@0.4.19: + version "0.4.19" + resolved "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" + integrity sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q== + dependencies: + sax ">=0.6.0" + xmlbuilder "~9.0.1" + +xmlbuilder@~9.0.1: + version "9.0.7" + resolved "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz#132ee63d2ec5565c557e20f4c22df9aca686b10d" + integrity sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0= + xmlchars@^2.1.1: version "2.2.0" resolved "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb" @@ -9884,7 +9949,7 @@ xtend@~4.0.1: resolved "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ== -"y18n@^3.2.1 || ^4.0.0", y18n@^4.0.0: +y18n@^4.0.0: version "4.0.0" resolved "https://registry.npmjs.org/y18n/-/y18n-4.0.0.tgz#95ef94f85ecc81d007c264e190a120f0a3c8566b" integrity sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w== @@ -9894,14 +9959,6 @@ yallist@^3.0.0, yallist@^3.0.2, yallist@^3.0.3: resolved "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd" integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g== -yargs-parser@^11.1.1: - version "11.1.1" - resolved "https://registry.npmjs.org/yargs-parser/-/yargs-parser-11.1.1.tgz#879a0865973bca9f6bab5cbdf3b1c67ec7d3bcf4" - integrity sha512-C6kB/WJDiaxONLJQnF8ccx9SEeoTTLek8RVbaOIsrAUS8VrBEXfmeSnCZxygc+XC2sNMBIwOOnfcxiynjHsVSQ== - dependencies: - camelcase "^5.0.0" - decamelize "^1.2.0" - yargs-parser@^13.1.2: version "13.1.2" resolved "https://registry.npmjs.org/yargs-parser/-/yargs-parser-13.1.2.tgz#130f09702ebaeef2650d54ce6e3e5706f7a4fb38" @@ -9926,24 +9983,6 @@ yargs-parser@^18.1.3: camelcase "^5.0.0" decamelize "^1.2.0" -yargs@^12.0.5: - version "12.0.5" - resolved "https://registry.npmjs.org/yargs/-/yargs-12.0.5.tgz#05f5997b609647b64f66b81e3b4b10a368e7ad13" - integrity sha512-Lhz8TLaYnxq/2ObqHDql8dX8CJi97oHxrjUcYtzKbbykPtVW9WB+poxI+NM2UIzsMgNCZTIf0AQwsjK5yMAqZw== - dependencies: - cliui "^4.0.0" - decamelize "^1.2.0" - find-up "^3.0.0" - get-caller-file "^1.0.1" - os-locale "^3.0.0" - require-directory "^2.1.1" - require-main-filename "^1.0.1" - set-blocking "^2.0.0" - string-width "^2.0.0" - which-module "^2.0.0" - y18n "^3.2.1 || ^4.0.0" - yargs-parser "^11.1.1" - yargs@^13.3.0: version "13.3.2" resolved "https://registry.npmjs.org/yargs/-/yargs-13.3.2.tgz#ad7ffefec1aa59565ac915f82dccb38a9c31a2dd" From 776c68774af711d8267ba4960dd3dba62bf18e60 Mon Sep 17 00:00:00 2001 From: Adrian Smijulj Date: Mon, 7 Sep 2020 12:03:43 +0200 Subject: [PATCH 16/26] chore: update dev dependencies --- packages/fields-storage-dynamodb/package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/fields-storage-dynamodb/package.json b/packages/fields-storage-dynamodb/package.json index 2f140c8..559dffc 100644 --- a/packages/fields-storage-dynamodb/package.json +++ b/packages/fields-storage-dynamodb/package.json @@ -14,7 +14,8 @@ "aws-sdk": "^2.729.0" }, "devDependencies": { - "@shelf/jest-dynamodb": "^1.7.0" + "@shelf/jest-dynamodb": "^1.7.0", + "uniqid": "^5.2.0" }, "publishConfig": { "access": "public" From 3afe739189259c357388c6046c00565cbd3bc80c Mon Sep 17 00:00:00 2001 From: Adrian Smijulj Date: Mon, 7 Sep 2020 12:05:54 +0200 Subject: [PATCH 17/26] fix: exclude GSI fields if null in batchWrite operations --- .../src/DynamoDbDriver.js | 46 +++++++++++++++++-- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/packages/fields-storage-dynamodb/src/DynamoDbDriver.js b/packages/fields-storage-dynamodb/src/DynamoDbDriver.js index db2cee0..a10f479 100644 --- a/packages/fields-storage-dynamodb/src/DynamoDbDriver.js +++ b/packages/fields-storage-dynamodb/src/DynamoDbDriver.js @@ -2,6 +2,28 @@ import { DocumentClient } from "aws-sdk/clients/dynamodb"; import BatchProcess from "./BatchProcess"; import QueryGenerator from "./QueryGenerator"; +const propertyIsPartOfUniqueKey = (property, keys) => { + for (let i = 0; i < keys.length; i++) { + const key = keys[i]; + if (!key.unique) { + continue; + } + + let fields = keys[i].fields; + if (!Array.isArray(fields)) { + continue; + } + + for (let j = 0; j < fields.length; j++) { + let field = fields[j]; + if (field.name === property) { + return true; + } + } + } + return false; +}; + class DynamoDbDriver { constructor({ documentClient, tableName } = {}) { this.batchProcesses = {}; @@ -35,7 +57,7 @@ class DynamoDbDriver { return true; } - async update({ query, data, name, batch, instance }) { + async update({ query, data, name, batch, instance, keys }) { if (!batch) { const update = { UpdateExpression: "SET ", @@ -69,7 +91,24 @@ class DynamoDbDriver { // call is part of an model instance update (not part of SomeModel.save() call), where we can access the // toStorage function. So, if that's the case, we'll call it with the skipDifferenceCheck flag enabled. // Normally we wouldn't have to do all of this dancing, but it's how DynamoDB works, there's no way around it. - let Item = instance ? await instance.toStorage({ skipDifferenceCheck: true }) : data; + const storageData = instance + ? await instance.toStorage({ skipDifferenceCheck: true }) + : data; + + // The only problem with the above approach is that it may insert null values into GSI columns, + // which immediately gets rejected by DynamoDB. Let's remove those here. + const Item = {}; + for (let property in storageData) { + // Check if key is a part of a unique index. If so, and is null, remove it from data. + if (!propertyIsPartOfUniqueKey(property, keys)) { + Item[property] = storageData[property]; + continue; + } + + if (storageData[property] !== null) { + Item[property] = storageData[property]; + } + } batchProcess.addOperation("PutRequest", { TableName: this.tableName || name, @@ -100,6 +139,7 @@ class DynamoDbDriver { const batchProcess = this.getBatchProcess(batch); batchProcess.addOperation("DeleteRequest", { + TableName: this.tableName || name, Key: query }); @@ -127,7 +167,7 @@ class DynamoDbDriver { }); const { Items } = await this.documentClient.query(queryParams).promise(); - return [Items] + return [Items]; } const batchProcess = this.getBatchProcess(batch); From 226ac0a29e043d5d81b2e7b8b8fb73f3bab8fb23 Mon Sep 17 00:00:00 2001 From: Adrian Smijulj Date: Mon, 7 Sep 2020 12:06:49 +0200 Subject: [PATCH 18/26] fix: export Batch class --- packages/fields-storage/src/index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/fields-storage/src/index.js b/packages/fields-storage/src/index.js index 3f88632..459bd26 100644 --- a/packages/fields-storage/src/index.js +++ b/packages/fields-storage/src/index.js @@ -10,5 +10,6 @@ export { default as getStorageName } from "./getStorageName"; export { default as hasStorageName } from "./hasStorageName"; export { default as withPrimaryKey } from "./withPrimaryKey"; export { default as withKey } from "./withKey"; +export { default as Batch } from "./Batch"; export { default as withUniqueKey } from "./withUniqueKey"; export { default as getKeys } from "./getKeys"; From 18c32afcedb0414ffc9f9187512cdfab727564a5 Mon Sep 17 00:00:00 2001 From: Adrian Smijulj Date: Mon, 7 Sep 2020 12:07:57 +0200 Subject: [PATCH 19/26] feat: introduce Batch class --- packages/fields-storage/src/Batch.js | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 packages/fields-storage/src/Batch.js diff --git a/packages/fields-storage/src/Batch.js b/packages/fields-storage/src/Batch.js new file mode 100644 index 0000000..cc891c8 --- /dev/null +++ b/packages/fields-storage/src/Batch.js @@ -0,0 +1,27 @@ +class Batch { + constructor(...operations) { + this.operations = operations; + this.type = "batch"; + this.id = Math.random() + .toString(36) + .slice(-6); // e.g. tfz58m + } + + push(...items) { + this.operations.push(...items); + } + + async execute() { + const promises = []; + for (let i = 0; i < this.operations.length; i++) { + const [model, operation, args] = this.operations[i]; + promises.push(model[operation]({ ...args, batch: this })); + } + + await Promise.all(promises); + + return this; + } +} + +export default Batch; From 45fa74c5d43f3de3fd8d6b853e382e2c258e18cb Mon Sep 17 00:00:00 2001 From: Adrian Smijulj Date: Mon, 7 Sep 2020 12:08:19 +0200 Subject: [PATCH 20/26] feat: create DynamoDb driver --- packages/fields-storage-dynamodb/src/BatchProcess.js | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/fields-storage-dynamodb/src/BatchProcess.js b/packages/fields-storage-dynamodb/src/BatchProcess.js index cbae420..b5e0772 100644 --- a/packages/fields-storage-dynamodb/src/BatchProcess.js +++ b/packages/fields-storage-dynamodb/src/BatchProcess.js @@ -36,16 +36,14 @@ class BatchProcess { }; for (let i = 0; i < this.operations.length; i++) { - let [type, params] = this.operations[i]; + let [type, { TableName, ...rest }] = this.operations[i]; - if (!batchWriteParams.RequestItems[params.TableName]) { - batchWriteParams.RequestItems[params.TableName] = []; + if (!batchWriteParams.RequestItems[TableName]) { + batchWriteParams.RequestItems[TableName] = []; } - batchWriteParams.RequestItems[params.TableName].push({ - [type]: { - Item: params.Item - } + batchWriteParams.RequestItems[TableName].push({ + [type]: rest }); } From bb7e041b35f632338e46f50a92c424d99c64e23c Mon Sep 17 00:00:00 2001 From: Adrian Smijulj Date: Mon, 7 Sep 2020 12:08:34 +0200 Subject: [PATCH 21/26] feat: introduce Transaction class --- packages/fields-storage/src/Transaction.js | 23 ++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 packages/fields-storage/src/Transaction.js diff --git a/packages/fields-storage/src/Transaction.js b/packages/fields-storage/src/Transaction.js new file mode 100644 index 0000000..0f80459 --- /dev/null +++ b/packages/fields-storage/src/Transaction.js @@ -0,0 +1,23 @@ +// @flow +import Batch from "./Batch"; + +class Transaction extends Batch { + constructor(...operations) { + super(...operations); + this.type = "transaction"; + } + + async execute() { + for (let i = 0; i < this.operations.length; i++) { + const item = this.operations[i]; + if (Array.isArray(item)) { + await this.operations[i][0]({ ...this.operations[i][1], batch: this }); + } else { + await this.operations[i]({ batch: this }); + } + } + return this; + } +} + +export default Transaction; From b485b067d7539da41e7383f0d3ebd4f2fa60511c Mon Sep 17 00:00:00 2001 From: Adrian Smijulj Date: Mon, 7 Sep 2020 12:08:58 +0200 Subject: [PATCH 22/26] fix: find entries in storage pool by primary key, not `id` --- packages/fields-storage/src/StoragePool.js | 39 ++++++++++++---------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/packages/fields-storage/src/StoragePool.js b/packages/fields-storage/src/StoragePool.js index 45cbddd..1b82feb 100644 --- a/packages/fields-storage/src/StoragePool.js +++ b/packages/fields-storage/src/StoragePool.js @@ -1,6 +1,20 @@ // @flow import { getName } from "@commodo/name"; import StoragePoolEntry from "./StoragePoolEntry"; +import getPrimaryKey from "./getPrimaryKey"; + +function getPoolItemId(model, data) { + const primaryKey = getPrimaryKey(model); + const output = { namespace: model.getStorageName(), id: [] }; + + for (let i = 0; i < primaryKey.fields.length; i++) { + let field = primaryKey.fields[i]; + output.id.push(data ? data[field.name] : model[field.name]); + } + + output.id.join(":"); + return output; +} class StoragePool { pool: {}; @@ -13,43 +27,32 @@ class StoragePool { } add(model: $Subtype): this { - const namespace = getName(model); + const { namespace, id } = getPoolItemId(model); if (!this.getPool()[namespace]) { this.getPool()[namespace] = {}; } - this.getPool()[namespace][model.id] = new StoragePoolEntry(model); + this.getPool()[namespace][id] = new StoragePoolEntry(model); return this; } - has(model, id): boolean { - const namespace = getName(model); - if (!this.getPool()[namespace]) { - return false; - } - - const modelId = id || model.id; - return typeof this.getPool()[namespace][modelId] !== "undefined"; - } - remove(model): this { - const namespace = getName(model); + const { namespace, id } = getPoolItemId(model); if (!this.getPool()[namespace]) { return this; } - delete this.getPool()[namespace][model.id]; + delete this.getPool()[namespace][id]; return this; } - get(model: Class<$Subtype> | CreateModel, id: ?mixed): ?CreateModel { - const namespace = getName(model); + get(model, data: any = null) { + const { namespace, id } = getPoolItemId(model, data); if (!this.getPool()[namespace]) { return undefined; } - const modelId = id || model.id; - const poolEntry: StoragePoolEntry = this.getPool()[namespace][modelId]; + const poolEntry: StoragePoolEntry = this.getPool()[namespace][id]; if (poolEntry) { return poolEntry.getModel(); } From 1ed2340a16e48d7c6b8ee15f290dd446dce3b75d Mon Sep 17 00:00:00 2001 From: Adrian Smijulj Date: Mon, 7 Sep 2020 12:09:54 +0200 Subject: [PATCH 23/26] fix: add IndexName if unique key query was recognized --- .../src/statements/createKeyConditionExpressionArgs.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/fields-storage-dynamodb/src/statements/createKeyConditionExpressionArgs.js b/packages/fields-storage-dynamodb/src/statements/createKeyConditionExpressionArgs.js index 36f6ec2..8d64f01 100644 --- a/packages/fields-storage-dynamodb/src/statements/createKeyConditionExpressionArgs.js +++ b/packages/fields-storage-dynamodb/src/statements/createKeyConditionExpressionArgs.js @@ -20,5 +20,9 @@ export default ({ query, sort, key }) => { output.ScanIndexForward = true; } + if (!key.primary) { + output.IndexName = key.name; + } + return output; }; From 4689f6dcb3b6ece82384e934d87730e3322aab2c Mon Sep 17 00:00:00 2001 From: Adrian Smijulj Date: Mon, 7 Sep 2020 12:10:34 +0200 Subject: [PATCH 24/26] chore: rename `maxPerPage` to `maxLimit` --- .../__tests__/maxPerPage.test.js | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/packages/fields-storage/__tests__/maxPerPage.test.js b/packages/fields-storage/__tests__/maxPerPage.test.js index 4c61746..5e7f298 100644 --- a/packages/fields-storage/__tests__/maxPerPage.test.js +++ b/packages/fields-storage/__tests__/maxPerPage.test.js @@ -4,69 +4,69 @@ import { withName } from "@commodo/name"; import { compose } from "ramda"; import { CustomDriver } from "./resources/CustomDriver"; -const EntityWithoutMaxPerPage = compose( +const EntityWithoutMaxLimit = compose( withFields({ name: string() }), - withName("EntityWithoutMaxPerPage"), + withName("EntityWithoutMaxLimit"), withStorage({ driver: new CustomDriver() }) )(); -const EntityWithMaxPerPage = compose( +const EntityWithMaxLimit = compose( withFields({ name: string() }), - withName("EntityWithMaxPerPage"), + withName("EntityWithMaxLimit"), withStorage({ driver: new CustomDriver(), - maxPerPage: 500 + maxLimit: 500 }) )(); -const EntityWithMaxPerPageSetToNull = compose( +const EntityWithMaxLimitSetToNull = compose( withFields({ name: string() }), - withName("EntityWithMaxPerPage"), + withName("EntityWithMaxLimit"), withStorage({ driver: new CustomDriver(), - maxPerPage: null + maxLimit: null }) )(); -describe("maxPerPage test", () => { - test("must throw errors if maxPerPage config parameter was exceeded", async () => { - await EntityWithoutMaxPerPage.find({ limit: 99 }); - await EntityWithoutMaxPerPage.find({ limit: 100 }); +describe("maxLimit test", () => { + test("must throw errors if maxLimit config parameter was exceeded", async () => { + await EntityWithoutMaxLimit.find({ limit: 99 }); + await EntityWithoutMaxLimit.find({ limit: 100 }); let error = null; try { - await EntityWithoutMaxPerPage.find({ limit: 101 }); + await EntityWithoutMaxLimit.find({ limit: 101 }); } catch (e) { error = e; } expect(error.message).toBe("Cannot query for more than 100 models per page."); - await EntityWithMaxPerPage.find({ limit: 499 }); - await EntityWithMaxPerPage.find({ limit: 500 }); + await EntityWithMaxLimit.find({ limit: 499 }); + await EntityWithMaxLimit.find({ limit: 500 }); error = null; try { - await EntityWithMaxPerPage.find({ limit: 501 }); + await EntityWithMaxLimit.find({ limit: 501 }); } catch (e) { error = e; } expect(error.message).toBe("Cannot query for more than 500 models per page."); - await EntityWithMaxPerPageSetToNull.find({ limit: 100 }); + await EntityWithMaxLimitSetToNull.find({ limit: 100 }); error = null; try { - await EntityWithoutMaxPerPage.find({ limit: 101 }); + await EntityWithoutMaxLimit.find({ limit: 101 }); } catch (e) { error = e; } From 256466a9a1f7e91e44e002db179952b1e7893bd5 Mon Sep 17 00:00:00 2001 From: Adrian Smijulj Date: Mon, 7 Sep 2020 12:13:44 +0200 Subject: [PATCH 25/26] feat: introduce additional static method BREAKING CHANGE --- packages/fields-storage/src/withStorage.js | 270 ++++++++++----------- 1 file changed, 135 insertions(+), 135 deletions(-) diff --git a/packages/fields-storage/src/withStorage.js b/packages/fields-storage/src/withStorage.js index d3d212c..51158c7 100644 --- a/packages/fields-storage/src/withStorage.js +++ b/packages/fields-storage/src/withStorage.js @@ -2,8 +2,8 @@ import { getName as defaultGetName } from "@commodo/name"; import getStorageName from "./getStorageName"; import getPrimaryKey from "./getPrimaryKey"; +import getKeys from "./getKeys"; import { withStaticProps, withProps } from "repropose"; -import cloneDeep from "lodash.clonedeep"; import { withHooks } from "@commodo/hooks"; import type { SaveParams } from "@commodo/fields-storage/types"; import WithStorageError from "./WithStorageError"; @@ -11,12 +11,16 @@ import Collection from "./Collection"; import StoragePool from "./StoragePool"; import FieldsStorageAdapter from "./FieldsStorageAdapter"; import { decodeCursor, encodeCursor } from "./cursor"; + +// TODO: check faster alternative. +import cloneDeep from "lodash.clonedeep"; + interface IStorageDriver {} type Configuration = { storagePool?: StoragePool, driver?: IStorageDriver, - maxPerPage: ?number + maxLimit: ?number }; const defaults = { @@ -35,7 +39,7 @@ const hook = async (name, { options, model }) => { await model.hook(name, { model, options }); }; -const registerSaveUpdateCreateHooks = async (prefix, { existing, model, options }) => { +const triggerSaveUpdateCreateHooks = async (prefix, { existing, model, options }) => { await hook(prefix + "Save", { model, options }); if (existing) { await hook(prefix + "Update", { model, options }); @@ -76,27 +80,12 @@ function cursorFrom(data, keys) { const withStorage = (configuration: Configuration) => { return baseFn => { - let fn = withHooks({ - delete() { - if (!this.id) { - throw new WithStorageError( - "Cannot delete before saving to storage.", - WithStorageError.CANNOT_DELETE_NO_ID - ); - } - } - })(baseFn); - - fn = withProps(props => ({ + let fn = withProps(props => ({ __withStorage: { existing: false, processing: false, fieldsStorageAdapter: new FieldsStorageAdapter() }, - // generateId, - // isId(value) { - // return typeof value === "string" && !!value.match(/^[a-zA-Z0-9]*$/); - // }, isExisting() { return this.__withStorage.existing; }, @@ -104,7 +93,7 @@ const withStorage = (configuration: Configuration) => { this.__withStorage.existing = existing; return this; }, - async save(options: ?SaveParams): Promise { + async save(rawArgs: ?SaveParams): Promise { const primaryKey = getPrimaryKey(this); if (!primaryKey) { throw Error( @@ -112,7 +101,7 @@ const withStorage = (configuration: Configuration) => { ); } - options = { ...options, ...defaults.save }; + const args = { ...defaults.save, ...cloneDeep(rawArgs) }; if (this.__withStorage.processing) { return; @@ -122,72 +111,78 @@ const withStorage = (configuration: Configuration) => { const existing = this.isExisting(); - await registerSaveUpdateCreateHooks("before", { existing, model: this, options }); + await triggerSaveUpdateCreateHooks("before", { + existing, + model: this, + options: args + }); try { - await hook("__save", { model: this, options }); + await hook("__save", { model: this, options: args }); if (existing) { - await hook("__update", { model: this, options }); + await hook("__update", { model: this, options: args }); } else { - await hook("__create", { model: this, options }); + await hook("__create", { model: this, options: args }); } - options.validation !== false && (await this.validate()); + args.validation !== false && (await this.validate()); - await registerSaveUpdateCreateHooks("__before", { + await triggerSaveUpdateCreateHooks("__before", { existing, model: this, - options + options: args }); if (this.isDirty()) { - // if (!this.id) { - // this.id = this.constructor.generateId(); - // } + const data = await this.toStorage(); if (existing) { - // const { getId } = options; - // await this.getStorageDriver().update(model[ - // { - // name: getName(this), - // query: - // typeof getId === "function" ? getId(this) : { id: this.id }, - // data: await this.toStorage() - // } - // ]); - await this.getStorageDriver().update({ model: this, primaryKey }); + const query = {}; + for (let i = 0; i < primaryKey.fields.length; i++) { + let field = primaryKey.fields[i]; + query[field.name] = this[field.name]; + } + + await this.constructor.update({ + batch: args.batch, + instance: this, + data, + query + }); } else { - await this.getStorageDriver().create({ model: this, primaryKey }); + await this.constructor.create({ + batch: args.batch, + instance: this, + data + }); } } - await registerSaveUpdateCreateHooks("__after", { + await triggerSaveUpdateCreateHooks("__after", { existing, model: this, - options + options: args }); this.setExisting(); this.clean(); this.constructor.getStoragePool().add(this); - } catch (e) { - if (!existing) { - // this.getField("id").reset(); - } - throw e; } finally { this.__withStorage.processing = null; } - await registerSaveUpdateCreateHooks("after", { existing, model: this, options }); + await triggerSaveUpdateCreateHooks("after", { + existing, + model: this, + options: args + }); }, /** * Deletes current and all linked models (if autoDelete on the attribute was enabled). - * @param options */ - async delete(options: ?Object) { + async delete(rawArgs) { const primaryKey = getPrimaryKey(this); if (!primaryKey) { throw Error( @@ -201,28 +196,28 @@ const withStorage = (configuration: Configuration) => { this.__withStorage.processing = "delete"; - options = { ...options, ...defaults.delete }; - - let getId; - - ({ getId, ...options } = options); + const args = { ...defaults.delete, ...cloneDeep(rawArgs) }; try { - await this.hook("delete", { options, model: this }); + await this.hook("delete", { options: args, model: this }); - options.validation !== false && (await this.validate()); + args.validation !== false && (await this.validate()); - await this.hook("beforeDelete", { options, model: this }); + await this.hook("beforeDelete", { options: args, model: this }); - await this.getStorageDriver().delete({ - name: getName(this), - options: { - query: typeof getId === "function" ? getId(this) : { id: this.id }, - ...options - } + const query = {}; + for (let i = 0; i < primaryKey.fields.length; i++) { + let field = primaryKey.fields[i]; + query[field.name] = this[field.name]; + } + + await this.constructor.delete({ + batch: args.batch, + instance: this, + query }); - await this.hook("afterDelete", { options, model: this }); + await this.hook("afterDelete", { options: args, model: this }); this.constructor.getStoragePool().remove(this); } finally { @@ -252,7 +247,7 @@ const withStorage = (configuration: Configuration) => { skipDifferenceCheck }); } - }))(fn); + }))(baseFn); fn = withStaticProps(() => { const __withStorage = { @@ -293,76 +288,80 @@ const withStorage = (configuration: Configuration) => { }, /** - * Finds a single model matched by given ID. - * @param id - * @param options + * Inserts an entry into the database. + */ + async create(args) { + const { data, batch, ...rest } = cloneDeep(args); + + return this.getStorageDriver().create({ + ...rest, + model: this, + name: getName(this), + keys: getKeys(this), + primaryKey: getPrimaryKey(this), + data, + batch + }); + }, + + /** + * Updates an existing entry in the database. */ - // async findById(id: mixed, options: ?Object): Promise { - // if (!id || !this.isId(id)) { - // return null; - // } - // - // const pooled = this.getStoragePool().get(this, id); - // if (pooled) { - // return pooled; - // } - // - // if (!options) { - // options = {}; - // } - // - // const newParams = { ...options, query: { id } }; - // return await this.findOne(newParams); - // }, + async update(args = {}) { + const { data, query, batch, ...rest } = cloneDeep(args); + + return this.getStorageDriver().update({ + ...rest, + model: this, + name: getName(this), + keys: getKeys(this), + primaryKey: getPrimaryKey(this), + data, + query, + batch + }); + }, + + /** + * Deletes an existing entry in the database. + */ + async delete(args = {}) { + const { data, query, batch, ...rest } = cloneDeep(args); + + return this.getStorageDriver().delete({ + ...rest, + model: this, + name: getName(this), + keys: getKeys(this), + primaryKey: getPrimaryKey(this), + data, + query, + batch + }); + }, /** * Finds one model matched by given query parameters. * @param rawArgs */ - async findOne(rawArgs: ?Object): Promise> { + async findOne(rawArgs) { + // TODO: don't load if not necessary, check storage poll first. if (!rawArgs) { rawArgs = {}; } const args = cloneDeep(rawArgs); - // - // { - // name: getName(this), - // options: prepared - // } - // - const result = await this.getStorageDriver().findOne({ - model: this, - args - }); + args.limit = 1; + const [result] = await this.find(args); return result; - if (result) { - const pooled = this.getStoragePool().get(this, result.id); - if (pooled) { - return pooled; - } - - const model: $Subtype = new this(); - model.setExisting(); - await model.populateFromStorage(((result: any): Object)); - this.getStoragePool().add(model); - return model; - } - return null; }, - // isId(value) { - // return typeof value === "string" && !!value.match(/^[0-9a-fA-F]{24}$/); - // }, - // generateId, - async find(options: ?FindParams) { - if (!options) { - options = {}; - } - const maxPerPage = this.__withStorage.maxPerPage || 100; + async find(args = {}) { + const maxLimit = this.__withStorage.maxLimit || 100; let { + batch, query = {}, sort, limit, @@ -371,17 +370,17 @@ const withStorage = (configuration: Configuration) => { totalCount: countTotal = false, defaultSortField = "id", ...other - } = options; + } = args; if (!sort) { sort = {}; } - limit = Number.isInteger(limit) && limit > 0 ? limit : maxPerPage; + limit = Number.isInteger(limit) && limit > 0 ? limit : maxLimit; - if (limit > maxPerPage) { + if (limit > maxLimit) { throw new WithStorageError( - `Cannot query for more than ${maxPerPage} models per page.`, + `Cannot query for more than ${maxLimit} models per page.`, WithStorageError.MAX_PER_PAGE_EXCEEDED ); } @@ -436,10 +435,14 @@ const withStorage = (configuration: Configuration) => { } const params = { query, sort, limit: limit + 1, ...other }; - let [results, meta] = await this.getStorageDriver().find({ + let [results, meta = {}] = await this.getStorageDriver().find({ + ...params, + model: this, name: getName(this), - args: params, - model: this + keys: getKeys(this), + primaryKey: getPrimaryKey(this), + query, + batch }); // Have we reached the last record? @@ -455,12 +458,9 @@ const withStorage = (configuration: Configuration) => { let totalCount = null; if (countTotal) { totalCount = await this.getStorageDriver().count({ - model: this, - name: getName(this), - args: { - query: originalQuery, - ...other - }, + query: originalQuery, + ...other, + name: getName(this) }); } @@ -491,7 +491,7 @@ const withStorage = (configuration: Configuration) => { const result: ?Array = results; if (result instanceof Array) { for (let i = 0; i < result.length; i++) { - const pooled = this.getStoragePool().get(this, result[i].id); + const pooled = this.getStoragePool().get(this, result[i]); if (pooled) { collection.push(pooled); } else { From 5676b840f3e095402c95ade4e0c099605584383b Mon Sep 17 00:00:00 2001 From: Adrian Smijulj Date: Mon, 7 Sep 2020 12:32:52 +0200 Subject: [PATCH 26/26] ci: add beta release action --- .github/workflows/betaRelease.yml | 51 +++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 .github/workflows/betaRelease.yml diff --git a/.github/workflows/betaRelease.yml b/.github/workflows/betaRelease.yml new file mode 100644 index 0000000..9134abd --- /dev/null +++ b/.github/workflows/betaRelease.yml @@ -0,0 +1,51 @@ +name: Next Release + +on: + push: + branches: [ beta ] + +env: + GH_TOKEN: ${{ secrets.GH_TOKEN }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + +jobs: + build-test-release: + name: Build, test and release + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-node@v2-beta + with: + node-version: '12' + + - name: Create ".npmrc" file in the project root + run: echo "//registry.npmjs.org/:_authToken=\${NPM_TOKEN}" > .npmrc + + - name: Install dependencies + run: yarn --frozen-lockfile + + - name: Check dependencies + run: yarn adio + + - name: Build packages + run: yarn build + + # - name: Run tests + # run: yarn test:dist + + - name: Prepare packages + run: yarn prepack + + - name: Set git email + run: git config --global user.email "info@webiny.com" + + - name: Set git username + run: git config --global user.name "webiny" + + - name: Create a release on GitHub + run: yarn lerna version --conventional-prerelease --yes + + - name: Release packages to NPM + run: yarn lerna publish from-package --dist-tag beta --yes +