From b8a7559ec91ec083aa946b517ee4c8906b93de7b Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Thu, 16 Mar 2023 15:49:53 -0700 Subject: [PATCH 1/7] Remove old test files --- pkg/mongoproxy/plugins/authz/after | 9 --------- pkg/mongoproxy/plugins/authz/before | 9 --------- 2 files changed, 18 deletions(-) delete mode 100644 pkg/mongoproxy/plugins/authz/after delete mode 100644 pkg/mongoproxy/plugins/authz/before diff --git a/pkg/mongoproxy/plugins/authz/after b/pkg/mongoproxy/plugins/authz/after deleted file mode 100644 index 0c5213c..0000000 --- a/pkg/mongoproxy/plugins/authz/after +++ /dev/null @@ -1,9 +0,0 @@ -goos: linux -goarch: amd64 -pkg: github.com/wish/mongoproxy/pkg/mongoproxy/plugins/authz -cpu: AMD Ryzen 7 PRO 5850U with Radeon Graphics -BenchmarkPluginDefaultPolicies/0-16 733518 1600 ns/op 1152 B/op 10 allocs/op -BenchmarkPluginDefaultPolicies/1-16 756729 1693 ns/op 1152 B/op 10 allocs/op -BenchmarkPluginDefaultPolicies/2-16 839948 1394 ns/op 976 B/op 7 allocs/op -PASS -ok github.com/wish/mongoproxy/pkg/mongoproxy/plugins/authz 3.735s diff --git a/pkg/mongoproxy/plugins/authz/before b/pkg/mongoproxy/plugins/authz/before deleted file mode 100644 index a5740a5..0000000 --- a/pkg/mongoproxy/plugins/authz/before +++ /dev/null @@ -1,9 +0,0 @@ -goos: linux -goarch: amd64 -pkg: github.com/wish/mongoproxy/pkg/mongoproxy/plugins/authz -cpu: AMD Ryzen 7 PRO 5850U with Radeon Graphics -BenchmarkPluginDefaultPolicies/0-16 750019 1610 ns/op 1152 B/op 10 allocs/op -BenchmarkPluginDefaultPolicies/1-16 671475 1555 ns/op 1152 B/op 10 allocs/op -BenchmarkPluginDefaultPolicies/2-16 992031 1317 ns/op 976 B/op 7 allocs/op -PASS -ok github.com/wish/mongoproxy/pkg/mongoproxy/plugins/authz 3.653s From bd2749b2c8142d8bdaf09617a05ac236832d49c7 Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Thu, 16 Mar 2023 16:32:36 -0700 Subject: [PATCH 2/7] Update ttlcache --- go.mod | 4 +- go.sum | 10 +- .../ReneKroon/ttlcache/v2/.travis.yml | 23 -- .../ReneKroon/ttlcache/v2/CHANGELOG.md | 34 --- .../jellydator/ttlcache/v2/CHANGELOG.md | 94 ++++++ .../ttlcache/v2/LICENSE | 2 +- .../ttlcache/v2/Readme.md | 47 ++- .../ttlcache/v2/cache.go | 278 +++++++++++++----- .../ttlcache/v2/evictionreason_enumer.go | 0 .../ttlcache/v2/go.mod | 4 +- .../ttlcache/v2/go.sum | 35 +-- .../ttlcache/v2/item.go | 0 .../ttlcache/v2/metrics.go | 0 .../ttlcache/v2/priority_queue.go | 14 + vendor/golang.org/x/sync/AUTHORS | 3 - vendor/golang.org/x/sync/CONTRIBUTORS | 3 - vendor/golang.org/x/sync/errgroup/errgroup.go | 74 ++++- .../x/sync/singleflight/singleflight.go | 11 +- vendor/modules.txt | 6 +- 19 files changed, 437 insertions(+), 205 deletions(-) delete mode 100644 vendor/github.com/ReneKroon/ttlcache/v2/.travis.yml delete mode 100644 vendor/github.com/ReneKroon/ttlcache/v2/CHANGELOG.md create mode 100644 vendor/github.com/jellydator/ttlcache/v2/CHANGELOG.md rename vendor/github.com/{ReneKroon => jellydator}/ttlcache/v2/LICENSE (97%) rename vendor/github.com/{ReneKroon => jellydator}/ttlcache/v2/Readme.md (77%) rename vendor/github.com/{ReneKroon => jellydator}/ttlcache/v2/cache.go (63%) rename vendor/github.com/{ReneKroon => jellydator}/ttlcache/v2/evictionreason_enumer.go (100%) rename vendor/github.com/{ReneKroon => jellydator}/ttlcache/v2/go.mod (73%) rename vendor/github.com/{ReneKroon => jellydator}/ttlcache/v2/go.sum (63%) rename vendor/github.com/{ReneKroon => jellydator}/ttlcache/v2/item.go (100%) rename vendor/github.com/{ReneKroon => jellydator}/ttlcache/v2/metrics.go (100%) rename vendor/github.com/{ReneKroon => jellydator}/ttlcache/v2/priority_queue.go (82%) delete mode 100644 vendor/golang.org/x/sync/AUTHORS delete mode 100644 vendor/golang.org/x/sync/CONTRIBUTORS diff --git a/go.mod b/go.mod index 0e820df..53ef387 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,9 @@ go 1.13 require ( github.com/HdrHistogram/hdrhistogram-go v1.1.0 // indirect - github.com/ReneKroon/ttlcache/v2 v2.3.0 github.com/cespare/xxhash/v2 v2.1.1 github.com/getsentry/sentry-go v0.9.0 + github.com/jellydator/ttlcache/v2 v2.11.1 github.com/jessevdk/go-flags v1.4.0 github.com/json-iterator/go v1.1.11 github.com/miekg/dns v1.1.41 // indirect @@ -20,7 +20,7 @@ require ( go.mongodb.org/mongo-driver v1.5.1 go.uber.org/automaxprocs v1.3.0 golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c // indirect - golang.org/x/sync v0.0.0-20210220032951-036812b2e83c + golang.org/x/sync v0.1.0 golang.org/x/sys v0.0.0-20210426080607-c94f62235c83 // indirect golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 gopkg.in/fsnotify.v1 v1.4.7 diff --git a/go.sum b/go.sum index 104b4c7..2693107 100644 --- a/go.sum +++ b/go.sum @@ -11,8 +11,6 @@ github.com/HdrHistogram/hdrhistogram-go v1.1.0 h1:6dpdDPTRoo78HxAJ6T1HfMiKSnqhgR github.com/HdrHistogram/hdrhistogram-go v1.1.0/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= -github.com/ReneKroon/ttlcache/v2 v2.3.0 h1:qZnUjRKIrbKHH6vF5T7Y9Izn5ObfTZfyYpGhvz2BKPo= -github.com/ReneKroon/ttlcache/v2 v2.3.0/go.mod h1:zbo6Pv/28e21Z8CzzqgYRArQYGYtjONRxaAKGxzQvG4= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= @@ -25,7 +23,6 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/alvaroloes/enumer v1.1.2/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT0m65DJ+Wo= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= @@ -213,6 +210,8 @@ github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/ github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk= github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g= github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= +github.com/jellydator/ttlcache/v2 v2.11.1 h1:AZGME43Eh2Vv3giG6GeqeLeFXxwxn1/qHItqWZl6U64= +github.com/jellydator/ttlcache/v2 v2.11.1/go.mod h1:RtE5Snf0/57e+2cLWFYWCCsLas2Hy3c5Z4n14XmSvTI= github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -329,7 +328,6 @@ github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnh github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1/go.mod h1:eD5JxqMiuNYyFNmyY9rkJ/slN8y59oEu4Ei7F8OoKWQ= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= @@ -547,8 +545,9 @@ golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -610,7 +609,6 @@ golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= diff --git a/vendor/github.com/ReneKroon/ttlcache/v2/.travis.yml b/vendor/github.com/ReneKroon/ttlcache/v2/.travis.yml deleted file mode 100644 index c8f0d21..0000000 --- a/vendor/github.com/ReneKroon/ttlcache/v2/.travis.yml +++ /dev/null @@ -1,23 +0,0 @@ -language: go - -go: - - "1.15.x" - - "1.14.x" - -git: - depth: 1 - -install: - - go install -race std - - go install golang.org/x/tools/cmd/cover - - go install golang.org/x/lint/golint - - export PATH=$HOME/gopath/bin:$PATH - - go get golang.org/x/tools/cmd/cover - - go get github.com/mattn/goveralls - -script: - - golint . - - go test -cover -race -count=1 -timeout=30s -run . - - go test -covermode=count -coverprofile=coverage.out -timeout=90s -run . - - '[ ! -z "$COVERALLS_TOKEN" ] && $HOME/gopath/bin/goveralls -coverprofile=coverage.out -service=travis-ci -repotoken $COVERALLS_TOKEN' - - cd bench; go test -run=Bench.* -bench=. -benchmem; cd .. \ No newline at end of file diff --git a/vendor/github.com/ReneKroon/ttlcache/v2/CHANGELOG.md b/vendor/github.com/ReneKroon/ttlcache/v2/CHANGELOG.md deleted file mode 100644 index 7ba8bd4..0000000 --- a/vendor/github.com/ReneKroon/ttlcache/v2/CHANGELOG.md +++ /dev/null @@ -1,34 +0,0 @@ -# 2.3.0 (February 2021) - -## API changes: - -* #38: Added func (cache *Cache) SetExpirationReasonCallback(callback ExpireReasonCallback) This wil function will replace SetExpirationCallback(..) in the next major version. - -# 2.2.0 (January 2021) - -## API changes: - -* #37 : a GetMetrics call is now available for some information on hits/misses etc. -* #34 : Errors are now const - -# 2.1.0 (October 2020) - -## API changes - -* `SetCacheSizeLimit(limit int)` a call was contributed to set a cache limit. #35 - -# 2.0.0 (July 2020) - -## Fixes #29, #30, #31 - -## Behavioural changes - -* `Remove(key)` now also calls the expiration callback when it's set -* `Count()` returns zero when the cache is closed - -## API changes - -* `SetLoaderFunction` allows you to provide a function to retrieve data on missing cache keys. -* Operations that affect item behaviour such as `Close`, `Set`, `SetWithTTL`, `Get`, `Remove`, `Purge` now return an error with standard errors `ErrClosed` an `ErrNotFound` instead of a bool or nothing -* `SkipTTLExtensionOnHit` replaces `SkipTtlExtensionOnHit` to satisfy golint -* The callback types are now exported diff --git a/vendor/github.com/jellydator/ttlcache/v2/CHANGELOG.md b/vendor/github.com/jellydator/ttlcache/v2/CHANGELOG.md new file mode 100644 index 0000000..dbfa620 --- /dev/null +++ b/vendor/github.com/jellydator/ttlcache/v2/CHANGELOG.md @@ -0,0 +1,94 @@ +# 2.11.0 (December 2021) + +#64: @DoubeDi added a method `GetItems` to retrieve all items in the cache. This method also triggers all callbacks associated with a normal `Get` + +## API changes: + +// GetItems returns a copy of all items in the cache. Returns nil when the cache has been closed. +func (cache *Cache) GetItems() map[string]interface{} { + +# 2.10.0 (December 2021) + +#62 : @nikhilk1701 found a memory leak where removed items are not directly eligible for garbage collection. There are no API changes. + +# 2.9.0 (October 2021) + +#55,#56,#57 : @chenyahui was on fire and greatly improved the peformance of the library. He also got rid of the blocking call to expirationNotification, making the code run twice as fast in the benchmarks! + +# 2.8.1 (September 2021) + +#53 : Avoids recalculation of TTL value returned in API when TTL is extended. by @iczc + +# 2.8.0 (August 2021) + +#51 : The call GetWithTTL(key string) (interface{}, time.Duration, error) is added so that you can retrieve an item, and also know the remaining TTL. Thanks to @asgarciap for contributing. + +# 2.7.0 (June 2021) + +#46 : got panic + +A panic occured in a line that checks the maximum amount of items in the cache. While not definite root cause has been found, there is indeed the possibility of crashing an empty cache if the cache limit is set to 'zero' which codes for infinite. This would lead to removal of the first item in the cache which would panic on an empty cache. + +Fixed this by applying the global cache lock to all configuration options as well. + +# 2.6.0 (May 2021) + +#44 : There are no API changes, but a contribution was made to use https://pkg.go.dev/golang.org/x/sync/singleflight as a way to provide everybody waiting for a key with that key when it's fetched. + +This removes some complexity from the code and will make sure that all callers will get a return value even if there's high concurrency and low TTL (as proven by the test that was added). + +# 2.5.0 (May 2021) + +## API changes: + +* #39 : Allow custom loader function for each key via `GetByLoader` + +Introduce the `SimpleCache` interface for quick-start and basic usage. + +# 2.4.0 (April 2021) + +## API changes: + +* #42 : Add option to get list of keys +* #40: Allow 'Touch' on items without other operation + +// Touch resets the TTL of the key when it exists, returns ErrNotFound if the key is not present. +func (cache *Cache) Touch(key string) error + +// GetKeys returns all keys of items in the cache. Returns nil when the cache has been closed. +func (cache *Cache) GetKeys() []string + +# 2.3.0 (February 2021) + +## API changes: + +* #38: Added func (cache *Cache) SetExpirationReasonCallback(callback ExpireReasonCallback) This wil function will replace SetExpirationCallback(..) in the next major version. + +# 2.2.0 (January 2021) + +## API changes: + +* #37 : a GetMetrics call is now available for some information on hits/misses etc. +* #34 : Errors are now const + +# 2.1.0 (October 2020) + +## API changes + +* `SetCacheSizeLimit(limit int)` a call was contributed to set a cache limit. #35 + +# 2.0.0 (July 2020) + +## Fixes #29, #30, #31 + +## Behavioural changes + +* `Remove(key)` now also calls the expiration callback when it's set +* `Count()` returns zero when the cache is closed + +## API changes + +* `SetLoaderFunction` allows you to provide a function to retrieve data on missing cache keys. +* Operations that affect item behaviour such as `Close`, `Set`, `SetWithTTL`, `Get`, `Remove`, `Purge` now return an error with standard errors `ErrClosed` an `ErrNotFound` instead of a bool or nothing +* `SkipTTLExtensionOnHit` replaces `SkipTtlExtensionOnHit` to satisfy golint +* The callback types are now exported diff --git a/vendor/github.com/ReneKroon/ttlcache/v2/LICENSE b/vendor/github.com/jellydator/ttlcache/v2/LICENSE similarity index 97% rename from vendor/github.com/ReneKroon/ttlcache/v2/LICENSE rename to vendor/github.com/jellydator/ttlcache/v2/LICENSE index b3b587d..f36a3b9 100644 --- a/vendor/github.com/ReneKroon/ttlcache/v2/LICENSE +++ b/vendor/github.com/jellydator/ttlcache/v2/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2018 Rene Kroon +Copyright (c) 2022 Jellydator Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/vendor/github.com/ReneKroon/ttlcache/v2/Readme.md b/vendor/github.com/jellydator/ttlcache/v2/Readme.md similarity index 77% rename from vendor/github.com/ReneKroon/ttlcache/v2/Readme.md rename to vendor/github.com/jellydator/ttlcache/v2/Readme.md index 08ea517..9c736cd 100644 --- a/vendor/github.com/ReneKroon/ttlcache/v2/Readme.md +++ b/vendor/github.com/jellydator/ttlcache/v2/Readme.md @@ -1,7 +1,7 @@ # TTLCache - an in-memory cache with expiration -[![Documentation](https://img.shields.io/badge/go.dev-reference-007d9c?logo=go&logoColor=white&style=flat-square)](https://pkg.go.dev/github.com/ReneKroon/ttlcache/v2) -[![Release](https://img.shields.io/github/release/ReneKroon/ttlcache.svg?label=Release)](https://github.com/ReneKroon/ttlcache/releases) +**Although v2 of ttlcache is not yet deprecated, v3 should be used as it +contains quite a few additions and improvements.** TTLCache is a simple key/value cache in golang with the following functions: @@ -15,16 +15,43 @@ TTLCache is a simple key/value cache in golang with the following functions: Note (issue #25): by default, due to historic reasons, the TTL will be reset on each cache hit and you need to explicitly configure the cache to use a TTL that will not get extended. -[![Build Status](https://travis-ci.org/ReneKroon/ttlcache.svg?branch=master)](https://travis-ci.org/ReneKroon/ttlcache) -[![Go Report Card](https://goreportcard.com/badge/github.com/ReneKroon/ttlcache)](https://goreportcard.com/report/github.com/ReneKroon/ttlcache) -[![Coverage Status](https://coveralls.io/repos/github/ReneKroon/ttlcache/badge.svg?branch=master)](https://coveralls.io/github/ReneKroon/ttlcache?branch=master) -[![GitHub issues](https://img.shields.io/github/issues/ReneKroon/ttlcache.svg)](https://github.com/ReneKroon/ttlcache/issues) -[![license](https://img.shields.io/github/license/ReneKroon/ttlcache.svg?maxAge=2592000)](https://github.com/ReneKroon/ttlcache/LICENSE) - ## Usage -You can copy it as a full standalone demo program. +`go get github.com/jellydator/ttlcache/v2` + +You can copy it as a full standalone demo program. The first snippet is basic usage, where the second exploits more options in the cache. + +Basic: +```go +package main + +import ( + "fmt" + "time" + + "github.com/jellydator/ttlcache/v2" +) + +var notFound = ttlcache.ErrNotFound + +func main() { + var cache ttlcache.SimpleCache = ttlcache.NewCache() + + cache.SetTTL(time.Duration(10 * time.Second)) + cache.Set("MyKey", "MyValue") + cache.Set("MyNumber", 1000) + + if val, err := cache.Get("MyKey"); err != notFound { + fmt.Printf("Got it: %s\n", val) + } + + cache.Remove("MyNumber") + cache.Purge() + cache.Close() +} +``` +Advanced: ```go package main @@ -32,7 +59,7 @@ import ( "fmt" "time" - "github.com/ReneKroon/ttlcache/v2" + "github.com/jellydator/ttlcache/v2" ) var ( diff --git a/vendor/github.com/ReneKroon/ttlcache/v2/cache.go b/vendor/github.com/jellydator/ttlcache/v2/cache.go similarity index 63% rename from vendor/github.com/ReneKroon/ttlcache/v2/cache.go rename to vendor/github.com/jellydator/ttlcache/v2/cache.go index 89dd917..4b2f820 100644 --- a/vendor/github.com/ReneKroon/ttlcache/v2/cache.go +++ b/vendor/github.com/jellydator/ttlcache/v2/cache.go @@ -3,13 +3,15 @@ package ttlcache import ( "sync" "time" + + "golang.org/x/sync/singleflight" ) // CheckExpireCallback is used as a callback for an external check on item expiration type CheckExpireCallback func(key string, value interface{}) bool // ExpireCallback is used as a callback on item expiration or when notifying of an item new to the cache -// Note that ExpireReasonCallback will be the succesor of this function in the next major release. +// Note that ExpireReasonCallback will be the successor of this function in the next major release. type ExpireCallback func(key string, value interface{}) // ExpireReasonCallback is used as a callback on item expiration with extra information why the item expired. @@ -18,25 +20,44 @@ type ExpireReasonCallback func(key string, reason EvictionReason, value interfac // LoaderFunction can be supplied to retrieve an item where a cache miss occurs. Supply an item specific ttl or Duration.Zero type LoaderFunction func(key string) (data interface{}, ttl time.Duration, err error) +// SimpleCache interface enables a quick-start. Interface for basic usage. +type SimpleCache interface { + Get(key string) (interface{}, error) + GetWithTTL(key string) (interface{}, time.Duration, error) + Set(key string, data interface{}) error + SetTTL(ttl time.Duration) error + SetWithTTL(key string, data interface{}, ttl time.Duration) error + Remove(key string) error + Close() error + Purge() error +} + // Cache is a synchronized map of items that can auto-expire once stale type Cache struct { - mutex sync.Mutex - ttl time.Duration - items map[string]*item - loaderLock map[string]*sync.Cond - expireCallback ExpireCallback - expireReasonCallback ExpireReasonCallback - checkExpireCallback CheckExpireCallback - newItemCallback ExpireCallback + // mutex is shared for all operations that need to be safe + mutex sync.Mutex + // ttl is the global ttl for the cache, can be zero (is infinite) + ttl time.Duration + // actual item storage + items map[string]*item + // lock used to avoid fetching a remote item multiple times + loaderLock *singleflight.Group + expireCallback ExpireCallback + expireReasonCallback ExpireReasonCallback + checkExpireCallback CheckExpireCallback + newItemCallback ExpireCallback + // the queue is used to have an ordered structure to use for expiration and cleanup. priorityQueue *priorityQueue expirationNotification chan bool - expirationTime time.Time - skipTTLExtension bool - shutdownSignal chan (chan struct{}) - isShutDown bool - loaderFunction LoaderFunction - sizeLimit int - metrics Metrics + // hasNotified is used to not schedule new expiration processing when an request is already pending. + hasNotified bool + expirationTime time.Time + skipTTLExtension bool + shutdownSignal chan (chan struct{}) + isShutDown bool + loaderFunction LoaderFunction + sizeLimit int + metrics Metrics } // EvictionReason is an enum that explains why an item was evicted @@ -72,19 +93,25 @@ func (cache *Cache) getItem(key string) (*item, bool, bool) { return nil, false, false } - if item.ttl >= 0 && (item.ttl > 0 || cache.ttl > 0) { - if cache.ttl > 0 && item.ttl == 0 { - item.ttl = cache.ttl - } + // no need to change priority queue when skipTTLExtension is true or the item will not expire + if cache.skipTTLExtension || (item.ttl == 0 && cache.ttl == 0) { + return item, true, false + } - if !cache.skipTTLExtension { - item.touch() - } - cache.priorityQueue.update(item) + if item.ttl == 0 { + item.ttl = cache.ttl } + item.touch() + + oldExpireTime := cache.priorityQueue.root().expireAt + cache.priorityQueue.update(item) + nowExpireTime := cache.priorityQueue.root().expireAt + expirationNotification := false - if cache.expirationTime.After(time.Now().Add(item.ttl)) { + + // notify expiration only if the latest expire time is changed + if (oldExpireTime.IsZero() && !nowExpireTime.IsZero()) || oldExpireTime.After(nowExpireTime) { expirationNotification = true } return item, exists, expirationNotification @@ -95,9 +122,10 @@ func (cache *Cache) startExpirationProcessing() { for { var sleepTime time.Duration cache.mutex.Lock() + cache.hasNotified = false if cache.priorityQueue.Len() > 0 { - sleepTime = time.Until(cache.priorityQueue.items[0].expireAt) - if sleepTime < 0 && cache.priorityQueue.items[0].expireAt.IsZero() { + sleepTime = time.Until(cache.priorityQueue.root().expireAt) + if sleepTime < 0 && cache.priorityQueue.root().expireAt.IsZero() { sleepTime = time.Hour } else if sleepTime < 0 { sleepTime = time.Microsecond @@ -158,7 +186,6 @@ func (cache *Cache) removeItem(item *item, reason EvictionReason) { cache.checkExpirationCallback(item, reason) cache.priorityQueue.remove(item) delete(cache.items, item.key) - } func (cache *Cache) evictjob(reason EvictionReason) { @@ -230,6 +257,11 @@ func (cache *Cache) SetWithTTL(key string, data interface{}, ttl time.Duration) } item, exists, _ := cache.getItem(key) + oldExpireTime := time.Time{} + if !cache.priorityQueue.isEmpty() { + oldExpireTime = cache.priorityQueue.root().expireAt + } + if exists { item.data = data item.ttl = ttl @@ -242,43 +274,75 @@ func (cache *Cache) SetWithTTL(key string, data interface{}, ttl time.Duration) } cache.metrics.Inserted++ - if item.ttl >= 0 && (item.ttl > 0 || cache.ttl > 0) { - if cache.ttl > 0 && item.ttl == 0 { - item.ttl = cache.ttl - } - item.touch() + if item.ttl == 0 { + item.ttl = cache.ttl } + item.touch() + if exists { cache.priorityQueue.update(item) } else { cache.priorityQueue.push(item) } + nowExpireTime := cache.priorityQueue.root().expireAt + cache.mutex.Unlock() if !exists && cache.newItemCallback != nil { cache.newItemCallback(key, data) } - cache.expirationNotification <- true + + // notify expiration only if the latest expire time is changed + if (oldExpireTime.IsZero() && !nowExpireTime.IsZero()) || oldExpireTime.After(nowExpireTime) { + cache.notifyExpiration() + } return nil } // Get is a thread-safe way to lookup items -// Every lookup, also touches the item, hence extending it's life +// Every lookup, also touches the item, hence extending its life func (cache *Cache) Get(key string) (interface{}, error) { + return cache.GetByLoader(key, nil) +} + +// GetWithTTL has exactly the same behaviour as Get but also returns +// the remaining TTL for a specific item at the moment its retrieved +func (cache *Cache) GetWithTTL(key string) (interface{}, time.Duration, error) { + return cache.GetByLoaderWithTtl(key, nil) +} + +// GetByLoader can take a per key loader function (i.e. to propagate context) +func (cache *Cache) GetByLoader(key string, customLoaderFunction LoaderFunction) (interface{}, error) { + dataToReturn, _, err := cache.GetByLoaderWithTtl(key, customLoaderFunction) + + return dataToReturn, err +} + +// GetByLoaderWithTtl can take a per key loader function (i.e. to propagate context) +func (cache *Cache) GetByLoaderWithTtl(key string, customLoaderFunction LoaderFunction) (interface{}, time.Duration, error) { cache.mutex.Lock() if cache.isShutDown { cache.mutex.Unlock() - return nil, ErrClosed + return nil, 0, ErrClosed } cache.metrics.Hits++ item, exists, triggerExpirationNotification := cache.getItem(key) var dataToReturn interface{} + ttlToReturn := time.Duration(0) if exists { cache.metrics.Retrievals++ dataToReturn = item.data + if !cache.skipTTLExtension { + ttlToReturn = item.ttl + } else { + ttlToReturn = time.Until(item.expireAt) + } + if ttlToReturn < 0 { + ttlToReturn = 0 + } } var err error @@ -286,58 +350,65 @@ func (cache *Cache) Get(key string) (interface{}, error) { cache.metrics.Misses++ err = ErrNotFound } - if cache.loaderFunction == nil || exists { + + loaderFunction := cache.loaderFunction + if customLoaderFunction != nil { + loaderFunction = customLoaderFunction + } + + if loaderFunction == nil || exists { cache.mutex.Unlock() } - if cache.loaderFunction != nil && !exists { - if lock, ok := cache.loaderLock[key]; ok { - // if a lock is present then a fetch is in progress and we wait. - cache.mutex.Unlock() - lock.L.Lock() - lock.Wait() - lock.L.Unlock() - cache.mutex.Lock() - item, exists, triggerExpirationNotification = cache.getItem(key) - if exists { - dataToReturn = item.data - err = nil - } - cache.mutex.Unlock() - } else { - // if no lock is present we are the leader and should set the lock and fetch. - m := sync.NewCond(&sync.Mutex{}) - cache.loaderLock[key] = m - cache.mutex.Unlock() - // cache is not blocked during IO - dataToReturn, err = cache.invokeLoader(key) - cache.mutex.Lock() - m.Broadcast() - // cleanup so that we don't block consecutive access. - delete(cache.loaderLock, key) - cache.mutex.Unlock() + if loaderFunction != nil && !exists { + type loaderResult struct { + data interface{} + ttl time.Duration } - + ch := cache.loaderLock.DoChan(key, func() (interface{}, error) { + // cache is not blocked during io + invokeData, ttl, err := cache.invokeLoader(key, loaderFunction) + lr := &loaderResult{ + data: invokeData, + ttl: ttl, + } + return lr, err + }) + cache.mutex.Unlock() + res := <-ch + dataToReturn = res.Val.(*loaderResult).data + ttlToReturn = res.Val.(*loaderResult).ttl + err = res.Err } if triggerExpirationNotification { - cache.expirationNotification <- true + cache.notifyExpiration() } - return dataToReturn, err + return dataToReturn, ttlToReturn, err } -func (cache *Cache) invokeLoader(key string) (dataToReturn interface{}, err error) { - var ttl time.Duration +func (cache *Cache) notifyExpiration() { + cache.mutex.Lock() + if cache.hasNotified { + cache.mutex.Unlock() + return + } + cache.hasNotified = true + cache.mutex.Unlock() + + cache.expirationNotification <- true +} - dataToReturn, ttl, err = cache.loaderFunction(key) +func (cache *Cache) invokeLoader(key string, loaderFunction LoaderFunction) (dataToReturn interface{}, ttl time.Duration, err error) { + dataToReturn, ttl, err = loaderFunction(key) if err == nil { err = cache.SetWithTTL(key, dataToReturn, ttl) if err != nil { dataToReturn = nil } } - return dataToReturn, err + return dataToReturn, ttl, err } // Remove removes an item from the cache if it exists, triggers expiration callback when set. Can return ErrNotFound if the entry was not present. @@ -369,6 +440,41 @@ func (cache *Cache) Count() int { return length } +// GetKeys returns all keys of items in the cache. Returns nil when the cache has been closed. +func (cache *Cache) GetKeys() []string { + cache.mutex.Lock() + defer cache.mutex.Unlock() + + if cache.isShutDown { + return nil + } + keys := make([]string, len(cache.items)) + i := 0 + for k := range cache.items { + keys[i] = k + i++ + } + return keys +} + +// GetItems returns a copy of all items in the cache. Returns nil when the cache has been closed. +func (cache *Cache) GetItems() map[string]interface{} { + cache.mutex.Lock() + defer cache.mutex.Unlock() + + if cache.isShutDown { + return nil + } + items := make(map[string]interface{}, len(cache.items)) + for k := range cache.items { + item, exists, _ := cache.getItem(k) + if exists { + items[k] = item.data + } + } + return items +} + // SetTTL sets the global TTL value for items in the cache, which can be overridden at the item level. func (cache *Cache) SetTTL(ttl time.Duration) error { cache.mutex.Lock() @@ -379,28 +485,36 @@ func (cache *Cache) SetTTL(ttl time.Duration) error { } cache.ttl = ttl cache.mutex.Unlock() - cache.expirationNotification <- true + cache.notifyExpiration() return nil } // SetExpirationCallback sets a callback that will be called when an item expires func (cache *Cache) SetExpirationCallback(callback ExpireCallback) { + cache.mutex.Lock() + defer cache.mutex.Unlock() cache.expireCallback = callback } // SetExpirationReasonCallback sets a callback that will be called when an item expires, includes reason of expiry func (cache *Cache) SetExpirationReasonCallback(callback ExpireReasonCallback) { + cache.mutex.Lock() + defer cache.mutex.Unlock() cache.expireReasonCallback = callback } // SetCheckExpirationCallback sets a callback that will be called when an item is about to expire // in order to allow external code to decide whether the item expires or remains for another TTL cycle func (cache *Cache) SetCheckExpirationCallback(callback CheckExpireCallback) { + cache.mutex.Lock() + defer cache.mutex.Unlock() cache.checkExpireCallback = callback } // SetNewItemCallback sets a callback that will be called when a new item is added to the cache func (cache *Cache) SetNewItemCallback(callback ExpireCallback) { + cache.mutex.Lock() + defer cache.mutex.Unlock() cache.newItemCallback = callback } @@ -408,12 +522,16 @@ func (cache *Cache) SetNewItemCallback(callback ExpireCallback) { // no longer extend TTL of items when they are retrieved using Get, or when their expiration condition is evaluated // using SetCheckExpirationCallback. func (cache *Cache) SkipTTLExtensionOnHit(value bool) { + cache.mutex.Lock() + defer cache.mutex.Unlock() cache.skipTTLExtension = value } // SetLoaderFunction allows you to set a function to retrieve cache misses. The signature matches that of the Get function. // Additional Get calls on the same key block while fetching is in progress (groupcache style). func (cache *Cache) SetLoaderFunction(loader LoaderFunction) { + cache.mutex.Lock() + defer cache.mutex.Unlock() cache.loaderFunction = loader } @@ -434,6 +552,8 @@ func (cache *Cache) Purge() error { // If a new item is getting cached, the closes item to being timed out will be replaced // Set to 0 to turn off func (cache *Cache) SetCacheSizeLimit(limit int) { + cache.mutex.Lock() + defer cache.mutex.Unlock() cache.sizeLimit = limit } @@ -444,9 +564,9 @@ func NewCache() *Cache { cache := &Cache{ items: make(map[string]*item), - loaderLock: make(map[string]*sync.Cond), + loaderLock: &singleflight.Group{}, priorityQueue: newPriorityQueue(), - expirationNotification: make(chan bool), + expirationNotification: make(chan bool, 1), expirationTime: time.Now(), shutdownSignal: shutdownChan, isShutDown: false, @@ -465,6 +585,18 @@ func (cache *Cache) GetMetrics() Metrics { return cache.metrics } +// Touch resets the TTL of the key when it exists, returns ErrNotFound if the key is not present. +func (cache *Cache) Touch(key string) error { + cache.mutex.Lock() + defer cache.mutex.Unlock() + item, exists := cache.items[key] + if !exists { + return ErrNotFound + } + item.touch() + return nil +} + func min(duration time.Duration, second time.Duration) time.Duration { if duration < second { return duration diff --git a/vendor/github.com/ReneKroon/ttlcache/v2/evictionreason_enumer.go b/vendor/github.com/jellydator/ttlcache/v2/evictionreason_enumer.go similarity index 100% rename from vendor/github.com/ReneKroon/ttlcache/v2/evictionreason_enumer.go rename to vendor/github.com/jellydator/ttlcache/v2/evictionreason_enumer.go diff --git a/vendor/github.com/ReneKroon/ttlcache/v2/go.mod b/vendor/github.com/jellydator/ttlcache/v2/go.mod similarity index 73% rename from vendor/github.com/ReneKroon/ttlcache/v2/go.mod rename to vendor/github.com/jellydator/ttlcache/v2/go.mod index 510ef4c..f8086ee 100644 --- a/vendor/github.com/ReneKroon/ttlcache/v2/go.mod +++ b/vendor/github.com/jellydator/ttlcache/v2/go.mod @@ -1,12 +1,12 @@ -module github.com/ReneKroon/ttlcache/v2 +module github.com/jellydator/ttlcache/v2 go 1.15 require ( - github.com/alvaroloes/enumer v1.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/stretchr/testify v1.7.0 go.uber.org/goleak v1.1.10 golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/tools v0.0.0-20210112230658-8b4aab62c064 // indirect ) diff --git a/vendor/github.com/ReneKroon/ttlcache/v2/go.sum b/vendor/github.com/jellydator/ttlcache/v2/go.sum similarity index 63% rename from vendor/github.com/ReneKroon/ttlcache/v2/go.sum rename to vendor/github.com/jellydator/ttlcache/v2/go.sum index 168c7af..d1f5162 100644 --- a/vendor/github.com/ReneKroon/ttlcache/v2/go.sum +++ b/vendor/github.com/jellydator/ttlcache/v2/go.sum @@ -1,86 +1,57 @@ -github.com/alvaroloes/enumer v1.1.2 h1:5khqHB33TZy1GWCO/lZwcroBFh7u+0j40T83VUbfAMY= -github.com/alvaroloes/enumer v1.1.2/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT0m65DJ+Wo= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1 h1:/I3lTljEEDNYLho3/FUB7iD/oc2cEFgVmbHzV+O0PtU= -github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1/go.mod h1:eD5JxqMiuNYyFNmyY9rkJ/slN8y59oEu4Ei7F8OoKWQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.uber.org/goleak v1.1.10 h1:z+mqJhf6ss6BSfSM671tgKyZBFPTTJM+HLxnhPC3wu0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 h1:2M3HP5CCK1Si9FQhwnzYhXdG6DXeebvUHFpre8QvbyI= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.0 h1:8pl+sMODzuvGJkmj2W4kZihvVb5mKm8pB/X44PIQHv8= -golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11 h1:Yq9t9jnGoR+dBuitxdo9l6Q7xh/zOyNnYUtDKaQ3x0E= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d h1:szSOL78iTCl0LF1AMjhSWJj8tIM0KixlUUnBtYXsmd8= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210112230658-8b4aab62c064 h1:BmCFkEH4nJrYcAc2L08yX5RhYGD4j58PTMkEUDkpz2I= golang.org/x/tools v0.0.0-20210112230658-8b4aab62c064/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/vendor/github.com/ReneKroon/ttlcache/v2/item.go b/vendor/github.com/jellydator/ttlcache/v2/item.go similarity index 100% rename from vendor/github.com/ReneKroon/ttlcache/v2/item.go rename to vendor/github.com/jellydator/ttlcache/v2/item.go diff --git a/vendor/github.com/ReneKroon/ttlcache/v2/metrics.go b/vendor/github.com/jellydator/ttlcache/v2/metrics.go similarity index 100% rename from vendor/github.com/ReneKroon/ttlcache/v2/metrics.go rename to vendor/github.com/jellydator/ttlcache/v2/metrics.go diff --git a/vendor/github.com/ReneKroon/ttlcache/v2/priority_queue.go b/vendor/github.com/jellydator/ttlcache/v2/priority_queue.go similarity index 82% rename from vendor/github.com/ReneKroon/ttlcache/v2/priority_queue.go rename to vendor/github.com/jellydator/ttlcache/v2/priority_queue.go index 11b9c31..5d40548 100644 --- a/vendor/github.com/ReneKroon/ttlcache/v2/priority_queue.go +++ b/vendor/github.com/jellydator/ttlcache/v2/priority_queue.go @@ -14,6 +14,18 @@ type priorityQueue struct { items []*item } +func (pq *priorityQueue) isEmpty() bool { + return len(pq.items) == 0 +} + +func (pq *priorityQueue) root() *item { + if len(pq.items) == 0 { + return nil + } + + return pq.items[0] +} + func (pq *priorityQueue) update(item *item) { heap.Fix(pq, item.queueIndex) } @@ -66,6 +78,8 @@ func (pq *priorityQueue) Pop() interface{} { n := len(old) item := old[n-1] item.queueIndex = -1 + // de-reference the element to be popped for Garbage Collector to de-allocate the memory + old[n-1] = nil pq.items = old[0 : n-1] return item } diff --git a/vendor/golang.org/x/sync/AUTHORS b/vendor/golang.org/x/sync/AUTHORS deleted file mode 100644 index 15167cd..0000000 --- a/vendor/golang.org/x/sync/AUTHORS +++ /dev/null @@ -1,3 +0,0 @@ -# This source code refers to The Go Authors for copyright purposes. -# The master list of authors is in the main Go distribution, -# visible at http://tip.golang.org/AUTHORS. diff --git a/vendor/golang.org/x/sync/CONTRIBUTORS b/vendor/golang.org/x/sync/CONTRIBUTORS deleted file mode 100644 index 1c4577e..0000000 --- a/vendor/golang.org/x/sync/CONTRIBUTORS +++ /dev/null @@ -1,3 +0,0 @@ -# This source code was written by the Go contributors. -# The master list of contributors is in the main Go distribution, -# visible at http://tip.golang.org/CONTRIBUTORS. diff --git a/vendor/golang.org/x/sync/errgroup/errgroup.go b/vendor/golang.org/x/sync/errgroup/errgroup.go index 9857fe5..cbee7a4 100644 --- a/vendor/golang.org/x/sync/errgroup/errgroup.go +++ b/vendor/golang.org/x/sync/errgroup/errgroup.go @@ -8,22 +8,35 @@ package errgroup import ( "context" + "fmt" "sync" ) +type token struct{} + // A Group is a collection of goroutines working on subtasks that are part of // the same overall task. // -// A zero Group is valid and does not cancel on error. +// A zero Group is valid, has no limit on the number of active goroutines, +// and does not cancel on error. type Group struct { cancel func() wg sync.WaitGroup + sem chan token + errOnce sync.Once err error } +func (g *Group) done() { + if g.sem != nil { + <-g.sem + } + g.wg.Done() +} + // WithContext returns a new Group and an associated Context derived from ctx. // // The derived Context is canceled the first time a function passed to Go @@ -45,14 +58,48 @@ func (g *Group) Wait() error { } // Go calls the given function in a new goroutine. +// It blocks until the new goroutine can be added without the number of +// active goroutines in the group exceeding the configured limit. // -// The first call to return a non-nil error cancels the group; its error will be -// returned by Wait. +// The first call to return a non-nil error cancels the group's context, if the +// group was created by calling WithContext. The error will be returned by Wait. func (g *Group) Go(f func() error) { + if g.sem != nil { + g.sem <- token{} + } + g.wg.Add(1) + go func() { + defer g.done() + + if err := f(); err != nil { + g.errOnce.Do(func() { + g.err = err + if g.cancel != nil { + g.cancel() + } + }) + } + }() +} + +// TryGo calls the given function in a new goroutine only if the number of +// active goroutines in the group is currently below the configured limit. +// +// The return value reports whether the goroutine was started. +func (g *Group) TryGo(f func() error) bool { + if g.sem != nil { + select { + case g.sem <- token{}: + // Note: this allows barging iff channels in general allow barging. + default: + return false + } + } + g.wg.Add(1) go func() { - defer g.wg.Done() + defer g.done() if err := f(); err != nil { g.errOnce.Do(func() { @@ -63,4 +110,23 @@ func (g *Group) Go(f func() error) { }) } }() + return true +} + +// SetLimit limits the number of active goroutines in this group to at most n. +// A negative value indicates no limit. +// +// Any subsequent call to the Go method will block until it can add an active +// goroutine without exceeding the configured limit. +// +// The limit must not be modified while any goroutines in the group are active. +func (g *Group) SetLimit(n int) { + if n < 0 { + g.sem = nil + return + } + if len(g.sem) != 0 { + panic(fmt.Errorf("errgroup: modify limit while %v goroutines in the group are still active", len(g.sem))) + } + g.sem = make(chan token, n) } diff --git a/vendor/golang.org/x/sync/singleflight/singleflight.go b/vendor/golang.org/x/sync/singleflight/singleflight.go index 690eb85..8473fb7 100644 --- a/vendor/golang.org/x/sync/singleflight/singleflight.go +++ b/vendor/golang.org/x/sync/singleflight/singleflight.go @@ -52,10 +52,6 @@ type call struct { val interface{} err error - // forgotten indicates whether Forget was called with this call's key - // while the call was still in flight. - forgotten bool - // These fields are read and written with the singleflight // mutex held before the WaitGroup is done, and are read but // not written after the WaitGroup is done. @@ -148,10 +144,10 @@ func (g *Group) doCall(c *call, key string, fn func() (interface{}, error)) { c.err = errGoexit } - c.wg.Done() g.mu.Lock() defer g.mu.Unlock() - if !c.forgotten { + c.wg.Done() + if g.m[key] == c { delete(g.m, key) } @@ -204,9 +200,6 @@ func (g *Group) doCall(c *call, key string, fn func() (interface{}, error)) { // an earlier call to complete. func (g *Group) Forget(key string) { g.mu.Lock() - if c, ok := g.m[key]; ok { - c.forgotten = true - } delete(g.m, key) g.mu.Unlock() } diff --git a/vendor/modules.txt b/vendor/modules.txt index 5ab28bf..4086f83 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,7 +1,5 @@ # github.com/BurntSushi/toml v0.3.1 github.com/BurntSushi/toml -# github.com/ReneKroon/ttlcache/v2 v2.3.0 -github.com/ReneKroon/ttlcache/v2 # github.com/aws/aws-sdk-go v1.34.28 github.com/aws/aws-sdk-go/aws github.com/aws/aws-sdk-go/aws/awserr @@ -40,6 +38,8 @@ github.com/golang/protobuf/ptypes/duration github.com/golang/protobuf/ptypes/timestamp # github.com/golang/snappy v0.0.1 github.com/golang/snappy +# github.com/jellydator/ttlcache/v2 v2.11.1 +github.com/jellydator/ttlcache/v2 # github.com/jessevdk/go-flags v1.4.0 github.com/jessevdk/go-flags # github.com/jmespath/go-jmespath v0.4.0 @@ -173,7 +173,7 @@ golang.org/x/net/internal/iana golang.org/x/net/internal/socket golang.org/x/net/ipv4 golang.org/x/net/ipv6 -# golang.org/x/sync v0.0.0-20210220032951-036812b2e83c +# golang.org/x/sync v0.1.0 golang.org/x/sync/errgroup golang.org/x/sync/semaphore golang.org/x/sync/singleflight From 5c12d99f9dc0cba05df37909f60403ef3ad0832f Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Thu, 16 Mar 2023 16:33:03 -0700 Subject: [PATCH 3/7] Change cursorCache to not passively create entries on access This way we can more easily check if a cursor exists without accidentally creating an entry. --- pkg/mongoproxy/plugins/authz/plugin_test.go | 12 +++++-- pkg/mongoproxy/plugins/interface.go | 1 + pkg/mongoproxy/plugins/mongo/mongo.go | 9 +++++- pkg/mongoproxy/proxy.go | 36 +++++++++++++-------- 4 files changed, 41 insertions(+), 17 deletions(-) diff --git a/pkg/mongoproxy/plugins/authz/plugin_test.go b/pkg/mongoproxy/plugins/authz/plugin_test.go index 6400793..e44d080 100644 --- a/pkg/mongoproxy/plugins/authz/plugin_test.go +++ b/pkg/mongoproxy/plugins/authz/plugin_test.go @@ -22,7 +22,7 @@ type stubCursorCache struct { m map[int64]*plugins.CursorCacheEntry } -func (c *stubCursorCache) GetCursor(cursorID int64) *plugins.CursorCacheEntry { +func (c *stubCursorCache) CreateCursor(cursorID int64) *plugins.CursorCacheEntry { v, ok := c.m[cursorID] if !ok { v = plugins.NewCursorCacheEntry(cursorID) @@ -30,6 +30,14 @@ func (c *stubCursorCache) GetCursor(cursorID int64) *plugins.CursorCacheEntry { } return v } + +func (c *stubCursorCache) GetCursor(cursorID int64) *plugins.CursorCacheEntry { + v, ok := c.m[cursorID] + if !ok { + return nil + } + return v +} func (c *stubCursorCache) CloseCursor(cursorID int64) { delete(c.m, cursorID) } @@ -58,7 +66,7 @@ func TestPluginGetMore(t *testing.T) { p := plugins.BuildPipeline([]plugins.Plugin{d}, func(_ context.Context, r *plugins.Request) (bson.D, error) { switch r.Command.(type) { case *command.Find: - r.CursorCache.GetCursor(cursorID) + r.CursorCache.CreateCursor(cursorID) return bson.D{ {"ok", 1}, {"cursor", bson.D{{"id", cursorID}}}, diff --git a/pkg/mongoproxy/plugins/interface.go b/pkg/mongoproxy/plugins/interface.go index 2a2165f..9774a91 100644 --- a/pkg/mongoproxy/plugins/interface.go +++ b/pkg/mongoproxy/plugins/interface.go @@ -33,6 +33,7 @@ func NewCursorCacheEntry(id int64) *CursorCacheEntry { } type CursorCache interface { + CreateCursor(cursorID int64) *CursorCacheEntry GetCursor(cursorID int64) *CursorCacheEntry CloseCursor(cursorID int64) } diff --git a/pkg/mongoproxy/plugins/mongo/mongo.go b/pkg/mongoproxy/plugins/mongo/mongo.go index 7e92ca1..91599d7 100644 --- a/pkg/mongoproxy/plugins/mongo/mongo.go +++ b/pkg/mongoproxy/plugins/mongo/mongo.go @@ -302,7 +302,7 @@ func (p *MongoPlugin) Process(ctx context.Context, r *plugins.Request, next plug if cursorID, ok := cursorIDRaw.(int64); ok && cursorID > 0 { logrus.Tracef("Store cursor: %v %v", cursorID, cmdServer) // TODO: TTL from cmd - r.CursorCache.GetCursor(cursorID).Map[contextKeyServer] = cmdServer + r.CursorCache.CreateCursor(cursorID).Map[contextKeyServer] = cmdServer } } } @@ -530,6 +530,13 @@ func (p *MongoPlugin) Process(ctx context.Context, r *plugins.Request, next plug v, ok = bsonutil.Lookup(result, "cursorsKilled") if ok { cursorsKilled = append(cursorsKilled, v.(primitive.A)...) + for _, cursorIDRaw := range cursorsKilled { + fmt.Println("kill a cursor", cursorIDRaw) + cursorID, ok := cursorIDRaw.(int64) + if ok { + r.CursorCache.CloseCursor(cursorID) + } + } } v, ok = bsonutil.Lookup(result, "cursorsNotFound") if ok { diff --git a/pkg/mongoproxy/proxy.go b/pkg/mongoproxy/proxy.go index fe93ba4..66ff065 100644 --- a/pkg/mongoproxy/proxy.go +++ b/pkg/mongoproxy/proxy.go @@ -13,8 +13,8 @@ import ( "sync" "time" - "github.com/ReneKroon/ttlcache/v2" "github.com/getsentry/sentry-go" + "github.com/jellydator/ttlcache/v2" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" "github.com/sirupsen/logrus" @@ -91,24 +91,16 @@ func NewProxy(l net.Listener, cfg *config.Config) (*Proxy, error) { // Set up cursorCache p.cursorCache.SetTTL(p.cfg.IdleCursorTimeout) // default TTL -- config - p.cursorCache.SetLoaderFunction(func(key string) (interface{}, time.Duration, error) { - cursorID, err := strconv.ParseInt(key, 10, 64) - if err != nil { - return nil, time.Duration(0), err - } - - return plugins.NewCursorCacheEntry(cursorID), time.Duration(0), nil - }) // expiration handler to send killCursor commands p.cursorCache.SetExpirationReasonCallback(func(key string, reason ttlcache.EvictionReason, value interface{}) { logrus.Tracef("expire cursor %s", key) - i, err := strconv.ParseInt(key, 10, 64) - if err != nil { - return - } // If the cursor expired (we timed out waiting) we want to kill the downstream cursor as we remove it from the cache if reason == ttlcache.Expired { + i, err := strconv.ParseInt(key, 10, 64) + if err != nil { + return + } p.HandleMongo(context.TODO(), &plugins.Request{CursorCache: p, CC: p.internalCC}, bson.D{ {"killCursors", "admin"}, {"cursors", primitive.A{i}}, @@ -138,10 +130,26 @@ type Proxy struct { internalCC *plugins.ClientConnection } +func (p *Proxy) CreateCursor(cursorID int64) *plugins.CursorCacheEntry { + v, err := p.cursorCache.GetByLoader(strconv.FormatInt(cursorID, 10), func(key string) (interface{}, time.Duration, error) { + cursorID, err := strconv.ParseInt(key, 10, 64) + if err != nil { + return nil, time.Duration(0), err + } + + return plugins.NewCursorCacheEntry(cursorID), time.Duration(0), nil + }) + if err == ttlcache.ErrNotFound { + panic("can't get cursor") + } + + return v.(*plugins.CursorCacheEntry) +} + func (p *Proxy) GetCursor(cursorID int64) *plugins.CursorCacheEntry { v, err := p.cursorCache.Get(strconv.FormatInt(cursorID, 10)) if err == ttlcache.ErrNotFound { - panic("can't get cursor") + return nil } return v.(*plugins.CursorCacheEntry) From 8f4522bf0d9b7c1d99dea892d9fda4acc33d5018 Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Thu, 16 Mar 2023 16:33:57 -0700 Subject: [PATCH 4/7] Change authz killCursor behavior to always allow a user to kill cursors created on the same session. Fixes #26 --- pkg/mongoproxy/plugins/authz/plugin.go | 39 +++++++++++++++++++++----- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/pkg/mongoproxy/plugins/authz/plugin.go b/pkg/mongoproxy/plugins/authz/plugin.go index af32cae..5de7e73 100644 --- a/pkg/mongoproxy/plugins/authz/plugin.go +++ b/pkg/mongoproxy/plugins/authz/plugin.go @@ -2,6 +2,7 @@ package authz import ( "context" + "fmt" "log" "path" @@ -143,7 +144,7 @@ func (p *AuthzPlugin) Configure(d bson.D) error { return nil } -func (p *AuthzPlugin) resourcesForCommand(r *plugins.Request, c command.Command) map[authzlib.AuthorizationMethod][]authzlib.Resource { +func (p *AuthzPlugin) resourcesForCommand(r *plugins.Request, c command.Command) (map[authzlib.AuthorizationMethod][]authzlib.Resource, error) { resourceMap := make(map[authzlib.AuthorizationMethod][]authzlib.Resource) // Pick which commands we allow without authentication at all @@ -262,7 +263,11 @@ func (p *AuthzPlugin) resourcesForCommand(r *plugins.Request, c command.Command) } case *command.Explain: - resourceMap = p.resourcesForCommand(r, cmd.Cmd) + var err error + resourceMap, err = p.resourcesForCommand(r, cmd.Cmd) + if err != nil { + return nil, err + } switch cmd.Verbosity { case "queryPlanner": resourceMap[authzlib.Read] = append(resourceMap[authzlib.Read], authzlib.Resource{ @@ -365,9 +370,9 @@ func (p *AuthzPlugin) resourcesForCommand(r *plugins.Request, c command.Command) case *command.GetMore: cursorResources := r.CursorCache.GetCursor(cmd.CursorID).Map[contextKeyResources] if cr, ok := cursorResources.(map[authzlib.AuthorizationMethod][]authzlib.Resource); ok { - return cr + return cr, nil } - return nil + return nil, nil case *command.HostInfo: resourceMap[authzlib.Read] = []authzlib.Resource{ @@ -392,6 +397,23 @@ func (p *AuthzPlugin) resourcesForCommand(r *plugins.Request, c command.Command) } case *command.KillCursors: + selfCursors := true + for i, cursorIDRaw := range cmd.Cursors { + cursorID, ok := cursorIDRaw.(int64) + if !ok { + return nil, fmt.Errorf("field 'cursors' contains an element that is not of type long: %d: \"%v\"", i, cursorIDRaw) + } + if r.CursorCache.GetCursor(cursorID) == nil { + selfCursors = false + break + } + } + // If one of the cursors isn't from this client/connection then we require + // global write permissions + if selfCursors { + return nil, nil + } + resourceMap[authzlib.Delete] = []authzlib.Resource{ { Global: true, @@ -477,7 +499,7 @@ func (p *AuthzPlugin) resourcesForCommand(r *plugins.Request, c command.Command) } } - return resourceMap + return resourceMap, nil } // Process is the function executed when a message is called in the pipeline. @@ -487,10 +509,13 @@ func (p *AuthzPlugin) Process(ctx context.Context, r *plugins.Request, next plug return next(ctx, r) } - resourceMap := p.resourcesForCommand(r, r.Command) + resourceMap, err := p.resourcesForCommand(r, r.Command) + if err != nil { + return mongoerror.FailedToParse.ErrMessage(err.Error()), nil + } // If there is no resource; we don't allow the call through - if len(resourceMap) == 0 { + if resourceMap != nil && len(resourceMap) == 0 { return mongoerror.Unauthorized.ErrMessage("unauthorized no resource for " + r.CommandName), nil } From 4332f0d2ca23cb34e80b89c97cd6a4f429d25fc3 Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Fri, 17 Mar 2023 13:02:03 -0700 Subject: [PATCH 5/7] update go version --- .github/workflows/go.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index fc534f4..ec3a75f 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -23,7 +23,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v2 with: - go-version: 1.18 + go-version: '1.20' - name: install run: | From 16783dd0a07592b5fd99ea4d49ef24906707aaaf Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Fri, 17 Mar 2023 13:04:53 -0700 Subject: [PATCH 6/7] make fmt --- pkg/mongoproxy/plugins/authz/authzlib/resource.go | 1 + pkg/mongoproxy/plugins/authz/authzlib/utils.go | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/mongoproxy/plugins/authz/authzlib/resource.go b/pkg/mongoproxy/plugins/authz/authzlib/resource.go index 532d8c7..d0f337e 100644 --- a/pkg/mongoproxy/plugins/authz/authzlib/resource.go +++ b/pkg/mongoproxy/plugins/authz/authzlib/resource.go @@ -12,6 +12,7 @@ type Resource struct { } // getResource receives a map[string]string and returns the global, dbs, collections and fields +// // it is attempting to access. func getResource(resource map[string]string) (r Resource, err error) { if resource["Global"] == "*" { diff --git a/pkg/mongoproxy/plugins/authz/authzlib/utils.go b/pkg/mongoproxy/plugins/authz/authzlib/utils.go index 14ece23..1488c37 100644 --- a/pkg/mongoproxy/plugins/authz/authzlib/utils.go +++ b/pkg/mongoproxy/plugins/authz/authzlib/utils.go @@ -26,14 +26,15 @@ func appendArrayIfMissing(slice []string, other []string) []string { } // splitURI receives a uri and returns the dbs, collections and fields +// // it is attempting to access. // // uri examples: +// // db/coll/fld -> [db] [coll] [fld] // db/coll/* -> [db] [coll] [*] // db/coll/fld1,fld2 -> [db] [coll] [fld1 fld2] // db/*/fld1,fld2 -> [db] [*] [fld1 fld2] -// func splitURI(uri string) ([]string, []string, []string, error) { parts := strings.Split(uri, "/") if len(parts) > 3 { @@ -65,9 +66,11 @@ func splitURI(uri string) ([]string, []string, []string, error) { // (e.g. some tree) which can evaluate the permissions without having to expand to // all possible options // expandResource returns a slice of potential resources that can appear +// // in config given a single resource. // // Example: +// // db/coll/fld -> [db/coll/fld db/coll/* db/*/fld db/*/* // */coll/fld */*/fld */coll/* */*/*] func expandResource(r Resource) []Resource { From 705c6160ee325f2cb6ccae82119396c3cad76cd4 Mon Sep 17 00:00:00 2001 From: Thomas Jackson Date: Fri, 17 Mar 2023 14:01:54 -0700 Subject: [PATCH 7/7] Fix issue in getmore and update tests --- pkg/mongoproxy/plugins/authz/plugin.go | 8 ++++++-- pkg/mongoproxy/plugins/authz/plugin_test.go | 3 +-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pkg/mongoproxy/plugins/authz/plugin.go b/pkg/mongoproxy/plugins/authz/plugin.go index 5de7e73..6765342 100644 --- a/pkg/mongoproxy/plugins/authz/plugin.go +++ b/pkg/mongoproxy/plugins/authz/plugin.go @@ -368,8 +368,12 @@ func (p *AuthzPlugin) resourcesForCommand(r *plugins.Request, c command.Command) } case *command.GetMore: - cursorResources := r.CursorCache.GetCursor(cmd.CursorID).Map[contextKeyResources] - if cr, ok := cursorResources.(map[authzlib.AuthorizationMethod][]authzlib.Resource); ok { + cursorCacheEntry := r.CursorCache.GetCursor(cmd.CursorID) + // If we don't have this cursor, we'll disallow this further down (as we don't know what it is) + if cursorCacheEntry == nil { + return nil, fmt.Errorf("cursorID %d not found", cmd.CursorID) + } + if cr, ok := cursorCacheEntry.Map[contextKeyResources].(map[authzlib.AuthorizationMethod][]authzlib.Resource); ok { return cr, nil } return nil, nil diff --git a/pkg/mongoproxy/plugins/authz/plugin_test.go b/pkg/mongoproxy/plugins/authz/plugin_test.go index e44d080..03ad026 100644 --- a/pkg/mongoproxy/plugins/authz/plugin_test.go +++ b/pkg/mongoproxy/plugins/authz/plugin_test.go @@ -665,8 +665,7 @@ func TestPlugin(t *testing.T) { ///////////// { cmd: bson.D{{"killCursors", ""}, {"$db", "db"}}, - good: [][]plugins.ClientIdentity{nil, idents["global"]}, - bad: [][]plugins.ClientIdentity{idents["role1"]}, + good: [][]plugins.ClientIdentity{nil, idents["global"], idents["role1"]}, }, { cmd: bson.D{{"killCursors", ""}, {"$db", "authzcolcru"}},