Misc cleanup/fixes

.github/workflows/go.yml

@@ -23,7 +23,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.18
+        go-version: '1.20'
 
     - name: install
       run: |

go.mod

@@ -4,9 +4,9 @@ go 1.13
 
 require (
 	github.com/HdrHistogram/hdrhistogram-go v1.1.0 // indirect
-	github.com/ReneKroon/ttlcache/v2 v2.3.0
 	github.com/cespare/xxhash/v2 v2.1.1
 	github.com/getsentry/sentry-go v0.9.0
+	github.com/jellydator/ttlcache/v2 v2.11.1
 	github.com/jessevdk/go-flags v1.4.0
 	github.com/json-iterator/go v1.1.11
 	github.com/miekg/dns v1.1.41 // indirect
@@ -20,7 +20,7 @@ require (
 	go.mongodb.org/mongo-driver v1.5.1
 	go.uber.org/automaxprocs v1.3.0
 	golang.org/x/net v0.0.0-20210331212208-0fccb6fa2b5c // indirect
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
+	golang.org/x/sync v0.1.0
 	golang.org/x/sys v0.0.0-20210426080607-c94f62235c83 // indirect
 	golang.org/x/time v0.0.0-20201208040808-7e3f01d25324
 	gopkg.in/fsnotify.v1 v1.4.7

go.sum

@@ -11,8 +11,6 @@ github.com/HdrHistogram/hdrhistogram-go v1.1.0 h1:6dpdDPTRoo78HxAJ6T1HfMiKSnqhgR
 github.com/HdrHistogram/hdrhistogram-go v1.1.0/go.mod h1:yDgFjdqOqDEKOvasDdhWNXYg9BVp4O+o5f6V/ehm6Oo=
 github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY=
 github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0=
-github.com/ReneKroon/ttlcache/v2 v2.3.0 h1:qZnUjRKIrbKHH6vF5T7Y9Izn5ObfTZfyYpGhvz2BKPo=
-github.com/ReneKroon/ttlcache/v2 v2.3.0/go.mod h1:zbo6Pv/28e21Z8CzzqgYRArQYGYtjONRxaAKGxzQvG4=
 github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0=
 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
@@ -25,7 +23,6 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
-github.com/alvaroloes/enumer v1.1.2/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT0m65DJ+Wo=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
@@ -213,6 +210,8 @@ github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/
 github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk=
 github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g=
 github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw=
+github.com/jellydator/ttlcache/v2 v2.11.1 h1:AZGME43Eh2Vv3giG6GeqeLeFXxwxn1/qHItqWZl6U64=
+github.com/jellydator/ttlcache/v2 v2.11.1/go.mod h1:RtE5Snf0/57e+2cLWFYWCCsLas2Hy3c5Z4n14XmSvTI=
 github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA=
 github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
@@ -329,7 +328,6 @@ github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnh
 github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4=
 github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
-github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1/go.mod h1:eD5JxqMiuNYyFNmyY9rkJ/slN8y59oEu4Ei7F8OoKWQ=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
@@ -547,8 +545,9 @@ golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -610,7 +609,6 @@ golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3
 golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=

pkg/mongoproxy/plugins/authz/authzlib/resource.go

@@ -12,6 +12,7 @@ type Resource struct {
 }
 
 // getResource receives a map[string]string and returns the global, dbs, collections and fields
+//
 //	it is attempting to access.
 func getResource(resource map[string]string) (r Resource, err error) {
 	if resource["Global"] == "*" {

pkg/mongoproxy/plugins/authz/authzlib/utils.go

@@ -26,14 +26,15 @@ func appendArrayIfMissing(slice []string, other []string) []string {
 }
 
 // splitURI receives a uri and returns the dbs, collections and fields
+//
 //	it is attempting to access.
 //
 // uri examples:
+//
 //	db/coll/fld -> [db] [coll] [fld]
 //	db/coll/* -> [db] [coll] [*]
 //	db/coll/fld1,fld2 -> [db] [coll] [fld1 fld2]
 //	db/*/fld1,fld2 -> [db] [*] [fld1 fld2]
-//
 func splitURI(uri string) ([]string, []string, []string, error) {
 	parts := strings.Split(uri, "/")
 	if len(parts) > 3 {
@@ -65,9 +66,11 @@ func splitURI(uri string) ([]string, []string, []string, error) {
 // (e.g. some tree) which can evaluate the permissions without having to expand to
 // all possible options
 // expandResource returns a slice of potential resources that can appear
+//
 //	in config given a single resource.
 //
 // Example:
+//
 //	db/coll/fld -> [db/coll/fld db/coll/* db/*/fld db/*/*
 //					*/coll/fld */*/fld */coll/* */*/*]
 func expandResource(r Resource) []Resource {

pkg/mongoproxy/plugins/authz/plugin.go

@@ -2,6 +2,7 @@ package authz
 
 import (
 	"context"
+	"fmt"
 	"log"
 	"path"
 
@@ -143,7 +144,7 @@ func (p *AuthzPlugin) Configure(d bson.D) error {
 	return nil
 }
 
-func (p *AuthzPlugin) resourcesForCommand(r *plugins.Request, c command.Command) map[authzlib.AuthorizationMethod][]authzlib.Resource {
+func (p *AuthzPlugin) resourcesForCommand(r *plugins.Request, c command.Command) (map[authzlib.AuthorizationMethod][]authzlib.Resource, error) {
 	resourceMap := make(map[authzlib.AuthorizationMethod][]authzlib.Resource)
 
 	// Pick which commands we allow without authentication at all
@@ -262,7 +263,11 @@ func (p *AuthzPlugin) resourcesForCommand(r *plugins.Request, c command.Command)
 		}
 
 	case *command.Explain:
-		resourceMap = p.resourcesForCommand(r, cmd.Cmd)
+		var err error
+		resourceMap, err = p.resourcesForCommand(r, cmd.Cmd)
+		if err != nil {
+			return nil, err
+		}
 		switch cmd.Verbosity {
 		case "queryPlanner":
 			resourceMap[authzlib.Read] = append(resourceMap[authzlib.Read], authzlib.Resource{
@@ -363,11 +368,15 @@ func (p *AuthzPlugin) resourcesForCommand(r *plugins.Request, c command.Command)
 		}
 
 	case *command.GetMore:
-		cursorResources := r.CursorCache.GetCursor(cmd.CursorID).Map[contextKeyResources]
-		if cr, ok := cursorResources.(map[authzlib.AuthorizationMethod][]authzlib.Resource); ok {
-			return cr
+		cursorCacheEntry := r.CursorCache.GetCursor(cmd.CursorID)
+		// If we don't have this cursor, we'll disallow this further down (as we don't know what it is)
+		if cursorCacheEntry == nil {
+			return nil, fmt.Errorf("cursorID %d not found", cmd.CursorID)
+		}
+		if cr, ok := cursorCacheEntry.Map[contextKeyResources].(map[authzlib.AuthorizationMethod][]authzlib.Resource); ok {
+			return cr, nil
 		}
-		return nil
+		return nil, nil
 
 	case *command.HostInfo:
 		resourceMap[authzlib.Read] = []authzlib.Resource{
@@ -392,6 +401,23 @@ func (p *AuthzPlugin) resourcesForCommand(r *plugins.Request, c command.Command)
 		}
 
 	case *command.KillCursors:
+		selfCursors := true
+		for i, cursorIDRaw := range cmd.Cursors {
+			cursorID, ok := cursorIDRaw.(int64)
+			if !ok {
+				return nil, fmt.Errorf("field 'cursors' contains an element that is not of type long: %d: \"%v\"", i, cursorIDRaw)
+			}
+			if r.CursorCache.GetCursor(cursorID) == nil {
+				selfCursors = false
+				break
+			}
+		}
+		// If one of the cursors isn't from this client/connection then we require
+		// global write permissions
+		if selfCursors {
+			return nil, nil
+		}
+
 		resourceMap[authzlib.Delete] = []authzlib.Resource{
 			{
 				Global: true,
@@ -477,7 +503,7 @@ func (p *AuthzPlugin) resourcesForCommand(r *plugins.Request, c command.Command)
 		}
 	}
 
-	return resourceMap
+	return resourceMap, nil
 }
 
 // Process is the function executed when a message is called in the pipeline.
@@ -487,10 +513,13 @@ func (p *AuthzPlugin) Process(ctx context.Context, r *plugins.Request, next plug
 		return next(ctx, r)
 	}
 
-	resourceMap := p.resourcesForCommand(r, r.Command)
+	resourceMap, err := p.resourcesForCommand(r, r.Command)
+	if err != nil {
+		return mongoerror.FailedToParse.ErrMessage(err.Error()), nil
+	}
 
 	// If there is no resource; we don't allow the call through
-	if len(resourceMap) == 0 {
+	if resourceMap != nil && len(resourceMap) == 0 {
 		return mongoerror.Unauthorized.ErrMessage("unauthorized no resource for " + r.CommandName), nil
 	}
 

pkg/mongoproxy/plugins/authz/plugin_test.go

@@ -22,14 +22,22 @@ type stubCursorCache struct {
 	m map[int64]*plugins.CursorCacheEntry
 }
 
-func (c *stubCursorCache) GetCursor(cursorID int64) *plugins.CursorCacheEntry {
+func (c *stubCursorCache) CreateCursor(cursorID int64) *plugins.CursorCacheEntry {
 	v, ok := c.m[cursorID]
 	if !ok {
 		v = plugins.NewCursorCacheEntry(cursorID)
 		c.m[cursorID] = v
 	}
 	return v
 }
+
+func (c *stubCursorCache) GetCursor(cursorID int64) *plugins.CursorCacheEntry {
+	v, ok := c.m[cursorID]
+	if !ok {
+		return nil
+	}
+	return v
+}
 func (c *stubCursorCache) CloseCursor(cursorID int64) {
 	delete(c.m, cursorID)
 }
@@ -58,7 +66,7 @@ func TestPluginGetMore(t *testing.T) {
 	p := plugins.BuildPipeline([]plugins.Plugin{d}, func(_ context.Context, r *plugins.Request) (bson.D, error) {
 		switch r.Command.(type) {
 		case *command.Find:
-			r.CursorCache.GetCursor(cursorID)
+			r.CursorCache.CreateCursor(cursorID)
 			return bson.D{
 				{"ok", 1},
 				{"cursor", bson.D{{"id", cursorID}}},
@@ -657,8 +665,7 @@ func TestPlugin(t *testing.T) {
 		/////////////
 		{
 			cmd:  bson.D{{"killCursors", ""}, {"$db", "db"}},
-			good: [][]plugins.ClientIdentity{nil, idents["global"]},
-			bad:  [][]plugins.ClientIdentity{idents["role1"]},
+			good: [][]plugins.ClientIdentity{nil, idents["global"], idents["role1"]},
 		},
 		{
 			cmd:  bson.D{{"killCursors", ""}, {"$db", "authzcolcru"}},

pkg/mongoproxy/plugins/interface.go

@@ -33,6 +33,7 @@ func NewCursorCacheEntry(id int64) *CursorCacheEntry {
 }
 
 type CursorCache interface {
+	CreateCursor(cursorID int64) *CursorCacheEntry
 	GetCursor(cursorID int64) *CursorCacheEntry
 	CloseCursor(cursorID int64)
 }

pkg/mongoproxy/plugins/mongo/mongo.go

@@ -302,7 +302,7 @@ func (p *MongoPlugin) Process(ctx context.Context, r *plugins.Request, next plug
 				if cursorID, ok := cursorIDRaw.(int64); ok && cursorID > 0 {
 					logrus.Tracef("Store cursor: %v %v", cursorID, cmdServer)
 					// TODO: TTL from cmd
-					r.CursorCache.GetCursor(cursorID).Map[contextKeyServer] = cmdServer
+					r.CursorCache.CreateCursor(cursorID).Map[contextKeyServer] = cmdServer
 				}
 			}
 		}
@@ -530,6 +530,13 @@ func (p *MongoPlugin) Process(ctx context.Context, r *plugins.Request, next plug
 			v, ok = bsonutil.Lookup(result, "cursorsKilled")
 			if ok {
 				cursorsKilled = append(cursorsKilled, v.(primitive.A)...)
+				for _, cursorIDRaw := range cursorsKilled {
+					fmt.Println("kill a cursor", cursorIDRaw)
+					cursorID, ok := cursorIDRaw.(int64)
+					if ok {
+						r.CursorCache.CloseCursor(cursorID)
+					}
+				}
 			}
 			v, ok = bsonutil.Lookup(result, "cursorsNotFound")
 			if ok {

pkg/mongoproxy/proxy.go

@@ -13,8 +13,8 @@ import (
 	"sync"
 	"time"
 
-	"github.com/ReneKroon/ttlcache/v2"
 	"github.com/getsentry/sentry-go"
+	"github.com/jellydator/ttlcache/v2"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promauto"
 	"github.com/sirupsen/logrus"
@@ -91,24 +91,16 @@ func NewProxy(l net.Listener, cfg *config.Config) (*Proxy, error) {
 
 	// Set up cursorCache
 	p.cursorCache.SetTTL(p.cfg.IdleCursorTimeout) // default TTL -- config
-	p.cursorCache.SetLoaderFunction(func(key string) (interface{}, time.Duration, error) {
-		cursorID, err := strconv.ParseInt(key, 10, 64)
-		if err != nil {
-			return nil, time.Duration(0), err
-		}
-
-		return plugins.NewCursorCacheEntry(cursorID), time.Duration(0), nil
-	})
 	// expiration handler to send killCursor commands
 	p.cursorCache.SetExpirationReasonCallback(func(key string, reason ttlcache.EvictionReason, value interface{}) {
 		logrus.Tracef("expire cursor %s", key)
-		i, err := strconv.ParseInt(key, 10, 64)
-		if err != nil {
-			return
-		}
 
 		// If the cursor expired (we timed out waiting) we want to kill the downstream cursor as we remove it from the cache
 		if reason == ttlcache.Expired {
+			i, err := strconv.ParseInt(key, 10, 64)
+			if err != nil {
+				return
+			}
 			p.HandleMongo(context.TODO(), &plugins.Request{CursorCache: p, CC: p.internalCC}, bson.D{
 				{"killCursors", "admin"},
 				{"cursors", primitive.A{i}},
@@ -138,10 +130,26 @@ type Proxy struct {
 	internalCC *plugins.ClientConnection
 }
 
+func (p *Proxy) CreateCursor(cursorID int64) *plugins.CursorCacheEntry {
+	v, err := p.cursorCache.GetByLoader(strconv.FormatInt(cursorID, 10), func(key string) (interface{}, time.Duration, error) {
+		cursorID, err := strconv.ParseInt(key, 10, 64)
+		if err != nil {
+			return nil, time.Duration(0), err
+		}
+
+		return plugins.NewCursorCacheEntry(cursorID), time.Duration(0), nil
+	})
+	if err == ttlcache.ErrNotFound {
+		panic("can't get cursor")
+	}
+
+	return v.(*plugins.CursorCacheEntry)
+}
+
 func (p *Proxy) GetCursor(cursorID int64) *plugins.CursorCacheEntry {
 	v, err := p.cursorCache.Get(strconv.FormatInt(cursorID, 10))
 	if err == ttlcache.ErrNotFound {
-		panic("can't get cursor")
+		return nil
 	}
 
 	return v.(*plugins.CursorCacheEntry)