From 4791073331d2b167bad882a01ea13020b7e153dd Mon Sep 17 00:00:00 2001 From: yinwang-wish <92184841+yinwang-wish@users.noreply.github.com> Date: Wed, 26 Oct 2022 17:12:50 -0700 Subject: [PATCH 1/5] fix bugs --- pkg/mongoproxy/plugins/schema/example.json | 76 ++++++++++++++++++++ pkg/mongoproxy/plugins/schema/type_test.go | 20 ++++++ pkg/mongoproxy/plugins/schema/types.go | 82 +++++++++++++++++++--- pkg/mongoproxy/plugins/schema/util.go | 15 +++- 4 files changed, 179 insertions(+), 14 deletions(-) diff --git a/pkg/mongoproxy/plugins/schema/example.json b/pkg/mongoproxy/plugins/schema/example.json index 513b3fa..14c8caf 100644 --- a/pkg/mongoproxy/plugins/schema/example.json +++ b/pkg/mongoproxy/plugins/schema/example.json @@ -221,6 +221,24 @@ }, "enforceSchema": true }, + "requireonlyarray": { + "denyUnknownFields": false, + "fields": { + "a": { + "required": true, + "type": "string" + }, + "b": { + "required": false, + "type": "[]object" + }, + "c": { + "required": false, + "type": "object" + } + }, + "enforceSchema": true + }, "requireonlysuba": { "denyUnknownFields": true, "fields": { @@ -240,6 +258,64 @@ }, "enforceSchema": true }, + "requireonlysubddnosub": { + "denyUnknownFields": false, + "fields": { + "doc": { + "required": true, + "type": "object" + } + }, + "enforceSchema": true + }, + "requireonlysubddwithsub": { + "denyUnknownFields": false, + "fields": { + "doc": { + "required": true, + "subfields": { + "a": { + "required": true, + "type": "object" + } + }, + "type": "object" + } + }, + "enforceSchema": true + }, + "requireonlysubddwithsubdeny": { + "denyUnknownFields": true, + "fields": { + "doc": { + "required": true, + "subfields": { + "a": { + "required": true, + "type": "object" + } + }, + "type": "object" + } + }, + "enforceSchema": true + }, + "requireonlysubddwithsubarr": { + "denyUnknownFields": false, + "fields": { + "doc": { + "required": true, + "subfields": { + "a": { + "required": true, + "type": "[]string" + } + }, + "type": "object" + } + }, + "enforceSchema": true + }, "requireonlysub": { "denyUnknownFields": true, "fields": { diff --git a/pkg/mongoproxy/plugins/schema/type_test.go b/pkg/mongoproxy/plugins/schema/type_test.go index 5e3ee8f..23fda2f 100644 --- a/pkg/mongoproxy/plugins/schema/type_test.go +++ b/pkg/mongoproxy/plugins/schema/type_test.go @@ -299,6 +299,7 @@ var ( {DB: "testdb", Collection: "testcollection", In: bson.D{{"$addToSet", bson.D{{"name", 1}}}}, Err: true}, // addToSet unknown field {DB: "testdb", Collection: "testcollection", In: bson.D{{"$addToSet", bson.D{{"unknown", 1}}}}}, + {DB: "testdb", Collection: "requireonlyarray", In: bson.D{{"$addToSet", bson.D{{"a", "dsdes"}}}}}, // addToSet correct type {DB: "testdb", Collection: "testcollection", In: bson.D{{"$addToSet", bson.D{{"name", "name"}}}}}, @@ -401,6 +402,21 @@ var ( {"$setOnInsert", bson.D{{"a", "a"}}}, }, Upsert: true}, + {DB: "testdb", Collection: "requireonlysubddnosub", In: bson.D{ + {"$set", bson.D{{"doc.a.123", 1}}}, + }, Upsert: true}, + {DB: "testdb", Collection: "requireonlysubddwithsub", In: bson.D{ + {"$set", bson.D{{"doc.a.123", 1}}}, + }, Upsert: true}, + {DB: "testdb", Collection: "requireonlysubddwithsub", In: bson.D{ + {"$set", bson.D{{"doc.a.123", 1}}}, + }, Upsert: false}, + {DB: "testdb", Collection: "requireonlysubddwithsubdeny", In: bson.D{ + {"$set", bson.D{{"doc.a.123", 1}}}, + }, Upsert: true, Err: true}, + {DB: "testdb", Collection: "requireonlysubddwithsubarr", In: bson.D{ + {"$set", bson.D{{"doc.a.123", "test"}}}, + }, Upsert: true}, // incorrect type {DB: "testdb", Collection: "requireonlya", In: bson.D{ {"$set", bson.D{{"a", 1}}}, @@ -472,6 +488,10 @@ var ( // set correct type for array {DB: "testdb", Collection: "testcollection", In: bson.D{{"$set", bson.D{{"friends.$", "linda"}}}}}, + // test operator without $ sign + {DB: "testdb", Collection: "requireonlyarray", In: bson.D{{"a", bson.D{{"$eq", "a"}}}}}, + {DB: "testdb", Collection: "requireonlyarray", In: bson.D{{"b", bson.D{{"$in", bson.A{bson.D{{"a", "b"}}}}}}}}, + {DB: "testdb", Collection: "requireonlyarray", In: bson.D{{"c", bson.D{{"$nin", bson.D{{"a", "b"}}}}}}}, {DB: "testdb", Collection: "testcollection", In: bson.D{{"$set", bson.D{{"friends.0", "linda"}}}}}, {DB: "testdb", Collection: "testcollection", In: bson.D{{"$set", bson.D{{"friends", bson.A{"linda", "test"}}}}}}, {DB: "testdb", Collection: "testcollection", In: bson.D{{"$set", bson.D{{"friends.$[]", "linda"}}}}}, diff --git a/pkg/mongoproxy/plugins/schema/types.go b/pkg/mongoproxy/plugins/schema/types.go index 2f71d43..211f32f 100644 --- a/pkg/mongoproxy/plugins/schema/types.go +++ b/pkg/mongoproxy/plugins/schema/types.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "reflect" + "strconv" "strings" "github.com/sirupsen/logrus" @@ -206,6 +207,11 @@ func (c *Collection) GetField(names ...string) *CollectionField { if !ok { return nil } + vType := fmt.Sprint(v.Type) + if vType == "object" && v.SubFields == nil { + field = &v + break + } field = &v } else { var ( @@ -216,6 +222,10 @@ func (c *Collection) GetField(names ...string) *CollectionField { v, ok = field.remoteCollection.Fields[name] } else { v, ok = field.SubFields[name] + if _, err := strconv.ParseInt(name,10,64); err == nil { + ok = true + } + print("okokokok ,", ok) } if !ok { logrus.Debugf("can not find field in collection: %s", name) @@ -248,6 +258,7 @@ func (c *Collection) ValidateUpdate(ctx context.Context, obj bson.D, upsert bool insertFields bson.M // Insert fields (if we have them) -- only for upserts unsetFields bson.M // fields being unset renameFields bson.M // fields being renamed + flag bool ) if !c.EnforceSchema && !c.EnforceSchemaByCollectionLogOnly { @@ -262,12 +273,26 @@ func (c *Collection) ValidateUpdate(ctx context.Context, obj bson.D, upsert bool */ if !strings.HasPrefix(obj[0].Key, "$") || !SetContain(OpMap, obj[0].Key) { m := make(bson.M, len(obj)) - if upsert { - insertFields = handleObj(obj, m) - logrus.Debugf("insertFields: %s", insertFields) + var findOp = []string{"$in", "$nin", "$eq", "$gt", "$gte", "$lt", "$lte", "$ne"} + interfaceType := fmt.Sprint(reflect.TypeOf(obj[0].Value)) + if string(interfaceType) == "primitive.D" { + curObj := obj[0].Value.(primitive.D) + curMap := curObj.Map() + for _, s := range findOp { + if _, ok := curMap[s]; ok { + newObj := bson.D{{obj[0].Key, curMap[s]}} + upsertOrSetField(upsert, insertFields, setFields, newObj, m) + } + } } else { - setFields = handleObj(obj, m) - logrus.Debugf("setFields: %s", setFields) + if upsert { + insertFields = handleObj(obj, m) + logrus.Debugf("insertFields: %s", insertFields) + } else { + setFields = handleObj(obj, m) + fmt.Println(obj) + logrus.Debugf("setFields: %s", setFields) + } } } else { for _, e := range obj { @@ -359,8 +384,39 @@ func (c *Collection) ValidateUpdate(ctx context.Context, obj bson.D, upsert bool return fmt.Errorf("cannot set a required field with nil value: %f", v) } if f != nil && v != nil { - if err := f.Validate(ctx, v, c.DenyUnknownFields, true); err != nil { - return err + fString := fmt.Sprint(c.GetField(strings.Split(k, ".")[0]).Type) + if fString == "object" { + if c.GetField(strings.Split(k, ".")[0]).SubFields == nil { + flag = true + continue + } else { + for _, e := range obj { + newObj := e.Value.(bson.D) + for _, i := range newObj { + newKey := i.Key + newVal := i.Value + fObj := c.GetField(strings.Split(newKey, ".")...) + fObjStr := strings.Trim(fmt.Sprint(fObj), "&{") + fObjStr = strings.TrimSpace(fObjStr) + if fObj == nil || strings.HasPrefix(fObjStr, "") { + if !c.DenyUnknownFields { + flag = true + continue + } else { + logrus.Errorf("Can't set value for unknonw field %s", newKey) + break + } + } + if err := fObj.Validate(ctx, newVal, c.DenyUnknownFields, true); err != nil { + return err + } + } + } + } + } else { + if err := f.Validate(ctx, v, c.DenyUnknownFields, true); err != nil { + return err + } } } } @@ -369,12 +425,14 @@ func (c *Collection) ValidateUpdate(ctx context.Context, obj bson.D, upsert bool if upsert { doc := make(bson.M, len(setFields)+len(insertFields)) logrus.Debugf("upsert doc built") - for k, v := range setFields { - if err := SetValue(doc, strings.Split(k, "."), v); err != nil { - return err + if !flag { + for k, v := range setFields { + if err := SetValue(doc, strings.Split(k, "."), v); err != nil { + return err + } } + logrus.Debugf("finished setField setValue") } - logrus.Debugf("finished setField setValue") for k, v := range insertFields { if err := SetValue(doc, strings.Split(k, "."), v); err != nil { @@ -461,6 +519,8 @@ func (c *CollectionField) ValidateElement(ctx context.Context, d interface{}, va // ValidateInsert will validate the schema of the passed in object. func (c *CollectionField) Validate(ctx context.Context, v interface{}, denyUnknownFields, isUpdate bool) error { validateType := c.Type + fmt.Println("$%$%$%$%$%", validateType) + fmt.Println("$%$%$%$%$%", c.Name) interfaceType := fmt.Sprint(reflect.TypeOf(v)) if isUpdate { // array update is validating a scalar instead of [] if !strings.HasPrefix(interfaceType, "[]") && interfaceType != "primitive.A" { diff --git a/pkg/mongoproxy/plugins/schema/util.go b/pkg/mongoproxy/plugins/schema/util.go index 0918050..a95dbb5 100644 --- a/pkg/mongoproxy/plugins/schema/util.go +++ b/pkg/mongoproxy/plugins/schema/util.go @@ -3,12 +3,11 @@ package schema import ( "context" "fmt" - "reflect" - "regexp" - "github.com/sirupsen/logrus" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" + "reflect" + "regexp" ) var ( @@ -173,6 +172,16 @@ func handleObj(obj bson.D, m bson.M) bson.M { return m } +func upsertOrSetField(upsert bool, insertFields bson.M, setFields bson.M, obj bson.D, m bson.M) { + if upsert { + insertFields = handleObj(obj, m) + logrus.Debugf("insertFields: %s", insertFields) + } else { + setFields = handleObj(obj, m) + logrus.Debugf("setFields: %s", setFields) + } +} + func processArray(e bson.E) bson.E { if match := re.Find([]byte(e.Key)); match != nil { // detect array // remove positional info, so tree traverse can proceed From 6611c634468bfaff9d59da8a39889a18dbb1be35 Mon Sep 17 00:00:00 2001 From: yinwang-wish <92184841+yinwang-wish@users.noreply.github.com> Date: Wed, 26 Oct 2022 17:13:28 -0700 Subject: [PATCH 2/5] fix --- pkg/mongoproxy/plugins/schema/types.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/mongoproxy/plugins/schema/types.go b/pkg/mongoproxy/plugins/schema/types.go index 211f32f..7dc01f1 100644 --- a/pkg/mongoproxy/plugins/schema/types.go +++ b/pkg/mongoproxy/plugins/schema/types.go @@ -222,7 +222,7 @@ func (c *Collection) GetField(names ...string) *CollectionField { v, ok = field.remoteCollection.Fields[name] } else { v, ok = field.SubFields[name] - if _, err := strconv.ParseInt(name,10,64); err == nil { + if _, err := strconv.ParseInt(name, 10, 64); err == nil { ok = true } print("okokokok ,", ok) @@ -258,7 +258,7 @@ func (c *Collection) ValidateUpdate(ctx context.Context, obj bson.D, upsert bool insertFields bson.M // Insert fields (if we have them) -- only for upserts unsetFields bson.M // fields being unset renameFields bson.M // fields being renamed - flag bool + flag bool ) if !c.EnforceSchema && !c.EnforceSchemaByCollectionLogOnly { From f15bf23e3945af06b6cb2c17093a3fd78b6b8592 Mon Sep 17 00:00:00 2001 From: yinwang-wish <92184841+yinwang-wish@users.noreply.github.com> Date: Wed, 26 Oct 2022 17:18:41 -0700 Subject: [PATCH 3/5] update --- pkg/mongoproxy/plugins/schema/util.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/mongoproxy/plugins/schema/util.go b/pkg/mongoproxy/plugins/schema/util.go index a95dbb5..60ffd93 100644 --- a/pkg/mongoproxy/plugins/schema/util.go +++ b/pkg/mongoproxy/plugins/schema/util.go @@ -3,11 +3,12 @@ package schema import ( "context" "fmt" + "reflect" + "regexp" + "github.com/sirupsen/logrus" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/bson/primitive" - "reflect" - "regexp" ) var ( From 8979b4b763eaea4b49fdab3331203dd39b85cacd Mon Sep 17 00:00:00 2001 From: yinwang-wish <92184841+yinwang-wish@users.noreply.github.com> Date: Wed, 26 Oct 2022 17:22:58 -0700 Subject: [PATCH 4/5] fix --- pkg/mongoproxy/plugins/schema/types.go | 8 +++++++- pkg/mongoproxy/plugins/schema/util.go | 10 ---------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/pkg/mongoproxy/plugins/schema/types.go b/pkg/mongoproxy/plugins/schema/types.go index 7dc01f1..692b2f9 100644 --- a/pkg/mongoproxy/plugins/schema/types.go +++ b/pkg/mongoproxy/plugins/schema/types.go @@ -281,7 +281,13 @@ func (c *Collection) ValidateUpdate(ctx context.Context, obj bson.D, upsert bool for _, s := range findOp { if _, ok := curMap[s]; ok { newObj := bson.D{{obj[0].Key, curMap[s]}} - upsertOrSetField(upsert, insertFields, setFields, newObj, m) + if upsert { + insertFields = handleObj(newObj, m) + logrus.Debugf("insertFields: %s", insertFields) + } else { + setFields = handleObj(newObj, m) + logrus.Debugf("setFields: %s", setFields) + } } } } else { diff --git a/pkg/mongoproxy/plugins/schema/util.go b/pkg/mongoproxy/plugins/schema/util.go index 60ffd93..0918050 100644 --- a/pkg/mongoproxy/plugins/schema/util.go +++ b/pkg/mongoproxy/plugins/schema/util.go @@ -173,16 +173,6 @@ func handleObj(obj bson.D, m bson.M) bson.M { return m } -func upsertOrSetField(upsert bool, insertFields bson.M, setFields bson.M, obj bson.D, m bson.M) { - if upsert { - insertFields = handleObj(obj, m) - logrus.Debugf("insertFields: %s", insertFields) - } else { - setFields = handleObj(obj, m) - logrus.Debugf("setFields: %s", setFields) - } -} - func processArray(e bson.E) bson.E { if match := re.Find([]byte(e.Key)); match != nil { // detect array // remove positional info, so tree traverse can proceed From a043f7e8cf511cfe144ac2d6ba3edef542520616 Mon Sep 17 00:00:00 2001 From: yinwang-wish <92184841+yinwang-wish@users.noreply.github.com> Date: Wed, 26 Oct 2022 17:34:23 -0700 Subject: [PATCH 5/5] fix --- pkg/mongoproxy/plugins/schema/types.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/mongoproxy/plugins/schema/types.go b/pkg/mongoproxy/plugins/schema/types.go index 692b2f9..f829402 100644 --- a/pkg/mongoproxy/plugins/schema/types.go +++ b/pkg/mongoproxy/plugins/schema/types.go @@ -525,8 +525,6 @@ func (c *CollectionField) ValidateElement(ctx context.Context, d interface{}, va // ValidateInsert will validate the schema of the passed in object. func (c *CollectionField) Validate(ctx context.Context, v interface{}, denyUnknownFields, isUpdate bool) error { validateType := c.Type - fmt.Println("$%$%$%$%$%", validateType) - fmt.Println("$%$%$%$%$%", c.Name) interfaceType := fmt.Sprint(reflect.TypeOf(v)) if isUpdate { // array update is validating a scalar instead of [] if !strings.HasPrefix(interfaceType, "[]") && interfaceType != "primitive.A" {