From 20d7e4bc10f9625856bae43b4312c4be46c31a71 Mon Sep 17 00:00:00 2001 From: Sonui Date: Tue, 25 Feb 2025 18:18:26 +0800 Subject: [PATCH 01/39] feat(plugin): add key-value storage support for plugins --- internal/entity/plugin_kv_storage_entity.go | 32 ++ internal/migrations/init_data.go | 1 + internal/migrations/migrations.go | 1 + .../plugin_common/plugin_common_service.go | 5 + plugin/kv_storage.go | 344 ++++++++++++++++++ plugin/plugin.go | 12 + 6 files changed, 395 insertions(+) create mode 100644 internal/entity/plugin_kv_storage_entity.go create mode 100644 plugin/kv_storage.go diff --git a/internal/entity/plugin_kv_storage_entity.go b/internal/entity/plugin_kv_storage_entity.go new file mode 100644 index 000000000..c7e6efbe3 --- /dev/null +++ b/internal/entity/plugin_kv_storage_entity.go @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package entity + +type PluginKVStorage struct { + ID int `xorm:"not null pk autoincr INT(11) id"` + PluginSlugName string `xorm:"not null VARCHAR(128) UNIQUE(uk_psg) plugin_slug_name"` + Group string `xorm:"not null VARCHAR(128) UNIQUE(uk_psg) 'group'"` + Key string `xorm:"not null VARCHAR(128) UNIQUE(uk_psg) 'key'"` + Value string `xorm:"not null TEXT value"` +} + +func (PluginKVStorage) TableName() string { + return "plugin_kv_storage" +} diff --git a/internal/migrations/init_data.go b/internal/migrations/init_data.go index 7322f7388..8b853c4dc 100644 --- a/internal/migrations/init_data.go +++ b/internal/migrations/init_data.go @@ -74,6 +74,7 @@ var ( &entity.Badge{}, &entity.BadgeGroup{}, &entity.BadgeAward{}, + &entity.PluginKVStorage{}, } roles = []*entity.Role{ diff --git a/internal/migrations/migrations.go b/internal/migrations/migrations.go index 57f4778e5..ca7156495 100644 --- a/internal/migrations/migrations.go +++ b/internal/migrations/migrations.go @@ -101,6 +101,7 @@ var migrations = []Migration{ NewMigration("v1.4.1", "add question link", addQuestionLink, true), NewMigration("v1.4.2", "add the number of question links", addQuestionLinkedCount, true), NewMigration("v1.4.5", "add file record", addFileRecord, true), + NewMigration("v1.4.6", "add plugin kv storage", addPluginKVStorage, true), } func GetMigrations() []Migration { diff --git a/internal/service/plugin_common/plugin_common_service.go b/internal/service/plugin_common/plugin_common_service.go index c1b0ad442..de6d981d5 100644 --- a/internal/service/plugin_common/plugin_common_service.go +++ b/internal/service/plugin_common/plugin_common_service.go @@ -135,6 +135,11 @@ func (ps *PluginCommonService) GetUserPluginConfig(ctx context.Context, req *sch } func (ps *PluginCommonService) initPluginData() { + plugin.SetKVStorageDB(&plugin.Data{ + DB: ps.data.DB, + Cache: ps.data.Cache, + }) + // init plugin status pluginStatus, err := ps.configService.GetStringValue(context.TODO(), constant.PluginStatus) if err != nil { diff --git a/plugin/kv_storage.go b/plugin/kv_storage.go new file mode 100644 index 000000000..9714487fe --- /dev/null +++ b/plugin/kv_storage.go @@ -0,0 +1,344 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package plugin + +import ( + "context" + "encoding/json" + "fmt" + "math/rand" + "time" + + "github.com/apache/answer/internal/entity" + "github.com/segmentfault/pacman/log" + "xorm.io/builder" + "xorm.io/xorm" +) + +// define error +var ( + ErrKVKeyNotFound = fmt.Errorf("key not found in KV storage") + ErrKVGroupEmpty = fmt.Errorf("group name is empty") + ErrKVKeyEmpty = fmt.Errorf("key name is empty") + ErrKVKeyAndGroupEmpty = fmt.Errorf("both key and group are empty") + ErrKVTransactionFailed = fmt.Errorf("KV storage transaction failed") + ErrKVDataNotInitialized = fmt.Errorf("KV storage data not initialized") + ErrKVDBNotInitialized = fmt.Errorf("KV storage database connection not initialized") +) + +type KVOperator struct { + data *Data + session *xorm.Session + pluginSlugName string +} + +func (kv *KVOperator) checkDB() error { + if kv.data == nil { + return ErrKVDataNotInitialized + } + if kv.data.DB == nil { + return ErrKVDBNotInitialized + } + return nil +} + +func (kv *KVOperator) getSession(ctx context.Context) (session *xorm.Session, close func()) { + if kv.session != nil { + session = kv.session + } else { + session = kv.data.DB.NewSession().Context(ctx) + close = func() { + if session != nil { + session.Close() + } + } + } + return +} + +func (kv *KVOperator) getCacheTTL() time.Duration { + return 30*time.Minute + time.Duration(rand.Intn(300))*time.Second +} + +func (kv *KVOperator) getCacheKey(group, key string) string { + if group == "" { + return fmt.Sprintf("plugin_kv_storage:%s:key:%s", kv.pluginSlugName, key) + } + if key == "" { + return fmt.Sprintf("plugin_kv_storage:%s:group:%s", kv.pluginSlugName, group) + } + return fmt.Sprintf("plugin_kv_storage:%s:group:%s:key:%s", kv.pluginSlugName, group, key) +} + +func (kv *KVOperator) Get(ctx context.Context, group, key string) (string, error) { + // validate + if err := kv.checkDB(); err != nil { + return "", err + } + if key == "" { + return "", ErrKVKeyEmpty + } + + cacheKey := kv.getCacheKey(group, key) + if value, exist, err := kv.data.Cache.GetString(ctx, cacheKey); err == nil && exist { + return value, nil + } + + // query + data := entity.PluginKVStorage{} + query, close := kv.getSession(ctx) + defer close() + + query.Where(builder.Eq{ + "plugin_slug_name": kv.pluginSlugName, + "`group`": group, + "`key`": key, + }) + + has, err := query.Get(&data) + if err != nil { + return "", err + } + if !has { + return "", ErrKVKeyNotFound + } + + if err := kv.data.Cache.SetString(ctx, cacheKey, data.Value, kv.getCacheTTL()); err != nil { + log.Error(err) + } + + return data.Value, nil +} + +func (kv *KVOperator) Set(ctx context.Context, group, key, value string) error { + if err := kv.checkDB(); err != nil { + return err + } + + if key == "" { + return ErrKVKeyEmpty + } + + query, close := kv.getSession(ctx) + if close != nil { + defer close() + } + + data := &entity.PluginKVStorage{ + PluginSlugName: kv.pluginSlugName, + Group: group, + Key: key, + Value: value, + } + + kv.cleanCache(ctx, group, key) + + affected, err := query.Where(builder.Eq{ + "plugin_slug_name": kv.pluginSlugName, + "`group`": group, + "`key`": key, + }).Cols("value").Update(data) + if err != nil { + return err + } + + if affected == 0 { + _, err = query.Insert(data) + if err != nil { + return err + } + } + return nil +} + +func (kv *KVOperator) Del(ctx context.Context, group, key string) error { + if err := kv.checkDB(); err != nil { + return err + } + + if key == "" && group == "" { + return ErrKVKeyAndGroupEmpty + } + + kv.cleanCache(ctx, group, key) + + session, close := kv.getSession(ctx) + defer close() + + session.Where(builder.Eq{ + "plugin_slug_name": kv.pluginSlugName, + }) + if group != "" { + session.Where(builder.Eq{"`group`": group}) + } + if key != "" { + session.Where(builder.Eq{"`key`": key}) + } + + _, err := session.Delete(&entity.PluginKVStorage{}) + return err +} + +func (kv *KVOperator) cleanCache(ctx context.Context, group, key string) { + if key != "" { + if err := kv.data.Cache.Del(ctx, kv.getCacheKey("", key)); err != nil { + log.Warnf("Failed to delete cache for key %s: %v", key, err) + } + + if group != "" { + if err := kv.data.Cache.Del(ctx, kv.getCacheKey(group, key)); err != nil { + log.Warnf("Failed to delete cache for group %s, key %s: %v", group, key, err) + } + } + } + + if group != "" { + if err := kv.data.Cache.Del(ctx, kv.getCacheKey(group, "")); err != nil { + log.Warnf("Failed to delete cache for group %s: %v", group, err) + } + } +} + +func (kv *KVOperator) GetByGroup(ctx context.Context, group string, page, pageSize int) (map[string]string, error) { + if err := kv.checkDB(); err != nil { + return nil, err + } + + if group == "" { + return nil, ErrKVGroupEmpty + } + + if page < 1 { + page = 1 + } + if pageSize < 1 { + pageSize = 10 + } + + if pageSize > 100 { + pageSize = 100 + } + + cacheKey := kv.getCacheKey(group, "") + if value, exist, err := kv.data.Cache.GetString(ctx, cacheKey); err == nil && exist { + result := make(map[string]string) + if err := json.Unmarshal([]byte(value), &result); err == nil { + return result, nil + } + } + + query, close := kv.getSession(ctx) + defer close() + + var items []entity.PluginKVStorage + err := query.Where(builder.Eq{"plugin_slug_name": kv.pluginSlugName, "`group`": group}). + Limit(pageSize, (page-1)*pageSize). + OrderBy("id ASC"). + Find(&items) + if err != nil { + return nil, err + } + + result := make(map[string]string, len(items)) + for _, item := range items { + result[item.Key] = item.Value + if err := kv.data.Cache.SetString(ctx, kv.getCacheKey(group, item.Key), item.Value, kv.getCacheTTL()); err != nil { + log.Warnf("Failed to set cache for group %s, key %s: %v", group, item.Key, err) + } + } + + if resultJSON, err := json.Marshal(result); err == nil { + _ = kv.data.Cache.SetString(ctx, cacheKey, string(resultJSON), kv.getCacheTTL()) + } + + return result, nil +} + +func (kv *KVOperator) Tx(ctx context.Context, fn func(ctx context.Context, kv *KVOperator) error) error { + if err := kv.checkDB(); err != nil { + return fmt.Errorf("%w: %v", ErrKVTransactionFailed, err) + } + + var ( + txKv = kv + shouldCommit bool + ) + + if kv.session == nil { + session := kv.data.DB.NewSession().Context(ctx) + if err := session.Begin(); err != nil { + session.Close() + return fmt.Errorf("%w: begin transaction failed: %v", ErrKVTransactionFailed, err) + } + + defer func() { + if !shouldCommit { + if rollbackErr := session.Rollback(); rollbackErr != nil { + log.Errorf("rollback failed: %v", rollbackErr) + } + } + session.Close() + }() + + txKv = &KVOperator{ + session: session, + data: kv.data, + pluginSlugName: kv.pluginSlugName, + } + shouldCommit = true + } + + if err := fn(ctx, txKv); err != nil { + return fmt.Errorf("%w: %v", ErrKVTransactionFailed, err) + } + + if shouldCommit { + if err := txKv.session.Commit(); err != nil { + return fmt.Errorf("%w: commit failed: %v", ErrKVTransactionFailed, err) + } + } + return nil +} + +// PluginData defines the interface for plugins that need data storage capabilities +type KVStorage interface { + Info() Info + SetOperator(operator *KVOperator) +} + +var ( + _, + registerPluginKVStorage = func() (CallFn[KVStorage], RegisterFn[KVStorage]) { + callFn, registerFn := MakePlugin[KVStorage](false) + return callFn, func(p KVStorage) { + registerFn(p) + kvStoragePluginStack.plugins = append(kvStoragePluginStack.plugins, p) + } + }() + kvStoragePluginStack = &Stack[KVStorage]{} +) + +func SetKVStorageDB(data *Data) { + for _, p := range kvStoragePluginStack.plugins { + p.SetOperator(&KVOperator{ + data: data, + pluginSlugName: p.Info().SlugName, + }) + } +} diff --git a/plugin/plugin.go b/plugin/plugin.go index 36087c547..dc8c35ea1 100644 --- a/plugin/plugin.go +++ b/plugin/plugin.go @@ -23,13 +23,21 @@ import ( "encoding/json" "sync" + "github.com/segmentfault/pacman/cache" "github.com/segmentfault/pacman/i18n" + "xorm.io/xorm" "github.com/apache/answer/internal/base/handler" "github.com/apache/answer/internal/base/translator" "github.com/gin-gonic/gin" ) +// Data is defined here to avoid circular dependency with internal/base/data +type Data struct { + DB *xorm.Engine + Cache cache.Cache +} + // GinContext is a wrapper of gin.Context // We export it to make it easy to use in plugins type GinContext = gin.Context @@ -114,6 +122,10 @@ func Register(p Base) { if _, ok := p.(Importer); ok { registerImporter(p.(Importer)) } + + if _, ok := p.(KVStorage); ok { + registerPluginKVStorage(p.(KVStorage)) + } } type Stack[T Base] struct { From 42cc06886abcdfc808cc6c3b5c637f31c5799029 Mon Sep 17 00:00:00 2001 From: Sonui Date: Sat, 1 Mar 2025 11:22:40 +0800 Subject: [PATCH 02/39] refactor(plugin): improved initialization of the KV storage plugin --- .../plugin_common/plugin_common_service.go | 10 +++++--- plugin/kv_storage.go | 24 +++++++------------ plugin/plugin.go | 2 +- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/internal/service/plugin_common/plugin_common_service.go b/internal/service/plugin_common/plugin_common_service.go index de6d981d5..d3aa839b2 100644 --- a/internal/service/plugin_common/plugin_common_service.go +++ b/internal/service/plugin_common/plugin_common_service.go @@ -135,9 +135,13 @@ func (ps *PluginCommonService) GetUserPluginConfig(ctx context.Context, req *sch } func (ps *PluginCommonService) initPluginData() { - plugin.SetKVStorageDB(&plugin.Data{ - DB: ps.data.DB, - Cache: ps.data.Cache, + _ = plugin.CallKVStorage(func(k plugin.KVStorage) error { + k.SetOperator(plugin.NewKVOperator( + ps.data.DB, + ps.data.Cache, + k.Info().SlugName, + )) + return nil }) // init plugin status diff --git a/plugin/kv_storage.go b/plugin/kv_storage.go index 9714487fe..b4d940b80 100644 --- a/plugin/kv_storage.go +++ b/plugin/kv_storage.go @@ -323,22 +323,16 @@ type KVStorage interface { } var ( - _, - registerPluginKVStorage = func() (CallFn[KVStorage], RegisterFn[KVStorage]) { - callFn, registerFn := MakePlugin[KVStorage](false) - return callFn, func(p KVStorage) { - registerFn(p) - kvStoragePluginStack.plugins = append(kvStoragePluginStack.plugins, p) - } - }() - kvStoragePluginStack = &Stack[KVStorage]{} + CallKVStorage, + registerKVStorage = MakePlugin[KVStorage](true) ) -func SetKVStorageDB(data *Data) { - for _, p := range kvStoragePluginStack.plugins { - p.SetOperator(&KVOperator{ - data: data, - pluginSlugName: p.Info().SlugName, - }) +// NewKVOperator creates a new KV storage operator with the specified database engine, cache and plugin name. +// It returns a KVOperator instance that can be used to interact with the plugin's storage. +func NewKVOperator(db *xorm.Engine, cache cache.Cache, pluginSlugName string) *KVOperator { + return &KVOperator{ + data: &Data{DB: db, Cache: cache}, + pluginSlugName: pluginSlugName, + cacheTTL: 30 * time.Minute, } } diff --git a/plugin/plugin.go b/plugin/plugin.go index dc8c35ea1..a9e173100 100644 --- a/plugin/plugin.go +++ b/plugin/plugin.go @@ -124,7 +124,7 @@ func Register(p Base) { } if _, ok := p.(KVStorage); ok { - registerPluginKVStorage(p.(KVStorage)) + registerKVStorage(p.(KVStorage)) } } From 9f8cab56267f2f9e19eaa393e1c4be98e0090e19 Mon Sep 17 00:00:00 2001 From: Sonui Date: Sat, 1 Mar 2025 11:56:32 +0800 Subject: [PATCH 03/39] perf(plugin): remove unnecessary KV storage checks --- plugin/kv_storage.go | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/plugin/kv_storage.go b/plugin/kv_storage.go index b4d940b80..a3ab46d03 100644 --- a/plugin/kv_storage.go +++ b/plugin/kv_storage.go @@ -48,16 +48,6 @@ type KVOperator struct { pluginSlugName string } -func (kv *KVOperator) checkDB() error { - if kv.data == nil { - return ErrKVDataNotInitialized - } - if kv.data.DB == nil { - return ErrKVDBNotInitialized - } - return nil -} - func (kv *KVOperator) getSession(ctx context.Context) (session *xorm.Session, close func()) { if kv.session != nil { session = kv.session @@ -88,9 +78,6 @@ func (kv *KVOperator) getCacheKey(group, key string) string { func (kv *KVOperator) Get(ctx context.Context, group, key string) (string, error) { // validate - if err := kv.checkDB(); err != nil { - return "", err - } if key == "" { return "", ErrKVKeyEmpty } @@ -127,10 +114,6 @@ func (kv *KVOperator) Get(ctx context.Context, group, key string) (string, error } func (kv *KVOperator) Set(ctx context.Context, group, key, value string) error { - if err := kv.checkDB(); err != nil { - return err - } - if key == "" { return ErrKVKeyEmpty } @@ -168,10 +151,6 @@ func (kv *KVOperator) Set(ctx context.Context, group, key, value string) error { } func (kv *KVOperator) Del(ctx context.Context, group, key string) error { - if err := kv.checkDB(); err != nil { - return err - } - if key == "" && group == "" { return ErrKVKeyAndGroupEmpty } @@ -216,10 +195,6 @@ func (kv *KVOperator) cleanCache(ctx context.Context, group, key string) { } func (kv *KVOperator) GetByGroup(ctx context.Context, group string, page, pageSize int) (map[string]string, error) { - if err := kv.checkDB(); err != nil { - return nil, err - } - if group == "" { return nil, ErrKVGroupEmpty } @@ -231,10 +206,6 @@ func (kv *KVOperator) GetByGroup(ctx context.Context, group string, page, pageSi pageSize = 10 } - if pageSize > 100 { - pageSize = 100 - } - cacheKey := kv.getCacheKey(group, "") if value, exist, err := kv.data.Cache.GetString(ctx, cacheKey); err == nil && exist { result := make(map[string]string) @@ -271,10 +242,6 @@ func (kv *KVOperator) GetByGroup(ctx context.Context, group string, page, pageSi } func (kv *KVOperator) Tx(ctx context.Context, fn func(ctx context.Context, kv *KVOperator) error) error { - if err := kv.checkDB(); err != nil { - return fmt.Errorf("%w: %v", ErrKVTransactionFailed, err) - } - var ( txKv = kv shouldCommit bool From 42d6ad61114ab9f5a15ac1c9b26326f273b51039 Mon Sep 17 00:00:00 2001 From: Sonui Date: Sat, 1 Mar 2025 13:55:36 +0800 Subject: [PATCH 04/39] fix(plugin): rename 'close' to avoid builtin collision --- plugin/kv_storage.go | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/plugin/kv_storage.go b/plugin/kv_storage.go index a3ab46d03..0ec3d6409 100644 --- a/plugin/kv_storage.go +++ b/plugin/kv_storage.go @@ -48,18 +48,18 @@ type KVOperator struct { pluginSlugName string } -func (kv *KVOperator) getSession(ctx context.Context) (session *xorm.Session, close func()) { - if kv.session != nil { - session = kv.session - } else { +func (kv *KVOperator) getSession(ctx context.Context) (*xorm.Session, func()) { + session := kv.session + cleanup := func() {} + if session == nil { session = kv.data.DB.NewSession().Context(ctx) - close = func() { + cleanup = func() { if session != nil { session.Close() } } } - return + return session, cleanup } func (kv *KVOperator) getCacheTTL() time.Duration { @@ -89,8 +89,8 @@ func (kv *KVOperator) Get(ctx context.Context, group, key string) (string, error // query data := entity.PluginKVStorage{} - query, close := kv.getSession(ctx) - defer close() + query, cleanup := kv.getSession(ctx) + defer cleanup() query.Where(builder.Eq{ "plugin_slug_name": kv.pluginSlugName, @@ -118,10 +118,8 @@ func (kv *KVOperator) Set(ctx context.Context, group, key, value string) error { return ErrKVKeyEmpty } - query, close := kv.getSession(ctx) - if close != nil { - defer close() - } + query, cleanup := kv.getSession(ctx) + defer cleanup() data := &entity.PluginKVStorage{ PluginSlugName: kv.pluginSlugName, @@ -157,8 +155,8 @@ func (kv *KVOperator) Del(ctx context.Context, group, key string) error { kv.cleanCache(ctx, group, key) - session, close := kv.getSession(ctx) - defer close() + session, cleanup := kv.getSession(ctx) + defer cleanup() session.Where(builder.Eq{ "plugin_slug_name": kv.pluginSlugName, @@ -214,8 +212,8 @@ func (kv *KVOperator) GetByGroup(ctx context.Context, group string, page, pageSi } } - query, close := kv.getSession(ctx) - defer close() + query, cleanup := kv.getSession(ctx) + defer cleanup() var items []entity.PluginKVStorage err := query.Where(builder.Eq{"plugin_slug_name": kv.pluginSlugName, "`group`": group}). From 56f36ec5e605bfc74fd17e86198d657a1a3f45ce Mon Sep 17 00:00:00 2001 From: Sonui Date: Mon, 10 Mar 2025 00:28:57 +0800 Subject: [PATCH 05/39] refactor(plugin): improve KV storage with better caching and param handling --- plugin/kv_storage.go | 198 ++++++++++++++++++++++++------------------- 1 file changed, 112 insertions(+), 86 deletions(-) diff --git a/plugin/kv_storage.go b/plugin/kv_storage.go index 0ec3d6409..17617f6c9 100644 --- a/plugin/kv_storage.go +++ b/plugin/kv_storage.go @@ -20,32 +20,64 @@ package plugin import ( "context" - "encoding/json" "fmt" - "math/rand" + "math/rand/v2" "time" "github.com/apache/answer/internal/entity" + "github.com/segmentfault/pacman/cache" "github.com/segmentfault/pacman/log" "xorm.io/builder" "xorm.io/xorm" ) -// define error +// Error variables for KV storage operations var ( - ErrKVKeyNotFound = fmt.Errorf("key not found in KV storage") - ErrKVGroupEmpty = fmt.Errorf("group name is empty") - ErrKVKeyEmpty = fmt.Errorf("key name is empty") - ErrKVKeyAndGroupEmpty = fmt.Errorf("both key and group are empty") - ErrKVTransactionFailed = fmt.Errorf("KV storage transaction failed") - ErrKVDataNotInitialized = fmt.Errorf("KV storage data not initialized") - ErrKVDBNotInitialized = fmt.Errorf("KV storage database connection not initialized") + // ErrKVKeyNotFound is returned when the requested key does not exist in the KV storage + ErrKVKeyNotFound = fmt.Errorf("key not found in KV storage") + // ErrKVGroupEmpty is returned when a required group name is empty + ErrKVGroupEmpty = fmt.Errorf("group name is empty") + // ErrKVKeyEmpty is returned when a required key name is empty + ErrKVKeyEmpty = fmt.Errorf("key name is empty") + // ErrKVKeyAndGroupEmpty is returned when both key and group names are empty + ErrKVKeyAndGroupEmpty = fmt.Errorf("both key and group are empty") + // ErrKVTransactionFailed is returned when a KV storage transaction operation fails + ErrKVTransactionFailed = fmt.Errorf("KV storage transaction failed") ) +// KVParams is the parameters for KV storage operations +type KVParams struct { + Group string + Key string + Value string + Page int + PageSize int +} + +// KVOperator provides methods to interact with the key-value storage system for plugins type KVOperator struct { data *Data session *xorm.Session pluginSlugName string + cacheTTL time.Duration +} + +// KVStorageOption defines a function type that configures a KVOperator +type KVStorageOption func(*KVOperator) + +// WithCacheTTL is the option to set the cache TTL; the default value is 30 minutes. +// If ttl is less than 0, the cache will not be used +func WithCacheTTL(ttl time.Duration) KVStorageOption { + return func(kv *KVOperator) { + kv.cacheTTL = ttl + } +} + +// Option is used to set the options for the KV storage +func (kv *KVOperator) Option(opts ...KVStorageOption) { + for _, opt := range opts { + opt(kv) + } } func (kv *KVOperator) getSession(ctx context.Context) (*xorm.Session, func()) { @@ -62,28 +94,53 @@ func (kv *KVOperator) getSession(ctx context.Context) (*xorm.Session, func()) { return session, cleanup } -func (kv *KVOperator) getCacheTTL() time.Duration { - return 30*time.Minute + time.Duration(rand.Intn(300))*time.Second +func (kv *KVOperator) getCacheKey(params KVParams) string { + return fmt.Sprintf("plugin_kv_storage:%s:group:%s:key:%s", kv.pluginSlugName, params.Group, params.Key) +} + +func (kv *KVOperator) setCache(ctx context.Context, params KVParams) { + if kv.cacheTTL < 0 { + return + } + + ttl := kv.cacheTTL + if ttl > 10 { + ttl += time.Duration(float64(ttl) * 0.1 * (1 - rand.Float64())) + } + + cacheKey := kv.getCacheKey(params) + if err := kv.data.Cache.SetString(ctx, cacheKey, params.Value, ttl); err != nil { + log.Warnf("cache set failed: %v, key: %s", err, cacheKey) + } +} + +func (kv *KVOperator) getCache(ctx context.Context, params KVParams) (string, bool, error) { + if kv.cacheTTL < 0 { + return "", false, nil + } + + cacheKey := kv.getCacheKey(params) + return kv.data.Cache.GetString(ctx, cacheKey) } -func (kv *KVOperator) getCacheKey(group, key string) string { - if group == "" { - return fmt.Sprintf("plugin_kv_storage:%s:key:%s", kv.pluginSlugName, key) +func (kv *KVOperator) cleanCache(ctx context.Context, params KVParams) { + if kv.cacheTTL < 0 { + return } - if key == "" { - return fmt.Sprintf("plugin_kv_storage:%s:group:%s", kv.pluginSlugName, group) + + if err := kv.data.Cache.Del(ctx, kv.getCacheKey(params)); err != nil { + log.Warnf("Failed to delete cache for key %s: %v", params.Key, err) } - return fmt.Sprintf("plugin_kv_storage:%s:group:%s:key:%s", kv.pluginSlugName, group, key) } -func (kv *KVOperator) Get(ctx context.Context, group, key string) (string, error) { - // validate - if key == "" { +// Get retrieves a value from KV storage by group and key. +// Returns the value as a string or an error if the key is not found. +func (kv *KVOperator) Get(ctx context.Context, params KVParams) (string, error) { + if params.Key == "" { return "", ErrKVKeyEmpty } - cacheKey := kv.getCacheKey(group, key) - if value, exist, err := kv.data.Cache.GetString(ctx, cacheKey); err == nil && exist { + if value, exist, err := kv.getCache(ctx, params); err == nil && exist { return value, nil } @@ -94,8 +151,8 @@ func (kv *KVOperator) Get(ctx context.Context, group, key string) (string, error query.Where(builder.Eq{ "plugin_slug_name": kv.pluginSlugName, - "`group`": group, - "`key`": key, + "`group`": params.Group, + "`key`": params.Key, }) has, err := query.Get(&data) @@ -106,15 +163,15 @@ func (kv *KVOperator) Get(ctx context.Context, group, key string) (string, error return "", ErrKVKeyNotFound } - if err := kv.data.Cache.SetString(ctx, cacheKey, data.Value, kv.getCacheTTL()); err != nil { - log.Error(err) - } + kv.setCache(ctx, params) return data.Value, nil } -func (kv *KVOperator) Set(ctx context.Context, group, key, value string) error { - if key == "" { +// Set stores a value in KV storage with the specified group and key. +// Updates the value if it already exists. +func (kv *KVOperator) Set(ctx context.Context, params KVParams) error { + if params.Key == "" { return ErrKVKeyEmpty } @@ -123,17 +180,17 @@ func (kv *KVOperator) Set(ctx context.Context, group, key, value string) error { data := &entity.PluginKVStorage{ PluginSlugName: kv.pluginSlugName, - Group: group, - Key: key, - Value: value, + Group: params.Group, + Key: params.Key, + Value: params.Value, } - kv.cleanCache(ctx, group, key) + kv.cleanCache(ctx, params) affected, err := query.Where(builder.Eq{ "plugin_slug_name": kv.pluginSlugName, - "`group`": group, - "`key`": key, + "`group`": params.Group, + "`key`": params.Key, }).Cols("value").Update(data) if err != nil { return err @@ -148,12 +205,16 @@ func (kv *KVOperator) Set(ctx context.Context, group, key, value string) error { return nil } -func (kv *KVOperator) Del(ctx context.Context, group, key string) error { - if key == "" && group == "" { +// Del removes values from KV storage by group and/or key. +// If both group and key are provided, only that specific entry is deleted. +// If only group is provided, all entries in that group are deleted. +// At least one of group or key must be provided. +func (kv *KVOperator) Del(ctx context.Context, params KVParams) error { + if params.Key == "" && params.Group == "" { return ErrKVKeyAndGroupEmpty } - kv.cleanCache(ctx, group, key) + kv.cleanCache(ctx, params) session, cleanup := kv.getSession(ctx) defer cleanup() @@ -161,63 +222,35 @@ func (kv *KVOperator) Del(ctx context.Context, group, key string) error { session.Where(builder.Eq{ "plugin_slug_name": kv.pluginSlugName, }) - if group != "" { - session.Where(builder.Eq{"`group`": group}) + if params.Group != "" { + session.Where(builder.Eq{"`group`": params.Group}) } - if key != "" { - session.Where(builder.Eq{"`key`": key}) + if params.Key != "" { + session.Where(builder.Eq{"`key`": params.Key}) } _, err := session.Delete(&entity.PluginKVStorage{}) return err } -func (kv *KVOperator) cleanCache(ctx context.Context, group, key string) { - if key != "" { - if err := kv.data.Cache.Del(ctx, kv.getCacheKey("", key)); err != nil { - log.Warnf("Failed to delete cache for key %s: %v", key, err) - } - - if group != "" { - if err := kv.data.Cache.Del(ctx, kv.getCacheKey(group, key)); err != nil { - log.Warnf("Failed to delete cache for group %s, key %s: %v", group, key, err) - } - } - } - - if group != "" { - if err := kv.data.Cache.Del(ctx, kv.getCacheKey(group, "")); err != nil { - log.Warnf("Failed to delete cache for group %s: %v", group, err) - } - } -} - -func (kv *KVOperator) GetByGroup(ctx context.Context, group string, page, pageSize int) (map[string]string, error) { - if group == "" { +func (kv *KVOperator) GetByGroup(ctx context.Context, params KVParams) (map[string]string, error) { + if params.Group == "" { return nil, ErrKVGroupEmpty } - if page < 1 { - page = 1 + if params.Page < 1 { + params.Page = 1 } - if pageSize < 1 { - pageSize = 10 - } - - cacheKey := kv.getCacheKey(group, "") - if value, exist, err := kv.data.Cache.GetString(ctx, cacheKey); err == nil && exist { - result := make(map[string]string) - if err := json.Unmarshal([]byte(value), &result); err == nil { - return result, nil - } + if params.PageSize < 1 { + params.PageSize = 10 } query, cleanup := kv.getSession(ctx) defer cleanup() var items []entity.PluginKVStorage - err := query.Where(builder.Eq{"plugin_slug_name": kv.pluginSlugName, "`group`": group}). - Limit(pageSize, (page-1)*pageSize). + err := query.Where(builder.Eq{"plugin_slug_name": kv.pluginSlugName, "`group`": params.Group}). + Limit(params.PageSize, (params.Page-1)*params.PageSize). OrderBy("id ASC"). Find(&items) if err != nil { @@ -227,13 +260,6 @@ func (kv *KVOperator) GetByGroup(ctx context.Context, group string, page, pageSi result := make(map[string]string, len(items)) for _, item := range items { result[item.Key] = item.Value - if err := kv.data.Cache.SetString(ctx, kv.getCacheKey(group, item.Key), item.Value, kv.getCacheTTL()); err != nil { - log.Warnf("Failed to set cache for group %s, key %s: %v", group, item.Key, err) - } - } - - if resultJSON, err := json.Marshal(result); err == nil { - _ = kv.data.Cache.SetString(ctx, cacheKey, string(resultJSON), kv.getCacheTTL()) } return result, nil From beac023875eeb2007a866695789140b78dfc2f19 Mon Sep 17 00:00:00 2001 From: Sonui Date: Thu, 3 Apr 2025 14:01:11 +0800 Subject: [PATCH 06/39] docs(plugin): Add comments to KVOperator methods to improve code readability --- plugin/kv_storage.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/plugin/kv_storage.go b/plugin/kv_storage.go index 17617f6c9..9b449f8ff 100644 --- a/plugin/kv_storage.go +++ b/plugin/kv_storage.go @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package plugin import ( @@ -233,6 +234,8 @@ func (kv *KVOperator) Del(ctx context.Context, params KVParams) error { return err } +// GetByGroup retrieves all key-value pairs for a specific group with pagination support. +// Returns a map of keys to values or an error if the group is empty or not found. func (kv *KVOperator) GetByGroup(ctx context.Context, params KVParams) (map[string]string, error) { if params.Group == "" { return nil, ErrKVGroupEmpty @@ -265,6 +268,9 @@ func (kv *KVOperator) GetByGroup(ctx context.Context, params KVParams) (map[stri return result, nil } +// Tx executes a function within a transaction context. If the KVOperator already has a session, +// it will use that session. Otherwise, it creates a new transaction session. +// The transaction will be committed if the function returns nil, or rolled back if it returns an error. func (kv *KVOperator) Tx(ctx context.Context, fn func(ctx context.Context, kv *KVOperator) error) error { var ( txKv = kv @@ -307,7 +313,7 @@ func (kv *KVOperator) Tx(ctx context.Context, fn func(ctx context.Context, kv *K return nil } -// PluginData defines the interface for plugins that need data storage capabilities +// KVStorage defines the interface for plugins that need data storage capabilities type KVStorage interface { Info() Info SetOperator(operator *KVOperator) From 15ef81533ff0bcc781c99c223b89b7c656822e0d Mon Sep 17 00:00:00 2001 From: Sonui Date: Thu, 3 Apr 2025 14:01:53 +0800 Subject: [PATCH 07/39] feat(migrations): Add v26 migration file to support plugin KV storage --- internal/migrations/v26.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 internal/migrations/v26.go diff --git a/internal/migrations/v26.go b/internal/migrations/v26.go new file mode 100644 index 000000000..008a094a4 --- /dev/null +++ b/internal/migrations/v26.go @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package migrations + +import ( + "context" + + "github.com/apache/answer/internal/entity" + "xorm.io/xorm" +) + +func addPluginKVStorage(ctx context.Context, x *xorm.Engine) error { + return x.Context(ctx).Sync(new(entity.PluginKVStorage)) +} From d866ca71e1b0e705ff2a6c0db7771f5d57f065c0 Mon Sep 17 00:00:00 2001 From: Sonui Date: Wed, 7 May 2025 23:46:39 +0800 Subject: [PATCH 08/39] fix(plugin): set params.Value in Get method to ensure correct value retrieval --- plugin/kv_storage.go | 1 + 1 file changed, 1 insertion(+) diff --git a/plugin/kv_storage.go b/plugin/kv_storage.go index 9b449f8ff..d1ed3eaa6 100644 --- a/plugin/kv_storage.go +++ b/plugin/kv_storage.go @@ -164,6 +164,7 @@ func (kv *KVOperator) Get(ctx context.Context, params KVParams) (string, error) return "", ErrKVKeyNotFound } + params.Value = data.Value kv.setCache(ctx, params) return data.Value, nil From aac1532b7ebe94a28d9c11b2d85a391df7a8982e Mon Sep 17 00:00:00 2001 From: Sonui Date: Wed, 7 May 2025 23:49:56 +0800 Subject: [PATCH 09/39] chore(migrations): update migration version --- internal/migrations/migrations.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/migrations/migrations.go b/internal/migrations/migrations.go index ca7156495..212bf14bb 100644 --- a/internal/migrations/migrations.go +++ b/internal/migrations/migrations.go @@ -101,7 +101,7 @@ var migrations = []Migration{ NewMigration("v1.4.1", "add question link", addQuestionLink, true), NewMigration("v1.4.2", "add the number of question links", addQuestionLinkedCount, true), NewMigration("v1.4.5", "add file record", addFileRecord, true), - NewMigration("v1.4.6", "add plugin kv storage", addPluginKVStorage, true), + NewMigration("v1.5.1", "add plugin kv storage", addPluginKVStorage, true), } func GetMigrations() []Migration { From 757e89467a7b66b73b4b6ee16ab7967faeb5bd8b Mon Sep 17 00:00:00 2001 From: Shaobiao Lin Date: Tue, 6 May 2025 19:06:56 +0800 Subject: [PATCH 10/39] docs(command): remove redundant comments The codes are self-documenting enough. --- cmd/command.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/cmd/command.go b/cmd/command.go index fe17713e7..602a12299 100644 --- a/cmd/command.go +++ b/cmd/command.go @@ -82,7 +82,6 @@ func init() { } var ( - // rootCmd represents the base command when called without any subcommands rootCmd = &cobra.Command{ Use: "answer", Short: "Answer is a minimalist open source Q&A community.", @@ -92,7 +91,6 @@ To run answer, use: - 'answer run' to launch application.`, } - // runCmd represents the run command runCmd = &cobra.Command{ Use: "run", Short: "Run the application", @@ -105,7 +103,6 @@ To run answer, use: }, } - // initCmd represents the init command initCmd = &cobra.Command{ Use: "init", Short: "init answer application", @@ -135,7 +132,6 @@ To run answer, use: }, } - // upgradeCmd represents the upgrade command upgradeCmd = &cobra.Command{ Use: "upgrade", Short: "upgrade Answer version", @@ -157,7 +153,6 @@ To run answer, use: }, } - // dumpCmd represents the dump command dumpCmd = &cobra.Command{ Use: "dump", Short: "back up data", @@ -179,7 +174,6 @@ To run answer, use: }, } - // checkCmd represents the check command checkCmd = &cobra.Command{ Use: "check", Short: "checking the required environment", @@ -214,7 +208,6 @@ To run answer, use: }, } - // buildCmd used to build another answer with plugins buildCmd = &cobra.Command{ Use: "build", Short: "used to build answer with plugins", @@ -235,7 +228,6 @@ To run answer, use: }, } - // pluginCmd prints all plugins packed in the binary pluginCmd = &cobra.Command{ Use: "plugin", Short: "prints all plugins packed in the binary", @@ -249,7 +241,6 @@ To run answer, use: }, } - // configCmd set some config to default value configCmd = &cobra.Command{ Use: "config", Short: "set some config to default value", @@ -286,7 +277,6 @@ To run answer, use: }, } - // i18nCmd used to merge i18n files i18nCmd = &cobra.Command{ Use: "i18n", Short: "overwrite i18n files", From 4182d9f5061ca018d01c414acf9be67bb72aded3 Mon Sep 17 00:00:00 2001 From: Shaobiao Lin Date: Tue, 6 May 2025 19:59:15 +0800 Subject: [PATCH 11/39] docs(command): unify style of usage statements close #1330 --- cmd/command.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/cmd/command.go b/cmd/command.go index 602a12299..7d779893f 100644 --- a/cmd/command.go +++ b/cmd/command.go @@ -93,8 +93,8 @@ To run answer, use: runCmd = &cobra.Command{ Use: "run", - Short: "Run the application", - Long: `Run the application`, + Short: "Run Answer", + Long: `Start running Answer`, Run: func(_ *cobra.Command, _ []string) { cli.FormatAllPath(dataDirPath) fmt.Println("config file path: ", cli.GetConfigFilePath()) @@ -105,8 +105,8 @@ To run answer, use: initCmd = &cobra.Command{ Use: "init", - Short: "init answer application", - Long: `init answer application`, + Short: "Initialize Answer", + Long: `Initialize Answer with specified configuration`, Run: func(_ *cobra.Command, _ []string) { // check config file and database. if config file exists and database is already created, init done cli.InstallAllInitialEnvironment(dataDirPath) @@ -134,8 +134,8 @@ To run answer, use: upgradeCmd = &cobra.Command{ Use: "upgrade", - Short: "upgrade Answer version", - Long: `upgrade Answer version`, + Short: "Upgrade Answer", + Long: `Upgrade Answer to the latest version`, Run: func(_ *cobra.Command, _ []string) { log.SetLogger(log.NewStdLogger(os.Stdout)) cli.FormatAllPath(dataDirPath) @@ -155,8 +155,8 @@ To run answer, use: dumpCmd = &cobra.Command{ Use: "dump", - Short: "back up data", - Long: `back up data`, + Short: "Back up data", + Long: `Back up database into an SQL file`, Run: func(_ *cobra.Command, _ []string) { fmt.Println("Answer is backing up data") cli.FormatAllPath(dataDirPath) @@ -176,7 +176,7 @@ To run answer, use: checkCmd = &cobra.Command{ Use: "check", - Short: "checking the required environment", + Short: "Check the required environment", Long: `Check if the current environment meets the startup requirements`, Run: func(_ *cobra.Command, _ []string) { cli.FormatAllPath(dataDirPath) @@ -210,7 +210,7 @@ To run answer, use: buildCmd = &cobra.Command{ Use: "build", - Short: "used to build answer with plugins", + Short: "Build Answer with plugins", Long: `Build a new Answer with plugins that you need`, Run: func(_ *cobra.Command, _ []string) { fmt.Printf("try to build a new answer with plugins:\n%s\n", strings.Join(buildWithPlugins, "\n")) @@ -230,8 +230,8 @@ To run answer, use: pluginCmd = &cobra.Command{ Use: "plugin", - Short: "prints all plugins packed in the binary", - Long: `prints all plugins packed in the binary`, + Short: "Print all plugins packed in the binary", + Long: `Print all plugins packed in the binary`, Run: func(_ *cobra.Command, _ []string) { _ = plugin.CallBase(func(base plugin.Base) error { info := base.Info() @@ -243,8 +243,8 @@ To run answer, use: configCmd = &cobra.Command{ Use: "config", - Short: "set some config to default value", - Long: `set some config to default value`, + Short: "Set some config to default value", + Long: `Set some config to default value`, Run: func(_ *cobra.Command, _ []string) { cli.FormatAllPath(dataDirPath) @@ -279,7 +279,7 @@ To run answer, use: i18nCmd = &cobra.Command{ Use: "i18n", - Short: "overwrite i18n files", + Short: "Overwrite i18n files", Long: `Merge i18n files from plugins to original i18n files. It will overwrite the original i18n files`, Run: func(_ *cobra.Command, _ []string) { if err := cli.ReplaceI18nFilesLocal(i18nTargetPath); err != nil { From 9368da2c04919bd2549fe4c078c52d58a7a74a60 Mon Sep 17 00:00:00 2001 From: shuai Date: Mon, 12 May 2025 16:24:59 +0800 Subject: [PATCH 12/39] fix: code snippet caused the layout to break #1329 --- ui/src/index.scss | 1 + 1 file changed, 1 insertion(+) diff --git a/ui/src/index.scss b/ui/src/index.scss index 52abe4087..279dbf8e8 100644 --- a/ui/src/index.scss +++ b/ui/src/index.scss @@ -251,6 +251,7 @@ img[src=''] { border-radius: 0.25rem; padding: 1rem; max-height: 38rem; + white-space: normal; } blockquote { border-left: 0.25rem solid #ced4da; From 50bed824f6cf6bcc4dc567c5df72113b9b298811 Mon Sep 17 00:00:00 2001 From: shuai Date: Mon, 19 May 2025 10:33:24 +0800 Subject: [PATCH 13/39] feat: Supports separate setting of navigation background color --- i18n/en_US.yaml | 8 +- ui/src/components/Header/index.scss | 2 +- ui/src/components/Header/index.tsx | 12 +- ui/src/pages/Admin/Themes/index.tsx | 24 +- ui/src/stores/themeSetting.ts | 2 +- ui/src/utils/color.ts | 4 + ui/test.html | 463 ---------------------------- 7 files changed, 39 insertions(+), 476 deletions(-) delete mode 100644 ui/test.html diff --git a/i18n/en_US.yaml b/i18n/en_US.yaml index a22396cf9..6bae82b9d 100644 --- a/i18n/en_US.yaml +++ b/i18n/en_US.yaml @@ -1927,9 +1927,9 @@ ui: name: Name email: Email reputation: Reputation - created_at: Created Time - delete_at: Deleted Time - suspend_at: Suspended Time + created_at: Created time + delete_at: Deleted time + suspend_at: Suspended time status: Status role: Role action: Action @@ -2143,7 +2143,7 @@ ui: color_scheme: label: Color scheme navbar_style: - label: Navbar style + label: Navbar background style primary_color: label: Primary color text: Modify the colors used by your themes diff --git a/ui/src/components/Header/index.scss b/ui/src/components/Header/index.scss index 0bc6e2e37..1f362dccc 100644 --- a/ui/src/components/Header/index.scss +++ b/ui/src/components/Header/index.scss @@ -63,7 +63,7 @@ } // style for colored navbar - &.theme-colored { + &.theme-dark { .placeholder-search { padding-left: 42px; box-shadow: none; diff --git a/ui/src/components/Header/index.tsx b/ui/src/components/Header/index.tsx index 52bd09220..1003220e7 100644 --- a/ui/src/components/Header/index.tsx +++ b/ui/src/components/Header/index.tsx @@ -24,7 +24,7 @@ import { Link, NavLink, useLocation, useMatch } from 'react-router-dom'; import classnames from 'classnames'; -import { userCenter, floppyNavigation } from '@/utils'; +import { userCenter, floppyNavigation, isLight } from '@/utils'; import { loggedUserInfoStore, siteInfoStore, @@ -82,9 +82,12 @@ const Header: FC = () => { }, [location.pathname]); let navbarStyle = 'theme-colored'; + let themeMode = 'light'; const { theme, theme_config } = themeSettingStore((_) => _); if (theme_config?.[theme]?.navbar_style) { - navbarStyle = `theme-${theme_config[theme].navbar_style}`; + themeMode = isLight(theme_config[theme].navbar_style) ? 'light' : 'dark'; + console.log('isLightTheme', themeMode); + navbarStyle = `theme-${themeMode}`; } useEffect(() => { @@ -103,9 +106,12 @@ const Header: FC = () => { return (
{ navbar_style: { type: 'string', title: t('navbar_style.label'), - enum: ['colored', 'light'], - enumNames: ['Colored', 'Light'], - default: 'colored', + default: DEFAULT_THEME_COLOR, }, primary_color: { type: 'string', @@ -80,7 +78,19 @@ const Index: FC = () => { 'ui:widget': 'select', }, navbar_style: { - 'ui:widget': 'select', + 'ui:widget': 'input_group', + 'ui:options': { + inputType: 'color', + suffixBtnOptions: { + text: '', + variant: 'outline-secondary', + iconName: 'arrow-counterclockwise', + actionType: 'click', + title: t('reset', { keyPrefix: 'btns' }), + // eslint-disable-next-line @typescript-eslint/no-use-before-define + clickCallback: () => resetNavbarStyle(), + }, + }, }, primary_color: { 'ui:widget': 'input_group', @@ -102,6 +112,12 @@ const Index: FC = () => { const [formData, setFormData] = useState(initFormData(schema)); const { update: updateThemeSetting } = themeSettingStore((_) => _); + const resetNavbarStyle = () => { + const formMeta = { ...formData }; + formMeta.navbar_style.value = DEFAULT_THEME_COLOR; + setFormData({ ...formMeta }); + }; + const resetPrimaryScheme = () => { const formMeta = { ...formData }; formMeta.primary_color.value = DEFAULT_THEME_COLOR; diff --git a/ui/src/stores/themeSetting.ts b/ui/src/stores/themeSetting.ts index c48d9adf4..9f6802b86 100644 --- a/ui/src/stores/themeSetting.ts +++ b/ui/src/stores/themeSetting.ts @@ -36,7 +36,7 @@ const store = create((set) => ({ theme_options: [{ label: 'Default', value: 'default' }], theme_config: { default: { - navbar_style: 'colored', + navbar_style: DEFAULT_THEME_COLOR, primary_color: DEFAULT_THEME_COLOR, }, }, diff --git a/ui/src/utils/color.ts b/ui/src/utils/color.ts index 52cdfc1d1..6aa515648 100644 --- a/ui/src/utils/color.ts +++ b/ui/src/utils/color.ts @@ -60,3 +60,7 @@ export const shiftColor = (color, weight) => { } return tintColor(color, -weight); }; + +export const isLight = (color) => { + return Color(color).isLight(); +}; diff --git a/ui/test.html b/ui/test.html deleted file mode 100644 index 28577a55e..000000000 --- a/ui/test.html +++ /dev/null @@ -1,463 +0,0 @@ - - - - - - - 快速问答 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- -
- - -
- -
-
-
- - -
-
- -
- -
-
-
-
-
-
- ui.question.all_questions -
- -
- - ui.question.newest - - - ui.question.active - - - - ui.question.unanswered - - - ui.question.frequent - -
- -
- - ui.question.score -
-
-
- -
- -
-
- ui.btns.view -
- - ui.btns.card - - - ui.btns.compact - -
-
- -
-
- -
  • -
    -
    - - shuai - shuai - - 1 -
    - • - -
    - -
    - - What is a tag? - -
    - -
    - - When asking a question, - we need to choose tags. What are tags and why should I use them? - -
    - -
    - - - support - - -
    - -
    -
    -
    - - 0 - ui.counts.votes -
    -
    - - 1 - ui.counts.answers -
    - - - 0 - ui.counts.views - -
    -
    -
    • - -
  • -
    -
    - - shuai - shuai - - 1 -
    - • - -
    - -
    - - What is reputation and how do I - earn them? - -
    - -
    - - I see that each user has - reputation points, What is it and how do I earn them? - -
    - -
    - - - support - - -
    - -
    -
    -
    - - 0 - ui.counts.votes -
    -
    - - 1 - ui.counts.answers -
    - - - 0 - ui.counts.views - -
    -
    -
    • - -
    -
    - -
      - - - -
    • - 1 - - -
      • - - - -
    - -
    -
    -
    -
    - -
    -
    ui.question.hot_questions
    -
    - - - - -
    What is a tag?
    - -
    - - ui.question.x_answers -
    - -     - - - - -
    What is reputation and how do I earn them?
    - -
    - - ui.question.x_answers -
    - -     - -
    -
    - -
    -
    -
    - - -
    -
    -
    -
    - - \ No newline at end of file From 4ae8d8ac8c009dd787059ca208cdc481a6032632 Mon Sep 17 00:00:00 2001 From: shuai Date: Mon, 19 May 2025 16:45:17 +0800 Subject: [PATCH 14/39] fix: reset navbar_style default color --- internal/migrations/init.go | 2 +- internal/migrations/v5.go | 2 +- ui/src/components/Header/index.scss | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/migrations/init.go b/internal/migrations/init.go index b2c814e13..610e24d8b 100644 --- a/internal/migrations/init.go +++ b/internal/migrations/init.go @@ -233,7 +233,7 @@ func (m *Mentor) initSiteInfoLegalConfig() { } func (m *Mentor) initSiteInfoThemeConfig() { - themeConfig := `{"theme":"default","theme_config":{"default":{"navbar_style":"colored","primary_color":"#0033ff"}}}` + themeConfig := `{"theme":"default","theme_config":{"default":{"navbar_style":"#0033ff","primary_color":"#0033ff"}}}` _, m.err = m.engine.Context(m.ctx).Insert(&entity.SiteInfo{ Type: "theme", Content: themeConfig, diff --git a/internal/migrations/v5.go b/internal/migrations/v5.go index 9d422d743..91d12f159 100644 --- a/internal/migrations/v5.go +++ b/internal/migrations/v5.go @@ -50,7 +50,7 @@ func addThemeAndPrivateMode(ctx context.Context, x *xorm.Engine) error { } } - themeConfig := `{"theme":"default","theme_config":{"default":{"navbar_style":"colored","primary_color":"#0033ff"}}}` + themeConfig := `{"theme":"default","theme_config":{"default":{"navbar_style":"#0033ff","primary_color":"#0033ff"}}}` themeSiteInfo := &entity.SiteInfo{ Type: "theme", Content: themeConfig, diff --git a/ui/src/components/Header/index.scss b/ui/src/components/Header/index.scss index 1f362dccc..f1df512d4 100644 --- a/ui/src/components/Header/index.scss +++ b/ui/src/components/Header/index.scss @@ -84,7 +84,6 @@ // style for colored navbar &.theme-light { - background: rgb(255, 255, 255); .nav-link { color: rgba(0, 0, 0, 0.65); } @@ -93,7 +92,7 @@ box-shadow: none; color: var(--bs-body-color); background-color: rgba(255, 255, 255, 0.2); - border: $border-width $border-style #dee2e6; + border: $border-width $border-style rgba(0, 0, 0, 0.05); &:focus { border: $border-width $border-style $border-color; } From 95f5de544d9c08cfbe6e90ae38df5269247bc90c Mon Sep 17 00:00:00 2001 From: shuai Date: Mon, 19 May 2025 18:05:56 +0800 Subject: [PATCH 15/39] fix: customize renamed to apperance and move branding below apprance --- i18n/en_US.yaml | 1 + i18n/zh_CN.yaml | 1 + ui/src/common/constants.ts | 7 +++---- ui/src/router/routes.ts | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/i18n/en_US.yaml b/i18n/en_US.yaml index 6bae82b9d..dd9a2d580 100644 --- a/i18n/en_US.yaml +++ b/i18n/en_US.yaml @@ -1792,6 +1792,7 @@ ui: privileges: Privileges plugins: Plugins installed_plugins: Installed Plugins + apperance: Appearance website_welcome: Welcome to {{site_name}} user_center: login: Login diff --git a/i18n/zh_CN.yaml b/i18n/zh_CN.yaml index 64dfcf98d..923f0520d 100644 --- a/i18n/zh_CN.yaml +++ b/i18n/zh_CN.yaml @@ -1753,6 +1753,7 @@ ui: privileges: 特权 plugins: 插件 installed_plugins: 已安装插件 + apperance: 外观 website_welcome: 欢迎来到 {{site_name}} user_center: login: 登录 diff --git a/ui/src/common/constants.ts b/ui/src/common/constants.ts index 855d1a244..163d4989a 100644 --- a/ui/src/common/constants.ts +++ b/ui/src/common/constants.ts @@ -103,16 +103,16 @@ export const ADMIN_NAV_MENUS = [ icon: 'award-fill', }, { - name: 'customize', + name: 'apperance', icon: 'palette-fill', children: [ { name: 'themes', }, { - name: 'css_html', - path: 'css-html', + name: 'customize', }, + { name: 'branding' }, ], }, { @@ -121,7 +121,6 @@ export const ADMIN_NAV_MENUS = [ children: [ { name: 'general' }, { name: 'interface' }, - { name: 'branding' }, { name: 'smtp' }, { name: 'legal' }, { name: 'write' }, diff --git a/ui/src/router/routes.ts b/ui/src/router/routes.ts index 60ca7e91d..d9d20a44a 100644 --- a/ui/src/router/routes.ts +++ b/ui/src/router/routes.ts @@ -351,7 +351,7 @@ const routes: RouteNode[] = [ page: 'pages/Admin/Themes', }, { - path: 'css-html', + path: 'customize', page: 'pages/Admin/CssAndHtml', }, { From 17b88b6a4fa9fc7b5d5b8e355b0ae846e01ca327 Mon Sep 17 00:00:00 2001 From: shuai Date: Tue, 20 May 2025 11:49:05 +0800 Subject: [PATCH 16/39] fix: reanme css/html page title and fix some zh_cn translations --- i18n/en_US.yaml | 1 - i18n/zh_CN.yaml | 10 +++------- ui/src/pages/Admin/CssAndHtml/index.tsx | 2 +- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/i18n/en_US.yaml b/i18n/en_US.yaml index dd9a2d580..10134ffa7 100644 --- a/i18n/en_US.yaml +++ b/i18n/en_US.yaml @@ -1787,7 +1787,6 @@ ui: seo: SEO customize: Customize themes: Themes - css_html: CSS/HTML login: Login privileges: Privileges plugins: Plugins diff --git a/i18n/zh_CN.yaml b/i18n/zh_CN.yaml index 923f0520d..f1a9f67b5 100644 --- a/i18n/zh_CN.yaml +++ b/i18n/zh_CN.yaml @@ -1748,7 +1748,6 @@ ui: seo: SEO customize: 自定义 themes: 主题 - css_html: CSS/HTML login: 登录 privileges: 特权 plugins: 插件 @@ -2112,16 +2111,13 @@ ui: page_title: CSS 与 HTML custom_css: label: 自定义 CSS - text: > - + text: 这将作为 <link> 标签插入 head: label: 头部 - text: > - + text: 这将在 </head> 前插入 header: label: 页眉 - text: > - + text: 这将在 <body> 后插入 footer: label: 页脚 text: 这将在 </body> 之前插入。 diff --git a/ui/src/pages/Admin/CssAndHtml/index.tsx b/ui/src/pages/Admin/CssAndHtml/index.tsx index de376ad71..422659afc 100644 --- a/ui/src/pages/Admin/CssAndHtml/index.tsx +++ b/ui/src/pages/Admin/CssAndHtml/index.tsx @@ -150,7 +150,7 @@ const Index: FC = () => { return ( <> -

    {t('page_title')}

    +

    {t('customize', { keyPrefix: 'nav_menus' })}

    Date: Tue, 20 May 2025 14:30:33 +0800 Subject: [PATCH 17/39] fix: admin gravatar base URL is empty should use placehoder for default value --- ui/src/pages/Admin/SettingsUsers/index.tsx | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ui/src/pages/Admin/SettingsUsers/index.tsx b/ui/src/pages/Admin/SettingsUsers/index.tsx index e2714790e..e09809806 100644 --- a/ui/src/pages/Admin/SettingsUsers/index.tsx +++ b/ui/src/pages/Admin/SettingsUsers/index.tsx @@ -94,6 +94,9 @@ const Index: FC = () => { }, gravatar_base_url: { 'ui:widget': 'input', + 'ui:options': { + placeholder: 'https://www.gravatar.com/avatar/', + }, }, profile_editable: { 'ui:widget': 'legend', @@ -186,7 +189,7 @@ const Index: FC = () => { v = 'system'; } if (k === 'gravatar_base_url' && !v) { - v = 'https://www.gravatar.com/avatar/'; + v = ''; } formMeta[k] = { ...formData[k], value: v }; }); From c7ab8534f164ecef33c4e0b7c06bb6ddc466bc82 Mon Sep 17 00:00:00 2001 From: Sonui Date: Tue, 25 Feb 2025 18:18:26 +0800 Subject: [PATCH 18/39] feat(plugin): add key-value storage support for plugins --- internal/entity/plugin_kv_storage_entity.go | 32 ++ internal/migrations/init_data.go | 1 + internal/migrations/migrations.go | 1 + .../plugin_common/plugin_common_service.go | 5 + plugin/kv_storage.go | 344 ++++++++++++++++++ plugin/plugin.go | 12 + 6 files changed, 395 insertions(+) create mode 100644 internal/entity/plugin_kv_storage_entity.go create mode 100644 plugin/kv_storage.go diff --git a/internal/entity/plugin_kv_storage_entity.go b/internal/entity/plugin_kv_storage_entity.go new file mode 100644 index 000000000..c7e6efbe3 --- /dev/null +++ b/internal/entity/plugin_kv_storage_entity.go @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package entity + +type PluginKVStorage struct { + ID int `xorm:"not null pk autoincr INT(11) id"` + PluginSlugName string `xorm:"not null VARCHAR(128) UNIQUE(uk_psg) plugin_slug_name"` + Group string `xorm:"not null VARCHAR(128) UNIQUE(uk_psg) 'group'"` + Key string `xorm:"not null VARCHAR(128) UNIQUE(uk_psg) 'key'"` + Value string `xorm:"not null TEXT value"` +} + +func (PluginKVStorage) TableName() string { + return "plugin_kv_storage" +} diff --git a/internal/migrations/init_data.go b/internal/migrations/init_data.go index 7322f7388..8b853c4dc 100644 --- a/internal/migrations/init_data.go +++ b/internal/migrations/init_data.go @@ -74,6 +74,7 @@ var ( &entity.Badge{}, &entity.BadgeGroup{}, &entity.BadgeAward{}, + &entity.PluginKVStorage{}, } roles = []*entity.Role{ diff --git a/internal/migrations/migrations.go b/internal/migrations/migrations.go index 57f4778e5..ca7156495 100644 --- a/internal/migrations/migrations.go +++ b/internal/migrations/migrations.go @@ -101,6 +101,7 @@ var migrations = []Migration{ NewMigration("v1.4.1", "add question link", addQuestionLink, true), NewMigration("v1.4.2", "add the number of question links", addQuestionLinkedCount, true), NewMigration("v1.4.5", "add file record", addFileRecord, true), + NewMigration("v1.4.6", "add plugin kv storage", addPluginKVStorage, true), } func GetMigrations() []Migration { diff --git a/internal/service/plugin_common/plugin_common_service.go b/internal/service/plugin_common/plugin_common_service.go index c1b0ad442..de6d981d5 100644 --- a/internal/service/plugin_common/plugin_common_service.go +++ b/internal/service/plugin_common/plugin_common_service.go @@ -135,6 +135,11 @@ func (ps *PluginCommonService) GetUserPluginConfig(ctx context.Context, req *sch } func (ps *PluginCommonService) initPluginData() { + plugin.SetKVStorageDB(&plugin.Data{ + DB: ps.data.DB, + Cache: ps.data.Cache, + }) + // init plugin status pluginStatus, err := ps.configService.GetStringValue(context.TODO(), constant.PluginStatus) if err != nil { diff --git a/plugin/kv_storage.go b/plugin/kv_storage.go new file mode 100644 index 000000000..9714487fe --- /dev/null +++ b/plugin/kv_storage.go @@ -0,0 +1,344 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package plugin + +import ( + "context" + "encoding/json" + "fmt" + "math/rand" + "time" + + "github.com/apache/answer/internal/entity" + "github.com/segmentfault/pacman/log" + "xorm.io/builder" + "xorm.io/xorm" +) + +// define error +var ( + ErrKVKeyNotFound = fmt.Errorf("key not found in KV storage") + ErrKVGroupEmpty = fmt.Errorf("group name is empty") + ErrKVKeyEmpty = fmt.Errorf("key name is empty") + ErrKVKeyAndGroupEmpty = fmt.Errorf("both key and group are empty") + ErrKVTransactionFailed = fmt.Errorf("KV storage transaction failed") + ErrKVDataNotInitialized = fmt.Errorf("KV storage data not initialized") + ErrKVDBNotInitialized = fmt.Errorf("KV storage database connection not initialized") +) + +type KVOperator struct { + data *Data + session *xorm.Session + pluginSlugName string +} + +func (kv *KVOperator) checkDB() error { + if kv.data == nil { + return ErrKVDataNotInitialized + } + if kv.data.DB == nil { + return ErrKVDBNotInitialized + } + return nil +} + +func (kv *KVOperator) getSession(ctx context.Context) (session *xorm.Session, close func()) { + if kv.session != nil { + session = kv.session + } else { + session = kv.data.DB.NewSession().Context(ctx) + close = func() { + if session != nil { + session.Close() + } + } + } + return +} + +func (kv *KVOperator) getCacheTTL() time.Duration { + return 30*time.Minute + time.Duration(rand.Intn(300))*time.Second +} + +func (kv *KVOperator) getCacheKey(group, key string) string { + if group == "" { + return fmt.Sprintf("plugin_kv_storage:%s:key:%s", kv.pluginSlugName, key) + } + if key == "" { + return fmt.Sprintf("plugin_kv_storage:%s:group:%s", kv.pluginSlugName, group) + } + return fmt.Sprintf("plugin_kv_storage:%s:group:%s:key:%s", kv.pluginSlugName, group, key) +} + +func (kv *KVOperator) Get(ctx context.Context, group, key string) (string, error) { + // validate + if err := kv.checkDB(); err != nil { + return "", err + } + if key == "" { + return "", ErrKVKeyEmpty + } + + cacheKey := kv.getCacheKey(group, key) + if value, exist, err := kv.data.Cache.GetString(ctx, cacheKey); err == nil && exist { + return value, nil + } + + // query + data := entity.PluginKVStorage{} + query, close := kv.getSession(ctx) + defer close() + + query.Where(builder.Eq{ + "plugin_slug_name": kv.pluginSlugName, + "`group`": group, + "`key`": key, + }) + + has, err := query.Get(&data) + if err != nil { + return "", err + } + if !has { + return "", ErrKVKeyNotFound + } + + if err := kv.data.Cache.SetString(ctx, cacheKey, data.Value, kv.getCacheTTL()); err != nil { + log.Error(err) + } + + return data.Value, nil +} + +func (kv *KVOperator) Set(ctx context.Context, group, key, value string) error { + if err := kv.checkDB(); err != nil { + return err + } + + if key == "" { + return ErrKVKeyEmpty + } + + query, close := kv.getSession(ctx) + if close != nil { + defer close() + } + + data := &entity.PluginKVStorage{ + PluginSlugName: kv.pluginSlugName, + Group: group, + Key: key, + Value: value, + } + + kv.cleanCache(ctx, group, key) + + affected, err := query.Where(builder.Eq{ + "plugin_slug_name": kv.pluginSlugName, + "`group`": group, + "`key`": key, + }).Cols("value").Update(data) + if err != nil { + return err + } + + if affected == 0 { + _, err = query.Insert(data) + if err != nil { + return err + } + } + return nil +} + +func (kv *KVOperator) Del(ctx context.Context, group, key string) error { + if err := kv.checkDB(); err != nil { + return err + } + + if key == "" && group == "" { + return ErrKVKeyAndGroupEmpty + } + + kv.cleanCache(ctx, group, key) + + session, close := kv.getSession(ctx) + defer close() + + session.Where(builder.Eq{ + "plugin_slug_name": kv.pluginSlugName, + }) + if group != "" { + session.Where(builder.Eq{"`group`": group}) + } + if key != "" { + session.Where(builder.Eq{"`key`": key}) + } + + _, err := session.Delete(&entity.PluginKVStorage{}) + return err +} + +func (kv *KVOperator) cleanCache(ctx context.Context, group, key string) { + if key != "" { + if err := kv.data.Cache.Del(ctx, kv.getCacheKey("", key)); err != nil { + log.Warnf("Failed to delete cache for key %s: %v", key, err) + } + + if group != "" { + if err := kv.data.Cache.Del(ctx, kv.getCacheKey(group, key)); err != nil { + log.Warnf("Failed to delete cache for group %s, key %s: %v", group, key, err) + } + } + } + + if group != "" { + if err := kv.data.Cache.Del(ctx, kv.getCacheKey(group, "")); err != nil { + log.Warnf("Failed to delete cache for group %s: %v", group, err) + } + } +} + +func (kv *KVOperator) GetByGroup(ctx context.Context, group string, page, pageSize int) (map[string]string, error) { + if err := kv.checkDB(); err != nil { + return nil, err + } + + if group == "" { + return nil, ErrKVGroupEmpty + } + + if page < 1 { + page = 1 + } + if pageSize < 1 { + pageSize = 10 + } + + if pageSize > 100 { + pageSize = 100 + } + + cacheKey := kv.getCacheKey(group, "") + if value, exist, err := kv.data.Cache.GetString(ctx, cacheKey); err == nil && exist { + result := make(map[string]string) + if err := json.Unmarshal([]byte(value), &result); err == nil { + return result, nil + } + } + + query, close := kv.getSession(ctx) + defer close() + + var items []entity.PluginKVStorage + err := query.Where(builder.Eq{"plugin_slug_name": kv.pluginSlugName, "`group`": group}). + Limit(pageSize, (page-1)*pageSize). + OrderBy("id ASC"). + Find(&items) + if err != nil { + return nil, err + } + + result := make(map[string]string, len(items)) + for _, item := range items { + result[item.Key] = item.Value + if err := kv.data.Cache.SetString(ctx, kv.getCacheKey(group, item.Key), item.Value, kv.getCacheTTL()); err != nil { + log.Warnf("Failed to set cache for group %s, key %s: %v", group, item.Key, err) + } + } + + if resultJSON, err := json.Marshal(result); err == nil { + _ = kv.data.Cache.SetString(ctx, cacheKey, string(resultJSON), kv.getCacheTTL()) + } + + return result, nil +} + +func (kv *KVOperator) Tx(ctx context.Context, fn func(ctx context.Context, kv *KVOperator) error) error { + if err := kv.checkDB(); err != nil { + return fmt.Errorf("%w: %v", ErrKVTransactionFailed, err) + } + + var ( + txKv = kv + shouldCommit bool + ) + + if kv.session == nil { + session := kv.data.DB.NewSession().Context(ctx) + if err := session.Begin(); err != nil { + session.Close() + return fmt.Errorf("%w: begin transaction failed: %v", ErrKVTransactionFailed, err) + } + + defer func() { + if !shouldCommit { + if rollbackErr := session.Rollback(); rollbackErr != nil { + log.Errorf("rollback failed: %v", rollbackErr) + } + } + session.Close() + }() + + txKv = &KVOperator{ + session: session, + data: kv.data, + pluginSlugName: kv.pluginSlugName, + } + shouldCommit = true + } + + if err := fn(ctx, txKv); err != nil { + return fmt.Errorf("%w: %v", ErrKVTransactionFailed, err) + } + + if shouldCommit { + if err := txKv.session.Commit(); err != nil { + return fmt.Errorf("%w: commit failed: %v", ErrKVTransactionFailed, err) + } + } + return nil +} + +// PluginData defines the interface for plugins that need data storage capabilities +type KVStorage interface { + Info() Info + SetOperator(operator *KVOperator) +} + +var ( + _, + registerPluginKVStorage = func() (CallFn[KVStorage], RegisterFn[KVStorage]) { + callFn, registerFn := MakePlugin[KVStorage](false) + return callFn, func(p KVStorage) { + registerFn(p) + kvStoragePluginStack.plugins = append(kvStoragePluginStack.plugins, p) + } + }() + kvStoragePluginStack = &Stack[KVStorage]{} +) + +func SetKVStorageDB(data *Data) { + for _, p := range kvStoragePluginStack.plugins { + p.SetOperator(&KVOperator{ + data: data, + pluginSlugName: p.Info().SlugName, + }) + } +} diff --git a/plugin/plugin.go b/plugin/plugin.go index 36087c547..dc8c35ea1 100644 --- a/plugin/plugin.go +++ b/plugin/plugin.go @@ -23,13 +23,21 @@ import ( "encoding/json" "sync" + "github.com/segmentfault/pacman/cache" "github.com/segmentfault/pacman/i18n" + "xorm.io/xorm" "github.com/apache/answer/internal/base/handler" "github.com/apache/answer/internal/base/translator" "github.com/gin-gonic/gin" ) +// Data is defined here to avoid circular dependency with internal/base/data +type Data struct { + DB *xorm.Engine + Cache cache.Cache +} + // GinContext is a wrapper of gin.Context // We export it to make it easy to use in plugins type GinContext = gin.Context @@ -114,6 +122,10 @@ func Register(p Base) { if _, ok := p.(Importer); ok { registerImporter(p.(Importer)) } + + if _, ok := p.(KVStorage); ok { + registerPluginKVStorage(p.(KVStorage)) + } } type Stack[T Base] struct { From 8579bde60760a8a2bbaf7466fc6e6f8c6c332e7c Mon Sep 17 00:00:00 2001 From: Sonui Date: Sat, 1 Mar 2025 11:22:40 +0800 Subject: [PATCH 19/39] refactor(plugin): improved initialization of the KV storage plugin --- .../plugin_common/plugin_common_service.go | 10 +++++--- plugin/kv_storage.go | 24 +++++++------------ plugin/plugin.go | 2 +- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/internal/service/plugin_common/plugin_common_service.go b/internal/service/plugin_common/plugin_common_service.go index de6d981d5..d3aa839b2 100644 --- a/internal/service/plugin_common/plugin_common_service.go +++ b/internal/service/plugin_common/plugin_common_service.go @@ -135,9 +135,13 @@ func (ps *PluginCommonService) GetUserPluginConfig(ctx context.Context, req *sch } func (ps *PluginCommonService) initPluginData() { - plugin.SetKVStorageDB(&plugin.Data{ - DB: ps.data.DB, - Cache: ps.data.Cache, + _ = plugin.CallKVStorage(func(k plugin.KVStorage) error { + k.SetOperator(plugin.NewKVOperator( + ps.data.DB, + ps.data.Cache, + k.Info().SlugName, + )) + return nil }) // init plugin status diff --git a/plugin/kv_storage.go b/plugin/kv_storage.go index 9714487fe..b4d940b80 100644 --- a/plugin/kv_storage.go +++ b/plugin/kv_storage.go @@ -323,22 +323,16 @@ type KVStorage interface { } var ( - _, - registerPluginKVStorage = func() (CallFn[KVStorage], RegisterFn[KVStorage]) { - callFn, registerFn := MakePlugin[KVStorage](false) - return callFn, func(p KVStorage) { - registerFn(p) - kvStoragePluginStack.plugins = append(kvStoragePluginStack.plugins, p) - } - }() - kvStoragePluginStack = &Stack[KVStorage]{} + CallKVStorage, + registerKVStorage = MakePlugin[KVStorage](true) ) -func SetKVStorageDB(data *Data) { - for _, p := range kvStoragePluginStack.plugins { - p.SetOperator(&KVOperator{ - data: data, - pluginSlugName: p.Info().SlugName, - }) +// NewKVOperator creates a new KV storage operator with the specified database engine, cache and plugin name. +// It returns a KVOperator instance that can be used to interact with the plugin's storage. +func NewKVOperator(db *xorm.Engine, cache cache.Cache, pluginSlugName string) *KVOperator { + return &KVOperator{ + data: &Data{DB: db, Cache: cache}, + pluginSlugName: pluginSlugName, + cacheTTL: 30 * time.Minute, } } diff --git a/plugin/plugin.go b/plugin/plugin.go index dc8c35ea1..a9e173100 100644 --- a/plugin/plugin.go +++ b/plugin/plugin.go @@ -124,7 +124,7 @@ func Register(p Base) { } if _, ok := p.(KVStorage); ok { - registerPluginKVStorage(p.(KVStorage)) + registerKVStorage(p.(KVStorage)) } } From d12ac86d6605c2525c0cb0b08591b66c0d8a6e82 Mon Sep 17 00:00:00 2001 From: Sonui Date: Sat, 1 Mar 2025 11:56:32 +0800 Subject: [PATCH 20/39] perf(plugin): remove unnecessary KV storage checks --- plugin/kv_storage.go | 33 --------------------------------- 1 file changed, 33 deletions(-) diff --git a/plugin/kv_storage.go b/plugin/kv_storage.go index b4d940b80..a3ab46d03 100644 --- a/plugin/kv_storage.go +++ b/plugin/kv_storage.go @@ -48,16 +48,6 @@ type KVOperator struct { pluginSlugName string } -func (kv *KVOperator) checkDB() error { - if kv.data == nil { - return ErrKVDataNotInitialized - } - if kv.data.DB == nil { - return ErrKVDBNotInitialized - } - return nil -} - func (kv *KVOperator) getSession(ctx context.Context) (session *xorm.Session, close func()) { if kv.session != nil { session = kv.session @@ -88,9 +78,6 @@ func (kv *KVOperator) getCacheKey(group, key string) string { func (kv *KVOperator) Get(ctx context.Context, group, key string) (string, error) { // validate - if err := kv.checkDB(); err != nil { - return "", err - } if key == "" { return "", ErrKVKeyEmpty } @@ -127,10 +114,6 @@ func (kv *KVOperator) Get(ctx context.Context, group, key string) (string, error } func (kv *KVOperator) Set(ctx context.Context, group, key, value string) error { - if err := kv.checkDB(); err != nil { - return err - } - if key == "" { return ErrKVKeyEmpty } @@ -168,10 +151,6 @@ func (kv *KVOperator) Set(ctx context.Context, group, key, value string) error { } func (kv *KVOperator) Del(ctx context.Context, group, key string) error { - if err := kv.checkDB(); err != nil { - return err - } - if key == "" && group == "" { return ErrKVKeyAndGroupEmpty } @@ -216,10 +195,6 @@ func (kv *KVOperator) cleanCache(ctx context.Context, group, key string) { } func (kv *KVOperator) GetByGroup(ctx context.Context, group string, page, pageSize int) (map[string]string, error) { - if err := kv.checkDB(); err != nil { - return nil, err - } - if group == "" { return nil, ErrKVGroupEmpty } @@ -231,10 +206,6 @@ func (kv *KVOperator) GetByGroup(ctx context.Context, group string, page, pageSi pageSize = 10 } - if pageSize > 100 { - pageSize = 100 - } - cacheKey := kv.getCacheKey(group, "") if value, exist, err := kv.data.Cache.GetString(ctx, cacheKey); err == nil && exist { result := make(map[string]string) @@ -271,10 +242,6 @@ func (kv *KVOperator) GetByGroup(ctx context.Context, group string, page, pageSi } func (kv *KVOperator) Tx(ctx context.Context, fn func(ctx context.Context, kv *KVOperator) error) error { - if err := kv.checkDB(); err != nil { - return fmt.Errorf("%w: %v", ErrKVTransactionFailed, err) - } - var ( txKv = kv shouldCommit bool From d90810afd7e01e66358341014b2b787e1e1b71d2 Mon Sep 17 00:00:00 2001 From: Sonui Date: Sat, 1 Mar 2025 13:55:36 +0800 Subject: [PATCH 21/39] fix(plugin): rename 'close' to avoid builtin collision --- plugin/kv_storage.go | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/plugin/kv_storage.go b/plugin/kv_storage.go index a3ab46d03..0ec3d6409 100644 --- a/plugin/kv_storage.go +++ b/plugin/kv_storage.go @@ -48,18 +48,18 @@ type KVOperator struct { pluginSlugName string } -func (kv *KVOperator) getSession(ctx context.Context) (session *xorm.Session, close func()) { - if kv.session != nil { - session = kv.session - } else { +func (kv *KVOperator) getSession(ctx context.Context) (*xorm.Session, func()) { + session := kv.session + cleanup := func() {} + if session == nil { session = kv.data.DB.NewSession().Context(ctx) - close = func() { + cleanup = func() { if session != nil { session.Close() } } } - return + return session, cleanup } func (kv *KVOperator) getCacheTTL() time.Duration { @@ -89,8 +89,8 @@ func (kv *KVOperator) Get(ctx context.Context, group, key string) (string, error // query data := entity.PluginKVStorage{} - query, close := kv.getSession(ctx) - defer close() + query, cleanup := kv.getSession(ctx) + defer cleanup() query.Where(builder.Eq{ "plugin_slug_name": kv.pluginSlugName, @@ -118,10 +118,8 @@ func (kv *KVOperator) Set(ctx context.Context, group, key, value string) error { return ErrKVKeyEmpty } - query, close := kv.getSession(ctx) - if close != nil { - defer close() - } + query, cleanup := kv.getSession(ctx) + defer cleanup() data := &entity.PluginKVStorage{ PluginSlugName: kv.pluginSlugName, @@ -157,8 +155,8 @@ func (kv *KVOperator) Del(ctx context.Context, group, key string) error { kv.cleanCache(ctx, group, key) - session, close := kv.getSession(ctx) - defer close() + session, cleanup := kv.getSession(ctx) + defer cleanup() session.Where(builder.Eq{ "plugin_slug_name": kv.pluginSlugName, @@ -214,8 +212,8 @@ func (kv *KVOperator) GetByGroup(ctx context.Context, group string, page, pageSi } } - query, close := kv.getSession(ctx) - defer close() + query, cleanup := kv.getSession(ctx) + defer cleanup() var items []entity.PluginKVStorage err := query.Where(builder.Eq{"plugin_slug_name": kv.pluginSlugName, "`group`": group}). From 9ea153b4026ab8dccc34c71e7c513c4cd1bda030 Mon Sep 17 00:00:00 2001 From: Sonui Date: Mon, 10 Mar 2025 00:28:57 +0800 Subject: [PATCH 22/39] refactor(plugin): improve KV storage with better caching and param handling --- plugin/kv_storage.go | 198 ++++++++++++++++++++++++------------------- 1 file changed, 112 insertions(+), 86 deletions(-) diff --git a/plugin/kv_storage.go b/plugin/kv_storage.go index 0ec3d6409..17617f6c9 100644 --- a/plugin/kv_storage.go +++ b/plugin/kv_storage.go @@ -20,32 +20,64 @@ package plugin import ( "context" - "encoding/json" "fmt" - "math/rand" + "math/rand/v2" "time" "github.com/apache/answer/internal/entity" + "github.com/segmentfault/pacman/cache" "github.com/segmentfault/pacman/log" "xorm.io/builder" "xorm.io/xorm" ) -// define error +// Error variables for KV storage operations var ( - ErrKVKeyNotFound = fmt.Errorf("key not found in KV storage") - ErrKVGroupEmpty = fmt.Errorf("group name is empty") - ErrKVKeyEmpty = fmt.Errorf("key name is empty") - ErrKVKeyAndGroupEmpty = fmt.Errorf("both key and group are empty") - ErrKVTransactionFailed = fmt.Errorf("KV storage transaction failed") - ErrKVDataNotInitialized = fmt.Errorf("KV storage data not initialized") - ErrKVDBNotInitialized = fmt.Errorf("KV storage database connection not initialized") + // ErrKVKeyNotFound is returned when the requested key does not exist in the KV storage + ErrKVKeyNotFound = fmt.Errorf("key not found in KV storage") + // ErrKVGroupEmpty is returned when a required group name is empty + ErrKVGroupEmpty = fmt.Errorf("group name is empty") + // ErrKVKeyEmpty is returned when a required key name is empty + ErrKVKeyEmpty = fmt.Errorf("key name is empty") + // ErrKVKeyAndGroupEmpty is returned when both key and group names are empty + ErrKVKeyAndGroupEmpty = fmt.Errorf("both key and group are empty") + // ErrKVTransactionFailed is returned when a KV storage transaction operation fails + ErrKVTransactionFailed = fmt.Errorf("KV storage transaction failed") ) +// KVParams is the parameters for KV storage operations +type KVParams struct { + Group string + Key string + Value string + Page int + PageSize int +} + +// KVOperator provides methods to interact with the key-value storage system for plugins type KVOperator struct { data *Data session *xorm.Session pluginSlugName string + cacheTTL time.Duration +} + +// KVStorageOption defines a function type that configures a KVOperator +type KVStorageOption func(*KVOperator) + +// WithCacheTTL is the option to set the cache TTL; the default value is 30 minutes. +// If ttl is less than 0, the cache will not be used +func WithCacheTTL(ttl time.Duration) KVStorageOption { + return func(kv *KVOperator) { + kv.cacheTTL = ttl + } +} + +// Option is used to set the options for the KV storage +func (kv *KVOperator) Option(opts ...KVStorageOption) { + for _, opt := range opts { + opt(kv) + } } func (kv *KVOperator) getSession(ctx context.Context) (*xorm.Session, func()) { @@ -62,28 +94,53 @@ func (kv *KVOperator) getSession(ctx context.Context) (*xorm.Session, func()) { return session, cleanup } -func (kv *KVOperator) getCacheTTL() time.Duration { - return 30*time.Minute + time.Duration(rand.Intn(300))*time.Second +func (kv *KVOperator) getCacheKey(params KVParams) string { + return fmt.Sprintf("plugin_kv_storage:%s:group:%s:key:%s", kv.pluginSlugName, params.Group, params.Key) +} + +func (kv *KVOperator) setCache(ctx context.Context, params KVParams) { + if kv.cacheTTL < 0 { + return + } + + ttl := kv.cacheTTL + if ttl > 10 { + ttl += time.Duration(float64(ttl) * 0.1 * (1 - rand.Float64())) + } + + cacheKey := kv.getCacheKey(params) + if err := kv.data.Cache.SetString(ctx, cacheKey, params.Value, ttl); err != nil { + log.Warnf("cache set failed: %v, key: %s", err, cacheKey) + } +} + +func (kv *KVOperator) getCache(ctx context.Context, params KVParams) (string, bool, error) { + if kv.cacheTTL < 0 { + return "", false, nil + } + + cacheKey := kv.getCacheKey(params) + return kv.data.Cache.GetString(ctx, cacheKey) } -func (kv *KVOperator) getCacheKey(group, key string) string { - if group == "" { - return fmt.Sprintf("plugin_kv_storage:%s:key:%s", kv.pluginSlugName, key) +func (kv *KVOperator) cleanCache(ctx context.Context, params KVParams) { + if kv.cacheTTL < 0 { + return } - if key == "" { - return fmt.Sprintf("plugin_kv_storage:%s:group:%s", kv.pluginSlugName, group) + + if err := kv.data.Cache.Del(ctx, kv.getCacheKey(params)); err != nil { + log.Warnf("Failed to delete cache for key %s: %v", params.Key, err) } - return fmt.Sprintf("plugin_kv_storage:%s:group:%s:key:%s", kv.pluginSlugName, group, key) } -func (kv *KVOperator) Get(ctx context.Context, group, key string) (string, error) { - // validate - if key == "" { +// Get retrieves a value from KV storage by group and key. +// Returns the value as a string or an error if the key is not found. +func (kv *KVOperator) Get(ctx context.Context, params KVParams) (string, error) { + if params.Key == "" { return "", ErrKVKeyEmpty } - cacheKey := kv.getCacheKey(group, key) - if value, exist, err := kv.data.Cache.GetString(ctx, cacheKey); err == nil && exist { + if value, exist, err := kv.getCache(ctx, params); err == nil && exist { return value, nil } @@ -94,8 +151,8 @@ func (kv *KVOperator) Get(ctx context.Context, group, key string) (string, error query.Where(builder.Eq{ "plugin_slug_name": kv.pluginSlugName, - "`group`": group, - "`key`": key, + "`group`": params.Group, + "`key`": params.Key, }) has, err := query.Get(&data) @@ -106,15 +163,15 @@ func (kv *KVOperator) Get(ctx context.Context, group, key string) (string, error return "", ErrKVKeyNotFound } - if err := kv.data.Cache.SetString(ctx, cacheKey, data.Value, kv.getCacheTTL()); err != nil { - log.Error(err) - } + kv.setCache(ctx, params) return data.Value, nil } -func (kv *KVOperator) Set(ctx context.Context, group, key, value string) error { - if key == "" { +// Set stores a value in KV storage with the specified group and key. +// Updates the value if it already exists. +func (kv *KVOperator) Set(ctx context.Context, params KVParams) error { + if params.Key == "" { return ErrKVKeyEmpty } @@ -123,17 +180,17 @@ func (kv *KVOperator) Set(ctx context.Context, group, key, value string) error { data := &entity.PluginKVStorage{ PluginSlugName: kv.pluginSlugName, - Group: group, - Key: key, - Value: value, + Group: params.Group, + Key: params.Key, + Value: params.Value, } - kv.cleanCache(ctx, group, key) + kv.cleanCache(ctx, params) affected, err := query.Where(builder.Eq{ "plugin_slug_name": kv.pluginSlugName, - "`group`": group, - "`key`": key, + "`group`": params.Group, + "`key`": params.Key, }).Cols("value").Update(data) if err != nil { return err @@ -148,12 +205,16 @@ func (kv *KVOperator) Set(ctx context.Context, group, key, value string) error { return nil } -func (kv *KVOperator) Del(ctx context.Context, group, key string) error { - if key == "" && group == "" { +// Del removes values from KV storage by group and/or key. +// If both group and key are provided, only that specific entry is deleted. +// If only group is provided, all entries in that group are deleted. +// At least one of group or key must be provided. +func (kv *KVOperator) Del(ctx context.Context, params KVParams) error { + if params.Key == "" && params.Group == "" { return ErrKVKeyAndGroupEmpty } - kv.cleanCache(ctx, group, key) + kv.cleanCache(ctx, params) session, cleanup := kv.getSession(ctx) defer cleanup() @@ -161,63 +222,35 @@ func (kv *KVOperator) Del(ctx context.Context, group, key string) error { session.Where(builder.Eq{ "plugin_slug_name": kv.pluginSlugName, }) - if group != "" { - session.Where(builder.Eq{"`group`": group}) + if params.Group != "" { + session.Where(builder.Eq{"`group`": params.Group}) } - if key != "" { - session.Where(builder.Eq{"`key`": key}) + if params.Key != "" { + session.Where(builder.Eq{"`key`": params.Key}) } _, err := session.Delete(&entity.PluginKVStorage{}) return err } -func (kv *KVOperator) cleanCache(ctx context.Context, group, key string) { - if key != "" { - if err := kv.data.Cache.Del(ctx, kv.getCacheKey("", key)); err != nil { - log.Warnf("Failed to delete cache for key %s: %v", key, err) - } - - if group != "" { - if err := kv.data.Cache.Del(ctx, kv.getCacheKey(group, key)); err != nil { - log.Warnf("Failed to delete cache for group %s, key %s: %v", group, key, err) - } - } - } - - if group != "" { - if err := kv.data.Cache.Del(ctx, kv.getCacheKey(group, "")); err != nil { - log.Warnf("Failed to delete cache for group %s: %v", group, err) - } - } -} - -func (kv *KVOperator) GetByGroup(ctx context.Context, group string, page, pageSize int) (map[string]string, error) { - if group == "" { +func (kv *KVOperator) GetByGroup(ctx context.Context, params KVParams) (map[string]string, error) { + if params.Group == "" { return nil, ErrKVGroupEmpty } - if page < 1 { - page = 1 + if params.Page < 1 { + params.Page = 1 } - if pageSize < 1 { - pageSize = 10 - } - - cacheKey := kv.getCacheKey(group, "") - if value, exist, err := kv.data.Cache.GetString(ctx, cacheKey); err == nil && exist { - result := make(map[string]string) - if err := json.Unmarshal([]byte(value), &result); err == nil { - return result, nil - } + if params.PageSize < 1 { + params.PageSize = 10 } query, cleanup := kv.getSession(ctx) defer cleanup() var items []entity.PluginKVStorage - err := query.Where(builder.Eq{"plugin_slug_name": kv.pluginSlugName, "`group`": group}). - Limit(pageSize, (page-1)*pageSize). + err := query.Where(builder.Eq{"plugin_slug_name": kv.pluginSlugName, "`group`": params.Group}). + Limit(params.PageSize, (params.Page-1)*params.PageSize). OrderBy("id ASC"). Find(&items) if err != nil { @@ -227,13 +260,6 @@ func (kv *KVOperator) GetByGroup(ctx context.Context, group string, page, pageSi result := make(map[string]string, len(items)) for _, item := range items { result[item.Key] = item.Value - if err := kv.data.Cache.SetString(ctx, kv.getCacheKey(group, item.Key), item.Value, kv.getCacheTTL()); err != nil { - log.Warnf("Failed to set cache for group %s, key %s: %v", group, item.Key, err) - } - } - - if resultJSON, err := json.Marshal(result); err == nil { - _ = kv.data.Cache.SetString(ctx, cacheKey, string(resultJSON), kv.getCacheTTL()) } return result, nil From c7646bd60021f64560ac37415b0c8ea206aedfb8 Mon Sep 17 00:00:00 2001 From: Sonui Date: Thu, 3 Apr 2025 14:01:11 +0800 Subject: [PATCH 23/39] docs(plugin): Add comments to KVOperator methods to improve code readability --- plugin/kv_storage.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/plugin/kv_storage.go b/plugin/kv_storage.go index 17617f6c9..9b449f8ff 100644 --- a/plugin/kv_storage.go +++ b/plugin/kv_storage.go @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ + package plugin import ( @@ -233,6 +234,8 @@ func (kv *KVOperator) Del(ctx context.Context, params KVParams) error { return err } +// GetByGroup retrieves all key-value pairs for a specific group with pagination support. +// Returns a map of keys to values or an error if the group is empty or not found. func (kv *KVOperator) GetByGroup(ctx context.Context, params KVParams) (map[string]string, error) { if params.Group == "" { return nil, ErrKVGroupEmpty @@ -265,6 +268,9 @@ func (kv *KVOperator) GetByGroup(ctx context.Context, params KVParams) (map[stri return result, nil } +// Tx executes a function within a transaction context. If the KVOperator already has a session, +// it will use that session. Otherwise, it creates a new transaction session. +// The transaction will be committed if the function returns nil, or rolled back if it returns an error. func (kv *KVOperator) Tx(ctx context.Context, fn func(ctx context.Context, kv *KVOperator) error) error { var ( txKv = kv @@ -307,7 +313,7 @@ func (kv *KVOperator) Tx(ctx context.Context, fn func(ctx context.Context, kv *K return nil } -// PluginData defines the interface for plugins that need data storage capabilities +// KVStorage defines the interface for plugins that need data storage capabilities type KVStorage interface { Info() Info SetOperator(operator *KVOperator) From aa0493e4f0b94d8c58bfd1054f1725b2f738b598 Mon Sep 17 00:00:00 2001 From: Sonui Date: Thu, 3 Apr 2025 14:01:53 +0800 Subject: [PATCH 24/39] feat(migrations): Add v26 migration file to support plugin KV storage --- internal/migrations/v26.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 internal/migrations/v26.go diff --git a/internal/migrations/v26.go b/internal/migrations/v26.go new file mode 100644 index 000000000..008a094a4 --- /dev/null +++ b/internal/migrations/v26.go @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package migrations + +import ( + "context" + + "github.com/apache/answer/internal/entity" + "xorm.io/xorm" +) + +func addPluginKVStorage(ctx context.Context, x *xorm.Engine) error { + return x.Context(ctx).Sync(new(entity.PluginKVStorage)) +} From ed543e8825195d72d4d5eab1d12ec63d6d588de3 Mon Sep 17 00:00:00 2001 From: Sonui Date: Wed, 7 May 2025 23:46:39 +0800 Subject: [PATCH 25/39] fix(plugin): set params.Value in Get method to ensure correct value retrieval --- plugin/kv_storage.go | 1 + 1 file changed, 1 insertion(+) diff --git a/plugin/kv_storage.go b/plugin/kv_storage.go index 9b449f8ff..d1ed3eaa6 100644 --- a/plugin/kv_storage.go +++ b/plugin/kv_storage.go @@ -164,6 +164,7 @@ func (kv *KVOperator) Get(ctx context.Context, params KVParams) (string, error) return "", ErrKVKeyNotFound } + params.Value = data.Value kv.setCache(ctx, params) return data.Value, nil From 1baa3f0fb6848a5a0ff3b2d2f0b571acb799f060 Mon Sep 17 00:00:00 2001 From: Sonui Date: Wed, 7 May 2025 23:49:56 +0800 Subject: [PATCH 26/39] chore(migrations): update migration version --- internal/migrations/migrations.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/migrations/migrations.go b/internal/migrations/migrations.go index ca7156495..212bf14bb 100644 --- a/internal/migrations/migrations.go +++ b/internal/migrations/migrations.go @@ -101,7 +101,7 @@ var migrations = []Migration{ NewMigration("v1.4.1", "add question link", addQuestionLink, true), NewMigration("v1.4.2", "add the number of question links", addQuestionLinkedCount, true), NewMigration("v1.4.5", "add file record", addFileRecord, true), - NewMigration("v1.4.6", "add plugin kv storage", addPluginKVStorage, true), + NewMigration("v1.5.1", "add plugin kv storage", addPluginKVStorage, true), } func GetMigrations() []Migration { From 54f7682e1a2acf4ceeced8707297d5cbfe1edc09 Mon Sep 17 00:00:00 2001 From: Shaobiao Lin Date: Tue, 6 May 2025 19:06:56 +0800 Subject: [PATCH 27/39] docs(command): remove redundant comments The codes are self-documenting enough. --- cmd/command.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/cmd/command.go b/cmd/command.go index fe17713e7..602a12299 100644 --- a/cmd/command.go +++ b/cmd/command.go @@ -82,7 +82,6 @@ func init() { } var ( - // rootCmd represents the base command when called without any subcommands rootCmd = &cobra.Command{ Use: "answer", Short: "Answer is a minimalist open source Q&A community.", @@ -92,7 +91,6 @@ To run answer, use: - 'answer run' to launch application.`, } - // runCmd represents the run command runCmd = &cobra.Command{ Use: "run", Short: "Run the application", @@ -105,7 +103,6 @@ To run answer, use: }, } - // initCmd represents the init command initCmd = &cobra.Command{ Use: "init", Short: "init answer application", @@ -135,7 +132,6 @@ To run answer, use: }, } - // upgradeCmd represents the upgrade command upgradeCmd = &cobra.Command{ Use: "upgrade", Short: "upgrade Answer version", @@ -157,7 +153,6 @@ To run answer, use: }, } - // dumpCmd represents the dump command dumpCmd = &cobra.Command{ Use: "dump", Short: "back up data", @@ -179,7 +174,6 @@ To run answer, use: }, } - // checkCmd represents the check command checkCmd = &cobra.Command{ Use: "check", Short: "checking the required environment", @@ -214,7 +208,6 @@ To run answer, use: }, } - // buildCmd used to build another answer with plugins buildCmd = &cobra.Command{ Use: "build", Short: "used to build answer with plugins", @@ -235,7 +228,6 @@ To run answer, use: }, } - // pluginCmd prints all plugins packed in the binary pluginCmd = &cobra.Command{ Use: "plugin", Short: "prints all plugins packed in the binary", @@ -249,7 +241,6 @@ To run answer, use: }, } - // configCmd set some config to default value configCmd = &cobra.Command{ Use: "config", Short: "set some config to default value", @@ -286,7 +277,6 @@ To run answer, use: }, } - // i18nCmd used to merge i18n files i18nCmd = &cobra.Command{ Use: "i18n", Short: "overwrite i18n files", From c6189e50203e25086a80cab3a19f8751e4da77ee Mon Sep 17 00:00:00 2001 From: Shaobiao Lin Date: Tue, 6 May 2025 19:59:15 +0800 Subject: [PATCH 28/39] docs(command): unify style of usage statements close #1330 --- cmd/command.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/cmd/command.go b/cmd/command.go index 602a12299..7d779893f 100644 --- a/cmd/command.go +++ b/cmd/command.go @@ -93,8 +93,8 @@ To run answer, use: runCmd = &cobra.Command{ Use: "run", - Short: "Run the application", - Long: `Run the application`, + Short: "Run Answer", + Long: `Start running Answer`, Run: func(_ *cobra.Command, _ []string) { cli.FormatAllPath(dataDirPath) fmt.Println("config file path: ", cli.GetConfigFilePath()) @@ -105,8 +105,8 @@ To run answer, use: initCmd = &cobra.Command{ Use: "init", - Short: "init answer application", - Long: `init answer application`, + Short: "Initialize Answer", + Long: `Initialize Answer with specified configuration`, Run: func(_ *cobra.Command, _ []string) { // check config file and database. if config file exists and database is already created, init done cli.InstallAllInitialEnvironment(dataDirPath) @@ -134,8 +134,8 @@ To run answer, use: upgradeCmd = &cobra.Command{ Use: "upgrade", - Short: "upgrade Answer version", - Long: `upgrade Answer version`, + Short: "Upgrade Answer", + Long: `Upgrade Answer to the latest version`, Run: func(_ *cobra.Command, _ []string) { log.SetLogger(log.NewStdLogger(os.Stdout)) cli.FormatAllPath(dataDirPath) @@ -155,8 +155,8 @@ To run answer, use: dumpCmd = &cobra.Command{ Use: "dump", - Short: "back up data", - Long: `back up data`, + Short: "Back up data", + Long: `Back up database into an SQL file`, Run: func(_ *cobra.Command, _ []string) { fmt.Println("Answer is backing up data") cli.FormatAllPath(dataDirPath) @@ -176,7 +176,7 @@ To run answer, use: checkCmd = &cobra.Command{ Use: "check", - Short: "checking the required environment", + Short: "Check the required environment", Long: `Check if the current environment meets the startup requirements`, Run: func(_ *cobra.Command, _ []string) { cli.FormatAllPath(dataDirPath) @@ -210,7 +210,7 @@ To run answer, use: buildCmd = &cobra.Command{ Use: "build", - Short: "used to build answer with plugins", + Short: "Build Answer with plugins", Long: `Build a new Answer with plugins that you need`, Run: func(_ *cobra.Command, _ []string) { fmt.Printf("try to build a new answer with plugins:\n%s\n", strings.Join(buildWithPlugins, "\n")) @@ -230,8 +230,8 @@ To run answer, use: pluginCmd = &cobra.Command{ Use: "plugin", - Short: "prints all plugins packed in the binary", - Long: `prints all plugins packed in the binary`, + Short: "Print all plugins packed in the binary", + Long: `Print all plugins packed in the binary`, Run: func(_ *cobra.Command, _ []string) { _ = plugin.CallBase(func(base plugin.Base) error { info := base.Info() @@ -243,8 +243,8 @@ To run answer, use: configCmd = &cobra.Command{ Use: "config", - Short: "set some config to default value", - Long: `set some config to default value`, + Short: "Set some config to default value", + Long: `Set some config to default value`, Run: func(_ *cobra.Command, _ []string) { cli.FormatAllPath(dataDirPath) @@ -279,7 +279,7 @@ To run answer, use: i18nCmd = &cobra.Command{ Use: "i18n", - Short: "overwrite i18n files", + Short: "Overwrite i18n files", Long: `Merge i18n files from plugins to original i18n files. It will overwrite the original i18n files`, Run: func(_ *cobra.Command, _ []string) { if err := cli.ReplaceI18nFilesLocal(i18nTargetPath); err != nil { From 9331690b0d4da9900ef51faf138533e5f26d8c22 Mon Sep 17 00:00:00 2001 From: shuai Date: Tue, 20 May 2025 15:06:24 +0800 Subject: [PATCH 29/39] fix: When copying text and images, can the images only appear at the end of the article --- ui/src/components/Editor/ToolBars/image.tsx | 74 +++++++++++++++------ ui/src/components/Editor/utils/index.ts | 10 +++ 2 files changed, 64 insertions(+), 20 deletions(-) diff --git a/ui/src/components/Editor/ToolBars/image.tsx b/ui/src/components/Editor/ToolBars/image.tsx index 322da08d7..766301f2f 100644 --- a/ui/src/components/Editor/ToolBars/image.tsx +++ b/ui/src/components/Editor/ToolBars/image.tsx @@ -228,30 +228,64 @@ const Image = ({ editorInstance }) => { return; } event.preventDefault(); - - let innerText = ''; - const allPTag = new DOMParser() - .parseFromString( - htmlStr.replace( - /]*)>/gi, - `

    ![${t('image.text')}]($3)\n\n

    `, - ), - 'text/html', - ) - .querySelectorAll('body p'); - - allPTag.forEach((p, index) => { - const text = p.textContent || ''; - if (text !== '') { - if (index === allPTag.length - 1) { - innerText += `${p.textContent}`; + const parser = new DOMParser(); + const doc = parser.parseFromString(htmlStr, 'text/html'); + const { body } = doc; + + let markdownText = ''; + + function traverse(node) { + if (node.nodeType === Node.TEXT_NODE) { + // text node + markdownText += node.textContent; + } else if (node.nodeType === Node.ELEMENT_NODE) { + // element node + const tagName = node.tagName.toLowerCase(); + + if (tagName === 'img') { + // img node + const src = node.getAttribute('src'); + const alt = node.getAttribute('alt') || t('image.text'); + markdownText += `![${alt}](${src})`; + } else if (tagName === 'br') { + // br node + markdownText += '\n'; } else { - innerText += `${p.textContent}${text.endsWith('\n') ? '' : '\n\n'}`; + for (let i = 0; i < node.childNodes.length; i += 1) { + traverse(node.childNodes[i]); + } + } + + const blockLevelElements = [ + 'p', + 'div', + 'h1', + 'h2', + 'h3', + 'h4', + 'h5', + 'h6', + 'ul', + 'ol', + 'li', + 'blockquote', + 'pre', + 'table', + 'thead', + 'tbody', + 'tr', + 'th', + 'td', + ]; + if (blockLevelElements.includes(tagName)) { + markdownText += '\n\n'; } } - }); + } + + traverse(body); - editor.replaceSelection(innerText); + editor.replaceSelection(markdownText); }; const handleClick = () => { if (!link.value) { diff --git a/ui/src/components/Editor/utils/index.ts b/ui/src/components/Editor/utils/index.ts index 3977367c7..61e95fbb8 100644 --- a/ui/src/components/Editor/utils/index.ts +++ b/ui/src/components/Editor/utils/index.ts @@ -172,6 +172,16 @@ export const useEditor = ({ placeholder(placeholderText), EditorView.lineWrapping, editableCompartment.of(EditorView.editable.of(true)), + EditorView.domEventHandlers({ + paste(event) { + const clipboard = event.clipboardData as DataTransfer; + const htmlStr = clipboard.getData('text/html'); + const imgRegex = + /]*)>/; + + return Boolean(htmlStr.match(imgRegex)); + }, + }), ], }); From 89df65c8846625538df527c804dc24a843260959 Mon Sep 17 00:00:00 2001 From: shuai Date: Tue, 20 May 2025 15:33:46 +0800 Subject: [PATCH 30/39] fix: Copy content avoids multiple consecutive newlines --- ui/src/components/Editor/ToolBars/image.tsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ui/src/components/Editor/ToolBars/image.tsx b/ui/src/components/Editor/ToolBars/image.tsx index 766301f2f..c9950b069 100644 --- a/ui/src/components/Editor/ToolBars/image.tsx +++ b/ui/src/components/Editor/ToolBars/image.tsx @@ -285,6 +285,10 @@ const Image = ({ editorInstance }) => { traverse(body); + markdownText = markdownText.replace(/[\n\s]+/g, (match) => { + return match.length > 1 ? '\n\n' : match; + }); + editor.replaceSelection(markdownText); }; const handleClick = () => { From 84cc97dc783116f17aac5e6ca7e83becb5e006c0 Mon Sep 17 00:00:00 2001 From: shuai Date: Fri, 23 May 2025 15:05:15 +0800 Subject: [PATCH 31/39] fix: pre tag background color adjustment, remove the background color of the quoted content --- ui/src/components/Editor/Viewer.tsx | 2 +- ui/src/index.scss | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/ui/src/components/Editor/Viewer.tsx b/ui/src/components/Editor/Viewer.tsx index 3f41831dc..58187a034 100644 --- a/ui/src/components/Editor/Viewer.tsx +++ b/ui/src/components/Editor/Viewer.tsx @@ -78,7 +78,7 @@ const Index = ({ value }, ref) => {
    diff --git a/ui/src/index.scss b/ui/src/index.scss index 279dbf8e8..97aa7b1fd 100644 --- a/ui/src/index.scss +++ b/ui/src/index.scss @@ -247,17 +247,16 @@ img[src=''] { } } pre { - background-color: var(--an-e9ecef); + background-color: var(--bs-gray-100); border-radius: 0.25rem; padding: 1rem; max-height: 38rem; - white-space: normal; + white-space: pre-wrap; } blockquote { border-left: 0.25rem solid #ced4da; padding: 1rem; color: #6c757d; - background-color: var(--an-e9ecef); p { color: var(--bs-body-color); } From def0559a5513d711d31bbf066fe4442c7ed30c7d Mon Sep 17 00:00:00 2001 From: DanielAuerX <72256996+DanielAuerX@users.noreply.github.com> Date: Fri, 23 May 2025 09:07:14 +0200 Subject: [PATCH 32/39] Fix file record and delete files (branding and avatar) (#1335) - [x] create file_record table - [x] avatar and branding files are added to file_record - [x] branding files are being deleted - [x] avatar files are being deleted - [x] reload latest avatar (frontend) after backend state is being updated problems addressed in the pr: - clean up job fails, because it cannot access file_record table - avatar and branding files are not added to the file_record table - avatar and branding files are never deleted - after an avatar is being updated/deleted, the old file is still being requested due to browser caching. This is causing error logs ("no such file or directory") in the backend. cf. conversation in [pr 1326](https://github.com/apache/answer/pull/1326) --------- Co-authored-by: broccoli Co-authored-by: LinkinStars --- cmd/wire_gen.go | 8 +-- internal/base/middleware/avatar.go | 4 +- .../controller_admin/siteinfo_controller.go | 14 +++++- internal/migrations/init_data.go | 1 + internal/repo/file_record/file_record_repo.go | 15 ++++++ internal/repo/site_info/siteinfo_repo.go | 15 ++++++ internal/repo/user/user_repo.go | 15 ++++++ internal/service/content/user_service.go | 45 ++++++++++++++++- .../file_record/file_record_service.go | 38 +++++++++++++- internal/service/siteinfo/siteinfo_service.go | 49 +++++++++++++++++++ .../siteinfo_common/siteinfo_service.go | 12 +++++ internal/service/uploader/upload.go | 18 +++++-- internal/service/user_common/user.go | 11 +++++ 13 files changed, 232 insertions(+), 13 deletions(-) diff --git a/cmd/wire_gen.go b/cmd/wire_gen.go index 702f60df3..f98deb424 100644 --- a/cmd/wire_gen.go +++ b/cmd/wire_gen.go @@ -181,7 +181,9 @@ func initApplication(debug bool, serverConf *conf.Server, dbConf *data.Database, metaCommonService := metacommon.NewMetaCommonService(metaRepo) questionCommon := questioncommon.NewQuestionCommon(questionRepo, answerRepo, voteRepo, followRepo, tagCommonService, userCommon, collectionCommon, answerCommon, metaCommonService, configService, activityQueueService, revisionRepo, siteInfoCommonService, dataData) eventQueueService := event_queue.NewEventQueueService() - userService := content.NewUserService(userRepo, userActiveActivityRepo, activityRepo, emailService, authService, siteInfoCommonService, userRoleRelService, userCommon, userExternalLoginService, userNotificationConfigRepo, userNotificationConfigService, questionCommon, eventQueueService) + fileRecordRepo := file_record.NewFileRecordRepo(dataData) + fileRecordService := file_record2.NewFileRecordService(fileRecordRepo, revisionRepo, serviceConf, siteInfoCommonService, userCommon) + userService := content.NewUserService(userRepo, userActiveActivityRepo, activityRepo, emailService, authService, siteInfoCommonService, userRoleRelService, userCommon, userExternalLoginService, userNotificationConfigRepo, userNotificationConfigService, questionCommon, eventQueueService, fileRecordService) captchaRepo := captcha.NewCaptchaRepo(dataData) captchaService := action.NewCaptchaService(captchaRepo) userController := controller.NewUserController(authService, userService, captchaService, emailService, siteInfoCommonService, userNotificationConfigService) @@ -239,7 +241,7 @@ func initApplication(debug bool, serverConf *conf.Server, dbConf *data.Database, reasonService := reason2.NewReasonService(reasonRepo) reasonController := controller.NewReasonController(reasonService) themeController := controller_admin.NewThemeController() - siteInfoService := siteinfo.NewSiteInfoService(siteInfoRepo, siteInfoCommonService, emailService, tagCommonService, configService, questionCommon) + siteInfoService := siteinfo.NewSiteInfoService(siteInfoRepo, siteInfoCommonService, emailService, tagCommonService, configService, questionCommon, fileRecordService) siteInfoController := controller_admin.NewSiteInfoController(siteInfoService) controllerSiteInfoController := controller.NewSiteInfoController(siteInfoCommonService) notificationCommon := notificationcommon.NewNotificationCommon(dataData, notificationRepo, userCommon, activityRepo, followRepo, objService, notificationQueueService, userExternalLoginRepo, siteInfoCommonService) @@ -248,8 +250,6 @@ func initApplication(debug bool, serverConf *conf.Server, dbConf *data.Database, notificationController := controller.NewNotificationController(notificationService, rankService) dashboardService := dashboard.NewDashboardService(questionRepo, answerRepo, commentCommonRepo, voteRepo, userRepo, reportRepo, configService, siteInfoCommonService, serviceConf, reviewService, revisionRepo, dataData) dashboardController := controller.NewDashboardController(dashboardService) - fileRecordRepo := file_record.NewFileRecordRepo(dataData) - fileRecordService := file_record2.NewFileRecordService(fileRecordRepo, revisionRepo, serviceConf, siteInfoCommonService) uploaderService := uploader.NewUploaderService(serviceConf, siteInfoCommonService, fileRecordService) uploadController := controller.NewUploadController(uploaderService) activityActivityRepo := activity.NewActivityRepo(dataData, configService) diff --git a/internal/base/middleware/avatar.go b/internal/base/middleware/avatar.go index 1d2464173..98430638b 100644 --- a/internal/base/middleware/avatar.go +++ b/internal/base/middleware/avatar.go @@ -21,6 +21,7 @@ package middleware import ( "fmt" + "net/http" "net/url" "os" "path" @@ -62,7 +63,8 @@ func (am *AvatarMiddleware) AvatarThumb() gin.HandlerFunc { filePath, err = am.uploaderService.AvatarThumbFile(ctx, filename, size) if err != nil { log.Error(err) - ctx.Abort() + ctx.AbortWithStatus(http.StatusNotFound) + return } } avatarFile, err := os.ReadFile(filePath) diff --git a/internal/controller_admin/siteinfo_controller.go b/internal/controller_admin/siteinfo_controller.go index cd1b2ab68..8a92daba3 100644 --- a/internal/controller_admin/siteinfo_controller.go +++ b/internal/controller_admin/siteinfo_controller.go @@ -28,6 +28,7 @@ import ( "github.com/apache/answer/internal/schema" "github.com/apache/answer/internal/service/siteinfo" "github.com/gin-gonic/gin" + "github.com/segmentfault/pacman/log" ) // SiteInfoController site info controller @@ -274,8 +275,17 @@ func (sc *SiteInfoController) UpdateBranding(ctx *gin.Context) { if handler.BindAndCheck(ctx, req) { return } - err := sc.siteInfoService.SaveSiteBranding(ctx, req) - handler.HandleResponse(ctx, err, nil) + currentBranding, getBrandingErr := sc.siteInfoService.GetSiteBranding(ctx) + if getBrandingErr == nil { + cleanUpErr := sc.siteInfoService.CleanUpRemovedBrandingFiles(ctx, req, currentBranding) + if cleanUpErr != nil { + log.Errorf("failed to clean up removed branding file(s): %v", cleanUpErr) + } + } else { + log.Errorf("failed to get current site branding: %v", getBrandingErr) + } + saveErr := sc.siteInfoService.SaveSiteBranding(ctx, req) + handler.HandleResponse(ctx, saveErr, nil) } // UpdateSiteWrite update site write info diff --git a/internal/migrations/init_data.go b/internal/migrations/init_data.go index 8b853c4dc..96151625d 100644 --- a/internal/migrations/init_data.go +++ b/internal/migrations/init_data.go @@ -74,6 +74,7 @@ var ( &entity.Badge{}, &entity.BadgeGroup{}, &entity.BadgeAward{}, + &entity.FileRecord{}, &entity.PluginKVStorage{}, } diff --git a/internal/repo/file_record/file_record_repo.go b/internal/repo/file_record/file_record_repo.go index ed081be40..ce486c7ab 100644 --- a/internal/repo/file_record/file_record_repo.go +++ b/internal/repo/file_record/file_record_repo.go @@ -82,3 +82,18 @@ func (fr *fileRecordRepo) UpdateFileRecord(ctx context.Context, fileRecord *enti } return } + +// GetFileRecordByURL gets a file record by its url +func (fr *fileRecordRepo) GetFileRecordByURL(ctx context.Context, fileURL string) (record *entity.FileRecord, err error) { + record = &entity.FileRecord{} + session := fr.data.DB.Context(ctx) + exists, err := session.Where("file_url = ? AND status = ?", fileURL, entity.FileRecordStatusAvailable).Get(record) + if err != nil { + err = errors.InternalServer(reason.DatabaseError).WithError(err).WithStack() + return + } + if !exists { + return + } + return record, nil +} diff --git a/internal/repo/site_info/siteinfo_repo.go b/internal/repo/site_info/siteinfo_repo.go index 420e483f3..5f95b7486 100644 --- a/internal/repo/site_info/siteinfo_repo.go +++ b/internal/repo/site_info/siteinfo_repo.go @@ -101,3 +101,18 @@ func (sr *siteInfoRepo) setCache(ctx context.Context, siteType string, siteInfo log.Error(err) } } + +func (sr *siteInfoRepo) IsBrandingFileUsed(ctx context.Context, filePath string) (bool, error) { + siteInfo := &entity.SiteInfo{} + count, err := sr.data.DB.Context(ctx). + Table("site_info"). + Where(builder.Eq{"type": "branding"}). + And(builder.Like{"content", "%" + filePath + "%"}). + Count(&siteInfo) + + if err != nil { + return false, errors.InternalServer(reason.DatabaseError).WithError(err).WithStack() + } + + return count > 0, nil +} diff --git a/internal/repo/user/user_repo.go b/internal/repo/user/user_repo.go index f56e00b89..a85cd79a1 100644 --- a/internal/repo/user/user_repo.go +++ b/internal/repo/user/user_repo.go @@ -33,6 +33,7 @@ import ( "github.com/apache/answer/plugin" "github.com/segmentfault/pacman/errors" "github.com/segmentfault/pacman/log" + "xorm.io/builder" "xorm.io/xorm" ) @@ -380,3 +381,17 @@ func decorateByUserCenterUser(original *entity.User, ucUser *plugin.UserCenterBa original.Status = int(ucUser.Status) } } + +func (ur *userRepo) IsAvatarFileUsed(ctx context.Context, filePath string) (bool, error) { + user := &entity.User{} + count, err := ur.data.DB.Context(ctx). + Table("user"). + Where(builder.Like{"avatar", "%" + filePath + "%"}). + Count(&user) + + if err != nil { + return false, errors.InternalServer(reason.DatabaseError).WithError(err).WithStack() + } + + return count > 0, nil +} diff --git a/internal/service/content/user_service.go b/internal/service/content/user_service.go index b59790c11..ece3a86de 100644 --- a/internal/service/content/user_service.go +++ b/internal/service/content/user_service.go @@ -23,9 +23,10 @@ import ( "context" "encoding/json" "fmt" + "time" + "github.com/apache/answer/internal/service/event_queue" "github.com/apache/answer/pkg/token" - "time" "github.com/apache/answer/internal/base/constant" questioncommon "github.com/apache/answer/internal/service/question_common" @@ -41,6 +42,7 @@ import ( "github.com/apache/answer/internal/service/activity_common" "github.com/apache/answer/internal/service/auth" "github.com/apache/answer/internal/service/export" + "github.com/apache/answer/internal/service/file_record" "github.com/apache/answer/internal/service/role" "github.com/apache/answer/internal/service/siteinfo_common" usercommon "github.com/apache/answer/internal/service/user_common" @@ -67,6 +69,7 @@ type UserService struct { userNotificationConfigService *user_notification_config.UserNotificationConfigService questionService *questioncommon.QuestionCommon eventQueueService event_queue.EventQueueService + fileRecordService *file_record.FileRecordService } func NewUserService(userRepo usercommon.UserRepo, @@ -82,6 +85,7 @@ func NewUserService(userRepo usercommon.UserRepo, userNotificationConfigService *user_notification_config.UserNotificationConfigService, questionService *questioncommon.QuestionCommon, eventQueueService event_queue.EventQueueService, + fileRecordService *file_record.FileRecordService, ) *UserService { return &UserService{ userCommonService: userCommonService, @@ -97,6 +101,7 @@ func NewUserService(userRepo usercommon.UserRepo, userNotificationConfigService: userNotificationConfigService, questionService: questionService, eventQueueService: eventQueueService, + fileRecordService: fileRecordService, } } @@ -355,6 +360,9 @@ func (us *UserService) UpdateInfo(ctx context.Context, req *schema.UpdateInfoReq } cond := us.formatUserInfoForUpdateInfo(oldUserInfo, req, siteUsers) + + us.cleanUpRemovedAvatar(ctx, oldUserInfo.Avatar, cond.Avatar) + err = us.userRepo.UpdateInfo(ctx, cond) if err != nil { return nil, err @@ -363,6 +371,41 @@ func (us *UserService) UpdateInfo(ctx context.Context, req *schema.UpdateInfoReq return nil, err } +func (us *UserService) cleanUpRemovedAvatar( + ctx context.Context, + oldAvatarJSON string, + newAvatarJSON string, +) { + if oldAvatarJSON == newAvatarJSON { + return + } + + var oldAvatar, newAvatar schema.AvatarInfo + + _ = json.Unmarshal([]byte(oldAvatarJSON), &oldAvatar) + _ = json.Unmarshal([]byte(newAvatarJSON), &newAvatar) + + if len(oldAvatar.Custom) == 0 { + return + } + + // clean up if old is custom and it's either removed or replaced + if oldAvatar.Custom != newAvatar.Custom { + fileRecord, err := us.fileRecordService.GetFileRecordByURL(ctx, oldAvatar.Custom) + if err != nil { + log.Error(err) + return + } + if fileRecord == nil { + log.Warn("no file record found for old avatar url:", oldAvatar.Custom) + return + } + if err := us.fileRecordService.DeleteAndMoveFileRecord(ctx, fileRecord); err != nil { + log.Error(err) + } + } +} + func (us *UserService) formatUserInfoForUpdateInfo( oldUserInfo *entity.User, req *schema.UpdateInfoRequest, siteUsersConf *schema.SiteUsersResp) *entity.User { avatar, _ := json.Marshal(req.Avatar) diff --git a/internal/service/file_record/file_record_service.go b/internal/service/file_record/file_record_service.go index abb983768..29097ba8c 100644 --- a/internal/service/file_record/file_record_service.go +++ b/internal/service/file_record/file_record_service.go @@ -24,6 +24,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "time" "github.com/apache/answer/internal/base/constant" @@ -31,6 +32,7 @@ import ( "github.com/apache/answer/internal/service/revision" "github.com/apache/answer/internal/service/service_config" "github.com/apache/answer/internal/service/siteinfo_common" + usercommon "github.com/apache/answer/internal/service/user_common" "github.com/apache/answer/pkg/checker" "github.com/apache/answer/pkg/dir" "github.com/apache/answer/pkg/writer" @@ -44,6 +46,7 @@ type FileRecordRepo interface { GetFileRecordPage(ctx context.Context, page, pageSize int, cond *entity.FileRecord) ( fileRecordList []*entity.FileRecord, total int64, err error) DeleteFileRecord(ctx context.Context, id int) (err error) + GetFileRecordByURL(ctx context.Context, fileURL string) (record *entity.FileRecord, err error) } // FileRecordService file record service @@ -52,6 +55,7 @@ type FileRecordService struct { revisionRepo revision.RevisionRepo serviceConfig *service_config.ServiceConfig siteInfoService siteinfo_common.SiteInfoCommonService + userService *usercommon.UserCommon } // NewFileRecordService new file record service @@ -60,12 +64,14 @@ func NewFileRecordService( revisionRepo revision.RevisionRepo, serviceConfig *service_config.ServiceConfig, siteInfoService siteinfo_common.SiteInfoCommonService, + userService *usercommon.UserCommon, ) *FileRecordService { return &FileRecordService{ fileRecordRepo: fileRecordRepo, revisionRepo: revisionRepo, serviceConfig: serviceConfig, siteInfoService: siteInfoService, + userService: userService, } } @@ -104,6 +110,21 @@ func (fs *FileRecordService) CleanOrphanUploadFiles(ctx context.Context) { if fileRecord.CreatedAt.AddDate(0, 0, 2).After(time.Now()) { continue } + if isBrandingOrAvatarFile(fileRecord.FilePath) { + if strings.Contains(fileRecord.FilePath, constant.BrandingSubPath+"/") { + if fs.siteInfoService.IsBrandingFileUsed(ctx, fileRecord.FilePath) { + continue + } + } else if strings.Contains(fileRecord.FilePath, constant.AvatarSubPath+"/") { + if fs.userService.IsAvatarFileUsed(ctx, fileRecord.FilePath) { + continue + } + } + if err := fs.DeleteAndMoveFileRecord(ctx, fileRecord); err != nil { + log.Error(err) + } + continue + } if checker.IsNotZeroString(fileRecord.ObjectID) { _, exist, err := fs.revisionRepo.GetLastRevisionByObjectID(ctx, fileRecord.ObjectID) if err != nil { @@ -129,7 +150,7 @@ func (fs *FileRecordService) CleanOrphanUploadFiles(ctx context.Context) { } } // Delete and move the file record - if err := fs.deleteAndMoveFileRecord(ctx, fileRecord); err != nil { + if err := fs.DeleteAndMoveFileRecord(ctx, fileRecord); err != nil { log.Error(err) } } @@ -137,6 +158,10 @@ func (fs *FileRecordService) CleanOrphanUploadFiles(ctx context.Context) { } } +func isBrandingOrAvatarFile(filePath string) bool { + return strings.Contains(filePath, constant.BrandingSubPath+"/") || strings.Contains(filePath, constant.AvatarSubPath+"/") +} + func (fs *FileRecordService) PurgeDeletedFiles(ctx context.Context) { deletedPath := filepath.Join(fs.serviceConfig.UploadPath, constant.DeletedSubPath) log.Infof("purge deleted files: %s", deletedPath) @@ -152,7 +177,7 @@ func (fs *FileRecordService) PurgeDeletedFiles(ctx context.Context) { return } -func (fs *FileRecordService) deleteAndMoveFileRecord(ctx context.Context, fileRecord *entity.FileRecord) error { +func (fs *FileRecordService) DeleteAndMoveFileRecord(ctx context.Context, fileRecord *entity.FileRecord) error { // Delete the file record if err := fs.fileRecordRepo.DeleteFileRecord(ctx, fileRecord.ID); err != nil { return fmt.Errorf("delete file record error: %v", err) @@ -170,3 +195,12 @@ func (fs *FileRecordService) deleteAndMoveFileRecord(ctx context.Context, fileRe log.Debugf("delete and move file: %s", fileRecord.FileURL) return nil } + +func (fs *FileRecordService) GetFileRecordByURL(ctx context.Context, fileURL string) (record *entity.FileRecord, err error) { + record, err = fs.fileRecordRepo.GetFileRecordByURL(ctx, fileURL) + if err != nil { + log.Errorf("error retrieving file record by URL: %v", err) + return + } + return +} diff --git a/internal/service/siteinfo/siteinfo_service.go b/internal/service/siteinfo/siteinfo_service.go index 5141faeee..a0f4891c4 100644 --- a/internal/service/siteinfo/siteinfo_service.go +++ b/internal/service/siteinfo/siteinfo_service.go @@ -22,6 +22,7 @@ package siteinfo import ( "context" "encoding/json" + errpkg "errors" "fmt" "strings" @@ -33,6 +34,7 @@ import ( "github.com/apache/answer/internal/schema" "github.com/apache/answer/internal/service/config" "github.com/apache/answer/internal/service/export" + "github.com/apache/answer/internal/service/file_record" questioncommon "github.com/apache/answer/internal/service/question_common" "github.com/apache/answer/internal/service/siteinfo_common" tagcommon "github.com/apache/answer/internal/service/tag_common" @@ -49,6 +51,7 @@ type SiteInfoService struct { tagCommonService *tagcommon.TagCommonService configService *config.ConfigService questioncommon *questioncommon.QuestionCommon + fileRecordService *file_record.FileRecordService } func NewSiteInfoService( @@ -58,6 +61,7 @@ func NewSiteInfoService( tagCommonService *tagcommon.TagCommonService, configService *config.ConfigService, questioncommon *questioncommon.QuestionCommon, + fileRecordService *file_record.FileRecordService, ) *SiteInfoService { plugin.RegisterGetSiteURLFunc(func() string { @@ -76,6 +80,7 @@ func NewSiteInfoService( tagCommonService: tagCommonService, configService: configService, questioncommon: questioncommon, + fileRecordService: fileRecordService, } } @@ -438,3 +443,47 @@ func (s *SiteInfoService) UpdatePrivilegesConfig(ctx context.Context, req *schem } return } + +func (s *SiteInfoService) CleanUpRemovedBrandingFiles( + ctx context.Context, + newBranding *schema.SiteBrandingReq, + currentBranding *schema.SiteBrandingResp, +) error { + var allErrors []error + currentFiles := map[string]string{ + "logo": currentBranding.Logo, + "mobile_logo": currentBranding.MobileLogo, + "square_icon": currentBranding.SquareIcon, + "favicon": currentBranding.Favicon, + } + + newFiles := map[string]string{ + "logo": newBranding.Logo, + "mobile_logo": newBranding.MobileLogo, + "square_icon": newBranding.SquareIcon, + "favicon": newBranding.Favicon, + } + + for key, currentFile := range currentFiles { + newFile := newFiles[key] + if currentFile != "" && currentFile != newFile { + fileRecord, err := s.fileRecordService.GetFileRecordByURL(ctx, currentFile) + if err != nil { + allErrors = append(allErrors, err) + continue + } + if fileRecord == nil { + err := errpkg.New("file record is nil for key " + key) + allErrors = append(allErrors, err) + continue + } + if err := s.fileRecordService.DeleteAndMoveFileRecord(ctx, fileRecord); err != nil { + allErrors = append(allErrors, err) + } + } + } + if len(allErrors) > 0 { + return errpkg.Join(allErrors...) + } + return nil +} diff --git a/internal/service/siteinfo_common/siteinfo_service.go b/internal/service/siteinfo_common/siteinfo_service.go index f715bc9b0..0c896c2b0 100644 --- a/internal/service/siteinfo_common/siteinfo_service.go +++ b/internal/service/siteinfo_common/siteinfo_service.go @@ -35,6 +35,7 @@ import ( type SiteInfoRepo interface { SaveByType(ctx context.Context, siteType string, data *entity.SiteInfo) (err error) GetByType(ctx context.Context, siteType string) (siteInfo *entity.SiteInfo, exist bool, err error) + IsBrandingFileUsed(ctx context.Context, filePath string) (bool, error) } // siteInfoCommonService site info common service @@ -56,6 +57,7 @@ type SiteInfoCommonService interface { GetSiteTheme(ctx context.Context) (resp *schema.SiteThemeResp, err error) GetSiteSeo(ctx context.Context) (resp *schema.SiteSeoResp, err error) GetSiteInfoByType(ctx context.Context, siteType string, resp interface{}) (err error) + IsBrandingFileUsed(ctx context.Context, filePath string) bool } // NewSiteInfoCommonService new site info common service @@ -233,3 +235,13 @@ func (s *siteInfoCommonService) GetSiteInfoByType(ctx context.Context, siteType _ = json.Unmarshal([]byte(siteInfo.Content), resp) return nil } + +func (s *siteInfoCommonService) IsBrandingFileUsed(ctx context.Context, filePath string) bool { + used, err := s.siteInfoRepo.IsBrandingFileUsed(ctx, filePath) + if err != nil { + log.Errorf("error checking if branding file is used: %v", err) + // will try again with the next clean up + return true + } + return used +} diff --git a/internal/service/uploader/upload.go b/internal/service/uploader/upload.go index d5cc2dfbe..2ae5369df 100644 --- a/internal/service/uploader/upload.go +++ b/internal/service/uploader/upload.go @@ -127,7 +127,13 @@ func (us *uploaderService) UploadAvatarFile(ctx *gin.Context, userID string) (ur newFilename := fmt.Sprintf("%s%s", uid.IDStr12(), fileExt) avatarFilePath := path.Join(constant.AvatarSubPath, newFilename) - return us.uploadImageFile(ctx, fileHeader, avatarFilePath) + url, err = us.uploadImageFile(ctx, fileHeader, avatarFilePath) + if err != nil { + return "", err + } + us.fileRecordService.AddFileRecord(ctx, userID, avatarFilePath, url, string(plugin.UserAvatar)) + return url, nil + } func (us *uploaderService) AvatarThumbFile(ctx *gin.Context, fileName string, size int) (url string, err error) { @@ -149,7 +155,7 @@ func (us *uploaderService) AvatarThumbFile(ctx *gin.Context, fileName string, si filePath := fmt.Sprintf("%s/%s/%s", us.serviceConfig.UploadPath, constant.AvatarSubPath, fileName) avatarFile, err = os.ReadFile(filePath) if err != nil { - return "", errors.InternalServer(reason.UnknownError).WithError(err).WithStack() + return "", errors.NotFound(reason.UnknownError).WithError(err) } reader := bytes.NewReader(avatarFile) img, err := imaging.Decode(reader) @@ -282,7 +288,13 @@ func (us *uploaderService) UploadBrandingFile(ctx *gin.Context, userID string) ( newFilename := fmt.Sprintf("%s%s", uid.IDStr12(), fileExt) avatarFilePath := path.Join(constant.BrandingSubPath, newFilename) - return us.uploadImageFile(ctx, fileHeader, avatarFilePath) + url, err = us.uploadImageFile(ctx, fileHeader, avatarFilePath) + if err != nil { + return "", err + } + us.fileRecordService.AddFileRecord(ctx, userID, avatarFilePath, url, string(plugin.AdminBranding)) + return url, nil + } func (us *uploaderService) uploadImageFile(ctx *gin.Context, file *multipart.FileHeader, fileSubPath string) ( diff --git a/internal/service/user_common/user.go b/internal/service/user_common/user.go index 7f50eafe5..3df99261d 100644 --- a/internal/service/user_common/user.go +++ b/internal/service/user_common/user.go @@ -60,6 +60,7 @@ type UserRepo interface { GetByEmail(ctx context.Context, email string) (userInfo *entity.User, exist bool, err error) GetUserCount(ctx context.Context) (count int64, err error) SearchUserListByName(ctx context.Context, name string, limit int, onlyStaff bool) (userList []*entity.User, err error) + IsAvatarFileUsed(ctx context.Context, filePath string) (bool, error) } // UserCommon user service @@ -245,3 +246,13 @@ func (us *UserCommon) CacheLoginUserInfo(ctx context.Context, userID string, use } return accessToken, userCacheInfo, nil } + +func (us *UserCommon) IsAvatarFileUsed(ctx context.Context, filePath string) bool { + used, err := us.userRepo.IsAvatarFileUsed(ctx, filePath) + if err != nil { + log.Errorf("error checking if branding file is used: %v", err) + // will try again with the next clean up + return true + } + return used +} From afdf6ce778c2e977bc69e47a16a8791bd2d4b558 Mon Sep 17 00:00:00 2001 From: Luffy <52o@qq52o.cn> Date: Thu, 22 May 2025 11:45:52 +0800 Subject: [PATCH 33/39] fix: Unified display name and username length checks --- docs/docs.go | 8 +++++--- docs/swagger.json | 8 +++++--- docs/swagger.yaml | 4 +++- i18n/af_ZA.yaml | 4 ++-- i18n/ar_SA.yaml | 4 ++-- i18n/ca_ES.yaml | 4 ++-- i18n/cs_CZ.yaml | 8 ++++---- i18n/cy_GB.yaml | 8 ++++---- i18n/el_GR.yaml | 4 ++-- i18n/en_US.yaml | 8 ++++---- i18n/fa_IR.yaml | 4 ++-- i18n/fi_FI.yaml | 4 ++-- i18n/he_IL.yaml | 4 ++-- i18n/hi_IN.yaml | 8 ++++---- i18n/hu_HU.yaml | 4 ++-- i18n/id_ID.yaml | 8 ++++---- i18n/ko_KR.yaml | 4 ++-- i18n/ml_IN.yaml | 8 ++++---- i18n/nl_NL.yaml | 4 ++-- i18n/no_NO.yaml | 4 ++-- i18n/ro_RO.yaml | 4 ++-- i18n/ru_RU.yaml | 4 ++-- i18n/sk_SK.yaml | 4 ++-- i18n/sr_SP.yaml | 4 ++-- i18n/sv_SE.yaml | 8 ++++---- i18n/te_IN.yaml | 8 ++++---- i18n/tr_TR.yaml | 8 ++++---- i18n/zh_TW.yaml | 4 ++-- internal/schema/backyard_user_schema.go | 4 ++-- internal/schema/user_schema.go | 2 +- ui/src/hooks/useChangeProfileModal/index.tsx | 4 ++-- 31 files changed, 86 insertions(+), 80 deletions(-) diff --git a/docs/docs.go b/docs/docs.go index 4f5fe86ec..82ab3c779 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -8263,7 +8263,7 @@ const docTemplate = `{ "display_name": { "type": "string", "maxLength": 30, - "minLength": 4 + "minLength": 2 }, "email": { "type": "string", @@ -8274,7 +8274,8 @@ const docTemplate = `{ }, "username": { "type": "string", - "maxLength": 30 + "maxLength": 30, + "minLength": 2 } } }, @@ -11084,7 +11085,8 @@ const docTemplate = `{ }, "display_name": { "type": "string", - "maxLength": 30 + "maxLength": 30, + "minLength": 2 }, "location": { "type": "string", diff --git a/docs/swagger.json b/docs/swagger.json index 5a48fd0de..689255a42 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -8236,7 +8236,7 @@ "display_name": { "type": "string", "maxLength": 30, - "minLength": 4 + "minLength": 2 }, "email": { "type": "string", @@ -8247,7 +8247,8 @@ }, "username": { "type": "string", - "maxLength": 30 + "maxLength": 30, + "minLength": 2 } } }, @@ -11057,7 +11058,8 @@ }, "display_name": { "type": "string", - "maxLength": 30 + "maxLength": 30, + "minLength": 2 }, "location": { "type": "string", diff --git a/docs/swagger.yaml b/docs/swagger.yaml index c06ce9499..a399b3bfd 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -527,7 +527,7 @@ definitions: properties: display_name: maxLength: 30 - minLength: 4 + minLength: 2 type: string email: maxLength: 500 @@ -536,6 +536,7 @@ definitions: type: string username: maxLength: 30 + minLength: 2 type: string required: - display_name @@ -2485,6 +2486,7 @@ definitions: type: string display_name: maxLength: 30 + minLength: 2 type: string location: maxLength: 100 diff --git a/i18n/af_ZA.yaml b/i18n/af_ZA.yaml index 0d1cf1cf8..f421ba9af 100644 --- a/i18n/af_ZA.yaml +++ b/i18n/af_ZA.yaml @@ -678,12 +678,12 @@ ui: display_name: label: Display Name msg: Display name cannot be empty. - msg_range: Display name up to 30 characters. + msg_range: Display name must be 2-30 characters in length. username: label: Username caption: People can mention you as "@username". msg: Username cannot be empty. - msg_range: Username up to 30 characters. + msg_range: Username must be 2-30 characters in length. character: 'Must use the character set "a-z", "0-9", " - . _"' avatar: label: Profile Image diff --git a/i18n/ar_SA.yaml b/i18n/ar_SA.yaml index 1f298b937..094a05523 100644 --- a/i18n/ar_SA.yaml +++ b/i18n/ar_SA.yaml @@ -678,12 +678,12 @@ ui: display_name: label: Display Name msg: Display name cannot be empty. - msg_range: Display name up to 30 characters. + msg_range: Display name must be 2-30 characters in length. username: label: Username caption: People can mention you as "@username". msg: Username cannot be empty. - msg_range: Username up to 30 characters. + msg_range: Username must be 2-30 characters in length. character: 'Must use the character set "a-z", "0-9", " - . _"' avatar: label: Profile Image diff --git a/i18n/ca_ES.yaml b/i18n/ca_ES.yaml index 1f298b937..094a05523 100644 --- a/i18n/ca_ES.yaml +++ b/i18n/ca_ES.yaml @@ -678,12 +678,12 @@ ui: display_name: label: Display Name msg: Display name cannot be empty. - msg_range: Display name up to 30 characters. + msg_range: Display name must be 2-30 characters in length. username: label: Username caption: People can mention you as "@username". msg: Username cannot be empty. - msg_range: Username up to 30 characters. + msg_range: Username must be 2-30 characters in length. character: 'Must use the character set "a-z", "0-9", " - . _"' avatar: label: Profile Image diff --git a/i18n/cs_CZ.yaml b/i18n/cs_CZ.yaml index e7561c9e9..e7fda9afd 100644 --- a/i18n/cs_CZ.yaml +++ b/i18n/cs_CZ.yaml @@ -1275,12 +1275,12 @@ ui: display_name: label: Display name msg: Display name cannot be empty. - msg_range: Display name up to 30 characters. + msg_range: Display name must be 2-30 characters in length. username: label: Username caption: People can mention you as "@username". msg: Username cannot be empty. - msg_range: Username up to 30 characters. + msg_range: Username must be 2-30 characters in length. character: 'Must use the character set "a-z", "0-9", " - . _"' avatar: label: Profile image @@ -1852,10 +1852,10 @@ ui: fields: display_name: label: Display name - msg_range: Display name up to 30 characters. + msg_range: Display name must be 2-30 characters in length. username: label: Username - msg_range: Username up to 30 characters. + msg_range: Username must be 2-30 characters in length. email: label: Email msg_invalid: Invalid Email Address. diff --git a/i18n/cy_GB.yaml b/i18n/cy_GB.yaml index 09987e25b..ed65acbb0 100644 --- a/i18n/cy_GB.yaml +++ b/i18n/cy_GB.yaml @@ -1275,12 +1275,12 @@ ui: display_name: label: Display name msg: Display name cannot be empty. - msg_range: Display name up to 30 characters. + msg_range: Display name must be 2-30 characters in length. username: label: Username caption: People can mention you as "@username". msg: Username cannot be empty. - msg_range: Username up to 30 characters. + msg_range: Username must be 2-30 characters in length. character: 'Must use the character set "a-z", "0-9", " - . _"' avatar: label: Profile image @@ -1852,10 +1852,10 @@ ui: fields: display_name: label: Display name - msg_range: Display name up to 30 characters. + msg_range: Display name must be 2-30 characters in length. username: label: Username - msg_range: Username up to 30 characters. + msg_range: Username must be 2-30 characters in length. email: label: Email msg_invalid: Invalid Email Address. diff --git a/i18n/el_GR.yaml b/i18n/el_GR.yaml index 1f298b937..094a05523 100644 --- a/i18n/el_GR.yaml +++ b/i18n/el_GR.yaml @@ -678,12 +678,12 @@ ui: display_name: label: Display Name msg: Display name cannot be empty. - msg_range: Display name up to 30 characters. + msg_range: Display name must be 2-30 characters in length. username: label: Username caption: People can mention you as "@username". msg: Username cannot be empty. - msg_range: Username up to 30 characters. + msg_range: Username must be 2-30 characters in length. character: 'Must use the character set "a-z", "0-9", " - . _"' avatar: label: Profile Image diff --git a/i18n/en_US.yaml b/i18n/en_US.yaml index a22396cf9..191581361 100644 --- a/i18n/en_US.yaml +++ b/i18n/en_US.yaml @@ -1300,12 +1300,12 @@ ui: display_name: label: Display name msg: Display name cannot be empty. - msg_range: Display name up to 30 characters. + msg_range: Display name must be 2-30 characters in length. username: label: Username caption: People can mention you as "@username". msg: Username cannot be empty. - msg_range: Username up to 30 characters. + msg_range: Username must be 2-30 characters in length. character: 'Must use the character set "a-z", "0-9", " - . _"' avatar: label: Profile image @@ -1892,10 +1892,10 @@ ui: fields: display_name: label: Display name - msg_range: Display name up to 30 characters. + msg_range: Display name must be 2-30 characters in length. username: label: Username - msg_range: Username up to 30 characters. + msg_range: Username must be 2-30 characters in length. email: label: Email msg_invalid: Invalid Email Address. diff --git a/i18n/fa_IR.yaml b/i18n/fa_IR.yaml index 5838f4cc8..429aa8563 100644 --- a/i18n/fa_IR.yaml +++ b/i18n/fa_IR.yaml @@ -1852,10 +1852,10 @@ ui: fields: display_name: label: Display name - msg_range: Display name up to 30 characters. + msg_range: Display name must be 2-30 characters in length. username: label: Username - msg_range: Username up to 30 characters. + msg_range: Username must be 2-30 characters in length. email: label: Email msg_invalid: Invalid Email Address. diff --git a/i18n/fi_FI.yaml b/i18n/fi_FI.yaml index 1f298b937..094a05523 100644 --- a/i18n/fi_FI.yaml +++ b/i18n/fi_FI.yaml @@ -678,12 +678,12 @@ ui: display_name: label: Display Name msg: Display name cannot be empty. - msg_range: Display name up to 30 characters. + msg_range: Display name must be 2-30 characters in length. username: label: Username caption: People can mention you as "@username". msg: Username cannot be empty. - msg_range: Username up to 30 characters. + msg_range: Username must be 2-30 characters in length. character: 'Must use the character set "a-z", "0-9", " - . _"' avatar: label: Profile Image diff --git a/i18n/he_IL.yaml b/i18n/he_IL.yaml index 1f298b937..094a05523 100644 --- a/i18n/he_IL.yaml +++ b/i18n/he_IL.yaml @@ -678,12 +678,12 @@ ui: display_name: label: Display Name msg: Display name cannot be empty. - msg_range: Display name up to 30 characters. + msg_range: Display name must be 2-30 characters in length. username: label: Username caption: People can mention you as "@username". msg: Username cannot be empty. - msg_range: Username up to 30 characters. + msg_range: Username must be 2-30 characters in length. character: 'Must use the character set "a-z", "0-9", " - . _"' avatar: label: Profile Image diff --git a/i18n/hi_IN.yaml b/i18n/hi_IN.yaml index c2d3781d8..8dac7f47e 100644 --- a/i18n/hi_IN.yaml +++ b/i18n/hi_IN.yaml @@ -1275,12 +1275,12 @@ ui: display_name: label: Display name msg: Display name cannot be empty. - msg_range: Display name up to 30 characters. + msg_range: Display name must be 2-30 characters in length. username: label: Username caption: People can mention you as "@username". msg: Username cannot be empty. - msg_range: Username up to 30 characters. + msg_range: Username must be 2-30 characters in length. character: 'Must use the character set "a-z", "0-9", " - . _"' avatar: label: Profile image @@ -1852,10 +1852,10 @@ ui: fields: display_name: label: Display name - msg_range: Display name up to 30 characters. + msg_range: Display name must be 2-30 characters in length. username: label: Username - msg_range: Username up to 30 characters. + msg_range: Username must be 2-30 characters in length. email: label: Email msg_invalid: Invalid Email Address. diff --git a/i18n/hu_HU.yaml b/i18n/hu_HU.yaml index 1f298b937..094a05523 100644 --- a/i18n/hu_HU.yaml +++ b/i18n/hu_HU.yaml @@ -678,12 +678,12 @@ ui: display_name: label: Display Name msg: Display name cannot be empty. - msg_range: Display name up to 30 characters. + msg_range: Display name must be 2-30 characters in length. username: label: Username caption: People can mention you as "@username". msg: Username cannot be empty. - msg_range: Username up to 30 characters. + msg_range: Username must be 2-30 characters in length. character: 'Must use the character set "a-z", "0-9", " - . _"' avatar: label: Profile Image diff --git a/i18n/id_ID.yaml b/i18n/id_ID.yaml index ff969bfa0..03a707c8e 100644 --- a/i18n/id_ID.yaml +++ b/i18n/id_ID.yaml @@ -1275,12 +1275,12 @@ ui: display_name: label: Display name msg: Display name cannot be empty. - msg_range: Display name up to 30 characters. + msg_range: Display name must be 2-30 characters in length. username: label: Username caption: People can mention you as "@username". msg: Username cannot be empty. - msg_range: Username up to 30 characters. + msg_range: Username must be 2-30 characters in length. character: 'Must use the character set "a-z", "0-9", " - . _"' avatar: label: Profile image @@ -1852,10 +1852,10 @@ ui: fields: display_name: label: Display name - msg_range: Display name up to 30 characters. + msg_range: Display name must be 2-30 characters in length. username: label: Username - msg_range: Username up to 30 characters. + msg_range: Username must be 2-30 characters in length. email: label: Email msg_invalid: Invalid Email Address. diff --git a/i18n/ko_KR.yaml b/i18n/ko_KR.yaml index e1381004e..543ef3b7f 100644 --- a/i18n/ko_KR.yaml +++ b/i18n/ko_KR.yaml @@ -1852,10 +1852,10 @@ ui: fields: display_name: label: Display name - msg_range: Display name up to 30 characters. + msg_range: Display name must be 2-30 characters in length. username: label: Username - msg_range: Username up to 30 characters. + msg_range: Username must be 2-30 characters in length. email: label: 이메일 msg_invalid: Invalid Email Address. diff --git a/i18n/ml_IN.yaml b/i18n/ml_IN.yaml index 9a3f74593..a1a6da696 100644 --- a/i18n/ml_IN.yaml +++ b/i18n/ml_IN.yaml @@ -1275,12 +1275,12 @@ ui: display_name: label: Display name msg: Display name cannot be empty. - msg_range: Display name up to 30 characters. + msg_range: Display name must be 2-30 characters in length. username: label: Username caption: People can mention you as "@username". msg: Username cannot be empty. - msg_range: Username up to 30 characters. + msg_range: Username must be 2-30 characters in length. character: 'Must use the character set "a-z", "0-9", " - . _"' avatar: label: Profile image @@ -1852,10 +1852,10 @@ ui: fields: display_name: label: Display name - msg_range: Display name up to 30 characters. + msg_range: Display name must be 2-30 characters in length. username: label: Username - msg_range: Username up to 30 characters. + msg_range: Username must be 2-30 characters in length. email: label: Email msg_invalid: Invalid Email Address. diff --git a/i18n/nl_NL.yaml b/i18n/nl_NL.yaml index 1f298b937..094a05523 100644 --- a/i18n/nl_NL.yaml +++ b/i18n/nl_NL.yaml @@ -678,12 +678,12 @@ ui: display_name: label: Display Name msg: Display name cannot be empty. - msg_range: Display name up to 30 characters. + msg_range: Display name must be 2-30 characters in length. username: label: Username caption: People can mention you as "@username". msg: Username cannot be empty. - msg_range: Username up to 30 characters. + msg_range: Username must be 2-30 characters in length. character: 'Must use the character set "a-z", "0-9", " - . _"' avatar: label: Profile Image diff --git a/i18n/no_NO.yaml b/i18n/no_NO.yaml index 90b00b6c9..a0cd01396 100644 --- a/i18n/no_NO.yaml +++ b/i18n/no_NO.yaml @@ -678,12 +678,12 @@ ui: display_name: label: Display Name msg: Display name cannot be empty. - msg_range: Display name up to 30 characters. + msg_range: Display name must be 2-30 characters in length. username: label: Username caption: People can mention you as "@username". msg: Username cannot be empty. - msg_range: Username up to 30 characters. + msg_range: Username must be 2-30 characters in length. character: 'Must use the character set "a-z", "0-9", " - . _"' avatar: label: Profile Image diff --git a/i18n/ro_RO.yaml b/i18n/ro_RO.yaml index 80b0ebdb1..e7ba8455a 100644 --- a/i18n/ro_RO.yaml +++ b/i18n/ro_RO.yaml @@ -1852,10 +1852,10 @@ ui: fields: display_name: label: Display name - msg_range: Display name up to 30 characters. + msg_range: Display name must be 2-30 characters in length. username: label: Username - msg_range: Username up to 30 characters. + msg_range: Username must be 2-30 characters in length. email: label: Email msg_invalid: Invalid Email Address. diff --git a/i18n/ru_RU.yaml b/i18n/ru_RU.yaml index d2507f99d..58e80e985 100644 --- a/i18n/ru_RU.yaml +++ b/i18n/ru_RU.yaml @@ -1852,10 +1852,10 @@ ui: fields: display_name: label: Display name - msg_range: Display name up to 30 characters. + msg_range: Display name must be 2-30 characters in length. username: label: Username - msg_range: Username up to 30 characters. + msg_range: Username must be 2-30 characters in length. email: label: Email msg_invalid: Invalid Email Address. diff --git a/i18n/sk_SK.yaml b/i18n/sk_SK.yaml index 414e6a9dd..1de09bd70 100644 --- a/i18n/sk_SK.yaml +++ b/i18n/sk_SK.yaml @@ -1852,10 +1852,10 @@ ui: fields: display_name: label: Display name - msg_range: Display name up to 30 characters. + msg_range: Display name must be 2-30 characters in length. username: label: Username - msg_range: Username up to 30 characters. + msg_range: Username must be 2-30 characters in length. email: label: Email msg_invalid: Invalid Email Address. diff --git a/i18n/sr_SP.yaml b/i18n/sr_SP.yaml index 1f298b937..094a05523 100644 --- a/i18n/sr_SP.yaml +++ b/i18n/sr_SP.yaml @@ -678,12 +678,12 @@ ui: display_name: label: Display Name msg: Display name cannot be empty. - msg_range: Display name up to 30 characters. + msg_range: Display name must be 2-30 characters in length. username: label: Username caption: People can mention you as "@username". msg: Username cannot be empty. - msg_range: Username up to 30 characters. + msg_range: Username must be 2-30 characters in length. character: 'Must use the character set "a-z", "0-9", " - . _"' avatar: label: Profile Image diff --git a/i18n/sv_SE.yaml b/i18n/sv_SE.yaml index a2ecc2b19..910313f97 100644 --- a/i18n/sv_SE.yaml +++ b/i18n/sv_SE.yaml @@ -1275,12 +1275,12 @@ ui: display_name: label: Visningsnamn msg: Display name cannot be empty. - msg_range: Display name up to 30 characters. + msg_range: Display name must be 2-30 characters in length. username: label: Användarnamn caption: People can mention you as "@username". msg: Username cannot be empty. - msg_range: Username up to 30 characters. + msg_range: Username must be 2-30 characters in length. character: 'Must use the character set "a-z", "0-9", " - . _"' avatar: label: Profilbild @@ -1852,10 +1852,10 @@ ui: fields: display_name: label: Visningsnamn - msg_range: Display name up to 30 characters. + msg_range: Display name must be 2-30 characters in length. username: label: Användarnamn - msg_range: Username up to 30 characters. + msg_range: Username must be 2-30 characters in length. email: label: Email msg_invalid: Ogiltig e-postadress. diff --git a/i18n/te_IN.yaml b/i18n/te_IN.yaml index 425239fc3..cc5ce3380 100644 --- a/i18n/te_IN.yaml +++ b/i18n/te_IN.yaml @@ -1275,12 +1275,12 @@ ui: display_name: label: Display name msg: Display name cannot be empty. - msg_range: Display name up to 30 characters. + msg_range: Display name must be 2-30 characters in length. username: label: Username caption: People can mention you as "@username". msg: Username cannot be empty. - msg_range: Username up to 30 characters. + msg_range: Username must be 2-30 characters in length. character: 'Must use the character set "a-z", "0-9", " - . _"' avatar: label: Profile image @@ -1852,10 +1852,10 @@ ui: fields: display_name: label: Display name - msg_range: Display name up to 30 characters. + msg_range: Display name must be 2-30 characters in length. username: label: Username - msg_range: Username up to 30 characters. + msg_range: Username must be 2-30 characters in length. email: label: Email msg_invalid: Invalid Email Address. diff --git a/i18n/tr_TR.yaml b/i18n/tr_TR.yaml index f1bf034a7..0b845a0ef 100644 --- a/i18n/tr_TR.yaml +++ b/i18n/tr_TR.yaml @@ -1275,12 +1275,12 @@ ui: display_name: label: Display name msg: Display name cannot be empty. - msg_range: Display name up to 30 characters. + msg_range: Display name must be 2-30 characters in length. username: label: Username caption: People can mention you as "@username". msg: Username cannot be empty. - msg_range: Username up to 30 characters. + msg_range: Username must be 2-30 characters in length. character: 'Must use the character set "a-z", "0-9", " - . _"' avatar: label: Profile image @@ -1852,10 +1852,10 @@ ui: fields: display_name: label: Display name - msg_range: Display name up to 30 characters. + msg_range: Display name must be 2-30 characters in length. username: label: Username - msg_range: Username up to 30 characters. + msg_range: Username must be 2-30 characters in length. email: label: Email msg_invalid: Invalid Email Address. diff --git a/i18n/zh_TW.yaml b/i18n/zh_TW.yaml index 72ab17795..4e02c70d4 100644 --- a/i18n/zh_TW.yaml +++ b/i18n/zh_TW.yaml @@ -1852,10 +1852,10 @@ ui: fields: display_name: label: Display name - msg_range: Display name up to 30 characters. + msg_range: Display name must be 2-30 characters in length. username: label: Username - msg_range: Username up to 30 characters. + msg_range: Username must be 2-30 characters in length. email: label: Email msg_invalid: Invalid Email Address. diff --git a/internal/schema/backyard_user_schema.go b/internal/schema/backyard_user_schema.go index b6e29f65d..923a20ed9 100644 --- a/internal/schema/backyard_user_schema.go +++ b/internal/schema/backyard_user_schema.go @@ -111,8 +111,8 @@ type UpdateUserRoleReq struct { // EditUserProfileReq edit user profile request type EditUserProfileReq struct { UserID string `validate:"required" json:"user_id"` - DisplayName string `validate:"required,gte=4,lte=30" json:"display_name"` - Username string `validate:"omitempty,gt=3,lte=30" json:"username"` + DisplayName string `validate:"required,gte=2,lte=30" json:"display_name"` + Username string `validate:"omitempty,gte=2,lte=30" json:"username"` Email string `validate:"required,email,gt=0,lte=500" json:"email"` LoginUserID string `json:"-"` IsAdmin bool `json:"-"` diff --git a/internal/schema/user_schema.go b/internal/schema/user_schema.go index 3abbbdb36..7c0af1dc4 100644 --- a/internal/schema/user_schema.go +++ b/internal/schema/user_schema.go @@ -238,7 +238,7 @@ func (u *UserModifyPasswordReq) Check() (errFields []*validator.FormErrorField, } type UpdateInfoRequest struct { - DisplayName string `validate:"omitempty,gt=0,lte=30" json:"display_name"` + DisplayName string `validate:"omitempty,gte=2,lte=30" json:"display_name"` Username string `validate:"omitempty,gte=2,lte=30" json:"username"` Avatar AvatarInfo `json:"avatar"` Bio string `validate:"omitempty,gt=0,lte=4096" json:"bio"` diff --git a/ui/src/hooks/useChangeProfileModal/index.tsx b/ui/src/hooks/useChangeProfileModal/index.tsx index 6f85e25d8..5fec7babc 100644 --- a/ui/src/hooks/useChangeProfileModal/index.tsx +++ b/ui/src/hooks/useChangeProfileModal/index.tsx @@ -69,7 +69,7 @@ const useChangeProfileModal = (props: IProps = {}, userData) => { 'ui:options': { inputType: 'text', validator: (value) => { - const MIN_LENGTH = 3; + const MIN_LENGTH = 2; const MAX_LENGTH = 30; if (value.length < MIN_LENGTH || value.length > MAX_LENGTH) { return t('form.fields.display_name.msg_range'); @@ -82,7 +82,7 @@ const useChangeProfileModal = (props: IProps = {}, userData) => { 'ui:options': { inputType: 'text', validator: (value) => { - const MIN_LENGTH = 3; + const MIN_LENGTH = 2; const MAX_LENGTH = 30; if (value.length < MIN_LENGTH || value.length > MAX_LENGTH) { return t('form.fields.username.msg_range'); From 6381a2192e9a5118fe89c3defdbff890dec4697c Mon Sep 17 00:00:00 2001 From: Giorgio Bonvicini <144213540+Giorgio-Bonvicini-R4P@users.noreply.github.com> Date: Fri, 23 May 2025 17:57:51 +0200 Subject: [PATCH 34/39] fix: update the visit cookie if it does not match the visit token Previous behavior was to keep any existing visit cookies, which caused problems like #1334 --- internal/controller/user_controller.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/internal/controller/user_controller.go b/internal/controller/user_controller.go index c8ffd7fef..49b9b23c6 100644 --- a/internal/controller/user_controller.go +++ b/internal/controller/user_controller.go @@ -717,9 +717,12 @@ func (uc *UserController) SearchUserListByName(ctx *gin.Context) { } func (uc *UserController) setVisitCookies(ctx *gin.Context, visitToken string, force bool) { - cookie, err := ctx.Cookie(constant.UserVisitCookiesCacheKey) - if err == nil && len(cookie) > 0 && !force { - return + if !force { + cookie, _ := ctx.Cookie(constant.UserVisitCookiesCacheKey) + // If the cookie is the same as the visitToken, no need to set it again + if cookie == visitToken { + return + } } general, err := uc.siteInfoCommonService.GetSiteGeneral(ctx) if err != nil { From 503e1cf7d48673d23b40c214b0205e1eda38f544 Mon Sep 17 00:00:00 2001 From: shuai Date: Tue, 27 May 2025 18:26:46 +0800 Subject: [PATCH 35/39] fix: dropdown header use h6 --- ui/src/components/QuestionList/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/src/components/QuestionList/index.tsx b/ui/src/components/QuestionList/index.tsx index fcb0994f2..a0f8b129c 100644 --- a/ui/src/components/QuestionList/index.tsx +++ b/ui/src/components/QuestionList/index.tsx @@ -120,7 +120,7 @@ const QuestionList: FC = ({ - + {t('view', { keyPrefix: 'btns' })} From b20f3a876249ab44a02b5070581bab031164f6a7 Mon Sep 17 00:00:00 2001 From: shuai Date: Tue, 27 May 2025 18:27:44 +0800 Subject: [PATCH 36/39] fix: dropdown header use h6 --- ui/template/sort-btns.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/template/sort-btns.html b/ui/template/sort-btns.html index a981d1505..1db002dde 100644 --- a/ui/template/sort-btns.html +++ b/ui/template/sort-btns.html @@ -48,9 +48,9 @@
    -
    +
    {{translator ($.language) "ui.btns.view" }} -
    + {{translator ($.language) "ui.btns.card" }} From 6df436c7183fdbaccdc9b7d6659001d51f0f4ef8 Mon Sep 17 00:00:00 2001 From: Sonui Date: Mon, 26 May 2025 00:27:06 +0800 Subject: [PATCH 37/39] feat(tests): add comprehensive tests for KV storage plugin --- plugin/plugin_test/kv_storage_test.go | 412 +++++++++++++++++++++++++ plugin/plugin_test/plugin_main_test.go | 207 +++++++++++++ 2 files changed, 619 insertions(+) create mode 100644 plugin/plugin_test/kv_storage_test.go create mode 100644 plugin/plugin_test/plugin_main_test.go diff --git a/plugin/plugin_test/kv_storage_test.go b/plugin/plugin_test/kv_storage_test.go new file mode 100644 index 000000000..0dcd5b46a --- /dev/null +++ b/plugin/plugin_test/kv_storage_test.go @@ -0,0 +1,412 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package plugin_test + +import ( + "context" + "fmt" + "math/rand" + "sync" + "testing" + "time" + + "github.com/apache/answer/plugin" + "github.com/segmentfault/pacman/log" + _ "modernc.org/sqlite" +) + +var ( + testPlugin *TestKVStoragePlugin +) + +// Helper functions for testing +func mustSet(t *testing.T, kv *plugin.KVOperator, ctx context.Context, group, key, value string) { + if err := kv.Set(ctx, plugin.KVParams{Group: group, Key: key, Value: value}); err != nil { + t.Fatalf("Failed to set %s/%s: %v", group, key, err) + } +} + +func mustGet(t *testing.T, kv *plugin.KVOperator, ctx context.Context, group, key, expected string) { + val, err := kv.Get(ctx, plugin.KVParams{Group: group, Key: key}) + if err != nil { + t.Fatalf("Failed to get %s/%s: %v", group, key, err) + } + if val != expected { + t.Errorf("Expected '%s' for %s/%s, got '%s'", expected, group, key, val) + } +} + +func mustDel(t *testing.T, kv *plugin.KVOperator, ctx context.Context, group, key string) { + if err := kv.Del(ctx, plugin.KVParams{Group: group, Key: key}); err != nil { + t.Fatalf("Failed to delete %s/%s: %v", group, key, err) + } +} + +func assertNotFound(t *testing.T, kv *plugin.KVOperator, ctx context.Context, group, key string) { + val, err := kv.Get(ctx, plugin.KVParams{Group: group, Key: key}) + if err != plugin.ErrKVKeyNotFound { + t.Errorf("Expected ErrKVKeyNotFound for %s/%s, got: %v", group, key, err) + } + if val != "" { + t.Errorf("Expected empty value for %s/%s, got: '%s'", group, key, val) + } +} + +func assertError(t *testing.T, err error, expected error, msg string) { + if err != expected { + t.Errorf("%s: expected %v, got %v", msg, expected, err) + } +} + +// TestKVStoragePlugin implements KVStorage interface for testing +type TestKVStoragePlugin struct { + operator *plugin.KVOperator +} + +// Info returns plugin information +func (p *TestKVStoragePlugin) Info() plugin.Info { + return plugin.Info{ + Name: plugin.MakeTranslator("test_kv_storage_name"), + SlugName: "test_kv_storage", + Description: plugin.MakeTranslator("test_kv_storage_desc"), + Author: "Answer Team", + Version: "1.0.0", + Link: "https://github.com/apache/answer", + } +} + +// SetOperator sets KV operator +func (p *TestKVStoragePlugin) SetOperator(operator *plugin.KVOperator) { + p.operator = operator +} + +// setupTestEnvironment sets up test environment +func setupTestEnvironment() { + // Initialize only once + if testPlugin != nil { + return + } + + // Create and register test plugin + testPlugin = &TestKVStoragePlugin{} + plugin.Register(testPlugin) + + // Enable plugin + plugin.StatusManager.Enable("test_kv_storage", true) + + // Initialize plugin data, refer to plugin_common_service.go implementation + _ = plugin.CallKVStorage(func(k plugin.KVStorage) error { + k.SetOperator(plugin.NewKVOperator( + testDataSource.DB, + testDataSource.Cache, + k.Info().SlugName, + )) + return nil + }) +} + +// Test basic operations including CRUD and edge cases +func TestBasicOperations(t *testing.T) { + setupTestEnvironment() + kv := testPlugin.operator + ctx := context.Background() + + t.Run("BasicCRUD", func(t *testing.T) { + // Set/Get + mustSet(t, kv, ctx, "group1", "key1", "value1") + mustGet(t, kv, ctx, "group1", "key1", "value1") + + // Update + mustSet(t, kv, ctx, "group1", "key1", "new_value") + mustGet(t, kv, ctx, "group1", "key1", "new_value") + + // Delete + mustDel(t, kv, ctx, "group1", "key1") + assertNotFound(t, kv, ctx, "group1", "key1") + + // Group operation + mustSet(t, kv, ctx, "group1", "key2", "value2") + mustSet(t, kv, ctx, "group1", "key3", "value3") + groupData, err := kv.GetByGroup(ctx, plugin.KVParams{Group: "group1", Page: 1, PageSize: 10}) + if err != nil { + t.Fatalf("Failed to get group data: %v", err) + } + + // the groupData should only have key2 and key3 because key1 is deleted + if len(groupData) != 2 { + t.Errorf("Expected 2 items, got %d", len(groupData)) + } + if groupData["key2"] != "value2" || groupData["key3"] != "value3" { + t.Errorf("Unexpected group data: %v", groupData) + } + }) + + t.Run("EdgeCases", func(t *testing.T) { + // Empty key + err := kv.Set(ctx, plugin.KVParams{Group: "group", Key: "", Value: "value"}) + assertError(t, err, plugin.ErrKVKeyEmpty, "Empty key test") + + // Empty group query + _, err = kv.GetByGroup(ctx, plugin.KVParams{Group: "", Page: 1, PageSize: 10}) + assertError(t, err, plugin.ErrKVGroupEmpty, "Empty group test") + + // Non-existent key + assertNotFound(t, kv, ctx, "non_exist_group", "non_exist_key") + + // Cache penetration protection + key := fmt.Sprintf("non_exist_key_%d", time.Now().UnixNano()) + assertNotFound(t, kv, ctx, "cache_penetration", key) + }) + + t.Run("CacheConsistency", func(t *testing.T) { + mustSet(t, kv, ctx, "cache_group", "cache_key", "cache_value") + mustGet(t, kv, ctx, "cache_group", "cache_key", "cache_value") + + // Update and verify immediate consistency + mustSet(t, kv, ctx, "cache_group", "cache_key", "updated_value") + mustGet(t, kv, ctx, "cache_group", "cache_key", "updated_value") + }) +} + +// Test transactions including rollback and nested transactions +func TestTransactions(t *testing.T) { + setupTestEnvironment() + kv := testPlugin.operator + ctx := context.Background() + + t.Run("SuccessfulTransaction", func(t *testing.T) { + err := kv.Tx(ctx, func(ctx context.Context, txKv *plugin.KVOperator) error { + if err := txKv.Set(ctx, plugin.KVParams{Group: "tx_group", Key: "tx_key1", Value: "tx_value1"}); err != nil { + return err + } + if err := txKv.Set(ctx, plugin.KVParams{Group: "tx_group", Key: "tx_key2", Value: "tx_value2"}); err != nil { + return err + } + return nil + }) + if err != nil { + t.Fatalf("Successful transaction failed: %v", err) + } + + mustGet(t, kv, ctx, "tx_group", "tx_key1", "tx_value1") + mustGet(t, kv, ctx, "tx_group", "tx_key2", "tx_value2") + }) + + t.Run("TransactionRollback", func(t *testing.T) { + err := kv.Tx(ctx, func(ctx context.Context, txKv *plugin.KVOperator) error { + if err := txKv.Set(ctx, plugin.KVParams{Group: "tx_group", Key: "tx_key3", Value: "tx_value3"}); err != nil { + return err + } + return fmt.Errorf("mock error") + }) + if err == nil { + t.Error("Expected transaction to fail but it succeeded") + } + + assertNotFound(t, kv, ctx, "tx_group", "tx_key3") + }) + + t.Run("NestedTransactions", func(t *testing.T) { + err := kv.Tx(ctx, func(ctx context.Context, txKv *plugin.KVOperator) error { + if err := txKv.Set(ctx, plugin.KVParams{Group: "nested", Key: "key1", Value: "value1"}); err != nil { + return err + } + + return txKv.Tx(ctx, func(ctx context.Context, nestedKv *plugin.KVOperator) error { + if err := nestedKv.Set(ctx, plugin.KVParams{Group: "nested", Key: "key2", Value: "value2"}); err != nil { + return err + } + return fmt.Errorf("mock nested error") + }) + }) + if err == nil { + t.Error("Expected nested transaction to fail but it succeeded") + } + + // Verify outer transaction also rolled back + assertNotFound(t, kv, ctx, "nested", "key1") + assertNotFound(t, kv, ctx, "nested", "key2") + }) +} + +// Test pagination in GetByGroup +func TestPagination(t *testing.T) { + setupTestEnvironment() + kv := testPlugin.operator + ctx := context.Background() + totalItems := 25 + + for i := range totalItems { + mustSet(t, kv, ctx, "pagination", fmt.Sprintf("key%d", i), fmt.Sprintf("value%d", i)) + } + + // Test pagination + page1, err := kv.GetByGroup(ctx, plugin.KVParams{Group: "pagination", Page: 1, PageSize: 10}) + if err != nil { + t.Fatalf("Failed to get page 1: %v", err) + } + if len(page1) != 10 { + t.Errorf("Page 1: expected 10 items, got %d", len(page1)) + } + + page2, err := kv.GetByGroup(ctx, plugin.KVParams{Group: "pagination", Page: 2, PageSize: 10}) + if err != nil { + t.Fatalf("Failed to get page 2: %v", err) + } + if len(page2) != 10 { + t.Errorf("Page 2: expected 10 items, got %d", len(page2)) + } + + page3, err := kv.GetByGroup(ctx, plugin.KVParams{Group: "pagination", Page: 3, PageSize: 10}) + if err != nil { + t.Fatalf("Failed to get page 3: %v", err) + } + if len(page3) != 5 { + t.Errorf("Page 3: expected 5 items, got %d", len(page3)) + } + + // Verify different keys on different pages + for i := range 10 { + key := fmt.Sprintf("key%d", i) + if _, ok := page1[key]; !ok { + t.Errorf("Pagination test failed, key %s should be on page 1", key) + } + } + for i := range 10 { + key := fmt.Sprintf("key%d", i+10) + if _, ok := page2[key]; !ok { + t.Errorf("Pagination test failed, key %s should be on page 2", key) + } + } +} + +// Test concurrent operations and performance +func TestConcurrency(t *testing.T) { + setupTestEnvironment() + kv := testPlugin.operator + ctx := context.Background() + + t.Run("BasicConcurrency", func(t *testing.T) { + parallel := 10 + var wg sync.WaitGroup + wg.Add(parallel) + + for i := range parallel { + go func(index int) { + defer wg.Done() + time.Sleep(time.Duration(rand.Intn(200)) * time.Millisecond) + mustSet(t, kv, ctx, "concurrent", fmt.Sprintf("key%d", index), "value") + }(i) + } + wg.Wait() + + // Verify results + wg.Add(parallel) + for i := range parallel { + go func(index int) { + defer wg.Done() + time.Sleep(time.Duration(rand.Intn(200)) * time.Millisecond) + mustGet(t, kv, ctx, "concurrent", fmt.Sprintf("key%d", index), "value") + }(i) + } + wg.Wait() + }) + + t.Run("StressTest", func(t *testing.T) { + if testing.Short() { + t.Skip("Skipping stress test in short mode") + } + + totalOps := 1000 + workerCount := 20 + prefix := "stress_test" + opsPerWorker := totalOps / workerCount + + log.Info("Starting KV storage stress test...") + startTime := time.Now() + + // Concurrent write test + var wg sync.WaitGroup + errorCount := int64(0) + + for w := range workerCount { + wg.Add(1) + go func(workerID int) { + defer wg.Done() + startIdx := workerID * opsPerWorker + + for i := range opsPerWorker { + i := startIdx + i + err := kv.Set(ctx, plugin.KVParams{ + Group: prefix, + Key: fmt.Sprintf("key%d", i), + Value: fmt.Sprintf("value%d", i), + }) + if err != nil { + log.Warnf("Write error: %v", err) + errorCount++ + } + } + }(w) + } + wg.Wait() + + writeTime := time.Since(startTime) + + // Verify data integrity + groupData, err := kv.GetByGroup(ctx, plugin.KVParams{Group: prefix, Page: 1, PageSize: totalOps}) + if err != nil { + t.Fatalf("Failed to verify data: %v", err) + } + if len(groupData) != totalOps { + t.Errorf("Data loss: expected %d items, got %d", totalOps, len(groupData)) + } + + // Concurrent read test + startTime = time.Now() + readErrors := int64(0) + + wg.Add(workerCount) + for range workerCount { + go func() { + defer wg.Done() + for range opsPerWorker { + keyIdx := rand.Intn(totalOps) + key := fmt.Sprintf("key%d", keyIdx) + expected := fmt.Sprintf("value%d", keyIdx) + + val, err := kv.Get(ctx, plugin.KVParams{Group: prefix, Key: key}) + if err != nil { + readErrors++ + } else if val != expected { + t.Errorf("Data inconsistency: key=%s, expected=%s, got=%s", key, expected, val) + } + } + }() + } + wg.Wait() + + readTime := time.Since(startTime) + + log.Infof("Stress test completed:") + log.Infof(" Write: %d ops in %v (%.1f ops/sec), %d errors", totalOps, writeTime, float64(totalOps)/writeTime.Seconds(), errorCount) + log.Infof(" Read: %d ops in %v (%.1f ops/sec), %d errors", totalOps, readTime, float64(totalOps)/readTime.Seconds(), readErrors) + }) +} diff --git a/plugin/plugin_test/plugin_main_test.go b/plugin/plugin_test/plugin_main_test.go new file mode 100644 index 000000000..4f92bebe9 --- /dev/null +++ b/plugin/plugin_test/plugin_main_test.go @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package plugin_test + +import ( + "context" + "database/sql" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/apache/answer/internal/base/data" + "github.com/apache/answer/internal/migrations" + "github.com/ory/dockertest/v3" + "github.com/ory/dockertest/v3/docker" + "github.com/segmentfault/pacman/cache" + "github.com/segmentfault/pacman/log" + "xorm.io/xorm" + "xorm.io/xorm/schemas" +) + +var ( + mysqlDBSetting = TestDBSetting{ + Driver: string(schemas.MYSQL), + ImageName: "mariadb", + ImageVersion: "10.4.7", + ENV: []string{"MYSQL_ROOT_PASSWORD=root", "MYSQL_DATABASE=answer", "MYSQL_ROOT_HOST=%"}, + PortID: "3306/tcp", + Connection: "root:root@(localhost:%s)/answer?parseTime=true", // port is not fixed, it will be got by port id + } + postgresDBSetting = TestDBSetting{ + Driver: string(schemas.POSTGRES), + ImageName: "postgres", + ImageVersion: "14", + ENV: []string{"POSTGRES_USER=root", "POSTGRES_PASSWORD=root", "POSTGRES_DB=answer", "LISTEN_ADDRESSES='*'"}, + PortID: "5432/tcp", + Connection: "host=localhost port=%s user=root password=root dbname=answer sslmode=disable", + } + sqlite3DBSetting = TestDBSetting{ + Driver: string(schemas.SQLITE), + Connection: filepath.Join(os.TempDir(), "answer-test-data.db"), + } + dbSettingMapping = map[string]TestDBSetting{ + mysqlDBSetting.Driver: mysqlDBSetting, + sqlite3DBSetting.Driver: sqlite3DBSetting, + postgresDBSetting.Driver: postgresDBSetting, + } + // after all test down will execute tearDown function to clean-up + tearDown func() + // testDataSource used for repo testing + testDataSource *data.Data + testCache cache.Cache +) + +func TestMain(t *testing.M) { + dbSetting, ok := dbSettingMapping[os.Getenv("TEST_DB_DRIVER")] + if !ok { + // Use sqlite3 to test. + dbSetting = dbSettingMapping[string(schemas.SQLITE)] + } + if dbSetting.Driver == string(schemas.SQLITE) { + os.RemoveAll(dbSetting.Connection) + } + + defer func() { + if tearDown != nil { + tearDown() + } + }() + if err := initTestDataSource(dbSetting); err != nil { + panic(err) + } + log.Info("init test database successfully") + + if ret := t.Run(); ret != 0 { + panic(ret) + } +} + +type TestDBSetting struct { + Driver string + ImageName string + ImageVersion string + ENV []string + PortID string + Connection string +} + +func initTestDataSource(dbSetting TestDBSetting) error { + connection, imageCleanUp, err := initDatabaseImage(dbSetting) + if err != nil { + return err + } + dbSetting.Connection = connection + + dbEngine, err := initDatabase(dbSetting) + if err != nil { + return err + } + + newCache, err := initCache() + if err != nil { + return err + } + + newData, dbCleanUp, err := data.NewData(dbEngine, newCache) + if err != nil { + return err + } + testDataSource = newData + testCache = newCache + + tearDown = func() { + dbCleanUp() + log.Info("cleanup test database successfully") + imageCleanUp() + log.Info("cleanup test database image successfully") + } + return nil +} + +func initDatabaseImage(dbSetting TestDBSetting) (connection string, cleanup func(), err error) { + // sqlite3 don't need to set up image + if dbSetting.Driver == string(schemas.SQLITE) { + return dbSetting.Connection, func() { + log.Info("remove database", dbSetting.Connection) + err = os.Remove(dbSetting.Connection) + if err != nil { + log.Error(err) + } + }, nil + } + pool, err := dockertest.NewPool("") + pool.MaxWait = time.Minute * 5 + if err != nil { + return "", nil, fmt.Errorf("could not connect to docker: %s", err) + } + + //resource, err := pool.Run(dbSetting.ImageName, dbSetting.ImageVersion, dbSetting.ENV) + resource, err := pool.RunWithOptions(&dockertest.RunOptions{ + Repository: dbSetting.ImageName, + Tag: dbSetting.ImageVersion, + Env: dbSetting.ENV, + }, func(config *docker.HostConfig) { + config.AutoRemove = true + config.RestartPolicy = docker.RestartPolicy{Name: "no"} + }) + if err != nil { + return "", nil, fmt.Errorf("could not pull resource: %s", err) + } + + connection = fmt.Sprintf(dbSetting.Connection, resource.GetPort(dbSetting.PortID)) + if err := pool.Retry(func() error { + db, err := sql.Open(dbSetting.Driver, connection) + if err != nil { + return err + } + return db.Ping() + }); err != nil { + return "", nil, fmt.Errorf("could not connect to database: %s", err) + } + return connection, func() { _ = pool.Purge(resource) }, nil +} + +func initDatabase(dbSetting TestDBSetting) (dbEngine *xorm.Engine, err error) { + dataConf := &data.Database{Driver: dbSetting.Driver, Connection: dbSetting.Connection} + dbEngine, err = data.NewDB(true, dataConf) + if err != nil { + return nil, fmt.Errorf("connection to database failed: %s", err) + } + if err := migrations.NewMentor(context.TODO(), dbEngine, &migrations.InitNeedUserInputData{ + Language: "en_US", + SiteName: "ANSWER", + SiteURL: "http://127.0.0.1:8080/", + ContactEmail: "answer@answer.com", + AdminName: "admin", + AdminPassword: "admin", + AdminEmail: "answer@answer.com", + }).InitDB(); err != nil { + return nil, fmt.Errorf("migrations init database failed: %s", err) + } + return dbEngine, nil +} + +func initCache() (newCache cache.Cache, err error) { + newCache, _, err = data.NewCache(&data.CacheConf{}) + return newCache, err +} From 4175b06039e3059b5e6ecf4cc81b2be5a07c52a8 Mon Sep 17 00:00:00 2001 From: Sonui Date: Mon, 26 May 2025 00:56:08 +0800 Subject: [PATCH 38/39] perf(tests): replace panic with os.Exit in TestMain for graceful test termination --- plugin/plugin_test/plugin_main_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/plugin_test/plugin_main_test.go b/plugin/plugin_test/plugin_main_test.go index 4f92bebe9..add11460b 100644 --- a/plugin/plugin_test/plugin_main_test.go +++ b/plugin/plugin_test/plugin_main_test.go @@ -92,7 +92,7 @@ func TestMain(t *testing.M) { log.Info("init test database successfully") if ret := t.Run(); ret != 0 { - panic(ret) + os.Exit(ret) } } From bd032dff36d3a6431dcece86e9687d1f0c2ef87c Mon Sep 17 00:00:00 2001 From: LinkinStars Date: Wed, 28 May 2025 10:16:27 +0800 Subject: [PATCH 39/39] upgrade(docs): upgrade to 1.5.1 --- Makefile | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 62981b789..d88d5085b 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ .PHONY: build clean ui -VERSION=1.5.0 +VERSION=1.5.1 BIN=answer DIR_SRC=./cmd/answer DOCKER_CMD=docker diff --git a/README.md b/README.md index 3e1a1bcfe..8f5ae243b 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ To learn more about the project, visit [answer.apache.org](https://answer.apache ### Running with docker ```bash -docker run -d -p 9080:80 -v answer-data:/data --name answer apache/answer:1.5.0 +docker run -d -p 9080:80 -v answer-data:/data --name answer apache/answer:1.5.1 ``` For more information, see [Installation](https://answer.apache.org/docs/installation).