From ca4a02d5df32f9066c8620689f3c8bf8823c0165 Mon Sep 17 00:00:00 2001 From: Srijan Verma Date: Sat, 18 Apr 2026 12:34:02 +0530 Subject: [PATCH 1/9] feat: Implemented evictallkeysrandom --- core/eviction.go | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/core/eviction.go b/core/eviction.go index b162ab3..d724dc4 100644 --- a/core/eviction.go +++ b/core/eviction.go @@ -7,6 +7,8 @@ whenevr a cache is full, we will be evicting the first key which we do find package core +import "github.com/sharpsalt/Velox-In-Memory-Database/config" + //Evcits the first key it found while iterating the map //TODP: Make it efficient by doing thrugh somehting func evictFirst(){ @@ -16,9 +18,36 @@ func evictFirst(){ } } +//Randomly removes keys to make space for the new data added +//The number of keys removed will be sufficient to free up least 10% space +func evictAllKeysRandom(){ + evictCount:=int64(config.EvictionRatio*float64(config.KeysLimit)) + //Iteration of Golang dictionary can be considered as a random + //because it depends on the has of the inserted key + for k:= range store{ + Del(k) + evictCount-- + if evictCount==0{ + break + } + } +} +/* +How do we know that our eviction is working correctly?? + +which means when we are doing large number of key sets , we are putting lots of keys in db , let's say we put kelimit to 100 +, how do we sure that we are not breachng it , so every dbs supports statistics +*/ + //TODO: Make the eveiction strategy configuration dirven //TODO: Support multiple eviction strategies func evict(){ - evictFirst() + // evictFirst() + switch config.EvictionStrategy{ + case "simple-first": + evictfirst() + case "allkets-random": + evictAllkeysRandom() + } } From 4ca2614b7f5d505ec0971c5fe1f07fb39b8a38cc Mon Sep 17 00:00:00 2001 From: Srijan Verma Date: Thu, 23 Apr 2026 06:05:10 +0530 Subject: [PATCH 2/9] feat: Implemented INFI,CLIENT,LATENCY of eval each --- core/eval.go | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/core/eval.go b/core/eval.go index d07d166..8e23448 100644 --- a/core/eval.go +++ b/core/eval.go @@ -140,7 +140,10 @@ func evalGET(args []string)[]byte{ return RESP_NIL } - + //if key already expired then return nil + if hasExpired(obj){ + return RESP_NIL + } //return te RESP encoded value // c.Write(Encode(obj.Value,false)) // return nil @@ -184,6 +187,20 @@ func evalTTL(args []string) []byte{ return RESP_MINUS_2 } + exp.isExpirySet:=getExpiry(obj) + if !isExpirySet{ + return RESP_MINUS_1 + } + + //if key expired i.e key does not exist hence return -2 + if uint64(time.Now().UnixMilli())>exp{ + return RESP_MINUS_2 + } + + //compute the time remaining for the key to expire and + //return the RESP encoded form of it + durationMS:=exp-uint64(time.Now().UnixMilli()) + // c.Write(Encode(int64(durationMS/1000),false)) // return nil return Encode(int64(durationMs/1000), false) @@ -220,7 +237,9 @@ func evalEXPIRE(args []string) []byte{ return RESP_ZERO } - obj.ExpiresAt = time.Now().UnixMilli() + exDurationSec*1000 + // obj.ExpiresAt = time.Now().UnixMilli() + exDurationSec*1000 + + setExpiry(obj,exDurationSec*1000) //1 print krenge if the timeout was set // c.Write([]byte(":1\r\n")) @@ -234,6 +253,14 @@ func evalBGREWRITEAOF(args []string) []byte{ return RESP_OK } +func evalINFO(args []string)[]byte{ + var info []byte + buf:=bytes.NewBuffer(info) + for i := range KeyspaceStat{ + buf.WriteString(fmt.Sprintf("db%d:keys=%d,expire=0,avg_ttl=0\r\n",i,KeyspaceStat[i]["keys"])) + } +} + // func EvalAndRespond(cmd *Rediscmd,c net.Conn)error{ func EvalAndRespond(cmds []*RedisCmd, c io.ReadWriter) error{ //It's job is like depending on what job is sent to us @@ -260,6 +287,12 @@ func EvalAndRespond(cmds []*RedisCmd, c io.ReadWriter) error{ buf.Write(evalBGREWRITEAOF(cmd.Args)) case "INCR": buf.Write(evalPING(cmd.Args)) + case "INFO": + buf.Write(evalINFO(cmd.Args)) + case "CLIENT": + buf.Write(evalCLIENT(cmd.Args)) + case "LATENCY": + buf.Write(evalLATENCY(cmd.Args)) default: buf.Write(evalPING(cmd.Args)) } From ca9034b1cd27df9bd85620bb5a75a083c3d1e9a7 Mon Sep 17 00:00:00 2001 From: Srijan Verma Date: Thu, 23 Apr 2026 06:06:31 +0530 Subject: [PATCH 3/9] feat: Implemented getCurrentClock(),getIdleTime(),populateEvictionPool(),evictAllKeysLRU() --- core/eviction.go | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/core/eviction.go b/core/eviction.go index d724dc4..60c4094 100644 --- a/core/eviction.go +++ b/core/eviction.go @@ -18,6 +18,49 @@ func evictFirst(){ } } +/* +The approximated LRU algorithm +*/ +func getCurrentClock() uint32{ + return uint32(time.Now().Unix())&0x00FFFFF // it would give us 24 bit clock resolution of the time +} + +func getIdleTime(LastAccessdAt uint32) uint32{ + /* + it gives me current clock and if it is greater than lastaccessedat then for anytimestamp then diff, other (max-last)*c + */ + c:=getCurrentClock() + if c>=LastAccessdAt{ + return c-LastAccessdAt + } + return (0x00FFFFF-LastAccessdAt)*c +} + +func populateEvictionPool(){ + sampleSize:=5 + for k:=range store{ + ePool.Push(k,store[k].lastaccessedat) + sampleSize-- + if sampleSize==0{ + break + } + } +} + +//TODO: no need to populate everytime, should populate +//only when the number of keys to evict is less than what we have in the pool +funct evictAllKeysLRU(){ + populateEvictionPool() + evictCount:=int16(config.EvictionRatio=float64(config.KeysLimit)) + for i:=;i0 ;i++{ + item:=ePool.Pop() + if item==nil{ + return + } + Del(item.key) + } +} + //Randomly removes keys to make space for the new data added //The number of keys removed will be sufficient to free up least 10% space func evictAllKeysRandom(){ @@ -49,5 +92,7 @@ func evict(){ evictfirst() case "allkets-random": evictAllkeysRandom() + case: "allkeys-lru": + evictAllKeysLRU() } } From f3b6d91b35817c72edf3e87111b289a2049005b8 Mon Sep 17 00:00:00 2001 From: Srijan Verma Date: Thu, 23 Apr 2026 06:07:11 +0530 Subject: [PATCH 4/9] feat: Implemented hasExpire() --- core/expire.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/core/expire.go b/core/expire.go index 0b01f22..3390c28 100644 --- a/core/expire.go +++ b/core/expire.go @@ -5,6 +5,19 @@ import ( "time" ) +func hasExpired(obj *Obj) bool{ + exp,ok:=expires[obj] + if !ok{ + return false + } + return exp<=uint64(time.Now().UnixMilli()) +} + +func getExpiry(obj *Obj)(uint64,bool){ + exp,ok:=expires[obj] + return exp,ok +} + //Delete all the expired keys- the active way //Sampling approach: https://redis.io/commands/expire/ From 6f55d55ec39538e50e799b78082b3593a1cd6ad2 Mon Sep 17 00:00:00 2001 From: Srijan Verma Date: Thu, 23 Apr 2026 06:07:59 +0530 Subject: [PATCH 5/9] feat: replaced ExpireAt with LastAccessedAt --- core/object.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/core/object.go b/core/object.go index fb83dfd..b2095d7 100644 --- a/core/object.go +++ b/core/object.go @@ -4,8 +4,21 @@ package core //TODO: Change ExpiresAt it to LRU Bits as handled by Redis type Obj struct{ TypeEncoding uint8 - Value interface{} - ExpiresAt int64 + Value interface{} //which means we acn put literally anything,anyvalue and which would work fine + // ExpiresAt int64 + /* + Earlier we use to have expireAt here, we don't need as we need LastAccessdAt + since golang doesn't support bitfield so LastAccessedAt is storing everytime the key is getting accessed + */ + + /* + Redis allot 24 bits to these bits, but we will use 32 bits because + golang does not support bitfields and we need not make this super-compplicated + by merging TypeEncoding + LastAccessedAt in one 32 bit integer + but nonthelss, we can benchmarks and see how that fares + For noe, we continue with 32 bit integer to stre th LastAccessedAt + */ + LastAccessedAt uint32 } /* C and c++ give us ways to apply bit fields on set, means in each set i can assign a fix number of bits From d7cdaa913470253dcb96e56a7867189b20234912 Mon Sep 17 00:00:00 2001 From: Srijan Verma Date: Thu, 23 Apr 2026 06:08:36 +0530 Subject: [PATCH 6/9] feat: Implemented setExpiry --- core/store.go | 43 ++++++++++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/core/store.go b/core/store.go index 4170c03..8054364 100644 --- a/core/store.go +++ b/core/store.go @@ -2,36 +2,46 @@ package core import ( "time" - + "store.go" "github.com/sharpsalt/Velox-In-Memory-Database/config" ) var store map[string]*Obj //the best datastrcuture to hold key value is hash table so we are using it -type Obj struct{ - Value interface{} //which means we acn put literally anything,anyvalue and which would work fine - ExpiresAt int64 //is time ke baad expire hojayega wo -} +var expires map[*Obj]uint64; //similar to redis like redis has key value dictionary, adn it has expires +//so we are storing object pointer walong with expiration time + func init(){ store=make(map[string]*Obj) + expires=make(map[*Obj]uint64) +} + +func setExpiry(obj *Obj,expDurationMs int64){ + expires[obj]=uint64(time.Now().UnixMilli())+uint64(expDurationMs) } //eralier we used to take (value interface{},DurationMs int64) //but now we are also stroing type encoding so, 1 for object type and 1 for encdojgn -func NewObj(value interface{},DurationMs int64,oType uint8,oEnc uint8) *Obj{ +func NewObj(value interface{},expDurationMs int64,oType uint8,oEnc uint8) *Obj{ //creating a new object, setting things up and returning another object //since we want to store abolution expires instead of doing it multiple time that's why we have created this fucntion var expiresAt int64=-1 - if DurationMs>0{ - expiresAt=time.Now().UnixMilli()+DurationMs + if expDurationMs>0{ + /* + when we say setExpiry? + + it means we have to create an enrty of particular object that is expired dictionary + */ + setExpiry(obj,expDurationMs) } return &Obj{ Value: value, TypeEncoding: oType|oEnc, - ExpiresAt: expiresAt, + // ExpiresAt: expiresAt, + LastAccessedAt: getCurrentClock(), } } @@ -47,7 +57,13 @@ func Put(k string, obj *Obj){ if len(store) >= config.KeysLimit{ evict() } + obj.LastAccessedAt=getCurrentClock() store[k] = obj + if KeyspaceStat[0]==nil{ + KeyspaceStat[0]=make(map[string]int) + } + KeyspaceStat[0]["keys"]++ + //actually you can use grafana to visualize it, like it's easy , you just have to knwo how things work } func Get(k string) *Obj{ @@ -62,16 +78,21 @@ func Get(k string) *Obj{ periodicaly it is mvoing forward and sample randomly 20 keys and seees the expiration and delete the required one and phir se whi loop chalao */ - delete(store,k) - return nil + if hasExpired(v){ + Del(k) + return nil + } } } + v.LastAccessedAt=getCurrentClock() return v } func Del(k string ) bool{ if _,ok:=store[k];ok{ delete(store,k) + delete(expire,_) + KeyspaceStat[0]["keys"]-- return true } return false From 461c28c409884929a12e22361ff838dff8fa39bd Mon Sep 17 00:00:00 2001 From: Srijan Verma Date: Thu, 23 Apr 2026 06:09:36 +0530 Subject: [PATCH 7/9] feat: setted up configfor eviction,aof,keyslimit,etc --- config/main.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 config/main.go diff --git a/config/main.go b/config/main.go new file mode 100644 index 0000000..b8182bf --- /dev/null +++ b/config/main.go @@ -0,0 +1,14 @@ +package config + +var Host string="0.0.0.0" +var Port int=7379 + +var KeysLimit int=100 //like ye hum threshold set krdiye hai like our database still support atmax this many keys + +//will evict EvictionRation of keys whenever evictionruns +//it would dictate whenever my eviction is triggered, how many keys i will be evicting. +var EvictionRatio flaot64=0.40 //Generally aise real scenario me kabhi itna to nahi hi krte hai but still + +var EvictionStrategy string="allkeys-random" +var AOFFile string="./velox.aof" + From a0f44052b7235e52e4e630d46317d6fe42616f8a Mon Sep 17 00:00:00 2001 From: Srijan Verma Date: Thu, 23 Apr 2026 06:10:25 +0530 Subject: [PATCH 8/9] feat: Implemented evictionPool from scratch --- core/evictionpool.go | 80 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 core/evictionpool.go diff --git a/core/evictionpool.go b/core/evictionpool.go new file mode 100644 index 0000000..9a2d721 --- /dev/null +++ b/core/evictionpool.go @@ -0,0 +1,80 @@ +package core + +import{ + "sort" +} + + +type PoolItem struct{ + key string + lastaccessedat uint32 +} + +//TODO: when last accessed at of object changes +//updates the poolItem corresponding to that +type EvictionPool struct{ + pool []*PoolItem //it is basically an array of pool items + keyset map[string]*PoolItem //na keyset , like the key which are present. +} + +type ByIdleTime []*PoolItem + +func (a ByIdleTime) len() int{ + return len(a) +} + +func (a ByIdleTime) Swap(i int,j int){ + a[i],a[j]=a[j],a[i] +} + +func (a ByIdleTime) Less(i int,j int) bool{//basically it is a comparator function which i am suing to sort the time + return getIdleTime(a[i].lastaccessedat)>getIdleTime(a[j].lastaccessedat) +} + +//TODO: Make the Implementation efficient to not need repeated sorting +func (pq *EvictionPool) Push(key string,lastaccessedat uint32){ + _,ok:=pq.keyset[key] + if ok{ + //while pushing it into eviction pool if it already exists then we don;t have to push it again it in eveiction pool + return + } + ietm:=&PoolItem(key:key,lastaccessedat:lastaccessedat) + if len(pq.pool)pq.pool[len(pq.pool)-1].lastaccessedat{ + //if i have no space in eviction pool but the element which i have smapled is worse than my current + //i will create space for that by removing the 1st one and adding new element by appending it + ///so this way we are ensuring that our pool contains best possible candidates to be evcited + pq.pool=pq.pool[1:] + pq.keyset[key]=item + pq.pool=append(pq.pool,item) + } +} + +func (pq *EvictionPool) Pop() *PoolItem{ + if len(pq.pool)==0{ + return nil + } + item:=pq.pool[0] + pq.pool=pq.pool[1:] + return item +} + +func newEvictionPool(size int) *EvictionPool{ + return &EvictionPool{ + pool: make([]*PoolItem,size) + keyset: make(map[string]*PoolItem) + } +} + +var ePoolSizeMax int=16 +var ePool *EvictionPool=newEvictionPool(0) + + From d5e620d713031fea1413f7738eead37f36238d28 Mon Sep 17 00:00:00 2001 From: Srijan Verma Date: Thu, 23 Apr 2026 06:10:56 +0530 Subject: [PATCH 9/9] feat: Implemented UpdateDBState --- core/stats.go | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 core/stats.go diff --git a/core/stats.go b/core/stats.go new file mode 100644 index 0000000..e9e4233 --- /dev/null +++ b/core/stats.go @@ -0,0 +1,10 @@ +package core + +var KeyspaceStat [4]map[string]int //just all global object +//eg i support 4 databases within my redis +//in redis you can have 16 databases in redis itself, by deault it goes from db0,db1,db2,...db15 + +//for which db,which metric,which value +func UpdatDBStat(num int,metric string,value int){ + KeyspaceStat[num][metric]=value +}