diff --git a/config/main.go b/config/main.go new file mode 100644 index 0000000..b8182bf --- /dev/null +++ b/config/main.go @@ -0,0 +1,14 @@ +package config + +var Host string="0.0.0.0" +var Port int=7379 + +var KeysLimit int=100 //like ye hum threshold set krdiye hai like our database still support atmax this many keys + +//will evict EvictionRation of keys whenever evictionruns +//it would dictate whenever my eviction is triggered, how many keys i will be evicting. +var EvictionRatio flaot64=0.40 //Generally aise real scenario me kabhi itna to nahi hi krte hai but still + +var EvictionStrategy string="allkeys-random" +var AOFFile string="./velox.aof" + diff --git a/core/eval.go b/core/eval.go index d07d166..8e23448 100644 --- a/core/eval.go +++ b/core/eval.go @@ -140,7 +140,10 @@ func evalGET(args []string)[]byte{ return RESP_NIL } - + //if key already expired then return nil + if hasExpired(obj){ + return RESP_NIL + } //return te RESP encoded value // c.Write(Encode(obj.Value,false)) // return nil @@ -184,6 +187,20 @@ func evalTTL(args []string) []byte{ return RESP_MINUS_2 } + exp.isExpirySet:=getExpiry(obj) + if !isExpirySet{ + return RESP_MINUS_1 + } + + //if key expired i.e key does not exist hence return -2 + if uint64(time.Now().UnixMilli())>exp{ + return RESP_MINUS_2 + } + + //compute the time remaining for the key to expire and + //return the RESP encoded form of it + durationMS:=exp-uint64(time.Now().UnixMilli()) + // c.Write(Encode(int64(durationMS/1000),false)) // return nil return Encode(int64(durationMs/1000), false) @@ -220,7 +237,9 @@ func evalEXPIRE(args []string) []byte{ return RESP_ZERO } - obj.ExpiresAt = time.Now().UnixMilli() + exDurationSec*1000 + // obj.ExpiresAt = time.Now().UnixMilli() + exDurationSec*1000 + + setExpiry(obj,exDurationSec*1000) //1 print krenge if the timeout was set // c.Write([]byte(":1\r\n")) @@ -234,6 +253,14 @@ func evalBGREWRITEAOF(args []string) []byte{ return RESP_OK } +func evalINFO(args []string)[]byte{ + var info []byte + buf:=bytes.NewBuffer(info) + for i := range KeyspaceStat{ + buf.WriteString(fmt.Sprintf("db%d:keys=%d,expire=0,avg_ttl=0\r\n",i,KeyspaceStat[i]["keys"])) + } +} + // func EvalAndRespond(cmd *Rediscmd,c net.Conn)error{ func EvalAndRespond(cmds []*RedisCmd, c io.ReadWriter) error{ //It's job is like depending on what job is sent to us @@ -260,6 +287,12 @@ func EvalAndRespond(cmds []*RedisCmd, c io.ReadWriter) error{ buf.Write(evalBGREWRITEAOF(cmd.Args)) case "INCR": buf.Write(evalPING(cmd.Args)) + case "INFO": + buf.Write(evalINFO(cmd.Args)) + case "CLIENT": + buf.Write(evalCLIENT(cmd.Args)) + case "LATENCY": + buf.Write(evalLATENCY(cmd.Args)) default: buf.Write(evalPING(cmd.Args)) } diff --git a/core/eviction.go b/core/eviction.go index b162ab3..60c4094 100644 --- a/core/eviction.go +++ b/core/eviction.go @@ -7,6 +7,8 @@ whenevr a cache is full, we will be evicting the first key which we do find package core +import "github.com/sharpsalt/Velox-In-Memory-Database/config" + //Evcits the first key it found while iterating the map //TODP: Make it efficient by doing thrugh somehting func evictFirst(){ @@ -16,9 +18,81 @@ func evictFirst(){ } } +/* +The approximated LRU algorithm +*/ +func getCurrentClock() uint32{ + return uint32(time.Now().Unix())&0x00FFFFF // it would give us 24 bit clock resolution of the time +} + +func getIdleTime(LastAccessdAt uint32) uint32{ + /* + it gives me current clock and if it is greater than lastaccessedat then for anytimestamp then diff, other (max-last)*c + */ + c:=getCurrentClock() + if c>=LastAccessdAt{ + return c-LastAccessdAt + } + return (0x00FFFFF-LastAccessdAt)*c +} + +func populateEvictionPool(){ + sampleSize:=5 + for k:=range store{ + ePool.Push(k,store[k].lastaccessedat) + sampleSize-- + if sampleSize==0{ + break + } + } +} + +//TODO: no need to populate everytime, should populate +//only when the number of keys to evict is less than what we have in the pool +funct evictAllKeysLRU(){ + populateEvictionPool() + evictCount:=int16(config.EvictionRatio=float64(config.KeysLimit)) + for i:=;i0 ;i++{ + item:=ePool.Pop() + if item==nil{ + return + } + Del(item.key) + } +} + +//Randomly removes keys to make space for the new data added +//The number of keys removed will be sufficient to free up least 10% space +func evictAllKeysRandom(){ + evictCount:=int64(config.EvictionRatio*float64(config.KeysLimit)) + //Iteration of Golang dictionary can be considered as a random + //because it depends on the has of the inserted key + for k:= range store{ + Del(k) + evictCount-- + if evictCount==0{ + break + } + } +} +/* +How do we know that our eviction is working correctly?? + +which means when we are doing large number of key sets , we are putting lots of keys in db , let's say we put kelimit to 100 +, how do we sure that we are not breachng it , so every dbs supports statistics +*/ + //TODO: Make the eveiction strategy configuration dirven //TODO: Support multiple eviction strategies func evict(){ - evictFirst() + // evictFirst() + switch config.EvictionStrategy{ + case "simple-first": + evictfirst() + case "allkets-random": + evictAllkeysRandom() + case: "allkeys-lru": + evictAllKeysLRU() + } } diff --git a/core/evictionpool.go b/core/evictionpool.go new file mode 100644 index 0000000..9a2d721 --- /dev/null +++ b/core/evictionpool.go @@ -0,0 +1,80 @@ +package core + +import{ + "sort" +} + + +type PoolItem struct{ + key string + lastaccessedat uint32 +} + +//TODO: when last accessed at of object changes +//updates the poolItem corresponding to that +type EvictionPool struct{ + pool []*PoolItem //it is basically an array of pool items + keyset map[string]*PoolItem //na keyset , like the key which are present. +} + +type ByIdleTime []*PoolItem + +func (a ByIdleTime) len() int{ + return len(a) +} + +func (a ByIdleTime) Swap(i int,j int){ + a[i],a[j]=a[j],a[i] +} + +func (a ByIdleTime) Less(i int,j int) bool{//basically it is a comparator function which i am suing to sort the time + return getIdleTime(a[i].lastaccessedat)>getIdleTime(a[j].lastaccessedat) +} + +//TODO: Make the Implementation efficient to not need repeated sorting +func (pq *EvictionPool) Push(key string,lastaccessedat uint32){ + _,ok:=pq.keyset[key] + if ok{ + //while pushing it into eviction pool if it already exists then we don;t have to push it again it in eveiction pool + return + } + ietm:=&PoolItem(key:key,lastaccessedat:lastaccessedat) + if len(pq.pool)pq.pool[len(pq.pool)-1].lastaccessedat{ + //if i have no space in eviction pool but the element which i have smapled is worse than my current + //i will create space for that by removing the 1st one and adding new element by appending it + ///so this way we are ensuring that our pool contains best possible candidates to be evcited + pq.pool=pq.pool[1:] + pq.keyset[key]=item + pq.pool=append(pq.pool,item) + } +} + +func (pq *EvictionPool) Pop() *PoolItem{ + if len(pq.pool)==0{ + return nil + } + item:=pq.pool[0] + pq.pool=pq.pool[1:] + return item +} + +func newEvictionPool(size int) *EvictionPool{ + return &EvictionPool{ + pool: make([]*PoolItem,size) + keyset: make(map[string]*PoolItem) + } +} + +var ePoolSizeMax int=16 +var ePool *EvictionPool=newEvictionPool(0) + + diff --git a/core/expire.go b/core/expire.go index 0b01f22..3390c28 100644 --- a/core/expire.go +++ b/core/expire.go @@ -5,6 +5,19 @@ import ( "time" ) +func hasExpired(obj *Obj) bool{ + exp,ok:=expires[obj] + if !ok{ + return false + } + return exp<=uint64(time.Now().UnixMilli()) +} + +func getExpiry(obj *Obj)(uint64,bool){ + exp,ok:=expires[obj] + return exp,ok +} + //Delete all the expired keys- the active way //Sampling approach: https://redis.io/commands/expire/ diff --git a/core/object.go b/core/object.go index fb83dfd..b2095d7 100644 --- a/core/object.go +++ b/core/object.go @@ -4,8 +4,21 @@ package core //TODO: Change ExpiresAt it to LRU Bits as handled by Redis type Obj struct{ TypeEncoding uint8 - Value interface{} - ExpiresAt int64 + Value interface{} //which means we acn put literally anything,anyvalue and which would work fine + // ExpiresAt int64 + /* + Earlier we use to have expireAt here, we don't need as we need LastAccessdAt + since golang doesn't support bitfield so LastAccessedAt is storing everytime the key is getting accessed + */ + + /* + Redis allot 24 bits to these bits, but we will use 32 bits because + golang does not support bitfields and we need not make this super-compplicated + by merging TypeEncoding + LastAccessedAt in one 32 bit integer + but nonthelss, we can benchmarks and see how that fares + For noe, we continue with 32 bit integer to stre th LastAccessedAt + */ + LastAccessedAt uint32 } /* C and c++ give us ways to apply bit fields on set, means in each set i can assign a fix number of bits diff --git a/core/stats.go b/core/stats.go new file mode 100644 index 0000000..e9e4233 --- /dev/null +++ b/core/stats.go @@ -0,0 +1,10 @@ +package core + +var KeyspaceStat [4]map[string]int //just all global object +//eg i support 4 databases within my redis +//in redis you can have 16 databases in redis itself, by deault it goes from db0,db1,db2,...db15 + +//for which db,which metric,which value +func UpdatDBStat(num int,metric string,value int){ + KeyspaceStat[num][metric]=value +} diff --git a/core/store.go b/core/store.go index 4170c03..8054364 100644 --- a/core/store.go +++ b/core/store.go @@ -2,36 +2,46 @@ package core import ( "time" - + "store.go" "github.com/sharpsalt/Velox-In-Memory-Database/config" ) var store map[string]*Obj //the best datastrcuture to hold key value is hash table so we are using it -type Obj struct{ - Value interface{} //which means we acn put literally anything,anyvalue and which would work fine - ExpiresAt int64 //is time ke baad expire hojayega wo -} +var expires map[*Obj]uint64; //similar to redis like redis has key value dictionary, adn it has expires +//so we are storing object pointer walong with expiration time + func init(){ store=make(map[string]*Obj) + expires=make(map[*Obj]uint64) +} + +func setExpiry(obj *Obj,expDurationMs int64){ + expires[obj]=uint64(time.Now().UnixMilli())+uint64(expDurationMs) } //eralier we used to take (value interface{},DurationMs int64) //but now we are also stroing type encoding so, 1 for object type and 1 for encdojgn -func NewObj(value interface{},DurationMs int64,oType uint8,oEnc uint8) *Obj{ +func NewObj(value interface{},expDurationMs int64,oType uint8,oEnc uint8) *Obj{ //creating a new object, setting things up and returning another object //since we want to store abolution expires instead of doing it multiple time that's why we have created this fucntion var expiresAt int64=-1 - if DurationMs>0{ - expiresAt=time.Now().UnixMilli()+DurationMs + if expDurationMs>0{ + /* + when we say setExpiry? + + it means we have to create an enrty of particular object that is expired dictionary + */ + setExpiry(obj,expDurationMs) } return &Obj{ Value: value, TypeEncoding: oType|oEnc, - ExpiresAt: expiresAt, + // ExpiresAt: expiresAt, + LastAccessedAt: getCurrentClock(), } } @@ -47,7 +57,13 @@ func Put(k string, obj *Obj){ if len(store) >= config.KeysLimit{ evict() } + obj.LastAccessedAt=getCurrentClock() store[k] = obj + if KeyspaceStat[0]==nil{ + KeyspaceStat[0]=make(map[string]int) + } + KeyspaceStat[0]["keys"]++ + //actually you can use grafana to visualize it, like it's easy , you just have to knwo how things work } func Get(k string) *Obj{ @@ -62,16 +78,21 @@ func Get(k string) *Obj{ periodicaly it is mvoing forward and sample randomly 20 keys and seees the expiration and delete the required one and phir se whi loop chalao */ - delete(store,k) - return nil + if hasExpired(v){ + Del(k) + return nil + } } } + v.LastAccessedAt=getCurrentClock() return v } func Del(k string ) bool{ if _,ok:=store[k];ok{ delete(store,k) + delete(expire,_) + KeyspaceStat[0]["keys"]-- return true } return false