Object statistics lru

config/main.go

@@ -0,0 +1,14 @@
+package config
+
+var Host string="0.0.0.0"
+var Port int=7379
+
+var KeysLimit int=100 //like ye hum threshold set krdiye hai like our database still support atmax this many keys 
+
+//will evict EvictionRation of keys whenever evictionruns
+//it would dictate whenever my eviction is triggered, how many keys i will be evicting.
+var EvictionRatio flaot64=0.40 //Generally aise real scenario me kabhi itna to nahi hi krte hai but still
+
+var EvictionStrategy string="allkeys-random"
+var AOFFile string="./velox.aof"
+

core/eval.go

@@ -140,7 +140,10 @@ func evalGET(args []string)[]byte{
 		return RESP_NIL
 	}
 
-
+	//if key already expired then return nil
+	if hasExpired(obj){
+		return RESP_NIL
+	}
 	//return te RESP encoded value 
 	// c.Write(Encode(obj.Value,false))
 	// return nil
@@ -184,6 +187,20 @@ func evalTTL(args []string) []byte{
 		return RESP_MINUS_2
 	}
 
+	exp.isExpirySet:=getExpiry(obj)
+	if !isExpirySet{
+		return RESP_MINUS_1
+	}
+
+	//if key expired i.e key does not exist hence return -2
+	if uint64(time.Now().UnixMilli())>exp{
+		return RESP_MINUS_2
+	}
+
+	//compute the time remaining for the key to expire and 
+	//return the RESP encoded form of it 
+	durationMS:=exp-uint64(time.Now().UnixMilli())
+
 	// c.Write(Encode(int64(durationMS/1000),false))
 	// return nil
 	return Encode(int64(durationMs/1000), false)
@@ -220,7 +237,9 @@ func evalEXPIRE(args []string) []byte{
 		return RESP_ZERO
 	}
 
-	obj.ExpiresAt = time.Now().UnixMilli() + exDurationSec*1000
+	// obj.ExpiresAt = time.Now().UnixMilli() + exDurationSec*1000
+
+	setExpiry(obj,exDurationSec*1000)
 
 	//1 print krenge if the timeout was set
 	// c.Write([]byte(":1\r\n"))
@@ -234,6 +253,14 @@ func evalBGREWRITEAOF(args []string) []byte{
 	return RESP_OK
 }
 
+func evalINFO(args []string)[]byte{
+	var info []byte
+	buf:=bytes.NewBuffer(info)
+	for i := range KeyspaceStat{
+		buf.WriteString(fmt.Sprintf("db%d:keys=%d,expire=0,avg_ttl=0\r\n",i,KeyspaceStat[i]["keys"]))
+	}
+}
+
 // func EvalAndRespond(cmd *Rediscmd,c net.Conn)error{
 func EvalAndRespond(cmds []*RedisCmd, c io.ReadWriter) error{
 	//It's job is like depending on what job is sent to us
@@ -260,6 +287,12 @@ func EvalAndRespond(cmds []*RedisCmd, c io.ReadWriter) error{
 			buf.Write(evalBGREWRITEAOF(cmd.Args))
 		case "INCR":
 			buf.Write(evalPING(cmd.Args))
+		case "INFO":
+			buf.Write(evalINFO(cmd.Args))
+		case "CLIENT":
+			buf.Write(evalCLIENT(cmd.Args))
+		case "LATENCY":
+			buf.Write(evalLATENCY(cmd.Args))
 		default:
 			buf.Write(evalPING(cmd.Args))
 		}

core/eviction.go

@@ -7,6 +7,8 @@ whenevr a cache is full, we will be evicting the first key which we do find
 
 package core
 
+import "github.com/sharpsalt/Velox-In-Memory-Database/config"
+
 //Evcits the first key it found while iterating the map 
 //TODP: Make it efficient by doing thrugh somehting
 func evictFirst(){
@@ -16,9 +18,81 @@ func evictFirst(){
 	}
 }
 
+/*
+The approximated LRU algorithm
+*/
+func getCurrentClock() uint32{
+	return uint32(time.Now().Unix())&0x00FFFFF // it would give us 24 bit clock resolution of the time
+}
+
+func getIdleTime(LastAccessdAt uint32) uint32{
+	/*
+	it gives me current clock and if it is greater than lastaccessedat then for anytimestamp  then diff, other (max-last)*c
+	*/
+	c:=getCurrentClock()
+	if c>=LastAccessdAt{
+		return c-LastAccessdAt
+	}
+	return (0x00FFFFF-LastAccessdAt)*c
+}
+
+func populateEvictionPool(){
+	sampleSize:=5
+	for k:=range store{
+		ePool.Push(k,store[k].lastaccessedat)
+		sampleSize--
+		if sampleSize==0{
+			break
+		}
+	}
+}
+
+//TODO: no need to populate everytime, should populate 
+//only when the number of keys to evict is less than what we have in the pool
+funct evictAllKeysLRU(){
+	populateEvictionPool()
+	evictCount:=int16(config.EvictionRatio=float64(config.KeysLimit))
+	for i:=;i<int(evictCount) && len(ePool.pool)>0 ;i++{
+		item:=ePool.Pop()
+		if item==nil{
+			return
+		}
+		Del(item.key)
+	}
+}
+
+//Randomly removes keys to make space for the new data added
+//The number of keys removed will be sufficient to free up least 10% space
+func evictAllKeysRandom(){
+	evictCount:=int64(config.EvictionRatio*float64(config.KeysLimit))
+	//Iteration of Golang dictionary can be considered as a random
+	//because it depends on the has of the inserted key 
+	for k:= range store{
+		Del(k)
+		evictCount--
+		if evictCount==0{
+			break
+		}
+	}
+}
+/*
+How do we know that our eviction is working correctly??
+
+which means when we are doing large number of key sets , we are putting lots of keys in db , let's say we put kelimit to 100
+, how do we sure that we are not breachng it , so every dbs supports statistics
+*/
+
 
 //TODO: Make the eveiction strategy configuration dirven
 //TODO: Support multiple eviction strategies
 func evict(){
-	evictFirst()
+	// evictFirst()
+	switch config.EvictionStrategy{
+	case "simple-first":
+		evictfirst()
+	case "allkets-random":
+		evictAllkeysRandom()
+	case: "allkeys-lru":
+		evictAllKeysLRU()
+	}
 }

core/evictionpool.go

@@ -0,0 +1,80 @@
+package core
+
+import{
+	"sort"
+}
+
+
+type PoolItem struct{
+	key string
+	lastaccessedat uint32
+}
+
+//TODO: when last accessed at of object changes
+//updates the poolItem corresponding to that
+type EvictionPool struct{
+	pool []*PoolItem   //it is basically an array of pool items 
+	keyset map[string]*PoolItem  //na keyset , like the key which are present.
+}
+
+type ByIdleTime []*PoolItem
+
+func (a ByIdleTime) len() int{
+	return len(a)
+}
+
+func (a ByIdleTime) Swap(i int,j int){
+	a[i],a[j]=a[j],a[i]
+}
+
+func (a ByIdleTime) Less(i int,j int) bool{//basically it is a comparator function which i am suing to sort the time
+	return getIdleTime(a[i].lastaccessedat)>getIdleTime(a[j].lastaccessedat)
+}
+
+//TODO: Make the Implementation efficient to not need repeated sorting
+func (pq *EvictionPool) Push(key string,lastaccessedat uint32){
+	_,ok:=pq.keyset[key]
+	if ok{
+		//while pushing it into eviction pool if it already exists then we don;t have to push it again it in eveiction pool
+		return
+	}
+	ietm:=&PoolItem(key:key,lastaccessedat:lastaccessedat)
+	if len(pq.pool)<ePoolSizeMax{
+		//which means there is some space for adding element
+		//remeber eviction pool is needed to be sorted by idle time
+		pq.keyset[key]=items
+		pq.pool=append(pq.pool,item)
+
+
+		//Performance bottleneck
+		sort.Sort(ByIdleTime(pq.pool))
+	}else if lastaccessedat>pq.pool[len(pq.pool)-1].lastaccessedat{
+		//if i have no space in eviction pool but the element which i have smapled is worse than my current 
+		//i will create space for that by removing the 1st one and adding new element by appending it 
+		///so this way we are ensuring that our pool contains best possible candidates to be evcited
+		pq.pool=pq.pool[1:]
+		pq.keyset[key]=item
+		pq.pool=append(pq.pool,item)
+	}
+}
+
+func (pq *EvictionPool) Pop() *PoolItem{
+	if len(pq.pool)==0{
+		return nil
+	}
+	item:=pq.pool[0]
+	pq.pool=pq.pool[1:]
+	return item
+}
+
+func newEvictionPool(size int) *EvictionPool{
+	return &EvictionPool{
+		pool: make([]*PoolItem,size)
+		keyset: make(map[string]*PoolItem)
+	}
+}
+
+var ePoolSizeMax int=16
+var ePool *EvictionPool=newEvictionPool(0)
+
+

core/expire.go

@@ -5,6 +5,19 @@ import (
 	"time"
 )
 
+func hasExpired(obj *Obj) bool{
+	exp,ok:=expires[obj]
+	if !ok{
+		return false
+	}
+	return exp<=uint64(time.Now().UnixMilli())
+}
+
+func getExpiry(obj *Obj)(uint64,bool){
+	exp,ok:=expires[obj]
+	return exp,ok
+}
+
 //Delete all the expired keys- the active way
 //Sampling approach: https://redis.io/commands/expire/
 

core/object.go

@@ -4,8 +4,21 @@ package core
 //TODO: Change ExpiresAt it to LRU Bits as handled by Redis
 type Obj struct{
 	TypeEncoding uint8
-	Value        interface{}
-	ExpiresAt    int64
+	Value        interface{} //which means we acn put literally anything,anyvalue and which would work fine
+	// ExpiresAt    int64
+	/*
+	Earlier we use to have expireAt here, we don't need as we need LastAccessdAt
+	since golang doesn't support bitfield so LastAccessedAt is storing everytime the key is getting accessed
+	*/
+
+	/*
+	Redis allot 24 bits to these bits, but we will use 32 bits because
+	golang does not support bitfields and we need not make this super-compplicated 
+	by merging TypeEncoding + LastAccessedAt in one 32 bit integer
+	but nonthelss, we can benchmarks and see how that fares
+	For noe, we continue with 32 bit integer to stre th LastAccessedAt
+	*/
+	LastAccessedAt uint32
 }
 /*
 C and c++ give us ways to apply bit fields on set, means in each set i can assign a fix number of bits

core/stats.go

@@ -0,0 +1,10 @@
+package core
+
+var KeyspaceStat [4]map[string]int //just all global object
+//eg i support 4 databases within my redis
+//in redis you can have 16 databases in redis itself, by deault it goes from db0,db1,db2,...db15
+
+//for which db,which metric,which value
+func UpdatDBStat(num int,metric string,value int){
+	KeyspaceStat[num][metric]=value
+}