diff --git a/src/aof.c b/src/aof.c index 6f8e53712cc..5d8e3ab2c01 100644 --- a/src/aof.c +++ b/src/aof.c @@ -1345,7 +1345,7 @@ int rewriteAppendOnlyFileRio(rio *aof) { expiretime = getExpire(db,&key); /* Save the key and associated value */ - if (o->type == OBJ_STRING) { + if (o->type == OBJ_STRING || o->type == OBJ_STRING_PMEM) { /* Emit a SET command */ char cmd[]="*3\r\n$3\r\nSET\r\n"; if (rioWrite(aof,cmd,sizeof(cmd)-1) == 0) goto werr; diff --git a/src/bitops.c b/src/bitops.c index f506a881bfe..5a89f85163b 100644 --- a/src/bitops.c +++ b/src/bitops.c @@ -483,7 +483,7 @@ robj *lookupStringForBitCommand(client *c, size_t maxbit) { o = createObject(OBJ_STRING,sdsnewlen(NULL, byte+1)); dbAdd(c->db,c->argv[1],o); } else { - if (checkType(c,o,OBJ_STRING)) return NULL; + if (checkTypeStringvariant(c,o)) return NULL; o = dbUnshareStringValue(c->db,c->argv[1],o); o->ptr = sdsgrowzero(o->ptr,byte+1); } @@ -504,7 +504,7 @@ robj *lookupStringForBitCommand(client *c, size_t maxbit) { * If the source object is NULL the function is guaranteed to return NULL * and set 'len' to 0. */ unsigned char *getObjectReadOnlyString(robj *o, long *len, char *llbuf) { - serverAssert(o->type == OBJ_STRING); + serverAssert(o->type == OBJ_STRING || o->type == OBJ_STRING_PMEM); unsigned char *p = NULL; /* Set the 'p' pointer to the string, that can be just a stack allocated @@ -572,7 +572,7 @@ void getbitCommand(client *c) { return; if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.czero)) == NULL || - checkType(c,o,OBJ_STRING)) return; + checkTypeStringvariant(c,o)) return; byte = bitoffset >> 3; bit = 7 - (bitoffset & 0x7); @@ -635,7 +635,7 @@ void bitopCommand(client *c) { continue; } /* Return an error if one of the keys is not a string. */ - if (checkType(c,o,OBJ_STRING)) { + if (checkTypeStringvariant(c,o)) { unsigned long i; for (i = 0; i < j; i++) { if (objects[i]) @@ -774,7 +774,7 @@ void bitcountCommand(client *c) { /* Lookup, check for type, and return 0 for non existing keys. */ if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.czero)) == NULL || - checkType(c,o,OBJ_STRING)) return; + checkTypeStringvariant(c,o)) return; p = getObjectReadOnlyString(o,&strlen,llbuf); /* Parse start/end range if any. */ @@ -838,7 +838,7 @@ void bitposCommand(client *c) { addReplyLongLong(c, bit ? -1 : 0); return; } - if (checkType(c,o,OBJ_STRING)) return; + if (checkTypeStringvariant(c,o)) return; p = getObjectReadOnlyString(o,&strlen,llbuf); /* Parse start/end range if any. */ @@ -1000,7 +1000,7 @@ void bitfieldGeneric(client *c, int flags) { /* Lookup for read is ok if key doesn't exit, but errors * if it's not a string. */ o = lookupKeyRead(c->db,c->argv[1]); - if (o != NULL && checkType(c,o,OBJ_STRING)) { + if (o != NULL && checkTypeStringvariant(c,o)) { zfree(ops); return; } diff --git a/src/db.c b/src/db.c index dc4a0b63e11..d989f1638b4 100644 --- a/src/db.c +++ b/src/db.c @@ -350,7 +350,7 @@ int dbDelete(redisDb *db, robj *key) { * using an sdscat() call to append some data, or anything else. */ robj *dbUnshareStringValue(redisDb *db, robj *key, robj *o) { - serverAssert(o->type == OBJ_STRING); + serverAssert(o->type == OBJ_STRING || o->type == OBJ_STRING_PMEM); if (o->refcount != 1 || o->encoding != OBJ_ENCODING_RAW) { robj *decoded = getDecodedObject(o); o = createRawStringObject(decoded->ptr, sdslen(decoded->ptr)); @@ -926,6 +926,7 @@ char* getObjectTypeName(robj *o) { } else { switch(o->type) { case OBJ_STRING: type = "string"; break; + case OBJ_STRING_PMEM: type = "string"; break; case OBJ_LIST: type = "list"; break; case OBJ_SET: type = "set"; break; case OBJ_ZSET: type = "zset"; break; diff --git a/src/debug.c b/src/debug.c index d79226bf266..7f180c702cc 100644 --- a/src/debug.c +++ b/src/debug.c @@ -125,7 +125,7 @@ void xorObjectDigest(redisDb *db, robj *keyobj, unsigned char *digest, robj *o) char buf[128]; /* Save the key and associated value */ - if (o->type == OBJ_STRING) { + if (o->type == OBJ_STRING || o->type == OBJ_STRING_PMEM) { mixStringObjectDigest(digest,o); } else if (o->type == OBJ_LIST) { listTypeIterator *li = listTypeInitIterator(o,0,LIST_TAIL); @@ -553,7 +553,7 @@ NULL val = dictGetVal(de); key = dictGetKey(de); - if (val->type != OBJ_STRING || !sdsEncodedObject(val)) { + if ((val->type != OBJ_STRING && val->type != OBJ_STRING_PMEM) || !sdsEncodedObject(val)) { addReplyError(c,"Not an sds encoded string."); } else { addReplyStatusFormat(c, @@ -833,7 +833,7 @@ void _serverAssertPrintClientInfo(const client *c) { char buf[128]; char *arg; - if (c->argv[j]->type == OBJ_STRING && sdsEncodedObject(c->argv[j])) { + if ((c->argv[j]->type == OBJ_STRING || c->argv[j]->type == OBJ_STRING_PMEM) && sdsEncodedObject(c->argv[j])) { arg = (char*) c->argv[j]->ptr; } else { snprintf(buf,sizeof(buf),"Object type: %u, encoding: %u", @@ -849,7 +849,7 @@ void serverLogObjectDebugInfo(const robj *o) { serverLog(LL_WARNING,"Object type: %d", o->type); serverLog(LL_WARNING,"Object encoding: %d", o->encoding); serverLog(LL_WARNING,"Object refcount: %d", o->refcount); - if (o->type == OBJ_STRING && sdsEncodedObject(o)) { + if ((o->type == OBJ_STRING || o->type == OBJ_STRING_PMEM) && sdsEncodedObject(o)) { serverLog(LL_WARNING,"Object raw string len: %zu", sdslen(o->ptr)); if (sdslen(o->ptr) < 4096) { sds repr = sdscatrepr(sdsempty(),o->ptr,sdslen(o->ptr)); diff --git a/src/defrag.c b/src/defrag.c index d8e5cfa83c2..c6445923e36 100644 --- a/src/defrag.c +++ b/src/defrag.c @@ -109,7 +109,7 @@ robj *activeDefragStringOb(robj* ob, long *defragged) { return NULL; /* try to defrag robj (only if not an EMBSTR type (handled below). */ - if (ob->type!=OBJ_STRING || ob->encoding!=OBJ_ENCODING_EMBSTR) { + if ((ob->type!=OBJ_STRING && ob->type!=OBJ_STRING_PMEM) || ob->encoding!=OBJ_ENCODING_EMBSTR) { if ((ret = activeDefragAlloc(ob))) { ob = ret; (*defragged)++; @@ -117,7 +117,7 @@ robj *activeDefragStringOb(robj* ob, long *defragged) { } /* try to defrag string object */ - if (ob->type == OBJ_STRING) { + if ((ob->type == OBJ_STRING) || (ob->type == OBJ_STRING_PMEM)) { if(ob->encoding==OBJ_ENCODING_RAW) { sds newsds = activeDefragSds((sds)ob->ptr); if (newsds) { @@ -833,7 +833,7 @@ long defragKey(redisDb *db, dictEntry *de) { ob = newob; } - if (ob->type == OBJ_STRING) { + if ((ob->type == OBJ_STRING) || (ob->type == OBJ_STRING_PMEM)) { /* Already handled in activeDefragStringOb. */ } else if (ob->type == OBJ_LIST) { if (ob->encoding == OBJ_ENCODING_QUICKLIST) { diff --git a/src/gopher.c b/src/gopher.c index 38e44f75472..84eb03ca8f3 100644 --- a/src/gopher.c +++ b/src/gopher.c @@ -53,7 +53,7 @@ void processGopherRequest(client *c) { robj *o = lookupKeyRead(c->db,keyname); /* If there is no such key, return with a Gopher error. */ - if (o == NULL || o->type != OBJ_STRING) { + if (o == NULL || (o->type != OBJ_STRING && o->type != OBJ_STRING_PMEM)) { char *errstr; if (o == NULL) errstr = "Error: no content at the specified key"; diff --git a/src/hyperloglog.c b/src/hyperloglog.c index 721f492a180..ec8551f755d 100644 --- a/src/hyperloglog.c +++ b/src/hyperloglog.c @@ -1150,7 +1150,7 @@ int isHLLObjectOrReply(client *c, robj *o) { struct hllhdr *hdr; /* Key exists, check type */ - if (checkType(c,o,OBJ_STRING)) + if (checkTypeStringvariant(c,o)) return C_ERR; /* Error already sent. */ if (!sdsEncodedObject(o)) goto invalid; diff --git a/src/module.c b/src/module.c index 96481b92c6b..8963188e53a 100644 --- a/src/module.c +++ b/src/module.c @@ -2038,6 +2038,7 @@ int RM_KeyType(RedisModuleKey *key) { * defines as desired. */ switch(key->value->type) { case OBJ_STRING: return REDISMODULE_KEYTYPE_STRING; + case OBJ_STRING_PMEM: return REDISMODULE_KEYTYPE_STRING; case OBJ_LIST: return REDISMODULE_KEYTYPE_LIST; case OBJ_SET: return REDISMODULE_KEYTYPE_SET; case OBJ_ZSET: return REDISMODULE_KEYTYPE_ZSET; @@ -2057,6 +2058,7 @@ size_t RM_ValueLength(RedisModuleKey *key) { if (key == NULL || key->value == NULL) return 0; switch(key->value->type) { case OBJ_STRING: return stringObjectLen(key->value); + case OBJ_STRING_PMEM: return stringObjectLen(key->value); case OBJ_LIST: return listTypeLength(key->value); case OBJ_SET: return setTypeSize(key->value); case OBJ_ZSET: return zsetLength(key->value); @@ -2202,7 +2204,7 @@ char *RM_StringDMA(RedisModuleKey *key, size_t *len, int mode) { return emptystring; } - if (key->value->type != OBJ_STRING) return NULL; + if (key->value->type != OBJ_STRING && key->value->type != OBJ_STRING_PMEM) return NULL; /* For write access, and even for read access if the object is encoded, * we unshare the string (that has the side effect of decoding it). */ @@ -2227,7 +2229,7 @@ char *RM_StringDMA(RedisModuleKey *key, size_t *len, int mode) { * unless the new length value requested is zero. */ int RM_StringTruncate(RedisModuleKey *key, size_t newlen) { if (!(key->mode & REDISMODULE_WRITE)) return REDISMODULE_ERR; - if (key->value && key->value->type != OBJ_STRING) return REDISMODULE_ERR; + if (key->value && key->value->type != OBJ_STRING && key->value->type != OBJ_STRING_PMEM) return REDISMODULE_ERR; if (newlen > 512*1024*1024) return REDISMODULE_ERR; /* Empty key and new len set to 0. Just return REDISMODULE_OK without diff --git a/src/networking.c b/src/networking.c index a0cd03107e0..c02bea9f6d8 100644 --- a/src/networking.c +++ b/src/networking.c @@ -49,7 +49,7 @@ size_t sdsZmallocSize(sds s) { /* Return the amount of memory used by the sds string at object->ptr * for a string object. */ size_t getStringObjectSdsUsedMemory(robj *o) { - serverAssertWithInfo(NULL,o,o->type == OBJ_STRING); + serverAssertWithInfo(NULL,o,o->type == OBJ_STRING || o->type == OBJ_STRING_PMEM); switch(o->encoding) { case OBJ_ENCODING_RAW: return sdsZmallocSize(o->ptr); case OBJ_ENCODING_EMBSTR: return zmalloc_size(o)-sizeof(robj); @@ -1718,7 +1718,7 @@ int processMultibulkBuffer(client *c) { sdsclear(c->querybuf); } else { c->argv[c->argc++] = - createStringObject(c->querybuf+c->qb_pos,c->bulklen); + createStringObjectOptim(c->querybuf+c->qb_pos,c->bulklen); c->qb_pos += c->bulklen+2; } c->bulklen = -1; diff --git a/src/object.c b/src/object.c index 4d25cd79df3..4a25dceb660 100644 --- a/src/object.c +++ b/src/object.c @@ -38,6 +38,7 @@ #define OBJ_MEMORY_GENERAL 0 #define OBJ_MEMORY_DRAM 1 +#define OBJ_MEMORY_PMEM 2 /* ===================== Creation and parsing of objects ==================== */ @@ -81,9 +82,54 @@ robj *createRawStringObject(const char *ptr, size_t len) { return createObject(OBJ_STRING, sdsnewlen(ptr,len)); } +static robj *createRawStringOptim(const char *ptr, size_t len) { + int location; + int type; + sds sds_ptr = sdsnewlen_with_info(ptr ,len, &location); + if (location == OBJ_MEMORY_DRAM) type = OBJ_STRING; + else if (location == OBJ_MEMORY_PMEM) type = OBJ_STRING_PMEM; + else serverAssert(NULL); + return createObject(type, sds_ptr); +} + /* Create a string object with encoding OBJ_ENCODING_EMBSTR, that is * an object where the sds string is actually an unmodifiable string * allocated in the same chunk as the object itself. */ +static robj *createEmbeddedStringObjectOptim(const char *ptr, size_t len) { + int location; + robj *o = zmalloc_with_info(sizeof(robj)+sizeof(struct sdshdr8)+len+1, &location); + struct sdshdr8 *sh = (void*)(o+1); + if (location == OBJ_MEMORY_DRAM) + o->type = OBJ_STRING; + else if (location == OBJ_MEMORY_PMEM) + o->type = OBJ_STRING_PMEM; + else + serverAssert(NULL); + + o->encoding = OBJ_ENCODING_EMBSTR; + o->ptr = sh+1; + o->refcount = 1; + + if (server.maxmemory_policy & MAXMEMORY_FLAG_LFU) { + o->lru = (LFUGetTimeInMinutes()<<8) | LFU_INIT_VAL; + } else { + o->lru = LRU_CLOCK(); + } + + sh->len = len; + sh->alloc = len; + sh->flags = SDS_TYPE_8; + if (ptr == SDS_NOINIT) + sh->buf[len] = '\0'; + else if (ptr) { + memcpy(sh->buf,ptr,len); + sh->buf[len] = '\0'; + } else { + memset(sh->buf,0,len+1); + } + return o; +} + robj *createEmbeddedStringObject(const char *ptr, size_t len) { robj *o = zmalloc(sizeof(robj)+sizeof(struct sdshdr8)+len+1); struct sdshdr8 *sh = (void*)(o+1); @@ -126,6 +172,13 @@ robj *createStringObject(const char *ptr, size_t len) { return createRawStringObject(ptr,len); } +robj *createStringObjectOptim(const char *ptr, size_t len) { + if (len <= OBJ_ENCODING_EMBSTR_SIZE_LIMIT) + return createEmbeddedStringObjectOptim(ptr,len); + else + return createRawStringOptim(ptr,len); +} + /* Create a string object from a long long value. When possible returns a * shared integer object, or at least an integer encoded one. * @@ -196,7 +249,7 @@ robj *createStringObjectFromLongDouble(long double value, int humanfriendly) { robj *dupStringObject(const robj *o) { robj *d; - serverAssert(o->type == OBJ_STRING); + serverAssert(o->type == OBJ_STRING || o->type == OBJ_STRING_PMEM); switch(o->encoding) { case OBJ_ENCODING_RAW: @@ -281,9 +334,9 @@ robj *createModuleObject(moduleType *mt, void *value) { return createObject(OBJ_MODULE,mv); } -void freeStringObjectOptim(robj *o) { +void freeStringObjectOptim(robj *o, int variant) { if (o->encoding == OBJ_ENCODING_RAW) { - sdsfreeOptim(o->ptr); + sdsfreeOptim(o->ptr, variant); } } @@ -370,7 +423,8 @@ void incrRefCount(robj *o) { static void _decrRefCount(robj *o, int on_dram) { if (o->refcount == 1) { switch(o->type) { - case OBJ_STRING: freeStringObjectOptim(o); break; + case OBJ_STRING: freeStringObjectOptim(o, OBJ_MEMORY_GENERAL); break; + case OBJ_STRING_PMEM: freeStringObjectOptim(o, OBJ_MEMORY_PMEM); break; case OBJ_LIST: freeListObject(o); break; case OBJ_SET: freeSetObject(o); break; case OBJ_ZSET: freeZsetObject(o); break; @@ -379,7 +433,11 @@ static void _decrRefCount(robj *o, int on_dram) { case OBJ_STREAM: freeStreamObject(o); break; default: serverPanic("Unknown object type"); break; } - if (on_dram == OBJ_MEMORY_GENERAL || o->encoding == OBJ_ENCODING_EMBSTR) { + if (o->encoding == OBJ_ENCODING_EMBSTR) { + if (o->type == OBJ_STRING) zfree(o); + else if (o->type== OBJ_STRING_PMEM) zfree_pmem(o); + else zfree(o); + } else if (on_dram == OBJ_MEMORY_GENERAL) { zfree(o); } else { zfree_dram(o); @@ -430,12 +488,20 @@ int checkType(client *c, robj *o, int type) { return 0; } +int checkTypeStringvariant(client *c, robj *o) { + if (o->type == OBJ_STRING || o->type == OBJ_STRING_PMEM) { + return 0; + } + addReply(c,shared.wrongtypeerr); + return 1; +} + int isSdsRepresentableAsLongLong(sds s, long long *llval) { return string2ll(s,sdslen(s),llval) ? C_OK : C_ERR; } int isObjectRepresentableAsLongLong(robj *o, long long *llval) { - serverAssertWithInfo(NULL,o,o->type == OBJ_STRING); + serverAssertWithInfo(NULL,o,o->type == OBJ_STRING || o->type == OBJ_STRING_PMEM); if (o->encoding == OBJ_ENCODING_INT) { if (llval) *llval = (long) o->ptr; return C_OK; @@ -466,7 +532,7 @@ robj *tryObjectEncoding(robj *o) { * in this function. Other types use encoded memory efficient * representations but are handled by the commands implementing * the type. */ - serverAssertWithInfo(NULL,o,o->type == OBJ_STRING); + serverAssertWithInfo(NULL,o,o->type == OBJ_STRING || o->type == OBJ_STRING_PMEM); /* We try some specialized encoding only for objects that are * RAW or EMBSTR encoded, in other words objects that are still @@ -545,7 +611,7 @@ robj *getDecodedObject(robj *o) { incrRefCount(o); return o; } - if (o->type == OBJ_STRING && o->encoding == OBJ_ENCODING_INT) { + if ((o->type == OBJ_STRING || o->type == OBJ_STRING_PMEM) && o->encoding == OBJ_ENCODING_INT) { char buf[32]; ll2string(buf,32,(long)o->ptr); @@ -568,7 +634,7 @@ robj *getDecodedObject(robj *o) { #define REDIS_COMPARE_COLL (1<<1) int compareStringObjectsWithFlags(robj *a, robj *b, int flags) { - serverAssertWithInfo(NULL,a,a->type == OBJ_STRING && b->type == OBJ_STRING); + serverAssertWithInfo(NULL,a,(a->type == OBJ_STRING || a->type == OBJ_STRING_PMEM) && (b->type == OBJ_STRING || b->type == OBJ_STRING_PMEM)); char bufa[128], bufb[128], *astr, *bstr; size_t alen, blen, minlen; @@ -625,7 +691,7 @@ int equalStringObjects(robj *a, robj *b) { } size_t stringObjectLen(robj *o) { - serverAssertWithInfo(NULL,o,o->type == OBJ_STRING); + serverAssertWithInfo(NULL,o,o->type == OBJ_STRING || o->type == OBJ_STRING_PMEM); if (sdsEncodedObject(o)) { return sdslen(o->ptr); } else { @@ -639,7 +705,7 @@ int getDoubleFromObject(const robj *o, double *target) { if (o == NULL) { value = 0; } else { - serverAssertWithInfo(NULL,o,o->type == OBJ_STRING); + serverAssertWithInfo(NULL,o,o->type == OBJ_STRING || o->type == OBJ_STRING_PMEM); if (sdsEncodedObject(o)) { if (!string2d(o->ptr, sdslen(o->ptr), &value)) return C_ERR; @@ -673,7 +739,7 @@ int getLongDoubleFromObject(robj *o, long double *target) { if (o == NULL) { value = 0; } else { - serverAssertWithInfo(NULL,o,o->type == OBJ_STRING); + serverAssertWithInfo(NULL,o,o->type == OBJ_STRING || o->type == OBJ_STRING_PMEM); if (sdsEncodedObject(o)) { if (!string2ld(o->ptr, sdslen(o->ptr), &value)) return C_ERR; @@ -707,7 +773,7 @@ int getLongLongFromObject(robj *o, long long *target) { if (o == NULL) { value = 0; } else { - serverAssertWithInfo(NULL,o,o->type == OBJ_STRING); + serverAssertWithInfo(NULL,o,o->type == OBJ_STRING || o->type == OBJ_STRING_PMEM); if (sdsEncodedObject(o)) { if (string2ll(o->ptr,sdslen(o->ptr),&value) == 0) return C_ERR; } else if (o->encoding == OBJ_ENCODING_INT) { @@ -802,7 +868,7 @@ size_t objectComputeSize(robj *o, size_t sample_size) { struct dictEntry *de; size_t asize = 0, elesize = 0, samples = 0; - if (o->type == OBJ_STRING) { + if (o->type == OBJ_STRING || o->type == OBJ_STRING_PMEM) { if(o->encoding == OBJ_ENCODING_INT) { asize = sizeof(*o); } else if(o->encoding == OBJ_ENCODING_RAW) { diff --git a/src/rdb.c b/src/rdb.c index 5cec208c5a2..968dc113c13 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -630,6 +630,7 @@ int rdbLoadBinaryFloatValue(rio *rdb, float *val) { int rdbSaveObjectType(rio *rdb, robj *o) { switch (o->type) { case OBJ_STRING: + case OBJ_STRING_PMEM: return rdbSaveType(rdb,RDB_TYPE_STRING); case OBJ_LIST: if (o->encoding == OBJ_ENCODING_QUICKLIST) @@ -759,7 +760,7 @@ size_t rdbSaveStreamConsumers(rio *rdb, streamCG *cg) { ssize_t rdbSaveObject(rio *rdb, robj *o, robj *key) { ssize_t n = 0, nwritten = 0; - if (o->type == OBJ_STRING) { + if (o->type == OBJ_STRING || o->type == OBJ_STRING_PMEM) { /* Save a string value */ if ((n = rdbSaveStringObject(rdb,o)) == -1) return -1; nwritten += n; diff --git a/src/sds.c b/src/sds.c index e90776c00e4..3c722224682 100644 --- a/src/sds.c +++ b/src/sds.c @@ -43,6 +43,7 @@ const char *SDS_NOINIT = "SDS_NOINIT"; #define SDS_GENERAL_VARIANT 0 #define SDS_DRAM_VARIANT 1 +#define SDS_PMEM_VARIANT 2 static inline int sdsHdrSize(char type) { switch(type&SDS_TYPE_MASK) { @@ -107,7 +108,7 @@ static inline char sdsReqType(size_t string_size) { * You can print the string with printf() as there is an implicit \0 at the * end of the string. However the string is binary safe and can contain * \0 characters in the middle, as the length is stored in the sds header. */ -static sds _sdsnewlen(const void *init, size_t initlen, int on_dram) { +static sds _sdsnewlen(const void *init, size_t initlen, int on_dram, int* info_variant) { void *sh; sds s; char type = sdsReqType(initlen); @@ -117,8 +118,15 @@ static sds _sdsnewlen(const void *init, size_t initlen, int on_dram) { int hdrlen = sdsHdrSize(type); unsigned char *fp; /* flags pointer. */ - sh = (on_dram == SDS_DRAM_VARIANT) ? s_dram_malloc(hdrlen+initlen+1) - : s_malloc(hdrlen+initlen+1); + if (on_dram == SDS_DRAM_VARIANT){ + sh = s_dram_malloc(hdrlen+initlen+1); + if (info_variant) *info_variant = SDS_DRAM_VARIANT; + } else if (info_variant){ + sh = s_malloc_with_info(hdrlen+initlen+1, info_variant); + } else { + sh = s_malloc(hdrlen+initlen+1); + } + if (sh == NULL) return NULL; if (init==SDS_NOINIT) init = NULL; @@ -166,12 +174,16 @@ static sds _sdsnewlen(const void *init, size_t initlen, int on_dram) { return s; } +sds sdsnewlen_with_info(const void *init, size_t initlen, int*info) { + return _sdsnewlen(init, initlen, SDS_GENERAL_VARIANT, info); +} + sds sdsnewlen(const void *init, size_t initlen) { - return _sdsnewlen(init, initlen, SDS_GENERAL_VARIANT); + return _sdsnewlen(init, initlen, SDS_GENERAL_VARIANT, NULL); } static sds sdsdramnewlen(const void *init, size_t initlen) { - return _sdsnewlen(init, initlen, SDS_DRAM_VARIANT); + return _sdsnewlen(init, initlen, SDS_DRAM_VARIANT, NULL); } /* Create an empty (zero length) sds string. Even in this case the string @@ -203,9 +215,10 @@ void sdsfree(sds s) { s_free((char*)s-sdsHdrSize(s[-1])); } -void sdsfreeOptim(sds s) { +void sdsfreeOptim(sds s, int type) { if (s == NULL) return; - s_free((char*)s-sdsHdrSizeOptim(s)); + if (type == SDS_PMEM_VARIANT) s_free_pmem((char*)s-sdsHdrSizeOptim(s)); + else s_free((char*)s-sdsHdrSizeOptim(s)); } /* Set the sds string length to the length as obtained with strlen(), so diff --git a/src/sds.h b/src/sds.h index 9cd9ebb335f..cc95e3e04f0 100644 --- a/src/sds.h +++ b/src/sds.h @@ -219,12 +219,13 @@ static inline void sdssetalloc(sds s, size_t newlen) { } sds sdsnewlen(const void *init, size_t initlen); +sds sdsnewlen_with_info(const void *init, size_t initlen, int* info); sds sdsnew(const char *init); sds sdsempty(void); sds sdsdramempty(void); sds sdsdup(const sds s); void sdsfree(sds s); -void sdsfreeOptim(sds s); +void sdsfreeOptim(sds s, int type); sds sdsgrowzero(sds s, size_t len); sds sdscatlen(sds s, const void *t, size_t len); sds sdscat(sds s, const char *t); diff --git a/src/sdsalloc.h b/src/sdsalloc.h index 7c7a6a80f09..73c931116a1 100644 --- a/src/sdsalloc.h +++ b/src/sdsalloc.h @@ -43,6 +43,9 @@ #define s_malloc zmalloc #define s_realloc zrealloc #define s_free zfree +#define s_free_dram zfree_dram +#define s_free_pmem zfree_pmem #define s_dram_malloc zmalloc_dram +#define s_malloc_with_info zmalloc_with_info #endif diff --git a/src/server.h b/src/server.h index 534c1c71fa4..8b4ba92e9d5 100644 --- a/src/server.h +++ b/src/server.h @@ -467,6 +467,7 @@ typedef long long ustime_t; /* microsecond time type. */ * encoding version. */ #define OBJ_MODULE 5 /* Module object. */ #define OBJ_STREAM 6 /* Stream object. */ +#define OBJ_STRING_PMEM 7 /* String(VALUE) on PMEM object. */ /* Extract encver / signature from a module type ID. */ #define REDISMODULE_TYPE_ENCVER_BITS 10 @@ -1756,13 +1757,14 @@ void incrRefCount(robj *o); robj *makeObjectShared(robj *o); robj *resetRefCount(robj *obj); void freeStringObject(robj *o); -void freeStringObjectOptim(robj *o); +void freeStringObjectOptim(robj *o, int variant); void freeListObject(robj *o); void freeSetObject(robj *o); void freeZsetObject(robj *o); void freeHashObject(robj *o); robj *createObject(int type, void *ptr); robj *createStringObject(const char *ptr, size_t len); +robj *createStringObjectOptim(const char *ptr, size_t len); robj *createRawStringObject(const char *ptr, size_t len); robj *createEmbeddedStringObject(const char *ptr, size_t len); robj *dupStringObject(const robj *o); @@ -1785,6 +1787,7 @@ robj *createStreamObject(void); robj *createModuleObject(moduleType *mt, void *value); int getLongFromObjectOrReply(client *c, robj *o, long *target, const char *msg); int checkType(client *c, robj *o, int type); +int checkTypeStringvariant(client *c, robj *o); int getLongLongFromObjectOrReply(client *c, robj *o, long long *target, const char *msg); int getDoubleFromObjectOrReply(client *c, robj *o, double *target, const char *msg); int getDoubleFromObject(const robj *o, double *target); diff --git a/src/slowlog.c b/src/slowlog.c index 1d715e39b77..2fd32353281 100644 --- a/src/slowlog.c +++ b/src/slowlog.c @@ -62,7 +62,7 @@ slowlogEntry *slowlogCreateEntry(client *c, robj **argv, int argc, long long dur argc-slargc+1)); } else { /* Trim too long strings as well... */ - if (argv[j]->type == OBJ_STRING && + if ((argv[j]->type == OBJ_STRING || argv[j]->type == OBJ_STRING_PMEM) && sdsEncodedObject(argv[j]) && sdslen(argv[j]->ptr) > SLOWLOG_ENTRY_MAX_STRING) { diff --git a/src/sort.c b/src/sort.c index f269a773149..c55a9662491 100644 --- a/src/sort.c +++ b/src/sort.c @@ -119,7 +119,7 @@ robj *lookupKeyByPattern(redisDb *db, robj *pattern, robj *subst, int writeflag) * is a new object with refcount already incremented. */ o = hashTypeGetValueObject(o, fieldobj->ptr); } else { - if (o->type != OBJ_STRING) goto noobj; + if (o->type != OBJ_STRING && o->type != OBJ_STRING_PMEM ) goto noobj; /* Every object that this function returns needs to have its refcount * increased. sortCommand decreases it again. */ diff --git a/src/t_string.c b/src/t_string.c index 5306069bf8f..6ed1a386ad8 100644 --- a/src/t_string.c +++ b/src/t_string.c @@ -167,7 +167,7 @@ int getGenericCommand(client *c) { if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.null[c->resp])) == NULL) return C_OK; - if (o->type != OBJ_STRING) { + if (o->type != OBJ_STRING && o->type != OBJ_STRING_PMEM ) { addReply(c,shared.wrongtypeerr); return C_ERR; } else { @@ -219,7 +219,7 @@ void setrangeCommand(client *c) { size_t olen; /* Key exists, check type */ - if (checkType(c,o,OBJ_STRING)) + if (checkTypeStringvariant(c,o)) return; /* Return existing string length when setting nothing */ @@ -259,7 +259,7 @@ void getrangeCommand(client *c) { if (getLongLongFromObjectOrReply(c,c->argv[3],&end,NULL) != C_OK) return; if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.emptybulk)) == NULL || - checkType(c,o,OBJ_STRING)) return; + checkTypeStringvariant(c,o)) return; if (o->encoding == OBJ_ENCODING_INT) { str = llbuf; @@ -298,7 +298,7 @@ void mgetCommand(client *c) { if (o == NULL) { addReplyNull(c); } else { - if (o->type != OBJ_STRING) { + if (o->type != OBJ_STRING && o->type != OBJ_STRING_PMEM) { addReplyNull(c); } else { addReplyBulk(c,o); @@ -348,7 +348,7 @@ void incrDecrCommand(client *c, long long incr) { robj *o, *new; o = lookupKeyWrite(c->db,c->argv[1]); - if (o != NULL && checkType(c,o,OBJ_STRING)) return; + if (o != NULL && checkTypeStringvariant(c,o)) return; if (getLongLongFromObjectOrReply(c,o,&value,NULL) != C_OK) return; oldvalue = value; @@ -408,7 +408,7 @@ void incrbyfloatCommand(client *c) { robj *o, *new, *aux1, *aux2; o = lookupKeyWrite(c->db,c->argv[1]); - if (o != NULL && checkType(c,o,OBJ_STRING)) return; + if (o != NULL && checkTypeStringvariant(c,o)) return; if (getLongDoubleFromObjectOrReply(c,o,&value,NULL) != C_OK || getLongDoubleFromObjectOrReply(c,c->argv[2],&incr,NULL) != C_OK) return; @@ -453,7 +453,7 @@ void appendCommand(client *c) { totlen = stringObjectLen(c->argv[2]); } else { /* Key exists, check type */ - if (checkType(c,o,OBJ_STRING)) + if (checkTypeStringvariant(c,o)) return; /* "append" is an argument, so always an sds */ @@ -476,7 +476,7 @@ void appendCommand(client *c) { void strlenCommand(client *c) { robj *o; if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.czero)) == NULL || - checkType(c,o,OBJ_STRING)) return; + checkTypeStringvariant(c,o)) return; addReplyLongLong(c,stringObjectLen(o)); } diff --git a/src/zmalloc.c b/src/zmalloc.c index e204655d8c1..18ad2a6a287 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -57,8 +57,8 @@ void zlibc_free(void *ptr) { #endif #endif -#define DRAM_LOCATION 0 -#define PMEM_LOCATION 1 +#define DRAM_LOCATION 1 +#define PMEM_LOCATION 2 /* Explicitly override malloc/free etc when using tcmalloc. */ #if defined(USE_TCMALLOC) @@ -97,7 +97,7 @@ static int zmalloc_is_pmem(void * ptr) { return DRAM_LOCATION; } -static void zfree_pmem(void *ptr) { +void zfree_pmem(void *ptr) { (void)(ptr); zmalloc_pmem_not_available(); } @@ -183,7 +183,7 @@ static int zmalloc_is_pmem(void * ptr) { return (temp_kind == MEMKIND_DEFAULT) ? DRAM_LOCATION : PMEM_LOCATION; } -static void zfree_pmem(void *ptr) { +void zfree_pmem(void *ptr) { #ifndef HAVE_MALLOC_SIZE void *realptr; size_t oldsize; @@ -266,6 +266,15 @@ void *zmalloc(size_t size) { return (size < pmem_threshold) ? zmalloc_dram(size) : zmalloc_pmem(size); } +void *zmalloc_with_info(size_t size, int* location) { + if (size < pmem_threshold) { + *location = DRAM_LOCATION; + return zmalloc_dram(size); + } + *location = PMEM_LOCATION; + return zmalloc_pmem(size); +} + /* Allocation and free functions that bypass the thread cache * and go straight to the allocator arena bins. * Currently implemented only for jemalloc. Used for online defragmentation. */ @@ -336,7 +345,7 @@ void *zrealloc_dram(void *ptr, size_t size) { } void *zrealloc(void *ptr, size_t size) { - if (!zmalloc_is_pmem(ptr)) { + if (zmalloc_is_pmem(ptr) == DRAM_LOCATION) { return zrealloc_dram(ptr, size); } else { return zrealloc_pmem(ptr, size); @@ -379,7 +388,7 @@ void zfree_dram(void *ptr) { } void zfree(void *ptr) { - if (!zmalloc_is_pmem(ptr)) { + if (zmalloc_is_pmem(ptr) == DRAM_LOCATION) { zfree_dram(ptr); } else { zfree_pmem(ptr); diff --git a/src/zmalloc.h b/src/zmalloc.h index 4b862839904..ecb595205f0 100644 --- a/src/zmalloc.h +++ b/src/zmalloc.h @@ -91,6 +91,7 @@ void *zcalloc(size_t size); void *zrealloc(void *ptr, size_t size); void zfree(void *ptr); void zfree_dram(void *ptr); +void zfree_pmem(void *ptr); char *zstrdup(const char *s); size_t zmalloc_used_memory(void); size_t zmalloc_used_pmem_memory(void); @@ -105,6 +106,7 @@ size_t zmalloc_get_memory_size(void); void zlibc_free(void *ptr); void zmalloc_set_threshold(size_t threshold); size_t zmalloc_get_threshold(void); +void *zmalloc_with_info(size_t size, int* location); void *zmalloc_dram(size_t size); void *zcalloc_dram(size_t size); void *zrealloc_dram(void *ptr, size_t size);