From 5785a9ffe36a83c29a2bc9a8503fb39e1e65521d Mon Sep 17 00:00:00 2001 From: Surabhi Singh Date: Wed, 24 Feb 2021 09:58:44 -0800 Subject: [PATCH 1/5] Add CompactionFilter, RepairDb, Flush and GetLiveFilesMetadata 1. Add support to specify delegates required by CompactionFilter. 2. Add Flush and FlushOptions to be able to do on-demand flush 3. Add RepairDb to be able to ingest sstfiles written by other rocksdb instances 4. Add methods to retrieve and parse LiveFiles Metadata --- RocksDbSharp/CompactionFilter.cs | 20 +++ RocksDbSharp/FlushOptions.cs | 20 +++ RocksDbSharp/LiveFilesMetadata.cs | 13 ++ RocksDbSharp/Native.Extensions.cs | 29 ++++ RocksDbSharp/RocksDb.cs | 103 ++++++++++++- RocksDbSharp/RocksDbSharp.csproj | 6 +- tests/RocksDbSharpTest/AdditionalTests.cs | 141 ++++++++++++++++++ .../RocksDbSharpTest/CompactionFilterTest.cs | 140 +++++++++++++++++ 8 files changed, 467 insertions(+), 5 deletions(-) create mode 100644 RocksDbSharp/CompactionFilter.cs create mode 100644 RocksDbSharp/FlushOptions.cs create mode 100644 RocksDbSharp/LiveFilesMetadata.cs create mode 100644 RocksDbSharp/Native.Extensions.cs create mode 100644 tests/RocksDbSharpTest/AdditionalTests.cs create mode 100644 tests/RocksDbSharpTest/CompactionFilterTest.cs diff --git a/RocksDbSharp/CompactionFilter.cs b/RocksDbSharp/CompactionFilter.cs new file mode 100644 index 0000000..516eb63 --- /dev/null +++ b/RocksDbSharp/CompactionFilter.cs @@ -0,0 +1,20 @@ +using System; + +namespace RocksDbSharp +{ + public class CompactionFilter + { + public IntPtr Handle; + private readonly NameDelegate getNameDelegate; + private readonly FilterDelegate filterDelegate; + private readonly DestructorDelegate destroyDelegate; + + public CompactionFilter(NameDelegate nameDelegate, FilterDelegate filterDelegate, DestructorDelegate destroyDelegate, IntPtr state) + { + this.getNameDelegate = nameDelegate; + this.filterDelegate = filterDelegate; + this.destroyDelegate = destroyDelegate; + Handle = Native.Instance.rocksdb_compactionfilter_create(state, destroyDelegate, filterDelegate, getNameDelegate); + } + } +} diff --git a/RocksDbSharp/FlushOptions.cs b/RocksDbSharp/FlushOptions.cs new file mode 100644 index 0000000..c83d835 --- /dev/null +++ b/RocksDbSharp/FlushOptions.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace RocksDbSharp +{ + public class FlushOptions: OptionsHandle + { + public FlushOptions() + { + Native.Instance.rocksdb_flushoptions_create(); + } + + public FlushOptions SetWaitForFlush(bool waitForFlush) + { + Native.Instance.rocksdb_flushoptions_set_wait(Handle, waitForFlush); + return this; + } + } +} diff --git a/RocksDbSharp/LiveFilesMetadata.cs b/RocksDbSharp/LiveFilesMetadata.cs new file mode 100644 index 0000000..c2638e5 --- /dev/null +++ b/RocksDbSharp/LiveFilesMetadata.cs @@ -0,0 +1,13 @@ +namespace RocksDbSharp +{ + public class LiveFileMetadata + { + public string FileName; + public int FileLevel; + public ulong FileSize; + public string SmallestKeyInFile; + public string LargestKeyInFile; + public ulong NumEntriesInFile; + public ulong NumDeletionsInFile; + } +} diff --git a/RocksDbSharp/Native.Extensions.cs b/RocksDbSharp/Native.Extensions.cs new file mode 100644 index 0000000..5c19901 --- /dev/null +++ b/RocksDbSharp/Native.Extensions.cs @@ -0,0 +1,29 @@ +// Native class for custom additions missing in rocksdb package + +using char_ptr_ptr = System.IntPtr; +using const_size_t = System.UIntPtr; +using rocksdb_t_ptr = System.IntPtr; +using System; + +namespace RocksDbSharp +{ + public partial class Native + { + public abstract void rocksdb_compact_file( + rocksdb_t_ptr db, + string[] file_list, + const_size_t list_len, + int output_level, out char_ptr_ptr errptr); + + public void rocksdb_compact_file( + rocksdb_t_ptr db, + string[] file_list, + const_size_t list_len, + int output_level) + { + rocksdb_compact_file(db, file_list, list_len, output_level, out char_ptr_ptr errptr); + if (errptr != IntPtr.Zero) + throw new RocksDbException(errptr); + } + } +} \ No newline at end of file diff --git a/RocksDbSharp/RocksDb.cs b/RocksDbSharp/RocksDb.cs index 770898b..e365681 100644 --- a/RocksDbSharp/RocksDb.cs +++ b/RocksDbSharp/RocksDb.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Runtime.InteropServices; using System.Text; +using System.Threading; using Transitional; namespace RocksDbSharp @@ -45,6 +46,11 @@ public static RocksDb Open(OptionsHandle options, string path) return new RocksDb(db, optionsReferences: null, cfOptionsRefs: null); } + public static void RepairDB(OptionsHandle options, string path) + { + Native.Instance.rocksdb_repair_db(options.Handle, path); + } + public static RocksDb OpenReadOnly(OptionsHandle options, string path, bool errorIfLogFileExists) { IntPtr db = Native.Instance.rocksdb_open_for_read_only(options.Handle, path, errorIfLogFileExists); @@ -168,7 +174,7 @@ public long Get(byte[] key, long keyLength, byte[] buffer, long offset, long len } } - public KeyValuePair[] MultiGet(byte[][] keys, ColumnFamilyHandle[] cf = null, ReadOptions readOptions = null) + public KeyValuePair[] MultiGet(byte[][] keys, ColumnFamilyHandle[] cf = null, ReadOptions readOptions = null) { return Native.Instance.rocksdb_multi_get(Handle, (readOptions ?? DefaultReadOptions).Handle, keys); } @@ -261,7 +267,7 @@ public void DropColumnFamily(string name) Native.Instance.rocksdb_drop_column_family(Handle, cf.Handle); columnFamilies.Remove(name); } - + public ColumnFamilyHandle GetDefaultColumnFamily() { return GetColumnFamily(ColumnFamilies.DefaultName); @@ -306,5 +312,98 @@ public void CompactRange(string start, string limit, ColumnFamilyHandle cf = nul encoding = Encoding.UTF8; CompactRange(start == null ? null : encoding.GetBytes(start), limit == null ? null : encoding.GetBytes(limit), cf); } + + public void Flush(FlushOptions flushOptions) + { + Native.Instance.rocksdb_flush(Handle, flushOptions.Handle); + } + + public List GetLiveFilesMetadata() + { + IntPtr buffer = Native.Instance.rocksdb_livefiles(Handle); + if (buffer == IntPtr.Zero) + { + return new List(); + } + + try + { + List filesMetadata = new List(); + + int fileCount = Native.Instance.rocksdb_livefiles_count(buffer); + for (int index = 0; index < fileCount; index++) + { + IntPtr fileMetadata = Native.Instance.rocksdb_livefiles_name(buffer, index); + string fileName = Marshal.PtrToStringAnsi(fileMetadata); + + int level = Native.Instance.rocksdb_livefiles_level(buffer, index); + + UIntPtr fS = Native.Instance.rocksdb_livefiles_size(buffer, index); + ulong fileSize = fS.ToUInt64(); + + var smallestKeyPtr = Native.Instance.rocksdb_livefiles_smallestkey(buffer, index, out var smallestKeySize); + string smallestKey = Marshal.PtrToStringAnsi(smallestKeyPtr); + + var largestKeyPtr = Native.Instance.rocksdb_livefiles_largestkey(buffer, index, out var largestKeySize); + string largestKey = Marshal.PtrToStringAnsi(largestKeyPtr); + + ulong entries = Native.Instance.rocksdb_livefiles_entries(buffer, index); + + ulong deletions = Native.Instance.rocksdb_livefiles_deletions(buffer, index); + + filesMetadata.Add(new LiveFileMetadata() + { + FileName = fileName, + FileLevel = level, + FileSize = fileSize, + SmallestKeyInFile = smallestKey, + LargestKeyInFile = largestKey, + NumEntriesInFile = entries, + NumDeletionsInFile = deletions + }); + } + + return filesMetadata; + } + finally + { + Native.Instance.rocksdb_livefiles_destroy(buffer); + buffer = IntPtr.Zero; + } + } + + /// + /// Lean API to just get Live file names. Refer to GetLiveFilesMetadata() for the complete metadata + /// + /// + public List GetLiveFileNames() + { + IntPtr buffer = Native.Instance.rocksdb_livefiles(Handle); + if (buffer == IntPtr.Zero) + { + return new List(); + } + + try + { + List liveFiles = new List(); + + int fileCount = Native.Instance.rocksdb_livefiles_count(buffer); + + for (int index = 0; index < fileCount; index++) + { + IntPtr fileMetadata = Native.Instance.rocksdb_livefiles_name(buffer, index); + string fileName = Marshal.PtrToStringAnsi(fileMetadata); + liveFiles.Add(fileName); + } + + return liveFiles; + } + finally + { + Native.Instance.rocksdb_livefiles_destroy(buffer); + buffer = IntPtr.Zero; + } + } } } diff --git a/RocksDbSharp/RocksDbSharp.csproj b/RocksDbSharp/RocksDbSharp.csproj index 9f03eff..11f6cb4 100644 --- a/RocksDbSharp/RocksDbSharp.csproj +++ b/RocksDbSharp/RocksDbSharp.csproj @@ -1,5 +1,5 @@  - + RocksDbSharp netstandard1.6;net40;net45 @@ -31,7 +31,7 @@ TRACE - + @@ -42,6 +42,6 @@ - + \ No newline at end of file diff --git a/tests/RocksDbSharpTest/AdditionalTests.cs b/tests/RocksDbSharpTest/AdditionalTests.cs new file mode 100644 index 0000000..8495fc0 --- /dev/null +++ b/tests/RocksDbSharpTest/AdditionalTests.cs @@ -0,0 +1,141 @@ +using System; +using Xunit; +using System.IO; +using RocksDbSharp; +using System.Linq; + +namespace RocksDbSharpTest +{ + public class AdditionalTests + { + public const string ReplacedText = "REPLACEMENT_TEXT"; + + [Fact] + public void TestFlush() + { + var dbName = "TestFlushDB"; + + DeleteDb(dbName); + + var options = new DbOptions().SetCreateIfMissing(true); + + using (var db = RocksDb.Open(options, dbName)) + { + db.Put("key", "value"); + } + + var sstFiles = Directory.EnumerateFiles(dbName).Where(s => s.EndsWith(".sst", StringComparison.OrdinalIgnoreCase)); + + Assert.True(!sstFiles.Any()); + + DeleteDb(dbName); + + FlushOptions flushOptions = new FlushOptions().SetWaitForFlush(true); + + using (var db = RocksDb.Open(options, dbName)) + { + db.Put("key", "value"); + db.Flush(flushOptions); + } + + sstFiles = Directory.EnumerateFiles(dbName).Where(s => s.EndsWith(".sst", StringComparison.OrdinalIgnoreCase)); + Assert.True(sstFiles.Any()); + } + + [Fact] + public void TestRepairDB() + { + var dbName = "TestFlushDB"; + DeleteDb(dbName); + var options = new DbOptions().SetCreateIfMissing(true); + var flushOptions = new FlushOptions().SetWaitForFlush(true); + + using (var db = RocksDb.Open(options, dbName)) + { + db.Put("key0", "value0"); + db.Flush(flushOptions); + } + + var firstSSTfile = Directory.EnumerateFiles(dbName).Where(s => s.EndsWith(".sst", StringComparison.OrdinalIgnoreCase)).FirstOrDefault(); + var checkKey = "key1"; + var checkValue = "value0"; + using (var db = RocksDb.Open(options, dbName)) + { + db.Put(checkKey, checkValue); + db.Flush(flushOptions); + } + + File.Delete($"{firstSSTfile}"); + RocksDb.RepairDB(options, dbName); + + using (var db = RocksDb.Open(options, dbName)) + { + Assert.Equal(db.Get("key0"), null); + Assert.Equal(db.Get(checkKey), checkValue); + } + } + + [Fact] + public void TestLiveFiles() + { + var dbName = "TestLiveFiles"; + DeleteDb(dbName); + var options = new DbOptions().SetCreateIfMissing(true); + var flushOptions = new FlushOptions().SetWaitForFlush(true); + + using (var db = RocksDb.Open(options, dbName)) + { + db.Put("key0", "value0"); + db.Put("key1", "value0"); + db.Flush(flushOptions); + + db.Put("key7", "value0"); + db.Put("key8", "value0"); + + db.Flush(flushOptions); + + var files = db.GetLiveFilesMetadata(); + var fileNames = files.Select(file => file.FileName); + var fileList = Directory.EnumerateFiles(dbName); + + Assert.True(fileList.All(file => fileList.Contains(file))); + Assert.Equal(db.Get("key0"), "value0"); + } + } + + [Fact] + public void TestLiveFileNames() + { + var dbName = "TestLiveFiles"; + DeleteDb(dbName); + var options = new DbOptions().SetCreateIfMissing(true); + var flushOptions = new FlushOptions().SetWaitForFlush(true); + + using (var db = RocksDb.Open(options, dbName)) + { + db.Put("key0", "value0"); + db.Put("key1", "value0"); + db.Flush(flushOptions); + + db.Put("key7", "value0"); + db.Put("key8", "value0"); + + db.Flush(flushOptions); + + var files = db.GetLiveFileNames(); + var fileList = Directory.EnumerateFiles(dbName); + + Assert.True(fileList.All(file => fileList.Contains(file))); + Assert.Equal(db.Get("key0"), "value0"); + } + } + + public static void DeleteDb(string dbName) + { + if (Directory.Exists(dbName)) + { + Directory.Delete(dbName, true); + } + } + } +} diff --git a/tests/RocksDbSharpTest/CompactionFilterTest.cs b/tests/RocksDbSharpTest/CompactionFilterTest.cs new file mode 100644 index 0000000..5014961 --- /dev/null +++ b/tests/RocksDbSharpTest/CompactionFilterTest.cs @@ -0,0 +1,140 @@ +using RocksDbSharp; +using System; +using System.IO; +using System.Linq; +using System.Runtime.InteropServices; +using System.Text; +using System.Threading.Tasks; +using Xunit; + +namespace RocksDbSharpTest +{ + public class CompactionFilterTest + { + [Fact] + public void TestAllDelegates() + { + var dbName = "TestCompactionFilterDb"; + FlushOptions flushOptions = new FlushOptions().SetWaitForFlush(true); + DbOptions options = new DbOptions().SetCreateIfMissing(true); + var filter = new CompactionFilter(DefaultNameDelegate, TestFilterDelegate, DefaultDestructorDelegate, IntPtr.Zero); + options.SetCompactionFilter(filter.Handle); + options.SetDisableAutoCompactions(1); + + SetupAndPopulate(dbName, flushOptions, options); + } + + private static void SetupAndPopulate(string dbName, FlushOptions flushOptions, DbOptions options) + { + DeleteDb(dbName); + using (var db = RocksDb.Open(options, dbName)) + { + byte[] key = Encoding.UTF8.GetBytes("keep_a"); + byte[] value = Encoding.UTF8.GetBytes("1"); + + db.Put(key, value); + + key = Encoding.UTF8.GetBytes("keep_b"); + value = Encoding.UTF8.GetBytes("2"); + + db.Put(key, value); + + key = Encoding.UTF8.GetBytes("Remove_c"); + value = Encoding.UTF8.GetBytes("3"); + + db.Put(key, value); + + key = Encoding.UTF8.GetBytes("Replace_d"); + value = Encoding.UTF8.GetBytes("4"); + + db.Put(key, value); + + db.Flush(flushOptions); + db.CompactRange(null, null, null); + + key = Encoding.UTF8.GetBytes("keep_a"); + var result = db.Get(key); + Assert.NotNull(result); + + key = Encoding.UTF8.GetBytes("Remove_c"); + result = db.Get(key); + Assert.Null(result); + + key = Encoding.UTF8.GetBytes("Replace_d"); + result = db.Get(key); + Assert.NotNull(result); + Assert.Equal(result, Encoding.UTF8.GetBytes("8")); + } + } + + public char TestFilterDelegate(IntPtr p0, int level, IntPtr key, UIntPtr key_length, IntPtr existing_value, UIntPtr value_length, IntPtr new_value, IntPtr new_value_length, IntPtr value_changed) + { + int keyLength = (int)key_length.ToUInt64(); + byte[] keyValue = new byte[keyLength]; + Marshal.Copy(key, keyValue, 0, keyLength); + + int valueLength = (int)value_length.ToUInt64(); + byte[] existingValue = new byte[valueLength]; + Marshal.Copy(existing_value, existingValue, 0, valueLength); + + var keyStr = Encoding.UTF8.GetString(keyValue); + if (keyStr.StartsWith("Remove")) + { + return (char)1; + } + + if (keyStr.StartsWith("Replace")) + { + var valueStr = Encoding.UTF8.GetString(existingValue); + int val = Convert.ToInt32(valueStr); + byte[] newValue = Encoding.UTF8.GetBytes($"{val * 2}"); + + if (newValue != null && newValue.Length != 0) + { + IntPtr buffer = Marshal.AllocHGlobal(newValue.Length); + try + { + Marshal.Copy(newValue, 0, buffer, newValue.Length); + Marshal.WriteIntPtr(new_value, buffer); + Marshal.WriteByte(value_changed, 1); + + var intSize = IntPtr.Size; + switch (intSize) + { + case 4: + Marshal.WriteInt32(new_value_length, newValue.Length); + break; + case 8: + Marshal.WriteInt64(new_value_length, newValue.Length); + break; + } + } + finally + { + Marshal.FreeHGlobal(buffer); + } + } + } + + return (char)0; + } + + public unsafe void DefaultDestructorDelegate(IntPtr state) + { + Marshal.FreeHGlobal(state); + } + + public unsafe IntPtr DefaultNameDelegate(IntPtr state) + { + return Marshal.StringToHGlobalAnsi("DefaultCompactionFilter"); + } + + public static void DeleteDb(string dbName) + { + if (Directory.Exists(dbName)) + { + Directory.Delete(dbName, true); + } + } + } +} From 51ab55632c131c99ec245049ba4a3597cac6f85d Mon Sep 17 00:00:00 2001 From: Surabhi Singh Date: Wed, 24 Feb 2021 10:03:43 -0800 Subject: [PATCH 2/5] Delete Native.Extensions.cs Removing it until the api is added to c.h --- RocksDbSharp/Native.Extensions.cs | 29 ----------------------------- 1 file changed, 29 deletions(-) delete mode 100644 RocksDbSharp/Native.Extensions.cs diff --git a/RocksDbSharp/Native.Extensions.cs b/RocksDbSharp/Native.Extensions.cs deleted file mode 100644 index 5c19901..0000000 --- a/RocksDbSharp/Native.Extensions.cs +++ /dev/null @@ -1,29 +0,0 @@ -// Native class for custom additions missing in rocksdb package - -using char_ptr_ptr = System.IntPtr; -using const_size_t = System.UIntPtr; -using rocksdb_t_ptr = System.IntPtr; -using System; - -namespace RocksDbSharp -{ - public partial class Native - { - public abstract void rocksdb_compact_file( - rocksdb_t_ptr db, - string[] file_list, - const_size_t list_len, - int output_level, out char_ptr_ptr errptr); - - public void rocksdb_compact_file( - rocksdb_t_ptr db, - string[] file_list, - const_size_t list_len, - int output_level) - { - rocksdb_compact_file(db, file_list, list_len, output_level, out char_ptr_ptr errptr); - if (errptr != IntPtr.Zero) - throw new RocksDbException(errptr); - } - } -} \ No newline at end of file From be2044f6e04f9217274d247665c5f88a923e9a37 Mon Sep 17 00:00:00 2001 From: Surabhi Singh Date: Wed, 24 Feb 2021 11:48:04 -0800 Subject: [PATCH 3/5] Split metadata class Split metadata about live files to be able to populate only a few fields --- RocksDbSharp/LiveFilesMetadata.cs | 14 +++++- RocksDbSharp/RocksDb.cs | 52 +++++++++++++++-------- tests/RocksDbSharpTest/AdditionalTests.cs | 37 ++++++++++------ 3 files changed, 70 insertions(+), 33 deletions(-) diff --git a/RocksDbSharp/LiveFilesMetadata.cs b/RocksDbSharp/LiveFilesMetadata.cs index c2638e5..d4504d4 100644 --- a/RocksDbSharp/LiveFilesMetadata.cs +++ b/RocksDbSharp/LiveFilesMetadata.cs @@ -1,13 +1,23 @@ namespace RocksDbSharp { public class LiveFileMetadata + { + public FileMetadata FileMetadata; + public FileDataMetadata FileDataMetadata; + } + + public class FileMetadata { public string FileName; public int FileLevel; - public ulong FileSize; + public ulong FileSize; + } + + public class FileDataMetadata + { public string SmallestKeyInFile; public string LargestKeyInFile; public ulong NumEntriesInFile; public ulong NumDeletionsInFile; - } + } } diff --git a/RocksDbSharp/RocksDb.cs b/RocksDbSharp/RocksDb.cs index e365681..462e496 100644 --- a/RocksDbSharp/RocksDb.cs +++ b/RocksDbSharp/RocksDb.cs @@ -318,12 +318,18 @@ public void Flush(FlushOptions flushOptions) Native.Instance.rocksdb_flush(Handle, flushOptions.Handle); } - public List GetLiveFilesMetadata() + + /// + /// Returns metadata about the file and data in the file. + /// + /// setting it to true only populates FileName, Filesize and filelevel; By default it is false + /// LiveFilesMetadata or null in case of failure + public List GetLiveFilesMetadata(bool populateFileMetadataOnly=false) { IntPtr buffer = Native.Instance.rocksdb_livefiles(Handle); if (buffer == IntPtr.Zero) { - return new List(); + return null; } try @@ -333,6 +339,9 @@ public List GetLiveFilesMetadata() int fileCount = Native.Instance.rocksdb_livefiles_count(buffer); for (int index = 0; index < fileCount; index++) { + LiveFileMetadata liveFileMetadata = new LiveFileMetadata(); + + FileMetadata metadata = new FileMetadata(); IntPtr fileMetadata = Native.Instance.rocksdb_livefiles_name(buffer, index); string fileName = Marshal.PtrToStringAnsi(fileMetadata); @@ -341,26 +350,33 @@ public List GetLiveFilesMetadata() UIntPtr fS = Native.Instance.rocksdb_livefiles_size(buffer, index); ulong fileSize = fS.ToUInt64(); - var smallestKeyPtr = Native.Instance.rocksdb_livefiles_smallestkey(buffer, index, out var smallestKeySize); - string smallestKey = Marshal.PtrToStringAnsi(smallestKeyPtr); + metadata.FileName = fileName; + metadata.FileLevel = level; + metadata.FileSize = fileSize; - var largestKeyPtr = Native.Instance.rocksdb_livefiles_largestkey(buffer, index, out var largestKeySize); - string largestKey = Marshal.PtrToStringAnsi(largestKeyPtr); + liveFileMetadata.FileMetadata = metadata; - ulong entries = Native.Instance.rocksdb_livefiles_entries(buffer, index); + if (!populateFileMetadataOnly) + { + FileDataMetadata fileDataMetadata = new FileDataMetadata(); + var smallestKeyPtr = Native.Instance.rocksdb_livefiles_smallestkey(buffer, index, out var smallestKeySize); + string smallestKey = Marshal.PtrToStringAnsi(smallestKeyPtr); - ulong deletions = Native.Instance.rocksdb_livefiles_deletions(buffer, index); + var largestKeyPtr = Native.Instance.rocksdb_livefiles_largestkey(buffer, index, out var largestKeySize); + string largestKey = Marshal.PtrToStringAnsi(largestKeyPtr); - filesMetadata.Add(new LiveFileMetadata() - { - FileName = fileName, - FileLevel = level, - FileSize = fileSize, - SmallestKeyInFile = smallestKey, - LargestKeyInFile = largestKey, - NumEntriesInFile = entries, - NumDeletionsInFile = deletions - }); + ulong entries = Native.Instance.rocksdb_livefiles_entries(buffer, index); + ulong deletions = Native.Instance.rocksdb_livefiles_deletions(buffer, index); + + fileDataMetadata.SmallestKeyInFile = smallestKey; + fileDataMetadata.LargestKeyInFile = largestKey; + fileDataMetadata.NumEntriesInFile = entries; + fileDataMetadata.NumDeletionsInFile = deletions; + + liveFileMetadata.FileDataMetadata = fileDataMetadata; + } + + filesMetadata.Add(liveFileMetadata); } return filesMetadata; diff --git a/tests/RocksDbSharpTest/AdditionalTests.cs b/tests/RocksDbSharpTest/AdditionalTests.cs index 8495fc0..4aaadfe 100644 --- a/tests/RocksDbSharpTest/AdditionalTests.cs +++ b/tests/RocksDbSharpTest/AdditionalTests.cs @@ -83,24 +83,35 @@ public void TestLiveFiles() var options = new DbOptions().SetCreateIfMissing(true); var flushOptions = new FlushOptions().SetWaitForFlush(true); - using (var db = RocksDb.Open(options, dbName)) { - db.Put("key0", "value0"); - db.Put("key1", "value0"); - db.Flush(flushOptions); + using (var db = RocksDb.Open(options, dbName)) + { + var files = db.GetLiveFilesMetadata(); - db.Put("key7", "value0"); - db.Put("key8", "value0"); + Assert.True(files.Count == 0); + } + } - db.Flush(flushOptions); + { + using (var db = RocksDb.Open(options, dbName)) + { + db.Put("key0", "value0"); + db.Put("key1", "value0"); + db.Flush(flushOptions); - var files = db.GetLiveFilesMetadata(); - var fileNames = files.Select(file => file.FileName); - var fileList = Directory.EnumerateFiles(dbName); + db.Put("key7", "value0"); + db.Put("key8", "value0"); - Assert.True(fileList.All(file => fileList.Contains(file))); - Assert.Equal(db.Get("key0"), "value0"); - } + db.Flush(flushOptions); + + var files = db.GetLiveFilesMetadata(); + var fileNames = files.Select(file => file.FileMetadata.FileName); + var fileList = Directory.EnumerateFiles(dbName); + + Assert.True(fileList.All(file => fileList.Contains(file))); + Assert.Equal(db.Get("key0"), "value0"); + } + } } [Fact] From 6c6bdf6e6bd4b718df729a9224f0c9bee4d00dd0 Mon Sep 17 00:00:00 2001 From: Surabhi Singh Date: Wed, 24 Feb 2021 20:48:36 -0800 Subject: [PATCH 4/5] fix formatting --- RocksDbSharp/CompactionFilter.cs | 5 ++++- RocksDbSharp/RocksDb.cs | 14 ++++++++++---- tests/RocksDbSharpTest/AdditionalTests.cs | 8 ++++++-- tests/RocksDbSharpTest/CompactionFilterTest.cs | 5 ++++- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/RocksDbSharp/CompactionFilter.cs b/RocksDbSharp/CompactionFilter.cs index 516eb63..99513fe 100644 --- a/RocksDbSharp/CompactionFilter.cs +++ b/RocksDbSharp/CompactionFilter.cs @@ -9,7 +9,10 @@ public class CompactionFilter private readonly FilterDelegate filterDelegate; private readonly DestructorDelegate destroyDelegate; - public CompactionFilter(NameDelegate nameDelegate, FilterDelegate filterDelegate, DestructorDelegate destroyDelegate, IntPtr state) + public CompactionFilter(NameDelegate nameDelegate, + FilterDelegate filterDelegate, + DestructorDelegate destroyDelegate, + IntPtr state) { this.getNameDelegate = nameDelegate; this.filterDelegate = filterDelegate; diff --git a/RocksDbSharp/RocksDb.cs b/RocksDbSharp/RocksDb.cs index 462e496..7d5dd54 100644 --- a/RocksDbSharp/RocksDb.cs +++ b/RocksDbSharp/RocksDb.cs @@ -322,7 +322,8 @@ public void Flush(FlushOptions flushOptions) /// /// Returns metadata about the file and data in the file. /// - /// setting it to true only populates FileName, Filesize and filelevel; By default it is false + /// setting it to true only populates FileName, + /// Filesize and filelevel; By default it is false /// LiveFilesMetadata or null in case of failure public List GetLiveFilesMetadata(bool populateFileMetadataOnly=false) { @@ -359,10 +360,14 @@ public List GetLiveFilesMetadata(bool populateFileMetadataOnly if (!populateFileMetadataOnly) { FileDataMetadata fileDataMetadata = new FileDataMetadata(); - var smallestKeyPtr = Native.Instance.rocksdb_livefiles_smallestkey(buffer, index, out var smallestKeySize); + var smallestKeyPtr = Native.Instance.rocksdb_livefiles_smallestkey(buffer, + index, + out var smallestKeySize); string smallestKey = Marshal.PtrToStringAnsi(smallestKeyPtr); - var largestKeyPtr = Native.Instance.rocksdb_livefiles_largestkey(buffer, index, out var largestKeySize); + var largestKeyPtr = Native.Instance.rocksdb_livefiles_largestkey(buffer, + index, + out var largestKeySize); string largestKey = Marshal.PtrToStringAnsi(largestKeyPtr); ulong entries = Native.Instance.rocksdb_livefiles_entries(buffer, index); @@ -389,7 +394,8 @@ public List GetLiveFilesMetadata(bool populateFileMetadataOnly } /// - /// Lean API to just get Live file names. Refer to GetLiveFilesMetadata() for the complete metadata + /// Lean API to just get Live file names. + /// Refer to GetLiveFilesMetadata() for the complete metadata /// /// public List GetLiveFileNames() diff --git a/tests/RocksDbSharpTest/AdditionalTests.cs b/tests/RocksDbSharpTest/AdditionalTests.cs index 4aaadfe..79db352 100644 --- a/tests/RocksDbSharpTest/AdditionalTests.cs +++ b/tests/RocksDbSharpTest/AdditionalTests.cs @@ -38,7 +38,8 @@ public void TestFlush() db.Flush(flushOptions); } - sstFiles = Directory.EnumerateFiles(dbName).Where(s => s.EndsWith(".sst", StringComparison.OrdinalIgnoreCase)); + sstFiles = Directory.EnumerateFiles(dbName) + .Where(s => s.EndsWith(".sst", StringComparison.OrdinalIgnoreCase)); Assert.True(sstFiles.Any()); } @@ -56,7 +57,10 @@ public void TestRepairDB() db.Flush(flushOptions); } - var firstSSTfile = Directory.EnumerateFiles(dbName).Where(s => s.EndsWith(".sst", StringComparison.OrdinalIgnoreCase)).FirstOrDefault(); + var firstSSTfile = Directory.EnumerateFiles(dbName) + .Where(s => s.EndsWith(".sst", StringComparison.OrdinalIgnoreCase)) + .FirstOrDefault(); + var checkKey = "key1"; var checkValue = "value0"; using (var db = RocksDb.Open(options, dbName)) diff --git a/tests/RocksDbSharpTest/CompactionFilterTest.cs b/tests/RocksDbSharpTest/CompactionFilterTest.cs index 5014961..ec0b4a2 100644 --- a/tests/RocksDbSharpTest/CompactionFilterTest.cs +++ b/tests/RocksDbSharpTest/CompactionFilterTest.cs @@ -17,7 +17,10 @@ public void TestAllDelegates() var dbName = "TestCompactionFilterDb"; FlushOptions flushOptions = new FlushOptions().SetWaitForFlush(true); DbOptions options = new DbOptions().SetCreateIfMissing(true); - var filter = new CompactionFilter(DefaultNameDelegate, TestFilterDelegate, DefaultDestructorDelegate, IntPtr.Zero); + var filter = new CompactionFilter(DefaultNameDelegate, + TestFilterDelegate, + DefaultDestructorDelegate, + IntPtr.Zero); options.SetCompactionFilter(filter.Handle); options.SetDisableAutoCompactions(1); From 9b6d9b463215d2d5eee3657c12d70632c53940f7 Mon Sep 17 00:00:00 2001 From: Surabhi Singh Date: Wed, 24 Feb 2021 20:50:21 -0800 Subject: [PATCH 5/5] Update RocksDb.cs --- RocksDbSharp/RocksDb.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/RocksDbSharp/RocksDb.cs b/RocksDbSharp/RocksDb.cs index 7d5dd54..96af4cd 100644 --- a/RocksDbSharp/RocksDb.cs +++ b/RocksDbSharp/RocksDb.cs @@ -4,7 +4,6 @@ using System.Linq; using System.Runtime.InteropServices; using System.Text; -using System.Threading; using Transitional; namespace RocksDbSharp