rgw: add RGWDedup integration tests

src/common/options/rgw.yaml.in

@@ -3838,4 +3838,90 @@ options:
   flags:
   - startup
   with_legacy: true
-
+- name: rgw_enable_dedup_threads
+  type: bool
+  level: advanced
+  desc: Enable RGW deduplication
+  default: false
+  services:
+  - rgw
+  with_legacy: true
+- name: rgw_dedup_num_workers
+  type: uint
+  level: advanced
+  desc: Number of workers for deduplication
+  default: 3
+  services:
+  - rgw
+  with_legacy: true
+- name: rgw_dedup_chunk_algo
+  type: str
+  level: advanced
+  desc: Chunking Algorithm for deduplication
+  default: fastcdc
+  services:
+  - rgw
+  enum_values:
+  - fastcdc
+  - fixed
+  with_legacy: true
+- name: rgw_dedup_chunk_size
+  type: uint
+  level: advanced
+  desc: Chunk size for deduplication
+  default: 16384
+  services:
+  - rgw
+  with_legacy: true
+- name: rgw_dedup_fp_algo
+  type: str
+  level: advanced
+  desc: Fingerprint Algorithm for deduplication
+  default: sha1
+  services:
+  - rgw
+  enum_values:
+  - sha1
+  - sha256
+  - sha512
+  with_legacy: true
+- name: rgw_dedup_threshold
+  type: uint
+  level: advanced
+  desc: Chunk count threshold for deduplication
+  default: 3
+  services:
+  - rgw
+  with_legacy: true
+- name: rgw_dedup_scrub_ratio
+  type: uint
+  level: advanced
+  desc: Scrub ratio for deduplication
+  default: 10
+  services:
+  - rgw
+  with_legacy: true
+- name: rgw_dedup_cold_pool_name
+  type: str
+  level: advanced
+  desc: Cold pool name for RGWDedup
+  default: default-cold-pool
+  services:
+  - rgw
+  with_legacy: true
+- name: rgw_dedup_fpmanager_memory_limit
+  type: uint
+  level: advanced
+  desc: FPManager memory limit for deduplication
+  default: 4294967296
+  services:
+  - rgw
+  with_legacy: true
+- name: rgw_dedup_fpmanager_low_watermark
+  type: uint
+  level: advanced
+  desc: A low-watermark of fpmap in FPManager
+  default: 50
+  services:
+  - rgw
+  with_legacy: true

src/osd/PrimaryLogPG.cc

@@ -2573,6 +2573,11 @@ PrimaryLogPG::cache_result_t PrimaryLogPG::maybe_handle_manifest_detail(
     return cache_result_t::HANDLED_PROXY;
   case object_manifest_t::TYPE_CHUNKED:
     {
+      // in case of metadata handling ops don't need to promote chunk objects
+      if (op->may_read() && !op->may_read_data() && !op->may_write()) {
+        return cache_result_t::NOOP;
+      }
+
       if (can_proxy_chunked_read(op, obc)) {
 	map<hobject_t,FlushOpRef>::iterator p = flush_ops.find(obc->obs.oi.soid);
         if (p != flush_ops.end()) {

src/rgw/CMakeLists.txt

@@ -148,6 +148,10 @@ set(librgw_common_srcs
   rgw_bucket_encryption.cc
   rgw_tracer.cc
   rgw_lua_background.cc
+  rgw_dedup.cc
+  rgw_fp_manager.cc
+  rgw_dedup_manager.cc
+  rgw_dedup_worker.cc
   driver/rados/cls_fifo_legacy.cc
   driver/rados/rgw_bucket.cc
   driver/rados/rgw_bucket_sync.cc
@@ -261,6 +265,8 @@ target_link_libraries(rgw_common
     cls_timeindex_client
     cls_user_client
     cls_version_client
+    cls_cas_client
+    cls_cas_internal
     librados
     rt
     ICU::uc

src/rgw/driver/rados/rgw_rados.cc

@@ -65,6 +65,7 @@
 
 #include "rgw_gc.h"
 #include "rgw_lc.h"
+#include "rgw_dedup.h"
 
 #include "rgw_object_expirer_core.h"
 #include "rgw_sync.h"
@@ -1070,6 +1071,13 @@ void RGWRados::finalize()
   if (run_notification_thread) {
     rgw::notify::shutdown();
   }
+
+  if (use_dedup) {
+    if (dedup.get()) {
+      dedup->finalize();
+      dedup.reset();
+    }
+  }
 }
 
 /** 
@@ -1197,6 +1205,17 @@ int RGWRados::init_complete(const DoutPrefixProvider *dpp, optional_yield y)
     obj_expirer->start_processor();
   }
 
+  if (use_dedup) {
+    dedup = std::make_shared<RGWDedup>();
+    if (dedup->initialize(cct, this->driver) < 0) {
+      ldpp_dout(dpp, 0) << "initialing RGWDedup failed" << dendl;
+    } else {
+      dedup->start_dedup_manager();
+    }
+  } else {
+    ldpp_dout(dpp, 5) << "note: RGWDedup not initialized" << dendl;
+  }
+
   auto& current_period = svc.zone->get_current_period();
   auto& zonegroup = svc.zone->get_zonegroup();
   auto& zone_params = svc.zone->get_zone_params();

src/rgw/driver/rados/rgw_rados.h

@@ -58,6 +58,7 @@ struct RGWZoneGroup;
 struct RGWZoneParams;
 class RGWReshard;
 class RGWReshardWait;
+class RGWDedup;
 
 struct get_obj_data;
 
@@ -369,12 +370,14 @@ class RGWRados
   RGWGC* gc{nullptr};
   RGWLC* lc{nullptr};
   RGWObjectExpirer* obj_expirer{nullptr};
+  std::shared_ptr<RGWDedup> dedup;
   bool use_gc_thread{false};
   bool use_lc_thread{false};
   bool quota_threads{false};
   bool run_sync_thread{false};
   bool run_reshard_thread{false};
   bool run_notification_thread{false};
+  bool use_dedup;
 
   RGWMetaNotifier* meta_notifier{nullptr};
   RGWDataNotifier* data_notifier{nullptr};
@@ -523,6 +526,11 @@ class RGWRados
     return *this;
   }
 
+  RGWRados& set_use_dedup(bool _use_dedup) {
+    use_dedup = _use_dedup;
+    return *this;
+  }
+
   librados::IoCtx* get_lc_pool_ctx() {
     return &lc_pool_ctx;
   }

src/rgw/rgw_appmain.cc

@@ -243,6 +243,7 @@ int rgw::AppMain::init_storage()
           run_sync,
           g_conf().get_val<bool>("rgw_dynamic_resharding"),
           true, null_yield, // run notification thread
+          g_conf()->rgw_enable_dedup_threads,
           g_conf()->rgw_cache_enabled);
   if (!env.driver) {
     return -EIO;

src/rgw/rgw_dedup.cc

@@ -0,0 +1,58 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-                                                      
+// vim: ts=8 sw=2 smarttab ft=cpp
+
+#include "rgw_dedup.h"
+
+#define dout_subsys ceph_subsys_rgw
+
+using namespace std;
+
+
+int RGWDedup::initialize(CephContext* _cct, rgw::sal::RadosStore* _store)
+{
+  cct = _cct;
+  store = _store;
+  dedup_manager = make_unique<RGWDedupManager>(this, cct, store);
+  if (dedup_manager->initialize() < 0) {
+    return -1;
+  }
+  return 0;
+}
+
+void RGWDedup::finalize()
+{
+  stop_dedup_manager();
+  dedup_manager.reset();
+}
+
+void RGWDedup::start_dedup_manager()
+{
+  assert(dedup_manager.get());
+  dedup_manager->set_down_flag(false);
+  dedup_manager->create("dedup_manager");
+}
+
+void RGWDedup::stop_dedup_manager()
+{
+  assert(dedup_manager.get());
+  if (!dedup_manager->get_down_flag()) {
+    dedup_manager->stop();
+    dedup_manager->join();
+  }
+  ldpp_dout(this, 2) << "stop RGWDedup" << dendl;
+}
+
+unsigned RGWDedup::get_subsys() const
+{
+  return dout_subsys;
+}
+
+CephContext* RGWDedup::get_cct() const
+{
+  return cct;
+}
+
+std::ostream& RGWDedup::gen_prefix(std::ostream& out) const
+{
+  return out << "RGWDedup: ";
+}

src/rgw/rgw_dedup.h

@@ -0,0 +1,43 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-                                            
+// vim: ts=8 sw=2 smarttab ft=cpp
+
+#ifndef CEPH_RGW_DEDUP_H
+#define CEPH_RGW_DEDUP_H
+
+
+#include <string>
+#include <atomic>
+#include <sstream>
+
+#include "include/types.h"
+#include "common/Cond.h"
+#include "common/Thread.h"
+#include "rgw_dedup_manager.h"
+
+using namespace std;
+
+class RGWDedup : public DoutPrefixProvider
+{
+  CephContext* cct;
+  rgw::sal::RadosStore* store;
+  unique_ptr<RGWDedupManager> dedup_manager;
+
+public:
+  RGWDedup() : cct(nullptr), store(nullptr) {}
+  RGWDedup(const RGWDedup& rhs) = delete;
+  RGWDedup& operator=(const RGWDedupManager& rhs) = delete;
+  virtual ~RGWDedup() override {}
+
+  int initialize(CephContext* _cct, rgw::sal::RadosStore* _store);
+  void finalize();
+
+  void start_dedup_manager();
+  void stop_dedup_manager();
+
+  CephContext* get_cct() const override;
+  unsigned get_subsys() const override;
+  std::ostream& gen_prefix(std::ostream& out) const override;
+};
+
+#endif
+