HDDS-14960. OM Web UI dashboard for Ozone Snapshot

hadoop-hdds/framework/src/main/resources/webapps/static/ozone.js

@@ -256,6 +256,8 @@
       ioLinkHref: '@',
       scanner: '<',
       scannerLinkHref: '@',
+      snapshot: '@',
+      snapshotLinkHref: '@'
     },
     templateUrl: 'static/templates/menu.html',
     controller: function($http) {

hadoop-hdds/framework/src/main/resources/webapps/static/templates/menu.html

@@ -58,5 +58,6 @@
         </li>
         <li ng-show="$ctrl.iostatus"><a ng-href="{{$ctrl.ioLinkHref}}">IO Status</a></li>
         <li ng-show="$ctrl.scanner"><a ng-href="{{$ctrl.scannerLinkHref}}">Data Scanner</a></li>
+        <li ng-show="$ctrl.snapshot"><a ng-href="{{$ctrl.snapshotLinkHref}}">Ozone Snapshot</a></li>
     </ul>
 </div><!--/.nav-collapse -->

hadoop-hdds/server-scm/src/main/resources/webapps/scm/scm-overview.html

@@ -391,4 +391,4 @@ <h2>Safemode rules statuses</h2>
         <td>{{typestat.value[1]}}</td>
     </tr>
     </tbody>
-</table>
+</table>

hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/SnapshotDiffJob.java

@@ -94,7 +94,7 @@ public SnapshotDiffJob(long creationTime,
     this.largestEntryKey = largestEntryKey;
   }
 
-  public static Codec<SnapshotDiffJob> getCodec() {
+  public static Codec<SnapshotDiffJob> codec() {
     return CODEC;
   }
 
@@ -228,10 +228,10 @@ public String toString() {
       sb.append(", reason: ").append(reason);
     }
     if (status.equals(JobStatus.IN_PROGRESS) && subStatus != null) {
-      sb.append(", subStatus: ").append(status);
+      sb.append(", subStatus: ").append(subStatus);
       if (subStatus.equals(SubStatus.OBJECT_ID_MAP_GEN_FSO) ||
           subStatus.equals(SubStatus.OBJECT_ID_MAP_GEN_OBS)) {
-        sb.append(String.format(", keysProcessedPercent: %.2f", keysProcessedPct));
+        sb.append(String.format(", keysProcessedPct: %.2f", keysProcessedPct));
       }
     }
     return sb.toString();

hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOmSnapshotDiffJobCodec.java

@@ -30,7 +30,7 @@
 public class TestOmSnapshotDiffJobCodec {
   private final OldSnapshotDiffJobCodecForTesting oldCodec
       = new OldSnapshotDiffJobCodecForTesting();
-  private final Codec<SnapshotDiffJob> newCodec = SnapshotDiffJob.getCodec();
+  private final Codec<SnapshotDiffJob> newCodec = SnapshotDiffJob.codec();
 
   @Test
   public void testOldJsonSerializedDataCanBeReadByNewCodec() throws Exception {

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmSnapshotManager.java

@@ -425,7 +425,7 @@ private static CodecRegistry createCodecRegistryForSnapDiff() {
     // DiffReportEntry codec for Diff Report.
     registry.addCodec(SnapshotDiffReportOzone.DiffReportEntry.class,
         SnapshotDiffReportOzone.getDiffReportEntryCodec());
-    registry.addCodec(SnapshotDiffJob.class, SnapshotDiffJob.getCodec());
+    registry.addCodec(SnapshotDiffJob.class, SnapshotDiffJob.codec());
     return registry.build();
   }
 

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManager.java

@@ -95,9 +95,12 @@
 import java.util.function.BiFunction;
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
+import javax.management.ObjectName;
 import org.apache.commons.io.file.PathUtils;
 import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hdds.HddsUtils;
 import org.apache.hadoop.hdds.StringUtils;
+import org.apache.hadoop.hdds.annotation.InterfaceAudience;
 import org.apache.hadoop.hdds.conf.OzoneConfiguration;
 import org.apache.hadoop.hdds.utils.NativeLibraryNotLoadedException;
 import org.apache.hadoop.hdds.utils.db.CodecRegistry;
@@ -111,6 +114,7 @@
 import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksDB;
 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
 import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
+import org.apache.hadoop.metrics2.util.MBeans;
 import org.apache.hadoop.ozone.OFSPath;
 import org.apache.hadoop.ozone.OzoneConsts;
 import org.apache.hadoop.ozone.om.OMMetadataManager;
@@ -147,7 +151,8 @@
 /**
  * Class to generate snapshot diff.
  */
-public class SnapshotDiffManager implements AutoCloseable {
+@InterfaceAudience.Private
+public class SnapshotDiffManager implements AutoCloseable, SnapshotDiffManagerMXBean {
   private static final Logger LOG =
       LoggerFactory.getLogger(SnapshotDiffManager.class);
   private static final Map<DiffType, String> DIFF_TYPE_STRING_MAP =
@@ -177,6 +182,7 @@ public class SnapshotDiffManager implements AutoCloseable {
    */
   private final PersistentMap<String, SnapshotDiffJob> snapDiffJobTable;
   private final ExecutorService snapDiffExecutor;
+  private ObjectName snapshotDiffManagerBeanName;
 
   /**
    * Directory to keep hardlinks of SST files for a snapDiff job temporarily.
@@ -280,6 +286,7 @@ public SnapshotDiffManager(ManagedRocksDB db,
     // When we build snapDiff HA aware, we will revisit this.
     // Details: https://github.com/apache/ozone/pull/4438#discussion_r1149788226
     this.loadJobsOnStartUp();
+    this.registerMXBean();
   }
 
   @VisibleForTesting
@@ -1514,8 +1521,34 @@ void loadJobsOnStartUp() {
     }
   }
 
+  @Override
+  public List<SnapshotDiffJob> getSnapshotDiffJobs() {
+    List<SnapshotDiffJob> jobs = new ArrayList<>();
+    try (ClosableIterator<Map.Entry<String, SnapshotDiffJob>> iterator =
+             snapDiffJobTable.iterator()) {
+      while (iterator.hasNext()) {
+        jobs.add(iterator.next().getValue());
+      }
+    }
+    return jobs;
+  }
+
+  private void registerMXBean() {
+    this.snapshotDiffManagerBeanName = HddsUtils.registerWithJmxProperties(
+        "OzoneManager", "SnapshotDiffManager",
+        Collections.emptyMap(), this);
+  }
+
+  private void unregisterMXBean() {
+    if (this.snapshotDiffManagerBeanName != null) {
+      MBeans.unregister(this.snapshotDiffManagerBeanName);
+      this.snapshotDiffManagerBeanName = null;
+    }
+  }
+
   @Override
   public void close() {
+    unregisterMXBean();
     if (snapDiffExecutor != null) {
       closeExecutorService(snapDiffExecutor, "SnapDiffExecutor");
     }

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/SnapshotDiffManagerMXBean.java

@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.snapshot;
+
+import java.util.List;
+import org.apache.hadoop.hdds.annotation.InterfaceAudience;
+import org.apache.hadoop.ozone.om.helpers.SnapshotDiffJob;
+
+/**
+ * JMX interface for SnapshotDiffManager.
+ */
+@InterfaceAudience.Private
+public interface SnapshotDiffManagerMXBean {
+  /**
+   * Returns all snapshot diff jobs.
+   * @return list of snapshot diff jobs
+   */
+  List<SnapshotDiffJob> getSnapshotDiffJobs();
+}

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/db/SnapshotDiffDBDefinition.java

@@ -51,7 +51,7 @@ public final class SnapshotDiffDBDefinition extends DBDefinition.WithMap {
   public static final String SNAP_DIFF_JOB_TABLE_NAME = "snap-diff-job-table";
 
   public static final DBColumnFamilyDefinition<String, SnapshotDiffJob> SNAP_DIFF_JOB_TABLE_DEF
-      = new DBColumnFamilyDefinition<>(SNAP_DIFF_JOB_TABLE_NAME, StringCodec.get(), SnapshotDiffJob.getCodec());
+      = new DBColumnFamilyDefinition<>(SNAP_DIFF_JOB_TABLE_NAME, StringCodec.get(), SnapshotDiffJob.codec());
   /**
    * Global table to keep the diff report. Each key is prefixed by the jobId
    * to improve look up and clean up. JobId comes from snap-diff-job-table.

hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/index.html

@@ -49,7 +49,9 @@
             <a class="navbar-brand" href="#">Ozone Manager</a>
         </div>
         <navmenu
-                metrics="{ 'OM metrics' : '#!/metrics/ozoneManager', 'Rpc metrics' : '#!/metrics/rpc'}"></navmenu>
+                metrics="{ 'OM metrics' : '#!/metrics/ozoneManager', 'Rpc metrics' : '#!/metrics/rpc'}"
+                snapshot="true"
+                snapshot-link-href="#!/snapshots"></navmenu>
     </div>
 </header>
 

hadoop-ozone/ozone-manager/src/main/resources/webapps/ozoneManager/om-snapshots.html

@@ -0,0 +1,165 @@
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<h1>Usage Statistics</h1>
+<table class="table table-bordered table-striped">
+    <thead>
+    <tr>
+        <th>Metric</th>
+        <th>Value</th>
+    </tr>
+    </thead>
+    <tbody>
+    <tr>
+        <td>Number of Active Snapshots</td>
+        <td>{{$ctrl.snapshotUsageMetrics.NumSnapshotActive}}</td>
+    </tr>
+    <tr>
+        <td>Number of Deleted Snapshots</td>
+        <td>{{$ctrl.snapshotUsageMetrics.NumSnapshotDeleted}}</td>
+    </tr>
+    <tr>
+        <td>Snapshot Cache Size</td>
+        <td>{{$ctrl.snapshotUsageMetrics.NumSnapshotCacheSize}}</td>
+    </tr>
+    </tbody>
+</table>
+
+<div style="margin-bottom: 20px;"></div>
+
+<h1>Snapshot Diff Jobs</h1>
+
+<div class="row">
+    <div class="col-md-6 text-left">
+        <label>Show: </label>
+        <select class="form-select" ng-model="RecordsToDisplay" ng-change="UpdateRecordsToShow()">
+            <option value="10" ng-selected="{true}">10</option>
+            <option value="20">20</option>
+            <option value="50">50</option>
+            <option value="All">All</option>
+        </select>
+    </div>
+    <div class="col-md-6 text-right">
+        <label>Search: </label> <input type="text" ng-model="search">
+    </div>
+</div>
+
+<table class="table table-bordered table-striped">
+    <thead>
+    <tr>
+        <th ng-click="columnSort('jobId')" class="nodeStatusInfo">
+            <span ng-class="{'sorting' : (columnName != 'jobId'), 'sortasc' : (columnName == 'jobId' && !reverse), 'sortdesc':(columnName == 'jobId' && reverse)}">Job ID</span>
+        </th>
+        <th ng-click="columnSort('status')" class="nodeStatusInfo">
+            <span ng-class="{'sorting' : (columnName != 'status'), 'sortasc' : (columnName == 'status' && !reverse), 'sortdesc':(columnName == 'status' && reverse)}">Status</span>
+        </th>
+        <th ng-click="columnSort('volume')" class="nodeStatusInfo">
+            <span ng-class="{'sorting' : (columnName != 'volume'), 'sortasc' : (columnName == 'volume' && !reverse), 'sortdesc':(columnName == 'volume' && reverse)}">Volume</span>
+        </th>
+        <th ng-click="columnSort('bucket')" class="nodeStatusInfo">
+            <span ng-class="{'sorting' : (columnName != 'bucket'), 'sortasc' : (columnName == 'bucket' && !reverse), 'sortdesc':(columnName == 'bucket' && reverse)}">Bucket</span>
+        </th>
+        <th ng-click="columnSort('fromSnapshot')" class="nodeStatusInfo">
+            <span ng-class="{'sorting' : (columnName != 'fromSnapshot'), 'sortasc' : (columnName == 'fromSnapshot' && !reverse), 'sortdesc':(columnName == 'fromSnapshot' && reverse)}">From Snapshot</span>
+        </th>
+        <th ng-click="columnSort('toSnapshot')" class="nodeStatusInfo">
+            <span ng-class="{'sorting' : (columnName != 'toSnapshot'), 'sortasc' : (columnName == 'toSnapshot' && !reverse), 'sortdesc':(columnName == 'toSnapshot' && reverse)}">To Snapshot</span>
+        </th>
+        <th ng-click="columnSort('keysProcessedPct')" class="nodeStatusInfo">
+            <span ng-class="{'sorting' : (columnName != 'keysProcessedPct'), 'sortasc' : (columnName == 'keysProcessedPct' && !reverse), 'sortdesc':(columnName == 'keysProcessedPct' && reverse)}">Progress</span>
+        </th>
+        <th ng-click="columnSort('totalDiffEntries')" class="nodeStatusInfo">
+            <span ng-class="{'sorting' : (columnName != 'totalDiffEntries'), 'sortasc' : (columnName == 'totalDiffEntries' && !reverse), 'sortdesc':(columnName == 'totalDiffEntries' && reverse)}">Total Diff Entries</span>
+        </th>
+        <th ng-click="columnSort('creationTime')" class="nodeStatusInfo">
+            <span ng-class="{'sorting' : (columnName != 'creationTime'), 'sortasc' : (columnName == 'creationTime' && !reverse), 'sortdesc':(columnName == 'creationTime' && reverse)}">Creation Time</span>
+        </th>
+    </tr>
+    </thead>
+    <tbody>
+    <tr ng-repeat="job in $ctrl.snapshotDiffJobs|filter:search|orderBy:columnName:reverse">
+        <td>{{job.jobId}}</td>
+        <td>
+            {{job.status}}
+            <div ng-if="job.status === 'FAILED' && job.reason">
+                <span class="label label-danger">Reason: {{job.reason}}</span>
+            </div>
+            <div ng-if="job.status === 'IN_PROGRESS' && job.subStatus">
+                <span class="label label-info">Sub-status: {{job.subStatus}}</span>
+            </div>
+        </td>
+        <td>{{job.volume}}</td>
+        <td>{{job.bucket}}</td>
+        <td>{{job.fromSnapshot}}</td>
+        <td>{{job.toSnapshot}}</td>
+        <td>
+            <span ng-if="job.status === 'DONE'">Completed</span>
+            <span ng-if="job.status !== 'DONE'">{{job.keysProcessedPct | number:2}}%</span>
+        </td>
+        <td>{{job.totalDiffEntries}}</td>
+        <td>{{job.creationTime | date:'yyyy-MM-dd HH:mm:ss'}}</td>
+    </tr>
+    </tbody>
+</table>
+
+<div class="row">
+    <div class="col-md-6 text-left">
+        <label>Page:</label>
+        <select class="form-select" ng-model="currentPage" ng-change="handlePagination(currentPage, false)">
+            <option ng-repeat="page in getPagesArray()" ng-value="page">{{page}}</option>
+        </select>
+        <span>of {{lastIndex}}. </span>
+        <span ng-if="$ctrl.snapshotDiffJobs && $ctrl.snapshotDiffJobs.length > 0">
+            Showing {{getCurrentPageFirstItemIndex()}} to {{getCurrentPageLastItemIndex()}} of the total {{totalItems}} entries.
+        </span>
+    </div>
+    <div class="col-md-6 text-right">
+        <nav aria-label="..." ng-show="RecordsToDisplay !== 'All'">
+            <ul class="pagination" style="margin: 0; padding: 0">
+                <li class="page-item" ng-class="{disabled:currentPage==1}"
+                    ng-click="handlePagination(currentPage-1,(currentPage==1))">
+                    <span class="page-link" tabindex="-1">Previous</span>
+                </li>
+                <li class="page-item active">
+                    <span class="page-link">{{currentPage}} </span>
+                </li>
+                <li class="page-item" ng-class="{disabled:lastIndex==currentPage}"
+                    ng-click="handlePagination(currentPage+1, (lastIndex==currentPage))">
+                    <span class="page-link" tabindex="-1">Next</span>
+                </li>
+            </ul>
+        </nav>
+    </div>
+</div>
+
+<div style="margin-bottom: 20px;"></div>
+
+<h1>Snapshot Internal Metrics</h1>
+
+<table class="table table-bordered table-striped">
+    <thead>
+    <tr>
+        <th>Metric Name</th>
+        <th>Value</th>
+    </tr>
+    </thead>
+    <tbody>
+    <tr ng-repeat="metric in $ctrl.snapshotMetrics">
+        <td>{{metric.key}}</td>
+        <td>{{metric.value}}</td>
+    </tr>
+    </tbody>
+</table>