[FSTORE-618] Expand current java docstrings to cover new hsfs functionalities (#932)

davitbzh · web-flow · commit 6c1229782ccf · 2023-03-09T17:13:59.000+01:00
diff --git a/java/pom.xml b/java/pom.xml
@@ -398,6 +398,9 @@
                     <source>1.8</source>
                     <target>1.8</target>
                     <sourcepath>${delombok.output}</sourcepath>
+                    <sourceFileExcludes>
+                        <exclude>**/MainClass.java</exclude>
+                    </sourceFileExcludes>
                 </configuration>
 
                 <executions>
diff --git a/java/src/main/java/com/logicalclocks/hsfs/Feature.java b/java/src/main/java/com/logicalclocks/hsfs/Feature.java
@@ -133,6 +133,23 @@ public Feature(String name, String type, String onlineType, Boolean primary, Boo
     this.description = description;
   }
 
+  /**
+   * Returns true if the feature has a complex type.
+   *
+   * <pre>
+   * {@code // get feature store handle
+   *        FeatureStore fs = ...;
+   *        // get feature group handle
+   *        FeatureGroup fg = ...;
+   *        // get feature
+   *        Feature ft = fg.getFeature("feature_name");
+   *        // check if the feature has a complex type.
+   *        ft.isComplex();
+   * }
+   * </pre>
+   *
+   * @return boolean
+   */
   @JsonIgnore
   public boolean isComplex() {
     return Constants.COMPLEX_FEATURE_TYPES.stream().anyMatch(c -> type.toUpperCase().startsWith(c));
@@ -198,6 +215,7 @@ public Filter ge(Feature value) {
    * @deprecated
    * `in` method is deprecated. Use `isin` instead.
    */
+  @Deprecated
   public Filter in(Collection<?> collection) {
     return isin(collection);
   }
diff --git a/java/src/main/java/com/logicalclocks/hsfs/FeatureGroup.java b/java/src/main/java/com/logicalclocks/hsfs/FeatureGroup.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/FeatureStore.java b/java/src/main/java/com/logicalclocks/hsfs/FeatureStore.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/FeatureView.java b/java/src/main/java/com/logicalclocks/hsfs/FeatureView.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/StorageConnector.java b/java/src/main/java/com/logicalclocks/hsfs/StorageConnector.java
@@ -77,6 +77,20 @@ public abstract class StorageConnector {
 
   protected StorageConnectorApi storageConnectorApi = new StorageConnectorApi();
 
+  /**
+   * Reads a query or a path into a dataframe using the storage connector.
+   *
+   * @param query By default, the storage connector will read the table configured together with the connector, if any.
+   *              It's possible to overwrite this by passing a SQL query here.
+   * @param dataFormat When reading from object stores such as S3, HopsFS and ADLS, specify the file format to be read,
+   *                  e.g. `csv`, `parquet`.
+   * @param options Any additional key/value options to be passed to the connector.
+   * @param path Path to be read from within the bucket of the storage connector. Not relevant for JDBC or database
+   *             based connectors such as Snowflake, JDBC or Redshift.
+   * @return DataFrame.
+   * @throws FeatureStoreException If ...
+   * @throws IOException If ...
+   */
   public Object read(String query, String dataFormat, Map<String, String> options, String path)
       throws FeatureStoreException, IOException {
     return SparkEngine.getInstance().read(this, dataFormat, options, path);
diff --git a/java/src/main/java/com/logicalclocks/hsfs/StreamFeatureGroup.java b/java/src/main/java/com/logicalclocks/hsfs/StreamFeatureGroup.java
diff --git a/java/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupEngine.java b/java/src/main/java/com/logicalclocks/hsfs/engine/FeatureGroupEngine.java
@@ -58,9 +58,10 @@ public class FeatureGroupEngine {
    *                           key.
    * @param writeOptions Additional write options as key-value pairs, defaults to empty Map.
    * @return Feature Group metadata object
-   * @throws FeatureStoreException FeatureStoreException
-   * @throws IOException IOException
-   * @throws ParseException ParseException
+   * @throws FeatureStoreException In case Client is not connected to Hopsworks, unable to identify date format and/or
+   *                               no commit information was found for  the feature group;
+   * @throws IOException Generic IO exception.
+   * @throws ParseException In case it's unable to parse HUDI and or statistics commit date string to date type.
    */
   public FeatureGroup save(FeatureGroup featureGroup, Dataset<Row> dataset, List<String> partitionKeys,
                            String hudiPrecombineKey, Map<String, String> writeOptions)
diff --git a/java/src/main/java/com/logicalclocks/hsfs/engine/SparkEngine.java b/java/src/main/java/com/logicalclocks/hsfs/engine/SparkEngine.java
@@ -257,8 +257,8 @@ public void registerHudiTemporaryTable(HudiFeatureGroupAlias hudiFeatureGroupAli
    * @param writeOptions Additional write options as key-value pairs, defaults to empty Map
    * @param saveMode org.apache.spark.sql.saveMode: Append, Overwrite, ErrorIfExists, Ignore
    * @return Spark dataframe
-   * @throws FeatureStoreException  FeatureStoreException
-   * @throws IOException IOException
+   * @throws FeatureStoreException If Client is not connected to Hopsworks
+   * @throws IOException Generic IO exception.
    */
   public Dataset<Row>[] write(TrainingDataset trainingDataset, Query query, Map<String, String> queryReadOptions,
                     Map<String, String> writeOptions, SaveMode saveMode) throws FeatureStoreException, IOException {
@@ -561,8 +561,8 @@ public <S> StreamingQuery writeStreamDataframe(FeatureGroupBase featureGroupBase
    * @param featureGroupBase FeatureGroupBase Feature Group base metadata object
    * @param dataset Spark DataFrame or RDD.
    * @return Spark DataFrame.
-   * @throws FeatureStoreException FeatureStoreException
-   * @throws IOException IOException
+   * @throws FeatureStoreException If Client is not connected to Hopsworks
+   * @throws IOException Generic IO exception.
    */
   public Dataset<Row> encodeComplexFeatures(FeatureGroupBase featureGroupBase, Dataset<Row> dataset)
           throws FeatureStoreException, IOException {
@@ -584,8 +584,8 @@ public Dataset<Row> encodeComplexFeatures(FeatureGroupBase featureGroupBase, Dat
    * @param featureGroupBase FeatureGroupBase Feature Group base metadata object
    * @param dataset  Spark DataFrame or RDD.
    * @return Spark DataFrame.
-   * @throws FeatureStoreException FeatureStoreException
-   * @throws IOException IOException
+   * @throws FeatureStoreException If Client is not connected to Hopsworks
+   * @throws IOException Generic IO exception.
    */
   private Dataset<Row> onlineFeatureGroupToAvro(FeatureGroupBase featureGroupBase, Dataset<Row> dataset)
       throws FeatureStoreException, IOException {
@@ -676,7 +676,7 @@ public String profile(Dataset<Row> df) {
   }
 
   public void setupConnectorHadoopConf(StorageConnector storageConnector)
-          throws FeatureStoreException, IOException {
+          throws IOException {
     if (storageConnector == null) {
       return;
     }
diff --git a/java/src/main/java/com/logicalclocks/hsfs/engine/hudi/HudiEngine.java b/java/src/main/java/com/logicalclocks/hsfs/engine/hudi/HudiEngine.java
@@ -211,7 +211,7 @@ private FeatureGroupCommit getLastCommitMetadata(SparkSession sparkSession, Stri
 
   private Map<String, String> setupHudiWriteOpts(FeatureGroupBase featureGroup, HudiOperationType operation,
                                                 Map<String, String> writeOptions)
-      throws IOException, FeatureStoreException {
+      throws FeatureStoreException {
     Map<String, String> hudiArgs = new HashMap<String, String>();
 
     hudiArgs.put(HUDI_TABLE_STORAGE_TYPE, HUDI_COPY_ON_WRITE);
diff --git a/java/src/main/java/com/logicalclocks/hsfs/metadata/FeatureGroupBase.java b/java/src/main/java/com/logicalclocks/hsfs/metadata/FeatureGroupBase.java
@@ -176,8 +176,8 @@ public <T> T read() throws FeatureStoreException, IOException {
    *
    * @param name  name of the tag
    * @param value value of the tag. The value of a tag can be any valid json - primitives, arrays or json objects.
-   * @throws FeatureStoreException FeatureStoreException
-   * @throws IOException IOException
+   * @throws FeatureStoreException If Client is not connected to Hopsworks,
+   * @throws IOException Generic IO exception.
    */
   public void addTag(String name, Object value) throws FeatureStoreException, IOException {
     featureGroupBaseEngine.addTag(this, name, value);
@@ -187,8 +187,8 @@ public void addTag(String name, Object value) throws FeatureStoreException, IOEx
    * Get all tags of the feature group.
    *
    * @return map of tag name and values. The value of a tag can be any valid json - primitives, arrays or json objects
-   * @throws FeatureStoreException FeatureStoreException
-   * @throws IOException IOException
+   * @throws FeatureStoreException If Client is not connected to Hopsworks,
+   * @throws IOException Generic IO exception.
    */
   @JsonIgnore
   public Map<String, Object> getTags() throws FeatureStoreException, IOException {
@@ -212,8 +212,8 @@ public Object getTag(String name) throws FeatureStoreException, IOException {
    * Delete a tag of the feature group.
    *
    * @param name name of the tag to be deleted
-   * @throws FeatureStoreException FeatureStoreException
-   * @throws IOException IOException
+   * @throws FeatureStoreException If Client is not connected to Hopsworks,
+   * @throws IOException Generic IO exception.
    */
   public void deleteTag(String name) throws FeatureStoreException, IOException {
     featureGroupBaseEngine.deleteTag(this, name);
@@ -223,8 +223,8 @@ public void deleteTag(String name) throws FeatureStoreException, IOException {
    * Update the description of the feature group.
    *
    * @param description feature group description.
-   * @throws FeatureStoreException FeatureStoreException
-   * @throws IOException IOException
+   * @throws FeatureStoreException If Client is not connected to Hopsworks,
+   * @throws IOException Generic IO exception.
    */
   public void updateDescription(String description) throws FeatureStoreException, IOException {
     featureGroupBaseEngine.updateDescription(this, description, this.getClass());
@@ -235,8 +235,8 @@ public void updateDescription(String description) throws FeatureStoreException,
    *
    * @param featureName Name of the feature
    * @param description Description of the feature
-   * @throws FeatureStoreException FeatureStoreException
-   * @throws IOException IOException
+   * @throws FeatureStoreException If Client is not connected to Hopsworks,
+   * @throws IOException Generic IO exception.
    */
   public void updateFeatureDescription(String featureName, String description)
       throws FeatureStoreException, IOException {
@@ -250,9 +250,10 @@ public void updateFeatureDescription(String featureName, String description)
    * Currently only feature description updates are supported.
    *
    * @param features List of Feature metadata objects
-   * @throws FeatureStoreException FeatureStoreException
-   * @throws IOException IOException
-   * @throws ParseException ParseException
+   * @throws FeatureStoreException If Client is not connected to Hopsworks, unable to identify date format and/or
+   *                               no commit information was found for this feature group;
+   * @throws IOException Generic IO exception.
+   * @throws ParseException In case it's unable to parse date string to date type.
    */
   public void updateFeatures(List<Feature> features) throws FeatureStoreException, IOException, ParseException {
     featureGroupBaseEngine.appendFeatures(this, features, this.getClass());
@@ -263,9 +264,10 @@ public void updateFeatures(List<Feature> features) throws FeatureStoreException,
    * Currently only feature description updates are supported.
    *
    * @param feature Feature metadata object
-   * @throws FeatureStoreException FeatureStoreException
-   * @throws IOException IOException
-   * @throws ParseException ParseException
+   * @throws FeatureStoreException If Client is not connected to Hopsworks, unable to identify date format and/or
+   *                               no commit information was found for this feature group;
+   * @throws IOException Generic IO exception.
+   * @throws ParseException In case it's unable to parse date string to date type.
    */
   public void updateFeatures(Feature feature) throws FeatureStoreException, IOException, ParseException {
     featureGroupBaseEngine.appendFeatures(this, Collections.singletonList(feature), this.getClass());
@@ -276,9 +278,10 @@ public void updateFeatures(Feature feature) throws FeatureStoreException, IOExce
    * It is only possible to append features to a feature group. Removing features is considered a breaking change.
    *
    * @param features list of Feature metadata objects
-   * @throws FeatureStoreException FeatureStoreException
-   * @throws IOException IOException
-   * @throws ParseException ParseException
+   * @throws FeatureStoreException If Client is not connected to Hopsworks, unable to identify date format and/or
+   *                               no commit information was found for this feature group;
+   * @throws IOException Generic IO exception.
+   * @throws ParseException In case it's unable to parse date string to date type.
    */
   public void appendFeatures(List<Feature> features) throws FeatureStoreException, IOException, ParseException {
     featureGroupBaseEngine.appendFeatures(this, new ArrayList<>(features), this.getClass());
@@ -289,9 +292,10 @@ public void appendFeatures(List<Feature> features) throws FeatureStoreException,
    * It is only possible to append features to a feature group. Removing features is considered a breaking change.
    *
    * @param features List of Feature metadata objects
-   * @throws FeatureStoreException FeatureStoreException
-   * @throws IOException IOException
-   * @throws ParseException ParseException
+   * @throws FeatureStoreException If Client is not connected to Hopsworks, unable to identify date format and/or
+   *                               no commit information was found for this feature group;
+   * @throws IOException Generic IO exception.
+   * @throws ParseException In case it's unable to parse date string to date type.
    */
   public void appendFeatures(Feature features) throws FeatureStoreException, IOException, ParseException {
     List<Feature> featureList = new ArrayList<>();
@@ -304,8 +308,8 @@ public void appendFeatures(Feature features) throws FeatureStoreException, IOExc
    * Change the `enabled`, `histograms`, `correlations` or `columns` attributes and persist
    * the changes by calling this method.
    *
-   * @throws FeatureStoreException FeatureStoreException
-   * @throws IOException IOException
+   * @throws FeatureStoreException If Client is not connected to Hopsworks,
+   * @throws IOException Generic IO exception.
    */
   public void updateStatisticsConfig() throws FeatureStoreException, IOException {
     featureGroupBaseEngine.updateStatisticsConfig(this, this.getClass());
@@ -315,8 +319,8 @@ public void updateStatisticsConfig() throws FeatureStoreException, IOException {
    * Recompute the statistics for the feature group and save them to the feature store.
    *
    * @return statistics object of computed statistics
-   * @throws FeatureStoreException FeatureStoreException
-   * @throws IOException IOException
+   * @throws FeatureStoreException If Client is not connected to Hopsworks,
+   * @throws IOException Generic IO exception.
    */
   public Statistics computeStatistics() throws FeatureStoreException, IOException {
     if (statisticsConfig.getEnabled()) {
@@ -332,8 +336,8 @@ public Statistics computeStatistics() throws FeatureStoreException, IOException
    * Get the last statistics commit for the feature group.
    *
    * @return statistics object of latest commit
-   * @throws FeatureStoreException FeatureStoreException
-   * @throws IOException IOException
+   * @throws FeatureStoreException If Client is not connected to Hopsworks,
+   * @throws IOException Generic IO exception.
    */
   @JsonIgnore
   public Statistics getStatistics() throws FeatureStoreException, IOException {
@@ -345,8 +349,8 @@ public Statistics getStatistics() throws FeatureStoreException, IOException {
    *
    * @param commitTime commit time in the format "YYYYMMDDhhmmss"
    * @return statistics object for the commit time
-   * @throws FeatureStoreException FeatureStoreException
-   * @throws IOException IOException
+   * @throws FeatureStoreException If Client is not connected to Hopsworks and/or unable to identify date format.
+   * @throws IOException Generic IO exception.
    */
   @JsonIgnore
   public Statistics getStatistics(String commitTime) throws FeatureStoreException, IOException {
@@ -371,8 +375,8 @@ public void unloadSubject() {
    *
    * @param filter Filter metadata object
    * @return Query object
-   * @throws FeatureStoreException FeatureStoreException
-   * @throws IOException IOException
+   * @throws FeatureStoreException If Client is not connected to Hopsworks.
+   * @throws IOException Generic IO exception.
    */
   public Query filter(Filter filter) throws FeatureStoreException, IOException {
     return this.selectAll().filter(filter);
@@ -383,8 +387,8 @@ public Query filter(Filter filter) throws FeatureStoreException, IOException {
    *
    * @param filter Filter metadata object
    * @return Query object
-   * @throws FeatureStoreException FeatureStoreException
-   * @throws IOException IOException
+   * @throws FeatureStoreException If Client is not connected to Hopsworks.
+   * @throws IOException Generic IO exception.
    */
   public Query filter(FilterLogic filter) throws FeatureStoreException, IOException {
     return this.selectAll().filter(filter);
@@ -395,7 +399,7 @@ public Query filter(FilterLogic filter) throws FeatureStoreException, IOExceptio
    *
    * @param name feature name
    * @return Feature metadata object
-   * @throws FeatureStoreException FeatureStoreException
+   * @throws FeatureStoreException If Client is not connected to Hopsworks.
    */
   @JsonIgnore
   public Feature getFeature(String name) throws FeatureStoreException {
diff --git a/python/hsfs/feature_group.py b/python/hsfs/feature_group.py
@@ -1574,9 +1574,9 @@ def insert(
         or insert data from a dataframe into the existing feature group.
 
         Incrementally insert data to a feature group or overwrite all  data contained in the feature group. By
-        default, the data is inserted into the offline storag as well as the online storage if the feature group is
-        `online_enabled=True`. To insert only into the online storage, set `storage="online"`, or oppositely
-        `storage="offline"`.
+        default, the data is inserted into the offline storage as well as the online storage if the feature group is
+        `online_enabled=True`. To insert only into the online or offline storage set `storage="online"` or
+        `storage="offline"` respectively.
 
         The `features` dataframe can be a Spark DataFrame or RDD, a Pandas DataFrame,
         or a two-dimensional Numpy array or a two-dimensional Python nested list.
@@ -1992,7 +1992,7 @@ def commit_delete_record(
         write_options: Optional[Dict[Any, Any]] = {},
     ):
         """Drops records present in the provided DataFrame and commits it as update to this
-        Feature group. This method can only be used on time travel enabled feature groups
+        Feature group. This method can only be used on time travel enabled feature groups.
 
         # Arguments
             delete_df: dataFrame containing records to be deleted.
