fixes according to review

Author: dgrinenko

docs/SendGrid-batchsource.md

@@ -2,21 +2,22 @@
 
 Description
 -----------
-This plugin used to query SendGrid v3 API.
-
+Plugin fetches data from SendGrid. SendGrid is a cloud-based service that assists businesses with email delivery. For 
+the end user SendGrid provides information about existing Marketing Campaigns, Email Analytics, Bounces, Spam Reports.
+ 
 Properties
 ----------
 ### General
 
 **Reference Name:** Name used to uniquely identify this source for lineage, annotating metadata, etc.
 
-**Authentication type:** The way, how user would like to be authenticated to the SendGrid acoount
+**Authentication type:** The way, how user would like to be authenticated to the SendGrid account
 
 **API Key:** The SendGrid API Key taken from the SendGrid account
 
-**Username:** Login name for SendGrid
+**Username:** Login name for the SendGrid account
 
-**Password:** Login password for the username specified above
+**Password:** Password for the SendGrid account
 
 **Data Source Types:** List of data source groups
 
@@ -25,7 +26,7 @@ Available:
 - Stats Fields
 - Suppressions Fields
 
-**Data Source:** One of the above sources picked from list
+**Data Source:**  SendGrid source object
 
 Available:
 - Marketing Campaigns Fields
@@ -42,7 +43,18 @@ Available:
   - Bounces
   - Global Unsubscribes
   - Group Unsubscribes
-**Data Source Fields:** 
+  
+**Data Source Fields:** The list of fields available for the retrieval
+
+- *Automation:* id, name, status, type, message_count, created_at, updated_at, live_at
+- *SingleSends:* id, name, status, created_at, updated_at,
+- *Senders:* id, nickname, address, address_2, city, country, state, zip, locked, created_at, updated_at, from(email, name), verified(status, reason), reply_to(name, email)
+- *Contacts:* id, first_name, last_name, list_ids, created_at, updated_at, email, Segments, id, name, parent_list_id, created_at, updated_at, sample_updated_at, contacts_count
+- *GlobalStats:* date ,blocks ,bounce_drops ,bounces ,clicks ,deferred ,invalid_emails ,opens ,processed ,requests ,spam_report_drops ,spam_reports ,unique_clicks ,unique_opens ,unsubscribe_drops ,unsubscribes
+- *CategoryStats:* name, type, date, blocks, bounce_drops, bounces, clicks, deferred, delivered, invalid_emails, opens, processed, requests, spam_report_drops, spam_reports, unique_clicks, unique_opens, unsubscribe_drops, unsubscribes
+- *AdvancedStats:* name, type, date, clicks, opens, unique_clicks, unique_opens
+- *Bounces:* created, email, reason, status
+- *GroupUnsubscribes:* id ,name ,description ,is_default ,last_email_send_at ,unsubscribes
 
 **Start Date:** The date in format YYYY-MM-DD, starting from which the data is requested
 

pom.xml

@@ -28,13 +28,29 @@
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <cdap.version>6.1.0-SNAPSHOT</cdap.version>
-        <cdap.plugin.version>2.2.0-SNAPSHOT</cdap.plugin.version>
-        <hadoop.version>2.3.0</hadoop.version>
-        <hydrator.version>2.2.0</hydrator.version>
+        <hadoop.version>2.8.0</hadoop.version>
+        <hydrator.version>2.3.0-SNAPSHOT</hydrator.version>
         <gson.version>2.2.4</gson.version>
         <common.logging.version>1.2</common.logging.version>
     </properties>
 
+    <repositories>
+        <repository>
+            <id>sonatype</id>
+            <url>https://oss.sonatype.org/content/groups/public</url>
+        </repository>
+        <repository>
+            <id>sonatype-snapshots</id>
+            <url>https://oss.sonatype.org/content/repositories/snapshots</url>
+        </repository>
+    </repositories>
+    <pluginRepositories>
+        <pluginRepository>
+            <id>sonatype</id>
+            <url>https://oss.sonatype.org/content/groups/public/</url>
+        </pluginRepository>
+    </pluginRepositories>
+
 
     <dependencies>
         <dependency>
@@ -84,7 +100,7 @@
         <dependency>
             <groupId>io.cdap.plugin</groupId>
             <artifactId>hydrator-common</artifactId>
-            <version>${cdap.plugin.version}</version>
+            <version>${hydrator.version}</version>
         </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>
@@ -213,11 +229,7 @@
                <extensions>true</extensions>
                <configuration>
                    <instructions>
-                       <_exportcontents>
-                           io.cdap.plugin.*;
-                           org.apache.commons.lang;
-                           org.apache.commons.logging.*;
-                       </_exportcontents>
+                       <_exportcontents>io.cdap.plugin.sendgrid.*</_exportcontents>
                        <Embed-Dependency>*;inline=false;scope=compile</Embed-Dependency>
                        <Embed-Transitive>true</Embed-Transitive>
                        <Embed-Directory>lib</Embed-Directory>

src/main/java/io/cdap/plugin/sendgrid/common/SendGridClient.java

@@ -31,7 +31,6 @@
 import java.io.IOException;
 import java.lang.reflect.ParameterizedType;
 import java.lang.reflect.Type;
-import java.net.ConnectException;
 import java.util.ArrayList;
 import java.util.Base64;
 import java.util.List;
@@ -85,7 +84,7 @@ public SendGridClient(SendGridBatchSourceConfig config) {
     } else if (config.getAuthType() == SendGridAuthType.BASIC) {
       sendGrid = new SendGridAPIClient(config.getAuthUserName(), config.getAuthPassword());
     } else {
-      throw new IllegalArgumentException(String.format("Invalid %s authentication method",
+      throw new IllegalArgumentException(String.format("Invalid authentication method '%s'",
         SendGridClient.class.getCanonicalName()));
     }
   }
@@ -107,10 +106,10 @@ public SendGridClient(String username, String password) {
    * @param endpoint relative uri to be queried
    * @param arguments arguments for the query
    * @return query body
-   * @throws ConnectException if any issue with query the API happen
+   * @throws IOException if any issue with query the API happen
    */
   private String makeApiRequest(Method method, String endpoint, @Nullable Map<String, String> arguments)
-    throws ConnectException {
+    throws IOException {
 
     Request request = new Request();
     request.setMethod(method);
@@ -124,8 +123,7 @@ private String makeApiRequest(Method method, String endpoint, @Nullable Map<Stri
     try {
       response = sendGrid.api(request);
     } catch (IOException e) {
-      throw new ConnectException(String.format("Request to SendGrid API \"%s\"failed with: %s", endpoint,
-        e.getMessage()));
+      throw new IOException(String.format("Request to SendGrid API \"%s\"", endpoint), e);
     }
 
     return response.getBody();
@@ -136,13 +134,17 @@ private String makeApiRequest(Method method, String endpoint, @Nullable Map<Stri
    *
    * @param objectInfo objects definition
    * @param arguments query arguments
-   * @throws ConnectException if any validation issue
+   * @throws IllegalArgumentException if any validation issue
    */
-  private void checkIncomingArguments(ObjectInfo objectInfo, Map<String, String> arguments) throws ConnectException {
+  private void checkIncomingArguments(ObjectInfo objectInfo, Map<String, String> arguments)
+    throws IllegalArgumentException {
+
     if (objectInfo.getRequiredArguments() != null && !objectInfo.getRequiredArguments().isEmpty()) {
       if (arguments == null || arguments.isEmpty()) {
-        throw new ConnectException(String.format("Object %s require input arguments to be passed, nothing found",
-          objectInfo.getCdapObjectName()));
+        throw new IllegalArgumentException(String.format(
+          "Object '%s' require input arguments to be passed, nothing found",
+          objectInfo.getCdapObjectName()
+        ));
       }
       List<String> exceptions = new ArrayList<>();
 
@@ -155,14 +157,17 @@ private void checkIncomingArguments(ObjectInfo objectInfo, Map<String, String> a
           arguments.keySet().stream()
             .filter(x::equals)
             .findFirst()
-            .orElseThrow(() -> new ConnectException(String.format("Object %s require %s argument, but nothing provided",
-              objectInfo.getCdapObjectName(), x)));
-        } catch (ConnectException e) {
+            .orElseThrow(() -> new IllegalArgumentException(String.format(
+              "Object '%s' require %s argument, but nothing provided",
+              objectInfo.getCdapObjectName(),
+              x
+            )));
+        } catch (IllegalArgumentException e) {
           exceptions.add(e.getMessage());
         }
       });
       if (!exceptions.isEmpty()) {
-        throw new ConnectException(exceptions.stream().collect(Collectors.joining(System.lineSeparator())));
+        throw new IllegalArgumentException(exceptions.stream().collect(Collectors.joining(System.lineSeparator())));
       }
     }
   }
@@ -173,9 +178,12 @@ private void checkIncomingArguments(ObjectInfo objectInfo, Map<String, String> a
    * @param objectInfo objects definition
    * @param arguments query arguments
    * @return object representation of the query
-   * @throws ConnectException if any issue with query the API happen
+   * @throws IOException if any issue with query the API happen
+   * @throws IllegalStateException unsupported response type caught
+   * @throws IllegalArgumentException if any validation issue
    */
-  public List<IBaseObject> getObject(ObjectInfo objectInfo, Map<String, String> arguments) throws ConnectException {
+  public List<IBaseObject> getObject(ObjectInfo objectInfo, Map<String, String> arguments)
+    throws IOException, IllegalStateException, IllegalArgumentException {
     checkIncomingArguments(objectInfo, arguments);
 
     String endpoint = objectInfo.getSendGridAPIUrl();
@@ -222,8 +230,10 @@ public Type getOwnerType() {
 
       return gson.fromJson(response, typeToken);
     } else {
-      throw new ConnectException(String.format("Unsupported API Response type: %s",
-        objectInfo.getApiResponseType().name()));
+      throw new IllegalStateException(String.format(
+        "Unsupported API Response type: %s",
+        objectInfo.getApiResponseType().name()
+      ));
     }
   }
 
@@ -232,13 +242,13 @@ public Type getOwnerType() {
    *
    * @param objectInfo objects definition
    * @return object representation of the query
-   * @throws ConnectException if any issue with query the API happen
+   * @throws IOException if any issue with query the API happen
+   * @throws IllegalStateException unsupported response type caught
+   * @throws IllegalArgumentException if any validation issue
    */
-  public List<IBaseObject> getObject(ObjectInfo objectInfo) throws ConnectException {
+  public List<IBaseObject> getObject(ObjectInfo objectInfo)
+    throws IOException, IllegalStateException, IllegalArgumentException {
+
     return getObject(objectInfo, null);
   }
-
-
-
-
 }

src/main/java/io/cdap/plugin/sendgrid/common/config/BaseSourceConfig.java

@@ -17,18 +17,22 @@
 
 import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+
 import io.cdap.cdap.api.annotation.Description;
 import io.cdap.cdap.api.annotation.Macro;
 import io.cdap.cdap.api.annotation.Name;
 import io.cdap.cdap.api.data.schema.Schema;
 import io.cdap.cdap.etl.api.FailureCollector;
 import io.cdap.plugin.common.ReferencePluginConfig;
+import io.cdap.plugin.sendgrid.common.helpers.ObjectDefinition;
 import io.cdap.plugin.sendgrid.common.helpers.ObjectHelper;
 import io.cdap.plugin.sendgrid.common.objects.SendGridAuthType;
 
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 
 import javax.annotation.Nullable;
 
@@ -47,76 +51,83 @@ public class BaseSourceConfig extends ReferencePluginConfig {
   public static final String PROPERTY_DATA_SOURCE_SUPPRESSIONS = PROPERTY_DATA_SOURCE + "Suppressions";
 
   public static final String PROPERTY_DATA_SOURCE_FIELDS = "dataSourceFields";
+  public static final String PROPERTY_STAT_CATEGORIES = "statCategories";
   public static final String PROPERTY_START_DATE = "startDate";
   public static final String PROPERTY_END_DATE = "endDate";
 
   @Name(PROPERTY_AUTH_TYPE)
-  @Description("Auth type")
+  @Description("The way, how user would like to be authenticated to the SendGrid account")
   @Macro
   private String authType;
 
   @Name((PROPERTY_SENDGRID_API_KEY))
-  @Description("Api key")
+  @Description("The SendGrid API Key taken from the SendGrid account")
   @Macro
   @Nullable
   private String sendGridApiKey;
 
   @Name(PROPERTY_AUTH_USERNAME)
-  @Description("Username")
+  @Description("Login name for the SendGrid account")
   @Macro
   @Nullable
   private String authUserName;
 
   @Name(PROPERTY_AUTH_PASSWORD)
-  @Description("Password")
+  @Description("Password for the SendGrid account")
   @Macro
   @Nullable
   private String authPassword;
 
   @Name(PROPERTY_DATA_SOURCE_TYPES)
-  @Description("Data source categories")
+  @Description("List of data source groups")
   @Macro
   private String dataSourceTypes;
 
   @Name(PROPERTY_DATA_SOURCE_MARKETING)
-  @Description("Data source marketing objects")
+  @Description("SendGrid source objects for the Marketing group")
   @Macro
   @Nullable
   private String dataSourceMarketing;
 
   @Name(PROPERTY_DATA_SOURCE_STATS)
-  @Description("Data source stats objects")
+  @Description("SendGrid source objects for the Statistics group")
   @Macro
   @Nullable
   private String dataSourceStats;
 
   @Name(PROPERTY_DATA_SOURCE_SUPPRESSIONS)
-  @Description("Data source suppression objects")
+  @Description("SendGrid source objects for the Suppressions group")
   @Macro
   @Nullable
   private String dataSourceSuppressions;
 
   @Name(PROPERTY_DATA_SOURCE_FIELDS)
-  @Description("Data source objects fields")
+  @Description("The list of fields available for the retrieval")
   @Macro
   @Nullable
   private String dataSourceFields;
 
   @Name(PROPERTY_START_DATE)
-  @Description("Request information starting from")
+  @Description("The date in format YYYY-MM-DD, starting from which the data is requested")
   @Nullable
   @Macro
   private String startDate;
 
   @Name(PROPERTY_END_DATE)
-  @Description("End period of the requested information")
+  @Description("The date in format YYYY-MM-DD, the end date for the requested data")
   @Nullable
   @Macro
   private String endDate;
 
-  private Schema schema;
-  private List<String> dataSource;
-  private Boolean multiObjectMode;
+  @Name(PROPERTY_STAT_CATEGORIES)
+  @Description("List of requested categories for the CategoryStats request")
+  @Nullable
+  @Macro
+  private String statCategories;
+
+  private transient Schema schema;
+  private transient List<String> dataSource;
+  private transient Boolean multiObjectMode;
 
   /**
    * Constructor
@@ -193,6 +204,25 @@ public Schema getSchema(List<String> dataSource) {
     return ObjectHelper.buildSchema(dataSource, getFields(), isMultiObjectMode());
   }
 
+  /**
+   * Returns query properties required for some SendGrid Objects
+   * marked with {@link ObjectDefinition#RequiredArguments()}
+   */
+  public Map<String, String> getRequestArguments() {
+    ImmutableMap.Builder<String, String> builder = new ImmutableMap.Builder<>();
+
+    if (!Strings.isNullOrEmpty(startDate)) {
+      builder.put(PROPERTY_START_DATE, startDate);
+    }
+    if (!Strings.isNullOrEmpty(endDate)) {
+      builder.put(PROPERTY_END_DATE, endDate);
+    }
+    if (!Strings.isNullOrEmpty(statCategories)) {
+      builder.put(PROPERTY_END_DATE, statCategories);
+    }
+    return builder.build();
+  }
+
   public void validate(FailureCollector failureCollector) {
 
   }
@@ -207,7 +237,7 @@ public SendGridAuthType getAuthType() {
       case "basic":
         return SendGridAuthType.BASIC;
       default:
-        return SendGridAuthType.UNKNOWN;
+        throw new IllegalArgumentException(String.format("Authentication using '%s' is not supported", authType));
     }
   }