Add Javadoc

Based on #61

Author: XoMEX

README.md

@@ -18,9 +18,24 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+/**
+ * Common functionality for crawler main entry points. Provides functionality to parse command line
+ * arguments and start the worker/controller as specified by the user.
+ */
 public class CommonMain {
     private static final Logger LOGGER = LogManager.getLogger();
 
+    /**
+     * Main entry point for the application. Uses JCommander to parse the controller/worker command
+     * and parse the arguments into the respective configuration object. Then starts the
+     * controller/worker.
+     *
+     * @param args Command line arguments
+     * @param controllerCommandConfig Configuration for the controller. Will be filled by JCommander
+     *     from the command line if first argument is "controller".
+     * @param workerCommandConfig Configuration for the worker. Will be filled by JCommander from
+     *     the command line if first argument is "worker".
+     */
     public static void main(
             String[] args,
             ControllerCommandConfig controllerCommandConfig,
@@ -71,6 +86,15 @@ public static void main(
         }
     }
 
+    /**
+     * Convenience method to start the application with just a controller configuration. Creates a
+     * default worker configuration. See {@link #main(String[], ControllerCommandConfig,
+     * WorkerCommandConfig)} for details.
+     *
+     * @param args Command line arguments
+     * @param controllerConfig Configuration for the controller. Will be filled by JCommander from
+     *     the command line if first argument is "controller".
+     */
     public static void main(String[] args, ControllerCommandConfig controllerConfig) {
         main(args, controllerConfig, new WorkerCommandConfig());
     }

src/main/java/de/rub/nds/crawler/CommonMain.java

@@ -26,6 +26,12 @@
 import org.apache.commons.validator.routines.UrlValidator;
 import org.quartz.CronScheduleBuilder;
 
+/**
+ * Configuration class for controller instances used to parse command line parameters. Contains
+ * settings for the controller's behavior, including scan parameters, target selection, and
+ * notification settings. This abstract class provides the base configuration, while specific
+ * scanner implementations must extend it to provide scanner-specific configuration.
+ */
 public abstract class ControllerCommandConfig {
 
     @ParametersDelegate private final RabbitMqDelegate rabbitMqDelegate;
@@ -123,6 +129,7 @@ public void validate() {
         }
     }
 
+    /** Validator that ensures parameter values are positive integers. */
     public static class PositiveInteger implements IParameterValidator {
         public void validate(String name, String value) throws ParameterException {
             int n = Integer.parseInt(value);
@@ -133,6 +140,7 @@ public void validate(String name, String value) throws ParameterException {
         }
     }
 
+    /** Validator that ensures parameter values are valid cron expressions. */
     public static class CronSyntax implements IParameterValidator {
         public void validate(String name, String value) throws ParameterException {
             CronScheduleBuilder.cronSchedule(value);

src/main/java/de/rub/nds/crawler/config/ControllerCommandConfig.java

@@ -13,6 +13,11 @@
 import de.rub.nds.crawler.config.delegate.MongoDbDelegate;
 import de.rub.nds.crawler.config.delegate.RabbitMqDelegate;
 
+/**
+ * Configuration class for worker instances used to parse command line parameters. Contains settings
+ * for the worker's behavior, including thread counts and timeouts, as well as MongoDB and RabbitMQ
+ * connection settings.
+ */
 public class WorkerCommandConfig {
 
     @ParametersDelegate private final RabbitMqDelegate rabbitMqDelegate;

src/main/java/de/rub/nds/crawler/config/WorkerCommandConfig.java

@@ -10,16 +10,17 @@
 
 import com.beust.jcommander.Parameter;
 
+/** Configuration delegate that holds MongoDB connection settings. */
 public class MongoDbDelegate {
 
     @Parameter(
             names = "-mongoDbHost",
-            description = "Host of the MongoDB instance this crawler saves to.")
+            description = "Host of the MongoDB instance this crawler saves results to.")
     private String mongoDbHost;
 
     @Parameter(
             names = "-mongoDbPort",
-            description = "Port of the MongoDB instance this crawler saves to.")
+            description = "Port of the MongoDB instance this crawler saves results to.")
     private int mongoDbPort;
 
     @Parameter(
@@ -29,12 +30,12 @@ public class MongoDbDelegate {
 
     @Parameter(
             names = "-mongoDbPass",
-            description = "The passwort to be used to authenticate with MongoDB.")
+            description = "The password to be used to authenticate with MongoDB.")
     private String mongoDbPass;
 
     @Parameter(
             names = "-mongoDbPassFile",
-            description = "The passwort to be used to authenticate with MongoDB.")
+            description = "The password to be used to authenticate with MongoDB.")
     private String mongoDbPassFile;
 
     @Parameter(

src/main/java/de/rub/nds/crawler/config/delegate/MongoDbDelegate.java

@@ -10,24 +10,31 @@
 
 import com.beust.jcommander.Parameter;
 
+/** Configuration delegate that holds RabbitMQ connection settings. */
 public class RabbitMqDelegate {
 
-    @Parameter(names = "-rabbitMqHost")
+    @Parameter(names = "-rabbitMqHost", description = "Host of the RabbitMQ instance")
     private String rabbitMqHost;
 
-    @Parameter(names = "-rabbitMqPort")
+    @Parameter(names = "-rabbitMqPort", description = "Port of the RabbitMQ instance")
     private int rabbitMqPort;
 
-    @Parameter(names = "-rabbitMqUser")
+    @Parameter(names = "-rabbitMqUser", description = "Username for RabbitMQ authentication")
     private String rabbitMqUser;
 
-    @Parameter(names = "-rabbitMqPass")
+    @Parameter(
+            names = "-rabbitMqPass",
+            description =
+                    "Password for RabbitMQ authentication. Alternatively use -rabbitMqPassFile")
     private String rabbitMqPass;
 
-    @Parameter(names = "-rabbitMqPassFile")
+    @Parameter(
+            names = "-rabbitMqPassFile",
+            description =
+                    "File containing the password for RabbitMQ authentication. Alternatively use -rabbitMqPass")
     private String rabbitMqPassFile;
 
-    @Parameter(names = "-rabbitMqTLS")
+    @Parameter(names = "-rabbitMqTLS", description = "Use TLS for the RabbitMQ connection")
     private boolean rabbitMqTLS;
 
     public String getRabbitMqHost() {

src/main/java/de/rub/nds/crawler/config/delegate/RabbitMqDelegate.java

@@ -8,6 +8,10 @@
  */
 package de.rub.nds.crawler.constant;
 
+/**
+ * Enumeration of different Crux list sizes available for scanning. Each enum constant represents a
+ * specific list of top websites, with the value indicating the number of entries in that list.
+ */
 public enum CruxListNumber {
     TOP_1k(1000),
     TOP_5K(5000),

src/main/java/de/rub/nds/crawler/constant/CruxListNumber.java

@@ -8,6 +8,10 @@
  */
 package de.rub.nds.crawler.constant;
 
+/**
+ * Enumeration of possible job status values. Indicates the current state or final result of a scan
+ * job.
+ */
 public enum JobStatus {
     /** Job is waiting to be executed. */
     TO_BE_EXECUTED(false),
@@ -42,6 +46,11 @@ public enum JobStatus {
         this.isError = isError;
     }
 
+    /**
+     * Get whether this status represents an error condition.
+     *
+     * @return True if this status is an error, false otherwise
+     */
     public boolean isError() {
         return isError;
     }

src/main/java/de/rub/nds/crawler/constant/JobStatus.java

@@ -12,20 +12,33 @@
 import de.rub.nds.crawler.data.ScanTarget;
 import de.rub.nds.crawler.util.CanceallableThreadPoolExecutor;
 import de.rub.nds.scanner.core.execution.NamedThreadFactory;
-import java.util.concurrent.*;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingDeque;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.bson.Document;
 
+/**
+ * A worker that scans all targets for a single scan ID. Instances are managed using the {@link
+ * BulkScanWorkerManager}.
+ *
+ * @param <T> The specific ScanConfig type used by this worker
+ */
 public abstract class BulkScanWorker<T extends ScanConfig> {
     private static final Logger LOGGER = LogManager.getLogger();
     private final AtomicInteger activeJobs = new AtomicInteger(0);
     private final AtomicBoolean initialized = new AtomicBoolean(false);
     private final AtomicBoolean shouldCleanupSelf = new AtomicBoolean(false);
     private final Object initializationLock = new Object();
+
+    /** The bulk scan ID for this worker. This is unique across all workers. */
     protected final String bulkScanId;
+
+    /** The scan configuration for this worker */
     protected final T scanConfig;
 
     /**
@@ -34,6 +47,15 @@ public abstract class BulkScanWorker<T extends ScanConfig> {
      */
     private final ThreadPoolExecutor timeoutExecutor;
 
+    /**
+     * Creates a new bulk scan worker. This should only be called by the {@link
+     * BulkScanWorkerManager}.
+     *
+     * @param bulkScanId The ID of the bulk scan this worker is associated with
+     * @param scanConfig The scan configuration for this worker
+     * @param parallelScanThreads The number of parallel scan threads to use, i.e., how many {@link
+     *     ScanTarget}s to handle in parallel.
+     */
     protected BulkScanWorker(String bulkScanId, T scanConfig, int parallelScanThreads) {
         this.bulkScanId = bulkScanId;
         this.scanConfig = scanConfig;
@@ -48,6 +70,13 @@ protected BulkScanWorker(String bulkScanId, T scanConfig, int parallelScanThread
                         new NamedThreadFactory("crawler-worker: scan executor"));
     }
 
+    /**
+     * Handles a scan target by submitting it to the executor. If init was not called, it will
+     * initialize itself. In this case it will also clean up itself if all jobs are done.
+     *
+     * @param scanTarget The target to scan.
+     * @return A future that resolves to the scan result once the scan is done.
+     */
     public Future<Document> handle(ScanTarget scanTarget) {
         // if we initialized ourself, we also clean up ourself
         shouldCleanupSelf.weakCompareAndSetAcquire(false, init());
@@ -62,8 +91,21 @@ public Future<Document> handle(ScanTarget scanTarget) {
                 });
     }
 
+    /**
+     * Scans a target and returns the result as a Document. This is the core scanning functionality
+     * that must be implemented by subclasses.
+     *
+     * @param scanTarget The target to scan
+     * @return The scan result as a Document
+     */
     public abstract Document scan(ScanTarget scanTarget);
 
+    /**
+     * Initializes this worker if it hasn't been initialized yet. This method is thread-safe and
+     * will only initialize once.
+     *
+     * @return True if this call performed the initialization, false if already initialized
+     */
     public final boolean init() {
         // synchronize such that no thread runs before being initialized
         // but only synchronize if not already initialized
@@ -78,6 +120,13 @@ public final boolean init() {
         return false;
     }
 
+    /**
+     * Cleans up this worker if it has been initialized and has no active jobs. This method is
+     * thread-safe and will only clean up once. If there are still active jobs, it will enqueue the
+     * cleanup for later.
+     *
+     * @return True if this call performed the cleanup, false otherwise
+     */
     public final boolean cleanup() {
         // synchronize such that init and cleanup do not run simultaneously
         // but only synchronize if already initialized
@@ -99,7 +148,17 @@ public final boolean cleanup() {
         return false;
     }
 
+    /**
+     * Performs the actual initialization of this worker. This method is called exactly once by
+     * {@link #init()} when initialization is needed. Subclasses must implement this method to
+     * initialize their specific resources.
+     */
     protected abstract void initInternal();
 
+    /**
+     * Performs the actual cleanup of this worker. This method is called exactly once by {@link
+     * #cleanup()} when cleanup is needed. Subclasses must implement this method to clean up their
+     * specific resources.
+     */
     protected abstract void cleanupInternal();
 }

src/main/java/de/rub/nds/crawler/core/BulkScanWorker.java

@@ -22,10 +22,29 @@
 import org.apache.logging.log4j.Logger;
 import org.bson.Document;
 
+/**
+ * Each ScanJob has its own BulkScanWorker. This class manages the BulkScanWorkers and ensures that
+ * each BulkScanWorker is only created once and is cleaned up after it is not used anymore.
+ *
+ * <p>More concretely: If a scan with ID 1 is started, a worker is created. This worker will exist
+ * as long as scan targets for scan ID 1 are processed. If the worker does not receive jobs for more
+ * than 30 Minutes, it is considered stale. Upon receiving a new job, stale workers are removed.
+ * I.e. if after 30 mins a new ID 1 job arrives, the worker persists. If a new ID 2 job arrives, a
+ * worker for that is created, and stale workers are removed.
+ *
+ * <p>This class manages the mechanism above and acts as a singleton factory and manager for
+ * BulkScanWorker instances.
+ */
 public class BulkScanWorkerManager {
     private static final Logger LOGGER = LogManager.getLogger();
     private static volatile BulkScanWorkerManager instance;
 
+    /**
+     * Gets the singleton instance of the BulkScanWorkerManager. Creates the instance if it doesn't
+     * exist yet.
+     *
+     * @return The singleton instance
+     */
     public static BulkScanWorkerManager getInstance() {
         if (instance == null) {
             synchronized (BulkScanWorkerManager.class) {
@@ -37,6 +56,17 @@ public static BulkScanWorkerManager getInstance() {
         return instance;
     }
 
+    /**
+     * Static convenience method to handle a scan job. See also {@link #handle(ScanJobDescription,
+     * int, int)}.
+     *
+     * @param scanJobDescription The scan job to handle
+     * @param parallelConnectionThreads The number of parallel connection threads to use (used to
+     *     create worker if it does not exist)
+     * @param parallelScanThreads The number of parallel scan threads to use (used to create worker
+     *     if it does not exist)
+     * @return A future that returns the scan result when the target is scanned is done
+     */
     public static Future<Document> handleStatic(
             ScanJobDescription scanJobDescription,
             int parallelConnectionThreads,
@@ -62,6 +92,19 @@ private BulkScanWorkerManager() {
                         .build();
     }
 
+    /**
+     * Gets or creates a bulk scan worker for the specified bulk scan. Workers are cached and reused
+     * to ensure thread limits.
+     *
+     * @param bulkScanId The ID of the bulk scan to get the worker for (used as key in the cache)
+     * @param scanConfig The scan configuration (used to create worker if it does not exist)
+     * @param parallelConnectionThreads The number of parallel connection threads to use (used to
+     *     create worker if it does not exist)
+     * @param parallelScanThreads The number of parallel scan threads to use (used to create worker
+     *     if it does not exist)
+     * @return A bulk scan worker for the specified bulk scan
+     * @throws UncheckedException If a worker cannot be created
+     */
     public BulkScanWorker<?> getBulkScanWorker(
             String bulkScanId,
             ScanConfig scanConfig,
@@ -83,6 +126,17 @@ public BulkScanWorker<?> getBulkScanWorker(
         }
     }
 
+    /**
+     * Handles a scan job by creating or retrieving the appropriate worker and submitting the scan
+     * target for processing.
+     *
+     * @param scanJobDescription The scan job to handle
+     * @param parallelConnectionThreads The number of parallel connection threads to use (used to
+     *     create worker if it does not exist)
+     * @param parallelScanThreads The number of parallel scan threads to use (used to create worker
+     *     if it does not exist)
+     * @return A future that returns the scan result when the target is scanned is done
+     */
     public Future<Document> handle(
             ScanJobDescription scanJobDescription,
             int parallelConnectionThreads,