TS/31571 Send log to Teamscale via a custom Appender

agent/src/dist/logging/logback.debug.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <configuration>
-	<define name="defaultLogDir" class="com.teamscale.jacoco.agent.util.LogDirectoryPropertyDefiner" />
+	<define name="defaultLogDir" class="com.teamscale.jacoco.agent.logging.LogDirectoryPropertyDefiner" />
 
 	<appender name="CONSOLE" class="shadow.ch.qos.logback.core.ConsoleAppender">
 		<encoder>

agent/src/dist/logging/logback.rolling-file.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <configuration>
-    <define name="defaultLogDir" class="com.teamscale.jacoco.agent.util.LogDirectoryPropertyDefiner"/>
+    <define name="defaultLogDir" class="com.teamscale.jacoco.agent.logging.LogDirectoryPropertyDefiner"/>
 
 	<appender name="RollingFile" class="shadow.ch.qos.logback.core.rolling.RollingFileAppender">
 		<!-- TODO: adjust the path where the log file should be written -->

agent/src/main/java/com/teamscale/jacoco/agent/Agent.java

@@ -34,7 +34,7 @@
 import java.util.stream.Stream;
 
 import static com.teamscale.jacoco.agent.upload.teamscale.TeamscaleUploader.RETRY_UPLOAD_FILE_SUFFIX;
-import static com.teamscale.jacoco.agent.util.LoggingUtils.wrap;
+import static com.teamscale.jacoco.agent.logging.LoggingUtils.wrap;
 
 /**
  * A wrapper around the JaCoCo Java agent that automatically triggers a dump and XML conversion based on a time

agent/src/main/java/com/teamscale/jacoco/agent/AgentBase.java

@@ -1,7 +1,7 @@
 package com.teamscale.jacoco.agent;
 
 import com.teamscale.jacoco.agent.options.AgentOptions;
-import com.teamscale.jacoco.agent.util.LoggingUtils;
+import com.teamscale.jacoco.agent.logging.LoggingUtils;
 import org.eclipse.jetty.server.Server;
 import org.eclipse.jetty.server.ServerConnector;
 import org.eclipse.jetty.servlet.ServletContextHandler;
@@ -114,10 +114,17 @@
 	 */
 	void registerShutdownHook() {
 		Runtime.getRuntime().addShutdownHook(new Thread(() -> {
-			stopServer();
-			prepareShutdown();
-			logger.info("CQSE JaCoCo agent successfully shut down.");
-			PreMain.closeLoggingResources();
+			try {
+				logger.info("CQSE JaCoCo agent is shutting down...");
+				stopServer();
+				prepareShutdown();
+				logger.info("CQSE JaCoCo agent successfully shut down.");
+			} catch (Exception e) {
+				logger.error("Exception during agent shutdown.", e);
+			} finally {
+				// Try to flush logging resources also in case of an exception during shutdown
+				PreMain.closeLoggingResources();
+			}
 		}));
 	}
 

agent/src/main/java/com/teamscale/jacoco/agent/Main.java

@@ -7,7 +7,7 @@
 import com.teamscale.jacoco.agent.commandline.Validator;
 import com.teamscale.jacoco.agent.convert.ConvertCommand;
 import com.teamscale.jacoco.agent.util.AgentUtils;
-import com.teamscale.jacoco.agent.util.LoggingUtils;
+import com.teamscale.jacoco.agent.logging.LoggingUtils;
 import org.conqat.lib.commons.string.StringUtils;
 import org.jacoco.core.JaCoCo;
 import org.slf4j.Logger;

agent/src/main/java/com/teamscale/jacoco/agent/PreMain.java

@@ -2,6 +2,7 @@
 
 import com.teamscale.client.HttpUtils;
 import com.teamscale.jacoco.agent.configuration.AgentOptionReceiveException;
+import com.teamscale.jacoco.agent.logging.LogToTeamscaleAppender;
 import com.teamscale.jacoco.agent.options.AgentOptionParseException;
 import com.teamscale.jacoco.agent.options.AgentOptions;
 import com.teamscale.jacoco.agent.options.AgentOptionsParser;
@@ -12,13 +13,14 @@
 import com.teamscale.jacoco.agent.testimpact.TestwiseCoverageAgent;
 import com.teamscale.jacoco.agent.upload.UploaderException;
 import com.teamscale.jacoco.agent.util.AgentUtils;
-import com.teamscale.jacoco.agent.util.DebugLogDirectoryPropertyDefiner;
-import com.teamscale.jacoco.agent.util.LogDirectoryPropertyDefiner;
-import com.teamscale.jacoco.agent.util.LoggingUtils;
+import com.teamscale.jacoco.agent.logging.DebugLogDirectoryPropertyDefiner;
+import com.teamscale.jacoco.agent.logging.LogDirectoryPropertyDefiner;
+import com.teamscale.jacoco.agent.logging.LoggingUtils;
+import com.teamscale.report.util.ILogger;
 import org.conqat.lib.commons.collections.CollectionUtils;
+import org.conqat.lib.commons.collections.Pair;
 import org.conqat.lib.commons.filesystem.FileSystemUtils;
 import org.conqat.lib.commons.string.StringUtils;
-import org.jetbrains.annotations.NotNull;
 import org.slf4j.Logger;
 
 import java.io.File;
@@ -31,6 +33,8 @@
 import java.util.List;
 import java.util.Optional;
 
+import static com.teamscale.jacoco.agent.logging.LoggingUtils.getLoggerContext;
+
 /** Container class for the premain entry point for the agent. */
 public class PreMain {
 
@@ -55,44 +59,63 @@
 	 * Entry point for the agent, called by the JVM.
 	 */
 	public static void premain(String options, Instrumentation instrumentation) throws Exception {
 		if (System.getProperty(LOCKING_SYSTEM_PROPERTY) != null) {
 			return;
 		}
 		System.setProperty(LOCKING_SYSTEM_PROPERTY, "true");
 
 		String environmentConfigId = System.getenv(CONFIG_ID_ENVIRONMENT_VARIABLE);
 		String environmentConfigFile = System.getenv(CONFIG_FILE_ENVIRONMENT_VARIABLE);
 		if (StringUtils.isEmpty(options) && environmentConfigId == null && environmentConfigFile == null) {
 			// profiler was registered globally and no config was set explicitly by the user, thus ignore this process
 			// and don't profile anything
 			return;
 		}
 
-		AgentOptions agentOptions;
+		AgentOptions agentOptions = null;
 		try {
-			agentOptions = getAndApplyAgentOptions(options, environmentConfigId, environmentConfigFile);
+			Pair<AgentOptions, List<Exception>> parseResult = getAndApplyAgentOptions(options, environmentConfigId,
+					environmentConfigFile);
+			agentOptions = parseResult.getFirst();
+
+			// After parsing everything and configuring logging, we now
+			// can throw the caught exceptions.
+			for (Exception exception : parseResult.getSecond()) {
+				throw exception;
+			}
+		} catch (AgentOptionParseException e) {
+			getLoggerContext().getLogger(PreMain.class).error(e.getMessage(), e);
+
+			// Flush logs to Teamscale, if configured.
+			closeLoggingResources();
+
+			// Unregister the profiler from Teamscale.
+			if (agentOptions != null && agentOptions.configurationViaTeamscale != null) {
+				agentOptions.configurationViaTeamscale.unregisterProfiler();
+			}
+
+			throw e;
 		} catch (AgentOptionReceiveException e) {
 			// When Teamscale is not available, we don't want to fail hard to still allow for testing even if no
 			// coverage is collected (see TS-33237)
 			return;
 		}
 
 		Logger logger = LoggingUtils.getLogger(Agent.class);
 
 		logger.info("Teamscale Java profiler version " + AgentUtils.VERSION);
 		logger.info("Starting JaCoCo's agent");
 		JacocoAgentOptionsBuilder agentBuilder = new JacocoAgentOptionsBuilder(agentOptions);
 		JaCoCoPreMain.premain(agentBuilder.createJacocoAgentOptions(), instrumentation, logger);
 
 		if (agentOptions.configurationViaTeamscale != null) {
 			agentOptions.configurationViaTeamscale.startHeartbeatThreadAndRegisterShutdownHook();
 		}
 		AgentBase agent = createAgent(agentOptions, instrumentation);
 		agent.registerShutdownHook();
 	}
 
-	@NotNull
-	private static AgentOptions getAndApplyAgentOptions(String options, String environmentConfigId,
+	private static Pair<AgentOptions, List<Exception>> getAndApplyAgentOptions(String options, String environmentConfigId,
 			String environmentConfigFile) throws AgentOptionParseException, IOException, AgentOptionReceiveException {
 
 		DelayedLogger delayedLogger = new DelayedLogger();
@@ -110,10 +133,14 @@
 		if (credentials == null) {
 			delayedLogger.warn("Did not find a teamscale.properties file!");
 		}
+
+		Pair<AgentOptions, List<Exception>> parseResult;
 		AgentOptions agentOptions;
 		try {
-			agentOptions = AgentOptionsParser.parse(options, environmentConfigId, environmentConfigFile, credentials,
+			parseResult = AgentOptionsParser.parse(
+					options, environmentConfigId, environmentConfigFile, credentials,
 					delayedLogger);
+			agentOptions = parseResult.getFirst();
 		} catch (AgentOptionParseException e) {
 			try (LoggingUtils.LoggingResources ignored = initializeFallbackLogging(options, delayedLogger)) {
 				delayedLogger.errorAndStdErr("Failed to parse agent options: " + e.getMessage(), e);
@@ -134,7 +161,8 @@
 		Logger logger = LoggingUtils.getLogger(Agent.class);
 		delayedLogger.logTo(logger);
 		HttpUtils.setShouldValidateSsl(agentOptions.shouldValidateSsl());
-		return agentOptions;
+
+		return parseResult;
 	}
 
 	private static void attemptLogAndThrow(DelayedLogger delayedLogger) {
@@ -153,6 +181,12 @@
 			loggingResources = LoggingUtils.initializeLogging(agentOptions.getLoggingConfig());
 			logger.info("Logging to " + new LogDirectoryPropertyDefiner().getPropertyValue());
 		}
+
+		if (agentOptions.getTeamscaleServerOptions().isConfiguredForServerConnection()) {
+			if (LogToTeamscaleAppender.addTeamscaleAppenderTo(getLoggerContext(), agentOptions)) {
+				logger.info("Logs are being forwarded to Teamscale at " + agentOptions.getTeamscaleServerOptions().url);
+			}
+		}
 	}
 
 	/** Closes the opened logging contexts. */
@@ -226,7 +260,7 @@
 
 	/** Creates a fallback logger using the given config file. */
 	private static LoggingUtils.LoggingResources createFallbackLoggerFromConfig(String configLocation,
-			DelayedLogger delayedLogger) {
+			ILogger delayedLogger) {
 		try {
 			return LoggingUtils.initializeLogging(
 					new FilePatternResolver(delayedLogger).parsePath(AgentOptionsParser.LOGGING_CONFIG_OPTION,

agent/src/main/java/com/teamscale/jacoco/agent/ResourceBase.java

@@ -3,7 +3,7 @@
 import com.teamscale.client.CommitDescriptor;
 import com.teamscale.client.TeamscaleServer;
 import com.teamscale.jacoco.agent.testimpact.TestwiseCoverageAgent;
-import com.teamscale.jacoco.agent.util.LoggingUtils;
+import com.teamscale.jacoco.agent.logging.LoggingUtils;
 import com.teamscale.report.testwise.model.RevisionInfo;
 import org.conqat.lib.commons.string.StringUtils;
 import org.slf4j.Logger;

agent/src/main/java/com/teamscale/jacoco/agent/commit_resolution/git_properties/GitMultiProjectPropertiesLocator.java

@@ -3,7 +3,7 @@
 import com.teamscale.jacoco.agent.options.ProjectAndCommit;
 import com.teamscale.jacoco.agent.upload.teamscale.DelayedTeamscaleMultiProjectUploader;
 import com.teamscale.jacoco.agent.util.DaemonThreadFactory;
-import com.teamscale.jacoco.agent.util.LoggingUtils;
+import com.teamscale.jacoco.agent.logging.LoggingUtils;
 import org.jetbrains.annotations.Nullable;
 import org.jetbrains.annotations.VisibleForTesting;
 import org.slf4j.Logger;

agent/src/main/java/com/teamscale/jacoco/agent/commit_resolution/git_properties/GitPropertiesLocatingTransformer.java

@@ -1,6 +1,6 @@
 package com.teamscale.jacoco.agent.commit_resolution.git_properties;
 
-import com.teamscale.jacoco.agent.util.LoggingUtils;
+import com.teamscale.jacoco.agent.logging.LoggingUtils;
 import com.teamscale.report.util.ClasspathWildcardIncludeFilter;
 import org.conqat.lib.commons.collections.Pair;
 import org.conqat.lib.commons.string.StringUtils;

agent/src/main/java/com/teamscale/jacoco/agent/commit_resolution/git_properties/GitSingleProjectPropertiesLocator.java

@@ -2,7 +2,7 @@
 
 import com.teamscale.jacoco.agent.upload.delay.DelayedUploader;
 import com.teamscale.jacoco.agent.util.DaemonThreadFactory;
-import com.teamscale.jacoco.agent.util.LoggingUtils;
+import com.teamscale.jacoco.agent.logging.LoggingUtils;
 import org.checkerframework.checker.nullness.qual.Nullable;
 import org.slf4j.Logger;
 

agent/src/main/java/com/teamscale/jacoco/agent/commit_resolution/sapnwdi/NwdiMarkerClassLocatingTransformer.java

@@ -3,7 +3,7 @@
 import com.teamscale.client.CommitDescriptor;
 import com.teamscale.jacoco.agent.options.sapnwdi.DelayedSapNwdiMultiUploader;
 import com.teamscale.jacoco.agent.options.sapnwdi.SapNwdiApplication;
-import com.teamscale.jacoco.agent.util.LoggingUtils;
+import com.teamscale.jacoco.agent.logging.LoggingUtils;
 import com.teamscale.report.util.ClasspathWildcardIncludeFilter;
 import org.conqat.lib.commons.string.StringUtils;
 import org.slf4j.Logger;

agent/src/main/java/com/teamscale/jacoco/agent/configuration/ConfigurationViaTeamscale.java

@@ -7,7 +7,7 @@
 import com.teamscale.client.ProfilerRegistration;
 import com.teamscale.client.TeamscaleServiceGenerator;
 import com.teamscale.jacoco.agent.options.AgentOptionParseException;
-import com.teamscale.jacoco.agent.util.LoggingUtils;
+import com.teamscale.jacoco.agent.logging.LoggingUtils;
 import com.teamscale.report.util.ILogger;
 import okhttp3.HttpUrl;
 import okhttp3.ResponseBody;
@@ -60,6 +60,10 @@ public static ConfigurationViaTeamscale retrieve(ILogger logger, String configur
 			ProcessInformation processInformation = new ProcessInformationRetriever(logger).getProcessInformation();
 			Response<ProfilerRegistration> response = teamscaleClient.registerProfiler(configurationId,
 					processInformation).execute();
+			if (response.code() == 405) {
+				response = teamscaleClient.registerProfilerLegacy(configurationId,
+						processInformation).execute();
+			}
 			if (!response.isSuccessful()) {
 				if (response.code() >= 400 && response.code() < 500) {
 					throw new AgentOptionParseException(
@@ -109,6 +113,9 @@ public void startHeartbeatThreadAndRegisterShutdownHook() {
 	private void sendHeartbeat() {
 		try {
 			Response<ResponseBody> response = teamscaleClient.sendHeartbeat(profilerId, profilerInfo).execute();
+			if (response.code() == 405) {
+				response = teamscaleClient.sendHeartbeatLegacy(profilerId, profilerInfo).execute();
+			}
 			if (!response.isSuccessful()) {
 				LoggingUtils.getLogger(this)
 						.error("Failed to send heartbeat. Teamscale responded with: " + response.errorBody().string());
@@ -118,9 +125,13 @@ private void sendHeartbeat() {
 		}
 	}
 
-	private void unregisterProfiler() {
+	/** Unregisters the profiler in Teamscale (marks it as shut down). */
+	public void unregisterProfiler() {
 		try {
 			Response<ResponseBody> response = teamscaleClient.unregisterProfiler(profilerId).execute();
+			if (response.code() == 405) {
+				response = teamscaleClient.unregisterProfilerLegacy(profilerId).execute();
+			}
 			if (!response.isSuccessful()) {
 				LoggingUtils.getLogger(this)
 						.error("Failed to unregister profiler. Teamscale responded with: " + response.errorBody()
@@ -131,5 +142,7 @@ private void unregisterProfiler() {
 		}
 	}
 
-
+	public String getProfilerId() {
+		return profilerId;
+	}
 }

agent/src/main/java/com/teamscale/jacoco/agent/configuration/ProcessInformationRetriever.java

@@ -41,7 +41,6 @@ private String getHostName() {
 		}
 	}
 
-
 	/**
 	 * Returns a string that <i>probably</i> contains the PID.
 	 * <p>

agent/src/main/java/com/teamscale/jacoco/agent/convert/Converter.java

@@ -3,7 +3,7 @@
 import com.teamscale.client.TestDetails;
 import com.teamscale.jacoco.agent.options.AgentOptionParseException;
 import com.teamscale.jacoco.agent.util.Benchmark;
-import com.teamscale.jacoco.agent.util.LoggingUtils;
+import com.teamscale.jacoco.agent.logging.LoggingUtils;
 import com.teamscale.report.ReportUtils;
 import com.teamscale.report.jacoco.EmptyReportException;
 import com.teamscale.report.jacoco.JaCoCoXmlReportGenerator;
@@ -26,7 +26,7 @@
 import java.nio.file.Paths;
 import java.util.List;
 
-import static com.teamscale.jacoco.agent.util.LoggingUtils.wrap;
+import static com.teamscale.jacoco.agent.logging.LoggingUtils.wrap;
 
 /** Converts one .exec binary coverage file to XML. */
 public class Converter {

agent/src/main/java/com/teamscale/jacoco/agent/util/DebugLogDirectoryPropertyDefiner.java → agent/src/main/java/com/teamscale/jacoco/agent/logging/DebugLogDirectoryPropertyDefiner.java

@@ -1,4 +1,4 @@
-package com.teamscale.jacoco.agent.util;
+package com.teamscale.jacoco.agent.logging;
 
 import java.nio.file.Path;
 

agent/src/main/java/com/teamscale/jacoco/agent/util/LogDirectoryPropertyDefiner.java → agent/src/main/java/com/teamscale/jacoco/agent/logging/LogDirectoryPropertyDefiner.java

@@ -1,6 +1,7 @@
-package com.teamscale.jacoco.agent.util;
+package com.teamscale.jacoco.agent.logging;
 
 import ch.qos.logback.core.PropertyDefinerBase;
+import com.teamscale.jacoco.agent.util.AgentUtils;
 
 import java.nio.file.Path;