chore: Complex start up check

api-catalog-services/src/main/java/org/zowe/apiml/apicatalog/ApiCatalogApplication.java

@@ -28,6 +28,7 @@
 @ComponentScan(value = {
     "org.zowe.apiml.apicatalog",
     "org.zowe.apiml.product.compatibility",
+    "org.zowe.apiml.product.eureka.web",
     "org.zowe.apiml.product.security",
     "org.zowe.apiml.product.web",
     "org.zowe.apiml.product.gateway",

apiml-common/src/main/java/org/zowe/apiml/product/config/ServerAddressPropertiesUpdater.java

@@ -147,7 +147,7 @@ public void postProcessEnvironment(ConfigurableEnvironment environment, SpringAp
      * @return processor to create new customizer beans or an empty processor if there are no additional address.
      */
     @Bean
-    public BeanDefinitionRegistryPostProcessor registerAdditionalTomcatConnectors() {
+    public static BeanDefinitionRegistryPostProcessor registerAdditionalTomcatConnectors() {
         if (ADDITIONAL_NETWORKS.isEmpty()) {
             return registry -> {};
         }

apiml-common/src/main/java/org/zowe/apiml/product/eureka/client/ApimlPeerEurekaNode.java

@@ -462,8 +462,9 @@ public ProcessingResult process(List<ReplicationTask> tasks) {
                     logNetworkErrorSample(null, "; retrying after delay.", e);
                     return ProcessingResult.TransientError;
                 } else {
+                    log.warn("Cannot replicate to another DS instance: {}", e.getMessage());
                     logNetworkErrorSample(null, "; not re-trying this exception because it does not seem to be a network exception.", e);
-                    return ProcessingResult.PermanentError;
+                    return ProcessingResult.TransientError;
                 }
             }
             return ProcessingResult.Success;

apiml-common/src/main/java/org/zowe/apiml/product/eureka/web/EurekaRegistryVersionEndpoint.java

@@ -0,0 +1,75 @@
+/*
+ * This program and the accompanying materials are made available under the terms of the
+ * Eclipse Public License v2.0 which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-v20.html
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Copyright Contributors to the Zowe Project.
+ */
+
+package org.zowe.apiml.product.eureka.web;
+
+import com.netflix.discovery.EurekaClient;
+import com.netflix.discovery.EurekaEvent;
+import jakarta.annotation.PostConstruct;
+import lombok.Builder;
+import lombok.RequiredArgsConstructor;
+import lombok.Value;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.boot.actuate.endpoint.annotation.Endpoint;
+import org.springframework.boot.actuate.endpoint.annotation.ReadOperation;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
+import org.springframework.context.event.EventListener;
+import org.springframework.stereotype.Component;
+
+import java.util.regex.Pattern;
+
+import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON;
+
+@Component
+@RequiredArgsConstructor
+@Endpoint(id = "eurekaversion")
+@ConditionalOnMissingBean(name = "modulithConfig")
+@Slf4j
+public class EurekaRegistryVersionEndpoint {
+
+    private static final Pattern VERSION_PATTERN = Pattern.compile("^.*_([0-9]+)_.*$");
+
+    private Long version = -1L;
+
+    private final EurekaClient eurekaClient;
+
+    @PostConstruct
+    void registerListener() {
+        eurekaClient.registerEventListener(event -> {
+            onRegistryUpdate(event);
+        });
+    }
+
+    @EventListener
+    void onRegistryUpdate(EurekaEvent event) {
+        var hashCode = eurekaClient.getApplications().getAppsHashCode();
+        var matcher = VERSION_PATTERN.matcher(hashCode);
+        if (matcher.matches()) {
+            version = Long.parseLong(matcher.group(1));
+            log.debug("New Eureka registry version: {}", this.version);
+        } else {
+            log.debug("Unexpected Eureka registry hashCode: {}", hashCode);
+        }
+    }
+
+    @ReadOperation(produces = APPLICATION_JSON)
+    public VersionDto status() {
+        return VersionDto.builder().version(this.version).build();
+    }
+
+    @Builder
+    @Value
+    static class VersionDto {
+
+        private Long version;
+
+    }
+
+}

apiml-security-common/src/main/java/org/zowe/apiml/security/common/error/AuthExceptionHandler.java

@@ -109,7 +109,7 @@ private <E extends Exception> Map.Entry<Class<E>, ExceptionHandler<E>> entry(Cla
         entry(WebClientResponseException.BadRequest.class,
             (ex, ctx) -> handleBadRequest(ctx.requestUri, ctx.function, ex, "org.zowe.apiml.security.login.invalidInput")),
         entry(AccessDeniedException.class,
-            (ex, ctx) ->  handleForbidden(ctx.function, ex)
+            (ex, ctx) ->  handleForbidden(ctx.requestUri, ctx.function, ex)
         ),
         entry(GatewayNotAvailableException.class,
             (ex, ctx) -> handleGatewayNotAvailable(ctx.function, ex, ctx.requestUri)
@@ -270,9 +270,9 @@ private void handleRuntimeException(String uri, BiConsumer<ApiMessageView, HttpS
         writeErrorResponse("org.zowe.apiml.common.internalRequestError", HttpStatus.INTERNAL_SERVER_ERROR, function, uri, ExceptionUtils.getMessage(ex), ExceptionUtils.getRootCauseMessage(ex));
     }
 
-    private void handleForbidden(BiConsumer<ApiMessageView, HttpStatus> function, AccessDeniedException ex) {
+    private void handleForbidden(String uri, BiConsumer<ApiMessageView, HttpStatus> function, AccessDeniedException ex) {
         log.debug(MESSAGE_FORMAT, HttpStatus.FORBIDDEN.value(), ex.getMessage());
-        writeErrorResponse("org.zowe.apiml.security.forbidden", HttpStatus.FORBIDDEN, function);
+        writeErrorResponse("org.zowe.apiml.security.forbidden", HttpStatus.FORBIDDEN, function, uri);
     }
 
     private void handleGatewayNotAvailable(BiConsumer<ApiMessageView, HttpStatus> function, GatewayNotAvailableException ex, String uri) {

apiml/src/main/java/org/zowe/apiml/ApimlApplication.java

@@ -25,6 +25,7 @@
     scanBasePackages = {
         "org.zowe.apiml.filter",
         "org.zowe.apiml.gateway",
+        "org.zowe.apiml.product.eureka.web",
         "org.zowe.apiml.product.web",
         "org.zowe.apiml.product.gateway",
         "org.zowe.apiml.product.version",

apiml/src/main/java/org/zowe/apiml/ApimlEurekaRegistryVersionEndpoint.java

@@ -0,0 +1,59 @@
+/*
+ * This program and the accompanying materials are made available under the terms of the
+ * Eclipse Public License v2.0 which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-v20.html
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Copyright Contributors to the Zowe Project.
+ */
+
+package org.zowe.apiml;
+
+import com.netflix.eureka.registry.PeerAwareInstanceRegistry;
+import lombok.Builder;
+import lombok.RequiredArgsConstructor;
+import lombok.Value;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.boot.actuate.endpoint.annotation.Endpoint;
+import org.springframework.boot.actuate.endpoint.annotation.ReadOperation;
+import org.springframework.stereotype.Component;
+
+import java.util.regex.Pattern;
+
+import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON;
+
+@Component
+@RequiredArgsConstructor
+@Endpoint(id = "eurekaversion")
+@Slf4j
+public class ApimlEurekaRegistryVersionEndpoint {
+
+    private static final Pattern VERSION_PATTERN = Pattern.compile("^.*_([0-9]+)_.*$");
+
+    private final PeerAwareInstanceRegistry peerAwareInstanceRegistry;
+
+    @ReadOperation(produces = APPLICATION_JSON)
+    public VersionDto status() {
+        long version = -1;
+        var hashCode = peerAwareInstanceRegistry.getApplications().getAppsHashCode();
+        var matcher = VERSION_PATTERN.matcher(hashCode);
+        if (matcher.matches()) {
+            version = Long.parseLong(matcher.group(1));
+            log.debug("New Eureka registry version: {}", version);
+        } else {
+            log.debug("Unexpected Eureka registry hashCode: {}", hashCode);
+        }
+        return VersionDto.builder().version(version).build();
+    }
+
+    @Builder
+    @Value
+    static class VersionDto {
+
+        private Long version;
+
+    }
+
+
+}

apiml/src/main/java/org/zowe/apiml/EurekaConfiguration.java

@@ -72,7 +72,6 @@
 import org.springframework.boot.web.servlet.FilterRegistrationBean;
 import org.springframework.cloud.client.actuator.HasFeatures;
 import org.springframework.cloud.netflix.eureka.EurekaConstants;
-import org.springframework.cloud.netflix.eureka.EurekaInstanceConfigBean;
 import org.springframework.cloud.netflix.eureka.server.*;
 import org.springframework.context.annotation.*;
 import org.springframework.core.Ordered;
@@ -88,8 +87,6 @@
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Set;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 import java.util.regex.Pattern;
 
 /**
@@ -208,32 +205,6 @@ PeerReplicationResource peerReplicationResource() {
         return new PeerReplicationResource();
     }
 
-    @Bean
-    PeerAwareInstanceRegistry peerAwareInstanceRegistry(ServerCodecs serverCodecs,
-                                                        EurekaServerHttpClientFactory eurekaServerHttpClientFactory,
-                                                        EurekaInstanceConfigBean eurekaInstanceConfigBean) {
-        if (eurekaInstanceConfigBean.isAsyncClientInitialization()) {
-            if (log.isDebugEnabled()) {
-                log.debug("Initializing client asynchronously...");
-            }
-
-            ExecutorService executorService = Executors.newSingleThreadExecutor();
-            executorService.submit(() -> {
-                this.eurekaClient.getApplications();
-                if (log.isDebugEnabled()) {
-                    log.debug("Asynchronous client initialization done.");
-                }
-            });
-        } else {
-            this.eurekaClient.getApplications(); // force initialization
-        }
-
-        return new InstanceRegistry(this.eurekaServerConfig, this.eurekaClientConfig, serverCodecs, this.eurekaClient,
-            eurekaServerHttpClientFactory,
-            this.instanceRegistryProperties.getExpectedNumberOfClientsSendingRenews(),
-            this.instanceRegistryProperties.getDefaultOpenForTrafficCount());
-    }
-
     @Bean
     @ConditionalOnMissingBean
     EurekaServerContext eurekaServerContext(ServerCodecs serverCodecs, PeerAwareInstanceRegistry registry,

apiml/src/main/java/org/zowe/apiml/EurekaHealthIndicatorApiml.java

@@ -0,0 +1,65 @@
+/*
+ * This program and the accompanying materials are made available under the terms of the
+ * Eclipse Public License v2.0 which accompanies this distribution, and is available at
+ * https://www.eclipse.org/legal/epl-v20.html
+ *
+ * SPDX-License-Identifier: EPL-2.0
+ *
+ * Copyright Contributors to the Zowe Project.
+ */
+
+package org.zowe.apiml;
+
+import org.springframework.boot.actuate.health.Health;
+import org.springframework.boot.actuate.health.Status;
+import org.springframework.cloud.client.discovery.DiscoveryClient;
+import org.springframework.cloud.netflix.eureka.EurekaHealthIndicator;
+import org.springframework.context.annotation.Primary;
+import org.springframework.stereotype.Component;
+
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * This class is replacement of org.springframework.cloud.netflix.eureka.EurekaHealthIndicator, because it is using
+ * a different Eureka client
+ */
+@Primary
+@Component("eurekaHealthIndicator")
+public class EurekaHealthIndicatorApiml extends EurekaHealthIndicator {
+
+    private final DiscoveryClient discoveryClient;
+
+    public EurekaHealthIndicatorApiml(DiscoveryClient discoveryClient) {
+        super(null, null, null);
+        this.discoveryClient = discoveryClient;
+    }
+
+    @Override
+    public String getName() {
+        return "eureka";
+    }
+
+    @Override
+    public Health health() {
+        Health.Builder builder = Health.unknown();
+        Status status = getStatus(builder);
+        return builder.status(status).withDetail("applications", getApplications()).build();
+    }
+
+    private Status getStatus(Health.Builder builder) {
+        if (discoveryClient.getServices().isEmpty()) {
+            return new Status("UP", "Eureka registry is not available at the moment");
+        }
+        return new Status("UP", "Eureka is ready to use");
+    }
+
+    private Map<String, Object> getApplications() {
+        return discoveryClient.getServices().stream()
+            .collect(Collectors.toMap(
+                String::toLowerCase,
+                serviceId -> discoveryClient.getInstances(serviceId).size())
+            );
+    }
+
+}