diff --git a/.github/actions/validate-apiml-healthy/action.yml b/.github/actions/validate-apiml-healthy/action.yml deleted file mode 100644 index c546b5ab74..0000000000 --- a/.github/actions/validate-apiml-healthy/action.yml +++ /dev/null @@ -1,83 +0,0 @@ -name: "Validate APIML Healthy" -description: "Validates APIML health using Gateway, Caching Service and Authentication Service health status" - -inputs: - gateway-host: - description: "Gateway hostname" - required: false - default: "localhost" - zaas-host: - description: "ZAAS hostname" - required: false - default: "localhost" - caching-service: - description: "Check Caching Service health" - required: false - default: "false" - api-catalog: - description: "Check API Catalog health" - required: false - default: "false" - discoverable-client: - description: "Check Discoverable Client health" - required: false - default: "false" - -runs: - using: "composite" - - # if ZAAS Service and input services are not healthy after 3 minutes then exit - steps: - - name: Validate APIML setup - shell: bash - run: | - set +e # curl -s doesn't swallow the error alone - attemptCounter=1 - maxAttempts=18 - valid="false" - until [ $attemptCounter -gt $maxAttempts ]; do - sleep 10 - gwHealth=$(curl -k -s https://${{ inputs.gateway-host }}:10010/application/health) - zaasHealth=$(curl -k -s https://${{ inputs.zaas-host }}:10023/application/health) - echo "Polling for GW health: $attemptCounter" - echo $gwHealth - - gatewayUp=$(echo $gwHealth | jq -r '.status') - authUp=$(echo $gwHealth | jq -r '.components.gateway.details.zaas') - acUp="$(echo $gwHealth | jq -r '.components.gateway.details.apicatalog')" - - csUp="DOWN" - csHealth="$(echo $zaasHealth | jq -r '[.components.discoveryComposite.components.discoveryClient.details.services[]] | index("cachingservice")')" - if [ "$csHealth" != "null" ]; then - csUp="UP" - fi - - dcUp="DOWN" - dcHealth="$(echo $zaasHealth | jq -r '[.components.discoveryComposite.components.discoveryClient.details.services[]] | index("discoverableclient")')" - if [ "$dcHealth" != "null" ]; then - dcUp="UP" - fi - - if [ "$gatewayUp" = "UP" ] && [ "$authUp" = "UP" ] && [ "$csUp" == "UP" ]; then - echo ">>>>>APIML is ready" - valid="true" - if [ ${{ inputs.caching-service }} != "false" ] && [ "$csUp" != "UP" ]; then - valid="false" - fi - if [ ${{ inputs.api-catalog }} != "false" ] && [ "$acUp" != "UP" ]; then - valid="false" - fi - if [ ${{ inputs.discoverable-client }} != "false" ] && [ "$dcUp" != "UP" ]; then - valid="false" - fi - fi - - if [ "$valid" == "true" ]; then - break - fi - attemptCounter=$((attemptCounter+1)) - done - if [ "$valid" != "true" ]; then - echo ">>>>>APIML is not ready" - exit 1 - fi diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index c4fde90f19..4aa4d360f8 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -98,10 +98,19 @@ jobs: - uses: ./.github/actions/setup + - name: Run Startup Check + if: always() + run: > + ./gradlew runStartUpCheck --info -Denvironment.config=-docker-modulith -Denvironment.modulith=true + -Ddiscoverableclient.instances=1 -DcentralGateway.instances=0 + -Partifactory_user=$ARTIFACTORY_USERNAME -Partifactory_password=$ARTIFACTORY_PASSWORD + env: + ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }} + ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }} + - name: Run Modulith CI Tests - timeout-minutes: 4 run: > - ENV_CONFIG=docker-modulith ./gradlew runStartUpCheck :integration-tests:runContainerModulithTests --info + ENV_CONFIG=docker-modulith ./gradlew :integration-tests:runContainerModulithTests --info -Ddiscoverableclient.instances=1 -Denvironment.config=-docker-modulith -Denvironment.modulith=true -Partifactory_user=${{ secrets.ARTIFACTORY_USERNAME }} -Partifactory_password=${{ secrets.ARTIFACTORY_PASSWORD }} - uses: ./.github/actions/dump-jacoco @@ -138,7 +147,7 @@ jobs: APIML_GATEWAY_SERVICESTOLIMITREQUESTRATE: discoverableclient APIML_GATEWAY_COOKIENAMEFORRATELIMIT: apimlAuthenticationToken APIML_SECURITY_AUTH_PROVIDER: saf - APIML_DISCOVERY_ALLPEERSURLS: https://apiml:10011/eureka,https://apiml-2:10011/eureka + APIML_DISCOVERY_ALLPEERSURLS: https://apiml-2:10011/eureka,https://apiml:10011/eureka APIML_SERVICE_HOSTNAME: apiml logbackService: ZWEAGW1 apiml-2: @@ -156,26 +165,10 @@ jobs: APIML_DISCOVERY_ALLPEERSURLS: https://apiml:10011/eureka,https://apiml-2:10011/eureka APIML_SERVICE_HOSTNAME: apiml-2 logbackService: ZWEAGW2 - api-catalog-services: - image: ghcr.io/balhar-jakub/api-catalog-services:${{ github.run_id }}-${{ github.run_number }} - volumes: - - /api-defs:/api-defs - env: - APIML_SERVICE_HOSTNAME: api-catalog-services - APIML_SERVICE_DISCOVERYSERVICEURLS: https://apiml:10011/eureka,https://apiml-2:10011/eureka - APIML_SERVICE_GATEWAYHOSTNAME: https://apiml:10010 - api-catalog-services-2: - image: ghcr.io/balhar-jakub/api-catalog-services:${{ github.run_id }}-${{ github.run_number }} - volumes: - - /api-defs:/api-defs - env: - APIML_SERVICE_HOSTNAME: api-catalog-services-2 - APIML_SERVICE_DISCOVERYSERVICEURLS: https://apiml-2:10011/eureka,https://apiml:10011/eureka - APIML_SERVICE_GATEWAYHOSTNAME: https://apiml-2:10010 discoverable-client: image: ghcr.io/balhar-jakub/discoverable-client:${{ github.run_id }}-${{ github.run_number }} env: - APIML_SERVICE_DISCOVERYSERVICEURLS: https://apiml:10011/eureka + APIML_SERVICE_DISCOVERYSERVICEURLS: https://apiml:10011/eureka,https://apiml-2:10011/eureka steps: - uses: actions/checkout@v4 with: @@ -185,10 +178,11 @@ jobs: - name: Run Startup Check if: always() - timeout-minutes: 4 run: > - ./gradlew runStartUpCheck --info -Denvironment.config=-docker-modulith-ha -Ddiscoverableclient.instances=1 -Denvironment.gwCount=2 + ./gradlew runStartUpCheck --info -Denvironment.config=-docker-modulith-ha -Ddiscoverableclient.instances=1 + -Denvironment.gwCount=2 -Dgateway.instances=2 -Ddiscovery.instances=2 -Dcaching.instances=2 -Dapicatalog.instances=2 -Partifactory_user=$ARTIFACTORY_USERNAME -Partifactory_password=$ARTIFACTORY_PASSWORD -Denvironment.modulith=true + -Dzosmf.instances=0 env: ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }} ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }} @@ -234,13 +228,13 @@ jobs: SPRING_CLOUD_GATEWAY_SERVER_WEBFLUX_HTTPCLIENT_WEBSOCKET_MAXFRAMEPAYLOADLENGTH: 16348 APIML_GATEWAY_SERVICESTOLIMITREQUESTRATE: discoverableclient APIML_GATEWAY_COOKIENAMEFORRATELIMIT: apimlAuthenticationToken - APIML_SERVICE_DISCOVERYSERVICEURLS: https://apiml:10011/eureka,https://apiml-2:10011/eureka - APIML_DISCOVERY_ALLPEERSURLS: https://apiml:10011/eureka,https://apiml-2:10011/eureka + APIML_SERVICE_DISCOVERYSERVICEURLS: https://apiml-2:10011/eureka,https://apiml-3:10011/eureka,https://apiml:10011/eureka + APIML_DISCOVERY_ALLPEERSURLS: https://apiml-2:10011/eureka,https://apiml-3:10011/eureka,https://apiml:10011/eureka APIML_SERVICE_HOSTNAME: apiml logbackService: ZWEAGW1 APIML_HEALTH_PROTECTED: false CACHING_STORAGE_MODE: infinispan - CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "apiml[7600],apiml-2[7600]" + CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "apiml[7600],apiml-2[7600],apiml-3[7600]" JGROUPS_BIND_PORT: 7600 JGROUPS_BIND_ADDRESS: apiml JGROUPS_KEYEXCHANGE_PORT: 7601 @@ -255,13 +249,13 @@ jobs: SPRING_CLOUD_GATEWAY_SERVER_WEBFLUX_HTTPCLIENT_WEBSOCKET_MAXFRAMEPAYLOADLENGTH: 16348 APIML_GATEWAY_SERVICESTOLIMITREQUESTRATE: discoverableclient APIML_GATEWAY_COOKIENAMEFORRATELIMIT: apimlAuthenticationToken - APIML_SERVICE_DISCOVERYSERVICEURLS: https://apiml:10011/eureka,https://apiml-2:10011/eureka,https://apiml-3:10011/eureka - APIML_DISCOVERY_ALLPEERSURLS: https://apiml:10011/eureka,https://apiml-2:10011/eureka,https://apiml-3:10011/eureka + APIML_SERVICE_DISCOVERYSERVICEURLS: https://apiml:10011/eureka,https://apiml-3:10011/eureka,https://apiml-2:10011/eureka + APIML_DISCOVERY_ALLPEERSURLS: https://apiml:10011/eureka,https://apiml-3:10011/eureka,https://apiml-2:10011/eureka APIML_SERVICE_HOSTNAME: apiml-2 logbackService: ZWEAGW2 APIML_HEALTH_PROTECTED: false CACHING_STORAGE_MODE: infinispan - CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "apiml[7600],apiml-2[7600]" + CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "apiml[7600],apiml-2[7600],apiml-3[7600]" JGROUPS_BIND_PORT: 7600 JGROUPS_BIND_ADDRESS: apiml-2 JGROUPS_KEYEXCHANGE_PORT: 7601 @@ -282,7 +276,7 @@ jobs: logbackService: ZWEAGW3 APIML_HEALTH_PROTECTED: false CACHING_STORAGE_MODE: infinispan - CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "apiml[7600],apiml-2[7600]" + CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "apiml[7600],apiml-2[7600],apiml-3[7600]" JGROUPS_BIND_PORT: 7600 JGROUPS_BIND_ADDRESS: apiml-3 JGROUPS_KEYEXCHANGE_PORT: 7601 @@ -295,6 +289,7 @@ jobs: env: APIML_SERVICE_HOSTNAME: discoverable-client-2 APIML_SERVICE_DISCOVERYSERVICEURLS: https://apiml-2:10011/eureka,https://apiml:10011/eureka + logbackService: ZWEADC2 mock-services: image: ghcr.io/balhar-jakub/mock-services:${{ github.run_id }}-${{ github.run_number }} steps: @@ -306,7 +301,6 @@ jobs: - name: Run Startup Check if: always() - timeout-minutes: 4 run: > ./gradlew runStartUpCheck --info -Denvironment.config=-docker-modulith-ha -Ddiscoverableclient.instances=2 -Partifactory_user=$ARTIFACTORY_USERNAME -Partifactory_password=$ARTIFACTORY_PASSWORD -Denvironment.modulith=true @@ -386,13 +380,6 @@ jobs: timeout-minutes: 15 services: - api-catalog-services-2: - image: ghcr.io/balhar-jakub/api-catalog-services-standalone:${{ github.run_id }}-${{ github.run_number }} - volumes: - - /api-defs:/api-defs - env: - APIML_SERVICE_HOSTNAME: api-catalog-services-2 - APIML_HEALTH_PROTECTED: false api-catalog-services: image: ghcr.io/balhar-jakub/api-catalog-services:${{ github.run_id }}-${{ github.run_number }} volumes: @@ -406,12 +393,6 @@ jobs: SPRING_SERVLET_MULTIPART_MAXREQUESTSIZE: 1024MB discovery-service: image: ghcr.io/balhar-jakub/discovery-service:${{ github.run_id }}-${{ github.run_number }} -# needs to run in isolation from another DS for multi-tenancy setup - discovery-service-2: - image: ghcr.io/balhar-jakub/discovery-service:${{ github.run_id }}-${{ github.run_number }} - env: - APIML_SERVICE_HOSTNAME: discovery-service-2 - APIML_DISCOVERY_ALLPEERSURLS: https://discovery-service-2:10011/eureka gateway-service: image: ghcr.io/balhar-jakub/gateway-service:${{ github.run_id }}-${{ github.run_number }} env: @@ -427,6 +408,7 @@ jobs: APIML_GATEWAY_COOKIENAMEFORRATELIMIT: apimlAuthenticationToken EUREKA_CLIENT_INSTANCEINFOREPLICATIONINTERVALSECONDS: 1 EUREKA_CLIENT_REGISTRYFETCHINTERVALSECONDS: 1 + logbackService: ZWEAGW1 options: --memory 640m --memory-swap 640m zaas-service: image: ghcr.io/balhar-jakub/zaas-service:${{ github.run_id }}-${{ github.run_number }} @@ -437,6 +419,29 @@ jobs: mock-services: image: ghcr.io/balhar-jakub/mock-services:${{ github.run_id }}-${{ github.run_number }} + # needs to run in isolation from another DS for multi-tenancy setup + discovery-service-2: + image: ghcr.io/balhar-jakub/discovery-service:${{ github.run_id }}-${{ github.run_number }} + env: + APIML_SERVICE_HOSTNAME: discovery-service-2 + APIML_DISCOVERY_ALLPEERSURLS: https://discovery-service-2:10011/eureka + logbackService: ZWEADS2 + gateway-service-2: + image: ghcr.io/balhar-jakub/gateway-service:${{ github.run_id }}-${{ github.run_number }} + env: + APIML_SERVICE_HOSTNAME: gateway-service-2 + APIML_SERVICE_DISCOVERYSERVICEURLS: https://discovery-service-2:10011/eureka/ + logbackService: ZWEAGW2 + options: --memory 640m --memory-swap 640m + zaas-service-2: + image: ghcr.io/balhar-jakub/zaas-service:${{ github.run_id }}-${{ github.run_number }} + env: + APIML_SERVICE_HOSTNAME: zaas-service-2 + APIML_SERVICE_DISCOVERYSERVICEURLS: https://discovery-service-2:10011/eureka/ + APIML_SECURITY_X509_ENABLED: true + APIML_SECURITY_X509_ACCEPTFORWARDEDCERT: true + APIML_SECURITY_X509_CERTIFICATESURL: https://gateway-service-2:10010/gateway/certificates + logbackService: ZWEAZS2 steps: - uses: actions/checkout@v4 with: @@ -446,9 +451,9 @@ jobs: - name: Run Startup Check if: always() - timeout-minutes: 4 run: > - ./gradlew runStartUpCheck --info -Denvironment.config=-docker + ./gradlew runStartUpCheck --info -Denvironment.config=-docker -DcentralGateway.instances=0 + -DcentralHosts=discovery-service-2,gateway-service-2,zaas-service-2 env: ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }} ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }} @@ -466,6 +471,7 @@ jobs: with: name: ContainerCITests-${{ env.JOB_ID }} path: | + integration-tests/build/test-results/runStartUpCheck/binary/** integration-tests/build/reports/** results/** @@ -479,13 +485,14 @@ jobs: services: api-catalog-services-2: - image: ghcr.io/balhar-jakub/api-catalog-services-standalone:${{ github.run_id }}-${{ github.run_number }} + image: ghcr.io/balhar-jakub/api-catalog-services:${{ github.run_id }}-${{ github.run_number }} volumes: - /api-defs:/api-defs env: APIML_SERVICE_HOSTNAME: api-catalog-services-2 APIML_HEALTH_PROTECTED: false APIML_SERVICE_DISCOVERYSERVICEURLS: https://discovery-service-2:10011/eureka/,https://discovery-service:10011/eureka/ + logbackService: ZWEAAC2 api-catalog-services: image: ghcr.io/balhar-jakub/api-catalog-services:${{ github.run_id }}-${{ github.run_number }} volumes: @@ -504,12 +511,13 @@ jobs: image: ghcr.io/balhar-jakub/discovery-service:${{ github.run_id }}-${{ github.run_number }} env: APIML_SERVICE_HOSTNAME: discovery-service - APIML_DISCOVERY_ALLPEERSURLS: https://discovery-service-2:10011/eureka + APIML_DISCOVERY_ALLPEERSURLS: https://discovery-service-2:10011/eureka,https://discovery-service:10011/eureka discovery-service-2: image: ghcr.io/balhar-jakub/discovery-service:${{ github.run_id }}-${{ github.run_number }} env: APIML_SERVICE_HOSTNAME: discovery-service-2 - APIML_DISCOVERY_ALLPEERSURLS: https://discovery-service:10011/eureka + APIML_DISCOVERY_ALLPEERSURLS: https://discovery-service:10011/eureka,https://discovery-service-2:10011/eureka + logbackService: ZWEADS2 gateway-service: image: ghcr.io/balhar-jakub/gateway-service:${{ github.run_id }}-${{ github.run_number }} env: @@ -539,6 +547,7 @@ jobs: APIML_SECURITY_AUTH_PROVIDER: saf APIML_SERVICE_HOSTNAME: zaas-service-2 APIML_SERVICE_DISCOVERYSERVICEURLS: https://discovery-service-2:10011/eureka/,https://discovery-service:10011/eureka/ + logbackService: ZWEAZS2 steps: - uses: actions/checkout@v4 with: @@ -548,10 +557,10 @@ jobs: - name: Run Startup Check if: always() - timeout-minutes: 4 run: > ./gradlew runStartUpCheck --info -Denvironment.config=-ha -Ddiscoverableclient.instances=1 -Partifactory_user=$ARTIFACTORY_USERNAME -Partifactory_password=$ARTIFACTORY_PASSWORD + -Dcaching.instances=1 -Dzosmf.instances=0 env: ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }} ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }} @@ -568,6 +577,7 @@ jobs: with: name: CITestsSAFProviderHA-${{ env.JOB_ID }} path: | + integration-tests/build/test-results/runStartUpCheck/binary/** integration-tests/build/reports/** results/** @@ -607,6 +617,7 @@ jobs: APIML_SECURITY_X509_ACCEPTFORWARDEDCERT: true APIML_SECURITY_X509_CERTIFICATESURL: https://gateway-service-2:10010/gateway/certificates APIML_SERVICE_HOSTNAME: zaas-service-2 + logbackService: ZWEAZS2 central-gateway-service: image: ghcr.io/balhar-jakub/gateway-service:${{ github.run_id }}-${{ github.run_number }} env: @@ -626,6 +637,16 @@ jobs: - uses: ./.github/actions/setup + - name: Run Startup Check + if: always() + run: > + ./gradlew runStartUpCheck --info -Denvironment.config=-docker + -Partifactory_user=$ARTIFACTORY_USERNAME -Partifactory_password=$ARTIFACTORY_PASSWORD + -Ddiscovery.instances=1 -Dapicatalog.instances=0 + env: + ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }} + ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }} + - name: Run CI Tests run: > ./gradlew :integration-tests:runGatewayProxyTest --info -Denvironment.config=-docker @@ -641,6 +662,7 @@ jobs: with: name: GatewayProxy-${{ env.JOB_ID }} path: | + integration-tests/build/test-results/runStartUpCheck/binary/** integration-tests/build/reports/** results/** @@ -658,6 +680,10 @@ jobs: image: ghcr.io/balhar-jakub/discovery-service:${{ github.run_id }}-${{ github.run_number }} volumes: - /api-defs:/api-defs + api-catalog-services: + image: ghcr.io/balhar-jakub/api-catalog-services:${{ github.run_id }}-${{ github.run_number }} + volumes: + - /api-defs:/api-defs zaas-service: image: ghcr.io/balhar-jakub/zaas-service:${{ github.run_id }}-${{ github.run_number }} env: @@ -699,10 +725,13 @@ jobs: env: APIML_SERVICE_HOSTNAME: api-catalog-services-2 APIML_SERVICE_DISCOVERYSERVICEURLS: https://discovery-service-2:10011/eureka/ + logbackService: ZWEAAC2 discovery-service-2: image: ghcr.io/balhar-jakub/discovery-service:${{ github.run_id }}-${{ github.run_number }} env: + APIML_DISCOVERY_ALLPEERSURLS: https://discovery-service-2:10011/eureka APIML_SERVICE_HOSTNAME: discovery-service-2 + logbackService: ZWEADS2 zaas-service-2: image: ghcr.io/balhar-jakub/zaas-service:${{ github.run_id }}-${{ github.run_number }} env: @@ -711,7 +740,7 @@ jobs: APIML_SECURITY_X509_ENABLED: true APIML_SECURITY_X509_ACCEPTFORWARDEDCERT: true APIML_SECURITY_X509_CERTIFICATESURL: https://central-gateway-service:10010/gateway/certificates - logbackService: ZWEAAZ1 + logbackService: ZWEAZS2 mock-services: image: ghcr.io/balhar-jakub/mock-services:${{ github.run_id }}-${{ github.run_number }} @@ -722,13 +751,23 @@ jobs: - uses: ./.github/actions/setup + - name: Run Startup Check + if: always() + run: > + ./gradlew runStartUpCheck --info -Denvironment.config=-docker + -Ddiscoverableclient.instances=0 -Dgateway.instances=2 + -DcentralHosts=central-gateway-service,api-catalog-services-2,discovery-service-2 + -Partifactory_user=$ARTIFACTORY_USERNAME -Partifactory_password=$ARTIFACTORY_PASSWORD + env: + ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }} + ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }} + - name: Run CI Tests - timeout-minutes: 4 run: > - ./gradlew runStartUpCheck :integration-tests:runGatewayCentralRegistryTest --info + ./gradlew :integration-tests:runGatewayCentralRegistryTest --info -Denvironment.config=-docker -Dgateway.instances=2 -Ddiscoverableclient.instances=0 -Partifactory_user=${{ secrets.ARTIFACTORY_USERNAME }} -Partifactory_password=${{ secrets.ARTIFACTORY_PASSWORD }} - + -DcentralHosts=central-gateway-service,api-catalog-services-2,discovery-service-2 - name: Dump CGW jacoco data run: > java -jar ./scripts/jacococli.jar dump --address gateway-service --port 6300 --destfile ./results/gateway-service.exec @@ -739,6 +778,7 @@ jobs: with: name: GatewayCentralRegistry-${{ env.JOB_ID }} path: | + integration-tests/build/test-results/runStartUpCheck/binary/** integration-tests/build/reports/** results/** @@ -804,9 +844,10 @@ jobs: - name: Run Startup Check if: always() - timeout-minutes: 4 run: > - ./gradlew runStartUpCheck --info -Denvironment.config=-docker-modulith -Ddiscoverableclient.instances=0 -Denvironment.modulith=true -Dgateway.instances=2 -Dgateway.host=central-gateway-service + ./gradlew runStartUpCheck --info -Denvironment.config=-docker-modulith -Ddiscoverableclient.instances=0 + -Denvironment.modulith=true -Dgateway.host=central-gateway-service + -DcentralHosts=central-gateway-service - name: Run CI Tests timeout-minutes: 4 @@ -822,6 +863,7 @@ jobs: with: name: ModulithCentralRegistry-${{ env.JOB_ID }} path: | + integration-tests/build/test-results/runStartUpCheck/binary/** integration-tests/build/reports/** - uses: ./.github/actions/teardown @@ -859,6 +901,16 @@ jobs: - uses: ./.github/actions/setup + - name: Run Startup Check + if: always() + run: > + ./gradlew runStartUpCheck --info -Denvironment.config=-docker + -Partifactory_user=$ARTIFACTORY_USERNAME -Partifactory_password=$ARTIFACTORY_PASSWORD + -Ddiscovery.instances=1 -DcentralGateway.instances=0 + env: + ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }} + ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }} + - name: Run CI Tests run: > ./gradlew :integration-tests:runRegistrationTests --info -Denvironment.config=-docker @@ -871,6 +923,7 @@ jobs: with: name: ContainerCITestsRegistration-${{ env.JOB_ID }} path: | + integration-tests/build/test-results/runStartUpCheck/binary/** integration-tests/build/reports/** - uses: ./.github/actions/teardown @@ -891,6 +944,11 @@ jobs: JGROUPS_BIND_PORT: "7600" central-gateway-service: image: ghcr.io/balhar-jakub/gateway-service:${{ github.run_id }}-${{ github.run_number }} + env: + APIML_SERVICE_APIMLID: central-apiml + APIML_SERVICE_HOSTNAME: central-gateway-service + APIML_SERVICE_EXTERNALURL: https://central-gateway-service:10010 + logbackService: ZWEAGW2 discoverable-client: image: ghcr.io/balhar-jakub/discoverable-client:${{ github.run_id }}-${{ github.run_number }} api-catalog-services: @@ -930,14 +988,24 @@ jobs: - uses: ./.github/actions/setup + - name: Run Startup Check + if: always() + run: > + ./gradlew runStartUpCheck --info -Denvironment.config=-docker + -Ddiscoverableclient.instances=1 -Ddiscovery.instances=1 + -Partifactory_user=$ARTIFACTORY_USERNAME -Partifactory_password=$ARTIFACTORY_PASSWORD + env: + ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }} + ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }} + - name: Build with Gradle run: > - ./gradlew runStartUpCheck :integration-tests:runZaasTest --info -Denvironment.config=-docker + ./gradlew :integration-tests:runZaasTest --info -Denvironment.config=-docker -Partifactory_user=${{ secrets.ARTIFACTORY_USERNAME }} -Partifactory_password=${{ secrets.ARTIFACTORY_PASSWORD }} -Doidc.client.id=${{ secrets.OIDC_CLIENT_ID }} -Doidc.client.secret=${{ secrets.OIDC_CLIENT_SECRET }} -Doidc.test.user=${{ secrets.OIDC_TEST_USER }} -Doidc.test.pass=${{ secrets.OIDC_TEST_PASS }} -Doidc.test.alt_user=${{ secrets.OIDC_WINNIE_USER }} -Doidc.test.alt_pass=${{ secrets.OIDC_WINNIE_PASS }} -Doidc.host=${{secrets.OIDC_HOST}} - -Ddiscoverableclient.instances=1 + -Ddiscoverableclient.instances=1 -Ddiscovery.instances=1 - name: Dump DC jacoco data run: > @@ -949,6 +1017,7 @@ jobs: with: name: ContainerCITestsZaas-${{ env.JOB_ID }} path: | + integration-tests/build/test-results/runStartUpCheck/binary/** integration-tests/build/reports/** results/** @@ -996,6 +1065,16 @@ jobs: - uses: ./.github/actions/setup + - name: Run Startup Check + if: always() + run: > + ./gradlew runStartUpCheck --info -Denvironment.config=-docker + -Partifactory_user=$ARTIFACTORY_USERNAME -Partifactory_password=$ARTIFACTORY_PASSWORD + -Ddiscovery.instances=1 -DcentralGateway.instances=0 + env: + ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }} + ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }} + - name: Build with Gradle run: > ./gradlew :integration-tests:runZosmfAuthTest --info -Denvironment.config=-docker @@ -1007,6 +1086,7 @@ jobs: with: name: ContainerCITestsZosmfWithoutJwt-${{ env.JOB_ID }} path: | + integration-tests/build/test-results/runStartUpCheck/binary/** integration-tests/build/reports/** - uses: ./.github/actions/teardown @@ -1044,6 +1124,16 @@ jobs: - uses: ./.github/actions/setup + - name: Run Startup Check + if: always() + run: > + ./gradlew runStartUpCheck --info -Denvironment.config=-docker-modulith -Denvironment.modulith=true + -Partifactory_user=$ARTIFACTORY_USERNAME -Partifactory_password=$ARTIFACTORY_PASSWORD + -DcentralGateway.instances=0 + env: + ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }} + ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }} + - name: Build with Gradle run: > ./gradlew :integration-tests:runZosmfAuthTest --info @@ -1056,6 +1146,7 @@ jobs: with: name: ContainerCITestsZosmfWithoutJwtModulith-${{ env.JOB_ID }} path: | + integration-tests/build/test-results/runStartUpCheck/binary/** integration-tests/build/reports/** - uses: ./.github/actions/teardown @@ -1078,9 +1169,18 @@ jobs: chmod +x run-redis.sh ./run-redis.sh -l -t -a ${{ env.JOB_ID }} - - uses: ./.github/actions/validate-apiml-healthy - with: - caching-service: "true" + - name: Run Startup Check + run: > + ./gradlew runStartUpCheck --info + -DtlsConfiguration.clientKeyStore=../docker/redis/redis-containers/keystore/all-services.keystore.p12 + -DtlsConfiguration.keyAlias=apimtst + -DtlsConfiguration.keyStore=../docker/redis/redis-containers/keystore/all-services.keystore.p12 + -DtlsConfiguration.trustStore=../docker/redis/redis-containers/keystore/all-services.truststore.p12 + -Ddiscoverableclient.instances=0 -Dapicatalog.instances=0 -Ddiscovery.instances=1 -DcentralGateway.instances=0 + -Partifactory_user=$ARTIFACTORY_USERNAME -Partifactory_password=$ARTIFACTORY_PASSWORD + env: + ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }} + ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }} - name: Run Caching Service tests run: > @@ -1115,6 +1215,10 @@ jobs: if: always() run: docker logs discovery-service + - name: Output Mock services container logs + if: always() + run: docker logs mock-services + # Coverage results are not stored in this job as it would not provide much additional data - name: Store results uses: actions/upload-artifact@v4 @@ -1122,6 +1226,7 @@ jobs: with: name: ContainerCITestsWithRedisReplica-${{ env.JOB_ID }} path: | + integration-tests/build/test-results/runStartUpCheck/binary/** integration-tests/build/reports/** - uses: ./.github/actions/teardown @@ -1144,9 +1249,18 @@ jobs: chmod +x run-redis.sh ./run-redis.sh -l -s -t -a ${{ env.JOB_ID }} - - uses: ./.github/actions/validate-apiml-healthy - with: - caching-service: "true" + - name: Run Startup Check + run: > + ./gradlew runStartUpCheck --info + -DtlsConfiguration.clientKeyStore=../docker/redis/redis-containers/keystore/all-services.keystore.p12 + -DtlsConfiguration.keyAlias=apimtst + -DtlsConfiguration.keyStore=../docker/redis/redis-containers/keystore/all-services.keystore.p12 + -DtlsConfiguration.trustStore=../docker/redis/redis-containers/keystore/all-services.truststore.p12 + -Ddiscoverableclient.instances=0 -Dapicatalog.instances=0 -Ddiscovery.instances=1 -DcentralGateway.instances=0 + -Partifactory_user=$ARTIFACTORY_USERNAME -Partifactory_password=$ARTIFACTORY_PASSWORD + env: + ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }} + ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }} - name: Run Caching Service tests run: > @@ -1187,6 +1301,10 @@ jobs: if: always() run: docker logs discovery-service + - name: Output Mock services container logs + if: always() + run: docker logs mock-services + # Coverage results are not stored in this job as it would not provide much additional data - name: Store results uses: actions/upload-artifact@v4 @@ -1194,6 +1312,7 @@ jobs: with: name: ContainerCITestsWithRedisSentinel-${{ env.JOB_ID }} path: | + integration-tests/build/test-results/runStartUpCheck/binary/** integration-tests/build/reports/** - uses: ./.github/actions/teardown @@ -1209,6 +1328,13 @@ jobs: image: ghcr.io/balhar-jakub/api-catalog-services:${{ github.run_id }}-${{ github.run_number }} volumes: - /api-defs:/api-defs + api-catalog-services-2: + image: ghcr.io/balhar-jakub/api-catalog-services:${{ github.run_id }}-${{ github.run_number }} + volumes: + - /api-defs:/api-defs + env: + APIML_SERVICE_HOSTNAME: api-catalog-services-2 + logbackService: ZWEAAC2 caching-service: image: ghcr.io/balhar-jakub/caching-service:${{ github.run_id }}-${{ github.run_number }} discoverable-client: @@ -1225,6 +1351,7 @@ jobs: - /api-defs:/api-defs env: APIML_DISCOVERY_ALLPEERSURLS: https://discovery-service:10011/eureka,https://discovery-service-2:10011/eureka + logbackService: ZWEADC2 discovery-service-2: image: ghcr.io/balhar-jakub/discovery-service:${{ github.run_id }}-${{ github.run_number }} volumes: @@ -1232,10 +1359,14 @@ jobs: env: APIML_SERVICE_HOSTNAME: discovery-service-2 APIML_DISCOVERY_ALLPEERSURLS: https://discovery-service-2:10011/eureka,https://discovery-service:10011/eureka + logbackService: ZWEADS2 zaas-service: image: ghcr.io/balhar-jakub/zaas-service:${{ github.run_id }}-${{ github.run_number }} zaas-service-2: image: ghcr.io/balhar-jakub/zaas-service:${{ github.run_id }}-${{ github.run_number }} + env: + APIML_SERVICE_HOSTNAME: zaas-service-2 + logbackService: ZWEAZS2 gateway-service: image: ghcr.io/balhar-jakub/gateway-service:${{ github.run_id }}-${{ github.run_number }} env: @@ -1266,22 +1397,35 @@ jobs: apt-cache policy docker-ce apt install -y docker-ce + - name: Run Startup Check + if: always() + run: > + ./gradlew runStartUpCheck --info -Denvironment.config=-ha + -Ddiscoverableclient.instances=2 -Dcaching.instances=1 + -Partifactory_user=$ARTIFACTORY_USERNAME -Partifactory_password=$ARTIFACTORY_PASSWORD + env: + ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }} + ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }} + - name: Run HA Tests run: > - ./gradlew runStartUpCheck runDeterministicLbHaTests --info - -Denvironment.config=-ha -Ddiscoverableclient.instances=2 + ./gradlew runDeterministicLbHaTests --info + -Denvironment.config=-ha -Ddiscoverableclient.instances=2 -Dcaching.instances=1 -Partifactory_user=$ARTIFACTORY_USERNAME -Partifactory_password=$ARTIFACTORY_PASSWORD env: ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }} ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }} + - name: Get Container Logs if: ${{ false }} # Debug of containers run: | docker ps -a docker ps -q | xargs -L 1 docker logs + - name: Correct Permisions run: | chmod 755 -R .gradle + # Coverage results are not stored in this job as it would not provide much additional data - name: Store results uses: actions/upload-artifact@v4 @@ -1289,6 +1433,7 @@ jobs: with: name: DeterministicLbHaTest-${{ env.JOB_ID }} path: | + integration-tests/build/test-results/runStartUpCheck/binary/** integration-tests/build/reports/** - uses: ./.github/actions/teardown @@ -1309,6 +1454,7 @@ jobs: env: APIML_SERVICE_HOSTNAME: discoverable-client-2 APIML_SERVICE_DISCOVERYSERVICEURLS: https://apiml:10011/eureka,https://apiml-2:10011/eureka + logbackService: ZWEADC2 mock-services: image: ghcr.io/balhar-jakub/mock-services:${{ github.run_id }}-${{ github.run_number }} env: @@ -1349,9 +1495,10 @@ jobs: - name: Run Startup Check if: always() - timeout-minutes: 4 run: > - ./gradlew runStartUpCheck --info -Denvironment.config=-docker-modulith-ha -Ddiscoverableclient.instances=2 -Denvironment.gwCount=2 + ./gradlew runStartUpCheck --info -Denvironment.config=-docker-modulith-ha + -Ddiscoverableclient.instances=2 -Denvironment.gwCount=2 -Dgateway.instances=2 -Ddiscovery.instances=2 + -Dcaching.instances=2 -Dapicatalog.instances=2 -Partifactory_user=$ARTIFACTORY_USERNAME -Partifactory_password=$ARTIFACTORY_PASSWORD -Denvironment.modulith=true env: ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }} @@ -1380,6 +1527,7 @@ jobs: with: name: DeterministicLbHaTestModulith-${{ env.JOB_ID }} path: | + integration-tests/build/test-results/runStartUpCheck/binary/** integration-tests/build/reports/** - uses: ./.github/actions/teardown @@ -1395,17 +1543,24 @@ jobs: image: ghcr.io/balhar-jakub/api-catalog-services:${{ github.run_id }}-${{ github.run_number }} volumes: - /api-defs:/api-defs + env: + APIML_SERVICE_DISCOVERYSERVICEURLS: https://discovery-service:10011/eureka/,https://discovery-service-2:10011/eureka/ caching-service: image: ghcr.io/balhar-jakub/caching-service:${{ github.run_id }}-${{ github.run_number }} + env: + APIML_SERVICE_DISCOVERYSERVICEURLS: https://discovery-service:10011/eureka/,https://discovery-service-2:10011/eureka/ discoverable-client: image: ghcr.io/balhar-jakub/discoverable-client:${{ github.run_id }}-${{ github.run_number }} env: APIML_SERVICE_CUSTOMMETADATA_APIML_LB_TYPE: authentication + APIML_SERVICE_DISCOVERYSERVICEURLS: https://discovery-service:10011/eureka/,https://discovery-service-2:10011/eureka/ discoverable-client-2: image: ghcr.io/balhar-jakub/discoverable-client:${{ github.run_id }}-${{ github.run_number }} env: APIML_SERVICE_HOSTNAME: discoverable-client-2 APIML_SERVICE_CUSTOMMETADATA_APIML_LB_TYPE: authentication + APIML_SERVICE_DISCOVERYSERVICEURLS: https://discovery-service-2:10011/eureka/,https://discovery-service:10011/eureka/ + logbackService: ZWEADC2 mock-services: image: ghcr.io/balhar-jakub/mock-services:${{ github.run_id }}-${{ github.run_number }} discovery-service: @@ -1413,18 +1568,25 @@ jobs: volumes: - /api-defs:/api-defs env: - APIML_DISCOVERY_ALLPEERSURLS: https://discovery-service:10011/eureka,https://discovery-service-2:10011/eureka + APIML_DISCOVERY_ALLPEERSURLS: https://discovery-service-2:10011/eureka,https://discovery-service:10011/eureka discovery-service-2: image: ghcr.io/balhar-jakub/discovery-service:${{ github.run_id }}-${{ github.run_number }} volumes: - /api-defs:/api-defs env: APIML_SERVICE_HOSTNAME: discovery-service-2 - APIML_DISCOVERY_ALLPEERSURLS: https://discovery-service-2:10011/eureka,https://discovery-service:10011/eureka + APIML_DISCOVERY_ALLPEERSURLS: https://discovery-service:10011/eureka,https://discovery-service-2:10011/eureka + logbackService: ZWEADS2 zaas-service: image: ghcr.io/balhar-jakub/zaas-service:${{ github.run_id }}-${{ github.run_number }} + env: + APIML_SERVICE_DISCOVERYSERVICEURLS: https://discovery-service:10011/eureka/,https://discovery-service-2:10011/eureka/ zaas-service-2: image: ghcr.io/balhar-jakub/zaas-service:${{ github.run_id }}-${{ github.run_number }} + env: + APIML_SERVICE_HOSTNAME: zaas-service-2 + APIML_SERVICE_DISCOVERYSERVICEURLS: https://discovery-service-2:10011/eureka/,https://discovery-service:10011/eureka/ + logbackService: ZWEAZS2 gateway-service: image: ghcr.io/balhar-jakub/gateway-service:${{ github.run_id }}-${{ github.run_number }} env: @@ -1434,7 +1596,7 @@ jobs: image: ghcr.io/balhar-jakub/gateway-service:${{ github.run_id }}-${{ github.run_number }} env: APIML_SERVICE_HOSTNAME: gateway-service-2 - APIML_SERVICE_DISCOVERYSERVICEURLS: https://discovery-service:10011/eureka/,https://discovery-service-2:10011/eureka/ + APIML_SERVICE_DISCOVERYSERVICEURLS: https://discovery-service-2:10011/eureka/,https://discovery-service:10011/eureka/ APIML_SERVICE_CUSTOMMETADATA_APIML_LB_TYPE: authentication logbackService: ZWEAGW2 steps: @@ -1455,6 +1617,16 @@ jobs: apt-cache policy docker-ce apt install -y docker-ce + - name: Run Startup Check + if: always() + run: > + ./gradlew runStartUpCheck --info -Denvironment.config=-ha + -Partifactory_user=$ARTIFACTORY_USERNAME -Partifactory_password=$ARTIFACTORY_PASSWORD + -Dapicatalog.instances=1 -Dcaching.instances=1 + env: + ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }} + ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }} + - name: Run HA Tests run: > ./gradlew runStickySessionLbHaTests --info -Denvironment.config=-ha @@ -1462,14 +1634,17 @@ jobs: env: ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }} ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }} + - name: Get Container Logs if: ${{ false }} # Debug of containers run: | docker ps -a docker ps -q | xargs -L 1 docker logs + - name: Correct Permisions run: | chmod 755 -R .gradle + # Coverage results are not stored in this job as it would not provide much additional data - name: Store results uses: actions/upload-artifact@v4 @@ -1477,6 +1652,7 @@ jobs: with: name: StickySessionLbHaTest-${{ env.JOB_ID }} path: | + integration-tests/build/test-results/runStartUpCheck/binary/** integration-tests/build/reports/** - uses: ./.github/actions/teardown @@ -1492,17 +1668,20 @@ jobs: image: ghcr.io/balhar-jakub/discoverable-client:${{ github.run_id }}-${{ github.run_number }} env: APIML_SERVICE_CUSTOMMETADATA_APIML_LB_TYPE: authentication + APIML_SERVICE_DISCOVERYSERVICEURLS: https://apiml:10011/eureka,https://apiml-2:10011/eureka discoverable-client-2: image: ghcr.io/balhar-jakub/discoverable-client:${{ github.run_id }}-${{ github.run_number }} env: APIML_SERVICE_HOSTNAME: discoverable-client-2 APIML_SERVICE_CUSTOMMETADATA_APIML_LB_TYPE: authentication + APIML_SERVICE_DISCOVERYSERVICEURLS: https://apiml-2:10011/eureka,https://apiml:10011/eureka + logbackService: ZWEADC2 mock-services: image: ghcr.io/balhar-jakub/mock-services:${{ github.run_id }}-${{ github.run_number }} apiml: image: ghcr.io/balhar-jakub/apiml:${{ github.run_id }}-${{ github.run_number }} env: - APIML_DISCOVERY_ALLPEERSURLS: https://apiml:10011/eureka,https://apiml-2:10011/eureka + APIML_DISCOVERY_ALLPEERSURLS: https://apiml-2:10011/eureka,https://apiml:10011/eureka APIML_SERVICE_HOSTNAME: apiml APIML_SERVICE_CUSTOMMETADATA_APIML_LB_TYPE: authentication logbackService: ZWEAGW1 @@ -1531,6 +1710,16 @@ jobs: apt-cache policy docker-ce apt install -y docker-ce + - name: Run Startup Check + if: always() + run: > + ./gradlew runStartUpCheck --info -Denvironment.config=-docker-modulith-ha -Denvironment.modulith=true + -Partifactory_user=$ARTIFACTORY_USERNAME -Partifactory_password=$ARTIFACTORY_PASSWORD + -Dgateway.instances=2 -Ddiscovery.instances=2 -Dcaching.instances=2 -Dapicatalog.instances=2 + env: + ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }} + ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }} + - name: Run HA Tests run: > ./gradlew runStickySessionLbHaTests --info -Denvironment.config=-docker-modulith-ha @@ -1553,6 +1742,7 @@ jobs: with: name: StickySessionLbHaTestModulith-${{ env.JOB_ID }} path: | + integration-tests/build/test-results/runStartUpCheck/binary/** integration-tests/build/reports/** - uses: ./.github/actions/teardown @@ -1571,13 +1761,17 @@ jobs: image: ghcr.io/balhar-jakub/api-catalog-services:${{ github.run_id }}-${{ github.run_number }} volumes: - /api-defs:/api-defs + env: + APIML_SERVICE_DISCOVERYSERVICEURLS: https://discovery-service:10011/eureka/,https://discovery-service-2:10011/eureka/ api-catalog-services-2: image: ghcr.io/balhar-jakub/api-catalog-services:${{ github.run_id }}-${{ github.run_number }} volumes: - /api-defs:/api-defs env: APIML_SERVICE_HOSTNAME: api-catalog-services-2 + APIML_SERVICE_DISCOVERYSERVICEURLS: https://discovery-service-2:10011/eureka/,https://discovery-service:10011/eureka/ APIML_HEALTH_PROTECTED: false + logbackService: ZWEAAC2 caching-service: image: ghcr.io/balhar-jakub/caching-service:${{ github.run_id }}-${{ github.run_number }} env: @@ -1590,6 +1784,7 @@ jobs: APIML_HEALTH_PROTECTED: false MANAGEMENT_ENDPOINTS_WEB_EXPOSURE_INCLUDE: "*" MANAGEMENT_ENDPOINT_SHUTDOWN_ACCESS: unrestricted + APIML_SERVICE_DISCOVERYSERVICEURLS: https://discovery-service:10011/eureka/,https://discovery-service-2:10011/eureka/ caching-service-2: image: ghcr.io/balhar-jakub/caching-service:${{ github.run_id }}-${{ github.run_number }} env: @@ -1602,6 +1797,8 @@ jobs: APIML_HEALTH_PROTECTED: false MANAGEMENT_ENDPOINTS_WEB_EXPOSURE_INCLUDE: "*" MANAGEMENT_ENDPOINT_SHUTDOWN_ACCESS: unrestricted + APIML_SERVICE_DISCOVERYSERVICEURLS: https://discovery-service-2:10011/eureka/,https://discovery-service:10011/eureka/ + logbackService: ZWEACS2 caching-service-3: image: ghcr.io/balhar-jakub/caching-service:${{ github.run_id }}-${{ github.run_number }} env: @@ -1614,14 +1811,22 @@ jobs: APIML_HEALTH_PROTECTED: false MANAGEMENT_ENDPOINTS_WEB_EXPOSURE_INCLUDE: "*" MANAGEMENT_ENDPOINT_SHUTDOWN_ACCESS: unrestricted + APIML_SERVICE_DISCOVERYSERVICEURLS: https://discovery-service:10011/eureka/,https://discovery-service-2:10011/eureka/ + logbackService: ZWEACS3 discoverable-client: image: ghcr.io/balhar-jakub/discoverable-client:${{ github.run_id }}-${{ github.run_number }} + env: + APIML_SERVICE_DISCOVERYSERVICEURLS: https://discovery-service:10011/eureka/,https://discovery-service-2:10011/eureka/ discoverable-client-2: image: ghcr.io/balhar-jakub/discoverable-client:${{ github.run_id }}-${{ github.run_number }} env: APIML_SERVICE_HOSTNAME: discoverable-client-2 + APIML_SERVICE_DISCOVERYSERVICEURLS: https://discovery-service-2:10011/eureka/,https://discovery-service:10011/eureka/ + logbackService: ZWEADC2 mock-services: image: ghcr.io/balhar-jakub/mock-services:${{ github.run_id }}-${{ github.run_number }} + env: + APIML_SERVICE_DISCOVERYSERVICEURLS: https://discovery-service:10011/eureka/,https://discovery-service-2:10011/eureka/ discovery-service: image: ghcr.io/balhar-jakub/discovery-service:${{ github.run_id }}-${{ github.run_number }} volumes: @@ -1635,14 +1840,18 @@ jobs: env: APIML_SERVICE_HOSTNAME: discovery-service-2 APIML_DISCOVERY_ALLPEERSURLS: https://discovery-service-2:10011/eureka,https://discovery-service:10011/eureka + logbackService: ZWEADS2 zaas-service: image: ghcr.io/balhar-jakub/zaas-service:${{ github.run_id }}-${{ github.run_number }} env: APIML_SERVICE_HOSTNAME: zaas-service + APIML_SERVICE_DISCOVERYSERVICEURLS: https://discovery-service:10011/eureka,https://discovery-service-2:10011/eureka zaas-service-2: image: ghcr.io/balhar-jakub/zaas-service:${{ github.run_id }}-${{ github.run_number }} env: APIML_SERVICE_HOSTNAME: zaas-service-2 + APIML_SERVICE_DISCOVERYSERVICEURLS: https://discovery-service-2:10011/eureka/,https://discovery-service:10011/eureka/ + logbackService: ZWEAZS2 gateway-service: image: ghcr.io/balhar-jakub/gateway-service:${{ github.run_id }}-${{ github.run_number }} env: @@ -1662,7 +1871,6 @@ jobs: - name: Run Startup Check if: always() - timeout-minutes: 4 run: > ./gradlew runStartUpCheck --info -Denvironment.config=-ha -Ddiscoverableclient.instances=2 -Partifactory_user=$ARTIFACTORY_USERNAME -Partifactory_password=$ARTIFACTORY_PASSWORD @@ -1683,7 +1891,7 @@ jobs: if: ${{ 'discovery-chaotic' == matrix.type }} run: > ./gradlew :integration-tests:runChaoticHATests --tests org.zowe.apiml.integration.ha.DiscoveryChaoticTest - --info -Denvironment.config=-ha -Ddiscoverableclient.instances=1 + --info -Denvironment.config=-ha -Partifactory_user=$ARTIFACTORY_USERNAME -Partifactory_password=$ARTIFACTORY_PASSWORD env: ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }} @@ -1739,6 +1947,7 @@ jobs: with: name: CITestsHA-${{ env.JOB_ID }}-${{ matrix.type }} path: | + integration-tests/build/test-results/runStartUpCheck/binary/** integration-tests/build/reports/** - uses: ./.github/actions/teardown @@ -1761,11 +1970,10 @@ jobs: JGROUPS_BIND_PORT: "7600" SERVER_SSL_KEYSTORETYPE: "PKCS12" CACHING_STORAGE_INFINISPAN_PERSISTENCE_DATALOCATION: "data_replica" - CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "caching-service-2[7600]" + CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "caching-service[7600],caching-service-2[7600]" CACHING_STORAGE_MODE: "infinispan" APIML_SERVICE_PORT: "10022" JGROUPS_BIND_ADDRESS: "caching-service" - caching-service-2: image: ghcr.io/balhar-jakub/caching-service:${{ github.run_id }}-${{ github.run_number }} env: @@ -1773,11 +1981,11 @@ jobs: JGROUPS_BIND_PORT: "7600" SERVER_SSL_KEYSTORETYPE: "PKCS12" CACHING_STORAGE_INFINISPAN_PERSISTENCE_DATALOCATION: "data" - CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "caching-service[7600]" + CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "caching-service[7600],caching-service-2[7600]" CACHING_STORAGE_MODE: "infinispan" JGROUPS_BIND_ADDRESS: "caching-service-2" APIML_SERVICE_HOSTNAME: "caching-service-2" - + logbackService: ZWEACS2 discoverable-client: image: ghcr.io/balhar-jakub/discoverable-client:${{ github.run_id }}-${{ github.run_number }} discovery-service: @@ -1801,6 +2009,16 @@ jobs: - uses: ./.github/actions/setup + - name: Run Startup Check + if: always() + run: > + ./gradlew runStartUpCheck --info -Denvironment.config=-docker + -Partifactory_user=$ARTIFACTORY_USERNAME -Partifactory_password=$ARTIFACTORY_PASSWORD + -Ddiscovery.instances=1 -DcentralGateway.instances=0 + env: + ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }} + ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }} + - name: Build with Gradle run: > ./gradlew :integration-tests:runInfinispanServiceTests --info -Denvironment.config=-docker @@ -1815,6 +2033,7 @@ jobs: with: name: CITestsWithInfinispan-${{ env.JOB_ID }} path: | + integration-tests/build/test-results/runStartUpCheck/binary/** integration-tests/build/reports/** results/** @@ -1832,29 +2051,28 @@ jobs: env: APIML_SECURITY_X509_ENABLED: true APIML_SECURITY_SSL_NONSTRICTVERIFYSSLCERTIFICATESOFSERVICES: true + APIML_DISCOVERY_ALLPEERSURLS: https://apiml-2:10011/eureka,https://apiml:10011/eureka ZWE_CACHING_SERVICE_PERSISTENT: 'infinispan' JGROUPS_BIND_PORT: "7600" SERVER_SSL_KEYSTORETYPE: "PKCS12" - CACHING_STORAGE_INFINISPAN_PERSISTENCE_DATALOCATION: "data_replica" - CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "apiml-2[7600]" + CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "apiml[7600],apiml-2[7600]" CACHING_STORAGE_MODE: "infinispan" JGROUPS_BIND_ADDRESS: "apiml" - apiml-2: image: ghcr.io/balhar-jakub/apiml:${{ github.run_id }}-${{ github.run_number }} env: APIML_SECURITY_X509_ENABLED: true APIML_GATEWAY_SERVICESTOLIMITREQUESTRATE: discoverableclient APIML_SECURITY_SSL_NONSTRICTVERIFYSSLCERTIFICATESOFSERVICES: true + APIML_DISCOVERY_ALLPEERSURLS: https://apiml:10011/eureka,https://apiml-2:10011/eureka ZWE_CACHING_SERVICE_PERSISTENT: 'infinispan' JGROUPS_BIND_PORT: "7600" SERVER_SSL_KEYSTORETYPE: "PKCS12" - CACHING_STORAGE_INFINISPAN_PERSISTENCE_DATALOCATION: "data" - CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "apiml[7600]" + CACHING_STORAGE_INFINISPAN_INITIALHOSTS: "apiml[7600],apiml-2[7600]" CACHING_STORAGE_MODE: "infinispan" JGROUPS_BIND_ADDRESS: "apiml-2" APIML_SERVICE_HOSTNAME: "apiml-2" - + logbackService: ZWEAGW2 api-catalog-services: image: ghcr.io/balhar-jakub/api-catalog-services:${{ github.run_id }}-${{ github.run_number }} volumes: @@ -1864,6 +2082,8 @@ jobs: discoverable-client: image: ghcr.io/balhar-jakub/discoverable-client:${{ github.run_id }}-${{ github.run_number }} + env: + APIML_SERVICE_DISCOVERYSERVICEURLS: https://apiml:10011/eureka,https://apiml-2:10011/eureka mock-services: image: ghcr.io/balhar-jakub/mock-services:${{ github.run_id }}-${{ github.run_number }} steps: @@ -1873,6 +2093,16 @@ jobs: - uses: ./.github/actions/setup + - name: Run Startup Check + if: always() + run: > + ./gradlew runStartUpCheck --info -Denvironment.config=-docker-modulith + -Denvironment.modulith=true -Ddiscoverableclient.instances=1 -DcentralGateway.instances=0 + -Partifactory_user=$ARTIFACTORY_USERNAME -Partifactory_password=$ARTIFACTORY_PASSWORD + env: + ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }} + ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }} + - name: Build with Gradle run: > ENV_CONFIG=docker-modulith ./gradlew :integration-tests:runModulithInfinispanServiceTests --info @@ -1888,6 +2118,7 @@ jobs: with: name: CITestsModulithWithInfinispan-${{ env.JOB_ID }} path: | + integration-tests/build/test-results/runStartUpCheck/binary/** integration-tests/build/reports/** results/** @@ -1940,6 +2171,7 @@ jobs: env: APIML_SERVICE_HOSTNAME: discovery-service-2 APIML_DISCOVERY_ALLPEERSURLS: https://discovery-service-2:10011/eureka + logbackService: ZWEADS2 gateway-service-2: image: ghcr.io/balhar-jakub/gateway-service:${{ github.run_id }}-${{ github.run_number }} env: @@ -1960,6 +2192,7 @@ jobs: APIML_SERVICE_DISCOVERYSERVICEURLS: https://discovery-service-2:10011/eureka/ APIML_HEALTH_PROTECTED: false APIML_SERVICE_HOSTNAME: zaas-service-2 + logbackService: ZWEAZS2 steps: - uses: actions/checkout@v4 @@ -1978,10 +2211,13 @@ jobs: ~/.cache/Cypress api-catalog-ui/frontend/node_modules key: my-cache-${{ runner.os }}-${{ hashFiles('api-catalog-ui/frontend/*.json') }} + - name: Run startup check - timeout-minutes: 4 run: > - ./gradlew runStartUpCheck --info --scan -Denvironment.config=-ha -Ddiscoverableclient.instances=1 -Dgateway.instances=1 + ./gradlew runStartUpCheck --info --scan -Denvironment.config=-ha -Ddiscoverableclient.instances=1 + -Dgateway.instances=1 -Dcaching.instances=1 -Dapicatalog.instances=1 + -DcentralHosts=api-catalog-services,discoverable-client,discovery-service,gateway-service,zaas-service,caching-service + - name: Show status when APIML is not ready yet if: failure() shell: bash @@ -1990,24 +2226,29 @@ jobs: apt install -y apt-transport-https ca-certificates curl software-properties-common curl -k -s https://gateway-service:10010/application/health curl -k -s https://gateway-service-2:10010/application/health + - name: Cypress run API Catalog run: | cd api-catalog-ui/frontend export CYPRESS_AUTH0_USERNAME=${{ secrets.OIDC_WINNIE_USER }} export CYPRESS_AUTH0_PASSWORD=${{ secrets.OIDC_WINNIE_PASS }} npm run cy:e2e:ci + - name: Dump CGW jacoco data run: > java -jar ./scripts/jacococli.jar dump --address gateway-service --port 6300 --destfile ./results/gateway-service.exec + - name: Correct Permissions run: | chmod 755 -R .gradle + - name: Store results uses: actions/upload-artifact@v4 if: always() with: name: E2EUITests-${{ env.JOB_ID }} path: | + integration-tests/build/test-results/runStartUpCheck/binary/** results/** - name: Upload screenshots API Catalog @@ -2090,11 +2331,13 @@ jobs: ~/.cache/Cypress api-catalog-ui/frontend/node_modules key: my-cache-${{ runner.os }}-${{ hashFiles('api-catalog-ui/frontend/*.json') }} + - name: Run startup check - timeout-minutes: 4 run: > ./gradlew runStartUpCheck --info --scan -Denvironment.config=-docker-modulith-ha -Ddiscoverableclient.instances=1 -Denvironment.modulith=true -Denvironment.gwCount=1 -Dgateway.host=gateway-service,gateway-service-2 -Ddiscovery.host=gateway-service -Ddiscovery.additionalHost=gateway-service-2 + -DcentralHosts=discoverable-client,gateway-service + - name: Show status when APIML is not ready yet if: failure() shell: bash @@ -2103,24 +2346,29 @@ jobs: apt install -y apt-transport-https ca-certificates curl software-properties-common curl -k -s https://gateway-service:10010/application/health curl -k -s https://gateway-service-2:10010/application/health + - name: Cypress run API Catalog run: | cd api-catalog-ui/frontend export CYPRESS_AUTH0_USERNAME=${{ secrets.OIDC_WINNIE_USER }} export CYPRESS_AUTH0_PASSWORD=${{ secrets.OIDC_WINNIE_PASS }} npm run cy:e2e:ci:modulith + - name: Dump CGW jacoco data run: > java -jar ./scripts/jacococli.jar dump --address gateway-service --port 6310 --destfile ./results/gateway-service.exec + - name: Correct Permissions run: | chmod 755 -R .gradle + - name: Store results uses: actions/upload-artifact@v4 if: always() with: name: E2EUITestsModulith-${{ env.JOB_ID }} path: | + integration-tests/build/test-results/runStartUpCheck/binary/** results/** - name: Upload screenshots API Catalog @@ -2163,6 +2411,17 @@ jobs: - uses: ./.github/actions/setup + - name: Run Startup Check + if: always() + run: > + ./gradlew runStartUpCheck --info -Denvironment.config=-docker + -Partifactory_user=$ARTIFACTORY_USERNAME -Partifactory_password=$ARTIFACTORY_PASSWORD + -Ddiscovery.instances=1 -DserviceIdReplaced=discoverableclient:sampleclient + -DcentralGateway.instances=0 + env: + ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }} + ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }} + - name: Run Service ID Prefix Replacer Tests run: > ./gradlew :integration-tests:runIdPrefixReplacerTests --info -Denvironment.config=-docker @@ -2174,6 +2433,7 @@ jobs: with: name: CITestsServicePrefixReplacer-${{ env.JOB_ID }} path: | + integration-tests/build/test-results/runStartUpCheck/binary/** integration-tests/build/reports/** results/** @@ -2205,6 +2465,16 @@ jobs: - uses: ./.github/actions/setup + - name: Run Startup Check + if: always() + run: > + ./gradlew runStartUpCheck --info -Denvironment.config=-docker-modulith + -Denvironment.modulith=true -Ddiscoverableclient.instances=0 -DcentralGateway.instances=0 + -Partifactory_user=$ARTIFACTORY_USERNAME -Partifactory_password=$ARTIFACTORY_PASSWORD + env: + ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }} + ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }} + - name: Run Node.JS and Python Services Tests run: > ./gradlew :integration-tests:runNodeAndPythonSampleServiceTests --info -Denvironment.config=-docker-modulith @@ -2216,6 +2486,7 @@ jobs: with: name: CITestsNodeJsAndPythonServices-${{ env.JOB_ID }} path: | + integration-tests/build/test-results/runStartUpCheck/binary/** integration-tests/build/reports/** results/** diff --git a/api-catalog-services/src/main/java/org/zowe/apiml/apicatalog/ApiCatalogApplication.java b/api-catalog-services/src/main/java/org/zowe/apiml/apicatalog/ApiCatalogApplication.java index a259418794..1007763dcc 100644 --- a/api-catalog-services/src/main/java/org/zowe/apiml/apicatalog/ApiCatalogApplication.java +++ b/api-catalog-services/src/main/java/org/zowe/apiml/apicatalog/ApiCatalogApplication.java @@ -28,6 +28,7 @@ @ComponentScan(value = { "org.zowe.apiml.apicatalog", "org.zowe.apiml.product.compatibility", + "org.zowe.apiml.product.eureka.web", "org.zowe.apiml.product.security", "org.zowe.apiml.product.web", "org.zowe.apiml.product.gateway", diff --git a/apiml-common/src/main/java/org/zowe/apiml/product/config/ServerAddressPropertiesUpdater.java b/apiml-common/src/main/java/org/zowe/apiml/product/config/ServerAddressPropertiesUpdater.java index 61056dbe82..9dcf7f0380 100644 --- a/apiml-common/src/main/java/org/zowe/apiml/product/config/ServerAddressPropertiesUpdater.java +++ b/apiml-common/src/main/java/org/zowe/apiml/product/config/ServerAddressPropertiesUpdater.java @@ -147,7 +147,7 @@ public void postProcessEnvironment(ConfigurableEnvironment environment, SpringAp * @return processor to create new customizer beans or an empty processor if there are no additional address. */ @Bean - public BeanDefinitionRegistryPostProcessor registerAdditionalTomcatConnectors() { + public static BeanDefinitionRegistryPostProcessor registerAdditionalTomcatConnectors() { if (ADDITIONAL_NETWORKS.isEmpty()) { return registry -> {}; } diff --git a/apiml-common/src/main/java/org/zowe/apiml/product/eureka/client/ApimlPeerEurekaNode.java b/apiml-common/src/main/java/org/zowe/apiml/product/eureka/client/ApimlPeerEurekaNode.java index 259194b042..199873d091 100644 --- a/apiml-common/src/main/java/org/zowe/apiml/product/eureka/client/ApimlPeerEurekaNode.java +++ b/apiml-common/src/main/java/org/zowe/apiml/product/eureka/client/ApimlPeerEurekaNode.java @@ -462,8 +462,9 @@ public ProcessingResult process(List tasks) { logNetworkErrorSample(null, "; retrying after delay.", e); return ProcessingResult.TransientError; } else { + log.warn("Cannot replicate to another DS instance: {}", e.getMessage()); logNetworkErrorSample(null, "; not re-trying this exception because it does not seem to be a network exception.", e); - return ProcessingResult.PermanentError; + return ProcessingResult.TransientError; } } return ProcessingResult.Success; diff --git a/apiml-common/src/main/java/org/zowe/apiml/product/eureka/web/EurekaRegistryVersionEndpoint.java b/apiml-common/src/main/java/org/zowe/apiml/product/eureka/web/EurekaRegistryVersionEndpoint.java new file mode 100644 index 0000000000..c7d062903a --- /dev/null +++ b/apiml-common/src/main/java/org/zowe/apiml/product/eureka/web/EurekaRegistryVersionEndpoint.java @@ -0,0 +1,75 @@ +/* + * This program and the accompanying materials are made available under the terms of the + * Eclipse Public License v2.0 which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-v20.html + * + * SPDX-License-Identifier: EPL-2.0 + * + * Copyright Contributors to the Zowe Project. + */ + +package org.zowe.apiml.product.eureka.web; + +import com.netflix.discovery.EurekaClient; +import com.netflix.discovery.EurekaEvent; +import jakarta.annotation.PostConstruct; +import lombok.Builder; +import lombok.RequiredArgsConstructor; +import lombok.Value; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.actuate.endpoint.annotation.Endpoint; +import org.springframework.boot.actuate.endpoint.annotation.ReadOperation; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; + +import java.util.regex.Pattern; + +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; + +@Component +@RequiredArgsConstructor +@Endpoint(id = "eurekaversion") +@ConditionalOnMissingBean(name = "modulithConfig") +@Slf4j +public class EurekaRegistryVersionEndpoint { + + private static final Pattern VERSION_PATTERN = Pattern.compile("^.*_([0-9]+)_.*$"); + + private Long version = -1L; + + private final EurekaClient eurekaClient; + + @PostConstruct + void registerListener() { + eurekaClient.registerEventListener(event -> { + onRegistryUpdate(event); + }); + } + + @EventListener + void onRegistryUpdate(EurekaEvent event) { + var hashCode = eurekaClient.getApplications().getAppsHashCode(); + var matcher = VERSION_PATTERN.matcher(hashCode); + if (matcher.matches()) { + version = Long.parseLong(matcher.group(1)); + log.debug("New Eureka registry version: {}", this.version); + } else { + log.debug("Unexpected Eureka registry hashCode: {}", hashCode); + } + } + + @ReadOperation(produces = APPLICATION_JSON) + public VersionDto status() { + return VersionDto.builder().version(this.version).build(); + } + + @Builder + @Value + static class VersionDto { + + private Long version; + + } + +} diff --git a/apiml-security-common/src/main/java/org/zowe/apiml/security/common/error/AuthExceptionHandler.java b/apiml-security-common/src/main/java/org/zowe/apiml/security/common/error/AuthExceptionHandler.java index ce2e951d7e..938fdbc454 100644 --- a/apiml-security-common/src/main/java/org/zowe/apiml/security/common/error/AuthExceptionHandler.java +++ b/apiml-security-common/src/main/java/org/zowe/apiml/security/common/error/AuthExceptionHandler.java @@ -109,7 +109,7 @@ private Map.Entry, ExceptionHandler> entry(Cla entry(WebClientResponseException.BadRequest.class, (ex, ctx) -> handleBadRequest(ctx.requestUri, ctx.function, ex, "org.zowe.apiml.security.login.invalidInput")), entry(AccessDeniedException.class, - (ex, ctx) -> handleForbidden(ctx.function, ex) + (ex, ctx) -> handleForbidden(ctx.requestUri, ctx.function, ex) ), entry(GatewayNotAvailableException.class, (ex, ctx) -> handleGatewayNotAvailable(ctx.function, ex, ctx.requestUri) @@ -270,9 +270,9 @@ private void handleRuntimeException(String uri, BiConsumer function, AccessDeniedException ex) { + private void handleForbidden(String uri, BiConsumer function, AccessDeniedException ex) { log.debug(MESSAGE_FORMAT, HttpStatus.FORBIDDEN.value(), ex.getMessage()); - writeErrorResponse("org.zowe.apiml.security.forbidden", HttpStatus.FORBIDDEN, function); + writeErrorResponse("org.zowe.apiml.security.forbidden", HttpStatus.FORBIDDEN, function, uri); } private void handleGatewayNotAvailable(BiConsumer function, GatewayNotAvailableException ex, String uri) { diff --git a/apiml/src/main/java/org/zowe/apiml/ApimlApplication.java b/apiml/src/main/java/org/zowe/apiml/ApimlApplication.java index f149da2e39..049d9ec85d 100644 --- a/apiml/src/main/java/org/zowe/apiml/ApimlApplication.java +++ b/apiml/src/main/java/org/zowe/apiml/ApimlApplication.java @@ -25,6 +25,7 @@ scanBasePackages = { "org.zowe.apiml.filter", "org.zowe.apiml.gateway", + "org.zowe.apiml.product.eureka.web", "org.zowe.apiml.product.web", "org.zowe.apiml.product.gateway", "org.zowe.apiml.product.version", diff --git a/apiml/src/main/java/org/zowe/apiml/ApimlEurekaRegistryVersionEndpoint.java b/apiml/src/main/java/org/zowe/apiml/ApimlEurekaRegistryVersionEndpoint.java new file mode 100644 index 0000000000..f48dda8be2 --- /dev/null +++ b/apiml/src/main/java/org/zowe/apiml/ApimlEurekaRegistryVersionEndpoint.java @@ -0,0 +1,59 @@ +/* + * This program and the accompanying materials are made available under the terms of the + * Eclipse Public License v2.0 which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-v20.html + * + * SPDX-License-Identifier: EPL-2.0 + * + * Copyright Contributors to the Zowe Project. + */ + +package org.zowe.apiml; + +import com.netflix.eureka.registry.PeerAwareInstanceRegistry; +import lombok.Builder; +import lombok.RequiredArgsConstructor; +import lombok.Value; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.actuate.endpoint.annotation.Endpoint; +import org.springframework.boot.actuate.endpoint.annotation.ReadOperation; +import org.springframework.stereotype.Component; + +import java.util.regex.Pattern; + +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; + +@Component +@RequiredArgsConstructor +@Endpoint(id = "eurekaversion") +@Slf4j +public class ApimlEurekaRegistryVersionEndpoint { + + private static final Pattern VERSION_PATTERN = Pattern.compile("^.*_([0-9]+)_.*$"); + + private final PeerAwareInstanceRegistry peerAwareInstanceRegistry; + + @ReadOperation(produces = APPLICATION_JSON) + public VersionDto status() { + long version = -1; + var hashCode = peerAwareInstanceRegistry.getApplications().getAppsHashCode(); + var matcher = VERSION_PATTERN.matcher(hashCode); + if (matcher.matches()) { + version = Long.parseLong(matcher.group(1)); + log.debug("New Eureka registry version: {}", version); + } else { + log.debug("Unexpected Eureka registry hashCode: {}", hashCode); + } + return VersionDto.builder().version(version).build(); + } + + @Builder + @Value + static class VersionDto { + + private Long version; + + } + + +} diff --git a/apiml/src/main/java/org/zowe/apiml/EurekaConfiguration.java b/apiml/src/main/java/org/zowe/apiml/EurekaConfiguration.java index cda82fc122..1b0b65bf43 100644 --- a/apiml/src/main/java/org/zowe/apiml/EurekaConfiguration.java +++ b/apiml/src/main/java/org/zowe/apiml/EurekaConfiguration.java @@ -72,7 +72,6 @@ import org.springframework.boot.web.servlet.FilterRegistrationBean; import org.springframework.cloud.client.actuator.HasFeatures; import org.springframework.cloud.netflix.eureka.EurekaConstants; -import org.springframework.cloud.netflix.eureka.EurekaInstanceConfigBean; import org.springframework.cloud.netflix.eureka.server.*; import org.springframework.context.annotation.*; import org.springframework.core.Ordered; @@ -88,8 +87,6 @@ import java.util.Collections; import java.util.HashSet; import java.util.Set; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.regex.Pattern; /** @@ -208,32 +205,6 @@ PeerReplicationResource peerReplicationResource() { return new PeerReplicationResource(); } - @Bean - PeerAwareInstanceRegistry peerAwareInstanceRegistry(ServerCodecs serverCodecs, - EurekaServerHttpClientFactory eurekaServerHttpClientFactory, - EurekaInstanceConfigBean eurekaInstanceConfigBean) { - if (eurekaInstanceConfigBean.isAsyncClientInitialization()) { - if (log.isDebugEnabled()) { - log.debug("Initializing client asynchronously..."); - } - - ExecutorService executorService = Executors.newSingleThreadExecutor(); - executorService.submit(() -> { - this.eurekaClient.getApplications(); - if (log.isDebugEnabled()) { - log.debug("Asynchronous client initialization done."); - } - }); - } else { - this.eurekaClient.getApplications(); // force initialization - } - - return new InstanceRegistry(this.eurekaServerConfig, this.eurekaClientConfig, serverCodecs, this.eurekaClient, - eurekaServerHttpClientFactory, - this.instanceRegistryProperties.getExpectedNumberOfClientsSendingRenews(), - this.instanceRegistryProperties.getDefaultOpenForTrafficCount()); - } - @Bean @ConditionalOnMissingBean EurekaServerContext eurekaServerContext(ServerCodecs serverCodecs, PeerAwareInstanceRegistry registry, diff --git a/apiml/src/main/java/org/zowe/apiml/EurekaHealthIndicatorApiml.java b/apiml/src/main/java/org/zowe/apiml/EurekaHealthIndicatorApiml.java new file mode 100644 index 0000000000..faa7861533 --- /dev/null +++ b/apiml/src/main/java/org/zowe/apiml/EurekaHealthIndicatorApiml.java @@ -0,0 +1,65 @@ +/* + * This program and the accompanying materials are made available under the terms of the + * Eclipse Public License v2.0 which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-v20.html + * + * SPDX-License-Identifier: EPL-2.0 + * + * Copyright Contributors to the Zowe Project. + */ + +package org.zowe.apiml; + +import org.springframework.boot.actuate.health.Health; +import org.springframework.boot.actuate.health.Status; +import org.springframework.cloud.client.discovery.DiscoveryClient; +import org.springframework.cloud.netflix.eureka.EurekaHealthIndicator; +import org.springframework.context.annotation.Primary; +import org.springframework.stereotype.Component; + +import java.util.Map; +import java.util.stream.Collectors; + +/** + * This class is replacement of org.springframework.cloud.netflix.eureka.EurekaHealthIndicator, because it is using + * a different Eureka client + */ +@Primary +@Component("eurekaHealthIndicator") +public class EurekaHealthIndicatorApiml extends EurekaHealthIndicator { + + private final DiscoveryClient discoveryClient; + + public EurekaHealthIndicatorApiml(DiscoveryClient discoveryClient) { + super(null, null, null); + this.discoveryClient = discoveryClient; + } + + @Override + public String getName() { + return "eureka"; + } + + @Override + public Health health() { + Health.Builder builder = Health.unknown(); + Status status = getStatus(builder); + return builder.status(status).withDetail("applications", getApplications()).build(); + } + + private Status getStatus(Health.Builder builder) { + if (discoveryClient.getServices().isEmpty()) { + return new Status("UP", "Eureka registry is not available at the moment"); + } + return new Status("UP", "Eureka is ready to use"); + } + + private Map getApplications() { + return discoveryClient.getServices().stream() + .collect(Collectors.toMap( + String::toLowerCase, + serviceId -> discoveryClient.getInstances(serviceId).size()) + ); + } + +} diff --git a/apiml/src/main/java/org/zowe/apiml/ModulithConfig.java b/apiml/src/main/java/org/zowe/apiml/ModulithConfig.java index 5997e4708c..6fcafcba26 100644 --- a/apiml/src/main/java/org/zowe/apiml/ModulithConfig.java +++ b/apiml/src/main/java/org/zowe/apiml/ModulithConfig.java @@ -30,7 +30,7 @@ import org.apache.catalina.Context; import org.apache.catalina.Host; import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.InitializingBean; +import org.apache.commons.lang3.Strings; import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.context.event.ApplicationReadyEvent; @@ -99,7 +99,7 @@ name = "ClientCert", description = "Client certificate X509" ) -public class ModulithConfig implements InitializingBean { +public class ModulithConfig { private final ApplicationContext applicationContext; private final Map instances = new HashMap<>(); @@ -121,7 +121,10 @@ public class ModulithConfig implements InitializingBean { private String ipAddress; @Value("${apiml.service.port:10010}") - private int port; + private int gatewayPort; + + @Value("${apiml.internal-discovery.port:10011}") + private int discoveryPort; @Value("${server.attlsServer.enabled:false}") private boolean isServerAttlsEnabled; @@ -136,7 +139,13 @@ ApplicationInfo applicationInfo() { .authServiceId(CoreService.GATEWAY.getServiceId()).build(); } + private int getPort(String serviceId) { + return Strings.CI.equals(serviceId, CoreService.DISCOVERY.getServiceId()) ? discoveryPort : gatewayPort; + } + private InstanceInfo getInstanceInfo(String serviceId) { + int port = getPort(serviceId); + var leaseInfo = LeaseInfo.Builder.newBuilder() .setDurationInSecs(90) .setRegistrationTimestamp(System.currentTimeMillis()) @@ -196,11 +205,6 @@ static ApimlInstanceRegistry getRegistry() { .orElse(null); } - @Override - public void afterPropertiesSet() throws Exception { - createLocalInstances(); - } - void createLocalInstances() { instances.put(CoreService.GATEWAY.getServiceId(), getInstanceInfo(CoreService.GATEWAY.getServiceId())); instances.put(CoreService.DISCOVERY.getServiceId(), getInstanceInfo(CoreService.DISCOVERY.getServiceId())); @@ -214,6 +218,8 @@ void createLocalInstances() { @EventListener(ApplicationReadyEvent.class) public void onApplicationStart() { + createLocalInstances(); + log.info("Initialize timer for static services peer-replicated heartbeats"); eventPublisher.publishEvent(new ApiCatalogServiceAvailableEvent(new Object())); diff --git a/apiml/src/main/resources/application.yml b/apiml/src/main/resources/application.yml index aed6b48525..cd5ee81a40 100644 --- a/apiml/src/main/resources/application.yml +++ b/apiml/src/main/resources/application.yml @@ -16,7 +16,7 @@ eureka: server: max-threads-for-peer-replication: 6 useReadOnlyResponseCache: false - peer-node-read-timeout-ms: 8000 + peer-node-read-timeout-ms: 15000 spring: cloud: gateway: diff --git a/caching-service-package/src/main/resources/bin/start.sh b/caching-service-package/src/main/resources/bin/start.sh index 1afab7214a..5c00b99fe8 100755 --- a/caching-service-package/src/main/resources/bin/start.sh +++ b/caching-service-package/src/main/resources/bin/start.sh @@ -288,6 +288,7 @@ _BPX_JOBNAME=${ZWE_zowe_job_prefix}${CACHING_CODE} ${JAVA_BIN_DIR}java \ -Dcaching.storage.vsam.name=${VSAM_FILE_NAME} \ -Djgroups.bind.address=${ZWE_configs_storage_infinispan_jgroups_host:-${ZWE_haInstance_hostname:-localhost}} \ -Djgroups.bind.port=${ZWE_configs_storage_infinispan_jgroups_port:-7600} \ + -Djgroups.keyExchange.bind.address=${ZWE_configs_storage_infinispan_jgroups_keyExchange_host:-${ZWE_haInstance_hostname:-localhost}} \ -Djgroups.keyExchange.port=${ZWE_configs_storage_infinispan_jgroups_keyExchange_port:-7601} \ -Djgroups.tcp.diag.enabled=${ZWE_configs_storage_infinispan_jgroups_tcp_diag_enabled:-false} \ -Dcaching.storage.infinispan.initialHosts=${ZWE_configs_storage_infinispan_initialHosts:-localhost[7600]} \ diff --git a/caching-service/src/main/java/org/zowe/apiml/caching/api/CachingEurekaRegistryVersionEndpoint.java b/caching-service/src/main/java/org/zowe/apiml/caching/api/CachingEurekaRegistryVersionEndpoint.java new file mode 100644 index 0000000000..3921e917a7 --- /dev/null +++ b/caching-service/src/main/java/org/zowe/apiml/caching/api/CachingEurekaRegistryVersionEndpoint.java @@ -0,0 +1,63 @@ +/* + * This program and the accompanying materials are made available under the terms of the + * Eclipse Public License v2.0 which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-v20.html + * + * SPDX-License-Identifier: EPL-2.0 + * + * Copyright Contributors to the Zowe Project. + */ + +package org.zowe.apiml.caching.api; + +import lombok.Builder; +import lombok.RequiredArgsConstructor; +import lombok.Value; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.actuate.endpoint.annotation.Endpoint; +import org.springframework.boot.actuate.endpoint.annotation.ReadOperation; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.stereotype.Component; +import org.zowe.apiml.eurekaservice.client.ApiMediationClient; + +import java.util.regex.Pattern; + +import static io.netty.handler.codec.http.HttpHeaders.Values.APPLICATION_JSON; + +@Component +@RequiredArgsConstructor +@Endpoint(id = "eurekaversion") +@ConditionalOnMissingBean(name = "modulithConfig") +@Slf4j +public class CachingEurekaRegistryVersionEndpoint { + + private static final Pattern VERSION_PATTERN = Pattern.compile("^.*_([0-9]+)_.*$"); + + private final ApiMediationClient apiMediationClient; + + @ReadOperation(produces = APPLICATION_JSON) + public VersionDto status() { + long version = -1; + var eurekaClient = apiMediationClient.getEurekaClient(); + if (eurekaClient != null) { + var hashCode = eurekaClient.getApplications().getAppsHashCode(); + var matcher = VERSION_PATTERN.matcher(hashCode); + if (matcher.matches()) { + version = Long.parseLong(matcher.group(1)); + log.debug("New Eureka registry version: {}", version); + } else { + log.debug("Unexpected Eureka registry hashCode: {}", hashCode); + } + } + return VersionDto.builder().version(version).build(); + } + + @Builder + @Value + static class VersionDto { + + private Long version; + + } + +} diff --git a/caching-service/src/main/java/org/zowe/apiml/caching/config/SpringSecurityConfig.java b/caching-service/src/main/java/org/zowe/apiml/caching/config/SpringSecurityConfig.java index 1a17b8a1dd..f4b72ed80c 100644 --- a/caching-service/src/main/java/org/zowe/apiml/caching/config/SpringSecurityConfig.java +++ b/caching-service/src/main/java/org/zowe/apiml/caching/config/SpringSecurityConfig.java @@ -47,6 +47,7 @@ public SecurityWebFilterChain securityWebFilterChain(ServerHttpSecurity http) { var antMatchersToIgnore = new ArrayList(); antMatchersToIgnore.add("/cachingservice/application/info"); + antMatchersToIgnore.add("/cachingservice/application/eurekaversion"); antMatchersToIgnore.add("/cachingservice/v3/api-docs"); if (!isHealthEndpointProtected) { antMatchersToIgnore.add("/cachingservice/application/health"); diff --git a/caching-service/src/main/java/org/zowe/apiml/caching/health/ApimlCachesEndpoint.java b/caching-service/src/main/java/org/zowe/apiml/caching/health/ApimlCachesEndpoint.java new file mode 100644 index 0000000000..fca8676047 --- /dev/null +++ b/caching-service/src/main/java/org/zowe/apiml/caching/health/ApimlCachesEndpoint.java @@ -0,0 +1,42 @@ +/* + * This program and the accompanying materials are made available under the terms of the + * Eclipse Public License v2.0 which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-v20.html + * + * SPDX-License-Identifier: EPL-2.0 + * + * Copyright Contributors to the Zowe Project. + */ + +package org.zowe.apiml.caching.health; + +import lombok.experimental.Delegate; +import org.springframework.boot.actuate.cache.CachesEndpoint; +import org.springframework.boot.actuate.endpoint.annotation.Endpoint; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.cache.CacheManager; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; + +import java.util.Collections; +import java.util.Map; + +@Component +@Endpoint(id = "caches") +public class ApimlCachesEndpoint extends CachesEndpoint { + + @Delegate + private CachesEndpoint cachesEndpoint = new CachesEndpoint(Collections.emptyMap()); + + public ApimlCachesEndpoint() { + super(Collections.emptyMap()); + } + + @EventListener + public void onApplicationEvent(ApplicationReadyEvent event) { + var context = event.getApplicationContext(); + Map current = context.getBeansOfType(CacheManager.class); + this.cachesEndpoint = new CachesEndpoint(current); + } + +} diff --git a/caching-service/src/main/java/org/zowe/apiml/caching/health/CachesHealthIndicator.java b/caching-service/src/main/java/org/zowe/apiml/caching/health/CachesHealthIndicator.java index 652771773f..121da5ebc0 100644 --- a/caching-service/src/main/java/org/zowe/apiml/caching/health/CachesHealthIndicator.java +++ b/caching-service/src/main/java/org/zowe/apiml/caching/health/CachesHealthIndicator.java @@ -10,46 +10,78 @@ package org.zowe.apiml.caching.health; -import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.StringUtils; +import org.infinispan.remoting.transport.Address; import org.infinispan.spring.embedded.provider.SpringEmbeddedCacheManager; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.actuate.health.AbstractHealthIndicator; import org.springframework.boot.actuate.health.Health; import org.springframework.boot.actuate.health.Status; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.cache.CacheManager; +import org.springframework.context.event.EventListener; import org.springframework.stereotype.Component; import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; @Component -@RequiredArgsConstructor @ConditionalOnProperty(name = "caching.storage.mode", havingValue = "infinispan") public class CachesHealthIndicator extends AbstractHealthIndicator { - private final CacheManager cacheManager; + @Value("${caching.storage.infinispan.initialHosts:}") + private String initialHosts; + + private final AtomicReference cacheManager = new AtomicReference<>(); @Override protected void doHealthCheck(Health.Builder builder) { + var cm = cacheManager.get(); + if (cm == null) { + builder.unknown(); + return; + } + boolean health = true; - if (cacheManager instanceof SpringEmbeddedCacheManager springEmbeddedCacheManager) { + if (cm instanceof SpringEmbeddedCacheManager springEmbeddedCacheManager) { var nativeCacheManager = springEmbeddedCacheManager.getNativeCacheManager(); - var status = nativeCacheManager.getStatus(); - var infinispan = new HashMap(); + + var status = nativeCacheManager.getStatus(); infinispan.put("status", status); health &= status.allowInvocations(); var caches = new HashMap(); - for (String cacheName : cacheManager.getCacheNames()) { + for (String cacheName : cm.getCacheNames()) { var cacheStatus = nativeCacheManager.getCache(cacheName).getStatus(); caches.put(cacheName, cacheStatus); health &= cacheStatus.allowInvocations(); } infinispan.put("caches", caches); + + var initialHostsArray = StringUtils.split(initialHosts, ","); + boolean allMembers = initialHostsArray.length <= nativeCacheManager.getMembers().size(); + var cluster = Map.of( + "status", allMembers ? Status.UP : Status.DOWN, + "address", nativeCacheManager.getAddress().toString(), + "initialHosts", initialHostsArray, + "members", nativeCacheManager.getMembers().stream().map(Address::toString).toList() + ); + infinispan.put("cluster", cluster); + health &= allMembers; + builder.withDetail("infinispan", infinispan); } builder.status(health ? Status.UP : Status.DOWN); } + @EventListener + public void onApplicationEvent(ApplicationReadyEvent event) { + var context = event.getApplicationContext(); + cacheManager.set(context.getBean(CacheManager.class)); + } + } diff --git a/caching-service/src/main/java/org/zowe/apiml/caching/health/CachingHealthIndicator.java b/caching-service/src/main/java/org/zowe/apiml/caching/health/CachingHealthIndicator.java index 8ee0af1d4f..7df00c1c7e 100644 --- a/caching-service/src/main/java/org/zowe/apiml/caching/health/CachingHealthIndicator.java +++ b/caching-service/src/main/java/org/zowe/apiml/caching/health/CachingHealthIndicator.java @@ -41,6 +41,8 @@ public class CachingHealthIndicator extends AbstractHealthIndicator implements A @Override protected void doHealthCheck(Health.Builder builder) { + builder.up(); + boolean gatewayUp = Optional.ofNullable(apiMediationClient.getEurekaClient()) .map(eurekaClient -> eurekaClient.getApplication(CoreService.GATEWAY.getServiceId())) .map(Application::getInstances) diff --git a/caching-service/src/main/java/org/zowe/apiml/caching/service/infinispan/config/InfinispanConfig.java b/caching-service/src/main/java/org/zowe/apiml/caching/service/infinispan/config/InfinispanConfig.java index ec75a4cfa1..7f22a5c32b 100644 --- a/caching-service/src/main/java/org/zowe/apiml/caching/service/infinispan/config/InfinispanConfig.java +++ b/caching-service/src/main/java/org/zowe/apiml/caching/service/infinispan/config/InfinispanConfig.java @@ -13,7 +13,7 @@ import jakarta.annotation.PostConstruct; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.infinispan.commons.api.CacheContainerAdmin; +import org.apache.commons.lang3.Strings; import org.infinispan.commons.dataconversion.MediaType; import org.infinispan.configuration.cache.CacheMode; import org.infinispan.configuration.cache.ConfigurationBuilder; @@ -22,6 +22,7 @@ import org.infinispan.lock.EmbeddedClusteredLockManagerFactory; import org.infinispan.lock.api.ClusteredLock; import org.infinispan.lock.api.ClusteredLockManager; +import org.infinispan.manager.CacheContainer; import org.infinispan.manager.DefaultCacheManager; import org.infinispan.partitionhandling.AvailabilityException; import org.springframework.beans.factory.InitializingBean; @@ -40,9 +41,10 @@ import java.io.IOException; import java.io.InputStream; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Arrays; -import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; +import java.util.regex.Pattern; import static org.zowe.apiml.security.SecurityUtils.formatKeyringUrl; import static org.zowe.apiml.security.SecurityUtils.isKeyring; @@ -54,14 +56,13 @@ public class InfinispanConfig implements InitializingBean { private static final String KEYRING_PASSWORD = "password"; - private static final String SERVER_SSL_KEY_STORE_TYPE = "server.ssl.keyStoreType"; - private static final String SERVER_SSL_KEY_STORE = "server.ssl.keyStore"; - private static final String SERVER_SSL_KEY_STORE_PASSWORD = "server.ssl.keyStorePassword"; private static final String ZWE_HAINSTANCE_ID = "ZWE_haInstance_id"; private static final String LOCK_ZOWE_INVALIDATED = "zoweInvalidatedTokenLock"; - private static final String CACHE_ZOWE = "zoweCache"; - private static final String CACHE_ZOWE_INVALIDATED_TOKEN = "zoweInvalidatedTokenCache"; + public static final String CACHE_ZOWE = "zoweCache"; + public static final String CACHE_ZOWE_INVALIDATED_TOKEN = "zoweInvalidatedTokenCache"; + + private static Pattern MEMBER = Pattern.compile("^\\s*([^\\[\\s]+)\\s*\\[\\s*([0-9]+)\\s*\\]\\s*$"); @Value("${caching.storage.infinispan.initialHosts}") private String initialHosts; @@ -75,11 +76,23 @@ public class InfinispanConfig implements InitializingBean { @Value("${server.ssl.keyStorePassword}") private String keyStorePass; + @Value("${server.ssl.trustStoreType}") + private String trustStoreType; + + @Value("${server.ssl.trustStore}") + private String trustStore; + + @Value("${server.ssl.trustStorePassword}") + private String trustStorePass; + @Value("${jgroups.bind.port}") private String port; - @Value("${jgroups.bind.address}") - private String address; + @Value("${jgroups.bind.address:0.0.0.0}") + private String jgroupAddress; + + @Value("${jgroups.keyExchange.address:0.0.0.0}") + private String exchangeAddress; @Value("${jgroups.keyExchange.port:7601}") private String keyExchangePort; @@ -90,6 +103,9 @@ public class InfinispanConfig implements InitializingBean { @Value("${server.attlsServer.enabled:false}") private boolean isServerAttlsEnabled; + @Value("${apiml.service.hostname:localhost}") + private String hostname; + private AtomicReference zoweInvalidatedTokenLock = new AtomicReference<>(); @Override @@ -103,6 +119,10 @@ void updateKeyring() { keyStore = formatKeyringUrl(keyStore); if (StringUtils.isBlank(keyStorePass)) keyStorePass = KEYRING_PASSWORD; } + if (isKeyring(trustStore)) { + trustStore = formatKeyringUrl(trustStore); + if (StringUtils.isBlank(trustStorePass)) trustStorePass = KEYRING_PASSWORD; + } } static String getRootFolder() { @@ -120,26 +140,8 @@ static String getRootFolder() { } } - @Bean(destroyMethod = "stop") - synchronized DefaultCacheManager cacheManager(ResourceLoader resourceLoader) { - System.setProperty("jgroups.tcpping.initial_hosts", initialHosts); - System.setProperty("jgroups.bind.port", port); - System.setProperty("jgroups.bind.address", address); - System.setProperty("jgroups.keyExchange.port", keyExchangePort); - System.setProperty("jgroups.tcp.diag.enabled", String.valueOf(Boolean.parseBoolean(tcpDiagEnabled))); - - var oldKeyStoreType = Optional.ofNullable(System.getProperty(SERVER_SSL_KEY_STORE_TYPE)); - var oldKeyStore = Optional.ofNullable(System.getProperty(SERVER_SSL_KEY_STORE)); - var oldKeyStorePassword = Optional.ofNullable(System.getProperty(SERVER_SSL_KEY_STORE_PASSWORD)); - - if (!isServerAttlsEnabled) { - System.setProperty(SERVER_SSL_KEY_STORE_TYPE, keyStoreType); - System.setProperty(SERVER_SSL_KEY_STORE, keyStore); - System.setProperty(SERVER_SSL_KEY_STORE_PASSWORD, keyStorePass); - } - + private ConfigurationBuilderHolder getCacheManagerConfig(ResourceLoader resourceLoader) { ConfigurationBuilderHolder holder; - var infinispanConfigFile = isServerAttlsEnabled ? "infinispan-attls.xml" : "infinispan.xml"; try (InputStream configurationStream = resourceLoader.getResource("classpath:" + infinispanConfigFile).getInputStream()) { holder = new ParserRegistry().parse(configurationStream, MediaType.APPLICATION_XML); @@ -150,29 +152,72 @@ synchronized DefaultCacheManager cacheManager(ResourceLoader resourceLoader) { holder.newConfigurationBuilder("default") .persistence() .addSoftIndexFileStore() - .clustering().cacheMode(CacheMode.DIST_SYNC); - - DefaultCacheManager cacheManager = new DefaultCacheManager(holder, true); + .clustering().cacheMode(CacheMode.REPL_SYNC); + holder.getGlobalConfigurationBuilder().defaultCacheName("default"); + return holder; + } + private ConfigurationBuilder getCacheConfig() { ConfigurationBuilder builder = new ConfigurationBuilder(); builder .encoding().mediaType(MediaType.APPLICATION_JBOSS_MARSHALLING_TYPE) .persistence().addSoftIndexFileStore().clustering() - .clustering().cacheMode(CacheMode.DIST_SYNC); + .clustering().cacheMode(CacheMode.REPL_SYNC); + return builder; + } - var caches = Arrays.asList(CACHE_ZOWE, CACHE_ZOWE_INVALIDATED_TOKEN, "zosmfAuthenticationEndpoint", "invalidatedJwtTokens", "validationJwtToken", "zosmfInfo", "zosmfJwtEndpoint", "trustedCertificates", "parseOIDCToken", "validationOIDCToken"); - caches.forEach(cacheName -> cacheManager.administration() - .withFlags(CacheContainerAdmin.AdminFlag.VOLATILE) - .getOrCreateCache(cacheName, builder.build())); + private boolean isMe(String member) { + var matcher = MEMBER.matcher(member); + if (matcher.matches()) { + var hostname = matcher.group(1); + var port = matcher.group(2); + return + Strings.CI.equals(hostname, this.hostname) && + Strings.CS.equals(port, this.port); + } + return false; + } - oldKeyStoreType.ifPresent(kst -> System.setProperty(SERVER_SSL_KEY_STORE_TYPE, kst)); - oldKeyStore.ifPresent(ks -> System.setProperty(SERVER_SSL_KEY_STORE, ks)); - oldKeyStorePassword.ifPresent(p -> System.setProperty(SERVER_SSL_KEY_STORE_PASSWORD, p)); + private String getOrderedInitialHosts() { + var local = new ArrayList(); + var response = new ArrayList(); + for (String member : StringUtils.split(initialHosts, ",")) { + if (isMe(member)) { + local.add(member); + } else { + response.add(member); + } + } + response.addAll(local); + return StringUtils.join(response, ","); + } + + @Bean(destroyMethod = "stop") + synchronized LazyCacheManager cacheManager(ResourceLoader resourceLoader) { + var orderedInitialHosts = getOrderedInitialHosts(); + log.debug("Ordered initial hosts list: {}", orderedInitialHosts); + + System.setProperty("jgroups.tcpping.initial_hosts", getOrderedInitialHosts()); + System.setProperty("jgroups.bind.port", port); + System.setProperty("jgroups.bind.address", jgroupAddress); + System.setProperty("jgroups.keyExchange.port", keyExchangePort); + System.setProperty("jgroups.keyExchange.address", exchangeAddress); + System.setProperty("jgroups.tcp.diag.enabled", String.valueOf(Boolean.parseBoolean(tcpDiagEnabled))); + + System.setProperty("infinispan.ssl.keyStoreType", keyStoreType); + System.setProperty("infinispan.ssl.keyStore", keyStore); + System.setProperty("infinispan.ssl.keyStorePassword", keyStorePass); + + System.setProperty("infinispan.ssl.trustStoreType", keyStoreType); + System.setProperty("infinispan.ssl.trustStore", keyStore); + System.setProperty("infinispan.ssl.trustStorePassword", keyStorePass); + + var caches = Arrays.asList(CACHE_ZOWE, CACHE_ZOWE_INVALIDATED_TOKEN, "zosmfAuthenticationEndpoint", "invalidatedJwtTokens", "validationJwtToken", "zosmfInfo", "zosmfJwtEndpoint", "trustedCertificates", "parseOIDCToken", "validationOIDCToken"); - return cacheManager; + return new LazyCacheManager(getCacheManagerConfig(resourceLoader), getCacheConfig(), caches); } - private ClusteredLock lock(DefaultCacheManager cacheManager) { + private ClusteredLock lock(CacheContainer cacheManager) { ClusteredLock lock = zoweInvalidatedTokenLock.get(); if (lock != null) { return lock; @@ -181,8 +226,8 @@ private ClusteredLock lock(DefaultCacheManager cacheManager) { try { synchronized (zoweInvalidatedTokenLock) { lock = zoweInvalidatedTokenLock.get(); - if (lock == null) { - ClusteredLockManager clm = EmbeddedClusteredLockManagerFactory.from(cacheManager); + if (lock == null && cacheManager instanceof LazyCacheManager lazyCacheManager) { + ClusteredLockManager clm = EmbeddedClusteredLockManagerFactory.from(lazyCacheManager.getOriginal()); // it can throw AvailabilityException clm.defineLock(LOCK_ZOWE_INVALIDATED); lock = clm.get(LOCK_ZOWE_INVALIDATED); @@ -199,8 +244,7 @@ private ClusteredLock lock(DefaultCacheManager cacheManager) { @Bean public Storage storage(DefaultCacheManager cacheManager) { return new InfinispanStorage( - cacheManager.getCache(CACHE_ZOWE), - cacheManager.getCache(CACHE_ZOWE_INVALIDATED_TOKEN), + cacheManager, () -> lock(cacheManager) ); } diff --git a/caching-service/src/main/java/org/zowe/apiml/caching/service/infinispan/config/LazyCacheManager.java b/caching-service/src/main/java/org/zowe/apiml/caching/service/infinispan/config/LazyCacheManager.java new file mode 100644 index 0000000000..5cb5c13652 --- /dev/null +++ b/caching-service/src/main/java/org/zowe/apiml/caching/service/infinispan/config/LazyCacheManager.java @@ -0,0 +1,305 @@ +/* + * This program and the accompanying materials are made available under the terms of the + * Eclipse Public License v2.0 which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-v20.html + * + * SPDX-License-Identifier: EPL-2.0 + * + * Copyright Contributors to the Zowe Project. + */ + +package org.zowe.apiml.caching.service.infinispan.config; + +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.infinispan.Cache; +import org.infinispan.commons.api.CacheContainerAdmin; +import org.infinispan.commons.configuration.ClassAllowList; +import org.infinispan.configuration.cache.Configuration; +import org.infinispan.configuration.cache.ConfigurationBuilder; +import org.infinispan.configuration.global.GlobalConfiguration; +import org.infinispan.configuration.parsing.ConfigurationBuilderHolder; +import org.infinispan.health.Health; +import org.infinispan.lifecycle.ComponentStatus; +import org.infinispan.manager.CacheContainer; +import org.infinispan.manager.CacheManagerInfo; +import org.infinispan.manager.DefaultCacheManager; +import org.infinispan.manager.EmbeddedCacheManager; +import org.infinispan.remoting.transport.Address; +import org.infinispan.stats.CacheContainerStats; + +import javax.security.auth.Subject; +import java.io.IOException; +import java.util.*; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.atomic.AtomicReference; + +@Slf4j +public class LazyCacheManager extends DefaultCacheManager { + + private static final int RETRY = 2; + + private final AtomicReference cacheManager = new AtomicReference<>(); + private final CacheInitializer cacheInitializer; + + public LazyCacheManager( + ConfigurationBuilderHolder cacheManagerConfig, + ConfigurationBuilder cacheConfig, + Collection cacheNames + ) { + super(false); + cacheInitializer = new CacheInitializer(cacheManagerConfig, cacheConfig, new ArrayList<>(cacheNames)); + } + + private DefaultCacheManager getCacheManager() { + var container = cacheManager.get(); + if (container == null) { + synchronized (this) { + container = cacheManager.get(); + if (container == null) { + container = cacheInitializer.getDefaultCacheManager(); + } + } + if (container == null) { + throw new IllegalStateException("Cache container is not initialized yet"); + } + } + return container; + } + + @Override + public Configuration defineConfiguration(String cacheName, Configuration configuration) { + return getCacheManager().defineConfiguration(cacheName, configuration); + } + + @Override + public Configuration defineConfiguration(String cacheName, String templateCacheName, Configuration configurationOverride) { + return getCacheManager().defineConfiguration(cacheName, templateCacheName, configurationOverride); + } + + @Override + public void undefineConfiguration(String configurationName) { + getCacheManager().undefineConfiguration(configurationName); + } + + @Override + public String getClusterName() { + return getCacheManager().getClusterName(); + } + + @Override + public List
getMembers() { + return getCacheManager().getMembers(); + } + + @Override + public Address getAddress() { + return getCacheManager().getAddress(); + } + + @Override + public Address getCoordinator() { + return getCacheManager().getCoordinator(); + } + + @Override + public boolean isCoordinator() { + return getCacheManager().isCoordinator(); + } + + @Override + public ComponentStatus getStatus() { + return getCacheManager().getStatus(); + } + + @Override + public GlobalConfiguration getCacheManagerConfiguration() { + return getCacheManager().getCacheManagerConfiguration(); + } + + @Override + public Configuration getCacheConfiguration(String name) { + return getCacheManager().getCacheConfiguration(name); + } + + @Override + public Configuration getDefaultCacheConfiguration() { + return getCacheManager().getDefaultCacheConfiguration(); + } + + @Override + public Set getAccessibleCacheNames() { + return getCacheManager().getAccessibleCacheNames(); + } + + @Override + public boolean isRunning(String cacheName) { + return getCacheManager().isRunning(cacheName); + } + + @Override + public boolean isDefaultRunning() { + return getCacheManager().isDefaultRunning(); + } + + @Override + public boolean cacheExists(String cacheName) { + return getCacheManager().cacheExists(cacheName); + } + + @Override + public boolean cacheConfigurationExists(String name) { + return getCacheManager().cacheConfigurationExists(name); + } + + @Override + public Cache getCache() { + return getCacheManager().getCache(); + } + + @Override + public Cache getCache(String cacheName) { + return getCacheManager().getCache(cacheName); + } + + @Override + public Cache createCache(String name, Configuration configuration) { + return getCacheManager().createCache(name, configuration); + } + + @Override + public Cache getCache(String cacheName, boolean createIfAbsent) { + return getCacheManager().getCache(cacheName, createIfAbsent); + } + + @Override + public EmbeddedCacheManager startCaches(String... cacheNames) { + return getCacheManager().startCaches(cacheNames); + } + + @Override + public void addCacheDependency(String from, String to) { + getCacheManager().addCacheDependency(from, to); + } + + @Override + public CacheContainerStats getStats() { + return getCacheManager().getStats(); + } + + @Override + public Health getHealth() { + return getCacheManager().getHealth(); + } + + @Override + public CacheManagerInfo getCacheManagerInfo() { + return getCacheManager().getCacheManagerInfo(); + } + + @Override + public ClassAllowList getClassWhiteList() { + return getCacheManager().getClassWhiteList(); + } + + @Override + public ClassAllowList getClassAllowList() { + return getCacheManager().getClassAllowList(); + } + + @Override + public Subject getSubject() { + return getCacheManager().getSubject(); + } + + @Override + public EmbeddedCacheManager withSubject(Subject subject) { + return getCacheManager().withSubject(subject); + } + + @Override + public Set getCacheNames() { + return getCacheManager().getCacheNames(); + } + + @Override + public void start() { + getCacheManager().start(); + } + + @Override + public void stop() { + getCacheManager().stop(); + } + + public T getOriginal() { + return (T) getCacheManager(); + } + + @Override + public void close() throws IOException { + getCacheManager().close(); + } + + @Override + public CompletionStage addListenerAsync(Object listener) { + return getCacheManager().addListenerAsync(listener); + } + + @Override + public CompletionStage removeListenerAsync(Object listener) { + return getCacheManager().removeListenerAsync(listener); + } + + @RequiredArgsConstructor + class CacheInitializer { + + private DefaultCacheManager underInit; + + private final ConfigurationBuilderHolder cacheManagerConfig; + private final ConfigurationBuilder cacheConfig; + private final Collection cacheNames; + + public DefaultCacheManager getDefaultCacheManager() { + if (underInit == null) { + underInit = new DefaultCacheManager(cacheManagerConfig, true); + } + + for (int i = 0; i < 1 + RETRY; i++) { + if (createCaches()) { + break; + } + } + + if (cacheNames.isEmpty()) { + cacheManager.set(underInit); + } + + return underInit; + } + + private boolean createCaches() { + for (Iterator i = cacheNames.iterator(); i.hasNext();) { + String cacheName = i.next(); + if (createCache(cacheName)) { + i.remove(); + } + } + return cacheNames.isEmpty(); + } + + private boolean createCache(String cacheName) { + try { + underInit.administration() + .withFlags(CacheContainerAdmin.AdminFlag.VOLATILE) + .getOrCreateCache(cacheName, cacheConfig.build()); + return true; + } catch (Exception e) { + log.warn("Error during initialization of cache {}", cacheName, e); + return false; + } + } + + } + +} diff --git a/caching-service/src/main/java/org/zowe/apiml/caching/service/infinispan/storage/InfinispanStorage.java b/caching-service/src/main/java/org/zowe/apiml/caching/service/infinispan/storage/InfinispanStorage.java index fb987c7235..6661cbd0a9 100644 --- a/caching-service/src/main/java/org/zowe/apiml/caching/service/infinispan/storage/InfinispanStorage.java +++ b/caching-service/src/main/java/org/zowe/apiml/caching/service/infinispan/storage/InfinispanStorage.java @@ -13,8 +13,10 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.infinispan.lock.api.ClusteredLock; +import org.infinispan.manager.DefaultCacheManager; import org.zowe.apiml.cache.Storage; import org.zowe.apiml.cache.StorageException; import org.zowe.apiml.caching.model.KeyValue; @@ -31,35 +33,36 @@ import java.util.function.Supplier; import java.util.stream.Collectors; +import static org.zowe.apiml.caching.service.infinispan.config.InfinispanConfig.CACHE_ZOWE; +import static org.zowe.apiml.caching.service.infinispan.config.InfinispanConfig.CACHE_ZOWE_INVALIDATED_TOKEN; + @Slf4j +@RequiredArgsConstructor public class InfinispanStorage implements Storage { - - private final ConcurrentMap cache; - private final ConcurrentMap> tokenCache; - private final Supplier lockSupplier; private static final ObjectMapper objectMapper = new ObjectMapper(); - public InfinispanStorage( - ConcurrentMap cache, - ConcurrentMap> tokenCache, - Supplier lockSupplier - ) { - this.cache = cache; - this.tokenCache = tokenCache; - this.lockSupplier = lockSupplier; - } + private final DefaultCacheManager defaultCacheManager; + private final Supplier lockSupplier; static { objectMapper.registerModule(new JavaTimeModule()); } + private ConcurrentMap getCache() { + return defaultCacheManager.getCache(CACHE_ZOWE); + } + + private ConcurrentMap> getTokenCache() { + return defaultCacheManager.getCache(CACHE_ZOWE_INVALIDATED_TOKEN); + } + @Override public KeyValue create(String serviceId, KeyValue toCreate) { toCreate.setServiceId(serviceId); log.info("Writing record: {}|{}|{}", serviceId, toCreate.getKey(), toCreate.getValue()); - KeyValue serviceCache = cache.putIfAbsent(serviceId + toCreate.getKey(), toCreate); + KeyValue serviceCache = getCache().putIfAbsent(serviceId + toCreate.getKey(), toCreate); if (serviceCache != null) { throw new StorageException(Messages.DUPLICATE_KEY.getKey(), Messages.DUPLICATE_KEY.getStatus(), toCreate.getKey()); @@ -75,12 +78,12 @@ public KeyValue storeMapItem(String serviceId, String mapKey, KeyValue toCreate) try { String cacheKey = serviceId + mapKey; log.info("Storing the item into token cache: {} -> {}|{}", cacheKey, toCreate.getKey(), toCreate.getValue()); - Map tokenCacheItem = tokenCache.get(cacheKey); + Map tokenCacheItem = getTokenCache().get(cacheKey); if (tokenCacheItem == null) { tokenCacheItem = new HashMap<>(); } tokenCacheItem.put(toCreate.getKey(), toCreate.getValue()); - tokenCache.put(cacheKey, tokenCacheItem); + getTokenCache().put(cacheKey, tokenCacheItem); } finally { lock.unlock(); } @@ -93,7 +96,7 @@ public KeyValue storeMapItem(String serviceId, String mapKey, KeyValue toCreate) @Override public Map getAllMapItems(String serviceId, String mapKey) { log.info("Reading all records from token cache for service {} under the {} key.", serviceId, mapKey); - return tokenCache.get(serviceId + mapKey); + return getTokenCache().get(serviceId + mapKey); } @Override @@ -106,11 +109,11 @@ public Map> getAllMaps(String serviceId) { * It is difficult to support and also slower (see exchanging lambdas between nodes). */ Map> result = new HashMap<>(); - for (String key : tokenCache.keySet()) { + for (String key : getTokenCache().keySet()) { if (!key.startsWith(serviceId)) continue; String newKey = key.substring(serviceId.length()); - result.put(newKey, tokenCache.get(key)); + result.put(newKey, getTokenCache().get(key)); } return result; @@ -119,7 +122,7 @@ public Map> getAllMaps(String serviceId) { @Override public KeyValue read(String serviceId, String key) { log.info("Reading record for service {} under key {}", serviceId, key); - KeyValue serviceCache = cache.get(serviceId + key); + KeyValue serviceCache = getCache().get(serviceId + key); if (serviceCache != null) { return serviceCache; } else { @@ -131,7 +134,7 @@ public KeyValue read(String serviceId, String key) { public KeyValue update(String serviceId, KeyValue toUpdate) { toUpdate.setServiceId(serviceId); log.info("Updating record for service {} under key {}", serviceId, toUpdate); - KeyValue serviceCache = cache.put(serviceId + toUpdate.getKey(), toUpdate); + KeyValue serviceCache = getCache().put(serviceId + toUpdate.getKey(), toUpdate); if (serviceCache == null) { throw new StorageException(Messages.KEY_NOT_IN_CACHE.getKey(), Messages.KEY_NOT_IN_CACHE.getStatus(), toUpdate.getKey(), serviceId); } @@ -142,7 +145,7 @@ public KeyValue update(String serviceId, KeyValue toUpdate) { @Override public KeyValue delete(String serviceId, String toDelete) { log.info("Removing record for service {} under key {}", serviceId, toDelete); - KeyValue entry = cache.remove(serviceId + toDelete); + KeyValue entry = getCache().remove(serviceId + toDelete); if (entry != null) { return entry; } else { @@ -154,7 +157,7 @@ public KeyValue delete(String serviceId, String toDelete) { public Map readForService(String serviceId) { log.info("Reading all records for service {} ", serviceId); Map result = new HashMap<>(); - cache.forEach((key, value) -> { + getCache().forEach((key, value) -> { if (serviceId.equals(value.getServiceId())) { result.put(value.getKey(), value); } @@ -165,9 +168,9 @@ public Map readForService(String serviceId) { @Override public void deleteForService(String serviceId) { log.info("Removing all records for service {} ", serviceId); - cache.forEach((key, value) -> { + getCache().forEach((key, value) -> { if (value.getServiceId().equals(serviceId)) { - cache.remove(key); + getCache().remove(key); } }); } @@ -188,7 +191,7 @@ public void removeNonRelevantTokens(String serviceId, String mapKey) { } private void removeToken(String serviceId, String mapKey) { - Map map = tokenCache.get(serviceId + mapKey); + Map map = getTokenCache().get(serviceId + mapKey); if (map != null && !map.isEmpty()) { Map result = map.entrySet().stream().filter(entry -> { try { @@ -199,7 +202,7 @@ private void removeToken(String serviceId, String mapKey) { return true; } }).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - tokenCache.put(serviceId + mapKey, result); + getTokenCache().put(serviceId + mapKey, result); } } @@ -210,14 +213,14 @@ public void removeNonRelevantRules(String serviceId, String mapKey) { if (Boolean.TRUE.equals(r)) { try { long timestamp = System.currentTimeMillis(); - Map map = tokenCache.get(serviceId + mapKey); + Map map = getTokenCache().get(serviceId + mapKey); if (map != null && !map.isEmpty()) { Map result = map.entrySet().stream().filter(entry -> { long delta = timestamp - Long.parseLong(entry.getValue()); long deltaToDays = TimeUnit.MILLISECONDS.toDays(delta); return deltaToDays <= 90; }).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - tokenCache.put(serviceId + mapKey, result); + getTokenCache().put(serviceId + mapKey, result); } } finally { lock.unlock(); diff --git a/caching-service/src/main/resources/application.yml b/caching-service/src/main/resources/application.yml index b77595132f..41badcad5b 100644 --- a/caching-service/src/main/resources/application.yml +++ b/caching-service/src/main/resources/application.yml @@ -33,7 +33,6 @@ logging: org.zowe.apiml.caching: WARN org.springdoc: WARN org.springframework: WARN - org.springframework.boot: OFF org.infinispan: ERROR org.jgroups: ERROR com.netflix: WARN diff --git a/caching-service/src/main/resources/infinispan-attls.xml b/caching-service/src/main/resources/infinispan-attls.xml index 9353be28d2..444fa0a59a 100644 --- a/caching-service/src/main/resources/infinispan-attls.xml +++ b/caching-service/src/main/resources/infinispan-attls.xml @@ -10,7 +10,7 @@ - ) ReflectionTestUtils.getField(cachesHealthIndicator, "cacheManager")) + .set(mock(CacheManager.class)); var builder = mock(Health.Builder.class); cachesHealthIndicator.doHealthCheck(builder); verify(builder, never()).withDetail(any(), any()); @@ -49,7 +66,6 @@ void givenUnsupportedCacheManager_whenBuildHealth_thenNoDetailsAdded() { } - @Nested @ExtendWith(MockitoExtension.class) class SupportedCacheManager { @@ -88,7 +104,9 @@ void givenNonRunningCache_whenBuildHealth_thenItIsDown( doReturn(cache).when(nativeCacheManager).getCache(CACHE_1); doReturn(cacheStatus).when(cache).getStatus(); - var cachesHealthIndicator = new CachesHealthIndicator(cacheManager); + var cachesHealthIndicator = new CachesHealthIndicator(); + ((AtomicReference) ReflectionTestUtils.getField(cachesHealthIndicator, "cacheManager")) + .set(cacheManager); var builder = mock(Health.Builder.class); cachesHealthIndicator.doHealthCheck(builder); diff --git a/caching-service/src/test/java/org/zowe/apiml/caching/service/infinispan/storage/InfinispanStorageTest.java b/caching-service/src/test/java/org/zowe/apiml/caching/service/infinispan/storage/InfinispanStorageTest.java index 050ee042ca..a68142d260 100644 --- a/caching-service/src/test/java/org/zowe/apiml/caching/service/infinispan/storage/InfinispanStorageTest.java +++ b/caching-service/src/test/java/org/zowe/apiml/caching/service/infinispan/storage/InfinispanStorageTest.java @@ -12,23 +12,28 @@ import org.infinispan.AdvancedCache; import org.infinispan.Cache; +import org.infinispan.CacheSet; +import org.infinispan.commons.util.IteratorMapper; import org.infinispan.lock.api.ClusteredLock; +import org.infinispan.manager.DefaultCacheManager; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; -import org.zowe.apiml.caching.model.KeyValue; import org.zowe.apiml.cache.StorageException; +import org.zowe.apiml.caching.model.KeyValue; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; +import java.util.Set; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.*; +import static org.zowe.apiml.caching.service.infinispan.config.InfinispanConfig.CACHE_ZOWE; +import static org.zowe.apiml.caching.service.infinispan.config.InfinispanConfig.CACHE_ZOWE_INVALIDATED_TOKEN; class InfinispanStorageTest { @@ -47,7 +52,17 @@ void setup() { cache = mock(Cache.class); tokenCache = mock(AdvancedCache.class); lock = mock(ClusteredLock.class); - storage = new InfinispanStorage(cache, tokenCache, () -> lock); + storage = new InfinispanStorage(createCacheManager(cache, tokenCache), () -> lock); + } + + private DefaultCacheManager createCacheManager( + Map cache, + Map> tokenCache + ) { + var defaultCacheManager = mock(DefaultCacheManager.class); + doReturn(cache).when(defaultCacheManager).getCache(CACHE_ZOWE); + doReturn(tokenCache).when(defaultCacheManager).getCache(CACHE_ZOWE_INVALIDATED_TOKEN); + return defaultCacheManager; } @Nested @@ -92,6 +107,7 @@ void whenDelete_thenExceptionIsThrown() { @Nested class WhenEntryExists { + KeyValue keyValue; @BeforeEach @@ -123,16 +139,16 @@ void cacheIsUpdated() { @Test void itemIsDeleted() { - ConcurrentMap cache = new ConcurrentHashMap<>(); - InfinispanStorage storage = new InfinispanStorage(cache, tokenCache, () -> lock); + Cache cache = createCache(); + InfinispanStorage storage = new InfinispanStorage(createCacheManager(cache, tokenCache), () -> lock); assertNull(storage.create(serviceId1, TO_CREATE)); assertEquals(TO_CREATE, storage.delete(serviceId1, TO_CREATE.getKey())); } @Test void returnAll() { - ConcurrentMap cache = new ConcurrentHashMap<>(); - InfinispanStorage storage = new InfinispanStorage(cache, tokenCache, () -> lock); + Cache cache = createCache(); + InfinispanStorage storage = new InfinispanStorage(createCacheManager(cache, tokenCache), () -> lock); storage.create(serviceId1, new KeyValue("key", "value")); storage.create(serviceId1, new KeyValue("key2", "value2")); assertEquals(2, storage.readForService(serviceId1).size()); @@ -140,8 +156,8 @@ void returnAll() { @Test void removeAll() { - ConcurrentMap cache = new ConcurrentHashMap<>(); - InfinispanStorage storage = new InfinispanStorage(cache, tokenCache, () -> lock); + Cache cache = createCache(); + InfinispanStorage storage = new InfinispanStorage(createCacheManager(cache, tokenCache), () -> lock); storage.create(serviceId1, new KeyValue("key", "value")); storage.create(serviceId1, new KeyValue("key2", "value2")); assertEquals(2, storage.readForService(serviceId1).size()); @@ -153,6 +169,7 @@ void removeAll() { @Nested class WhenStoreToken { + KeyValue keyValue; @BeforeEach @@ -173,7 +190,7 @@ void createStoreWithEntry() { void addToken() { HashMap hashMap = new HashMap<>(); hashMap.put("key", "token"); - InfinispanStorage storage = new InfinispanStorage(cache, tokenCache, () -> lock); + InfinispanStorage storage = new InfinispanStorage(createCacheManager(cache, tokenCache), () -> lock); when(tokenCache.get(anyString())).thenAnswer(invocation -> hashMap); assertNull(storage.storeMapItem(serviceId1, "invalidTokens", new KeyValue("newkey", "newvalue"))); verify(tokenCache, times(1)).put(serviceId1 + "invalidTokens", hashMap); @@ -183,12 +200,13 @@ void addToken() { void updateToken() { HashMap hashMap = new HashMap(); hashMap.put("key", "token"); - InfinispanStorage storage = new InfinispanStorage(cache, tokenCache, () -> lock); + InfinispanStorage storage = new InfinispanStorage(createCacheManager(cache, tokenCache), () -> lock); when(tokenCache.get(serviceId1 + "invalidTokens")).thenReturn(hashMap); KeyValue keyValue = new KeyValue("key", "token2"); assertNull(storage.storeMapItem(serviceId1, "invalidTokens", keyValue)); verify(tokenCache, times(1)).put(serviceId1 + "invalidTokens", hashMap); } + } @Nested @@ -203,12 +221,14 @@ void returnTokenList() { when(tokenCache.get(serviceId1 + "invalidTokens")).thenReturn(expectedMap); assertEquals(2, storage.getAllMapItems(serviceId1, "invalidTokens").size()); } + } @Nested class WhenRetrieveInvalidTokensAndRules { InfinispanStorage underTest; + @BeforeEach void createStorage() { Map tokensService1 = new HashMap(); @@ -219,11 +239,11 @@ void createStorage() { Map rulesService1 = new HashMap(); rulesService1.put("key1", "rule1"); rulesService1.put("key2", "rule2"); - ConcurrentMap> tokenCache = new ConcurrentHashMap<>(); + Cache> tokenCache = createCache(); tokenCache.put(serviceId1 + "invalidTokens", tokensService1); tokenCache.put(serviceId1 + "invalidTokenRules", rulesService1); tokenCache.put(serviceId2 + "invalidTokens", tokensService2); - underTest = new InfinispanStorage(cache, tokenCache, () -> lock); + underTest = new InfinispanStorage(createCacheManager(cache, tokenCache), () -> lock); } @@ -253,12 +273,13 @@ void returnNoneForUnknownService() { Map> result = underTest.getAllMaps("unknown_service"); assertEquals(0, result.size()); - } + } @Nested class WhenEvictNonRelevantTokensAndRules { + InfinispanStorage underTest; @BeforeEach @@ -271,12 +292,14 @@ void createStorage() { rulesService.put("key1", "1595282400000"); Map rulesUsers = new HashMap(); rulesUsers.put("key1", "1595282400000"); - ConcurrentMap> tokenCache = new ConcurrentHashMap<>(); + Cache cache = createCache(); + Cache> tokenCache = createCache(); tokenCache.put(serviceId1 + "invalidTokens", tokensService); tokenCache.put(serviceId1 + "invalidScopes", rulesService); tokenCache.put(serviceId1 + "invalidUsers", rulesUsers); - underTest = new InfinispanStorage(cache, tokenCache, () -> lock); + underTest = new InfinispanStorage(createCacheManager(cache, tokenCache), () -> lock); } + @Test void thenEvictItems() { CompletableFuture cmpl = new CompletableFuture<>(); @@ -293,4 +316,25 @@ void thenEvictItems() { } + private Cache createCache() { + var data = new HashMap(); + var cache = mock(Cache.class); + doAnswer(answer -> data.put(answer.getArgument(0), answer.getArgument(1))).when(cache).put(any(), any()); + doAnswer(answer -> data.putIfAbsent(answer.getArgument(0), answer.getArgument(1))).when(cache).putIfAbsent(any(), any()); + doAnswer(answer -> data.get(answer.getArgument(0))).when(cache).get(any()); + doAnswer(answer -> data.remove(answer.getArgument(0))).when(cache).remove(any()); + doAnswer(answer -> { + new HashMap<>(data).forEach(answer.getArgument(0)); + return null; + }).when(cache).forEach(any()); + doAnswer(answer -> createCacheSet(data.keySet())).when(cache).keySet(); + return cache; + } + + private CacheSet createCacheSet(Set keys) { + var set = mock(CacheSet.class); + doReturn(new IteratorMapper(keys.iterator(), Function.identity())).when(set).iterator(); + return set; + } + } diff --git a/config/docker/api-catalog-services.yml b/config/docker/api-catalog-services.yml index cfc21dff51..d58044362e 100644 --- a/config/docker/api-catalog-services.yml +++ b/config/docker/api-catalog-services.yml @@ -42,3 +42,9 @@ spring: output: ansi: enabled: always + +management: + endpoints: + web: + exposure: + include: "*" diff --git a/config/docker/caching-service.yml b/config/docker/caching-service.yml index 3f70772e1a..9ae1164025 100644 --- a/config/docker/caching-service.yml +++ b/config/docker/caching-service.yml @@ -16,3 +16,9 @@ spring: enabled: always profiles: include: debug + +management: + endpoints: + web: + exposure: + include: "*" diff --git a/config/docker/discoverable-client.yml b/config/docker/discoverable-client.yml index 07ba48795b..635fdf3c44 100644 --- a/config/docker/discoverable-client.yml +++ b/config/docker/discoverable-client.yml @@ -28,3 +28,9 @@ server: keyStorePassword: password trustStore: /docker/all-services.truststore.p12 trustStorePassword: password + +management: + endpoints: + web: + exposure: + include: "*" diff --git a/config/local/api-catalog-service.yml b/config/local/api-catalog-service.yml index 35501aaf96..8c6fcff9d0 100644 --- a/config/local/api-catalog-service.yml +++ b/config/local/api-catalog-service.yml @@ -53,6 +53,12 @@ spring: ansi: enabled: always +management: + endpoints: + web: + exposure: + include: "*" + --- spring.config.activate.on-profile: standalone diff --git a/config/local/apiml-service-2.yml b/config/local/apiml-service-2.yml index cc4b5cf480..375c1d65e7 100644 --- a/config/local/apiml-service-2.yml +++ b/config/local/apiml-service-2.yml @@ -79,3 +79,8 @@ jgroups: port: 7601 address: localhost +management: + endpoints: + web: + exposure: + include: "*" diff --git a/config/local/apiml-service.yml b/config/local/apiml-service.yml index 971f500f7c..fc1224456b 100644 --- a/config/local/apiml-service.yml +++ b/config/local/apiml-service.yml @@ -72,3 +72,9 @@ spring: httpclient: websocket: max-frame-payload-length: 3145728 + +management: + endpoints: + web: + exposure: + include: "*" diff --git a/config/local/caching-service-2.yml b/config/local/caching-service-2.yml index 5ce41aa935..3a1e42f842 100644 --- a/config/local/caching-service-2.yml +++ b/config/local/caching-service-2.yml @@ -16,3 +16,9 @@ apiml: spring: profiles: include: dev + +management: + endpoints: + web: + exposure: + include: "*" diff --git a/config/local/caching-service.yml b/config/local/caching-service.yml index 768e2f50e2..acbf6ada22 100644 --- a/config/local/caching-service.yml +++ b/config/local/caching-service.yml @@ -14,3 +14,9 @@ caching: spring: profiles: include: dev + +management: + endpoints: + web: + exposure: + include: "*" diff --git a/config/local/discoverable-client.yml b/config/local/discoverable-client.yml index edb3c4a74a..19363728e7 100644 --- a/config/local/discoverable-client.yml +++ b/config/local/discoverable-client.yml @@ -27,3 +27,9 @@ server: keyStorePassword: password trustStore: keystore/localhost/localhost.truststore.p12 trustStorePassword: password + +management: + endpoints: + web: + exposure: + include: "*" diff --git a/config/local/discovery-service.yml b/config/local/discovery-service.yml index d1e3e31ab9..8c069cb16e 100644 --- a/config/local/discovery-service.yml +++ b/config/local/discovery-service.yml @@ -14,6 +14,12 @@ apiml: spring.output.ansi.enabled: always +management: + endpoints: + web: + exposure: + include: "*" + --- spring.config.activate.on-profile: https diff --git a/config/local/gateway-service.yml b/config/local/gateway-service.yml index ee139ab68c..20ea8ffc5f 100644 --- a/config/local/gateway-service.yml +++ b/config/local/gateway-service.yml @@ -40,3 +40,8 @@ spring: httpclient: websocket: max-frame-payload-length: 3145728 +management: + endpoints: + web: + exposure: + include: "*" diff --git a/config/local/zaas-service.yml b/config/local/zaas-service.yml index c9b75c501c..561d026a8f 100644 --- a/config/local/zaas-service.yml +++ b/config/local/zaas-service.yml @@ -58,6 +58,12 @@ server: trustStorePassword: password trustStoreType: PKCS12 +management: + endpoints: + web: + exposure: + include: "*" + --- spring.config.activate.on-profile: attls diff --git a/discoverable-client/build.gradle b/discoverable-client/build.gradle index a122c8da84..0ca01c0fb2 100644 --- a/discoverable-client/build.gradle +++ b/discoverable-client/build.gradle @@ -89,6 +89,7 @@ dependencies { implementation libs.spring.boot.starter.actuator implementation libs.spring.doc implementation libs.spring.boot.starter.graphql + implementation libs.eureka.jersey.client testImplementation libs.spring.boot.starter.test testImplementation(testFixtures(project(":apiml-common"))) diff --git a/discoverable-client/src/main/java/org/zowe/apiml/client/api/ClientEurekaRegistryVersionEndpoint.java b/discoverable-client/src/main/java/org/zowe/apiml/client/api/ClientEurekaRegistryVersionEndpoint.java new file mode 100644 index 0000000000..a8fa04fdd8 --- /dev/null +++ b/discoverable-client/src/main/java/org/zowe/apiml/client/api/ClientEurekaRegistryVersionEndpoint.java @@ -0,0 +1,61 @@ +/* + * This program and the accompanying materials are made available under the terms of the + * Eclipse Public License v2.0 which accompanies this distribution, and is available at + * https://www.eclipse.org/legal/epl-v20.html + * + * SPDX-License-Identifier: EPL-2.0 + * + * Copyright Contributors to the Zowe Project. + */ + +package org.zowe.apiml.client.api; + +import lombok.Builder; +import lombok.RequiredArgsConstructor; +import lombok.Value; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.actuate.endpoint.annotation.Endpoint; +import org.springframework.boot.actuate.endpoint.annotation.ReadOperation; +import org.springframework.stereotype.Component; +import org.zowe.apiml.eurekaservice.client.ApiMediationClient; + +import java.util.regex.Pattern; + +import static io.netty.handler.codec.http.HttpHeaders.Values.APPLICATION_JSON; + +@Component +@RequiredArgsConstructor +@Endpoint(id = "eurekaversion") +@Slf4j +public class ClientEurekaRegistryVersionEndpoint { + + private static final Pattern VERSION_PATTERN = Pattern.compile("^.*_([0-9]+)_.*$"); + + private final ApiMediationClient apiMediationClient; + + @ReadOperation(produces = APPLICATION_JSON) + public VersionDto status() { + long version = -1; + var eurekaClient = apiMediationClient.getEurekaClient(); + if (eurekaClient != null) { + var hashCode = eurekaClient.getApplications().getAppsHashCode(); + var matcher = VERSION_PATTERN.matcher(hashCode); + if (matcher.matches()) { + version = Long.parseLong(matcher.group(1)); + log.debug("New Eureka registry version: {}", version); + } else { + log.debug("Unexpected Eureka registry hashCode: {}", hashCode); + } + } + return VersionDto.builder().version(version).build(); + } + + @Builder + @Value + static class VersionDto { + + private Long version; + + } + +} diff --git a/discovery-service/build.gradle b/discovery-service/build.gradle index b2b8638157..065c3c2b43 100644 --- a/discovery-service/build.gradle +++ b/discovery-service/build.gradle @@ -55,6 +55,7 @@ gitProperties { dependencies { api project(':apiml-tomcat-common') + api project(':apiml-common') api project(':security-service-client-spring') implementation libs.spring.boot.starter.security diff --git a/discovery-service/src/main/java/org/zowe/apiml/discovery/ApimlInstanceRegistry.java b/discovery-service/src/main/java/org/zowe/apiml/discovery/ApimlInstanceRegistry.java index accb1f0ab3..1e0d056adb 100644 --- a/discovery-service/src/main/java/org/zowe/apiml/discovery/ApimlInstanceRegistry.java +++ b/discovery-service/src/main/java/org/zowe/apiml/discovery/ApimlInstanceRegistry.java @@ -29,8 +29,14 @@ import org.zowe.apiml.exception.MetadataValidationException; import org.zowe.apiml.util.EurekaUtils; -import java.lang.invoke.*; -import java.lang.reflect.*; +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.lang.invoke.WrongMethodTypeException; +import java.lang.reflect.Constructor; +import java.lang.reflect.Field; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.regex.Pattern; @@ -169,6 +175,7 @@ public void peerAwareHeartbeat(InstanceInfo instanceInfo) { try { replicateToPeersMethodHandle.invokeWithArguments(this, Action.Heartbeat, instanceInfo.getAppName(), instanceInfo.getId(), instanceInfo, null, false); } catch (Throwable e) { + log.warn("Unexpected error occurred while replicateToPeers: {}", instanceInfo.getInstanceId(), e); throw new IllegalStateException(EXCEPTION_MESSAGE, e); } } @@ -182,7 +189,7 @@ public void peerAwareHeartbeat(InstanceInfo instanceInfo) { */ public void registerStatically(InstanceInfo instanceInfo, boolean isReplication, boolean peerReplicate) { // the maximum lease duration time (Eureka bug: overflow of int during conversion to ms) - int leaseDuration = Integer.MAX_VALUE / 1000; + int leaseDuration = 1; // temporary register (do not increase count of service to avoid threshold) synchronized (lock) { diff --git a/discovery-service/src/main/java/org/zowe/apiml/discovery/DiscoveryServiceApplication.java b/discovery-service/src/main/java/org/zowe/apiml/discovery/DiscoveryServiceApplication.java index 255602013d..809a4b6bd4 100644 --- a/discovery-service/src/main/java/org/zowe/apiml/discovery/DiscoveryServiceApplication.java +++ b/discovery-service/src/main/java/org/zowe/apiml/discovery/DiscoveryServiceApplication.java @@ -30,6 +30,7 @@ @SpringBootApplication @ComponentScan({ "org.zowe.apiml.discovery", + "org.zowe.apiml.product.eureka.web", "org.zowe.apiml.product.config", "org.zowe.apiml.product.security", "org.zowe.apiml.product.web", diff --git a/discovery-service/src/main/java/org/zowe/apiml/discovery/config/EurekaConfig.java b/discovery-service/src/main/java/org/zowe/apiml/discovery/config/EurekaConfig.java index 6316729b93..104e1b1851 100644 --- a/discovery-service/src/main/java/org/zowe/apiml/discovery/config/EurekaConfig.java +++ b/discovery-service/src/main/java/org/zowe/apiml/discovery/config/EurekaConfig.java @@ -22,8 +22,10 @@ import jakarta.ws.rs.client.ClientRequestFilter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.NoSuchBeanDefinitionException; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; +import org.springframework.beans.factory.support.BeanDefinitionRegistryPostProcessor; import org.springframework.cloud.netflix.eureka.server.InstanceRegistryProperties; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; @@ -43,12 +45,31 @@ @Slf4j public class EurekaConfig { + private static final String PEER_AWARE_INSTANCE_REGISTRY = "peerAwareInstanceRegistry"; + @Value("${apiml.discovery.serviceIdPrefixReplacer:#{null}}") private String tuple; @Value("${apiml.discovery.maxPeerRetries:10}") private int maxPeerRetries; + /** + * This is a fix of impossible overriding of the original bean. + * + * @return bean definition processor to remove original bean peerAwareInstanceRegistry + */ + @Bean + public static BeanDefinitionRegistryPostProcessor deleteEurekaPeerAwareInstanceRegistry() { + return registry -> { + try { + registry.removeBeanDefinition(PEER_AWARE_INSTANCE_REGISTRY); + log.debug("The overridden bean {} is still in the registry. It is redundant and will be removed.", PEER_AWARE_INSTANCE_REGISTRY); + } catch (NoSuchBeanDefinitionException ex) { + log.debug("The overridden bean {} is not found in the registry.", PEER_AWARE_INSTANCE_REGISTRY); + } + }; + } + @Bean @Primary public ApimlInstanceRegistry apimlInstanceRegistry( @@ -92,9 +113,8 @@ public ApimlInstanceRegistry apimlInstanceRegistry( serverContext.initialize(); log.info("Initialized server context"); - // Copy registry from neighboring eureka node - //int registryCount = apimlInstanceRegistry.syncUp(); - //apimlInstanceRegistry.openForTraffic(applicationInfoManager, registryCount); + int registryCount = apimlInstanceRegistry.syncUp(); + apimlInstanceRegistry.openForTraffic(applicationInfoManager, registryCount); // Register all monitoring statistics. EurekaMonitors.registerAllStats(); diff --git a/discovery-service/src/main/java/org/zowe/apiml/discovery/eureka/RefreshablePeerEurekaNodes.java b/discovery-service/src/main/java/org/zowe/apiml/discovery/eureka/RefreshablePeerEurekaNodes.java index f0a3576efb..6ae9ee0b86 100644 --- a/discovery-service/src/main/java/org/zowe/apiml/discovery/eureka/RefreshablePeerEurekaNodes.java +++ b/discovery-service/src/main/java/org/zowe/apiml/discovery/eureka/RefreshablePeerEurekaNodes.java @@ -42,16 +42,16 @@ import org.glassfish.jersey.client.ClientProperties; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.cloud.context.environment.EnvironmentChangeEvent; +import org.springframework.cloud.netflix.eureka.server.EurekaServerConfigBean; import org.springframework.context.ApplicationListener; import org.zowe.apiml.product.eureka.client.ApimlPeerEurekaNode; import javax.net.ssl.SSLContext; -import java.net.InetAddress; -import java.net.MalformedURLException; -import java.net.URL; -import java.net.UnknownHostException; +import java.net.*; import java.util.Collection; +import java.util.Objects; import java.util.Set; +import java.util.stream.Stream; import static com.netflix.discovery.util.DiscoveryBuildInfo.buildVersion; @@ -91,8 +91,28 @@ public PeerEurekaNode createPeerEurekaNode(String peerEurekaNodeUrl) { return new ApimlPeerEurekaNode(registry, targetHost, peerEurekaNodeUrl, replicationClient, serverConfig, maxPeerRetries); } + private int getPort(int defaultPort) { + var propertyResolver = ((EurekaServerConfigBean) this.serverConfig).getPropertyResolver(); + return Stream.of( + "apiml.internal-discovery.port", + "apiml.service.port" + ) + .map(propertyResolver::getProperty) + .filter(Objects::nonNull) + .map(Integer::parseInt) + .findFirst() + .orElse(defaultPort); + } + + @Override + public boolean isThisMyUrl(String url) { + int urlPort = URI.create(url).getPort(); + int instancePort = getPort(urlPort); + return (urlPort == instancePort) && super.isThisMyUrl(url); + } + private Jersey3ReplicationClient createReplicationClient(EurekaServerConfig config, - ServerCodecs serverCodecs, String serviceUrl, Collection additionalFilters) { + ServerCodecs serverCodecs, String serviceUrl, Collection additionalFilters) { String name = Jersey3ReplicationClient.class.getSimpleName() + ": " + serviceUrl + "apps/: "; EurekaJersey3Client jerseyClient; diff --git a/discovery-service/src/main/resources/application.yml b/discovery-service/src/main/resources/application.yml index a7db48734f..bf869fa22f 100644 --- a/discovery-service/src/main/resources/application.yml +++ b/discovery-service/src/main/resources/application.yml @@ -98,6 +98,7 @@ eureka: server: max-threads-for-peer-replication: 6 useReadOnlyResponseCache: false + peer-node-read-timeout-ms: 15000 management: endpoints: diff --git a/docker/redis/compose/apiml.env.template b/docker/redis/compose/apiml.env.template index d6c8863bb6..029ebdf9c7 100644 --- a/docker/redis/compose/apiml.env.template +++ b/docker/redis/compose/apiml.env.template @@ -29,5 +29,3 @@ CACHING_STORAGE_REDIS_SENTINEL_ENABLED: {SENTINEL} {NOT_LINUX_SETTING}{SENTINEL_SETTING}CACHING_STORAGE_REDIS_SENTINEL_NODES_0: sentinelpassword@redis-sentinel-1:26379 {NOT_LINUX_SETTING}{SENTINEL_SETTING}CACHING_STORAGE_REDIS_SENTINEL_NODES_1: sentinelpassword@redis-sentinel-2:26380 {NOT_LINUX_SETTING}{SENTINEL_SETTING}CACHING_STORAGE_REDIS_SENTINEL_NODES_2: sentinelpassword@redis-sentinel-3:26381 - -ZOSMF_APPLIEDAPARS: AuthenticateApar diff --git a/gateway-service/src/main/java/org/zowe/apiml/gateway/GatewayServiceApplication.java b/gateway-service/src/main/java/org/zowe/apiml/gateway/GatewayServiceApplication.java index 492fae786b..880d98207f 100644 --- a/gateway-service/src/main/java/org/zowe/apiml/gateway/GatewayServiceApplication.java +++ b/gateway-service/src/main/java/org/zowe/apiml/gateway/GatewayServiceApplication.java @@ -20,6 +20,7 @@ "org.zowe.apiml.gateway", "org.zowe.apiml.product.web", "org.zowe.apiml.product.config", + "org.zowe.apiml.product.eureka.web", "org.zowe.apiml.product.gateway", "org.zowe.apiml.product.version", "org.zowe.apiml.product.logging", diff --git a/gradle/versions.gradle b/gradle/versions.gradle index a0511a4a2a..608521d37d 100644 --- a/gradle/versions.gradle +++ b/gradle/versions.gradle @@ -185,6 +185,7 @@ dependencyResolutionManagement { library('infinispan_spring_boot3_starter_embedded', 'org.infinispan', 'infinispan-spring-boot3-starter-embedded').versionRef('infinispan') library('infinispan_lock', 'org.infinispan', 'infinispan-clustered-lock').versionRef('infinispan') library('infinispan_jboss_marshalling', 'org.infinispan', 'infinispan-jboss-marshalling').versionRef('infinispan') + library('infinispan_annotations', 'org.infinispan', 'infinispan-component-annotations').versionRef('infinispan') library('jackson_annotations', 'com.fasterxml.jackson.core', 'jackson-annotations').versionRef('jacksonAnnotations') library('jackson_core', 'com.fasterxml.jackson.core', 'jackson-core').versionRef('jacksonCore') library('jackson_databind', 'com.fasterxml.jackson.core', 'jackson-databind').versionRef('jacksonDatabind') @@ -289,7 +290,7 @@ dependencyResolutionManagement { library('mockito_inline', 'org.mockito', 'mockito-inline').versionRef('mockitoInline') - bundle('infinispan', [ 'infinispan_spring_boot3_starter_embedded', 'infinispan_lock', 'infinispan_jboss_marshalling' ]) + bundle('infinispan', [ 'infinispan_spring_boot3_starter_embedded', 'infinispan_lock', 'infinispan_jboss_marshalling', 'infinispan_annotations' ]) bundle('jaxb', ['jaxbApi', 'jaxbImpl']) bundle('modulith', ['modulith', 'modulith_core', 'jmolecules']) } diff --git a/integration-tests/src/test/java/org/zowe/apiml/startup/impl/ApiMediationLayerStartupChecker.java b/integration-tests/src/test/java/org/zowe/apiml/startup/impl/ApiMediationLayerStartupChecker.java index d16e49c048..d792d567e2 100644 --- a/integration-tests/src/test/java/org/zowe/apiml/startup/impl/ApiMediationLayerStartupChecker.java +++ b/integration-tests/src/test/java/org/zowe/apiml/startup/impl/ApiMediationLayerStartupChecker.java @@ -12,34 +12,34 @@ import com.jayway.jsonpath.DocumentContext; import com.jayway.jsonpath.JsonPath; -import com.jayway.jsonpath.PathNotFoundException; -import lombok.AllArgsConstructor; +import io.netty.handler.codec.http.HttpHeaderValues; +import lombok.Data; import lombok.extern.slf4j.Slf4j; +import net.minidev.json.JSONArray; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.Strings; +import org.apache.http.HttpHeaders; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; -import org.zowe.apiml.util.config.ConfigReader; -import org.zowe.apiml.util.config.Credentials; -import org.zowe.apiml.util.config.DiscoverableClientConfiguration; -import org.zowe.apiml.util.config.DiscoveryServiceConfiguration; -import org.zowe.apiml.util.config.GatewayServiceConfiguration; -import org.zowe.apiml.util.config.SslContext; -import org.zowe.apiml.util.http.HttpClientUtils; +import org.springframework.web.util.DefaultUriBuilderFactory; +import org.zowe.apiml.product.constants.CoreService; +import org.zowe.apiml.util.config.*; import org.zowe.apiml.util.http.HttpRequestUtils; import java.io.IOException; -import java.util.ArrayList; -import java.util.Base64; -import java.util.List; +import java.net.URI; +import java.util.*; +import java.util.concurrent.Callable; +import java.util.function.Predicate; +import java.util.stream.Collectors; import static java.util.concurrent.TimeUnit.MINUTES; import static java.util.concurrent.TimeUnit.SECONDS; import static org.awaitility.Awaitility.await; -import static org.zowe.apiml.util.config.ConfigReader.IS_MODULITH_ENABLED; /** * Checks and waits until the testing environment is ready to be tested. @@ -47,189 +47,392 @@ @Slf4j public class ApiMediationLayerStartupChecker { - private final GatewayServiceConfiguration gatewayConfiguration; - private final DiscoverableClientConfiguration discoverableClientConfiguration; - private final DiscoveryServiceConfiguration discoveryServiceConfiguration; - private final Credentials credentials; - private final List servicesToCheck = new ArrayList<>(); - private final String healthEndpoint = "/application/health"; - - - public ApiMediationLayerStartupChecker() { - gatewayConfiguration = ConfigReader.environmentConfiguration().getGatewayServiceConfiguration(); - credentials = ConfigReader.environmentConfiguration().getCredentials(); - discoverableClientConfiguration = ConfigReader.environmentConfiguration().getDiscoverableClientConfiguration(); - discoveryServiceConfiguration = ConfigReader.environmentConfiguration().getDiscoveryServiceConfiguration(); - - servicesToCheck.add(new Service("Gateway", "$.status")); - if (!IS_MODULITH_ENABLED) { - servicesToCheck.add(new Service("ZAAS", "$.components.gateway.details.zaas")); + private static final long POOL_INTERVAL = 5; + + private static final boolean IS_MODULITH_ENABLED = Boolean.parseBoolean(System.getProperty("environment.modulith")); + private static final Credentials CREDENTIALS = ConfigReader.environmentConfiguration().getCredentials(); + private static final String CREDENTIALS_HEADER = "Basic " + Base64.getEncoder().encodeToString(String.format("%s:%s", CREDENTIALS.getUser(), CREDENTIALS.getPassword()).getBytes()); + private static final String APPLICATION_JSON_HEADER = HttpHeaderValues.APPLICATION_JSON.toString(); + + void initSsl() { + if (SslContext.sslClientCertValid == null) { + TlsConfiguration tlsCfg = ConfigReader.environmentConfiguration().getTlsConfiguration(); + SslContextConfigurer sslContextConfigurer = new SslContextConfigurer(tlsCfg.getKeyStorePassword(), tlsCfg.getClientKeystore(), tlsCfg.getKeyStore()); + try { + SslContext.prepareSslAuthentication(sslContextConfigurer); + } catch (Exception e) { + throw new RuntimeException(e); + } } - servicesToCheck.add(new Service("Api Catalog", "$.components.gateway.details.apicatalog")); - servicesToCheck.add(new Service("Discovery Service", "$.components.gateway.details.discovery")); } public void waitUntilReady() { - long poolInterval = 5; - await() - .atMost(10, MINUTES) - .pollDelay(0, SECONDS) - .pollInterval(poolInterval, SECONDS) - .until(this::areAllServicesUp); + log.debug("Modulith: {}", IS_MODULITH_ENABLED); + initSsl(); + ApimlInstance.load().forEach(ApimlInstance::waitUntilReady); } - private DocumentContext getDocumentAsContext(HttpGet request) { - try { - final HttpResponse response = HttpClientUtils.client().execute(request); - if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { - log.warn("Unexpected HTTP status code: {} for URI: {}. Message: {}", response.getStatusLine().getStatusCode(), request.getURI().toString(), EntityUtils.toString(response.getEntity())); - return null; - } - final String jsonResponse = EntityUtils.toString(response.getEntity()); - log.debug("URI: {}, JsonResponse is {}", request.getURI().toString(), jsonResponse); + @Data + private static class Instance { + + private final String scheme; + private final String hostname; + private final String serviceId; + private final int port; + private final ServiceConfiguration serviceConfiguration; + + private Instance(String hostname, ServiceConfiguration serviceConfiguration) { + this.scheme = serviceConfiguration.getScheme(); + this.hostname = hostname; + this.serviceId = serviceConfiguration.getServiceId(); + this.port = serviceConfiguration.getPort(); + this.serviceConfiguration = serviceConfiguration; + } - if (StringUtils.isNotEmpty(jsonResponse)) { - return JsonPath.parse(jsonResponse); + private Instance(Instance instance, String serviceId) { + this.scheme = instance.getScheme(); + this.hostname = instance.getHostname(); + this.serviceId = serviceId; + this.port = instance.getPort(); + this.serviceConfiguration = instance.getServiceConfiguration(); + } + + private static List getAllHosts(ServiceConfiguration serviceConfiguration) { + List hosts = new ArrayList<>(); + if (serviceConfiguration == null) { + return hosts; + } + if (StringUtils.isNotBlank(serviceConfiguration.getHost())) { + hosts.addAll(Arrays.asList(serviceConfiguration.getHost().split("[,;]"))); } - return null; - } catch (IOException e) { - log.warn("Check failed on getting the document: {}", e.getMessage()); - return null; + if (serviceConfiguration instanceof DiscoveryServiceConfiguration discoveryServiceConfiguration) { + String additionalHost = discoveryServiceConfiguration.getAdditionalHost(); + if (StringUtils.isNotBlank(additionalHost)) { + hosts.addAll(Arrays.asList(additionalHost.split("[,;]"))); + } + } + return hosts; } - } - private boolean areAllServicesUp() { - try { - var gatewayHosts = gatewayConfiguration.getHost().split(","); - var requestToGateway1 = HttpRequestUtils.getRequest(gatewayHosts[0], healthEndpoint); - // If second one does not exist, redundant call and check to same gateway - var requestToGateway2 = HttpRequestUtils.getRequest(gatewayHosts.length > 1 ? gatewayHosts[1] : gatewayHosts[0], healthEndpoint); + private static Instance of(Instance instance, String serviceId) { + return new Instance(instance, serviceId); + } - requestToGateway1.addHeader("Authorization", "Basic " + Base64.getEncoder().encodeToString(String.format("%s:%s", credentials.getUser(), credentials.getPassword()).getBytes())); - requestToGateway2.addHeader("Authorization", "Basic " + Base64.getEncoder().encodeToString(String.format("%s:%s", credentials.getUser(), credentials.getPassword()).getBytes())); - DocumentContext context1 = getDocumentAsContext(requestToGateway1); - DocumentContext context2 = getDocumentAsContext(requestToGateway2); + private static List of(ServiceConfiguration serviceConfiguration) { + return getAllHosts(serviceConfiguration).stream() + .filter(StringUtils::isNotBlank) + .map(String::trim) + .map(String::toLowerCase) + .map(host -> new Instance(host, serviceConfiguration)) + .toList(); + } - if (context1 == null || context2 == null) { - return false; + private static List of(ServiceConfiguration serviceConfiguration, String countProperty) { + List allInstances = of(serviceConfiguration); + String countString = System.getProperty(countProperty); + if (StringUtils.isNotBlank(countString)) { + try { + int count = Integer.parseInt(countString); + if (count >= 0 && count <= allInstances.size()) { + return allInstances.subList(0, count); + } + log.warn("Invalid count of services {}: {}", serviceConfiguration.getServiceId(), countString); + } catch (NumberFormatException e) { + log.warn("Invalid service instances value: {} -> {}", countProperty, countString); + } } + return allInstances; + } - boolean areAllServicesUp = true; - for (Service toCheck : servicesToCheck) { - boolean isUp = isServiceUp(context1, toCheck.path); - logDebug(toCheck.name + " is {}", isUp); + private String getUrl(String basePath) { + return new DefaultUriBuilderFactory().builder() + .scheme("https") + .host(this.hostname) + .port(this.port) + .path(this.serviceConfiguration.getServletContext() + basePath) + .toUriString(); + } - if (!isUp) { - areAllServicesUp = false; - } - } - if (!IS_MODULITH_ENABLED && !isAuthUp()) { - areAllServicesUp = false; - } + public String getHealthEndpointUrl() { + return getUrl("application/health"); + } - String allComponents = context1.read("$.components.discoveryComposite.components.discoveryClient.details.services").toString(); - boolean isTestApplicationUp = allComponents.toLowerCase().contains("discoverableclient"); - boolean needsTestApplication = discoverableClientConfiguration.getInstances() > 0; + public String getEurekaVersionUrl() { + return getUrl("application/eurekaversion"); + } - log.debug("Discoverable Client is {}", isTestApplicationUp); - log.debug("Needs Discoverable Client: {}", needsTestApplication); - isTestApplicationUp = !needsTestApplication || isTestApplicationUp; + public String getInstanceId() { + String prefix = this.serviceConfiguration.isStaticallyRegistred() ? "STATIC-" : ""; + return prefix + (this.hostname + ":" + this.serviceId + ":" + this.port).toLowerCase(); + } - Integer amountOfActiveGateways1 = context1.read("$.components.gateway.details.gatewayCount"); - Integer amountOfActiveGateways2 = context2.read("$.components.gateway.details.gatewayCount"); - var expectedGatewayCount = Integer.getInteger("environment.gwCount", gatewayConfiguration.getInstances()); + } - boolean isValidAmountOfGatewaysUp = amountOfActiveGateways1 != null && amountOfActiveGateways2 != null && - amountOfActiveGateways1 >= expectedGatewayCount && amountOfActiveGateways2 >= expectedGatewayCount; - log.debug("There are {} gateways in GW1 and {} in GW2", amountOfActiveGateways1, amountOfActiveGateways2); + private static class ApimlInstance { + + private int minimumEurekaVersion = -1; + + private final List allInstances; + private final List additionalRegistration = new ArrayList<>(); + + private ApimlInstance(Predicate instanceMatcher) { + var config = ConfigReader.environmentConfiguration(); + + var instances = new ArrayList(); + instances.addAll(Instance.of(config.getDiscoveryServiceConfiguration(), "discovery.instances")); + instances.addAll(Instance.of(config.getApiCatalogServiceConfiguration(), "apicatalog.instances")); + instances.addAll(Instance.of(config.getGatewayServiceConfiguration(), "gateway.instances")); + instances.addAll(Instance.of(config.getDiscoverableClientConfiguration(), "discoverableclient.instances")); + instances.addAll(Instance.of(config.getCachingServiceConfiguration(), "caching.instances")); + instances.addAll(Instance.of(config.getZosmfServiceConfiguration(), "zosmf.instances")); + instances.addAll(Instance.of(config.getZaasConfiguration(), "zaas.instances")); + instances.addAll(Instance.of(config.getCentralGatewayServiceConfiguration(), "centralGateway.instances")); + allInstances = instances.stream() + .map(i -> { + String replacement = System.getProperty("serviceIdReplaced", ""); + for (String rule : StringUtils.split(replacement, "[,;]")) { + var ids = rule.split(":"); + if (Strings.CI.equals(i.getServiceId(), ids[0])) { + return Instance.of(i, ids[1]); + } + } + return i; + }) + .filter(instanceMatcher) + .collect(Collectors.toList()); + } - if (!isValidAmountOfGatewaysUp) { - log.debug("Expecting at least {} gateways", gatewayConfiguration.getInstances()); - callEurekaApps(); - return false; + private List getInstancesRegistryCheck(Instance ds) { + var instances = new ArrayList<>(allInstances); + instances.addAll(additionalRegistration); + if (IS_MODULITH_ENABLED) { + return instances.stream().filter(instance -> + !Strings.CI.equalsAny(instance.getServiceId(), + CoreService.API_CATALOG.getServiceId(), + CoreService.DISCOVERY.getServiceId(), + CoreService.CACHING.getServiceId() + ) || + Strings.CI.equals(instance.getHostname(), ds.getHostname()) + ).toList(); } + return instances; + } - // Consider properly the case with multiple gateway services running on different ports. - if (gatewayConfiguration.getInternalPorts() != null && !gatewayConfiguration.getInternalPorts().isEmpty()) { - String[] internalPorts = gatewayConfiguration.getInternalPorts().split(","); - String[] hosts = gatewayConfiguration.getHost().split(","); + private List getInstancesEurekaVersionCheck() { + var instances = allInstances.stream(); + if (IS_MODULITH_ENABLED) { + instances = instances.filter(instance -> !Strings.CI.equalsAny(instance.getServiceId(), + CoreService.API_CATALOG.getServiceId(), + CoreService.GATEWAY.getServiceId(), + CoreService.CACHING.getServiceId() + )); + } + instances = instances.filter(instance -> !Strings.CI.equalsAny(instance.getServiceId(), + CoreService.DISCOVERY.getServiceId(), // the source of version + ConfigReader.environmentConfiguration().getZosmfServiceConfiguration().getServiceId() // does not support the endpoint + )); + return instances.toList(); + } - for (int i = 0; i < Math.min(internalPorts.length, hosts.length); i++) { - log.debug("Trying to access the Gateway at port {}", internalPorts[i]); - requestToGateway1 = HttpRequestUtils.getRequest(healthEndpoint); - requestToGateway1.addHeader("Authorization", "Basic " + Base64.getEncoder().encodeToString(String.format("%s:%s", credentials.getUser(), credentials.getPassword()).getBytes())); - var response = HttpClientUtils.client().execute(requestToGateway1); + private List getInstancesHealthCheck() { + if (IS_MODULITH_ENABLED) { + return allInstances.stream().filter(instance -> !Strings.CI.equalsAny(instance.getServiceId(), + CoreService.API_CATALOG.getServiceId(), + CoreService.CACHING.getServiceId() + )).toList(); + } + return allInstances; + } - if (response.getStatusLine().getStatusCode() != 200) { - log.debug("Response from gateway at {} was: {}", requestToGateway1.getURI(), response.getEntity() != null ? EntityUtils.toString(response.getEntity()) : "undefined"); - throw new IOException(); - } + private void awaitFor(Callable check, int durationMin) { + await() + .atMost(durationMin, MINUTES) + .pollDelay(0, SECONDS) + .pollInterval(POOL_INTERVAL, SECONDS) + .until(check); + } + private DocumentContext getDocumentAsContext(URI uri, boolean basicAuth) { + HttpGet request = new HttpGet(uri); + request.addHeader(HttpHeaders.ACCEPT, APPLICATION_JSON_HEADER); + if (basicAuth) { + request.addHeader(HttpHeaders.AUTHORIZATION, CREDENTIALS_HEADER); + } + try (CloseableHttpClient client = HttpClients.custom().setSSLContext(SslContext.sslClientCertValid).build()) { + final HttpResponse response = client.execute(request); + if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { + log.warn("Unexpected HTTP status code: {} for URI: {}. Message: {}", response.getStatusLine().getStatusCode(), request.getURI().toString(), EntityUtils.toString(response.getEntity())); + return null; } + final String jsonResponse = EntityUtils.toString(response.getEntity()); + log.debug("URI: {}, JsonResponse is {}", request.getURI().toString(), jsonResponse); + if (StringUtils.isNotEmpty(jsonResponse)) { + return JsonPath.parse(jsonResponse); + } + return null; + } catch (IOException e) { + log.warn("Check failed on getting the document: {}", e.getMessage()); + return null; } + } - var result = areAllServicesUp && isTestApplicationUp; - if (!result) { - log.debug("API ML is not ready, check which services are missing in the above messages"); + private boolean areAllInstanceOnInEureka(Instance ds, DocumentContext documentContext) { + var expectedInstances = getInstancesRegistryCheck(ds); + Set onboarded = ((JSONArray) documentContext.read("applications.application.*.instance.*.instanceId")).stream() + .map(String.class::cast) + .map(String::toLowerCase) + .collect(Collectors.toSet()); + Set expectedInstanceIds = expectedInstances.stream() + .map(Instance::getInstanceId) + .map(String::toLowerCase) + .collect(Collectors.toSet()); + + List missing = expectedInstanceIds.stream().filter(id -> !onboarded.contains(id)).sorted().toList(); + if (missing.isEmpty()) { + return true; } - return result; - } catch (PathNotFoundException | IOException e) { - log.warn("Check failed on retrieving the information from document: {}", e.getMessage()); + log.debug("{} services has not onboarded on {} yet: {}", missing.size(), ds.getInstanceId(), StringUtils.join(missing, ", ")); return false; } - } - private void callEurekaApps() { - HttpGet requestToEurekaApps = new HttpGet(HttpRequestUtils.getUriFromService(discoveryServiceConfiguration, "/eureka/apps")); - CloseableHttpClient client = HttpClients.custom().setSSLContext(SslContext.sslClientCertValid).build(); - try (client) { - var response = client.execute(requestToEurekaApps); - var entity = response.getEntity(); - if (entity != null) { - log.debug("eureka/apps: {}", EntityUtils.toString(entity)); - } else { - log.debug("eureka/apps entity is null"); + private boolean areAllInstancesOnboarded() { + for (var ds : get(CoreService.DISCOVERY)) { + var documentContext = getDocumentAsContext(HttpRequestUtils.getUri( + ds.getScheme(), ds.getHostname(), ds.getPort(), "/eureka/apps" + ), ds.getServiceConfiguration().isBasicAuthenticationSupported()); + if (documentContext == null || !areAllInstanceOnInEureka(ds, documentContext)) { + return false; + } } - } catch (Exception e) { - log.error("Cannot call Eureka apps", e); + return true; } - } - private boolean isAuthUp() { - HttpGet requestToZaas; - if (!IS_MODULITH_ENABLED) { - requestToZaas = new HttpGet(HttpRequestUtils.getUriFromZaas(healthEndpoint).get()); - } else { - requestToZaas = new HttpGet(HttpRequestUtils.getUriFromGateway(healthEndpoint)); + private int getEurekaVersion(Instance instance) { + var documentContext = getDocumentAsContext(URI.create(instance.getEurekaVersionUrl()), instance.getServiceConfiguration().isBasicAuthenticationSupported()); + if (documentContext != null) { + return documentContext.read("version"); + } + log.debug("Eurekaversion endpoint is not accessible on {}", instance.getInstanceId()); + return -1; } - requestToZaas.addHeader("Authorization", "Basic " + Base64.getEncoder().encodeToString(String.format("%s:%s", credentials.getUser(), credentials.getPassword()).getBytes())); - DocumentContext zaasContext = getDocumentAsContext(requestToZaas); - if (zaasContext == null) { + + private boolean areAllInstancesRegistryUpToDate() { + List notUpdated = new ArrayList<>(); + for (Instance instance : getInstancesEurekaVersionCheck()) { + int version = getEurekaVersion(instance); + if (version < this.minimumEurekaVersion) { + notUpdated.add(String.format("%s (%d / %s)", instance.getInstanceId(), version, this.minimumEurekaVersion)); + } + } + if (notUpdated.isEmpty()) { + return true; + } + + log.debug("There are instances that has not been updated yet: {}", StringUtils.join(notUpdated, ", ")); return false; } - boolean isUp; - if (!IS_MODULITH_ENABLED) { - isUp = isServiceUp(zaasContext, "$.components.zaas.details.auth"); - } else { - isUp = isServiceUp(zaasContext, "$.components.gateway.details.auth"); + + boolean areDiscoveryInSync() { + int sharedVersion = -1; + for (var discoveryServiceInstance : get(CoreService.DISCOVERY)) { + int version = getEurekaVersion(discoveryServiceInstance); + log.debug("Version at {} is {}", discoveryServiceInstance.getInstanceId(), version); + if (version < 0) { + // eureka is not initialized yet + return false; + } + if (sharedVersion < 0) { + // first fetched value + sharedVersion = version; + } + if (sharedVersion != version) { + // versions are not in sync + return false; + } + } + + this.minimumEurekaVersion = sharedVersion; + return true; } - logDebug("Authentication Service is {}", isUp); - return isUp; - } - private boolean isServiceUp(DocumentContext documentContext, String path) { - return documentContext.read(path).equals("UP"); - } + boolean areAllInstancesAreUp() { + List downInstances = new ArrayList<>(); + for (var instance : getInstancesHealthCheck()) { + var documentContext = getDocumentAsContext(URI.create(instance.getHealthEndpointUrl()), instance.getServiceConfiguration().isBasicAuthenticationSupported()); + String status = "N/A"; + if (documentContext != null) { + status = documentContext.read("status"); + } + if (!"UP".equals(status)) { + downInstances.add(String.format("%s (%s)", instance.getInstanceId(), status)); + } + } + if (!downInstances.isEmpty()) { + log.debug("Some instances are still down: {}", downInstances); + } + return downInstances.isEmpty(); + } - private void logDebug(String logMessage, boolean state) { - log.debug(logMessage, state ? "UP" : "DOWN"); - } + boolean isAuthUp() { + var serviceId = IS_MODULITH_ENABLED ? CoreService.GATEWAY : CoreService.ZAAS; + var key = "$.components.zaas.details.auth"; + List downZaasInstances = new ArrayList<>(); + for (var instance : get(serviceId)) { + var documentContext = getDocumentAsContext(URI.create(instance.getHealthEndpointUrl()), instance.getServiceConfiguration().isBasicAuthenticationSupported()); + var status = "N/A"; + if (documentContext != null) { + try { + status = documentContext.read(key); + } catch (Exception e) { + log.debug("Cannot parse {} on {}", key, instance.getInstanceId()); + } + } + if (!"UP".equals(status)) { + downZaasInstances.add(String.format("%s (%s)", instance.getInstanceId(), status)); + } + } + if (!downZaasInstances.isEmpty()) { + log.debug("Authentication Service is not ready: {}", downZaasInstances); + } + return downZaasInstances.isEmpty(); + } + + public void waitUntilReady() { + awaitFor(this::areAllInstancesOnboarded, 8); + awaitFor(this::areDiscoveryInSync, 3); + awaitFor(this::areAllInstancesRegistryUpToDate, 3); + awaitFor(this::areAllInstancesAreUp, 2); + awaitFor(this::isAuthUp, 1); + } + + public List get(CoreService type) { + return allInstances.stream().filter(instance -> type.getServiceId().equals(instance.getServiceId())).toList(); + } + + public static List load() { + String centralHostsConfig = System.getProperty("centralHosts"); + if (StringUtils.isBlank(centralHostsConfig)) { + var apiml = new ApimlInstance(x -> true); + log.debug("All instances = {}", apiml.allInstances); + return Collections.singletonList(apiml); + } + + var centralHosts = Arrays.stream(centralHostsConfig.split("[,;]")) + .map(String::trim) + .map(String::toLowerCase) + .collect(Collectors.toSet()); + var domain = new ApimlInstance(instance -> !centralHosts.contains(instance.getHostname().toLowerCase())); + var central = new ApimlInstance(instance -> centralHosts.contains(instance.getHostname().toLowerCase())); + central.additionalRegistration.addAll(domain.get(CoreService.GATEWAY)); + + log.debug("Domain = {}", domain.allInstances); + log.debug("Central = {}", central.allInstances); + + return Arrays.asList(domain, central); + } - @AllArgsConstructor - private class Service { - String name; - String path; } + } diff --git a/integration-tests/src/test/java/org/zowe/apiml/util/config/ApiCatalogServiceConfiguration.java b/integration-tests/src/test/java/org/zowe/apiml/util/config/ApiCatalogServiceConfiguration.java index 7e30cbb3f2..15a0ec3318 100644 --- a/integration-tests/src/test/java/org/zowe/apiml/util/config/ApiCatalogServiceConfiguration.java +++ b/integration-tests/src/test/java/org/zowe/apiml/util/config/ApiCatalogServiceConfiguration.java @@ -13,6 +13,7 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import org.zowe.apiml.product.constants.CoreService; @Data @AllArgsConstructor @@ -23,4 +24,15 @@ public class ApiCatalogServiceConfiguration implements ServiceConfiguration { private String host; private int port; private int instances; + + @Override + public String getServiceId() { + return CoreService.API_CATALOG.getServiceId(); + } + + @Override + public String getServletContext() { + return "/" + getServiceId() + "/"; + } + } diff --git a/integration-tests/src/test/java/org/zowe/apiml/util/config/CachingServiceConfiguration.java b/integration-tests/src/test/java/org/zowe/apiml/util/config/CachingServiceConfiguration.java index 3916496779..4421223e62 100644 --- a/integration-tests/src/test/java/org/zowe/apiml/util/config/CachingServiceConfiguration.java +++ b/integration-tests/src/test/java/org/zowe/apiml/util/config/CachingServiceConfiguration.java @@ -13,15 +13,26 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import org.zowe.apiml.product.constants.CoreService; @Data @AllArgsConstructor @NoArgsConstructor public class CachingServiceConfiguration implements ServiceConfiguration { - private String url; private String scheme; private String host; private int port; + private String url; + + @Override + public String getServiceId() { + return CoreService.CACHING.getServiceId(); + } + + @Override + public String getServletContext() { + return "/" + getServiceId() + "/"; + } } diff --git a/integration-tests/src/test/java/org/zowe/apiml/util/config/CentralGatewayServiceConfiguration.java b/integration-tests/src/test/java/org/zowe/apiml/util/config/CentralGatewayServiceConfiguration.java index 322b967735..5dda9b9c58 100644 --- a/integration-tests/src/test/java/org/zowe/apiml/util/config/CentralGatewayServiceConfiguration.java +++ b/integration-tests/src/test/java/org/zowe/apiml/util/config/CentralGatewayServiceConfiguration.java @@ -13,6 +13,7 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import org.zowe.apiml.product.constants.CoreService; @Data @AllArgsConstructor @@ -23,4 +24,9 @@ public class CentralGatewayServiceConfiguration implements ServiceConfiguration private String host; private int port; + @Override + public String getServiceId() { + return CoreService.GATEWAY.getServiceId(); + } + } diff --git a/integration-tests/src/test/java/org/zowe/apiml/util/config/DiscoverableClientConfiguration.java b/integration-tests/src/test/java/org/zowe/apiml/util/config/DiscoverableClientConfiguration.java index cb2d706cfa..840d2ca38a 100644 --- a/integration-tests/src/test/java/org/zowe/apiml/util/config/DiscoverableClientConfiguration.java +++ b/integration-tests/src/test/java/org/zowe/apiml/util/config/DiscoverableClientConfiguration.java @@ -21,9 +21,26 @@ @NoArgsConstructor @AllArgsConstructor public class DiscoverableClientConfiguration implements ServiceConfiguration { + private String scheme; private String applId; private String host; private int port; private int instances; + + @Override + public String getServiceId() { + return "discoverableclient"; + } + + @Override + public String getServletContext() { + return "/" + getServiceId() + "/"; + } + + @Override + public boolean isBasicAuthenticationSupported() { + return false; + } + } diff --git a/integration-tests/src/test/java/org/zowe/apiml/util/config/DiscoveryServiceConfiguration.java b/integration-tests/src/test/java/org/zowe/apiml/util/config/DiscoveryServiceConfiguration.java index 43843846eb..396c031ae6 100644 --- a/integration-tests/src/test/java/org/zowe/apiml/util/config/DiscoveryServiceConfiguration.java +++ b/integration-tests/src/test/java/org/zowe/apiml/util/config/DiscoveryServiceConfiguration.java @@ -13,6 +13,7 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import org.zowe.apiml.product.constants.CoreService; @Data @AllArgsConstructor @@ -26,5 +27,11 @@ public class DiscoveryServiceConfiguration implements ServiceConfiguration { private int port; private int additionalPort; private int instances; + + @Override + public String getServiceId() { + return CoreService.DISCOVERY.getServiceId(); + } + } diff --git a/integration-tests/src/test/java/org/zowe/apiml/util/config/GatewayServiceConfiguration.java b/integration-tests/src/test/java/org/zowe/apiml/util/config/GatewayServiceConfiguration.java index d67ba797f4..c37d8086fa 100644 --- a/integration-tests/src/test/java/org/zowe/apiml/util/config/GatewayServiceConfiguration.java +++ b/integration-tests/src/test/java/org/zowe/apiml/util/config/GatewayServiceConfiguration.java @@ -14,12 +14,14 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import org.zowe.apiml.product.constants.CoreService; @Data @AllArgsConstructor @NoArgsConstructor @JsonIgnoreProperties(ignoreUnknown = true) public class GatewayServiceConfiguration implements ServiceConfiguration { + private String scheme; private String host; private String dvipaHost; @@ -31,4 +33,9 @@ public class GatewayServiceConfiguration implements ServiceConfiguration { private int bucketCapacity; private String authProvider; private Integer connectionTimeout; + + public String getServiceId() { + return CoreService.GATEWAY.getServiceId(); + } + } diff --git a/integration-tests/src/test/java/org/zowe/apiml/util/config/ServiceConfiguration.java b/integration-tests/src/test/java/org/zowe/apiml/util/config/ServiceConfiguration.java index 58687487ae..d92f0b5005 100644 --- a/integration-tests/src/test/java/org/zowe/apiml/util/config/ServiceConfiguration.java +++ b/integration-tests/src/test/java/org/zowe/apiml/util/config/ServiceConfiguration.java @@ -13,7 +13,23 @@ public interface ServiceConfiguration { String getScheme(); + String getHost(); + int getPort(); + String getServiceId(); + + default boolean isStaticallyRegistred() { + return false; + } + + default String getServletContext() { + return "/"; + } + + default boolean isBasicAuthenticationSupported() { + return true; + } + } diff --git a/integration-tests/src/test/java/org/zowe/apiml/util/config/ZaasConfiguration.java b/integration-tests/src/test/java/org/zowe/apiml/util/config/ZaasConfiguration.java index 2933c6785f..9c9a522c33 100644 --- a/integration-tests/src/test/java/org/zowe/apiml/util/config/ZaasConfiguration.java +++ b/integration-tests/src/test/java/org/zowe/apiml/util/config/ZaasConfiguration.java @@ -13,6 +13,7 @@ import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import org.zowe.apiml.product.constants.CoreService; @Data @AllArgsConstructor @@ -22,4 +23,10 @@ public class ZaasConfiguration implements ServiceConfiguration { private String host; private int port; private int instances; + + @Override + public String getServiceId() { + return CoreService.ZAAS.getServiceId(); + } + } diff --git a/integration-tests/src/test/java/org/zowe/apiml/util/config/ZosmfServiceConfiguration.java b/integration-tests/src/test/java/org/zowe/apiml/util/config/ZosmfServiceConfiguration.java index 306a268d6a..690b49bc29 100644 --- a/integration-tests/src/test/java/org/zowe/apiml/util/config/ZosmfServiceConfiguration.java +++ b/integration-tests/src/test/java/org/zowe/apiml/util/config/ZosmfServiceConfiguration.java @@ -18,9 +18,16 @@ @AllArgsConstructor @NoArgsConstructor public class ZosmfServiceConfiguration implements ServiceConfiguration { + private String scheme; private String host; private int port; private String serviceId; private String contextRoot; + + @Override + public boolean isStaticallyRegistred() { + return true; + } + } diff --git a/mock-services/build.gradle b/mock-services/build.gradle index b9a6333285..1f73b394e7 100644 --- a/mock-services/build.gradle +++ b/mock-services/build.gradle @@ -86,6 +86,7 @@ dependencies { implementation libs.jjwt.impl implementation libs.jjwt.jackson implementation libs.nimbus.jose.jwt + implementation libs.spring.boot.starter.actuator testImplementation libs.spring.boot.starter.test diff --git a/mock-services/src/main/resources/application.yml b/mock-services/src/main/resources/application.yml index b194287ab8..09b09be80d 100644 --- a/mock-services/src/main/resources/application.yml +++ b/mock-services/src/main/resources/application.yml @@ -49,7 +49,12 @@ zss: "[zoweson@zowe.com]": USER "[auth0|690dc03fcb09ab253ab55dfe]": USER - +management: + endpoints: + web: + base-path: /application + exposure: + include: "*" --- spring.config.activate.on-profile: attlsServer diff --git a/onboarding-enabler-spring/src/main/java/org/zowe/apiml/enable/config/EnableApiDiscoveryConfig.java b/onboarding-enabler-spring/src/main/java/org/zowe/apiml/enable/config/EnableApiDiscoveryConfig.java index fbc7010df9..e35f786370 100644 --- a/onboarding-enabler-spring/src/main/java/org/zowe/apiml/enable/config/EnableApiDiscoveryConfig.java +++ b/onboarding-enabler-spring/src/main/java/org/zowe/apiml/enable/config/EnableApiDiscoveryConfig.java @@ -10,7 +10,7 @@ package org.zowe.apiml.enable.config; -import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.beans.factory.ObjectProvider; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.context.annotation.Bean; @@ -21,7 +21,6 @@ import org.zowe.apiml.eurekaservice.client.EurekaClientProvider; import org.zowe.apiml.eurekaservice.client.config.ApiMediationServiceConfig; import org.zowe.apiml.eurekaservice.client.impl.ApiMediationClientImpl; -import org.zowe.apiml.eurekaservice.client.impl.DiscoveryClientProvider; import org.zowe.apiml.message.core.MessageService; import org.zowe.apiml.message.yaml.YamlMessageServiceInstance; import org.zowe.apiml.product.logging.annotations.EnableApimlLogger; @@ -41,37 +40,18 @@ public MessageService messageServiceDiscovery() { return messageService; } - @ConditionalOnMissingBean({EurekaClientProvider.class, EurekaClientConfigProvider.class}) @Bean("apiMediationClient") - public ApiMediationClient defaultApiMediationClient() { - return new ApiMediationClientImpl(); - } - - @ConditionalOnBean(EurekaClientProvider.class) - @ConditionalOnMissingBean(EurekaClientConfigProvider.class) - @Bean("apiMediationClient") - public ApiMediationClient apiMediationClientWithProvider(EurekaClientProvider eurekaClientProvider) { - if (eurekaClientProvider == null) { - return new ApiMediationClientImpl(); - } - return new ApiMediationClientImpl(eurekaClientProvider); - } - - @ConditionalOnBean({EurekaClientProvider.class, EurekaClientConfigProvider.class}) - @Bean("apiMediationClient") - public ApiMediationClient apiMediationClientWithConfig(EurekaClientProvider eurekaClientProvider, EurekaClientConfigProvider eurekaClientConfigProvider) { - if (eurekaClientProvider != null) { - if (eurekaClientConfigProvider != null) { - return new ApiMediationClientImpl(eurekaClientProvider, eurekaClientConfigProvider); - } else { - return new ApiMediationClientImpl(eurekaClientProvider); - } + public ApiMediationClient apiMediationClient(ObjectProvider clientProvider, + ObjectProvider clientConfigProvider) { + EurekaClientProvider eurekaClientProvider = clientProvider.getIfAvailable(); + EurekaClientConfigProvider eurekaClientConfigProvider = clientConfigProvider.getIfAvailable(); + + if (eurekaClientProvider != null && eurekaClientConfigProvider != null) { + return new ApiMediationClientImpl(eurekaClientProvider, eurekaClientConfigProvider); + } else if (eurekaClientProvider != null) { + return new ApiMediationClientImpl(eurekaClientProvider); } else { - if (eurekaClientConfigProvider != null) { - return new ApiMediationClientImpl(new DiscoveryClientProvider(), eurekaClientConfigProvider); - } else { - return new ApiMediationClientImpl(); - } + return new ApiMediationClientImpl(); } } diff --git a/zaas-service/src/main/java/org/zowe/apiml/zaas/config/CacheConfig.java b/zaas-service/src/main/java/org/zowe/apiml/zaas/config/CacheConfig.java index 061d3f5e07..5d55da0ba3 100644 --- a/zaas-service/src/main/java/org/zowe/apiml/zaas/config/CacheConfig.java +++ b/zaas-service/src/main/java/org/zowe/apiml/zaas/config/CacheConfig.java @@ -28,6 +28,7 @@ import org.ehcache.jsr107.EhcacheCachingProvider; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.cache.CacheManager; @@ -97,9 +98,23 @@ public void afterPropertiesSet() { @Primary @Bean("cacheManager") + @ConditionalOnBean(name = "modulithConfig") @ConditionalOnProperty(value = "apiml.caching.enabled", havingValue = "true", matchIfMissing = true) @ConditionalOnProperty(name = "caching.storage.mode", havingValue = "inMemory", matchIfMissing = true) - public CacheManager cacheManager() { + public CacheManager cacheManagerModulith() { + return createCacheManager(); + } + + // fix for Redis IT (see setting CACHING_STORAGE_MODE='redis' for all service). This property is not related to microservices at all + @Primary + @Bean("cacheManager") + @ConditionalOnMissingBean(name = "modulithConfig") + @ConditionalOnProperty(value = "apiml.caching.enabled", havingValue = "true", matchIfMissing = true) + public CacheManager cacheManagerZaas() { + return createCacheManager(); + } + + public CacheManager createCacheManager() { var caches = new HashMap>(); var invalidatedJwtTokensConf = CacheConfigurationBuilder.newCacheConfigurationBuilder(