From a5fbabee0bfba536703239d0704eab0b7c42800e Mon Sep 17 00:00:00 2001 From: Steve Fan <19037626d@connect.polyu.hk> Date: Wed, 3 Nov 2021 17:25:01 +0800 Subject: [PATCH 1/3] Create maven.yml --- .github/workflows/maven.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/maven.yml diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml new file mode 100644 index 00000000..0951e926 --- /dev/null +++ b/.github/workflows/maven.yml @@ -0,0 +1,30 @@ +# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time +# For more information see: https://help.github.com/actions/language-and-framework-guides/building-and-testing-java-with-maven + +name: Java CI with Maven + +on: + push: + branches: [ vlab-dev ] + pull_request: + branches: [ vlab-dev ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up JDK 11 + uses: actions/setup-java@v2 + with: + java-version: '11' + distribution: 'adopt' + cache: maven + - name: Build with Maven + run: mvn package + - uses: actions/upload-artifact@v2 + with: + name: output + path: target/*.jar From 399ac665f8605a6a49a9b3097055abd6091c7bbe Mon Sep 17 00:00:00 2001 From: Steve Fan <19037626d@connect.polyu.hk> Date: Thu, 16 Dec 2021 20:36:17 +0800 Subject: [PATCH 2/3] add base dockerfile --- Dockerfile.base | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 Dockerfile.base diff --git a/Dockerfile.base b/Dockerfile.base new file mode 100644 index 00000000..57ac50a8 --- /dev/null +++ b/Dockerfile.base @@ -0,0 +1,21 @@ +# Example of custom Java runtime using jlink in a multi-stage container build +FROM eclipse-temurin:11-alpine as jre-build +RUN apk --no-cache add binutils + +# Create a custom Java runtime +RUN $JAVA_HOME/bin/jlink \ + --add-modules ALL-MODULE-PATH \ + --strip-debug \ + --no-man-pages \ + --no-header-files \ + --compress=2 \ + --output /javaruntime + +# Define your base image +FROM alpine:3.15.0 +ENV JAVA_HOME=/opt/java/openjdk +ENV PATH "${JAVA_HOME}/bin:${PATH}" +COPY --from=jre-build /javaruntime $JAVA_HOME +ENV VLAB_USER=vlab +RUN apk --no-cache add shadow fuse-overlayfs +RUN useradd -m -d /home/$VLAB_USER -s /bin/nologin $VLAB_USER From cd21123b6d365e75f3c195d26819877e343f69a2 Mon Sep 17 00:00:00 2001 From: Fan Chun Yin <19037626d@connect.polyu.hk> Date: Sun, 20 Mar 2022 00:12:59 +0800 Subject: [PATCH 3/3] FYP patch --- .gitignore | 25 + build.gradle.kts | 173 +++++ gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 59821 bytes gradle/wrapper/gradle-wrapper.properties | 5 + gradlew | 234 ++++++ gradlew.bat | 89 +++ pom.xml | 441 ------------ settings.gradle.kts | 14 + .../VLabControllerApplication.java | 163 +++-- .../VLabControllerConfiguration.java | 24 +- .../vlabcontroller/api/BaseController.java | 12 +- .../vlabcontroller/api/ConfigController.java | 9 +- .../vlabcontroller/api/ProxyController.java | 115 ++- .../api/ProxyRouteController.java | 42 +- .../auth/AuthenticationBackendFactory.java | 54 +- .../auth/IAuthenticationBackend.java | 6 +- .../auth/UserLogoutHandler.java | 6 +- .../impl/KeycloakAuthenticationBackend.java | 88 +-- .../impl/OpenIDAuthenticationBackend.java | 207 +++--- .../impl/SimpleAuthenticationBackend.java | 52 +- .../impl/WebServiceAuthenticationBackend.java | 59 +- .../AuthenticationFailureHandler.java | 4 +- .../backend/AbstractContainerBackend.java | 176 ++--- .../backend/ContainerBackendFactory.java | 43 +- .../backend/kubernetes/KubernetesBackend.java | 680 +++++++++--------- .../backend/kubernetes/PodPatcher.java | 32 +- .../impl/DefaultProxyLogoutStrategy.java | 12 +- .../impl/DefaultTargetMappingStrategy.java | 2 +- .../impl/RedisSessionProxyLogoutStrategy.java | 19 +- .../impl/URLConnectionTestStrategy.java | 98 +-- .../config/ProxyEngagementProperties.java | 20 + .../config/ProxyKeycloakProperties.java | 20 + .../config/ProxyKubernetesProperties.java | 31 + .../config/ProxyOAuth2Properties.java | 13 + .../config/ProxyOpenIDProperties.java | 23 + .../config/ProxyProperties.java | 101 +++ .../ProxyUsageStatsHikariProperties.java | 18 + .../config/ProxyUsageStatsProperties.java | 29 + .../config/ProxyUserProperties.java | 18 + .../config/ProxyWebServiceProperties.java | 13 + .../config/ServerProperties.java | 18 + .../controllers/AdminController.java | 72 +- .../controllers/AppController.java | 133 ++-- .../controllers/BaseController.java | 122 ++-- .../controllers/ControlPanelController.java | 30 +- .../controllers/FileBrowserController.java | 74 +- .../controllers/IndexController.java | 38 +- .../converter/QuantityConverter.java | 16 + .../vlabcontroller/entity/LabInstance.java | 23 + .../vlabcontroller/entity/SessionData.java | 18 + .../comp/vlabcontroller/entity/User.java | 25 + .../vlabcontroller/event/AuthFailedEvent.java | 9 +- .../vlabcontroller/event/ProxyStartEvent.java | 14 +- .../event/ProxyStartFailedEvent.java | 9 +- .../vlabcontroller/event/ProxyStopEvent.java | 14 +- .../vlabcontroller/event/UserLoginEvent.java | 9 +- .../vlabcontroller/event/UserLogoutEvent.java | 10 +- .../log/AbstractLogStorage.java | 21 +- .../vlabcontroller/log/FileLogStorage.java | 4 +- .../vlabcontroller/log/LogStorageFactory.java | 19 +- .../comp/vlabcontroller/log/S3LogStorage.java | 45 +- .../model/runtime/ContainerGroup.java | 16 +- .../model/runtime/HeartbeatStatus.java | 20 +- .../model/runtime/PortMappingMetadata.java | 13 +- .../vlabcontroller/model/runtime/Proxy.java | 34 +- .../model/runtime/ProxyMappingMetadata.java | 15 +- .../model/runtime/RuntimeSetting.java | 9 +- .../model/spec/ContainerSpec.java | 97 +-- .../model/spec/EntryPointSpec.java | 25 +- .../model/spec/EvaluatorSpec.java | 19 + .../vlabcontroller/model/spec/ProxySpec.java | 115 +-- .../model/spec/ProxySpecKubernetes.java | 38 +- .../model/spec/ResourceSpec.java | 20 +- .../model/spec/RuntimeSettingSpec.java | 24 +- .../repository/UserRepository.java | 14 + .../security/APISecurityConfig.java | 33 +- .../CustomFirewallSecurityConfig.java | 9 +- .../security/KeycloakRoleSecurityConfig.java | 22 +- .../security/UISecurityConfig.java | 24 +- .../security/WebSecurityConfig.java | 72 +- .../service/FileUpdateService.java | 46 +- .../service/HeartbeatService.java | 285 ++++---- .../vlabcontroller/service/LogService.java | 32 +- .../vlabcontroller/service/ProxyService.java | 97 ++- .../service/UserActionEventsListener.java | 76 ++ .../vlabcontroller/service/UserService.java | 152 ++-- .../spec/EngagementProperties.java | 38 - .../spec/FileBrowserProperties.java | 15 - .../spec/StatCollectorProperties.java | 28 - .../ExpressionAwareContainerSpec.java | 19 +- .../expression/SpecExpressionContext.java | 20 +- .../expression/SpecExpressionResolver.java | 34 +- .../spec/impl/DefaultSpecMergeStrategy.java | 36 +- .../spec/impl/DefaultSpecProvider.java | 51 -- .../impl/VLabControllerSpecMergeStrategy.java | 5 +- .../spec/setting/SettingSpecMapper.java | 9 +- .../spec/setting/SettingTypeRegistry.java | 10 +- .../setting/type/AbstractSettingType.java | 6 +- .../stat/StatCollectorRegistry.java | 44 +- .../stat/impl/AbstractDbCollector.java | 10 +- .../stat/impl/InfluxDBCollector.java | 29 +- .../stat/impl/JDBCCollector.java | 64 +- .../vlabcontroller/stat/impl/Micrometer.java | 29 +- .../vlabcontroller/ui/AuthController.java | 11 +- .../vlabcontroller/ui/ErrorController.java | 29 +- .../comp/vlabcontroller/ui/FaviconConfig.java | 30 +- .../ui/TemplateResolverConfig.java | 33 +- .../util/ChannelActiveListener.java | 16 +- .../vlabcontroller/util/ConfigFileHelper.java | 61 +- .../util/ConfigUpdateListener.java | 17 +- .../util/DelegatingStreamSinkConduit.java | 111 +-- .../util/DelegatingStreamSourceConduit.java | 103 +-- .../vlabcontroller/util/DurationUtil.java | 20 + .../vlabcontroller/util/PortAllocator.java | 20 +- .../util/ProxyMappingManager.java | 103 +-- .../vlabcontroller/util/RFC6335Validator.java | 7 + .../util/RedisSessionHelper.java | 9 +- .../comp/vlabcontroller/util/Retrying.java | 43 +- .../vlabcontroller/util/SessionHelper.java | 29 +- .../util/StartupEventListener.java | 11 +- .../model/runtime/MongoTest.java | 22 + .../runtime/ProxyMappingMetadataTest.java | 85 +-- 122 files changed, 3159 insertions(+), 3248 deletions(-) create mode 100644 build.gradle.kts create mode 100644 gradle/wrapper/gradle-wrapper.jar create mode 100644 gradle/wrapper/gradle-wrapper.properties create mode 100755 gradlew create mode 100644 gradlew.bat delete mode 100644 pom.xml create mode 100644 settings.gradle.kts create mode 100644 src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyEngagementProperties.java create mode 100644 src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyKeycloakProperties.java create mode 100644 src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyKubernetesProperties.java create mode 100644 src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyOAuth2Properties.java create mode 100644 src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyOpenIDProperties.java create mode 100644 src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyProperties.java create mode 100644 src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyUsageStatsHikariProperties.java create mode 100644 src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyUsageStatsProperties.java create mode 100644 src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyUserProperties.java create mode 100644 src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyWebServiceProperties.java create mode 100644 src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ServerProperties.java create mode 100644 src/main/java/hk/edu/polyu/comp/vlabcontroller/converter/QuantityConverter.java create mode 100644 src/main/java/hk/edu/polyu/comp/vlabcontroller/entity/LabInstance.java create mode 100644 src/main/java/hk/edu/polyu/comp/vlabcontroller/entity/SessionData.java create mode 100644 src/main/java/hk/edu/polyu/comp/vlabcontroller/entity/User.java create mode 100644 src/main/java/hk/edu/polyu/comp/vlabcontroller/model/spec/EvaluatorSpec.java create mode 100644 src/main/java/hk/edu/polyu/comp/vlabcontroller/repository/UserRepository.java create mode 100644 src/main/java/hk/edu/polyu/comp/vlabcontroller/service/UserActionEventsListener.java delete mode 100644 src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/EngagementProperties.java delete mode 100644 src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/FileBrowserProperties.java delete mode 100644 src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/StatCollectorProperties.java delete mode 100644 src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/impl/DefaultSpecProvider.java create mode 100644 src/main/java/hk/edu/polyu/comp/vlabcontroller/util/DurationUtil.java create mode 100644 src/main/java/hk/edu/polyu/comp/vlabcontroller/util/RFC6335Validator.java create mode 100644 src/test/java/hk/edu/polyu/comp/vlabcontroller/model/runtime/MongoTest.java diff --git a/.gitignore b/.gitignore index 74257689..60e2c583 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,28 @@ logs .idea *.iml + +.gradle +**/build/ +!src/**/build/ + +# Ignore Gradle GUI config +gradle-app.setting + +# Avoid ignoring Gradle wrapper jar file (.jar files are usually ignored) +!gradle-wrapper.jar + +# Avoid ignore Gradle wrappper properties +!gradle-wrapper.properties + +# Cache of project +.gradletasknamecache + +# Eclipse Gradle plugin generated files +# Eclipse Core +.project +# JDT-specific (Eclipse Java Development Tools) +.classpath + +*.log +*.gz \ No newline at end of file diff --git a/build.gradle.kts b/build.gradle.kts new file mode 100644 index 00000000..94e2a595 --- /dev/null +++ b/build.gradle.kts @@ -0,0 +1,173 @@ +//import com.ewerk.gradle.plugins.tasks.QuerydslCompile + +plugins { + application + idea + id("com.google.cloud.tools.jib") version "3.2.0" + id("io.freefair.lombok") version "6.4.1" + id("io.spring.dependency-management") version "1.0.11.RELEASE" + id("org.springframework.boot") version "2.6.6" +} + +group = "hk.edu.polyu.comp.vlabcontroller" +version = "1.0.3" +description = "VLabController" +java.sourceCompatibility = JavaVersion.VERSION_11 + +val springCloudVersion by extra("2021.0.1") + +configurations { + implementation.configure { + exclude(module = "spring-boot-starter-tomcat") + exclude("org.apache.tomcat") + } + compileOnly { + extendsFrom(configurations.annotationProcessor.get()) + } +} + +springBoot { + buildInfo() +} + +repositories { + maven(url = "https://repo.spring.io/release") + mavenCentral() +} + +dependencyManagement { + imports { + mavenBom("org.springframework.cloud:spring-cloud-dependencies:$springCloudVersion") + } +} + +dependencies { + var springBoot = run { + compileOnly("org.springframework.boot", "spring-boot-devtools") + implementation("org.springframework.boot", "spring-boot-configuration-processor") + implementation("org.springframework.boot", "spring-boot-starter-actuator") + implementation("org.springframework.boot", "spring-boot-starter-data-mongodb") + implementation("org.springframework.boot", "spring-boot-starter-data-redis") + implementation("org.springframework.boot", "spring-boot-starter-jdbc") + implementation("org.springframework.boot", "spring-boot-starter-mail") + implementation("org.springframework.boot", "spring-boot-starter-security") + implementation("org.springframework.boot", "spring-boot-starter-thymeleaf") + implementation("org.springframework.boot", "spring-boot-starter-undertow") + implementation("org.springframework.boot", "spring-boot-starter-web") + implementation("org.springframework.boot", "spring-boot-starter-websocket") + implementation("org.springframework.cloud", "spring-cloud-context") + implementation("org.springframework.data", "spring-data-commons") + implementation("org.springframework.security", "spring-security-oauth2-client") + implementation("org.springframework.security", "spring-security-oauth2-jose") + implementation("org.springframework.security.oauth.boot", "spring-security-oauth2-autoconfigure") + implementation("org.springframework.session", "spring-session-data-redis") + +// compile("org.springframework.data:spring-data-mongodb") + + testImplementation("org.springframework.boot", "spring-boot-starter-test") + testImplementation("org.springframework.boot", "spring-boot-starter-webflux") + testImplementation("org.springframework.security", "spring-security-test") + } + + var database = run { + implementation("mysql", "mysql-connector-java", "8.0.27") + implementation("org.postgresql", "postgresql", "42.2.24") + implementation("org.xerial", "sqlite-jdbc", "3.36.0.3") + implementation("org.mongodb:mongodb-driver-sync:4.4.2") + implementation("org.mongodb:bson:4.4.2") + } + + var javax = run { + implementation("javax.inject", "javax.inject", "1") + implementation("javax.json", "javax.json-api", "1.1.4") + implementation("javax.xml.bind", "jaxb-api", "2.3.1") + } + + var queryDsl = run { + annotationProcessor("com.querydsl:querydsl-apt:5.0.0:general") + implementation("com.querydsl:querydsl-mongodb") + } + + implementation("com.amazonaws", "aws-java-sdk-s3", "1.12.90") + implementation("com.fasterxml.jackson.datatype", "jackson-datatype-jsr353", "2.13.0") + implementation("com.google.guava", "guava", "31.1-jre") + implementation("io.fabric8", "kubernetes-client", "5.9.0") + implementation("io.micrometer", "micrometer-registry-influx", "1.7.5") + implementation("io.micrometer", "micrometer-registry-prometheus", "1.7.5") + implementation("io.vavr", "vavr", "0.10.4") + implementation("org.apache.commons", "commons-lang3", "3.12.0") + implementation("org.glassfish", "javax.json", "1.1.4") + implementation("org.jboss.xnio", "xnio-api", "3.8.4.Final") + implementation("org.keycloak", "keycloak-spring-security-adapter", "15.0.2") + implementation("org.thymeleaf.extras", "thymeleaf-extras-springsecurity5", "3.0.4.RELEASE") + implementation("com.ea.async:ea-async:1.2.3") + + testImplementation("junit", "junit", "4.13.2") +} + +jib { + from { + image = "ghcr.io/stevefan1999/vlab-controller-base" + } + to { + image = "ghcr.io/endangeredf1sh/vlab-controller:$version" + auth { + username = System.getenv("REGISTRY_USERNAME") + password = System.getenv("REGISTRY_PASSWORD") + } + } + container { + appRoot = "/opt/vlab-controller" + workingDirectory = "/opt/vlab-controller" + environment = mapOf( + "VLAB_USER" to "vlab", + "PROXY_TEMPLATEPATH" to "/opt/vlab-controller/resources/templates", + "SERVER_ERROR_WHITELABEL_ENABLED" to "false", + "TZ" to "Asia/Hong_Kong" + ) + labels.put("maintainer", mapOf( + "Aiden ZHANG Wenyi" to "im.endangeredfish@gmail.com", + "Fan Chun Yin" to "stevefan1999@gmail.com" + ).map { "${it.key} <${it.value}>" }.joinToString { "," }) + user = "vlab:vlab" + args = listOf( + "--spring.jmx.enabled=false", + "--spring.config.location=/etc/vlab-controller/config/application.yml" + ) + jvmFlags = listOf( + "-server", + "-Djava.awt.headless=true", + "-XX:+UseStringDeduplication" + ) + } + extraDirectories { + paths { + path { + setFrom("resources/templates") + into = "/opt/vlab-controller/resources/templates" + } + } + } +} + +tasks.withType { + options.encoding = "UTF-8" +} + +tasks.withType { + useJUnitPlatform() +} + +tasks.getByName("jar") { + enabled = false +} + +val runEaAsyncInstrumentation by tasks.registering(JavaExec::class) { + mainClass.set("com.ea.async.instrumentation.Main") + classpath = sourceSets.main.get().compileClasspath + args = listOf(buildDir.path) +} + +val compileJava by tasks.existing(JavaCompile::class) { + finalizedBy(runEaAsyncInstrumentation) +} \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000000000000000000000000000000000000..41d9927a4d4fb3f96a785543079b8df6723c946b GIT binary patch literal 59821 zcma&NV|1p`(k7gaZQHhOJ9%QKV?D8LCmq{1JGRYE(y=?XJw0>InKkE~^UnAEs2gk5 zUVGPCwX3dOb!}xiFmPB95NK!+5D<~S0s;d1zn&lrfAn7 zC?Nb-LFlib|DTEqB8oDS5&$(u1<5;wsY!V`2F7^=IR@I9so5q~=3i_(hqqG<9SbL8Q(LqDrz+aNtGYWGJ2;p*{a-^;C>BfGzkz_@fPsK8{pTT~_VzB$E`P@> z7+V1WF2+tSW=`ZRj3&0m&d#x_lfXq`bb-Y-SC-O{dkN2EVM7@!n|{s+2=xSEMtW7( zz~A!cBpDMpQu{FP=y;sO4Le}Z)I$wuFwpugEY3vEGfVAHGqZ-<{vaMv-5_^uO%a{n zE_Zw46^M|0*dZ`;t%^3C19hr=8FvVdDp1>SY>KvG!UfD`O_@weQH~;~W=fXK_!Yc> z`EY^PDJ&C&7LC;CgQJeXH2 zjfM}2(1i5Syj)Jj4EaRyiIl#@&lC5xD{8hS4Wko7>J)6AYPC-(ROpVE-;|Z&u(o=X z2j!*>XJ|>Lo+8T?PQm;SH_St1wxQPz)b)Z^C(KDEN$|-6{A>P7r4J1R-=R7|FX*@! zmA{Ja?XE;AvisJy6;cr9Q5ovphdXR{gE_7EF`ji;n|RokAJ30Zo5;|v!xtJr+}qbW zY!NI6_Wk#6pWFX~t$rAUWi?bAOv-oL6N#1>C~S|7_e4 zF}b9(&a*gHk+4@J26&xpiWYf2HN>P;4p|TD4f586umA2t@cO1=Fx+qd@1Ae#Le>{-?m!PnbuF->g3u)7(n^llJfVI%Q2rMvetfV5 z6g|sGf}pV)3_`$QiKQnqQ<&ghOWz4_{`rA1+7*M0X{y(+?$|{n zs;FEW>YzUWg{sO*+D2l6&qd+$JJP_1Tm;To<@ZE%5iug8vCN3yH{!6u5Hm=#3HJ6J zmS(4nG@PI^7l6AW+cWAo9sFmE`VRcM`sP7X$^vQY(NBqBYU8B|n-PrZdNv8?K?kUTT3|IE`-A8V*eEM2=u*kDhhKsmVPWGns z8QvBk=BPjvu!QLtlF0qW(k+4i+?H&L*qf262G#fks9}D5-L{yiaD10~a;-j!p!>5K zl@Lh+(9D{ePo_S4F&QXv|q_yT`GIPEWNHDD8KEcF*2DdZD;=J6u z|8ICSoT~5Wd!>g%2ovFh`!lTZhAwpIbtchDc{$N%<~e$E<7GWsD42UdJh1fD($89f2on`W`9XZJmr*7lRjAA8K0!(t8-u>2H*xn5cy1EG{J;w;Q-H8Yyx+WW(qoZZM7p(KQx^2-yI6Sw?k<=lVOVwYn zY*eDm%~=|`c{tUupZ^oNwIr!o9T;H3Fr|>NE#By8SvHb&#;cyBmY1LwdXqZwi;qn8 zK+&z{{95(SOPXAl%EdJ3jC5yV^|^}nOT@M0)|$iOcq8G{#*OH7=DlfOb; z#tRO#tcrc*yQB5!{l5AF3(U4>e}nEvkoE_XCX=a3&A6Atwnr&`r&f2d%lDr8f?hBB zr1dKNypE$CFbT9I?n){q<1zHmY>C=5>9_phi79pLJG)f=#dKdQ7We8emMjwR*qIMF zE_P-T*$hX#FUa%bjv4Vm=;oxxv`B*`weqUn}K=^TXjJG=UxdFMSj-QV6fu~;- z|IsUq`#|73M%Yn;VHJUbt<0UHRzbaF{X@76=8*-IRx~bYgSf*H(t?KH=?D@wk*E{| z2@U%jKlmf~C^YxD=|&H?(g~R9-jzEb^y|N5d`p#2-@?BUcHys({pUz4Zto7XwKq2X zSB~|KQGgv_Mh@M!*{nl~2~VV_te&E7K39|WYH zCxfd|v_4!h$Ps2@atm+gj14Ru)DhivY&(e_`eA)!O1>nkGq|F-#-6oo5|XKEfF4hR z%{U%ar7Z8~B!foCd_VRHr;Z1c0Et~y8>ZyVVo9>LLi(qb^bxVkbq-Jq9IF7!FT`(- zTMrf6I*|SIznJLRtlP)_7tQ>J`Um>@pP=TSfaPB(bto$G1C zx#z0$=zNpP-~R);kM4O)9Mqn@5Myv5MmmXOJln312kq#_94)bpSd%fcEo7cD#&|<` zrcal$(1Xv(nDEquG#`{&9Ci~W)-zd_HbH-@2F6+|a4v}P!w!Q*h$#Zu+EcZeY>u&?hn#DCfC zVuye5@Ygr+T)0O2R1*Hvlt>%rez)P2wS}N-i{~IQItGZkp&aeY^;>^m7JT|O^{`78 z$KaK0quwcajja;LU%N|{`2o&QH@u%jtH+j!haGj;*ZCR*`UgOXWE>qpXqHc?g&vA& zt-?_g8k%ZS|D;()0Lf!>7KzTSo-8hUh%OA~i76HKRLudaNiwo*E9HxmzN4y>YpZNO zUE%Q|H_R_UmX=*f=2g=xyP)l-DP}kB@PX|(Ye$NOGN{h+fI6HVw`~Cd0cKqO;s6aiYLy7sl~%gs`~XaL z^KrZ9QeRA{O*#iNmB7_P!=*^pZiJ5O@iE&X2UmUCPz!)`2G3)5;H?d~3#P|)O(OQ_ zua+ZzwWGkWflk4j^Lb=x56M75_p9M*Q50#(+!aT01y80x#rs9##!;b-BH?2Fu&vx} za%4!~GAEDsB54X9wCF~juV@aU}fp_(a<`Ig0Pip8IjpRe#BR?-niYcz@jI+QY zBU9!8dAfq@%p;FX)X=E7?B=qJJNXlJ&7FBsz;4&|*z{^kEE!XbA)(G_O6I9GVzMAF z8)+Un(6od`W7O!!M=0Z)AJuNyN8q>jNaOdC-zAZ31$Iq%{c_SYZe+(~_R`a@ zOFiE*&*o5XG;~UjsuW*ja-0}}rJdd@^VnQD!z2O~+k-OSF%?hqcFPa4e{mV1UOY#J zTf!PM=KMNAzbf(+|AL%K~$ahX0Ol zbAxKu3;v#P{Qia{_WzHl`!@!8c#62XSegM{tW1nu?Ee{sQq(t{0TSq67YfG;KrZ$n z*$S-+R2G?aa*6kRiTvVxqgUhJ{ASSgtepG3hb<3hlM|r>Hr~v_DQ>|Nc%&)r0A9go z&F3Ao!PWKVq~aWOzLQIy&R*xo>}{UTr}?`)KS&2$3NR@a+>+hqK*6r6Uu-H};ZG^| zfq_Vl%YE1*uGwtJ>H*Y(Q9E6kOfLJRlrDNv`N;jnag&f<4#UErM0ECf$8DASxMFF& zK=mZgu)xBz6lXJ~WZR7OYw;4&?v3Kk-QTs;v1r%XhgzSWVf|`Sre2XGdJb}l1!a~z zP92YjnfI7OnF@4~g*LF>G9IZ5c+tifpcm6#m)+BmnZ1kz+pM8iUhwag`_gqr(bnpy zl-noA2L@2+?*7`ZO{P7&UL~ahldjl`r3=HIdo~Hq#d+&Q;)LHZ4&5zuDNug@9-uk; z<2&m#0Um`s=B}_}9s&70Tv_~Va@WJ$n~s`7tVxi^s&_nPI0`QX=JnItlOu*Tn;T@> zXsVNAHd&K?*u~a@u8MWX17VaWuE0=6B93P2IQ{S$-WmT+Yp!9eA>@n~=s>?uDQ4*X zC(SxlKap@0R^z1p9C(VKM>nX8-|84nvIQJ-;9ei0qs{}X>?f%&E#%-)Bpv_p;s4R+ z;PMpG5*rvN&l;i{^~&wKnEhT!S!LQ>udPzta#Hc9)S8EUHK=%x+z@iq!O{)*XM}aI zBJE)vokFFXTeG<2Pq}5Na+kKnu?Ch|YoxdPb&Z{07nq!yzj0=xjzZj@3XvwLF0}Pa zn;x^HW504NNfLY~w!}5>`z=e{nzGB>t4ntE>R}r7*hJF3OoEx}&6LvZz4``m{AZxC zz6V+^73YbuY>6i9ulu)2`ozP(XBY5n$!kiAE_Vf4}Ih)tlOjgF3HW|DF+q-jI_0p%6Voc^e;g28* z;Sr4X{n(X7eEnACWRGNsHqQ_OfWhAHwnSQ87@PvPcpa!xr9`9+{QRn;bh^jgO8q@v zLekO@-cdc&eOKsvXs-eMCH8Y{*~3Iy!+CANy+(WXYS&6XB$&1+tB?!qcL@@) zS7XQ|5=o1fr8yM7r1AyAD~c@Mo`^i~hjx{N17%pDX?j@2bdBEbxY}YZxz!h#)q^1x zpc_RnoC3`V?L|G2R1QbR6pI{Am?yW?4Gy`G-xBYfebXvZ=(nTD7u?OEw>;vQICdPJBmi~;xhVV zisVvnE!bxI5|@IIlDRolo_^tc1{m)XTbIX^<{TQfsUA1Wv(KjJED^nj`r!JjEA%MaEGqPB z9YVt~ol3%e`PaqjZt&-)Fl^NeGmZ)nbL;92cOeLM2H*r-zA@d->H5T_8_;Jut0Q_G zBM2((-VHy2&eNkztIpHk&1H3M3@&wvvU9+$RO%fSEa_d5-qZ!<`-5?L9lQ1@AEpo* z3}Zz~R6&^i9KfRM8WGc6fTFD%PGdruE}`X$tP_*A)_7(uI5{k|LYc-WY*%GJ6JMmw zNBT%^E#IhekpA(i zcB$!EB}#>{^=G%rQ~2;gbObT9PQ{~aVx_W6?(j@)S$&Ja1s}aLT%A*mP}NiG5G93- z_DaRGP77PzLv0s32{UFm##C2LsU!w{vHdKTM1X)}W%OyZ&{3d^2Zu-zw?fT=+zi*q z^fu6CXQ!i?=ljsqSUzw>g#PMk>(^#ejrYp(C)7+@Z1=Mw$Rw!l8c9}+$Uz;9NUO(kCd#A1DX4Lbis0k; z?~pO(;@I6Ajp}PL;&`3+;OVkr3A^dQ(j?`by@A!qQam@_5(w6fG>PvhO`#P(y~2ue zW1BH_GqUY&>PggMhhi@8kAY;XWmj>y1M@c`0v+l~l0&~Kd8ZSg5#46wTLPo*Aom-5 z>qRXyWl}Yda=e@hJ%`x=?I42(B0lRiR~w>n6p8SHN~B6Y>W(MOxLpv>aB)E<1oEcw z%X;#DJpeDaD;CJRLX%u!t23F|cv0ZaE183LXxMq*uWn)cD_ zp!@i5zsmcxb!5uhp^@>U;K>$B|8U@3$65CmhuLlZ2(lF#hHq-<<+7ZN9m3-hFAPgA zKi;jMBa*59ficc#TRbH_l`2r>z(Bm_XEY}rAwyp~c8L>{A<0@Q)j*uXns^q5z~>KI z)43=nMhcU1ZaF;CaBo>hl6;@(2#9yXZ7_BwS4u>gN%SBS<;j{{+p}tbD8y_DFu1#0 zx)h&?`_`=ti_6L>VDH3>PPAc@?wg=Omdoip5j-2{$T;E9m)o2noyFW$5dXb{9CZ?c z);zf3U526r3Fl+{82!z)aHkZV6GM@%OKJB5mS~JcDjieFaVn}}M5rtPnHQVw0Stn- zEHs_gqfT8(0b-5ZCk1%1{QQaY3%b>wU z7lyE?lYGuPmB6jnMI6s$1uxN{Tf_n7H~nKu+h7=%60WK-C&kEIq_d4`wU(*~rJsW< zo^D$-(b0~uNVgC+$J3MUK)(>6*k?92mLgpod{Pd?{os+yHr&t+9ZgM*9;dCQBzE!V zk6e6)9U6Bq$^_`E1xd}d;5O8^6?@bK>QB&7l{vAy^P6FOEO^l7wK4K=lLA45gQ3$X z=$N{GR1{cxO)j;ZxKI*1kZIT9p>%FhoFbRK;M(m&bL?SaN zzkZS9xMf={o@gpG%wE857u@9dq>UKvbaM1SNtMA9EFOp7$BjJQVkIm$wU?-yOOs{i z1^(E(WwZZG{_#aIzfpGc@g5-AtK^?Q&vY#CtVpfLbW?g0{BEX4Vlk(`AO1{-D@31J zce}#=$?Gq+FZG-SD^z)-;wQg9`qEO}Dvo+S9*PUB*JcU)@S;UVIpN7rOqXmEIerWo zP_lk!@RQvyds&zF$Rt>N#_=!?5{XI`Dbo0<@>fIVgcU*9Y+ z)}K(Y&fdgve3ruT{WCNs$XtParmvV;rjr&R(V&_#?ob1LzO0RW3?8_kSw)bjom#0; zeNllfz(HlOJw012B}rgCUF5o|Xp#HLC~of%lg+!pr(g^n;wCX@Yk~SQOss!j9f(KL zDiI1h#k{po=Irl)8N*KU*6*n)A8&i9Wf#7;HUR^5*6+Bzh;I*1cICa|`&`e{pgrdc zs}ita0AXb$c6{tu&hxmT0faMG0GFc)unG8tssRJd%&?^62!_h_kn^HU_kBgp$bSew zqu)M3jTn;)tipv9Wt4Ll#1bmO2n?^)t^ZPxjveoOuK89$oy4(8Ujw{nd*Rs*<+xFi z{k*9v%sl?wS{aBSMMWdazhs0#gX9Has=pi?DhG&_0|cIyRG7c`OBiVG6W#JjYf7-n zIQU*Jc+SYnI8oG^Q8So9SP_-w;Y00$p5+LZ{l+81>v7|qa#Cn->312n=YQd$PaVz8 zL*s?ZU*t-RxoR~4I7e^c!8TA4g>w@R5F4JnEWJpy>|m5la2b#F4d*uoz!m=i1;`L` zB(f>1fAd~;*wf%GEbE8`EA>IO9o6TdgbIC%+en!}(C5PGYqS0{pa?PD)5?ds=j9{w za9^@WBXMZ|D&(yfc~)tnrDd#*;u;0?8=lh4%b-lFPR3ItwVJp};HMdEw#SXg>f-zU zEiaj5H=jzRSy(sWVd%hnLZE{SUj~$xk&TfheSch#23)YTcjrB+IVe0jJqsdz__n{- zC~7L`DG}-Dgrinzf7Jr)e&^tdQ}8v7F+~eF*<`~Vph=MIB|YxNEtLo1jXt#9#UG5` zQ$OSk`u!US+Z!=>dGL>%i#uV<5*F?pivBH@@1idFrzVAzttp5~>Y?D0LV;8Yv`wAa{hewVjlhhBM z_mJhU9yWz9Jexg@G~dq6EW5^nDXe(sU^5{}qbd0*yW2Xq6G37f8{{X&Z>G~dUGDFu zgmsDDZZ5ZmtiBw58CERFPrEG>*)*`_B75!MDsOoK`T1aJ4GZ1avI?Z3OX|Hg?P(xy zSPgO$alKZuXd=pHP6UZy0G>#BFm(np+dekv0l6gd=36FijlT8^kI5; zw?Z*FPsibF2d9T$_L@uX9iw*>y_w9HSh8c=Rm}f>%W+8OS=Hj_wsH-^actull3c@!z@R4NQ4qpytnwMaY z)>!;FUeY?h2N9tD(othc7Q=(dF zZAX&Y1ac1~0n(z}!9{J2kPPnru1?qteJPvA2m!@3Zh%+f1VQt~@leK^$&ZudOpS!+ zw#L0usf!?Df1tB?9=zPZ@q2sG!A#9 zKZL`2cs%|Jf}wG=_rJkwh|5Idb;&}z)JQuMVCZSH9kkG%zvQO01wBN)c4Q`*xnto3 zi7TscilQ>t_SLij{@Fepen*a(`upw#RJAx|JYYXvP1v8f)dTHv9pc3ZUwx!0tOH?c z^Hn=gfjUyo!;+3vZhxNE?LJgP`qYJ`J)umMXT@b z{nU(a^xFfofcxfHN-!Jn*{Dp5NZ&i9#9r{)s^lUFCzs5LQL9~HgxvmU#W|iNs0<3O z%Y2FEgvts4t({%lfX1uJ$w{JwfpV|HsO{ZDl2|Q$-Q?UJd`@SLBsMKGjFFrJ(s?t^ z2Llf`deAe@YaGJf)k2e&ryg*m8R|pcjct@rOXa=64#V9!sp=6tC#~QvYh&M~zmJ;% zr*A}V)Ka^3JE!1pcF5G}b&jdrt;bM^+J;G^#R08x@{|ZWy|547&L|k6)HLG|sN<~o z?y`%kbfRN_vc}pwS!Zr}*q6DG7;be0qmxn)eOcD%s3Wk`=@GM>U3ojhAW&WRppi0e zudTj{ufwO~H7izZJmLJD3uPHtjAJvo6H=)&SJ_2%qRRECN#HEU_RGa(Pefk*HIvOH zW7{=Tt(Q(LZ6&WX_Z9vpen}jqge|wCCaLYpiw@f_%9+-!l{kYi&gT@Cj#D*&rz1%e z@*b1W13bN8^j7IpAi$>`_0c!aVzLe*01DY-AcvwE;kW}=Z{3RJLR|O~^iOS(dNEnL zJJ?Dv^ab++s2v!4Oa_WFDLc4fMspglkh;+vzg)4;LS{%CR*>VwyP4>1Tly+!fA-k? z6$bg!*>wKtg!qGO6GQ=cAmM_RC&hKg$~(m2LdP{{*M+*OVf07P$OHp*4SSj9H;)1p z^b1_4p4@C;8G7cBCB6XC{i@vTB3#55iRBZiml^jc4sYnepCKUD+~k}TiuA;HWC6V3 zV{L5uUAU9CdoU+qsFszEwp;@d^!6XnX~KI|!o|=r?qhs`(-Y{GfO4^d6?8BC0xonf zKtZc1C@dNu$~+p#m%JW*J7alfz^$x`U~)1{c7svkIgQ3~RK2LZ5;2TAx=H<4AjC8{ z;)}8OfkZy7pSzVsdX|wzLe=SLg$W1+`Isf=o&}npxWdVR(i8Rr{uzE516a@28VhVr zVgZ3L&X(Q}J0R2{V(}bbNwCDD5K)<5h9CLM*~!xmGTl{Mq$@;~+|U*O#nc^oHnFOy z9Kz%AS*=iTBY_bSZAAY6wXCI?EaE>8^}WF@|}O@I#i69ljjWQPBJVk zQ_rt#J56_wGXiyItvAShJpLEMtW_)V5JZAuK#BAp6bV3K;IkS zK0AL(3ia99!vUPL#j>?<>mA~Q!mC@F-9I$9Z!96ZCSJO8FDz1SP3gF~m`1c#y!efq8QN}eHd+BHwtm%M5586jlU8&e!CmOC z^N_{YV$1`II$~cTxt*dV{-yp61nUuX5z?N8GNBuZZR}Uy_Y3_~@Y3db#~-&0TX644OuG^D3w_`?Yci{gTaPWST8`LdE)HK5OYv>a=6B%R zw|}>ngvSTE1rh`#1Rey0?LXTq;bCIy>TKm^CTV4BCSqdpx1pzC3^ca*S3fUBbKMzF z6X%OSdtt50)yJw*V_HE`hnBA)1yVN3Ruq3l@lY;%Bu+Q&hYLf_Z@fCUVQY-h4M3)- zE_G|moU)Ne0TMjhg?tscN7#ME6!Rb+y#Kd&-`!9gZ06o3I-VX1d4b1O=bpRG-tDK0 zSEa9y46s7QI%LmhbU3P`RO?w#FDM(}k8T`&>OCU3xD=s5N7}w$GntXF;?jdVfg5w9OR8VPxp5{uw zD+_;Gb}@7Vo_d3UV7PS65%_pBUeEwX_Hwfe2e6Qmyq$%0i8Ewn%F7i%=CNEV)Qg`r|&+$ zP6^Vl(MmgvFq`Zb715wYD>a#si;o+b4j^VuhuN>+sNOq6Qc~Y;Y=T&!Q4>(&^>Z6* zwliz!_16EDLTT;v$@W(s7s0s zi*%p>q#t)`S4j=Ox_IcjcllyT38C4hr&mlr6qX-c;qVa~k$MG;UqdnzKX0wo0Xe-_)b zrHu1&21O$y5828UIHI@N;}J@-9cpxob}zqO#!U%Q*ybZ?BH#~^fOT_|8&xAs_rX24 z^nqn{UWqR?MlY~klh)#Rz-*%&e~9agOg*fIN`P&v!@gcO25Mec23}PhzImkdwVT|@ zFR9dYYmf&HiUF4xO9@t#u=uTBS@k*97Z!&hu@|xQnQDkLd!*N`!0JN7{EUoH%OD85 z@aQ2(w-N)1_M{;FV)C#(a4p!ofIA3XG(XZ2E#%j_(=`IWlJAHWkYM2&(+yY|^2TB0 z>wfC-+I}`)LFOJ%KeBb1?eNxGKeq?AI_eBE!M~$wYR~bB)J3=WvVlT8ZlF2EzIFZt zkaeyj#vmBTGkIL9mM3cEz@Yf>j=82+KgvJ-u_{bBOxE5zoRNQW3+Ahx+eMGem|8xo zL3ORKxY_R{k=f~M5oi-Z>5fgqjEtzC&xJEDQ@`<)*Gh3UsftBJno-y5Je^!D?Im{j za*I>RQ=IvU@5WKsIr?kC$DT+2bgR>8rOf3mtXeMVB~sm%X7W5`s=Tp>FR544tuQ>9qLt|aUSv^io&z93luW$_OYE^sf8DB?gx z4&k;dHMWph>Z{iuhhFJr+PCZ#SiZ9e5xM$A#0yPtVC>yk&_b9I676n|oAH?VeTe*1 z@tDK}QM-%J^3Ns6=_vh*I8hE?+=6n9nUU`}EX|;Mkr?6@NXy8&B0i6h?7%D=%M*Er zivG61Wk7e=v;<%t*G+HKBqz{;0Biv7F+WxGirONRxJij zon5~(a`UR%uUzfEma99QGbIxD(d}~oa|exU5Y27#4k@N|=hE%Y?Y3H%rcT zHmNO#ZJ7nPHRG#y-(-FSzaZ2S{`itkdYY^ZUvyw<7yMBkNG+>$Rfm{iN!gz7eASN9-B3g%LIEyRev|3)kSl;JL zX7MaUL_@~4ot3$woD0UA49)wUeu7#lj77M4ar8+myvO$B5LZS$!-ZXw3w;l#0anYz zDc_RQ0Ome}_i+o~H=CkzEa&r~M$1GC!-~WBiHiDq9Sdg{m|G?o7g`R%f(Zvby5q4; z=cvn`M>RFO%i_S@h3^#3wImmWI4}2x4skPNL9Am{c!WxR_spQX3+;fo!y(&~Palyjt~Xo0uy6d%sX&I`e>zv6CRSm)rc^w!;Y6iVBb3x@Y=`hl9jft zXm5vilB4IhImY5b->x{!MIdCermpyLbsalx8;hIUia%*+WEo4<2yZ6`OyG1Wp%1s$ zh<|KrHMv~XJ9dC8&EXJ`t3ETz>a|zLMx|MyJE54RU(@?K&p2d#x?eJC*WKO9^d17# zdTTKx-Os3k%^=58Sz|J28aCJ}X2-?YV3T7ee?*FoDLOC214J4|^*EX`?cy%+7Kb3(@0@!Q?p zk>>6dWjF~y(eyRPqjXqDOT`4^Qv-%G#Zb2G?&LS-EmO|ixxt79JZlMgd^~j)7XYQ; z62rGGXA=gLfgy{M-%1gR87hbhxq-fL)GSfEAm{yLQP!~m-{4i_jG*JsvUdqAkoc#q6Yd&>=;4udAh#?xa2L z7mFvCjz(hN7eV&cyFb%(U*30H@bQ8-b7mkm!=wh2|;+_4vo=tyHPQ0hL=NR`jbsSiBWtG ztMPPBgHj(JTK#0VcP36Z`?P|AN~ybm=jNbU=^3dK=|rLE+40>w+MWQW%4gJ`>K!^- zx4kM*XZLd(E4WsolMCRsdvTGC=37FofIyCZCj{v3{wqy4OXX-dZl@g`Dv>p2`l|H^ zS_@(8)7gA62{Qfft>vx71stILMuyV4uKb7BbCstG@|e*KWl{P1$=1xg(7E8MRRCWQ1g)>|QPAZot~|FYz_J0T+r zTWTB3AatKyUsTXR7{Uu) z$1J5SSqoJWt(@@L5a)#Q6bj$KvuC->J-q1!nYS6K5&e7vNdtj- zj9;qwbODLgIcObqNRGs1l{8>&7W?BbDd!87=@YD75B2ep?IY|gE~t)$`?XJ45MG@2 zz|H}f?qtEb_p^Xs$4{?nA=Qko3Lc~WrAS`M%9N60FKqL7XI+v_5H-UDiCbRm`fEmv z$pMVH*#@wQqml~MZe+)e4Ts3Gl^!Z0W3y$;|9hI?9(iw29b7en0>Kt2pjFXk@!@-g zTb4}Kw!@u|V!wzk0|qM*zj$*-*}e*ZXs#Y<6E_!BR}3^YtjI_byo{F+w9H9?f%mnBh(uE~!Um7)tgp2Ye;XYdVD95qt1I-fc@X zXHM)BfJ?^g(s3K|{N8B^hamrWAW|zis$`6|iA>M-`0f+vq(FLWgC&KnBDsM)_ez1# zPCTfN8{s^K`_bum2i5SWOn)B7JB0tzH5blC?|x;N{|@ch(8Uy-O{B2)OsfB$q0@FR z27m3YkcVi$KL;;4I*S;Z#6VfZcZFn!D2Npv5pio)sz-`_H*#}ROd7*y4i(y(YlH<4 zh4MmqBe^QV_$)VvzWgMXFy`M(vzyR2u!xx&%&{^*AcVLrGa8J9ycbynjKR~G6zC0e zlEU>zt7yQtMhz>XMnz>ewXS#{Bulz$6HETn?qD5v3td>`qGD;Y8&RmkvN=24=^6Q@DYY zxMt}uh2cSToMkkIWo1_Lp^FOn$+47JXJ*#q=JaeiIBUHEw#IiXz8cStEsw{UYCA5v_%cF@#m^Y!=+qttuH4u}r6gMvO4EAvjBURtLf& z6k!C|OU@hv_!*qear3KJ?VzVXDKqvKRtugefa7^^MSWl0fXXZR$Xb!b6`eY4A1#pk zAVoZvb_4dZ{f~M8fk3o?{xno^znH1t;;E6K#9?erW~7cs%EV|h^K>@&3Im}c7nm%Y zbLozFrwM&tSNp|46)OhP%MJ(5PydzR>8)X%i3!^L%3HCoCF#Y0#9vPI5l&MK*_ z6G8Y>$`~c)VvQle_4L_AewDGh@!bKkJeEs_NTz(yilnM!t}7jz>fmJb89jQo6~)%% z@GNIJ@AShd&K%UdQ5vR#yT<-goR+D@Tg;PuvcZ*2AzSWN&wW$Xc+~vW)pww~O|6hL zBxX?hOyA~S;3rAEfI&jmMT4f!-eVm%n^KF_QT=>!A<5tgXgi~VNBXqsFI(iI$Tu3x0L{<_-%|HMG4Cn?Xs zq~fvBhu;SDOCD7K5(l&i7Py-;Czx5byV*3y%#-Of9rtz?M_owXc2}$OIY~)EZ&2?r zLQ(onz~I7U!w?B%LtfDz)*X=CscqH!UE=mO?d&oYvtj|(u)^yomS;Cd>Men|#2yuD zg&tf(*iSHyo;^A03p&_j*QXay9d}qZ0CgU@rnFNDIT5xLhC5_tlugv()+w%`7;ICf z>;<#L4m@{1}Og76*e zHWFm~;n@B1GqO8s%=qu)+^MR|jp(ULUOi~v;wE8SB6^mK@adSb=o+A_>Itjn13AF& zDZe+wUF9G!JFv|dpj1#d+}BO~s*QTe3381TxA%Q>P*J#z%( z5*8N^QWxgF73^cTKkkvgvIzf*cLEyyKw)Wf{#$n{uS#(rAA~>TS#!asqQ2m_izXe3 z7$Oh=rR;sdmVx3G)s}eImsb<@r2~5?vcw*Q4LU~FFh!y4r*>~S7slAE6)W3Up2OHr z2R)+O<0kKo<3+5vB}v!lB*`%}gFldc+79iahqEx#&Im@NCQU$@PyCZbcTt?K{;o@4 z312O9GB)?X&wAB}*-NEU zn@6`)G`FhT8O^=Cz3y+XtbwO{5+{4-&?z!esFts-C zypwgI^4#tZ74KC+_IW|E@kMI=1pSJkvg$9G3Va(!reMnJ$kcMiZ=30dTJ%(Ws>eUf z;|l--TFDqL!PZbLc_O(XP0QornpP;!)hdT#Ts7tZ9fcQeH&rhP_1L|Z_ha#JOroe^qcsLi`+AoBWHPM7}gD z+mHuPXd14M?nkp|nu9G8hPk;3=JXE-a204Fg!BK|$MX`k-qPeD$2OOqvF;C(l8wm13?>i(pz7kRyYm zM$IEzf`$}B%ezr!$(UO#uWExn%nTCTIZzq&8@i8sP#6r8 z*QMUzZV(LEWZb)wbmf|Li;UpiP;PlTQ(X4zreD`|`RG!7_wc6J^MFD!A=#K*ze>Jg z?9v?p(M=fg_VB0+c?!M$L>5FIfD(KD5ku*djwCp+5GVIs9^=}kM2RFsxx0_5DE%BF zykxwjWvs=rbi4xKIt!z$&v(`msFrl4n>a%NO_4`iSyb!UiAE&mDa+apc zPe)#!ToRW~rqi2e1bdO1RLN5*uUM@{S`KLJhhY-@TvC&5D(c?a(2$mW-&N%h5IfEM zdFI6`6KJiJQIHvFiG-34^BtO3%*$(-Ht_JU*(KddiUYoM{coadlG&LVvke&*p>Cac z^BPy2Zteiq1@ulw0e)e*ot7@A$RJui0$l^{lsCt%R;$){>zuRv9#w@;m=#d%%TJmm zC#%eFOoy$V)|3*d<OC1iP+4R7D z8FE$E8l2Y?(o-i6wG=BKBh0-I?i3WF%hqdD7VCd;vpk|LFP!Et8$@voH>l>U8BY`Q zC*G;&y6|!p=7`G$*+hxCv!@^#+QD3m>^azyZoLS^;o_|plQaj-wx^ zRV&$HcY~p)2|Zqp0SYU?W3zV87s6JP-@D~$t0 zvd;-YL~JWc*8mtHz_s(cXus#XYJc5zdC=&!4MeZ;N3TQ>^I|Pd=HPjVP*j^45rs(n zzB{U4-44=oQ4rNN6@>qYVMH4|GmMIz#z@3UW-1_y#eNa+Q%(41oJ5i(DzvMO^%|?L z^r_+MZtw0DZ0=BT-@?hUtA)Ijk~Kh-N8?~X5%KnRH7cb!?Yrd8gtiEo!v{sGrQk{X zvV>h{8-DqTyuAxIE(hb}jMVtga$;FIrrKm>ye5t%M;p!jcH1(Bbux>4D#MVhgZGd> z=c=nVb%^9T?iDgM&9G(mV5xShc-lBLi*6RShenDqB%`-2;I*;IHg6>#ovKQ$M}dDb z<$USN%LMqa5_5DR7g7@(oAoQ%!~<1KSQr$rmS{UFQJs5&qBhgTEM_Y7|0Wv?fbP`z z)`8~=v;B)+>Jh`V*|$dTxKe`HTBkho^-!!K#@i{9FLn-XqX&fQcGsEAXp)BV7(`Lk zC{4&+Pe-0&<)C0kAa(MTnb|L;ZB5i|b#L1o;J)+?SV8T*U9$Vxhy}dm3%!A}SK9l_6(#5(e*>8|;4gNKk7o_%m_ zEaS=Z(ewk}hBJ>v`jtR=$pm_Wq3d&DU+6`BACU4%qdhH1o^m8hT2&j<4Z8!v=rMCk z-I*?48{2H*&+r<{2?wp$kh@L@=rj8c`EaS~J>W?)trc?zP&4bsNagS4yafuDoXpi5`!{BVqJ1$ZC3`pf$`LIZ(`0&Ik+!_Xa=NJW`R2 zd#Ntgwz`JVwC4A61$FZ&kP)-{T|rGO59`h#1enAa`cWxRR8bKVvvN6jBzAYePrc&5 z+*zr3en|LYB2>qJp479rEALk5d*X-dfKn6|kuNm;2-U2+P3_rma!nWjZQ-y*q3JS? zBE}zE-!1ZBR~G%v!$l#dZ*$UV4$7q}xct}=on+Ba8{b>Y9h*f-GW0D0o#vJ0%ALg( ztG2+AjWlG#d;myA(i&dh8Gp?y9HD@`CTaDAy?c&0unZ%*LbLIg4;m{Kc?)ws3^>M+ zt5>R)%KIJV*MRUg{0$#nW=Lj{#8?dD$yhjBOrAeR#4$H_Dc(eyA4dNjZEz1Xk+Bqt zB&pPl+?R{w8GPv%VI`x`IFOj320F1=cV4aq0(*()Tx!VVxCjua;)t}gTr=b?zY+U! zkb}xjXZ?hMJN{Hjw?w&?gz8Ow`htX z@}WG*_4<%ff8(!S6bf3)p+8h2!Rory>@aob$gY#fYJ=LiW0`+~l7GI%EX_=8 z{(;0&lJ%9)M9{;wty=XvHbIx|-$g4HFij`J$-z~`mW)*IK^MWVN+*>uTNqaDmi!M8 zurj6DGd)g1g(f`A-K^v)3KSOEoZXImXT06apJum-dO_%oR)z6Bam-QC&CNWh7kLOE zcxLdVjYLNO2V?IXWa-ys30Jbxw(Xm?U1{4kDs9`gZQHh8X{*w9=H&Zz&-6RL?uq#R zxN+k~JaL|gdsdvY_u6}}MHC?a@ElFeipA1Lud#M~)pp2SnG#K{a@tSpvXM;A8gz9> zRVDV5T1%%!LsNRDOw~LIuiAiKcj<%7WpgjP7G6mMU1#pFo6a-1>0I5ZdhxnkMX&#L z=Vm}?SDlb_LArobqpnU!WLQE*yVGWgs^4RRy4rrJwoUUWoA~ZJUx$mK>J6}7{CyC4 zv=8W)kKl7TmAnM%m;anEDPv5tzT{A{ON9#FPYF6c=QIc*OrPp96tiY&^Qs+#A1H>Y z<{XtWt2eDwuqM zQ_BI#UIP;2-olOL4LsZ`vTPv-eILtuB7oWosoSefWdM}BcP>iH^HmimR`G`|+9waCO z&M375o@;_My(qYvPNz;N8FBZaoaw3$b#x`yTBJLc8iIP z--la{bzK>YPP|@Mke!{Km{vT8Z4|#An*f=EmL34?!GJfHaDS#41j~8c5KGKmj!GTh&QIH+DjEI*BdbSS2~6VTt}t zhAwNQNT6%c{G`If3?|~Fp7iwee(LaUS)X9@I29cIb61} z$@YBq4hSplr&liE@ye!y&7+7n$fb+8nS~co#^n@oCjCwuKD61x$5|0ShDxhQES5MP z(gH|FO-s6#$++AxnkQR!3YMgKcF)!&aqr^a3^{gAVT`(tY9@tqgY7@ z>>ul3LYy`R({OY7*^Mf}UgJl(N7yyo$ag;RIpYHa_^HKx?DD`%Vf1D0s^ zjk#OCM5oSzuEz(7X`5u~C-Y~n4B}_3*`5B&8tEdND@&h;H{R`o%IFpIJ4~Kw!kUjehGT8W!CD7?d8sg_$KKp%@*dW)#fI1#R<}kvzBVpaog_2&W%c_jJfP` z6)wE+$3+Hdn^4G}(ymPyasc1<*a7s2yL%=3LgtZLXGuA^jdM^{`KDb%%}lr|ONDsl zy~~jEuK|XJ2y<`R{^F)Gx7DJVMvpT>gF<4O%$cbsJqK1;v@GKXm*9l3*~8^_xj*Gs z=Z#2VQ6`H@^~#5Pv##@CddHfm;lbxiQnqy7AYEH(35pTg^;u&J2xs-F#jGLuDw2%z z`a>=0sVMM+oKx4%OnC9zWdbpq*#5^yM;og*EQKpv`^n~-mO_vj=EgFxYnga(7jO?G z`^C87B4-jfB_RgN2FP|IrjOi;W9AM1qS}9W@&1a9Us>PKFQ9~YE!I~wTbl!m3$Th? z)~GjFxmhyyGxN}t*G#1^KGVXm#o(K0xJyverPe}mS=QgJ$#D}emQDw+dHyPu^&Uv> z4O=3gK*HLFZPBY|!VGq60Of6QrAdj`nj1h!$?&a;Hgaj{oo{l0P3TzpJK_q_eW8Ng zP6QF}1{V;xlolCs?pGegPoCSxx@bshb#3ng4Fkp4!7B0=&+1%187izf@}tvsjZ6{m z4;K>sR5rm97HJrJ`w}Y`-MZN$Wv2N%X4KW(N$v2@R1RkRJH2q1Ozs0H`@ zd5)X-{!{<+4Nyd=hQ8Wm3CCd}ujm*a?L79ztfT7@&(?B|!pU5&%9Rl!`i;suAg0+A zxb&UYpo-z}u6CLIndtH~C|yz&!OV_I*L;H#C7ie_5uB1fNRyH*<^d=ww=gxvE%P$p zRHKI{^{nQlB9nLhp9yj-so1is{4^`{Xd>Jl&;dX;J)#- z=fmE5GiV?-&3kcjM1+XG7&tSq;q9Oi4NUuRrIpoyp*Fn&nVNFdUuGQ_g)g>VzXGdneB7`;!aTUE$t* z5iH+8XPxrYl)vFo~+vmcU-2) zq!6R(T0SsoDnB>Mmvr^k*{34_BAK+I=DAGu){p)(ndZqOFT%%^_y;X(w3q-L``N<6 zw9=M zoQ8Lyp>L_j$T20UUUCzYn2-xdN}{e@$8-3vLDN?GbfJ>7*qky{n!wC#1NcYQr~d51 zy;H!am=EI#*S&TCuP{FA3CO)b0AAiN*tLnDbvKwxtMw-l;G2T@EGH)YU?-B`+Y=!$ zypvDn@5V1Tr~y~U0s$ee2+CL3xm_BmxD3w}d_Pd@S%ft#v~_j;6sC6cy%E|dJy@wj z`+(YSh2CrXMxI;yVy*=O@DE2~i5$>nuzZ$wYHs$y`TAtB-ck4fQ!B8a;M=CxY^Nf{ z+UQhn0jopOzvbl(uZZ1R-(IFaprC$9hYK~b=57@ zAJ8*pH%|Tjotzu5(oxZyCQ{5MAw+6L4)NI!9H&XM$Eui-DIoDa@GpNI=I4}m>Hr^r zZjT?xDOea}7cq+TP#wK1p3}sbMK{BV%(h`?R#zNGIP+7u@dV5#zyMau+w}VC1uQ@p zrFUjrJAx6+9%pMhv(IOT52}Dq{B9njh_R`>&j&5Sbub&r*hf4es)_^FTYdDX$8NRk zMi=%I`)hN@N9>X&Gu2RmjKVsUbU>TRUM`gwd?CrL*0zxu-g#uNNnnicYw=kZ{7Vz3 zULaFQ)H=7%Lm5|Z#k?<{ux{o4T{v-e zTLj?F(_qp{FXUzOfJxEyKO15Nr!LQYHF&^jMMBs z`P-}WCyUYIv>K`~)oP$Z85zZr4gw>%aug1V1A)1H(r!8l&5J?ia1x_}Wh)FXTxZUE zs=kI}Ix2cK%Bi_Hc4?mF^m`sr6m8M(n?E+k7Tm^Gn}Kf= zfnqoyVU^*yLypz?s+-XV5(*oOBwn-uhwco5b(@B(hD|vtT8y7#W{>RomA_KchB&Cd zcFNAD9mmqR<341sq+j+2Ra}N5-3wx5IZqg6Wmi6CNO#pLvYPGNER}Q8+PjvIJ42|n zc5r@T*p)R^U=d{cT2AszQcC6SkWiE|hdK)m{7ul^mU+ED1R8G#)#X}A9JSP_ubF5p z8Xxcl;jlGjPwow^p+-f_-a~S;$lztguPE6SceeUCfmRo=Qg zKHTY*O_ z;pXl@z&7hniVYVbGgp+Nj#XP^Aln2T!D*{(Td8h{8Dc?C)KFfjPybiC`Va?Rf)X>y z;5?B{bAhPtbmOMUsAy2Y0RNDQ3K`v`gq)#ns_C&ec-)6cq)d^{5938T`Sr@|7nLl; zcyewuiSUh7Z}q8iIJ@$)L3)m)(D|MbJm_h&tj^;iNk%7K-YR}+J|S?KR|29K?z-$c z<+C4uA43yfSWBv*%z=-0lI{ev`C6JxJ};A5N;lmoR(g{4cjCEn33 z-ef#x^uc%cM-f^_+*dzE?U;5EtEe;&8EOK^K}xITa?GH`tz2F9N$O5;)`Uof4~l+t z#n_M(KkcVP*yMYlk_~5h89o zlf#^qjYG8Wovx+f%x7M7_>@r7xaXa2uXb?_*=QOEe_>ErS(v5-i)mrT3&^`Oqr4c9 zDjP_6T&NQMD`{l#K&sHTm@;}ed_sQ88X3y`ON<=$<8Qq{dOPA&WAc2>EQ+U8%>yWR zK%(whl8tB;{C)yRw|@Gn4%RhT=bbpgMZ6erACc>l5^p)9tR`(2W-D*?Ph6;2=Fr|G- zdF^R&aCqyxqWy#P7#G8>+aUG`pP*ow93N=A?pA=aW0^^+?~#zRWcf_zlKL8q8-80n zqGUm=S8+%4_LA7qrV4Eq{FHm9#9X15%ld`@UKyR7uc1X*>Ebr0+2yCye6b?i=r{MPoqnTnYnq z^?HWgl+G&@OcVx4$(y;{m^TkB5Tnhx2O%yPI=r*4H2f_6Gfyasq&PN^W{#)_Gu7e= zVHBQ8R5W6j;N6P3O(jsRU;hkmLG(Xs_8=F&xh@`*|l{~0OjUVlgm z7opltSHg7Mb%mYamGs*v1-#iW^QMT**f+Nq*AzIvFT~Ur3KTD26OhIw1WQsL(6nGg znHUo-4e15cXBIiyqN};5ydNYJ6zznECVVR44%(P0oW!yQ!YH)FPY?^k{IrtrLo7Zo`?sg%%oMP9E^+H@JLXicr zi?eoI?LODRPcMLl90MH32rf8btf69)ZE~&4d%(&D{C45egC6bF-XQ;6QKkbmqW>_H z{86XDZvjiN2wr&ZPfi;^SM6W+IP0);50m>qBhzx+docpBkkiY@2bSvtPVj~E`CfEu zhQG5G>~J@dni5M5Jmv7GD&@%UR`k3ru-W$$onI259jM&nZ)*d3QFF?Mu?{`+nVzkx z=R*_VH=;yeU?9TzQ3dP)q;P)4sAo&k;{*Eky1+Z!10J<(cJC3zY9>bP=znA=<-0RR zMnt#<9^X7BQ0wKVBV{}oaV=?JA=>R0$az^XE%4WZcA^Em>`m_obQyKbmf-GA;!S-z zK5+y5{xbkdA?2NgZ0MQYF-cfOwV0?3Tzh8tcBE{u%Uy?Ky4^tn^>X}p>4&S(L7amF zpWEio8VBNeZ=l!%RY>oVGOtZh7<>v3?`NcHlYDPUBRzgg z0OXEivCkw<>F(>1x@Zk=IbSOn+frQ^+jI*&qdtf4bbydk-jgVmLAd?5ImK+Sigh?X zgaGUlbf^b-MH2@QbqCawa$H1Vb+uhu{zUG9268pa{5>O&Vq8__Xk5LXDaR1z$g;s~;+Ae82wq#l;wo08tX(9uUX6NJWq1vZLh3QbP$# zL`udY|Qp*4ER`_;$%)2 zmcJLj|FD`(;ts0bD{}Ghq6UAVpEm#>j`S$wHi0-D_|)bEZ}#6) zIiqH7Co;TB`<6KrZi1SF9=lO+>-_3=Hm%Rr7|Zu-EzWLSF{9d(H1v*|UZDWiiqX3} zmx~oQ6%9~$=KjPV_ejzz7aPSvTo+3@-a(OCCoF_u#2dHY&I?`nk zQ@t8#epxAv@t=RUM09u?qnPr6=Y5Pj;^4=7GJ`2)Oq~H)2V)M1sC^S;w?hOB|0zXT zQdf8$)jslO>Q}(4RQ$DPUF#QUJm-k9ysZFEGi9xN*_KqCs9Ng(&<;XONBDe1Joku? z*W!lx(i&gvfXZ4U(AE@)c0FI2UqrFLOO$&Yic|`L;Vyy-kcm49hJ^Mj^H9uY8Fdm2 z?=U1U_5GE_JT;Tx$2#I3rAAs(q@oebIK=19a$N?HNQ4jw0ljtyGJ#D}z3^^Y=hf^Bb--297h6LQxi0-`TB|QY2QPg92TAq$cEQdWE ze)ltSTVMYe0K4wte6;^tE+^>|a>Hit_3QDlFo!3Jd`GQYTwlR#{<^MzG zK!vW&))~RTKq4u29bc<+VOcg7fdorq-kwHaaCQe6tLB{|gW1_W_KtgOD0^$^|`V4C# z*D_S9Dt_DIxpjk3my5cBFdiYaq||#0&0&%_LEN}BOxkb3v*d$4L|S|z z!cZZmfe~_Y`46v=zul=aixZTQCOzb(jx>8&a%S%!(;x{M2!*$od2!Pwfs>RZ-a%GOZdO88rS)ZW~{$656GgW)$Q=@!x;&Nn~!K)lr4gF*%qVO=hlodHA@2)keS2 zC}7O=_64#g&=zY?(zhzFO3)f5=+`dpuyM!Q)zS&otpYB@hhn$lm*iK2DRt+#1n|L%zjM}nB*$uAY^2JIw zV_P)*HCVq%F))^)iaZD#R9n^{sAxBZ?Yvi1SVc*`;8|F2X%bz^+s=yS&AXjysDny)YaU5RMotF-tt~FndTK ziRve_5b!``^ZRLG_ks}y_ye0PKyKQSsQCJuK5()b2ThnKPFU?An4;dK>)T^4J+XjD zEUsW~H?Q&l%K4<1f5^?|?lyCQe(O3?!~OU{_Wxs#|Ff8?a_WPQUKvP7?>1()Cy6oLeA zjEF^d#$6Wb${opCc^%%DjOjll%N2=GeS6D-w=Ap$Ux2+0v#s#Z&s6K*)_h{KFfgKjzO17@p1nKcC4NIgt+3t}&}F z@cV; zZ1r#~?R@ZdSwbFNV(fFl2lWI(Zf#nxa<6f!nBZD>*K)nI&Fun@ngq@Ge!N$O< zySt*mY&0moUXNPe~Fg=%gIu)tJ;asscQ!-AujR@VJBRoNZNk;z4hs4T>Ud!y=1NwGs-k zlTNeBOe}=)Epw=}+dfX;kZ32h$t&7q%Xqdt-&tlYEWc>>c3(hVylsG{Ybh_M8>Cz0ZT_6B|3!_(RwEJus9{;u-mq zW|!`{BCtnao4;kCT8cr@yeV~#rf76=%QQs(J{>Mj?>aISwp3{^BjBO zLV>XSRK+o=oVDBnbv?Y@iK)MiFSl{5HLN@k%SQZ}yhPiu_2jrnI?Kk?HtCv>wN$OM zSe#}2@He9bDZ27hX_fZey=64#SNU#1~=icK`D>a;V-&Km>V6ZdVNj7d2 z-NmAoOQm_aIZ2lXpJhlUeJ95eZt~4_S zIfrDs)S$4UjyxKSaTi#9KGs2P zfSD>(y~r+bU4*#|r`q+be_dopJzKK5JNJ#rR978ikHyJKD>SD@^Bk$~D0*U38Y*IpYcH>aaMdZq|YzQ-Ixd(_KZK!+VL@MWGl zG!k=<%Y-KeqK%``uhx}0#X^@wS+mX@6Ul@90#nmYaKh}?uw>U;GS4fn3|X%AcV@iY z8v+ePk)HxSQ7ZYDtlYj#zJ?5uJ8CeCg3efmc#|a%2=u>+vrGGRg$S@^mk~0f;mIu! zWMA13H1<@hSOVE*o0S5D8y=}RiL#jQpUq42D}vW$z*)VB*FB%C?wl%(3>ANaY)bO@ zW$VFutemwy5Q*&*9HJ603;mJJkB$qp6yxNOY0o_4*y?2`qbN{m&*l{)YMG_QHXXa2 z+hTmlA;=mYwg{Bfusl zyF&}ib2J;#q5tN^e)D62fWW*Lv;Rnb3GO-JVtYG0CgR4jGujFo$Waw zSNLhc{>P~>{KVZE1Vl1!z)|HFuN@J7{`xIp_)6>*5Z27BHg6QIgqLqDJTmKDM+ON* zK0Fh=EG`q13l z+m--9UH0{ZGQ%j=OLO8G2WM*tgfY}bV~>3Grcrpehjj z6Xe<$gNJyD8td3EhkHjpKk}7?k55Tu7?#;5`Qcm~ki;BeOlNr+#PK{kjV>qfE?1No zMA07}b>}Dv!uaS8Hym0TgzxBxh$*RX+Fab6Gm02!mr6u}f$_G4C|^GSXJMniy^b`G z74OC=83m0G7L_dS99qv3a0BU({t$zHQsB-RI_jn1^uK9ka_%aQuE2+~J2o!7`735Z zb?+sTe}Gd??VEkz|KAPMfj(1b{om89p5GIJ^#Aics_6DD%WnNGWAW`I<7jT|Af|8g zZA0^)`p8i#oBvX2|I&`HC8Pn&0>jRuMF4i0s=}2NYLmgkZb=0w9tvpnGiU-gTUQhJ zR6o4W6ZWONuBZAiN77#7;TR1^RKE(>>OL>YU`Yy_;5oj<*}ac99DI(qGCtn6`949f ziMpY4k>$aVfffm{dNH=-=rMg|u?&GIToq-u;@1-W&B2(UOhC-O2N5_px&cF-C^tWp zXvChm9@GXEcxd;+Q6}u;TKy}$JF$B`Ty?|Y3tP$N@Rtoy(*05Wj-Ks32|2y2ZM>bM zi8v8E1os!yorR!FSeP)QxtjIKh=F1ElfR8U7StE#Ika;h{q?b?Q+>%78z^>gTU5+> zxQ$a^rECmETF@Jl8fg>MApu>btHGJ*Q99(tMqsZcG+dZ6Yikx7@V09jWCiQH&nnAv zY)4iR$Ro223F+c3Q%KPyP9^iyzZsP%R%-i^MKxmXQHnW6#6n7%VD{gG$E;7*g86G< zu$h=RN_L2(YHO3@`B<^L(q@^W_0#U%mLC9Q^XEo3LTp*~(I%?P_klu-c~WJxY1zTI z^PqntLIEmdtK~E-v8yc&%U+jVxW5VuA{VMA4Ru1sk#*Srj0Pk#tZuXxkS=5H9?8eb z)t38?JNdP@#xb*yn=<*_pK9^lx%;&yH6XkD6-JXgdddZty8@Mfr9UpGE!I<37ZHUe z_Rd+LKsNH^O)+NW8Ni-V%`@J_QGKA9ZCAMSnsN>Ych9VW zCE7R_1FVy}r@MlkbxZ*TRIGXu`ema##OkqCM9{wkWQJg^%3H${!vUT&vv2250jAWN zw=h)C!b2s`QbWhBMSIYmWqZ_~ReRW;)U#@C&ThctSd_V!=HA=kdGO-Hl57an|M1XC?~3f0{7pyjWY}0mChU z2Fj2(B*r(UpCKm-#(2(ZJD#Y|Or*Vc5VyLpJ8gO1;fCm@EM~{DqpJS5FaZ5%|ALw) zyumBl!i@T57I4ITCFmdbxhaOYud}i!0YkdiNRaQ%5$T5>*HRBhyB~<%-5nj*b8=i= z(8g(LA50%0Zi_eQe}Xypk|bt5e6X{aI^jU2*c?!p*$bGk=?t z+17R){lx~Z{!B34Zip~|A;8l@%*Gc}kT|kC0*Ny$&fI3@%M! zqk_zvN}7bM`x@jqFOtaxI?*^Im5ix@=`QEv;__i;Tek-&7kGm6yP17QANVL>*d0B=4>i^;HKb$k8?DYFMr38IX4azK zBbwjF%$>PqXhJh=*7{zH5=+gi$!nc%SqFZlwRm zmpctOjZh3bwt!Oc>qVJhWQf>`HTwMH2ibK^eE*j!&Z`-bs8=A`Yvnb^?p;5+U=Fb8 z@h>j_3hhazd$y^Z-bt%3%E3vica%nYnLxW+4+?w{%|M_=w^04U{a6^22>M_?{@mXP zS|Qjcn4&F%WN7Z?u&I3fU(UQVw4msFehxR*80dSb=a&UG4zDQp&?r2UGPy@G?0FbY zVUQ?uU9-c;f9z06$O5FO1TOn|P{pLcDGP?rfdt`&uw|(Pm@$n+A?)8 zP$nG(VG&aRU*(_5z#{+yVnntu`6tEq>%9~n^*ao}`F6ph_@6_8|AfAXtFfWee_14` zKKURYV}4}=UJmxv7{RSz5QlwZtzbYQs0;t3?kx*7S%nf-aY&lJ@h?-BAn%~0&&@j) zQd_6TUOLXErJ`A3vE?DJIbLE;s~s%eVt(%fMzUq^UfZV9c?YuhO&6pwKt>j(=2CkgTNEq7&c zfeGN+%5DS@b9HO>zsoRXv@}(EiA|t5LPi}*R3?(-=iASADny<{D0WiQG>*-BSROk4vI6%$R>q64J&v-T+(D<_(b!LD z9GL;DV;;N3!pZYg23mcg81tx>7)=e%f|i{6Mx0GczVpc}{}Mg(W_^=Wh0Rp+xXgX` z@hw|5=Je&nz^Xa>>vclstYt;8c2PY)87Ap;z&S&`yRN>yQVV#K{4&diVR7Rm;S{6m z6<+;jwbm`==`JuC6--u6W7A@o4&ZpJV%5+H)}toy0afF*!)AaG5=pz_i9}@OG%?$O z2cec6#@=%xE3K8;^ps<2{t4SnqH+#607gAHP-G4^+PBiC1s>MXf&bQ|Pa;WBIiErV z?3VFpR9JFl9(W$7p3#xe(Bd?Z93Uu~jHJFo7U3K_x4Ej-=N#=a@f;kPV$>;hiN9i9 z<6elJl?bLI$o=|d6jlihA4~bG;Fm2eEnlGxZL`#H%Cdes>uJfMJ4>@1SGGeQ81DwxGxy7L5 zm05Ik*WpSgZvHh@Wpv|2i|Y#FG?Y$hbRM5ZF0Z7FB3cY0+ei#km9mDSPI}^!<<`vr zuv$SPg2vU{wa)6&QMY)h1hbbxvR2cc_6WcWR`SH& z&KuUQcgu}!iW2Wqvp~|&&LSec9>t(UR_|f$;f-fC&tSO-^-eE0B~Frttnf+XN(#T) z^PsuFV#(pE#6ztaI8(;ywN%CtZh?w&;_)w_s@{JiA-SMjf&pQk+Bw<}f@Q8-xCQMwfaf zMgHsAPU=>>Kw~uDFS(IVRN{$ak(SV(hrO!UqhJ?l{lNnA1>U24!=>|q_p404Xd>M# z7?lh^C&-IfeIr`Dri9If+bc%oU0?|Rh8)%BND5;_9@9tuM)h5Kcw6}$Ca7H_n)nOf0pd`boCXItb`o11 zb`)@}l6I_h>n+;`g+b^RkYs7;voBz&Gv6FLmyvY|2pS)z#P;t8k;lS>49a$XeVDc4 z(tx2Pe3N%Gd(!wM`E7WRBZy)~vh_vRGt&esDa0NCua)rH#_39*H0!gIXpd>~{rGx+ zJKAeXAZ-z5n=mMVqlM5Km;b;B&KSJlScD8n?2t}kS4Wf9@MjIZSJ2R?&=zQn zs_`=+5J$47&mP4s{Y{TU=~O_LzSrXvEP6W?^pz<#Y*6Fxg@$yUGp31d(h+4x>xpb< zH+R639oDST6F*0iH<9NHC^Ep*8D4-%p2^n-kD6YEI<6GYta6-I;V^ZH3n5}syTD=P z3b6z=jBsdP=FlXcUe@I|%=tY4J_2j!EVNEzph_42iO3yfir|Dh>nFl&Lu9!;`!zJB zCis9?_(%DI?$CA(00pkzw^Up`O;>AnPc(uE$C^a9868t$m?5Q)CR%!crI$YZpiYK6m= z!jv}82He`QKF;10{9@roL2Q7CF)OeY{~dBp>J~X#c-Z~{YLAxNmn~kWQW|2u!Yq00 zl5LKbzl39sVCTpm9eDW_T>Z{x@s6#RH|P zA~_lYas7B@SqI`N=>x50Vj@S)QxouKC(f6Aj zz}7e5e*5n?j@GO;mCYEo^Jp_*BmLt3!N)(T>f#L$XHQWzZEVlJo(>qH@7;c%fy zS-jm^Adju9Sm8rOKTxfTU^!&bg2R!7C_-t+#mKb_K?0R72%26ASF;JWA_prJ8_SVW zOSC7C&CpSrgfXRp8r)QK34g<~!1|poTS7F;)NseFsbwO$YfzEeG3oo!qe#iSxQ2S# z1=Fxc9J;2)pCab-9o-m8%BLjf(*mk#JJX3k9}S7Oq)dV0jG)SOMbw7V^Z<5Q0Cy$< z^U0QUVd4(96W03OA1j|x%{sd&BRqIERDb6W{u1p1{J(a;fd6lnWzjeS`d?L3-0#o7 z{Qv&L7!Tm`9|}u=|IbwS_jgH(_V@o`S*R(-XC$O)DVwF~B&5c~m!zl14ydT6sK+Ly zn+}2hQ4RTC^8YvrQ~vk$f9u=pTN{5H_yTOcza9SVE&nt_{`ZC8zkmFji=UyD`G4~f zUfSTR=Kju>6u+y&|Bylb*W&^P|8fvEbQH3+w*DrKq|9xMzq2OiZyM=;(?>~4+O|jn zC_Et05oc>e%}w4ye2Fm%RIR??VvofwZS-}BL@X=_4jdHp}FlMhW_IW?Zh`4$z*Wr!IzQHa3^?1|);~VaWmsIcmc6 zJs{k0YW}OpkfdoTtr4?9F6IX6$!>hhA+^y_y@vvA_Gr7u8T+i-< zDX(~W5W{8mfbbM-en&U%{mINU#Q8GA`byo)iLF7rMVU#wXXY`a3ji3m{4;x53216i z`zA8ap?>_}`tQj7-%$K78uR}R$|@C2)qgop$}o=g(jOv0ishl!E(R73N=i0~%S)6+ z1xFP7|H0yt3Z_Re*_#C2m3_X{=zi1C&3CM7e?9-Y5lCtAlA%RFG9PDD=Quw1dfYnZ zdUL)#+m`hKx@PT`r;mIx_RQ6Txbti+&;xQorP;$H=R2r)gPMO9>l+!p*Mt04VH$$M zSLwJ81IFjQ5N!S#;MyBD^IS`2n04kuYbZ2~4%3%tp0jn^**BZQ05ELp zY%yntZ=52s6U5Y93Aao)v~M3y?6h7mZcVGp63pK*d&!TRjW99rUU;@s#3kYB76Bs$|LRwkH>L!0Xe zE=dz1o}phhnOVYZFsajQsRA^}IYZnk9Wehvo>gHPA=TPI?2A`plIm8=F1%QiHx*Zn zi)*Y@)$aXW0v1J|#+R2=$ysooHZ&NoA|Wa}htd`=Eud!(HD7JlT8ug|yeBZmpry(W z)pS>^1$N#nuo3PnK*>Thmaxz4pLcY?PP2r3AlhJ7jw(TI8V#c}>Ym;$iPaw+83L+* z!_QWpYs{UWYcl0u z(&(bT0Q*S_uUX9$jC;Vk%oUXw=A-1I+!c18ij1CiUlP@pfP9}CHAVm{!P6AEJ(7Dn z?}u#}g`Q?`*|*_0Rrnu8{l4PP?yCI28qC~&zlwgLH2AkfQt1?B#3AOQjW&10%@@)Q zDG?`6$8?Nz(-sChL8mRs#3z^uOA>~G=ZIG*mgUibWmgd{a|Tn4nkRK9O^37E(()Q% zPR0#M4e2Q-)>}RSt1^UOCGuv?dn|IT3#oW_$S(YR+jxAzxCD_L25p_dt|^>g+6Kgj zJhC8n)@wY;Y7JI6?wjU$MQU|_Gw*FIC)x~^Eq1k41BjLmr}U>6#_wxP0-2Ka?uK14u5M-lAFSX$K1K{WH!M1&q}((MWWUp#Uhl#n_yT5dFs4X`>vmM& z*1!p0lACUVqp&sZG1GWATvZEENs^0_7Ymwem~PlFN3hTHVBv(sDuP;+8iH07a)s(# z%a7+p1QM)YkS7>kbo${k2N1&*%jFP*7UABJ2d||c!eSXWM*<4(_uD7;1XFDod@cT$ zP>IC%^fbC${^QrUXy$f)yBwY^g@}}kngZKa1US!lAa+D=G4wklukaY8AEW%GL zh40pnuv*6D>9`_e14@wWD^o#JvxYVG-~P)+<)0fW zP()DuJN?O*3+Ab!CP-tGr8S4;JN-Ye^9D%(%8d{vb_pK#S1z)nZzE^ezD&%L6nYbZ z*62>?u)xQe(Akd=e?vZbyb5)MMNS?RheZDHU?HK<9;PBHdC~r{MvF__%T)-9ifM#cR#2~BjVJYbA>xbPyl9yNX zX)iFVvv-lfm`d?tbfh^j*A|nw)RszyD<#e>llO8X zou=q3$1|M@Ob;F|o4H0554`&y9T&QTa3{yn=w0BLN~l;XhoslF-$4KGNUdRe?-lcV zS4_WmftU*XpP}*wFM^oKT!D%_$HMT#V*j;9weoOq0mjbl1271$F)`Q(C z76*PAw3_TE{vntIkd=|(zw)j^!@j ^tV@s0U~V+mu)vv`xgL$Z9NQLnuRdZ;95D|1)!0Aybwv}XCE#xz1k?ZC zxAU)v@!$Sm*?)t2mWrkevNFbILU9&znoek=d7jn*k+~ptQ)6z`h6e4B&g?Q;IK+aH z)X(BH`n2DOS1#{AJD-a?uL)@Vl+`B=6X3gF(BCm>Q(9+?IMX%?CqgpsvK+b_de%Q> zj-GtHKf!t@p2;Gu*~#}kF@Q2HMevg~?0{^cPxCRh!gdg7MXsS}BLtG_a0IY0G1DVm z2F&O-$Dzzc#M~iN`!j38gAn`6*~h~AP=s_gy2-#LMFoNZ0<3q+=q)a|4}ur7F#><%j1lnr=F42Mbti zi-LYs85K{%NP8wE1*r4Mm+ZuZ8qjovmB;f##!E*M{*A(4^~vg!bblYi1M@7tq^L8- zH7tf_70iWXqcSQgENGdEjvLiSLicUi3l0H*sx=K!!HLxDg^K|s1G}6Tam|KBV>%YeU)Q>zxQe;ddnDTWJZ~^g-kNeycQ?u242mZs`i8cP)9qW`cwqk)Jf?Re0=SD=2z;Gafh(^X-=WJ$i7Z9$Pao56bTwb+?p>L3bi9 zP|qi@;H^1iT+qnNHBp~X>dd=Us6v#FPDTQLb9KTk%z{&OWmkx3uY(c6JYyK3w|z#Q zMY%FPv%ZNg#w^NaW6lZBU+}Znwc|KF(+X0RO~Q6*O{T-P*fi@5cPGLnzWMSyoOPe3 z(J;R#q}3?z5Ve%crTPZQFLTW81cNY-finw!LH9wr$(C)p_@v?(y#b-R^Pv!}_#7t+A?pHEUMY zoQZIwSETTKeS!W{H$lyB1^!jn4gTD{_mgG?#l1Hx2h^HrpCXo95f3utP-b&%w80F} zXFs@Jp$lbIL64@gc?k*gJ;OForPaapOH7zNMB60FdNP<*9<@hEXJk9Rt=XhHR-5_$Ck-R?+1py&J3Y9^sBBZuj?GwSzua;C@9)@JZpaI zE?x6{H8@j9P06%K_m%9#nnp0Li;QAt{jf-7X%Pd2jHoI4As-9!UR=h6Rjc z!3{UPWiSeLG&>1V5RlM@;5HhQW_&-wL2?%k@dvRS<+@B6Yaj*NG>qE5L*w~1ATP$D zmWu6(OE=*EHqy{($~U4zjxAwpPn42_%bdH9dMphiUU|) z*+V@lHaf%*GcXP079>vy5na3h^>X=n;xc;VFx)`AJEk zYZFlS#Nc-GIHc}j06;cOU@ zAD7Egkw<2a8TOcfO9jCp4U4oI*`|jpbqMWo(={gG3BjuM3QTGDG`%y|xithFck}0J zG}N#LyhCr$IYP`#;}tdm-7^9=72+CBfBsOZ0lI=LC_a%U@(t3J_I1t(UdiJ^@NubM zvvA0mGvTC%{fj53M^|Ywv$KbW;n8B-x{9}Z!K6v-tw&Xe_D2{7tX?eVk$sA*0826( zuGz!K7$O#;K;1w<38Tjegl)PmRso`fc&>fAT5s z7hzQe-_`lx`}2=c)jz6;yn(~F6#M@z_7@Z(@GWbIAo6A2&;aFf&>CVHpqoPh5#~=G zav`rZ3mSL2qwNL+Pg>aQv;%V&41e|YU$!fQ9Ksle!XZERpjAowHtX zi#0lnw{(zmk&}t`iFEMmx-y7FWaE*vA{Hh&>ieZg{5u0-3@a8BY)Z47E`j-H$dadu zIP|PXw1gjO@%aSz*O{GqZs_{ke|&S6hV{-dPkl*V|3U4LpqhG0eVdqfeNX28hrafI zE13WOsRE|o?24#`gQJs@v*EwL{@3>Ffa;knvI4@VEG2I>t-L(KRS0ShZ9N!bwXa}e zI0}@2#PwFA&Y9o}>6(ZaSaz>kw{U=@;d{|dYJ~lyjh~@bBL>n}#@KjvXUOhrZ`DbnAtf5bz3LD@0RpmAyC-4cgu<7rZo&C3~A_jA*0)v|Ctcdu} zt@c7nQ6hSDC@76c4hI&*v|5A0Mj4eQ4kVb0$5j^*$@psB zdouR@B?l6E%a-9%i(*YWUAhxTQ(b@z&Z#jmIb9`8bZ3Um3UW!@w4%t0#nxsc;*YrG z@x$D9Yj3EiA(-@|IIzi@!E$N)j?gedGJpW!7wr*7zKZwIFa>j|cy<(1`VV_GzWN=1 zc%OO)o*RRobvTZE<9n1s$#V+~5u8ZwmDaysD^&^cxynksn!_ypmx)Mg^8$jXu5lMo zK3K_8GJh#+7HA1rO2AM8cK(#sXd2e?%3h2D9GD7!hxOEKJZK&T`ZS0e*c9c36Y-6yz2D0>Kvqy(EuiQtUQH^~M*HY!$e z20PGLb2Xq{3Ceg^sn+99K6w)TkprP)YyNU(+^PGU8}4&Vdw*u;(`Bw!Um76gL_aMT z>*82nmA8Tp;~hwi0d3S{vCwD};P(%AVaBr=yJ zqB?DktZ#)_VFh_X69lAHQw(ZNE~ZRo2fZOIP;N6fD)J*3u^YGdgwO(HnI4pb$H#9) zizJ<>qI*a6{+z=j+SibowDLKYI*Je2Y>~=*fL@i*f&8**s~4l&B&}$~nwhtbOTr=G zFx>{y6)dpJPqv={_@*!q0=jgw3^j`qi@!wiWiT_$1`SPUgaG&9z9u9=m5C8`GpMaM zyMRSv2llS4F}L?233!)f?mvcYIZ~U z7mPng^=p)@Z*Fp9owSYA`Fe4OjLiJ`rdM`-U(&z1B1`S`ufK_#T@_BvenxDQU`deH$X5eMVO=;I4EJjh6?kkG2oc6AYF6|(t)L0$ukG}Zn=c+R`Oq;nC)W^ z{ek!A?!nCsfd_5>d&ozG%OJmhmnCOtARwOq&p!FzWl7M))YjqK8|;6sOAc$w2%k|E z`^~kpT!j+Y1lvE0B)mc$Ez_4Rq~df#vC-FmW;n#7E)>@kMA6K30!MdiC19qYFnxQ* z?BKegU_6T37%s`~Gi2^ewVbciy-m5%1P3$88r^`xN-+VdhhyUj4Kzg2 zlKZ|FLUHiJCZL8&<=e=F2A!j@3D@_VN%z?J;uw9MquL`V*f^kYTrpoWZ6iFq00uO+ zD~Zwrs!e4cqGedAtYxZ76Bq3Ur>-h(m1~@{x@^*YExmS*vw9!Suxjlaxyk9P#xaZK z)|opA2v#h=O*T42z>Mub2O3Okd3GL86KZM2zlfbS z{Vps`OO&3efvt->OOSpMx~i7J@GsRtoOfQ%vo&jZ6^?7VhBMbPUo-V^Znt%-4k{I# z8&X)=KY{3lXlQg4^FH^{jw0%t#2%skLNMJ}hvvyd>?_AO#MtdvH;M^Y?OUWU6BdMX zJ(h;PM9mlo@i)lWX&#E@d4h zj4Z0Czj{+ipPeW$Qtz_A52HA<4$F9Qe4CiNQSNE2Q-d1OPObk4?7-&`={{yod5Iy3kB=PK3%0oYSr`Gca120>CHbC#SqE*ivL2R(YmI1A|nAT?JmK*2qj_3p#?0h)$#ixdmP?UejCg9%AS2 z8I(=_QP(a(s)re5bu-kcNQc-&2{QZ%KE*`NBx|v%K2?bK@Ihz_e<5Y(o(gQ-h+s&+ zjpV>uj~?rfJ!UW5Mop~ro^|FP3Z`@B6A=@f{Wn78cm`)3&VJ!QE+P9&$;3SDNH>hI z_88;?|LHr%1kTX0t*xzG-6BU=LRpJFZucRBQ<^zy?O5iH$t>o}C}Fc+kM1EZu$hm% zTTFKrJkXmCylFgrA;QAA(fX5Sia5TNo z?=Ujz7$Q?P%kM$RKqRQisOexvV&L+bolR%`u`k;~!o(HqgzV9I6w9|g*5SVZN6+kT9H$-3@%h%k7BBnB zPn+wmPYNG)V2Jv`&$LoI*6d0EO^&Nh`E* z&1V^!!Szd`8_uf%OK?fuj~! z%p9QLJ?V*T^)72<6p1ONqpmD?Wm((40>W?rhjCDOz?#Ei^sXRt|GM3ULLnoa8cABQ zA)gCqJ%Q5J%D&nJqypG-OX1`JLT+d`R^|0KtfGQU+jw79la&$GHTjKF>*8BI z0}l6TC@XB6`>7<&{6WX2kX4k+0SaI`$I8{{mMHB}tVo*(&H2SmZLmW* z+P8N>(r}tR?f!O)?)df>HIu>$U~e~tflVmwk*+B1;TuqJ+q_^`jwGwCbCgSevBqj$ z<`Fj*izeO)_~fq%wZ0Jfvi6<3v{Afz;l5C^C7!i^(W>%5!R=Ic7nm(0gJ~9NOvHyA zqWH2-6w^YmOy(DY{VrN6ErvZREuUMko@lVbdLDq*{A+_%F>!@6Z)X9kR1VI1+Ler+ zLUPtth=u~23=CqZoAbQ`uGE_91kR(8Ie$mq1p`q|ilkJ`Y-ob_=Nl(RF=o7k{47*I)F%_XMBz9uwRH8q1o$TkV@8Pwl zzi`^7i;K6Ak7o58a_D-V0AWp;H8pSjbEs$4BxoJkkC6UF@QNL)0$NU;Wv0*5 z0Ld;6tm7eR%u=`hnUb)gjHbE2cP?qpo3f4w%5qM0J*W_Kl6&z4YKX?iD@=McR!gTyhpGGYj!ljQm@2GL^J70`q~4CzPv@sz`s80FgiuxjAZ zLq61rHv1O>>w1qOEbVBwGu4%LGS!!muKHJ#JjfT>g`aSn>83Af<9gM3XBdY)Yql|{ zUds}u*;5wuus)D>HmexkC?;R&*Z`yB4;k;4T*(823M&52{pOd1yXvPJ3PPK{Zs>6w zztXy*HSH0scZHn7qIsZ8y-zftJ*uIW;%&-Ka0ExdpijI&xInDg-Bv-Q#Islcbz+R! zq|xz?3}G5W@*7jSd`Hv9q^5N*yN=4?Lh=LXS^5KJC=j|AJ5Y(f_fC-c4YQNtvAvn|(uP9@5Co{dL z?7|=jqTzD8>(6Wr&(XYUEzT~-VVErf@|KeFpKjh=v51iDYN_`Kg&XLOIG;ZI8*U$@ zKig{dy?1H}UbW%3jp@7EVSD>6c%#abQ^YfcO(`)*HuvNc|j( zyUbYozBR15$nNU$0ZAE%ivo4viW?@EprUZr6oX=4Sc!-WvrpJdF`3SwopKPyX~F>L zJ>N>v=_plttTSUq6bYu({&rkq)d94m5n~Sk_MO*gY*tlkPFd2m=Pi>MK)ObVV@Sgs zmXMNMvvcAuz+<$GLR2!j4w&;{)HEkxl{$B^*)lUKIn&p5_huD6+%WDoH4`p}9mkw$ zXCPw6Y7tc%rn$o_vy>%UNBC`0@+Ih-#T05AT)ooKt?94^ROI5;6m2pIM@@tdT=&WP z{u09xEVdD}{(3v}8AYUyT82;LV%P%TaJa%f)c36?=90z>Dzk5mF2}Gs0jYCmufihid8(VFcZWs8#59;JCn{!tHu5kSBbm zL`F{COgE01gg-qcP2Lt~M9}mALg@i?TZp&i9ZM^G<3`WSDh}+Ceb3Q!QecJ|N;Xrs z{wH{D8wQ2+mEfBX#M8)-32+~q4MRVr1UaSPtw}`iwx@x=1Xv-?UT{t}w}W(J&WKAC zrZ%hssvf*T!rs}}#atryn?LB=>0U%PLwA9IQZt$$UYrSw`7++}WR7tfE~*Qg)vRrM zT;(1>Zzka?wIIz8vfrG86oc^rjM@P7^i8D~b(S23AoKYj9HBC(6kq9g`1gN@|9^xO z{~h zbxGMHqGZ@eJ17bgES?HQnwp|G#7I>@p~o2zxWkgZUYSUeB*KT{1Q z*J3xZdWt`eBsA}7(bAHNcMPZf_BZC(WUR5B8wUQa=UV^e21>|yp+uop;$+#JwXD!> zunhJVCIKgaol0AM_AwJNl}_k&q|uD?aTE@{Q*&hxZ=k_>jcwp}KwG6mb5J*pV@K+- zj*`r0WuEU_8O=m&1!|rj9FG7ad<2px63;Gl z9lJrXx$~mPnuiqIH&n$jSt*ReG}1_?r4x&iV#3e_z+B4QbhHwdjiGu^J3vcazPi`| zaty}NFSWe=TDry*a*4XB)F;KDI$5i9!!(5p@5ra4*iW;FlGFV0P;OZXF!HCQ!oLm1 zsK+rY-FnJ?+yTBd0}{*Y6su|hul)wJ>RNQ{eau*;wWM{vWM`d0dTC-}Vwx6@cd#P? zx$Qyk^2*+_ZnMC}q0)+hE-q)PKoox#;pc%DNJ&D5+if6X4j~p$A7-s&AjDkSEV)aM z(<3UOw*&f)+^5F0Mpzw3zB1ZHl*B?C~Cx) zuNg*>5RM9F5{EpU@a2E7hAE`m<89wbQ2Lz&?Egu-^sglNXG5Q;{9n(%&*kEb0vApd zRHrY@22=pkFN81%x)~acZeu`yvK zovAVJNykgxqkEr^hZksHkpxm>2I8FTu2%+XLs@?ym0n;;A~X>i32{g6NOB@o4lk8{ zB}7Z2MNAJi>9u=y%s4QUXaNdt@SlAZr54!S6^ETWoik6gw=k-itu_}Yl_M9!l+Rbv z(S&WD`{_|SE@@(|Wp7bq1Zq}mc4JAG?mr2WN~6}~u`7M_F@J9`sr0frzxfuqSF~mA z$m$(TWAuCIE99yLSwi%R)8geQhs;6VBlRhJb(4Cx zu)QIF%_W9+21xI45U>JknBRaZ9nYkgAcK6~E|Zxo!B&z9zQhjsi^fgwZI%K@rYbMq znWBXg1uCZ+ljGJrsW7@x3h2 z;kn!J!bwCeOrBx;oPkZ}FeP%wExyf4=XMp)N8*lct~SyfK~4^-75EZFpHYO5AnuRM z!>u?>Vj3+j=uiHc<=cD~JWRphDSwxFaINB42-{@ZJTWe85>-RcQ&U%?wK)vjz z5u5fJYkck##j(bP7W0*RdW#BmAIK`D3=(U~?b`cJ&U2jHj}?w6 z_4BM)#EoJ6)2?pcR4AqBd)qAUn@RtNQq})FIQoBK4ie+GB(Vih2D|Ds>RJo2zE~C- z7mI)7p)5(-O6JRh6a@VZ5~piVC+Xv=O-)=0eTMSJsRE^c1@bPQWlr}E31VqO-%739 zdcmE{`1m;5LH8w|7euK>>>U#Iod8l1yivC>;YWsg=z#07E%cU9x1yw#3l6AcIm%79 zGi^zH6rM#CZMow(S(8dcOq#5$kbHnQV6s?MRsU3et!!YK5H?OV9vf2qy-UHCn>}2d zTwI(A_fzmmCtE@10yAGgU7R&|Fl$unZJ_^0BgCEDE6(B*SzfkapE9#0N6adc>}dtH zJ#nt^F~@JMJg4=Pv}OdUHyPt-<<9Z&c0@H@^4U?KwZM&6q0XjXc$>K3c&3iXLD9_%(?)?2kmZ=Ykb;)M`Tw=%_d=e@9eheGG zk0<`4so}r={C{zr|6+_1mA_=a56(XyJq||g6Es1E6%fPg#l{r+vk9;)r6VB7D84nu zE0Z1EIxH{Y@}hT+|#$0xn+CdMy6Uhh80eK~nfMEIpM z`|G1v!USmx81nY8XkhEOSWto}pc#{Ut#`Pqb}9j$FpzkQ7`0<-@5D_!mrLah98Mpr zz(R7;ZcaR-$aKqUaO!j z=7QT;Bu0cvYBi+LDfE_WZ`e@YaE_8CCxoRc?Y_!Xjnz~Gl|aYjN2&NtT5v4#q3od2 zkCQZHe#bn(5P#J**Fj4Py%SaaAKJsmV6}F_6Z7V&n6QAu8UQ#9{gkq+tB=VF_Q6~^ zf(hXvhJ#tC(eYm6g|I>;55Lq-;yY*COpTp4?J}hGQ42MIVI9CgEC{3hYw#CZfFKVG zgD(steIg8veyqX%pYMoulq zMUmbj8I`t>mC`!kZ@A>@PYXy*@NprM@e}W2Q+s?XIRM-U1FHVLM~c60(yz1<46-*j zW*FjTnBh$EzI|B|MRU11^McTPIGVJrzozlv$1nah_|t4~u}Ht^S1@V8r@IXAkN;lH z_s|WHlN90k4X}*#neR5bX%}?;G`X!1#U~@X6bbhgDYKJK17~oFF0&-UB#()c$&V<0 z7o~Pfye$P@$)Lj%T;axz+G1L_YQ*#(qO zQND$QTz(~8EF1c3<%;>dAiD$>8j@7WS$G_+ktE|Z?Cx<}HJb=!aChR&4z ziD&FwsiZ)wxS4k6KTLn>d~!DJ^78yb>?Trmx;GLHrbCBy|Bip<@sWdAfP0I~;(Ybr zoc-@j?wA!$ zIP0m3;LZy+>dl#&Ymws@7|{i1+OFLYf@+8+)w}n?mHUBCqg2=-Hb_sBb?=q))N7Ej zDIL9%@xQFOA!(EQmchHiDN%Omrr;WvlPIN5gW;u#ByV)x2aiOd2smy&;vA2+V!u|D zc~K(OVI8} z0t|e0OQ7h23e01O;%SJ}Q#yeDh`|jZR7j-mL(T4E;{w^}2hzmf_6PF|`gWVj{I?^2T3MBK>{?nMXed4kgNox2DP!jvP9v`;pa6AV)OD zDt*Vd-x7s{-;E?E5}3p-V;Y#dB-@c5vTWfS7<=>E+tN$ME`Z7K$px@!%{5{uV`cH80|IzU! zDs9=$%75P^QKCRQ`mW7$q9U?mU@vrFMvx)NNDrI(uk>xwO;^($EUvqVev#{W&GdtR z0ew;Iwa}(-5D28zABlC{WnN{heSY5Eq5Fc=TN^9X#R}0z53!xP85#@;2E=&oNYHyo z46~#Sf!1M1X!rh}ioe`>G2SkPH{5nCoP`GT@}rH;-LP1Q7U_ypw4+lwsqiBql80aA zJE<(88yw$`xzNiSnU(hsyJqHGac<}{Av)x9lQ=&py9djsh0uc}6QkmKN3{P!TEy;P zzLDVQj4>+0r<9B0owxBt5Uz`!M_VSS|{(?`_e+qD9b=vZHoo6>?u;!IP zM7sqoyP>kWY|=v06gkhaGRUrO8n@zE?Yh8$om@8%=1}*!2wdIWsbrCg@;6HfF?TEN z+B_xtSvT6H3in#8e~jvD7eE|LTQhO_>3b823&O_l$R$CFvP@3~)L7;_A}JpgN@ax{ z2d9Ra)~Yh%75wsmHK8e87yAn-ZMiLo6#=<&PgdFsJw1bby-j&3%&4=9dQFltFR(VB z@=6XmyNN4yr^^o$ON8d{PQ=!OX17^CrdM~7D-;ZrC!||<+FEOxI_WI3 zCA<35va%4v>gcEX-@h8esj=a4szW7x z{0g$hwoWRQG$yK{@3mqd-jYiVofJE!Wok1*nV7Gm&Ssq#hFuvj1sRyHg(6PFA5U*Q z8Rx>-blOs=lb`qa{zFy&n4xY;sd$fE+<3EI##W$P9M{B3c3Si9gw^jlPU-JqD~Cye z;wr=XkV7BSv#6}DrsXWFJ3eUNrc%7{=^sP>rp)BWKA9<}^R9g!0q7yWlh;gr_TEOD|#BmGq<@IV;ue zg+D2}cjpp+dPf&Q(36sFU&K8}hA85U61faW&{lB`9HUl-WWCG|<1XANN3JVAkRYvr5U z4q6;!G*MTdSUt*Mi=z_y3B1A9j-@aK{lNvxK%p23>M&=KTCgR!Ee8c?DAO2_R?Bkaqr6^BSP!8dHXxj%N1l+V$_%vzHjq zvu7p@%Nl6;>y*S}M!B=pz=aqUV#`;h%M0rUHfcog>kv3UZAEB*g7Er@t6CF8kHDmK zTjO@rejA^ULqn!`LwrEwOVmHx^;g|5PHm#B6~YD=gjJ!043F+&#_;D*mz%Q60=L9O zve|$gU&~As5^uz@2-BfQ!bW)Khn}G+Wyjw-19qI#oB(RSNydn0t~;tAmK!P-d{b-@ z@E5|cdgOS#!>%#Rj6ynkMvaW@37E>@hJP^82zk8VXx|3mR^JCcWdA|t{0nPmYFOxN z55#^-rlqobcr==<)bi?E?SPymF*a5oDDeSdO0gx?#KMoOd&G(2O@*W)HgX6y_aa6i zMCl^~`{@UR`nMQE`>n_{_aY5nA}vqU8mt8H`oa=g0SyiLd~BxAj2~l$zRSDHxvDs; zI4>+M$W`HbJ|g&P+$!U7-PHX4RAcR0szJ*(e-417=bO2q{492SWrqDK+L3#ChUHtz z*@MP)e^%@>_&#Yk^1|tv@j4%3T)diEXATx4K*hcO`sY$jk#jN5WD<=C3nvuVs zRh||qDHnc~;Kf59zr0;c7VkVSUPD%NnnJC_l3F^#f_rDu8l}l8qcAz0FFa)EAt32I zUy_JLIhU_J^l~FRH&6-iv zSpG2PRqzDdMWft>Zc(c)#tb%wgmWN%>IOPmZi-noqS!^Ft zb81pRcQi`X#UhWK70hy4tGW1mz|+vI8c*h@fFGJtW3r>qV>1Z0r|L>7I3un^gcep$ zAAWfZHRvB|E*kktY$qQP_$YG60C z@X~tTQjB3%@`uz!qxtxF+LE!+=nrS^07hn`EgAp!h|r03h7B!$#OZW#ACD+M;-5J!W+{h z|6I;5cNnE(Y863%1(oH}_FTW})8zYb$7czPg~Szk1+_NTm6SJ0MS_|oSz%e(S~P-& zSFp;!k?uFayytV$8HPwuyELSXOs^27XvK-DOx-Dl!P|28DK6iX>p#Yb%3`A&CG0X2 zS43FjN%IB}q(!hC$fG}yl1y9W&W&I@KTg6@K^kpH8=yFuP+vI^+59|3%Zqnb5lTDAykf9S#X`3N(X^SpdMyWQGOQRjhiwlj!0W-yD<3aEj^ z&X%=?`6lCy~?`&WSWt?U~EKFcCG_RJ(Qp7j=$I%H8t)Z@6Vj zA#>1f@EYiS8MRHZphpMA_5`znM=pzUpBPO)pXGYpQ6gkine{ z6u_o!P@Q+NKJ}k!_X7u|qfpAyIJb$_#3@wJ<1SE2Edkfk9C!0t%}8Yio09^F`YGzp zaJHGk*-ffsn85@)%4@`;Fv^8q(-Wk7r=Q8pT&hD`5(f?M{gfzGbbwh8(}G#|#fDuk z7v1W)5H9wkorE0ZZjL0Q1=NRGY>zwgfm81DdoaVwNH;or{{e zSyybt)m<=zXoA^RALYG-2touH|L*BLvmm9cdMmn+KGopyR@4*=&0 z&4g|FLoreZOhRmh=)R0bg~T2(8V_q7~42-zvb)+y959OAv!V$u(O z3)%Es0M@CRFmG{5sovIq4%8Ahjk#*5w{+)+MWQoJI_r$HxL5km1#6(e@{lK3Udc~n z0@g`g$s?VrnQJ$!oPnb?IHh-1qA`Rz$)Ai<6w$-MJW-gKNvOhL+XMbE7&mFt`x1KY z>k4(!KbbpZ`>`K@1J<(#vVbjx@Z@(6Q}MF#Mnbr-f55)vXj=^j+#)=s+ThMaV~E`B z8V=|W_fZWDwiso8tNMTNse)RNBGi=gVwgg%bOg8>mbRN%7^Um-7oj4=6`$|(K7!+t^90a{$1 z8Z>}<#!bm%ZEFQ{X(yBZMc>lCz0f1I2w9SquGh<9<=AO&g6BZte6hn>Qmvv;Rt)*c zJfTr2=~EnGD8P$v3R|&1RCl&7)b+`=QGapiPbLg_pxm`+HZurtFZ;wZ=`Vk*do~$wBxoW&=j0OTbQ=Q%S8XJ%~qoa3Ea|au5 zo}_(P;=!y z-AjFrERh%8la!z6Fn@lR?^E~H12D? z8#ht=1F;7@o4$Q8GDj;sSC%Jfn01xgL&%F2wG1|5ikb^qHv&9hT8w83+yv&BQXOQy zMVJSBL(Ky~p)gU3#%|blG?I zR9rP^zUbs7rOA0X52Ao=GRt@C&zlyjNLv-}9?*x{y(`509qhCV*B47f2hLrGl^<@S zuRGR!KwHei?!CM10pBKpDIoBNyRuO*>3FU?HjipIE#B~y3FSfOsMfj~F9PNr*H?0o zHyYB^G(YyNh{SxcE(Y-`x5jFMKb~HO*m+R%rq|ic4fzJ#USpTm;X7K+E%xsT_3VHK ze?*uc4-FsILUH;kL>_okY(w`VU*8+l>o>JmiU#?2^`>arnsl#)*R&nf_%>A+qwl%o z{l(u)M?DK1^mf260_oteV3#E_>6Y4!_hhVDM8AI6MM2V*^_M^sQ0dmHu11fy^kOqX zqzps-c5efIKWG`=Es(9&S@K@)ZjA{lj3ea7_MBPk(|hBFRjHVMN!sNUkrB;(cTP)T97M$ z0Dtc&UXSec<+q?y>5=)}S~{Z@ua;1xt@=T5I7{`Z=z_X*no8s>mY;>BvEXK%b`a6(DTS6t&b!vf_z#HM{Uoy z_5fiB(zpkF{})ruka$iX*~pq1ZxD?q68dIoIZSVls9kFGsTwvr4{T_LidcWtt$u{k zJlW7moRaH6+A5hW&;;2O#$oKyEN8kx z`LmG)Wfq4ykh+q{I3|RfVpkR&QH_x;t41UwxzRFXt^E2B$domKT@|nNW`EHwyj>&< zJatrLQ=_3X%vd%nHh^z@vIk(<5%IRAa&Hjzw`TSyVMLV^L$N5Kk_i3ey6byDt)F^U zuM+Ub4*8+XZpnnPUSBgu^ijLtQD>}K;eDpe1bNOh=fvIfk`&B61+S8ND<(KC%>y&? z>opCnY*r5M+!UrWKxv0_QvTlJc>X#AaI^xoaRXL}t5Ej_Z$y*|w*$6D+A?Lw-CO-$ zitm^{2Ct82-<0IW)0KMNvJHgBrdsIR0v~=H?n6^}l{D``Me90`^o|q!olsF?UX3YS zq^6Vu>Ijm>>PaZI8G@<^NGw{Cx&%|PwYrfwR!gX_%AR=L3BFsf8LxI|K^J}deh0Zd zV?$3r--FEX`#INxsOG6_=!v)DI>0q|BxT)z-G6kzA01M?rba+G_mwNMQD1mbVbNTW zmBi*{s_v_Ft9m2Avg!^78(QFu&n6mbRJ2bAv!b;%yo{g*9l2)>tsZJOOp}U~8VUH`}$8p_}t*XIOehezolNa-a2x0BS})Y9}& z*TPgua{Ewn-=wVrmJUeU39EKx+%w%=ixQWKDLpwaNJs65#6o7Ln7~~X+p_o2BR1g~ zVCfxLzxA{HlWAI6^H;`juI=&r1jQrUv_q0Z1Ja-tjdktrrP>GOC*#p?*xfQU5MqjM zsBe!9lh(u8)w$e@Z|>aUHI5o;MGw*|Myiz3-f0;pHg~Q#%*Kx8MxH%AluVXjG2C$) zWL-K63@Q`#y9_k_+}eR(x4~dp7oV-ek0H>Igy8p#i4GN{>#v=pFYUQT(g&b$OeTy- zX_#FDgNF8XyfGY6R!>inYn8IR2RDa&O!(6NIHrC0H+Qpam1bNa=(`SRKjixBTtm&e z`j9porEci!zdlg1RI0Jw#b(_Tb@RQK1Zxr_%7SUeH6=TrXt3J@js`4iDD0=I zoHhK~I7^W8^Rcp~Yaf>2wVe|Hh1bXa_A{oZ9eG$he;_xYvTbTD#moBy zY57-f2Ef1TP^lBi&p5_s7WGG9|0T}dlfxOxXvScJO1Cnq`c`~{Dp;{;l<-KkCDE+p zmexJkd}zCgE{eF=)K``-qC~IT6GcRog_)!X?fK^F8UDz$(zFUrwuR$qro5>qqn>+Z z%<5>;_*3pZ8QM|yv9CAtrAx;($>4l^_$_-L*&?(77!-=zvnCVW&kUcZMb6;2!83si z518Y%R*A3JZ8Is|kUCMu`!vxDgaWjs7^0j(iTaS4HhQ)ldR=r)_7vYFUr%THE}cPF z{0H45FJ5MQW^+W>P+eEX2kLp3zzFe*-pFVAdDZRybv?H|>`9f$AKVjFWJ=wegO7hO zOIYCtd?Vj{EYLT*^gl35|HbMX|NAEUf2ra9dy1=O;figB>La=~eA^#>O6n4?EMugV zbbt{Dbfef5l^(;}5kZ@!XaWwF8z0vUr6r|+QN*|WpF z^*osUHzOnE$lHuWYO$G7>}Y)bY0^9UY4eDV`E{s+{}Z$O$2*lMEYl zTA`ki(<0(Yrm~}15V-E^e2W6`*`%ydED-3G@$UFm6$ZtLx z+av`BhsHcAWqdxPWfu2*%{}|Sptax4_=NpDMeWy$* zZM6__s`enB$~0aT1BU^2k`J9F%+n+lL_|8JklWOCVYt*0%o*j4w1CsB_H^tVpYT_LLyKuyk=CV6~1M<7~^FylL*+AIFf3h>J=x$ygY-BG}4LJ z8XxYPY!v7dO3PVwEoY=`)6krokmR^|Mg5ztX_^#QR}ibr^X-|_St#rtv3gukh0(#A=};NPlNz57ZDFJ9hf#NP50zS)+Fo=StX)i@ zWS?W}i6LjB>kAB~lupAPyIjFb)izFgRq*iS*(Jt509jNr3r72{Gj`5DGoj;J&k5G@Rm!dJ($ox>SbxR)fc zz|Phug;~A7!p@?|mMva@rWuf2fSDK_ZxN3vVmlYz>rrf?LpiNs)^z!y{As@`55JC~ zS*GD3#N-ptY!2<613UelAJ;M4EEI$dm)`8#n$|o{ce^dlyoUY3bsy2hgnj-;ovubb zg2h1rZA6Ot}K_cpYBpIuF&CyK~5R0Wv;kG|3A^8K3nk{rw$Be8u@aos#qvKQKJyVU$cX6biw&Ep#+q7upFX z%qo&`WZ){<%zh@BTl{MO@v9#;t+cb7so0Uz49Fmo1e4>y!vUyIHadguZS0T7-x#_drMXz*16*c zymR0u^`ZQpXN}2ofegbpSedL%F9aypdQcrzjzPlBW0j zMlPzC&ePZ@Cq!?d%9oQNEg0`rHALm8l#lUdXMVEqDvb(AID~H(?H9z!e9G98fG@IzhajKr)3{L_Clu1(Bwg`RM!-(MOuZi zbeDsj9I3(~EITsE=3Z)a|l_rn8W92U0DB70gF7YYfO0j!)h?QobY1lSR>0 z_TVw@$eP~3k8r9;%g%RlZzCJ2%f}DvY`rsZ$;ak&^~-`i%B%+O!pnADeVyV!dHj|} zzOj#q4eRx9Q8c2Z7vy9L&fGLj+3_?fp}+8o`Xpwyi(81H|7P8#65%FIS*lOi={o&v z4NV$xu7az4Nb50dRGZv<tdZCx4Ek<_o3!mAT} zL5l*|K3Qr-)W8paaG z&R6{ped_4e2cy}ejD0!dt{*PaC*^L@eB%(1Fmc%Y#4)~!jF#lCGfj#E??4LG-T;!M z>Uha}f;W>ib_ZL-I7-v9KZQls^G!-JmL^w;=^}?!RXK;m4$#MwI2AH-l7M2-0 zVMK8k^+4+>2S0k^N_40EDa#`7c;2!&3-o6MHsnBfRnq@>E@)=hDulVq-g5SQWDWbt zj6H5?QS2gRZ^Zvbs~cW|8jagJV|;^zqC0e=D1oUsQPJ3MCb+eRGw(XgIY9y8v_tXq z9$(xWntWpx_Uronmvho{JfyYdV{L1N$^s^|-Nj`Ll`lUsiWTjm&8fadUGMXreJGw$ zQ**m+Tj|(XG}DyUKY~2?&9&n6SJ@9VKa9Hcayv{ar^pNr0WHy zP$bQv&8O!vd;GoT!pLwod-42qB^`m!b7nP@YTX}^+1hzA$}LSLh}Ln|?`%8xGMazw z8WT!LoYJ-Aq3=2p6ZSP~uMgSSWv3f`&-I06tU}WhZsA^6nr&r17hjQIZE>^pk=yZ% z06}dfR$85MjWJPq)T?OO(RxoaF+E#4{Z7)i9}Xsb;Nf+dzig61HO;@JX1Lf9)R5j9)Oi6vPL{H z&UQ9ln=$Q8jnh6-t;`hKM6pHftdd?$=1Aq16jty4-TF~`Gx=C&R242uxP{Y@Q~%O3 z*(16@x+vJsbW@^3tzY=-5MHi#(kB};CU%Ep`mVY1j$MAPpYJBB3x$ue`%t}wZ-@CG z(lBv36{2HMjxT)2$n%(UtHo{iW9>4HX4>)%k8QNnzIQYXrm-^M%#Qk%9odbUrZDz1YPdY`2Z4w~p!5tb^m(mUfk}kZ9+EsmenQ)5iwiaulcy zCJ#2o4Dz?@%)aAKfVXYMF;3t@aqNh2tBBlBkCdj`F31b=h93y(46zQ-YK@+zX5qM9 z&=KkN&3@Ptp*>UD$^q-WpG|9O)HBXz{D>p!`a36aPKkgz7uxEo0J>-o+4HHVD9!Hn z${LD0d{tuGsW*wvZoHc8mJroAs(3!FK@~<}Pz1+vY|Gw}Lwfxp{4DhgiQ_SSlV)E| zZWZxYZLu2EB1=g_y@(ieCQC_1?WNA0J0*}eMZfxCCs>oL;?kHdfMcKB+A)Qull$v( z2x6(38utR^-(?DG>d1GyU()8>ih3ud0@r&I$`ZSS<*1n6(76=OmP>r_JuNCdS|-8U zxGKXL1)Lc2kWY@`_kVBt^%7t9FyLVYX(g%a6>j=yURS1!V<9ieT$$5R+yT!I>}jI5 z?fem|T=Jq;BfZmsvqz_Ud*m5;&xE66*o*S22vf-L+MosmUPPA}~wy`kntf8rIeP-m;;{`xe}9E~G7J!PYoVH_$q~NzQab?F8vWUja5BJ!T5%5IpyqI#Dkps0B;gQ*z?c#N>spFw|wRE$gY?y4wQbJ zku2sVLh({KQz6e0yo+X!rV#8n8<;bHWd{ZLL_(*9Oi)&*`LBdGWz>h zx+p`Wi00u#V$f=CcMmEmgFjw+KnbK3`mbaKfoCsB{;Q^oJgj*LWnd_(dk9Kcssbj` z?*g8l`%{*LuY!Ls*|Tm`1Gv-tRparW8q4AK(5pfJFY5>@qO( zcY>pt*na>LlB^&O@YBDnWLE$x7>pMdSmb-?qMh79eB+Wa{)$%}^kX@Z3g>fytppz! zl%>pMD(Yw+5=!UgYHLD69JiJ;YhiGeEyZM$Au{ff;i zCBbNQfO{d!b7z^F732XX&qhEsJA1UZtJjJEIPyDq+F`LeAUU_4`%2aTX#3NG3%W8u zC!7OvlB?QJ4s2#Ok^_8SKcu&pBd}L?vLRT8Kow#xARt`5&Cg=ygYuz>>c z4)+Vv$;<$l=is&E{k&4Lf-Lzq#BHuWc;wDfm4Fbd5Sr!40s{UpKT$kzmUi{V0t1yp zPOf%H8ynE$x@dQ_!+ISaI}#%72UcYm7~|D*(Fp8xiFAj$CmQ4oH3C+Q8W=Y_9Sp|B z+k<%5=y{eW=YvTivV(*KvC?qxo)xqcEU9(Te=?ITts~;xA0Jph-vpd4@Zw#?r2!`? zB3#XtIY^wxrpjJv&(7Xjvm>$TIg2ZC&+^j(gT0R|&4cb)=92-2Hti1`& z=+M;*O%_j3>9zW|3h{0Tfh5i)Fa;clGNJpPRcUmgErzC{B+zACiPHbff3SmsCZ&X; zp=tgI=zW-t(5sXFL8;ITHw0?5FL3+*z5F-KcLN130l=jAU6%F=DClRPrzO|zY+HD`zlZ-)JT}X?2g!o zxg4Ld-mx6&*-N0-MQ(z+zJo8c`B39gf{-h2vqH<=^T&o1Dgd>4BnVht+JwLcrjJl1 zsP!8`>3-rSls07q2i1hScM&x0lQyBbk(U=#3hI7Bkh*kj6H*&^p+J?OMiT_3*vw5R zEl&p|QQHZq6f~TlAeDGy(^BC0vUK?V&#ezC0*#R-h}_8Cw8-*${mVfHssathC8%VA zUE^Qd!;Rvym%|f@?-!sEj|73Vg8!$$zj_QBZAOraF5HCFKl=(Ac|_p%-P;6z<2WSf zz(9jF2x7ZR{w+p)ETCW06PVt0YnZ>gW9^sr&~`%a_7j-Ful~*4=o|&TM@k@Px2z>^ t{*Ed16F~3V5p+(suF-++X8+nHtT~NSfJ>UC3v)>lEpV}<+rIR_{{yMcG_L>v literal 0 HcmV?d00001 diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000..41dfb879 --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-7.4-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew new file mode 100755 index 00000000..7ff1072a --- /dev/null +++ b/gradlew @@ -0,0 +1,234 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit + +APP_NAME="Gradle" +APP_BASE_NAME=${0##*/} + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='' + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + +# Collect all arguments for the java command; +# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of +# shell script including quotes and variable substitutions, so put them in +# double quotes to make sure that they get re-expanded; and +# * put everything else in single quotes, so that it's not re-expanded. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 00000000..107acd32 --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,89 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/pom.xml b/pom.xml deleted file mode 100644 index c3a7e022..00000000 --- a/pom.xml +++ /dev/null @@ -1,441 +0,0 @@ - - 4.0.0 - - hk.edu.polyu.comp.vl - vlabcontroller - 1.0.3 - - VLabController - jar - - - org.springframework.boot - spring-boot-starter-parent - 2.5.6 - - - - - UTF-8 - 1.18.2 - 11 - - 2021.0.0-M3 - - - - - repository.spring.milestone - Spring Milestone Repository - https://repo.spring.io/milestone - - - - - - - org.springframework.cloud - spring-cloud-dependencies - ${spring-cloud.version} - pom - import - - - - - - - javax.json - javax.json-api - 1.1.4 - - - org.glassfish - javax.json - 1.1.4 - - - com.fasterxml.jackson.datatype - jackson-datatype-jsr353 - 2.13.0 - - - org.springframework.boot - spring-boot-starter-mail - - - - org.springframework.boot - spring-boot-configuration-processor - true - - - - - org.apache.commons - commons-collections4 - 4.4 - - - org.apache.commons - commons-compress - 1.21 - - - commons-beanutils - commons-beanutils - 1.9.4 - - - com.google.guava - guava - 31.0.1-jre - - - junit - junit - test - - - org.springframework.data - spring-data-commons - ${project.parent.version} - compile - - - org.jboss.xnio - xnio-api - 3.8.4.Final - compile - - - - - org.springframework.boot - spring-boot-starter-web - - - org.springframework.boot - spring-boot-starter-tomcat - - - - - org.springframework.boot - spring-boot-starter-websocket - - - org.springframework.boot - spring-boot-starter-security - - - org.springframework.boot - spring-boot-starter-undertow - - - org.springframework.boot - spring-boot-starter-thymeleaf - - - org.springframework.boot - spring-boot-starter-test - test - - - org.springframework.boot - spring-boot-starter-webflux - test - - - org.springframework.boot - spring-boot-starter-data-redis - - - - - org.springframework.security.oauth.boot - spring-security-oauth2-autoconfigure - - - org.springframework.security - spring-security-oauth2-client - - - org.springframework.security - spring-security-oauth2-jose - - - org.springframework.security - spring-security-test - test - - - org.springframework.boot - spring-boot-starter-actuator - - - org.springframework.session - spring-session-data-redis - - - org.springframework.cloud - spring-cloud-context - - - - - org.springframework.boot - spring-boot-starter-jdbc - - - com.h2database - h2 - - - - - org.keycloak - keycloak-spring-security-adapter - 15.0.2 - - - - - com.spotify - docker-client - 8.16.0 - - - - org.glassfish.jersey.inject - jersey-hk2 - 3.0.3 - - - - - - - - - org.postgresql - postgresql - - - mysql - mysql-connector-java - - - - io.micrometer - micrometer-registry-prometheus - - - io.micrometer - micrometer-registry-influx - - - - - io.fabric8 - kubernetes-client - 5.9.0 - - - - - org.thymeleaf.extras - thymeleaf-extras-springsecurity5 - - - - - com.amazonaws - aws-java-sdk-s3 - 1.12.90 - - - - - org.projectlombok - lombok - 1.18.22 - provided - - - - com.pivovarit - throwing-function - 1.5.1 - - - - org.springframework.boot - spring-boot-devtools - provided - - - - - - - - - maven-clean-plugin - 3.1.0 - - - - maven-compiler-plugin - 3.8.0 - - - maven-surefire-plugin - 2.22.1 - - - maven-jar-plugin - 3.0.2 - - - maven-install-plugin - 2.5.2 - - - maven-deploy-plugin - 2.8.2 - - - - maven-site-plugin - 3.7.1 - - - maven-project-info-reports-plugin - 3.0.0 - - - - - - com.google.cloud.tools - jib-maven-plugin - 3.1.4 - - - ghcr.io/stevefan1999/vlab-controller-base - - - ghcr.io/endangeredf1sh/vlab-controller:${project.version} - - ${env.REGISTRY_USERNAME} - ${env.REGISTRY_PASSWORD} - - - - /opt/vlab-controller - /opt/vlab-controller - - vlab - /opt/vlab-controller/resources/templates - false - Asia/Hong_Kong - - - - Aiden ZHANG Wenyi <im.endangeredfish@gmail.com>, Fan Chun Yin <stevefan1999@gmail.com> - - - vlab:vlab - - --spring.jmx.enabled=false - --spring.config.location=/etc/vlab-controller/config/application.yml - - - - - - resources/templates - /opt/vlab-controller/resources/templates - - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - - 11 - 11 - - - org.projectlombok - lombok - 1.18.22 - - - - - - - - org.springframework.boot - spring-boot-maven-plugin - ${project.parent.version} - - ${repackage.classifier} - - - - - build-info - - - - - - - org.codehaus.mojo - versions-maven-plugin - 2.8.1 - - - org.apache.commons:commons-collections4 - - - - - - org.apache.maven.plugins - maven-dependency-plugin - 3.2.0 - - - net.nicoulaj.maven.plugins - checksum-maven-plugin - 1.5 - - - - attach-artifact-checksums - - artifacts - - - - - - true - - SHA-256 - MD5 - - - - - - diff --git a/settings.gradle.kts b/settings.gradle.kts new file mode 100644 index 00000000..845d7bca --- /dev/null +++ b/settings.gradle.kts @@ -0,0 +1,14 @@ +/* + * This file was generated by the Gradle 'init' task. + * + * This project uses @Incubating APIs which are subject to change. + */ + +rootProject.name = "vlabcontroller" + +pluginManagement { + repositories { + maven { url = uri("https://repo.spring.io/release") } + gradlePluginPortal() + } +} \ No newline at end of file diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/VLabControllerApplication.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/VLabControllerApplication.java index 4db12fc5..39c4500e 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/VLabControllerApplication.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/VLabControllerApplication.java @@ -1,17 +1,21 @@ package hk.edu.polyu.comp.vlabcontroller; import com.fasterxml.jackson.datatype.jsr353.JSR353Module; +import hk.edu.polyu.comp.vlabcontroller.config.ProxyProperties; +import hk.edu.polyu.comp.vlabcontroller.config.ServerProperties; import hk.edu.polyu.comp.vlabcontroller.util.ProxyMappingManager; import io.undertow.Handlers; import io.undertow.servlet.api.ServletSessionConfig; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.boot.SpringApplication; import org.springframework.boot.actuate.health.Health; import org.springframework.boot.actuate.health.HealthIndicator; import org.springframework.boot.actuate.redis.RedisHealthIndicator; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.ConfigurationPropertiesScan; +import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.boot.web.embedded.undertow.UndertowServletWebServerFactory; import org.springframework.boot.web.server.PortInUseException; import org.springframework.boot.web.servlet.FilterRegistrationBean; @@ -32,38 +36,37 @@ import java.net.UnknownHostException; import java.nio.file.Files; import java.nio.file.Paths; +import java.util.Arrays; import java.util.Objects; +import java.util.Optional; import java.util.Properties; +import java.util.function.Predicate; +@Slf4j @SpringBootApplication +@ConfigurationPropertiesScan +@EnableConfigurationProperties @ComponentScan("hk.edu.polyu.comp") +@RequiredArgsConstructor public class VLabControllerApplication { public static final String CONFIG_FILENAME = "application.yml"; public static final String CONFIG_DEMO_PROFILE = "demo"; - private final Logger log = LogManager.getLogger(getClass()); private final Environment environment; private final ProxyMappingManager mappingManager; private final DefaultCookieSerializer defaultCookieSerializer; - - public VLabControllerApplication(Environment environment, ProxyMappingManager mappingManager, DefaultCookieSerializer defaultCookieSerializer) { - this.environment = environment; - this.mappingManager = mappingManager; - this.defaultCookieSerializer = defaultCookieSerializer; - } + private final ServerProperties serverProps; + private final ProxyProperties proxyProperties; public static void main(String[] args) { - SpringApplication app = new SpringApplication(VLabControllerApplication.class); - - String configFilename = System.getenv("SPRING_CONFIG_LOCATION"); - for (String arg : args) { - String pattern = "spring.config.location="; - int idx = arg.indexOf(pattern); - if (idx > -1) configFilename = arg.substring(idx + pattern.length()); - break; - } - if (configFilename == null) configFilename = CONFIG_FILENAME; - boolean hasExternalConfig = Files.exists(Paths.get(configFilename)); - if (!hasExternalConfig) app.setAdditionalProfiles(CONFIG_DEMO_PROFILE); + var app = new SpringApplication(VLabControllerApplication.class); + + var configFilename = Optional.ofNullable(System.getenv("SPRING_CONFIG_LOCATION")) + .filter(Predicate.not(String::isBlank)) + .or(() -> Arrays.stream(args) + .filter(x -> x.contains("spring.config.location")) + .map(x -> x.split("=")[1]).findFirst()) + .orElse(CONFIG_FILENAME); + if (!Files.exists(Paths.get(configFilename))) app.setAdditionalProfiles(CONFIG_DEMO_PROFILE); setDefaultProperties(app); @@ -79,67 +82,63 @@ public static void main(String[] args) { } private static void setDefaultProperties(SpringApplication app) { - Properties properties = new Properties(); - - // use in-memory session storage by default. Can be overwritten in application.yml - properties.put("spring.session.store-type", "none"); - // required for proper working of the SP_USER_INITIATED_LOGOUT session attribute in the UserService - properties.put("spring.session.redis.flush-mode", "IMMEDIATE"); - - // disable multi-part handling by Spring. We don't need this anywhere in the application. - // When enabled this will cause problems when proxying file-uploads to apps. - properties.put("spring.servlet.multipart.enabled", "false"); - - // disable logging of requests, since this reads part of the requests and therefore undertow is unable to correctly handle those requests - properties.put("logging.level.org.springframework.web.servlet.DispatcherServlet", "INFO"); - - properties.put("spring.application.name", "VLabController"); - - // Metrics configuration - // ==================== - - // disable all supported exporters by default - // Note: if we upgrade to Spring Boot 2.4.0 we can use properties.put("management.metrics.export.defaults.enabled", "false"); - properties.put("management.metrics.export.prometheus.enabled", "false"); - properties.put("management.metrics.export.influx.enabled", "false"); - // set actuator to port 9090 (can be overwritten) - properties.put("management.server.port", "9090"); - // enable prometheus endpoint by default (but not the exporter) - properties.put("management.endpoint.prometheus.enabled", "true"); - // include prometheus and health endpoint in exposure - properties.put("management.endpoints.web.exposure.include", "health,prometheus"); - - // ==================== - - // Health configuration - // ==================== - - // enable redisSession check for the readiness probe - properties.put("management.endpoint.health.group.readiness.include", "readinessProbe,redisSession"); - // disable ldap health endpoint - properties.put("management.health.ldap.enabled", false); - // disable default redis health endpoint since it's managed by redisSession - properties.put("management.health.redis.enabled", "false"); - // enable Kubernetes probes - properties.put("management.endpoint.health.probes.enabled", true); - - // ==================== - - app.setDefaultProperties(properties); + app.setDefaultProperties(new Properties() {{ + // use in-memory session storage by default. Can be overwritten in application.yml + put("spring.session.store-type", "none"); + // required for proper working of the SP_USER_INITIATED_LOGOUT session attribute in the UserService + put("spring.session.redis.flush-mode", "IMMEDIATE"); + + // disable multi-part handling by Spring. We don't need this anywhere in the application. + // When enabled this will cause problems when proxying file-uploads to apps. + put("spring.servlet.multipart.enabled", "false"); + + // disable logging of requests, since this reads part of the requests and therefore undertow is unable to correctly handle those requests + put("logging.level.org.springframework.web.servlet.DispatcherServlet", "INFO"); + + put("spring.application.name", "VLabController"); + + // ==================== + // Metrics configuration + // ==================== + + // disable all supported exporters by default + // Note: if we upgrade to Spring Boot 2.4.0 we can use put("management.metrics.export.defaults.enabled", "false"); + put("management.metrics.export.prometheus.enabled", "false"); + put("management.metrics.export.influx.enabled", "false"); + // set actuator to port 9090 (can be overwritten) + put("management.server.port", "9090"); + // enable prometheus endpoint by default (but not the exporter) + put("management.endpoint.prometheus.enabled", "true"); + // include prometheus and health endpoint in exposure + put("management.endpoints.web.exposure.include", "health,prometheus"); + + // ==================== + // Health configuration + // ==================== + + // enable redisSession check for the readiness probe + put("management.endpoint.health.group.readiness.include", "readinessProbe,redisSession"); + // disable ldap health endpoint + put("management.health.ldap.enabled", false); + // disable default redis health endpoint since it's managed by redisSession + put("management.health.redis.enabled", "false"); + // enable Kubernetes probes + put("management.endpoint.health.probes.enabled", true); + }}); // See: https://github.com/keycloak/keycloak/pull/7053 System.setProperty("jdk.serialSetFilterAfterRead", "true"); } @PostConstruct public void init() { - if (environment.getProperty("server.use-forward-headers") != null) { + if (serverProps.isUseForwardHeaders()) { log.warn("WARNING: Using server.use-forward-headers will not work in this VLabController release, you need to change your configuration to use another property. See https://shinyproxy.io/documentation/security/#forward-headers on how to change your configuration."); } - String sameSiteCookie = environment.getProperty("proxy.same-site-cookie", "Lax"); + var sameSiteCookie = proxyProperties.getSameSiteCookie(); log.debug("Setting sameSiteCookie policy to {}", sameSiteCookie); defaultCookieSerializer.setSameSite(sameSiteCookie); - String proxyIdentifier = environment.getProperty("proxy.identifier-value"); + var proxyIdentifier = proxyProperties.getIdentifierValue(); if (proxyIdentifier != null && !proxyIdentifier.isEmpty()) { defaultCookieSerializer.setCookieName("SESSION_" + proxyIdentifier.toUpperCase()); } @@ -149,32 +148,30 @@ public void init() { @Bean public UndertowServletWebServerFactory servletContainer() { - UndertowServletWebServerFactory factory = new UndertowServletWebServerFactory(); + var factory = new UndertowServletWebServerFactory(); factory.addDeploymentInfoCustomizers(info -> { info.setPreservePathOnForward(false); // required for the /api/route/{id}/ endpoint to work properly - if (Boolean.valueOf(environment.getProperty("logging.requestdump", "false"))) { - info.addOuterHandlerChainWrapper(defaultHandler -> Handlers.requestDump(defaultHandler)); + if (Boolean.parseBoolean(environment.getProperty("logging.requestdump", "false"))) { + info.addOuterHandlerChainWrapper(Handlers::requestDump); } - info.addInnerHandlerChainWrapper(defaultHandler -> { - return mappingManager.createHttpHandler(defaultHandler); - }); - ServletSessionConfig sessionConfig = new ServletSessionConfig(); + info.addInnerHandlerChainWrapper(mappingManager::createHttpHandler); + var sessionConfig = new ServletSessionConfig(); sessionConfig.setHttpOnly(true); - sessionConfig.setSecure(Boolean.valueOf(environment.getProperty("server.secureCookies", "false"))); + sessionConfig.setSecure(serverProps.isSecureCookies()); info.setServletSessionConfig(sessionConfig); }); try { - factory.setAddress(InetAddress.getByName(environment.getProperty("proxy.bind-address", "0.0.0.0"))); + factory.setAddress(InetAddress.getByName(proxyProperties.getBindAddress())); } catch (UnknownHostException e) { throw new IllegalArgumentException("Invalid bind address specified", e); } - factory.setPort(Integer.parseInt(environment.getProperty("proxy.port", "8080"))); + factory.setPort(proxyProperties.getPort()); return factory; } @Bean public FilterRegistrationBean registration2(FormContentFilter filter) { - FilterRegistrationBean registration = new FilterRegistrationBean<>(filter); + var registration = new FilterRegistrationBean<>(filter); registration.setEnabled(false); return registration; } @@ -217,7 +214,7 @@ public Health health() { @Bean @ConditionalOnProperty(name = "spring.session.store-type", havingValue = "redis") public SessionRegistry sessionRegistry(FindByIndexNameSessionRepository sessionRepository) { - return new SpringSessionBackedSessionRegistry(sessionRepository); + return new SpringSessionBackedSessionRegistry<>(sessionRepository); } @Bean diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/VLabControllerConfiguration.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/VLabControllerConfiguration.java index 1ab0dceb..59a19ce9 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/VLabControllerConfiguration.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/VLabControllerConfiguration.java @@ -1,27 +1,31 @@ package hk.edu.polyu.comp.vlabcontroller; +import hk.edu.polyu.comp.vlabcontroller.config.ProxyProperties; import hk.edu.polyu.comp.vlabcontroller.service.HeartbeatService; +import lombok.RequiredArgsConstructor; +import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Lazy; -import org.springframework.core.env.Environment; +import org.springframework.data.mongodb.repository.config.EnableMongoRepositories; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; import javax.annotation.PostConstruct; @Configuration +@RequiredArgsConstructor(onConstructor_ = {@Lazy}) +@RefreshScope +@EnableMongoRepositories +@EnableScheduling public class VLabControllerConfiguration { - private final HeartbeatService heartbeatService; - private final Environment environment; - - public VLabControllerConfiguration(@Lazy HeartbeatService heartbeatService, Environment environment) { - this.heartbeatService = heartbeatService; - this.environment = environment; - } + private final ProxyProperties proxyProperties; + private final ThreadPoolTaskScheduler threadPoolTaskScheduler; @PostConstruct public void init() { + threadPoolTaskScheduler.setPoolSize(2048); // Enable heartbeat unless explicitly disabled. - boolean enabled = Boolean.valueOf(environment.getProperty("proxy.heartbeat-enabled", "true")); - heartbeatService.setEnabled(enabled); + heartbeatService.setEnabled(proxyProperties.isHeartbeatEnabled()); } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/api/BaseController.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/api/BaseController.java index 96124bde..ff6e9afe 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/api/BaseController.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/api/BaseController.java @@ -1,7 +1,9 @@ package hk.edu.polyu.comp.vlabcontroller.api; +import hk.edu.polyu.comp.vlabcontroller.config.ProxyProperties; +import lombok.Setter; import lombok.experimental.StandardException; -import org.springframework.core.env.Environment; +import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.http.HttpStatus; import org.springframework.ui.ModelMap; import org.springframework.web.bind.annotation.ControllerAdvice; @@ -11,13 +13,13 @@ import javax.inject.Inject; +@RefreshScope public class BaseController { - - @Inject - private Environment environment; + @Setter(onMethod_ = {@Inject}) + protected ProxyProperties proxyProperties; protected void prepareMap(ModelMap map) { - map.put("title", environment.getProperty("proxy.title", "VLabController")); + map.put("title", proxyProperties.getTitle()); } @StandardException diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/api/ConfigController.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/api/ConfigController.java index f18f9f5e..56152932 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/api/ConfigController.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/api/ConfigController.java @@ -2,6 +2,7 @@ import hk.edu.polyu.comp.vlabcontroller.event.ConfigUpdateEvent; import hk.edu.polyu.comp.vlabcontroller.util.ConfigFileHelper; +import lombok.RequiredArgsConstructor; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.context.ApplicationEventPublisher; import org.springframework.http.HttpStatus; @@ -13,18 +14,14 @@ @ConditionalOnExpression("${proxy.config.enable-refresh-api:false}") @RestController +@RequiredArgsConstructor public class ConfigController { private final ApplicationEventPublisher publisher; private final ConfigFileHelper configFileHelper; - public ConfigController(ApplicationEventPublisher publisher, ConfigFileHelper configFileHelper) { - this.publisher = publisher; - this.configFileHelper = configFileHelper; - } - @PostMapping(value = "/api/config/refresh") public ResponseEntity refresh() throws NoSuchAlgorithmException { - String hash = configFileHelper.getConfigHash(); + var hash = configFileHelper.getConfigHash(); publisher.publishEvent(new ConfigUpdateEvent(this)); return new ResponseEntity<>(hash, HttpStatus.OK); } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/api/ProxyController.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/api/ProxyController.java index 87bcbaea..9d86496c 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/api/ProxyController.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/api/ProxyController.java @@ -1,25 +1,28 @@ package hk.edu.polyu.comp.vlabcontroller.api; +import com.google.common.collect.Sets; import hk.edu.polyu.comp.vlabcontroller.model.runtime.Proxy; import hk.edu.polyu.comp.vlabcontroller.model.runtime.RuntimeSetting; import hk.edu.polyu.comp.vlabcontroller.model.spec.ProxySpec; import hk.edu.polyu.comp.vlabcontroller.service.ProxyService; +import io.vavr.Function1; +import lombok.RequiredArgsConstructor; import org.springframework.http.HttpStatus; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.*; +import java.time.Duration; import java.util.List; +import java.util.Map; +import java.util.Optional; import java.util.Set; @RestController +@RequiredArgsConstructor public class ProxyController extends BaseController { private final ProxyService proxyService; - public ProxyController(ProxyService proxyService) { - this.proxyService = proxyService; - } - @GetMapping(value = "/api/proxyspec", produces = MediaType.APPLICATION_JSON_VALUE) public List listProxySpecs() { return proxyService.getProxySpecs(null, false); @@ -27,9 +30,8 @@ public List listProxySpecs() { @GetMapping(value = "/api/proxyspec/{proxySpecId}", produces = MediaType.APPLICATION_JSON_VALUE) public ResponseEntity getProxySpec(@PathVariable String proxySpecId) { - ProxySpec spec = proxyService.findProxySpec(s -> s.getId().equals(proxySpecId), false); - if (spec == null) return new ResponseEntity<>(HttpStatus.NOT_FOUND); - return new ResponseEntity<>(spec, HttpStatus.OK); + return findProxySpecByIdAndACL(proxySpecId) + .map(ResponseEntity::ok).orElse(ResponseEntity.notFound().build()); } @GetMapping(value = "/api/proxy", produces = MediaType.APPLICATION_JSON_VALUE) @@ -39,34 +41,101 @@ public List listProxies() { @GetMapping(value = "/api/proxy/{proxyId}", produces = MediaType.APPLICATION_JSON_VALUE) public ResponseEntity getProxy(@PathVariable String proxyId) { - Proxy proxy = proxyService.findProxy(p -> p.getId().equals(proxyId), false); - if (proxy == null) return new ResponseEntity<>(HttpStatus.NOT_FOUND); - return new ResponseEntity<>(proxy, HttpStatus.OK); + return findProxyByIdAndACL(proxyId, false) + .map(ResponseEntity::ok).orElse(ResponseEntity.notFound().build()); + } + + @PostMapping(value = "/api/proxy/{proxyId}/metadata", produces = MediaType.APPLICATION_JSON_VALUE) + public ResponseEntity> setMetadata( + @PathVariable String proxyId, @RequestBody Map payload, + @RequestParam(required = false, defaultValue = "true") boolean override + ) { + return findProxyByIdAndACL(proxyId, true) + .map((Function1>>) proxy -> { + var metadata = proxy.getMetadata(); + var duplicates = Sets.intersection(metadata.keySet(), payload.keySet()); + var shouldPut = duplicates.isEmpty() || override; + if (shouldPut) metadata.putAll(payload); + return shouldPut ? ResponseEntity.ok(metadata) : ResponseEntity.status(HttpStatus.CONFLICT).body(Map.of("conflicts", duplicates)); + }) + .orElse(ResponseEntity.notFound().build()); + } + + @PostMapping(value = "/api/proxy/{proxyId}/metadata/{key}", produces = MediaType.APPLICATION_JSON_VALUE) + public ResponseEntity> setMetadata( + @PathVariable String proxyId, @PathVariable String key, @RequestBody Object value, + @RequestParam(required = false, defaultValue = "true") boolean override + ) { + return findProxyByIdAndACL(proxyId, true) + .map((Function1>>) proxy -> { + var metadata = proxy.getMetadata(); + var shouldPut = !metadata.containsKey(key) || override; + if (shouldPut) metadata.put(key, value); + return shouldPut ? ResponseEntity.ok(metadata) : ResponseEntity.status(HttpStatus.CONFLICT).build(); + }).orElse(ResponseEntity.notFound().build()); + } + + @GetMapping(value = "/api/proxy/{proxyId}/metadata", produces = MediaType.APPLICATION_JSON_VALUE) + public ResponseEntity> getMetadata(@PathVariable String proxyId) { + return findProxyByIdAndACL(proxyId, true) + .flatMap(proxy -> Optional.ofNullable(proxy.getMetadata()).map(ResponseEntity::ok)) + .orElse(ResponseEntity.notFound().build()); + } + + @GetMapping(value = "/api/proxy/{proxyId}/metadata/{key}", produces = MediaType.APPLICATION_JSON_VALUE) + public ResponseEntity getMetadata(@PathVariable String proxyId, @PathVariable String key) { + return findProxyByIdAndACL(proxyId, true) + .flatMap(proxy -> Optional.ofNullable(proxy.getMetadata().get(key)).map(ResponseEntity::ok)) + .orElse(ResponseEntity.notFound().build()); + } + + @DeleteMapping(value = "/api/proxy/{proxyId}/metadata/{key}", produces = MediaType.APPLICATION_JSON_VALUE) + public ResponseEntity deleteMetadata( + @PathVariable String proxyId, @PathVariable String key, + @RequestParam(required = false, defaultValue = "true") boolean silentIfNotExist + ) { + return findProxyByIdAndACL(proxyId, true) + .map((Function1>) proxy -> { + var metadata = proxy.getMetadata(); + var shouldRemove = metadata.containsKey(key) || silentIfNotExist; + if (shouldRemove) metadata.remove(key); + return shouldRemove ? ResponseEntity.ok().build() : ResponseEntity.notFound().build(); + }) + .orElse(ResponseEntity.notFound().build()); } @PostMapping(value = "/api/proxy/{proxySpecId}", produces = MediaType.APPLICATION_JSON_VALUE) public ResponseEntity startProxy(@PathVariable String proxySpecId, @RequestBody(required = false) Set runtimeSettings) { - ProxySpec baseSpec = proxyService.findProxySpec(s -> s.getId().equals(proxySpecId), false); - if (baseSpec == null) return new ResponseEntity<>(HttpStatus.NOT_FOUND); - - ProxySpec spec = proxyService.resolveProxySpec(baseSpec, null, runtimeSettings); - Proxy proxy = proxyService.startProxy(spec, false); - return new ResponseEntity<>(proxy, HttpStatus.CREATED); + return findProxySpecByIdAndACL(proxySpecId) + .map(baseSpec -> { + var spec = proxyService.resolveProxySpec(baseSpec, null, runtimeSettings); + var proxy = proxyService.startProxy(spec, false); + return ResponseEntity.status(HttpStatus.CREATED).body(proxy); + }).orElse(ResponseEntity.notFound().build()); } @PostMapping(value = "/api/proxy", produces = MediaType.APPLICATION_JSON_VALUE) public ResponseEntity startProxy(@RequestBody ProxySpec proxySpec) { - ProxySpec spec = proxyService.resolveProxySpec(null, proxySpec, null); - Proxy proxy = proxyService.startProxy(spec, false); - return new ResponseEntity<>(proxy, HttpStatus.CREATED); + var spec = proxyService.resolveProxySpec(null, proxySpec, null); + var proxy = proxyService.startProxy(spec, false); + return ResponseEntity.status(HttpStatus.CREATED).body(proxy); } @DeleteMapping(value = "/api/proxy/{proxyId}", produces = MediaType.APPLICATION_JSON_VALUE) public ResponseEntity stopProxy(@PathVariable String proxyId) { - Proxy proxy = proxyService.findProxy(p -> p.getId().equals(proxyId), false); - if (proxy == null) return new ResponseEntity<>(HttpStatus.NOT_FOUND); + return findProxyByIdAndACL(proxyId, false) + .map(proxy -> { + proxyService.stopProxy(proxy, true, false, Duration.ZERO); + return ResponseEntity.ok("Proxy stopped"); + }) + .orElse(ResponseEntity.notFound().build()); + } + + private Optional findProxyByIdAndACL(String proxyId, boolean ignoreAccessControl) { + return Optional.ofNullable(proxyService.findProxy(p -> p.getId().equals(proxyId), ignoreAccessControl)); + } - proxyService.stopProxy(proxy, true, false, 0); - return new ResponseEntity<>("Proxy stopped", HttpStatus.OK); + private Optional findProxySpecByIdAndACL(String specId) { + return Optional.ofNullable(proxyService.findProxySpec(p -> p.getId().equals(specId), false)); } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/api/ProxyRouteController.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/api/ProxyRouteController.java index 9b40dd28..124bf3c9 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/api/ProxyRouteController.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/api/ProxyRouteController.java @@ -1,12 +1,12 @@ package hk.edu.polyu.comp.vlabcontroller.api; -import hk.edu.polyu.comp.vlabcontroller.model.runtime.Proxy; +import hk.edu.polyu.comp.vlabcontroller.config.ServerProperties; import hk.edu.polyu.comp.vlabcontroller.service.ProxyService; import hk.edu.polyu.comp.vlabcontroller.service.UserService; import hk.edu.polyu.comp.vlabcontroller.util.ProxyMappingManager; import hk.edu.polyu.comp.vlabcontroller.util.SessionHelper; -import org.apache.commons.lang.StringUtils; -import org.springframework.core.env.Environment; +import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.RequestMapping; @@ -14,37 +14,31 @@ import javax.servlet.http.HttpServletResponse; @Controller +@RequiredArgsConstructor public class ProxyRouteController extends BaseController { private final UserService userService; private final ProxyService proxyService; private final ProxyMappingManager mappingManager; - private final Environment environment; - - public ProxyRouteController(UserService userService, ProxyService proxyService, ProxyMappingManager mappingManager, Environment environment) { - this.userService = userService; - this.proxyService = proxyService; - this.mappingManager = mappingManager; - this.environment = environment; - } + private final ServerProperties serverProperties; @RequestMapping(value = "/api/route/**") public void route(HttpServletRequest request, HttpServletResponse response) { try { - String baseURL = SessionHelper.getContextPath(environment, true) + "api/route/"; - String mapping = request.getRequestURI().substring(baseURL.length()).replaceAll("/{2,}", "/"); - String proxyId = mappingManager.getProxyId(mapping); - String prefix = proxyId; + var baseURL = SessionHelper.getContextPath(serverProperties, true) + "api/route/"; + var mapping = request.getRequestURI().substring(baseURL.length()).replaceAll("/{2,}", "/"); + var proxyId = mappingManager.getProxyId(mapping); + var prefix = proxyId; if (proxyId != null) { - boolean isAdmin = userService.isAdmin(); - Proxy proxy = proxyService.findProxy(p -> proxyId.equals(p.getId()), true); - String[] path = mapping.split("/"); - String mappingType = path.length > 1 ? path[1] : ""; - int targetPort = -1; - boolean hasAccess = userService.isOwner(proxy); + var isAdmin = userService.isAdmin(); + var proxy = proxyService.findProxy(p -> proxyId.equals(p.getId()), true); + var path = mapping.split("/"); + var mappingType = path.length > 1 ? path[1] : ""; + var targetPort = -1; + var hasAccess = userService.isOwner(proxy); if (("/" + mappingType).equals(mappingManager.getProxyPortMappingsEndpoint())) { - String portString = path[2]; + var portString = path[2]; if (portString != null) { - int port = Integer.parseInt(portString); + var port = Integer.parseInt(portString); if (port < 0 || port > 65535) { response.sendError(404, "Invalid port"); } else { @@ -54,7 +48,7 @@ public void route(HttpServletRequest request, HttpServletResponse response) { } } if (hasAccess || isAdmin) { - String subPath = StringUtils.substringAfter(mapping, prefix); + var subPath = StringUtils.substringAfter(mapping, prefix); if (subPath.trim().isEmpty()) { response.sendRedirect(request.getRequestURI() + "/"); return; diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/auth/AuthenticationBackendFactory.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/auth/AuthenticationBackendFactory.java index e81b1391..6c88d24c 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/auth/AuthenticationBackendFactory.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/auth/AuthenticationBackendFactory.java @@ -1,59 +1,53 @@ package hk.edu.polyu.comp.vlabcontroller.auth; import hk.edu.polyu.comp.vlabcontroller.auth.impl.*; +import hk.edu.polyu.comp.vlabcontroller.config.ProxyProperties; +import lombok.RequiredArgsConstructor; import org.springframework.beans.factory.config.AbstractFactoryBean; +import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Primary; -import org.springframework.core.env.Environment; import org.springframework.stereotype.Service; +import java.util.concurrent.atomic.AtomicBoolean; + +import static io.vavr.API.*; + /** * Instantiates an appropriate authentication backend depending on the application configuration. */ @Service(value = "authenticationBackend") @Primary +@RequiredArgsConstructor +@RefreshScope public class AuthenticationBackendFactory extends AbstractFactoryBean { - private final Environment environment; + private final ProxyProperties proxyProperties; private final ApplicationContext applicationContext; // These backends register some beans of their own, so must be instantiated here. private final KeycloakAuthenticationBackend keycloakBackend; - public AuthenticationBackendFactory(Environment environment, ApplicationContext applicationContext, KeycloakAuthenticationBackend keycloakBackend) { - this.environment = environment; - this.applicationContext = applicationContext; - this.keycloakBackend = keycloakBackend; - } - @Override public Class getObjectType() { return IAuthenticationBackend.class; } @Override - protected IAuthenticationBackend createInstance() throws Exception { - IAuthenticationBackend backend = null; - - String type = environment.getProperty("proxy.authentication", "none"); - switch (type) { - case NoAuthenticationBackend.NAME: - backend = new NoAuthenticationBackend(); - break; - case SimpleAuthenticationBackend.NAME: - backend = new SimpleAuthenticationBackend(); - break; - case OpenIDAuthenticationBackend.NAME: - backend = new OpenIDAuthenticationBackend(); - break; - case KeycloakAuthenticationBackend.NAME: + protected IAuthenticationBackend createInstance() { + var regBeans = new AtomicBoolean(true); + var backend = Match(proxyProperties.getAuthentication()).of( + Case($(NoAuthenticationBackend.NAME), NoAuthenticationBackend::new), + Case($(SimpleAuthenticationBackend.NAME), SimpleAuthenticationBackend::new), + Case($(OpenIDAuthenticationBackend.NAME), OpenIDAuthenticationBackend::new), + Case($(WebServiceAuthenticationBackend.NAME), WebServiceAuthenticationBackend::new), + Case($(KeycloakAuthenticationBackend.NAME), () -> { + regBeans.set(false); return keycloakBackend; - case WebServiceAuthenticationBackend.NAME: - backend = new WebServiceAuthenticationBackend(); - break; - default: + }), + Case($(), type -> { throw new RuntimeException("Unknown authentication type:" + type); - } - - applicationContext.getAutowireCapableBeanFactory().autowireBean(backend); + }) + ); + if (regBeans.get()) applicationContext.getAutowireCapableBeanFactory().autowireBean(backend); return backend; } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/auth/IAuthenticationBackend.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/auth/IAuthenticationBackend.java index 290960d5..75ebc682 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/auth/IAuthenticationBackend.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/auth/IAuthenticationBackend.java @@ -1,12 +1,14 @@ package hk.edu.polyu.comp.vlabcontroller.auth; import hk.edu.polyu.comp.vlabcontroller.model.spec.ContainerSpec; +import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.security.config.annotation.authentication.builders.AuthenticationManagerBuilder; import org.springframework.security.config.annotation.web.builders.HttpSecurity; import org.springframework.security.config.annotation.web.configurers.ExpressionUrlAuthorizationConfigurer.AuthorizedUrl; -import java.util.List; +import java.util.Map; +@RefreshScope public interface IAuthenticationBackend { /** @@ -46,7 +48,7 @@ default void customizeContainer(ContainerSpec spec) { // Default: do nothing. } - default void customizeContainerEnv(List env) { + default void customizeContainerEnv(Map env) { // Default: do nothing. } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/auth/UserLogoutHandler.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/auth/UserLogoutHandler.java index f8ff4e06..b1e6e878 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/auth/UserLogoutHandler.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/auth/UserLogoutHandler.java @@ -1,6 +1,7 @@ package hk.edu.polyu.comp.vlabcontroller.auth; import hk.edu.polyu.comp.vlabcontroller.service.UserService; +import lombok.RequiredArgsConstructor; import org.springframework.security.core.Authentication; import org.springframework.security.web.authentication.logout.LogoutHandler; import org.springframework.stereotype.Component; @@ -9,13 +10,10 @@ import javax.servlet.http.HttpServletResponse; @Component +@RequiredArgsConstructor public class UserLogoutHandler implements LogoutHandler { private final UserService userService; - public UserLogoutHandler(UserService userService) { - this.userService = userService; - } - @Override public void logout(HttpServletRequest request, HttpServletResponse response, Authentication authentication) { userService.logout(authentication); diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/auth/impl/KeycloakAuthenticationBackend.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/auth/impl/KeycloakAuthenticationBackend.java index 7015966c..279ca0f9 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/auth/impl/KeycloakAuthenticationBackend.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/auth/impl/KeycloakAuthenticationBackend.java @@ -2,11 +2,10 @@ import hk.edu.polyu.comp.vlabcontroller.auth.IAuthenticationBackend; import hk.edu.polyu.comp.vlabcontroller.auth.impl.keycloak.AuthenticationFailureHandler; +import hk.edu.polyu.comp.vlabcontroller.config.ProxyProperties; +import lombok.RequiredArgsConstructor; import org.keycloak.adapters.AdapterDeploymentContext; -import org.keycloak.adapters.KeycloakConfigResolver; -import org.keycloak.adapters.KeycloakDeployment; import org.keycloak.adapters.KeycloakDeploymentBuilder; -import org.keycloak.adapters.spi.HttpFacade.Request; import org.keycloak.adapters.spi.KeycloakAccount; import org.keycloak.adapters.springsecurity.AdapterDeploymentContextFactoryBean; import org.keycloak.adapters.springsecurity.account.KeycloakRole; @@ -21,10 +20,10 @@ import org.keycloak.representations.IDToken; import org.keycloak.representations.adapters.config.AdapterConfig; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Lazy; -import org.springframework.core.env.Environment; import org.springframework.security.authentication.AuthenticationManager; import org.springframework.security.config.annotation.authentication.builders.AuthenticationManagerBuilder; import org.springframework.security.config.annotation.web.builders.HttpSecurity; @@ -49,28 +48,24 @@ import javax.servlet.ServletException; import java.io.Serializable; +import java.util.List; +import java.util.Map; import java.util.*; import java.util.stream.Collectors; +import static io.vavr.API.*; @Component +@RequiredArgsConstructor(onConstructor_ = {@Lazy}) +@RefreshScope public class KeycloakAuthenticationBackend implements IAuthenticationBackend { public static final String NAME = "keycloak"; - final Environment environment; + final ProxyProperties proxyProperties; final WebSecurityConfigurerAdapter webSecurityConfigurerAdapter; final ApplicationContext ctx; final AuthenticationManager authenticationManager; - - @Lazy - public KeycloakAuthenticationBackend(Environment environment, WebSecurityConfigurerAdapter webSecurityConfigurerAdapter, ApplicationContext ctx, AuthenticationManager authenticationManager) { - this.environment = environment; - this.webSecurityConfigurerAdapter = webSecurityConfigurerAdapter; - this.ctx = ctx; - this.authenticationManager = authenticationManager; - } - @Override public String getName() { return NAME; @@ -118,7 +113,7 @@ protected KeycloakAuthenticationProcessingFilter keycloakAuthenticationProcessin new RequestHeaderRequestMatcher(KeycloakAuthenticationProcessingFilter.AUTHORIZATION_HEADER) ); - KeycloakAuthenticationProcessingFilter filter = new KeycloakAuthenticationProcessingFilter(authenticationManager, requestMatcher); + var filter = new KeycloakAuthenticationProcessingFilter(authenticationManager, requestMatcher); filter.setSessionAuthenticationStrategy(sessionAuthenticationStrategy()); filter.setAuthenticationFailureHandler(keycloakAuthenticationFailureHandler()); // Fix: call afterPropertiesSet manually, because Spring doesn't invoke it for some reason. @@ -130,7 +125,7 @@ protected KeycloakAuthenticationProcessingFilter keycloakAuthenticationProcessin @Bean @ConditionalOnProperty(name = "proxy.authentication", havingValue = "keycloak") protected KeycloakPreAuthActionsFilter keycloakPreAuthActionsFilter() { - KeycloakPreAuthActionsFilter filter = new KeycloakPreAuthActionsFilter(httpSessionManager()); + var filter = new KeycloakPreAuthActionsFilter(httpSessionManager()); // Fix: call afterPropertiesSet manually, because Spring doesn't invoke it for some reason. filter.setApplicationContext(ctx); try { @@ -164,22 +159,17 @@ public KeycloakAuthenticationFailureHandler keycloakAuthenticationFailureHandler @Bean @ConditionalOnProperty(name = "proxy.authentication", havingValue = "keycloak") protected AdapterDeploymentContext adapterDeploymentContext() throws Exception { - AdapterConfig cfg = new AdapterConfig(); - cfg.setRealm(environment.getProperty("proxy.keycloak.realm")); - cfg.setAuthServerUrl(environment.getProperty("proxy.keycloak.auth-server-url")); - cfg.setResource(environment.getProperty("proxy.keycloak.resource")); - cfg.setSslRequired(environment.getProperty("proxy.keycloak.ssl-required", "external")); - cfg.setUseResourceRoleMappings(Boolean.parseBoolean(environment.getProperty("proxy.keycloak.use-resource-role-mappings", "false"))); - Map credentials = new HashMap<>(); - credentials.put("secret", environment.getProperty("proxy.keycloak.credentials-secret")); - cfg.setCredentials(credentials); - KeycloakDeployment dep = KeycloakDeploymentBuilder.build(cfg); - AdapterDeploymentContextFactoryBean factoryBean = new AdapterDeploymentContextFactoryBean(new KeycloakConfigResolver() { - @Override - public KeycloakDeployment resolve(Request facade) { - return dep; - } - }); + var cfg = new AdapterConfig(); + var keycloak = proxyProperties.getKeycloak(); + + cfg.setRealm(keycloak.getRealm()); + cfg.setAuthServerUrl(keycloak.getAuthServerUrl()); + cfg.setResource(keycloak.getResource()); + cfg.setSslRequired(keycloak.getSslRequired()); + cfg.setUseResourceRoleMappings(keycloak.isUseResourceRoleMappings()); + cfg.setCredentials(Map.of("secret", keycloak.getCredentialsSecret())); + var dep = KeycloakDeploymentBuilder.build(cfg); + var factoryBean = new AdapterDeploymentContextFactoryBean(facade -> dep); factoryBean.afterPropertiesSet(); return factoryBean.getObject(); } @@ -192,13 +182,13 @@ protected KeycloakAuthenticationProvider keycloakAuthenticationProvider() { return new KeycloakAuthenticationProvider() { @Override public Authentication authenticate(Authentication authentication) throws AuthenticationException { - KeycloakAuthenticationToken token = (KeycloakAuthenticationToken) super.authenticate(authentication); + var token = (KeycloakAuthenticationToken) super.authenticate(authentication); List auth = token.getAuthorities().stream() - .map(t -> t.getAuthority().toUpperCase()) - .map(a -> a.startsWith("ROLE_") ? a : "ROLE_" + a) - .map(KeycloakRole::new) - .collect(Collectors.toList()); - String nameAttribute = environment.getProperty("proxy.keycloak.name-attribute", IDToken.NAME).toLowerCase(); + .map(t -> t.getAuthority().toUpperCase()) + .map(a -> a.startsWith("ROLE_") ? a : "ROLE_" + a) + .map(KeycloakRole::new) + .collect(Collectors.toList()); + var nameAttribute = proxyProperties.getKeycloak().getNameAttribute().toLowerCase(); return new KeycloakAuthenticationToken2(token.getAccount(), token.isInteractive(), nameAttribute, auth); } }; @@ -221,20 +211,14 @@ public KeycloakAuthenticationToken2(KeycloakAccount account, boolean interactive @Override public String getName() { - IDToken token = getAccount().getKeycloakSecurityContext().getIdToken(); - if (token == null) { - token = getAccount().getKeycloakSecurityContext().getToken(); - } - switch (nameAttribute) { - case IDToken.PREFERRED_USERNAME: - return token.getPreferredUsername(); - case IDToken.NICKNAME: - return token.getNickName(); - case IDToken.EMAIL: - return token.getEmail(); - default: - return token.getName(); - } + var ctx = getAccount().getKeycloakSecurityContext(); + var token = Optional.ofNullable(ctx.getIdToken()).orElseGet(ctx::getToken); + return Match(nameAttribute).of( + Case($(IDToken.PREFERRED_USERNAME), token::getPreferredUsername), + Case($(IDToken.NICKNAME), token::getNickName), + Case($(IDToken.EMAIL), token::getEmail), + Case($(), token::getName) + ); } } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/auth/impl/OpenIDAuthenticationBackend.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/auth/impl/OpenIDAuthenticationBackend.java index afce92d9..c8fa1a3c 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/auth/impl/OpenIDAuthenticationBackend.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/auth/impl/OpenIDAuthenticationBackend.java @@ -1,24 +1,23 @@ package hk.edu.polyu.comp.vlabcontroller.auth.impl; import hk.edu.polyu.comp.vlabcontroller.auth.IAuthenticationBackend; +import hk.edu.polyu.comp.vlabcontroller.config.ProxyProperties; +import hk.edu.polyu.comp.vlabcontroller.config.ServerProperties; import hk.edu.polyu.comp.vlabcontroller.security.FixedDefaultOAuth2AuthorizationRequestResolver; import hk.edu.polyu.comp.vlabcontroller.util.SessionHelper; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; import net.minidev.json.JSONArray; import net.minidev.json.parser.JSONParser; import net.minidev.json.parser.ParseException; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.springframework.core.env.Environment; +import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.security.config.annotation.authentication.builders.AuthenticationManagerBuilder; import org.springframework.security.config.annotation.web.builders.HttpSecurity; import org.springframework.security.config.annotation.web.configurers.ExpressionUrlAuthorizationConfigurer.AuthorizedUrl; -import org.springframework.security.core.Authentication; -import org.springframework.security.core.AuthenticationException; import org.springframework.security.core.GrantedAuthority; import org.springframework.security.core.authority.SimpleGrantedAuthority; import org.springframework.security.core.authority.mapping.GrantedAuthoritiesMapper; import org.springframework.security.core.context.SecurityContextHolder; -import org.springframework.security.oauth2.client.OAuth2AuthorizedClient; import org.springframework.security.oauth2.client.oidc.userinfo.OidcUserRequest; import org.springframework.security.oauth2.client.oidc.userinfo.OidcUserService; import org.springframework.security.oauth2.client.registration.ClientRegistration; @@ -36,19 +35,17 @@ import org.springframework.security.oauth2.core.oidc.user.DefaultOidcUser; import org.springframework.security.oauth2.core.oidc.user.OidcUser; import org.springframework.security.oauth2.core.oidc.user.OidcUserAuthority; -import org.springframework.security.web.authentication.AuthenticationFailureHandler; import org.springframework.web.context.request.RequestContextHolder; import org.springframework.web.context.request.ServletRequestAttributes; import org.springframework.web.servlet.support.ServletUriComponentsBuilder; import javax.inject.Inject; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.IOException; import java.util.*; import java.util.stream.Collectors; +import java.util.stream.Stream; +@Slf4j +@RefreshScope public class OpenIDAuthenticationBackend implements IAuthenticationBackend { public static final String NAME = "openid"; @@ -56,12 +53,13 @@ public class OpenIDAuthenticationBackend implements IAuthenticationBackend { private static final String REG_ID = "vlab"; private static final String ENV_TOKEN_NAME = "VLAB_OIDC_ACCESS_TOKEN"; - private final Logger log = LogManager.getLogger(OpenIDAuthenticationBackend.class); - private OAuth2AuthorizedClientRepository oAuth2AuthorizedClientRepository; - @Inject - private Environment environment; + @Setter(onMethod_ = {@Inject}) + private ProxyProperties proxyProperties; + + @Setter(onMethod_ = {@Inject}) + private ServerProperties serverProperties; @Override public String getName() { @@ -75,32 +73,26 @@ public boolean hasAuthorization() { @Override public void configureHttpSecurity(HttpSecurity http, AuthorizedUrl anyRequestConfigurer) throws Exception { - ClientRegistrationRepository clientRegistrationRepo = createClientRepo(); + var clientRegistrationRepo = createClientRepo(); oAuth2AuthorizedClientRepository = new HttpSessionOAuth2AuthorizedClientRepository(); anyRequestConfigurer.authenticated(); http - .oauth2Login() - .loginPage("/login") - .clientRegistrationRepository(clientRegistrationRepo) - .authorizedClientRepository(oAuth2AuthorizedClientRepository) - .authorizationEndpoint() - .authorizationRequestResolver(new FixedDefaultOAuth2AuthorizationRequestResolver(clientRegistrationRepo, OAuth2AuthorizationRequestRedirectFilter.DEFAULT_AUTHORIZATION_REQUEST_BASE_URI)) - .and() - .failureHandler(new AuthenticationFailureHandler() { - - @Override - public void onAuthenticationFailure(HttpServletRequest request, HttpServletResponse response, - AuthenticationException exception) throws IOException, ServletException { - log.error(exception); - response.sendRedirect(ServletUriComponentsBuilder.fromCurrentContextPath().path("/auth-error").build().toUriString()); - } - - }) - .userInfoEndpoint() - .userAuthoritiesMapper(createAuthoritiesMapper()) - .oidcUserService(createOidcUserService()); + .oauth2Login() + .loginPage("/login") + .clientRegistrationRepository(clientRegistrationRepo) + .authorizedClientRepository(oAuth2AuthorizedClientRepository) + .authorizationEndpoint() + .authorizationRequestResolver(new FixedDefaultOAuth2AuthorizationRequestResolver(clientRegistrationRepo, OAuth2AuthorizationRequestRedirectFilter.DEFAULT_AUTHORIZATION_REQUEST_BASE_URI)) + .and() + .failureHandler((request, response, exception) -> { + log.error("an error occured: {}", exception); + response.sendRedirect(ServletUriComponentsBuilder.fromCurrentContextPath().path("/auth-error").build().toUriString()); + }) + .userInfoEndpoint() + .userAuthoritiesMapper(createAuthoritiesMapper()) + .oidcUserService(createOidcUserService()); } @Override @@ -109,117 +101,98 @@ public void configureAuthenticationManagerBuilder(AuthenticationManagerBuilder a } public String getLoginRedirectURI() { - return SessionHelper.getContextPath(environment, false) - + OAuth2AuthorizationRequestRedirectFilter.DEFAULT_AUTHORIZATION_REQUEST_BASE_URI - + "/" + REG_ID; + return SessionHelper.getContextPath(serverProperties, false) + + OAuth2AuthorizationRequestRedirectFilter.DEFAULT_AUTHORIZATION_REQUEST_BASE_URI + + "/" + REG_ID; } @Override public String getLogoutSuccessURL() { - String logoutURL = environment.getProperty("proxy.openid.logout-url"); - if (logoutURL == null || logoutURL.trim().isEmpty()) + var logoutURL = proxyProperties.getOpenID().getLogoutUrl(); + if (logoutURL == null || logoutURL.isBlank()) logoutURL = IAuthenticationBackend.super.getLogoutSuccessURL(); return logoutURL; } @Override - public void customizeContainerEnv(List env) { - Authentication auth = SecurityContextHolder.getContext().getAuthentication(); + public void customizeContainerEnv(Map env) { + var auth = SecurityContextHolder.getContext().getAuthentication(); if (auth == null) return; - OidcUser user = (OidcUser) auth.getPrincipal(); - HttpServletRequest request = ((ServletRequestAttributes) RequestContextHolder.getRequestAttributes()).getRequest(); - OAuth2AuthorizedClient client = oAuth2AuthorizedClientRepository.loadAuthorizedClient(REG_ID, auth, request); + var user = (OidcUser) auth.getPrincipal(); + var request = ((ServletRequestAttributes) RequestContextHolder.getRequestAttributes()).getRequest(); + var client = oAuth2AuthorizedClientRepository.loadAuthorizedClient(REG_ID, auth, request); if (client == null || client.getAccessToken() == null) return; - env.add(ENV_TOKEN_NAME + "=" + client.getAccessToken().getTokenValue()); + env.put(ENV_TOKEN_NAME, client.getAccessToken().getTokenValue()); } protected ClientRegistrationRepository createClientRepo() { - Set scopes = new HashSet<>(); - scopes.add("openid"); - scopes.add("email"); - - for (int i = 0; ; i++) { - String scope = environment.getProperty(String.format("proxy.openid.scopes[%d]", i)); - if (scope == null) break; - else scopes.add(scope); - } - - ClientRegistration client = ClientRegistration.withRegistrationId(REG_ID) + var openID = proxyProperties.getOpenID(); + return new InMemoryClientRegistrationRepository( + ClientRegistration.withRegistrationId(REG_ID) .authorizationGrantType(AuthorizationGrantType.AUTHORIZATION_CODE) .clientName(REG_ID) - .redirectUriTemplate("{baseUrl}/login/oauth2/code/{registrationId}") - .scope(scopes.toArray(new String[scopes.size()])) - .userNameAttributeName(environment.getProperty("proxy.openid.username-attribute", "email")) - .authorizationUri(environment.getProperty("proxy.openid.auth-url")) - .tokenUri(environment.getProperty("proxy.openid.token-url")) - .jwkSetUri(environment.getProperty("proxy.openid.jwks-url")) - .clientId(environment.getProperty("proxy.openid.client-id")) - .clientSecret(environment.getProperty("proxy.openid.client-secret")) - .build(); - - return new InMemoryClientRegistrationRepository(Collections.singletonList(client)); + .redirectUri("{baseUrl}/login/oauth2/code/{registrationId}") + .scope(Stream.concat(Stream.of("openid", "email"), openID.getScopes().stream()).collect(Collectors.toSet())) + .userNameAttributeName(openID.getUsernameAttribute()) + .authorizationUri(openID.getAuthUrl()) + .tokenUri(openID.getTokenUrl()) + .jwkSetUri(openID.getJwksUrl()) + .clientId(openID.getClientId()) + .clientSecret(openID.getClientSecret()) + .build() + ); } protected GrantedAuthoritiesMapper createAuthoritiesMapper() { - String rolesClaimName = environment.getProperty("proxy.openid.roles-claim"); - if (rolesClaimName == null || rolesClaimName.isEmpty()) { - return authorities -> authorities; - } else { - return authorities -> { - Set mappedAuthorities = new HashSet<>(); - for (GrantedAuthority auth : authorities) { - if (auth instanceof OidcUserAuthority) { - OidcIdToken idToken = ((OidcUserAuthority) auth).getIdToken(); - - if (log.isDebugEnabled()) { - String lineSep = System.getProperty("line.separator"); - String claims = idToken.getClaims().entrySet().stream() - .map(e -> String.format("%s -> %s", e.getKey(), e.getValue())) - .collect(Collectors.joining(lineSep)); - log.debug(String.format("Checking for roles in claim '%s'. Available claims in ID token (%d):%s%s", - rolesClaimName, idToken.getClaims().size(), lineSep, claims)); - } + var rolesClaimName = proxyProperties.getOpenID().getRolesClaim(); + if (rolesClaimName != null && !rolesClaimName.isEmpty()) { + return authorities -> authorities.stream() + .filter(OidcUserAuthority.class::isInstance) + .map(OidcUserAuthority.class::cast) + .map(OidcUserAuthority::getIdToken) + .flatMap(idToken -> { + var claims = idToken.getClaims(); + if (log.isDebugEnabled()) { + var lineSep = System.getProperty("line.separator"); + var claims_ = claims.entrySet().stream() + .map(e -> String.format("%s -> %s", e.getKey(), e.getValue())) + .collect(Collectors.joining(lineSep)); + log.debug(String.format("Checking for roles in claim '%s'. Available claims in ID token (%d):%s%s", + rolesClaimName, claims.size(), lineSep, claims_)); + } - Object claimValue = idToken.getClaims().get(rolesClaimName); - if (claimValue == null) { - log.debug("No matching claim found."); - } else { - log.debug(String.format("Matching claim found: %s -> %s (%s)", rolesClaimName, claimValue, claimValue.getClass())); - } + var claimValue = claims.get(rolesClaimName); + if (claimValue == null) { + log.debug("No matching claim found."); + } else { + log.debug(String.format("Matching claim found: %s -> %s (%s)", rolesClaimName, claimValue, claimValue.getClass())); + } - // Workaround: in some cases, getClaimAsStringList fails to parse?? - List roles = idToken.getClaimAsStringList(rolesClaimName); - if (roles == null && claimValue instanceof String) { - List parsedRoles = new ArrayList<>(); + // Workaround: in some cases, getClaimAsStringList fails to parse?? + return Optional.ofNullable(idToken.getClaimAsStringList(rolesClaimName)) + .map(Collection::stream) + .orElseGet(() -> { try { - Object value = new JSONParser(JSONParser.MODE_PERMISSIVE).parse((String) claimValue); + var value = new JSONParser(JSONParser.MODE_PERMISSIVE).parse((String) claimValue); if (value instanceof List) { - List valueList = (List) value; - valueList.forEach(o -> parsedRoles.add(o.toString())); + return ((List) value).stream().map(Object::toString); } } catch (ParseException e) { // Unable to parse JSON } - roles = parsedRoles; - } - if (roles == null) { if (log.isDebugEnabled()) log.debug("Failed to parse claim value as an array: " + claimValue); - continue; - } - - for (String role : roles) { - String mappedRole = role.toUpperCase().startsWith("ROLE_") ? role : "ROLE_" + role; - mappedAuthorities.add(new SimpleGrantedAuthority(mappedRole.toUpperCase())); - } - if (log.isDebugEnabled()) log.debug("The following roles were successfully parsed: " + roles); - } - } - return mappedAuthorities; - }; + return Stream.empty(); + }) + .map(role -> role.toUpperCase().startsWith("ROLE_") ? role : "ROLE_" + role) + .map(String::toUpperCase) + .map(SimpleGrantedAuthority::new); + }) + .collect(Collectors.toSet()); } + return authorities -> authorities; } protected OidcUserService createOidcUserService() { @@ -233,7 +206,7 @@ public OidcUser loadUser(OidcUserRequest userRequest) throws OAuth2Authenticatio } catch (IllegalArgumentException ex) { throw new OAuth2AuthenticationException(new OAuth2Error(OAuth2ErrorCodes.INVALID_REQUEST), "Error while loading user info", ex); } - String nameAttributeKey = environment.getProperty("proxy.openid.username-attribute", "email"); + var nameAttributeKey = proxyProperties.getOpenID().getUsernameAttribute(); return new CustomNameOidcUser(new HashSet<>(user.getAuthorities()), user.getIdToken(), user.getUserInfo(), nameAttributeKey); } }; @@ -254,7 +227,7 @@ public CustomNameOidcUser(Set authorities, OidcIdToken idToken @Override public String getName() { if (isEmailsAttribute) { - Object emails = getAttributes().get(ID_ATTR_EMAILS); + var emails = getAttributes().get(ID_ATTR_EMAILS); if (emails instanceof String[]) return ((String[]) emails)[0]; else if (emails instanceof JSONArray) return ((JSONArray) emails).get(0).toString(); else return emails.toString(); diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/auth/impl/SimpleAuthenticationBackend.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/auth/impl/SimpleAuthenticationBackend.java index 81c31a8d..c5067800 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/auth/impl/SimpleAuthenticationBackend.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/auth/impl/SimpleAuthenticationBackend.java @@ -1,25 +1,26 @@ package hk.edu.polyu.comp.vlabcontroller.auth.impl; import hk.edu.polyu.comp.vlabcontroller.auth.IAuthenticationBackend; -import org.springframework.core.env.Environment; +import hk.edu.polyu.comp.vlabcontroller.config.ProxyProperties; +import lombok.Setter; +import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.security.config.annotation.authentication.builders.AuthenticationManagerBuilder; -import org.springframework.security.config.annotation.authentication.configurers.provisioning.InMemoryUserDetailsManagerConfigurer; import org.springframework.security.config.annotation.web.builders.HttpSecurity; import org.springframework.security.config.annotation.web.configurers.ExpressionUrlAuthorizationConfigurer.AuthorizedUrl; import javax.inject.Inject; -import java.util.Arrays; /** * Simple authentication method where user/password combinations are * provided by the application.yml file. */ +@RefreshScope public class SimpleAuthenticationBackend implements IAuthenticationBackend { public static final String NAME = "simple"; - @Inject - private Environment environment; + @Setter(onMethod_ = {@Inject}) + private ProxyProperties proxyProperties; @Override public String getName() { @@ -38,39 +39,12 @@ public void configureHttpSecurity(HttpSecurity http, AuthorizedUrl anyRequestCon @Override public void configureAuthenticationManagerBuilder(AuthenticationManagerBuilder auth) throws Exception { - InMemoryUserDetailsManagerConfigurer userDetails = auth.inMemoryAuthentication(); - int i = 0; - SimpleUser user = loadUser(i++); - while (user != null) { - userDetails.withUser(user.name).password("{noop}" + user.password).roles(user.roles); - user = loadUser(i++); - } - } - - private SimpleUser loadUser(int index) { - String userName = environment.getProperty(String.format("proxy.users[%d].name", index)); - if (userName == null) return null; - String password = environment.getProperty(String.format("proxy.users[%d].password", index)); - String[] roles = environment.getProperty(String.format("proxy.users[%d].groups", index), String[].class); - if (roles == null) { - roles = new String[0]; - } else { - roles = Arrays.stream(roles).map(s -> s.toUpperCase()).toArray(i -> new String[i]); - } - return new SimpleUser(userName, password, roles); - } - - private static class SimpleUser { - - public String name; - public String password; - public String[] roles; - - public SimpleUser(String name, String password, String[] roles) { - this.name = name; - this.password = password; - this.roles = roles; - } - + var userDetails = auth.inMemoryAuthentication(); + proxyProperties.getUsers().stream() + .filter(x -> x.getName() != null) + .forEach(user -> userDetails + .withUser(user.getName()) + .password("{noop}" + user.getPassword()) + .roles(user.getGroups().stream().map(String::toUpperCase).toArray(String[]::new))); } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/auth/impl/WebServiceAuthenticationBackend.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/auth/impl/WebServiceAuthenticationBackend.java index ede754f1..26871f94 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/auth/impl/WebServiceAuthenticationBackend.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/auth/impl/WebServiceAuthenticationBackend.java @@ -1,38 +1,34 @@ package hk.edu.polyu.comp.vlabcontroller.auth.impl; -import com.google.common.collect.Lists; import hk.edu.polyu.comp.vlabcontroller.auth.IAuthenticationBackend; -import org.springframework.core.env.Environment; +import hk.edu.polyu.comp.vlabcontroller.config.ProxyProperties; +import lombok.Setter; +import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.http.*; import org.springframework.security.authentication.AuthenticationServiceException; import org.springframework.security.authentication.BadCredentialsException; -import org.springframework.security.authentication.rcp.RemoteAuthenticationException; -import org.springframework.security.authentication.rcp.RemoteAuthenticationManager; import org.springframework.security.authentication.rcp.RemoteAuthenticationProvider; import org.springframework.security.config.annotation.authentication.builders.AuthenticationManagerBuilder; import org.springframework.security.config.annotation.web.builders.HttpSecurity; import org.springframework.security.config.annotation.web.configurers.ExpressionUrlAuthorizationConfigurer.AuthorizedUrl; -import org.springframework.security.core.GrantedAuthority; import org.springframework.web.client.HttpClientErrorException; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; import javax.inject.Inject; -import java.util.Arrays; -import java.util.Collection; +import java.util.List; /** * Web service authentication method where user/password combinations are * checked by a HTTP call to a remote web service. */ +@RefreshScope public class WebServiceAuthenticationBackend implements IAuthenticationBackend { public static final String NAME = "webservice"; - private static final String PROPERTY_PREFIX = "proxy.webservice."; - - @Inject - private Environment environment; + @Setter(onMethod_ = {@Inject}) + private ProxyProperties proxyProperties; @Override public String getName() { @@ -51,33 +47,28 @@ public void configureHttpSecurity(HttpSecurity http, AuthorizedUrl anyRequestCon @Override public void configureAuthenticationManagerBuilder(AuthenticationManagerBuilder auth) throws Exception { - RemoteAuthenticationProvider authenticationProvider = new RemoteAuthenticationProvider(); - authenticationProvider.setRemoteAuthenticationManager(new RemoteAuthenticationManager() { - - @Override - public Collection attemptAuthentication(String username, String password) - throws RemoteAuthenticationException { - RestTemplate restTemplate = new RestTemplate(); + var authenticationProvider = new RemoteAuthenticationProvider(); + authenticationProvider.setRemoteAuthenticationManager((username, password) -> { + var restTemplate = new RestTemplate(); - HttpHeaders headers = new HttpHeaders(); - headers.setAccept(Arrays.asList(MediaType.APPLICATION_JSON)); - headers.setContentType(MediaType.APPLICATION_JSON); + var headers = new HttpHeaders(); + headers.setAccept(List.of(MediaType.APPLICATION_JSON)); + headers.setContentType(MediaType.APPLICATION_JSON); - try { - String body = String.format(environment.getProperty(PROPERTY_PREFIX + "authentication-request-body", ""), username, password); - String loginUrl = environment.getProperty(PROPERTY_PREFIX + "authentication-url"); - ResponseEntity result = restTemplate.exchange(loginUrl, HttpMethod.POST, new HttpEntity<>(body, headers), String.class); - if (result.getStatusCode() == HttpStatus.OK) { - return Lists.newArrayList(); - } - throw new AuthenticationServiceException("Unknown response received " + result); - } catch (HttpClientErrorException e) { - throw new BadCredentialsException("Invalid username or password"); - } catch (RestClientException e) { - throw new AuthenticationServiceException("Internal error " + e.getMessage()); + try { + var body = String.format(proxyProperties.getWebService().getAuthenticationRequestBody(), username, password); + var loginUrl = proxyProperties.getWebService().getAuthenticationUrl(); + var result = restTemplate.exchange(loginUrl, HttpMethod.POST, new HttpEntity<>(body, headers), String.class); + if (result.getStatusCode() == HttpStatus.OK) { + return List.of(); } - + throw new AuthenticationServiceException("Unknown response received " + result); + } catch (HttpClientErrorException e) { + throw new BadCredentialsException("Invalid username or password"); + } catch (RestClientException e) { + throw new AuthenticationServiceException("Internal error " + e.getMessage()); } + }); auth.authenticationProvider(authenticationProvider); } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/auth/impl/keycloak/AuthenticationFailureHandler.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/auth/impl/keycloak/AuthenticationFailureHandler.java index b2ebdb5e..23e69154 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/auth/impl/keycloak/AuthenticationFailureHandler.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/auth/impl/keycloak/AuthenticationFailureHandler.java @@ -20,9 +20,9 @@ public void onAuthenticationFailure(HttpServletRequest request, HttpServletRespo // We now set a flag in the session indicating the reason of the Keycloak error. // The error page can then properly handle this. - Object obj = request.getAttribute("org.keycloak.adapters.spi.AuthenticationError"); + var obj = request.getAttribute("org.keycloak.adapters.spi.AuthenticationError"); if (obj instanceof org.keycloak.adapters.OIDCAuthenticationError) { - OIDCAuthenticationError authError = (OIDCAuthenticationError) obj; + var authError = (OIDCAuthenticationError) obj; request.getSession().setAttribute(SP_KEYCLOAK_ERROR_REASON, authError.getReason()); } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/backend/AbstractContainerBackend.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/backend/AbstractContainerBackend.java index af760a29..ccbe6475 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/backend/AbstractContainerBackend.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/backend/AbstractContainerBackend.java @@ -4,12 +4,13 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import com.google.common.base.Charsets; +import com.google.common.collect.Maps; import hk.edu.polyu.comp.vlabcontroller.VLabControllerApplication; import hk.edu.polyu.comp.vlabcontroller.VLabControllerException; import hk.edu.polyu.comp.vlabcontroller.auth.IAuthenticationBackend; import hk.edu.polyu.comp.vlabcontroller.backend.strategy.IProxyTargetMappingStrategy; import hk.edu.polyu.comp.vlabcontroller.backend.strategy.IProxyTestStrategy; +import hk.edu.polyu.comp.vlabcontroller.config.ProxyProperties; import hk.edu.polyu.comp.vlabcontroller.model.runtime.ContainerGroup; import hk.edu.polyu.comp.vlabcontroller.model.runtime.Proxy; import hk.edu.polyu.comp.vlabcontroller.model.runtime.ProxyStatus; @@ -17,8 +18,9 @@ import hk.edu.polyu.comp.vlabcontroller.service.UserService; import hk.edu.polyu.comp.vlabcontroller.spec.expression.ExpressionAwareContainerSpec; import hk.edu.polyu.comp.vlabcontroller.spec.expression.SpecExpressionResolver; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import io.vavr.Function1; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Lazy; import org.springframework.core.env.Environment; @@ -28,26 +30,20 @@ import java.io.IOException; import java.io.OutputStream; import java.math.BigInteger; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.time.Duration; import java.util.*; import java.util.function.BiConsumer; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.stream.Collectors; -public abstract class AbstractContainerBackend implements IContainerBackend { - - protected static final String PROPERTY_INTERNAL_NETWORKING = "internal-networking"; - protected static final String PROPERTY_URL = "url"; - protected static final String PROPERTY_CERT_PATH = "cert-path"; - protected static final String PROPERTY_CONTAINER_PROTOCOL = "container-protocol"; - protected static final String PROPERTY_PRIVILEGED = "privileged"; - - protected static final String DEFAULT_TARGET_PROTOCOL = "http"; +import static io.vavr.API.unchecked; +@Slf4j +public abstract class AbstractContainerBackend implements IContainerBackend { protected static final String ENV_VAR_USER_NAME = "VLAB_USERNAME"; protected static final String ENV_VAR_USER_GROUPS = "VLAB_USERGROUPS"; @@ -58,31 +54,29 @@ public abstract class AbstractContainerBackend implements IContainerBackend { protected static final String RUNTIME_LABEL_CREATED_TIMESTAMP = "comp.polyu.edu.hk/vl-proxy-created-timestamp"; protected static final String RUNTIME_LABEL_PROXIED_APP = "comp.polyu.edu.hk/vl-proxied-app"; protected static final String RUNTIME_LABEL_INSTANCE = "comp.polyu.edu.hk/vl-instance"; + protected static final String RUNTIME_LABEL_EVALUATOR = "comp.polyu.edu.hk/is-evaluator"; - protected final Logger log = LogManager.getLogger(getClass()); - @Inject + @Setter(onMethod_ = {@Inject}) protected IProxyTargetMappingStrategy mappingStrategy; - @Inject + @Setter(onMethod_ = {@Inject}) protected IProxyTestStrategy testStrategy; - @Inject + @Setter(onMethod_ = {@Inject}) protected UserService userService; - @Inject + @Setter(onMethod_ = {@Inject}) protected Environment environment; - @Inject + @Setter(onMethod_ = {@Inject}) protected SpecExpressionResolver expressionResolver; - @Inject - @Lazy + @Setter(onMethod_ = {@Inject, @Lazy}) // Note: lazy needed to work around early initialization conflict protected IAuthenticationBackend authBackend; + @Setter(onMethod_ = {@Inject}) + protected ProxyProperties proxyProperties; + protected String instanceId = null; - private boolean useInternalNetwork; - private boolean privileged; + @Override public void initialize() throws VLabControllerException { - // If this application runs as a container itself, things like port publishing can be omitted. - useInternalNetwork = Boolean.parseBoolean(getProperty(PROPERTY_INTERNAL_NETWORKING, "false")); - privileged = Boolean.parseBoolean(getProperty(PROPERTY_PRIVILEGED, "false")); try { instanceId = calculateInstanceId(); log.info("Hash of config is: " + instanceId); @@ -95,7 +89,7 @@ public void initialize() throws VLabControllerException { public void startProxy(Proxy proxy) throws VLabControllerException { proxy.setId(UUID.randomUUID().toString()); proxy.setStatus(ProxyStatus.Starting); - proxy.setCreatedTimestamp(System.currentTimeMillis()); + proxy.setCreatedTimestamp(Duration.ofMillis(System.currentTimeMillis())); try { try { @@ -108,41 +102,47 @@ public void startProxy(Proxy proxy) throws VLabControllerException { throw new VLabControllerException("Container did not respond in time"); } - proxy.setStartupTimestamp(System.currentTimeMillis()); + proxy.setStartupTimestamp(Duration.ofMillis(System.currentTimeMillis())); proxy.setStatus(ProxyStatus.Up); } catch (VLabControllerException e) { try { stopProxy(proxy); } catch (Exception ex) { - log.error(ex); + log.error("an error occured: {}", ex); } throw e; } } protected void doStartProxy(Proxy proxy) throws Exception { - var eSpecs = proxy.getSpec().getContainerSpecs().stream() - .map(spec -> { + Function1 applySpecToProxy = spec -> { if (authBackend != null) authBackend.customizeContainer(spec); // add labels need for App Recovery and maintenance - spec.addRuntimeLabel(RUNTIME_LABEL_PROXIED_APP, true, "true"); spec.addRuntimeLabel(RUNTIME_LABEL_INSTANCE, true, instanceId); - spec.addRuntimeLabel(RUNTIME_LABEL_PROXY_ID, true, proxy.getId()); spec.addRuntimeLabel(RUNTIME_LABEL_PROXY_SPEC_ID, true, proxy.getSpec().getId()); spec.addRuntimeLabel(RUNTIME_LABEL_USER_ID, true, proxy.getUserId()); spec.addRuntimeLabel(RUNTIME_LABEL_CREATED_TIMESTAMP, true, String.valueOf(proxy.getCreatedTimestamp())); - String[] groups = userService.getGroups(userService.getCurrentAuth()); + var groups = userService.getGroups(userService.getCurrentAuth()); spec.addRuntimeLabel(RUNTIME_LABEL_USER_GROUPS, false, String.join(",", groups)); - return new ExpressionAwareContainerSpec(spec, proxy, expressionResolver); - }) - .map(ContainerSpec.class::cast) - .collect(Collectors.toList()); + return (ContainerSpec) new ExpressionAwareContainerSpec(spec, proxy, expressionResolver); + }; - ContainerGroup c = startContainer(eSpecs, proxy); + var eSpecs = proxy.getSpec().getContainerSpecs().stream() + .map(applySpecToProxy) + .peek(spec -> spec.addRuntimeLabel(RUNTIME_LABEL_PROXIED_APP, true, "true")) + .collect(Collectors.toList()); + Optional.ofNullable(proxy.getSpec().getEvaluator()).ifPresent(evaluator -> { + var spec = applySpecToProxy.apply(evaluator); + spec.addRuntimeLabel(RUNTIME_LABEL_EVALUATOR, true, "true"); + spec.getEnv().put("CONTROLLER_HOST", proxyProperties.getServiceName()); + spec.addRuntimeLabel(RUNTIME_LABEL_EVALUATOR, true, "true"); + eSpecs.add(spec); + }); + var c = startContainer(eSpecs, proxy); proxy.setContainerGroup(c); } @@ -167,81 +167,29 @@ public BiConsumer getOutputAttacher(Proxy proxy) { return null; } - protected String getProperty(String key) { - return getProperty(key, null); - } - - protected String getProperty(String key, String defaultValue) { - return environment.getProperty(getPropertyPrefix() + key, defaultValue); - } - - protected abstract String getPropertyPrefix(); - - protected Long memoryToBytes(String memory) { - if (memory == null || memory.isEmpty()) return null; - Matcher matcher = Pattern.compile("(\\d+)([bkmg]?)").matcher(memory.toLowerCase()); - if (!matcher.matches()) throw new IllegalArgumentException("Invalid memory argument: " + memory); - long mem = Long.parseLong(matcher.group(1)); - String unit = matcher.group(2); - switch (unit) { - case "k": - mem *= 1024; - break; - case "m": - mem *= 1024 * 1024; - break; - case "g": - mem *= 1024 * 1024 * 1024; - break; - default: - } - return mem; + protected Map buildEnv(ContainerSpec containerSpec, Proxy proxy) { + return new HashMap<>() {{ + put(ENV_VAR_USER_NAME, proxy.getUserId()); + put(ENV_VAR_USER_GROUPS, String.join(",", userService.getGroups(userService.getCurrentAuth()))); + Optional.ofNullable(containerSpec.getEnvFile()) + .filter(x -> Files.isRegularFile(Paths.get(x))) + .map(unchecked(FileInputStream::new)) + .map(unchecked(x -> new Properties() {{ load(x); }})) + .map(Maps::fromProperties) + .ifPresent(this::putAll); + Optional.ofNullable(containerSpec.getEnv()).ifPresent(this::putAll); + // Allow the authentication backend to add values to the environment, if needed. + Optional.ofNullable(authBackend).ifPresent(x -> x.customizeContainerEnv(this)); + }}; } - protected List buildEnv(ContainerSpec containerSpec, Proxy proxy) throws IOException { - List env = new ArrayList<>(); - env.add(String.format("%s=%s", ENV_VAR_USER_NAME, proxy.getUserId())); - - String[] groups = userService.getGroups(userService.getCurrentAuth()); - env.add(String.format("%s=%s", ENV_VAR_USER_GROUPS, String.join(",", groups))); - - String envFile = containerSpec.getEnvFile(); - if (envFile != null && Files.isRegularFile(Paths.get(envFile))) { - Properties envProps = new Properties(); - envProps.load(new FileInputStream(envFile)); - for (Map.Entry key : envProps.entrySet()) { - env.add(String.format("%s=%s", key.getKey(), key.getValue())); - } - } - - if (containerSpec.getEnv() != null) { - for (Map.Entry entry : containerSpec.getEnv().entrySet()) { - env.add(String.format("%s=%s", entry.getKey(), entry.getValue())); - } - } - - // Allow the authentication backend to add values to the environment, if needed. - if (authBackend != null) authBackend.customizeContainerEnv(env); - - return env; - } - - protected boolean isUseInternalNetwork() { - return useInternalNetwork; - } - - protected boolean isPrivileged() { - return privileged; - } - - private File getPathToConfigFile() { - String path = environment.getProperty("spring.config.location"); + var path = environment.getProperty("spring.config.location"); if (path != null) { return Paths.get(path).toFile(); } - File file = Paths.get(VLabControllerApplication.CONFIG_FILENAME).toFile(); + var file = Paths.get(VLabControllerApplication.CONFIG_FILENAME).toFile(); if (file.exists()) { return file; } @@ -260,23 +208,23 @@ private String calculateInstanceId() throws IOException, NoSuchAlgorithmExceptio * dump it again into YAML. We also sort the keys of maps and properties so that * the order does not matter for the resulting hash. */ - ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory()); + var objectMapper = new ObjectMapper(new YAMLFactory()); objectMapper.configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true); objectMapper.configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true); - File file = getPathToConfigFile(); + var file = getPathToConfigFile(); if (file == null) { // this should only happen in tests instanceId = "unknown-instance-id"; return instanceId; } - Object parsedConfig = objectMapper.readValue(file, Object.class); - String canonicalConfigFile = objectMapper.writeValueAsString(parsedConfig); + var parsedConfig = objectMapper.readValue(file, Object.class); + var canonicalConfigFile = objectMapper.writeValueAsString(parsedConfig); - MessageDigest digest = MessageDigest.getInstance("SHA-1"); + var digest = MessageDigest.getInstance("SHA-1"); digest.reset(); - digest.update(canonicalConfigFile.getBytes(Charsets.UTF_8)); + digest.update(canonicalConfigFile.getBytes(StandardCharsets.UTF_8)); instanceId = String.format("%040x", new BigInteger(1, digest.digest())); return instanceId; } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/backend/ContainerBackendFactory.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/backend/ContainerBackendFactory.java index 6ba2ac3c..a98be535 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/backend/ContainerBackendFactory.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/backend/ContainerBackendFactory.java @@ -1,28 +1,29 @@ package hk.edu.polyu.comp.vlabcontroller.backend; import hk.edu.polyu.comp.vlabcontroller.backend.kubernetes.KubernetesBackend; -import org.springframework.beans.BeansException; +import hk.edu.polyu.comp.vlabcontroller.config.ProxyProperties; +import lombok.RequiredArgsConstructor; +import lombok.Setter; import org.springframework.beans.factory.config.AbstractFactoryBean; +import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; -import org.springframework.core.env.Environment; import org.springframework.stereotype.Service; +import javax.inject.Inject; +import java.util.Arrays; +import java.util.Objects; + +import static io.vavr.API.unchecked; + @Service +@RequiredArgsConstructor +@RefreshScope public class ContainerBackendFactory extends AbstractFactoryBean implements ApplicationContextAware { - - private static final String PROPERTY_CONTAINER_BACKEND = "proxy.container-backend"; - protected final Environment environment; + @Setter(onMethod_ = {@Inject}) private ApplicationContext applicationContext; - - public ContainerBackendFactory(Environment environment) { - this.environment = environment; - } - - @Override - public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { - this.applicationContext = applicationContext; - } + @Setter(onMethod_ = {@Inject}) + private ProxyProperties proxyProperties; @Override public Class getObjectType() { @@ -31,8 +32,8 @@ public Class getObjectType() { @Override protected IContainerBackend createInstance() throws Exception { - String backendName = environment.getProperty(PROPERTY_CONTAINER_BACKEND); - IContainerBackend backend = ContainerBackend.createFor(backendName); + var backendName = proxyProperties.getContainerBackend(); + var backend = ContainerBackend.createFor(backendName); applicationContext.getAutowireCapableBeanFactory().autowireBean(backend); backend.initialize(); return backend; @@ -49,11 +50,11 @@ private enum ContainerBackend { this.type = type; } - public static IContainerBackend createFor(String name) throws Exception { - for (ContainerBackend cb : values()) { - if (cb.name.equalsIgnoreCase(name)) return cb.type.newInstance(); - } - return null; + public static IContainerBackend createFor(String name) { + return Arrays.stream(values()) + .filter(cb -> cb.name.equalsIgnoreCase(name)).map(unchecked(cb -> cb.type.newInstance())) + .filter(Objects::nonNull) + .findFirst().orElse(null); } } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/backend/kubernetes/KubernetesBackend.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/backend/kubernetes/KubernetesBackend.java index 9a047f8c..e7e58135 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/backend/kubernetes/KubernetesBackend.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/backend/kubernetes/KubernetesBackend.java @@ -13,6 +13,7 @@ import hk.edu.polyu.comp.vlabcontroller.model.spec.ContainerSpec; import hk.edu.polyu.comp.vlabcontroller.service.ProxyService; import hk.edu.polyu.comp.vlabcontroller.spec.expression.SpecExpressionContext; +import hk.edu.polyu.comp.vlabcontroller.util.RFC6335Validator; import hk.edu.polyu.comp.vlabcontroller.util.Retrying; import io.fabric8.kubernetes.api.model.*; import io.fabric8.kubernetes.client.ConfigBuilder; @@ -20,41 +21,42 @@ import io.fabric8.kubernetes.client.KubernetesClient; import io.fabric8.kubernetes.client.internal.readiness.Readiness; import io.fabric8.kubernetes.client.utils.Serialization; -import javax.json.JsonPatch; -import org.apache.commons.io.IOUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import io.vavr.Function0; +import io.vavr.Function1; +import io.vavr.control.Try; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.cloud.context.config.annotation.RefreshScope; +import org.springframework.cloud.context.scope.refresh.RefreshScopeRefreshedEvent; +import org.springframework.context.event.EventListener; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; import javax.inject.Inject; +import javax.json.JsonPatch; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.OutputStream; import java.net.URI; import java.nio.file.Files; import java.nio.file.Paths; +import java.time.Duration; import java.util.*; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ScheduledFuture; import java.util.function.BiConsumer; import java.util.function.Function; +import java.util.function.Predicate; +import java.util.regex.Pattern; import java.util.stream.Collectors; +import java.util.stream.Stream; -import static com.pivovarit.function.ThrowingFunction.unchecked; +import static io.vavr.API.unchecked; +import static java.lang.Boolean.parseBoolean; +import static org.apache.commons.lang3.StringUtils.startsWithIgnoreCase; +@Slf4j +@RefreshScope public class KubernetesBackend extends AbstractContainerBackend { - - private static final String PROPERTY_PREFIX = "proxy.kubernetes."; - - private static final String PROPERTY_NAMESPACE = "namespace"; - private static final String PROPERTY_API_VERSION = "api-version"; - private static final String PROPERTY_IMG_PULL_POLICY = "image-pull-policy"; - private static final String PROPERTY_IMG_PULL_SECRETS = "image-pull-secrets"; - private static final String PROPERTY_IMG_PULL_SECRET = "image-pull-secret"; - private static final String PROPERTY_NODE_SELECTOR = "node-selector"; - private static final String PROPERTY_UID_NAMESPACE = "custom-namespace"; - private static final String PROPERTY_NAMESPACE_PREFIX = "namespace-prefix"; - - private static final String DEFAULT_NAMESPACE = "default"; - private static final String DEFAULT_API_VERSION = "v1"; - private static final String PARAM_POD = "pod"; private static final String PARAM_CONTAINER = "container"; private static final String PARAM_SERVICE = "service"; @@ -63,82 +65,111 @@ public class KubernetesBackend extends AbstractContainerBackend { private static final String SECRET_KEY_REF = "secretKeyRef"; - private final Logger log = LogManager.getLogger(KubernetesBackend.class); + @Setter(onMethod_ = {@Inject}) + private Retrying retrying; - @Inject + @Setter(onMethod_ = {@Inject}) private PodPatcher podPatcher; - @Inject + @Setter(onMethod_ = {@Inject}) private ProxyService proxyService; - @Inject + @Setter(onMethod_ = {@Inject}) private ObjectMapper objectMapper; + @Setter(onMethod_ = {@Inject}) + private ThreadPoolTaskScheduler taskScheduler; + private KubernetesClient kubeClient; + private Optional> cleanupPodsFuture = Optional.empty(); + + Function0 cleanupPodsTask = () -> () -> Optional.ofNullable(getFailedAndUnknownPods()) + .map(PodList::getItems) + .filter(Predicate.not(List::isEmpty)) + .stream() + .flatMap(Collection::stream) + .map(Pod::getMetadata) + .map(ObjectMeta::getLabels) + .map(x -> x.get("comp.polyu.edu.hk/vl-proxy-id")) + .forEach(proxyId -> { + proxyService.stopProxy(proxyService.getProxy(proxyId), true, true, Duration.ZERO); + log.error("Cleaned error proxy {}", proxyId); + }); + + @EventListener + public void onRefreshScopeRefreshed(final RefreshScopeRefreshedEvent event) { + log.debug("backend refreshed"); + } + @Override public void initialize() throws VLabControllerException { super.initialize(); var configBuilder = new ConfigBuilder(); - var masterUrl = getProperty(PROPERTY_URL); - if (masterUrl != null) configBuilder.withMasterUrl(masterUrl); + Optional.ofNullable(proxyProperties.getKubernetes().getUrl()).map(configBuilder::withMasterUrl); attachTLSCerts(configBuilder); kubeClient = new DefaultKubernetesClient(configBuilder.build()); cleanBeforeStart(); - var cleanFailedThread = new Thread(new ErrorPodsCleaner(), ErrorPodsCleaner.class.getSimpleName()); - cleanFailedThread.setDaemon(true); - cleanFailedThread.start(); + log.info("Enable failed and unknown phase pods detection & cleaning"); + startCleanupPods(); } private void attachTLSCerts(ConfigBuilder configBuilder) { - var certPath = getProperty(PROPERTY_CERT_PATH); + var certPath = proxyProperties.getKubernetes().getCertPath(); + Function> loadFile = file -> Optional.of(Paths.get(certPath, file)).filter(Files::exists).map(Object::toString); if (certPath != null && Files.isDirectory(Paths.get(certPath))) { - var certFilePath = Paths.get(certPath, "ca.pem"); - if (Files.exists(certFilePath)) configBuilder.withCaCertFile(certFilePath.toString()); - certFilePath = Paths.get(certPath, "cert.pem"); - if (Files.exists(certFilePath)) configBuilder.withClientCertFile(certFilePath.toString()); - certFilePath = Paths.get(certPath, "key.pem"); - if (Files.exists(certFilePath)) configBuilder.withClientKeyFile(certFilePath.toString()); + loadFile.apply("ca.pem").ifPresent(configBuilder::withCaCertFile); + loadFile.apply("cert.pem").ifPresent(configBuilder::withClientCertFile); + loadFile.apply("key.pem").ifPresent(configBuilder::withClientKeyFile); } } public void initialize(KubernetesClient client) { super.initialize(); kubeClient = client; - var cleanFailedThread = new Thread(new ErrorPodsCleaner(), ErrorPodsCleaner.class.getSimpleName()); - cleanFailedThread.setDaemon(true); - cleanFailedThread.start(); + startCleanupPods(); + } + + void startCleanupPods() { + cleanupPodsFuture.ifPresent(x -> x.cancel(true)); + cleanupPodsFuture = Optional.of(taskScheduler.scheduleAtFixedRate(cleanupPodsTask.apply(), Duration.ofSeconds(30))); + } + + Stream evalExpressionForConfig(List data, Class type, SpecExpressionContext context) { + Function1 evalExpressionOnConfig = x -> unchecked(objectMapper::writeValueAsString) + .andThen(y -> expressionResolver.evaluateToString(y, context)) + .andThen(unchecked(y -> objectMapper.readValue(y, type))) + .apply(x); + + return data.stream() + .map(x -> Try.of(() -> evalExpressionOnConfig.apply(x))) + .peek(x -> x.onFailure(e -> log.error("an error occured: {}", e))) + .filter(Predicate.not(Try::isFailure)) + .map(Try::get); } @Override protected ContainerGroup startContainer(List specs, Proxy proxy) throws Exception { - var containerGroup = new ContainerGroup(); - containerGroup.setSpecs(specs); - containerGroup.setId(UUID.randomUUID().toString()); - - var identifierLabel = environment.getProperty("proxy.identifier-label", "comp.polyu.edu.hk/vl-identifier"); - var identifierValue = environment.getProperty("proxy.identifier-value", "default-identifier"); + var containerGroup = new ContainerGroup(UUID.randomUUID().toString(), new ArrayList<>(), new HashMap<>()); - var kubeNamespace = getProperty(PROPERTY_NAMESPACE, DEFAULT_NAMESPACE); - var namespacePrefix = getProperty(PROPERTY_NAMESPACE_PREFIX); - var uidNamespace = Boolean.parseBoolean(getProperty(PROPERTY_UID_NAMESPACE, "false")); + var kubernetes = proxyProperties.getKubernetes(); + var kubeNamespace = kubernetes.getNamespace(); + var namespacePrefix = kubernetes.getNamespacePrefix(); + var uidNamespace = kubernetes.isCustomNamespace(); log.debug("UserID Namespace Mode: {}", uidNamespace); if (uidNamespace) { kubeNamespace = Strings.isNullOrEmpty(namespacePrefix) ? proxy.getUserId() : String.format("%s-%s", namespacePrefix, proxy.getUserId()); } proxy.setNamespace(kubeNamespace); - var apiVersion = getProperty(PROPERTY_API_VERSION, DEFAULT_API_VERSION); - - var imagePullSecrets = Optional.ofNullable(getProperty(PROPERTY_IMG_PULL_SECRET)) - .map(List::of) - .or(() -> Optional.ofNullable(getProperty(PROPERTY_IMG_PULL_SECRETS)).map(x -> x.split(",")).map(List::of)) - .orElse(List.of()) - .stream().map(LocalObjectReference::new).collect(Collectors.toList()); + var imagePullSecrets = Optional.ofNullable(kubernetes.getImagePullSecret()) + .map(List::of) + .orElseGet(kubernetes::getImagePullSecrets) + .stream().map(LocalObjectReference::new).collect(Collectors.toList()); log.debug("imagePullSecrets: {}", imagePullSecrets); @@ -148,105 +179,99 @@ protected ContainerGroup startContainer(List specs, Proxy proxy) // Handle runtime labels var runtimeLabels = specs.stream() - .flatMap(x -> x.getRuntimeLabels().entrySet().stream()) - .filter(p -> p.getValue().getFirst()) - .collect(Collectors.toMap(Map.Entry::getKey, m -> m.getValue().getSecond(), (v1, v2) -> v2)); + .flatMap(x -> x.getRuntimeLabels().entrySet().stream()) + .filter(p -> p.getValue().getFirst()) + .collect(Collectors.toMap(Map.Entry::getKey, m -> m.getValue().getSecond(), (v1, v2) -> v2)); var runtimeAnnotations = specs.stream() - .flatMap(x -> x.getRuntimeLabels().entrySet().stream()) - .filter(p -> !p.getValue().getFirst()) - .collect(Collectors.toMap(Map.Entry::getKey, m -> m.getValue().getSecond(), (v1, v2) -> v2)); + .flatMap(x -> x.getRuntimeLabels().entrySet().stream()) + .filter(p -> !p.getValue().getFirst()) + .collect(Collectors.toMap(Map.Entry::getKey, m -> m.getValue().getSecond(), (v1, v2) -> v2)); + var identifierLabel = proxyProperties.getIdentifierLabel(); + var identifierValue = proxyProperties.getIdentifierValue(); var objectMetaBuilder = new ObjectMetaBuilder() - .withNamespace(kubeNamespace) - .withName("vl-pod-" + containerGroup.getId()) - .addToLabels(specLabels) - .addToLabels(identifierLabel, identifierValue) - .addToLabels(runtimeLabels) - .addToAnnotations(runtimeAnnotations); + .withNamespace(kubeNamespace) + .withName("vl-pod-" + containerGroup.getId()) + .addToLabels(specLabels) + .addToLabels(identifierLabel, identifierValue) + .addToLabels(runtimeLabels) + .addToAnnotations(runtimeAnnotations); var podBuilder = new PodBuilder() - .withApiVersion(apiVersion) + .withApiVersion(kubernetes.getApiVersion()) .withKind("Pod") .withMetadata(objectMetaBuilder.build()); var containers = specs.stream() - .map(unchecked(spec -> { - var volumeMounts = spec.getVolumeMounts(); - if (proxy.isAdmin()) { - var adminVolumeMounts = spec.getAdminVolumeMounts(); - if (!adminVolumeMounts.isEmpty()) { - volumeMounts.addAll(adminVolumeMounts); - log.debug("Admin VolumeMount loaded: {}", adminVolumeMounts); - } + .map(unchecked(spec -> { + var volumeMounts = spec.getVolumeMounts(); + if (proxy.isAdmin()) { + var adminVolumeMounts = spec.getAdminVolumeMounts(); + if (!adminVolumeMounts.isEmpty()) { + volumeMounts.addAll(adminVolumeMounts); + log.debug("Admin VolumeMount loaded: {}", adminVolumeMounts); } - var envVars = buildEnv(spec, proxy).stream() - .map(envString -> { - var e = envString.split("="); - if (e.length == 1) e = new String[]{e[0], ""}; - if (e.length > 2) e[1] = envString.substring(envString.indexOf('=') + 1); - if (!e[1].toLowerCase().startsWith(SECRET_KEY_REF.toLowerCase())) { - return Optional.of(new EnvVar(e[0], e[1], null)); - } - var ref = e[1].split(":"); - if (ref.length != 3) { - log.warn(String.format("Invalid secret key reference: %s. Expected format: '%s::'", envString, SECRET_KEY_REF)); - return Optional.empty(); - } - var secretKeyRef = new SecretKeySelectorBuilder() - .withName(ref[1]) - .withKey(ref[2]) - .build(); - var envVarSourceBuilder = new EnvVarSourceBuilder() - .withSecretKeyRef(secretKeyRef); - return Optional.of(new EnvVar(e[0], null, envVarSourceBuilder.build())); + } + var envVars = buildEnv(spec, proxy).entrySet().stream() + .map(e -> { + var value = e.getValue(); + if (startsWithIgnoreCase(value, SECRET_KEY_REF)) { + var ref = Pattern.compile(String.format("%s:(?\\S+?):(?\\S+)", SECRET_KEY_REF)).matcher(value); + if (!ref.matches()) { + log.warn(String.format("Invalid secret key reference: %s. Expected format: '%s::'", e, SECRET_KEY_REF)); + return Optional.empty(); + } + return Optional.of(new EnvVar(e.getKey(), null, new EnvVarSourceBuilder() + .withSecretKeyRef(new SecretKeySelectorBuilder() + .withName(ref.group("name")) + .withKey(ref.group("key")) + .build()).build())); + } + + return Optional.of(new EnvVar(e.getKey(), value, null)); + }) + .flatMap(Optional::stream) + .collect(Collectors.toList()); + + var security = new SecurityContextBuilder() + .withPrivileged(kubernetes.isPrivileged() || spec.isPrivileged()) + .build(); + + var resources = spec.getResources(); + var containerBuilder = new ContainerBuilder() + .withImage(spec.getImage()) + .withCommand(spec.getCmd()) + .withName( + Optional.ofNullable(spec.getName()).filter(Predicate.not(String::isBlank)) + .orElse(String.format("vl-container-%s", UUID.randomUUID())) + ) + .withPorts( + spec.getPortMapping().entrySet().stream() + .map(e -> { + var builder = new ContainerPortBuilder(); + if (RFC6335Validator.valid(e.getKey())) builder = builder.withName(e.getKey()); + builder = builder.withContainerPort(e.getValue()); + return builder.build(); }) - .flatMap(Optional::stream) - .collect(Collectors.toList()); - - var security = new SecurityContextBuilder() - .withPrivileged(isPrivileged() || spec.isPrivileged()) - .build(); - - var toQuantity = (Function) (String x) -> Optional.ofNullable(x).map(Quantity::new).orElse(null); - var containerBuilder = new ContainerBuilder() - .withImage(spec.getImage()) - .withCommand(spec.getCmd()) - .withName(String.format("vl-container-%s", UUID.randomUUID())) - .withPorts( - spec.getPortMapping().values().stream() - .map(p -> new ContainerPortBuilder().withContainerPort(p).build()) - .collect(Collectors.toList()) - ) - .withVolumeMounts(volumeMounts) - .withSecurityContext(security) - .withResources( - new ResourceRequirementsBuilder() - .addToRequests(spec.getResources().getRequests().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, v -> toQuantity.apply(v.getValue())))) - .addToLimits(spec.getResources().getLimits().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, v -> toQuantity.apply(v.getValue())))) - .build() - ) - .withEnv(envVars); - - var imagePullPolicy = getProperty(PROPERTY_IMG_PULL_POLICY); - if (imagePullPolicy != null) containerBuilder.withImagePullPolicy(imagePullPolicy); - - return containerBuilder.build(); - })) - .collect(Collectors.toList()); - containerGroup.getParameters().put(PARAM_CONTAINER, containers); - SpecExpressionContext context = SpecExpressionContext.create(proxy, proxy.getSpec()); - List volumes = proxy.getSpec().getKubernetes().getVolumes().stream().map(volume -> { - try { - String volumeString = objectMapper.writeValueAsString(volume); - volumeString = expressionResolver.evaluateToString(volumeString, context); - return objectMapper.readValue(volumeString, Volume.class); - } catch (Exception e) { - log.error(e); - return null; - } - }).collect(Collectors.toList()); + .collect(Collectors.toList()) + ) + .withVolumeMounts(volumeMounts) + .withSecurityContext(security) + .withResources(resources.asResourceRequirements()) + .withEnv(envVars); + + var imagePullPolicy = kubernetes.getImagePullPolicy(); + if (imagePullPolicy != null) containerBuilder.withImagePullPolicy(imagePullPolicy); + + return containerBuilder.build(); + })) + .collect(Collectors.toList()); + containerGroup.getParameters().put(PARAM_CONTAINER, containers); + var context = SpecExpressionContext.create(proxy, proxy.getSpec()); + var volumes = evalExpressionForConfig(proxy.getSpec().getKubernetes().getVolumes(), Volume.class, context) + .collect(Collectors.toList()); log.debug("containers created: {}", containers.size()); log.debug("volumes created: {}", volumes.size()); @@ -255,51 +280,42 @@ protected ContainerGroup startContainer(List specs, Proxy proxy) podSpec.setVolumes(volumes); podSpec.setImagePullSecrets(imagePullSecrets); - var nodeSelectorString = getProperty(PROPERTY_NODE_SELECTOR); + if (proxy.getSpec().isSecure()) { + podSpec.setRuntimeClassName(proxyProperties.getKubernetes().getSecureRuntimeName()); + } + + var nodeSelectorString = kubernetes.getNodeSelector(); if (nodeSelectorString != null) { podSpec.setNodeSelector(Splitter.on(",").withKeyValueSeparator("=").split(nodeSelectorString)); } log.debug("nodeSelectorString: {}", nodeSelectorString); - var startupPod = podBuilder - .withSpec(podSpec) - .build(); - - JsonPatch patch = readPatchFromSpec(proxy); - Pod patchedPod = podPatcher.patchWithDebug(startupPod, patch); - final String effectiveKubeNamespace = patchedPod.getMetadata().getNamespace(); // use the namespace of the patched Pod, in case the patch changes the namespace. + var patchedPod = podPatcher.patchWithDebug(podBuilder.withSpec(podSpec).build(), readPatchFromSpec(proxy)); + final var effectiveKubeNamespace = patchedPod.getMetadata().getNamespace(); // use the namespace of the patched Pod, in case the patch changes the namespace. containerGroup.getParameters().put(PARAM_NAMESPACE, effectiveKubeNamespace); - var pvcs = proxy.getSpec().getKubernetes().getPersistentVolumeClaims().stream().map(pvc -> { - try { - String pvcString = objectMapper.writeValueAsString(pvc); - pvcString = expressionResolver.evaluateToString(pvcString, context); - var expressionPVC = objectMapper.readValue(pvcString, PersistentVolumeClaim.class); - var labelCache = expressionPVC.getMetadata().getLabels(); - if (labelCache == null) { - labelCache = new HashMap<>(); - } + var pvcs = evalExpressionForConfig(proxy.getSpec().getKubernetes().getPersistentVolumeClaims(), PersistentVolumeClaim.class, context) + .peek(expressionPVC -> { + var labelCache = Optional.ofNullable(expressionPVC.getMetadata().getLabels()).orElseGet(HashMap::new); labelCache.putAll(specLabels); labelCache.putAll(runtimeLabels); labelCache.put(identifierLabel, identifierValue); expressionPVC.getMetadata().setLabels(labelCache); - return kubeClient.persistentVolumeClaims().inNamespace(effectiveKubeNamespace).createOrReplace(expressionPVC); - } catch (Exception e) { - log.error(e); - return null; - } - }).collect(Collectors.toList()); - containerGroup.getParameters().put(PARAM_PVC, pvcs.stream().filter(Objects::nonNull).collect(Collectors.toList())); - log.debug("created {} PVCs", pvcs.stream().filter(Objects::nonNull).count()); + }) + .map(expressionPVC -> kubeClient.persistentVolumeClaims().inNamespace(effectiveKubeNamespace).createOrReplace(expressionPVC)) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + log.debug("created {} PVCs", pvcs.size()); + containerGroup.getParameters().put(PARAM_PVC, pvcs); // create additional manifests -> use the effective (i.e. patched) namespace if no namespace is provided createAdditionalManifests(proxy, effectiveKubeNamespace, specLabels, runtimeLabels); var startedPod = kubeClient - .pods() - .inNamespace(effectiveKubeNamespace) - .create(patchedPod); + .pods() + .inNamespace(effectiveKubeNamespace) + .create(patchedPod); log.debug("pod started"); @@ -308,121 +324,129 @@ protected ContainerGroup startContainer(List specs, Proxy proxy) log.debug("pod registered"); // If SP runs inside the cluster, it can access pods directly and doesn't need any port publishing service. - var service = makeServiceIfNecessary(specs, proxy, containerGroup, apiVersion, effectiveKubeNamespace, specLabels, runtimeLabels); - containerGroup.getParameters().put(PARAM_SERVICE, service); + if (!proxyProperties.getKubernetes().isInternalNetworking()) { + var ports = specs.stream() + .flatMap(x -> x.getPortMapping().entrySet().stream()) + .map(e -> { + var builder = new ServicePortBuilder(); + if (RFC6335Validator.valid(e.getKey())) builder = builder.withName(e.getKey()).withNewTargetPort(e.getKey()); + else builder = builder.withNewTargetPort(e.getValue()); + return builder.build(); + }) + .collect(Collectors.toList()); + var service = startService(effectiveKubeNamespace, new ServiceBuilder() + .withApiVersion(kubernetes.getApiVersion()) + .withKind("Service") + .withNewMetadata() + .withName("vl-service-" + containerGroup.getId()) + .addToLabels(identifierLabel, identifierValue) + .addToLabels(specLabels) + .addToLabels(runtimeLabels) + .endMetadata() + .withNewSpec() + .addToSelector(RUNTIME_LABEL_PROXY_ID, proxy.getId()) + .withType("NodePort") + .withPorts(ports) + .endSpec() + .build() + ); + containerGroup.getParameters().put(PARAM_SERVICE, service); + log.debug("service registered"); + calculateProxyRoutes(specs, proxy, containerGroup, service); + } - log.debug("service registered"); + specs.stream() + .filter(spec -> parseBoolean(spec.getRuntimeLabels().get(RUNTIME_LABEL_EVALUATOR).getSecond())) + .findAny() + .ifPresent(evaluatorSpec -> startService(effectiveKubeNamespace, new ServiceBuilder() + .withApiVersion(kubernetes.getApiVersion()) + .withKind("Service") + .withNewMetadata() + .addToLabels(identifierLabel, identifierValue) + .addToLabels(specLabels) + .addToLabels(runtimeLabels) + .withName("vl-evaluator-" + containerGroup.getId()) + .endMetadata() + .withNewSpec() + .addToSelector(RUNTIME_LABEL_PROXY_ID, proxy.getId()) + .withType("ClusterIP") + .withPorts(List.of(new ServicePortBuilder().withName("rpc").withPort(80).withNewTargetPort("rpc").build())) + .endSpec() + .build() + )); - calculateProxyRoutes(specs, proxy, containerGroup, service); return containerGroup; } private void createAdditionalManifests(Proxy proxy, String namespace, Map specLabels, Map runtimeLabels) { - for (HasMetadata fullObject : getAdditionManifestsAsObjects(proxy, namespace)) { - if (kubeClient.resource(fullObject).fromServer().get() == null) { - String identifierLabel = environment.getProperty("proxy.identifier-label", "comp.polyu.edu.hk/vl-identifier"); - String identifierValue = environment.getProperty("proxy.identifier-value", "default-identifier"); - ObjectMeta cache = fullObject.getMetadata(); - Map labels = cache.getLabels(); - if (labels == null) { - labels = new HashMap<>(); - } + getAdditionManifestsAsObjects(proxy, namespace).stream() + .filter(fullObject -> kubeClient.resource(fullObject).fromServer().get() == null) + .forEach(fullObject -> { + var identifierLabel = proxyProperties.getIdentifierLabel(); + var identifierValue = proxyProperties.getIdentifierValue(); + var cache = fullObject.getMetadata(); + var labels = Optional.ofNullable(cache.getLabels()).orElseGet(HashMap::new); labels.put(identifierLabel, identifierValue); labels.putAll(specLabels); labels.putAll(runtimeLabels); cache.setLabels(labels); fullObject.setMetadata(cache); kubeClient.resource(fullObject).createOrReplace(); - } - } + }); } private JsonPatch readPatchFromSpec(Proxy proxy) throws JsonProcessingException { - String patchAsString = proxy.getSpec().getKubernetes().getPodPatches(); + var patchAsString = proxy.getSpec().getKubernetes().getPodPatches(); if (patchAsString == null) { return null; } // resolve expressions - SpecExpressionContext context = SpecExpressionContext.create(proxy, proxy.getSpec()); - String expressionAwarePatch = expressionResolver.evaluateToString(patchAsString, context); + var context = SpecExpressionContext.create(proxy, proxy.getSpec()); + var expressionAwarePatch = expressionResolver.evaluateToString(patchAsString, context); - ObjectMapper yamlReader = new ObjectMapper(new YAMLFactory()); + var yamlReader = new ObjectMapper(new YAMLFactory()); yamlReader.registerModule(new JSR353Module()); return yamlReader.readValue(expressionAwarePatch, JsonPatch.class); } - - private Pod waitUntilPodReadyOrDie(Pod startedPod) { - var totalWaitMs = Integer.parseInt(environment.getProperty("proxy.kubernetes.pod-wait-time", "60000")); - var maxTries = totalWaitMs / 1000; - boolean result = Retrying.retry(i -> { - var pod = kubeClient.resource(startedPod).fromServer().get(); - if (!Readiness.isPodReady(pod)) { - if (i > 1) - log.debug(String.format("Container not ready yet, trying again (%d/%d)", i, maxTries)); - return false; - } - return true; - }, maxTries, 1000); - if (!result){ - throw new VLabControllerException("Container did not become ready in time"); - } - return kubeClient.resource(startedPod).fromServer().get(); + private Pod waitUntilPodReadyOrDie(Pod startedPod) throws ExecutionException, InterruptedException { + var maxTries = (int) proxyProperties.getKubernetes().getPodWaitTime().toSeconds(); + var retry = retrying.retry(i -> { + if (Readiness.isPodReady(kubeClient.resource(startedPod).fromServer().get())) return true; + if (i > 1) log.debug(String.format("Container not ready yet, trying again (%d/%d)", i, maxTries)); + return false; + }, maxTries, Duration.ofSeconds(1)); + if (retry.get()) return kubeClient.resource(startedPod).fromServer().get(); + throw new VLabControllerException("Container did not become ready in time"); } // Calculate proxy routes for all configured ports. - private void calculateProxyRoutes(List specs, Proxy proxy, ContainerGroup containerGroup, Service service) throws Exception { - for (var entry : specs.stream() - .flatMap(x -> x.getPortMapping().entrySet().stream()) - .collect(Collectors.toList())) { - var servicePort = service == null ? -1 : service.getSpec().getPorts().stream() - .filter(p -> p.getPort().equals(entry.getValue())).map(ServicePort::getNodePort) - .findAny().orElse(-1); - - var mapping = mappingStrategy.createMapping(entry.getKey(), containerGroup, proxy); - var target = calculateTarget(containerGroup, entry.getValue(), servicePort); - log.debug("adding {} to {}", target, mapping); - proxy.getTargets().put(mapping, target); - } + private void calculateProxyRoutes(List specs, Proxy proxy, ContainerGroup containerGroup, Service service) { + var targetMaps = specs.stream() + .flatMap(x -> x.getPortMapping().entrySet().stream()) + .collect(Collectors.toMap( + entry -> mappingStrategy.createMapping(entry.getKey(), containerGroup, proxy), + unchecked(entry -> calculateTarget(containerGroup, entry.getValue(), + Optional.ofNullable(service).flatMap(x -> + x.getSpec().getPorts().stream() + .filter(p -> p.getPort().equals(entry.getValue())) + .map(ServicePort::getNodePort) + .findAny() + ).orElse(-1) + )))); + log.debug("adding target maps: {}", targetMaps); + proxy.getTargets().putAll(targetMaps); } - private Service makeServiceIfNecessary(List specs, Proxy proxy, ContainerGroup containerGroup, String apiVersion, String effectiveKubeNamespace, Map specLabels, Map runtimeLabels) { - Service service = null; - if (!isUseInternalNetwork()) { - String identifierLabel = environment.getProperty("proxy.identifier-label", "comp.polyu.edu.hk/vl-identifier"); - String identifierValue = environment.getProperty("proxy.identifier-value", "default-identifier"); - - var servicePorts = specs.stream() - .flatMap(x -> x.getPortMapping().values().stream()) - .map(p -> new ServicePortBuilder().withPort(p).build()) - .collect(Collectors.toList()); - - var startupService = new ServiceBuilder() - .withApiVersion(apiVersion) - .withKind("Service") - .withNewMetadata() - .withName("vl-service-" + containerGroup.getId()) - .addToLabels(identifierLabel, identifierValue) - .addToLabels(specLabels) - .addToLabels(runtimeLabels) - .endMetadata() - .withNewSpec() - .addToSelector(RUNTIME_LABEL_PROXY_ID, proxy.getId()) - .withType("NodePort") - .withPorts(servicePorts) - .endSpec() - .build(); - kubeClient.services().inNamespace(effectiveKubeNamespace).createOrReplace(startupService); - // Workaround: waitUntilReady appears to be buggy. - Retrying.retry(i -> isServiceReady(kubeClient.resource(startupService).fromServer().get()), 60, 1000); - - service = kubeClient.resource(startupService).fromServer().get(); - } - return service; + private Service startService(String effectiveKubeNamespace, Service startupService) { + kubeClient.services().inNamespace(effectiveKubeNamespace).createOrReplace(startupService); + retrying.retry(i -> isServiceReady(kubeClient.resource(startupService).fromServer().get()), 60, Duration.ofSeconds(1)); + return kubeClient.resource(startupService).fromServer().get(); } + /** * Converts the additional manifests of the spec into HasMetadata objects. * When the resource has no namespace definition, the provided namespace @@ -430,41 +454,34 @@ private Service makeServiceIfNecessary(List specs, Proxy proxy, C */ private List getAdditionManifestsAsObjects(Proxy proxy, String namespace) { var context = SpecExpressionContext.create(proxy, proxy.getSpec()); - - var result = new ArrayList(); - for (var manifest : proxy.getSpec().getKubernetes().getAdditionalManifests()) { - var expressionManifest = expressionResolver.evaluateToString(manifest, context); - HasMetadata object = Serialization.unmarshal(new ByteArrayInputStream(expressionManifest.getBytes())); // used to determine whether the manifest has specified a namespace - - var fullObject = kubeClient.load(new ByteArrayInputStream(expressionManifest.getBytes())).get().get(0); - if (object.getMetadata().getNamespace() == null) { - // the load method (in some cases) automatically sets a namespace when no namespace is provided - // therefore we overwrite this namespace with the namespace of the pod. - fullObject.getMetadata().setNamespace(namespace); - } - result.add(fullObject); - } - return result; + return proxy.getSpec().getKubernetes().getAdditionalManifests().stream() + .map(manifest -> expressionResolver.evaluateToString(manifest, context).getBytes()) + .map(bs -> { + HasMetadata object = Serialization.unmarshal(new ByteArrayInputStream(bs)); // used to determine whether the manifest has specified a namespace + var fullObject = kubeClient.load(new ByteArrayInputStream(bs)).get().get(0); + if (object.getMetadata().getNamespace() == null) { + // the load method (in some cases) automatically sets a namespace when no namespace is provided + // therefore we overwrite this namespace with the namespace of the pod. + fullObject.getMetadata().setNamespace(namespace); + } + return fullObject; + }).collect(Collectors.toList()); } private boolean isServiceReady(Service service) { - if (service == null) { - return false; - } - if (service.getStatus() == null) { - return false; - } - return service.getStatus().getLoadBalancer() != null; + return Optional.ofNullable(service).map(Service::getStatus) + .map(ServiceStatus::getLoadBalancer).isPresent(); } protected URI calculateTarget(ContainerGroup containerGroup, int containerPort, int servicePort) throws Exception { - var targetProtocol = getProperty(PROPERTY_CONTAINER_PROTOCOL, DEFAULT_TARGET_PROTOCOL); + var kubernetes = proxyProperties.getKubernetes(); + var targetProtocol = kubernetes.getContainerProtocol(); String targetHostName; int targetPort; var pod = (Pod) containerGroup.getParameters().get(PARAM_POD); - if (isUseInternalNetwork()) { + if (kubernetes.isInternalNetworking()) { targetHostName = pod.getStatus().getPodIP(); targetPort = containerPort; } else { @@ -476,7 +493,6 @@ protected URI calculateTarget(ContainerGroup containerGroup, int containerPort, } @Override - @SuppressWarnings("unchecked") protected void doStopProxy(Proxy proxy) throws VLabControllerException { var kubeNamespace = proxy.getNamespace(); if (kubeNamespace == null) { @@ -488,95 +504,67 @@ protected void doStopProxy(Proxy proxy) throws VLabControllerException { kubeClient.persistentVolumeClaims().inNamespace(kubeNamespace).withLabel(RUNTIME_LABEL_PROXY_ID, proxy.getId()).delete(); // delete additional manifests - for (var fullObject : getAdditionManifestsAsObjects(proxy, kubeNamespace)) { - kubeClient.resource(fullObject).delete(); - } + getAdditionManifestsAsObjects(proxy, kubeNamespace).forEach(fullObject -> kubeClient.resource(fullObject).delete()); } @Override - @SuppressWarnings("unchecked") public BiConsumer getOutputAttacher(Proxy proxy) { var containerGroup = proxy.getContainerGroup(); - var containers = (List) containerGroup.getParameters().get(PARAM_CONTAINER); + var parameters = containerGroup.getParameters(); + var containers = (List) parameters.get(PARAM_CONTAINER); if (containers.isEmpty()) return null; return (stdOut, stdErr) -> { try { - var kubeNamespace = containerGroup.getParameters().get(PARAM_NAMESPACE).toString(); - if (kubeNamespace == null) { - kubeNamespace = getProperty(PROPERTY_NAMESPACE, DEFAULT_NAMESPACE); + var kubeNamespace = Optional.ofNullable(parameters.get(PARAM_NAMESPACE).toString()) + .orElseGet(() -> proxyProperties.getKubernetes().getNamespace()); + try (var watcher = kubeClient.pods().inNamespace(kubeNamespace).withName("vl-pod-" + containerGroup.getId()).watchLog()) { + watcher.getOutput().transferTo(stdOut); } - var watcher = kubeClient.pods().inNamespace(kubeNamespace).withName("vl-pod-" + containerGroup.getId()).watchLog(); - IOUtils.copy(watcher.getOutput(), stdOut); } catch (IOException e) { log.error("Error while attaching to container output", e); } }; } - @Override - protected String getPropertyPrefix() { - return PROPERTY_PREFIX; - } - public void cleanBeforeStart() { - var identifierLabel = environment.getProperty("proxy.identifier-label", "comp.polyu.edu.hk/vl-identifier"); - var identifierValue = environment.getProperty("proxy.identifier-value", "default-identifier"); - var orphanPods = kubeClient.pods().inAnyNamespace().withLabel(identifierLabel, identifierValue).list(); - if (orphanPods != null) { - for (var pod : orphanPods.getItems()) { - var namespace = pod.getMetadata().getNamespace(); - kubeClient.pods().inNamespace(namespace).delete(pod); - } - log.info("Cleaned {} pods", orphanPods.getItems().size()); - } - var orphanServices = kubeClient.services().inAnyNamespace().withLabel(identifierLabel, identifierValue).list(); - if (orphanServices != null) { - for (var service : orphanServices.getItems()) { - var namespace = service.getMetadata().getNamespace(); - kubeClient.services().inNamespace(namespace).delete(service); - } - log.info("Cleaned " + orphanServices.getItems().size() + " services"); - } - var orphanPVCs = kubeClient.persistentVolumeClaims().inAnyNamespace().withLabel(identifierLabel, identifierValue).list(); - if (orphanPVCs != null) { - for (var pvc : orphanPVCs.getItems()) { - var namespace = pvc.getMetadata().getNamespace(); - kubeClient.persistentVolumeClaims().inNamespace(namespace).delete(pvc); - } - log.info("Cleaned " + orphanPVCs.getItems().size() + " PersistentVolumeClaims"); - } + var identifierLabel = proxyProperties.getIdentifierLabel(); + var identifierValue = proxyProperties.getIdentifierValue(); + Optional.ofNullable(kubeClient.pods().inAnyNamespace().withLabel(identifierLabel, identifierValue).list()) + .ifPresent(orphanPods -> { + orphanPods.getItems().forEach(pod -> { + var namespace = pod.getMetadata().getNamespace(); + kubeClient.pods().inNamespace(namespace).delete(pod); + }); + log.info("Cleaned {} pods", orphanPods.getItems().size()); + }); + + Optional.ofNullable(kubeClient.services().inAnyNamespace().withLabel(identifierLabel, identifierValue).list()) + .ifPresent(orphanServices -> { + orphanServices.getItems().forEach(service -> { + var namespace = service.getMetadata().getNamespace(); + kubeClient.services().inNamespace(namespace).delete(service); + }); + log.info("Cleaned {} services", orphanServices.getItems().size()); + }); + + Optional.ofNullable(kubeClient.persistentVolumeClaims().inAnyNamespace().withLabel(identifierLabel, identifierValue).list()) + .ifPresent(orphanPVCs -> { + orphanPVCs.getItems().forEach(pvc -> { + var namespace = pvc.getMetadata().getNamespace(); + kubeClient.persistentVolumeClaims().inNamespace(namespace).delete(pvc); + }); + log.info("Cleaned {} PersistentVolumeClaims", orphanPVCs.getItems().size()); + }); } public PodList getFailedAndUnknownPods() { - var identifierLabel = environment.getProperty("proxy.identifier-label", "comp.polyu.edu.hk/vl-identifier"); - var identifierValue = environment.getProperty("proxy.identifier-value", "default-identifier"); + var identifierLabel = proxyProperties.getIdentifierLabel(); + var identifierValue = proxyProperties.getIdentifierValue(); return kubeClient.pods().inAnyNamespace() - .withLabel(identifierLabel, identifierValue) - .withoutField("status.phase", "Pending") - .withoutField("status.phase", "Running") - .withoutField("status.phase", "Succeeded") - .list(); - } - - private class ErrorPodsCleaner implements Runnable { - @Override - public void run() { - log.info("Enable failed and unknown phase pods detection & cleaning"); - while (true) { - var failedPods = getFailedAndUnknownPods(); - if (failedPods != null && !failedPods.getItems().isEmpty()) { - for (var pod : failedPods.getItems()) { - var proxyId = pod.getMetadata().getLabels().get("comp.polyu.edu.hk/vl-proxy-id"); - proxyService.stopProxy(proxyService.getProxy(proxyId), true, true, 0); - log.error("Cleaned error proxy {}", proxyId); - } - } - try { - Thread.sleep(30000); - } catch (Exception e) { - log.error(e); - } - } - } + .withLabel(identifierLabel, identifierValue) + .withoutField("status.phase", "Pending") + .withoutField("status.phase", "Running") + .withoutField("status.phase", "Succeeded") + .list(); } } \ No newline at end of file diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/backend/kubernetes/PodPatcher.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/backend/kubernetes/PodPatcher.java index d9176975..54af6f2e 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/backend/kubernetes/PodPatcher.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/backend/kubernetes/PodPatcher.java @@ -3,35 +3,33 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.datatype.jsr353.JSR353Module; +import hk.edu.polyu.comp.vlabcontroller.config.ProxyProperties; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.client.internal.SerializationUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.springframework.core.env.Environment; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; -import javax.inject.Inject; import javax.json.JsonPatch; import javax.json.JsonStructure; +@Slf4j @Component +@RequiredArgsConstructor +@RefreshScope public class PodPatcher { - - private static final String DEBUG_PROPERTY = "proxy.kubernetes.debug-patches"; private final ObjectMapper mapper = new ObjectMapper(); - private final Logger log = LogManager.getLogger(getClass()); - private final Environment environment; - private boolean loggingEnabled = false; + private final ProxyProperties proxyProperties; - public PodPatcher(Environment environment) { - this.environment = environment; + public boolean isLoggingEnabled() { + return proxyProperties.getKubernetes().isDebugPatches(); } @PostConstruct public void init() { mapper.registerModule(new JSR353Module()); - loggingEnabled = Boolean.valueOf(environment.getProperty(DEBUG_PROPERTY, "false")); } /** @@ -44,9 +42,9 @@ public Pod patch(Pod pod, JsonPatch patch) { // 1. convert Pod to javax.json.JsonValue object. // This conversion does not actually convert to a string, but some internal // representation of Jackson. - JsonStructure podAsJsonValue = mapper.convertValue(pod, JsonStructure.class); + var podAsJsonValue = mapper.convertValue(pod, JsonStructure.class); // 2. apply patch - JsonStructure patchedPodAsJsonValue = patch.apply(podAsJsonValue); + var patchedPodAsJsonValue = patch.apply(podAsJsonValue); // 3. convert back to a pod return mapper.convertValue(patchedPodAsJsonValue, Pod.class); } @@ -56,11 +54,11 @@ public Pod patch(Pod pod, JsonPatch patch) { * enabled the original and patched specification will be logged as YAML. */ public Pod patchWithDebug(Pod pod, JsonPatch patch) throws JsonProcessingException { - if (loggingEnabled) { + if (isLoggingEnabled()) { log.info("Original Pod: " + SerializationUtils.dumpAsYaml(pod)); } - Pod patchedPod = patch(pod, patch); - if (loggingEnabled) { + var patchedPod = patch(pod, patch); + if (isLoggingEnabled()) { log.info("Patched Pod: " + SerializationUtils.dumpAsYaml(patchedPod)); } return patchedPod; diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/backend/strategy/impl/DefaultProxyLogoutStrategy.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/backend/strategy/impl/DefaultProxyLogoutStrategy.java index 84040eed..26f75d24 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/backend/strategy/impl/DefaultProxyLogoutStrategy.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/backend/strategy/impl/DefaultProxyLogoutStrategy.java @@ -2,26 +2,24 @@ import hk.edu.polyu.comp.vlabcontroller.backend.strategy.IProxyLogoutStrategy; import hk.edu.polyu.comp.vlabcontroller.service.ProxyService; +import lombok.RequiredArgsConstructor; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.stereotype.Component; +import java.time.Duration; + /** * Default logout behaviour: stop all proxies owned by the user. */ @Component @ConditionalOnMissingBean(RedisSessionProxyLogoutStrategy.class) +@RequiredArgsConstructor public class DefaultProxyLogoutStrategy implements IProxyLogoutStrategy { private final ProxyService proxyService; - public DefaultProxyLogoutStrategy(ProxyService proxyService) { - this.proxyService = proxyService; - } - @Override public void onLogout(String userId, boolean expired) { - for (var proxy : proxyService.getProxies(p -> p.getUserId().equals(userId), true)) { - proxyService.stopProxy(proxy, true, true, 0); - } + proxyService.getProxies(p -> p.getUserId().equals(userId), true).forEach(proxy -> proxyService.stopProxy(proxy, true, true, Duration.ZERO)); } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/backend/strategy/impl/DefaultTargetMappingStrategy.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/backend/strategy/impl/DefaultTargetMappingStrategy.java index 89388c56..8f91a0bd 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/backend/strategy/impl/DefaultTargetMappingStrategy.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/backend/strategy/impl/DefaultTargetMappingStrategy.java @@ -11,7 +11,7 @@ public class DefaultTargetMappingStrategy implements IProxyTargetMappingStrategy public static final String DEFAULT_MAPPING_KEY = "default"; public String createMapping(String mappingKey, ContainerGroup containerGroup, Proxy proxy) { - String mapping = proxy.getId(); + var mapping = proxy.getId(); if (!mappingKey.equalsIgnoreCase(DEFAULT_MAPPING_KEY)) { // For non-default mappings, also append the mapping key mapping += "/" + mappingKey; diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/backend/strategy/impl/RedisSessionProxyLogoutStrategy.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/backend/strategy/impl/RedisSessionProxyLogoutStrategy.java index 89d58109..94927718 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/backend/strategy/impl/RedisSessionProxyLogoutStrategy.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/backend/strategy/impl/RedisSessionProxyLogoutStrategy.java @@ -1,30 +1,25 @@ package hk.edu.polyu.comp.vlabcontroller.backend.strategy.impl; import hk.edu.polyu.comp.vlabcontroller.backend.strategy.IProxyLogoutStrategy; -import hk.edu.polyu.comp.vlabcontroller.model.runtime.Proxy; import hk.edu.polyu.comp.vlabcontroller.service.ProxyService; import hk.edu.polyu.comp.vlabcontroller.util.RedisSessionHelper; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; +import java.time.Duration; +@Slf4j @Component @ConditionalOnProperty(prefix = "spring.session", name = "store-type", havingValue = "redis") +@RequiredArgsConstructor(onConstructor_ = {@Lazy}) public class RedisSessionProxyLogoutStrategy implements IProxyLogoutStrategy { - private final Logger log = LogManager.getLogger(RedisSessionProxyLogoutStrategy.class); private final ProxyService proxyService; private final RedisSessionHelper redisSessionHelper; - @Lazy - public RedisSessionProxyLogoutStrategy(ProxyService proxyService, RedisSessionHelper redisSessionHelper) { - this.proxyService = proxyService; - this.redisSessionHelper = redisSessionHelper; - } - @PostConstruct private void init() { log.info("Enabled redis session logout strategy."); @@ -35,8 +30,6 @@ public void onLogout(String userId, boolean expired) { if (redisSessionHelper.getSessionByUsername(userId).size() > 1 - (expired ? 1 : 0)) { return; } - for (Proxy proxy : proxyService.getProxies(p -> p.getUserId().equals(userId), true)) { - proxyService.stopProxy(proxy, true, true, 0); - } + proxyService.getProxies(p -> p.getUserId().equals(userId), true).forEach(proxy -> proxyService.stopProxy(proxy, true, true, Duration.ZERO)); } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/backend/strategy/impl/URLConnectionTestStrategy.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/backend/strategy/impl/URLConnectionTestStrategy.java index 6be84ee1..0bed02e8 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/backend/strategy/impl/URLConnectionTestStrategy.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/backend/strategy/impl/URLConnectionTestStrategy.java @@ -1,82 +1,62 @@ package hk.edu.polyu.comp.vlabcontroller.backend.strategy.impl; import hk.edu.polyu.comp.vlabcontroller.backend.strategy.IProxyTestStrategy; +import hk.edu.polyu.comp.vlabcontroller.config.ProxyProperties; import hk.edu.polyu.comp.vlabcontroller.model.runtime.Proxy; import hk.edu.polyu.comp.vlabcontroller.model.runtime.ProxyStatus; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import hk.edu.polyu.comp.vlabcontroller.util.DurationUtil; +import hk.edu.polyu.comp.vlabcontroller.util.Retrying; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.context.annotation.Primary; -import org.springframework.core.env.Environment; import org.springframework.stereotype.Component; import java.net.HttpURLConnection; -import java.net.URI; import java.net.URL; -import java.util.function.IntPredicate; +import java.time.Duration; +import java.util.Optional; +import java.util.stream.Stream; + +import static io.vavr.API.unchecked; /** * This component tests the responsiveness of containers by making an HTTP GET request to the container's published port (default 3838). * If this request does not receive non-error (5xx) response within a configured time limit, the container is considered to be unresponsive. */ +@Slf4j @Component @Primary +@RequiredArgsConstructor +@RefreshScope public class URLConnectionTestStrategy implements IProxyTestStrategy { - - private final Environment environment; - private final Logger log = LogManager.getLogger(URLConnectionTestStrategy.class); - - public URLConnectionTestStrategy(Environment environment) { - this.environment = environment; - } - - private static boolean retry(IntPredicate job, int tries, int waitTime, boolean retryOnException) { - boolean retVal = false; - RuntimeException exception = null; - for (int currentTry = 1; currentTry <= tries; currentTry++) { - try { - if (job.test(currentTry)) { - retVal = true; - exception = null; - break; - } - } catch (RuntimeException e) { - if (retryOnException) exception = e; - else throw e; - } - try { - Thread.sleep(waitTime); - } catch (InterruptedException ignore) { - } - } - if (exception == null) return retVal; - else throw exception; - } + private final ProxyProperties proxyProperties; + private final Retrying retrying; @Override public boolean testProxy(Proxy proxy) { - - int totalWaitMs = Integer.parseInt(environment.getProperty("proxy.container-wait-time", "20000")); - int waitMs = Math.min(2000, totalWaitMs); - int maxTries = totalWaitMs / waitMs; - int timeoutMs = Integer.parseInt(environment.getProperty("proxy.container-wait-timeout", "5000")); - - if (proxy.getTargets().isEmpty()) return false; - URI targetURI = proxy.getTargets().values().iterator().next(); - int failedResponseCode = -1; - return retry(i -> { - try { - if (proxy.getStatus() == ProxyStatus.Stopping || proxy.getStatus() == ProxyStatus.Stopped) return true; - URL testURL = new URL(targetURI.toString()); - HttpURLConnection connection = ((HttpURLConnection) testURL.openConnection()); - connection.setConnectTimeout(timeoutMs); - connection.setInstanceFollowRedirects(false); - int responseCode = connection.getResponseCode(); - if (responseCode < 500) return true; - } catch (Exception e) { - if (i > 1) - log.warn(String.format("Container unresponsive, trying again (%d/%d): %s", i, maxTries, targetURI)); - } - return false; - }, maxTries, waitMs, false); + var totalWaitMs = proxyProperties.getContainerWaitTime(); + + var waitMs = DurationUtil.atLeast(Duration.ofSeconds(2)).apply(totalWaitMs); + var maxTries = (int) totalWaitMs.dividedBy(waitMs); + var timeoutMs = proxyProperties.getContainerWaitTimeout(); + return Optional.ofNullable(proxy.getTargets()) + .map(x -> x.values().iterator().next()) + .map(x -> retrying.retry(i -> { + try { + if (Stream.of(ProxyStatus.Stopping, ProxyStatus.Stopped).anyMatch(y -> y == proxy.getStatus())) return true; + var connection = (HttpURLConnection) new URL(x.toString()).openConnection(); + connection.setConnectTimeout((int) timeoutMs.toMillis()); + connection.setInstanceFollowRedirects(false); + var responseCode = connection.getResponseCode(); + if (responseCode < 500) return true; + } catch (Exception e) { + if (i > 1) + log.warn(String.format("Container unresponsive, trying again (%d/%d): %s", i, maxTries, x)); + } + return false; + }, maxTries, waitMs, false)) + .map(unchecked(x -> x.get())) + .orElse(false); } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyEngagementProperties.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyEngagementProperties.java new file mode 100644 index 00000000..f7199256 --- /dev/null +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyEngagementProperties.java @@ -0,0 +1,20 @@ +package hk.edu.polyu.comp.vlabcontroller.config; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ProxyEngagementProperties { + boolean enabled = true; + List filterPath = new ArrayList<>(); + int idleRetry = 3; + int threshold = 230; + Duration maxAge = Duration.ofHours(4); +} diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyKeycloakProperties.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyKeycloakProperties.java new file mode 100644 index 00000000..1b258ca9 --- /dev/null +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyKeycloakProperties.java @@ -0,0 +1,20 @@ +package hk.edu.polyu.comp.vlabcontroller.config; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.keycloak.representations.IDToken; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ProxyKeycloakProperties { + String realm; + String claim; + String authServerUrl; + String resource; + String sslRequired = "external"; + boolean useResourceRoleMappings = false; + String credentialsSecret; + String nameAttribute = IDToken.NAME; +} diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyKubernetesProperties.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyKubernetesProperties.java new file mode 100644 index 00000000..5115c57b --- /dev/null +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyKubernetesProperties.java @@ -0,0 +1,31 @@ +package hk.edu.polyu.comp.vlabcontroller.config; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ProxyKubernetesProperties { + Duration podWaitTime = Duration.ofMinutes(1); + boolean debugPatches = false; + boolean internalNetworking = false; + boolean privileged = false; + String url; + String certPath; + String containerProtocol = "http"; + String namespace = "default"; + String apiVersion = "v1"; + String imagePullPolicy; + List imagePullSecrets = new ArrayList<>(); + String imagePullSecret; + String nodeSelector; + boolean customNamespace = false; + String namespacePrefix; + String secureRuntimeName = "kata-qemu"; +} diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyOAuth2Properties.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyOAuth2Properties.java new file mode 100644 index 00000000..48031314 --- /dev/null +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyOAuth2Properties.java @@ -0,0 +1,13 @@ +package hk.edu.polyu.comp.vlabcontroller.config; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ProxyOAuth2Properties { + String resourceId; + String jwksUrl; +} diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyOpenIDProperties.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyOpenIDProperties.java new file mode 100644 index 00000000..83bb2206 --- /dev/null +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyOpenIDProperties.java @@ -0,0 +1,23 @@ +package hk.edu.polyu.comp.vlabcontroller.config; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.ArrayList; +import java.util.List; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ProxyOpenIDProperties { + String logoutUrl; + String usernameAttribute = "email"; + String authUrl; + String tokenUrl; + String jwksUrl; + String clientId; + String clientSecret; + String rolesClaim; + List scopes = new ArrayList<>(); +} diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyProperties.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyProperties.java new file mode 100644 index 00000000..c4b479c7 --- /dev/null +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyProperties.java @@ -0,0 +1,101 @@ +package hk.edu.polyu.comp.vlabcontroller.config; + +import hk.edu.polyu.comp.vlabcontroller.model.spec.ProxySpec; +import hk.edu.polyu.comp.vlabcontroller.spec.IProxySpecProvider; +import hk.edu.polyu.comp.vlabcontroller.util.SessionHelper; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.context.properties.NestedConfigurationProperty; +import org.springframework.cloud.context.config.annotation.RefreshScope; +import org.springframework.context.annotation.Primary; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.inject.Inject; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +@Data +@RefreshScope +@Component +@Primary +@ConfigurationProperties(prefix = "proxy") +@AllArgsConstructor +@NoArgsConstructor +public class ProxyProperties implements IProxySpecProvider { + List adminGroups = new ArrayList<>(); + String allowedRole; + String authentication = "none"; + String bindAddress = "0.0.0.0"; + String containerBackend = "kubernetes"; + String containerLogPath; + String containerLogS3AccessKey; + String containerLogS3AccessSecret; + String containerLogS3Endpoint = "https://s3-eu-west-1.amazonaws.com"; + String domain; + String faviconPath; + String identifierLabel = "comp.polyu.edu.hk/vl-identifier"; + String identifierValue = "default-identifier"; + String landingPage = "/"; + String logoUrl; + Duration maxAge = Duration.ofHours(4); + String sameSiteCookie = "Lax"; + String templatePath; + String title = "VLabController"; + boolean containerLogS3SSE = false; + boolean heartbeatEnabled = true; + boolean hideNavbar; + String supportMailToAddress; + int containerQuantityLimit = 2; + Duration containerWaitTime = Duration.ofSeconds(20); + Duration containerWaitTimeout = Duration.ofSeconds(5); + Duration heartbeatRate = Duration.ofSeconds(10); + Duration heartbeatTimeout = Duration.ofSeconds(60); + Duration waitTimeout = Duration.ofSeconds(5); + short port = 8080; + List specs = new ArrayList<>(); + ProxySpec fileBrowser; + String serviceName; + + @NestedConfigurationProperty ProxyKubernetesProperties kubernetes = new ProxyKubernetesProperties(); + @NestedConfigurationProperty ProxyKeycloakProperties keycloak = new ProxyKeycloakProperties(); + @NestedConfigurationProperty ProxyOpenIDProperties openID = new ProxyOpenIDProperties(); + @NestedConfigurationProperty ProxyOAuth2Properties oauth2 = new ProxyOAuth2Properties(); + @NestedConfigurationProperty ProxyWebServiceProperties webService = new ProxyWebServiceProperties(); + @NestedConfigurationProperty ProxyUsageStatsProperties usageStats = new ProxyUsageStatsProperties(); + @NestedConfigurationProperty ProxyEngagementProperties engagement = new ProxyEngagementProperties(); + @NestedConfigurationProperty List users = new ArrayList<>(); + + public ProxySpec getSpec(String id) { + return Optional.ofNullable(id).filter(Predicate.not(String::isBlank)) + .flatMap(x -> specs.stream().filter(s -> x.equals(s.getId())).findAny()) + .orElse(null); + } + + @PostConstruct + public void afterPropertiesSet() { + this.specs.stream().collect(Collectors.groupingBy(ProxySpec::getId)).forEach((id, duplicateSpecs) -> { + if (duplicateSpecs.size() > 1) + throw new IllegalArgumentException("Configuration error: spec with id '${id}' is defined multiple times"); + }); + } + + private static ServerProperties serverProperties; + + @Inject + public void setServerProperties(ServerProperties serverProperties) { + ProxyProperties.serverProperties = serverProperties; + } + + public static String getPublicPath(String appName) { + var contextPath = SessionHelper.getContextPath(serverProperties, true); + return contextPath + "app_direct/" + appName + "/"; + } +} + diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyUsageStatsHikariProperties.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyUsageStatsHikariProperties.java new file mode 100644 index 00000000..6b908098 --- /dev/null +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyUsageStatsHikariProperties.java @@ -0,0 +1,18 @@ +package hk.edu.polyu.comp.vlabcontroller.config; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.time.Duration; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ProxyUsageStatsHikariProperties { + Duration connectionTimeout; + Duration idleTimeout; + Duration maxLifetime; + int minimumIdle; + int maximumPoolSize = 1; +} diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyUsageStatsProperties.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyUsageStatsProperties.java new file mode 100644 index 00000000..506ab563 --- /dev/null +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyUsageStatsProperties.java @@ -0,0 +1,29 @@ +package hk.edu.polyu.comp.vlabcontroller.config; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.springframework.boot.context.properties.NestedConfigurationProperty; + +import java.util.Objects; +import java.util.stream.Stream; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ProxyUsageStatsProperties { + String username = "monetdb"; + String password = "monetdb"; + @NestedConfigurationProperty ProxyUsageStatsPropertiesUrls url = new ProxyUsageStatsPropertiesUrls(); + @NestedConfigurationProperty ProxyUsageStatsHikariProperties hikari = new ProxyUsageStatsHikariProperties(); + + @Data public static class ProxyUsageStatsPropertiesUrls { + String influx = ""; + String jdbc = ""; + String micrometer = ""; + + public boolean backendExists() { + return !Stream.of(influx, jdbc, micrometer).filter(Objects::nonNull).allMatch(String::isBlank); + } + } +} diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyUserProperties.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyUserProperties.java new file mode 100644 index 00000000..4a351641 --- /dev/null +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyUserProperties.java @@ -0,0 +1,18 @@ +package hk.edu.polyu.comp.vlabcontroller.config; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.ArrayList; +import java.util.List; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ProxyUserProperties { + String name; + String password; + List roles = new ArrayList<>(); + List groups = new ArrayList<>(); +} diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyWebServiceProperties.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyWebServiceProperties.java new file mode 100644 index 00000000..7695d597 --- /dev/null +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ProxyWebServiceProperties.java @@ -0,0 +1,13 @@ +package hk.edu.polyu.comp.vlabcontroller.config; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ProxyWebServiceProperties { + String authenticationRequestBody; + String authenticationUrl; +} diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ServerProperties.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ServerProperties.java new file mode 100644 index 00000000..afd33dd8 --- /dev/null +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/config/ServerProperties.java @@ -0,0 +1,18 @@ +package hk.edu.polyu.comp.vlabcontroller.config; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@Data +@ConfigurationProperties(prefix = "server") +@AllArgsConstructor +@NoArgsConstructor +public class ServerProperties { + String frameOptions = "disable"; + @Value("${server.servlet.context-path:}") String servletContextPath; + boolean secureCookies = false; + boolean useForwardHeaders; +} diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/controllers/AdminController.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/controllers/AdminController.java index 92055d88..1cd20c84 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/controllers/AdminController.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/controllers/AdminController.java @@ -1,69 +1,55 @@ package hk.edu.polyu.comp.vlabcontroller.controllers; -import hk.edu.polyu.comp.vlabcontroller.auth.IAuthenticationBackend; -import hk.edu.polyu.comp.vlabcontroller.model.runtime.HeartbeatStatus; import hk.edu.polyu.comp.vlabcontroller.model.runtime.Proxy; import hk.edu.polyu.comp.vlabcontroller.service.HeartbeatService; -import hk.edu.polyu.comp.vlabcontroller.service.ProxyService; -import hk.edu.polyu.comp.vlabcontroller.service.UserService; +import hk.edu.polyu.comp.vlabcontroller.util.DurationUtil; +import lombok.RequiredArgsConstructor; +import org.apache.commons.lang3.time.DurationUtils; import org.springframework.context.annotation.Lazy; -import org.springframework.core.env.Environment; import org.springframework.stereotype.Controller; import org.springframework.ui.ModelMap; import org.springframework.web.bind.annotation.RequestMapping; import javax.servlet.http.HttpServletRequest; -import java.util.HashMap; +import java.time.Duration; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; @Controller +@RequiredArgsConstructor(onConstructor_ = {@Lazy}) public class AdminController extends BaseController { - private final HeartbeatService heartbeatService; - protected AdminController(ProxyService proxyService, UserService userService, Environment environment, IAuthenticationBackend authenticationBackend, @Lazy HeartbeatService heartbeatService) { - super(proxyService, userService, environment, authenticationBackend); - this.heartbeatService = heartbeatService; - } - - @RequestMapping("/admin") private String admin(ModelMap map, HttpServletRequest request) { prepareMap(map, request); - List proxies = proxyService.getProxies(null, false); - Map proxyUptimes = new HashMap<>(); - for (Proxy proxy : proxies) { - long uptimeSec = 0; - // if the proxy hasn't started up yet, the uptime should be zero - if (proxy.getStartupTimestamp() > 0) { - uptimeSec = (System.currentTimeMillis() - proxy.getStartupTimestamp()) / 1000; - } - String uptime = String.format("%d:%02d:%02d", uptimeSec / 3600, (uptimeSec % 3600) / 60, uptimeSec % 60); - proxyUptimes.put(proxy.getId(), uptime); - } + var websocketHeartbeats = heartbeatService.getWebsocketHeartbeats(); + var heartbeatRate = proxyProperties.getHeartbeatRate(); - Map proxyHeartbeats = heartbeatService.getProxyHeartbeats(); - Map websocketHeartbeats = heartbeatService.getWebsocketHeartbeats(); - long heartbeatRate = Long.parseLong(environment.getProperty("proxy.heartbeat-rate", "60000")); - - Map lastActive = new HashMap<>(); - proxyHeartbeats.forEach((k, v) -> { - long httpRequestActiveTimestamp = v; - HeartbeatStatus hbs = websocketHeartbeats.get(k); - if (hbs != null) { - long websocketActiveTimestamp = hbs.getLastRecordTimestamp() - hbs.getTerminateCounter() * heartbeatRate; - lastActive.put(k, Math.max(websocketActiveTimestamp, httpRequestActiveTimestamp)); - } else { - lastActive.put(k, httpRequestActiveTimestamp); - } - }); - - map.put("proxies", proxies); - map.put("proxyUptimes", proxyUptimes); - map.put("lastActive", lastActive); + map.put("proxies", proxyService.getProxies(null, false)); + map.put("proxyUptimes", getUptimes(proxyService.getProxies(null, false))); + map.put("lastActive", heartbeatService.getProxyHeartbeats().entrySet().stream().collect( + Collectors.toMap(Map.Entry::getKey, x -> DurationUtil.max(x.getValue(), + Optional.ofNullable(websocketHeartbeats.get(x.getKey())) + .map(hbs -> hbs.getLastRecordTimestamp().minus(heartbeatRate.multipliedBy(hbs.getTerminateCounter()))) + .orElse(Duration.ofMillis(1L).negated())) + ))); return "admin"; } + + static Map getUptimes(List proxies) { + return proxies.stream() + .collect(Collectors.toMap(Proxy::getId, proxy -> { + // if the proxy hasn't started up yet, the uptime should be zero + var uptime = proxy.getStartupTimestamp(); + if (DurationUtils.isPositive(uptime)) { + uptime = Duration.ofMillis(System.currentTimeMillis()).minus(uptime); + } + return String.format("%d:%02d:%02d", uptime.toHours(), uptime.toMinutesPart(), uptime.toSecondsPart()); + })); + } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/controllers/AppController.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/controllers/AppController.java index a3f9718b..49ffdc65 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/controllers/AppController.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/controllers/AppController.java @@ -1,19 +1,14 @@ package hk.edu.polyu.comp.vlabcontroller.controllers; -import com.google.common.base.Strings; import hk.edu.polyu.comp.vlabcontroller.VLabControllerException; -import hk.edu.polyu.comp.vlabcontroller.auth.IAuthenticationBackend; import hk.edu.polyu.comp.vlabcontroller.model.runtime.Proxy; -import hk.edu.polyu.comp.vlabcontroller.model.spec.ProxySpec; import hk.edu.polyu.comp.vlabcontroller.model.spec.EntryPointSpec; -import hk.edu.polyu.comp.vlabcontroller.service.ProxyService; -import hk.edu.polyu.comp.vlabcontroller.service.UserService; import hk.edu.polyu.comp.vlabcontroller.util.ProxyMappingManager; +import hk.edu.polyu.comp.vlabcontroller.util.Retrying; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.apache.http.client.utils.URIBuilder; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.springframework.context.annotation.Lazy; -import org.springframework.core.env.Environment; import org.springframework.stereotype.Controller; import org.springframework.ui.ModelMap; import org.springframework.web.bind.annotation.ModelAttribute; @@ -30,29 +25,21 @@ import java.net.URISyntaxException; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; -import java.time.Duration; import java.util.Base64; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; import static hk.edu.polyu.comp.vlabcontroller.controllers.FileBrowserController.awaitReadyHelper; +@Slf4j @Controller +@RequiredArgsConstructor(onConstructor_ = {@Lazy}) public class AppController extends BaseController { - - private final Logger log = LogManager.getLogger(AppController.class); - private final ProxyMappingManager mappingManager; - - protected AppController(ProxyService proxyService, UserService userService, Environment environment, @Lazy IAuthenticationBackend authenticationBackend, ProxyMappingManager mappingManager) { - super(proxyService, userService, environment, authenticationBackend); - this.mappingManager = mappingManager; - } - + private final Retrying retrying; @RequestMapping(value = "/app/**", method = RequestMethod.GET) public String app(ModelMap map, HttpServletRequest request, @@ -60,10 +47,11 @@ public String app(ModelMap map, HttpServletRequest request, @ModelAttribute("md") String markdownEncodedUrl) { prepareMap(map, request); - Proxy proxy = findUserProxy(request); - if (proxy == null && !userService.isAdmin()) { - int containerLimit = environment.getProperty("proxy.container-quantity-limit", Integer.class, 2); - int proxies = proxyService.getProxies(p -> p.getUserId().equals(userService.getCurrentUserId()) && !p.getSpec().getId().equals("filebrowser"), false).size(); + var proxy = findUserProxy(request); + var hasProxy = proxy == null; + if (hasProxy && !userService.isAdmin()) { + int containerLimit = proxyProperties.getContainerQuantityLimit(); + var proxies = proxyService.getProxies(p -> p.getUserId().equals(userService.getCurrentUserId()) && !p.getSpec().getId().equals("filebrowser"), false).size(); if (proxies >= containerLimit) { return "limit-error"; } @@ -71,23 +59,23 @@ public String app(ModelMap map, HttpServletRequest request, awaitReady(proxy); map.put("appTitle", getAppTitle(request)); - String baseDomain = environment.getProperty("proxy.domain"); + var baseDomain = proxyProperties.getDomain(); map.put("baseDomain", baseDomain); - if (!Strings.isNullOrEmpty(innerURI)) { + if (innerURI != null && !innerURI.isEmpty()) { innerURI = new String(Base64.getDecoder().decode(innerURI), StandardCharsets.UTF_8); map.put("subDomainMode", true); map.put("iframeURL", innerURI); - map.put("container", (proxy == null) ? "" : innerURI); + map.put("container", hasProxy ? "" : innerURI); } else { - map.put("container", (proxy == null) ? "" : buildContainerPath(request)); + map.put("container", hasProxy ? "" : buildContainerPath(request)); } - map.put("proxyId", (proxy == null) ? "" : proxy.getId()); - map.put("startTime", (proxy == null) ? System.currentTimeMillis() : proxy.getStartupTimestamp()); - map.put("maxAge", Duration.parse(environment.getProperty("proxy.engagement.max-age", "PT4H")).toMillis()); + map.put("proxyId", hasProxy ? "" : proxy.getId()); + map.put("startTime", hasProxy ? System.currentTimeMillis() : proxy.getStartupTimestamp()); + map.put("maxAge", proxyProperties.getMaxAge().toMillis()); String markdownURL; try { markdownURL = new String(Base64.getUrlDecoder().decode(markdownEncodedUrl), StandardCharsets.UTF_8); - Pattern urlPattern = Pattern.compile("(https?)://[-A-Za-z0-9+&@#/%?=~_|!:,.;]+[-A-Za-z0-9+&@#/%=~_|]"); + var urlPattern = Pattern.compile("(https?)://[-A-Za-z0-9+&@#/%?=~_|!:,.;]+[-A-Za-z0-9+&@#/%=~_|]"); if (urlPattern.matcher(markdownURL).matches()) { markdownURL = URLEncoder.encode(markdownURL, StandardCharsets.UTF_8.toString()); } else { @@ -104,47 +92,43 @@ public String app(ModelMap map, HttpServletRequest request, @ResponseBody public Map startApp(HttpServletRequest request) { try { - Proxy proxy = getOrStart(request); - String containerPath = buildContainerPath(request); + var proxy = getOrStart(request); + var containerPath = buildContainerPath(request); Map response = new HashMap<>(); response.put("containerPath", containerPath); response.put("proxyId", proxy.getId()); return response; - } catch (IllegalArgumentException e) { - log.error(e.getMessage()); - log.debug(e); - Map response = new HashMap<>(); - response.put("error_code", "404"); - response.put("error_message", "Unable to find application: " + getAppName(request)); - return response; - } catch (VLabControllerException e) { + } catch (IllegalArgumentException | VLabControllerException e) { log.error(e.getMessage() + ": " + getAppName(request)); - log.debug(e); - Map response = new HashMap<>(); - response.put("error_code", "404"); - response.put("error_message", "Failed to start application: " + getAppName(request)); - return response; + log.debug("an error occured: {}", e); + return Map.ofEntries( + Map.entry("error_code", "404"), + Map.entry("error_message", + (e instanceof IllegalArgumentException + ? "Unable to find application: " + : "Failed to start application: ") + getAppName(request)) + ); } } @RequestMapping(value = "/app_direct/**") public void appDirect(HttpServletRequest request, HttpServletResponse response) throws IOException { - Proxy proxy = findUserProxy(request); + var proxy = findUserProxy(request); awaitReady(proxy); - String mapping = getProxyEndpoint(proxy); - String appPort = getAppPort(request); - String subPath = request.getRequestURI(); + var mapping = getProxyEndpoint(proxy); + var appPort = getAppPort(request); + var subPath = request.getRequestURI(); subPath = subPath.substring(subPath.indexOf("/app_direct/") + 12); subPath = subPath.substring(getAppName(request).length()); - int port = -1; + var port = -1; if (appPort != null) { port = Integer.parseInt(appPort); subPath = subPath.substring(("/port/" + appPort).length()); } - if (subPath.trim().isEmpty()) { + if (subPath.isBlank()) { try { response.sendRedirect(request.getRequestURI() + "/"); } catch (Exception e) { @@ -170,19 +154,18 @@ private String subDomainRedirection(ModelMap map, HttpServletRequest request, RedirectAttributes redirectAttributes, @ModelAttribute("md") String markdownUrl) { try { - String[] servletPath = request.getServletPath().substring("/redirect/".length()).split("/", -1); - String subDomain = servletPath[0]; - String path = servletPath[1]; - String baseDomain = environment.getProperty("proxy.domain"); - String[] args = subDomain.split("--"); - String appID = args[args.length - 2]; - ProxySpec spec = proxyService.getProxySpec(appID); - - @SuppressWarnings("unchecked") - List apps = (List) spec.getSettings().get("entrypoint"); - - EntryPointSpec entryPointSpec = apps.stream().filter(p -> args[0].equals(Integer.toString(p.getPort()))).collect(Collectors.toList()).get(0); - URIBuilder innerURI = new URIBuilder(); + var servletPath = request.getServletPath().substring("/redirect/".length()).split("/", -1); + var subDomain = servletPath[0]; + var path = servletPath[1]; + var baseDomain = proxyProperties.getDomain(); + var args = subDomain.split("--"); + var appID = args[args.length - 2]; + var spec = proxyService.getProxySpec(appID); + + var apps = (List) spec.getSettings().get("entrypoint"); + + var entryPointSpec = apps.stream().filter(p -> args[0].equals(Integer.toString(p.getPort()))).collect(Collectors.toList()).get(0); + var innerURI = new URIBuilder(); innerURI.setScheme("https"); innerURI.setHost(subDomain + "." + baseDomain); innerURI.setPath(path); @@ -197,31 +180,31 @@ private String subDomainRedirection(ModelMap map, HttpServletRequest request, } private Proxy getOrStart(HttpServletRequest request) { - Proxy proxy = findUserProxy(request); + var proxy = findUserProxy(request); if (proxy == null) { - String specId = getAppName(request); - ProxySpec spec = proxyService.getProxySpec(specId); + var specId = getAppName(request); + var spec = proxyService.getProxySpec(specId); if (spec == null) throw new IllegalArgumentException("Unknown proxy spec: " + specId); - ProxySpec resolvedSpec = proxyService.resolveProxySpec(spec, null, null); + var resolvedSpec = proxyService.resolveProxySpec(spec, null, null); proxy = proxyService.startProxy(resolvedSpec, false); } return proxy; } private boolean awaitReady(Proxy proxy) { - return awaitReadyHelper(proxy, environment.getProperty("proxy.container-wait-time", "20000")); + return awaitReadyHelper(proxy, proxyProperties.getContainerWaitTime(), retrying); } private String buildContainerPath(HttpServletRequest request) { - String appName = getAppName(request); + var appName = getAppName(request); if (appName == null) return ""; - String queryString = ServletUriComponentsBuilder.fromRequest(request).replaceQueryParam("sp_hide_navbar").build().getQuery(); + var queryString = ServletUriComponentsBuilder.fromRequest(request).replaceQueryParam("sp_hide_navbar").build().getQuery(); queryString = (queryString == null) ? "" : "?" + queryString; - Pattern containerPathPattern = Pattern.compile(".*?/app[^/]*/[^/]*/?(.*)"); - Matcher matcher = containerPathPattern.matcher(request.getRequestURI()); - String containerPath = matcher.find() ? matcher.group(1) + queryString : queryString; + var containerPathPattern = Pattern.compile(".*?/app[^/]*/[^/]*/?(.*)"); + var matcher = containerPathPattern.matcher(request.getRequestURI()); + var containerPath = matcher.find() ? matcher.group(1) + queryString : queryString; return getContextPath() + "app_direct/" + appName + "/" + containerPath; } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/controllers/BaseController.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/controllers/BaseController.java index 5c038da6..d5a74ae6 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/controllers/BaseController.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/controllers/BaseController.java @@ -1,55 +1,48 @@ package hk.edu.polyu.comp.vlabcontroller.controllers; import hk.edu.polyu.comp.vlabcontroller.auth.IAuthenticationBackend; +import hk.edu.polyu.comp.vlabcontroller.config.ProxyProperties; +import hk.edu.polyu.comp.vlabcontroller.config.ServerProperties; import hk.edu.polyu.comp.vlabcontroller.model.runtime.Proxy; -import hk.edu.polyu.comp.vlabcontroller.model.spec.ProxySpec; import hk.edu.polyu.comp.vlabcontroller.service.ProxyService; import hk.edu.polyu.comp.vlabcontroller.service.UserService; import hk.edu.polyu.comp.vlabcontroller.util.SessionHelper; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; import org.apache.http.client.utils.URIBuilder; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.springframework.context.annotation.Lazy; -import org.springframework.core.env.Environment; +import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.security.authentication.AnonymousAuthenticationToken; -import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.ui.ModelMap; import org.springframework.util.StreamUtils; +import javax.inject.Inject; import javax.servlet.http.HttpServletRequest; import java.io.IOException; -import java.io.InputStream; import java.net.URISyntaxException; import java.net.URL; import java.net.URLConnection; -import java.security.Principal; -import java.util.*; -import java.util.regex.Matcher; +import java.util.Base64; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; import java.util.regex.Pattern; +@RefreshScope +@Slf4j public abstract class BaseController { - - private static final Logger logger = LogManager.getLogger(BaseController.class); private static final Pattern appPattern = Pattern.compile(".*?/app[^/]*/([^/]*)/?.*"); private static final Pattern portPattern = Pattern.compile(".*/app[^/]*/[^/]*/port/([0-9]{1,5}).*"); private static final Map imageCache = new HashMap<>(); - - final ProxyService proxyService; - final UserService userService; - final Environment environment; - final IAuthenticationBackend authenticationBackend; - - @Lazy - protected BaseController(ProxyService proxyService, UserService userService, Environment environment, IAuthenticationBackend authenticationBackend) { - this.proxyService = proxyService; - this.userService = userService; - this.environment = environment; - this.authenticationBackend = authenticationBackend; - } + + @Setter(onMethod_ = {@Inject}) protected ProxyService proxyService; + @Setter(onMethod_ = {@Inject}) protected UserService userService; + @Setter(onMethod_ = {@Inject}) protected ProxyProperties proxyProperties; + @Setter(onMethod_ = {@Inject}) protected ServerProperties serverProperties; + @Setter(onMethod_ = {@Inject}) protected IAuthenticationBackend authenticationBackend; protected String getUserName(HttpServletRequest request) { - Principal principal = request.getUserPrincipal(); + var principal = request.getUserPrincipal(); return (principal == null) ? request.getSession().getId() : principal.getName(); } @@ -58,7 +51,7 @@ protected String getAppPort(HttpServletRequest request) { } protected String getAppPort(String uri) { - Matcher matcher = portPattern.matcher(uri); + var matcher = portPattern.matcher(uri); return matcher.matches() ? matcher.group(1) : null; } @@ -67,24 +60,24 @@ protected String getAppName(HttpServletRequest request) { } protected String getAppName(String uri) { - Matcher matcher = appPattern.matcher(uri); + var matcher = appPattern.matcher(uri); return matcher.matches() ? matcher.group(1) : null; } protected String getAppTitle(HttpServletRequest request) { - String appName = getAppName(request); + var appName = getAppName(request); if (appName == null || appName.isEmpty()) return ""; - ProxySpec spec = proxyService.getProxySpec(appName); + var spec = proxyService.getProxySpec(appName); if (spec == null || spec.getDisplayName() == null || spec.getDisplayName().isEmpty()) return appName; else return spec.getDisplayName(); } protected String getContextPath() { - return SessionHelper.getContextPath(environment, true); + return SessionHelper.getContextPath(serverProperties, true); } protected Proxy findUserProxy(HttpServletRequest request) { - String appName = getAppName(request); + var appName = getAppName(request); if (appName == null) return null; return proxyService.findProxy(p -> appName.equals(p.getSpec().getId()) && userService.isOwner(p), false); } @@ -95,36 +88,39 @@ protected String getProxyEndpoint(Proxy proxy) { } protected void prepareMap(ModelMap map, HttpServletRequest request) { - map.put("title", environment.getProperty("proxy.title", "VLabController")); - map.put("logo", resolveImageURI(environment.getProperty("proxy.logo-url"))); - map.put("instance", environment.getProperty("proxy.identifier-value", "default-identifier")); - map.put("enableSubDomainMode", !environment.getProperty("proxy.domain", "").isEmpty()); - String authURL = environment.getProperty("proxy.keycloak.auth-server-url", ""); - String realm = environment.getProperty("proxy.keycloak.realm", ""); - String accountManagementUrl = null; - try { - URIBuilder uriBuilder = new URIBuilder(authURL); - List pathSegments = uriBuilder.getPathSegments(); - pathSegments.removeIf(String::isBlank); - pathSegments.add("realms"); - pathSegments.add(realm); - pathSegments.add("account"); - uriBuilder.setPathSegments(pathSegments); - accountManagementUrl = uriBuilder.build().toString(); - } catch (URISyntaxException e) { - logger.error("Keycloak URL syntax error"); + map.put("title", proxyProperties.getTitle()); + map.put("logo", proxyProperties.getLogoUrl()); + map.put("instance", proxyProperties.getIdentifierValue()); + map.put("enableSubDomainMode", !proxyProperties.getDomain().isEmpty()); + var keycloak = proxyProperties.getKeycloak(); + if (proxyProperties.getAuthentication().equals("keycloak") && keycloak != null) { + var authURL = keycloak.getAuthServerUrl(); + var realm = keycloak.getRealm(); + String accountManagementUrl = null; + try { + var uriBuilder = new URIBuilder(authURL); + var pathSegments = uriBuilder.getPathSegments(); + pathSegments.removeIf(String::isBlank); + pathSegments.add("realms"); + pathSegments.add(realm); + pathSegments.add("account"); + uriBuilder.setPathSegments(pathSegments); + accountManagementUrl = uriBuilder.build().toString(); + } catch (URISyntaxException e) { + log.error("Keycloak URL syntax error"); + } + map.put("accountManagementUrl", accountManagementUrl); } - map.put("accountManagementUrl", accountManagementUrl); - String hideNavBarParam = request.getParameter("sp_hide_navbar"); + var hideNavBarParam = request.getParameter("sp_hide_navbar"); if (Objects.equals(hideNavBarParam, "true")) { map.put("showNavbar", false); } else { - map.put("showNavbar", !Boolean.parseBoolean(environment.getProperty("proxy.hide-navbar"))); + map.put("showNavbar", !proxyProperties.isHideNavbar()); } - Authentication authentication = SecurityContextHolder.getContext().getAuthentication(); - boolean isLoggedIn = authentication != null && !(authentication instanceof AnonymousAuthenticationToken) && authentication.isAuthenticated(); + var authentication = SecurityContextHolder.getContext().getAuthentication(); + var isLoggedIn = authentication != null && !(authentication instanceof AnonymousAuthenticationToken) && authentication.isAuthenticated(); map.put("isLoggedIn", isLoggedIn); map.put("isAdmin", userService.isAdmin(authentication)); map.put("isSupportEnabled", isLoggedIn && getSupportAddress() != null); @@ -132,25 +128,25 @@ protected void prepareMap(ModelMap map, HttpServletRequest request) { } protected String getSupportAddress() { - return environment.getProperty("proxy.support.mail-to-address"); + return proxyProperties.getSupportMailToAddress(); } protected String resolveImageURI(String resourceURI) { if (resourceURI == null || resourceURI.isEmpty()) return resourceURI; if (imageCache.containsKey(resourceURI)) return imageCache.get(resourceURI); - String resolvedValue = resourceURI; + var resolvedValue = resourceURI; if (resourceURI.toLowerCase().startsWith("file://")) { - String mimetype = URLConnection.guessContentTypeFromName(resourceURI); + var mimetype = URLConnection.guessContentTypeFromName(resourceURI); if (mimetype == null) { - logger.warn("Cannot determine mimetype for resource: " + resourceURI); + log.warn("Cannot determine mimetype for resource: " + resourceURI); } else { - try (InputStream input = new URL(resourceURI).openConnection().getInputStream()) { - byte[] data = StreamUtils.copyToByteArray(input); - String encoded = Base64.getEncoder().encodeToString(data); + try (var input = new URL(resourceURI).openConnection().getInputStream()) { + var data = StreamUtils.copyToByteArray(input); + var encoded = Base64.getEncoder().encodeToString(data); resolvedValue = String.format("data:%s;base64,%s", mimetype, encoded); } catch (IOException e) { - logger.warn("Failed to convert file URI to data URI: " + resourceURI, e); + log.warn("Failed to convert file URI to data URI: " + resourceURI, e); } } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/controllers/ControlPanelController.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/controllers/ControlPanelController.java index fd72c68d..376e4d37 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/controllers/ControlPanelController.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/controllers/ControlPanelController.java @@ -1,44 +1,22 @@ package hk.edu.polyu.comp.vlabcontroller.controllers; -import hk.edu.polyu.comp.vlabcontroller.auth.IAuthenticationBackend; -import hk.edu.polyu.comp.vlabcontroller.model.runtime.Proxy; -import hk.edu.polyu.comp.vlabcontroller.service.ProxyService; -import hk.edu.polyu.comp.vlabcontroller.service.UserService; -import org.springframework.context.annotation.Lazy; -import org.springframework.core.env.Environment; import org.springframework.stereotype.Controller; import org.springframework.ui.ModelMap; import org.springframework.web.bind.annotation.RequestMapping; import javax.servlet.http.HttpServletRequest; -import java.util.HashMap; -import java.util.List; -import java.util.Map; @Controller public class ControlPanelController extends BaseController { - protected ControlPanelController(ProxyService proxyService, UserService userService, Environment environment, @Lazy IAuthenticationBackend authenticationBackend) { - super(proxyService, userService, environment, authenticationBackend); - } - @RequestMapping("/controlpanel") private String panel(ModelMap map, HttpServletRequest request) { prepareMap(map, request); - String username = getUserName(request); - List proxies = proxyService.getProxies(p -> p.getUserId().equals(username), false); + var username = getUserName(request); + var proxies = proxyService.getProxies(p -> p.getUserId().equals(username), false); - Map proxyUptimes = new HashMap<>(); - for (Proxy proxy : proxies) { - long uptimeSec = 0; - // if the proxy hasn't started up yet, the uptime should be zero - if (proxy.getStartupTimestamp() > 0) { - uptimeSec = (System.currentTimeMillis() - proxy.getStartupTimestamp()) / 1000; - } - String uptime = String.format("%d:%02d:%02d", uptimeSec / 3600, (uptimeSec % 3600) / 60, uptimeSec % 60); - proxyUptimes.put(proxy.getId(), uptime); - } + var proxyUptimes = AdminController.getUptimes(proxies); - int containerLimit = environment.getProperty("proxy.container-quantity-limit", Integer.class, 2); + int containerLimit = proxyProperties.getContainerQuantityLimit(); map.put("withFileBrowser", proxyService.findProxy(p -> p.getSpec().getId().equals("filebrowser"), false) != null); map.put("containerLimit", containerLimit); map.put("proxies", proxies); diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/controllers/FileBrowserController.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/controllers/FileBrowserController.java index 92cb25df..86eb178d 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/controllers/FileBrowserController.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/controllers/FileBrowserController.java @@ -1,18 +1,16 @@ package hk.edu.polyu.comp.vlabcontroller.controllers; import hk.edu.polyu.comp.vlabcontroller.VLabControllerException; -import hk.edu.polyu.comp.vlabcontroller.auth.IAuthenticationBackend; +import hk.edu.polyu.comp.vlabcontroller.config.ProxyProperties; import hk.edu.polyu.comp.vlabcontroller.model.runtime.Proxy; import hk.edu.polyu.comp.vlabcontroller.model.runtime.ProxyStatus; import hk.edu.polyu.comp.vlabcontroller.model.spec.ProxySpec; -import hk.edu.polyu.comp.vlabcontroller.service.ProxyService; -import hk.edu.polyu.comp.vlabcontroller.service.UserService; -import hk.edu.polyu.comp.vlabcontroller.spec.FileBrowserProperties; +import hk.edu.polyu.comp.vlabcontroller.util.DurationUtil; import hk.edu.polyu.comp.vlabcontroller.util.Retrying; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.context.annotation.Lazy; -import org.springframework.core.env.Environment; import org.springframework.stereotype.Controller; import org.springframework.ui.ModelMap; import org.springframework.web.bind.annotation.RequestMapping; @@ -21,28 +19,24 @@ import org.springframework.web.servlet.support.ServletUriComponentsBuilder; import javax.servlet.http.HttpServletRequest; +import java.time.Duration; import java.util.HashMap; import java.util.Map; import java.util.regex.Pattern; +@Slf4j @Controller +@RequiredArgsConstructor(onConstructor_ = {@Lazy}) +@RefreshScope public class FileBrowserController extends BaseController { - - private final Logger log = LogManager.getLogger(FileBrowserController.class); - - private final FileBrowserProperties fileBrowserProperties; - - protected FileBrowserController(ProxyService proxyService, UserService userService, Environment environment, @Lazy IAuthenticationBackend authenticationBackend, FileBrowserProperties fileBrowserProperties) { - super(proxyService, userService, environment, authenticationBackend); - this.fileBrowserProperties = fileBrowserProperties; - } - + private final ProxyProperties proxyProperties; + private final Retrying retrying; @RequestMapping(value = "/filebrowser/**") public String fileBrowser(ModelMap map, HttpServletRequest request) { prepareMap(map, request); - String id = "filebrowser"; - Proxy proxy = proxyService.findProxy(p -> p.getSpec().getId().equals(id) && userService.isOwner(p), false); + var id = "filebrowser"; + var proxy = proxyService.findProxy(p -> p.getSpec().getId().equals(id) && userService.isOwner(p), false); awaitReady(proxy); map.put("appTitle", "File Browser"); map.put("container", (proxy == null) ? "" : buildContainerPath(request)); @@ -52,25 +46,26 @@ public String fileBrowser(ModelMap map, HttpServletRequest request) { @RequestMapping(value = "/filebrowser/**", method = RequestMethod.POST) @ResponseBody public Map startFileBrowser(HttpServletRequest request) { - String id = "filebrowser"; - Proxy proxy = proxyService.findProxy(p -> p.getSpec().getId().equals(id) && userService.isOwner(p), false); + var id = "filebrowser"; + var proxy = proxyService.findProxy(p -> p.getSpec().getId().equals(id) && userService.isOwner(p), false); if (proxy == null) { - if (fileBrowserProperties != null) { - ProxySpec spec = fileBrowserSpecTranslate(fileBrowserProperties); - ProxySpec resolvedSpec = proxyService.resolveProxySpec(spec, null, null); + var fileBrowser = proxyProperties.getFileBrowser(); + if (fileBrowser != null) { + var spec = fileBrowserSpecTranslate(fileBrowser); + var resolvedSpec = proxyService.resolveProxySpec(spec, null, null); try { proxy = proxyService.startProxy(resolvedSpec, false); } catch (VLabControllerException e) { - String errorMessage = "Failed to start file browser"; + var errorMessage = "Failed to start file browser"; log.error(errorMessage); - log.debug(e); + log.debug("error details: {}", e); Map response = new HashMap<>(); response.put("error_code", "404"); response.put("error_message", errorMessage); return response; } } else { - String errorMessage = "Missing file browser spec"; + var errorMessage = "Missing file browser spec"; log.error(errorMessage); Map response = new HashMap<>(); response.put("error_code", "404"); @@ -79,43 +74,38 @@ public Map startFileBrowser(HttpServletRequest request) { } } awaitReady(proxy); - String containerPath = buildContainerPath(request); + var containerPath = buildContainerPath(request); Map response = new HashMap<>(); response.put("containerPath", containerPath); return response; } - private ProxySpec fileBrowserSpecTranslate(FileBrowserProperties fbp) { - ProxySpec spec = new ProxySpec(); - fbp.copy(spec); - spec.setId("filebrowser"); - spec.setDisplayName("File Browser"); - return spec; + private ProxySpec fileBrowserSpecTranslate(ProxySpec fbp) { + return fbp.copyBuilder().id("filebrowser").displayName("File Browser").build(); } private boolean awaitReady(Proxy proxy) { - return awaitReadyHelper(proxy, environment.getProperty("proxy.container-wait-time", "20000")); + return awaitReadyHelper(proxy, proxyProperties.getContainerWaitTime(), retrying); } - static boolean awaitReadyHelper(Proxy proxy, String property) { + static boolean awaitReadyHelper(Proxy proxy, Duration delay, Retrying retrying) { if (proxy == null) return false; if (proxy.getStatus() == ProxyStatus.Up) return true; if (proxy.getStatus() == ProxyStatus.Stopping || proxy.getStatus() == ProxyStatus.Stopped) return false; - int totalWaitMs = Integer.parseInt(property); - int waitMs = Math.min(500, totalWaitMs); - int maxTries = totalWaitMs / waitMs; - Retrying.retry(i -> proxy.getStatus() != ProxyStatus.Starting, maxTries, waitMs); + var waitMs = DurationUtil.atLeast(Duration.ofMillis(500)).apply(delay); + var maxTries = (int) delay.dividedBy(waitMs); + retrying.retry(i -> proxy.getStatus() != ProxyStatus.Starting, maxTries, waitMs); return (proxy.getStatus() == ProxyStatus.Up); } private String buildContainerPath(HttpServletRequest request) { - String queryString = ServletUriComponentsBuilder.fromRequest(request).replaceQueryParam("sp_hide_navbar").build().getQuery(); + var queryString = ServletUriComponentsBuilder.fromRequest(request).replaceQueryParam("sp_hide_navbar").build().getQuery(); queryString = (queryString == null) ? "" : "?" + queryString; var containerPathPattern = Pattern.compile(".*?/filebrowser[/]*(.*)"); var matcher = containerPathPattern.matcher(request.getRequestURI()); - String containerPath = matcher.find() ? matcher.group(1) + queryString : queryString; + var containerPath = matcher.find() ? matcher.group(1) + queryString : queryString; return getContextPath() + "app_direct/filebrowser" + "/" + containerPath; } } \ No newline at end of file diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/controllers/IndexController.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/controllers/IndexController.java index 4aeab871..5a21288a 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/controllers/IndexController.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/controllers/IndexController.java @@ -1,47 +1,31 @@ package hk.edu.polyu.comp.vlabcontroller.controllers; -import hk.edu.polyu.comp.vlabcontroller.auth.IAuthenticationBackend; import hk.edu.polyu.comp.vlabcontroller.model.spec.ProxySpec; -import hk.edu.polyu.comp.vlabcontroller.service.ProxyService; -import hk.edu.polyu.comp.vlabcontroller.service.UserService; -import org.springframework.context.annotation.Lazy; -import org.springframework.core.env.Environment; import org.springframework.stereotype.Controller; import org.springframework.ui.ModelMap; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.servlet.view.RedirectView; import javax.servlet.http.HttpServletRequest; -import java.util.HashMap; import java.util.Map; +import java.util.stream.Collectors; @Controller public class IndexController extends BaseController { - protected IndexController(ProxyService proxyService, UserService userService, Environment environment, @Lazy IAuthenticationBackend authenticationBackend) { - super(proxyService, userService, environment, authenticationBackend); - } - @RequestMapping("/") private Object index(ModelMap map, HttpServletRequest request) { - String landingPage = environment.getProperty("proxy.landing-page", "/"); + var landingPage = proxyProperties.getLandingPage(); if (!landingPage.equals("/")) return new RedirectView(landingPage); - prepareMap(map, request); - - ProxySpec[] apps = proxyService.getProxySpecs(null, false).toArray(new ProxySpec[0]); - map.put("apps", apps); - - Map appLogos = new HashMap<>(); - map.put("appLogos", appLogos); - - boolean displayAppLogos = false; - for (ProxySpec app : apps) { - if (app.getLogoURL() != null) { - displayAppLogos = true; - appLogos.put(app, resolveImageURI(app.getLogoURL())); - } - } - map.put("displayAppLogos", displayAppLogos); + var apps = proxyService.getProxySpecs(null, false); + var appLogos = apps.stream() + .filter(x -> x.getLogoURL() != null) + .collect(Collectors.toMap(x -> x, x -> resolveImageURI(x.getLogoURL()))); + map.putAll(Map.ofEntries( + Map.entry("apps", apps.toArray(ProxySpec[]::new)), + Map.entry("appLogos", appLogos), + Map.entry("displayAppLogos", !appLogos.isEmpty()) + )); return "index"; } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/converter/QuantityConverter.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/converter/QuantityConverter.java new file mode 100644 index 00000000..0a4303e6 --- /dev/null +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/converter/QuantityConverter.java @@ -0,0 +1,16 @@ +package hk.edu.polyu.comp.vlabcontroller.converter; + +import io.fabric8.kubernetes.api.model.Quantity; +import org.springframework.boot.context.properties.ConfigurationPropertiesBinding; +import org.springframework.core.convert.converter.Converter; +import org.springframework.stereotype.Component; + +@Component +@ConfigurationPropertiesBinding +public class QuantityConverter implements Converter { + + @Override + public Quantity convert(String from) { + return new Quantity(from); + } +} \ No newline at end of file diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/entity/LabInstance.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/entity/LabInstance.java new file mode 100644 index 00000000..8e6677a5 --- /dev/null +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/entity/LabInstance.java @@ -0,0 +1,23 @@ +package hk.edu.polyu.comp.vlabcontroller.entity; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.joda.time.DateTime; +import org.springframework.data.annotation.CreatedDate; +import org.springframework.data.annotation.Id; + +import java.util.HashSet; +import java.util.Set; + +@AllArgsConstructor +@NoArgsConstructor +@Builder(toBuilder = true) +@Data +public class LabInstance { + @Id private String id; + @CreatedDate private DateTime startedAt; + private DateTime completedAt; + @Builder.Default private Set progress = new HashSet<>(); +} diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/entity/SessionData.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/entity/SessionData.java new file mode 100644 index 00000000..19fc40be --- /dev/null +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/entity/SessionData.java @@ -0,0 +1,18 @@ +package hk.edu.polyu.comp.vlabcontroller.entity; + +import com.querydsl.core.annotations.QueryEntity; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.joda.time.DateTime; + +@AllArgsConstructor +@NoArgsConstructor +@Data +@Builder(toBuilder = true) +@QueryEntity +public class SessionData { + private DateTime loggedInAt; + private DateTime loggedOutAt; +} diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/entity/User.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/entity/User.java new file mode 100644 index 00000000..b1987766 --- /dev/null +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/entity/User.java @@ -0,0 +1,25 @@ +package hk.edu.polyu.comp.vlabcontroller.entity; + +import com.querydsl.core.annotations.QueryEntity; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.springframework.data.annotation.Id; +import org.springframework.data.mongodb.core.mapping.Document; + +import java.util.HashMap; +import java.util.LinkedList; +import java.util.Map; + +@AllArgsConstructor +@NoArgsConstructor +@Data +@Builder(toBuilder = true) +@QueryEntity +@Document +public class User { + @Id private String id; + @Builder.Default private LinkedList labs = new LinkedList<>(); + @Builder.Default private Map session = new HashMap<>(); +} diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/event/AuthFailedEvent.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/event/AuthFailedEvent.java index dd2a3b9c..f2ca23a9 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/event/AuthFailedEvent.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/event/AuthFailedEvent.java @@ -1,14 +1,19 @@ package hk.edu.polyu.comp.vlabcontroller.event; +import lombok.Builder; +import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.ToString; import org.springframework.context.ApplicationEvent; +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) public class AuthFailedEvent extends ApplicationEvent { - @Getter private final String userId; - @Getter private final String sessionId; + @Builder public AuthFailedEvent(Object source, String userId, String sessionId) { super(source); this.userId = userId; diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/event/ProxyStartEvent.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/event/ProxyStartEvent.java index cdcfa2f0..16abb9ef 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/event/ProxyStartEvent.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/event/ProxyStartEvent.java @@ -1,20 +1,26 @@ package hk.edu.polyu.comp.vlabcontroller.event; +import lombok.Builder; +import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.ToString; import org.springframework.context.ApplicationEvent; import java.time.Duration; +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) public class ProxyStartEvent extends ApplicationEvent { - @Getter + private final String proxyId; private final String userId; - @Getter private final String specId; - @Getter private final Duration startupTime; - public ProxyStartEvent(Object source, String userId, String specId, Duration startupTime) { + @Builder + public ProxyStartEvent(Object source, String proxyId, String userId, String specId, Duration startupTime) { super(source); + this.proxyId = proxyId; this.userId = userId; this.specId = specId; this.startupTime = startupTime; diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/event/ProxyStartFailedEvent.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/event/ProxyStartFailedEvent.java index 2a245da2..0acf874f 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/event/ProxyStartFailedEvent.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/event/ProxyStartFailedEvent.java @@ -1,14 +1,19 @@ package hk.edu.polyu.comp.vlabcontroller.event; +import lombok.Builder; +import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.ToString; import org.springframework.context.ApplicationEvent; +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) public class ProxyStartFailedEvent extends ApplicationEvent { - @Getter private final String userId; - @Getter private final String specId; + @Builder public ProxyStartFailedEvent(Object source, String userId, String specId) { super(source); this.userId = userId; diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/event/ProxyStopEvent.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/event/ProxyStopEvent.java index b855c123..c8484e5c 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/event/ProxyStopEvent.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/event/ProxyStopEvent.java @@ -1,20 +1,26 @@ package hk.edu.polyu.comp.vlabcontroller.event; +import lombok.Builder; +import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.ToString; import org.springframework.context.ApplicationEvent; import java.time.Duration; +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) public class ProxyStopEvent extends ApplicationEvent { - @Getter + private final String proxyId; private final String userId; - @Getter private final String specId; - @Getter private final Duration usageTime; - public ProxyStopEvent(Object source, String userId, String specId, Duration usageTime) { + @Builder + public ProxyStopEvent(Object source, String proxyId, String userId, String specId, Duration usageTime) { super(source); + this.proxyId = proxyId; this.userId = userId; this.specId = specId; this.usageTime = usageTime; diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/event/UserLoginEvent.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/event/UserLoginEvent.java index 3dc7732b..237f96f2 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/event/UserLoginEvent.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/event/UserLoginEvent.java @@ -1,14 +1,19 @@ package hk.edu.polyu.comp.vlabcontroller.event; +import lombok.Builder; +import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.ToString; import org.springframework.context.ApplicationEvent; +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) public class UserLoginEvent extends ApplicationEvent { - @Getter private final String userId; - @Getter private final String sessionId; + @Builder public UserLoginEvent(Object source, String userId, String sessionId) { super(source); this.userId = userId; diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/event/UserLogoutEvent.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/event/UserLogoutEvent.java index d7e118f6..32bd0b4c 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/event/UserLogoutEvent.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/event/UserLogoutEvent.java @@ -1,14 +1,17 @@ package hk.edu.polyu.comp.vlabcontroller.event; +import lombok.Builder; +import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.ToString; import org.springframework.context.ApplicationEvent; +@Getter +@ToString +@EqualsAndHashCode(callSuper = false) public class UserLogoutEvent extends ApplicationEvent { - @Getter private final String userId; - @Getter private final String sessionId; - @Getter private final Boolean wasExpired; /** @@ -17,6 +20,7 @@ public class UserLogoutEvent extends ApplicationEvent { * @param sessionId * @param wasExpired whether the user is logged autoamtically because the session has expired */ + @Builder public UserLogoutEvent(Object source, String userId, String sessionId, Boolean wasExpired) { super(source); this.userId = userId; diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/log/AbstractLogStorage.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/log/AbstractLogStorage.java index dad42119..d7d9d77a 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/log/AbstractLogStorage.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/log/AbstractLogStorage.java @@ -1,40 +1,43 @@ package hk.edu.polyu.comp.vlabcontroller.log; +import hk.edu.polyu.comp.vlabcontroller.config.ProxyProperties; import hk.edu.polyu.comp.vlabcontroller.model.runtime.Proxy; -import org.springframework.core.env.Environment; +import lombok.Setter; +import org.springframework.cloud.context.config.annotation.RefreshScope; import javax.inject.Inject; import java.io.IOException; import java.text.SimpleDateFormat; import java.util.Date; +@RefreshScope public abstract class AbstractLogStorage implements ILogStorage { private static final String PARAM_LOG_PATHS = "log_paths"; - @Inject - protected Environment environment; + @Setter(onMethod_ = {@Inject}) + protected ProxyProperties proxyProperties; protected String containerLogPath; @Override public void initialize() throws IOException { - containerLogPath = environment.getProperty("proxy.container-log-path"); + containerLogPath = proxyProperties.getContainerLogPath(); } @Override public String getStorageLocation() { - return containerLogPath; + return proxyProperties.getContainerLogPath(); } @Override public String[] getLogs(Proxy proxy) throws IOException { - String[] paths = (String[]) proxy.getContainerGroup().getParameters().get(PARAM_LOG_PATHS); + var paths = (String[]) proxy.getContainerGroup().getParameters().get(PARAM_LOG_PATHS); if (paths == null) { - String timestamp = new SimpleDateFormat("yyyyMMdd").format(new Date()); + var timestamp = new SimpleDateFormat("yyyyMMdd").format(new Date()); paths = new String[]{ - String.format("%s/%s_%s_%s_stdout.log", containerLogPath, proxy.getSpec().getId(), proxy.getId(), timestamp), - String.format("%s/%s_%s_%s_stderr.log", containerLogPath, proxy.getSpec().getId(), proxy.getId(), timestamp) + String.format("%s/%s_%s_%s_stdout.log", getStorageLocation(), proxy.getSpec().getId(), proxy.getId(), timestamp), + String.format("%s/%s_%s_%s_stderr.log", getStorageLocation(), proxy.getSpec().getId(), proxy.getId(), timestamp) }; proxy.getContainerGroup().getParameters().put(PARAM_LOG_PATHS, paths); } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/log/FileLogStorage.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/log/FileLogStorage.java index 6aaa46b8..fe7bde01 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/log/FileLogStorage.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/log/FileLogStorage.java @@ -9,7 +9,7 @@ import java.nio.file.Paths; import java.util.Arrays; -import static com.pivovarit.function.ThrowingFunction.unchecked; +import static io.vavr.API.unchecked; public class FileLogStorage extends AbstractLogStorage { @@ -21,7 +21,7 @@ public void initialize() throws IOException { @Override public OutputStream[] createOutputStreams(Proxy proxy) throws IOException { - return Arrays.stream(getLogs(proxy)).map(unchecked(FileOutputStream::new)).toArray(OutputStream[]::new); + return Arrays.stream(getLogs(proxy)).map(unchecked(x -> new FileOutputStream(x))).toArray(OutputStream[]::new); } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/log/LogStorageFactory.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/log/LogStorageFactory.java index 564488ec..93c9ade4 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/log/LogStorageFactory.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/log/LogStorageFactory.java @@ -1,22 +1,21 @@ package hk.edu.polyu.comp.vlabcontroller.log; +import hk.edu.polyu.comp.vlabcontroller.config.ProxyProperties; +import lombok.RequiredArgsConstructor; import org.springframework.beans.factory.config.AbstractFactoryBean; +import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Primary; -import org.springframework.core.env.Environment; import org.springframework.stereotype.Service; @Service(value = "logStorage") @Primary +@RequiredArgsConstructor +@RefreshScope public class LogStorageFactory extends AbstractFactoryBean { - private final Environment environment; + private final ProxyProperties proxyProperties; private final ApplicationContext applicationContext; - public LogStorageFactory(Environment environment, ApplicationContext applicationContext) { - this.environment = environment; - this.applicationContext = applicationContext; - } - @Override public Class getObjectType() { return ILogStorage.class; @@ -24,10 +23,10 @@ public Class getObjectType() { @Override protected ILogStorage createInstance() throws Exception { - ILogStorage storage = null; + ILogStorage storage; - String containerLogPath = environment.getProperty("proxy.container-log-path"); - if (containerLogPath == null || containerLogPath.trim().isEmpty()) { + var containerLogPath = proxyProperties.getContainerLogPath(); + if (containerLogPath == null || containerLogPath.isBlank()) { storage = new NoopLogStorage(); } else if (containerLogPath.toLowerCase().startsWith("s3://")) { storage = new S3LogStorage(); diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/log/S3LogStorage.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/log/S3LogStorage.java index 5bee157c..5813f941 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/log/S3LogStorage.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/log/S3LogStorage.java @@ -7,20 +7,17 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.S3Object; import com.amazonaws.services.s3.transfer.TransferManager; import com.amazonaws.services.s3.transfer.TransferManagerBuilder; import hk.edu.polyu.comp.vlabcontroller.model.runtime.Proxy; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import lombok.extern.slf4j.Slf4j; import org.bouncycastle.util.Arrays; import java.io.*; //TODO Optimize flushing behaviour +@Slf4j public class S3LogStorage extends AbstractLogStorage { - - private final Logger log = LogManager.getLogger(S3LogStorage.class); private AmazonS3 s3; private TransferManager transferMgr; private String bucketName; @@ -31,15 +28,15 @@ public class S3LogStorage extends AbstractLogStorage { public void initialize() throws IOException { super.initialize(); - String accessKey = environment.getProperty("proxy.container-log-s3-access-key"); - String accessSecret = environment.getProperty("proxy.container-log-s3-access-secret"); - String endpoint = environment.getProperty("proxy.container-log-s3-endpoint", "https://s3-eu-west-1.amazonaws.com"); - enableSSE = Boolean.valueOf(environment.getProperty("proxy.container-log-s3-sse", "false")); + var accessKey = proxyProperties.getContainerLogS3AccessKey(); + var accessSecret = proxyProperties.getContainerLogS3AccessSecret(); + var endpoint = proxyProperties.getContainerLogS3Endpoint(); + enableSSE = proxyProperties.isContainerLogS3SSE(); - String subPath = containerLogPath.substring("s3://".length()).trim(); + var subPath = containerLogPath.substring("s3://".length()).trim(); if (subPath.endsWith("/")) subPath = subPath.substring(0, subPath.length() - 1); - int bucketPathIndex = subPath.indexOf("/"); + var bucketPathIndex = subPath.indexOf("/"); if (bucketPathIndex == -1) { bucketName = subPath; bucketPath = ""; @@ -60,10 +57,10 @@ public void initialize() throws IOException { @Override public OutputStream[] createOutputStreams(Proxy proxy) throws IOException { - String[] paths = getLogs(proxy); - OutputStream[] streams = new OutputStream[2]; - for (int i = 0; i < streams.length; i++) { - String fileName = paths[i].substring(paths[i].lastIndexOf("/") + 1); + var paths = getLogs(proxy); + var streams = new OutputStream[2]; + for (var i = 0; i < streams.length; i++) { + var fileName = paths[i].substring(paths[i].lastIndexOf("/") + 1); // TODO kubernetes never flushes. So perform timed flushes, and also flush upon container shutdown streams[i] = new BufferedOutputStream(new S3OutputStream(bucketPath + fileName), 1024 * 1024); } @@ -71,15 +68,15 @@ public OutputStream[] createOutputStreams(Proxy proxy) throws IOException { } private void doUpload(String key, byte[] bytes) throws IOException { - byte[] bytesToUpload = bytes; + var bytesToUpload = bytes; - byte[] originalBytes = getContent(key); + var originalBytes = getContent(key); if (originalBytes != null) { bytesToUpload = Arrays.copyOf(originalBytes, originalBytes.length + bytes.length); System.arraycopy(bytes, 0, bytesToUpload, originalBytes.length, bytes.length); } - ObjectMetadata metadata = new ObjectMetadata(); + var metadata = new ObjectMetadata(); metadata.setContentLength(bytesToUpload.length); if (enableSSE) metadata.setSSEAlgorithm(ObjectMetadata.AES_256_SERVER_SIDE_ENCRYPTION); @@ -96,11 +93,11 @@ private void doUpload(String key, byte[] bytes) throws IOException { private byte[] getContent(String key) throws IOException { if (s3.doesObjectExist(bucketName, key)) { - S3Object o = s3.getObject(bucketName, key); - ByteArrayOutputStream out = new ByteArrayOutputStream(); + var o = s3.getObject(bucketName, key); + var out = new ByteArrayOutputStream(); try (InputStream in = o.getObjectContent()) { - byte[] buffer = new byte[40 * 1024]; - int len = 0; + var buffer = new byte[40 * 1024]; + var len = 0; while ((len = in.read(buffer)) > 0) { out.write(buffer, 0, len); } @@ -122,13 +119,13 @@ public S3OutputStream(String s3Key) { @Override public void write(int b) throws IOException { // Warning: highly inefficient. Always write arrays. - byte[] bytesToCopy = new byte[]{(byte) b}; + var bytesToCopy = new byte[]{(byte) b}; write(bytesToCopy, 0, 1); } @Override public void write(byte[] b, int off, int len) throws IOException { - byte[] bytesToCopy = new byte[len]; + var bytesToCopy = new byte[len]; System.arraycopy(b, off, bytesToCopy, 0, len); doUpload(s3Key, bytesToCopy); } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/runtime/ContainerGroup.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/runtime/ContainerGroup.java index d86bdf2f..2ed8ec07 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/runtime/ContainerGroup.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/runtime/ContainerGroup.java @@ -2,20 +2,18 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import hk.edu.polyu.comp.vlabcontroller.model.spec.ContainerSpec; -import lombok.Getter; -import lombok.Setter; +import lombok.*; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +@Data @Builder(toBuilder = true) +@AllArgsConstructor +@NoArgsConstructor public class ContainerGroup { - @Getter @Setter private String id; - @Getter @Setter private List specs = new ArrayList<>(); - @Setter private Map parameters = new HashMap<>(); - - @JsonIgnore public Map getParameters() { - return parameters; - } + private String id; + @Singular private List specs = new ArrayList<>(); + @Getter(onMethod_ = {@JsonIgnore}) @Singular private Map parameters = new HashMap<>(); } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/runtime/HeartbeatStatus.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/runtime/HeartbeatStatus.java index d155823e..a95fc654 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/runtime/HeartbeatStatus.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/runtime/HeartbeatStatus.java @@ -1,22 +1,18 @@ package hk.edu.polyu.comp.vlabcontroller.model.runtime; -import lombok.Getter; -import lombok.Setter; +import lombok.*; +import java.time.Duration; + +@Data @Builder(toBuilder = true) @AllArgsConstructor public class HeartbeatStatus { - @Getter - private long startRecordTimestamp; - @Getter - @Setter - private long lastRecordTimestamp; - @Getter - @Setter + private Duration startRecordTimestamp; + private Duration lastRecordTimestamp; private int totalPayloadLength; - @Getter private int terminateCounter; public HeartbeatStatus() { - this.startRecordTimestamp = System.currentTimeMillis(); + this.startRecordTimestamp = Duration.ofMillis(System.currentTimeMillis()); this.lastRecordTimestamp = this.startRecordTimestamp; } @@ -25,7 +21,7 @@ public void increaseCounter() { } public void clearAll() { - startRecordTimestamp = System.currentTimeMillis(); + startRecordTimestamp = Duration.ofMillis(System.currentTimeMillis()); totalPayloadLength = 0; terminateCounter = 0; } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/runtime/PortMappingMetadata.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/runtime/PortMappingMetadata.java index cfbbe52f..1cba2d7c 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/runtime/PortMappingMetadata.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/runtime/PortMappingMetadata.java @@ -1,17 +1,14 @@ package hk.edu.polyu.comp.vlabcontroller.model.runtime; import io.undertow.server.handlers.proxy.LoadBalancingProxyClient; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; +import lombok.*; import java.net.URI; @ToString -@AllArgsConstructor +@Data @Builder(toBuilder = true) public class PortMappingMetadata { - @Getter @Setter private String portMapping; - @Getter @Setter private URI target; - @Getter @Setter private LoadBalancingProxyClient loadBalancingProxyClient; + private String portMapping; + private URI target; + private LoadBalancingProxyClient loadBalancingProxyClient; } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/runtime/Proxy.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/runtime/Proxy.java index fa55e11e..9285b6ab 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/runtime/Proxy.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/runtime/Proxy.java @@ -1,42 +1,26 @@ package hk.edu.polyu.comp.vlabcontroller.model.runtime; import hk.edu.polyu.comp.vlabcontroller.model.spec.ProxySpec; -import lombok.Getter; -import lombok.Setter; +import lombok.*; import java.net.URI; +import java.time.Duration; import java.util.HashMap; import java.util.Map; +@Data @Builder(toBuilder = true) +@AllArgsConstructor +@NoArgsConstructor public class Proxy { - @Getter - @Setter private String id; - @Getter - @Setter private ProxySpec spec; - @Getter - @Setter private ProxyStatus status; - @Getter - @Setter - private long startupTimestamp; - @Getter - @Setter - private long createdTimestamp; - @Getter - @Setter + private Duration startupTimestamp; + private Duration createdTimestamp; private String userId; - @Getter - @Setter private boolean admin; - @Getter - @Setter private String namespace; - @Getter - @Setter private ContainerGroup containerGroup; - @Getter - @Setter - private Map targets = new HashMap<>(); + @Builder.Default private Map metadata = new HashMap<>(); + @Singular private Map targets = new HashMap<>(); } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/runtime/ProxyMappingMetadata.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/runtime/ProxyMappingMetadata.java index 9a5318b9..321a0368 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/runtime/ProxyMappingMetadata.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/runtime/ProxyMappingMetadata.java @@ -1,18 +1,19 @@ package hk.edu.polyu.comp.vlabcontroller.model.runtime; -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; -import org.apache.commons.lang.StringUtils; +import lombok.*; +import org.apache.commons.lang3.StringUtils; import java.net.URI; import java.util.ArrayList; import java.util.List; @ToString +@Data @Builder(toBuilder = true) +@AllArgsConstructor +@NoArgsConstructor public class ProxyMappingMetadata { - @Getter private URI defaultTarget; - @Getter @Setter private List portMappingMetadataList = new ArrayList<>(); + @Setter(AccessLevel.NONE) private URI defaultTarget; + @Singular("portMappingMetadata") private List portMappingMetadataList = new ArrayList<>(); public void setDefaultTarget(URI defaultTarget) { // Can't be updated if set @@ -26,7 +27,7 @@ public boolean containsExactMappingPath(String path) { } public boolean containsMappingPathPrefix(String prefix) { - String path = StringUtils.removeEnd(prefix, "/"); + var path = StringUtils.removeEnd(prefix, "/"); return portMappingMetadataList.stream().anyMatch(p -> p.getPortMapping().startsWith(path)); } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/runtime/RuntimeSetting.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/runtime/RuntimeSetting.java index 0a5f4d31..206b6a43 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/runtime/RuntimeSetting.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/runtime/RuntimeSetting.java @@ -1,13 +1,10 @@ package hk.edu.polyu.comp.vlabcontroller.model.runtime; -import lombok.Getter; -import lombok.Setter; +import lombok.Builder; +import lombok.Data; +@Data @Builder(toBuilder = true) public class RuntimeSetting { - @Getter - @Setter private String name; - @Getter - @Setter private Object value; } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/spec/ContainerSpec.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/spec/ContainerSpec.java index 79476c0b..6201e3b2 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/spec/ContainerSpec.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/spec/ContainerSpec.java @@ -1,60 +1,38 @@ package hk.edu.polyu.comp.vlabcontroller.model.spec; import com.fasterxml.jackson.annotation.JsonIgnore; +import hk.edu.polyu.comp.vlabcontroller.config.ProxyProperties; import io.fabric8.kubernetes.api.model.VolumeMount; -import lombok.Getter; -import lombok.Setter; -import lombok.extern.log4j.Log4j2; +import lombok.*; +import lombok.experimental.SuperBuilder; +import lombok.extern.slf4j.Slf4j; import org.springframework.data.util.Pair; import java.util.*; import java.util.stream.Collectors; -@Log4j2 +@Slf4j +@Data +@SuperBuilder(toBuilder = true) +@NoArgsConstructor +@AllArgsConstructor public class ContainerSpec { - @Getter - @Setter private String image; - @Getter - @Setter - private List cmd = new ArrayList<>(); - @Getter - @Setter - private Map env = new HashMap<>(); - @Getter - @Setter + private String name; + @Singular("cmd") private List cmd = new ArrayList<>(); + @Singular("env") private Map env = new HashMap<>(); private String envFile; - @Getter - @Setter private String network; - @Getter - @Setter - private List networkConnections = new ArrayList<>(); - @Getter - @Setter - private List dns = new ArrayList<>(); - @Getter - @Setter - private List entryPoints = new ArrayList<>(); - @Getter - @Setter - private Map portMapping = new HashMap<>(); - @Getter - @Setter + @Singular private List networkConnections = new ArrayList<>(); + @Singular("dns") private List dns = new ArrayList<>(); + @Singular private List entryPoints = new ArrayList<>(); + @Singular("portMapping") private Map portMapping = new HashMap<>(); private boolean privileged; - @Getter - @Setter - private ResourceSpec resources = new ResourceSpec(); - private List volumeMount; - @Getter - @Setter - private List volumeMounts = new ArrayList<>(); - @Getter - @Setter - private List adminVolumeMounts = new ArrayList<>(); - @Getter - @Setter - private Map settings = new HashMap<>(); + @Builder.Default private ResourceSpec resources = new ResourceSpec(); + @Deprecated @Singular("DEPRECATED_volumeMount") private List volumeMount = new ArrayList<>(); + @Singular private List volumeMounts = new ArrayList<>(); + @Singular private List adminVolumeMounts = new ArrayList<>(); + @Singular private Map settings = new HashMap<>(); /** * RuntimeLabels are labels which are calculated at runtime and contain metadata about the proxy. @@ -65,8 +43,7 @@ public class ContainerSpec { * In practice, safe labels are saved as Kubernetes labels and non-safe labels are saved as * Kubernetes annotations. */ - @Setter - private Map> runtimeLabels = new HashMap<>(); + @Setter private Map> runtimeLabels = new HashMap<>(); @JsonIgnore public Map> getRuntimeLabels() { @@ -81,33 +58,27 @@ public void addRuntimeLabel(String key, Boolean safe, String value) { } } - @Deprecated(since="1.0.2", forRemoval = true) + @Deprecated(since = "1.0.2", forRemoval = true) public void setVolumeMount(List volumeMount) { log.warn("containerSpec[].volumeMount is deprecated in 1.0.2+, unavailable in 1.1+, use containerSpec[].volumeMounts instead"); setVolumeMounts(volumeMount); this.volumeMount = volumeMounts; } - @Deprecated(since="1.0.2", forRemoval = true) + @Deprecated(since = "1.0.2", forRemoval = true) public List getVolumeMount() { return volumeMount; } - public void copy(ContainerSpec target) { - target.setImage(image); - target.getCmd().addAll(cmd); - target.getEnv().putAll(env); - target.setEnvFile(envFile); - target.setNetwork(network); - target.getNetworkConnections().addAll(networkConnections); - target.getDns().addAll(dns); - target.getEntryPoints().addAll(entryPoints); - target.getPortMapping().putAll(portMapping); - target.getPortMapping().putAll(entryPoints.stream().collect(Collectors.toMap(x -> String.format("port_mappings/%d", x.getPort()), EntryPointSpec::getPort))); - target.setResources(resources); - target.setPrivileged(privileged); - target.getVolumeMounts().addAll(volumeMounts); - target.getAdminVolumeMounts().addAll(adminVolumeMounts); - target.getSettings().putAll(settings); + public void populatePublicPathById(String id) { + var map = getEnv().entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + map.put("PUBLIC_PATH", ProxyProperties.getPublicPath(id)); + setEnv(Collections.unmodifiableMap(map)); + } + + public ContainerSpec copy() { + return this.toBuilder() + .portMapping(entryPoints.stream().collect(Collectors.toMap(x -> String.format("port_mappings/%d", x.getPort()), EntryPointSpec::getPort))) + .build(); } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/spec/EntryPointSpec.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/spec/EntryPointSpec.java index aa942685..b647f5d9 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/spec/EntryPointSpec.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/spec/EntryPointSpec.java @@ -1,28 +1,19 @@ package hk.edu.polyu.comp.vlabcontroller.model.spec; -import lombok.Getter; -import lombok.Setter; +import lombok.*; import java.util.HashMap; import java.util.Map; +@Data +@Builder(toBuilder = true) +@AllArgsConstructor +@NoArgsConstructor(access = AccessLevel.PRIVATE) public class EntryPointSpec { - @Getter - @Setter private String displayName; - @Getter - @Setter private String description; - @Getter - @Setter private int port; - @Getter - @Setter - private String path = ""; - @Getter - @Setter - private boolean disableSubdomain = false; - @Getter - @Setter - private Map parameters = new HashMap<>(); + @Builder.Default private String path = ""; + private boolean disableSubdomain; + @Singular private Map parameters = new HashMap<>(); } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/spec/EvaluatorSpec.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/spec/EvaluatorSpec.java new file mode 100644 index 00000000..7a843280 --- /dev/null +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/spec/EvaluatorSpec.java @@ -0,0 +1,19 @@ +package hk.edu.polyu.comp.vlabcontroller.model.spec; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.Singular; +import lombok.experimental.SuperBuilder; + +import java.util.ArrayList; +import java.util.List; + +@Data +@SuperBuilder(toBuilder = true) +@NoArgsConstructor +@AllArgsConstructor +public class EvaluatorSpec extends ContainerSpec { + @Singular + private List goals = new ArrayList<>(); +} diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/spec/ProxySpec.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/spec/ProxySpec.java index 6fdce6bb..7f43181b 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/spec/ProxySpec.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/spec/ProxySpec.java @@ -1,8 +1,6 @@ package hk.edu.polyu.comp.vlabcontroller.model.spec; -import hk.edu.polyu.comp.vlabcontroller.spec.impl.DefaultSpecProvider; -import lombok.Getter; -import lombok.Setter; +import lombok.*; import java.util.ArrayList; import java.util.HashMap; @@ -10,72 +8,81 @@ import java.util.Map; import java.util.stream.Collectors; +@Data +@Builder(toBuilder = true) +@NoArgsConstructor +@AllArgsConstructor public class ProxySpec { - @Getter - @Setter private String id; - @Getter - @Setter private String displayName; - @Getter - @Setter private String description; - @Getter - @Setter private String logoURL; - @Getter - @Setter - private List accessGroups = new ArrayList<>(); - @Getter - private List containerSpecs = new ArrayList<>(); - @Getter - @Setter - private List runtimeSettingSpecs = new ArrayList<>(); - @Getter - @Setter - private Map labels = new HashMap<>(); - @Getter - private Map settings = new HashMap<>(); - @Getter - @Setter - private ProxySpecKubernetes kubernetes = new ProxySpecKubernetes(); - @Getter - @Setter + @Singular private List tags = new ArrayList<>(); + @Singular private List accessGroups = new ArrayList<>(); + @Singular private List containerSpecs = new ArrayList<>(); + @Singular private List runtimeSettingSpecs = new ArrayList<>(); + @Singular private Map labels = new HashMap<>(); + @Setter(AccessLevel.PACKAGE) @Singular private Map settings = new HashMap<>(); + @Builder.Default private ProxySpecKubernetes kubernetes = new ProxySpecKubernetes(); + private boolean isSecure; private String defaultTutorialLink; + private EvaluatorSpec evaluator; public void setContainerSpecs(List containerSpecs) { this.containerSpecs = containerSpecs; - var entryPoints = containerSpecs.stream().flatMap(x -> x.getEntryPoints().stream()).collect(Collectors.toList()); + var entryPoints = containerSpecs.stream().filter(x -> x.getEntryPoints() != null).flatMap(x -> x.getEntryPoints().stream()).collect(Collectors.toList()); settings.put("entrypoint", entryPoints); } - public void copy(ProxySpec target) { - target.setId(id); - target.setDisplayName(displayName); - target.setDescription(description); - target.setLogoURL(logoURL); - target.setDefaultTutorialLink(defaultTutorialLink); - - target.getAccessGroups().addAll(accessGroups); + public void populateContainerSpecPublicPathById() { + containerSpecs.forEach(x -> x.populatePublicPathById(id)); + } - for (ContainerSpec spec : containerSpecs) { - ContainerSpec copy = new ContainerSpec(); - spec.copy(copy); - copy.getEnv().put("PUBLIC_PATH", DefaultSpecProvider.getPublicPath(id)); - target.getContainerSpecs().add(copy); - } + public ProxySpecBuilder copyToBuilder(ProxySpecBuilder builder) { + var self = this.copy(); + self.kubernetes = self.kubernetes.copy(); - for (RuntimeSettingSpec spec : runtimeSettingSpecs) { - RuntimeSettingSpec copy = new RuntimeSettingSpec(); - spec.copy(copy); - target.getRuntimeSettingSpecs().add(copy); - } + return builder + .clearContainerSpecs() + .clearRuntimeSettingSpecs() + .id(id) + .displayName(displayName) + .description(description) + .logoURL(logoURL) + .accessGroups(accessGroups) + .containerSpecs( + self.containerSpecs.stream() + .peek(x -> x.populatePublicPathById(builder.id)) + .collect(Collectors.toList()) + ) +// .runtimeSettingSpecs(self.runtimeSettingSpecs) + .labels(labels) + .settings(settings) + .kubernetes(self.kubernetes) + .defaultTutorialLink(defaultTutorialLink) + .tags(tags) + ; + } - target.getLabels().putAll(labels); - target.getSettings().putAll(settings); - ProxySpecKubernetes proxySpecKubernetesCopy = new ProxySpecKubernetes(); - kubernetes.copy(proxySpecKubernetesCopy); - target.setKubernetes(proxySpecKubernetesCopy); + public ProxySpecBuilder copyBuilder() { + return this.toBuilder() + .clearContainerSpecs() + .clearRuntimeSettingSpecs() + .containerSpecs( + containerSpecs.stream() + .map(ContainerSpec::copy) + .peek(x -> x.populatePublicPathById(id)) + .collect(Collectors.toList()) + ) +// .runtimeSettingSpecs( +// runtimeSettingSpecs.stream() +// .map(RuntimeSettingSpec::copy) +// .collect(Collectors.toList()) +// ) + .kubernetes(kubernetes.copy()); } + public ProxySpec copy() { + return copyBuilder().build(); + } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/spec/ProxySpecKubernetes.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/spec/ProxySpecKubernetes.java index f0f74ae5..ed1f9050 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/spec/ProxySpecKubernetes.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/spec/ProxySpecKubernetes.java @@ -2,30 +2,32 @@ import io.fabric8.kubernetes.api.model.PersistentVolumeClaim; import io.fabric8.kubernetes.api.model.Volume; -import lombok.Getter; -import lombok.Setter; +import lombok.*; import java.util.ArrayList; import java.util.List; +@Data @Builder(toBuilder = true) +@AllArgsConstructor @NoArgsConstructor public class ProxySpecKubernetes { - @Getter - @Setter - private List volumes = new ArrayList<>(); - @Getter - @Setter + @Singular private List volumes = new ArrayList<>(); private String podPatches; - @Getter - @Setter - private List additionalManifests = new ArrayList<>(); - @Getter - @Setter - private List persistentVolumeClaims = new ArrayList<>(); + @Singular private List additionalManifests = new ArrayList<>(); + @Singular private List persistentVolumeClaims = new ArrayList<>(); - public void copy(ProxySpecKubernetes target){ - target.getVolumes().addAll(volumes); - target.setPodPatches(podPatches); - target.getAdditionalManifests().addAll(additionalManifests); - target.getPersistentVolumeClaims().addAll(persistentVolumeClaims); + public ProxySpecKubernetesBuilder copyToBuilder(ProxySpecKubernetesBuilder builder) { + return builder + .volumes(volumes) + .podPatches(podPatches) + .additionalManifests(additionalManifests) + .persistentVolumeClaims(persistentVolumeClaims); + } + + public ProxySpecKubernetesBuilder copyBuilder() { + return this.toBuilder(); + } + + public ProxySpecKubernetes copy() { + return copyBuilder().build(); } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/spec/ResourceSpec.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/spec/ResourceSpec.java index cf9139a5..7000697c 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/spec/ResourceSpec.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/spec/ResourceSpec.java @@ -1,13 +1,25 @@ package hk.edu.polyu.comp.vlabcontroller.model.spec; -import lombok.Getter; -import lombok.Setter; +import io.fabric8.kubernetes.api.model.Quantity; +import io.fabric8.kubernetes.api.model.ResourceRequirements; +import io.fabric8.kubernetes.api.model.ResourceRequirementsBuilder; +import lombok.*; import java.util.HashMap; import java.util.Map; +@Data +@Builder(toBuilder = true) +@AllArgsConstructor +@NoArgsConstructor public class ResourceSpec { - @Getter @Setter private Map limits = new HashMap<>(); - @Getter @Setter private Map requests = new HashMap<>(); + @Singular Map limits = new HashMap<>(); + @Singular Map requests = new HashMap<>(); + public ResourceRequirements asResourceRequirements() { + return new ResourceRequirementsBuilder() + .addToRequests(getRequests()) + .addToLimits(getLimits()) + .build(); + } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/spec/RuntimeSettingSpec.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/spec/RuntimeSettingSpec.java index 2d48a89a..7e8c5b07 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/spec/RuntimeSettingSpec.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/model/spec/RuntimeSettingSpec.java @@ -1,28 +1,20 @@ package hk.edu.polyu.comp.vlabcontroller.model.spec; -import lombok.Getter; -import lombok.Setter; +import lombok.*; import java.util.HashMap; import java.util.Map; +@Data +@Builder(toBuilder = true) +@AllArgsConstructor +@NoArgsConstructor public class RuntimeSettingSpec { - @Getter - @Setter private String name; - @Getter - @Setter private String type; - @Getter - @Setter - private Map config; + @Singular("config") private Map config = new HashMap<>(); - public void copy(RuntimeSettingSpec target) { - target.setName(name); - target.setType(type); - if (config != null) { - if (target.getConfig() == null) target.setConfig(new HashMap<>()); - target.getConfig().putAll(config); - } + public RuntimeSettingSpec copy() { + return this.toBuilder().build(); } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/repository/UserRepository.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/repository/UserRepository.java new file mode 100644 index 00000000..af3e2304 --- /dev/null +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/repository/UserRepository.java @@ -0,0 +1,14 @@ +package hk.edu.polyu.comp.vlabcontroller.repository; + +import hk.edu.polyu.comp.vlabcontroller.entity.QUser; +import hk.edu.polyu.comp.vlabcontroller.entity.User; +import org.springframework.data.mongodb.repository.MongoRepository; +import org.springframework.data.querydsl.QuerydslPredicateExecutor; + +public interface UserRepository extends MongoRepository, QuerydslPredicateExecutor { + default User findUserByIdOrCreate(String uid) { + return this + .findOne(QUser.user.id.eq(uid)) + .orElse(User.builder().id(uid).build()); + } +} diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/security/APISecurityConfig.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/security/APISecurityConfig.java index 3264fa81..dde049c1 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/security/APISecurityConfig.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/security/APISecurityConfig.java @@ -1,10 +1,12 @@ package hk.edu.polyu.comp.vlabcontroller.security; +import hk.edu.polyu.comp.vlabcontroller.config.ProxyProperties; +import lombok.RequiredArgsConstructor; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.core.env.Environment; import org.springframework.security.authentication.UsernamePasswordAuthenticationToken; import org.springframework.security.config.annotation.web.builders.HttpSecurity; import org.springframework.security.core.Authentication; @@ -16,7 +18,7 @@ import org.springframework.security.oauth2.provider.token.store.JwtAccessTokenConverter; import org.springframework.security.oauth2.provider.token.store.jwk.JwkTokenStore; -import javax.inject.Inject; +import javax.servlet.http.Cookie; import javax.servlet.http.HttpServletRequest; import java.util.Arrays; import java.util.Map; @@ -24,13 +26,10 @@ @Configuration @ConditionalOnProperty(name = "proxy.oauth2.resource-id") @EnableResourceServer +@RequiredArgsConstructor +@RefreshScope public class APISecurityConfig extends ResourceServerConfigurerAdapter { - - private final Environment environment; - - public APISecurityConfig(Environment environment) { - this.environment = environment; - } + private final ProxyProperties proxyProperties; @Override public void configure(HttpSecurity http) throws Exception { @@ -41,20 +40,20 @@ public void configure(HttpSecurity http) throws Exception { public void configure(ResourceServerSecurityConfigurer resources) throws Exception { resources .tokenExtractor(new CookieTokenExtractor()) - .resourceId(environment.getProperty("proxy.oauth2.resource-id")); + .resourceId(proxyProperties.getOauth2().getResourceId()); } @Bean public JwtAccessTokenConverter jwtAccessTokenConverter() { - JwtAccessTokenConverter converter = new JwtAccessTokenConverter(); - DefaultAccessTokenConverter tokenConverter = new DefaultAccessTokenConverter(); + var converter = new JwtAccessTokenConverter(); + var tokenConverter = new DefaultAccessTokenConverter(); tokenConverter.setUserTokenConverter(new DefaultUserAuthenticationConverter() { @Override public Authentication extractAuthentication(Map map) { - Authentication auth = super.extractAuthentication(map); + var auth = super.extractAuthentication(map); if (auth == null) { // If 'user_name' is not available, use 'sub' instead. - String principal = String.valueOf(map.get("sub")); + var principal = String.valueOf(map.get("sub")); return new UsernamePasswordAuthenticationToken(principal, "N/A", null); } return auth; @@ -67,13 +66,13 @@ public Authentication extractAuthentication(Map map) { @Bean @ConditionalOnMissingBean(TokenStore.class) public TokenStore jwkTokenStore() { - return new JwkTokenStore(environment.getProperty("proxy.oauth2.jwks-url"), jwtAccessTokenConverter()); + return new JwkTokenStore(proxyProperties.getOauth2().getJwksUrl(), jwtAccessTokenConverter()); } @Bean @ConditionalOnMissingBean(ResourceServerTokenServices.class) public DefaultTokenServices jwkTokenServices(TokenStore jwkTokenStore) { - DefaultTokenServices services = new DefaultTokenServices(); + var services = new DefaultTokenServices(); services.setTokenStore(jwkTokenStore); return services; } @@ -85,11 +84,11 @@ public DefaultTokenServices jwkTokenServices(TokenStore jwkTokenStore) { private static class CookieTokenExtractor extends BearerTokenExtractor { @Override protected String extractToken(HttpServletRequest request) { - String token = super.extractToken(request); + var token = super.extractToken(request); if (token == null && request.getCookies() != null) { token = Arrays.stream(request.getCookies()) .filter(c -> c.getName().equals("access_token")).findAny() - .map(c -> c.getValue()).orElse(null); + .map(Cookie::getValue).orElse(null); } return token; } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/security/CustomFirewallSecurityConfig.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/security/CustomFirewallSecurityConfig.java index 6399bae4..5eb50cfa 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/security/CustomFirewallSecurityConfig.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/security/CustomFirewallSecurityConfig.java @@ -1,22 +1,19 @@ package hk.edu.polyu.comp.vlabcontroller.security; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Configuration; import org.springframework.security.config.annotation.web.builders.HttpSecurity; import org.springframework.security.config.annotation.web.builders.WebSecurity; import org.springframework.security.web.firewall.StrictHttpFirewall; +@Slf4j @Configuration public class CustomFirewallSecurityConfig implements ICustomSecurityConfig { - - private final Logger log = LogManager.getLogger(CustomFirewallSecurityConfig.class); - @Override public void apply(WebSecurity web) throws Exception { log.info("Enable customized firewall"); ICustomSecurityConfig.super.apply(web); - StrictHttpFirewall customStrictHttpFirewall = new StrictHttpFirewall(); + var customStrictHttpFirewall = new StrictHttpFirewall(); customStrictHttpFirewall.setAllowUrlEncodedDoubleSlash(true); web.httpFirewall(customStrictHttpFirewall); } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/security/KeycloakRoleSecurityConfig.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/security/KeycloakRoleSecurityConfig.java index eb5fae3e..16823be2 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/security/KeycloakRoleSecurityConfig.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/security/KeycloakRoleSecurityConfig.java @@ -1,24 +1,22 @@ package hk.edu.polyu.comp.vlabcontroller.security; import com.google.common.base.Strings; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import hk.edu.polyu.comp.vlabcontroller.config.ProxyProperties; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.context.annotation.Configuration; -import org.springframework.core.env.Environment; import org.springframework.security.config.annotation.web.builders.HttpSecurity; import org.springframework.security.config.annotation.web.builders.WebSecurity; +@Slf4j @Configuration @ConditionalOnProperty(name = "proxy.authentication", havingValue = "keycloak") +@RequiredArgsConstructor +@RefreshScope public class KeycloakRoleSecurityConfig implements ICustomSecurityConfig { - private final Logger log = LogManager.getLogger(getClass()); - - final Environment environment; - - public KeycloakRoleSecurityConfig(Environment environment) { - this.environment = environment; - } + private final ProxyProperties proxyProperties; @Override public void apply(WebSecurity web) throws Exception { @@ -28,8 +26,8 @@ public void apply(WebSecurity web) throws Exception { @Override public void apply(HttpSecurity http) throws Exception { ICustomSecurityConfig.super.apply(http); - String[] uriArray = new String[]{"/api/**", "/app/**", "/app_direct/**", "/filebrowser", "/controlpanel", environment.getProperty("proxy.landing-page")}; - String role = environment.getProperty("proxy.allowed-role"); + var uriArray = new String[]{"/api/**", "/app/**", "/app_direct/**", "/filebrowser", "/controlpanel", proxyProperties.getLandingPage()}; + var role = proxyProperties.getAllowedRole(); if (!Strings.isNullOrEmpty(role)) { log.info("Enable allowed roles mode"); http.authorizeRequests().antMatchers("/").anonymous(); diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/security/UISecurityConfig.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/security/UISecurityConfig.java index 96e1af9b..f78e52ff 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/security/UISecurityConfig.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/security/UISecurityConfig.java @@ -4,38 +4,32 @@ import hk.edu.polyu.comp.vlabcontroller.model.spec.ProxySpec; import hk.edu.polyu.comp.vlabcontroller.service.ProxyService; import hk.edu.polyu.comp.vlabcontroller.service.UserService; +import lombok.RequiredArgsConstructor; import org.springframework.security.config.annotation.web.builders.HttpSecurity; import org.springframework.stereotype.Component; -import java.util.List; +import java.util.stream.Collectors; @Component +@RequiredArgsConstructor public class UISecurityConfig implements ICustomSecurityConfig { - private final ProxyService proxyService; private final IAuthenticationBackend auth; private final UserService userService; - public UISecurityConfig(ProxyService proxyService, IAuthenticationBackend auth, UserService userService) { - this.proxyService = proxyService; - this.auth = auth; - this.userService = userService; - } - @Override public void apply(HttpSecurity http) throws Exception { if (auth.hasAuthorization()) { - // Limit access to the app pages according to spec permissions - for (ProxySpec spec : proxyService.getProxySpecs(null, true)) { - List groups = spec.getAccessGroups(); - if (groups.isEmpty()) continue; - String[] appGroups = groups.stream().map(String::toUpperCase).toArray(String[]::new); - http.authorizeRequests().antMatchers("/app/" + spec.getId()).hasAnyRole(appGroups); + var convertedMatches = proxyService.getProxySpecs(null, true).stream() + .filter(x -> !x.getAccessGroups().isEmpty()) + .collect(Collectors.toMap(ProxySpec::getId, x -> x.getAccessGroups().stream().map(String::toUpperCase))); + for (var entry : convertedMatches.entrySet()) { + http.authorizeRequests().antMatchers("/app/" + entry.getKey()).hasAnyRole(entry.getValue().toArray(String[]::new)); } // Limit access to the admin pages - http.authorizeRequests().antMatchers("/admin").hasAnyRole(userService.getAdminGroups()); + http.authorizeRequests().antMatchers("/admin").hasAnyRole(userService.getAdminGroups().toArray(String[]::new)); } } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/security/WebSecurityConfig.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/security/WebSecurityConfig.java index 0c6c950c..bccadca3 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/security/WebSecurityConfig.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/security/WebSecurityConfig.java @@ -2,11 +2,14 @@ import hk.edu.polyu.comp.vlabcontroller.auth.IAuthenticationBackend; import hk.edu.polyu.comp.vlabcontroller.auth.UserLogoutHandler; +import hk.edu.polyu.comp.vlabcontroller.config.ServerProperties; +import io.vavr.control.Option; +import lombok.RequiredArgsConstructor; +import lombok.Setter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.core.env.Environment; import org.springframework.security.authentication.AuthenticationEventPublisher; import org.springframework.security.authentication.AuthenticationManager; import org.springframework.security.config.annotation.authentication.builders.AuthenticationManagerBuilder; @@ -15,37 +18,29 @@ import org.springframework.security.config.annotation.web.builders.WebSecurity; import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; -import org.springframework.security.config.annotation.web.configurers.ExpressionUrlAuthorizationConfigurer; import org.springframework.security.web.authentication.www.BasicAuthenticationFilter; import org.springframework.security.web.header.writers.StaticHeadersWriter; import org.springframework.security.web.util.matcher.AntPathRequestMatcher; -import javax.inject.Inject; import java.util.List; +import static io.vavr.API.*; + @Configuration @EnableWebSecurity +@RequiredArgsConstructor public class WebSecurityConfig extends WebSecurityConfigurerAdapter { - - @Inject - private UserLogoutHandler logoutHandler; - - @Inject - private IAuthenticationBackend auth; - - @Inject - private AuthenticationEventPublisher eventPublisher; - - @Inject - private Environment environment; - - @Autowired(required = false) + private final UserLogoutHandler logoutHandler; + private final IAuthenticationBackend auth; + private final AuthenticationEventPublisher eventPublisher; + private final ServerProperties serverProperties; + @Setter(onMethod_ = {@Autowired(required = false)}) private List customConfigs; @Override public void configure(WebSecurity web) { if (customConfigs != null) { - for (ICustomSecurityConfig cfg : customConfigs) { + for (var cfg : customConfigs) { try { cfg.apply(web); } catch (Exception e) { @@ -64,26 +59,21 @@ protected void configure(HttpSecurity http) throws Exception { http.csrf().requireCsrfProtectionMatcher(new AntPathRequestMatcher("/login", "POST")); // Always set header: X-Content-Type-Options=nosniff - http.headers().contentTypeOptions(); - - String frameOptions = environment.getProperty("server.frameOptions", "disable"); - switch (frameOptions.toUpperCase()) { - case "DISABLE": - http.headers().frameOptions().disable(); - break; - case "DENY": - http.headers().frameOptions().deny(); - break; - case "SAMEORIGIN": - http.headers().frameOptions().sameOrigin(); - break; - default: - if (frameOptions.toUpperCase().startsWith("ALLOW-FROM")) { - http.headers() - .frameOptions().disable() - .addHeaderWriter(new StaticHeadersWriter("X-Frame-Options", frameOptions)); + var headers = http.headers(); + var frameOptionsConfig = headers.frameOptions(); + headers.contentTypeOptions(); + + var frameOptions = serverProperties.getFrameOptions(); + Match(frameOptions.toUpperCase()).of( + Case($("DISABLE"), () -> run(frameOptionsConfig::disable)), + Case($("DENY"), () -> run(frameOptionsConfig::deny)), + Case($("SAMEORIGIN"), () -> run(frameOptionsConfig::sameOrigin)), + Case($(), cappedFrameOptions -> run(() -> { + if (cappedFrameOptions.startsWith("ALLOW-FROM")) { + frameOptionsConfig.disable().addHeaderWriter(new StaticHeadersWriter("X-Frame-Options", frameOptions)); } - } + })) + ); // Allow public access to health endpoint http.authorizeRequests().antMatchers("/actuator/health").permitAll(); @@ -92,9 +82,7 @@ protected void configure(HttpSecurity http) throws Exception { http.authorizeRequests().antMatchers("/actuator/prometheus").permitAll(); // Note: call early, before http.authorizeRequests().anyRequest().fullyAuthenticated(); - if (customConfigs != null) { - for (ICustomSecurityConfig cfg : customConfigs) cfg.apply(http); - } + for (var cfg : Option.of(customConfigs).getOrElse(List.of())) cfg.apply(http); if (auth.hasAuthorization()) { @@ -120,12 +108,10 @@ protected void configure(HttpSecurity http) throws Exception { if (auth.hasAuthorization()) { // The `anyRequest` method may only be called once. // Therefore we call it here, make our changes to it and forward it to the various authentication backends - ExpressionUrlAuthorizationConfigurer.AuthorizedUrl anyRequestConfigurer = http.authorizeRequests().anyRequest(); + var anyRequestConfigurer = http.authorizeRequests().anyRequest(); anyRequestConfigurer.fullyAuthenticated(); auth.configureHttpSecurity(http, anyRequestConfigurer); } - - } @Bean diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/service/FileUpdateService.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/service/FileUpdateService.java index 92c42bba..dbe9b778 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/service/FileUpdateService.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/service/FileUpdateService.java @@ -2,23 +2,26 @@ import hk.edu.polyu.comp.vlabcontroller.event.ConfigUpdateEvent; import hk.edu.polyu.comp.vlabcontroller.util.ConfigFileHelper; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Value; import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.context.ApplicationEventPublisher; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; import org.springframework.stereotype.Service; import javax.annotation.PostConstruct; -import java.security.NoSuchAlgorithmException; +import java.util.Optional; +import java.util.concurrent.ScheduledFuture; +@Slf4j @RefreshScope @Service -public class FileUpdateService extends Thread { - protected final Logger log = LogManager.getLogger(getClass()); - +@RequiredArgsConstructor +public class FileUpdateService { private final ConfigFileHelper configFileHelper; private final ApplicationEventPublisher publisher; + private final ThreadPoolTaskScheduler taskScheduler; @Value("${proxy.config.interval:5000}") private int interval; @@ -26,33 +29,22 @@ public class FileUpdateService extends Thread { @Value("${proxy.config.auto-update:true}") private boolean configAutoUpdate; - public FileUpdateService(ConfigFileHelper configFileHelper, ApplicationEventPublisher publisher) { - this.configFileHelper = configFileHelper; - this.publisher = publisher; - } + private Optional> configUpdateFuture = Optional.empty(); + private String configHashCache; @PostConstruct public void start() { + var self = this; if (configAutoUpdate) { log.info("Starting configuration auto detection, interval: {}ms", interval); - super.start(); - } - } - - @Override - public void run() { - try { - String before = configFileHelper.getConfigHash(); - while (true) { - String after = configFileHelper.getConfigHash(); - if (!before.equals(after)) { - publisher.publishEvent(new ConfigUpdateEvent(this)); + configUpdateFuture.ifPresent(x -> x.cancel(true)); + configUpdateFuture = Optional.of(taskScheduler.scheduleAtFixedRate(() -> { + var hash = configFileHelper.getConfigHash(); + if (configHashCache != null && !configHashCache.equals(hash)) { + publisher.publishEvent(new ConfigUpdateEvent(self)); } - before = after; - Thread.sleep(interval); - } - } catch (NoSuchAlgorithmException | InterruptedException e) { - e.printStackTrace(); + configHashCache = hash; + }, interval)); } } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/service/HeartbeatService.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/service/HeartbeatService.java index 6cece26c..b8d11786 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/service/HeartbeatService.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/service/HeartbeatService.java @@ -1,94 +1,153 @@ package hk.edu.polyu.comp.vlabcontroller.service; +import hk.edu.polyu.comp.vlabcontroller.config.ProxyProperties; import hk.edu.polyu.comp.vlabcontroller.model.runtime.HeartbeatStatus; -import hk.edu.polyu.comp.vlabcontroller.model.runtime.Proxy; import hk.edu.polyu.comp.vlabcontroller.model.runtime.ProxyStatus; -import hk.edu.polyu.comp.vlabcontroller.spec.EngagementProperties; import hk.edu.polyu.comp.vlabcontroller.util.ChannelActiveListener; import hk.edu.polyu.comp.vlabcontroller.util.DelegatingStreamSinkConduit; import hk.edu.polyu.comp.vlabcontroller.util.DelegatingStreamSourceConduit; import io.undertow.server.HttpServerExchange; import io.undertow.server.protocol.http.HttpServerConnection; -import lombok.extern.log4j.Log4j2; -import org.springframework.core.env.Environment; +import io.vavr.Function0; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.time.DurationUtils; +import org.springframework.cloud.context.config.annotation.RefreshScope; +import org.springframework.cloud.context.scope.refresh.RefreshScopeRefreshedEvent; +import org.springframework.context.event.EventListener; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; import org.springframework.stereotype.Service; import org.xnio.StreamConnection; -import org.xnio.conduits.ConduitStreamSinkChannel; -import org.xnio.conduits.ConduitStreamSourceChannel; import javax.annotation.PostConstruct; -import javax.annotation.Resource; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -@Log4j2 +import java.time.Duration; +import java.util.*; +import java.util.concurrent.ScheduledFuture; +import java.util.function.Consumer; + +@Slf4j @Service +@RequiredArgsConstructor +@RefreshScope public class HeartbeatService { - - private static final String PROP_ENABLED = "proxy.heartbeat-enabled"; - private static final String PROP_RATE = "proxy.heartbeat-rate"; - private static final String PROP_TIMEOUT = "proxy.heartbeat-timeout"; - private static final byte[] WEBSOCKET_PING = {(byte) 0b10001001, (byte) 0b00000000}; private static final byte WEBSOCKET_PONG = (byte) 0b10001010; - private final Map proxyHeartbeats = Collections.synchronizedMap(new HashMap<>()); + @Getter + private final Map proxyHeartbeats = Collections.synchronizedMap(new HashMap<>()); + @Getter private final Map websocketHeartbeats = Collections.synchronizedMap(new HashMap<>()); - private final ScheduledExecutorService heartbeatExecutor = Executors.newScheduledThreadPool(3); + @Setter private volatile boolean enabled; private final ProxyService proxyService; - private final Environment environment; + private final ProxyProperties proxyProperties; + private final ThreadPoolTaskScheduler taskScheduler; + + private List> runningFutures = new ArrayList<>(); - @Resource - private EngagementProperties engagementProperties; + private ScheduledFuture idleDetectionFuture; - public HeartbeatService(ProxyService proxyService, Environment environment) { - this.proxyService = proxyService; - this.environment = environment; + @EventListener + public void onRefreshScopeRefreshed(final RefreshScopeRefreshedEvent event) { + log.debug("heartbeat service refreshed"); } @PostConstruct public void init() { - enabled = Boolean.parseBoolean(environment.getProperty(PROP_ENABLED, "false")); - if (!enabled) { - enabled = environment.getProperty(PROP_RATE) != null || environment.getProperty(PROP_TIMEOUT) != null; + enabled = proxyProperties.isHeartbeatEnabled() || DurationUtils.isPositive(proxyProperties.getHeartbeatRate()) || DurationUtils.isPositive(proxyProperties.getHeartbeatTimeout()); + + Runnable idleDetection = () -> { + try { + log.debug("running idle detection"); + var currentTimestamp = Duration.ofMillis(System.currentTimeMillis()); + proxyService.getProxies(null, true).stream() + .filter(proxy -> proxy.getStatus() == ProxyStatus.Up) + .filter(proxy -> !proxy.getSpec().getId().equals("filebrowser")) + .forEach(proxy -> { + var id = proxy.getId(); + Consumer deleteProxy = time -> { + proxyHeartbeats.remove(id); + websocketHeartbeats.remove(id); + proxyService.stopProxy(proxy, true, true, time); + }; + + var engagement = proxyProperties.getEngagement(); + if (currentTimestamp.minus(proxy.getStartupTimestamp()).compareTo(engagement.getMaxAge()) > 0) { + log.info(String.format("Releasing timeout proxy [user: %s] [spec: %s] [id: %s] [duration: %dhr]", proxy.getUserId(), proxy.getSpec().getId(), id, engagement.getMaxAge().toHours())); + deleteProxy.accept(Duration.ZERO); + return; + } + // websocket idle termination + if (!engagement.isEnabled()) { + return; + } + var idleRetryLimit = engagement.getIdleRetry(); + var webSocketHeartbeatStatus = websocketHeartbeats.get(id); + var isPureHttp = webSocketHeartbeatStatus == null; + Function0 isIdled = () -> webSocketHeartbeatStatus.getTerminateCounter() >= idleRetryLimit; + + // 230 bytes per second default (10% load, 2300 bytes/sec when working on vscode) + var threshold = engagement.getThreshold(); + + if (!isPureHttp) { + // idle + var duration = currentTimestamp.minus(webSocketHeartbeatStatus.getStartRecordTimestamp()); + var rate = webSocketHeartbeatStatus.getTotalPayloadLength() / duration.toSeconds(); + if (rate < threshold) { + webSocketHeartbeatStatus.increaseCounter(); + log.debug("proxy {} websocket idle detected ({}/{})! average speed={} bytes/sec, threshold={} bytes/sec", id, webSocketHeartbeatStatus.getTerminateCounter(), idleRetryLimit, rate, threshold); + } + // active + else { + log.debug("proxy {} websocket active, average speed={} bytes/sec, threshold={} bytes/sec", id, rate, threshold); + webSocketHeartbeatStatus.clearAll(); + } + + webSocketHeartbeatStatus.setLastRecordTimestamp(Duration.ofMillis(System.currentTimeMillis())); + } + + var proxySilence = currentTimestamp.minus(Optional.ofNullable(proxyHeartbeats.get(id)).orElseGet(proxy::getStartupTimestamp)); + if ((proxySilence.compareTo(proxyProperties.getHeartbeatTimeout()) > 0) && (isPureHttp || isIdled.apply())) { + var silence = isPureHttp ? proxySilence : proxyProperties.getHeartbeatRate().multipliedBy(webSocketHeartbeatStatus.getTerminateCounter() - 1); + log.info("Releasing {} proxy [user: {}] [spec: {}] [id: {}] [silence: {}ms]", + isPureHttp ? "inactive" : "idled", + proxy.getUserId(), + proxy.getSpec().getId(), + id, + silence); + deleteProxy.accept(silence); + } + log.debug("proxy {} received HTTP requests {} ms ago, inactive threshold={} ms", id, proxySilence, proxyProperties.getHeartbeatTimeout()); + }); + } catch (Throwable t) { + log.error("Error in " + this.getClass().getSimpleName(), t); + } + }; + + if (idleDetectionFuture != null) { + idleDetectionFuture.cancel(true); + idleDetectionFuture = null; } if (enabled) { log.debug("Idle detection enabled"); - Thread cleanupThread = new Thread(new InactiveProxyKiller(), InactiveProxyKiller.class.getSimpleName()); - cleanupThread.setDaemon(true); - cleanupThread.start(); + idleDetectionFuture = taskScheduler.scheduleAtFixedRate(idleDetection, proxyProperties.getHeartbeatRate()); } } - public void setEnabled(boolean enabled) { - this.enabled = enabled; - } - - public Map getWebsocketHeartbeats() { - return websocketHeartbeats; - } - - public Map getProxyHeartbeats() { - return proxyHeartbeats; - } - public void attachHeartbeatChecker(HttpServerExchange exchange, String proxyId) { if (exchange.isUpgrade()) { // For websockets, attach a ping-pong listener to the underlying TCP channel. - HeartbeatConnector connector = new HeartbeatConnector(proxyId); + var connector = new HeartbeatConnector(proxyId); // Delay the wrapping, because Undertow will make changes to the channel while the upgrade is being performed. - HttpServerConnection httpConn = (HttpServerConnection) exchange.getConnection(); - heartbeatExecutor.schedule(() -> connector.wrapChannels(httpConn.getChannel()), 3000, TimeUnit.MILLISECONDS); + var httpConn = (HttpServerConnection) exchange.getConnection(); + runningFutures.add(taskScheduler.scheduleAtFixedRate(() -> connector.wrapChannels(httpConn.getChannel()), Duration.ofSeconds(3))); } else { // request URI prefix filter // exchange.getRequestPath() == /proxy_endpoint// @@ -96,8 +155,8 @@ public void attachHeartbeatChecker(HttpServerExchange exchange, String proxyId) // e.g access http:////app/app_name/static/js/example.js // exchange.getRequestPath() == /proxy_endpoint//static/js/example.js // exchange.getRelativePath() == /static/js/example.js - for (String path : engagementProperties.getFilterPath()) { - String relativeRequestPath = exchange.getRelativePath(); + for (var path : proxyProperties.getEngagement().getFilterPath()) { + var relativeRequestPath = exchange.getRelativePath(); log.debug("Client requests {} to proxy {}", relativeRequestPath, proxyId); if (relativeRequestPath.startsWith(path)) { log.debug("Matched prefix {} of proxy {}", path, proxyId); @@ -110,17 +169,9 @@ public void attachHeartbeatChecker(HttpServerExchange exchange, String proxyId) } private void heartbeatReceived(String proxyId) { - Proxy proxy = proxyService.getProxy(proxyId); + var proxy = proxyService.getProxy(proxyId); if (log.isDebugEnabled()) log.debug("Heartbeat received for proxy " + proxyId); - if (proxy != null) proxyHeartbeats.put(proxyId, System.currentTimeMillis()); - } - - private long getHeartbeatRate() { - return Long.parseLong(environment.getProperty(PROP_RATE, "10000")); - } - - private long getHeartbeatTimeout() { - return Long.parseLong(environment.getProperty(PROP_TIMEOUT, "60000")); + if (proxy != null) proxyHeartbeats.put(proxyId, Duration.ofMillis(System.currentTimeMillis())); } private class HeartbeatConnector { @@ -134,25 +185,25 @@ public HeartbeatConnector(String proxyId) { private void wrapChannels(StreamConnection streamConn) { if (!streamConn.isOpen()) return; - ConduitStreamSinkChannel sinkChannel = streamConn.getSinkChannel(); - ChannelActiveListener writeListener = new ChannelActiveListener(); - DelegatingStreamSinkConduit conduitWrapper = new DelegatingStreamSinkConduit(sinkChannel.getConduit(), writeListener); + var sinkChannel = streamConn.getSinkChannel(); + var writeListener = new ChannelActiveListener(); + var conduitWrapper = new DelegatingStreamSinkConduit(sinkChannel.getConduit(), writeListener); sinkChannel.setConduit(conduitWrapper); - ConduitStreamSourceChannel sourceChannel = streamConn.getSourceChannel(); - DelegatingStreamSourceConduit srcConduitWrapper = new DelegatingStreamSourceConduit(sourceChannel.getConduit(), data -> checkPong(data)); + var sourceChannel = streamConn.getSourceChannel(); + var srcConduitWrapper = new DelegatingStreamSourceConduit(sourceChannel.getConduit(), this::checkPong); sourceChannel.setConduit(srcConduitWrapper); - heartbeatExecutor.schedule(() -> sendPing(writeListener, streamConn), getHeartbeatRate(), TimeUnit.MILLISECONDS); + runningFutures.add(taskScheduler.scheduleAtFixedRate(() -> sendPing(writeListener, streamConn), proxyProperties.getHeartbeatRate())); } private void sendPing(ChannelActiveListener writeListener, StreamConnection streamConn) { - if (writeListener.isActive(getHeartbeatRate())) { + if (writeListener.isActive(proxyProperties.getHeartbeatRate())) { // active means that data was written to the channel in the least heartbeat interval // therefore we don't send a ping now to not cause collisions // reschedule ping - heartbeatExecutor.schedule(() -> sendPing(writeListener, streamConn), getHeartbeatRate(), TimeUnit.MILLISECONDS); + runningFutures.add(taskScheduler.scheduleAtFixedRate(() -> sendPing(writeListener, streamConn), proxyProperties.getHeartbeatRate())); // mark as we received a heartbeat // heartbeatReceived(proxyId); return; @@ -166,7 +217,7 @@ private void sendPing(ChannelActiveListener writeListener, StreamConnection stre // Ignore failure, keep trying as long as the stream connection is valid. } - heartbeatExecutor.schedule(() -> sendPing(writeListener, streamConn), getHeartbeatRate(), TimeUnit.MILLISECONDS); + runningFutures.add(taskScheduler.scheduleAtFixedRate(() -> sendPing(writeListener, streamConn), proxyProperties.getHeartbeatRate())); } private void checkPong(byte[] response) { @@ -179,7 +230,7 @@ private void checkPong(byte[] response) { // payload length analyzer // https://datatracker.ietf.org/doc/html/rfc6455#section-5.2 - int payloadLength = response[1] & 0x7F; + var payloadLength = response[1] & 0x7F; if (payloadLength == 126) { if (response.length < 4) { // handle broken packet @@ -202,7 +253,7 @@ private void checkPong(byte[] response) { } log.debug("Websocket packet received, length={} bytes", payloadLength); - Proxy proxy = proxyService.getProxy(proxyId); + var proxy = proxyService.getProxy(proxyId); // if a proxy is terminated manually before status block created, stop checkPong. if (proxy == null || (proxy.getStatus() == ProxyStatus.Stopping || proxy.getStatus() == ProxyStatus.Stopped)) { @@ -210,95 +261,9 @@ private void checkPong(byte[] response) { return; } - HeartbeatStatus heartbeatStatus = websocketHeartbeats.computeIfAbsent(proxyId, k -> new HeartbeatStatus()); - int lastLength = heartbeatStatus.getTotalPayloadLength(); + var heartbeatStatus = websocketHeartbeats.computeIfAbsent(proxyId, k -> new HeartbeatStatus()); + var lastLength = heartbeatStatus.getTotalPayloadLength(); heartbeatStatus.setTotalPayloadLength(lastLength + payloadLength); } } - - private class InactiveProxyKiller implements Runnable { - @Override - public void run() { - long cleanupInterval = getHeartbeatRate(); - long heartbeatTimeout = getHeartbeatTimeout(); - - while (true) { - try { - long currentTimestamp = System.currentTimeMillis(); - for (Proxy proxy : proxyService.getProxies(null, true)) { - if (proxy.getStatus() != ProxyStatus.Up) continue; - else if (proxy.getSpec().getId().equals("filebrowser")) continue; - - // reached max-age limitation - if (currentTimestamp - proxy.getStartupTimestamp() > engagementProperties.getMaxAge().toMillis()) { - log.info(String.format("Releasing timeout proxy [user: %s] [spec: %s] [id: %s] [duration: %dhr]", proxy.getUserId(), proxy.getSpec().getId(), proxy.getId(), engagementProperties.getMaxAge().toHours())); - proxyHeartbeats.remove(proxy.getId()); - websocketHeartbeats.remove(proxy.getId()); - proxyService.stopProxy(proxy, true, true, 0); - continue; - } - - // websocket idle termination - boolean isPureHttp = false; - boolean isIdled = false; - int idleRetryLimit = engagementProperties.getIdleRetry(); - if (engagementProperties.isEnabled()) { - HeartbeatStatus heartbeatStatus = websocketHeartbeats.get(proxy.getId()); - - // 230 bytes per second default (10% load, 2300 bytes/sec when working on vscode) - int threshold = engagementProperties.getThreshold(); - - if (heartbeatStatus == null) { - isPureHttp = true; - } else { - long duration = currentTimestamp - heartbeatStatus.getStartRecordTimestamp(); - // idle - double rate = heartbeatStatus.getTotalPayloadLength() / (duration / 1000.0); - if (rate < threshold) { - heartbeatStatus.increaseCounter(); - log.debug("proxy {} websocket idle detected ({}/{})! average speed={} bytes/sec, threshold={} bytes/sec", proxy.getId(), heartbeatStatus.getTerminateCounter(), idleRetryLimit, rate, threshold); - } - // active - else { - log.debug("proxy {} websocket active, average speed={} bytes/sec, threshold={} bytes/sec", proxy.getId(), rate, threshold); - heartbeatStatus.clearAll(); - } - - // idle confirmed - if (heartbeatStatus.getTerminateCounter() >= idleRetryLimit) { - isIdled = true; - } - - heartbeatStatus.setLastRecordTimestamp(System.currentTimeMillis()); - } - - Long lastHeartbeat = proxyHeartbeats.get(proxy.getId()); - if (lastHeartbeat == null) lastHeartbeat = proxy.getStartupTimestamp(); - long proxySilence = currentTimestamp - lastHeartbeat; - if ((proxySilence > heartbeatTimeout) && (isPureHttp | isIdled)) { - long silence = isPureHttp ? proxySilence : cleanupInterval * (heartbeatStatus.getTerminateCounter() - 1); - log.info("Releasing {} proxy [user: {}] [spec: {}] [id: {}] [silence: {}ms]", - isPureHttp ? "inactive" : "idled", - proxy.getUserId(), - proxy.getSpec().getId(), - proxy.getId(), - silence); - - proxyHeartbeats.remove(proxy.getId()); - websocketHeartbeats.remove(proxy.getId()); - proxyService.stopProxy(proxy, true, true, silence); - } - log.debug("proxy {} received HTTP requests {} ms ago, inactive threshold={} ms", proxy.getId(), proxySilence, heartbeatTimeout); - } - } - Thread.sleep(cleanupInterval); - } catch (InterruptedException e) { - log.error("Inactive proxy killer was interrupted, stop cleanup work"); - break; - } catch (Throwable t) { - log.error("Error in " + this.getClass().getSimpleName(), t); - } - } - } - } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/service/LogService.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/service/LogService.java index e5d52b1b..81ae861c 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/service/LogService.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/service/LogService.java @@ -3,9 +3,9 @@ import hk.edu.polyu.comp.vlabcontroller.log.ILogStorage; import hk.edu.polyu.comp.vlabcontroller.log.NoopLogStorage; import hk.edu.polyu.comp.vlabcontroller.model.runtime.Proxy; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.springframework.core.env.Environment; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import javax.annotation.PostConstruct; @@ -16,20 +16,14 @@ import java.util.concurrent.Executors; import java.util.function.BiConsumer; +@Slf4j @Service +@RequiredArgsConstructor public class LogService { - private static final String PARAM_STREAMS = "streams"; - private final Logger log = LogManager.getLogger(LogService.class); - final Environment environment; final ILogStorage logStorage; private ExecutorService executor; - private boolean loggingEnabled; - - public LogService(Environment environment, ILogStorage logStorage) { - this.environment = environment; - this.logStorage = logStorage; - } + @Getter private boolean loggingEnabled; @PostConstruct public void init() { @@ -51,16 +45,12 @@ public void shutdown() { if (executor != null) executor.shutdown(); } - public boolean isLoggingEnabled() { - return loggingEnabled; - } - public void attachToOutput(Proxy proxy, BiConsumer outputAttacher) { if (!isLoggingEnabled()) return; executor.submit(() -> { try { - OutputStream[] streams = logStorage.createOutputStreams(proxy); + var streams = logStorage.createOutputStreams(proxy); if (streams == null || streams.length < 2) { log.error("Failed to attach logging of proxy " + proxy.getId() + ": no output streams defined"); } else { @@ -79,15 +69,15 @@ public void attachToOutput(Proxy proxy, BiConsumer o public void detach(Proxy proxy) { if (!isLoggingEnabled()) return; - OutputStream[] streams = (OutputStream[]) proxy.getContainerGroup().getParameters().get(PARAM_STREAMS); + var streams = (OutputStream[]) proxy.getContainerGroup().getParameters().get(PARAM_STREAMS); if (streams == null || streams.length < 2) { log.warn("Cannot detach container logging: streams not found"); return; } - for (int i = 0; i < streams.length; i++) { + for (OutputStream stream : streams) { try { - streams[i].flush(); - streams[i].close(); + stream.flush(); + stream.close(); } catch (IOException e) { log.error("Failed to close container logging streams", e); } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/service/ProxyService.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/service/ProxyService.java index 29fdfcaf..e11f1bdd 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/service/ProxyService.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/service/ProxyService.java @@ -13,25 +13,22 @@ import hk.edu.polyu.comp.vlabcontroller.spec.IProxySpecProvider; import hk.edu.polyu.comp.vlabcontroller.spec.ProxySpecException; import hk.edu.polyu.comp.vlabcontroller.util.ProxyMappingManager; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.time.DurationUtils; import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.annotation.Lazy; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; import org.springframework.security.access.AccessDeniedException; import org.springframework.stereotype.Service; import javax.annotation.PreDestroy; -import java.io.OutputStream; -import java.net.URI; import java.time.Duration; import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Map.Entry; import java.util.Set; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.function.BiConsumer; +import java.util.concurrent.Future; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -47,12 +44,12 @@ * checks before manipulating proxies. *

*/ +@Slf4j @Service +@RequiredArgsConstructor(onConstructor_ = {@Lazy}) public class ProxyService { - - private final Logger log = LogManager.getLogger(ProxyService.class); private final List activeProxies = Collections.synchronizedList(new ArrayList<>()); - private final ExecutorService containerKiller = Executors.newSingleThreadExecutor(); + private final ThreadPoolTaskScheduler taskScheduler; private final IProxySpecProvider baseSpecProvider; private final IProxySpecMergeStrategy specMergeStrategy; @@ -61,29 +58,17 @@ public class ProxyService { private final UserService userService; private final LogService logService; private final ApplicationEventPublisher applicationEventPublisher; - - @Lazy - public ProxyService(IProxySpecProvider baseSpecProvider, IProxySpecMergeStrategy specMergeStrategy, IContainerBackend backend, ProxyMappingManager mappingManager, UserService userService, LogService logService, ApplicationEventPublisher applicationEventPublisher) { - this.baseSpecProvider = baseSpecProvider; - this.specMergeStrategy = specMergeStrategy; - this.backend = backend; - this.mappingManager = mappingManager; - this.userService = userService; - this.logService = logService; - this.applicationEventPublisher = applicationEventPublisher; - } + private List> containerKillerFutures = new ArrayList<>(); @PreDestroy public void shutdown() { - try { - containerKiller.shutdown(); - } finally { - for (Proxy proxy : activeProxies) { - try { - backend.stopProxy(proxy); - } catch (Exception exception) { - exception.printStackTrace(); - } + containerKillerFutures.forEach(x -> x.cancel(true)); + + for (var proxy : activeProxies) { + try { + backend.stopProxy(proxy); + } catch (Exception exception) { + exception.printStackTrace(); } } } @@ -119,9 +104,9 @@ public ProxySpec findProxySpec(Predicate filter, boolean ignoreAccess */ public List getProxySpecs(Predicate filter, boolean ignoreAccessControl) { return baseSpecProvider.getSpecs().stream() - .filter(spec -> ignoreAccessControl || userService.canAccess(spec)) - .filter(spec -> filter == null || filter.test(spec)) - .collect(Collectors.toList()); + .filter(spec -> ignoreAccessControl || userService.canAccess(spec)) + .filter(spec -> filter == null || filter.test(spec)) + .collect(Collectors.toList()); } /** @@ -168,11 +153,11 @@ public Proxy findProxy(Predicate filter, boolean ignoreAccessControl) { * @return A List of matching proxies, may be empty. */ public List getProxies(Predicate filter, boolean ignoreAccessControl) { - boolean isAdmin = userService.isAdmin(); + var isAdmin = userService.isAdmin(); List matches = new ArrayList<>(); synchronized (activeProxies) { - for (Proxy proxy : activeProxies) { - boolean hasAccess = ignoreAccessControl || isAdmin || userService.isOwner(proxy); + for (var proxy : activeProxies) { + var hasAccess = ignoreAccessControl || isAdmin || userService.isOwner(proxy); if (hasAccess && (filter == null || filter.test(proxy))) matches.add(proxy); } } @@ -192,11 +177,12 @@ public Proxy startProxy(ProxySpec spec, boolean ignoreAccessControl) throws VLab throw new AccessDeniedException(String.format("Cannot start proxy %s: access denied", spec.getId())); } - Proxy proxy = new Proxy(); - proxy.setStatus(ProxyStatus.New); - proxy.setUserId(userService.getCurrentUserId()); - proxy.setSpec(spec); - proxy.setAdmin(userService.isAdmin()); + var proxy = Proxy.builder() + .status(ProxyStatus.New) + .userId(userService.getCurrentUserId()) + .spec(spec.copy()) + .admin(userService.isAdmin()) + .build(); activeProxies.add(proxy); try { @@ -204,16 +190,17 @@ public Proxy startProxy(ProxySpec spec, boolean ignoreAccessControl) throws VLab } finally { if (proxy.getStatus() != ProxyStatus.Up) { activeProxies.remove(proxy); - applicationEventPublisher.publishEvent(new ProxyStartFailedEvent(this, proxy.getUserId(), spec.getId())); + var event = ProxyStartFailedEvent.builder().source(this).specId(spec.getId()).userId(proxy.getUserId()).build(); + applicationEventPublisher.publishEvent(event); } } - for (Entry target : proxy.getTargets().entrySet()) { + for (var target : proxy.getTargets().entrySet()) { mappingManager.addMapping(proxy.getId(), target.getKey(), target.getValue()); } if (logService.isLoggingEnabled()) { - BiConsumer outputAttacher = backend.getOutputAttacher(proxy); + var outputAttacher = backend.getOutputAttacher(proxy); if (outputAttacher == null) { log.warn("Cannot log proxy output: " + backend.getClass() + " does not support output attaching."); } else { @@ -222,7 +209,11 @@ public Proxy startProxy(ProxySpec spec, boolean ignoreAccessControl) throws VLab } log.info(String.format("Proxy activated [user: %s] [spec: %s] [id: %s]", proxy.getUserId(), spec.getId(), proxy.getId())); - applicationEventPublisher.publishEvent(new ProxyStartEvent(this, proxy.getUserId(), spec.getId(), Duration.ofMillis(proxy.getStartupTimestamp() - proxy.getCreatedTimestamp()))); + var event = ProxyStartEvent.builder() + .source(this).proxyId(proxy.getId()).specId(spec.getId()).userId(proxy.getUserId()) + .startupTime(proxy.getStartupTimestamp().minus(proxy.getCreatedTimestamp())) + .build(); + applicationEventPublisher.publishEvent(event); return proxy; } @@ -235,7 +226,7 @@ public Proxy startProxy(ProxySpec spec, boolean ignoreAccessControl) throws VLab * @param ignoreAccessControl True to allow access to any proxy, regardless of the current security context. * @param silenceOffset Milliseconds to subtract idle silence period, report accurate usage time. */ - public void stopProxy(Proxy proxy, boolean async, boolean ignoreAccessControl, long silenceOffset) { + public void stopProxy(Proxy proxy, boolean async, boolean ignoreAccessControl, Duration silenceOffset) { if (!ignoreAccessControl && !userService.isAdmin() && !userService.isOwner(proxy)) { throw new AccessDeniedException(String.format("Cannot stop proxy %s: access denied", proxy.getId())); } @@ -247,16 +238,18 @@ public void stopProxy(Proxy proxy, boolean async, boolean ignoreAccessControl, l backend.stopProxy(proxy); logService.detach(proxy); log.info(String.format("Proxy released [user: %s] [spec: %s] [id: %s]", proxy.getUserId(), proxy.getSpec().getId(), proxy.getId())); - if (proxy.getStartupTimestamp() > 0) { - applicationEventPublisher.publishEvent(new ProxyStopEvent(this, proxy.getUserId(), - proxy.getSpec().getId(), - Duration.ofMillis(System.currentTimeMillis() - proxy.getStartupTimestamp() - silenceOffset))); + if (DurationUtils.isPositive(proxy.getStartupTimestamp())) { + var event = ProxyStopEvent.builder() + .usageTime(Duration.ofMillis(System.currentTimeMillis()).minus(proxy.getStartupTimestamp()).minus(silenceOffset)) + .source(this).proxyId(proxy.getId()).userId(proxy.getUserId()).specId(proxy.getSpec().getId()) + .build(); + applicationEventPublisher.publishEvent(event); } } catch (Exception e) { log.error("Failed to release proxy " + proxy.getId(), e); } }; - if (async) containerKiller.submit(releaser); + if (async) containerKillerFutures.add(taskScheduler.submit(releaser)); else releaser.run(); mappingManager.removeProxyMapping(proxy.getId()); diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/service/UserActionEventsListener.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/service/UserActionEventsListener.java new file mode 100644 index 00000000..61f276fd --- /dev/null +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/service/UserActionEventsListener.java @@ -0,0 +1,76 @@ +package hk.edu.polyu.comp.vlabcontroller.service; + +import hk.edu.polyu.comp.vlabcontroller.entity.LabInstance; +import hk.edu.polyu.comp.vlabcontroller.entity.SessionData; +import hk.edu.polyu.comp.vlabcontroller.entity.User; +import hk.edu.polyu.comp.vlabcontroller.event.ProxyStartEvent; +import hk.edu.polyu.comp.vlabcontroller.event.ProxyStopEvent; +import hk.edu.polyu.comp.vlabcontroller.event.UserLoginEvent; +import hk.edu.polyu.comp.vlabcontroller.event.UserLogoutEvent; +import hk.edu.polyu.comp.vlabcontroller.repository.UserRepository; +import lombok.RequiredArgsConstructor; +import org.joda.time.DateTime; +import org.springframework.cloud.context.config.annotation.RefreshScope; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; + +import java.util.Optional; + +@RefreshScope +@Component +@RequiredArgsConstructor +public class UserActionEventsListener { + private final UserRepository repository; + + @EventListener + public void onProxyStart(ProxyStartEvent event) { + var time = new DateTime(event.getTimestamp()); + var user = this.repository.findUserByIdOrCreate(event.getUserId()); + var labs = user.getLabs(); + labs.stream().filter(x -> x.getId().equals(event.getProxyId())).findAny() + .ifPresentOrElse( + lab -> lab.setStartedAt(time), + () -> labs.addFirst(LabInstance.builder().id(event.getProxyId()).startedAt(time).build()) + ); + this.repository.save(user); + } + + @EventListener + public void onProxyStop(ProxyStopEvent event) { + var time = new DateTime(event.getTimestamp()); + User user = this.repository.findUserByIdOrCreate(event.getUserId()); + var labs = user.getLabs(); + labs.stream().filter(x -> x.getId().equals(event.getProxyId())).findAny() + .ifPresentOrElse( + lab -> lab.setStartedAt(time), + () -> labs.addFirst(LabInstance.builder().id(event.getProxyId()).completedAt(time).build()) + ); + this.repository.save(user); + } + + @EventListener + public void onUserLogin(UserLoginEvent event) { + var time = new DateTime(event.getTimestamp()); + var user = this.repository.findUserByIdOrCreate(event.getUserId()); + var sessions = user.getSession(); + Optional.ofNullable(sessions.get(event.getSessionId())) + .ifPresentOrElse( + session -> session.setLoggedInAt(time), + () -> sessions.put(event.getSessionId(), SessionData.builder().loggedInAt(time).build()) + ); + this.repository.save(user); + } + + @EventListener + public void onUserLogout(UserLogoutEvent event) { + var time = new DateTime(event.getTimestamp()); + var user = this.repository.findUserByIdOrCreate(event.getUserId()); + var sessions = user.getSession(); + Optional.ofNullable(sessions.get(event.getSessionId())) + .ifPresentOrElse( + session -> session.setLoggedOutAt(time), + () -> sessions.put(event.getSessionId(), SessionData.builder().loggedOutAt(time).build()) + ); + this.repository.save(user); + } +} \ No newline at end of file diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/service/UserService.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/service/UserService.java index f97acef7..88616535 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/service/UserService.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/service/UserService.java @@ -2,24 +2,24 @@ import hk.edu.polyu.comp.vlabcontroller.auth.IAuthenticationBackend; import hk.edu.polyu.comp.vlabcontroller.backend.strategy.IProxyLogoutStrategy; +import hk.edu.polyu.comp.vlabcontroller.config.ProxyProperties; import hk.edu.polyu.comp.vlabcontroller.event.AuthFailedEvent; import hk.edu.polyu.comp.vlabcontroller.event.UserLoginEvent; import hk.edu.polyu.comp.vlabcontroller.event.UserLogoutEvent; import hk.edu.polyu.comp.vlabcontroller.model.runtime.Proxy; import hk.edu.polyu.comp.vlabcontroller.model.spec.ProxySpec; import hk.edu.polyu.comp.vlabcontroller.util.SessionHelper; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import io.vavr.control.Option; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.context.ApplicationEventPublisher; import org.springframework.context.annotation.Lazy; import org.springframework.context.event.EventListener; -import org.springframework.core.env.Environment; import org.springframework.security.authentication.AnonymousAuthenticationToken; import org.springframework.security.authentication.event.AbstractAuthenticationFailureEvent; import org.springframework.security.authentication.event.AuthenticationSuccessEvent; import org.springframework.security.core.Authentication; -import org.springframework.security.core.GrantedAuthority; -import org.springframework.security.core.context.SecurityContext; import org.springframework.security.core.context.SecurityContextHolder; import org.springframework.security.web.session.HttpSessionCreatedEvent; import org.springframework.security.web.session.HttpSessionDestroyedEvent; @@ -27,29 +27,24 @@ import org.springframework.web.context.request.RequestContextHolder; import org.springframework.web.context.request.ServletRequestAttributes; -import javax.servlet.http.HttpSession; -import java.util.*; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Predicate; +import java.util.stream.Collectors; +@Slf4j @Service +@RequiredArgsConstructor(onConstructor_ = {@Lazy}) +@RefreshScope public class UserService { - private final static String ATTRIBUTE_USER_INITIATED_LOGOUT = "SP_USER_INITIATED_LOGOUT"; - - private final Logger log = LogManager.getLogger(UserService.class); private final Map userInitiatedLogoutMap = new HashMap<>(); - private final Environment environment; + private final ProxyProperties proxyProperties; private final IAuthenticationBackend authBackend; private final IProxyLogoutStrategy logoutStrategy; private final ApplicationEventPublisher applicationEventPublisher; - @Lazy - public UserService(Environment environment, IAuthenticationBackend authBackend, IProxyLogoutStrategy logoutStrategy, ApplicationEventPublisher applicationEventPublisher) { - this.environment = environment; - this.authBackend = authBackend; - this.logoutStrategy = logoutStrategy; - this.applicationEventPublisher = applicationEventPublisher; - } - public Authentication getCurrentAuth() { return SecurityContextHolder.getContext().getAuthentication(); } @@ -58,36 +53,23 @@ public String getCurrentUserId() { return getUserId(getCurrentAuth()); } - public String[] getAdminGroups() { - Set adminGroups = new HashSet<>(); - - // Support for old, non-array notation - String singleGroup = environment.getProperty("proxy.admin-groups"); - if (singleGroup != null && !singleGroup.isEmpty()) adminGroups.add(singleGroup.toUpperCase()); - - for (int i = 0; ; i++) { - String groupName = environment.getProperty(String.format("proxy.admin-groups[%s]", i)); - if (groupName == null || groupName.isEmpty()) break; - adminGroups.add(groupName.toUpperCase()); - } - - return adminGroups.toArray(new String[adminGroups.size()]); + public Collection getAdminGroups() { + return proxyProperties.getAdminGroups().stream() + .filter(Predicate.not(String::isBlank)) + .map(String::toUpperCase) + .collect(Collectors.toSet()); } - public String[] getGroups() { + public Collection getGroups() { return getGroups(getCurrentAuth()); } - public String[] getGroups(Authentication auth) { - List groups = new ArrayList<>(); - if (auth != null) { - for (GrantedAuthority grantedAuth : auth.getAuthorities()) { - String authName = grantedAuth.getAuthority().toUpperCase(); - if (authName.startsWith("ROLE_")) authName = authName.substring(5); - groups.add(authName); - } - } - return groups.toArray(new String[groups.size()]); + public Collection getGroups(Authentication auth) { + return auth.getAuthorities().stream().map(grantedAuth -> { + var authName = grantedAuth.getAuthority().toUpperCase(); + if (authName.startsWith("ROLE_")) authName = authName.substring(5); + return authName; + }).collect(Collectors.toList()); } public boolean isAdmin() { @@ -95,10 +77,7 @@ public boolean isAdmin() { } public boolean isAdmin(Authentication auth) { - for (String adminGroup : getAdminGroups()) { - if (isMember(auth, adminGroup)) return true; - } - return false; + return getAdminGroups().stream().anyMatch(adminGroup -> isMember(auth, adminGroup)); } public boolean canAccess(ProxySpec spec) { @@ -108,12 +87,8 @@ public boolean canAccess(ProxySpec spec) { public boolean canAccess(Authentication auth, ProxySpec spec) { if (auth == null || spec == null) return false; if (auth instanceof AnonymousAuthenticationToken) return !authBackend.hasAuthorization(); - List groups = spec.getAccessGroups(); - if (groups.isEmpty()) return true; - for (String group : groups) { - if (isMember(auth, group)) return true; - } - return false; + var groups = spec.getAccessGroups(); + return groups.isEmpty() || groups.stream().anyMatch(group -> isMember(auth, group)); } public boolean isOwner(Proxy proxy) { @@ -127,10 +102,7 @@ public boolean isOwner(Authentication auth, Proxy proxy) { private boolean isMember(Authentication auth, String groupName) { if (auth == null || auth instanceof AnonymousAuthenticationToken || groupName == null) return false; - for (String group : getGroups(auth)) { - if (group.equalsIgnoreCase(groupName)) return true; - } - return false; + return getGroups(auth).stream().anyMatch(group -> group.equalsIgnoreCase(groupName)); } private String getUserId(Authentication auth) { @@ -144,10 +116,10 @@ private String getUserId(Authentication auth) { @EventListener public void onAbstractAuthenticationFailureEvent(AbstractAuthenticationFailureEvent event) { - Authentication source = event.getAuthentication(); + var source = event.getAuthentication(); Exception e = event.getException(); log.info(String.format("Authentication failure [user: %s] [error: %s]", source.getName(), e.getMessage())); - String userId = getUserId(source); + var userId = getUserId(source); applicationEventPublisher.publishEvent(new AuthFailedEvent( this, @@ -156,14 +128,14 @@ public void onAbstractAuthenticationFailureEvent(AbstractAuthenticationFailureEv } public void logout(Authentication auth) { - String userId = getUserId(auth); + var userId = getUserId(auth); if (userId == null) return; if (logoutStrategy != null) logoutStrategy.onLogout(userId, false); log.info(String.format("User logged out [user: %s]", userId)); - HttpSession session = ((ServletRequestAttributes) RequestContextHolder.currentRequestAttributes()).getRequest().getSession(); - String sessionId = session.getId(); + var session = ((ServletRequestAttributes) RequestContextHolder.currentRequestAttributes()).getRequest().getSession(); + var sessionId = session.getId(); userInitiatedLogoutMap.put(sessionId, "true"); applicationEventPublisher.publishEvent(new UserLogoutEvent( this, @@ -174,11 +146,11 @@ public void logout(Authentication auth) { @EventListener public void onAuthenticationSuccessEvent(AuthenticationSuccessEvent event) { - Authentication auth = event.getAuthentication(); - String userName = auth.getName(); + var auth = event.getAuthentication(); + var userName = auth.getName(); - HttpSession session = ((ServletRequestAttributes) RequestContextHolder.currentRequestAttributes()).getRequest().getSession(); - boolean firstLogin = session.getAttribute("firstLogin") == null || (Boolean) session.getAttribute("firstLogin"); + var session = ((ServletRequestAttributes) RequestContextHolder.currentRequestAttributes()).getRequest().getSession(); + var firstLogin = session.getAttribute("firstLogin") == null || (Boolean) session.getAttribute("firstLogin"); if (firstLogin) { session.setAttribute("firstLogin", false); } else { @@ -187,11 +159,8 @@ public void onAuthenticationSuccessEvent(AuthenticationSuccessEvent event) { log.info(String.format("User logged in [user: %s]", userName)); - String userId = getUserId(auth); - applicationEventPublisher.publishEvent(new UserLoginEvent( - this, - userId, - RequestContextHolder.currentRequestAttributes().getSessionId())); + var userId = getUserId(auth); + applicationEventPublisher.publishEvent(UserLoginEvent.builder().source(this).userId(userId).sessionId(RequestContextHolder.currentRequestAttributes().getSessionId()).build()); } @EventListener @@ -203,47 +172,42 @@ public void onHttpSessionDestroyedEvent(HttpSessionDestroyedEvent event) { Session Attributes set in logout() cannot be fetched here but these two session instances have same sessionId, an additional Map can be used as workaround */ - String userInitiatedLogout = userInitiatedLogoutMap.remove(event.getId()); + var userInitiatedLogout = userInitiatedLogoutMap.remove(event.getId()); if (userInitiatedLogout != null && userInitiatedLogout.equals("true")) { // user initiated the logout // event already handled by the logout() function above -> ignore it } else { // user did not initiated the logout -> session expired // not already handled by any other handler + var eventBuilder = UserLogoutEvent.builder().source(this); + + var sid = Option.none(); + var uid = Option.none(); + if (!event.getSecurityContexts().isEmpty()) { - SecurityContext securityContext = event.getSecurityContexts().get(0); + var securityContext = event.getSecurityContexts().get(0); if (securityContext == null) return; - String userId = securityContext.getAuthentication().getName(); - + var userId = securityContext.getAuthentication().getName(); logoutStrategy.onLogout(userId, true); log.info(String.format("HTTP session expired [user: %s]", userId)); - applicationEventPublisher.publishEvent(new UserLogoutEvent( - this, - userId, - event.getSession().getId(), - true - )); + uid = Option.some(userId); + sid = Option.some(RequestContextHolder.currentRequestAttributes().getSessionId()); } else if (authBackend.getName().equals("none")) { - log.info(String.format("Anonymous user logged out [user: %s]", event.getSession().getId())); - applicationEventPublisher.publishEvent(new UserLogoutEvent( - this, - event.getSession().getId(), - event.getSession().getId(), - true - )); + var id = event.getSession().getId(); + log.info(String.format("Anonymous user logged out [user: %s]", id)); + sid = uid = Option.some(id); } + applicationEventPublisher.publishEvent(eventBuilder.userId(uid.get()).sessionId(sid.get()).wasExpired(true).build()); } } @EventListener public void onHttpSessionCreated(HttpSessionCreatedEvent event) { if (authBackend.getName().equals("none")) { - log.info(String.format("Anonymous user logged in [user: %s]", event.getSession().getId())); - applicationEventPublisher.publishEvent(new UserLoginEvent( - this, - event.getSession().getId(), - event.getSession().getId())); + var id = event.getSession().getId(); + log.info(String.format("Anonymous user logged in [user: %s]", id)); + applicationEventPublisher.publishEvent(UserLoginEvent.builder().source(this).userId(id).sessionId(id).build()); } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/EngagementProperties.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/EngagementProperties.java deleted file mode 100644 index 8101ba32..00000000 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/EngagementProperties.java +++ /dev/null @@ -1,38 +0,0 @@ -package hk.edu.polyu.comp.vlabcontroller.spec; - - -import lombok.Getter; -import lombok.Setter; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.context.config.annotation.RefreshScope; -import org.springframework.context.annotation.Configuration; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; - -@RefreshScope -@EnableConfigurationProperties -@Configuration -@ConfigurationProperties(prefix = "proxy.engagement") -public class EngagementProperties { - @Getter - @Setter - private boolean enabled = true; - @Getter - @Setter - private List filterPath = new ArrayList<>(); - @Getter - @Setter - private int idleRetry = 3; - @Getter - @Setter - private int threshold = 230; - @Getter - private Duration maxAge = Duration.ofHours(4); - - public void setMaxAge(String duration) { - this.maxAge = Duration.parse(duration); - } -} diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/FileBrowserProperties.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/FileBrowserProperties.java deleted file mode 100644 index 8e998735..00000000 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/FileBrowserProperties.java +++ /dev/null @@ -1,15 +0,0 @@ -package hk.edu.polyu.comp.vlabcontroller.spec; - -import hk.edu.polyu.comp.vlabcontroller.model.spec.ProxySpec; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.context.config.annotation.RefreshScope; -import org.springframework.context.annotation.Configuration; - -@RefreshScope -@EnableConfigurationProperties -@Configuration -@ConfigurationProperties(prefix = "proxy.filebrowser") -public class FileBrowserProperties extends ProxySpec { - -} \ No newline at end of file diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/StatCollectorProperties.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/StatCollectorProperties.java deleted file mode 100644 index a9f1bec1..00000000 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/StatCollectorProperties.java +++ /dev/null @@ -1,28 +0,0 @@ -package hk.edu.polyu.comp.vlabcontroller.spec; - -import lombok.Getter; -import lombok.Setter; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.context.config.annotation.RefreshScope; -import org.springframework.context.annotation.Configuration; - -@RefreshScope -@EnableConfigurationProperties -@Configuration -@ConfigurationProperties(prefix = "proxy.usage-stats-url") -public class StatCollectorProperties { - @Getter - @Setter - private String influxURL = ""; - @Getter - @Setter - private String jdbcURL = ""; - @Getter - @Setter - private String micrometerURL = ""; - - public boolean backendExists() { - return !influxURL.isEmpty() || !jdbcURL.isEmpty() || !micrometerURL.isEmpty(); - } -} diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/expression/ExpressionAwareContainerSpec.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/expression/ExpressionAwareContainerSpec.java index 7c5c009c..d3fd3369 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/expression/ExpressionAwareContainerSpec.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/expression/ExpressionAwareContainerSpec.java @@ -1,14 +1,13 @@ package hk.edu.polyu.comp.vlabcontroller.spec.expression; import hk.edu.polyu.comp.vlabcontroller.model.runtime.Proxy; -import hk.edu.polyu.comp.vlabcontroller.model.spec.EntryPointSpec; import hk.edu.polyu.comp.vlabcontroller.model.spec.ContainerSpec; +import hk.edu.polyu.comp.vlabcontroller.model.spec.EntryPointSpec; import hk.edu.polyu.comp.vlabcontroller.model.spec.ResourceSpec; import io.fabric8.kubernetes.api.model.VolumeMount; import io.fabric8.kubernetes.api.model.VolumeMountBuilder; import org.springframework.data.util.Pair; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -43,9 +42,8 @@ public List getCmd() { } public Map getEnv() { - Map env = new HashMap<>(); - source.getEnv().entrySet().stream().forEach(e -> env.put(e.getKey(), resolve(e.getValue()))); - return env; + return source.getEnv().entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, x -> resolve(x.getValue()))); } public String getEnvFile() { @@ -74,10 +72,8 @@ public Map getPortMapping() { @Override public ResourceSpec getResources() { - ResourceSpec resourceSpec = new ResourceSpec(); - source.getResources().getLimits().forEach((key, value) -> resourceSpec.getLimits().put(key, resolve(value))); - source.getResources().getRequests().forEach((key, value) -> resourceSpec.getRequests().put(key, resolve(value))); - return resourceSpec; + var resources = source.getResources(); + return ResourceSpec.builder().limits(resources.getLimits()).requests(resources.getRequests()).build(); } public boolean isPrivileged() { @@ -91,9 +87,8 @@ public Map> getRuntimeLabels() { } public Map getSettings() { - Map settings = new HashMap<>(); - source.getSettings().entrySet().stream().forEach(e -> settings.put(e.getKey(), resolve(e.getValue()))); - return settings; + return source.getSettings().entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, x -> resolve(x.getValue()))); } public List getVolumeMounts() { diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/expression/SpecExpressionContext.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/expression/SpecExpressionContext.java index 2bfa0853..0e081d07 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/expression/SpecExpressionContext.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/expression/SpecExpressionContext.java @@ -3,16 +3,19 @@ import hk.edu.polyu.comp.vlabcontroller.model.runtime.Proxy; import hk.edu.polyu.comp.vlabcontroller.model.spec.ContainerSpec; import hk.edu.polyu.comp.vlabcontroller.model.spec.ProxySpec; +import lombok.Getter; public class SpecExpressionContext { - + @Getter private ContainerSpec containerSpec; + @Getter private ProxySpec proxySpec; + @Getter private Proxy proxy; public static SpecExpressionContext create(Object... objects) { - SpecExpressionContext ctx = new SpecExpressionContext(); - for (Object o : objects) { + var ctx = new SpecExpressionContext(); + for (var o : objects) { if (o instanceof ContainerSpec) { ctx.containerSpec = (ContainerSpec) o; } else if (o instanceof ProxySpec) { @@ -24,15 +27,4 @@ public static SpecExpressionContext create(Object... objects) { return ctx; } - public ContainerSpec getContainerSpec() { - return containerSpec; - } - - public ProxySpec getProxySpec() { - return proxySpec; - } - - public Proxy getProxy() { - return proxy; - } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/expression/SpecExpressionResolver.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/expression/SpecExpressionResolver.java index a00d0665..719277af 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/expression/SpecExpressionResolver.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/expression/SpecExpressionResolver.java @@ -1,11 +1,10 @@ package hk.edu.polyu.comp.vlabcontroller.spec.expression; +import lombok.RequiredArgsConstructor; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.ApplicationContext; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.expression.*; -import org.springframework.core.convert.ConversionService; -import org.springframework.expression.Expression; import org.springframework.expression.ExpressionParser; import org.springframework.expression.ParserContext; import org.springframework.expression.spel.standard.SpelExpressionParser; @@ -21,6 +20,7 @@ * Note: inspired by org.springframework.context.expression.StandardBeanExpressionResolver */ @Component +@RequiredArgsConstructor public class SpecExpressionResolver { private final Map evaluationCache = new ConcurrentHashMap<>(8); @@ -40,33 +40,29 @@ public String getExpressionSuffix() { return StandardBeanExpressionResolver.DEFAULT_EXPRESSION_SUFFIX; } }; - private final ExpressionParser expressionParser; + private final ExpressionParser expressionParser = new SpelExpressionParser(); private final ApplicationContext appContext; - public SpecExpressionResolver(ApplicationContext appContext) { - this.expressionParser = new SpelExpressionParser(); - this.appContext = appContext; - } - public Object evaluate(String expression, SpecExpressionContext context) { if (expression == null) return null; if (expression.isEmpty()) return ""; - Expression expr = this.expressionParser.parseExpression(expression, this.beanExpressionParserContext); + var expr = this.expressionParser.parseExpression(expression, this.beanExpressionParserContext); ConfigurableBeanFactory beanFactory = ((ConfigurableApplicationContext) appContext).getBeanFactory(); - StandardEvaluationContext sec = evaluationCache.get(context); + var sec = evaluationCache.get(context); if (sec == null) { - sec = new StandardEvaluationContext(); - sec.setRootObject(context); - sec.addPropertyAccessor(new BeanExpressionContextAccessor()); - sec.addPropertyAccessor(new BeanFactoryAccessor()); - sec.addPropertyAccessor(new MapAccessor()); - sec.addPropertyAccessor(new EnvironmentAccessor()); - sec.setBeanResolver(new BeanFactoryResolver(appContext)); - sec.setTypeLocator(new StandardTypeLocator(beanFactory.getBeanClassLoader())); - ConversionService conversionService = beanFactory.getConversionService(); + sec = new StandardEvaluationContext() {{ + setRootObject(context); + addPropertyAccessor(new BeanExpressionContextAccessor()); + addPropertyAccessor(new BeanFactoryAccessor()); + addPropertyAccessor(new MapAccessor()); + addPropertyAccessor(new EnvironmentAccessor()); + setBeanResolver(new BeanFactoryResolver(appContext)); + setTypeLocator(new StandardTypeLocator(beanFactory.getBeanClassLoader())); + }}; + var conversionService = beanFactory.getConversionService(); if (conversionService != null) sec.setTypeConverter(new StandardTypeConverter(conversionService)); evaluationCache.put(context, sec); } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/impl/DefaultSpecMergeStrategy.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/impl/DefaultSpecMergeStrategy.java index e85bff7e..b31c4138 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/impl/DefaultSpecMergeStrategy.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/impl/DefaultSpecMergeStrategy.java @@ -5,8 +5,11 @@ import hk.edu.polyu.comp.vlabcontroller.spec.IProxySpecMergeStrategy; import hk.edu.polyu.comp.vlabcontroller.spec.ProxySpecException; import hk.edu.polyu.comp.vlabcontroller.spec.setting.SettingTypeRegistry; +import lombok.RequiredArgsConstructor; +import lombok.val; import org.springframework.stereotype.Component; +import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -14,40 +17,29 @@ * This default merge strategy allows any combination of base spec, runtime spec and runtime settings. */ @Component +@RequiredArgsConstructor public class DefaultSpecMergeStrategy implements IProxySpecMergeStrategy { private final SettingTypeRegistry settingTypeRegistry; - public DefaultSpecMergeStrategy(SettingTypeRegistry settingTypeRegistry) { - this.settingTypeRegistry = settingTypeRegistry; - } - @Override public ProxySpec merge(ProxySpec baseSpec, ProxySpec runtimeSpec, Set runtimeSettings) throws ProxySpecException { - if (baseSpec == null && runtimeSpec == null) + val hasBase = baseSpec != null; + val hasRuntime = runtimeSpec != null; + if (!(hasBase || hasRuntime)) throw new ProxySpecException("No base or runtime proxy spec provided"); - ProxySpec finalSpec = new ProxySpec(); - copySpec(baseSpec, finalSpec); - copySpec(runtimeSpec, finalSpec); + var finalSpec = (hasBase && hasRuntime) + ? runtimeSpec.copyToBuilder(baseSpec.copyBuilder()).build() + : (hasBase ? baseSpec : runtimeSpec); - if (runtimeSettings != null) { - for (RuntimeSetting setting : runtimeSettings) { - settingTypeRegistry.applySetting(setting, finalSpec); - } + for (var setting : Optional.ofNullable(runtimeSettings).orElse(Set.of())) { + settingTypeRegistry.applySetting(setting, finalSpec); } if (finalSpec.getId() == null) { - var id = UUID.randomUUID().toString(); - finalSpec.setId(id); - for (var containerSpec : finalSpec.getContainerSpecs()) { - containerSpec.getEnv().put("PUBLIC_PATH", DefaultSpecProvider.getPublicPath(id)); - } + finalSpec.setId(UUID.randomUUID().toString()); + finalSpec.populateContainerSpecPublicPathById(); } return finalSpec; } - - protected void copySpec(ProxySpec from, ProxySpec to) { - if (from == null || to == null) return; - from.copy(to); - } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/impl/DefaultSpecProvider.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/impl/DefaultSpecProvider.java deleted file mode 100644 index 84a75efd..00000000 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/impl/DefaultSpecProvider.java +++ /dev/null @@ -1,51 +0,0 @@ -package hk.edu.polyu.comp.vlabcontroller.spec.impl; - -import hk.edu.polyu.comp.vlabcontroller.model.spec.ProxySpec; -import hk.edu.polyu.comp.vlabcontroller.spec.IProxySpecProvider; -import hk.edu.polyu.comp.vlabcontroller.util.SessionHelper; -import lombok.Getter; -import lombok.Setter; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.context.annotation.Primary; -import org.springframework.core.env.Environment; -import org.springframework.stereotype.Component; - -import javax.annotation.PostConstruct; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -@Component -@Primary -@ConfigurationProperties(prefix = "proxy") -public class DefaultSpecProvider implements IProxySpecProvider { - @Getter - @Setter - private List specs = new ArrayList<>(); - - public ProxySpec getSpec(String id) { - if (id == null || id.isEmpty()) return null; - return specs.stream().filter(s -> id.equals(s.getId())).findAny().orElse(null); - } - - @PostConstruct - public void afterPropertiesSet() { - this.specs.stream().collect(Collectors.groupingBy(ProxySpec::getId)).forEach((id, duplicateSpecs) -> { - if (duplicateSpecs.size() > 1) - throw new IllegalArgumentException(String.format("Configuration error: spec with id '%s' is defined multiple times", id)); - }); - } - - private static Environment environment; - - @Autowired - public void setEnvironment(Environment env) { - DefaultSpecProvider.environment = env; - } - - public static String getPublicPath(String appName) { - String contextPath = SessionHelper.getContextPath(environment, true); - return contextPath + "app_direct/" + appName + "/"; - } -} diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/impl/VLabControllerSpecMergeStrategy.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/impl/VLabControllerSpecMergeStrategy.java index 2f0636b1..fe5cc806 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/impl/VLabControllerSpecMergeStrategy.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/impl/VLabControllerSpecMergeStrategy.java @@ -19,10 +19,7 @@ public ProxySpec merge(ProxySpec baseSpec, ProxySpec runtimeSpec, Set= target.getContainerSpecs().size()) doFail(spec, "container index too high"); targetObject = target.getContainerSpecs().get(index); if (nameParts.length < 2) doFail(spec, "no container field specified"); diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/setting/SettingTypeRegistry.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/setting/SettingTypeRegistry.java index 2bc88ed9..ea949b74 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/setting/SettingTypeRegistry.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/setting/SettingTypeRegistry.java @@ -4,6 +4,7 @@ import hk.edu.polyu.comp.vlabcontroller.model.spec.ProxySpec; import hk.edu.polyu.comp.vlabcontroller.model.spec.RuntimeSettingSpec; import hk.edu.polyu.comp.vlabcontroller.spec.ProxySpecException; +import lombok.Setter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -22,8 +23,7 @@ */ @Component public class SettingTypeRegistry { - - @Autowired(required = false) + @Setter(onMethod_ = {@Autowired(required = false)}) private Map typeMap = new HashMap<>(); public RuntimeSettingSpec resolveSpec(RuntimeSetting setting, ProxySpec proxySpec) { @@ -31,7 +31,7 @@ public RuntimeSettingSpec resolveSpec(RuntimeSetting setting, ProxySpec proxySpe } public IRuntimeSettingType resolveSpecType(RuntimeSettingSpec settingSpec) { - String type = settingSpec.getType(); + var type = settingSpec.getType(); if (type == null || type.isEmpty()) { //TODO try to determine the type via the spec config type = "setting.type.string"; @@ -40,10 +40,10 @@ public IRuntimeSettingType resolveSpecType(RuntimeSettingSpec settingSpec) { } public void applySetting(RuntimeSetting setting, ProxySpec targetSpec) throws ProxySpecException { - RuntimeSettingSpec settingSpec = resolveSpec(setting, targetSpec); + var settingSpec = resolveSpec(setting, targetSpec); if (settingSpec == null) return; - IRuntimeSettingType type = resolveSpecType(settingSpec); + var type = resolveSpecType(settingSpec); if (type == null) throw new ProxySpecException("Unknown setting type: " + settingSpec.getType()); type.apply(setting, settingSpec, targetSpec); diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/setting/type/AbstractSettingType.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/setting/type/AbstractSettingType.java index 3007c473..fba7377b 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/setting/type/AbstractSettingType.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/spec/setting/type/AbstractSettingType.java @@ -8,6 +8,7 @@ import hk.edu.polyu.comp.vlabcontroller.spec.ProxySpecException; import hk.edu.polyu.comp.vlabcontroller.spec.setting.IRuntimeSettingType; import hk.edu.polyu.comp.vlabcontroller.spec.setting.SettingSpecMapper; +import lombok.Setter; /** * Example runtime settings: @@ -34,13 +35,12 @@ * Each class translates into several settings, e.g. cpu & memory */ public abstract class AbstractSettingType implements IRuntimeSettingType { - - @Inject + @Setter(onMethod_ = {@Inject}) protected SettingSpecMapper mapper; @Override public void apply(RuntimeSetting setting, RuntimeSettingSpec settingSpec, ProxySpec targetSpec) throws ProxySpecException { - Object value = getValue(setting, settingSpec); + var value = getValue(setting, settingSpec); if (value == null) return; mapper.mapValue(value, settingSpec, targetSpec); } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/stat/StatCollectorRegistry.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/stat/StatCollectorRegistry.java index 946473a3..94243161 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/stat/StatCollectorRegistry.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/stat/StatCollectorRegistry.java @@ -1,62 +1,56 @@ package hk.edu.polyu.comp.vlabcontroller.stat; -import hk.edu.polyu.comp.vlabcontroller.spec.StatCollectorProperties; +import hk.edu.polyu.comp.vlabcontroller.config.ProxyProperties; import hk.edu.polyu.comp.vlabcontroller.stat.impl.InfluxDBCollector; import hk.edu.polyu.comp.vlabcontroller.stat.impl.JDBCCollector; import hk.edu.polyu.comp.vlabcontroller.stat.impl.Micrometer; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.support.BeanDefinitionBuilder; import org.springframework.beans.factory.support.DefaultListableBeanFactory; +import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.context.ApplicationContext; import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.core.env.Environment; import java.util.function.Consumer; +@Slf4j @Configuration +@RequiredArgsConstructor +@RefreshScope class StatCollectorFactory { - - private final Logger log = LogManager.getLogger(StatCollectorFactory.class); - - private final Environment environment; private final ApplicationContext applicationContext; - private final StatCollectorProperties statCollectorProperties; - - public StatCollectorFactory(Environment environment, ApplicationContext applicationContext, StatCollectorProperties statCollectorProperties) { - this.environment = environment; - this.applicationContext = applicationContext; - this.statCollectorProperties = statCollectorProperties; - } + private final ProxyProperties proxyProperties; @Bean public IStatCollector statsCollector() { // create beans manually, spring will not create beans automatically when null returned - if (!statCollectorProperties.backendExists()) { + var url = proxyProperties.getUsageStats().getUrl(); + if (!url.backendExists()) { log.info("Disabled. Usage statistics will not be processed."); return null; } - ConfigurableApplicationContext configurableApplicationContext = (ConfigurableApplicationContext) applicationContext; - DefaultListableBeanFactory defaultListableBeanFactory = (DefaultListableBeanFactory) configurableApplicationContext.getAutowireCapableBeanFactory(); + var configurableApplicationContext = (ConfigurableApplicationContext) applicationContext; + var defaultListableBeanFactory = (DefaultListableBeanFactory) configurableApplicationContext.getAutowireCapableBeanFactory(); - Consumer> createBean = (Class klass) -> { - BeanDefinitionBuilder beanDefinitionBuilder = BeanDefinitionBuilder.genericBeanDefinition(klass); + var createBean = (Consumer>) (Class klass) -> { + var beanDefinitionBuilder = BeanDefinitionBuilder.genericBeanDefinition(klass); defaultListableBeanFactory.registerBeanDefinition(klass.getName() + "Bean", beanDefinitionBuilder.getBeanDefinition()); }; - if (statCollectorProperties.getInfluxURL().contains("/write?db=")) { + if (url.getInflux().contains("/write?db=")) { createBean.accept(InfluxDBCollector.class); - log.info("Influx DB backend enabled, sending usage statics to {}", statCollectorProperties.getInfluxURL()); + log.info("Influx DB backend enabled, sending usage statics to {}", url.getInflux()); } - if (statCollectorProperties.getJdbcURL().contains("jdbc")) { + if (url.getJdbc().contains("jdbc")) { createBean.accept(JDBCCollector.class); - log.info("JDBC backend enabled, sending usage statistics to {}", statCollectorProperties.getJdbcURL()); + log.info("JDBC backend enabled, sending usage statistics to {}", url.getJdbc()); } - if (statCollectorProperties.getMicrometerURL().contains("micrometer")) { + if (url.getMicrometer().contains("micrometer")) { createBean.accept(Micrometer.class); log.info("Prometheus (Micrometer) backend enabled"); } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/stat/impl/AbstractDbCollector.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/stat/impl/AbstractDbCollector.java index 913b05bf..879ae448 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/stat/impl/AbstractDbCollector.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/stat/impl/AbstractDbCollector.java @@ -1,12 +1,20 @@ package hk.edu.polyu.comp.vlabcontroller.stat.impl; -import hk.edu.polyu.comp.vlabcontroller.stat.IStatCollector; +import hk.edu.polyu.comp.vlabcontroller.config.ProxyProperties; import hk.edu.polyu.comp.vlabcontroller.event.*; +import hk.edu.polyu.comp.vlabcontroller.stat.IStatCollector; +import lombok.Setter; +import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.context.event.EventListener; +import javax.inject.Inject; import java.io.IOException; +@RefreshScope public abstract class AbstractDbCollector implements IStatCollector { + @Setter(onMethod_ = {@Inject}) + protected ProxyProperties proxyProperties; + @EventListener public void onUserLogoutEvent(UserLogoutEvent event) throws IOException { writeToDb(event.getTimestamp(), event.getUserId(), "Logout", null, String.valueOf(event.getWasExpired())); diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/stat/impl/InfluxDBCollector.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/stat/impl/InfluxDBCollector.java index c13114e1..b33dc5a6 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/stat/impl/InfluxDBCollector.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/stat/impl/InfluxDBCollector.java @@ -1,10 +1,5 @@ package hk.edu.polyu.comp.vlabcontroller.stat.impl; -import org.apache.commons.io.IOUtils; -import org.springframework.core.env.Environment; - -import javax.annotation.PostConstruct; -import javax.inject.Inject; import java.io.ByteArrayOutputStream; import java.io.DataOutputStream; import java.io.IOException; @@ -19,39 +14,33 @@ * usage-stats-url: http://localhost:8086/write?db=usagestats */ public class InfluxDBCollector extends AbstractDbCollector { - - private String destination; - @Inject - private Environment environment; - - @PostConstruct - public void init() { - destination = environment.getProperty("proxy.usage-stats-url.influx-url"); + public String getDestination() { + return proxyProperties.getUsageStats().getUrl().getInflux(); } @Override protected void writeToDb(long timestamp, String userId, String type, String specId, String info) throws IOException { - String identifier = environment.getProperty("proxy.identifier-value", "default-identifier"); - String body = String.format("event,username=%s,type=%s,identifier=%s specid=\"%s\",info=\"%s\"", + var identifier = proxyProperties.getIdentifierValue(); + var body = String.format("event,username=%s,type=%s,identifier=%s specid=\"%s\",info=\"%s\"", userId.replace(" ", "\\ "), type.replace(" ", "\\ "), identifier.replace(" ", "\\ "), Optional.ofNullable(specId).orElse(""), Optional.ofNullable(info).orElse("")); - HttpURLConnection conn = (HttpURLConnection) new URL(destination).openConnection(); + var conn = (HttpURLConnection) new URL(getDestination()).openConnection(); conn.setRequestMethod("POST"); conn.setDoOutput(true); - try (DataOutputStream dos = new DataOutputStream(conn.getOutputStream())) { + try (var dos = new DataOutputStream(conn.getOutputStream())) { dos.write(body.getBytes(StandardCharsets.UTF_8)); dos.flush(); } - int responseCode = conn.getResponseCode(); + var responseCode = conn.getResponseCode(); if (responseCode == 204) { // All is well. } else { - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - IOUtils.copy(conn.getErrorStream(), bos); + var bos = new ByteArrayOutputStream(); + conn.getErrorStream().transferTo(bos); throw new IOException(bos.toString()); } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/stat/impl/JDBCCollector.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/stat/impl/JDBCCollector.java index e7bf5107..d3ed37ce 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/stat/impl/JDBCCollector.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/stat/impl/JDBCCollector.java @@ -1,13 +1,9 @@ package hk.edu.polyu.comp.vlabcontroller.stat.impl; import com.zaxxer.hikari.HikariDataSource; -import org.springframework.core.env.Environment; import javax.annotation.PostConstruct; -import javax.inject.Inject; import java.io.IOException; -import java.sql.Connection; -import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Timestamp; @@ -32,57 +28,55 @@ * varchar(128), data text ); */ public class JDBCCollector extends AbstractDbCollector { - private HikariDataSource ds; - @Inject - private Environment environment; - @PostConstruct public void init() { - String baseURL = environment.getProperty("proxy.usage-stats-url.jdbc-url"); - String username = environment.getProperty("proxy.usage-stats-username", "monetdb"); - String password = environment.getProperty("proxy.usage-stats-password", "monetdb"); - ds = new HikariDataSource(); - ds.setJdbcUrl(baseURL); - ds.setUsername(username); - ds.setPassword(password); - ds.addDataSourceProperty("useJDBCCompliantTimezoneShift", "true"); - ds.addDataSourceProperty("serverTimezone", "UTC"); + var usageStats = proxyProperties.getUsageStats(); + var baseURL = usageStats.getUrl().getJdbc(); + var username = usageStats.getUsername(); + var password = usageStats.getPassword(); + ds = new HikariDataSource() {{ + setJdbcUrl(baseURL); + setUsername(username); + setPassword(password); + addDataSourceProperty("useJDBCCompliantTimezoneShift", "true"); + addDataSourceProperty("serverTimezone", "UTC"); + }}; - Long connectionTimeout = environment.getProperty("proxy.usage-stats-hikari.connection-timeout", Long.class); - if (connectionTimeout != null) { - ds.setConnectionTimeout(connectionTimeout); + var hikari = usageStats.getHikari(); + var connectionTimeout = hikari.getConnectionTimeout(); + if (!connectionTimeout.isNegative()) { + ds.setConnectionTimeout(connectionTimeout.toMillis()); } - Long idleTimeout = environment.getProperty("proxy.usage-stats-hikari.idle-timeout", Long.class); - if (idleTimeout != null) { - ds.setIdleTimeout(idleTimeout); + var idleTimeout = hikari.getIdleTimeout(); + if (!idleTimeout.isNegative()) { + ds.setIdleTimeout(idleTimeout.toMillis()); } - Long maxLifetime = environment.getProperty("proxy.usage-stats-hikari.max-lifetime", Long.class); - if (maxLifetime != null) { - ds.setMaxLifetime(maxLifetime); + var maxLifetime = hikari.getMaxLifetime(); + if (!maxLifetime.isNegative()) { + ds.setMaxLifetime(maxLifetime.toMillis()); } - Integer minimumIdle = environment.getProperty("proxy.usage-stats-hikari.minimum-idle", Integer.class); - if (minimumIdle != null) { + var minimumIdle = hikari.getMinimumIdle(); + if (minimumIdle >= 0) { ds.setMinimumIdle(minimumIdle); } - Integer maximumPoolSize = environment.getProperty("proxy.usage-stats-hikari.maximum-pool-size", Integer.class); - if (maximumPoolSize != null) { + var maximumPoolSize = hikari.getMaximumPoolSize(); + if (maximumPoolSize >= 0) { ds.setMaximumPoolSize(maximumPoolSize); } - } @Override protected void writeToDb(long timestamp, String userId, String type, String specId, String info) throws IOException { - String identifier = environment.getProperty("proxy.identifier-value", "default-identifier"); - String sql = "INSERT INTO event(event_time, username, type, specid, identifier, info) VALUES (?,?,?,?,?,?)"; - try (Connection con = ds.getConnection()) { - try (PreparedStatement stmt = con.prepareStatement(sql)) { + var identifier = proxyProperties.getIdentifierValue(); + var sql = "INSERT INTO event(event_time, username, type, specid, identifier, info) VALUES (?,?,?,?,?,?)"; + try (var con = ds.getConnection()) { + try (var stmt = con.prepareStatement(sql)) { stmt.setTimestamp(1, new Timestamp(timestamp)); stmt.setString(2, userId); stmt.setString(3, type); diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/stat/impl/Micrometer.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/stat/impl/Micrometer.java index 92095ecb..b702bc87 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/stat/impl/Micrometer.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/stat/impl/Micrometer.java @@ -1,30 +1,27 @@ package hk.edu.polyu.comp.vlabcontroller.stat.impl; +import hk.edu.polyu.comp.vlabcontroller.event.*; import hk.edu.polyu.comp.vlabcontroller.service.ProxyService; import hk.edu.polyu.comp.vlabcontroller.stat.IStatCollector; -import hk.edu.polyu.comp.vlabcontroller.event.*; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.MeterRegistry; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; import org.springframework.context.event.EventListener; import javax.annotation.PostConstruct; import javax.inject.Inject; +@Slf4j public class Micrometer implements IStatCollector { - - private final Logger logger = LogManager.getLogger(getClass()); - @Inject + @Setter(onMethod_ = {@Inject}) private MeterRegistry registry; - @Inject + @Setter(onMethod_ = {@Inject}) private ProxyService proxyService; - private Counter appStartFailedCounter; + private Counter appStartFailedCounter; private Counter authFailedCounter; - private Counter userLogins; - private Counter userLogouts; @PostConstruct @@ -39,14 +36,14 @@ public void init() { @EventListener public void onUserLogoutEvent(UserLogoutEvent event) { - logger.debug("UserLogoutEvent [user: {}, sessionId: {}, expired: {}]", event.getUserId(), event.getSessionId(), event.getWasExpired()); + log.debug("UserLogoutEvent [user: {}, sessionId: {}, expired: {}]", event.getUserId(), event.getSessionId(), event.getWasExpired()); userLogouts.increment(); registry.counter("userIdLogouts", "user.id", event.getUserId()).increment(); } @EventListener public void onUserLoginEvent(UserLoginEvent event) { - logger.debug("UserLoginEvent [user: {}, sessionId: {}]", event.getUserId(), event.getSessionId()); + log.debug("UserLoginEvent [user: {}, sessionId: {}]", event.getUserId(), event.getSessionId()); userLogins.increment(); registry.counter("userIdLogins", "user.id", event.getUserId()).increment(); registry.counter("userIdLogouts", "user.id", event.getUserId()).increment(0); @@ -54,27 +51,27 @@ public void onUserLoginEvent(UserLoginEvent event) { @EventListener public void onProxyStartEvent(ProxyStartEvent event) { - logger.debug("ProxyStartEvent [user: {}, startupTime: {}]", event.getUserId(), event.getStartupTime()); + log.debug("ProxyStartEvent [user: {}, startupTime: {}]", event.getUserId(), event.getStartupTime()); registry.counter("appStarts", "spec.id", event.getSpecId(), "user.id", event.getUserId()).increment(); registry.timer("startupTime", "spec.id", event.getSpecId(), "user.id", event.getUserId()).record(event.getStartupTime()); } @EventListener public void onProxyStopEvent(ProxyStopEvent event) { - logger.debug("ProxyStopEvent [user: {}, usageTime: {}]", event.getUserId(), event.getUsageTime()); + log.debug("ProxyStopEvent [user: {}, usageTime: {}]", event.getUserId(), event.getUsageTime()); registry.counter("appStops", "spec.id", event.getSpecId(), "user.id", event.getUserId()).increment(); registry.timer("usageTime", "spec.id", event.getSpecId(), "user.id", event.getUserId()).record(event.getUsageTime()); } @EventListener public void onProxyStartFailedEvent(ProxyStartFailedEvent event) { - logger.debug("ProxyStartFailedEvent [user: {}, specId: {}]", event.getUserId(), event.getSpecId()); + log.debug("ProxyStartFailedEvent [user: {}, specId: {}]", event.getUserId(), event.getSpecId()); appStartFailedCounter.increment(); } @EventListener public void onAuthFailedEvent(AuthFailedEvent event) { - logger.debug("AuthFailedEvent [user: {}, sessionId: {}]", event.getUserId(), event.getSessionId()); + log.debug("AuthFailedEvent [user: {}, sessionId: {}]", event.getUserId(), event.getSessionId()); authFailedCounter.increment(); } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/ui/AuthController.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/ui/AuthController.java index 4eac19af..d8fc5536 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/ui/AuthController.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/ui/AuthController.java @@ -3,6 +3,7 @@ import hk.edu.polyu.comp.vlabcontroller.api.BaseController; import hk.edu.polyu.comp.vlabcontroller.auth.IAuthenticationBackend; import hk.edu.polyu.comp.vlabcontroller.auth.impl.OpenIDAuthenticationBackend; +import lombok.RequiredArgsConstructor; import org.springframework.core.env.Environment; import org.springframework.stereotype.Controller; import org.springframework.ui.ModelMap; @@ -13,16 +14,10 @@ import java.util.Optional; @Controller +@RequiredArgsConstructor public class AuthController extends BaseController { - - private final Environment environment; - private final IAuthenticationBackend auth; - - public AuthController(Environment environment, IAuthenticationBackend auth) { - this.environment = environment; - this.auth = auth; - } + private final Environment environment; @GetMapping(value = "/login") public Object getLoginPage(@RequestParam Optional error, ModelMap map) { diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/ui/ErrorController.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/ui/ErrorController.java index 3d49c413..1ce1dd53 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/ui/ErrorController.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/ui/ErrorController.java @@ -2,7 +2,7 @@ import hk.edu.polyu.comp.vlabcontroller.api.BaseController; import hk.edu.polyu.comp.vlabcontroller.auth.impl.keycloak.AuthenticationFailureHandler; -import lombok.extern.log4j.Log4j2; +import lombok.extern.slf4j.Slf4j; import org.keycloak.adapters.OIDCAuthenticationError; import org.keycloak.adapters.springsecurity.authentication.KeycloakCookieBasedRedirect; import org.springframework.http.HttpStatus; @@ -21,37 +21,36 @@ import java.util.HashMap; import java.util.Map; -@Log4j2 +@Slf4j @Controller @RequestMapping("/error") public class ErrorController extends BaseController implements org.springframework.boot.web.servlet.error.ErrorController { - @RequestMapping(produces = "text/html") public String handleError(ModelMap map, HttpServletRequest request, HttpServletResponse response) { // handle keycloak errors - Object obj = request.getSession().getAttribute(AuthenticationFailureHandler.SP_KEYCLOAK_ERROR_REASON); + var obj = request.getSession().getAttribute(AuthenticationFailureHandler.SP_KEYCLOAK_ERROR_REASON); if (obj instanceof OIDCAuthenticationError.Reason) { request.getSession().removeAttribute(AuthenticationFailureHandler.SP_KEYCLOAK_ERROR_REASON); - OIDCAuthenticationError.Reason reason = (OIDCAuthenticationError.Reason) obj; + var reason = (OIDCAuthenticationError.Reason) obj; if (reason == OIDCAuthenticationError.Reason.INVALID_STATE_COOKIE || reason == OIDCAuthenticationError.Reason.STALE_TOKEN) { // These errors are typically caused by users using wrong bookmarks (e.g. bookmarks with states in) // or when some cookies got stale. However, the user is logged into the IDP, therefore it's enough to // send the user to the main page, and they will get logged in automatically. - response.addCookie(KeycloakCookieBasedRedirect.createCookieFromRedirectUrl((String) null)); + response.addCookie(KeycloakCookieBasedRedirect.createCookieFromRedirectUrl(null)); return "redirect:/"; } else { return "redirect:/auth-error"; } } - Throwable exception = (Throwable) request.getAttribute("javax.servlet.error.exception"); + var exception = (Throwable) request.getAttribute("javax.servlet.error.exception"); if (exception == null) { exception = (Throwable) request.getAttribute("SPRING_SECURITY_LAST_EXCEPTION"); } - String[] msg = createMsgStack(exception); + var msg = createMsgStack(exception); if (exception == null) { msg[0] = HttpStatus.valueOf(response.getStatus()).getReasonPhrase(); } @@ -62,7 +61,7 @@ public String handleError(ModelMap map, HttpServletRequest request, HttpServletR if (isIllegalStateException(exception)) { log.warn("No state cookie on login attempt, force redirect to homepage"); - response.addCookie(KeycloakCookieBasedRedirect.createCookieFromRedirectUrl((String) null)); + response.addCookie(KeycloakCookieBasedRedirect.createCookieFromRedirectUrl(null)); return "redirect:/"; } @@ -77,8 +76,8 @@ public String handleError(ModelMap map, HttpServletRequest request, HttpServletR @RequestMapping(consumes = "application/json", produces = "application/json") @ResponseBody public ResponseEntity> error(HttpServletRequest request, HttpServletResponse response) { - Throwable exception = (Throwable) request.getAttribute("javax.servlet.error.exception"); - String[] msg = createMsgStack(exception); + var exception = (Throwable) request.getAttribute("javax.servlet.error.exception"); + var msg = createMsgStack(exception); Map map = new HashMap<>(); map.put("message", msg[0]); @@ -92,16 +91,16 @@ public String getErrorPath() { } private String[] createMsgStack(Throwable exception) { - String message = ""; - String stackTrace = ""; + var message = ""; + var stackTrace = ""; if (exception instanceof NestedServletException && exception.getCause() instanceof Exception) { exception = exception.getCause(); } if (exception != null) { if (exception.getMessage() != null) message = exception.getMessage(); - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - try (PrintWriter writer = new PrintWriter(bos)) { + var bos = new ByteArrayOutputStream(); + try (var writer = new PrintWriter(bos)) { exception.printStackTrace(writer); } stackTrace = bos.toString(); diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/ui/FaviconConfig.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/ui/FaviconConfig.java index 849a6c95..2b13e505 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/ui/FaviconConfig.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/ui/FaviconConfig.java @@ -1,11 +1,13 @@ package hk.edu.polyu.comp.vlabcontroller.ui; -import org.apache.logging.log4j.LogManager; +import hk.edu.polyu.comp.vlabcontroller.config.ProxyProperties; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.core.Ordered; -import org.springframework.core.env.Environment; import org.springframework.http.MediaType; import org.springframework.http.MediaTypeFactory; import org.springframework.util.FileCopyUtils; @@ -16,40 +18,36 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.IOException; -import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Collections; +@Slf4j @Configuration +@RequiredArgsConstructor +@RefreshScope public class FaviconConfig { - private static final String CONTENT_TYPE_ICO = "image/x-icon"; - - private final Environment environment; - - public FaviconConfig(Environment environment) { - this.environment = environment; - } + private final ProxyProperties proxyProperties; @Bean @ConditionalOnProperty(name = "proxy.favicon-path") public SimpleUrlHandlerMapping customFaviconHandlerMapping() { byte[] cachedIcon = null; - Path iconPath = Paths.get(environment.getProperty("proxy.favicon-path")); + var iconPath = Paths.get(proxyProperties.getFaviconPath()); if (Files.isRegularFile(iconPath)) { - try (InputStream input = Files.newInputStream(iconPath)) { + try (var input = Files.newInputStream(iconPath)) { cachedIcon = FileCopyUtils.copyToByteArray(input); } catch (IOException e) { throw new IllegalArgumentException("Cannot read favicon: " + iconPath, e); } } else { - LogManager.getLogger(FaviconConfig.class).error("Invalid favicon path: " + iconPath); + log.error("Invalid favicon path: " + iconPath); } - SimpleUrlHandlerMapping mapping = new SimpleUrlHandlerMapping(); + var mapping = new SimpleUrlHandlerMapping(); mapping.setOrder(Ordered.HIGHEST_PRECEDENCE); mapping.setUrlMap(Collections.singletonMap("**/favicon.???", new CachedFaviconHttpRequestHandler(cachedIcon, iconPath))); return mapping; @@ -75,10 +73,10 @@ public void handleRequest(HttpServletRequest request, HttpServletResponse respon } private String getContentType() { - String fileName = iconPath.getFileName().toString().toLowerCase(); + var fileName = iconPath.getFileName().toString().toLowerCase(); if (fileName.endsWith(".ico")) return CONTENT_TYPE_ICO; - MediaType mediaType = MediaTypeFactory.getMediaType(fileName).orElse(MediaType.APPLICATION_OCTET_STREAM); + var mediaType = MediaTypeFactory.getMediaType(fileName).orElse(MediaType.APPLICATION_OCTET_STREAM); return mediaType.toString(); } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/ui/TemplateResolverConfig.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/ui/TemplateResolverConfig.java index cd38720b..003d0f1f 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/ui/TemplateResolverConfig.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/ui/TemplateResolverConfig.java @@ -1,38 +1,35 @@ package hk.edu.polyu.comp.vlabcontroller.ui; +import hk.edu.polyu.comp.vlabcontroller.config.ProxyProperties; +import lombok.RequiredArgsConstructor; +import org.springframework.cloud.context.config.annotation.RefreshScope; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.core.env.Environment; import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry; import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; import org.thymeleaf.templateresolver.FileTemplateResolver; -import javax.inject.Inject; - @Configuration +@RequiredArgsConstructor +@RefreshScope public class TemplateResolverConfig implements WebMvcConfigurer { - private final Environment environment; - - public TemplateResolverConfig(Environment environment) { - this.environment = environment; - } + private final ProxyProperties proxyProperties; @Override public void addResourceHandlers(ResourceHandlerRegistry registry) { registry.addResourceHandler("/assets/**") - .addResourceLocations("file:" + environment.getProperty("proxy.template-path") + "/assets/"); + .addResourceLocations("file:" + proxyProperties.getTemplatePath() + "/assets/"); } @Bean public FileTemplateResolver templateResolver() { - FileTemplateResolver resolver = new FileTemplateResolver(); - resolver.setPrefix(environment.getProperty("proxy.template-path") + "/"); - - resolver.setSuffix(".html"); - resolver.setTemplateMode("HTML5"); - resolver.setCacheable(false); - resolver.setCheckExistence(true); - resolver.setOrder(1); - return resolver; + return new FileTemplateResolver() {{ + setPrefix(proxyProperties.getTemplatePath() + "/"); + setSuffix(".html"); + setTemplateMode("HTML5"); + setCacheable(false); + setCheckExistence(true); + setOrder(1); + }}; } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/ChannelActiveListener.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/ChannelActiveListener.java index ca3a3f57..898b8131 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/ChannelActiveListener.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/ChannelActiveListener.java @@ -1,30 +1,28 @@ package hk.edu.polyu.comp.vlabcontroller.util; +import java.time.Duration; + /** * A listener that keeps track of whether a channel is active. */ public class ChannelActiveListener implements Runnable { - private long lastWrite = 0; + private Duration lastWrite = Duration.ZERO; @Override public void run() { - lastWrite = System.currentTimeMillis(); + lastWrite = Duration.ofMillis(System.currentTimeMillis()); } /** * Checks whether the channel was active in the provided period. */ - public boolean isActive(long period) { - long diff = System.currentTimeMillis() - lastWrite; + public boolean isActive(Duration period) { + var diff = Duration.ofMillis(System.currentTimeMillis()).minus(lastWrite); // make sure the period is at least 5 seconds // this ensures that when the socket is active, the ping is delayed for at least 5 seconds - if (period < 5000) { - period = 5000; - } - - return diff <= period; + return diff.compareTo(DurationUtil.atLeast(Duration.ofSeconds(5)).apply(period)) <= 0; } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/ConfigFileHelper.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/ConfigFileHelper.java index ca122add..bb384230 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/ConfigFileHelper.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/ConfigFileHelper.java @@ -4,56 +4,47 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import com.google.common.base.Charsets; import hk.edu.polyu.comp.vlabcontroller.VLabControllerApplication; +import io.vavr.CheckedFunction1; +import lombok.RequiredArgsConstructor; import org.springframework.core.env.Environment; import org.springframework.stereotype.Component; import java.io.File; -import java.io.IOException; import java.math.BigInteger; +import java.nio.charset.StandardCharsets; import java.nio.file.Paths; import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; +import java.util.Optional; @Component +@RequiredArgsConstructor public class ConfigFileHelper { private final Environment environment; - public ConfigFileHelper(Environment environment) { - this.environment = environment; - } - private File getConfigFile() { - String path = environment.getProperty("spring.config.location"); - path = path == null ? VLabControllerApplication.CONFIG_FILENAME : path; - File file = Paths.get(path).toFile(); - if (file.exists()) { - return file; - } - return null; + return Optional.ofNullable(environment.getProperty("spring.config.location")) + .or(() -> Optional.of(VLabControllerApplication.CONFIG_FILENAME)) + .map(path -> Paths.get(path).toFile()) + .filter(File::exists) + .orElse(null); } - public String getConfigHash() throws NoSuchAlgorithmException { - ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory()); - objectMapper.configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true); - objectMapper.configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true); - File file = getConfigFile(); - String configHash; - if (file == null) { - configHash = "unknown"; - return configHash; - } - try { - Object parsedConfig = objectMapper.readValue(file, Object.class); - String canonicalConfigFile = objectMapper.writeValueAsString(parsedConfig); - MessageDigest digest = MessageDigest.getInstance("SHA-1"); - digest.reset(); - digest.update(canonicalConfigFile.getBytes(Charsets.UTF_8)); - configHash = String.format("%040x", new BigInteger(1, digest.digest())); - return configHash; - } catch (IOException e) { - return "illegal"; - } + public String getConfigHash() { + var objectMapper = new ObjectMapper(new YAMLFactory()) {{ + configure(SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS, true); + configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true); + }}; + return Optional.ofNullable(getConfigFile()) + .map(CheckedFunction1.lift(file -> { + var parsedConfig = objectMapper.readValue(file, Object.class); + var canonicalConfigFile = objectMapper.writeValueAsString(parsedConfig); + var digest = MessageDigest.getInstance("SHA-1"); + digest.reset(); + digest.update(canonicalConfigFile.getBytes(StandardCharsets.UTF_8)); + return String.format("%040x", new BigInteger(1, digest.digest())); + })) + .map(x -> x.getOrElse("illegal")) + .orElse("unknown"); } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/ConfigUpdateListener.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/ConfigUpdateListener.java index 2cafb623..d3fbcb19 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/ConfigUpdateListener.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/ConfigUpdateListener.java @@ -1,8 +1,8 @@ package hk.edu.polyu.comp.vlabcontroller.util; import hk.edu.polyu.comp.vlabcontroller.event.ConfigUpdateEvent; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.cloud.context.environment.EnvironmentChangeEvent; import org.springframework.cloud.context.refresh.ContextRefresher; import org.springframework.context.annotation.Configuration; @@ -10,28 +10,23 @@ import java.security.NoSuchAlgorithmException; +@Slf4j @Configuration +@RequiredArgsConstructor public class ConfigUpdateListener { - protected final Logger log = LogManager.getLogger(getClass()); - private final ConfigFileHelper configFileHelper; private final ContextRefresher contextRefresher; - public ConfigUpdateListener(ConfigFileHelper configFileHelper, ContextRefresher contextRefresher) { - this.configFileHelper = configFileHelper; - this.contextRefresher = contextRefresher; - } - @EventListener public void onUpdate(ConfigUpdateEvent event) throws NoSuchAlgorithmException { - String hash = configFileHelper.getConfigHash(); + var hash = configFileHelper.getConfigHash(); if (hash.equals("unknown")) { log.info("No active application.yml set"); } else if (hash.equals("illegal")) { log.error("application.yml syntax error"); } else { log.info("Config changed, new hash = " + hash); - new Thread(() -> contextRefresher.refresh()).start(); + new Thread(contextRefresher::refresh).start(); } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/DelegatingStreamSinkConduit.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/DelegatingStreamSinkConduit.java index 02a264aa..be139851 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/DelegatingStreamSinkConduit.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/DelegatingStreamSinkConduit.java @@ -1,102 +1,23 @@ package hk.edu.polyu.comp.vlabcontroller.util; -import org.xnio.XnioIoThread; -import org.xnio.XnioWorker; -import org.xnio.channels.StreamSourceChannel; +import lombok.RequiredArgsConstructor; +import lombok.experimental.Delegate; import org.xnio.conduits.StreamSinkConduit; -import org.xnio.conduits.WriteReadyHandler; import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; -import java.util.concurrent.TimeUnit; +@RequiredArgsConstructor public class DelegatingStreamSinkConduit implements StreamSinkConduit { + @SuppressWarnings("unused") + private interface Write { + int write(ByteBuffer src) throws IOException; + } + @Delegate(excludes=Write.class) private final StreamSinkConduit delegate; private final Runnable writeListener; - - public DelegatingStreamSinkConduit(StreamSinkConduit delegate, Runnable writeListener) { - this.delegate = delegate; - this.writeListener = writeListener; - } - - @Override - public void terminateWrites() throws IOException { - delegate.terminateWrites(); - } - - @Override - public boolean isWriteShutdown() { - return delegate.isWriteShutdown(); - } - - @Override - public void resumeWrites() { - delegate.resumeWrites(); - } - - @Override - public void suspendWrites() { - delegate.suspendWrites(); - } - - @Override - public void wakeupWrites() { - delegate.wakeupWrites(); - } - - @Override - public boolean isWriteResumed() { - return delegate.isWriteResumed(); - } - - @Override - public void awaitWritable() throws IOException { - delegate.awaitWritable(); - } - - @Override - public void awaitWritable(long time, TimeUnit timeUnit) throws IOException { - delegate.awaitWritable(time, timeUnit); - } - - @Override - public XnioIoThread getWriteThread() { - return delegate.getWriteThread(); - } - - @Override - public void setWriteReadyHandler(WriteReadyHandler handler) { - delegate.setWriteReadyHandler(handler); - } - - @Override - public void truncateWrites() throws IOException { - delegate.truncateWrites(); - } - - @Override - public boolean flush() throws IOException { - return delegate.flush(); - } - - @Override - public XnioWorker getWorker() { - return delegate.getWorker(); - } - - @Override - public long transferFrom(FileChannel src, long position, long count) throws IOException { - return delegate.transferFrom(src, position, count); - } - - @Override - public long transferFrom(StreamSourceChannel source, long count, ByteBuffer throughBuffer) throws IOException { - return delegate.transferFrom(source, count, throughBuffer); - } - @Override public int write(ByteBuffer src) throws IOException { if (writeListener != null) { @@ -108,20 +29,4 @@ public int write(ByteBuffer src) throws IOException { public int writeWithoutNotifying(ByteBuffer src) throws IOException { return delegate.write(src); } - - @Override - public long write(ByteBuffer[] srcs, int offs, int len) throws IOException { - return delegate.write(srcs, offs, len); - } - - @Override - public int writeFinal(ByteBuffer src) throws IOException { - return delegate.writeFinal(src); - } - - @Override - public long writeFinal(ByteBuffer[] srcs, int offset, int length) throws IOException { - return delegate.writeFinal(srcs, offset, length); - } - } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/DelegatingStreamSourceConduit.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/DelegatingStreamSourceConduit.java index 5af5e989..3f11317a 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/DelegatingStreamSourceConduit.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/DelegatingStreamSourceConduit.java @@ -1,111 +1,34 @@ package hk.edu.polyu.comp.vlabcontroller.util; -import org.xnio.XnioIoThread; -import org.xnio.XnioWorker; -import org.xnio.channels.StreamSinkChannel; -import org.xnio.conduits.ReadReadyHandler; +import lombok.RequiredArgsConstructor; +import lombok.experimental.Delegate; import org.xnio.conduits.StreamSourceConduit; import java.io.IOException; import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; -import java.util.concurrent.TimeUnit; import java.util.function.Consumer; +@RequiredArgsConstructor public class DelegatingStreamSourceConduit implements StreamSourceConduit { + @SuppressWarnings("unused") + private interface Read { + int read(ByteBuffer dst) throws IOException; + } + @Delegate(excludes=Read.class) private final StreamSourceConduit delegate; private final Consumer readListener; - public DelegatingStreamSourceConduit(StreamSourceConduit delegate, Consumer readListener) { - this.delegate = delegate; - this.readListener = readListener; - } - - @Override - public void terminateReads() throws IOException { - delegate.terminateReads(); - } - - @Override - public boolean isReadShutdown() { - return delegate.isReadShutdown(); - } - - @Override - public void resumeReads() { - delegate.resumeReads(); - } - - @Override - public void suspendReads() { - delegate.suspendReads(); - } - - @Override - public void wakeupReads() { - delegate.wakeupReads(); - } - - @Override - public boolean isReadResumed() { - return delegate.isReadResumed(); - } - - @Override - public void awaitReadable() throws IOException { - delegate.awaitReadable(); - } - - @Override - public void awaitReadable(long time, TimeUnit timeUnit) throws IOException { - delegate.awaitReadable(time, timeUnit); - } - - @Override - public XnioIoThread getReadThread() { - return delegate.getReadThread(); - } - - @Override - public void setReadReadyHandler(ReadReadyHandler handler) { - delegate.setReadReadyHandler(handler); - } - - @Override - public XnioWorker getWorker() { - return delegate.getWorker(); - } - - @Override - public long transferTo(long position, long count, FileChannel target) throws IOException { - return delegate.transferTo(position, count, target); - } - - @Override - public long transferTo(long count, ByteBuffer throughBuffer, StreamSinkChannel target) throws IOException { - return delegate.transferTo(count, throughBuffer, target); - } - @Override public int read(ByteBuffer dst) throws IOException { - if (readListener == null) { - return delegate.read(dst); - } else { - int read = delegate.read(dst); - ByteBuffer copy = dst.duplicate(); + var read = delegate.read(dst); + if (readListener != null) { + var copy = dst.duplicate(); copy.flip(); - byte[] data = new byte[copy.remaining()]; + var data = new byte[copy.remaining()]; copy.get(data); readListener.accept(data); - return read; } + return read; } - - @Override - public long read(ByteBuffer[] dsts, int offs, int len) throws IOException { - return delegate.read(dsts, offs, len); - } - - } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/DurationUtil.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/DurationUtil.java new file mode 100644 index 00000000..35a9b7fd --- /dev/null +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/DurationUtil.java @@ -0,0 +1,20 @@ +package hk.edu.polyu.comp.vlabcontroller.util; + +import io.vavr.Function1; + +import java.time.Duration; + +public class DurationUtil { + public static Duration max(Duration a, Duration b) { + return a.compareTo(b) > 0 ? a : b; + } + public static Duration min(Duration a, Duration b) { + return a.compareTo(b) < 0 ? a : b; + } + public static Function1 atLeast(Duration least) { + return x -> min(x, least).equals(x) ? least : x; + } + public static Function1 atMost(Duration most) { + return x -> max(x, most).equals(x) ? most : x; + } +} diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/PortAllocator.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/PortAllocator.java index d04f8900..8e10fac7 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/PortAllocator.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/PortAllocator.java @@ -1,9 +1,9 @@ package hk.edu.polyu.comp.vlabcontroller.util; import hk.edu.polyu.comp.vlabcontroller.VLabControllerException; +import lombok.Synchronized; import java.util.*; -import java.util.stream.Collectors; public class PortAllocator { @@ -18,12 +18,12 @@ public PortAllocator(int from, int to) { } public int allocate(String ownerId) { - int nextPort = range[0]; + var nextPort = range[0]; while (occupiedPorts.contains(nextPort)) nextPort++; if (range[1] > 0 && nextPort > range[1]) { throw new VLabControllerException("Cannot create container: all allocated ports are currently in use." - + " Please try again later or contact an administrator."); + + " Please try again later or contact an administrator."); } occupiedPorts.add(nextPort); @@ -36,16 +36,10 @@ public void release(int port) { occupiedPortOwners.remove(port); } + @Synchronized("occupiedPortOwners") public void release(String ownerId) { - synchronized (occupiedPortOwners) { - Set portsToRelease = occupiedPortOwners.entrySet().stream() - .filter(e -> e.getValue().equals(ownerId)) - .map(e -> e.getKey()) - .collect(Collectors.toSet()); - for (Integer port : portsToRelease) { - occupiedPorts.remove(port); - occupiedPortOwners.remove(port); - } - } + occupiedPortOwners.entrySet().stream() + .filter(e -> e.getValue().equals(ownerId)) + .map(Map.Entry::getKey).distinct().forEach(this::release); } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/ProxyMappingManager.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/ProxyMappingManager.java index fb64a9bb..fe199e3c 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/ProxyMappingManager.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/ProxyMappingManager.java @@ -15,43 +15,42 @@ import io.undertow.servlet.handlers.ServletRequestContext; import io.undertow.util.AttachmentKey; import io.undertow.util.PathMatcher; -import lombok.extern.log4j.Log4j2; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Component; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.IOException; -import java.lang.reflect.Field; import java.net.HttpURLConnection; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; import java.nio.ByteBuffer; +import java.time.Duration; import java.util.Collections; import java.util.HashMap; import java.util.Map; -import java.util.Map.Entry; +import java.util.Optional; +import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; /** * This component keeps track of which proxy mappings (i.e. URL endpoints) are currently registered, * and tells Undertow where they should proxy to. */ -@Log4j2 +@Slf4j @Component +@RequiredArgsConstructor public class ProxyMappingManager { - private static final String PROXY_INTERNAL_ENDPOINT = "/proxy_endpoint"; private static final String PROXY_PORT_MAPPINGS_ENDPOINT = "/port_mappings"; private static final AttachmentKey ATTACHMENT_KEY_DISPATCHER = AttachmentKey.create(ProxyMappingManager.class); private final Map proxyMappings = new HashMap<>(); // proxyId -> metadata private PathHandler pathHandler; private final HeartbeatService heartbeatService; - - public ProxyMappingManager(HeartbeatService heartbeatService) { - this.heartbeatService = heartbeatService; - } + private final Retrying retrying; public synchronized HttpHandler createHttpHandler(HttpHandler defaultHandler) { if (pathHandler == null) { @@ -67,27 +66,32 @@ public synchronized void addMapping(String proxyId, String mapping, URI target) if (proxyMappings.containsKey(proxyId)) { if (proxyMappings.get(proxyId).containsExactMappingPath(mapping)) return; } - ProxyMappingMetadata proxyMappingMetadata = proxyMappings.computeIfAbsent(proxyId, value -> new ProxyMappingMetadata()); + var proxyMappingMetadata = proxyMappings.computeIfAbsent(proxyId, __ -> ProxyMappingMetadata.builder().build()); + proxyMappingMetadata.setDefaultTarget(target); - LoadBalancingProxyClient proxyClient = new LoadBalancingProxyClient() { + var proxyClient = new LoadBalancingProxyClient() { @Override public void getConnection(ProxyTarget target, HttpServerExchange exchange, ProxyCallback callback, long timeout, TimeUnit timeUnit) { try { exchange.addResponseCommitListener(ex -> heartbeatService.attachHeartbeatChecker(ex, proxyId)); } catch (Exception e) { - log.error(e); + log.error("an error occured: {}", e); } super.getConnection(target, exchange, callback, timeout, timeUnit); } }; proxyClient.setMaxQueueSize(100); proxyClient.addHost(target); - - String path = PROXY_INTERNAL_ENDPOINT + "/" + mapping; + proxyMappingMetadata.getPortMappingMetadataList().add( + PortMappingMetadata.builder() + .portMapping(mapping) + .target(target) + .loadBalancingProxyClient(proxyClient) + .build() + ); + + var path = PROXY_INTERNAL_ENDPOINT + "/" + mapping; pathHandler.addPrefixPath(path, new ProxyHandler(proxyClient, ResponseCodeHandler.HANDLE_404)); - - proxyMappingMetadata.setDefaultTarget(target); - proxyMappingMetadata.getPortMappingMetadataList().add(new PortMappingMetadata(mapping, target, proxyClient)); log.debug("mapping {} was added, current mappings: {}", mapping, proxyMappings); } @@ -95,10 +99,11 @@ public synchronized void removeProxyMapping(String proxyId) { if (pathHandler == null) throw new IllegalStateException("Cannot change mappings: web server is not yet running."); if (proxyMappings.containsKey(proxyId)) { - ProxyMappingMetadata metadata = proxyMappings.get(proxyId); + var metadata = proxyMappings.get(proxyId); metadata.getPortMappingMetadataList().forEach(e -> { - e.getLoadBalancingProxyClient().closeCurrentConnections(); - e.getLoadBalancingProxyClient().removeHost(e.getTarget()); + var loadBalancingProxyClient = e.getLoadBalancingProxyClient(); + loadBalancingProxyClient.closeCurrentConnections(); + loadBalancingProxyClient.removeHost(e.getTarget()); pathHandler.removePrefixPath(PROXY_INTERNAL_ENDPOINT + "/" + e.getPortMapping()); }); proxyMappings.remove(proxyId); @@ -107,11 +112,9 @@ public synchronized void removeProxyMapping(String proxyId) { } public String getProxyId(String mapping) { - for (Entry e : proxyMappings.entrySet()) { - ProxyMappingMetadata metadata = e.getValue(); - if (metadata.containsMappingPathPrefix(mapping)) return e.getKey(); - } - return null; + return proxyMappings.entrySet().stream() + .filter(e -> e.getValue().containsMappingPathPrefix(mapping)) + .map(Map.Entry::getKey).findFirst().orElse(null); } public String getProxyPortMappingsEndpoint() { @@ -134,12 +137,12 @@ public String getProxyPortMappingsEndpoint() { * @throws ServletException If the dispatch fails for any other reason. */ public void dispatchAsync(String mapping, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException { - HttpServerExchange exchange = ServletRequestContext.current().getExchange(); + var exchange = ServletRequestContext.current().getExchange(); exchange.putAttachment(ATTACHMENT_KEY_DISPATCHER, this); - String queryString = request.getQueryString(); + var queryString = request.getQueryString(); queryString = (queryString == null) ? "" : "?" + queryString; - String targetPath = PROXY_INTERNAL_ENDPOINT + "/" + mapping + queryString; + var targetPath = PROXY_INTERNAL_ENDPOINT + "/" + mapping + queryString; request.startAsync(); request.getRequestDispatcher(targetPath).forward(request, response); @@ -163,58 +166,56 @@ public void dispatchAsync(String mapping, HttpServletRequest request, HttpServle * @throws ServletException If the dispatch fails for any other reason. * @throws URISyntaxException If URI syntax is not allowed. */ - public void dispatchAsync(Proxy proxy, String mapping, int port, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException, URISyntaxException { - HttpServerExchange exchange = ServletRequestContext.current().getExchange(); + public void dispatchAsync(Proxy proxy, String mapping, int port, HttpServletRequest request, HttpServletResponse response) throws IOException, ServletException, URISyntaxException, ExecutionException, InterruptedException { + var exchange = ServletRequestContext.current().getExchange(); exchange.putAttachment(ATTACHMENT_KEY_DISPATCHER, this); - String proxyId = proxy.getId(); - URI defaultTarget = proxyMappings.get(proxyId).getDefaultTarget(); - String port_mapping = proxyId + PROXY_PORT_MAPPINGS_ENDPOINT + "/" + port; - URI newTarget = new URI(defaultTarget.getScheme() + "://" + defaultTarget.getHost() + ":" + port); - int[] failedResponseCode = new int[1]; - boolean targetConnected = Retrying.retry(i -> { + var proxyId = proxy.getId(); + var defaultTarget = proxyMappings.get(proxyId).getDefaultTarget(); + var port_mapping = proxyId + PROXY_PORT_MAPPINGS_ENDPOINT + "/" + port; + var newTarget = new URI(defaultTarget.getScheme() + "://" + defaultTarget.getHost() + ":" + port); + var failedResponseCode = new int[1]; + var query = Optional.ofNullable(request.getQueryString()).map(x -> "?" + x).orElse(""); + var targetConnected = retrying.retry(i -> { try { - String query = request.getQueryString() == null ? "" : "?" + request.getQueryString(); log.debug("request protocol: {}, scheme: {}, headers: {}", request.getProtocol(), request.getScheme(), Collections.list(request.getHeaderNames())); // Handle websocket case if (request.getHeaders("Upgrade").hasMoreElements()) { return true; } - URL testURL = new URL(newTarget + mapping + query); + var testURL = new URL(newTarget + mapping + query); log.debug("Testing url of {}", testURL); - HttpURLConnection connection = (HttpURLConnection) testURL.openConnection(); + var connection = (HttpURLConnection) testURL.openConnection(); connection.setConnectTimeout(5000); connection.setInstanceFollowRedirects(false); - int responseCode = connection.getResponseCode(); + var responseCode = connection.getResponseCode(); log.debug("received connection from {}, status code: {}", testURL, responseCode); if (responseCode < 500) { log.debug("successfully connected to target {}", testURL); - }else{ + } else { failedResponseCode[0] = responseCode; } return true; - }catch (IOException ioe) { + } catch (IOException ioe) { failedResponseCode[0] = 404; log.debug("Trying to connect target URL ({}/{})", i, 5); } catch (Exception e) { failedResponseCode[0] = 500; - log.debug(e); + log.debug("an error occured: {}", e); log.debug("Trying to connect target URL ({}/{})", i, 5); } return false; - }, 5, 2000, true); + }, 5, Duration.ofSeconds(2), true); - if (!targetConnected) { + if (!targetConnected.get()) { response.sendError(failedResponseCode[0]); return; } addMapping(proxyId, port_mapping, newTarget); proxy.getTargets().put(port_mapping, newTarget); - String queryString = request.getQueryString(); - queryString = (queryString == null) ? "" : "?" + queryString; - String targetPath = PROXY_INTERNAL_ENDPOINT + "/" + port_mapping + mapping + queryString; + var targetPath = PROXY_INTERNAL_ENDPOINT + "/" + port_mapping + mapping + query; request.startAsync(); request.getRequestDispatcher(targetPath).forward(request, response); } @@ -228,10 +229,10 @@ public ProxyPathHandler(HttpHandler defaultHandler) { @SuppressWarnings("unchecked") @Override public void handleRequest(HttpServerExchange exchange) throws Exception { - Field field = PathHandler.class.getDeclaredField("pathMatcher"); + var field = PathHandler.class.getDeclaredField("pathMatcher"); field.setAccessible(true); - PathMatcher pathMatcher = (PathMatcher) field.get(this); - PathMatcher.PathMatch match = pathMatcher.match(exchange.getRelativePath()); + var pathMatcher = (PathMatcher) field.get(this); + var match = pathMatcher.match(exchange.getRelativePath()); // Note: this handler may never be accessed directly (because it bypasses Spring security). // Only allowed if the request was dispatched via this class. diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/RFC6335Validator.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/RFC6335Validator.java new file mode 100644 index 00000000..4ce7e027 --- /dev/null +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/RFC6335Validator.java @@ -0,0 +1,7 @@ +package hk.edu.polyu.comp.vlabcontroller.util; + +public class RFC6335Validator { + public static boolean valid(String input) { + return input.matches("^(?!.*--.*)[^\\W_]([^\\W_]|-)*(? sessionRepository; - public RedisSessionHelper(FindByIndexNameSessionRepository sessionRepository) { - this.sessionRepository = sessionRepository; - } - - public Map getSessionByUsername(String username) { + public Map getSessionByUsername(String username) { return sessionRepository.findByIndexNameAndIndexValue(FindByIndexNameSessionRepository.PRINCIPAL_NAME_INDEX_NAME, username); } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/Retrying.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/Retrying.java index 1832342c..4539c296 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/Retrying.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/Retrying.java @@ -1,34 +1,35 @@ package hk.edu.polyu.comp.vlabcontroller.util; +import io.vavr.control.Try; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Component; + +import java.time.Duration; +import java.util.concurrent.CompletableFuture; import java.util.function.IntPredicate; +import java.util.stream.IntStream; +@Component public class Retrying { - - public static boolean retry(IntPredicate job, int tries, int waitTime) { + @Async + public CompletableFuture retry(IntPredicate job, int tries, Duration waitTime) { return retry(job, tries, waitTime, false); } - public static boolean retry(IntPredicate job, int tries, int waitTime, boolean retryOnException) { - boolean retVal = false; - RuntimeException exception = null; - for (int currentTry = 1; currentTry <= tries; currentTry++) { + @Async + public CompletableFuture retry(IntPredicate job, int tries, Duration waitTime, boolean retryOnException) { + var result = Try.success(false); + for (var currentTry : (Iterable) () -> IntStream.rangeClosed(1, tries).iterator()) { + result = Try.of(() -> job.test(currentTry)) + .recoverWith(e -> retryOnException ? Try.success(false) : Try.failure(e)); + if (result.isFailure()) return CompletableFuture.failedFuture(result.getCause()); + if (result.get()) return CompletableFuture.completedFuture(true); try { - if (job.test(currentTry)) { - retVal = true; - exception = null; - break; - } - } catch (RuntimeException e) { - if (retryOnException) exception = e; - else throw e; - } - try { - Thread.sleep(waitTime); - } catch (InterruptedException ignore) { + Thread.sleep(waitTime.toMillis()); + } catch (InterruptedException ignored) { } } - if (exception == null) return retVal; - else throw exception; - + if (result.isFailure()) return CompletableFuture.failedFuture(result.getCause()); + return CompletableFuture.completedFuture(result.get()); } } diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/SessionHelper.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/SessionHelper.java index 9b490f63..c9950f44 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/SessionHelper.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/SessionHelper.java @@ -1,12 +1,11 @@ package hk.edu.polyu.comp.vlabcontroller.util; +import hk.edu.polyu.comp.vlabcontroller.config.ServerProperties; import io.undertow.server.HttpServerExchange; -import io.undertow.server.handlers.Cookie; import io.undertow.servlet.handlers.ServletRequestContext; -import io.undertow.util.HeaderValues; -import org.springframework.core.env.Environment; import org.springframework.security.authentication.AnonymousAuthenticationToken; import org.springframework.security.core.AuthenticatedPrincipal; +import org.springframework.security.core.Authentication; import org.springframework.security.core.context.SecurityContext; import javax.servlet.http.HttpSession; @@ -21,13 +20,13 @@ public class SessionHelper { * @return The current session ID, or null if no session is active. */ public static String getCurrentSessionId(boolean createIfMissing) { - ServletRequestContext context = ServletRequestContext.current(); + var context = ServletRequestContext.current(); if (context == null) return null; HttpSession session = context.getSession(); if (session != null) return session.getId(); - Cookie jSessionIdCookie = context.getExchange().getRequestCookies().get("JSESSIONID"); + var jSessionIdCookie = context.getExchange().getRequestCookie("JSESSIONID"); if (jSessionIdCookie != null) return jSessionIdCookie.getValue(); if (createIfMissing) return context.getCurrentServletContext().getSession(context.getExchange(), true).getId(); @@ -37,13 +36,13 @@ public static String getCurrentSessionId(boolean createIfMissing) { /** * Get the context path that has been configured for this instance. * - * @param environment The Spring environment containing the context-path setting. + * @param serverProperties The Spring configuration properties that resolves context-path * @param endWithSlash True to always end the context path with a slash. * @return The instance's context path, may be empty, never null. */ - public static String getContextPath(Environment environment, boolean endWithSlash) { - String contextPath = environment.getProperty("server.servlet.context-path"); - if (contextPath == null || contextPath.trim().equals("/") || contextPath.trim().isEmpty()) + public static String getContextPath(ServerProperties serverProperties, boolean endWithSlash) { + var contextPath = serverProperties.getServletContextPath(); + if (contextPath == null || contextPath.isBlank() || contextPath.trim().equals("/")) return endWithSlash ? "/" : ""; if (!contextPath.startsWith("/")) contextPath = "/" + contextPath; @@ -65,23 +64,23 @@ public static String getContextPath(Environment environment, boolean endWithSlas * @return An object containing information about the current user. */ public static SessionOwnerInfo createOwnerInfo(HttpServerExchange exchange) { - SessionOwnerInfo info = new SessionOwnerInfo(); + var info = new SessionOwnerInfo(); // Ideally, use the HTTP session information. info.principal = Optional.ofNullable(ServletRequestContext.current()) - .map(ctx -> ctx.getSession()) + .map(ServletRequestContext::getSession) .map(session -> (SecurityContext) session.getAttribute("SPRING_SECURITY_CONTEXT")) - .map(ctx -> ctx.getAuthentication()) + .map(SecurityContext::getAuthentication) .filter(auth -> !(auth instanceof AnonymousAuthenticationToken)) - .map(auth -> auth.getPrincipal()) + .map(Authentication::getPrincipal) .orElse(null); // Fallback: use the Authorization header, if present. - HeaderValues authHeader = exchange.getRequestHeaders().get("Authorization"); + var authHeader = exchange.getRequestHeaders().get("Authorization"); if (authHeader != null) info.authHeader = authHeader.getFirst(); // Fallback: use the JSESSIONID cookie, if present. - Cookie jSessionIdCookie = exchange.getRequestCookies().get("JSESSIONID"); + var jSessionIdCookie = exchange.getRequestCookie("JSESSIONID"); if (jSessionIdCookie != null) info.jSessionId = jSessionIdCookie.getValue(); // Final fallback: generate a JSESSIONID for this exchange. diff --git a/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/StartupEventListener.java b/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/StartupEventListener.java index 078aafe0..b65791c5 100644 --- a/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/StartupEventListener.java +++ b/src/main/java/hk/edu/polyu/comp/vlabcontroller/util/StartupEventListener.java @@ -1,16 +1,14 @@ package hk.edu.polyu.comp.vlabcontroller.util; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import lombok.extern.slf4j.Slf4j; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.boot.info.BuildProperties; import org.springframework.context.event.EventListener; import org.springframework.stereotype.Component; +@Slf4j @Component public class StartupEventListener { - private static final Logger LOGGER = LoggerFactory.getLogger(StartupEventListener.class); - private final BuildProperties buildProperties; public StartupEventListener(BuildProperties buildProperties) { @@ -19,9 +17,6 @@ public StartupEventListener(BuildProperties buildProperties) { @EventListener public void onStartup(ApplicationReadyEvent event) { - StringBuilder startupMsg = new StringBuilder("Started "); - startupMsg.append(buildProperties.getName()).append(" "); - startupMsg.append(buildProperties.getVersion()); - LOGGER.info(startupMsg.toString()); + log.info(String.format("Started %s %s", buildProperties.getName(), buildProperties.getVersion())); } } diff --git a/src/test/java/hk/edu/polyu/comp/vlabcontroller/model/runtime/MongoTest.java b/src/test/java/hk/edu/polyu/comp/vlabcontroller/model/runtime/MongoTest.java new file mode 100644 index 00000000..9d2d7b94 --- /dev/null +++ b/src/test/java/hk/edu/polyu/comp/vlabcontroller/model/runtime/MongoTest.java @@ -0,0 +1,22 @@ +package hk.edu.polyu.comp.vlabcontroller.model.runtime; + +import hk.edu.polyu.comp.vlabcontroller.entity.QUser; +import hk.edu.polyu.comp.vlabcontroller.entity.User; +import hk.edu.polyu.comp.vlabcontroller.repository.UserRepository; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; + +import static org.springframework.test.util.AssertionErrors.assertNotNull; + +@SpringBootTest +public class MongoTest { + @Autowired + UserRepository repository; + + @Test + public void testUserRepo() { + this.repository.insert(User.builder().id("test").build()); + assertNotNull("entity is null", this.repository.findOne(QUser.user.id.eq("test"))); + } +} diff --git a/src/test/java/hk/edu/polyu/comp/vlabcontroller/model/runtime/ProxyMappingMetadataTest.java b/src/test/java/hk/edu/polyu/comp/vlabcontroller/model/runtime/ProxyMappingMetadataTest.java index 96375881..e4c6e527 100644 --- a/src/test/java/hk/edu/polyu/comp/vlabcontroller/model/runtime/ProxyMappingMetadataTest.java +++ b/src/test/java/hk/edu/polyu/comp/vlabcontroller/model/runtime/ProxyMappingMetadataTest.java @@ -1,56 +1,57 @@ package hk.edu.polyu.comp.vlabcontroller.model.runtime; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.SpringBootTest; import java.net.URI; import java.net.URISyntaxException; +import java.util.List; -class ProxyMappingMetadataTest { +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@SpringBootTest +public class ProxyMappingMetadataTest { @Test - void containsExactTargetPath() throws URISyntaxException { - var metadata = new ProxyMappingMetadata(); - Assertions.assertFalse(metadata.containsExactMappingPath("test")); - metadata.getPortMappingMetadataList().add( - new PortMappingMetadata( - "1ca3dde2-8fdf-4fe4-8327-6849e4d77fcd/port_mappings/8000", - new URI("http://10.42.61.11:8000"), - null - )); - metadata.getPortMappingMetadataList().add( - new PortMappingMetadata( - "1ca3dde2-8fdf-4fe4-8327-6849e4d77fcd/port_mappings/8080", - new URI("http://10.42.61.11:8080"), - null - )); - Assertions.assertFalse(metadata.containsExactMappingPath("test")); - Assertions.assertTrue(metadata.containsExactMappingPath("1ca3dde2-8fdf-4fe4-8327-6849e4d77fcd/port_mappings/8000")); - Assertions.assertTrue(metadata.containsExactMappingPath("1ca3dde2-8fdf-4fe4-8327-6849e4d77fcd/port_mappings/8080")); + public void testContainsExactTargetPath() throws URISyntaxException { + var metadata = ProxyMappingMetadata.builder() + .portMappingMetadataList(List.of( + PortMappingMetadata.builder() + .portMapping("1ca3dde2-8fdf-4fe4-8327-6849e4d77fcd/port_mappings/8000") + .target(new URI("http://10.42.61.11:8000")) + .build(), + PortMappingMetadata.builder() + .portMapping("1ca3dde2-8fdf-4fe4-8327-6849e4d77fcd/port_mappings/8080") + .target(new URI("http://10.42.61.11:8080")) + .build() + )) + .build(); + assertFalse(metadata.containsExactMappingPath("test")); + assertTrue(metadata.containsExactMappingPath("1ca3dde2-8fdf-4fe4-8327-6849e4d77fcd/port_mappings/8000")); + assertTrue(metadata.containsExactMappingPath("1ca3dde2-8fdf-4fe4-8327-6849e4d77fcd/port_mappings/8080")); } @Test - void containsMappingPathPrefix() throws URISyntaxException { - var metadata = new ProxyMappingMetadata(); - Assertions.assertFalse(metadata.containsMappingPathPrefix("test")); - metadata.getPortMappingMetadataList().add( - new PortMappingMetadata( - "1ca3dde2-8fdf-4fe4-8327-6849e4d77fcd/port_mappings/8000", - new URI("http://10.42.61.11:8000"), - null - )); - metadata.getPortMappingMetadataList().add( - new PortMappingMetadata( - "1ca3dde2-8fdf-4fe4-8327-6849e4d77fcd/port_mappings/8080", - new URI("http://10.42.61.11:8080"), - null - )); - Assertions.assertFalse(metadata.containsMappingPathPrefix("test")); - Assertions.assertFalse(metadata.containsMappingPathPrefix("1ca3dde2-8fdf-4fe4-8327-6849e4d77fcd/port_mappings/8087")); - Assertions.assertFalse(metadata.containsMappingPathPrefix("1ca3dde2-8fdf-4fe4-8327-6849e4d77fcd/port8087")); - Assertions.assertTrue(metadata.containsMappingPathPrefix("1ca3dde2-8fdf-4fe4-8327-6849e4d77fcd")); - Assertions.assertTrue(metadata.containsMappingPathPrefix("1ca3dde2-8fdf-4fe4-8327-6849e4d77fcd/port_mappings")); - Assertions.assertTrue(metadata.containsMappingPathPrefix("1ca3dde2-8fdf-4fe4-8327-6849e4d77fcd/port_mappings/8080")); - Assertions.assertTrue(metadata.containsMappingPathPrefix("1ca3dde2-8fdf-4fe4-8327-6849e4d77fcd/port_mappings/8080/")); + public void testContainsMappingPathPrefix() throws URISyntaxException { + var metadata = ProxyMappingMetadata.builder() + .portMappingMetadataList(List.of( + PortMappingMetadata.builder() + .portMapping("1ca3dde2-8fdf-4fe4-8327-6849e4d77fcd/port_mappings/8000") + .target(new URI("http://10.42.61.11:8000")) + .build(), + PortMappingMetadata.builder() + .portMapping("1ca3dde2-8fdf-4fe4-8327-6849e4d77fcd/port_mappings/8080") + .target(new URI("http://10.42.61.11:8080")) + .build() + )) + .build(); + assertFalse(metadata.containsMappingPathPrefix("test")); + assertFalse(metadata.containsMappingPathPrefix("1ca3dde2-8fdf-4fe4-8327-6849e4d77fcd/port_mappings/8087")); + assertFalse(metadata.containsMappingPathPrefix("1ca3dde2-8fdf-4fe4-8327-6849e4d77fcd/port8087")); + assertTrue(metadata.containsMappingPathPrefix("1ca3dde2-8fdf-4fe4-8327-6849e4d77fcd")); + assertTrue(metadata.containsMappingPathPrefix("1ca3dde2-8fdf-4fe4-8327-6849e4d77fcd/port_mappings")); + assertTrue(metadata.containsMappingPathPrefix("1ca3dde2-8fdf-4fe4-8327-6849e4d77fcd/port_mappings/8080")); + assertTrue(metadata.containsMappingPathPrefix("1ca3dde2-8fdf-4fe4-8327-6849e4d77fcd/port_mappings/8080/")); } } \ No newline at end of file