From 3549fe86813fac0e0a2e868aa83f5ec1ccefd867 Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Wed, 5 Jul 2023 10:26:19 +0300 Subject: [PATCH 01/52] implement stub for neurosmt --- ksmt-neurosmt/build.gradle.kts | 12 ++++ .../ksmt/solver/neurosmt/KNeuroSMTSolver.kt | 59 +++++++++++++++++++ .../neurosmt/KNeuroSMTSolverConfiguration.kt | 44 ++++++++++++++ sandbox/build.gradle.kts | 27 +++++++++ sandbox/src/main/kotlin/Main.kt | 27 +++++++++ settings.gradle.kts | 2 + 6 files changed, 171 insertions(+) create mode 100644 ksmt-neurosmt/build.gradle.kts create mode 100644 ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/KNeuroSMTSolver.kt create mode 100644 ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/KNeuroSMTSolverConfiguration.kt create mode 100644 sandbox/build.gradle.kts create mode 100644 sandbox/src/main/kotlin/Main.kt diff --git a/ksmt-neurosmt/build.gradle.kts b/ksmt-neurosmt/build.gradle.kts new file mode 100644 index 000000000..51f6e9928 --- /dev/null +++ b/ksmt-neurosmt/build.gradle.kts @@ -0,0 +1,12 @@ +plugins { + id("io.ksmt.ksmt-base") + id("com.github.johnrengelman.shadow") version "7.1.2" +} + +repositories { + mavenCentral() +} + +dependencies { + implementation(project(":ksmt-core")) +} \ No newline at end of file diff --git a/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/KNeuroSMTSolver.kt b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/KNeuroSMTSolver.kt new file mode 100644 index 000000000..4b889b728 --- /dev/null +++ b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/KNeuroSMTSolver.kt @@ -0,0 +1,59 @@ +package io.ksmt.solver.neurosmt + +import io.ksmt.KContext +import io.ksmt.expr.KExpr +import io.ksmt.solver.KModel +import io.ksmt.solver.KSolver +import io.ksmt.solver.KSolverStatus +import io.ksmt.sort.KBoolSort +import kotlin.time.Duration + +class KNeuroSMTSolver(private val ctx: KContext) : KSolver { + override fun configure(configurator: KNeuroSMTSolverConfiguration.() -> Unit) { + TODO("Not yet implemented") + } + + override fun assert(expr: KExpr) { + // TODO("Not yet implemented") + } + + override fun assertAndTrack(expr: KExpr) { + TODO("Not yet implemented") + } + + override fun push() { + TODO("Not yet implemented") + } + + override fun pop(n: UInt) { + TODO("Not yet implemented") + } + + override fun check(timeout: Duration): KSolverStatus { + return KSolverStatus.SAT + } + + override fun checkWithAssumptions(assumptions: List>, timeout: Duration): KSolverStatus { + TODO("Not yet implemented") + } + + override fun model(): KModel { + TODO("Not yet implemented") + } + + override fun unsatCore(): List> { + TODO("Not yet implemented") + } + + override fun reasonOfUnknown(): String { + TODO("Not yet implemented") + } + + override fun interrupt() { + TODO("Not yet implemented") + } + + override fun close() { + // TODO("Not yet implemented") + } +} \ No newline at end of file diff --git a/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/KNeuroSMTSolverConfiguration.kt b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/KNeuroSMTSolverConfiguration.kt new file mode 100644 index 000000000..8bf4ba534 --- /dev/null +++ b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/KNeuroSMTSolverConfiguration.kt @@ -0,0 +1,44 @@ +package io.ksmt.solver.neurosmt + +import io.ksmt.solver.KSolverConfiguration + +interface KNeuroSMTSolverConfiguration : KSolverConfiguration { + fun setOption(option: String, value: Boolean) + fun setOption(option: String, value: Int) + fun setOption(option: String, value: Double) + fun setOption(option: String, value: String) + + override fun setBoolParameter(param: String, value: Boolean) { + setOption(param, value) + } + + override fun setIntParameter(param: String, value: Int) { + setOption(param, value) + } + + override fun setDoubleParameter(param: String, value: Double) { + setOption(param, value) + } + + override fun setStringParameter(param: String, value: String) { + setOption(param, value) + } +} + +class KNeuroSMTSolverConfigurationImpl(private val params: Any?) : KNeuroSMTSolverConfiguration { + override fun setOption(option: String, value: Boolean) { + TODO("Not yet implemented") + } + + override fun setOption(option: String, value: Int) { + TODO("Not yet implemented") + } + + override fun setOption(option: String, value: Double) { + TODO("Not yet implemented") + } + + override fun setOption(option: String, value: String) { + TODO("Not yet implemented") + } +} \ No newline at end of file diff --git a/sandbox/build.gradle.kts b/sandbox/build.gradle.kts new file mode 100644 index 000000000..19f55bb82 --- /dev/null +++ b/sandbox/build.gradle.kts @@ -0,0 +1,27 @@ +plugins { + kotlin("jvm") + application +} + +group = "org.example" +version = "unspecified" + +repositories { + mavenCentral() +} + +dependencies { + testImplementation("org.junit.jupiter:junit-jupiter-api:5.8.1") + testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.8.1") + + implementation(project(":ksmt-core")) + implementation(project(":ksmt-neurosmt")) +} + +tasks.getByName("test") { + useJUnitPlatform() +} + +application { + mainClass.set("MainKt") +} \ No newline at end of file diff --git a/sandbox/src/main/kotlin/Main.kt b/sandbox/src/main/kotlin/Main.kt new file mode 100644 index 000000000..0d8455531 --- /dev/null +++ b/sandbox/src/main/kotlin/Main.kt @@ -0,0 +1,27 @@ +import io.ksmt.KContext +import io.ksmt.solver.neurosmt.KNeuroSMTSolver +import io.ksmt.utils.getValue +import kotlin.time.Duration.Companion.seconds + +fun main() { + val ctx = KContext() + + with(ctx) { + // create symbolic variables + val a by boolSort + val b by intSort + val c by intSort + + // create an expression + val constraint = a and (b ge c + 3.expr) and (b * 2.expr gt 8.expr) and !a + + KNeuroSMTSolver(this).use { solver -> // create a Stub SMT solver instance + // assert expression + solver.assert(constraint) + + // check assertions satisfiability with timeout + val satisfiability = solver.check(timeout = 1.seconds) + println(satisfiability) // SAT + } + } +} \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts index c6d4d64de..a96f6ee2e 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -17,3 +17,5 @@ pluginManagement { } } } +include("ksmt-neurosmt") +include("sandbox") From dcce69b4f20421a82357b296913932decf2a9d4d Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Wed, 19 Jul 2023 15:45:23 +0300 Subject: [PATCH 02/52] set up gradle --- ksmt-neurosmt/build.gradle.kts | 2 +- ksmt-neurosmt/utils/build.gradle.kts | 44 ++++++++++++++++++++++++++++ sandbox/build.gradle.kts | 26 ++++++++++++++++ settings.gradle.kts | 2 ++ 4 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 ksmt-neurosmt/utils/build.gradle.kts diff --git a/ksmt-neurosmt/build.gradle.kts b/ksmt-neurosmt/build.gradle.kts index 51f6e9928..fe876a2e8 100644 --- a/ksmt-neurosmt/build.gradle.kts +++ b/ksmt-neurosmt/build.gradle.kts @@ -1,6 +1,5 @@ plugins { id("io.ksmt.ksmt-base") - id("com.github.johnrengelman.shadow") version "7.1.2" } repositories { @@ -9,4 +8,5 @@ repositories { dependencies { implementation(project(":ksmt-core")) + // implementation(project(":ksmt-z3")) } \ No newline at end of file diff --git a/ksmt-neurosmt/utils/build.gradle.kts b/ksmt-neurosmt/utils/build.gradle.kts new file mode 100644 index 000000000..04ab2409e --- /dev/null +++ b/ksmt-neurosmt/utils/build.gradle.kts @@ -0,0 +1,44 @@ +plugins { + kotlin("jvm") + application +} + +group = "org.example" +version = "unspecified" + +repositories { + mavenCentral() +} + +dependencies { + implementation(project(":ksmt-core")) + implementation(project(":ksmt-z3")) +} + +application { + mainClass.set("io.ksmt.solver.neurosmt.smt2converter.SMT2ConverterKt") +} + +tasks { + val fatJar = register("fatJar") { + dependsOn.addAll(listOf("compileJava", "compileKotlin", "processResources")) + + archiveFileName.set("convert-smt2.jar") + destinationDirectory.set(File(".")) + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + + manifest { + attributes(mapOf("Main-Class" to application.mainClass)) + } + + val sourcesMain = sourceSets.main.get() + val contents = configurations.runtimeClasspath.get() + .map { if (it.isDirectory) it else zipTree(it) } + sourcesMain.output + + from(contents) + } + + build { + dependsOn(fatJar) + } +} \ No newline at end of file diff --git a/sandbox/build.gradle.kts b/sandbox/build.gradle.kts index 19f55bb82..9a724d9c2 100644 --- a/sandbox/build.gradle.kts +++ b/sandbox/build.gradle.kts @@ -15,7 +15,9 @@ dependencies { testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.8.1") implementation(project(":ksmt-core")) + implementation(project(":ksmt-z3")) implementation(project(":ksmt-neurosmt")) + implementation(project(":ksmt-neurosmt:utils")) } tasks.getByName("test") { @@ -24,4 +26,28 @@ tasks.getByName("test") { application { mainClass.set("MainKt") +} + +tasks { + val fatJar = register("fatJar") { + dependsOn.addAll(listOf("compileJava", "compileKotlin", "processResources")) + + archiveFileName.set("sandbox.jar") + destinationDirectory.set(File(".")) + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + + manifest { + attributes(mapOf("Main-Class" to application.mainClass)) + } + + val sourcesMain = sourceSets.main.get() + val contents = configurations.runtimeClasspath.get() + .map { if (it.isDirectory) it else zipTree(it) } + sourcesMain.output + + from(contents) + } + + build { + dependsOn(fatJar) + } } \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts index a96f6ee2e..073762985 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -19,3 +19,5 @@ pluginManagement { } include("ksmt-neurosmt") include("sandbox") +include("ksmt-neurosmt:utils") +// findProject(":ksmt-neurosmt:utils")?.name = "utils" From 5f06f9d9ca9ed1208519b23e851263b5569a10d9 Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Wed, 19 Jul 2023 20:49:38 +0300 Subject: [PATCH 03/52] SMT formula graph extractor --- ksmt-neurosmt/utils/build.gradle.kts | 2 + .../smt2converter/FormulaGraphExtractor.kt | 69 +++++++++ .../neurosmt/smt2converter/SMT2Converter.kt | 61 ++++++++ .../solver/neurosmt/smt2converter/Utils.kt | 19 +++ sandbox/src/main/kotlin/Main.kt | 137 +++++++++++++++++- 5 files changed, 286 insertions(+), 2 deletions(-) create mode 100644 ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/FormulaGraphExtractor.kt create mode 100644 ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/SMT2Converter.kt create mode 100644 ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/Utils.kt diff --git a/ksmt-neurosmt/utils/build.gradle.kts b/ksmt-neurosmt/utils/build.gradle.kts index 04ab2409e..e3a6ad56f 100644 --- a/ksmt-neurosmt/utils/build.gradle.kts +++ b/ksmt-neurosmt/utils/build.gradle.kts @@ -13,6 +13,8 @@ repositories { dependencies { implementation(project(":ksmt-core")) implementation(project(":ksmt-z3")) + + implementation("me.tongfei:progressbar:0.9.4") } application { diff --git a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/FormulaGraphExtractor.kt b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/FormulaGraphExtractor.kt new file mode 100644 index 000000000..bc6b9933a --- /dev/null +++ b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/FormulaGraphExtractor.kt @@ -0,0 +1,69 @@ +package io.ksmt.solver.neurosmt.smt2converter + +import io.ksmt.KContext +import io.ksmt.expr.KApp +import io.ksmt.expr.KConst +import io.ksmt.expr.KExpr +import io.ksmt.expr.KInterpretedValue +import io.ksmt.expr.transformer.KNonRecursiveTransformer +import io.ksmt.sort.KBoolSort +import io.ksmt.sort.KSort +import java.io.OutputStream +import java.util.IdentityHashMap + +class FormulaGraphExtractor( + override val ctx: KContext, + val formula: KExpr, + outputStream: OutputStream +) : KNonRecursiveTransformer(ctx) { + + private val exprToVertexID = IdentityHashMap, Long>() + private var currentID = 0L + + private val writer = outputStream.bufferedWriter() + + override fun transformApp(expr: KApp): KExpr { + exprToVertexID[expr] = currentID++ + + when (expr) { + is KInterpretedValue<*> -> writeValue(expr) + is KConst<*> -> writeSymbolicVariable(expr) + else -> writeApp(expr) + } + + //println(expr::class) + //println("${expr.decl.name} ${expr.args.size}") + //outputStream.writer().write("${expr.decl.name} ${expr.args.size}\n") + //outputStream.writer().flush() + /* + println(expr.args) + println("${expr.decl} ${expr.decl.name} ${expr.decl.argSorts} ${expr.sort}") + for (child in expr.args) { + println((child as KApp<*, *>).decl.name) + } + */ + + return expr + } + + fun writeSymbolicVariable(symbol: KConst) { + writer.write("SYMBOLIC; ${symbol.sort}\n") + } + + fun writeValue(value: KInterpretedValue) { + writer.write("VALUE; ${value.decl.sort}\n") + } + + fun writeApp(expr: KApp) { + writer.write("${expr.decl.name};") + for (child in expr.args) { + writer.write(" ${exprToVertexID[child]}") + } + writer.newLine() + } + + fun extractGraph() { + apply(formula) + writer.close() + } +} \ No newline at end of file diff --git a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/SMT2Converter.kt b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/SMT2Converter.kt new file mode 100644 index 000000000..20a3961ea --- /dev/null +++ b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/SMT2Converter.kt @@ -0,0 +1,61 @@ +package io.ksmt.solver.neurosmt.smt2converter + +import io.ksmt.KContext +import io.ksmt.parser.KSMTLibParseException +import io.ksmt.solver.KSolverStatus +import io.ksmt.solver.z3.* +import me.tongfei.progressbar.ProgressBar +import java.io.FileOutputStream +import java.nio.file.Files +import java.nio.file.Path +import kotlin.io.path.isRegularFile +import kotlin.io.path.name + +fun main(args: Array) { + val inputRoot = args[0] + val outputRoot = args[1] + + val files = Files.walk(Path.of(inputRoot)).filter { it.isRegularFile() } + + var ok = 0; var fail = 0 + var sat = 0; var unsat = 0; var unknown = 0 + + val ctx = KContext() + + var curIdx = 0 + ProgressBar.wrap(files, "converting smt2 files").forEach { + if (!it.name.endsWith(".smt2")) { + return@forEach + } + + val answer = getAnswerForTest(it) + + when (answer) { + KSolverStatus.SAT -> sat++ + KSolverStatus.UNSAT -> unsat++ + KSolverStatus.UNKNOWN -> { + unknown++ + return@forEach + } + } + + with(ctx) { + val formula = try { + ok++ + mkAnd(KZ3SMTLibParser(ctx).parse(it)) + } catch (e: KSMTLibParseException) { + fail++ + return@forEach + } + + val extractor = FormulaGraphExtractor(ctx, formula, FileOutputStream("$outputRoot/$answer/$curIdx")) + extractor.extractGraph() + } + + curIdx++ + } + + println() + println("parsed: $ok; failed: $fail") + println("sat: $sat; unsat: $unsat; unknown: $unknown") +} \ No newline at end of file diff --git a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/Utils.kt b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/Utils.kt new file mode 100644 index 000000000..4968f62e3 --- /dev/null +++ b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/Utils.kt @@ -0,0 +1,19 @@ +package io.ksmt.solver.neurosmt.smt2converter + +import io.ksmt.solver.KSolverStatus +import java.io.File +import java.nio.file.Path + +fun getAnswerForTest(path: Path): KSolverStatus { + File(path.toUri()).useLines { lines -> + for (line in lines) { + when (line) { + "(set-info :status sat)" -> return KSolverStatus.SAT + "(set-info :status unsat)" -> return KSolverStatus.UNSAT + "(set-info :status unknown)" -> return KSolverStatus.UNKNOWN + } + } + } + + return KSolverStatus.UNKNOWN +} \ No newline at end of file diff --git a/sandbox/src/main/kotlin/Main.kt b/sandbox/src/main/kotlin/Main.kt index 0d8455531..d8920b910 100644 --- a/sandbox/src/main/kotlin/Main.kt +++ b/sandbox/src/main/kotlin/Main.kt @@ -1,19 +1,128 @@ import io.ksmt.KContext +import io.ksmt.expr.* +import io.ksmt.expr.transformer.KNonRecursiveTransformer +import io.ksmt.solver.KSolverStatus import io.ksmt.solver.neurosmt.KNeuroSMTSolver +import io.ksmt.solver.neurosmt.smt2converter.FormulaGraphExtractor +import io.ksmt.solver.neurosmt.smt2converter.getAnswerForTest +import io.ksmt.solver.z3.* +import io.ksmt.sort.* import io.ksmt.utils.getValue +import java.io.FileOutputStream +import java.nio.file.Files +import java.nio.file.Path +import kotlin.io.path.isRegularFile +import kotlin.io.path.name import kotlin.time.Duration.Companion.seconds +fun lol(a: Any) { + if (a is KConst<*>) { + println("const: ${a.decl.name}") + } + if (a is KInterpretedValue<*>) { + println("val: ${a.decl.name}") + } +} + +class Kek(override val ctx: KContext) : KNonRecursiveTransformer(ctx) { + override fun transformApp(expr: KApp): KExpr { + println("===") + //println(expr::class) + //lol(expr) + //println(expr) + /* + println(expr.args) + println("${expr.decl} ${expr.decl.name} ${expr.decl.argSorts} ${expr.sort}") + for (child in expr.args) { + println((child as KApp<*, *>).decl.name) + } + */ + return expr + } +} + fun main() { val ctx = KContext() with(ctx) { + val files = Files.walk(Path.of("../../neurosmt-benchmark/non-incremental/QF_BV")).filter { it.isRegularFile() } + var ok = 0; var fail = 0 + var sat = 0; var unsat = 0; var unk = 0 + + files.forEach { + println(it) + if (it.name.endsWith(".txt")) { + return@forEach + } + + when (getAnswerForTest(it)) { + KSolverStatus.SAT -> sat++ + KSolverStatus.UNSAT -> unsat++ + KSolverStatus.UNKNOWN -> unk++ + } + + return@forEach + + val formula = try { + val assertList = KZ3SMTLibParser(this).parse(it) + ok++ + mkAnd(assertList) + } catch (e: Exception) { + fail++ + return@forEach + } + + val extractor = FormulaGraphExtractor(this, formula, FileOutputStream("kek2.txt")) + extractor.extractGraph() + + println("$ok : $fail") + } + + println("$sat/$unsat/$unk") + + return@with + // create symbolic variables val a by boolSort val b by intSort val c by intSort + val d by realSort + val e by bv8Sort + val f by fp64Sort + val g by mkArraySort(realSort, boolSort) + val h by fp64Sort + + val x by intSort // create an expression - val constraint = a and (b ge c + 3.expr) and (b * 2.expr gt 8.expr) and !a + val constraint = a and (b ge c + 3.expr) and (b * 2.expr gt 8.expr) and + ((e eq mkBv(3.toByte())) or (d neq mkRealNum("271/100"))) and + (f eq 2.48.expr) and + (d + b.toRealExpr() neq mkRealNum("12/10")) and + (mkArraySelect(g, d) eq a) and + (mkArraySelect(g, d + mkRealNum(1)) neq a) and + (mkArraySelect(g, d - mkRealNum(1)) eq a) and + (d * d eq mkRealNum(4)) and + (mkBvMulExpr(e, e) eq mkBv(9.toByte())) // and (mkExistentialQuantifier(x eq 2.expr, listOf())) + //(mkFpMulExpr(mkFpRoundingModeExpr(KFpRoundingMode.RoundTowardZero), h, h) eq 2.0.expr) + + //val constraint = mkBvXorExpr(mkBvShiftLeftExpr(e, mkBv(1.toByte())), mkBvNotExpr(e)) eq + // mkBvLogicalShiftRightExpr(e, mkBv(1.toByte())) + + // (constraint as KAndExpr) + + //constraint.accept(Kek(this)) + //Kek(this).apply(constraint) + val extractor = FormulaGraphExtractor(this, constraint, FileOutputStream("kek.txt")) + //val extractor = io.ksmt.solver.neurosmt.preprocessing.FormulaGraphExtractor(this, constraint, System.err) + extractor.extractGraph() + + /*println("========") + constraint.apply { + println(this.args) + }*/ + + return@with KNeuroSMTSolver(this).use { solver -> // create a Stub SMT solver instance // assert expression @@ -21,7 +130,31 @@ fun main() { // check assertions satisfiability with timeout val satisfiability = solver.check(timeout = 1.seconds) - println(satisfiability) // SAT + // println(satisfiability) // SAT } + + KZ3Solver(this).use { solver -> + solver.assert(constraint) + + val satisfiability = solver.check(timeout = 180.seconds) + println(satisfiability) + + if (satisfiability == KSolverStatus.SAT) { + val model = solver.model() + println(model) + } + } + + val formula = """ + (declare-fun x () Real) + (declare-fun y () Real) + (declare-fun z () Real) + (assert (or (and (= y (+ x z)) (= x (+ y z))) (= 2.71 x))) + (check-sat) + """ + val assertions = KZ3SMTLibParser(this).parse(formula) + + println(assertions) + println(assertions[0].stringRepr) } } \ No newline at end of file From 4ae03cc5f446a98b34839772b2be80a842eb8dec Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Thu, 20 Jul 2023 14:53:49 +0300 Subject: [PATCH 04/52] some fixes for extractor --- .../neurosmt/smt2converter/SMT2Converter.kt | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/SMT2Converter.kt b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/SMT2Converter.kt index 20a3961ea..b43aac7f3 100644 --- a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/SMT2Converter.kt +++ b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/SMT2Converter.kt @@ -18,7 +18,7 @@ fun main(args: Array) { val files = Files.walk(Path.of(inputRoot)).filter { it.isRegularFile() } var ok = 0; var fail = 0 - var sat = 0; var unsat = 0; var unknown = 0 + var sat = 0; var unsat = 0; var skipped = 0 val ctx = KContext() @@ -30,32 +30,39 @@ fun main(args: Array) { val answer = getAnswerForTest(it) - when (answer) { - KSolverStatus.SAT -> sat++ - KSolverStatus.UNSAT -> unsat++ - KSolverStatus.UNKNOWN -> { - unknown++ - return@forEach - } + if (answer == KSolverStatus.UNKNOWN) { + skipped++ + return@forEach } with(ctx) { val formula = try { + val assertList = KZ3SMTLibParser(ctx).parse(it) ok++ - mkAnd(KZ3SMTLibParser(ctx).parse(it)) + mkAnd(assertList) } catch (e: KSMTLibParseException) { fail++ + e.printStackTrace() return@forEach } - val extractor = FormulaGraphExtractor(ctx, formula, FileOutputStream("$outputRoot/$answer/$curIdx")) + val outputStream = FileOutputStream("$outputRoot/$answer/$curIdx") + outputStream.write("; $it\n".encodeToByteArray()) + + val extractor = FormulaGraphExtractor(ctx, formula, outputStream) extractor.extractGraph() } + when (answer) { + KSolverStatus.SAT -> sat++ + KSolverStatus.UNSAT -> unsat++ + else -> { /* can't happen */ } + } + curIdx++ } println() - println("parsed: $ok; failed: $fail") - println("sat: $sat; unsat: $unsat; unknown: $unknown") + println("processed: $ok; failed: $fail") + println("sat: $sat; unsat: $unsat; skipped: $skipped") } \ No newline at end of file From c5c7e71619856a61008517a3f895af8b38cf08a4 Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Thu, 20 Jul 2023 19:00:24 +0300 Subject: [PATCH 05/52] another fix --- .../io/ksmt/solver/neurosmt/smt2converter/SMT2Converter.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/SMT2Converter.kt b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/SMT2Converter.kt index b43aac7f3..83ef0e39e 100644 --- a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/SMT2Converter.kt +++ b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/SMT2Converter.kt @@ -46,7 +46,7 @@ fun main(args: Array) { return@forEach } - val outputStream = FileOutputStream("$outputRoot/$answer/$curIdx") + val outputStream = FileOutputStream("$outputRoot/$curIdx-${answer.toString().lowercase()}") outputStream.write("; $it\n".encodeToByteArray()) val extractor = FormulaGraphExtractor(ctx, formula, outputStream) From 0236f09a307cb0dc255c77479f91456ae04a12d3 Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Fri, 21 Jul 2023 12:24:35 +0300 Subject: [PATCH 06/52] fuck python --- .../src/main/python/GraphDataloader.py | 105 ++++++++++++++++++ ksmt-neurosmt/src/main/python/GraphReader.py | 57 ++++++++++ ksmt-neurosmt/src/main/python/main.py | 72 ++++++++++++ ksmt-neurosmt/src/main/python/utils.py | 11 ++ 4 files changed, 245 insertions(+) create mode 100644 ksmt-neurosmt/src/main/python/GraphDataloader.py create mode 100644 ksmt-neurosmt/src/main/python/GraphReader.py create mode 100755 ksmt-neurosmt/src/main/python/main.py create mode 100644 ksmt-neurosmt/src/main/python/utils.py diff --git a/ksmt-neurosmt/src/main/python/GraphDataloader.py b/ksmt-neurosmt/src/main/python/GraphDataloader.py new file mode 100644 index 000000000..ee35f0402 --- /dev/null +++ b/ksmt-neurosmt/src/main/python/GraphDataloader.py @@ -0,0 +1,105 @@ +import os +import gc +import itertools + +import numpy as np +from tqdm import tqdm + +from sklearn.preprocessing import OrdinalEncoder + +import torch +from torch.utils.data import Dataset + +from torch_geometric.data import Data as Graph +from torch_geometric.loader import DataLoader + +from GraphReader import read_graph_from_file +from utils import train_val_test_indices + + +class GraphDataset(Dataset): + def __init__(self, node_sets, edge_sets, labels): + assert(len(node_sets) == len(edge_sets) and len(node_sets) == len(labels)) + + self.graphs = [Graph( + x=torch.tensor(nodes, dtype=torch.float), + edge_index=torch.tensor(edges).T if len(edges) else torch.tensor([[], []]), + y=torch.tensor([[label]], dtype=torch.float) + ) for nodes, edges, label in zip(node_sets, edge_sets, labels)] + + def __len__(self): + return len(self.graphs) + + def __getitem__(self, index): + return self.graphs[index] + + +def load_data(path_to_data): + all_operators, all_edges, all_labels = [], [], [] + + for it in tqdm(os.walk(path_to_data)): + for file_name in tqdm(it[2]): + cur_path = os.path.join(it[0], file_name) + + operators, edges = read_graph_from_file(cur_path) + + if len(edges) == 0: + continue + + all_operators.append(operators) + all_edges.append(edges) + if cur_path.endswith("-sat"): + all_labels.append(1) + elif cur_path.endswith("-unsat"): + all_labels.append(0) + else: + raise Exception(f"strange file path '{cur_path}'") + + assert (len(all_operators) == len(all_edges) and len(all_edges) == len(all_labels)) + + np.random.seed(24) + train_ind, val_ind, test_ind = train_val_test_indices(len(all_operators)) + + train_operators = [all_operators[i] for i in train_ind] + train_edges = [all_edges[i] for i in train_ind] + train_labels = [all_labels[i] for i in train_ind] + + val_operators = [all_operators[i] for i in val_ind] + val_edges = [all_edges[i] for i in val_ind] + val_labels = [all_labels[i] for i in val_ind] + + test_operators = [all_operators[i] for i in test_ind] + test_edges = [all_edges[i] for i in test_ind] + test_labels = [all_labels[i] for i in test_ind] + + # assert (len(train_operators) == len(train_edges) and len(train_edges) == len(train_labels)) + # assert (len(val_operators) == len(val_edges) and len(val_edges) == len(val_labels)) + # assert (len(test_operators) == len(test_edges) and len(test_edges) == len(test_labels)) + + del all_operators, all_edges, all_labels + gc.collect() + + encoder = OrdinalEncoder( + dtype=np.int, + handle_unknown="use_encoded_value", unknown_value=-1, + encoded_missing_value=-2 + ) + + encoder.fit(np.array(list(itertools.chain(*train_operators))).reshape(-1, 1)) + + def transform(op_list): + return encoder.transform(np.array(op_list).reshape(-1, 1)) + + train_operators = list(map(transform, train_operators)) + val_operators = list(map(transform, val_operators)) + test_operators = list(map(transform, test_operators)) + + train_ds = GraphDataset(train_operators, train_edges, train_labels) + val_ds = GraphDataset(val_operators, val_edges, val_labels) + test_ds = GraphDataset(test_operators, test_edges, test_labels) + + return ( + DataLoader(train_ds.graphs, batch_size=8), + DataLoader(val_ds.graphs, batch_size=8), + DataLoader(test_ds.graphs, batch_size=8) + ) diff --git a/ksmt-neurosmt/src/main/python/GraphReader.py b/ksmt-neurosmt/src/main/python/GraphReader.py new file mode 100644 index 000000000..6f00d43bf --- /dev/null +++ b/ksmt-neurosmt/src/main/python/GraphReader.py @@ -0,0 +1,57 @@ +from enum import Enum + + +class VertexType(Enum): + SYMBOLIC = 0 + VALUE = 1 + APP = 2 + + +def get_vertex_type(name): + if name == "SYMBOLIC": + return VertexType.SYMBOLIC + + if name == "VALUE": + return VertexType.VALUE + + return VertexType.APP + + +def process_line(line, v, operators, edges): + name, info = line.split(";") + info = info.strip() + vertex_type = get_vertex_type(name) + + if vertex_type == VertexType.APP: + operators.append(name) + + children = map(int, info.split(" ")) + for u in children: + edges.append([v, u]) + + else: + operators.append(name + ";" + info) + + +def read_graph_from_file(path): + operators, edges = [], [] + + with open(path, "r") as inf: + v = 0 + for line in inf.readlines(): + line = line.strip() + + if line.startswith(";"): + continue + + try: + process_line(line, v, operators, edges) + except Exception as e: + print(e, "\n") + print(path, "\n") + print(v, line, "\n") + raise e + + v += 1 + + return operators, edges diff --git a/ksmt-neurosmt/src/main/python/main.py b/ksmt-neurosmt/src/main/python/main.py new file mode 100755 index 000000000..83109b2ee --- /dev/null +++ b/ksmt-neurosmt/src/main/python/main.py @@ -0,0 +1,72 @@ +#!/usr/bin/python3 + +import sys +import os +import time + +from tqdm import tqdm, trange + +import torch +import torch.nn.functional as F +#from torch_geometric.data import Data +from torch_geometric.nn import GCNConv +from torch_geometric.utils import scatter + + +from GraphReader import read_graph_from_file +from GraphDataloader import load_data + + +class GCN(torch.nn.Module): + def __init__(self): + super().__init__() + self.conv1 = GCNConv(1, 16) + self.conv2 = GCNConv(16, 1) + + def forward(self, data): + x, edge_index = data.x, data.edge_index + + x = self.conv1(x, edge_index) + x = F.relu(x) + x = F.dropout(x, training=self.training) + x = self.conv2(x, edge_index) + + return x + + +if __name__ == "__main__": + tr, va, te = load_data(sys.argv[1]) + for batch in tr: + print(batch.num_graphs, batch) + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model = GCN().to(device) + optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=5e-4) + + model.train() + for epoch in trange(100): + for batch in tqdm(tr): + optimizer.zero_grad() + out = model(batch) + out = scatter(out, batch.batch, dim=0, reduce='mean') + loss = F.mse_loss(out, batch.y) + loss.backward() + optimizer.step() + + + #print(all_edges, all_expressions) + #print("\n\n\nFINISHED\n\n\n") + #time.sleep(5) + + """ + edge_index = torch.tensor([[0, 1, 1, 2], + [1, 0, 2, 1]], dtype=torch.long) + x = torch.tensor([[-1], [0], [1]], dtype=torch.float) + + print(torch.tensor(expressions).T) + data = Data( + x=torch.tensor(expressions).T, + edge_index=torch.tensor(edges) + ) + print(data, data.is_directed()) + """ diff --git a/ksmt-neurosmt/src/main/python/utils.py b/ksmt-neurosmt/src/main/python/utils.py new file mode 100644 index 000000000..24cbb57e5 --- /dev/null +++ b/ksmt-neurosmt/src/main/python/utils.py @@ -0,0 +1,11 @@ +import numpy as np + + +def train_val_test_indices(n, val_pct=0.15, test_pct=0.1): + perm = np.arange(n) + np.random.shuffle(perm) + + val_cnt = int(n * val_pct) + test_cnt = int(n * test_pct) + + return perm[val_cnt + test_cnt:], perm[:val_cnt], perm[val_cnt:val_cnt + test_cnt] From 13de179589e90b07f189d5791b6e3dca2505a706 Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Fri, 21 Jul 2023 14:25:52 +0300 Subject: [PATCH 07/52] yet another fix --- .../src/main/python/GraphDataloader.py | 12 ++++++++---- ksmt-neurosmt/src/main/python/main.py | 2 +- .../neurosmt/smt2converter/SMT2Converter.kt | 18 +++++++++++++++--- 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/ksmt-neurosmt/src/main/python/GraphDataloader.py b/ksmt-neurosmt/src/main/python/GraphDataloader.py index ee35f0402..fea91af3d 100644 --- a/ksmt-neurosmt/src/main/python/GraphDataloader.py +++ b/ksmt-neurosmt/src/main/python/GraphDataloader.py @@ -17,13 +17,16 @@ from utils import train_val_test_indices +BATCH_SIZE = 32 + + class GraphDataset(Dataset): def __init__(self, node_sets, edge_sets, labels): assert(len(node_sets) == len(edge_sets) and len(node_sets) == len(labels)) self.graphs = [Graph( x=torch.tensor(nodes, dtype=torch.float), - edge_index=torch.tensor(edges).T if len(edges) else torch.tensor([[], []]), + edge_index=torch.tensor(edges).T, y=torch.tensor([[label]], dtype=torch.float) ) for nodes, edges, label in zip(node_sets, edge_sets, labels)] @@ -44,6 +47,7 @@ def load_data(path_to_data): operators, edges = read_graph_from_file(cur_path) if len(edges) == 0: + print(f"w: formula with no edges; file '{cur_path}'") continue all_operators.append(operators) @@ -99,7 +103,7 @@ def transform(op_list): test_ds = GraphDataset(test_operators, test_edges, test_labels) return ( - DataLoader(train_ds.graphs, batch_size=8), - DataLoader(val_ds.graphs, batch_size=8), - DataLoader(test_ds.graphs, batch_size=8) + DataLoader(train_ds.graphs, batch_size=BATCH_SIZE), + DataLoader(val_ds.graphs, batch_size=BATCH_SIZE), + DataLoader(test_ds.graphs, batch_size=BATCH_SIZE) ) diff --git a/ksmt-neurosmt/src/main/python/main.py b/ksmt-neurosmt/src/main/python/main.py index 83109b2ee..e6f0983d0 100755 --- a/ksmt-neurosmt/src/main/python/main.py +++ b/ksmt-neurosmt/src/main/python/main.py @@ -44,7 +44,7 @@ def forward(self, data): optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=5e-4) model.train() - for epoch in trange(100): + for epoch in trange(50): for batch in tqdm(tr): optimizer.zero_grad() out = model(batch) diff --git a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/SMT2Converter.kt b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/SMT2Converter.kt index 83ef0e39e..175b12ff4 100644 --- a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/SMT2Converter.kt +++ b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/SMT2Converter.kt @@ -20,7 +20,7 @@ fun main(args: Array) { var ok = 0; var fail = 0 var sat = 0; var unsat = 0; var skipped = 0 - val ctx = KContext() + val ctx = KContext(simplificationMode = KContext.SimplificationMode.NO_SIMPLIFY) var curIdx = 0 ProgressBar.wrap(files, "converting smt2 files").forEach { @@ -38,8 +38,20 @@ fun main(args: Array) { with(ctx) { val formula = try { val assertList = KZ3SMTLibParser(ctx).parse(it) - ok++ - mkAnd(assertList) + when (assertList.size) { + 0 -> { + skipped++ + return@forEach + } + 1 -> { + ok++ + assertList[0] + } + else -> { + ok++ + mkAnd(assertList) + } + } } catch (e: KSMTLibParseException) { fail++ e.printStackTrace() From a6999eb8a67c9f7a12f004c6b2a80cfb2395b752 Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Mon, 24 Jul 2023 12:13:38 +0300 Subject: [PATCH 08/52] more fixes --- .../src/main/python/GraphDataloader.py | 83 +++++++++++++++---- ksmt-neurosmt/src/main/python/GraphReader.py | 14 ++-- ksmt-neurosmt/src/main/python/main.py | 77 +++++++++++++++-- ksmt-neurosmt/src/main/python/utils.py | 7 ++ 4 files changed, 152 insertions(+), 29 deletions(-) diff --git a/ksmt-neurosmt/src/main/python/GraphDataloader.py b/ksmt-neurosmt/src/main/python/GraphDataloader.py index fea91af3d..2b6e17d0e 100644 --- a/ksmt-neurosmt/src/main/python/GraphDataloader.py +++ b/ksmt-neurosmt/src/main/python/GraphDataloader.py @@ -14,21 +14,28 @@ from torch_geometric.loader import DataLoader from GraphReader import read_graph_from_file -from utils import train_val_test_indices +from utils import train_val_test_indices, align_sat_unsat_sizes -BATCH_SIZE = 32 +# torch.manual_seed(0) + +BATCH_SIZE = 1 # 32 class GraphDataset(Dataset): - def __init__(self, node_sets, edge_sets, labels): - assert(len(node_sets) == len(edge_sets) and len(node_sets) == len(labels)) + def __init__(self, node_sets, edge_sets, labels, depths): + assert ( + len(node_sets) == len(edge_sets) + and len(node_sets) == len(labels) + and len(labels) == len(depths) + ) self.graphs = [Graph( - x=torch.tensor(nodes, dtype=torch.float), - edge_index=torch.tensor(edges).T, - y=torch.tensor([[label]], dtype=torch.float) - ) for nodes, edges, label in zip(node_sets, edge_sets, labels)] + x=torch.tensor(nodes), + edge_index=torch.tensor(edges).t(), + y=torch.tensor([[label]], dtype=torch.float), + depth=depth + ) for nodes, edges, label, depth in zip(node_sets, edge_sets, labels, depths)] def __len__(self): return len(self.graphs) @@ -38,8 +45,47 @@ def __getitem__(self, index): def load_data(path_to_data): - all_operators, all_edges, all_labels = [], [], [] + sat_paths, unsat_paths = [], [] + for it in tqdm(os.walk(path_to_data)): + for file_name in tqdm(it[2]): + cur_path = os.path.join(it[0], file_name) + if cur_path.endswith("-sat"): + sat_paths.append(cur_path) + elif cur_path.endswith("-unsat"): + unsat_paths.append(cur_path) + else: + raise Exception(f"strange file path '{cur_path}'") + + np.random.seed(24) + sat_paths, unsat_paths = align_sat_unsat_sizes(sat_paths, unsat_paths) + all_operators, all_edges, all_labels, all_depths = [], [], [], [] + + for cur_path in tqdm(sat_paths): + operators, edges, depth = read_graph_from_file(cur_path) + + if len(edges) == 0: + print(f"w: formula with no edges; file '{cur_path}'") + continue + + all_operators.append(operators) + all_edges.append(edges) + all_labels.append(1) + all_depths.append(depth) + + for cur_path in tqdm(unsat_paths): + operators, edges, depth = read_graph_from_file(cur_path) + + if len(edges) == 0: + print(f"w: formula with no edges; file '{cur_path}'") + continue + + all_operators.append(operators) + all_edges.append(edges) + all_labels.append(0) + all_depths.append(depth) + + """ for it in tqdm(os.walk(path_to_data)): for file_name in tqdm(it[2]): cur_path = os.path.join(it[0], file_name) @@ -58,29 +104,36 @@ def load_data(path_to_data): all_labels.append(0) else: raise Exception(f"strange file path '{cur_path}'") + """ - assert (len(all_operators) == len(all_edges) and len(all_edges) == len(all_labels)) + assert ( + len(all_operators) == len(all_edges) + and len(all_edges) == len(all_labels) + and len(all_labels) == len(all_depths) + ) - np.random.seed(24) train_ind, val_ind, test_ind = train_val_test_indices(len(all_operators)) train_operators = [all_operators[i] for i in train_ind] train_edges = [all_edges[i] for i in train_ind] train_labels = [all_labels[i] for i in train_ind] + train_depths = [all_depths[i] for i in train_ind] val_operators = [all_operators[i] for i in val_ind] val_edges = [all_edges[i] for i in val_ind] val_labels = [all_labels[i] for i in val_ind] + val_depths = [all_depths[i] for i in val_ind] test_operators = [all_operators[i] for i in test_ind] test_edges = [all_edges[i] for i in test_ind] test_labels = [all_labels[i] for i in test_ind] + test_depths = [all_depths[i] for i in test_ind] # assert (len(train_operators) == len(train_edges) and len(train_edges) == len(train_labels)) # assert (len(val_operators) == len(val_edges) and len(val_edges) == len(val_labels)) # assert (len(test_operators) == len(test_edges) and len(test_edges) == len(test_labels)) - del all_operators, all_edges, all_labels + del all_operators, all_edges, all_labels, all_depths gc.collect() encoder = OrdinalEncoder( @@ -98,9 +151,9 @@ def transform(op_list): val_operators = list(map(transform, val_operators)) test_operators = list(map(transform, test_operators)) - train_ds = GraphDataset(train_operators, train_edges, train_labels) - val_ds = GraphDataset(val_operators, val_edges, val_labels) - test_ds = GraphDataset(test_operators, test_edges, test_labels) + train_ds = GraphDataset(train_operators, train_edges, train_labels, train_depths) + val_ds = GraphDataset(val_operators, val_edges, val_labels, val_depths) + test_ds = GraphDataset(test_operators, test_edges, test_labels, test_depths) return ( DataLoader(train_ds.graphs, batch_size=BATCH_SIZE), diff --git a/ksmt-neurosmt/src/main/python/GraphReader.py b/ksmt-neurosmt/src/main/python/GraphReader.py index 6f00d43bf..339fa9e71 100644 --- a/ksmt-neurosmt/src/main/python/GraphReader.py +++ b/ksmt-neurosmt/src/main/python/GraphReader.py @@ -17,24 +17,26 @@ def get_vertex_type(name): return VertexType.APP -def process_line(line, v, operators, edges): +def process_line(line, v, operators, edges, depth): name, info = line.split(";") info = info.strip() vertex_type = get_vertex_type(name) + depth.append(0) + assert (len(depth) == v + 1) if vertex_type == VertexType.APP: operators.append(name) children = map(int, info.split(" ")) for u in children: - edges.append([v, u]) - + edges.append([u, v]) + depth[v] = max(depth[v], depth[u] + 1) else: operators.append(name + ";" + info) def read_graph_from_file(path): - operators, edges = [], [] + operators, edges, depth = [], [], [] with open(path, "r") as inf: v = 0 @@ -45,7 +47,7 @@ def read_graph_from_file(path): continue try: - process_line(line, v, operators, edges) + process_line(line, v, operators, edges, depth) except Exception as e: print(e, "\n") print(path, "\n") @@ -54,4 +56,4 @@ def read_graph_from_file(path): v += 1 - return operators, edges + return operators, edges, max(depth) diff --git a/ksmt-neurosmt/src/main/python/main.py b/ksmt-neurosmt/src/main/python/main.py index e6f0983d0..19c14484c 100755 --- a/ksmt-neurosmt/src/main/python/main.py +++ b/ksmt-neurosmt/src/main/python/main.py @@ -7,6 +7,7 @@ from tqdm import tqdm, trange import torch +import torch.nn as nn import torch.nn.functional as F #from torch_geometric.data import Data from torch_geometric.nn import GCNConv @@ -17,15 +18,71 @@ from GraphDataloader import load_data +import torch +from torch.nn import Linear, Parameter +from torch_geometric.nn import MessagePassing +from torch_geometric.utils import add_self_loops, degree + +class GCNConv1(MessagePassing): + def __init__(self, in_channels, out_channels): + super().__init__(aggr='add', flow="source_to_target") # "Add" aggregation (Step 5). + self.lin = Linear(in_channels, out_channels, bias=False) + self.bias = Parameter(torch.empty(out_channels)) + + self.reset_parameters() + + def reset_parameters(self): + self.lin.reset_parameters() + self.bias.data.zero_() + + def forward(self, x, edge_index): + # x has shape [N, in_channels] + # edge_index has shape [2, E] + + # Step 1: Add self-loops to the adjacency matrix. + #edge_index, _ = add_self_loops(edge_index, num_nodes=x.size(0)) + + # Step 2: Linearly transform node feature matrix. + x = self.lin(x) + + # Step 3: Compute normalization. + row, col = edge_index + deg = degree(col, x.size(0), dtype=x.dtype) + deg_inv_sqrt = deg.pow(-0.5) + deg_inv_sqrt[deg_inv_sqrt == float('inf')] = 0 + norm = deg_inv_sqrt[row] * deg_inv_sqrt[col] + + # Step 4-5: Start propagating messages. + out = self.propagate(edge_index, x=x, norm=norm) + + # Step 6: Apply a final bias vector. + out += self.bias + + return out + + def message(self, x_j, norm): + # x_j has shape [E, out_channels] + + # Step 4: Normalize node features. + return norm.view(-1, 1) * x_j + + class GCN(torch.nn.Module): def __init__(self): super().__init__() - self.conv1 = GCNConv(1, 16) - self.conv2 = GCNConv(16, 1) + + self.embedding = nn.Embedding(1000, 16) + self.conv1 = GCNConv(16, 16, add_self_loops=False) + self.conv2 = GCNConv(16, 1, add_self_loops=False) + + #self.conv1 = GCNConv(16, 1) + #self.conv2 = GCNConv(16, 1) def forward(self, data): x, edge_index = data.x, data.edge_index + x = self.embedding(x.squeeze()) + #x[-2] = 1.23 x = self.conv1(x, edge_index) x = F.relu(x) x = F.dropout(x, training=self.training) @@ -38,20 +95,24 @@ def forward(self, data): tr, va, te = load_data(sys.argv[1]) for batch in tr: print(batch.num_graphs, batch) + #print(batch.ptr) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model = GCN().to(device) optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=5e-4) model.train() - for epoch in trange(50): - for batch in tqdm(tr): - optimizer.zero_grad() + for epoch in range(1): + for batch in tr: + #optimizer.zero_grad() out = model(batch) + #print(out[0].item(), out[-1].item(), batch[0].num_nodes, batch[0].num_edges, batch[0].depth.item()) + print(batch[0].depth.item(), batch[0].num_nodes, batch[0].num_edges) + out = scatter(out, batch.batch, dim=0, reduce='mean') - loss = F.mse_loss(out, batch.y) - loss.backward() - optimizer.step() + #loss = F.mse_loss(out, batch.y) + #loss.backward() + #optimizer.step() #print(all_edges, all_expressions) diff --git a/ksmt-neurosmt/src/main/python/utils.py b/ksmt-neurosmt/src/main/python/utils.py index 24cbb57e5..fbe53e3d4 100644 --- a/ksmt-neurosmt/src/main/python/utils.py +++ b/ksmt-neurosmt/src/main/python/utils.py @@ -9,3 +9,10 @@ def train_val_test_indices(n, val_pct=0.15, test_pct=0.1): test_cnt = int(n * test_pct) return perm[val_cnt + test_cnt:], perm[:val_cnt], perm[val_cnt:val_cnt + test_cnt] + + +def align_sat_unsat_sizes(sat_paths, unsat_paths): + if len(sat_paths) > len(unsat_paths): + return list(np.random.choice(sat_paths, len(unsat_paths), replace=False)), unsat_paths + else: + return sat_paths, list(np.random.choice(unsat_paths, len(sat_paths), replace=False)) From 98dc95846547985efdde9deeaf8b3d2c9a73720c Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Tue, 25 Jul 2023 17:43:01 +0300 Subject: [PATCH 09/52] aaaaaaaaaaaaa --- .../src/main/python/GraphDataloader.py | 132 ++++++++++-------- ksmt-neurosmt/src/main/python/GraphReader.py | 37 +++-- ksmt-neurosmt/src/main/python/main.py | 61 +++++++- ksmt-neurosmt/src/main/python/utils.py | 4 +- 4 files changed, 156 insertions(+), 78 deletions(-) diff --git a/ksmt-neurosmt/src/main/python/GraphDataloader.py b/ksmt-neurosmt/src/main/python/GraphDataloader.py index 2b6e17d0e..b139fbab7 100644 --- a/ksmt-neurosmt/src/main/python/GraphDataloader.py +++ b/ksmt-neurosmt/src/main/python/GraphDataloader.py @@ -13,29 +13,30 @@ from torch_geometric.data import Data as Graph from torch_geometric.loader import DataLoader -from GraphReader import read_graph_from_file +from GraphReader import read_graph_by_path from utils import train_val_test_indices, align_sat_unsat_sizes -# torch.manual_seed(0) - BATCH_SIZE = 1 # 32 +MAX_FORMULA_DEPTH = 2408 class GraphDataset(Dataset): - def __init__(self, node_sets, edge_sets, labels, depths): + def __init__(self, graph_data): + """ assert ( len(node_sets) == len(edge_sets) and len(node_sets) == len(labels) and len(labels) == len(depths) ) + """ self.graphs = [Graph( x=torch.tensor(nodes), edge_index=torch.tensor(edges).t(), y=torch.tensor([[label]], dtype=torch.float), depth=depth - ) for nodes, edges, label, depth in zip(node_sets, edge_sets, labels, depths)] + ) for nodes, edges, label, depth in graph_data] def __len__(self): return len(self.graphs) @@ -56,64 +57,48 @@ def load_data(path_to_data): else: raise Exception(f"strange file path '{cur_path}'") - np.random.seed(24) - sat_paths, unsat_paths = align_sat_unsat_sizes(sat_paths, unsat_paths) - - all_operators, all_edges, all_labels, all_depths = [], [], [], [] + if len(sat_paths) > 5000: + sat_paths = sat_paths[:5000] - for cur_path in tqdm(sat_paths): - operators, edges, depth = read_graph_from_file(cur_path) + if len(unsat_paths) > 5000: + sat_paths = unsat_paths[:5000] - if len(edges) == 0: - print(f"w: formula with no edges; file '{cur_path}'") - continue - - all_operators.append(operators) - all_edges.append(edges) - all_labels.append(1) - all_depths.append(depth) - - for cur_path in tqdm(unsat_paths): - operators, edges, depth = read_graph_from_file(cur_path) - - if len(edges) == 0: - print(f"w: formula with no edges; file '{cur_path}'") - continue + np.random.seed(24) + sat_paths, unsat_paths = align_sat_unsat_sizes(sat_paths, unsat_paths) - all_operators.append(operators) - all_edges.append(edges) - all_labels.append(0) - all_depths.append(depth) + graph_data = [] - """ - for it in tqdm(os.walk(path_to_data)): - for file_name in tqdm(it[2]): - cur_path = os.path.join(it[0], file_name) + def process_paths(paths, label): + for path in tqdm(paths): + operators, edges, depth = read_graph_by_path(path, max_depth=MAX_FORMULA_DEPTH) - operators, edges = read_graph_from_file(cur_path) + if depth > MAX_FORMULA_DEPTH: + continue if len(edges) == 0: - print(f"w: formula with no edges; file '{cur_path}'") + print(f"w: formula with no edges; file '{path}'") continue - all_operators.append(operators) - all_edges.append(edges) - if cur_path.endswith("-sat"): - all_labels.append(1) - elif cur_path.endswith("-unsat"): - all_labels.append(0) - else: - raise Exception(f"strange file path '{cur_path}'") - """ + graph_data.append((operators, edges, label, depth)) + + process_paths(sat_paths, 1) + process_paths(unsat_paths, 0) + """ assert ( len(all_operators) == len(all_edges) and len(all_edges) == len(all_labels) and len(all_labels) == len(all_depths) ) + """ - train_ind, val_ind, test_ind = train_val_test_indices(len(all_operators)) + train_ind, val_ind, test_ind = train_val_test_indices(len(graph_data)) + train_data = [graph_data[i] for i in train_ind] + val_data = [graph_data[i] for i in val_ind] + test_data = [graph_data[i] for i in test_ind] + + """ train_operators = [all_operators[i] for i in train_ind] train_edges = [all_edges[i] for i in train_ind] train_labels = [all_labels[i] for i in train_ind] @@ -128,32 +113,61 @@ def load_data(path_to_data): test_edges = [all_edges[i] for i in test_ind] test_labels = [all_labels[i] for i in test_ind] test_depths = [all_depths[i] for i in test_ind] + """ # assert (len(train_operators) == len(train_edges) and len(train_edges) == len(train_labels)) # assert (len(val_operators) == len(val_edges) and len(val_edges) == len(val_labels)) # assert (len(test_operators) == len(test_edges) and len(test_edges) == len(test_labels)) - del all_operators, all_edges, all_labels, all_depths + print("del start") + del graph_data gc.collect() + print("del end") encoder = OrdinalEncoder( - dtype=np.int, - handle_unknown="use_encoded_value", unknown_value=-1, - encoded_missing_value=-2 + dtype=int, + handle_unknown="use_encoded_value", unknown_value=1999, + encoded_missing_value=1998 ) - encoder.fit(np.array(list(itertools.chain(*train_operators))).reshape(-1, 1)) + print("enc fit start") + encoder.fit(np.array(list(itertools.chain( + *(list(zip(*train_data))[0]) + ))).reshape(-1, 1)) + print("enc fit end") + + """ + def transform_data(data): + for i in range(len(data)): + data[i][0] = encoder.transform(np.array(data[i][0]).reshape(-1, 1)) + """ + + def transform(data_for_one_graph): + nodes, edges, label, depth = data_for_one_graph + nodes = encoder.transform(np.array(nodes).reshape(-1, 1)) + return nodes, edges, label, depth + + print("transform start") + #transform_data(train_data) + #transform_data(val_data) + #transform_data(test_data) + + train_data = list(map(transform, train_data)) + val_data = list(map(transform, val_data)) + test_data = list(map(transform, test_data)) - def transform(op_list): - return encoder.transform(np.array(op_list).reshape(-1, 1)) + #train_operators = list(map(transform, train_operators)) + #val_operators = list(map(transform, val_operators)) + #test_operators = list(map(transform, test_operators)) + print("transform end") - train_operators = list(map(transform, train_operators)) - val_operators = list(map(transform, val_operators)) - test_operators = list(map(transform, test_operators)) + train_ds = GraphDataset(train_data) + val_ds = GraphDataset(val_data) + test_ds = GraphDataset(test_data) - train_ds = GraphDataset(train_operators, train_edges, train_labels, train_depths) - val_ds = GraphDataset(val_operators, val_edges, val_labels, val_depths) - test_ds = GraphDataset(test_operators, test_edges, test_labels, test_depths) + #train_ds = GraphDataset(train_operators, train_edges, train_labels, train_depths) + #val_ds = GraphDataset(val_operators, val_edges, val_labels, val_depths) + #test_ds = GraphDataset(test_operators, test_edges, test_labels, test_depths) return ( DataLoader(train_ds.graphs, batch_size=BATCH_SIZE), diff --git a/ksmt-neurosmt/src/main/python/GraphReader.py b/ksmt-neurosmt/src/main/python/GraphReader.py index 339fa9e71..02504bff0 100644 --- a/ksmt-neurosmt/src/main/python/GraphReader.py +++ b/ksmt-neurosmt/src/main/python/GraphReader.py @@ -35,25 +35,32 @@ def process_line(line, v, operators, edges, depth): operators.append(name + ";" + info) -def read_graph_from_file(path): +def read_graph_from_file(inf, max_depth): operators, edges, depth = [], [], [] - with open(path, "r") as inf: - v = 0 - for line in inf.readlines(): - line = line.strip() + v = 0 + for line in inf.readlines(): + line = line.strip() + + if line.startswith(";"): + continue - if line.startswith(";"): - continue + try: + process_line(line, v, operators, edges, depth) + except Exception as e: + print(e, "\n") + print(inf.name, "\n") + print(v, line, "\n") + raise e - try: - process_line(line, v, operators, edges, depth) - except Exception as e: - print(e, "\n") - print(path, "\n") - print(v, line, "\n") - raise e + if depth[v] > max_depth: + return None, None, depth[v] - v += 1 + v += 1 return operators, edges, max(depth) + + +def read_graph_by_path(path, max_depth): + with open(path, "r") as inf: + return read_graph_from_file(inf, max_depth) diff --git a/ksmt-neurosmt/src/main/python/main.py b/ksmt-neurosmt/src/main/python/main.py index 19c14484c..ee6598e7e 100755 --- a/ksmt-neurosmt/src/main/python/main.py +++ b/ksmt-neurosmt/src/main/python/main.py @@ -1,7 +1,7 @@ #!/usr/bin/python3 import sys -import os +import os; os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" import time from tqdm import tqdm, trange @@ -23,6 +23,12 @@ from torch_geometric.nn import MessagePassing from torch_geometric.utils import add_self_loops, degree +from Encoder import Encoder +from Decoder import Decoder +from Model import Model + + +""" class GCNConv1(MessagePassing): def __init__(self, in_channels, out_channels): super().__init__(aggr='add', flow="source_to_target") # "Add" aggregation (Step 5). @@ -89,14 +95,62 @@ def forward(self, data): x = self.conv2(x, edge_index) return x +""" if __name__ == "__main__": tr, va, te = load_data(sys.argv[1]) - for batch in tr: - print(batch.num_graphs, batch) + + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model = Model().to(device) + optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=5e-4) + + for epoch in trange(100): + model.train() + for batch in tqdm(tr): + optimizer.zero_grad() + batch = batch.to(device) + + out = model(batch) + out = out[batch.ptr[:-1]] + + loss = F.binary_cross_entropy_with_logits(out, batch.y) + loss.backward() + + optimizer.step() + + model.eval() + for batch in tqdm(va): + + + with torch.no_grad(): + batch = batch.to(device) + + out = model(batch) + out = out[batch.ptr[:-1]] + out = F.sigmoid(out) + print(out) + #loss = F.binary_cross_entropy_with_logits(out, batch.y) + + + + + + + + + + + + + + + + #for batch in tr: + # print(batch.num_graphs, batch) #print(batch.ptr) + """ device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model = GCN().to(device) optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=5e-4) @@ -114,6 +168,7 @@ def forward(self, data): #loss.backward() #optimizer.step() + """ #print(all_edges, all_expressions) #print("\n\n\nFINISHED\n\n\n") diff --git a/ksmt-neurosmt/src/main/python/utils.py b/ksmt-neurosmt/src/main/python/utils.py index fbe53e3d4..3db5a855c 100644 --- a/ksmt-neurosmt/src/main/python/utils.py +++ b/ksmt-neurosmt/src/main/python/utils.py @@ -14,5 +14,7 @@ def train_val_test_indices(n, val_pct=0.15, test_pct=0.1): def align_sat_unsat_sizes(sat_paths, unsat_paths): if len(sat_paths) > len(unsat_paths): return list(np.random.choice(sat_paths, len(unsat_paths), replace=False)), unsat_paths - else: + elif len(sat_paths) < len(unsat_paths): return sat_paths, list(np.random.choice(unsat_paths, len(sat_paths), replace=False)) + else: + return sat_paths, unsat_paths From 58791a4b3a06034b1b550490af00a8b504cd300d Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Wed, 26 Jul 2023 11:10:05 +0300 Subject: [PATCH 10/52] train baseline --- ksmt-neurosmt/src/main/python/Decoder.py | 17 +++++ ksmt-neurosmt/src/main/python/Encoder.py | 24 +++++++ .../src/main/python/GraphDataloader.py | 44 ++++++------ ksmt-neurosmt/src/main/python/Model.py | 20 ++++++ ksmt-neurosmt/src/main/python/main.py | 67 ++++++++++++++++--- ksmt-neurosmt/src/main/python/utils.py | 29 ++++++-- .../smt2converter/FormulaGraphExtractor.kt | 20 ++++-- 7 files changed, 181 insertions(+), 40 deletions(-) create mode 100644 ksmt-neurosmt/src/main/python/Decoder.py create mode 100644 ksmt-neurosmt/src/main/python/Encoder.py create mode 100644 ksmt-neurosmt/src/main/python/Model.py diff --git a/ksmt-neurosmt/src/main/python/Decoder.py b/ksmt-neurosmt/src/main/python/Decoder.py new file mode 100644 index 000000000..9ec645ae2 --- /dev/null +++ b/ksmt-neurosmt/src/main/python/Decoder.py @@ -0,0 +1,17 @@ +import torch.nn as nn + + +class Decoder(nn.Module): + def __init__(self, hidden_dim): + super().__init__() + + self.lin1 = nn.Linear(hidden_dim, hidden_dim) + self.act = nn.ReLU() + self.lin2 = nn.Linear(hidden_dim, 1) + + def forward(self, data): + data = self.lin1(data) + data = self.act(data) + data = self.lin2(data) + + return data diff --git a/ksmt-neurosmt/src/main/python/Encoder.py b/ksmt-neurosmt/src/main/python/Encoder.py new file mode 100644 index 000000000..c9c91eb45 --- /dev/null +++ b/ksmt-neurosmt/src/main/python/Encoder.py @@ -0,0 +1,24 @@ +import torch.nn as nn + +from torch_geometric.nn import GCNConv, GatedGraphConv, GATConv + + +class Encoder(nn.Module): + def __init__(self, hidden_dim): + super().__init__() + + self.embedding = nn.Embedding(2000, hidden_dim) + #self.conv = GatedGraphConv(64, num_layers=1, aggr="mean") + self.conv = GCNConv(hidden_dim, hidden_dim, add_self_loops=False) + #self.conv = GATConv(64, 64, add_self_loops=False) + + def forward(self, data): + x, edge_index, depth = data.x, data.edge_index, data.depth + + x = self.embedding(x.squeeze()) + + depth = depth.max() + for i in range(depth): + x = self.conv(x, edge_index) + + return x diff --git a/ksmt-neurosmt/src/main/python/GraphDataloader.py b/ksmt-neurosmt/src/main/python/GraphDataloader.py index b139fbab7..3995f7e3c 100644 --- a/ksmt-neurosmt/src/main/python/GraphDataloader.py +++ b/ksmt-neurosmt/src/main/python/GraphDataloader.py @@ -17,8 +17,9 @@ from utils import train_val_test_indices, align_sat_unsat_sizes -BATCH_SIZE = 1 # 32 +BATCH_SIZE = 1 MAX_FORMULA_DEPTH = 2408 +NUM_WORKERS = 4 class GraphDataset(Dataset): @@ -57,18 +58,7 @@ def load_data(path_to_data): else: raise Exception(f"strange file path '{cur_path}'") - if len(sat_paths) > 5000: - sat_paths = sat_paths[:5000] - - if len(unsat_paths) > 5000: - sat_paths = unsat_paths[:5000] - - np.random.seed(24) - sat_paths, unsat_paths = align_sat_unsat_sizes(sat_paths, unsat_paths) - - graph_data = [] - - def process_paths(paths, label): + def process_paths(paths, label, data): for path in tqdm(paths): operators, edges, depth = read_graph_by_path(path, max_depth=MAX_FORMULA_DEPTH) @@ -79,10 +69,21 @@ def process_paths(paths, label): print(f"w: formula with no edges; file '{path}'") continue - graph_data.append((operators, edges, label, depth)) + data.append((operators, edges, label, depth)) - process_paths(sat_paths, 1) - process_paths(unsat_paths, 0) + sat_data, unsat_data = [], [] + process_paths(sat_paths, 1, sat_data) + process_paths(unsat_paths, 0, unsat_data) + + np.random.seed(24) + sat_data, unsat_data = align_sat_unsat_sizes(sat_data, unsat_data) + + graph_data = sat_data + unsat_data + del sat_data, unsat_data + gc.collect() + + print("\nstats:") + print(f"overall: {sum(it[2] for it in graph_data) / len(graph_data)} | {len(graph_data)}") """ assert ( @@ -98,6 +99,11 @@ def process_paths(paths, label): val_data = [graph_data[i] for i in val_ind] test_data = [graph_data[i] for i in test_ind] + print(f"train: {sum(it[2] for it in train_data) / len(train_data)} | {len(train_data)}") + print(f"val: {sum(it[2] for it in val_data) / len(val_data)} | {len(val_data)}") + print(f"test: {sum(it[2] for it in test_data) / len(test_data)} | {len(test_data)}") + print(flush=True) + """ train_operators = [all_operators[i] for i in train_ind] train_edges = [all_edges[i] for i in train_ind] @@ -170,7 +176,7 @@ def transform(data_for_one_graph): #test_ds = GraphDataset(test_operators, test_edges, test_labels, test_depths) return ( - DataLoader(train_ds.graphs, batch_size=BATCH_SIZE), - DataLoader(val_ds.graphs, batch_size=BATCH_SIZE), - DataLoader(test_ds.graphs, batch_size=BATCH_SIZE) + DataLoader(train_ds.graphs, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS), + DataLoader(val_ds.graphs, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS), + DataLoader(test_ds.graphs, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS) ) diff --git a/ksmt-neurosmt/src/main/python/Model.py b/ksmt-neurosmt/src/main/python/Model.py new file mode 100644 index 000000000..a744b8870 --- /dev/null +++ b/ksmt-neurosmt/src/main/python/Model.py @@ -0,0 +1,20 @@ +import torch.nn as nn + +from Encoder import Encoder +from Decoder import Decoder + +EMBEDDING_DIM = 32 + + +class Model(nn.Module): + def __init__(self): + super().__init__() + + self.encoder = Encoder(hidden_dim=EMBEDDING_DIM) + self.decoder = Decoder(hidden_dim=EMBEDDING_DIM) + + def forward(self, data): + data = self.encoder(data) + data = self.decoder(data) + + return data diff --git a/ksmt-neurosmt/src/main/python/main.py b/ksmt-neurosmt/src/main/python/main.py index ee6598e7e..e68f25ced 100755 --- a/ksmt-neurosmt/src/main/python/main.py +++ b/ksmt-neurosmt/src/main/python/main.py @@ -2,6 +2,8 @@ import sys import os; os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" + +import numpy as np import time from tqdm import tqdm, trange @@ -27,6 +29,8 @@ from Decoder import Decoder from Model import Model +from sklearn.metrics import accuracy_score, classification_report + """ class GCNConv1(MessagePassing): @@ -103,7 +107,18 @@ def forward(self, data): device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model = Model().to(device) - optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=5e-4) + optimizer = torch.optim.Adam([p for p in model.parameters() if p is not None and p.requires_grad], lr=1e-4) + + def calc_grad_norm(): + grads = [ + p.grad.detach().flatten() for p in model.parameters() if p.grad is not None and p.requires_grad + ] + return torch.cat(grads).norm().item() + + for p in model.parameters(): + assert (p.requires_grad) + + criterion = nn.BCEWithLogitsLoss() for epoch in trange(100): model.train() @@ -115,22 +130,56 @@ def forward(self, data): out = out[batch.ptr[:-1]] loss = F.binary_cross_entropy_with_logits(out, batch.y) + #loss = criterion(out, batch.y) loss.backward() optimizer.step() - model.eval() - for batch in tqdm(va): + print("\n", flush=True) + print(f"grad norm: {calc_grad_norm()}") + def validate(dl): + model.eval() + #all_ans, correct_ans = 0, 0 + answers, targets = torch.tensor([]).to(device), torch.tensor([]).to(device) + losses = [] with torch.no_grad(): - batch = batch.to(device) + for batch in tqdm(dl): + batch = batch.to(device) + + out = model(batch) + out = out[batch.ptr[:-1]] + loss = F.binary_cross_entropy_with_logits(out, batch.y) + + out = F.sigmoid(out) + out = (out > 0.5) + + answers = torch.cat((answers, out)) + targets = torch.cat((targets, batch.y.to(torch.int).to(torch.bool))) + losses.append(loss.item()) + + #all_ans += len(batch.y) + #out: torch.Tensor = (batch.y.to(torch.int) == out.to(torch.bool)) + #correct_ans += out.int().sum().item() + + answers = torch.flatten(answers).detach().cpu().numpy() + targets = torch.flatten(targets).detach().cpu().numpy() + + #print(f"\n{correct_ans / all_ans}") + print(flush=True) + print(f"mean loss: {np.mean(losses)}") + print(f"acc: {accuracy_score(targets, answers)}", flush=True) + print(classification_report(targets, answers, digits=3, zero_division=0.0), flush=True) + + print() + print("train:") + validate(tr) + print("val:") + validate(va) + print() + - out = model(batch) - out = out[batch.ptr[:-1]] - out = F.sigmoid(out) - print(out) - #loss = F.binary_cross_entropy_with_logits(out, batch.y) diff --git a/ksmt-neurosmt/src/main/python/utils.py b/ksmt-neurosmt/src/main/python/utils.py index 3db5a855c..09cef7f19 100644 --- a/ksmt-neurosmt/src/main/python/utils.py +++ b/ksmt-neurosmt/src/main/python/utils.py @@ -11,10 +11,27 @@ def train_val_test_indices(n, val_pct=0.15, test_pct=0.1): return perm[val_cnt + test_cnt:], perm[:val_cnt], perm[val_cnt:val_cnt + test_cnt] -def align_sat_unsat_sizes(sat_paths, unsat_paths): - if len(sat_paths) > len(unsat_paths): - return list(np.random.choice(sat_paths, len(unsat_paths), replace=False)), unsat_paths - elif len(sat_paths) < len(unsat_paths): - return sat_paths, list(np.random.choice(unsat_paths, len(sat_paths), replace=False)) +def align_sat_unsat_sizes(sat_data, unsat_data): + sat_indices = list(range(len(sat_data))) + unsat_indices = list(range(len(unsat_data))) + + if len(sat_indices) > len(unsat_indices): + sat_indices = np.random.choice(np.array(sat_indices), len(unsat_indices), replace=False) + elif len(sat_indices) < len(unsat_indices): + unsat_indices = np.random.choice(np.array(unsat_indices), len(sat_indices), replace=False) + + return ( + list(np.array(sat_data, dtype=object)[sat_indices]), + list(np.array(unsat_data, dtype=object)[unsat_indices]) + ) + + """ + if len(sat_data) > len(unsat_data): + return list(np.random.choice(np.array(sat_data, dtype=object), len(unsat_data), replace=False)), unsat_data + elif len(sat_data) < len(unsat_data): + print(type(unsat_data[0])) + print(type(np.array(unsat_data, dtype=np.object)[0])) + return sat_data, list(np.random.choice(np.array(unsat_data, dtype=object), len(sat_data), replace=False)) else: - return sat_paths, unsat_paths + return sat_data, unsat_data + """ diff --git a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/FormulaGraphExtractor.kt b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/FormulaGraphExtractor.kt index bc6b9933a..4633749df 100644 --- a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/FormulaGraphExtractor.kt +++ b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/FormulaGraphExtractor.kt @@ -1,12 +1,12 @@ package io.ksmt.solver.neurosmt.smt2converter import io.ksmt.KContext -import io.ksmt.expr.KApp -import io.ksmt.expr.KConst -import io.ksmt.expr.KExpr -import io.ksmt.expr.KInterpretedValue +import io.ksmt.decl.KBitVecCustomSizeValueDecl +import io.ksmt.expr.* import io.ksmt.expr.transformer.KNonRecursiveTransformer import io.ksmt.sort.KBoolSort +import io.ksmt.sort.KBvCustomSizeSort +import io.ksmt.sort.KBvSort import io.ksmt.sort.KSort import java.io.OutputStream import java.util.IdentityHashMap @@ -47,11 +47,19 @@ class FormulaGraphExtractor( } fun writeSymbolicVariable(symbol: KConst) { - writer.write("SYMBOLIC; ${symbol.sort}\n") + when (symbol.sort) { + is KBoolSort -> writer.write("SYMBOLIC; Bool\n") + is KBvSort -> writer.write("SYMBOLIC; BitVec\n") + else -> error("unknown symbolic sort: ${symbol.sort}") + } } fun writeValue(value: KInterpretedValue) { - writer.write("VALUE; ${value.decl.sort}\n") + when (value.decl.sort) { + is KBoolSort -> writer.write("VALUE; Bool\n") + is KBvSort -> writer.write("VALUE; BitVec\n") + else -> error("unknown value sort: ${value.decl.sort}") + } } fun writeApp(expr: KApp) { From fab60e8d0640f0060b0cdbabe90f29f1b31ec3b0 Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Wed, 26 Jul 2023 11:10:05 +0300 Subject: [PATCH 11/52] train baseline --- ksmt-neurosmt/src/main/python/Decoder.py | 17 ++++ ksmt-neurosmt/src/main/python/Encoder.py | 24 ++++++ .../src/main/python/GraphDataloader.py | 44 ++++++----- ksmt-neurosmt/src/main/python/Model.py | 20 +++++ ksmt-neurosmt/src/main/python/main.py | 77 ++++++++++++++++--- ksmt-neurosmt/src/main/python/utils.py | 29 +++++-- .../smt2converter/FormulaGraphExtractor.kt | 20 +++-- 7 files changed, 191 insertions(+), 40 deletions(-) create mode 100644 ksmt-neurosmt/src/main/python/Decoder.py create mode 100644 ksmt-neurosmt/src/main/python/Encoder.py create mode 100644 ksmt-neurosmt/src/main/python/Model.py diff --git a/ksmt-neurosmt/src/main/python/Decoder.py b/ksmt-neurosmt/src/main/python/Decoder.py new file mode 100644 index 000000000..9ec645ae2 --- /dev/null +++ b/ksmt-neurosmt/src/main/python/Decoder.py @@ -0,0 +1,17 @@ +import torch.nn as nn + + +class Decoder(nn.Module): + def __init__(self, hidden_dim): + super().__init__() + + self.lin1 = nn.Linear(hidden_dim, hidden_dim) + self.act = nn.ReLU() + self.lin2 = nn.Linear(hidden_dim, 1) + + def forward(self, data): + data = self.lin1(data) + data = self.act(data) + data = self.lin2(data) + + return data diff --git a/ksmt-neurosmt/src/main/python/Encoder.py b/ksmt-neurosmt/src/main/python/Encoder.py new file mode 100644 index 000000000..c9c91eb45 --- /dev/null +++ b/ksmt-neurosmt/src/main/python/Encoder.py @@ -0,0 +1,24 @@ +import torch.nn as nn + +from torch_geometric.nn import GCNConv, GatedGraphConv, GATConv + + +class Encoder(nn.Module): + def __init__(self, hidden_dim): + super().__init__() + + self.embedding = nn.Embedding(2000, hidden_dim) + #self.conv = GatedGraphConv(64, num_layers=1, aggr="mean") + self.conv = GCNConv(hidden_dim, hidden_dim, add_self_loops=False) + #self.conv = GATConv(64, 64, add_self_loops=False) + + def forward(self, data): + x, edge_index, depth = data.x, data.edge_index, data.depth + + x = self.embedding(x.squeeze()) + + depth = depth.max() + for i in range(depth): + x = self.conv(x, edge_index) + + return x diff --git a/ksmt-neurosmt/src/main/python/GraphDataloader.py b/ksmt-neurosmt/src/main/python/GraphDataloader.py index b139fbab7..3995f7e3c 100644 --- a/ksmt-neurosmt/src/main/python/GraphDataloader.py +++ b/ksmt-neurosmt/src/main/python/GraphDataloader.py @@ -17,8 +17,9 @@ from utils import train_val_test_indices, align_sat_unsat_sizes -BATCH_SIZE = 1 # 32 +BATCH_SIZE = 1 MAX_FORMULA_DEPTH = 2408 +NUM_WORKERS = 4 class GraphDataset(Dataset): @@ -57,18 +58,7 @@ def load_data(path_to_data): else: raise Exception(f"strange file path '{cur_path}'") - if len(sat_paths) > 5000: - sat_paths = sat_paths[:5000] - - if len(unsat_paths) > 5000: - sat_paths = unsat_paths[:5000] - - np.random.seed(24) - sat_paths, unsat_paths = align_sat_unsat_sizes(sat_paths, unsat_paths) - - graph_data = [] - - def process_paths(paths, label): + def process_paths(paths, label, data): for path in tqdm(paths): operators, edges, depth = read_graph_by_path(path, max_depth=MAX_FORMULA_DEPTH) @@ -79,10 +69,21 @@ def process_paths(paths, label): print(f"w: formula with no edges; file '{path}'") continue - graph_data.append((operators, edges, label, depth)) + data.append((operators, edges, label, depth)) - process_paths(sat_paths, 1) - process_paths(unsat_paths, 0) + sat_data, unsat_data = [], [] + process_paths(sat_paths, 1, sat_data) + process_paths(unsat_paths, 0, unsat_data) + + np.random.seed(24) + sat_data, unsat_data = align_sat_unsat_sizes(sat_data, unsat_data) + + graph_data = sat_data + unsat_data + del sat_data, unsat_data + gc.collect() + + print("\nstats:") + print(f"overall: {sum(it[2] for it in graph_data) / len(graph_data)} | {len(graph_data)}") """ assert ( @@ -98,6 +99,11 @@ def process_paths(paths, label): val_data = [graph_data[i] for i in val_ind] test_data = [graph_data[i] for i in test_ind] + print(f"train: {sum(it[2] for it in train_data) / len(train_data)} | {len(train_data)}") + print(f"val: {sum(it[2] for it in val_data) / len(val_data)} | {len(val_data)}") + print(f"test: {sum(it[2] for it in test_data) / len(test_data)} | {len(test_data)}") + print(flush=True) + """ train_operators = [all_operators[i] for i in train_ind] train_edges = [all_edges[i] for i in train_ind] @@ -170,7 +176,7 @@ def transform(data_for_one_graph): #test_ds = GraphDataset(test_operators, test_edges, test_labels, test_depths) return ( - DataLoader(train_ds.graphs, batch_size=BATCH_SIZE), - DataLoader(val_ds.graphs, batch_size=BATCH_SIZE), - DataLoader(test_ds.graphs, batch_size=BATCH_SIZE) + DataLoader(train_ds.graphs, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS), + DataLoader(val_ds.graphs, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS), + DataLoader(test_ds.graphs, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS) ) diff --git a/ksmt-neurosmt/src/main/python/Model.py b/ksmt-neurosmt/src/main/python/Model.py new file mode 100644 index 000000000..a744b8870 --- /dev/null +++ b/ksmt-neurosmt/src/main/python/Model.py @@ -0,0 +1,20 @@ +import torch.nn as nn + +from Encoder import Encoder +from Decoder import Decoder + +EMBEDDING_DIM = 32 + + +class Model(nn.Module): + def __init__(self): + super().__init__() + + self.encoder = Encoder(hidden_dim=EMBEDDING_DIM) + self.decoder = Decoder(hidden_dim=EMBEDDING_DIM) + + def forward(self, data): + data = self.encoder(data) + data = self.decoder(data) + + return data diff --git a/ksmt-neurosmt/src/main/python/main.py b/ksmt-neurosmt/src/main/python/main.py index ee6598e7e..ed3ceb834 100755 --- a/ksmt-neurosmt/src/main/python/main.py +++ b/ksmt-neurosmt/src/main/python/main.py @@ -2,6 +2,8 @@ import sys import os; os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" + +import numpy as np import time from tqdm import tqdm, trange @@ -27,6 +29,8 @@ from Decoder import Decoder from Model import Model +from sklearn.metrics import accuracy_score, classification_report + """ class GCNConv1(MessagePassing): @@ -103,7 +107,28 @@ def forward(self, data): device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model = Model().to(device) - optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=5e-4) + optimizer = torch.optim.Adam([p for p in model.parameters() if p is not None and p.requires_grad], lr=1e-4) + + def calc_grad_norm(): + grads = [ + p.grad.detach().flatten() for p in model.parameters() if p.grad is not None and p.requires_grad + ] + return torch.cat(grads).norm().item() + + for p in model.parameters(): + assert (p.requires_grad) + + criterion = nn.BCEWithLogitsLoss() + + """ + for i, batch in enumerate(tr): + print(batch.y) + + if i >= 1000: + break + + print(flush=True) + """ for epoch in trange(100): model.train() @@ -115,22 +140,56 @@ def forward(self, data): out = out[batch.ptr[:-1]] loss = F.binary_cross_entropy_with_logits(out, batch.y) + #loss = criterion(out, batch.y) loss.backward() optimizer.step() - model.eval() - for batch in tqdm(va): + print("\n", flush=True) + print(f"grad norm: {calc_grad_norm()}") + def validate(dl): + model.eval() + #all_ans, correct_ans = 0, 0 + answers, targets = torch.tensor([]).to(device), torch.tensor([]).to(device) + losses = [] with torch.no_grad(): - batch = batch.to(device) + for batch in tqdm(dl): + batch = batch.to(device) + + out = model(batch) + out = out[batch.ptr[:-1]] + loss = F.binary_cross_entropy_with_logits(out, batch.y) + + out = F.sigmoid(out) + out = (out > 0.5) + + answers = torch.cat((answers, out)) + targets = torch.cat((targets, batch.y.to(torch.int).to(torch.bool))) + losses.append(loss.item()) + + #all_ans += len(batch.y) + #out: torch.Tensor = (batch.y.to(torch.int) == out.to(torch.bool)) + #correct_ans += out.int().sum().item() + + answers = torch.flatten(answers).detach().cpu().numpy() + targets = torch.flatten(targets).detach().cpu().numpy() + + #print(f"\n{correct_ans / all_ans}") + print(flush=True) + print(f"mean loss: {np.mean(losses)}") + print(f"acc: {accuracy_score(targets, answers)}", flush=True) + print(classification_report(targets, answers, digits=3, zero_division=0.0), flush=True) + + print() + print("train:") + validate(tr) + print("val:") + validate(va) + print() + - out = model(batch) - out = out[batch.ptr[:-1]] - out = F.sigmoid(out) - print(out) - #loss = F.binary_cross_entropy_with_logits(out, batch.y) diff --git a/ksmt-neurosmt/src/main/python/utils.py b/ksmt-neurosmt/src/main/python/utils.py index 3db5a855c..09cef7f19 100644 --- a/ksmt-neurosmt/src/main/python/utils.py +++ b/ksmt-neurosmt/src/main/python/utils.py @@ -11,10 +11,27 @@ def train_val_test_indices(n, val_pct=0.15, test_pct=0.1): return perm[val_cnt + test_cnt:], perm[:val_cnt], perm[val_cnt:val_cnt + test_cnt] -def align_sat_unsat_sizes(sat_paths, unsat_paths): - if len(sat_paths) > len(unsat_paths): - return list(np.random.choice(sat_paths, len(unsat_paths), replace=False)), unsat_paths - elif len(sat_paths) < len(unsat_paths): - return sat_paths, list(np.random.choice(unsat_paths, len(sat_paths), replace=False)) +def align_sat_unsat_sizes(sat_data, unsat_data): + sat_indices = list(range(len(sat_data))) + unsat_indices = list(range(len(unsat_data))) + + if len(sat_indices) > len(unsat_indices): + sat_indices = np.random.choice(np.array(sat_indices), len(unsat_indices), replace=False) + elif len(sat_indices) < len(unsat_indices): + unsat_indices = np.random.choice(np.array(unsat_indices), len(sat_indices), replace=False) + + return ( + list(np.array(sat_data, dtype=object)[sat_indices]), + list(np.array(unsat_data, dtype=object)[unsat_indices]) + ) + + """ + if len(sat_data) > len(unsat_data): + return list(np.random.choice(np.array(sat_data, dtype=object), len(unsat_data), replace=False)), unsat_data + elif len(sat_data) < len(unsat_data): + print(type(unsat_data[0])) + print(type(np.array(unsat_data, dtype=np.object)[0])) + return sat_data, list(np.random.choice(np.array(unsat_data, dtype=object), len(sat_data), replace=False)) else: - return sat_paths, unsat_paths + return sat_data, unsat_data + """ diff --git a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/FormulaGraphExtractor.kt b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/FormulaGraphExtractor.kt index bc6b9933a..4633749df 100644 --- a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/FormulaGraphExtractor.kt +++ b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/FormulaGraphExtractor.kt @@ -1,12 +1,12 @@ package io.ksmt.solver.neurosmt.smt2converter import io.ksmt.KContext -import io.ksmt.expr.KApp -import io.ksmt.expr.KConst -import io.ksmt.expr.KExpr -import io.ksmt.expr.KInterpretedValue +import io.ksmt.decl.KBitVecCustomSizeValueDecl +import io.ksmt.expr.* import io.ksmt.expr.transformer.KNonRecursiveTransformer import io.ksmt.sort.KBoolSort +import io.ksmt.sort.KBvCustomSizeSort +import io.ksmt.sort.KBvSort import io.ksmt.sort.KSort import java.io.OutputStream import java.util.IdentityHashMap @@ -47,11 +47,19 @@ class FormulaGraphExtractor( } fun writeSymbolicVariable(symbol: KConst) { - writer.write("SYMBOLIC; ${symbol.sort}\n") + when (symbol.sort) { + is KBoolSort -> writer.write("SYMBOLIC; Bool\n") + is KBvSort -> writer.write("SYMBOLIC; BitVec\n") + else -> error("unknown symbolic sort: ${symbol.sort}") + } } fun writeValue(value: KInterpretedValue) { - writer.write("VALUE; ${value.decl.sort}\n") + when (value.decl.sort) { + is KBoolSort -> writer.write("VALUE; Bool\n") + is KBvSort -> writer.write("VALUE; BitVec\n") + else -> error("unknown value sort: ${value.decl.sort}") + } } fun writeApp(expr: KApp) { From 829c45dd5744680c9f4412da857e3b7f8aa707cb Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Thu, 27 Jul 2023 15:02:35 +0300 Subject: [PATCH 12/52] fix major bug --- ksmt-neurosmt/src/main/python/Decoder.py | 5 ++++ ksmt-neurosmt/src/main/python/Encoder.py | 7 +++++ .../src/main/python/GraphDataloader.py | 21 ++++++++++---- ksmt-neurosmt/src/main/python/GraphReader.py | 10 +++---- ksmt-neurosmt/src/main/python/main.py | 28 ++++++++++++++----- 5 files changed, 53 insertions(+), 18 deletions(-) diff --git a/ksmt-neurosmt/src/main/python/Decoder.py b/ksmt-neurosmt/src/main/python/Decoder.py index 9ec645ae2..27eed810c 100644 --- a/ksmt-neurosmt/src/main/python/Decoder.py +++ b/ksmt-neurosmt/src/main/python/Decoder.py @@ -10,8 +10,13 @@ def __init__(self, hidden_dim): self.lin2 = nn.Linear(hidden_dim, 1) def forward(self, data): + #print("=" * 12) + #print(data.shape) + data = self.lin1(data) data = self.act(data) data = self.lin2(data) + #print(data.shape) + return data diff --git a/ksmt-neurosmt/src/main/python/Encoder.py b/ksmt-neurosmt/src/main/python/Encoder.py index c9c91eb45..6a7d26f81 100644 --- a/ksmt-neurosmt/src/main/python/Encoder.py +++ b/ksmt-neurosmt/src/main/python/Encoder.py @@ -1,3 +1,4 @@ +import torch import torch.nn as nn from torch_geometric.nn import GCNConv, GatedGraphConv, GATConv @@ -18,7 +19,13 @@ def forward(self, data): x = self.embedding(x.squeeze()) depth = depth.max() + #print(depth) for i in range(depth): x = self.conv(x, edge_index) + #x = self.conv(x, torch.tensor([[1, 2], [0, 0]], dtype=torch.long)) + + #print(x) + + x = x[data.ptr[1:] - 1] return x diff --git a/ksmt-neurosmt/src/main/python/GraphDataloader.py b/ksmt-neurosmt/src/main/python/GraphDataloader.py index 3995f7e3c..8afe3f98a 100644 --- a/ksmt-neurosmt/src/main/python/GraphDataloader.py +++ b/ksmt-neurosmt/src/main/python/GraphDataloader.py @@ -17,9 +17,10 @@ from utils import train_val_test_indices, align_sat_unsat_sizes -BATCH_SIZE = 1 -MAX_FORMULA_DEPTH = 2408 -NUM_WORKERS = 4 +BATCH_SIZE = 32 +MAX_FORMULA_SIZE = 10000 +MAX_FORMULA_DEPTH = 2500 # 2408 +NUM_WORKERS = 16 class GraphDataset(Dataset): @@ -58,11 +59,19 @@ def load_data(path_to_data): else: raise Exception(f"strange file path '{cur_path}'") + SHRINK = 10 ** 10 # 1000 + + if len(sat_paths) > SHRINK: + sat_paths = sat_paths[:SHRINK] + + if len(unsat_paths) > SHRINK: + unsat_paths = unsat_paths[:SHRINK] + def process_paths(paths, label, data): for path in tqdm(paths): - operators, edges, depth = read_graph_by_path(path, max_depth=MAX_FORMULA_DEPTH) + operators, edges, depth = read_graph_by_path(path, max_size=MAX_FORMULA_SIZE, max_depth=MAX_FORMULA_DEPTH) - if depth > MAX_FORMULA_DEPTH: + if depth is None: continue if len(edges) == 0: @@ -176,7 +185,7 @@ def transform(data_for_one_graph): #test_ds = GraphDataset(test_operators, test_edges, test_labels, test_depths) return ( - DataLoader(train_ds.graphs, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS), + DataLoader(train_ds.graphs, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS, shuffle=True, drop_last=True), DataLoader(val_ds.graphs, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS), DataLoader(test_ds.graphs, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS) ) diff --git a/ksmt-neurosmt/src/main/python/GraphReader.py b/ksmt-neurosmt/src/main/python/GraphReader.py index 02504bff0..62467a1ac 100644 --- a/ksmt-neurosmt/src/main/python/GraphReader.py +++ b/ksmt-neurosmt/src/main/python/GraphReader.py @@ -35,7 +35,7 @@ def process_line(line, v, operators, edges, depth): operators.append(name + ";" + info) -def read_graph_from_file(inf, max_depth): +def read_graph_from_file(inf, max_size, max_depth): operators, edges, depth = [], [], [] v = 0 @@ -53,14 +53,14 @@ def read_graph_from_file(inf, max_depth): print(v, line, "\n") raise e - if depth[v] > max_depth: - return None, None, depth[v] + if v >= max_size or depth[v] > max_depth: + return None, None, None v += 1 return operators, edges, max(depth) -def read_graph_by_path(path, max_depth): +def read_graph_by_path(path, max_size, max_depth): with open(path, "r") as inf: - return read_graph_from_file(inf, max_depth) + return read_graph_from_file(inf, max_size, max_depth) diff --git a/ksmt-neurosmt/src/main/python/main.py b/ksmt-neurosmt/src/main/python/main.py index ed3ceb834..f00db923a 100755 --- a/ksmt-neurosmt/src/main/python/main.py +++ b/ksmt-neurosmt/src/main/python/main.py @@ -130,14 +130,18 @@ def calc_grad_norm(): print(flush=True) """ - for epoch in trange(100): + with open("log.txt", "a") as f: + f.write("\n" + "=" * 12 + "\n") + + for epoch in trange(200): model.train() for batch in tqdm(tr): optimizer.zero_grad() batch = batch.to(device) out = model(batch) - out = out[batch.ptr[:-1]] + #out = out[batch.ptr[:-1]] + #out = out[batch.ptr[1:] - 1] loss = F.binary_cross_entropy_with_logits(out, batch.y) #loss = criterion(out, batch.y) @@ -159,10 +163,14 @@ def validate(dl): batch = batch.to(device) out = model(batch) - out = out[batch.ptr[:-1]] + #print(batch.ptr[1:] - 1) + #out = out[batch.ptr[:-1]] + #out = out[batch.ptr[1:] - 1] loss = F.binary_cross_entropy_with_logits(out, batch.y) out = F.sigmoid(out) + #print(out.detach().item(), batch.depth, batch.x.shape, batch.edge_index.shape) + #print(out.detach().item()) out = (out > 0.5) answers = torch.cat((answers, out)) @@ -177,18 +185,24 @@ def validate(dl): targets = torch.flatten(targets).detach().cpu().numpy() #print(f"\n{correct_ans / all_ans}") - print(flush=True) - print(f"mean loss: {np.mean(losses)}") + mean_loss = np.mean(losses) + print("\n", flush=True) + print(f"mean loss: {mean_loss}") print(f"acc: {accuracy_score(targets, answers)}", flush=True) print(classification_report(targets, answers, digits=3, zero_division=0.0), flush=True) + return mean_loss + print() print("train:") - validate(tr) + tr_loss = validate(tr) print("val:") - validate(va) + va_loss = validate(va) print() + with open("log.txt", "a") as f: + f.write(f"{epoch}: {tr_loss} | {va_loss}\n") + From 9091a71e004ddc8ca9ebc36322393995fac115f6 Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Thu, 27 Jul 2023 16:50:38 +0300 Subject: [PATCH 13/52] add roc-auc and more layers in decoder --- ksmt-neurosmt/src/main/python/Decoder.py | 16 +++++---- .../src/main/python/GraphDataloader.py | 15 ++++---- ksmt-neurosmt/src/main/python/GraphReader.py | 1 - ksmt-neurosmt/src/main/python/main.py | 34 +++++++++++++------ 4 files changed, 40 insertions(+), 26 deletions(-) diff --git a/ksmt-neurosmt/src/main/python/Decoder.py b/ksmt-neurosmt/src/main/python/Decoder.py index 27eed810c..bdb486d15 100644 --- a/ksmt-neurosmt/src/main/python/Decoder.py +++ b/ksmt-neurosmt/src/main/python/Decoder.py @@ -5,18 +5,20 @@ class Decoder(nn.Module): def __init__(self, hidden_dim): super().__init__() - self.lin1 = nn.Linear(hidden_dim, hidden_dim) self.act = nn.ReLU() - self.lin2 = nn.Linear(hidden_dim, 1) + self.linears = nn.ModuleList([nn.Linear(hidden_dim, hidden_dim) for _ in range(3)]) + self.out = nn.Linear(hidden_dim, 1) - def forward(self, data): + def forward(self, x): #print("=" * 12) #print(data.shape) - data = self.lin1(data) - data = self.act(data) - data = self.lin2(data) + for layer in self.linears: + x = layer(x) + x = self.act(x) + + x = self.out(x) #print(data.shape) - return data + return x diff --git a/ksmt-neurosmt/src/main/python/GraphDataloader.py b/ksmt-neurosmt/src/main/python/GraphDataloader.py index 8afe3f98a..a5f5d751d 100644 --- a/ksmt-neurosmt/src/main/python/GraphDataloader.py +++ b/ksmt-neurosmt/src/main/python/GraphDataloader.py @@ -19,7 +19,7 @@ BATCH_SIZE = 32 MAX_FORMULA_SIZE = 10000 -MAX_FORMULA_DEPTH = 2500 # 2408 +MAX_FORMULA_DEPTH = 2500 NUM_WORKERS = 16 @@ -91,9 +91,6 @@ def process_paths(paths, label, data): del sat_data, unsat_data gc.collect() - print("\nstats:") - print(f"overall: {sum(it[2] for it in graph_data) / len(graph_data)} | {len(graph_data)}") - """ assert ( len(all_operators) == len(all_edges) @@ -108,10 +105,12 @@ def process_paths(paths, label, data): val_data = [graph_data[i] for i in val_ind] test_data = [graph_data[i] for i in test_ind] - print(f"train: {sum(it[2] for it in train_data) / len(train_data)} | {len(train_data)}") - print(f"val: {sum(it[2] for it in val_data) / len(val_data)} | {len(val_data)}") - print(f"test: {sum(it[2] for it in test_data) / len(test_data)} | {len(test_data)}") - print(flush=True) + print("\nstats:") + print(f"overall: {sum(it[2] for it in graph_data) / len(graph_data)} | {len(graph_data)}") + print(f"train: {sum(it[2] for it in train_data) / len(train_data)} | {len(train_data)}") + print(f"val: {sum(it[2] for it in val_data) / len(val_data)} | {len(val_data)}") + print(f"test: {sum(it[2] for it in test_data) / len(test_data)} | {len(test_data)}") + print("\n", flush=True) """ train_operators = [all_operators[i] for i in train_ind] diff --git a/ksmt-neurosmt/src/main/python/GraphReader.py b/ksmt-neurosmt/src/main/python/GraphReader.py index 62467a1ac..bac9de5ab 100644 --- a/ksmt-neurosmt/src/main/python/GraphReader.py +++ b/ksmt-neurosmt/src/main/python/GraphReader.py @@ -22,7 +22,6 @@ def process_line(line, v, operators, edges, depth): info = info.strip() vertex_type = get_vertex_type(name) depth.append(0) - assert (len(depth) == v + 1) if vertex_type == VertexType.APP: operators.append(name) diff --git a/ksmt-neurosmt/src/main/python/main.py b/ksmt-neurosmt/src/main/python/main.py index f00db923a..0115133b4 100755 --- a/ksmt-neurosmt/src/main/python/main.py +++ b/ksmt-neurosmt/src/main/python/main.py @@ -29,7 +29,7 @@ from Decoder import Decoder from Model import Model -from sklearn.metrics import accuracy_score, classification_report +from sklearn.metrics import accuracy_score, roc_auc_score, classification_report """ @@ -152,12 +152,16 @@ def calc_grad_norm(): print("\n", flush=True) print(f"grad norm: {calc_grad_norm()}") - def validate(dl): + def validate(dl, val=False): model.eval() #all_ans, correct_ans = 0, 0 - answers, targets = torch.tensor([]).to(device), torch.tensor([]).to(device) + + probas = torch.tensor([]).to(device) + answers = torch.tensor([]).to(device) + targets = torch.tensor([]).to(device) losses = [] + with torch.no_grad(): for batch in tqdm(dl): batch = batch.to(device) @@ -169,6 +173,8 @@ def validate(dl): loss = F.binary_cross_entropy_with_logits(out, batch.y) out = F.sigmoid(out) + + probas = torch.cat((probas, out)) #print(out.detach().item(), batch.depth, batch.x.shape, batch.edge_index.shape) #print(out.detach().item()) out = (out > 0.5) @@ -181,27 +187,35 @@ def validate(dl): #out: torch.Tensor = (batch.y.to(torch.int) == out.to(torch.bool)) #correct_ans += out.int().sum().item() - answers = torch.flatten(answers).detach().cpu().numpy() - targets = torch.flatten(targets).detach().cpu().numpy() + probas = torch.flatten(probas).cpu().numpy() + answers = torch.flatten(answers).cpu().numpy() + targets = torch.flatten(targets).cpu().numpy() - #print(f"\n{correct_ans / all_ans}") mean_loss = np.mean(losses) + roc_auc = roc_auc_score(targets, probas) if val else None + print("\n", flush=True) print(f"mean loss: {mean_loss}") - print(f"acc: {accuracy_score(targets, answers)}", flush=True) + print(f"acc: {accuracy_score(targets, answers)}") + print(f"roc-auc: {roc_auc}") print(classification_report(targets, answers, digits=3, zero_division=0.0), flush=True) - return mean_loss + if val: + return mean_loss, roc_auc + else: + return mean_loss print() print("train:") tr_loss = validate(tr) print("val:") - va_loss = validate(va) + va_loss, roc_auc = validate(va, val=True) print() with open("log.txt", "a") as f: - f.write(f"{epoch}: {tr_loss} | {va_loss}\n") + f.write( + f"{str(epoch).rjust(3)}: {'{:.9f}'.format(tr_loss)} | {'{:.9f}'.format(va_loss)} | {'{:.9f}'.format(roc_auc)}\n" + ) From c5fdbc1be5d17fa38098b14c138007f8a8f6dadd Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Thu, 27 Jul 2023 19:36:36 +0300 Subject: [PATCH 14/52] some refactoring --- ksmt-neurosmt/src/main/python/Decoder.py | 5 - ksmt-neurosmt/src/main/python/Encoder.py | 5 +- .../src/main/python/GraphDataloader.py | 58 +----- ksmt-neurosmt/src/main/python/Model.py | 8 +- ksmt-neurosmt/src/main/python/main.py | 181 +----------------- ksmt-neurosmt/src/main/python/utils.py | 19 +- .../smt2converter/FormulaGraphExtractor.kt | 21 +- .../neurosmt/smt2converter/SMT2Converter.kt | 2 +- 8 files changed, 26 insertions(+), 273 deletions(-) diff --git a/ksmt-neurosmt/src/main/python/Decoder.py b/ksmt-neurosmt/src/main/python/Decoder.py index bdb486d15..39d776cdc 100644 --- a/ksmt-neurosmt/src/main/python/Decoder.py +++ b/ksmt-neurosmt/src/main/python/Decoder.py @@ -10,15 +10,10 @@ def __init__(self, hidden_dim): self.out = nn.Linear(hidden_dim, 1) def forward(self, x): - #print("=" * 12) - #print(data.shape) - for layer in self.linears: x = layer(x) x = self.act(x) x = self.out(x) - #print(data.shape) - return x diff --git a/ksmt-neurosmt/src/main/python/Encoder.py b/ksmt-neurosmt/src/main/python/Encoder.py index 6a7d26f81..b1bc5ca5b 100644 --- a/ksmt-neurosmt/src/main/python/Encoder.py +++ b/ksmt-neurosmt/src/main/python/Encoder.py @@ -9,8 +9,8 @@ def __init__(self, hidden_dim): super().__init__() self.embedding = nn.Embedding(2000, hidden_dim) - #self.conv = GatedGraphConv(64, num_layers=1, aggr="mean") self.conv = GCNConv(hidden_dim, hidden_dim, add_self_loops=False) + #self.conv = GATConv(64, 64, add_self_loops=False) def forward(self, data): @@ -19,13 +19,10 @@ def forward(self, data): x = self.embedding(x.squeeze()) depth = depth.max() - #print(depth) for i in range(depth): x = self.conv(x, edge_index) #x = self.conv(x, torch.tensor([[1, 2], [0, 0]], dtype=torch.long)) - #print(x) - x = x[data.ptr[1:] - 1] return x diff --git a/ksmt-neurosmt/src/main/python/GraphDataloader.py b/ksmt-neurosmt/src/main/python/GraphDataloader.py index a5f5d751d..6f18b6c7f 100644 --- a/ksmt-neurosmt/src/main/python/GraphDataloader.py +++ b/ksmt-neurosmt/src/main/python/GraphDataloader.py @@ -21,17 +21,11 @@ MAX_FORMULA_SIZE = 10000 MAX_FORMULA_DEPTH = 2500 NUM_WORKERS = 16 +SHRINK = 10 ** 10 # 1000 class GraphDataset(Dataset): def __init__(self, graph_data): - """ - assert ( - len(node_sets) == len(edge_sets) - and len(node_sets) == len(labels) - and len(labels) == len(depths) - ) - """ self.graphs = [Graph( x=torch.tensor(nodes), @@ -52,6 +46,7 @@ def load_data(path_to_data): for it in tqdm(os.walk(path_to_data)): for file_name in tqdm(it[2]): cur_path = os.path.join(it[0], file_name) + if cur_path.endswith("-sat"): sat_paths.append(cur_path) elif cur_path.endswith("-unsat"): @@ -59,8 +54,6 @@ def load_data(path_to_data): else: raise Exception(f"strange file path '{cur_path}'") - SHRINK = 10 ** 10 # 1000 - if len(sat_paths) > SHRINK: sat_paths = sat_paths[:SHRINK] @@ -91,14 +84,6 @@ def process_paths(paths, label, data): del sat_data, unsat_data gc.collect() - """ - assert ( - len(all_operators) == len(all_edges) - and len(all_edges) == len(all_labels) - and len(all_labels) == len(all_depths) - ) - """ - train_ind, val_ind, test_ind = train_val_test_indices(len(graph_data)) train_data = [graph_data[i] for i in train_ind] @@ -112,27 +97,6 @@ def process_paths(paths, label, data): print(f"test: {sum(it[2] for it in test_data) / len(test_data)} | {len(test_data)}") print("\n", flush=True) - """ - train_operators = [all_operators[i] for i in train_ind] - train_edges = [all_edges[i] for i in train_ind] - train_labels = [all_labels[i] for i in train_ind] - train_depths = [all_depths[i] for i in train_ind] - - val_operators = [all_operators[i] for i in val_ind] - val_edges = [all_edges[i] for i in val_ind] - val_labels = [all_labels[i] for i in val_ind] - val_depths = [all_depths[i] for i in val_ind] - - test_operators = [all_operators[i] for i in test_ind] - test_edges = [all_edges[i] for i in test_ind] - test_labels = [all_labels[i] for i in test_ind] - test_depths = [all_depths[i] for i in test_ind] - """ - - # assert (len(train_operators) == len(train_edges) and len(train_edges) == len(train_labels)) - # assert (len(val_operators) == len(val_edges) and len(val_edges) == len(val_labels)) - # assert (len(test_operators) == len(test_edges) and len(test_edges) == len(test_labels)) - print("del start") del graph_data gc.collect() @@ -150,39 +114,21 @@ def process_paths(paths, label, data): ))).reshape(-1, 1)) print("enc fit end") - """ - def transform_data(data): - for i in range(len(data)): - data[i][0] = encoder.transform(np.array(data[i][0]).reshape(-1, 1)) - """ - def transform(data_for_one_graph): nodes, edges, label, depth = data_for_one_graph nodes = encoder.transform(np.array(nodes).reshape(-1, 1)) return nodes, edges, label, depth print("transform start") - #transform_data(train_data) - #transform_data(val_data) - #transform_data(test_data) - train_data = list(map(transform, train_data)) val_data = list(map(transform, val_data)) test_data = list(map(transform, test_data)) - - #train_operators = list(map(transform, train_operators)) - #val_operators = list(map(transform, val_operators)) - #test_operators = list(map(transform, test_operators)) print("transform end") train_ds = GraphDataset(train_data) val_ds = GraphDataset(val_data) test_ds = GraphDataset(test_data) - #train_ds = GraphDataset(train_operators, train_edges, train_labels, train_depths) - #val_ds = GraphDataset(val_operators, val_edges, val_labels, val_depths) - #test_ds = GraphDataset(test_operators, test_edges, test_labels, test_depths) - return ( DataLoader(train_ds.graphs, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS, shuffle=True, drop_last=True), DataLoader(val_ds.graphs, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS), diff --git a/ksmt-neurosmt/src/main/python/Model.py b/ksmt-neurosmt/src/main/python/Model.py index a744b8870..758bf277f 100644 --- a/ksmt-neurosmt/src/main/python/Model.py +++ b/ksmt-neurosmt/src/main/python/Model.py @@ -13,8 +13,8 @@ def __init__(self): self.encoder = Encoder(hidden_dim=EMBEDDING_DIM) self.decoder = Decoder(hidden_dim=EMBEDDING_DIM) - def forward(self, data): - data = self.encoder(data) - data = self.decoder(data) + def forward(self, x): + x = self.encoder(x) + x = self.decoder(x) - return data + return x diff --git a/ksmt-neurosmt/src/main/python/main.py b/ksmt-neurosmt/src/main/python/main.py index 0115133b4..0212b73bd 100755 --- a/ksmt-neurosmt/src/main/python/main.py +++ b/ksmt-neurosmt/src/main/python/main.py @@ -2,105 +2,21 @@ import sys import os; os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" - -import numpy as np import time +import numpy as np from tqdm import tqdm, trange +from sklearn.metrics import accuracy_score, roc_auc_score, classification_report + import torch import torch.nn as nn import torch.nn.functional as F -#from torch_geometric.data import Data -from torch_geometric.nn import GCNConv -from torch_geometric.utils import scatter - -from GraphReader import read_graph_from_file from GraphDataloader import load_data - -import torch -from torch.nn import Linear, Parameter -from torch_geometric.nn import MessagePassing -from torch_geometric.utils import add_self_loops, degree - -from Encoder import Encoder -from Decoder import Decoder from Model import Model -from sklearn.metrics import accuracy_score, roc_auc_score, classification_report - - -""" -class GCNConv1(MessagePassing): - def __init__(self, in_channels, out_channels): - super().__init__(aggr='add', flow="source_to_target") # "Add" aggregation (Step 5). - self.lin = Linear(in_channels, out_channels, bias=False) - self.bias = Parameter(torch.empty(out_channels)) - - self.reset_parameters() - - def reset_parameters(self): - self.lin.reset_parameters() - self.bias.data.zero_() - - def forward(self, x, edge_index): - # x has shape [N, in_channels] - # edge_index has shape [2, E] - - # Step 1: Add self-loops to the adjacency matrix. - #edge_index, _ = add_self_loops(edge_index, num_nodes=x.size(0)) - - # Step 2: Linearly transform node feature matrix. - x = self.lin(x) - - # Step 3: Compute normalization. - row, col = edge_index - deg = degree(col, x.size(0), dtype=x.dtype) - deg_inv_sqrt = deg.pow(-0.5) - deg_inv_sqrt[deg_inv_sqrt == float('inf')] = 0 - norm = deg_inv_sqrt[row] * deg_inv_sqrt[col] - - # Step 4-5: Start propagating messages. - out = self.propagate(edge_index, x=x, norm=norm) - - # Step 6: Apply a final bias vector. - out += self.bias - - return out - - def message(self, x_j, norm): - # x_j has shape [E, out_channels] - - # Step 4: Normalize node features. - return norm.view(-1, 1) * x_j - - -class GCN(torch.nn.Module): - def __init__(self): - super().__init__() - - self.embedding = nn.Embedding(1000, 16) - self.conv1 = GCNConv(16, 16, add_self_loops=False) - self.conv2 = GCNConv(16, 1, add_self_loops=False) - - #self.conv1 = GCNConv(16, 1) - #self.conv2 = GCNConv(16, 1) - - def forward(self, data): - x, edge_index = data.x, data.edge_index - - x = self.embedding(x.squeeze()) - #x[-2] = 1.23 - x = self.conv1(x, edge_index) - x = F.relu(x) - x = F.dropout(x, training=self.training) - x = self.conv2(x, edge_index) - - return x -""" - if __name__ == "__main__": tr, va, te = load_data(sys.argv[1]) @@ -116,19 +32,7 @@ def calc_grad_norm(): return torch.cat(grads).norm().item() for p in model.parameters(): - assert (p.requires_grad) - - criterion = nn.BCEWithLogitsLoss() - - """ - for i, batch in enumerate(tr): - print(batch.y) - - if i >= 1000: - break - - print(flush=True) - """ + assert p.requires_grad with open("log.txt", "a") as f: f.write("\n" + "=" * 12 + "\n") @@ -140,11 +44,9 @@ def calc_grad_norm(): batch = batch.to(device) out = model(batch) - #out = out[batch.ptr[:-1]] #out = out[batch.ptr[1:] - 1] loss = F.binary_cross_entropy_with_logits(out, batch.y) - #loss = criterion(out, batch.y) loss.backward() optimizer.step() @@ -155,8 +57,6 @@ def calc_grad_norm(): def validate(dl, val=False): model.eval() - #all_ans, correct_ans = 0, 0 - probas = torch.tensor([]).to(device) answers = torch.tensor([]).to(device) targets = torch.tensor([]).to(device) @@ -167,26 +67,18 @@ def validate(dl, val=False): batch = batch.to(device) out = model(batch) - #print(batch.ptr[1:] - 1) - #out = out[batch.ptr[:-1]] #out = out[batch.ptr[1:] - 1] + loss = F.binary_cross_entropy_with_logits(out, batch.y) out = F.sigmoid(out) - probas = torch.cat((probas, out)) - #print(out.detach().item(), batch.depth, batch.x.shape, batch.edge_index.shape) - #print(out.detach().item()) out = (out > 0.5) answers = torch.cat((answers, out)) targets = torch.cat((targets, batch.y.to(torch.int).to(torch.bool))) losses.append(loss.item()) - #all_ans += len(batch.y) - #out: torch.Tensor = (batch.y.to(torch.int) == out.to(torch.bool)) - #correct_ans += out.int().sum().item() - probas = torch.flatten(probas).cpu().numpy() answers = torch.flatten(answers).cpu().numpy() targets = torch.flatten(targets).cpu().numpy() @@ -213,63 +105,8 @@ def validate(dl, val=False): print() with open("log.txt", "a") as f: - f.write( - f"{str(epoch).rjust(3)}: {'{:.9f}'.format(tr_loss)} | {'{:.9f}'.format(va_loss)} | {'{:.9f}'.format(roc_auc)}\n" - ) - - - - - - - - - - - - - - - - - - #for batch in tr: - # print(batch.num_graphs, batch) - #print(batch.ptr) - - """ - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - model = GCN().to(device) - optimizer = torch.optim.Adam(model.parameters(), lr=0.01, weight_decay=5e-4) - - model.train() - for epoch in range(1): - for batch in tr: - #optimizer.zero_grad() - out = model(batch) - #print(out[0].item(), out[-1].item(), batch[0].num_nodes, batch[0].num_edges, batch[0].depth.item()) - print(batch[0].depth.item(), batch[0].num_nodes, batch[0].num_edges) - - out = scatter(out, batch.batch, dim=0, reduce='mean') - #loss = F.mse_loss(out, batch.y) - #loss.backward() - #optimizer.step() - - """ - - #print(all_edges, all_expressions) - #print("\n\n\nFINISHED\n\n\n") - #time.sleep(5) - - """ - edge_index = torch.tensor([[0, 1, 1, 2], - [1, 0, 2, 1]], dtype=torch.long) - x = torch.tensor([[-1], [0], [1]], dtype=torch.float) + tr_loss = "{:.9f}".format(tr_loss) + va_loss = "{:.9f}".format(va_loss) + roc_auc = "{:.9f}".format(roc_auc) - print(torch.tensor(expressions).T) - data = Data( - x=torch.tensor(expressions).T, - edge_index=torch.tensor(edges) - ) - print(data, data.is_directed()) - """ + f.write(f"{str(epoch).rjust(3)}: {tr_loss} | {va_loss} | {roc_auc}\n") diff --git a/ksmt-neurosmt/src/main/python/utils.py b/ksmt-neurosmt/src/main/python/utils.py index 09cef7f19..877a3eb0c 100644 --- a/ksmt-neurosmt/src/main/python/utils.py +++ b/ksmt-neurosmt/src/main/python/utils.py @@ -1,12 +1,12 @@ import numpy as np -def train_val_test_indices(n, val_pct=0.15, test_pct=0.1): - perm = np.arange(n) +def train_val_test_indices(cnt, val_pct=0.15, test_pct=0.1): + perm = np.arange(cnt) np.random.shuffle(perm) - val_cnt = int(n * val_pct) - test_cnt = int(n * test_pct) + val_cnt = int(cnt * val_pct) + test_cnt = int(cnt * test_pct) return perm[val_cnt + test_cnt:], perm[:val_cnt], perm[val_cnt:val_cnt + test_cnt] @@ -24,14 +24,3 @@ def align_sat_unsat_sizes(sat_data, unsat_data): list(np.array(sat_data, dtype=object)[sat_indices]), list(np.array(unsat_data, dtype=object)[unsat_indices]) ) - - """ - if len(sat_data) > len(unsat_data): - return list(np.random.choice(np.array(sat_data, dtype=object), len(unsat_data), replace=False)), unsat_data - elif len(sat_data) < len(unsat_data): - print(type(unsat_data[0])) - print(type(np.array(unsat_data, dtype=np.object)[0])) - return sat_data, list(np.random.choice(np.array(unsat_data, dtype=object), len(sat_data), replace=False)) - else: - return sat_data, unsat_data - """ diff --git a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/FormulaGraphExtractor.kt b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/FormulaGraphExtractor.kt index 4633749df..9fb445188 100644 --- a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/FormulaGraphExtractor.kt +++ b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/FormulaGraphExtractor.kt @@ -1,15 +1,16 @@ package io.ksmt.solver.neurosmt.smt2converter import io.ksmt.KContext -import io.ksmt.decl.KBitVecCustomSizeValueDecl -import io.ksmt.expr.* +import io.ksmt.expr.KApp +import io.ksmt.expr.KConst +import io.ksmt.expr.KExpr +import io.ksmt.expr.KInterpretedValue import io.ksmt.expr.transformer.KNonRecursiveTransformer import io.ksmt.sort.KBoolSort -import io.ksmt.sort.KBvCustomSizeSort import io.ksmt.sort.KBvSort import io.ksmt.sort.KSort import java.io.OutputStream -import java.util.IdentityHashMap +import java.util.* class FormulaGraphExtractor( override val ctx: KContext, @@ -31,18 +32,6 @@ class FormulaGraphExtractor( else -> writeApp(expr) } - //println(expr::class) - //println("${expr.decl.name} ${expr.args.size}") - //outputStream.writer().write("${expr.decl.name} ${expr.args.size}\n") - //outputStream.writer().flush() - /* - println(expr.args) - println("${expr.decl} ${expr.decl.name} ${expr.decl.argSorts} ${expr.sort}") - for (child in expr.args) { - println((child as KApp<*, *>).decl.name) - } - */ - return expr } diff --git a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/SMT2Converter.kt b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/SMT2Converter.kt index 175b12ff4..2f1cfa3f6 100644 --- a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/SMT2Converter.kt +++ b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/SMT2Converter.kt @@ -3,7 +3,7 @@ package io.ksmt.solver.neurosmt.smt2converter import io.ksmt.KContext import io.ksmt.parser.KSMTLibParseException import io.ksmt.solver.KSolverStatus -import io.ksmt.solver.z3.* +import io.ksmt.solver.z3.KZ3SMTLibParser import me.tongfei.progressbar.ProgressBar import java.io.FileOutputStream import java.nio.file.Files From dcb2c1e2de4184c36949861a491d032b38b6751b Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Fri, 28 Jul 2023 19:08:34 +0300 Subject: [PATCH 15/52] sandboxing --- sandbox/build.gradle.kts | 1 + sandbox/src/main/kotlin/Main.kt | 211 +++++++++++++++++--------------- 2 files changed, 115 insertions(+), 97 deletions(-) diff --git a/sandbox/build.gradle.kts b/sandbox/build.gradle.kts index 9a724d9c2..ed33bad47 100644 --- a/sandbox/build.gradle.kts +++ b/sandbox/build.gradle.kts @@ -18,6 +18,7 @@ dependencies { implementation(project(":ksmt-z3")) implementation(project(":ksmt-neurosmt")) implementation(project(":ksmt-neurosmt:utils")) + implementation(project(":ksmt-runner")) } tasks.getByName("test") { diff --git a/sandbox/src/main/kotlin/Main.kt b/sandbox/src/main/kotlin/Main.kt index d8920b910..b78a58dc7 100644 --- a/sandbox/src/main/kotlin/Main.kt +++ b/sandbox/src/main/kotlin/Main.kt @@ -1,87 +1,104 @@ +import com.jetbrains.rd.framework.SerializationCtx +import com.jetbrains.rd.framework.Serializers +import com.jetbrains.rd.framework.UnsafeBuffer import io.ksmt.KContext import io.ksmt.expr.* -import io.ksmt.expr.transformer.KNonRecursiveTransformer +import io.ksmt.runner.serializer.AstSerializationCtx +import io.ksmt.solver.KModel +import io.ksmt.solver.KSolver +import io.ksmt.solver.KSolverConfiguration import io.ksmt.solver.KSolverStatus import io.ksmt.solver.neurosmt.KNeuroSMTSolver -import io.ksmt.solver.neurosmt.smt2converter.FormulaGraphExtractor -import io.ksmt.solver.neurosmt.smt2converter.getAnswerForTest import io.ksmt.solver.z3.* import io.ksmt.sort.* import io.ksmt.utils.getValue +import io.ksmt.utils.uncheckedCast +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream import java.io.FileOutputStream -import java.nio.file.Files -import java.nio.file.Path -import kotlin.io.path.isRegularFile -import kotlin.io.path.name +import java.io.InputStream +import java.io.OutputStream +import kotlin.time.Duration import kotlin.time.Duration.Companion.seconds -fun lol(a: Any) { - if (a is KConst<*>) { - println("const: ${a.decl.name}") - } - if (a is KInterpretedValue<*>) { - println("val: ${a.decl.name}") +fun serialize(ctx: KContext, expressions: List>, outputStream: OutputStream) { + val serializationCtx = AstSerializationCtx().apply { initCtx(ctx) } + val marshaller = AstSerializationCtx.marshaller(serializationCtx) + val emptyRdSerializationCtx = SerializationCtx(Serializers()) + + val buffer = UnsafeBuffer(ByteArray(10_000)) + + expressions.forEach { expr -> + marshaller.write(emptyRdSerializationCtx, buffer, expr) } + + outputStream.write(buffer.getArray()) + outputStream.flush() } -class Kek(override val ctx: KContext) : KNonRecursiveTransformer(ctx) { - override fun transformApp(expr: KApp): KExpr { - println("===") - //println(expr::class) - //lol(expr) - //println(expr) - /* - println(expr.args) - println("${expr.decl} ${expr.decl.name} ${expr.decl.argSorts} ${expr.sort}") - for (child in expr.args) { - println((child as KApp<*, *>).decl.name) +fun deserialize(ctx: KContext, inputStream: InputStream): List> { + val srcSerializationCtx = AstSerializationCtx().apply { initCtx(ctx) } + val srcMarshaller = AstSerializationCtx.marshaller(srcSerializationCtx) + val emptyRdSerializationCtx = SerializationCtx(Serializers()) + + val buffer = UnsafeBuffer(inputStream.readBytes()) + val expressions: MutableList> = mutableListOf() + + while (true) { + try { + expressions.add(srcMarshaller.read(emptyRdSerializationCtx, buffer).uncheckedCast()) + } catch (e : IllegalStateException) { + break } - */ - return expr } -} -fun main() { - val ctx = KContext() + return expressions +} - with(ctx) { - val files = Files.walk(Path.of("../../neurosmt-benchmark/non-incremental/QF_BV")).filter { it.isRegularFile() } - var ok = 0; var fail = 0 - var sat = 0; var unsat = 0; var unk = 0 - - files.forEach { - println(it) - if (it.name.endsWith(".txt")) { - return@forEach - } +class LogSolver(val ctx: KContext, val baseSolver: KSolver) : KSolver by baseSolver { + companion object { + var counter = 0L + } - when (getAnswerForTest(it)) { - KSolverStatus.SAT -> sat++ - KSolverStatus.UNSAT -> unsat++ - KSolverStatus.UNKNOWN -> unk++ - } + val stack = mutableListOf>>(mutableListOf()) - return@forEach + override fun assert(expr: KExpr) { + stack.last().add(expr) + baseSolver.assert(expr) + } - val formula = try { - val assertList = KZ3SMTLibParser(this).parse(it) - ok++ - mkAnd(assertList) - } catch (e: Exception) { - fail++ - return@forEach - } + override fun assertAndTrack(expr: KExpr) { + stack.last().add(expr) + baseSolver.assertAndTrack(expr) + } - val extractor = FormulaGraphExtractor(this, formula, FileOutputStream("kek2.txt")) - extractor.extractGraph() + override fun push() { + stack.add(mutableListOf()) + baseSolver.push() + } - println("$ok : $fail") + override fun pop(n: UInt) { + repeat(n.toInt()) { + stack.removeLast() } + baseSolver.pop(n) + } + + override fun check(timeout: Duration): KSolverStatus { + serialize(ctx, stack.flatten(), FileOutputStream("formulas/f-${counter++}.bin")) + return baseSolver.check(timeout) + } - println("$sat/$unsat/$unk") + override fun checkWithAssumptions(assumptions: List>, timeout: Duration): KSolverStatus { + serialize(ctx, stack.flatten() + assumptions, FileOutputStream("formulas/f-${counter++}.bin")) + return baseSolver.checkWithAssumptions(assumptions, timeout) + } +} - return@with +fun main() { + val ctx = KContext() + with(ctx) { // create symbolic variables val a by boolSort val b by intSort @@ -103,58 +120,58 @@ fun main() { (mkArraySelect(g, d + mkRealNum(1)) neq a) and (mkArraySelect(g, d - mkRealNum(1)) eq a) and (d * d eq mkRealNum(4)) and - (mkBvMulExpr(e, e) eq mkBv(9.toByte())) // and (mkExistentialQuantifier(x eq 2.expr, listOf())) - //(mkFpMulExpr(mkFpRoundingModeExpr(KFpRoundingMode.RoundTowardZero), h, h) eq 2.0.expr) + (mkBvMulExpr(e, e) eq mkBv(9.toByte())) - //val constraint = mkBvXorExpr(mkBvShiftLeftExpr(e, mkBv(1.toByte())), mkBvNotExpr(e)) eq - // mkBvLogicalShiftRightExpr(e, mkBv(1.toByte())) - - // (constraint as KAndExpr) - - //constraint.accept(Kek(this)) - //Kek(this).apply(constraint) - val extractor = FormulaGraphExtractor(this, constraint, FileOutputStream("kek.txt")) - //val extractor = io.ksmt.solver.neurosmt.preprocessing.FormulaGraphExtractor(this, constraint, System.err) - extractor.extractGraph() + val formula = """ + (declare-fun x () Real) + (declare-fun y () Real) + (declare-fun z () Real) + (declare-fun a () Int) + (assert (or (and (= y (+ x z)) (= x (+ y z))) (= 2.71 x))) + (assert (= a 2)) + (check-sat) + """ - /*println("========") - constraint.apply { - println(this.args) - }*/ + val assertions = mkAnd(KZ3SMTLibParser(this).parse(formula)) - return@with + val bvExpr = mkBvXorExpr(mkBvShiftLeftExpr(e, mkBv(1.toByte())), mkBvNotExpr(e)) eq + mkBvLogicalShiftRightExpr(e, mkBv(1.toByte())) - KNeuroSMTSolver(this).use { solver -> // create a Stub SMT solver instance - // assert expression - solver.assert(constraint) + val buf = ByteArrayOutputStream() + serialize(ctx, listOf(constraint, assertions, bvExpr), buf) + deserialize(ctx, ByteArrayInputStream(buf.toByteArray())).forEach { + println("nxt: $it") + } - // check assertions satisfiability with timeout - val satisfiability = solver.check(timeout = 1.seconds) - // println(satisfiability) // SAT + /* + KExprUninterpretedDeclCollector.collectUninterpretedDeclarations(mkAnd(assertions)).forEach { + println("${it.name} | ${it.argSorts} | ${it.sort}") } + */ KZ3Solver(this).use { solver -> - solver.assert(constraint) + LogSolver(this, solver).use { solver -> + solver.assert(constraint) - val satisfiability = solver.check(timeout = 180.seconds) - println(satisfiability) + val satisfiability = solver.check(timeout = 180.seconds) + println(satisfiability) - if (satisfiability == KSolverStatus.SAT) { - val model = solver.model() - println(model) + if (satisfiability == KSolverStatus.SAT) { + val model = solver.model() + println(model) + } } } - val formula = """ - (declare-fun x () Real) - (declare-fun y () Real) - (declare-fun z () Real) - (assert (or (and (= y (+ x z)) (= x (+ y z))) (= 2.71 x))) - (check-sat) - """ - val assertions = KZ3SMTLibParser(this).parse(formula) + /* + KNeuroSMTSolver(this).use { solver -> // create a Stub SMT solver instance + // assert expression + solver.assert(constraint) - println(assertions) - println(assertions[0].stringRepr) + // check assertions satisfiability with timeout + val satisfiability = solver.check(timeout = 1.seconds) + // println(satisfiability) // SAT + } + */ } } \ No newline at end of file From ce00214e605a910b98cd54798fa4f18b6c99c24b Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Tue, 1 Aug 2023 10:32:02 +0300 Subject: [PATCH 16/52] add pytorch-lightning and tensorboard --- ksmt-neurosmt/src/main/python/Encoder.py | 7 ++++-- .../src/main/python/GraphDataloader.py | 22 ++++++++++++------- ksmt-neurosmt/src/main/python/main.py | 22 +++++++++++++++++++ 3 files changed, 41 insertions(+), 10 deletions(-) diff --git a/ksmt-neurosmt/src/main/python/Encoder.py b/ksmt-neurosmt/src/main/python/Encoder.py index b1bc5ca5b..938a51486 100644 --- a/ksmt-neurosmt/src/main/python/Encoder.py +++ b/ksmt-neurosmt/src/main/python/Encoder.py @@ -4,24 +4,27 @@ from torch_geometric.nn import GCNConv, GatedGraphConv, GATConv +EMBEDDING_DIM = 2000 + + class Encoder(nn.Module): def __init__(self, hidden_dim): super().__init__() - self.embedding = nn.Embedding(2000, hidden_dim) + self.embedding = nn.Embedding(EMBEDDING_DIM, hidden_dim) self.conv = GCNConv(hidden_dim, hidden_dim, add_self_loops=False) #self.conv = GATConv(64, 64, add_self_loops=False) def forward(self, data): x, edge_index, depth = data.x, data.edge_index, data.depth + #edge_index = torch.tensor([[0, 0], [1, 2]], dtype=torch.long).to(edge_index.get_device()) x = self.embedding(x.squeeze()) depth = depth.max() for i in range(depth): x = self.conv(x, edge_index) - #x = self.conv(x, torch.tensor([[1, 2], [0, 0]], dtype=torch.long)) x = x[data.ptr[1:] - 1] diff --git a/ksmt-neurosmt/src/main/python/GraphDataloader.py b/ksmt-neurosmt/src/main/python/GraphDataloader.py index 6f18b6c7f..e676674bf 100644 --- a/ksmt-neurosmt/src/main/python/GraphDataloader.py +++ b/ksmt-neurosmt/src/main/python/GraphDataloader.py @@ -68,7 +68,7 @@ def process_paths(paths, label, data): continue if len(edges) == 0: - print(f"w: formula with no edges; file '{path}'") + print(f"w: ignoring formula without edges; file '{path}'") continue data.append((operators, edges, label, depth)) @@ -77,7 +77,6 @@ def process_paths(paths, label, data): process_paths(sat_paths, 1, sat_data) process_paths(unsat_paths, 0, unsat_data) - np.random.seed(24) sat_data, unsat_data = align_sat_unsat_sizes(sat_data, unsat_data) graph_data = sat_data + unsat_data @@ -125,12 +124,19 @@ def transform(data_for_one_graph): test_data = list(map(transform, test_data)) print("transform end") + print("create dataset start") train_ds = GraphDataset(train_data) val_ds = GraphDataset(val_data) test_ds = GraphDataset(test_data) - - return ( - DataLoader(train_ds.graphs, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS, shuffle=True, drop_last=True), - DataLoader(val_ds.graphs, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS), - DataLoader(test_ds.graphs, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS) - ) + print("create dataset end") + + try: + print("create dataloader start") + return ( + DataLoader(train_ds.graphs, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS, shuffle=True, drop_last=True), + DataLoader(val_ds.graphs, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS), + DataLoader(test_ds.graphs, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS) + ) + + finally: + print("create dataloader end\n", flush=True) diff --git a/ksmt-neurosmt/src/main/python/main.py b/ksmt-neurosmt/src/main/python/main.py index 0212b73bd..d7eca7e3b 100755 --- a/ksmt-neurosmt/src/main/python/main.py +++ b/ksmt-neurosmt/src/main/python/main.py @@ -12,15 +12,35 @@ import torch import torch.nn as nn import torch.nn.functional as F +from pytorch_lightning import Trainer, seed_everything +from pytorch_lightning.loggers import TensorBoardLogger from GraphDataloader import load_data from Model import Model +from LightningModel import LightningModel if __name__ == "__main__": + seed_everything(24, workers=True) + # torch.backends.cuda.matmul.allow_tf32 = True + tr, va, te = load_data(sys.argv[1]) + pl_model = LightningModel() + trainer = Trainer( + accelerator="auto", + # precision="16-mixed", + logger=TensorBoardLogger("../logs", name="neuro-smt"), + max_epochs=100, + log_every_n_steps=1, + enable_checkpointing=False, + barebones=False + ) + + trainer.fit(pl_model, tr, va) + + """ device = torch.device("cuda" if torch.cuda.is_available() else "cpu") model = Model().to(device) optimizer = torch.optim.Adam([p for p in model.parameters() if p is not None and p.requires_grad], lr=1e-4) @@ -110,3 +130,5 @@ def validate(dl, val=False): roc_auc = "{:.9f}".format(roc_auc) f.write(f"{str(epoch).rjust(3)}: {tr_loss} | {va_loss} | {roc_auc}\n") + + """ \ No newline at end of file From 2ffa9b4e7f17d0069235b71bd640f3004b2d91eb Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Tue, 1 Aug 2023 10:35:24 +0300 Subject: [PATCH 17/52] i forgot model --- .../src/main/python/LightningModel.py | 144 ++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 ksmt-neurosmt/src/main/python/LightningModel.py diff --git a/ksmt-neurosmt/src/main/python/LightningModel.py b/ksmt-neurosmt/src/main/python/LightningModel.py new file mode 100644 index 000000000..83e63c44a --- /dev/null +++ b/ksmt-neurosmt/src/main/python/LightningModel.py @@ -0,0 +1,144 @@ +from sklearn.metrics import classification_report + +import torch +import torch.nn.functional as F + +import pytorch_lightning as pl + +from torchmetrics.classification import BinaryAccuracy, BinaryAUROC + +from Model import Model + + +class LightningModel(pl.LightningModule): + def __init__(self): + super().__init__() + + self.model = Model() + + self.val_outputs = [] + self.val_targets = [] + + self.acc = BinaryAccuracy() + self.roc_auc = BinaryAUROC() + + #self.train_dl = train_dl + #self.val_dl = val_dl + #self.test_dl = test_dl + + def forward(self, x): + return self.model(x) + + def configure_optimizers(self): + params = [p for p in self.model.parameters() if p is not None and p.requires_grad] + optimizer = torch.optim.Adam(params, lr=1e-4) + + return optimizer + + def training_step(self, train_batch, batch_idx): + out = self.model(train_batch) + loss = F.binary_cross_entropy_with_logits(out, train_batch.y) + + out = F.sigmoid(out) + + self.log( + "train/loss", loss.item(), + prog_bar=True, logger=True, + on_step=True, on_epoch=True, + batch_size=train_batch.num_graphs + ) + self.log( + "train/acc", self.acc(out, train_batch.y), + prog_bar=True, logger=True, + on_step=False, on_epoch=True, + batch_size=train_batch.num_graphs + ) + + return loss + + def validation_step(self, val_batch, batch_idx): + out = self.model(val_batch) + loss = F.binary_cross_entropy_with_logits(out, val_batch.y) + + out = F.sigmoid(out) + + self.log( + "val/loss", loss.item(), + prog_bar=True, logger=True, + on_step=False, on_epoch=True, + batch_size=val_batch.num_graphs + ) + self.log( + "val/acc", self.acc(out, val_batch.y), + prog_bar=True, logger=True, + on_step=False, on_epoch=True, + batch_size=val_batch.num_graphs + ) + + self.val_outputs.append(out) + self.val_targets.append(val_batch.y) + + """ + probas = torch.flatten(probas).cpu().numpy() + answers = torch.flatten(answers).cpu().numpy() + targets = torch.flatten(targets).cpu().numpy() + + mean_loss = np.mean(losses) + roc_auc = roc_auc_score(targets, probas) if val else None + + print("\n", flush=True) + print(f"mean loss: {mean_loss}") + print(f"acc: {accuracy_score(targets, answers)}") + print(f"roc-auc: {roc_auc}") + print(classification_report(targets, answers, digits=3, zero_division=0.0), flush=True) + + if val: + return mean_loss, roc_auc + else: + return mean_loss + """ + + return loss + + def on_validation_epoch_end(self): + print("\n\n", flush=True) + + all_outputs = torch.flatten(torch.cat(self.val_outputs)) + all_targets = torch.flatten(torch.cat(self.val_targets)) + + self.val_outputs.clear() + self.val_targets.clear() + + logger = self.logger.experiment + + roc_auc = self.roc_auc(all_outputs, all_targets) + self.log( + "val/roc-auc", roc_auc, + prog_bar=True, logger=False, + on_step=False, on_epoch=True + ) + logger.add_scalar("val/roc-auc", roc_auc, self.current_epoch) + + all_outputs = all_outputs.cpu().numpy() + all_targets = all_targets.cpu().numpy() + + all_outputs = all_outputs > 0.5 + print(classification_report(all_targets, all_outputs, digits=3, zero_division=0.0)) + + print("\n", flush=True) + + """ + def backward(self, trainer, loss, optimizer, optimizer_idx): + loss.backward() + """ + + """ + def train_dataloader(self): + return self.train_dl + + def val_dataloader(self): + return self.val_dl + + def test_dataloader(self): + return self.test_dl + """ From 3a689f819a24ba45256da2c260995db8ae1685e9 Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Tue, 1 Aug 2023 11:41:32 +0300 Subject: [PATCH 18/52] adjust floating point precision --- ksmt-neurosmt/src/main/python/GraphDataloader.py | 2 +- ksmt-neurosmt/src/main/python/LightningModel.py | 8 ++++---- ksmt-neurosmt/src/main/python/main.py | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ksmt-neurosmt/src/main/python/GraphDataloader.py b/ksmt-neurosmt/src/main/python/GraphDataloader.py index e676674bf..e72d69e2f 100644 --- a/ksmt-neurosmt/src/main/python/GraphDataloader.py +++ b/ksmt-neurosmt/src/main/python/GraphDataloader.py @@ -21,7 +21,7 @@ MAX_FORMULA_SIZE = 10000 MAX_FORMULA_DEPTH = 2500 NUM_WORKERS = 16 -SHRINK = 10 ** 10 # 1000 +SHRINK = 10 ** (int(os.environ["SHRINK"]) if "SHRINK" in os.environ else 10) class GraphDataset(Dataset): diff --git a/ksmt-neurosmt/src/main/python/LightningModel.py b/ksmt-neurosmt/src/main/python/LightningModel.py index 83e63c44a..f2db7c03e 100644 --- a/ksmt-neurosmt/src/main/python/LightningModel.py +++ b/ksmt-neurosmt/src/main/python/LightningModel.py @@ -42,7 +42,7 @@ def training_step(self, train_batch, batch_idx): out = F.sigmoid(out) self.log( - "train/loss", loss.item(), + "train/loss", loss.detach().float(), prog_bar=True, logger=True, on_step=True, on_epoch=True, batch_size=train_batch.num_graphs @@ -63,7 +63,7 @@ def validation_step(self, val_batch, batch_idx): out = F.sigmoid(out) self.log( - "val/loss", loss.item(), + "val/loss", loss.float(), prog_bar=True, logger=True, on_step=False, on_epoch=True, batch_size=val_batch.num_graphs @@ -119,8 +119,8 @@ def on_validation_epoch_end(self): ) logger.add_scalar("val/roc-auc", roc_auc, self.current_epoch) - all_outputs = all_outputs.cpu().numpy() - all_targets = all_targets.cpu().numpy() + all_outputs = all_outputs.float().cpu().numpy() + all_targets = all_targets.float().cpu().numpy() all_outputs = all_outputs > 0.5 print(classification_report(all_targets, all_outputs, digits=3, zero_division=0.0)) diff --git a/ksmt-neurosmt/src/main/python/main.py b/ksmt-neurosmt/src/main/python/main.py index d7eca7e3b..4437919dc 100755 --- a/ksmt-neurosmt/src/main/python/main.py +++ b/ksmt-neurosmt/src/main/python/main.py @@ -1,7 +1,7 @@ #!/usr/bin/python3 import sys -import os; os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" +import os; os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"; os.environ["CUDA_VISIBLE_DEVICES"] = os.environ["GPU"] import time import numpy as np @@ -23,14 +23,14 @@ if __name__ == "__main__": seed_everything(24, workers=True) - # torch.backends.cuda.matmul.allow_tf32 = True + torch.set_float32_matmul_precision("medium") tr, va, te = load_data(sys.argv[1]) pl_model = LightningModel() trainer = Trainer( accelerator="auto", - # precision="16-mixed", + # precision="bf16-mixed", logger=TensorBoardLogger("../logs", name="neuro-smt"), max_epochs=100, log_every_n_steps=1, From 09ef95a45e7f61e1202385896e83de4de7f7ecf9 Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Tue, 1 Aug 2023 12:55:42 +0300 Subject: [PATCH 19/52] add checkpointing --- ksmt-neurosmt/src/main/python/main.py | 30 +++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/ksmt-neurosmt/src/main/python/main.py b/ksmt-neurosmt/src/main/python/main.py index 4437919dc..2d7e2b999 100755 --- a/ksmt-neurosmt/src/main/python/main.py +++ b/ksmt-neurosmt/src/main/python/main.py @@ -3,6 +3,7 @@ import sys import os; os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"; os.environ["CUDA_VISIBLE_DEVICES"] = os.environ["GPU"] import time +from argparse import ArgumentParser import numpy as np from tqdm import tqdm, trange @@ -21,11 +22,28 @@ from LightningModel import LightningModel +def get_args(): + parser = ArgumentParser(description="main training script") + parser.add_argument("--ds", required=True) + parser.add_argument("--ckpt", required=False) + + args = parser.parse_args() + print("args:") + for arg in vars(args): + print(arg, "=", getattr(args, arg)) + + print() + time.sleep(5) + + return args + + if __name__ == "__main__": seed_everything(24, workers=True) torch.set_float32_matmul_precision("medium") - tr, va, te = load_data(sys.argv[1]) + args = get_args() + tr, va, te = load_data(args.ds) pl_model = LightningModel() trainer = Trainer( @@ -34,11 +52,15 @@ logger=TensorBoardLogger("../logs", name="neuro-smt"), max_epochs=100, log_every_n_steps=1, - enable_checkpointing=False, - barebones=False + enable_checkpointing=True, + barebones=False, + default_root_dir=".." ) - trainer.fit(pl_model, tr, va) + if args.ckpt is None: + trainer.fit(pl_model, tr, va) + else: + trainer.fit(pl_model, tr, va, ckpt_path=args.ckpt) """ device = torch.device("cuda" if torch.cuda.is_available() else "cpu") From 6bfe3b6a2fa1cc1be47d8ec15f5993b83a5c08d3 Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Tue, 1 Aug 2023 13:00:56 +0300 Subject: [PATCH 20/52] refactor --- .../src/main/python/GraphDataloader.py | 4 +- .../src/main/python/LightningModel.py | 40 ------- ksmt-neurosmt/src/main/python/main.py | 103 +----------------- 3 files changed, 3 insertions(+), 144 deletions(-) diff --git a/ksmt-neurosmt/src/main/python/GraphDataloader.py b/ksmt-neurosmt/src/main/python/GraphDataloader.py index e72d69e2f..963b15efb 100644 --- a/ksmt-neurosmt/src/main/python/GraphDataloader.py +++ b/ksmt-neurosmt/src/main/python/GraphDataloader.py @@ -134,8 +134,8 @@ def transform(data_for_one_graph): print("create dataloader start") return ( DataLoader(train_ds.graphs, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS, shuffle=True, drop_last=True), - DataLoader(val_ds.graphs, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS), - DataLoader(test_ds.graphs, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS) + DataLoader(val_ds.graphs, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS, shuffle=False, drop_last=False), + DataLoader(test_ds.graphs, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS, shuffle=False, drop_last=False) ) finally: diff --git a/ksmt-neurosmt/src/main/python/LightningModel.py b/ksmt-neurosmt/src/main/python/LightningModel.py index f2db7c03e..9821b2c69 100644 --- a/ksmt-neurosmt/src/main/python/LightningModel.py +++ b/ksmt-neurosmt/src/main/python/LightningModel.py @@ -22,10 +22,6 @@ def __init__(self): self.acc = BinaryAccuracy() self.roc_auc = BinaryAUROC() - #self.train_dl = train_dl - #self.val_dl = val_dl - #self.test_dl = test_dl - def forward(self, x): return self.model(x) @@ -78,26 +74,6 @@ def validation_step(self, val_batch, batch_idx): self.val_outputs.append(out) self.val_targets.append(val_batch.y) - """ - probas = torch.flatten(probas).cpu().numpy() - answers = torch.flatten(answers).cpu().numpy() - targets = torch.flatten(targets).cpu().numpy() - - mean_loss = np.mean(losses) - roc_auc = roc_auc_score(targets, probas) if val else None - - print("\n", flush=True) - print(f"mean loss: {mean_loss}") - print(f"acc: {accuracy_score(targets, answers)}") - print(f"roc-auc: {roc_auc}") - print(classification_report(targets, answers, digits=3, zero_division=0.0), flush=True) - - if val: - return mean_loss, roc_auc - else: - return mean_loss - """ - return loss def on_validation_epoch_end(self): @@ -126,19 +102,3 @@ def on_validation_epoch_end(self): print(classification_report(all_targets, all_outputs, digits=3, zero_division=0.0)) print("\n", flush=True) - - """ - def backward(self, trainer, loss, optimizer, optimizer_idx): - loss.backward() - """ - - """ - def train_dataloader(self): - return self.train_dl - - def val_dataloader(self): - return self.val_dl - - def test_dataloader(self): - return self.test_dl - """ diff --git a/ksmt-neurosmt/src/main/python/main.py b/ksmt-neurosmt/src/main/python/main.py index 2d7e2b999..57b33e212 100755 --- a/ksmt-neurosmt/src/main/python/main.py +++ b/ksmt-neurosmt/src/main/python/main.py @@ -1,24 +1,16 @@ #!/usr/bin/python3 -import sys import os; os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"; os.environ["CUDA_VISIBLE_DEVICES"] = os.environ["GPU"] import time from argparse import ArgumentParser -import numpy as np -from tqdm import tqdm, trange - -from sklearn.metrics import accuracy_score, roc_auc_score, classification_report - import torch -import torch.nn as nn -import torch.nn.functional as F + from pytorch_lightning import Trainer, seed_everything from pytorch_lightning.loggers import TensorBoardLogger from GraphDataloader import load_data -from Model import Model from LightningModel import LightningModel @@ -61,96 +53,3 @@ def get_args(): trainer.fit(pl_model, tr, va) else: trainer.fit(pl_model, tr, va, ckpt_path=args.ckpt) - - """ - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - model = Model().to(device) - optimizer = torch.optim.Adam([p for p in model.parameters() if p is not None and p.requires_grad], lr=1e-4) - - def calc_grad_norm(): - grads = [ - p.grad.detach().flatten() for p in model.parameters() if p.grad is not None and p.requires_grad - ] - return torch.cat(grads).norm().item() - - for p in model.parameters(): - assert p.requires_grad - - with open("log.txt", "a") as f: - f.write("\n" + "=" * 12 + "\n") - - for epoch in trange(200): - model.train() - for batch in tqdm(tr): - optimizer.zero_grad() - batch = batch.to(device) - - out = model(batch) - #out = out[batch.ptr[1:] - 1] - - loss = F.binary_cross_entropy_with_logits(out, batch.y) - loss.backward() - - optimizer.step() - - print("\n", flush=True) - print(f"grad norm: {calc_grad_norm()}") - - def validate(dl, val=False): - model.eval() - - probas = torch.tensor([]).to(device) - answers = torch.tensor([]).to(device) - targets = torch.tensor([]).to(device) - losses = [] - - with torch.no_grad(): - for batch in tqdm(dl): - batch = batch.to(device) - - out = model(batch) - #out = out[batch.ptr[1:] - 1] - - loss = F.binary_cross_entropy_with_logits(out, batch.y) - - out = F.sigmoid(out) - probas = torch.cat((probas, out)) - out = (out > 0.5) - - answers = torch.cat((answers, out)) - targets = torch.cat((targets, batch.y.to(torch.int).to(torch.bool))) - losses.append(loss.item()) - - probas = torch.flatten(probas).cpu().numpy() - answers = torch.flatten(answers).cpu().numpy() - targets = torch.flatten(targets).cpu().numpy() - - mean_loss = np.mean(losses) - roc_auc = roc_auc_score(targets, probas) if val else None - - print("\n", flush=True) - print(f"mean loss: {mean_loss}") - print(f"acc: {accuracy_score(targets, answers)}") - print(f"roc-auc: {roc_auc}") - print(classification_report(targets, answers, digits=3, zero_division=0.0), flush=True) - - if val: - return mean_loss, roc_auc - else: - return mean_loss - - print() - print("train:") - tr_loss = validate(tr) - print("val:") - va_loss, roc_auc = validate(va, val=True) - print() - - with open("log.txt", "a") as f: - tr_loss = "{:.9f}".format(tr_loss) - va_loss = "{:.9f}".format(va_loss) - roc_auc = "{:.9f}".format(roc_auc) - - f.write(f"{str(epoch).rjust(3)}: {tr_loss} | {va_loss} | {roc_auc}\n") - - """ \ No newline at end of file From c787a291433e6b9136cf48834637f7dcbf6e81e2 Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Tue, 1 Aug 2023 13:13:50 +0300 Subject: [PATCH 21/52] fix backdoor --- sandbox/src/main/kotlin/Main.kt | 54 +++++++++++++++++++++++++++------ 1 file changed, 44 insertions(+), 10 deletions(-) diff --git a/sandbox/src/main/kotlin/Main.kt b/sandbox/src/main/kotlin/Main.kt index b78a58dc7..8bad14cbc 100644 --- a/sandbox/src/main/kotlin/Main.kt +++ b/sandbox/src/main/kotlin/Main.kt @@ -13,11 +13,11 @@ import io.ksmt.solver.z3.* import io.ksmt.sort.* import io.ksmt.utils.getValue import io.ksmt.utils.uncheckedCast -import java.io.ByteArrayInputStream -import java.io.ByteArrayOutputStream -import java.io.FileOutputStream -import java.io.InputStream -import java.io.OutputStream +import java.io.* +import java.nio.file.Files +import java.nio.file.Path +import java.util.concurrent.atomic.AtomicLong +import kotlin.io.path.isRegularFile import kotlin.time.Duration import kotlin.time.Duration.Companion.seconds @@ -26,7 +26,7 @@ fun serialize(ctx: KContext, expressions: List>, outputStream: val marshaller = AstSerializationCtx.marshaller(serializationCtx) val emptyRdSerializationCtx = SerializationCtx(Serializers()) - val buffer = UnsafeBuffer(ByteArray(10_000)) + val buffer = UnsafeBuffer(ByteArray(100_000)) expressions.forEach { expr -> marshaller.write(emptyRdSerializationCtx, buffer, expr) @@ -55,9 +55,26 @@ fun deserialize(ctx: KContext, inputStream: InputStream): List> return expressions } -class LogSolver(val ctx: KContext, val baseSolver: KSolver) : KSolver by baseSolver { +class LogSolver( + private val ctx: KContext, private val baseSolver: KSolver +) : KSolver by baseSolver { + companion object { - var counter = 0L + val counter = AtomicLong(0) + } + + init { + File("formulas").mkdirs() + } + + private fun getNewFileCounter(): Long { + return synchronized(counter) { + counter.getAndIncrement() + } + } + + private fun getNewFileName(): String { + return "formulas/f-${getNewFileCounter()}.bin" } val stack = mutableListOf>>(mutableListOf()) @@ -85,12 +102,12 @@ class LogSolver(val ctx: KContext, val baseSolver: KSo } override fun check(timeout: Duration): KSolverStatus { - serialize(ctx, stack.flatten(), FileOutputStream("formulas/f-${counter++}.bin")) + serialize(ctx, stack.flatten(), FileOutputStream(getNewFileName())) return baseSolver.check(timeout) } override fun checkWithAssumptions(assumptions: List>, timeout: Duration): KSolverStatus { - serialize(ctx, stack.flatten() + assumptions, FileOutputStream("formulas/f-${counter++}.bin")) + serialize(ctx, stack.flatten() + assumptions, FileOutputStream(getNewFileName())) return baseSolver.checkWithAssumptions(assumptions, timeout) } } @@ -98,6 +115,22 @@ class LogSolver(val ctx: KContext, val baseSolver: KSo fun main() { val ctx = KContext() + with(ctx) { + val files = Files.walk(Path.of("/home/stephen/Desktop/formulas")).filter { it.isRegularFile() } + + var ok = 0; var fail = 0 + files.forEach { + try { + println(deserialize(ctx, FileInputStream(it.toFile())).size) + ok++ + } catch (e: Exception) { + fail++ + } + } + println("$ok / $fail") + } + + /* with(ctx) { // create symbolic variables val a by boolSort @@ -174,4 +207,5 @@ fun main() { } */ } + */ } \ No newline at end of file From ad9d65dc4b35835a378e2ac9ff5e1b7343e8e6d4 Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Tue, 1 Aug 2023 14:52:03 +0300 Subject: [PATCH 22/52] align sat/unsat sizes --- .../src/main/python/GraphDataloader.py | 38 ++++++++++++++----- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/ksmt-neurosmt/src/main/python/GraphDataloader.py b/ksmt-neurosmt/src/main/python/GraphDataloader.py index 963b15efb..5faf6c2d0 100644 --- a/ksmt-neurosmt/src/main/python/GraphDataloader.py +++ b/ksmt-neurosmt/src/main/python/GraphDataloader.py @@ -60,7 +60,7 @@ def load_data(path_to_data): if len(unsat_paths) > SHRINK: unsat_paths = unsat_paths[:SHRINK] - def process_paths(paths, label, data): + def process_paths(paths, data, label): for path in tqdm(paths): operators, edges, depth = read_graph_by_path(path, max_size=MAX_FORMULA_SIZE, max_depth=MAX_FORMULA_DEPTH) @@ -74,11 +74,36 @@ def process_paths(paths, label, data): data.append((operators, edges, label, depth)) sat_data, unsat_data = [], [] - process_paths(sat_paths, 1, sat_data) - process_paths(unsat_paths, 0, unsat_data) + process_paths(sat_paths, sat_data, label=1) + process_paths(unsat_paths, unsat_data, label=0) sat_data, unsat_data = align_sat_unsat_sizes(sat_data, unsat_data) + def split_data(data): + train_ind, val_ind, test_ind = train_val_test_indices(len(data)) + + return [data[i] for i in train_ind], [data[i] for i in val_ind], [data[i] for i in test_ind] + + print("train/val/test split start") + + sat_train, sat_val, sat_test = split_data(sat_data) + unsat_train, unsat_val, unsat_test = split_data(unsat_data) + del sat_data, unsat_data + gc.collect() + + train_data = sat_train + unsat_train + val_data = sat_val + unsat_val + test_data = sat_test + unsat_test + del sat_train, sat_val, sat_test, unsat_train, unsat_val, unsat_test + gc.collect() + + np.random.shuffle(train_data) + np.random.shuffle(val_data) + np.random.shuffle(test_data) + + print("train/val/test split end") + + """ graph_data = sat_data + unsat_data del sat_data, unsat_data gc.collect() @@ -88,19 +113,14 @@ def process_paths(paths, label, data): train_data = [graph_data[i] for i in train_ind] val_data = [graph_data[i] for i in val_ind] test_data = [graph_data[i] for i in test_ind] + """ print("\nstats:") - print(f"overall: {sum(it[2] for it in graph_data) / len(graph_data)} | {len(graph_data)}") print(f"train: {sum(it[2] for it in train_data) / len(train_data)} | {len(train_data)}") print(f"val: {sum(it[2] for it in val_data) / len(val_data)} | {len(val_data)}") print(f"test: {sum(it[2] for it in test_data) / len(test_data)} | {len(test_data)}") print("\n", flush=True) - print("del start") - del graph_data - gc.collect() - print("del end") - encoder = OrdinalEncoder( dtype=int, handle_unknown="use_encoded_value", unknown_value=1999, From 4ce4ad8a5da7baf822812b58ace9acee60b02203 Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Tue, 1 Aug 2023 17:41:20 +0300 Subject: [PATCH 23/52] fix checkpointing --- ksmt-neurosmt/src/main/python/main.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ksmt-neurosmt/src/main/python/main.py b/ksmt-neurosmt/src/main/python/main.py index 57b33e212..99f294cd9 100755 --- a/ksmt-neurosmt/src/main/python/main.py +++ b/ksmt-neurosmt/src/main/python/main.py @@ -1,13 +1,13 @@ #!/usr/bin/python3 import os; os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"; os.environ["CUDA_VISIBLE_DEVICES"] = os.environ["GPU"] -import time from argparse import ArgumentParser import torch from pytorch_lightning import Trainer, seed_everything from pytorch_lightning.loggers import TensorBoardLogger +from pytorch_lightning.callbacks import ModelCheckpoint from GraphDataloader import load_data @@ -25,7 +25,6 @@ def get_args(): print(arg, "=", getattr(args, arg)) print() - time.sleep(5) return args @@ -42,6 +41,13 @@ def get_args(): accelerator="auto", # precision="bf16-mixed", logger=TensorBoardLogger("../logs", name="neuro-smt"), + callbacks=[ModelCheckpoint( + filename="epoch_{epoch:03d}_roc-auc_{val/roc-auc:.3f}", + monitor="val/roc-auc", + verbose=True, + save_last=True, save_top_k=3, mode="max", + auto_insert_metric_name=False, save_on_train_epoch_end=False + )], max_epochs=100, log_every_n_steps=1, enable_checkpointing=True, From ccf7177306b18ff1b33972271ddc502e7929bdd0 Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Tue, 1 Aug 2023 18:43:51 +0300 Subject: [PATCH 24/52] add validation script --- .../src/main/python/LightningModel.py | 43 +++++++++++++------ ksmt-neurosmt/src/main/python/main.py | 2 +- ksmt-neurosmt/src/main/python/validate.py | 40 +++++++++++++++++ 3 files changed, 70 insertions(+), 15 deletions(-) create mode 100644 ksmt-neurosmt/src/main/python/validate.py diff --git a/ksmt-neurosmt/src/main/python/LightningModel.py b/ksmt-neurosmt/src/main/python/LightningModel.py index 9821b2c69..09fc4075d 100644 --- a/ksmt-neurosmt/src/main/python/LightningModel.py +++ b/ksmt-neurosmt/src/main/python/LightningModel.py @@ -52,30 +52,33 @@ def training_step(self, train_batch, batch_idx): return loss - def validation_step(self, val_batch, batch_idx): - out = self.model(val_batch) - loss = F.binary_cross_entropy_with_logits(out, val_batch.y) + def shared_val_test_step(self, batch, batch_idx, metric_name): + out = self.model(batch) + loss = F.binary_cross_entropy_with_logits(out, batch.y) out = F.sigmoid(out) self.log( - "val/loss", loss.float(), + f"{metric_name}/loss", loss.float(), prog_bar=True, logger=True, on_step=False, on_epoch=True, - batch_size=val_batch.num_graphs + batch_size=batch.num_graphs ) self.log( - "val/acc", self.acc(out, val_batch.y), + f"{metric_name}/acc", self.acc(out, batch.y), prog_bar=True, logger=True, on_step=False, on_epoch=True, - batch_size=val_batch.num_graphs + batch_size=batch.num_graphs ) self.val_outputs.append(out) - self.val_targets.append(val_batch.y) + self.val_targets.append(batch.y) return loss + def validation_step(self, val_batch, batch_idx): + return self.shared_val_test_step(val_batch, batch_idx, "val") + def on_validation_epoch_end(self): print("\n\n", flush=True) @@ -85,15 +88,11 @@ def on_validation_epoch_end(self): self.val_outputs.clear() self.val_targets.clear() - logger = self.logger.experiment - - roc_auc = self.roc_auc(all_outputs, all_targets) self.log( - "val/roc-auc", roc_auc, - prog_bar=True, logger=False, + "val/roc-auc", self.roc_auc(all_outputs, all_targets), + prog_bar=True, logger=True, on_step=False, on_epoch=True ) - logger.add_scalar("val/roc-auc", roc_auc, self.current_epoch) all_outputs = all_outputs.float().cpu().numpy() all_targets = all_targets.float().cpu().numpy() @@ -102,3 +101,19 @@ def on_validation_epoch_end(self): print(classification_report(all_targets, all_outputs, digits=3, zero_division=0.0)) print("\n", flush=True) + + def test_step(self, test_batch, batch_idx): + return self.shared_val_test_step(test_batch, batch_idx, "test") + + def on_test_epoch_end(self): + all_outputs = torch.flatten(torch.cat(self.val_outputs)) + all_targets = torch.flatten(torch.cat(self.val_targets)) + + self.val_outputs.clear() + self.val_targets.clear() + + self.log( + "test/roc-auc", self.roc_auc(all_outputs, all_targets), + prog_bar=True, logger=True, + on_step=False, on_epoch=True + ) diff --git a/ksmt-neurosmt/src/main/python/main.py b/ksmt-neurosmt/src/main/python/main.py index 99f294cd9..f5b270b4e 100755 --- a/ksmt-neurosmt/src/main/python/main.py +++ b/ksmt-neurosmt/src/main/python/main.py @@ -48,7 +48,7 @@ def get_args(): save_last=True, save_top_k=3, mode="max", auto_insert_metric_name=False, save_on_train_epoch_end=False )], - max_epochs=100, + max_epochs=200, log_every_n_steps=1, enable_checkpointing=True, barebones=False, diff --git a/ksmt-neurosmt/src/main/python/validate.py b/ksmt-neurosmt/src/main/python/validate.py new file mode 100644 index 000000000..9302a7c3a --- /dev/null +++ b/ksmt-neurosmt/src/main/python/validate.py @@ -0,0 +1,40 @@ +#!/usr/bin/python3 + +import os; os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"; os.environ["CUDA_VISIBLE_DEVICES"] = os.environ["GPU"] +from argparse import ArgumentParser + +import torch + +from pytorch_lightning import Trainer, seed_everything + +from GraphDataloader import load_data + +from LightningModel import LightningModel + + +def get_args(): + parser = ArgumentParser(description="validation script") + parser.add_argument("--ds", required=True) + parser.add_argument("--ckpt", required=True) + + args = parser.parse_args() + print("args:") + for arg in vars(args): + print(arg, "=", getattr(args, arg)) + + print() + + return args + + +if __name__ == "__main__": + seed_everything(24, workers=True) + torch.set_float32_matmul_precision("medium") + + args = get_args() + tr, va, te = load_data(args.ds) + + trainer = Trainer() + + trainer.validate(LightningModel(), va, args.ckpt) + trainer.test(LightningModel(), te, args.ckpt) From 02a959b8bb004c6a93bd56ba241ab8ef52a45ec0 Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Wed, 2 Aug 2023 15:12:49 +0300 Subject: [PATCH 25/52] new logic of data loading --- .../src/main/python/GraphDataloader.py | 95 ++++++++++++--- .../src/main/python/create-ordinal-encoder.py | 50 ++++++++ ksmt-neurosmt/src/main/python/main.py | 4 +- .../src/main/python/prepare-train-val-test.py | 113 ++++++++++++++++++ ksmt-neurosmt/src/main/python/validate.py | 4 +- 5 files changed, 248 insertions(+), 18 deletions(-) create mode 100755 ksmt-neurosmt/src/main/python/create-ordinal-encoder.py create mode 100755 ksmt-neurosmt/src/main/python/prepare-train-val-test.py diff --git a/ksmt-neurosmt/src/main/python/GraphDataloader.py b/ksmt-neurosmt/src/main/python/GraphDataloader.py index 5faf6c2d0..a1aa6af5d 100644 --- a/ksmt-neurosmt/src/main/python/GraphDataloader.py +++ b/ksmt-neurosmt/src/main/python/GraphDataloader.py @@ -3,6 +3,7 @@ import itertools import numpy as np +import joblib from tqdm import tqdm from sklearn.preprocessing import OrdinalEncoder @@ -22,6 +23,7 @@ MAX_FORMULA_DEPTH = 2500 NUM_WORKERS = 16 SHRINK = 10 ** (int(os.environ["SHRINK"]) if "SHRINK" in os.environ else 10) +METADATA_PATH = "__meta" class GraphDataset(Dataset): @@ -41,9 +43,86 @@ def __getitem__(self, index): return self.graphs[index] -def load_data(path_to_data): +def load_data(paths_to_datasets, target): + data = [] + + print(f"loading {target}") + for path_to_dataset in paths_to_datasets: + print(f"loading data from '{path_to_dataset}'") + + with open(os.path.join(path_to_dataset, METADATA_PATH, target), "r") as f: + for path_to_sample in tqdm(list(f.readlines())): + path_to_sample = os.path.join(path_to_dataset, path_to_sample.strip()) + + operators, edges, depth = read_graph_by_path( + path_to_sample, max_size=MAX_FORMULA_SIZE, max_depth=MAX_FORMULA_DEPTH + ) + + if depth is None: + continue + + if len(edges) == 0: + print(f"w: ignoring formula without edges; file '{path_to_sample}'") + continue + + label = None + if path_to_sample.endswith("-sat"): + label = 1 + elif path_to_sample.endswith("-unsat"): + label = 0 + else: + raise Exception(f"strange file path '{path_to_sample}'") + + data.append((operators, edges, label, depth)) + + return data + + +def get_dataloader(paths_to_datasets, target, path_to_ordinal_encoder): + print(f"creating dataloader for {target}") + + data = load_data(paths_to_datasets, target) + + print("loading encoder") + encoder = joblib.load(path_to_ordinal_encoder) + + def transform(data_for_one_sample): + nodes, edges, label, depth = data_for_one_sample + nodes = encoder.transform(np.array(nodes).reshape(-1, 1)) + return nodes, edges, label, depth + + print("transforming") + data = list(map(transform, data)) + + print("creating dataset") + ds = GraphDataset(data) + + print("constructing dataloader\n", flush=True) + return DataLoader( + ds.graphs, + batch_size=BATCH_SIZE, num_workers=NUM_WORKERS, + shuffle=(target == "train"), drop_last=(target == "train") + ) + + +""" +if __name__ == "__main__": + print(get_dataloader(["../BV"], "train", "../enc.bin")) + print(get_dataloader(["../BV"], "val", "../enc.bin")) + print(get_dataloader(["../BV"], "test", "../enc.bin")) +""" + + +# deprecated? +def load_data_from_scratch(path_to_full_dataset): + return ( + get_dataloader(["../BV"], "train", "../enc.bin"), + get_dataloader(["../BV"], "val", "../enc.bin"), + get_dataloader(["../BV"], "test", "../enc.bin") + ) + sat_paths, unsat_paths = [], [] - for it in tqdm(os.walk(path_to_data)): + for it in tqdm(os.walk(path_to_full_dataset)): for file_name in tqdm(it[2]): cur_path = os.path.join(it[0], file_name) @@ -103,18 +182,6 @@ def split_data(data): print("train/val/test split end") - """ - graph_data = sat_data + unsat_data - del sat_data, unsat_data - gc.collect() - - train_ind, val_ind, test_ind = train_val_test_indices(len(graph_data)) - - train_data = [graph_data[i] for i in train_ind] - val_data = [graph_data[i] for i in val_ind] - test_data = [graph_data[i] for i in test_ind] - """ - print("\nstats:") print(f"train: {sum(it[2] for it in train_data) / len(train_data)} | {len(train_data)}") print(f"val: {sum(it[2] for it in val_data) / len(val_data)} | {len(val_data)}") diff --git a/ksmt-neurosmt/src/main/python/create-ordinal-encoder.py b/ksmt-neurosmt/src/main/python/create-ordinal-encoder.py new file mode 100755 index 000000000..a84424ac5 --- /dev/null +++ b/ksmt-neurosmt/src/main/python/create-ordinal-encoder.py @@ -0,0 +1,50 @@ +#!/usr/bin/python3 + +from argparse import ArgumentParser +import itertools + +import numpy as np +import joblib + +from sklearn.preprocessing import OrdinalEncoder + +from GraphDataloader import load_data + + +def create_ordinal_encoder(paths_to_datasets, path_to_ordinal_encoder): + data = load_data(paths_to_datasets, "train") + + encoder = OrdinalEncoder( + dtype=int, + handle_unknown="use_encoded_value", unknown_value=1999, + encoded_missing_value=1998 + ) + + print("fitting ordinal encoder") + encoder.fit(np.array(list(itertools.chain( + *(list(zip(*data))[0]) + ))).reshape(-1, 1)) + + print("dumping ordinal encoder") + joblib.dump(encoder, path_to_ordinal_encoder) + + +def get_args(): + parser = ArgumentParser(description="ordinal encoder preparing script") + parser.add_argument("--ds", required=True, nargs="+") + parser.add_argument("--oenc", required=True) + + args = parser.parse_args() + print("args:") + for arg in vars(args): + print(arg, "=", getattr(args, arg)) + + print() + + return args + + +if __name__ == "__main__": + args = get_args() + + create_ordinal_encoder(args.ds, args.oenc) diff --git a/ksmt-neurosmt/src/main/python/main.py b/ksmt-neurosmt/src/main/python/main.py index f5b270b4e..9802af8f4 100755 --- a/ksmt-neurosmt/src/main/python/main.py +++ b/ksmt-neurosmt/src/main/python/main.py @@ -9,7 +9,7 @@ from pytorch_lightning.loggers import TensorBoardLogger from pytorch_lightning.callbacks import ModelCheckpoint -from GraphDataloader import load_data +from GraphDataloader import load_data_from_scratch from LightningModel import LightningModel @@ -34,7 +34,7 @@ def get_args(): torch.set_float32_matmul_precision("medium") args = get_args() - tr, va, te = load_data(args.ds) + tr, va, te = load_data_from_scratch(args.ds) pl_model = LightningModel() trainer = Trainer( diff --git a/ksmt-neurosmt/src/main/python/prepare-train-val-test.py b/ksmt-neurosmt/src/main/python/prepare-train-val-test.py new file mode 100755 index 000000000..a221be97f --- /dev/null +++ b/ksmt-neurosmt/src/main/python/prepare-train-val-test.py @@ -0,0 +1,113 @@ +#!/usr/bin/python3 + +import os +from argparse import ArgumentParser + +import numpy as np +from tqdm import tqdm + +from pytorch_lightning import seed_everything + +from GraphReader import read_graph_by_path +from GraphDataloader import MAX_FORMULA_SIZE, MAX_FORMULA_DEPTH, SHRINK, METADATA_PATH +from utils import train_val_test_indices, align_sat_unsat_sizes + + +def create_split(path_to_dataset): + sat_paths, unsat_paths = [], [] + for root, dirs, files in os.walk(path_to_dataset, topdown=True): + if METADATA_PATH in dirs: + dirs.remove(METADATA_PATH) + + for file_name in files: + cur_path = os.path.join(root, file_name) + + if cur_path.endswith("-sat"): + sat_paths.append(cur_path) + elif cur_path.endswith("-unsat"): + unsat_paths.append(cur_path) + else: + raise Exception(f"strange file path '{cur_path}'") + + if len(sat_paths) > SHRINK: + sat_paths = sat_paths[:SHRINK] + + if len(unsat_paths) > SHRINK: + unsat_paths = unsat_paths[:SHRINK] + + def process_paths(paths): + correct_paths = [] + for path in tqdm(paths): + operators, edges, depth = read_graph_by_path(path, max_size=MAX_FORMULA_SIZE, max_depth=MAX_FORMULA_DEPTH) + + if depth is None: + continue + + if len(edges) == 0: + print(f"w: ignoring formula without edges; file '{path}'") + continue + + correct_paths.append(os.path.relpath(path, path_to_dataset)) + + return correct_paths + + sat_paths = process_paths(sat_paths) + unsat_paths = process_paths(unsat_paths) + + sat_paths, unsat_paths = align_sat_unsat_sizes(sat_paths, unsat_paths) + + def split_data(data): + train_ind, val_ind, test_ind = train_val_test_indices(len(data)) + + return [data[i] for i in train_ind], [data[i] for i in val_ind], [data[i] for i in test_ind] + + sat_train, sat_val, sat_test = split_data(sat_paths) + unsat_train, unsat_val, unsat_test = split_data(unsat_paths) + + train_data = sat_train + unsat_train + val_data = sat_val + unsat_val + test_data = sat_test + unsat_test + + np.random.shuffle(train_data) + np.random.shuffle(val_data) + np.random.shuffle(test_data) + + print("\nstats:", flush=True) + print(f"train: {len(train_data)}") + print(f"val: {len(val_data)}") + print(f"test: {len(test_data)}") + print(flush=True) + + meta_path = os.path.join(path_to_dataset, METADATA_PATH) + os.makedirs(meta_path, exist_ok=True) + + with open(os.path.join(meta_path, "train"), "w") as f: + f.write("\n".join(train_data) + "\n") + + with open(os.path.join(meta_path, "val"), "w") as f: + f.write("\n".join(val_data) + "\n") + + with open(os.path.join(meta_path, "test"), "w") as f: + f.write("\n".join(test_data) + "\n") + + +def get_args(): + parser = ArgumentParser(description="train/val/test splitting script") + parser.add_argument("--ds", required=True) + + args = parser.parse_args() + print("args:") + for arg in vars(args): + print(arg, "=", getattr(args, arg)) + + print() + + return args + + +if __name__ == "__main__": + seed_everything(24) + + args = get_args() + + create_split(args.ds) diff --git a/ksmt-neurosmt/src/main/python/validate.py b/ksmt-neurosmt/src/main/python/validate.py index 9302a7c3a..13301dd0c 100644 --- a/ksmt-neurosmt/src/main/python/validate.py +++ b/ksmt-neurosmt/src/main/python/validate.py @@ -7,7 +7,7 @@ from pytorch_lightning import Trainer, seed_everything -from GraphDataloader import load_data +from GraphDataloader import load_data_from_scratch from LightningModel import LightningModel @@ -32,7 +32,7 @@ def get_args(): torch.set_float32_matmul_precision("medium") args = get_args() - tr, va, te = load_data(args.ds) + tr, va, te = load_data_from_scratch(args.ds) trainer = Trainer() From 0bdd180cb2e54f9a0f68af06e6be364f6fe40450 Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Wed, 2 Aug 2023 16:01:18 +0300 Subject: [PATCH 26/52] adjust train/val scripts to new data loading process --- .../src/main/python/{main.py => train.py} | 18 +++++++++--------- ksmt-neurosmt/src/main/python/validate.py | 17 ++++++++++------- 2 files changed, 19 insertions(+), 16 deletions(-) rename ksmt-neurosmt/src/main/python/{main.py => train.py} (76%) diff --git a/ksmt-neurosmt/src/main/python/main.py b/ksmt-neurosmt/src/main/python/train.py similarity index 76% rename from ksmt-neurosmt/src/main/python/main.py rename to ksmt-neurosmt/src/main/python/train.py index 9802af8f4..d04bf457c 100755 --- a/ksmt-neurosmt/src/main/python/main.py +++ b/ksmt-neurosmt/src/main/python/train.py @@ -5,18 +5,19 @@ import torch -from pytorch_lightning import Trainer, seed_everything +from pytorch_lightning import Trainer from pytorch_lightning.loggers import TensorBoardLogger from pytorch_lightning.callbacks import ModelCheckpoint -from GraphDataloader import load_data_from_scratch +from GraphDataloader import get_dataloader from LightningModel import LightningModel def get_args(): parser = ArgumentParser(description="main training script") - parser.add_argument("--ds", required=True) + parser.add_argument("--ds", required=True, nargs="+") + parser.add_argument("--oenc", required=True) parser.add_argument("--ckpt", required=False) args = parser.parse_args() @@ -30,11 +31,13 @@ def get_args(): if __name__ == "__main__": - seed_everything(24, workers=True) + # seed_everything(24, workers=True) torch.set_float32_matmul_precision("medium") args = get_args() - tr, va, te = load_data_from_scratch(args.ds) + + train_dl = get_dataloader(args.ds, "train", args.oenc) + val_dl = get_dataloader(args.ds, "val", args.oenc) pl_model = LightningModel() trainer = Trainer( @@ -55,7 +58,4 @@ def get_args(): default_root_dir=".." ) - if args.ckpt is None: - trainer.fit(pl_model, tr, va) - else: - trainer.fit(pl_model, tr, va, ckpt_path=args.ckpt) + trainer.fit(pl_model, train_dl, val_dl, ckpt_path=args.ckpt) diff --git a/ksmt-neurosmt/src/main/python/validate.py b/ksmt-neurosmt/src/main/python/validate.py index 13301dd0c..45ce5cd4c 100644 --- a/ksmt-neurosmt/src/main/python/validate.py +++ b/ksmt-neurosmt/src/main/python/validate.py @@ -5,16 +5,17 @@ import torch -from pytorch_lightning import Trainer, seed_everything +from pytorch_lightning import Trainer -from GraphDataloader import load_data_from_scratch +from GraphDataloader import get_dataloader from LightningModel import LightningModel def get_args(): parser = ArgumentParser(description="validation script") - parser.add_argument("--ds", required=True) + parser.add_argument("--ds", required=True, nargs="+") + parser.add_argument("--oenc", required=True) parser.add_argument("--ckpt", required=True) args = parser.parse_args() @@ -28,13 +29,15 @@ def get_args(): if __name__ == "__main__": - seed_everything(24, workers=True) + # seed_everything(24, workers=True) torch.set_float32_matmul_precision("medium") args = get_args() - tr, va, te = load_data_from_scratch(args.ds) + + val_dl = get_dataloader(args.ds, "val", args.oenc) + test_dl = get_dataloader(args.ds, "test", args.oenc) trainer = Trainer() - trainer.validate(LightningModel(), va, args.ckpt) - trainer.test(LightningModel(), te, args.ckpt) + trainer.validate(LightningModel(), val_dl, args.ckpt) + trainer.test(LightningModel(), test_dl, args.ckpt) From bc590f9eeb58d7b0ba26585926e706ef3ef5e656 Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Wed, 2 Aug 2023 16:05:15 +0300 Subject: [PATCH 27/52] refactoring --- .../src/main/python/GraphDataloader.py | 130 ------------------ .../src/main/python/prepare-train-val-test.py | 5 +- 2 files changed, 4 insertions(+), 131 deletions(-) diff --git a/ksmt-neurosmt/src/main/python/GraphDataloader.py b/ksmt-neurosmt/src/main/python/GraphDataloader.py index a1aa6af5d..be79fe1c9 100644 --- a/ksmt-neurosmt/src/main/python/GraphDataloader.py +++ b/ksmt-neurosmt/src/main/python/GraphDataloader.py @@ -1,13 +1,9 @@ import os -import gc -import itertools import numpy as np import joblib from tqdm import tqdm -from sklearn.preprocessing import OrdinalEncoder - import torch from torch.utils.data import Dataset @@ -15,14 +11,12 @@ from torch_geometric.loader import DataLoader from GraphReader import read_graph_by_path -from utils import train_val_test_indices, align_sat_unsat_sizes BATCH_SIZE = 32 MAX_FORMULA_SIZE = 10000 MAX_FORMULA_DEPTH = 2500 NUM_WORKERS = 16 -SHRINK = 10 ** (int(os.environ["SHRINK"]) if "SHRINK" in os.environ else 10) METADATA_PATH = "__meta" @@ -103,127 +97,3 @@ def transform(data_for_one_sample): batch_size=BATCH_SIZE, num_workers=NUM_WORKERS, shuffle=(target == "train"), drop_last=(target == "train") ) - - -""" -if __name__ == "__main__": - print(get_dataloader(["../BV"], "train", "../enc.bin")) - print(get_dataloader(["../BV"], "val", "../enc.bin")) - print(get_dataloader(["../BV"], "test", "../enc.bin")) -""" - - -# deprecated? -def load_data_from_scratch(path_to_full_dataset): - return ( - get_dataloader(["../BV"], "train", "../enc.bin"), - get_dataloader(["../BV"], "val", "../enc.bin"), - get_dataloader(["../BV"], "test", "../enc.bin") - ) - - sat_paths, unsat_paths = [], [] - for it in tqdm(os.walk(path_to_full_dataset)): - for file_name in tqdm(it[2]): - cur_path = os.path.join(it[0], file_name) - - if cur_path.endswith("-sat"): - sat_paths.append(cur_path) - elif cur_path.endswith("-unsat"): - unsat_paths.append(cur_path) - else: - raise Exception(f"strange file path '{cur_path}'") - - if len(sat_paths) > SHRINK: - sat_paths = sat_paths[:SHRINK] - - if len(unsat_paths) > SHRINK: - unsat_paths = unsat_paths[:SHRINK] - - def process_paths(paths, data, label): - for path in tqdm(paths): - operators, edges, depth = read_graph_by_path(path, max_size=MAX_FORMULA_SIZE, max_depth=MAX_FORMULA_DEPTH) - - if depth is None: - continue - - if len(edges) == 0: - print(f"w: ignoring formula without edges; file '{path}'") - continue - - data.append((operators, edges, label, depth)) - - sat_data, unsat_data = [], [] - process_paths(sat_paths, sat_data, label=1) - process_paths(unsat_paths, unsat_data, label=0) - - sat_data, unsat_data = align_sat_unsat_sizes(sat_data, unsat_data) - - def split_data(data): - train_ind, val_ind, test_ind = train_val_test_indices(len(data)) - - return [data[i] for i in train_ind], [data[i] for i in val_ind], [data[i] for i in test_ind] - - print("train/val/test split start") - - sat_train, sat_val, sat_test = split_data(sat_data) - unsat_train, unsat_val, unsat_test = split_data(unsat_data) - del sat_data, unsat_data - gc.collect() - - train_data = sat_train + unsat_train - val_data = sat_val + unsat_val - test_data = sat_test + unsat_test - del sat_train, sat_val, sat_test, unsat_train, unsat_val, unsat_test - gc.collect() - - np.random.shuffle(train_data) - np.random.shuffle(val_data) - np.random.shuffle(test_data) - - print("train/val/test split end") - - print("\nstats:") - print(f"train: {sum(it[2] for it in train_data) / len(train_data)} | {len(train_data)}") - print(f"val: {sum(it[2] for it in val_data) / len(val_data)} | {len(val_data)}") - print(f"test: {sum(it[2] for it in test_data) / len(test_data)} | {len(test_data)}") - print("\n", flush=True) - - encoder = OrdinalEncoder( - dtype=int, - handle_unknown="use_encoded_value", unknown_value=1999, - encoded_missing_value=1998 - ) - - print("enc fit start") - encoder.fit(np.array(list(itertools.chain( - *(list(zip(*train_data))[0]) - ))).reshape(-1, 1)) - print("enc fit end") - - def transform(data_for_one_graph): - nodes, edges, label, depth = data_for_one_graph - nodes = encoder.transform(np.array(nodes).reshape(-1, 1)) - return nodes, edges, label, depth - - print("transform start") - train_data = list(map(transform, train_data)) - val_data = list(map(transform, val_data)) - test_data = list(map(transform, test_data)) - print("transform end") - - print("create dataset start") - train_ds = GraphDataset(train_data) - val_ds = GraphDataset(val_data) - test_ds = GraphDataset(test_data) - print("create dataset end") - - try: - print("create dataloader start") - return ( - DataLoader(train_ds.graphs, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS, shuffle=True, drop_last=True), - DataLoader(val_ds.graphs, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS, shuffle=False, drop_last=False), - DataLoader(test_ds.graphs, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS, shuffle=False, drop_last=False) - ) - - finally: - print("create dataloader end\n", flush=True) diff --git a/ksmt-neurosmt/src/main/python/prepare-train-val-test.py b/ksmt-neurosmt/src/main/python/prepare-train-val-test.py index a221be97f..3c73b343c 100755 --- a/ksmt-neurosmt/src/main/python/prepare-train-val-test.py +++ b/ksmt-neurosmt/src/main/python/prepare-train-val-test.py @@ -9,10 +9,13 @@ from pytorch_lightning import seed_everything from GraphReader import read_graph_by_path -from GraphDataloader import MAX_FORMULA_SIZE, MAX_FORMULA_DEPTH, SHRINK, METADATA_PATH +from GraphDataloader import MAX_FORMULA_SIZE, MAX_FORMULA_DEPTH, METADATA_PATH from utils import train_val_test_indices, align_sat_unsat_sizes +SHRINK = 10 ** (int(os.environ["SHRINK"]) if "SHRINK" in os.environ else 10) + + def create_split(path_to_dataset): sat_paths, unsat_paths = [], [] for root, dirs, files in os.walk(path_to_dataset, topdown=True): From bbc3c5ea86b3d9c42958a239a10e2f9c629e2724 Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Wed, 2 Aug 2023 18:46:04 +0300 Subject: [PATCH 28/52] add ksmt binary files processing --- ksmt-neurosmt/utils/build.gradle.kts | 28 ++++- .../FormulaGraphExtractor.kt | 18 ++- .../kotlin/io/ksmt/solver/neurosmt/Utils.kt | 109 ++++++++++++++++++ .../KSMTBinaryConverter.kt | 78 +++++++++++++ .../SMT2Converter.kt | 4 +- .../solver/neurosmt/smt2converter/Utils.kt | 19 --- 6 files changed, 226 insertions(+), 30 deletions(-) rename ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/{smt2converter => }/FormulaGraphExtractor.kt (71%) create mode 100644 ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/Utils.kt create mode 100644 ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/ksmtBinaryConverter/KSMTBinaryConverter.kt rename ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/{smt2converter => smt2Converter}/SMT2Converter.kt (93%) delete mode 100644 ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/Utils.kt diff --git a/ksmt-neurosmt/utils/build.gradle.kts b/ksmt-neurosmt/utils/build.gradle.kts index e3a6ad56f..8eb3e889e 100644 --- a/ksmt-neurosmt/utils/build.gradle.kts +++ b/ksmt-neurosmt/utils/build.gradle.kts @@ -13,16 +13,18 @@ repositories { dependencies { implementation(project(":ksmt-core")) implementation(project(":ksmt-z3")) + implementation(project(":ksmt-runner")) implementation("me.tongfei:progressbar:0.9.4") } application { - mainClass.set("io.ksmt.solver.neurosmt.smt2converter.SMT2ConverterKt") + // mainClass.set("io.ksmt.solver.neurosmt.smt2Converter.SMT2ConverterKt") + mainClass.set("io.ksmt.solver.neurosmt.ksmtBinaryConverter.KSMTBinaryConverterKt") } tasks { - val fatJar = register("fatJar") { + val smt2FatJar = register("smt2FatJar") { dependsOn.addAll(listOf("compileJava", "compileKotlin", "processResources")) archiveFileName.set("convert-smt2.jar") @@ -30,7 +32,25 @@ tasks { duplicatesStrategy = DuplicatesStrategy.EXCLUDE manifest { - attributes(mapOf("Main-Class" to application.mainClass)) + attributes(mapOf("Main-Class" to "io.ksmt.solver.neurosmt.smt2Converter.SMT2ConverterKt")) + } + + val sourcesMain = sourceSets.main.get() + val contents = configurations.runtimeClasspath.get() + .map { if (it.isDirectory) it else zipTree(it) } + sourcesMain.output + + from(contents) + } + + val ksmtFatJar = register("ksmtFatJar") { + dependsOn.addAll(listOf("compileJava", "compileKotlin", "processResources")) + + archiveFileName.set("convert-ksmt.jar") + destinationDirectory.set(File(".")) + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + + manifest { + attributes(mapOf("Main-Class" to "io.ksmt.solver.neurosmt.ksmtBinaryConverter.KSMTBinaryConverterKt")) } val sourcesMain = sourceSets.main.get() @@ -41,6 +61,6 @@ tasks { } build { - dependsOn(fatJar) + dependsOn(smt2FatJar, ksmtFatJar) } } \ No newline at end of file diff --git a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/FormulaGraphExtractor.kt b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/FormulaGraphExtractor.kt similarity index 71% rename from ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/FormulaGraphExtractor.kt rename to ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/FormulaGraphExtractor.kt index 9fb445188..8d430a3c1 100644 --- a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/FormulaGraphExtractor.kt +++ b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/FormulaGraphExtractor.kt @@ -1,4 +1,4 @@ -package io.ksmt.solver.neurosmt.smt2converter +package io.ksmt.solver.neurosmt import io.ksmt.KContext import io.ksmt.expr.KApp @@ -6,9 +6,7 @@ import io.ksmt.expr.KConst import io.ksmt.expr.KExpr import io.ksmt.expr.KInterpretedValue import io.ksmt.expr.transformer.KNonRecursiveTransformer -import io.ksmt.sort.KBoolSort -import io.ksmt.sort.KBvSort -import io.ksmt.sort.KSort +import io.ksmt.sort.* import java.io.OutputStream import java.util.* @@ -39,7 +37,11 @@ class FormulaGraphExtractor( when (symbol.sort) { is KBoolSort -> writer.write("SYMBOLIC; Bool\n") is KBvSort -> writer.write("SYMBOLIC; BitVec\n") - else -> error("unknown symbolic sort: ${symbol.sort}") + is KFpSort -> writer.write("SYMBOLIC; FP\n") + is KFpRoundingModeSort -> writer.write("SYMBOLIC; FP_RM\n") + is KArraySortBase<*> -> writer.write("SYMBOLIC; Array\n") + is KUninterpretedSort -> writer.write("SYMBOLIC; Unint\n") + else -> error("unknown symbolic sort: ${symbol.sort::class.simpleName}") } } @@ -47,7 +49,11 @@ class FormulaGraphExtractor( when (value.decl.sort) { is KBoolSort -> writer.write("VALUE; Bool\n") is KBvSort -> writer.write("VALUE; BitVec\n") - else -> error("unknown value sort: ${value.decl.sort}") + is KFpSort -> writer.write("VALUE; FP\n") + is KFpRoundingModeSort -> writer.write("VALUE; FP_RM\n") + is KArraySortBase<*> -> writer.write("VALUE; Array\n") + is KUninterpretedSort -> writer.write("VALUE; Unint\n") + else -> error("unknown value sort: ${value.decl.sort::class.simpleName}") } } diff --git a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/Utils.kt b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/Utils.kt new file mode 100644 index 000000000..32f7e0777 --- /dev/null +++ b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/Utils.kt @@ -0,0 +1,109 @@ +package io.ksmt.solver.neurosmt + +import com.jetbrains.rd.framework.SerializationCtx +import com.jetbrains.rd.framework.Serializers +import com.jetbrains.rd.framework.UnsafeBuffer +import io.ksmt.KContext +import io.ksmt.expr.KExpr +import io.ksmt.runner.serializer.AstSerializationCtx +import io.ksmt.solver.KSolverStatus +import io.ksmt.solver.z3.KZ3Solver +import io.ksmt.sort.KBoolSort +import io.ksmt.utils.uncheckedCast +import java.io.File +import java.io.InputStream +import java.io.OutputStream +import java.nio.file.Path +import kotlin.time.Duration +import kotlin.time.Duration.Companion.seconds + +fun getAnswerForTest(path: Path): KSolverStatus { + File(path.toUri()).useLines { lines -> + for (line in lines) { + when (line) { + "(set-info :status sat)" -> return KSolverStatus.SAT + "(set-info :status unsat)" -> return KSolverStatus.UNSAT + "(set-info :status unknown)" -> return KSolverStatus.UNKNOWN + } + } + } + + return KSolverStatus.UNKNOWN +} + +fun getAnswerForTest(ctx: KContext, formula: List>, timeout: Duration): KSolverStatus { + return KZ3Solver(ctx).use { solver -> + for (clause in formula) { + solver.assert(clause) + } + + solver.check(timeout = timeout) + } + + /* + with(ctx) { + val yicesStatus = KYicesSolver(this).use { solver -> + for (clause in formula) { + solver.assert(clause) + } + + solver.check(timeout = timeout) + } + + val bitwuzlaStatus = KBitwuzlaSolver(this).use { solver -> + for (clause in formula) { + solver.assert(clause) + } + + solver.check(timeout = timeout) + } + + if (yicesStatus == KSolverStatus.UNKNOWN) { + return bitwuzlaStatus + } + if (bitwuzlaStatus == KSolverStatus.UNKNOWN) { + return yicesStatus + } + + if (yicesStatus != bitwuzlaStatus) { + return KSolverStatus.UNKNOWN + } else { + return yicesStatus + } + } + */ +} + +fun serialize(ctx: KContext, expressions: List>, outputStream: OutputStream) { + val serializationCtx = AstSerializationCtx().apply { initCtx(ctx) } + val marshaller = AstSerializationCtx.marshaller(serializationCtx) + val emptyRdSerializationCtx = SerializationCtx(Serializers()) + + val buffer = UnsafeBuffer(ByteArray(100_000)) + + expressions.forEach { expr -> + marshaller.write(emptyRdSerializationCtx, buffer, expr) + } + + outputStream.write(buffer.getArray()) + outputStream.flush() +} + +fun deserialize(ctx: KContext, inputStream: InputStream): List> { + val srcSerializationCtx = AstSerializationCtx().apply { initCtx(ctx) } + val srcMarshaller = AstSerializationCtx.marshaller(srcSerializationCtx) + val emptyRdSerializationCtx = SerializationCtx(Serializers()) + + val buffer = UnsafeBuffer(inputStream.readBytes()) + val expressions: MutableList> = mutableListOf() + + while (true) { + try { + expressions.add(srcMarshaller.read(emptyRdSerializationCtx, buffer).uncheckedCast()) + } catch (e : IllegalStateException) { + break + } + } + + return expressions +} \ No newline at end of file diff --git a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/ksmtBinaryConverter/KSMTBinaryConverter.kt b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/ksmtBinaryConverter/KSMTBinaryConverter.kt new file mode 100644 index 000000000..f2103fa67 --- /dev/null +++ b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/ksmtBinaryConverter/KSMTBinaryConverter.kt @@ -0,0 +1,78 @@ +package io.ksmt.solver.neurosmt.ksmtBinaryConverter + +import io.ksmt.KContext +import io.ksmt.parser.KSMTLibParseException +import io.ksmt.solver.KSolverStatus +import io.ksmt.solver.neurosmt.FormulaGraphExtractor +import io.ksmt.solver.neurosmt.deserialize +import io.ksmt.solver.neurosmt.getAnswerForTest +import io.ksmt.solver.z3.KZ3SMTLibParser +import me.tongfei.progressbar.ProgressBar +import java.io.FileInputStream +import java.io.FileOutputStream +import java.nio.file.Files +import java.nio.file.Path +import kotlin.io.path.isRegularFile +import kotlin.io.path.name +import kotlin.time.Duration.Companion.seconds + +fun main(args: Array) { + val inputRoot = args[0] + val outputRoot = args[1] + val timeout = args[2].toInt().seconds + + val files = Files.walk(Path.of(inputRoot)).filter { it.isRegularFile() } + + var sat = 0; var unsat = 0; var skipped = 0 + + val ctx = KContext(simplificationMode = KContext.SimplificationMode.NO_SIMPLIFY) + + var curIdx = 0 + ProgressBar.wrap(files, "converting ksmt binary files").forEach { + val assertList = try { + deserialize(ctx, FileInputStream(it.toFile())) + } catch (e: Exception) { + skipped++ + return@forEach + } + + val answer = getAnswerForTest(ctx, assertList, timeout) + + if (answer == KSolverStatus.UNKNOWN) { + skipped++ + return@forEach + } + + with(ctx) { + val formula = when (assertList.size) { + 0 -> { + skipped++ + return@forEach + } + 1 -> { + assertList[0] + } + else -> { + mkAnd(assertList) + } + } + + val outputStream = FileOutputStream("$outputRoot/$curIdx-${answer.toString().lowercase()}") + outputStream.write("; $it\n".encodeToByteArray()) + + val extractor = FormulaGraphExtractor(ctx, formula, outputStream) + extractor.extractGraph() + } + + when (answer) { + KSolverStatus.SAT -> sat++ + KSolverStatus.UNSAT -> unsat++ + else -> { /* can't happen */ } + } + + curIdx++ + } + + println() + println("sat: $sat; unsat: $unsat; skipped: $skipped") +} \ No newline at end of file diff --git a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/SMT2Converter.kt b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2Converter/SMT2Converter.kt similarity index 93% rename from ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/SMT2Converter.kt rename to ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2Converter/SMT2Converter.kt index 2f1cfa3f6..f19d5abef 100644 --- a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/SMT2Converter.kt +++ b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2Converter/SMT2Converter.kt @@ -1,8 +1,10 @@ -package io.ksmt.solver.neurosmt.smt2converter +package io.ksmt.solver.neurosmt.smt2Converter import io.ksmt.KContext import io.ksmt.parser.KSMTLibParseException import io.ksmt.solver.KSolverStatus +import io.ksmt.solver.neurosmt.FormulaGraphExtractor +import io.ksmt.solver.neurosmt.getAnswerForTest import io.ksmt.solver.z3.KZ3SMTLibParser import me.tongfei.progressbar.ProgressBar import java.io.FileOutputStream diff --git a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/Utils.kt b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/Utils.kt deleted file mode 100644 index 4968f62e3..000000000 --- a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2converter/Utils.kt +++ /dev/null @@ -1,19 +0,0 @@ -package io.ksmt.solver.neurosmt.smt2converter - -import io.ksmt.solver.KSolverStatus -import java.io.File -import java.nio.file.Path - -fun getAnswerForTest(path: Path): KSolverStatus { - File(path.toUri()).useLines { lines -> - for (line in lines) { - when (line) { - "(set-info :status sat)" -> return KSolverStatus.SAT - "(set-info :status unsat)" -> return KSolverStatus.UNSAT - "(set-info :status unknown)" -> return KSolverStatus.UNKNOWN - } - } - } - - return KSolverStatus.UNKNOWN -} \ No newline at end of file From e0b644d6b633dbbae4857b1f433fd7df995228be Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Wed, 2 Aug 2023 18:47:55 +0300 Subject: [PATCH 29/52] refactoring --- .../kotlin/io/ksmt/solver/neurosmt/Utils.kt | 34 ------------------- .../KSMTBinaryConverter.kt | 3 -- 2 files changed, 37 deletions(-) diff --git a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/Utils.kt b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/Utils.kt index 32f7e0777..7466b8f20 100644 --- a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/Utils.kt +++ b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/Utils.kt @@ -15,7 +15,6 @@ import java.io.InputStream import java.io.OutputStream import java.nio.file.Path import kotlin.time.Duration -import kotlin.time.Duration.Companion.seconds fun getAnswerForTest(path: Path): KSolverStatus { File(path.toUri()).useLines { lines -> @@ -39,39 +38,6 @@ fun getAnswerForTest(ctx: KContext, formula: List>, timeout: Du solver.check(timeout = timeout) } - - /* - with(ctx) { - val yicesStatus = KYicesSolver(this).use { solver -> - for (clause in formula) { - solver.assert(clause) - } - - solver.check(timeout = timeout) - } - - val bitwuzlaStatus = KBitwuzlaSolver(this).use { solver -> - for (clause in formula) { - solver.assert(clause) - } - - solver.check(timeout = timeout) - } - - if (yicesStatus == KSolverStatus.UNKNOWN) { - return bitwuzlaStatus - } - if (bitwuzlaStatus == KSolverStatus.UNKNOWN) { - return yicesStatus - } - - if (yicesStatus != bitwuzlaStatus) { - return KSolverStatus.UNKNOWN - } else { - return yicesStatus - } - } - */ } fun serialize(ctx: KContext, expressions: List>, outputStream: OutputStream) { diff --git a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/ksmtBinaryConverter/KSMTBinaryConverter.kt b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/ksmtBinaryConverter/KSMTBinaryConverter.kt index f2103fa67..57ba3790f 100644 --- a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/ksmtBinaryConverter/KSMTBinaryConverter.kt +++ b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/ksmtBinaryConverter/KSMTBinaryConverter.kt @@ -1,19 +1,16 @@ package io.ksmt.solver.neurosmt.ksmtBinaryConverter import io.ksmt.KContext -import io.ksmt.parser.KSMTLibParseException import io.ksmt.solver.KSolverStatus import io.ksmt.solver.neurosmt.FormulaGraphExtractor import io.ksmt.solver.neurosmt.deserialize import io.ksmt.solver.neurosmt.getAnswerForTest -import io.ksmt.solver.z3.KZ3SMTLibParser import me.tongfei.progressbar.ProgressBar import java.io.FileInputStream import java.io.FileOutputStream import java.nio.file.Files import java.nio.file.Path import kotlin.io.path.isRegularFile -import kotlin.io.path.name import kotlin.time.Duration.Companion.seconds fun main(args: Array) { From 55464538bd4aff23c13f8ce902d1800c9a3ba325 Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Thu, 3 Aug 2023 11:21:56 +0300 Subject: [PATCH 30/52] refactoring 2 --- .../neurosmt/ksmtBinaryConverter/KSMTBinaryConverter.kt | 4 +++- .../io/ksmt/solver/neurosmt/smt2Converter/SMT2Converter.kt | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/ksmtBinaryConverter/KSMTBinaryConverter.kt b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/ksmtBinaryConverter/KSMTBinaryConverter.kt index 57ba3790f..2f59f0704 100644 --- a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/ksmtBinaryConverter/KSMTBinaryConverter.kt +++ b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/ksmtBinaryConverter/KSMTBinaryConverter.kt @@ -6,6 +6,7 @@ import io.ksmt.solver.neurosmt.FormulaGraphExtractor import io.ksmt.solver.neurosmt.deserialize import io.ksmt.solver.neurosmt.getAnswerForTest import me.tongfei.progressbar.ProgressBar +import java.io.File import java.io.FileInputStream import java.io.FileOutputStream import java.nio.file.Files @@ -54,7 +55,8 @@ fun main(args: Array) { } } - val outputStream = FileOutputStream("$outputRoot/$curIdx-${answer.toString().lowercase()}") + val outputFile = File("$outputRoot/$curIdx-${answer.toString().lowercase()}") + val outputStream = FileOutputStream(outputFile) outputStream.write("; $it\n".encodeToByteArray()) val extractor = FormulaGraphExtractor(ctx, formula, outputStream) diff --git a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2Converter/SMT2Converter.kt b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2Converter/SMT2Converter.kt index f19d5abef..e6fd1e89e 100644 --- a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2Converter/SMT2Converter.kt +++ b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2Converter/SMT2Converter.kt @@ -7,6 +7,7 @@ import io.ksmt.solver.neurosmt.FormulaGraphExtractor import io.ksmt.solver.neurosmt.getAnswerForTest import io.ksmt.solver.z3.KZ3SMTLibParser import me.tongfei.progressbar.ProgressBar +import java.io.File import java.io.FileOutputStream import java.nio.file.Files import java.nio.file.Path @@ -60,7 +61,8 @@ fun main(args: Array) { return@forEach } - val outputStream = FileOutputStream("$outputRoot/$curIdx-${answer.toString().lowercase()}") + val outputFile = File("$outputRoot/$curIdx-${answer.toString().lowercase()}") + val outputStream = FileOutputStream(outputFile) outputStream.write("; $it\n".encodeToByteArray()) val extractor = FormulaGraphExtractor(ctx, formula, outputStream) From 5c6bb72459c68dcdfbc4306b90c0f6ced4091b37 Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Wed, 9 Aug 2023 18:59:10 +0300 Subject: [PATCH 31/52] train/test/val split with groups to avoid data leaks --- .../src/main/python/GraphDataloader.py | 5 +- ksmt-neurosmt/src/main/python/GraphReader.py | 9 ++- .../src/main/python/check-split-leaks.py | 53 +++++++++++++ ksmt-neurosmt/src/main/python/utils.py | 75 ++++++++++++++++--- ksmt-neurosmt/src/main/python/validate.py | 0 .../KSMTBinaryConverter.kt | 20 ++++- 6 files changed, 148 insertions(+), 14 deletions(-) create mode 100755 ksmt-neurosmt/src/main/python/check-split-leaks.py mode change 100644 => 100755 ksmt-neurosmt/src/main/python/validate.py diff --git a/ksmt-neurosmt/src/main/python/GraphDataloader.py b/ksmt-neurosmt/src/main/python/GraphDataloader.py index be79fe1c9..7e1a1a35e 100644 --- a/ksmt-neurosmt/src/main/python/GraphDataloader.py +++ b/ksmt-neurosmt/src/main/python/GraphDataloader.py @@ -16,7 +16,7 @@ BATCH_SIZE = 32 MAX_FORMULA_SIZE = 10000 MAX_FORMULA_DEPTH = 2500 -NUM_WORKERS = 16 +NUM_WORKERS = 32 METADATA_PATH = "__meta" @@ -75,8 +75,11 @@ def load_data(paths_to_datasets, target): def get_dataloader(paths_to_datasets, target, path_to_ordinal_encoder): print(f"creating dataloader for {target}") + print("loading data") data = load_data(paths_to_datasets, target) + print(f"stats: {len(data)} overall; sat fraction is {sum(it[2] for it in data) / len(data)}") + print("loading encoder") encoder = joblib.load(path_to_ordinal_encoder) diff --git a/ksmt-neurosmt/src/main/python/GraphReader.py b/ksmt-neurosmt/src/main/python/GraphReader.py index bac9de5ab..2af2a319e 100644 --- a/ksmt-neurosmt/src/main/python/GraphReader.py +++ b/ksmt-neurosmt/src/main/python/GraphReader.py @@ -62,4 +62,11 @@ def read_graph_from_file(inf, max_size, max_depth): def read_graph_by_path(path, max_size, max_depth): with open(path, "r") as inf: - return read_graph_from_file(inf, max_size, max_depth) + try: + return read_graph_from_file(inf, max_size, max_depth) + + except Exception as e: + print(e) + print(f"path: '{path}'") + + raise e diff --git a/ksmt-neurosmt/src/main/python/check-split-leaks.py b/ksmt-neurosmt/src/main/python/check-split-leaks.py new file mode 100755 index 000000000..dbae54656 --- /dev/null +++ b/ksmt-neurosmt/src/main/python/check-split-leaks.py @@ -0,0 +1,53 @@ +#!/usr/bin/python3 + +import os +from argparse import ArgumentParser + +from tqdm import tqdm + +from GraphDataloader import METADATA_PATH + + +def get_groups_set(paths_to_datasets, target): + groups = set() + + print(f"loading {target}") + for path_to_dataset in paths_to_datasets: + print(f"loading data from '{path_to_dataset}'") + + with open(os.path.join(path_to_dataset, METADATA_PATH, target), "r") as f: + for path_to_sample in tqdm(list(f.readlines())): + path_to_sample = path_to_sample.strip() + path_to_parent = os.path.dirname(path_to_sample) + + groups.add(str(path_to_parent)) + + return groups + + +def get_args(): + parser = ArgumentParser(description="validation script") + parser.add_argument("--ds", required=True, nargs="+") + + args = parser.parse_args() + print("args:") + for arg in vars(args): + print(arg, "=", getattr(args, arg)) + + print() + + return args + + +if __name__ == "__main__": + args = get_args() + + train_groups = get_groups_set(args.ds, "train") + val_groups = get_groups_set(args.ds, "val") + test_groups = get_groups_set(args.ds, "test") + + assert train_groups.isdisjoint(val_groups) + assert val_groups.isdisjoint(test_groups) + assert test_groups.isdisjoint(train_groups) + + print("\nsuccess!") diff --git a/ksmt-neurosmt/src/main/python/utils.py b/ksmt-neurosmt/src/main/python/utils.py index 877a3eb0c..ae090781b 100644 --- a/ksmt-neurosmt/src/main/python/utils.py +++ b/ksmt-neurosmt/src/main/python/utils.py @@ -1,26 +1,81 @@ +import os + import numpy as np +from tqdm import tqdm + +from GraphReader import read_graph_by_path +from GraphDataloader import MAX_FORMULA_SIZE, MAX_FORMULA_DEPTH -def train_val_test_indices(cnt, val_pct=0.15, test_pct=0.1): +def train_val_test_indices(cnt, val_qty=0.15, test_qty=0.1): perm = np.arange(cnt) np.random.shuffle(perm) - val_cnt = int(cnt * val_pct) - test_cnt = int(cnt * test_pct) + val_cnt = int(cnt * val_qty) + test_cnt = int(cnt * test_qty) return perm[val_cnt + test_cnt:], perm[:val_cnt], perm[val_cnt:val_cnt + test_cnt] -def align_sat_unsat_sizes(sat_data, unsat_data): - sat_indices = list(range(len(sat_data))) - unsat_indices = list(range(len(unsat_data))) +def select_paths_with_suitable_samples_and_transform_to_paths_from_root(path_to_dataset_root, paths): + correct_paths = [] + for path in tqdm(paths): + operators, edges, depth = read_graph_by_path(path, max_size=MAX_FORMULA_SIZE, max_depth=MAX_FORMULA_DEPTH) + + if depth is None: + continue + + if len(edges) == 0: + print(f"w: ignoring formula without edges; file '{path}'") + continue + + correct_paths.append(os.path.relpath(path, path_to_dataset_root)) + + return correct_paths + + +def align_sat_unsat_sizes_with_upsamping(sat_data, unsat_data): + sat_cnt = len(sat_data) + unsat_cnt = len(unsat_data) + + sat_indices = list(range(sat_cnt)) + unsat_indices = list(range(unsat_cnt)) + + if sat_cnt < unsat_cnt: + sat_indices += list(np.random.choice(np.array(sat_indices), unsat_cnt - sat_cnt, replace=True)) + elif sat_cnt > unsat_cnt: + unsat_indices += list(np.random.choice(np.array(unsat_indices), sat_cnt - unsat_cnt, replace=False)) + + return ( + list(np.array(sat_data, dtype=object)[sat_indices]), + list(np.array(unsat_data, dtype=object)[unsat_indices]) + ) - if len(sat_indices) > len(unsat_indices): - sat_indices = np.random.choice(np.array(sat_indices), len(unsat_indices), replace=False) - elif len(sat_indices) < len(unsat_indices): - unsat_indices = np.random.choice(np.array(unsat_indices), len(sat_indices), replace=False) + +def align_sat_unsat_sizes_with_downsamping(sat_data, unsat_data): + sat_cnt = len(sat_data) + unsat_cnt = len(unsat_data) + + sat_indices = list(range(sat_cnt)) + unsat_indices = list(range(unsat_cnt)) + + if sat_cnt > unsat_cnt: + sat_indices = np.random.choice(np.array(sat_indices), unsat_cnt, replace=False) + elif sat_cnt < unsat_cnt: + unsat_indices = np.random.choice(np.array(unsat_indices), sat_cnt, replace=False) return ( list(np.array(sat_data, dtype=object)[sat_indices]), list(np.array(unsat_data, dtype=object)[unsat_indices]) ) + + +def align_sat_unsat_sizes(sat_data, unsat_data, mode): + if mode == "none": + return sat_data, unsat_data + elif mode == "upsample": + return align_sat_unsat_sizes_with_upsamping(sat_data, unsat_data) + elif mode == "downsample": + return align_sat_unsat_sizes_with_downsamping(sat_data, unsat_data) + else: + raise Exception(f"unknown sampling mode {mode}") diff --git a/ksmt-neurosmt/src/main/python/validate.py b/ksmt-neurosmt/src/main/python/validate.py old mode 100644 new mode 100755 diff --git a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/ksmtBinaryConverter/KSMTBinaryConverter.kt b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/ksmtBinaryConverter/KSMTBinaryConverter.kt index 2f59f0704..b268798b1 100644 --- a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/ksmtBinaryConverter/KSMTBinaryConverter.kt +++ b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/ksmtBinaryConverter/KSMTBinaryConverter.kt @@ -12,6 +12,7 @@ import java.io.FileOutputStream import java.nio.file.Files import java.nio.file.Path import kotlin.io.path.isRegularFile +import kotlin.io.path.name import kotlin.time.Duration.Companion.seconds fun main(args: Array) { @@ -21,12 +22,23 @@ fun main(args: Array) { val files = Files.walk(Path.of(inputRoot)).filter { it.isRegularFile() } + File(outputRoot).mkdirs() + var sat = 0; var unsat = 0; var skipped = 0 val ctx = KContext(simplificationMode = KContext.SimplificationMode.NO_SIMPLIFY) var curIdx = 0 ProgressBar.wrap(files, "converting ksmt binary files").forEach { + val relFile = it.toFile().relativeTo(File(inputRoot)) + val parentDirFile = if (relFile.parentFile == null) { + "." + } else { + relFile.parentFile.path + } + val outputDir = File(outputRoot, parentDirFile) + outputDir.mkdirs() + val assertList = try { deserialize(ctx, FileInputStream(it.toFile())) } catch (e: Exception) { @@ -34,7 +46,11 @@ fun main(args: Array) { return@forEach } - val answer = getAnswerForTest(ctx, assertList, timeout) + val answer = when { + it.name.endsWith("-sat") -> KSolverStatus.SAT + it.name.endsWith("-unsat") -> KSolverStatus.UNSAT + else -> getAnswerForTest(ctx, assertList, timeout) + } if (answer == KSolverStatus.UNKNOWN) { skipped++ @@ -55,7 +71,7 @@ fun main(args: Array) { } } - val outputFile = File("$outputRoot/$curIdx-${answer.toString().lowercase()}") + val outputFile = File("$outputDir/$curIdx-${answer.toString().lowercase()}") val outputStream = FileOutputStream(outputFile) outputStream.write("; $it\n".encodeToByteArray()) From 7893f94c4feb70c587eb4e55683598d44f83aac4 Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Thu, 10 Aug 2023 15:40:04 +0300 Subject: [PATCH 32/52] commit for main split file --- .../src/main/python/prepare-train-val-test.py | 202 +++++++++++++++--- 1 file changed, 172 insertions(+), 30 deletions(-) diff --git a/ksmt-neurosmt/src/main/python/prepare-train-val-test.py b/ksmt-neurosmt/src/main/python/prepare-train-val-test.py index 3c73b343c..3fe332526 100755 --- a/ksmt-neurosmt/src/main/python/prepare-train-val-test.py +++ b/ksmt-neurosmt/src/main/python/prepare-train-val-test.py @@ -4,21 +4,20 @@ from argparse import ArgumentParser import numpy as np -from tqdm import tqdm +from tqdm import trange from pytorch_lightning import seed_everything -from GraphReader import read_graph_by_path -from GraphDataloader import MAX_FORMULA_SIZE, MAX_FORMULA_DEPTH, METADATA_PATH -from utils import train_val_test_indices, align_sat_unsat_sizes +from GraphDataloader import METADATA_PATH +from utils import train_val_test_indices, align_sat_unsat_sizes, select_paths_with_suitable_samples_and_transform_to_paths_from_root -SHRINK = 10 ** (int(os.environ["SHRINK"]) if "SHRINK" in os.environ else 10) +SHRINK = 10 ** (int(os.environ["SHRINK"]) if "SHRINK" in os.environ else 20) -def create_split(path_to_dataset): +def classic_random_split(path_to_dataset_root, val_qty, test_qty, align_train_mode, align_val_mode, align_test_mode): sat_paths, unsat_paths = [], [] - for root, dirs, files in os.walk(path_to_dataset, topdown=True): + for root, dirs, files in os.walk(path_to_dataset_root, topdown=True): if METADATA_PATH in dirs: dirs.remove(METADATA_PATH) @@ -38,34 +37,134 @@ def create_split(path_to_dataset): if len(unsat_paths) > SHRINK: unsat_paths = unsat_paths[:SHRINK] - def process_paths(paths): - correct_paths = [] - for path in tqdm(paths): - operators, edges, depth = read_graph_by_path(path, max_size=MAX_FORMULA_SIZE, max_depth=MAX_FORMULA_DEPTH) + sat_paths = select_paths_with_suitable_samples_and_transform_to_paths_from_root(path_to_dataset_root, sat_paths) + unsat_paths = select_paths_with_suitable_samples_and_transform_to_paths_from_root(path_to_dataset_root, unsat_paths) - if depth is None: - continue + def split_data_to_train_val_test(data): + train_ind, val_ind, test_ind = train_val_test_indices(len(data), val_qty=val_qty, test_qty=test_qty) - if len(edges) == 0: - print(f"w: ignoring formula without edges; file '{path}'") - continue + return [data[i] for i in train_ind], [data[i] for i in val_ind], [data[i] for i in test_ind] + + sat_train, sat_val, sat_test = split_data_to_train_val_test(sat_paths) + unsat_train, unsat_val, unsat_test = split_data_to_train_val_test(unsat_paths) + + sat_train, unsat_train = align_sat_unsat_sizes(sat_train, unsat_train, align_train_mode) + sat_val, unsat_val = align_sat_unsat_sizes(sat_val, unsat_val, align_val_mode) + sat_test, unsat_test = align_sat_unsat_sizes(sat_test, unsat_test, align_test_mode) + + train_data = sat_train + unsat_train + val_data = sat_val + unsat_val + test_data = sat_test + unsat_test + + np.random.shuffle(train_data) + np.random.shuffle(val_data) + np.random.shuffle(test_data) + + return train_data, val_data, test_data + + +def grouped_random_split(path_to_dataset_root, val_qty, test_qty, align_train_mode, align_val_mode, align_test_mode): + def return_group_weight(path_to_group): + return len(os.listdir(path_to_group)) + + def calc_group_weights(path_to_dataset_root): + groups = os.listdir(path_to_dataset_root) + groups.remove(METADATA_PATH) + + weights = [return_group_weight(os.path.join(path_to_dataset_root, group)) for group in groups] + + return list(zip(groups, weights)) - correct_paths.append(os.path.relpath(path, path_to_dataset)) + groups = calc_group_weights(path_to_dataset_root) - return correct_paths + def pick_best_split(groups): + attempts = 100_000 - sat_paths = process_paths(sat_paths) - unsat_paths = process_paths(unsat_paths) + groups_cnt = len(groups) + samples_cnt = sum(g[1] for g in groups) - sat_paths, unsat_paths = align_sat_unsat_sizes(sat_paths, unsat_paths) + need_val = int(samples_cnt * val_qty) + need_test = int(samples_cnt * test_qty) + need_train = samples_cnt - need_val - need_test - def split_data(data): - train_ind, val_ind, test_ind = train_val_test_indices(len(data)) + best = None + + for _ in trange(attempts): + split = np.random.randint(3, size=groups_cnt) + + train_size = sum(groups[i][1] for i in range(groups_cnt) if split[i] == 0) + val_size = sum(groups[i][1] for i in range(groups_cnt) if split[i] == 1) + test_size = sum(groups[i][1] for i in range(groups_cnt) if split[i] == 2) + + cur_error = (train_size - need_train) ** 2 + (val_size - need_val) ** 2 + (test_size - need_test) ** 2 + + if best is None or best[0] > cur_error: + best = (cur_error, split) + + return best[1] + + split = pick_best_split(groups) + + train_data, val_data, test_data = [], [], [] + + for i in range(len(groups)): + cur_group = os.listdir(os.path.join(path_to_dataset_root, groups[i][0])) + cur_group = list(map(lambda sample: os.path.join(path_to_dataset_root, groups[i][0], sample), cur_group)) + + if split[i] == 0: + train_data += cur_group + elif split[i] == 1: + val_data += cur_group + elif split[i] == 2: + test_data += cur_group + + train_data = select_paths_with_suitable_samples_and_transform_to_paths_from_root(path_to_dataset_root, train_data) + val_data = select_paths_with_suitable_samples_and_transform_to_paths_from_root(path_to_dataset_root, val_data) + test_data = select_paths_with_suitable_samples_and_transform_to_paths_from_root(path_to_dataset_root, test_data) + + np.random.shuffle(train_data) + np.random.shuffle(val_data) + np.random.shuffle(test_data) + + return train_data, val_data, test_data + + """ + + sat_paths, unsat_paths = [], [] + for root, dirs, files in os.walk(path_to_dataset_root, topdown=True): + if METADATA_PATH in dirs: + dirs.remove(METADATA_PATH) + + for file_name in files: + cur_path = os.path.join(root, file_name) + + if cur_path.endswith("-sat"): + sat_paths.append(cur_path) + elif cur_path.endswith("-unsat"): + unsat_paths.append(cur_path) + else: + raise Exception(f"strange file path '{cur_path}'") + + if len(sat_paths) > SHRINK: + sat_paths = sat_paths[:SHRINK] + + if len(unsat_paths) > SHRINK: + unsat_paths = unsat_paths[:SHRINK] + + sat_paths = select_paths_with_suitable_samples_and_transform_to_paths_from_root(path_to_dataset_root, sat_paths) + unsat_paths = select_paths_with_suitable_samples_and_transform_to_paths_from_root(path_to_dataset_root, unsat_paths) + + def split_data_to_train_val_test(data): + train_ind, val_ind, test_ind = train_val_test_indices(len(data), val_qty=val_qty, test_qty=test_qty) return [data[i] for i in train_ind], [data[i] for i in val_ind], [data[i] for i in test_ind] - sat_train, sat_val, sat_test = split_data(sat_paths) - unsat_train, unsat_val, unsat_test = split_data(unsat_paths) + sat_train, sat_val, sat_test = split_data_to_train_val_test(sat_paths) + unsat_train, unsat_val, unsat_test = split_data_to_train_val_test(unsat_paths) + + sat_train, unsat_train = align_sat_unsat_sizes(sat_train, unsat_train, align_train_mode) + sat_val, unsat_val = align_sat_unsat_sizes(sat_val, unsat_val, align_val_mode) + sat_test, unsat_test = align_sat_unsat_sizes(sat_test, unsat_test, align_test_mode) train_data = sat_train + unsat_train val_data = sat_val + unsat_val @@ -75,29 +174,67 @@ def split_data(data): np.random.shuffle(val_data) np.random.shuffle(test_data) + return train_data, val_data, test_data + """ + + +def create_split(path_to_dataset_root, val_qty, test_qty, align_train_mode, align_val_mode, align_test_mode, grouped): + if grouped: + train_data, val_data, test_data = grouped_random_split( + path_to_dataset_root, + val_qty, test_qty, + align_train_mode, align_val_mode, align_test_mode + ) + + else: + train_data, val_data, test_data = classic_random_split( + path_to_dataset_root, + val_qty, test_qty, + align_train_mode, align_val_mode, align_test_mode + ) + print("\nstats:", flush=True) print(f"train: {len(train_data)}") print(f"val: {len(val_data)}") print(f"test: {len(test_data)}") print(flush=True) - meta_path = os.path.join(path_to_dataset, METADATA_PATH) + meta_path = os.path.join(path_to_dataset_root, METADATA_PATH) os.makedirs(meta_path, exist_ok=True) with open(os.path.join(meta_path, "train"), "w") as f: - f.write("\n".join(train_data) + "\n") + f.write("\n".join(train_data)) + + if len(train_data): + f.write("\n") with open(os.path.join(meta_path, "val"), "w") as f: - f.write("\n".join(val_data) + "\n") + f.write("\n".join(val_data)) + + if len(val_data): + f.write("\n") with open(os.path.join(meta_path, "test"), "w") as f: - f.write("\n".join(test_data) + "\n") + f.write("\n".join(test_data)) + + if len(test_data): + f.write("\n") def get_args(): parser = ArgumentParser(description="train/val/test splitting script") + parser.add_argument("--ds", required=True) + parser.add_argument("--val_qty", type=float, default=0.15) + parser.add_argument("--test_qty", type=float, default=0.1) + + parser.add_argument("--align_train", choices=["none", "upsample", "downsample"], default="upsample") + parser.add_argument("--align_val", choices=["none", "upsample", "downsample"], default="none") + parser.add_argument("--align_test", choices=["none", "upsample", "downsample"], default="none") + + parser.add_argument("--grouped", action="store_true") + args = parser.parse_args() print("args:") for arg in vars(args): @@ -113,4 +250,9 @@ def get_args(): args = get_args() - create_split(args.ds) + create_split( + args.ds, + args.val_qty, args.test_qty, + args.align_train, args.align_val, args.align_test, + args.grouped + ) From 40a483a24b90457769138cabb90269c03c01082e Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Thu, 10 Aug 2023 16:35:10 +0300 Subject: [PATCH 33/52] small refactor --- .../src/main/python/GraphDataloader.py | 2 +- .../src/main/python/prepare-train-val-test.py | 78 +++++-------------- 2 files changed, 22 insertions(+), 58 deletions(-) diff --git a/ksmt-neurosmt/src/main/python/GraphDataloader.py b/ksmt-neurosmt/src/main/python/GraphDataloader.py index 7e1a1a35e..850ddcae8 100644 --- a/ksmt-neurosmt/src/main/python/GraphDataloader.py +++ b/ksmt-neurosmt/src/main/python/GraphDataloader.py @@ -13,7 +13,7 @@ from GraphReader import read_graph_by_path -BATCH_SIZE = 32 +BATCH_SIZE = 1024 MAX_FORMULA_SIZE = 10000 MAX_FORMULA_DEPTH = 2500 NUM_WORKERS = 32 diff --git a/ksmt-neurosmt/src/main/python/prepare-train-val-test.py b/ksmt-neurosmt/src/main/python/prepare-train-val-test.py index 3fe332526..b9d5eb0b5 100755 --- a/ksmt-neurosmt/src/main/python/prepare-train-val-test.py +++ b/ksmt-neurosmt/src/main/python/prepare-train-val-test.py @@ -12,9 +12,6 @@ from utils import train_val_test_indices, align_sat_unsat_sizes, select_paths_with_suitable_samples_and_transform_to_paths_from_root -SHRINK = 10 ** (int(os.environ["SHRINK"]) if "SHRINK" in os.environ else 20) - - def classic_random_split(path_to_dataset_root, val_qty, test_qty, align_train_mode, align_val_mode, align_test_mode): sat_paths, unsat_paths = [], [] for root, dirs, files in os.walk(path_to_dataset_root, topdown=True): @@ -31,12 +28,6 @@ def classic_random_split(path_to_dataset_root, val_qty, test_qty, align_train_mo else: raise Exception(f"strange file path '{cur_path}'") - if len(sat_paths) > SHRINK: - sat_paths = sat_paths[:SHRINK] - - if len(unsat_paths) > SHRINK: - unsat_paths = unsat_paths[:SHRINK] - sat_paths = select_paths_with_suitable_samples_and_transform_to_paths_from_root(path_to_dataset_root, sat_paths) unsat_paths = select_paths_with_suitable_samples_and_transform_to_paths_from_root(path_to_dataset_root, unsat_paths) @@ -69,7 +60,8 @@ def return_group_weight(path_to_group): def calc_group_weights(path_to_dataset_root): groups = os.listdir(path_to_dataset_root) - groups.remove(METADATA_PATH) + if METADATA_PATH in groups: + groups.remove(METADATA_PATH) weights = [return_group_weight(os.path.join(path_to_dataset_root, group)) for group in groups] @@ -90,16 +82,16 @@ def pick_best_split(groups): best = None for _ in trange(attempts): - split = np.random.randint(3, size=groups_cnt) + cur_split = np.random.randint(3, size=groups_cnt) - train_size = sum(groups[i][1] for i in range(groups_cnt) if split[i] == 0) - val_size = sum(groups[i][1] for i in range(groups_cnt) if split[i] == 1) - test_size = sum(groups[i][1] for i in range(groups_cnt) if split[i] == 2) + train_size = sum(groups[i][1] for i in range(groups_cnt) if cur_split[i] == 0) + val_size = sum(groups[i][1] for i in range(groups_cnt) if cur_split[i] == 1) + test_size = sum(groups[i][1] for i in range(groups_cnt) if cur_split[i] == 2) cur_error = (train_size - need_train) ** 2 + (val_size - need_val) ** 2 + (test_size - need_test) ** 2 if best is None or best[0] > cur_error: - best = (cur_error, split) + best = (cur_error, cur_split) return best[1] @@ -122,60 +114,32 @@ def pick_best_split(groups): val_data = select_paths_with_suitable_samples_and_transform_to_paths_from_root(path_to_dataset_root, val_data) test_data = select_paths_with_suitable_samples_and_transform_to_paths_from_root(path_to_dataset_root, test_data) - np.random.shuffle(train_data) - np.random.shuffle(val_data) - np.random.shuffle(test_data) - - return train_data, val_data, test_data - - """ - - sat_paths, unsat_paths = [], [] - for root, dirs, files in os.walk(path_to_dataset_root, topdown=True): - if METADATA_PATH in dirs: - dirs.remove(METADATA_PATH) - - for file_name in files: - cur_path = os.path.join(root, file_name) - - if cur_path.endswith("-sat"): - sat_paths.append(cur_path) - elif cur_path.endswith("-unsat"): - unsat_paths.append(cur_path) - else: - raise Exception(f"strange file path '{cur_path}'") - - if len(sat_paths) > SHRINK: - sat_paths = sat_paths[:SHRINK] + def split_data_to_sat_unsat(data): + sat_data = list(filter(lambda path: path.endswith("-sat"), data)) + unsat_data = list(filter(lambda path: path.endswith("-unsat"), data)) - if len(unsat_paths) > SHRINK: - unsat_paths = unsat_paths[:SHRINK] + return sat_data, unsat_data - sat_paths = select_paths_with_suitable_samples_and_transform_to_paths_from_root(path_to_dataset_root, sat_paths) - unsat_paths = select_paths_with_suitable_samples_and_transform_to_paths_from_root(path_to_dataset_root, unsat_paths) - - def split_data_to_train_val_test(data): - train_ind, val_ind, test_ind = train_val_test_indices(len(data), val_qty=val_qty, test_qty=test_qty) + def align_data(data, mode): + sat_data, unsat_data = split_data_to_sat_unsat(data) + sat_data, unsat_data = align_sat_unsat_sizes(sat_data, unsat_data, mode) - return [data[i] for i in train_ind], [data[i] for i in val_ind], [data[i] for i in test_ind] + return sat_data + unsat_data - sat_train, sat_val, sat_test = split_data_to_train_val_test(sat_paths) - unsat_train, unsat_val, unsat_test = split_data_to_train_val_test(unsat_paths) + if align_train_mode != "none": + train_data = align_data(train_data, align_train_mode) - sat_train, unsat_train = align_sat_unsat_sizes(sat_train, unsat_train, align_train_mode) - sat_val, unsat_val = align_sat_unsat_sizes(sat_val, unsat_val, align_val_mode) - sat_test, unsat_test = align_sat_unsat_sizes(sat_test, unsat_test, align_test_mode) + if align_val_mode != "none": + val_data = align_data(val_data, align_val_mode) - train_data = sat_train + unsat_train - val_data = sat_val + unsat_val - test_data = sat_test + unsat_test + if align_test_mode != "none": + test_data = align_data(test_data, align_test_mode) np.random.shuffle(train_data) np.random.shuffle(val_data) np.random.shuffle(test_data) return train_data, val_data, test_data - """ def create_split(path_to_dataset_root, val_qty, test_qty, align_train_mode, align_val_mode, align_test_mode, grouped): From da66e8f23ae89a91e3071934471b0acc174ce86f Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Thu, 10 Aug 2023 17:41:39 +0300 Subject: [PATCH 34/52] add confusion matrix for val and test --- .../src/main/python/LightningModel.py | 36 ++++++++++++++----- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/ksmt-neurosmt/src/main/python/LightningModel.py b/ksmt-neurosmt/src/main/python/LightningModel.py index 09fc4075d..83d79ee53 100644 --- a/ksmt-neurosmt/src/main/python/LightningModel.py +++ b/ksmt-neurosmt/src/main/python/LightningModel.py @@ -5,7 +5,7 @@ import pytorch_lightning as pl -from torchmetrics.classification import BinaryAccuracy, BinaryAUROC +from torchmetrics.classification import BinaryAccuracy, BinaryAUROC, BinaryConfusionMatrix from Model import Model @@ -21,6 +21,7 @@ def __init__(self): self.acc = BinaryAccuracy() self.roc_auc = BinaryAUROC() + self.confusion_matrix = BinaryConfusionMatrix() def forward(self, x): return self.model(x) @@ -79,8 +80,28 @@ def shared_val_test_step(self, batch, batch_idx, metric_name): def validation_step(self, val_batch, batch_idx): return self.shared_val_test_step(val_batch, batch_idx, "val") + def print_confusion_matrix_and_classification_report(self, all_outputs, all_targets): + conf_mat = self.confusion_matrix(all_outputs, all_targets).detach().cpu().numpy() + + print(" +-------+-----------+-----------+") + print(" ", "|", "unsat", "|", str(conf_mat[0][0]).rjust(9, " "), "|", str(conf_mat[0][1]).rjust(9, " "), "|") + print("targets", "|", " sat", "|", str(conf_mat[1][0]).rjust(9, " "), "|", str(conf_mat[1][1]).rjust(9, " "), "|") + print(" +-------+-----------+-----------+") + print(" ", "|", " ", "|", " unsat ", "|", " sat ", "|") + print(" +-------+-----------+-----------+") + print(" preds", "\n", sep="") + + all_outputs = all_outputs.float().cpu().numpy() + all_targets = all_targets.float().cpu().numpy() + + all_outputs = all_outputs > 0.5 + print( + classification_report(all_targets, all_outputs, target_names=["unsat", "sat"], digits=3, zero_division=0.0), + flush=True + ) + def on_validation_epoch_end(self): - print("\n\n", flush=True) + print("\n", flush=True) all_outputs = torch.flatten(torch.cat(self.val_outputs)) all_targets = torch.flatten(torch.cat(self.val_targets)) @@ -94,13 +115,7 @@ def on_validation_epoch_end(self): on_step=False, on_epoch=True ) - all_outputs = all_outputs.float().cpu().numpy() - all_targets = all_targets.float().cpu().numpy() - - all_outputs = all_outputs > 0.5 - print(classification_report(all_targets, all_outputs, digits=3, zero_division=0.0)) - - print("\n", flush=True) + self.print_confusion_matrix_and_classification_report(all_outputs, all_targets) def test_step(self, test_batch, batch_idx): return self.shared_val_test_step(test_batch, batch_idx, "test") @@ -117,3 +132,6 @@ def on_test_epoch_end(self): prog_bar=True, logger=True, on_step=False, on_epoch=True ) + + print("\n") + self.print_confusion_matrix_and_classification_report(all_outputs, all_targets) From 24b933a3d76c426a7e6821a4851722dc2033e562 Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Fri, 11 Aug 2023 14:47:04 +0300 Subject: [PATCH 35/52] modify model input to save in onnx format --- ksmt-neurosmt/src/main/python/Decoder.py | 5 +- ksmt-neurosmt/src/main/python/Encoder.py | 17 +- .../src/main/python/LightningModel.py | 14 +- ksmt-neurosmt/src/main/python/Model.py | 4 +- .../src/main/python/create-ordinal-encoder.py | 4 + ksmt-neurosmt/src/main/python/sandbox.py | 161 ++++++++++++++++++ 6 files changed, 190 insertions(+), 15 deletions(-) create mode 100755 ksmt-neurosmt/src/main/python/sandbox.py diff --git a/ksmt-neurosmt/src/main/python/Decoder.py b/ksmt-neurosmt/src/main/python/Decoder.py index 39d776cdc..a3740ccfe 100644 --- a/ksmt-neurosmt/src/main/python/Decoder.py +++ b/ksmt-neurosmt/src/main/python/Decoder.py @@ -1,12 +1,15 @@ import torch.nn as nn +DECODER_LAYERS = 4 + + class Decoder(nn.Module): def __init__(self, hidden_dim): super().__init__() self.act = nn.ReLU() - self.linears = nn.ModuleList([nn.Linear(hidden_dim, hidden_dim) for _ in range(3)]) + self.linears = nn.ModuleList([nn.Linear(hidden_dim, hidden_dim) for _ in range(DECODER_LAYERS - 1)]) self.out = nn.Linear(hidden_dim, 1) def forward(self, x): diff --git a/ksmt-neurosmt/src/main/python/Encoder.py b/ksmt-neurosmt/src/main/python/Encoder.py index 938a51486..b98bd4f8e 100644 --- a/ksmt-neurosmt/src/main/python/Encoder.py +++ b/ksmt-neurosmt/src/main/python/Encoder.py @@ -16,16 +16,17 @@ def __init__(self, hidden_dim): #self.conv = GATConv(64, 64, add_self_loops=False) - def forward(self, data): - x, edge_index, depth = data.x, data.edge_index, data.depth - #edge_index = torch.tensor([[0, 0], [1, 2]], dtype=torch.long).to(edge_index.get_device()) + def forward(self, node_labels, edges, depths, root_ptrs): + # x, edge_index, depth = data.x, data.edge_index, data.depth + # edge_index = torch.tensor([[0, 0], [1, 2]], dtype=torch.long).to(edge_index.get_device()) - x = self.embedding(x.squeeze()) + node_features = self.embedding(node_labels.squeeze()) - depth = depth.max() + depth = depths.max() + node_features += depth * 0.0 for i in range(depth): - x = self.conv(x, edge_index) + node_features = self.conv(node_features, edges) - x = x[data.ptr[1:] - 1] + node_features = node_features[root_ptrs[1:] - 1] - return x + return node_features diff --git a/ksmt-neurosmt/src/main/python/LightningModel.py b/ksmt-neurosmt/src/main/python/LightningModel.py index 83d79ee53..f4eccc0c1 100644 --- a/ksmt-neurosmt/src/main/python/LightningModel.py +++ b/ksmt-neurosmt/src/main/python/LightningModel.py @@ -10,6 +10,12 @@ from Model import Model +def unpack_batch(batch): + node_labels, edges, depths, root_ptrs = batch.x, batch.edge_index, batch.depth, batch.ptr + + return node_labels, edges, depths, root_ptrs + + class LightningModel(pl.LightningModule): def __init__(self): super().__init__() @@ -23,8 +29,8 @@ def __init__(self): self.roc_auc = BinaryAUROC() self.confusion_matrix = BinaryConfusionMatrix() - def forward(self, x): - return self.model(x) + def forward(self, node_labels, edges, depths, root_ptrs): + return self.model(node_labels, edges, depths, root_ptrs) def configure_optimizers(self): params = [p for p in self.model.parameters() if p is not None and p.requires_grad] @@ -33,7 +39,7 @@ def configure_optimizers(self): return optimizer def training_step(self, train_batch, batch_idx): - out = self.model(train_batch) + out = self.model(*unpack_batch(train_batch)) loss = F.binary_cross_entropy_with_logits(out, train_batch.y) out = F.sigmoid(out) @@ -54,7 +60,7 @@ def training_step(self, train_batch, batch_idx): return loss def shared_val_test_step(self, batch, batch_idx, metric_name): - out = self.model(batch) + out = self.model(*unpack_batch(batch)) loss = F.binary_cross_entropy_with_logits(out, batch.y) out = F.sigmoid(out) diff --git a/ksmt-neurosmt/src/main/python/Model.py b/ksmt-neurosmt/src/main/python/Model.py index 758bf277f..bdf82781e 100644 --- a/ksmt-neurosmt/src/main/python/Model.py +++ b/ksmt-neurosmt/src/main/python/Model.py @@ -13,8 +13,8 @@ def __init__(self): self.encoder = Encoder(hidden_dim=EMBEDDING_DIM) self.decoder = Decoder(hidden_dim=EMBEDDING_DIM) - def forward(self, x): - x = self.encoder(x) + def forward(self, node_labels, edges, depths, root_ptrs): + x = self.encoder(node_labels, edges, depths, root_ptrs) x = self.decoder(x) return x diff --git a/ksmt-neurosmt/src/main/python/create-ordinal-encoder.py b/ksmt-neurosmt/src/main/python/create-ordinal-encoder.py index a84424ac5..0830b2ea9 100755 --- a/ksmt-neurosmt/src/main/python/create-ordinal-encoder.py +++ b/ksmt-neurosmt/src/main/python/create-ordinal-encoder.py @@ -28,6 +28,10 @@ def create_ordinal_encoder(paths_to_datasets, path_to_ordinal_encoder): print("dumping ordinal encoder") joblib.dump(encoder, path_to_ordinal_encoder) + with open(path_to_ordinal_encoder + ".cats", "w") as f: + for sample in encoder.categories_[0]: + f.write(str(sample) + "\n") + def get_args(): parser = ArgumentParser(description="ordinal encoder preparing script") diff --git a/ksmt-neurosmt/src/main/python/sandbox.py b/ksmt-neurosmt/src/main/python/sandbox.py new file mode 100755 index 000000000..f8e430320 --- /dev/null +++ b/ksmt-neurosmt/src/main/python/sandbox.py @@ -0,0 +1,161 @@ +#!/usr/bin/python3 + +import sys +# import os; os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"; os.environ["CUDA_VISIBLE_DEVICES"] = os.environ["GPU"] + +import torch + +from pytorch_lightning import Trainer +from pytorch_lightning.loggers import TensorBoardLogger +from pytorch_lightning.callbacks import ModelCheckpoint + +from GraphDataloader import get_dataloader + +from LightningModel import LightningModel + +from Model import EMBEDDING_DIM + + +if __name__ == "__main__": + + pl_model = LightningModel().load_from_checkpoint(sys.argv[1], map_location=torch.device("cpu")) + + """ + trainer = Trainer( + accelerator="auto", + # precision="bf16-mixed", + logger=TensorBoardLogger("../logs", name="neuro-smt"), + callbacks=[ModelCheckpoint( + filename="epoch_{epoch:03d}_roc-auc_{val/roc-auc:.3f}", + monitor="val/roc-auc", + verbose=True, + save_last=True, save_top_k=3, mode="max", + auto_insert_metric_name=False, save_on_train_epoch_end=False + )], + max_epochs=200, + log_every_n_steps=1, + enable_checkpointing=True, + barebones=False, + default_root_dir=".." + ) + """ + + MAX_SIZE = 3 + + node_labels = torch.tensor([[i] for i in range(MAX_SIZE)], dtype=torch.int32) + edges = torch.tensor([ + [i for i in range(MAX_SIZE - 1)], + [i for i in range(1, MAX_SIZE)] + ], dtype=torch.int64) + depths = torch.tensor([MAX_SIZE - 1], dtype=torch.int64) + root_ptrs = torch.tensor([0, MAX_SIZE], dtype=torch.int64) + + print(node_labels.shape, edges.shape, depths.shape, root_ptrs.shape) + + torch.onnx.export( + pl_model.model.encoder.embedding, + (node_labels,), + "embeddings.onnx", + opset_version=18, + input_names=["node_labels"], + output_names=["out"], + dynamic_axes={ + "node_labels": {0: "nodes_number"} + } + ) + + torch.onnx.export( + pl_model.model.encoder.conv, + (torch.rand((node_labels.shape[0], EMBEDDING_DIM)), edges), + "conv.onnx", + opset_version=18, + input_names=["node_features", "edges"], + output_names=["out"], + dynamic_axes={ + "node_features": {0: "nodes_number"}, + "edges": {1: "edges_number"} + } + ) + + torch.onnx.export( + pl_model.model.decoder, + (torch.rand((1, EMBEDDING_DIM)),), + "decoder.onnx", + opset_version=18, + input_names=["expr_features"], + output_names=["out"], + dynamic_axes={ + "expr_features": {0: "batch_size"} + } + ) + + """ + pl_model.to_onnx( + "kek.onnx", + (node_labels, edges, depths, root_ptrs), + opset_version=18, + input_names=["node_labels", "edges", "depths", "root_ptrs"], + output_names=["output"], + dynamic_axes={ + "node_labels": {0: "nodes_number"}, + "edges": {1: "edges_number"}, + "depths": {0: "batch_size"}, + "root_ptrs": {0: "batch_size_+_1"}, + }, + # verbose=True + ) + """ + + """ + torch.onnx.export(torch_model, + x, + '../Game_env/gnn_model.onnx', + opset_version=15, + export_params=True, + input_names = ['x', 'edge_index'], # the model's input names + output_names = ['output'], + dynamic_axes={'x' : {0 : 'nodes_number'}, # variable length axes + 'edge_index' : {1 : 'egdes_number'}, + }, + ) + """ + + """ + x = torch.randn(*shape, requires_grad=True).to(device) + torch_model = actor_model.eval() + torch_out = torch_model(x) + torch.onnx.export(torch_model, + x, + '../Game_env/actor_model.onnx', + opset_version=15, + export_params=True, + input_names = ['input'], # the model's input names + output_names = ['output'], + dynamic_axes={'input' : {0 : 'batch_size', + 1 : 'n_actions', + }, # variable length axes + 'output' : {0 : 'batch_size', + 1 : 'n_actions', + }, + }, + ) + algo_name = 'NN' + + if actor_model is not None and gnn_model is not None and self.use_gnn: + x_shape = [1, self.gnn_in_nfeatures] + edge_index_shape = [2, 1] + x = (torch.randn(*x_shape, requires_grad=True).to(device), torch.randint(0, 1, edge_index_shape).to(device)) + torch_model = gnn_model.eval() + torch_out = torch_model(*x) + torch.onnx.export(torch_model, + x, + '../Game_env/gnn_model.onnx', + opset_version=15, + export_params=True, + input_names = ['x', 'edge_index'], # the model's input names + output_names = ['output'], + dynamic_axes={'x' : {0 : 'nodes_number'}, # variable length axes + 'edge_index' : {1 : 'egdes_number'}, + }, + ) + """ From 7cb1af4ee3d340f28a2ade9e95d87d6033152057 Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Thu, 17 Aug 2023 12:23:29 +0300 Subject: [PATCH 36/52] add kotlin runtime for model --- ksmt-neurosmt/build.gradle.kts | 4 +- .../solver/neurosmt/runtime/ExprEncoder.kt | 118 ++++++++++++++ .../ksmt/solver/neurosmt/runtime/ONNXModel.kt | 61 ++++++++ sandbox/build.gradle.kts | 3 + sandbox/src/main/kotlin/Main.kt | 146 +++++++++++++++++- settings.gradle.kts | 2 +- 6 files changed, 325 insertions(+), 9 deletions(-) create mode 100644 ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ExprEncoder.kt create mode 100644 ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ONNXModel.kt diff --git a/ksmt-neurosmt/build.gradle.kts b/ksmt-neurosmt/build.gradle.kts index fe876a2e8..b2bc82195 100644 --- a/ksmt-neurosmt/build.gradle.kts +++ b/ksmt-neurosmt/build.gradle.kts @@ -1,5 +1,5 @@ plugins { - id("io.ksmt.ksmt-base") + kotlin("jvm") // id("io.ksmt.ksmt-base") -- need to be returned in future } repositories { @@ -9,4 +9,6 @@ repositories { dependencies { implementation(project(":ksmt-core")) // implementation(project(":ksmt-z3")) + + implementation("com.microsoft.onnxruntime:onnxruntime:1.15.1") } \ No newline at end of file diff --git a/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ExprEncoder.kt b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ExprEncoder.kt new file mode 100644 index 000000000..cdd794ffd --- /dev/null +++ b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ExprEncoder.kt @@ -0,0 +1,118 @@ +package io.ksmt.solver.neurosmt.runtime + +import ai.onnxruntime.OnnxTensor +import ai.onnxruntime.OrtEnvironment +import io.ksmt.KContext +import io.ksmt.expr.KApp +import io.ksmt.expr.KConst +import io.ksmt.expr.KExpr +import io.ksmt.expr.KInterpretedValue +import io.ksmt.expr.transformer.KNonRecursiveTransformer +import io.ksmt.sort.* +import java.nio.FloatBuffer +import java.nio.IntBuffer +import java.nio.LongBuffer +import java.util.* + +class ExprEncoder( + override val ctx: KContext, + val env: OrtEnvironment, + val ordinalEncoder: OrdinalEncoder, + val embeddingLayer: ONNXModel, + val convLayer: ONNXModel +) : KNonRecursiveTransformer(ctx) { + + private val exprToState = IdentityHashMap, OnnxTensor>() + + fun encodeExpr(expr: KExpr<*>): OnnxTensor { + apply(expr) + + return exprToState[expr] ?: error("expression state wasn't calculated yet") + } + + override fun transformApp(expr: KApp): KExpr { + when (expr) { + is KConst<*> -> calcSymbolicVariableState(expr) + is KInterpretedValue<*> -> calcValueState(expr) + else -> calcAppState(expr) + } + + return expr + } + + private fun getNodeEmbedding(key: String): OnnxTensor { + val nodeLabel = ordinalEncoder.getOrdinal(key) + val labelTensor = OnnxTensor.createTensor( + env, IntBuffer.allocate(1).put(nodeLabel).rewind(), longArrayOf(1, 1) + ) + + return embeddingLayer.forward(mapOf("node_labels" to labelTensor)) + } + + private fun createEdgeTensor(childrenCnt: Int): OnnxTensor { + val edges = listOf( + List(childrenCnt) { it + 1L }, + List(childrenCnt) { 0L } + ) + + val buffer = LongBuffer.allocate(childrenCnt * 2) + edges.forEach { row -> + row.forEach { node -> + buffer.put(node) + } + } + buffer.rewind() + + return OnnxTensor.createTensor(env, buffer, longArrayOf(2, childrenCnt.toLong())) + } + + private fun calcAppState(expr: KApp) { + val childrenStates = expr.args.map { exprToState[it] ?: error("expression state wasn't calculated yet") } + val childrenCnt = childrenStates.size + + val nodeEmbedding = getNodeEmbedding(expr.decl.name) + val embeddingSize = nodeEmbedding.info.shape.reduce { acc, l -> acc * l } + + val buffer = FloatBuffer.allocate((1 + childrenCnt) * embeddingSize.toInt()) + buffer.put(nodeEmbedding.floatBuffer) + childrenStates.forEach { + buffer.put(it.floatBuffer) + } + buffer.rewind() + val nodeFeatures = OnnxTensor.createTensor(env, buffer, longArrayOf(1L + childrenCnt, embeddingSize)) + + val edges = createEdgeTensor(childrenStates.size) + + val result = convLayer.forward(mapOf("node_features" to nodeFeatures, "edges" to edges)) + val newNodeFeatures = OnnxTensor.createTensor(env, result.floatBuffer.slice(0, embeddingSize.toInt()), longArrayOf(1L, embeddingSize)) + exprToState[expr] = newNodeFeatures + } + + private fun calcSymbolicVariableState(symbol: KConst) { + val key = when (symbol.decl.sort) { + is KBoolSort -> "SYMBOLIC;Bool" + is KBvSort -> "SYMBOLIC;BitVec" + is KFpSort -> "SYMBOLIC;FP" + is KFpRoundingModeSort -> "SYMBOLIC;FP_RM" + is KArraySortBase<*> -> "SYMBOLIC;Array" + is KUninterpretedSort -> "SYMBOLIC;Unint" + else -> error("unknown symbolic sort: ${symbol.decl.sort::class.simpleName}") + } + + exprToState[symbol] = getNodeEmbedding(key) + } + + private fun calcValueState(value: KInterpretedValue) { + val key = when (value.decl.sort) { + is KBoolSort -> "VALUE;Bool" + is KBvSort -> "VALUE;BitVec" + is KFpSort -> "VALUE;FP" + is KFpRoundingModeSort -> "VALUE;FP_RM" + is KArraySortBase<*> -> "VALUE;Array" + is KUninterpretedSort -> "VALUE;Unint" + else -> error("unknown value sort: ${value.decl.sort::class.simpleName}") + } + + exprToState[value] = getNodeEmbedding(key) + } +} \ No newline at end of file diff --git a/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ONNXModel.kt b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ONNXModel.kt new file mode 100644 index 000000000..8a98a05ab --- /dev/null +++ b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ONNXModel.kt @@ -0,0 +1,61 @@ +package io.ksmt.solver.neurosmt.runtime + +import ai.onnxruntime.OnnxTensor +import ai.onnxruntime.OrtEnvironment +import io.ksmt.KContext +import io.ksmt.expr.KExpr +import java.nio.file.Files +import java.nio.file.Path +import kotlin.math.exp +import kotlin.streams.asSequence + +class ONNXModel(env: OrtEnvironment, modelPath: String) : AutoCloseable { + val session = env.createSession(modelPath) + + fun forward(input: Map): OnnxTensor { + val result = session.run(input) + return result.get(0) as OnnxTensor + } + + override fun close() { + session.close() + } +} + +class NeuroSMTModelRunner( + ctx: KContext, + ordinalsPath: String, embeddingPath: String, convPath: String, decoderPath: String +) { + val env = OrtEnvironment.getEnvironment() + + val ordinalEncoder = OrdinalEncoder(ordinalsPath) + val embeddingLayer = ONNXModel(env, embeddingPath) + val convLayer = ONNXModel(env, convPath) + val encoder = ExprEncoder(ctx, env, ordinalEncoder, embeddingLayer, convLayer) + + val decoder = ONNXModel(env, decoderPath) + + fun run(expr: KExpr<*>): Float { + val exprFeatures = encoder.encodeExpr(expr) + val result = decoder.forward(mapOf("expr_features" to exprFeatures)) + val logit = result.floatBuffer[0] + + return 1f / (1f + exp(-logit)) + } +} + +const val UNKNOWN_VALUE = 1999 + +class OrdinalEncoder(ordinalsPath: String, private val unknownValue: Int = UNKNOWN_VALUE) { + private val lookup = HashMap() + + init { + Files.lines(Path.of(ordinalsPath)).asSequence().forEachIndexed { index, s -> + lookup[s] = index + } + } + + fun getOrdinal(s: String): Int { + return lookup[s] ?: unknownValue + } +} \ No newline at end of file diff --git a/sandbox/build.gradle.kts b/sandbox/build.gradle.kts index ed33bad47..88a376ea5 100644 --- a/sandbox/build.gradle.kts +++ b/sandbox/build.gradle.kts @@ -19,6 +19,9 @@ dependencies { implementation(project(":ksmt-neurosmt")) implementation(project(":ksmt-neurosmt:utils")) implementation(project(":ksmt-runner")) + + implementation("com.microsoft.onnxruntime:onnxruntime:1.15.1") + implementation("com.microsoft.onnxruntime:onnxruntime_gpu:1.15.1") } tasks.getByName("test") { diff --git a/sandbox/src/main/kotlin/Main.kt b/sandbox/src/main/kotlin/Main.kt index 8bad14cbc..7c6374ddb 100644 --- a/sandbox/src/main/kotlin/Main.kt +++ b/sandbox/src/main/kotlin/Main.kt @@ -1,25 +1,24 @@ +import ai.onnxruntime.OnnxTensor +import ai.onnxruntime.OrtEnvironment import com.jetbrains.rd.framework.SerializationCtx import com.jetbrains.rd.framework.Serializers import com.jetbrains.rd.framework.UnsafeBuffer import io.ksmt.KContext import io.ksmt.expr.* import io.ksmt.runner.serializer.AstSerializationCtx -import io.ksmt.solver.KModel import io.ksmt.solver.KSolver import io.ksmt.solver.KSolverConfiguration import io.ksmt.solver.KSolverStatus -import io.ksmt.solver.neurosmt.KNeuroSMTSolver +import io.ksmt.solver.neurosmt.runtime.NeuroSMTModelRunner import io.ksmt.solver.z3.* import io.ksmt.sort.* import io.ksmt.utils.getValue import io.ksmt.utils.uncheckedCast import java.io.* -import java.nio.file.Files -import java.nio.file.Path +import java.nio.FloatBuffer +import java.nio.LongBuffer import java.util.concurrent.atomic.AtomicLong -import kotlin.io.path.isRegularFile import kotlin.time.Duration -import kotlin.time.Duration.Companion.seconds fun serialize(ctx: KContext, expressions: List>, outputStream: OutputStream) { val serializationCtx = AstSerializationCtx().apply { initCtx(ctx) } @@ -113,6 +112,139 @@ class LogSolver( } fun main() { + + val ctx = KContext() + + val runner = NeuroSMTModelRunner(ctx, "usvm-enc-2.cats", "embeddings.onnx", "conv.onnx", "decoder.onnx") + + with(ctx) { + val a by boolSort + val b by intSort + val c by intSort + val d by realSort + val e by bv8Sort + val f by fp64Sort + val g by mkArraySort(realSort, boolSort) + val h by fp64Sort + + val expr = mkBvXorExpr(mkBvShiftLeftExpr(e, mkBv(1.toByte())), mkBvNotExpr(e)) eq + mkBvLogicalShiftRightExpr(e, mkBv(1.toByte())) + + println(runner.run(expr)) + } + + return + + val env = OrtEnvironment.getEnvironment() + //val session = env.createSession("kek.onnx") + val session = env.createSession("conv.onnx") + + println(session.inputNames) + for (info in session.inputInfo) { + println(info) + } + + println() + + println(session.outputNames) + for (info in session.outputInfo) { + println(info) + } + + println() + + println(session.metadata) + println() + + val nodeLabels = listOf(listOf(0L), listOf(1L), listOf(2L), listOf(3L), listOf(4L), listOf(5L), listOf(6L)) + val nodeFeatures = (1..7).map { (0..31).map { it / 31.toFloat() } } + val edges = listOf( + //listOf(0L, 1L, 0L), + //listOf(1L, 2L, 2L) + listOf(0L, 1L, 2L, 3L, 4L, 5L), + listOf(1L, 2L, 3L, 4L, 5L, 6L) + ) + val depths = listOf(8L) + val rootPtrs = listOf(0L, 7L) + + /* + val nodeLabels = listOf(listOf(0L), listOf(1L), listOf(1L), listOf(0L)) + val edges = listOf( + listOf(0L, 0L), + listOf(1L, 1L) + ) + val depths = listOf(1L, 1L) + val rootPtrs = listOf(0L, 1L, 2L) + */ + + val nodeLabelsBuffer = LongBuffer.allocate(nodeLabels.sumOf { it.size }) + nodeLabels.forEach { features -> + features.forEach { feature -> + nodeLabelsBuffer.put(feature) + } + } + nodeLabelsBuffer.rewind() + + val nodeFeaturesBuffer = FloatBuffer.allocate(nodeFeatures.sumOf { it.size }) + nodeFeatures.forEach { features -> + features.forEach { feature -> + nodeFeaturesBuffer.put(feature) + } + } + nodeFeaturesBuffer.rewind() + + val edgesBuffer = LongBuffer.allocate(edges.sumOf { it.size }) + edges.forEach { row -> + row.forEach { node -> + edgesBuffer.put(node) + } + } + edgesBuffer.rewind() + + val depthsBuffer = LongBuffer.allocate(depths.size) + depths.forEach { d -> + depthsBuffer.put(d) + } + depthsBuffer.rewind() + + val rootPtrsBuffer = LongBuffer.allocate(rootPtrs.size) + rootPtrs.forEach { r -> + rootPtrsBuffer.put(r) + } + rootPtrsBuffer.rewind() + + val nodeLabelsData = OnnxTensor.createTensor(env, nodeLabelsBuffer, listOf(nodeLabels.size.toLong(), nodeLabels[0].size.toLong()).toLongArray()) + val nodeFeaturesData = OnnxTensor.createTensor(env, nodeFeaturesBuffer, listOf(nodeFeatures.size.toLong(), nodeFeatures[0].size.toLong()).toLongArray()) + val edgesData = OnnxTensor.createTensor(env, edgesBuffer, listOf(edges.size.toLong(), edges[0].size.toLong()).toLongArray()) + val depthsData = OnnxTensor.createTensor(env, depthsBuffer, listOf(depths.size.toLong()).toLongArray()) + val rootPtrsData = OnnxTensor.createTensor(env, rootPtrsBuffer, listOf(rootPtrs.size.toLong()).toLongArray()) + + /* + val result = session.run(mapOf("node_labels" to nodeLabelsData, "edges" to edgesData, "depths" to depthsData, "root_ptrs" to rootPtrsData)) + val output = (result.get("output").get().value as Array<*>).map { + (it as FloatArray).toList() + } + */ + + var curFeatures = nodeFeaturesData + repeat(10) { + val result = session.run(mapOf("node_features" to curFeatures, "edges" to edgesData)) + curFeatures = result.get(0) as OnnxTensor + println(curFeatures.info.shape.toList()) + curFeatures.info.shape + println(curFeatures.floatBuffer.array().toList().subList(224 - 32, 224)) + } + + /* + val output = (result.get("out").get().value as Array<*>).map { + (it as FloatArray).toList() + } + + println(output) + */ + + + /* val ctx = KContext() with(ctx) { @@ -128,7 +260,7 @@ fun main() { } } println("$ok / $fail") - } + }*/ /* with(ctx) { diff --git a/settings.gradle.kts b/settings.gradle.kts index 073762985..850a5c3e8 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -18,6 +18,6 @@ pluginManagement { } } include("ksmt-neurosmt") -include("sandbox") include("ksmt-neurosmt:utils") // findProject(":ksmt-neurosmt:utils")?.name = "utils" +include("sandbox") From 59f78394639fb417b66685713a1c278fea95617c Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Fri, 18 Aug 2023 15:54:54 +0300 Subject: [PATCH 37/52] fix bug in runtime and change conv type --- .../solver/neurosmt/runtime/ExprEncoder.kt | 28 ++++ .../ksmt/solver/neurosmt/runtime/ONNXModel.kt | 4 +- ksmt-neurosmt/src/main/python/Encoder.py | 13 +- ksmt-neurosmt/src/main/python/sandbox.py | 27 ++++ .../solver/neurosmt/FormulaGraphExtractor.kt | 2 +- sandbox/build.gradle.kts | 2 + sandbox/src/main/kotlin/Main.kt | 130 ++++++++++++++++-- 7 files changed, 186 insertions(+), 20 deletions(-) diff --git a/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ExprEncoder.kt b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ExprEncoder.kt index cdd794ffd..135a84ab9 100644 --- a/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ExprEncoder.kt +++ b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ExprEncoder.kt @@ -37,6 +37,9 @@ class ExprEncoder( else -> calcAppState(expr) } + //print("calculated: ") + //println(exprToState[expr]?.floatBuffer?.array()?.toList()) + return expr } @@ -76,15 +79,40 @@ class ExprEncoder( val buffer = FloatBuffer.allocate((1 + childrenCnt) * embeddingSize.toInt()) buffer.put(nodeEmbedding.floatBuffer) childrenStates.forEach { + // println(it.floatBuffer.array().toList()) buffer.put(it.floatBuffer) } buffer.rewind() val nodeFeatures = OnnxTensor.createTensor(env, buffer, longArrayOf(1L + childrenCnt, embeddingSize)) + /* + println("+++++") + nodeFeatures.floatBuffer.array().toList().chunked(embeddingSize.toInt()).forEach { + println(it) + } + + */ + val edges = createEdgeTensor(childrenStates.size) + //val edges = createEdgeTensor(0) + //println("edges: ${edges.longBuffer.array().toList()}") + val result = convLayer.forward(mapOf("node_features" to nodeFeatures, "edges" to edges)) + + //print("fucking result: ") + //println(result.floatBuffer.array().toList()) + + /* + println("*****") + result.floatBuffer.array().toList().chunked(embeddingSize.toInt()).forEach { + println(it) + } + println("-----\n") + */ + val newNodeFeatures = OnnxTensor.createTensor(env, result.floatBuffer.slice(0, embeddingSize.toInt()), longArrayOf(1L, embeddingSize)) + // println(newNodeFeatures.floatBuffer.array().toList()) exprToState[expr] = newNodeFeatures } diff --git a/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ONNXModel.kt b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ONNXModel.kt index 8a98a05ab..6f7962808 100644 --- a/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ONNXModel.kt +++ b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ONNXModel.kt @@ -23,7 +23,7 @@ class ONNXModel(env: OrtEnvironment, modelPath: String) : AutoCloseable { } class NeuroSMTModelRunner( - ctx: KContext, + val ctx: KContext, ordinalsPath: String, embeddingPath: String, convPath: String, decoderPath: String ) { val env = OrtEnvironment.getEnvironment() @@ -31,11 +31,11 @@ class NeuroSMTModelRunner( val ordinalEncoder = OrdinalEncoder(ordinalsPath) val embeddingLayer = ONNXModel(env, embeddingPath) val convLayer = ONNXModel(env, convPath) - val encoder = ExprEncoder(ctx, env, ordinalEncoder, embeddingLayer, convLayer) val decoder = ONNXModel(env, decoderPath) fun run(expr: KExpr<*>): Float { + val encoder = ExprEncoder(ctx, env, ordinalEncoder, embeddingLayer, convLayer) val exprFeatures = encoder.encodeExpr(expr) val result = decoder.forward(mapOf("expr_features" to exprFeatures)) val logit = result.floatBuffer[0] diff --git a/ksmt-neurosmt/src/main/python/Encoder.py b/ksmt-neurosmt/src/main/python/Encoder.py index b98bd4f8e..4cfd6556a 100644 --- a/ksmt-neurosmt/src/main/python/Encoder.py +++ b/ksmt-neurosmt/src/main/python/Encoder.py @@ -1,20 +1,22 @@ import torch import torch.nn as nn -from torch_geometric.nn import GCNConv, GatedGraphConv, GATConv +from torch_geometric.nn import GCNConv, GATConv, TransformerConv, SAGEConv -EMBEDDING_DIM = 2000 +EMBEDDINGS_CNT = 2000 class Encoder(nn.Module): def __init__(self, hidden_dim): super().__init__() - self.embedding = nn.Embedding(EMBEDDING_DIM, hidden_dim) - self.conv = GCNConv(hidden_dim, hidden_dim, add_self_loops=False) + self.embedding = nn.Embedding(EMBEDDINGS_CNT, hidden_dim) + self.conv = SAGEConv(hidden_dim, hidden_dim, "mean", root_weight=False, project=True) - #self.conv = GATConv(64, 64, add_self_loops=False) + # self.conv = GATConv(hidden_dim, hidden_dim, add_self_loops=False) + # self.conv = TransformerConv(hidden_dim, hidden_dim, root_weight=False) + # self.conv = GCNConv(hidden_dim, hidden_dim, add_self_loops=False) def forward(self, node_labels, edges, depths, root_ptrs): # x, edge_index, depth = data.x, data.edge_index, data.depth @@ -23,7 +25,6 @@ def forward(self, node_labels, edges, depths, root_ptrs): node_features = self.embedding(node_labels.squeeze()) depth = depths.max() - node_features += depth * 0.0 for i in range(depth): node_features = self.conv(node_features, edges) diff --git a/ksmt-neurosmt/src/main/python/sandbox.py b/ksmt-neurosmt/src/main/python/sandbox.py index f8e430320..7bc58e159 100755 --- a/ksmt-neurosmt/src/main/python/sandbox.py +++ b/ksmt-neurosmt/src/main/python/sandbox.py @@ -19,6 +19,31 @@ if __name__ == "__main__": pl_model = LightningModel().load_from_checkpoint(sys.argv[1], map_location=torch.device("cpu")) + pl_model.eval() + pl_model.model.eval() + + print(pl_model.model.encoder.conv.lin_l.weight) + print(pl_model.model.encoder.conv.lin_l.bias) + + node_features = torch.tensor([ + [i / 32 for i in range(32)], + [-i / 32 for i in range(32)], + [i / 32 * (-1) ** i for i in range(32)] + ], dtype=torch.float) + edges = torch.tensor([ + [0, 1], + [2, 2] + ], dtype=torch.int64) + + conv = pl_model.model.encoder.conv + + print("#0") + print(node_features) + + for i in range(5): + print(f"#{i}") + node_features = conv(node_features, edges) + print(node_features) """ trainer = Trainer( @@ -40,6 +65,8 @@ ) """ + #sys.exit(0) + MAX_SIZE = 3 node_labels = torch.tensor([[i] for i in range(MAX_SIZE)], dtype=torch.int32) diff --git a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/FormulaGraphExtractor.kt b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/FormulaGraphExtractor.kt index 8d430a3c1..4fb6ab4db 100644 --- a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/FormulaGraphExtractor.kt +++ b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/FormulaGraphExtractor.kt @@ -34,7 +34,7 @@ class FormulaGraphExtractor( } fun writeSymbolicVariable(symbol: KConst) { - when (symbol.sort) { + when (symbol.decl.sort) { is KBoolSort -> writer.write("SYMBOLIC; Bool\n") is KBvSort -> writer.write("SYMBOLIC; BitVec\n") is KFpSort -> writer.write("SYMBOLIC; FP\n") diff --git a/sandbox/build.gradle.kts b/sandbox/build.gradle.kts index 88a376ea5..eee610672 100644 --- a/sandbox/build.gradle.kts +++ b/sandbox/build.gradle.kts @@ -22,6 +22,8 @@ dependencies { implementation("com.microsoft.onnxruntime:onnxruntime:1.15.1") implementation("com.microsoft.onnxruntime:onnxruntime_gpu:1.15.1") + + implementation("me.tongfei:progressbar:0.9.4") } tasks.getByName("test") { diff --git a/sandbox/src/main/kotlin/Main.kt b/sandbox/src/main/kotlin/Main.kt index 7c6374ddb..5c7413eae 100644 --- a/sandbox/src/main/kotlin/Main.kt +++ b/sandbox/src/main/kotlin/Main.kt @@ -2,6 +2,7 @@ import ai.onnxruntime.OnnxTensor import ai.onnxruntime.OrtEnvironment import com.jetbrains.rd.framework.SerializationCtx import com.jetbrains.rd.framework.Serializers +import com.jetbrains.rd.framework.SocketWire.Companion.timeout import com.jetbrains.rd.framework.UnsafeBuffer import io.ksmt.KContext import io.ksmt.expr.* @@ -9,16 +10,24 @@ import io.ksmt.runner.serializer.AstSerializationCtx import io.ksmt.solver.KSolver import io.ksmt.solver.KSolverConfiguration import io.ksmt.solver.KSolverStatus +import io.ksmt.solver.neurosmt.getAnswerForTest import io.ksmt.solver.neurosmt.runtime.NeuroSMTModelRunner import io.ksmt.solver.z3.* import io.ksmt.sort.* import io.ksmt.utils.getValue import io.ksmt.utils.uncheckedCast +import me.tongfei.progressbar.ProgressBar import java.io.* import java.nio.FloatBuffer import java.nio.LongBuffer +import java.nio.file.Files +import java.nio.file.Path import java.util.concurrent.atomic.AtomicLong +import kotlin.io.path.isRegularFile +import kotlin.io.path.name +import kotlin.io.path.pathString import kotlin.time.Duration +import kotlin.time.Duration.Companion.seconds fun serialize(ctx: KContext, expressions: List>, outputStream: OutputStream) { val serializationCtx = AstSerializationCtx().apply { initCtx(ctx) } @@ -111,12 +120,106 @@ class LogSolver( } } +const val THRESHOLD = 0.5 + fun main() { - val ctx = KContext() + val ctx = KContext(simplificationMode = KContext.SimplificationMode.NO_SIMPLIFY) + + val pathToDataset = "formulas" + val files = Files.walk(Path.of(pathToDataset)).filter { it.isRegularFile() }.toList() + + val runner = NeuroSMTModelRunner( + ctx, + ordinalsPath = "usvm-enc-2.cats", + embeddingPath = "embeddings.onnx", + convPath = "conv.onnx", + decoderPath = "decoder.onnx" + ) + + var sat = 0; var unsat = 0; var skipped = 0 + var ok = 0; var wa = 0 + + val confusionMatrix = mutableMapOf, Int>() + + files.forEachIndexed sample@{ ind, it -> + if (ind % 100 == 0) { + println("#$ind: $ok / $wa [$sat / $unsat / $skipped]") + println(confusionMatrix) + println() + } + + val sampleFile = File(it.pathString) + + val assertList = try { + deserialize(ctx, FileInputStream(sampleFile)) + } catch (e: Exception) { + skipped++ + return@sample + } + + val answer = when { + it.name.endsWith("-sat") -> KSolverStatus.SAT + it.name.endsWith("-unsat") -> KSolverStatus.UNSAT + else -> KSolverStatus.UNKNOWN + } + + if (answer == KSolverStatus.UNKNOWN) { + skipped++ + return@sample + } + + val prob = with(ctx) { + val formula = when (assertList.size) { + 0 -> { + skipped++ + return@sample + } + 1 -> { + assertList[0] + } + else -> { + mkAnd(assertList) + } + } + + runner.run(formula) + } + + val output = if (prob < THRESHOLD) { + KSolverStatus.UNSAT + } else { + KSolverStatus.SAT + } - val runner = NeuroSMTModelRunner(ctx, "usvm-enc-2.cats", "embeddings.onnx", "conv.onnx", "decoder.onnx") + when (answer) { + KSolverStatus.SAT -> sat++ + KSolverStatus.UNSAT -> unsat++ + else -> { /* can't happen */ } + } + if (output == answer) { + ok++ + } else { + wa++ + } + + confusionMatrix.compute(answer to output) { _, v -> + if (v == null) { + 1 + } else { + v + 1 + } + } + } + + println() + println("sat: $sat; unsat: $unsat; skipped: $skipped") + println("ok: $ok; wa: $wa") + + return + + /* with(ctx) { val a by boolSort val b by intSort @@ -130,11 +233,14 @@ fun main() { val expr = mkBvXorExpr(mkBvShiftLeftExpr(e, mkBv(1.toByte())), mkBvNotExpr(e)) eq mkBvLogicalShiftRightExpr(e, mkBv(1.toByte())) + val runner = NeuroSMTModelRunner(ctx, "usvm-enc-2.cats", "embeddings.onnx", "conv.onnx", "decoder.onnx") println(runner.run(expr)) } return + */ + val env = OrtEnvironment.getEnvironment() //val session = env.createSession("kek.onnx") val session = env.createSession("conv.onnx") @@ -156,16 +262,18 @@ fun main() { println(session.metadata) println() - val nodeLabels = listOf(listOf(0L), listOf(1L), listOf(2L), listOf(3L), listOf(4L), listOf(5L), listOf(6L)) - val nodeFeatures = (1..7).map { (0..31).map { it / 31.toFloat() } } + //val nodeLabels = listOf(listOf(0L), listOf(1L), listOf(2L), listOf(3L), listOf(4L), listOf(5L), listOf(6L)) + val nodeLabels = listOf(listOf(0L), listOf(1L), listOf(2L)) + //val nodeFeatures = (1..7).map { (0..31).map { it / 31.toFloat() } } + val nodeFeatures = (1..3).map { (0..31).map { it / 31.toFloat() } } val edges = listOf( - //listOf(0L, 1L, 0L), - //listOf(1L, 2L, 2L) - listOf(0L, 1L, 2L, 3L, 4L, 5L), - listOf(1L, 2L, 3L, 4L, 5L, 6L) + listOf(0L, 1L), + listOf(2L, 2L) + //listOf(0L, 1L, 2L, 3L, 4L, 5L), + //listOf(1L, 2L, 3L, 4L, 5L, 6L) ) - val depths = listOf(8L) - val rootPtrs = listOf(0L, 7L) + val depths = listOf(1L) + val rootPtrs = listOf(0L, 3L) /* val nodeLabels = listOf(listOf(0L), listOf(1L), listOf(1L), listOf(0L)) @@ -232,7 +340,7 @@ fun main() { curFeatures = result.get(0) as OnnxTensor println(curFeatures.info.shape.toList()) curFeatures.info.shape - println(curFeatures.floatBuffer.array().toList().subList(224 - 32, 224)) + println(curFeatures.floatBuffer.array().toList().subList(64, 96)) } /* From 65d8bba4039a556cf299afb5d9e771f16ba6424a Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Fri, 18 Aug 2023 17:45:05 +0300 Subject: [PATCH 38/52] move python files to new module --- ksmt-neurosmt/src/main/python/python.iml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 ksmt-neurosmt/src/main/python/python.iml diff --git a/ksmt-neurosmt/src/main/python/python.iml b/ksmt-neurosmt/src/main/python/python.iml new file mode 100644 index 000000000..12d9bcccd --- /dev/null +++ b/ksmt-neurosmt/src/main/python/python.iml @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file From 13b85fd7597f0b2779df1102a3fbf9c4a9275697 Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Tue, 22 Aug 2023 10:27:18 +0300 Subject: [PATCH 39/52] add node-wise features --- ksmt-neurosmt/src/main/python/Encoder.py | 19 +++-- .../src/main/python/GraphDataloader.py | 19 ++--- ksmt-neurosmt/src/main/python/GraphReader.py | 2 +- .../src/main/python/create-ordinal-encoder.py | 9 +- ksmt-neurosmt/src/main/python/python.iml | 6 +- ksmt-neurosmt/src/main/python/sandbox.py | 82 ------------------- ksmt-neurosmt/src/main/python/utils.py | 4 +- sandbox/build.gradle.kts | 2 +- 8 files changed, 32 insertions(+), 111 deletions(-) diff --git a/ksmt-neurosmt/src/main/python/Encoder.py b/ksmt-neurosmt/src/main/python/Encoder.py index 4cfd6556a..e628a0d7d 100644 --- a/ksmt-neurosmt/src/main/python/Encoder.py +++ b/ksmt-neurosmt/src/main/python/Encoder.py @@ -1,7 +1,7 @@ import torch import torch.nn as nn -from torch_geometric.nn import GCNConv, GATConv, TransformerConv, SAGEConv +from torch_geometric.nn import SAGEConv EMBEDDINGS_CNT = 2000 @@ -12,21 +12,22 @@ def __init__(self, hidden_dim): super().__init__() self.embedding = nn.Embedding(EMBEDDINGS_CNT, hidden_dim) - self.conv = SAGEConv(hidden_dim, hidden_dim, "mean", root_weight=False, project=True) + self.conv = SAGEConv(hidden_dim, hidden_dim, "mean", root_weight=True, project=True) - # self.conv = GATConv(hidden_dim, hidden_dim, add_self_loops=False) - # self.conv = TransformerConv(hidden_dim, hidden_dim, root_weight=False) # self.conv = GCNConv(hidden_dim, hidden_dim, add_self_loops=False) + # self.conv = GATConv(hidden_dim, hidden_dim, add_self_loops=False) # this can't be exported to ONNX + # self.conv = TransformerConv(hidden_dim, hidden_dim, root_weight=False) # this can't be exported to ONNX def forward(self, node_labels, edges, depths, root_ptrs): - # x, edge_index, depth = data.x, data.edge_index, data.depth - # edge_index = torch.tensor([[0, 0], [1, 2]], dtype=torch.long).to(edge_index.get_device()) - node_features = self.embedding(node_labels.squeeze()) depth = depths.max() - for i in range(depth): - node_features = self.conv(node_features, edges) + for i in range(1, depth + 1): + mask = (depths == i) + new_features = self.conv(node_features, edges) + + node_features = torch.clone(node_features) + node_features[mask] = new_features[mask] node_features = node_features[root_ptrs[1:] - 1] diff --git a/ksmt-neurosmt/src/main/python/GraphDataloader.py b/ksmt-neurosmt/src/main/python/GraphDataloader.py index 850ddcae8..dbb22e8b0 100644 --- a/ksmt-neurosmt/src/main/python/GraphDataloader.py +++ b/ksmt-neurosmt/src/main/python/GraphDataloader.py @@ -24,11 +24,11 @@ class GraphDataset(Dataset): def __init__(self, graph_data): self.graphs = [Graph( - x=torch.tensor(nodes), - edge_index=torch.tensor(edges).t(), + x=torch.tensor(nodes, dtype=torch.int32), + edge_index=torch.tensor(edges, dtype=torch.int64).t(), y=torch.tensor([[label]], dtype=torch.float), - depth=depth - ) for nodes, edges, label, depth in graph_data] + depth=torch.tensor(depths, dtype=torch.int32) + ) for nodes, edges, label, depths in graph_data] def __len__(self): return len(self.graphs) @@ -48,18 +48,17 @@ def load_data(paths_to_datasets, target): for path_to_sample in tqdm(list(f.readlines())): path_to_sample = os.path.join(path_to_dataset, path_to_sample.strip()) - operators, edges, depth = read_graph_by_path( + operators, edges, depths = read_graph_by_path( path_to_sample, max_size=MAX_FORMULA_SIZE, max_depth=MAX_FORMULA_DEPTH ) - if depth is None: + if operators is None: continue if len(edges) == 0: print(f"w: ignoring formula without edges; file '{path_to_sample}'") continue - label = None if path_to_sample.endswith("-sat"): label = 1 elif path_to_sample.endswith("-unsat"): @@ -67,7 +66,7 @@ def load_data(paths_to_datasets, target): else: raise Exception(f"strange file path '{path_to_sample}'") - data.append((operators, edges, label, depth)) + data.append((operators, edges, label, depths)) return data @@ -84,9 +83,9 @@ def get_dataloader(paths_to_datasets, target, path_to_ordinal_encoder): encoder = joblib.load(path_to_ordinal_encoder) def transform(data_for_one_sample): - nodes, edges, label, depth = data_for_one_sample + nodes, edges, label, depths = data_for_one_sample nodes = encoder.transform(np.array(nodes).reshape(-1, 1)) - return nodes, edges, label, depth + return nodes, edges, label, depths print("transforming") data = list(map(transform, data)) diff --git a/ksmt-neurosmt/src/main/python/GraphReader.py b/ksmt-neurosmt/src/main/python/GraphReader.py index 2af2a319e..4be7a28c7 100644 --- a/ksmt-neurosmt/src/main/python/GraphReader.py +++ b/ksmt-neurosmt/src/main/python/GraphReader.py @@ -57,7 +57,7 @@ def read_graph_from_file(inf, max_size, max_depth): v += 1 - return operators, edges, max(depth) + return operators, edges, depth def read_graph_by_path(path, max_size, max_depth): diff --git a/ksmt-neurosmt/src/main/python/create-ordinal-encoder.py b/ksmt-neurosmt/src/main/python/create-ordinal-encoder.py index 0830b2ea9..534d483ca 100755 --- a/ksmt-neurosmt/src/main/python/create-ordinal-encoder.py +++ b/ksmt-neurosmt/src/main/python/create-ordinal-encoder.py @@ -9,6 +9,7 @@ from sklearn.preprocessing import OrdinalEncoder from GraphDataloader import load_data +from Encoder import EMBEDDINGS_CNT def create_ordinal_encoder(paths_to_datasets, path_to_ordinal_encoder): @@ -16,8 +17,8 @@ def create_ordinal_encoder(paths_to_datasets, path_to_ordinal_encoder): encoder = OrdinalEncoder( dtype=int, - handle_unknown="use_encoded_value", unknown_value=1999, - encoded_missing_value=1998 + handle_unknown="use_encoded_value", unknown_value=EMBEDDINGS_CNT - 1, + encoded_missing_value=EMBEDDINGS_CNT - 2 ) print("fitting ordinal encoder") @@ -25,6 +26,10 @@ def create_ordinal_encoder(paths_to_datasets, path_to_ordinal_encoder): *(list(zip(*data))[0]) ))).reshape(-1, 1)) + for cat in encoder.categories_: + if len(cat) > EMBEDDINGS_CNT - 2: + print("w: too many categories") + print("dumping ordinal encoder") joblib.dump(encoder, path_to_ordinal_encoder) diff --git a/ksmt-neurosmt/src/main/python/python.iml b/ksmt-neurosmt/src/main/python/python.iml index 12d9bcccd..f4e6189e6 100644 --- a/ksmt-neurosmt/src/main/python/python.iml +++ b/ksmt-neurosmt/src/main/python/python.iml @@ -2,10 +2,8 @@ - - - - + + \ No newline at end of file diff --git a/ksmt-neurosmt/src/main/python/sandbox.py b/ksmt-neurosmt/src/main/python/sandbox.py index 7bc58e159..8afb28c3b 100755 --- a/ksmt-neurosmt/src/main/python/sandbox.py +++ b/ksmt-neurosmt/src/main/python/sandbox.py @@ -20,30 +20,6 @@ pl_model = LightningModel().load_from_checkpoint(sys.argv[1], map_location=torch.device("cpu")) pl_model.eval() - pl_model.model.eval() - - print(pl_model.model.encoder.conv.lin_l.weight) - print(pl_model.model.encoder.conv.lin_l.bias) - - node_features = torch.tensor([ - [i / 32 for i in range(32)], - [-i / 32 for i in range(32)], - [i / 32 * (-1) ** i for i in range(32)] - ], dtype=torch.float) - edges = torch.tensor([ - [0, 1], - [2, 2] - ], dtype=torch.int64) - - conv = pl_model.model.encoder.conv - - print("#0") - print(node_features) - - for i in range(5): - print(f"#{i}") - node_features = conv(node_features, edges) - print(node_features) """ trainer = Trainer( @@ -65,8 +41,6 @@ ) """ - #sys.exit(0) - MAX_SIZE = 3 node_labels = torch.tensor([[i] for i in range(MAX_SIZE)], dtype=torch.int32) @@ -77,8 +51,6 @@ depths = torch.tensor([MAX_SIZE - 1], dtype=torch.int64) root_ptrs = torch.tensor([0, MAX_SIZE], dtype=torch.int64) - print(node_labels.shape, edges.shape, depths.shape, root_ptrs.shape) - torch.onnx.export( pl_model.model.encoder.embedding, (node_labels,), @@ -132,57 +104,3 @@ # verbose=True ) """ - - """ - torch.onnx.export(torch_model, - x, - '../Game_env/gnn_model.onnx', - opset_version=15, - export_params=True, - input_names = ['x', 'edge_index'], # the model's input names - output_names = ['output'], - dynamic_axes={'x' : {0 : 'nodes_number'}, # variable length axes - 'edge_index' : {1 : 'egdes_number'}, - }, - ) - """ - - """ - x = torch.randn(*shape, requires_grad=True).to(device) - torch_model = actor_model.eval() - torch_out = torch_model(x) - torch.onnx.export(torch_model, - x, - '../Game_env/actor_model.onnx', - opset_version=15, - export_params=True, - input_names = ['input'], # the model's input names - output_names = ['output'], - dynamic_axes={'input' : {0 : 'batch_size', - 1 : 'n_actions', - }, # variable length axes - 'output' : {0 : 'batch_size', - 1 : 'n_actions', - }, - }, - ) - algo_name = 'NN' - - if actor_model is not None and gnn_model is not None and self.use_gnn: - x_shape = [1, self.gnn_in_nfeatures] - edge_index_shape = [2, 1] - x = (torch.randn(*x_shape, requires_grad=True).to(device), torch.randint(0, 1, edge_index_shape).to(device)) - torch_model = gnn_model.eval() - torch_out = torch_model(*x) - torch.onnx.export(torch_model, - x, - '../Game_env/gnn_model.onnx', - opset_version=15, - export_params=True, - input_names = ['x', 'edge_index'], # the model's input names - output_names = ['output'], - dynamic_axes={'x' : {0 : 'nodes_number'}, # variable length axes - 'edge_index' : {1 : 'egdes_number'}, - }, - ) - """ diff --git a/ksmt-neurosmt/src/main/python/utils.py b/ksmt-neurosmt/src/main/python/utils.py index ae090781b..4bf95b4ae 100644 --- a/ksmt-neurosmt/src/main/python/utils.py +++ b/ksmt-neurosmt/src/main/python/utils.py @@ -20,9 +20,9 @@ def train_val_test_indices(cnt, val_qty=0.15, test_qty=0.1): def select_paths_with_suitable_samples_and_transform_to_paths_from_root(path_to_dataset_root, paths): correct_paths = [] for path in tqdm(paths): - operators, edges, depth = read_graph_by_path(path, max_size=MAX_FORMULA_SIZE, max_depth=MAX_FORMULA_DEPTH) + operators, edges, _ = read_graph_by_path(path, max_size=MAX_FORMULA_SIZE, max_depth=MAX_FORMULA_DEPTH) - if depth is None: + if operators is None: continue if len(edges) == 0: diff --git a/sandbox/build.gradle.kts b/sandbox/build.gradle.kts index eee610672..3839eeb1a 100644 --- a/sandbox/build.gradle.kts +++ b/sandbox/build.gradle.kts @@ -21,7 +21,7 @@ dependencies { implementation(project(":ksmt-runner")) implementation("com.microsoft.onnxruntime:onnxruntime:1.15.1") - implementation("com.microsoft.onnxruntime:onnxruntime_gpu:1.15.1") + // implementation("com.microsoft.onnxruntime:onnxruntime_gpu:1.15.1") implementation("me.tongfei:progressbar:0.9.4") } From 6b8a1202a3a180b7bb0bb252328b57712cd986b0 Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Tue, 22 Aug 2023 14:12:37 +0300 Subject: [PATCH 40/52] refactor --- .../solver/neurosmt/runtime/ExprEncoder.kt | 45 ++++--------------- .../neurosmt/runtime/NeuroSMTModelRunner.kt | 28 ++++++++++++ .../ksmt/solver/neurosmt/runtime/ONNXModel.kt | 44 ------------------ .../solver/neurosmt/runtime/OrdinalEncoder.kt | 21 +++++++++ sandbox/src/main/kotlin/Main.kt | 5 ++- 5 files changed, 62 insertions(+), 81 deletions(-) create mode 100644 ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/NeuroSMTModelRunner.kt create mode 100644 ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/OrdinalEncoder.kt diff --git a/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ExprEncoder.kt b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ExprEncoder.kt index 135a84ab9..aa1b5f228 100644 --- a/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ExprEncoder.kt +++ b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ExprEncoder.kt @@ -31,14 +31,13 @@ class ExprEncoder( } override fun transformApp(expr: KApp): KExpr { - when (expr) { + val state = when (expr) { is KConst<*> -> calcSymbolicVariableState(expr) is KInterpretedValue<*> -> calcValueState(expr) else -> calcAppState(expr) } - //print("calculated: ") - //println(exprToState[expr]?.floatBuffer?.array()?.toList()) + exprToState[expr] = state return expr } @@ -69,7 +68,7 @@ class ExprEncoder( return OnnxTensor.createTensor(env, buffer, longArrayOf(2, childrenCnt.toLong())) } - private fun calcAppState(expr: KApp) { + private fun calcAppState(expr: KApp): OnnxTensor { val childrenStates = expr.args.map { exprToState[it] ?: error("expression state wasn't calculated yet") } val childrenCnt = childrenStates.size @@ -79,44 +78,18 @@ class ExprEncoder( val buffer = FloatBuffer.allocate((1 + childrenCnt) * embeddingSize.toInt()) buffer.put(nodeEmbedding.floatBuffer) childrenStates.forEach { - // println(it.floatBuffer.array().toList()) buffer.put(it.floatBuffer) } buffer.rewind() - val nodeFeatures = OnnxTensor.createTensor(env, buffer, longArrayOf(1L + childrenCnt, embeddingSize)) - - /* - println("+++++") - nodeFeatures.floatBuffer.array().toList().chunked(embeddingSize.toInt()).forEach { - println(it) - } - - */ + val nodeFeatures = OnnxTensor.createTensor(env, buffer, longArrayOf(1L + childrenCnt, embeddingSize)) val edges = createEdgeTensor(childrenStates.size) - - //val edges = createEdgeTensor(0) - //println("edges: ${edges.longBuffer.array().toList()}") - val result = convLayer.forward(mapOf("node_features" to nodeFeatures, "edges" to edges)) - //print("fucking result: ") - //println(result.floatBuffer.array().toList()) - - /* - println("*****") - result.floatBuffer.array().toList().chunked(embeddingSize.toInt()).forEach { - println(it) - } - println("-----\n") - */ - - val newNodeFeatures = OnnxTensor.createTensor(env, result.floatBuffer.slice(0, embeddingSize.toInt()), longArrayOf(1L, embeddingSize)) - // println(newNodeFeatures.floatBuffer.array().toList()) - exprToState[expr] = newNodeFeatures + return OnnxTensor.createTensor(env, result.floatBuffer.slice(0, embeddingSize.toInt()), longArrayOf(1L, embeddingSize)) } - private fun calcSymbolicVariableState(symbol: KConst) { + private fun calcSymbolicVariableState(symbol: KConst): OnnxTensor { val key = when (symbol.decl.sort) { is KBoolSort -> "SYMBOLIC;Bool" is KBvSort -> "SYMBOLIC;BitVec" @@ -127,10 +100,10 @@ class ExprEncoder( else -> error("unknown symbolic sort: ${symbol.decl.sort::class.simpleName}") } - exprToState[symbol] = getNodeEmbedding(key) + return getNodeEmbedding(key) } - private fun calcValueState(value: KInterpretedValue) { + private fun calcValueState(value: KInterpretedValue): OnnxTensor { val key = when (value.decl.sort) { is KBoolSort -> "VALUE;Bool" is KBvSort -> "VALUE;BitVec" @@ -141,6 +114,6 @@ class ExprEncoder( else -> error("unknown value sort: ${value.decl.sort::class.simpleName}") } - exprToState[value] = getNodeEmbedding(key) + return getNodeEmbedding(key) } } \ No newline at end of file diff --git a/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/NeuroSMTModelRunner.kt b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/NeuroSMTModelRunner.kt new file mode 100644 index 000000000..a9e5796de --- /dev/null +++ b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/NeuroSMTModelRunner.kt @@ -0,0 +1,28 @@ +package io.ksmt.solver.neurosmt.runtime + +import ai.onnxruntime.OrtEnvironment +import io.ksmt.KContext +import io.ksmt.expr.KExpr +import kotlin.math.exp + +class NeuroSMTModelRunner( + val ctx: KContext, + ordinalsPath: String, embeddingPath: String, convPath: String, decoderPath: String +) { + val env = OrtEnvironment.getEnvironment() + + val ordinalEncoder = OrdinalEncoder(ordinalsPath) + val embeddingLayer = ONNXModel(env, embeddingPath) + val convLayer = ONNXModel(env, convPath) + + val decoder = ONNXModel(env, decoderPath) + + fun run(expr: KExpr<*>): Float { + val encoder = ExprEncoder(ctx, env, ordinalEncoder, embeddingLayer, convLayer) + val exprFeatures = encoder.encodeExpr(expr) + val result = decoder.forward(mapOf("expr_features" to exprFeatures)) + val logit = result.floatBuffer[0] + + return 1f / (1f + exp(-logit)) // sigmoid calculation + } +} \ No newline at end of file diff --git a/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ONNXModel.kt b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ONNXModel.kt index 6f7962808..5711db6c4 100644 --- a/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ONNXModel.kt +++ b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ONNXModel.kt @@ -2,12 +2,6 @@ package io.ksmt.solver.neurosmt.runtime import ai.onnxruntime.OnnxTensor import ai.onnxruntime.OrtEnvironment -import io.ksmt.KContext -import io.ksmt.expr.KExpr -import java.nio.file.Files -import java.nio.file.Path -import kotlin.math.exp -import kotlin.streams.asSequence class ONNXModel(env: OrtEnvironment, modelPath: String) : AutoCloseable { val session = env.createSession(modelPath) @@ -20,42 +14,4 @@ class ONNXModel(env: OrtEnvironment, modelPath: String) : AutoCloseable { override fun close() { session.close() } -} - -class NeuroSMTModelRunner( - val ctx: KContext, - ordinalsPath: String, embeddingPath: String, convPath: String, decoderPath: String -) { - val env = OrtEnvironment.getEnvironment() - - val ordinalEncoder = OrdinalEncoder(ordinalsPath) - val embeddingLayer = ONNXModel(env, embeddingPath) - val convLayer = ONNXModel(env, convPath) - - val decoder = ONNXModel(env, decoderPath) - - fun run(expr: KExpr<*>): Float { - val encoder = ExprEncoder(ctx, env, ordinalEncoder, embeddingLayer, convLayer) - val exprFeatures = encoder.encodeExpr(expr) - val result = decoder.forward(mapOf("expr_features" to exprFeatures)) - val logit = result.floatBuffer[0] - - return 1f / (1f + exp(-logit)) - } -} - -const val UNKNOWN_VALUE = 1999 - -class OrdinalEncoder(ordinalsPath: String, private val unknownValue: Int = UNKNOWN_VALUE) { - private val lookup = HashMap() - - init { - Files.lines(Path.of(ordinalsPath)).asSequence().forEachIndexed { index, s -> - lookup[s] = index - } - } - - fun getOrdinal(s: String): Int { - return lookup[s] ?: unknownValue - } } \ No newline at end of file diff --git a/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/OrdinalEncoder.kt b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/OrdinalEncoder.kt new file mode 100644 index 000000000..7fc472d3e --- /dev/null +++ b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/OrdinalEncoder.kt @@ -0,0 +1,21 @@ +package io.ksmt.solver.neurosmt.runtime + +import java.nio.file.Files +import java.nio.file.Path +import kotlin.streams.asSequence + +const val UNKNOWN_VALUE = 1999 + +class OrdinalEncoder(ordinalsPath: String, private val unknownValue: Int = UNKNOWN_VALUE) { + private val lookup = HashMap() + + init { + Files.lines(Path.of(ordinalsPath)).asSequence().forEachIndexed { index, s -> + lookup[s] = index + } + } + + fun getOrdinal(s: String): Int { + return lookup[s] ?: unknownValue + } +} \ No newline at end of file diff --git a/sandbox/src/main/kotlin/Main.kt b/sandbox/src/main/kotlin/Main.kt index 5c7413eae..61d8807c1 100644 --- a/sandbox/src/main/kotlin/Main.kt +++ b/sandbox/src/main/kotlin/Main.kt @@ -124,7 +124,10 @@ const val THRESHOLD = 0.5 fun main() { - val ctx = KContext(simplificationMode = KContext.SimplificationMode.NO_SIMPLIFY) + val ctx = KContext( + astManagementMode = KContext.AstManagementMode.NO_GC, + simplificationMode = KContext.SimplificationMode.NO_SIMPLIFY + ) val pathToDataset = "formulas" val files = Files.walk(Path.of(pathToDataset)).filter { it.isRegularFile() }.toList() From 9b777cbd0b10fabd7932cd0f04c8713c89ea6a4e Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Wed, 23 Aug 2023 10:57:19 +0300 Subject: [PATCH 41/52] update target type + additional metrics --- .../src/main/python/GraphDataloader.py | 2 +- .../src/main/python/LightningModel.py | 84 +++++++++++-------- .../kotlin/io/ksmt/solver/neurosmt/Utils.kt | 2 +- 3 files changed, 50 insertions(+), 38 deletions(-) diff --git a/ksmt-neurosmt/src/main/python/GraphDataloader.py b/ksmt-neurosmt/src/main/python/GraphDataloader.py index dbb22e8b0..d76d552bf 100644 --- a/ksmt-neurosmt/src/main/python/GraphDataloader.py +++ b/ksmt-neurosmt/src/main/python/GraphDataloader.py @@ -26,7 +26,7 @@ def __init__(self, graph_data): self.graphs = [Graph( x=torch.tensor(nodes, dtype=torch.int32), edge_index=torch.tensor(edges, dtype=torch.int64).t(), - y=torch.tensor([[label]], dtype=torch.float), + y=torch.tensor([[label]], dtype=torch.int32), depth=torch.tensor(depths, dtype=torch.int32) ) for nodes, edges, label, depths in graph_data] diff --git a/ksmt-neurosmt/src/main/python/LightningModel.py b/ksmt-neurosmt/src/main/python/LightningModel.py index f4eccc0c1..c296e4f11 100644 --- a/ksmt-neurosmt/src/main/python/LightningModel.py +++ b/ksmt-neurosmt/src/main/python/LightningModel.py @@ -1,11 +1,12 @@ from sklearn.metrics import classification_report import torch +import torch.nn as nn import torch.nn.functional as F import pytorch_lightning as pl -from torchmetrics.classification import BinaryAccuracy, BinaryAUROC, BinaryConfusionMatrix +from torchmetrics.classification import BinaryAccuracy, BinaryConfusionMatrix, BinaryAUROC, BinaryPrecisionAtFixedRecall from Model import Model @@ -26,9 +27,13 @@ def __init__(self): self.val_targets = [] self.acc = BinaryAccuracy() - self.roc_auc = BinaryAUROC() self.confusion_matrix = BinaryConfusionMatrix() + self.roc_auc = BinaryAUROC() + self.precisions_at_recall = nn.ModuleList([ + BinaryPrecisionAtFixedRecall(rec) for rec in [0.75, 0.8, 0.85, 0.9, 0.95, 0.975, 0.99, 1.0] + ]) + def forward(self, node_labels, edges, depths, root_ptrs): return self.model(node_labels, edges, depths, root_ptrs) @@ -40,7 +45,7 @@ def configure_optimizers(self): def training_step(self, train_batch, batch_idx): out = self.model(*unpack_batch(train_batch)) - loss = F.binary_cross_entropy_with_logits(out, train_batch.y) + loss = F.binary_cross_entropy_with_logits(out, train_batch.y.to(torch.float)) out = F.sigmoid(out) @@ -59,20 +64,24 @@ def training_step(self, train_batch, batch_idx): return loss - def shared_val_test_step(self, batch, batch_idx, metric_name): + def shared_val_test_step(self, batch, batch_idx, target_name): out = self.model(*unpack_batch(batch)) - loss = F.binary_cross_entropy_with_logits(out, batch.y) + loss = F.binary_cross_entropy_with_logits(out, batch.y.to(torch.float)) out = F.sigmoid(out) + self.roc_auc.update(out, batch.y) + for precision_at_recall in self.precisions_at_recall: + precision_at_recall.update(out, batch.y) + self.log( - f"{metric_name}/loss", loss.float(), + f"{target_name}/loss", loss.float(), prog_bar=True, logger=True, on_step=False, on_epoch=True, batch_size=batch.num_graphs ) self.log( - f"{metric_name}/acc", self.acc(out, batch.y), + f"{target_name}/acc", self.acc(out, batch.y), prog_bar=True, logger=True, on_step=False, on_epoch=True, batch_size=batch.num_graphs @@ -83,6 +92,36 @@ def shared_val_test_step(self, batch, batch_idx, metric_name): return loss + def shared_val_test_epoch_end(self, target_name): + print("\n", flush=True) + + all_outputs = torch.flatten(torch.cat(self.val_outputs)) + all_targets = torch.flatten(torch.cat(self.val_targets)) + + self.val_outputs.clear() + self.val_targets.clear() + + self.print_confusion_matrix_and_classification_report(all_outputs, all_targets) + + self.log( + f"{target_name}/roc-auc", self.roc_auc.compute(), + prog_bar=True, logger=True, + on_step=False, on_epoch=True + ) + self.roc_auc.reset() + + for precision_at_recall in self.precisions_at_recall: + precision = precision_at_recall.compute()[0] + recall = precision_at_recall.min_recall + + self.log( + f"{target_name}/precision_at_{recall}", precision, + prog_bar=False, logger=True, + on_step=False, on_epoch=True + ) + + precision_at_recall.reset() + def validation_step(self, val_batch, batch_idx): return self.shared_val_test_step(val_batch, batch_idx, "val") @@ -107,37 +146,10 @@ def print_confusion_matrix_and_classification_report(self, all_outputs, all_targ ) def on_validation_epoch_end(self): - print("\n", flush=True) - - all_outputs = torch.flatten(torch.cat(self.val_outputs)) - all_targets = torch.flatten(torch.cat(self.val_targets)) - - self.val_outputs.clear() - self.val_targets.clear() - - self.log( - "val/roc-auc", self.roc_auc(all_outputs, all_targets), - prog_bar=True, logger=True, - on_step=False, on_epoch=True - ) - - self.print_confusion_matrix_and_classification_report(all_outputs, all_targets) + self.shared_val_test_epoch_end("val") def test_step(self, test_batch, batch_idx): return self.shared_val_test_step(test_batch, batch_idx, "test") def on_test_epoch_end(self): - all_outputs = torch.flatten(torch.cat(self.val_outputs)) - all_targets = torch.flatten(torch.cat(self.val_targets)) - - self.val_outputs.clear() - self.val_targets.clear() - - self.log( - "test/roc-auc", self.roc_auc(all_outputs, all_targets), - prog_bar=True, logger=True, - on_step=False, on_epoch=True - ) - - print("\n") - self.print_confusion_matrix_and_classification_report(all_outputs, all_targets) + self.shared_val_test_epoch_end("test") diff --git a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/Utils.kt b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/Utils.kt index 7466b8f20..1623cf479 100644 --- a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/Utils.kt +++ b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/Utils.kt @@ -61,7 +61,7 @@ fun deserialize(ctx: KContext, inputStream: InputStream): List> val emptyRdSerializationCtx = SerializationCtx(Serializers()) val buffer = UnsafeBuffer(inputStream.readBytes()) - val expressions: MutableList> = mutableListOf() + val expressions = mutableListOf>() while (true) { try { From ff4dc625058d2aa5eb8c4530f2878d0a6fcbd72a Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Wed, 23 Aug 2023 20:15:55 +0300 Subject: [PATCH 42/52] refactor graph extractor and some other stuff --- .../solver/neurosmt/runtime/ExprEncoder.kt | 34 +++++++------ .../src/main/python/prepare-train-val-test.py | 4 ++ ksmt-neurosmt/src/main/python/utils.py | 2 + .../solver/neurosmt/FormulaGraphExtractor.kt | 48 +++++++++++-------- 4 files changed, 53 insertions(+), 35 deletions(-) diff --git a/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ExprEncoder.kt b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ExprEncoder.kt index aa1b5f228..14c590907 100644 --- a/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ExprEncoder.kt +++ b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ExprEncoder.kt @@ -90,27 +90,31 @@ class ExprEncoder( } private fun calcSymbolicVariableState(symbol: KConst): OnnxTensor { - val key = when (symbol.decl.sort) { - is KBoolSort -> "SYMBOLIC;Bool" - is KBvSort -> "SYMBOLIC;BitVec" - is KFpSort -> "SYMBOLIC;FP" - is KFpRoundingModeSort -> "SYMBOLIC;FP_RM" - is KArraySortBase<*> -> "SYMBOLIC;Array" - is KUninterpretedSort -> "SYMBOLIC;Unint" - else -> error("unknown symbolic sort: ${symbol.decl.sort::class.simpleName}") + val sort = symbol.decl.sort + + val key = "SYMBOLIC;" + when (sort) { + is KBoolSort -> "Bool" + is KBvSort -> "BitVec" + is KFpSort -> "FP" + is KFpRoundingModeSort -> "FP_RM" + is KArraySortBase<*> -> "Array" + is KUninterpretedSort -> sort.name + else -> error("unknown symbolic sort: ${sort::class.simpleName}") } return getNodeEmbedding(key) } private fun calcValueState(value: KInterpretedValue): OnnxTensor { - val key = when (value.decl.sort) { - is KBoolSort -> "VALUE;Bool" - is KBvSort -> "VALUE;BitVec" - is KFpSort -> "VALUE;FP" - is KFpRoundingModeSort -> "VALUE;FP_RM" - is KArraySortBase<*> -> "VALUE;Array" - is KUninterpretedSort -> "VALUE;Unint" + val sort = value.decl.sort + + val key = "VALUE;" + when (sort) { + is KBoolSort -> "Bool" + is KBvSort -> "BitVec" + is KFpSort -> "FP" + is KFpRoundingModeSort -> "FP_RM" + is KArraySortBase<*> -> "Array" + is KUninterpretedSort -> sort.name else -> error("unknown value sort: ${value.decl.sort::class.simpleName}") } diff --git a/ksmt-neurosmt/src/main/python/prepare-train-val-test.py b/ksmt-neurosmt/src/main/python/prepare-train-val-test.py index b9d5eb0b5..94b077db6 100755 --- a/ksmt-neurosmt/src/main/python/prepare-train-val-test.py +++ b/ksmt-neurosmt/src/main/python/prepare-train-val-test.py @@ -79,6 +79,10 @@ def pick_best_split(groups): need_test = int(samples_cnt * test_qty) need_train = samples_cnt - need_val - need_test + print("picking best split with existing groups") + print(f"need: {need_train} (train) | {need_val} (val) | {need_test} (test)") + print(flush=True) + best = None for _ in trange(attempts): diff --git a/ksmt-neurosmt/src/main/python/utils.py b/ksmt-neurosmt/src/main/python/utils.py index 4bf95b4ae..ffef5283d 100644 --- a/ksmt-neurosmt/src/main/python/utils.py +++ b/ksmt-neurosmt/src/main/python/utils.py @@ -18,6 +18,8 @@ def train_val_test_indices(cnt, val_qty=0.15, test_qty=0.1): def select_paths_with_suitable_samples_and_transform_to_paths_from_root(path_to_dataset_root, paths): + print("\nloading paths", flush=True) + correct_paths = [] for path in tqdm(paths): operators, edges, _ = read_graph_by_path(path, max_size=MAX_FORMULA_SIZE, max_depth=MAX_FORMULA_DEPTH) diff --git a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/FormulaGraphExtractor.kt b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/FormulaGraphExtractor.kt index 4fb6ab4db..0acc2372c 100644 --- a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/FormulaGraphExtractor.kt +++ b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/FormulaGraphExtractor.kt @@ -30,39 +30,47 @@ class FormulaGraphExtractor( else -> writeApp(expr) } + writer.newLine() + return expr } - fun writeSymbolicVariable(symbol: KConst) { - when (symbol.decl.sort) { - is KBoolSort -> writer.write("SYMBOLIC; Bool\n") - is KBvSort -> writer.write("SYMBOLIC; BitVec\n") - is KFpSort -> writer.write("SYMBOLIC; FP\n") - is KFpRoundingModeSort -> writer.write("SYMBOLIC; FP_RM\n") - is KArraySortBase<*> -> writer.write("SYMBOLIC; Array\n") - is KUninterpretedSort -> writer.write("SYMBOLIC; Unint\n") - else -> error("unknown symbolic sort: ${symbol.sort::class.simpleName}") + private fun writeSymbolicVariable(symbol: KConst) { + writer.write("SYMBOLIC;") + + val sort = symbol.decl.sort + when (sort) { + is KBoolSort -> writer.write("Bool") + is KBvSort -> writer.write("BitVec") + is KFpSort -> writer.write("FP") + is KFpRoundingModeSort -> writer.write("FP_RM") + is KArraySortBase<*> -> writer.write("Array") + is KUninterpretedSort -> writer.write(sort.name) + else -> error("unknown symbolic sort: ${sort::class.simpleName}") } } - fun writeValue(value: KInterpretedValue) { - when (value.decl.sort) { - is KBoolSort -> writer.write("VALUE; Bool\n") - is KBvSort -> writer.write("VALUE; BitVec\n") - is KFpSort -> writer.write("VALUE; FP\n") - is KFpRoundingModeSort -> writer.write("VALUE; FP_RM\n") - is KArraySortBase<*> -> writer.write("VALUE; Array\n") - is KUninterpretedSort -> writer.write("VALUE; Unint\n") - else -> error("unknown value sort: ${value.decl.sort::class.simpleName}") + private fun writeValue(value: KInterpretedValue) { + writer.write("VALUE;") + + val sort = value.decl.sort + when (sort) { + is KBoolSort -> writer.write("Bool") + is KBvSort -> writer.write("BitVec") + is KFpSort -> writer.write("FP") + is KFpRoundingModeSort -> writer.write("FP_RM") + is KArraySortBase<*> -> writer.write("Array") + is KUninterpretedSort -> writer.write(sort.name) + else -> error("unknown value sort: ${sort::class.simpleName}") } } - fun writeApp(expr: KApp) { + private fun writeApp(expr: KApp) { writer.write("${expr.decl.name};") + for (child in expr.args) { writer.write(" ${exprToVertexID[child]}") } - writer.newLine() } fun extractGraph() { From 8460715f8d351aacd3e3d1160d5bdf776ca7a959 Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Thu, 24 Aug 2023 12:37:09 +0300 Subject: [PATCH 43/52] add standalone model runner for hand benchmarking --- ksmt-neurosmt/build.gradle.kts | 29 +++- .../neurosmt/runtime/standalone/CLArgs.kt | 30 ++++ .../standalone/StandaloneModelRunner.kt | 150 ++++++++++++++++++ 3 files changed, 207 insertions(+), 2 deletions(-) create mode 100644 ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/standalone/CLArgs.kt create mode 100644 ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/standalone/StandaloneModelRunner.kt diff --git a/ksmt-neurosmt/build.gradle.kts b/ksmt-neurosmt/build.gradle.kts index b2bc82195..19b0fd55c 100644 --- a/ksmt-neurosmt/build.gradle.kts +++ b/ksmt-neurosmt/build.gradle.kts @@ -1,5 +1,5 @@ plugins { - kotlin("jvm") // id("io.ksmt.ksmt-base") -- need to be returned in future + id("io.ksmt.ksmt-base") } repositories { @@ -8,7 +8,32 @@ repositories { dependencies { implementation(project(":ksmt-core")) - // implementation(project(":ksmt-z3")) + implementation(project("utils")) implementation("com.microsoft.onnxruntime:onnxruntime:1.15.1") + implementation("com.github.ajalt.clikt:clikt:3.5.2") +} + +tasks { + val standaloneModelRunnerFatJar = register("standaloneModelRunnerFatJar") { + dependsOn.addAll(listOf("compileJava", "compileKotlin", "processResources")) + + archiveFileName.set("standalone-model-runner.jar") + destinationDirectory.set(File(".")) + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + + manifest { + attributes(mapOf("Main-Class" to "io.ksmt.solver.neurosmt.runtime.standalone.StandaloneModelRunnerKt")) + } + + val sourcesMain = sourceSets.main.get() + val contents = configurations.runtimeClasspath.get() + .map { if (it.isDirectory) it else zipTree(it) } + sourcesMain.output + + from(contents) + } + + build { + dependsOn(standaloneModelRunnerFatJar) + } } \ No newline at end of file diff --git a/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/standalone/CLArgs.kt b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/standalone/CLArgs.kt new file mode 100644 index 000000000..92cd23848 --- /dev/null +++ b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/standalone/CLArgs.kt @@ -0,0 +1,30 @@ +package io.ksmt.solver.neurosmt.runtime.standalone + +import com.github.ajalt.clikt.core.CliktCommand +import com.github.ajalt.clikt.parameters.options.default +import com.github.ajalt.clikt.parameters.options.option +import com.github.ajalt.clikt.parameters.options.required +import com.github.ajalt.clikt.parameters.types.double + +class CLArgs : CliktCommand() { + override fun run() = Unit + + val datasetPath: String by + option("-D", "--data", help = "path to dataset").required() + + val ordinalsPath: String by + option("-o", "--ordinals", help = "path to ordinal encoder categories").required() + + val embeddingsPath: String by + option("-e", "--embeddings", help = "path to embeddings layer").required() + + val convPath: String by + option("-c", "--conv", help = "path to conv layer").required() + + val decoderPath: String by + option("-d", "--decoder", help = "path to decoder").required() + + val threshold: Double by + option("-t", "--threshold", help = "probability threshold for sat/unsat decision") + .double().default(0.5) +} \ No newline at end of file diff --git a/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/standalone/StandaloneModelRunner.kt b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/standalone/StandaloneModelRunner.kt new file mode 100644 index 000000000..10357695b --- /dev/null +++ b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/standalone/StandaloneModelRunner.kt @@ -0,0 +1,150 @@ +package io.ksmt.solver.neurosmt.runtime.standalone + +import com.github.ajalt.clikt.core.NoSuchParameter +import com.github.ajalt.clikt.core.PrintHelpMessage +import com.github.ajalt.clikt.core.UsageError +import io.ksmt.KContext +import io.ksmt.solver.KSolverStatus +import io.ksmt.solver.neurosmt.deserialize +import io.ksmt.solver.neurosmt.runtime.NeuroSMTModelRunner +import java.io.File +import java.io.FileInputStream +import java.nio.file.Files +import java.nio.file.Path +import kotlin.io.path.isRegularFile +import kotlin.io.path.name +import kotlin.io.path.pathString + +fun printStats( + sat: Int, unsat: Int, skipped: Int, + ok: Int, fail: Int, + confusionMatrix: Map, Int> +) { + println("$sat sat; $unsat unsat; $skipped skipped") + println("$ok ok; $fail failed") + println() + + val satToSat = confusionMatrix.getOrDefault(KSolverStatus.SAT to KSolverStatus.SAT, 0) + val unsatToUnsat = confusionMatrix.getOrDefault(KSolverStatus.UNSAT to KSolverStatus.UNSAT, 0) + val unsatToSat = confusionMatrix.getOrDefault(KSolverStatus.UNSAT to KSolverStatus.SAT, 0) + val satToUnsat = confusionMatrix.getOrDefault(KSolverStatus.SAT to KSolverStatus.UNSAT, 0) + + println("target vs output") + println(" sat vs sat : $satToSat") + println("unsat vs unsat : $unsatToUnsat") + println("unsat vs sat : $unsatToSat") + println(" sat vs unsat : $satToUnsat") + println() +} + +fun main(args: Array) { + val arguments = CLArgs() + try { + arguments.parse(args) + } catch (e: PrintHelpMessage) { + println(arguments.getFormattedHelp()) + return + } catch (e: NoSuchParameter) { + println(e.message) + return + } catch (e: UsageError) { + println(e.message) + return + } catch (e: Exception) { + println("Error!\n$e") + return + } + + val ctx = KContext( + astManagementMode = KContext.AstManagementMode.NO_GC, + simplificationMode = KContext.SimplificationMode.NO_SIMPLIFY + ) + + val files = Files.walk(Path.of(arguments.datasetPath)).filter { it.isRegularFile() }.toList() + + val runner = NeuroSMTModelRunner( + ctx, + ordinalsPath = arguments.ordinalsPath, + embeddingPath = arguments.embeddingsPath, + convPath = arguments.convPath, + decoderPath = arguments.decoderPath + ) + + var sat = 0; var unsat = 0; var skipped = 0 + var ok = 0; var fail = 0 + + val confusionMatrix = mutableMapOf, Int>() + + files.forEachIndexed sample@{ ind, it -> + if (ind % 100 == 0) { + println("\n#$ind:") + printStats(sat, unsat, skipped, ok, fail, confusionMatrix) + } + + val sampleFile = File(it.pathString) + + val assertList = try { + deserialize(ctx, FileInputStream(sampleFile)) + } catch (e: Exception) { + skipped++ + return@sample + } + + val answer = when { + it.name.endsWith("-sat") -> KSolverStatus.SAT + it.name.endsWith("-unsat") -> KSolverStatus.UNSAT + else -> KSolverStatus.UNKNOWN + } + + if (answer == KSolverStatus.UNKNOWN) { + skipped++ + return@sample + } + + val prob = with(ctx) { + val formula = when (assertList.size) { + 0 -> { + skipped++ + return@sample + } + 1 -> { + assertList[0] + } + else -> { + mkAnd(assertList) + } + } + + runner.run(formula) + } + + val output = if (prob < arguments.threshold) { + KSolverStatus.UNSAT + } else { + KSolverStatus.SAT + } + + when (answer) { + KSolverStatus.SAT -> sat++ + KSolverStatus.UNSAT -> unsat++ + else -> { /* can't happen */ } + } + + if (output == answer) { + ok++ + } else { + fail++ + } + + confusionMatrix.compute(answer to output) { _, v -> + if (v == null) { + 1 + } else { + v + 1 + } + } + } + + println() + printStats(sat, unsat, skipped, ok, fail, confusionMatrix) +} \ No newline at end of file From 8b42598a9ea36d48f3b7b9ff98da6cadc1d5ced2 Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Thu, 24 Aug 2023 12:49:08 +0300 Subject: [PATCH 44/52] add model export script --- ksmt-neurosmt/src/main/python/export-model.py | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 ksmt-neurosmt/src/main/python/export-model.py diff --git a/ksmt-neurosmt/src/main/python/export-model.py b/ksmt-neurosmt/src/main/python/export-model.py new file mode 100644 index 000000000..9a615a314 --- /dev/null +++ b/ksmt-neurosmt/src/main/python/export-model.py @@ -0,0 +1,62 @@ +#!/usr/bin/python3 + +import sys + +import torch + +from LightningModel import LightningModel + +from Model import EMBEDDING_DIM + + +if __name__ == "__main__": + + pl_model = LightningModel().load_from_checkpoint(sys.argv[1], map_location=torch.device("cpu")) + pl_model.eval() + + # input example + node_labels = torch.unsqueeze(torch.arange(0, 9).to(torch.int32), 1) + node_features = pl_model.model.encoder.embedding(node_labels.squeeze()) + edges = torch.tensor([ + [0, 0, 0, 1, 2, 3, 4, 4, 5, 6, 7], + [1, 2, 3, 3, 7, 4, 5, 6, 8, 8, 8] + ], dtype=torch.int64) + + torch.onnx.export( + pl_model.model.encoder.embedding, + (node_labels,), + sys.argv[2], # path to model file + opset_version=18, + input_names=["node_labels"], + output_names=["out"], + dynamic_axes={ + "node_labels": {0: "nodes_number"} + } + ) + + torch.onnx.export( + pl_model.model.encoder.conv, + (node_features, edges), + sys.argv[3], # path to model file + opset_version=18, + input_names=["node_features", "edges"], + output_names=["out"], + dynamic_axes={ + "node_features": {0: "nodes_number"}, + "edges": {1: "edges_number"} + } + ) + + torch.onnx.export( + pl_model.model.decoder, + (torch.rand((1, EMBEDDING_DIM)),), + sys.argv[4], # path to model file + opset_version=18, + input_names=["expr_features"], + output_names=["out"], + dynamic_axes={ + "expr_features": {0: "batch_size"} + } + ) + + print("success!") From 5cbfbb84562244493b93a874724cb7d76a2aaaa1 Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Thu, 24 Aug 2023 19:40:49 +0300 Subject: [PATCH 45/52] add type hints, some comments and global constants in external file --- ksmt-neurosmt/src/main/python/Decoder.py | 16 ++++++--- ksmt-neurosmt/src/main/python/Encoder.py | 27 ++++++++++---- .../src/main/python/GlobalConstants.py | 11 ++++++ .../src/main/python/GraphDataloader.py | 28 ++++++++------- ksmt-neurosmt/src/main/python/GraphReader.py | 35 +++++++++++-------- .../src/main/python/LightningModel.py | 17 ++++++--- ksmt-neurosmt/src/main/python/Model.py | 25 +++++++++---- .../src/main/python/check-split-leaks.py | 5 +-- .../src/main/python/create-ordinal-encoder.py | 4 +-- ksmt-neurosmt/src/main/python/export-model.py | 3 +- .../src/main/python/prepare-train-val-test.py | 2 +- ksmt-neurosmt/src/main/python/sandbox.py | 2 +- ksmt-neurosmt/src/main/python/train.py | 6 ++-- ksmt-neurosmt/src/main/python/utils.py | 19 ++++++---- ksmt-neurosmt/src/main/python/validate.py | 3 +- 15 files changed, 136 insertions(+), 67 deletions(-) create mode 100644 ksmt-neurosmt/src/main/python/GlobalConstants.py diff --git a/ksmt-neurosmt/src/main/python/Decoder.py b/ksmt-neurosmt/src/main/python/Decoder.py index a3740ccfe..7cb9d61e2 100644 --- a/ksmt-neurosmt/src/main/python/Decoder.py +++ b/ksmt-neurosmt/src/main/python/Decoder.py @@ -1,18 +1,26 @@ +import torch import torch.nn as nn - -DECODER_LAYERS = 4 +from GlobalConstants import DECODER_LAYERS class Decoder(nn.Module): - def __init__(self, hidden_dim): + def __init__(self, hidden_dim: int): super().__init__() self.act = nn.ReLU() self.linears = nn.ModuleList([nn.Linear(hidden_dim, hidden_dim) for _ in range(DECODER_LAYERS - 1)]) self.out = nn.Linear(hidden_dim, 1) - def forward(self, x): + def forward(self, x: torch.Tensor) -> torch.Tensor: + """ + decoder forward pass + + :param x: torch.Tensor of shape [batch size, hidden dimension size] (dtype=float) + :return: torch.Tensor of shape [batch size, 1] (dtype=float) -- + each element is a logit for probability of formula to be SAT + """ + for layer in self.linears: x = layer(x) x = self.act(x) diff --git a/ksmt-neurosmt/src/main/python/Encoder.py b/ksmt-neurosmt/src/main/python/Encoder.py index e628a0d7d..380b774f0 100644 --- a/ksmt-neurosmt/src/main/python/Encoder.py +++ b/ksmt-neurosmt/src/main/python/Encoder.py @@ -3,22 +3,35 @@ from torch_geometric.nn import SAGEConv - -EMBEDDINGS_CNT = 2000 +from GlobalConstants import EMBEDDINGS_CNT class Encoder(nn.Module): - def __init__(self, hidden_dim): + def __init__(self, hidden_dim: int): super().__init__() self.embedding = nn.Embedding(EMBEDDINGS_CNT, hidden_dim) self.conv = SAGEConv(hidden_dim, hidden_dim, "mean", root_weight=True, project=True) - # self.conv = GCNConv(hidden_dim, hidden_dim, add_self_loops=False) - # self.conv = GATConv(hidden_dim, hidden_dim, add_self_loops=False) # this can't be exported to ONNX - # self.conv = TransformerConv(hidden_dim, hidden_dim, root_weight=False) # this can't be exported to ONNX + # other options (not supported yet) + # self.conv = GATConv(hidden_dim, hidden_dim, add_self_loops=True) # this can't be exported to ONNX + # self.conv = TransformerConv(hidden_dim, hidden_dim, root_weight=True) # this can't be exported to ONNX + + def forward( + self, node_labels: torch.Tensor, edges: torch.Tensor, depths: torch.Tensor, root_ptrs: torch.Tensor + ) -> torch.Tensor: + """ + encoder forward pass + + :param node_labels: torch.Tensor of shape [number of nodes in batch, 1] (dtype=int32) + :param edges: torch.Tensor of shape [2, number of edges in batch] (dtype=int64) + :param depths: torch.Tensor of shape [number of nodes in batch] (dtype=int32) + :param root_ptrs: torch.Tensor of shape [batch size + 1] (dtype=int32) -- + pointers to root of graph for each expression + :return: torch.Tensor of shape [batch size, hidden dimension size] (dtype=float) -- + embeddings for each expression + """ - def forward(self, node_labels, edges, depths, root_ptrs): node_features = self.embedding(node_labels.squeeze()) depth = depths.max() diff --git a/ksmt-neurosmt/src/main/python/GlobalConstants.py b/ksmt-neurosmt/src/main/python/GlobalConstants.py new file mode 100644 index 000000000..bff231712 --- /dev/null +++ b/ksmt-neurosmt/src/main/python/GlobalConstants.py @@ -0,0 +1,11 @@ +BATCH_SIZE = 1024 +MAX_FORMULA_SIZE = 10000 +MAX_FORMULA_DEPTH = 2500 +NUM_WORKERS = 32 +METADATA_PATH = "__meta" + +EMBEDDINGS_CNT = 2000 +EMBEDDING_DIM = 32 +DECODER_LAYERS = 4 + +MAX_EPOCHS = 200 diff --git a/ksmt-neurosmt/src/main/python/GraphDataloader.py b/ksmt-neurosmt/src/main/python/GraphDataloader.py index d76d552bf..f4672e130 100644 --- a/ksmt-neurosmt/src/main/python/GraphDataloader.py +++ b/ksmt-neurosmt/src/main/python/GraphDataloader.py @@ -1,4 +1,5 @@ import os +from typing import Literal import numpy as np import joblib @@ -10,19 +11,13 @@ from torch_geometric.data import Data as Graph from torch_geometric.loader import DataLoader +from GlobalConstants import BATCH_SIZE, MAX_FORMULA_SIZE, MAX_FORMULA_DEPTH, NUM_WORKERS, METADATA_PATH from GraphReader import read_graph_by_path -BATCH_SIZE = 1024 -MAX_FORMULA_SIZE = 10000 -MAX_FORMULA_DEPTH = 2500 -NUM_WORKERS = 32 -METADATA_PATH = "__meta" - - +# torch Dataset class GraphDataset(Dataset): def __init__(self, graph_data): - self.graphs = [Graph( x=torch.tensor(nodes, dtype=torch.int32), edge_index=torch.tensor(edges, dtype=torch.int64).t(), @@ -37,7 +32,10 @@ def __getitem__(self, index): return self.graphs[index] -def load_data(paths_to_datasets, target): +# load all samples from all datasets and return them as a list of tuples +def load_data(paths_to_datasets: list[str], target: Literal["train", "val", "test"])\ + -> list[tuple[list[str], list[tuple[int, int]], int, list[int]]]: + data = [] print(f"loading {target}") @@ -52,7 +50,7 @@ def load_data(paths_to_datasets, target): path_to_sample, max_size=MAX_FORMULA_SIZE, max_depth=MAX_FORMULA_DEPTH ) - if operators is None: + if operators is None or edges is None or depths is None: continue if len(edges) == 0: @@ -71,7 +69,10 @@ def load_data(paths_to_datasets, target): return data -def get_dataloader(paths_to_datasets, target, path_to_ordinal_encoder): +# load samples from all datasets, transform them and return them in a Dataloader object +def get_dataloader(paths_to_datasets: list[str], target: Literal["train", "val", "test"], path_to_ordinal_encoder: str)\ + -> DataLoader: + print(f"creating dataloader for {target}") print("loading data") @@ -82,9 +83,12 @@ def get_dataloader(paths_to_datasets, target, path_to_ordinal_encoder): print("loading encoder") encoder = joblib.load(path_to_ordinal_encoder) - def transform(data_for_one_sample): + def transform(data_for_one_sample: tuple[list[str], list[tuple[int, int]], int, list[int]])\ + -> tuple[list[str], list[tuple[int, int]], int, list[int]]: + nodes, edges, label, depths = data_for_one_sample nodes = encoder.transform(np.array(nodes).reshape(-1, 1)) + return nodes, edges, label, depths print("transforming") diff --git a/ksmt-neurosmt/src/main/python/GraphReader.py b/ksmt-neurosmt/src/main/python/GraphReader.py index 4be7a28c7..741d7b642 100644 --- a/ksmt-neurosmt/src/main/python/GraphReader.py +++ b/ksmt-neurosmt/src/main/python/GraphReader.py @@ -1,4 +1,5 @@ from enum import Enum +from typing import Union, TextIO class VertexType(Enum): @@ -7,7 +8,7 @@ class VertexType(Enum): APP = 2 -def get_vertex_type(name): +def get_vertex_type(name: str): if name == "SYMBOLIC": return VertexType.SYMBOLIC @@ -17,50 +18,56 @@ def get_vertex_type(name): return VertexType.APP -def process_line(line, v, operators, edges, depth): +def process_line(line: str, cur_vertex: int, operators: list[str], edges: list[tuple[int, int]], depth: list[int]): name, info = line.split(";") info = info.strip() - vertex_type = get_vertex_type(name) + # depth[v] is a length of the longest path from vertex v to any sink in an expression graph depth.append(0) - if vertex_type == VertexType.APP: + if get_vertex_type(name) == VertexType.APP: operators.append(name) children = map(int, info.split(" ")) for u in children: - edges.append([u, v]) - depth[v] = max(depth[v], depth[u] + 1) + edges.append((u, cur_vertex)) + depth[cur_vertex] = max(depth[cur_vertex], depth[u] + 1) + else: operators.append(name + ";" + info) -def read_graph_from_file(inf, max_size, max_depth): +def read_graph_from_file(inf: TextIO, max_size: int, max_depth: int)\ + -> Union[tuple[list[str], list[tuple[int, int]], list[int]], tuple[None, None, None]]: + operators, edges, depth = [], [], [] - v = 0 + cur_vertex = 0 for line in inf.readlines(): line = line.strip() if line.startswith(";"): - continue + continue # lines starting with ";" are considered to be comments try: - process_line(line, v, operators, edges, depth) + process_line(line, cur_vertex, operators, edges, depth) + except Exception as e: print(e, "\n") print(inf.name, "\n") - print(v, line, "\n") + print(cur_vertex, line, "\n") raise e - if v >= max_size or depth[v] > max_depth: + if cur_vertex >= max_size or depth[cur_vertex] > max_depth: return None, None, None - v += 1 + cur_vertex += 1 return operators, edges, depth -def read_graph_by_path(path, max_size, max_depth): +def read_graph_by_path(path: str, max_size: int, max_depth: int)\ + -> Union[tuple[list[str], list[tuple[int, int]], list[int]], tuple[None, None, None]]: + with open(path, "r") as inf: try: return read_graph_from_file(inf, max_size, max_depth) diff --git a/ksmt-neurosmt/src/main/python/LightningModel.py b/ksmt-neurosmt/src/main/python/LightningModel.py index c296e4f11..88d495bae 100644 --- a/ksmt-neurosmt/src/main/python/LightningModel.py +++ b/ksmt-neurosmt/src/main/python/LightningModel.py @@ -1,3 +1,5 @@ +from typing import Literal + from sklearn.metrics import classification_report import torch @@ -8,6 +10,7 @@ from torchmetrics.classification import BinaryAccuracy, BinaryConfusionMatrix, BinaryAUROC, BinaryPrecisionAtFixedRecall +from GlobalConstants import EMBEDDING_DIM from Model import Model @@ -18,14 +21,19 @@ def unpack_batch(batch): class LightningModel(pl.LightningModule): + """ + PyTorch Lightning wrapper for model + """ + def __init__(self): super().__init__() - self.model = Model() + self.model = Model(hidden_dim=EMBEDDING_DIM) self.val_outputs = [] self.val_targets = [] + # different metrics self.acc = BinaryAccuracy() self.confusion_matrix = BinaryConfusionMatrix() @@ -34,6 +42,7 @@ def __init__(self): BinaryPrecisionAtFixedRecall(rec) for rec in [0.75, 0.8, 0.85, 0.9, 0.95, 0.975, 0.99, 1.0] ]) + # forward pass is just the same as in the original model def forward(self, node_labels, edges, depths, root_ptrs): return self.model(node_labels, edges, depths, root_ptrs) @@ -64,7 +73,7 @@ def training_step(self, train_batch, batch_idx): return loss - def shared_val_test_step(self, batch, batch_idx, target_name): + def shared_val_test_step(self, batch, batch_idx, target_name: Literal["val", "test"]): out = self.model(*unpack_batch(batch)) loss = F.binary_cross_entropy_with_logits(out, batch.y.to(torch.float)) @@ -92,7 +101,7 @@ def shared_val_test_step(self, batch, batch_idx, target_name): return loss - def shared_val_test_epoch_end(self, target_name): + def shared_val_test_epoch_end(self, target_name: Literal["val", "test"]): print("\n", flush=True) all_outputs = torch.flatten(torch.cat(self.val_outputs)) @@ -125,7 +134,7 @@ def shared_val_test_epoch_end(self, target_name): def validation_step(self, val_batch, batch_idx): return self.shared_val_test_step(val_batch, batch_idx, "val") - def print_confusion_matrix_and_classification_report(self, all_outputs, all_targets): + def print_confusion_matrix_and_classification_report(self, all_outputs: torch.Tensor, all_targets: torch.Tensor): conf_mat = self.confusion_matrix(all_outputs, all_targets).detach().cpu().numpy() print(" +-------+-----------+-----------+") diff --git a/ksmt-neurosmt/src/main/python/Model.py b/ksmt-neurosmt/src/main/python/Model.py index bdf82781e..91ff8a525 100644 --- a/ksmt-neurosmt/src/main/python/Model.py +++ b/ksmt-neurosmt/src/main/python/Model.py @@ -1,19 +1,32 @@ +import torch import torch.nn as nn from Encoder import Encoder from Decoder import Decoder -EMBEDDING_DIM = 32 - class Model(nn.Module): - def __init__(self): + def __init__(self, hidden_dim: int): super().__init__() - self.encoder = Encoder(hidden_dim=EMBEDDING_DIM) - self.decoder = Decoder(hidden_dim=EMBEDDING_DIM) + self.encoder = Encoder(hidden_dim=hidden_dim) + self.decoder = Decoder(hidden_dim=hidden_dim) + + def forward( + self, node_labels: torch.Tensor, edges: torch.Tensor, depths: torch.Tensor, root_ptrs: torch.Tensor + ) -> torch.Tensor: + """ + model full forward pass (encoder + decoder) + + :param node_labels: torch.Tensor of shape [number of nodes in batch, 1] (dtype=int32) + :param edges: torch.Tensor of shape [2, number of edges in batch] (dtype=int64) + :param depths: torch.Tensor of shape [number of nodes in batch] (dtype=int32) + :param root_ptrs: torch.Tensor of shape [batch size + 1] (dtype=int32) -- + pointers to root of graph for each expression + :return: torch.Tensor of shape [batch size, 1] (dtype=float) -- + each element is a logit for probability of formula to be SAT + """ - def forward(self, node_labels, edges, depths, root_ptrs): x = self.encoder(node_labels, edges, depths, root_ptrs) x = self.decoder(x) diff --git a/ksmt-neurosmt/src/main/python/check-split-leaks.py b/ksmt-neurosmt/src/main/python/check-split-leaks.py index dbae54656..141d7e739 100755 --- a/ksmt-neurosmt/src/main/python/check-split-leaks.py +++ b/ksmt-neurosmt/src/main/python/check-split-leaks.py @@ -2,13 +2,14 @@ import os from argparse import ArgumentParser +from typing import Literal from tqdm import tqdm -from GraphDataloader import METADATA_PATH +from GlobalConstants import METADATA_PATH -def get_groups_set(paths_to_datasets, target): +def get_groups_set(paths_to_datasets: list[str], target: Literal["train", "val", "test"]) -> set: groups = set() print(f"loading {target}") diff --git a/ksmt-neurosmt/src/main/python/create-ordinal-encoder.py b/ksmt-neurosmt/src/main/python/create-ordinal-encoder.py index 534d483ca..0ea1a6d3d 100755 --- a/ksmt-neurosmt/src/main/python/create-ordinal-encoder.py +++ b/ksmt-neurosmt/src/main/python/create-ordinal-encoder.py @@ -8,11 +8,11 @@ from sklearn.preprocessing import OrdinalEncoder +from GlobalConstants import EMBEDDINGS_CNT from GraphDataloader import load_data -from Encoder import EMBEDDINGS_CNT -def create_ordinal_encoder(paths_to_datasets, path_to_ordinal_encoder): +def create_ordinal_encoder(paths_to_datasets: list[str], path_to_ordinal_encoder: str): data = load_data(paths_to_datasets, "train") encoder = OrdinalEncoder( diff --git a/ksmt-neurosmt/src/main/python/export-model.py b/ksmt-neurosmt/src/main/python/export-model.py index 9a615a314..467e37d95 100644 --- a/ksmt-neurosmt/src/main/python/export-model.py +++ b/ksmt-neurosmt/src/main/python/export-model.py @@ -4,10 +4,9 @@ import torch +from GlobalConstants import EMBEDDING_DIM from LightningModel import LightningModel -from Model import EMBEDDING_DIM - if __name__ == "__main__": diff --git a/ksmt-neurosmt/src/main/python/prepare-train-val-test.py b/ksmt-neurosmt/src/main/python/prepare-train-val-test.py index 94b077db6..acede3f30 100755 --- a/ksmt-neurosmt/src/main/python/prepare-train-val-test.py +++ b/ksmt-neurosmt/src/main/python/prepare-train-val-test.py @@ -8,7 +8,7 @@ from pytorch_lightning import seed_everything -from GraphDataloader import METADATA_PATH +from GlobalConstants import METADATA_PATH from utils import train_val_test_indices, align_sat_unsat_sizes, select_paths_with_suitable_samples_and_transform_to_paths_from_root diff --git a/ksmt-neurosmt/src/main/python/sandbox.py b/ksmt-neurosmt/src/main/python/sandbox.py index 8afb28c3b..381836b9d 100755 --- a/ksmt-neurosmt/src/main/python/sandbox.py +++ b/ksmt-neurosmt/src/main/python/sandbox.py @@ -13,7 +13,7 @@ from LightningModel import LightningModel -from Model import EMBEDDING_DIM +from GlobalConstants import EMBEDDING_DIM if __name__ == "__main__": diff --git a/ksmt-neurosmt/src/main/python/train.py b/ksmt-neurosmt/src/main/python/train.py index d04bf457c..3b0dc8dfe 100755 --- a/ksmt-neurosmt/src/main/python/train.py +++ b/ksmt-neurosmt/src/main/python/train.py @@ -9,8 +9,8 @@ from pytorch_lightning.loggers import TensorBoardLogger from pytorch_lightning.callbacks import ModelCheckpoint +from GlobalConstants import MAX_EPOCHS from GraphDataloader import get_dataloader - from LightningModel import LightningModel @@ -31,7 +31,7 @@ def get_args(): if __name__ == "__main__": - # seed_everything(24, workers=True) + # enable usage of nvidia's TensorFloat if available torch.set_float32_matmul_precision("medium") args = get_args() @@ -51,7 +51,7 @@ def get_args(): save_last=True, save_top_k=3, mode="max", auto_insert_metric_name=False, save_on_train_epoch_end=False )], - max_epochs=200, + max_epochs=MAX_EPOCHS, log_every_n_steps=1, enable_checkpointing=True, barebones=False, diff --git a/ksmt-neurosmt/src/main/python/utils.py b/ksmt-neurosmt/src/main/python/utils.py index ffef5283d..b4b7d276e 100644 --- a/ksmt-neurosmt/src/main/python/utils.py +++ b/ksmt-neurosmt/src/main/python/utils.py @@ -3,11 +3,13 @@ import numpy as np from tqdm import tqdm +from GlobalConstants import MAX_FORMULA_SIZE, MAX_FORMULA_DEPTH from GraphReader import read_graph_by_path -from GraphDataloader import MAX_FORMULA_SIZE, MAX_FORMULA_DEPTH -def train_val_test_indices(cnt, val_qty=0.15, test_qty=0.1): +def train_val_test_indices(cnt: int, val_qty: float = 0.15, test_qty: float = 0.1)\ + -> tuple[np.ndarray, np.ndarray, np.ndarray]: + perm = np.arange(cnt) np.random.shuffle(perm) @@ -17,7 +19,10 @@ def train_val_test_indices(cnt, val_qty=0.15, test_qty=0.1): return perm[val_cnt + test_cnt:], perm[:val_cnt], perm[val_cnt:val_cnt + test_cnt] -def select_paths_with_suitable_samples_and_transform_to_paths_from_root(path_to_dataset_root, paths): +# select paths to suitable samples and transform them to paths from dataset root +def select_paths_with_suitable_samples_and_transform_to_paths_from_root(path_to_dataset_root: str, paths: list[str])\ + -> list[str]: + print("\nloading paths", flush=True) correct_paths = [] @@ -36,7 +41,7 @@ def select_paths_with_suitable_samples_and_transform_to_paths_from_root(path_to_ return correct_paths -def align_sat_unsat_sizes_with_upsamping(sat_data, unsat_data): +def align_sat_unsat_sizes_with_upsamping(sat_data: list[str], unsat_data: list[str]) -> tuple[list[str], list[str]]: sat_cnt = len(sat_data) unsat_cnt = len(unsat_data) @@ -46,7 +51,7 @@ def align_sat_unsat_sizes_with_upsamping(sat_data, unsat_data): if sat_cnt < unsat_cnt: sat_indices += list(np.random.choice(np.array(sat_indices), unsat_cnt - sat_cnt, replace=True)) elif sat_cnt > unsat_cnt: - unsat_indices += list(np.random.choice(np.array(unsat_indices), sat_cnt - unsat_cnt, replace=False)) + unsat_indices += list(np.random.choice(np.array(unsat_indices), sat_cnt - unsat_cnt, replace=True)) return ( list(np.array(sat_data, dtype=object)[sat_indices]), @@ -54,7 +59,7 @@ def align_sat_unsat_sizes_with_upsamping(sat_data, unsat_data): ) -def align_sat_unsat_sizes_with_downsamping(sat_data, unsat_data): +def align_sat_unsat_sizes_with_downsamping(sat_data: list[str], unsat_data: list[str]) -> tuple[list[str], list[str]]: sat_cnt = len(sat_data) unsat_cnt = len(unsat_data) @@ -72,7 +77,7 @@ def align_sat_unsat_sizes_with_downsamping(sat_data, unsat_data): ) -def align_sat_unsat_sizes(sat_data, unsat_data, mode): +def align_sat_unsat_sizes(sat_data: list[str], unsat_data: list[str], mode: str) -> tuple[list[str], list[str]]: if mode == "none": return sat_data, unsat_data elif mode == "upsample": diff --git a/ksmt-neurosmt/src/main/python/validate.py b/ksmt-neurosmt/src/main/python/validate.py index 45ce5cd4c..0d8588897 100755 --- a/ksmt-neurosmt/src/main/python/validate.py +++ b/ksmt-neurosmt/src/main/python/validate.py @@ -8,7 +8,6 @@ from pytorch_lightning import Trainer from GraphDataloader import get_dataloader - from LightningModel import LightningModel @@ -29,7 +28,7 @@ def get_args(): if __name__ == "__main__": - # seed_everything(24, workers=True) + # enable usage of nvidia's TensorFloat if available torch.set_float32_matmul_precision("medium") args = get_args() From ce79d98977644edc3a6817b8e9bbd2ad7ecb11e7 Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Fri, 25 Aug 2023 12:15:05 +0300 Subject: [PATCH 46/52] add model run time stats --- .../standalone/StandaloneModelRunner.kt | 31 ++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/standalone/StandaloneModelRunner.kt b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/standalone/StandaloneModelRunner.kt index 10357695b..556e78a62 100644 --- a/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/standalone/StandaloneModelRunner.kt +++ b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/standalone/StandaloneModelRunner.kt @@ -14,6 +14,10 @@ import java.nio.file.Path import kotlin.io.path.isRegularFile import kotlin.io.path.name import kotlin.io.path.pathString +import kotlin.math.roundToLong +import kotlin.math.sqrt +import kotlin.time.ExperimentalTime +import kotlin.time.measureTimedValue fun printStats( sat: Int, unsat: Int, skipped: Int, @@ -37,6 +41,16 @@ fun printStats( println() } +fun printModelRunTimeStats(modelRunTimes: List) { + val mean = modelRunTimes.average() + val std = sqrt(modelRunTimes.map { it - mean }.map { it * it }.average()) + + println("run time:") + println("mean: ${mean.roundToLong()}μs; std: ${std.roundToLong()}μs") + println() +} + +@OptIn(ExperimentalTime::class) fun main(args: Array) { val arguments = CLArgs() try { @@ -70,6 +84,8 @@ fun main(args: Array) { decoderPath = arguments.decoderPath ) + val modelRunTimes = mutableListOf() + var sat = 0; var unsat = 0; var skipped = 0 var ok = 0; var fail = 0 @@ -115,13 +131,18 @@ fun main(args: Array) { } } - runner.run(formula) + val timedVal = measureTimedValue { + runner.run(formula) + } + + modelRunTimes.add(timedVal.duration.inWholeMicroseconds) + timedVal.value } - val output = if (prob < arguments.threshold) { - KSolverStatus.UNSAT - } else { + val output = if (prob > arguments.threshold) { KSolverStatus.SAT + } else { + KSolverStatus.UNSAT } when (answer) { @@ -147,4 +168,6 @@ fun main(args: Array) { println() printStats(sat, unsat, skipped, ok, fail, confusionMatrix) + + printModelRunTimeStats(modelRunTimes) } \ No newline at end of file From 0cda4ac7624ea8f899b9a7bafe990801f5cc3b4c Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Fri, 25 Aug 2023 12:17:07 +0300 Subject: [PATCH 47/52] add requirements.txt --- ksmt-neurosmt/src/main/python/requirements.txt | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 ksmt-neurosmt/src/main/python/requirements.txt diff --git a/ksmt-neurosmt/src/main/python/requirements.txt b/ksmt-neurosmt/src/main/python/requirements.txt new file mode 100644 index 000000000..8517d1de2 --- /dev/null +++ b/ksmt-neurosmt/src/main/python/requirements.txt @@ -0,0 +1,7 @@ +tqdm +numpy +scikit-learn +torch +torch_geometric +pytorch-lightning +torchmetrics From 42ed4c5541e2ee5551eceb362398ce3dd3bd2834 Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Fri, 25 Aug 2023 12:33:21 +0300 Subject: [PATCH 48/52] add basic ksmt integration --- .../ksmt/solver/neurosmt/KNeuroSMTSolver.kt | 44 +++++++++++++++---- .../neurosmt/runtime/NeuroSMTModelRunner.kt | 6 +++ 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/KNeuroSMTSolver.kt b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/KNeuroSMTSolver.kt index 4b889b728..d5e943021 100644 --- a/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/KNeuroSMTSolver.kt +++ b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/KNeuroSMTSolver.kt @@ -5,36 +5,63 @@ import io.ksmt.expr.KExpr import io.ksmt.solver.KModel import io.ksmt.solver.KSolver import io.ksmt.solver.KSolverStatus +import io.ksmt.solver.neurosmt.runtime.NeuroSMTModelRunner import io.ksmt.sort.KBoolSort import kotlin.time.Duration -class KNeuroSMTSolver(private val ctx: KContext) : KSolver { +class KNeuroSMTSolver( + private val ctx: KContext, + ordinalsPath: String, embeddingPath: String, convPath: String, decoderPath: String, + private val threshold: Double = 0.5 +) : KSolver { + + private val modelRunner = NeuroSMTModelRunner(ctx, ordinalsPath, embeddingPath, convPath, decoderPath) + private val asserts = mutableListOf>>(mutableListOf()) + override fun configure(configurator: KNeuroSMTSolverConfiguration.() -> Unit) { TODO("Not yet implemented") } override fun assert(expr: KExpr) { - // TODO("Not yet implemented") + asserts.last().add(expr) } override fun assertAndTrack(expr: KExpr) { - TODO("Not yet implemented") + assert(expr) } override fun push() { - TODO("Not yet implemented") + asserts.add(mutableListOf()) } override fun pop(n: UInt) { - TODO("Not yet implemented") + repeat(n.toInt()) { + asserts.removeLast() + } } override fun check(timeout: Duration): KSolverStatus { - return KSolverStatus.SAT + val prob = with(ctx) { + modelRunner.run(mkAnd(asserts.flatten())) + } + + return if (prob > threshold) { + KSolverStatus.SAT + } else { + KSolverStatus.UNSAT + } } override fun checkWithAssumptions(assumptions: List>, timeout: Duration): KSolverStatus { - TODO("Not yet implemented") + val prob = with(ctx) { + modelRunner.run(mkAnd(asserts.flatten() + assumptions)) + } + + return if (prob > threshold) { + KSolverStatus.SAT + } else { + KSolverStatus.UNSAT + } } override fun model(): KModel { @@ -54,6 +81,7 @@ class KNeuroSMTSolver(private val ctx: KContext) : KSolver Date: Fri, 25 Aug 2023 15:34:36 +0300 Subject: [PATCH 49/52] add comments for kotlin code --- .../kotlin/io/ksmt/solver/neurosmt/runtime/ExprEncoder.kt | 3 +++ .../io/ksmt/solver/neurosmt/runtime/NeuroSMTModelRunner.kt | 2 ++ .../main/kotlin/io/ksmt/solver/neurosmt/runtime/ONNXModel.kt | 1 + .../kotlin/io/ksmt/solver/neurosmt/runtime/OrdinalEncoder.kt | 2 ++ .../neurosmt/runtime/standalone/StandaloneModelRunner.kt | 1 + ksmt-neurosmt/utils/build.gradle.kts | 5 ----- .../kotlin/io/ksmt/solver/neurosmt/FormulaGraphExtractor.kt | 1 + .../utils/src/main/kotlin/io/ksmt/solver/neurosmt/Utils.kt | 4 ++++ .../neurosmt/ksmtBinaryConverter/KSMTBinaryConverter.kt | 1 + .../io/ksmt/solver/neurosmt/smt2Converter/SMT2Converter.kt | 1 + 10 files changed, 16 insertions(+), 5 deletions(-) diff --git a/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ExprEncoder.kt b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ExprEncoder.kt index 14c590907..c7cf9798b 100644 --- a/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ExprEncoder.kt +++ b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ExprEncoder.kt @@ -14,6 +14,9 @@ import java.nio.IntBuffer import java.nio.LongBuffer import java.util.* +// expression encoder +// walks on an expression and calculates state for each node +// based on states for its children (which are already calculated at that moment) class ExprEncoder( override val ctx: KContext, val env: OrtEnvironment, diff --git a/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/NeuroSMTModelRunner.kt b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/NeuroSMTModelRunner.kt index 3f2499cf4..31a481c08 100644 --- a/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/NeuroSMTModelRunner.kt +++ b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/NeuroSMTModelRunner.kt @@ -5,6 +5,8 @@ import io.ksmt.KContext import io.ksmt.expr.KExpr import kotlin.math.exp +// wrapper for NeuroSMT model +// runs the whole model pipeline class NeuroSMTModelRunner( val ctx: KContext, ordinalsPath: String, embeddingPath: String, convPath: String, decoderPath: String diff --git a/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ONNXModel.kt b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ONNXModel.kt index 5711db6c4..928ed65e1 100644 --- a/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ONNXModel.kt +++ b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/ONNXModel.kt @@ -3,6 +3,7 @@ package io.ksmt.solver.neurosmt.runtime import ai.onnxruntime.OnnxTensor import ai.onnxruntime.OrtEnvironment +// wrapper for any exported ONNX model class ONNXModel(env: OrtEnvironment, modelPath: String) : AutoCloseable { val session = env.createSession(modelPath) diff --git a/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/OrdinalEncoder.kt b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/OrdinalEncoder.kt index 7fc472d3e..f3090a8a1 100644 --- a/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/OrdinalEncoder.kt +++ b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/OrdinalEncoder.kt @@ -6,6 +6,8 @@ import kotlin.streams.asSequence const val UNKNOWN_VALUE = 1999 +// wrapper for single-feature sklearn OrdinalEncoder (for each string we should provide its ordinal) +// used to convert strings to integers class OrdinalEncoder(ordinalsPath: String, private val unknownValue: Int = UNKNOWN_VALUE) { private val lookup = HashMap() diff --git a/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/standalone/StandaloneModelRunner.kt b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/standalone/StandaloneModelRunner.kt index 556e78a62..4f4622388 100644 --- a/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/standalone/StandaloneModelRunner.kt +++ b/ksmt-neurosmt/src/main/kotlin/io/ksmt/solver/neurosmt/runtime/standalone/StandaloneModelRunner.kt @@ -50,6 +50,7 @@ fun printModelRunTimeStats(modelRunTimes: List) { println() } +// standalone tool to run NeuroSMT model in kotlin @OptIn(ExperimentalTime::class) fun main(args: Array) { val arguments = CLArgs() diff --git a/ksmt-neurosmt/utils/build.gradle.kts b/ksmt-neurosmt/utils/build.gradle.kts index 8eb3e889e..9023512ad 100644 --- a/ksmt-neurosmt/utils/build.gradle.kts +++ b/ksmt-neurosmt/utils/build.gradle.kts @@ -18,11 +18,6 @@ dependencies { implementation("me.tongfei:progressbar:0.9.4") } -application { - // mainClass.set("io.ksmt.solver.neurosmt.smt2Converter.SMT2ConverterKt") - mainClass.set("io.ksmt.solver.neurosmt.ksmtBinaryConverter.KSMTBinaryConverterKt") -} - tasks { val smt2FatJar = register("smt2FatJar") { dependsOn.addAll(listOf("compileJava", "compileKotlin", "processResources")) diff --git a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/FormulaGraphExtractor.kt b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/FormulaGraphExtractor.kt index 0acc2372c..97ff03201 100644 --- a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/FormulaGraphExtractor.kt +++ b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/FormulaGraphExtractor.kt @@ -10,6 +10,7 @@ import io.ksmt.sort.* import java.io.OutputStream import java.util.* +// serializer of ksmt formula to simple format of formula's structure graph class FormulaGraphExtractor( override val ctx: KContext, val formula: KExpr, diff --git a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/Utils.kt b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/Utils.kt index 1623cf479..f5eee9357 100644 --- a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/Utils.kt +++ b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/Utils.kt @@ -16,6 +16,7 @@ import java.io.OutputStream import java.nio.file.Path import kotlin.time.Duration +// read .smt2 file and try to find answer inside it fun getAnswerForTest(path: Path): KSolverStatus { File(path.toUri()).useLines { lines -> for (line in lines) { @@ -30,6 +31,7 @@ fun getAnswerForTest(path: Path): KSolverStatus { return KSolverStatus.UNKNOWN } +// solve formula using Z3 solver fun getAnswerForTest(ctx: KContext, formula: List>, timeout: Duration): KSolverStatus { return KZ3Solver(ctx).use { solver -> for (clause in formula) { @@ -40,6 +42,7 @@ fun getAnswerForTest(ctx: KContext, formula: List>, timeout: Du } } +// serialize ksmt formula to binary file fun serialize(ctx: KContext, expressions: List>, outputStream: OutputStream) { val serializationCtx = AstSerializationCtx().apply { initCtx(ctx) } val marshaller = AstSerializationCtx.marshaller(serializationCtx) @@ -55,6 +58,7 @@ fun serialize(ctx: KContext, expressions: List>, outputStream: outputStream.flush() } +// deserialize ksmt formula from binary file fun deserialize(ctx: KContext, inputStream: InputStream): List> { val srcSerializationCtx = AstSerializationCtx().apply { initCtx(ctx) } val srcMarshaller = AstSerializationCtx.marshaller(srcSerializationCtx) diff --git a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/ksmtBinaryConverter/KSMTBinaryConverter.kt b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/ksmtBinaryConverter/KSMTBinaryConverter.kt index b268798b1..a87cb886e 100644 --- a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/ksmtBinaryConverter/KSMTBinaryConverter.kt +++ b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/ksmtBinaryConverter/KSMTBinaryConverter.kt @@ -15,6 +15,7 @@ import kotlin.io.path.isRegularFile import kotlin.io.path.name import kotlin.time.Duration.Companion.seconds +// tool to convert formulas from ksmt binary format to their structure graphs fun main(args: Array) { val inputRoot = args[0] val outputRoot = args[1] diff --git a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2Converter/SMT2Converter.kt b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2Converter/SMT2Converter.kt index e6fd1e89e..e14b8e0a9 100644 --- a/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2Converter/SMT2Converter.kt +++ b/ksmt-neurosmt/utils/src/main/kotlin/io/ksmt/solver/neurosmt/smt2Converter/SMT2Converter.kt @@ -14,6 +14,7 @@ import java.nio.file.Path import kotlin.io.path.isRegularFile import kotlin.io.path.name +// tool to convert formulas from .smt2 format to their structure graphs fun main(args: Array) { val inputRoot = args[0] val outputRoot = args[1] From dd840efce092a9a67b3b721c8d883d350382811d Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Fri, 25 Aug 2023 15:41:02 +0300 Subject: [PATCH 50/52] add model resource files --- ksmt-neurosmt/src/main/resources/conv.onnx | Bin 0 -> 18851 bytes ksmt-neurosmt/src/main/resources/decoder.onnx | Bin 0 -> 13907 bytes .../src/main/resources/embeddings.onnx | Bin 0 -> 256187 bytes .../src/main/resources/encoder-ordinals | 39 ++++++++++++++++++ 4 files changed, 39 insertions(+) create mode 100644 ksmt-neurosmt/src/main/resources/conv.onnx create mode 100644 ksmt-neurosmt/src/main/resources/decoder.onnx create mode 100644 ksmt-neurosmt/src/main/resources/embeddings.onnx create mode 100644 ksmt-neurosmt/src/main/resources/encoder-ordinals diff --git a/ksmt-neurosmt/src/main/resources/conv.onnx b/ksmt-neurosmt/src/main/resources/conv.onnx new file mode 100644 index 0000000000000000000000000000000000000000..e3bbbc3581dbbb28941be49e385feae3922d5546 GIT binary patch literal 18851 zcmbV!2RxSV_rFa~iHL+sR#wS~^4#Y%rJ+efip+=*O7jzv5F#z5rL>E7xX)>)sa=xx z(B3=c?=y;g>ht@){(8N7?&mt!bFTL}*IC!O@8?lb(NK<<9339%7o@9b+pc{(8>i$f zS;-WMYFM~`fOlYkPxOSyfGCM1G&rnXOhE9MplFH8pVz*@K2aJP)-{*bE&=1ld52Gk zj+hYb-CnoBk1<1qKPeV!5=EcTh#(&=8JXD5D`jOZ8cP&>1EOo+t4frjBYnc6y6MQS zl$Dd%NPfXqqp|hifY1rG%P9W!-caGsQHgauxEgBKZb3c~0kzYW{vI$?{A*m|Tm!v( zcvw`lPgr#Aqy|663{}2UEp#P{6MaG_1XwgykyTgjqayRq-%42pNpFp&KR)7ZQ_HAf zozeeb^{x8vzkji^{Us~g|6uhGr|LghjhECHU-COz78^-@4MZo{$BT!CFNs6CD%~S-XPrKO=@pe^V?P zN)&u11xM+~{AuOhl6n>``4@*88~ul89n)&?W9$dtrvLE$Zcf#IxzSI8cc_I`XG=Q# zR&d!mMn32lg4gn^1zBrCH8J2H;I=jgkFJGz~p#pD|-1y~l<7PY4aD@4y=W zCVqEjPmPv85&vbrx>~=a|4-7-l9l`yF3JC~w1)mqOaCMEk6@|!3+k^`=Axky9u_vq z$!SPf)IVl#Yp>hzhpaY^b;S9vm=z-Vx8q2D!CRx%PuOag{Q+M4*AomI{Fo~Vmi%8_ z>M5v39zUNtA^B$o{Y^Dg`up`~hZ^`xn`-eB#=nz79dBy-EAn@{^42i;iCnAeT5oFh zn;CUBBRlGM~DvV9c(uCNp&8M2apajNT98g2d)P8~J;I=p?y zZS%iClBvme3X(eRS(_*{er5i(WZhr6r|oz5^!c58)?x4)_x!I)s%|Z^QdaW6ZQSo1 zwHC@RJ-wD(Kl%Ms0O{6CeswFa^Rklvbu()NWbL~3liN@0*0Wl-8WV~Rh;$AO7#9#0 z9reSRb$_J9u9wkl$Dow{%&h)%Rk-UY;D~T z)%3f2sMK|i*!r7V58n^Dez&Q$AS5yxx<7kTXmG@j&+9h%7tK(id(8|<@bBcSEhv8{ zU)^HW@Vk84DsnaDs`~dsK;7ysQTCVH{+q;7yM8^_92^k!^MUGT*ZhZUsQk}`g}S7H zPiSa(%#?u0@V^V*Isu?IzSLXg?`T-7-(Lm*yYB&Day_4_kFg%PKJ$8&hx4QepD_R0 z&HK6X_?v2|^!IDcsp4N1lf7e|OljA#P66;=)mZrdRE^m?{x&(*y0Y%?lA~?i>i*Y`*6=)ZgP@WvUwg`Hy7!yS=W(QSZR|JsZ@~&(8egM^I z{!`V{BY5HulCFAvlA(MJT~c#?`u&F_jcR(#-l5-{&wn`21EP z6?wI!%`{Z>P~Sj?>kQFwS~pz2N0lq{KSTS)61G&nO(!0-ULA0CsvKuqx^wUTIdEz4Op-G`AP(F9kTTVB=!JnAw%HWS z)8D^@;jPqpiQR1Ib`4HR^9X)X!$gy|QR3|JtRoz+B@swS8->?!oO8N`-fU&5Op6TUsE2|rD^B9uDlLZ{E8;Ptej z_^zuh_4G31^s%X|m6%7P_bucGZBqD;rV8kmwN2Xljxl7nIRFOJoLO(-H7F6r`qfR+l3W%W?=6D-d)oYb&Uj(TfJXRuZx}vs%mLSp z9a%Vjf>fKohpi?a-0Rs8zSt`Wvhus~{;_X_o;{oKfHt1AYg})>)?*OA{dTAr4q3sf z>KHCk?}iq`{cu93mGCa*5e%Cc!=^@ypfX68G$Y&dF~3)^#L)%Yv^79kwB)dNf#A<) zY3j$e{B7P;-fA8IV@`Jy)~c33&yP=On6(`S7=9zYWly17$LaJ|o=M@8F{T%!fMfg> zq2;+WiVW_H&0VjFz4~;ZrTOz|+e8Uqs5~3?RAHrykHiHpo#|ytH~!XJ5%oG>6b1FZ zxM7q87pOO6PvuNXo6(FN&t9X(z2x!Uq6D;Rkml6+Toc~VRu5EO^~6JcZqm(&xu|4+ zA9g>PjH*Kl$!TOm^0+NtIAp@(KFs88xdviEu^!4ENGI(sld(t6OH%AO5XYD& z@t_vHV9Opiwtk97)v?EP4GMF)Q7eiqKp&7w(?mNfC&L>}n& zh+IBzkPbT0Tzq!nG1=O+;1B05`O2(hn3t)**f^P1Oz|KUfhe%J0}K!j!-kAC@N$bf zUszrZgJkC7E)5qLx^6hwuH6P@kIRbB9}MAB(^YtLgZ3;9x=i7>wBSa70gqf5iH}CR zW6q~n@NCUuq0+;I9kRdB@)d`QV;eW1r#iE6TtRydz8l4s9b4jdgCHoG6b_x$Wrfs9 ztHrI{6x(U_#^?#BMc*aE=ybC)Vt#=xZF4;(rpX@zS)b83+CvX)4^)#&&R250TH{ZN z(}14m32S%C;pFHJIKIbgF?Rk#`oa^i)y?4)v-z43KCCA+9W?=pCdWgP!gx>}x)9Q{ zr@#w)ZMar;jC59Ar1f$wMBP{oo;q(k4K!ILTW2@Du7$)48`F!Z@je|k z8NP^y=}#iHHg{>z^m3Z10s;k?vORgF*LvxVmX`KDMS4i+!B1q3bG9 z?qwKd-|7KPXE|`ko6kXcy(TSvd<5nlQD#j=BkrKm5$`nUg-f*5`8}Qh$Gs+)|1OYI zy84UlGaUJ8UMF0(xu1A^x+Mzd$BI_@>ZqA+hj~{sXhKy4X2m5z%-cpdRe7Os=J+Wy=a{e)pvbkI}QnMXSc)sq9ufeMmc2my5+l4{y`M+Y4|7TcZEL4%B{u zv+zgP1d#8Q#MX=E@aYO=yt!QsGfuc+WpI+%I%zJp%(RDulq?K7GZ&Dn$K_ z$0=UM2A_6o#)=t@aI5nG$T{4FgF4<59)9Uh`>yR5FML`ET`n)By(Y3~bU+i6)q`l< zyHt44D-E<;Yw(~I9Z+X=OmW59DmwcBFfrgZEVgmMhg1$ZxsfIVn-j8Z#I@nojeOGY`+1Fz5r=F7%x3)gzwS@fPC*>AgBEl z7Voy^R_cm~s||VBI%D#hZO-;SOIeWi9_ zVZQ^)56TF!v&yK6k~-gcLQ zy(eNphay^Db^>O)Me_97E_}LaH^FcEA4RWHi)d%p6gV{DtvE4k1u1>B;L?ZXRFd0M z9C7Sl+0QsQ^93k3ogD&aC&Sfd{W_0ZTjUw+{!*YY*u4_Ir?3mF=sToJ6-_w z${l30n!rg_j*sashNv*WhE@YXdGR-Rm~AHx3($d=qYB{K=iRV7ayF&EGr-~J^Jzzp zsi1IB4XeuC*h6N%==^dd$6RiP(Lyk-)Aq)5>e*XBc6%kyx5&l0+x(o$@#K1FO2H-`ET?#ySq z?W04kZhT(Vh?`H`BFHQDA+PhA9C|JdCzmBsSR)@ew?7};x|ItCen!}!y9Sqd8H0TA za*|kO@KFa%P8$D*pt;I`M|j+Ur~@&4Xa6y3+w%r(f4C18sHC8S=6rPPJe%Fdwc>b* zr+8xdF`<1-GhtjyBhH;V3fH{N;)bEk`E#qU(q20bfYe2nWzEWAv}(Gvzuqlsai%}) zE}6^^B2@9)gi;FnYRMsGYFM0|4xM~t`J$?~aL_-NpI<72olE`6>}6LN*|aa^H+V+j zE3;wPome!mtB@wW8$m5EnZwA(gK6!7NIpNu5c6L*g{&Bc#34=iqiX^=C+(-pMuk+V zqzhZWb_e|%ITWXA1B$ofq{%!(42hq`YgX(Pq?5i#7w<->vLq;r`a*kLB{+H0EYupI zjgm-hv^9Q0YnqIqC0BQXPlg2h&+daObM4{r1{wa?R~yp`iotz;HKjaJ$DCaRu}dAq z^Uh|hbjK6q`YeLd=FXH_F&Oi53!t#SG9Pn|ro(o9+2O1X&U3pBo2wD;TvO!AHZO(D zOCMmGsVk2+Ok+s>11IXc2wl6xio3l{Y0{oH*y8I`r~dEGz}pEmwP>O>o;vM`htvvS z+THe%2kIpV~>)X0y?vb0OL(+=9I!wz&BDTDp5r3rz%hpc&6$|H92s z<=k1!KC6HU!`~F23cn?6E*rz|N|Q-Bv?rfy9K|K(t)aui*;Lh10}Qkhu;0M}JolU* zdN~N#x}lwTc8E4kG~Ew|gPqyxh#^Z3+^4tg@`TNK_h|5qU1VSST(~?Z2rn8gqUSp- z(6{dJLHX5!5F*gU2{L z7AhqxNPbWxhpn5;+3(~~^I$Z)mak$^eMmFf$5D@GGq~fxf#g`=hK+X&hVGy2MDMO2 zVNG~*9I^ExbjVvzM}jrDht*;bH(!U4%pSNauUy>p;UaX?YmH-ehqG@*rC1QC!_KFR z;OuZaj&dH%tA`C2?{#3xwcSAl<}uj!U{jc(-G&SnSJJ|%UHIv!V06f8D)fm@5(jQg z5_T_|$2M(4DK_~wR1|1nOuRQ~1e~D@lJ3RoceF6LavH6hx|OzDnM3rV*8HtiPndVJ z2N)Jl=VhfXkbTa8mpoX)n-emG0S6nfc>M}wFUf{`{$_Y{h$bd_^}?Y;Z&34j#yD`# zev*~#%mW(^P`c? z_eCA~Ni5eWgbV$m3O&<0vwx@7Vp63*X{O$wQQQ(I2gaiR%~oQ&@kTuG$^i;AC@hql z_F3q@{~Rf}6u|P$FQl$AQc=-(FfGx&4J{v!#R4T${OUObD-~Y|;aU~cRx60!+Mj~Q zdpnDHSyp(@S5`cxmJMSXw4{!=W68#3Z?Q|x9YM)D)JbJ|PrkUx1Kti#=8E+uc<7K0 z2u-Z8`HTa_LuZ)s%it8sJey8|ZyZS0`J$+xr~||2wE=krSx~z%mUJ@ZF}v%04%Rfp zGs*YGOh+sHqQ@yjAoM@-+wC9=6Ueg@=kJQ6`6pcvfPilQ0 z3jA;Y-6rfw-Np zaj~12v-&)kG*K6y&iMux@>cPcd`Awn8pqn7Jn>Z96L4gr3|jUdh@qA9xJ{>)0rSOp~GG+hj z3jBCDuTQLO@s`cfz6Dz-zBGU~M|TkQzBR(WRo(GI3t!{`W;k-rRw4MR17@{2P@H_M zVe#j$Jz&*FD{Nu#NcyVJTH2wziR6>^fWh^HVy`PzFr>&G-P^U}*q&XnM|O&s|0o;s z!!qdBu1-`DHW*LLn+9ewi)q~E2t4;_PA7{+-+L85^>Avs6?a-4g=m}UOoKnrqju>y zByc+^5A-De^+Rdl1S|K`(V@vsFdSmffkmd=%E|%v z>U(oc>$gIh%QI36x+GolOb%^(>T$~vuK20MiRNsY0bMq3gY6rx2npjHc$bz)+Q!{z zP3U~y=^BagmtUd1xE*J&D&T%Y*OB8DB^=S(j`j9xpw9^dDy+N@;Q?NJabt77uBgj1 z-Oou=EBA`Z;{$o>!zwAIEr+EC2JrT?A$>^{=j@YE53d`=iK|Z0u95%@l*H&zR<2Tomd*XK3 zQ>{s(vhUKOuq^0e;fAvt#bRppJGzkAhf7PEu(fnPT{QOOhzZL<|M4==FCQ-i4Am%_ zwYNWdDP4w{XP?rNwT+~uW{oi^*_zs&&lHyF+rr(%YRK-VhU?GG2E8Y-G^58HI6iEN z_$=E0`b=Upu+&b zmbr?wc|{}#v=~h}rcEIEybkcp^;F=WBFtPnh{sQl=bLN1StT=`#%#3^cV@bCb=e?M zbPW+^?Aj!1J(4YU(OUwWKQHE!hY!;B7Ydxv(w#5e{7i16UVzdb8-8E?P~5opB8gi% zP>+RQ;PM<}UNEo=F6-z;KC;ts*5qNluxCU1G_3*Mcyi8H=UtfR$-kaFz zRBL>fQ!2!68G;XIHsJ0bJ8(byYtZ1i2YyHh;56YL$=q#(QXMZLV(|{RXFVS`nswz+ zLpx!hh8?c|9EIvPZ^NW5Le`6bHUuML?O&tcjWw0{qOH)?r;;u& zcnAIRThr++^4z{n5Go8kLJJ(^ap$Y2v~Fe!X&U8I*urL9)jNl_RR;6*13FMOcq)`e zDq#8c{=(5v<}HW4;ry;8(w$F_!aDaAI52!DU#v#1{Cbr1_n6R^ph~glI&-d;o`+$M ztH8PP0`(2*FRCdG5bSfh(x9w4ytH@*!UzTE8fAgem+p(g00Fl?+fDcNuF~s;W_;$- z0!s0J0p2@WqgII;k3Lo<299y(bhj4}Y}6A|Q!-HT(g9FDuFef6`#`$SBho$h72cSy zg0BykP$RiX_*BoA&0ppNM21KmCz@l6r(Y>`S|RmK*$fZ-3?XOrI8?VRfh+U=Ap3?o z9FVe0>|B)yQk`coCEy@Z{55LYa4=Cn)mD0@Uh zyfA19Z0*&91Dr>3*K0XAc;s{Xc(^IO2`!~opRUvUj_RCwx`HhC{=p4b*|5#lW7MpF z3ax2v$sWRbiz3Z&v(X@&Za0pdm+6VxU%Ui2>lD}_>5f-N%t7xE z3Fdw>qL{k_(MGEUjVw?XvhHUH(FZIs!)5@t)ILiGPJS)+&{M~Zo_nRM+P$WWai>M| zwX?-7J9a_e4HXdBq6p3{^+2+_8_)A;0(B)}`L#L$QXGIAXpohA78jaCs6d9Nt~ig z;Bc^TnSO-HKn9@^|gsbAJ@)< z?4Tv|aM)ZHu9kqlLL4Xg#_>t}6xP=aLzUApw5{`S-q!pqB+qf@Kjvj{b9HM9UzQI^ zjaA4jd>uVfIUrRqT+CIw`>|ZgAlw$~z%o0!w;XZOmABPMFl`dfkU?%5v7n;VHkm!z<_iZ(piFbVx{xuIWJBF#-KhuckhVAfbs zSSX!U(?Mv51@>y3EH{NyzwDsr>3$^FaWHioUy7d!ju*QeT*8AkjD#iW%`jEb1oILz zX}|+@?5TDa4EHvsj?-odH_9ht$Bo@F^`a?W&X7aJ)k-L<+L*Kawu03#A2_vUFkP6N z1-h%=7e~ma<4P@G9GYS&_;$9$;+UD>_D6=)@0d3}6-Qu@oHLFK$>jw{Zgo0SF2{QP z`q8;rjkt?b1X`R7#hk-Yd}D1JKJw%=RXF!&=jVu{hCB19GhSS3X^rEvKarv18tLw) zSH)8uhC%GAEG%-kM|Kkii-X>(qRM+^!TxpzHZgHydzIDT6+H{4CTerlv6@awgc%=n zm1DJ>Om10t0Sf18p|G$uPrE8m?)E~m|JD=d4pinUD@}H|(G~+0C$sVn4}7VjjNb;| zfX#uC&}PUE!FWS+E}Ax$X6#VG32L*&Yt6n2H@79>+*NT>|Gf9GId>+U($3_&O+C?a z(n$zzHJpPJ%7o-rjbZM^%Y3%UDe=1B7?_symTm~S6!T>Yn|idypn`=MQ`CuEH|2@R z36?my;G>w6;v+gvNaouPAA}Z~$yn6D5%1RADQKk?gK3ME@$nyvu~|?P%>0@Ptq!iI zn-+FxZg7wmw_3)l(##Q6lkjAz2M-A_$40h6JiEgHUat~_Nw+)Wvt(y3^tewY8@p5U zSY7m+nNHbCmuSz_M{qu3AAHjuNl!ek!!D~%@Nk|!E9Ch?UeZ$dyl^^Q@KnK0tuvuu z_!GG0GMMKpHKMJ-r-kserQ*|}snpioo#sE^T)eoiCB5$2o)b^>f!+&DP+>?TPV!79 zmv%0Abm~jkzkM{1?^p&p;c~3LwUWMg^b<#X8pdy(9u;SyCadmJ;CW>S#BI~(P~rVD zI+HL7GM|M}>5&B9d87x&XFehs>+O8MjT!f+j;v$uN^YegaJI+H;+Oja=z-cfai=4lRtA$+GHhNU%TSJzp>6R&+j=$Q6=RLh9hKXKr~JSUEx z@D&~}8qS6#?P0pYMw(C<##@K@^MxtxaM$xV;e}pX{&Z58yDVyn-O4@)%596q>fCwI zJ@zEICr3h4_p$u=>S=H1}U7ncmg54Jc-^+brh)f(PE_31GKA*{Rm^_ZmQAX+AzSOiD`R4$P^bgVQxJS2-1RLQX-#whT^K8-kLptE4M} z65*Wk3bEP7Asm-6maE;jQ>Kc*XD1o3;jk$<#Vu4g)MXL34_4r!jgw(-W=pQ>0T^I< zjaCgZV%3yRG-I`v&}QUzI=jV@SI^xbb?9q~lVt)}dx#v*y!Qo?>?Xprb<1(X_LJnh zIus8MGv&2cyl9O~g442&orL*^%3+?fRCLuEz=Dne29=eH{vT7JqpUqfU_bdR%r@T!lhpM@od+{1u9X+Q-t7q4_l`j4unHQy@I&#I=?-wP zg)L52?Se7t0($M~Ae=35pc%dAfn>8WrQJk~E4@;jG;Fn)l{g=4yBPB3^$qx_vo8xP zX5pkGi=emeI`O*uD1JC22#RCg(bX1Sv~%ty(RXrB7~$Fp4Z=tB?NPDNc6|dO!Lu9R z&>u-Ap2nitdqob4-OQU+ztD+iZsOF=9r$!;OX(3Ob+&$-Ns(vgll)yd-u%%69(zqj z7MAk#E9smut{1nPZ^vgF&Zlc}C1T84ZH8>LKHMU$|F zybbv~4yMT14X|P71M1gkC8XBejR`3=7Djt6V>LZFXk#}FRfN?-oQo^C=PK><|Jaf*C|JThv;A;AN0r^@ zUjT!obdJxf0wwjk8khBU8ep5z^57Er4SBrf;x^ka%KRCB-4i~n2 z#Aj-b)rZt|_I)WDz;Yj7LYJ&?sY-k8+lwTc;pNgOTPCj>Ob#M)3UfK^b!@*%<0}dH69;g z%2r`#gu1Z&(LN8(uiy@NyGg}K)Ut7Skcw2VeUjpl_{K@~&J(%^b zJ3KHI#IB`MJbJndFVtKCb4T>SRynz(+6o}-YZ)2by+PgQXrcS-ENb$oH+S-If<(^| zsCPpV6_eGlMfw%8$ur`avGI@-+*#a`)`1N^OoY`-W%7-76-HwyZ64h|+)!@e9@@!|dC znj3d!7}32kXFf0im%P1na6~B`v+$xFJEn^T+55=)!C{)XLPE3ryJ4@r4hU-tiK5MT z`%?p9vu`ym{ZI&vAM_MP8a;k)Z7TkZBL$;FVohB&X_n}iQWEK z2L)Z)(~HL)VAnx)aZt_ix5uh8PDw!ssXhmUJ4?R_8$w=F-!etkEYii~t0|aNdJ&Gj zE+*aOm&kI1DMpO%hX$1&D1F0uYJRde43~xr0i*VVSAHm)o7Xhy;>sv3b{&12Fc3yM z81TzCRy;hh2OghGwA$|zG>;Fb#ocF7yDzIDq$ag>F%Tf|kU?=D#bM}??MRcy<%{D@ zP5HXn7IChR0!p_&rhEIh2>M-fg#I-p@9k~x!NYa{bk*D~jw*E*N{>Vfi<8YbK)$!A zP~MpHrZ&Lv6eHX=a4cVRt|I@_t=VW&m2fTR1zF8hgN+sDEH0Dh!qmGIWp6=)=3b%9 zMWrHc@WGx_1iZ7V6nv#uV978gd@yTaakcql=s10`^hmdllsYwp3`+)M?qUY*-fQXg z`%av^4}@sXn-F^E0LA^$h$|Ma7nXWxVM6c}YP`k@cBeGqHaUuHwQjpKN;z5RbA3I$ zzWos@ZBK*wi;g^D#Rht(sf|na^uiu*ths8FCJ!I-kRlsp(@E!>@N$2c;O!%WLp6+e zxOo#ad_MqnmOXTub@MVLa0imS=}J3aKdxypCGej7q1@L*4FXKEA^n~eAGOfOX{kd6 z*YE?y`)--=PMZ_L^mtupGTW2WZ`oj3=h5iae?ILky$uWCHDnp*3&oD}q2;)m)Hdq~ zoSf5(cW!+Po93uNWTg_C+?T->3x=YWdprf5Iu84q?JKs4?TZa1@>pEepHqwH3Uk*w z)57JC;Kr`GR5rXJS_~aY`qL&!zs0YI(o1qwbibdl@^u#u8^2CiO^@MavMGnCzM`}5 zyGScyEYZ;ys}P z{EPg{6tZD0%yZ>CK>uAGLHFAsz+Nv-+J z>>Q8)2JHHEs<P8?WIagw35 zdU^?rI%5yd=Nj{pQ{^z>LmX6OzYyasw0NbT2j@;Hr<5!u-W}jfUCaVuWokJjHc$n7 zuV(yFbmz4d_Rw^RB6s`TiPu@C7AKEw#0N*1@m&XPTJGw^E7ZFQ2ENLycS?gUEq^0K z%zOfmcR6#^%wy2bzBe!UGDCXN@wDG;rNrIMjUu zsec{LU(yOGX3YhdKFJGAo;;$dbL4r~mg$gl0r|w9nzS#1kdve^g_X>FOU@VQF;fJ3hSCVc+RXBEi3r)}+&PTFTxLsd$ zJlZs$+J&3&v`u$taHknWv0nvui4)%1nhr9uM`~_rZWS{#??6N6^RViTHhx)ThWBfZ zHo5QHqG`veU_Pxswm9?x7NuVSxh3tmRlXK~-EAW%U;7Nvda5|SIt%Q&pDVsTZ2;U# z=*>+wZh_+K3&nok`WVo^N%5oi^J#1T3b+-$f+|kk2Uk;5G4R%9vZ-1MM`hw!<;zl9 zJ#Z<7Z!47sUe)6vr#xuQfWD-9DV$mx)Wn_ZtuR?DvN*zQ_1lw&hID-~hl zYc1ZqR0HSqQiPJ5$6(Q!tvoU+1iE_PfZpSEX_36K^j7=BFtqz%PIZZc(s2rOZ23!i zo8={Z2hJ@1@1FHK5r*2kLUR z8Hyzy;3t!~$t$F8-<=izT*o{uJV9(j_cd7AUtSq7XiG>UW1 zJK`|+HvGua7j{2tLe)hb;90^$2+Im2Cx0W%wA3NnJbRSSvg3-V6JV~b#?jTGkbSW! z_NY8b{YK4%+$$Y~>!+@h<*L1~s^X}4Rgyu?Chvz^lXjBn>o+jwn+r_6253~#49d<9 zho#M)!ls3bVA5SJym3a2o`>4q&Z!$C5NPb<-$=}l)BTrnWf`~kEW--Oa;npm|*0gtP$ zAXOCw;YcGB%nK!C zO6^PDQN_lg&~b+;-iS&P-Y=^b8&$;8ka;fb(PjzY*WnxWap`? zs}dA$vciNu-8g8K2KH!lk~+CG!o(04avuAJ+?rWY*$jPb?=K3oXE}0=ffWxq98&Ca zsu_Q1mqNLlF2Rh}TDT>qoGy+_;?tRfp@;D)7=BAmP;mW1vhUjRjL7bEIopi8n_ZR; z-Iz&Pq7r^+69kJYe6fFqA!&RqAy3!gxNPH2ikxvld|7;4i0`Gt%1bKY!MX&RaUfi@ zy0(Y{l}3Tv@b(-P(uVFEy#hs}tAd=5F3-Ak9r~~>UJdofg9UFOWA&usrdN;BjIrk2 zS<4aM&uxhAVVRKYb&{@TmWaV`EV-ZA<(k_s+NiB@2dcZ2l0VwuKs!~qo)kq_7mwi% zB^Th;+uY*)YvaWL(QpG_gc(VCuZT9kG0995^@{^Qo%Q*xNE2rj=RJ=MUN( zGrcvAPjlsq+xno6VFb9XI3W&c<%%~opOWWs2UL2$0v;3%rNmG(-VhWCH)O7p*^Mir zRzw?YwDCN2nybaj206isBvYK1nMJGfeW-l2ju5*qOQ=wvC^ji;i4tH7Of6l2xVu2i_Zi$V;gsy)9?Co)tr4Z<20hHq9-%PjOOR?$KS9$9idS zYA1Vf%jnZI!m{3%LNs(GIx4anO(l&r8vXfWNm1Tm6UO-lL`vjT3`Q9J^P?$$O;WCz sB&Q$=|OzQ)(Jx3aRX(t@SY^A}B@*bc*KOwikKO#s`+1*X|twA z>2`JcPlJWDUodUwf;9|>8E8;w7X3OFGze`!~ z{3#kq`utbZnv316IQxz~{}?@l^`G^@?#64uKX)ju+RP-^yq#`5F9E~0Y|_285c6{f zi!W09fl05EWcYfLcxp&@UIx9{pxbwN^(2^rLfRm~)q)fD4@8sb7qn_Z89jcu2L8HT zfMdHp()aVPrO%y=V2G|e`sMDyp&qtSJ!1$C{9?sRzV4n!AtBlp8N-ma;pl(d495J6JS6GGk0Lkjmt3fyV$d6Xl#ZgS#%KI?Jr&s#!7Q5P!q zoJyW=j?-?-)$~%W*>is5Fgoy73mZ;Vk?dnrP?A%@+YilWnmv^$tHxgVn5NEW;&(%2y#=-#zoXGpb@9GU91IMb zB93kKme_5x5}swTJHt4QI%=^|=_Aw-DCqcgr-^Jg{gG9O7@jT*^7W<4^Bf1v# z;bxy)XgXg4eO=eH{1^dEV^2#q4Y~z$wg>a1L<6Ys*$8=ugYdX?Exwy=j76&H5Y|tF zyDjR^*EK&=!Bq)I&AA7ma=ECqLPzwJO+l|$ZeWwXA2Tc$@ROQMj#<|eUId*0m+qZ; zR@oNX>hn-=yb}UX;)8HDMnm5#W}?f(PncRqG;ms>r001L(!Nkdoq?Vhr>9m93eI|hOpUz-a&=HA)}x9M+{Oz^qZKbi*}?G&d#D*l}HPS*QySUVWEBUn}zu(FvQL zpBBf?DWf^+lR2Zm32rx;f`i)SaOxCQo-C8ddKc{3ID8tb-Sel7^)h&=ITeP!I7cI| z_|unpI=sU-k}pyhR_wcre3xFAxT^Q&g>wU>Yxc-t{H#a)vZ}w(2=znM;BXuU1?oV| zv$dpRStGV?Dwk?BuP24^YOFPvK%Db}j+n|wpWD5mF_K+W^FviQbL6m?QnL=Eht;X2 zCQ48=Zi1eh^CY8QAEWI2OJHG?4ys8V^I674c=vt~9hFN4{j-B$10B~3Ok-O{}MDU z?m?2|SoZdACilcE^wRDET)k|>`S-o)uuUksH5Q5a)1qCq1bf9%Leu`W;FP_Q?G$V|X3bjiT(TD?B;2Lt?#ZYySQ!RRHWeKY714`$ zIbohSQ4-Zx22C^daKU^tin{!qc1@AT-M!uM-tx|%@yJVfBw4UK7G}8W{>2q_&h+~Yq9xM4vA@_qMUQ=yZk-hRA>_VL>5bu*304Y zm2u!5at*p^8sWI6Byphk2s}FQF+I&xVx9bYp`~a%S820W}`^#He7Sa{Z zw-^XlbG>j=jx0Zb=ip-+%tMFTV$!aBVSn&YzPxXp=;*wjMokO^FQdQozG(>j3?9Sd zzE#k==h`s)xC4j8r;&VrHN4g72L#I}iEqJ%^{l?z7%Dx$hRcCIJX}O5GZ9&wuAdebK16bSC3O~O#WLEnE;)d2Z^$s!%<~c72Lda9MiLR;=S>9 z)V4?yx4UeBHTLTT-;Exa_wlR{Z+=%YagQEwRL3}u$;EXOX43ZWyYb*HOFsB$oiy0r zjC-4Zpq9HbpkATH`*z-DG6GiAAxmk!^$VZ>{1YUB3)vV19H4@LaYW7#nb zqvcoAX}?^u<&9!qge?~Q`s4|f2SIyksZbwsPQ07*fF>98MYXS$;4{)z{9~fPD@t0R z@$nk+863#oy6#*w=obC?Jq|KUy3x2m74$Q@BgvF#^NN-zREx9VgKfsx(BBTUM_A|7r1D=u=Wr+J4~2$H-bG*0OybbWG#t{8;ksFn%vqHGJc z*pHUDTum0!mT976dJL(JiH9ZEJ)w`y3CP}`37UVd5Mmv+A;S$DQ%>Mx4WSGuRc)TI~**EZvZMK(Sl2| zaBKKzKC@L9clH~~K@;>aE4mwX^M6KNWNwM7@8ZPJqBSJ#WXpec7LuCUG4Yi3F>Ef1 zM1v~{=&!YfPD)+a>HGOaj6$4JC4F0>Iq=wKN$JeSXNnO$~r%L;-BV0 zOvAQd*55R~lsynzhKBQ(o;~n;`C59j(}BzDd(c{qL&DdK)u1}F6G<-Ir=vx37~Zmp zlQ$iOZPq=7-Y2cOWv)mkmhKkq<@(@{h}AfE-cOj5nSdWJOczD@DT3*^HJn;NpfOVc zPafC|k`GO=q*Q~~=BmMRS7#cm^gsxB(3Q_`9l~y#UFcKx1@hUYhLaXt#7>v1gg!ey zz@Xx9uvOm>UX8gzH$LvgZG9ZLm-Shgp7&l%*xAL)srCp@8+-_|B2IJrl@Yk)wi>E< zbr)1-ghTu0G;v|>Q@+rt3*2xopvs1WlwY`o%Epm;QP^Z?FOMB?zo=)qqt!n^Bk9A|Q zaW4ox`r!SHBv>3H3zz(r`0$QMabI8!O@6vn@Xu1?8Mlql%r^oqE_w%ZYIMjnYzZ4Q zno&s1C7Ra&6zUmI#uwL;M2KZ+pLD9Nx5T}{x5>FvFS2lP!BJCOv0U2(te#8Y)VR^0 ze>=r{9L;NM;Q2VPSzSB@h7nUw`O73 zhd(gJQ<3Ks?cmqPGR1@S(I{Cw04H)DYDJFcH6Jh1ynID$b}bMljBF4re@(+>+kG(2 z_5@7auoN9Ggn|DYGn6gqi4nKfbHFzZ$UCQw_5SN|zyTMY-OYvT3+@P&k2-U#u@1i; zcAKjEjx2sRZwTt27{om%9sq;E-aO)rGOSoEz`!y!eiIRh_tvg~dvcp4kM@UoRv+`g zm^V38n7WzX2(5JV@d%bq4#0l#eIQd^2Y(+PEL_^;#*ztEIQaHcGF$KttnDm#UtuBL zZT7%v-9s^0r#qhX%!a2~pDE9_l9Iu!^Y*9GFOx=ex!$odrhRJ zt4`oPYi%6fSVpZHAuvCoD+lbf=0c@zIHKsbRMF>?~E#ioHHMIYu&&m&LNj=P$&VOhI zpKkW>q|d!;0(y#rp;lQ3Opd?5ZI>6{u>+k7b@nTARBN|f|{*a z65Wnge|zECvCCz!K69P=y8NUK(}KZWNdeooU57pKGlf+}XW)viI*lH?3a-!p0?qk* zD0KD%QXH><88;GOW9EJMbaxtgFR;OxbML}wdp({o_KxtYQ#^dF`T{e?70{lfU`bk~ zEviI2uRQJN<_v*Oe*-2qpjv4l!?ZSJC%&-cEQpy2U9u+->4#`R>{ADry49unJS;gXU zeGT+538(T@(OtPoF(nZ-(x+ z*6jMnm)T*8pez|nw-zkKo;j8LELK8Zrz6nHZ;h}mQHD=j?m)gf3=JIi^U=b3=$bFV zqiHEDxxE5zJVH>>(LvkiDQLCgFeU7}NiESG=hkIDUUPE%cyUB$4BK{%Dz_e`8U*!LwSZRIfb`$jH)(i3di z9CP{wV)rEm9B$EqUEPgvg#UNm{=N&2iY=E~HrMm5JVRmsLuUbccZD&f+s` zU3$9h7#y3tjbuBk6YDEs?`hdz<02 zOrUV{ShMtO2UoGW@LrVt^@(14)~nUmwKxy^f}@UUfu1bub*kcty0_9_s+ zZ_~%m8pU*3eKnJ^8or;m3EZ}Maz(B$S6H7CBX(~C`M2BPV1gmO`=IVMz4)5gDirZN z+$mlhu}2sa*PAzAmSIoj$M9nMb@&$W1bW(jEmo-%s%dRQ5vx39#?;d8L^ z@eL@-+Kavh3gXL(R=#IA8&k#)M;$q>(k?j~+&Wx?A4lj@{UdWUaVw)hmjFl|HG?mV zck=3y9!u9hsN+o=6Kt@_pqe1NKLt`a zSHZiwrGeUs?f5_?UD#?61XIG*N-wenM18bD%d8&gE(8O9)+zm~vWCak$>F{Gu9!V* z8#)$7gZ+a0U@bQ`i*gs-aq?bs%x~|`a$)`u z2`gx3`|FOgkTxw%ZlYBK0b))Zqt;p#_VxHgE0zD!_>^B%Km7-t`1lCM4rHp{dWWnZ z-4k}zrBTb^PDtm9J^P1utkY|)F(tYe^eVE(h)3>}^$>-P&Y$7-Km&G-xl5Ov4^kJ3 zqTWrN>H7Qb{P9i}eYv`qY91TI{*Ysoc(f1QTz{V{RCdwPfv<$Worghl%0!;5X#-Oi zcQ8)xZaD1P8yeiM>h(#rH|oAqEp<>Eg%@50VPTqX=|WW{sJy(9j4cM!s4i>p^{stk zTn|fp_I8P|P}kh6ZMGYKuKfhQ%Pd*?ClEic4;2=c6^daw_Ncwr7N(w%<%PFzu-pn` z${ntR*+LsU^UvfTI}s0fzkt$~7_1mK5?tcjG4;|JNL;B{8po0NbnZ4`SeFnU9_7Q= zeayi!V>LN^E5NX?z8si01=A+Pi&`;uUV*k25OMq_%^p)tZ>n5Ty8bXJv~3VCCVr)m zzZ1lalS3fM6lmYvhcMAYiHlvDsd|_Vc-L9*+BQuwRjNvzo08#kCo4MeTOO}2YXomW z1nt^Hdg+lvl3_`Z@AgQjc%4RXl~%*M%OLNni0kDwoy58%K@H&{Wh2d)!u&{eBFFu`Uzp9WO zyZZ>iZ{2t-v#X@#`-^Epg&|9Wvc$##op@QpT^Q8Oo;)^g=1G-Ws3Dx|IIFmkb-F1} zSg{G~ODuRts{w9s(&1qXDlueg3%QL|=bYNdqNMvv*w`3WJKJreOiynLktyfLu0?nn6eO{SAl@0ak?e+j{lx3tYw%QaoWEpCix$=T&E7

T6E@`U>Qmstyk~eWW?Xd%(SJHW~F( z*8b!a4qanbpfXm*Eki@+wayl{% zfBmkYtjg86GW{5wG2P5phk2p%npB+dM4|MD>UIh>OM|ABFUjW0W-Of8P7l8h@uQ=ayx`urV(M?LDpdKpqiiD9Dx0o48bmb6VUEd ze{@NU#1$b?92E9d+?!ns9;;{K(}kn>teA}rH3?Gn%w3WZ26MUT$VsXy8bJ1jt6;ut zACg}&onB7eObcssY3-%Aut!-+**50|l{z=f$ez!|5lJ9Z_z6Dm*eR}WJR@xotWdXP zG#Whm3cKw|UX$SPL6t3#CWg3WS(EFY)sc2W(wd zL~-*6!^tk2OKMy<(5`N+r2oKx_3vzlsyr<&qy$hud<81aO*vqjC-^3w0@)M&Npfu; zZQS^S+CP7W`JL=|nZr#uRefB1U|A&2!X_xcxdBqFZE@X?SCAWAAqG^9A+OOk5Vi6; z?ezc*Y)mD?@ZtQ~F@?t+>%|q~d^QQq6z%5iM;i-k{=PMiqrVL!pY|M*9Xyb7+>b%$ z{VrTJ(ij7mo)NCCljnj;pVIz2d@!=2CkkVq!itn!$>QVXFwaRAyqzw?!JJTR=y_Bq z=uyST`<|vDOWRX+ zg0oarsfWprzDdqmjOXQ!fA~k`RjJ`=FZ$cGzhh7Kl4p;IlNcXL1XZ1xSl)OX3|DD+ z-Eg$P;Y;$ktIT5vNzK646Vv$jrelP?{iy7(KTd600jDBwNQXO^^?hSr2MYGyUI*o} z!Kf{SEY;eDzhMVxy5ba;etH7@@ie^Cz60HtnBvK0FKPIV6o?MD;>EEu`TX5r%u|?& z?k|nF&diw#x7f0M&@VLhl))8}l^o%1&$7XX$hCuu?3C?|nx(2-ZDog&D^k8)wFffB zxO3T|-dK}{rGfqvusCfel@Cfl{h7t|cDftNX?BP7rJq6md4T7okr8Y+y`5Lszw&&s z>xsl#K1rC>u!@H`x6|01EIMQ!hilH;vX5*dD)nlF8S8#+ZTM8{an}|Pm^%dh3#?tDn*OGx-e7#aW=ZGGgI$`WfM4HqW%Y(wW^>Y`zKe~v# zmS2}%wD7>=&J(G4R*L}f>5%j;jYr3CqH$3}aecyQ{eT>q;iBVHahn?<;HP>?C!Nw`H zcz09DfCaXw^dc30-N_OE2ei>L5vKS>V&wKzA4xrK#wKT$PD+dpK#=nf$qGkJf zC^o#nncIqaocDRUR&k9h$~95Gu8K8#525z4+AMqiC5Y>fqnt+s&R%(kx2-gW@dsn+ z!4#6(H;%?XeR4U{wT72kM1xY0W$AD84nAS^i`;_$04QaO2G*O%WoJ0P+OQdXpB|Jh znLQm+mUYA9&-YQ&8##Po7%$9PSqy*k5`>RW!o|VSpT$At>ruIK6WlEAk8;W87<~K; zFEySnfy<9^t5$(z%OM{)JoSV)EmRghZIWQ<&U1MEM1L{w-4T?kwTh0udSJC|1Z|J9 z<5F{Z*6ni@+QUD9_vTJKf8B6+VYCDMp6#WdUm}Hrj+Wdou#f^~nt_e57No4pwvnG8 zK>G+hIqpWr>L)>^haLr3Yhfg5qNed_8sNWLxNWV6>vn!2TiZ3BM}xK4;pGo%ld53k zav5IdR!h?K&qC6@f%wPbM+XN{!E?<=DI)wfjPUitw6|wbP49&mCNo2DJ$e)NUq6aF z9=l?4x@E_o>|LEa-c7u)^0Oj_#kpbvj$|{aX+s+X{vKimxGTkPBZo zXa=v!^JvtxnTPm=;p%o%?mXfu^qbudxr&Y8-?Iz0Q~-LDE3S|4gH3HKN)9(qMQ!`j z(yRg9u*z{dy0i~QNtd&fJ2n7S6^W{CMx*zyE0D}CnBV^veGDP4^a+LB>^k&Wl1Yye z?$Hd-gHRi~1h!=w;(=GIFmcd#y8Gb@Dkxg7sg@bcPNFQ; z3gcku^*UOadWAMj+7B6Cwe6CYj_#fWuD}rjNk2KXpQzZtZ(cssGqwnXj`{}_K-<< z@va>|ee+y6t}pk2-sJyBwEFo5 ztxIKbtY!`=yV`JONF!bC*&kik#G;V8m%7d=#Fx$CwEmnrZ_y^fHB$@E_C8FHu8+lA ziq(*@dOB9u{ep*Uw+kDD(IAsNhu~%=o>HBL%^O>(Ce9M`i3JfnInWooW&Ody!gDWoR^2BA4|@xf?gV( zINDSe6HZMfjqx4)zT?h(+P%*(#b*zCN1ni0`7&(xb2u1<7bWX)Y=()O?yR3l}xXy_rPN>E8eBk1f^s4Apah~ zq5Tx`h8P8-C))7_y^i@b(FQfbRU{kVI*Ci7UGTQYc!)Guj1S-$Onx+ke7+QbProS4 zy*LdoUvCl%XRX4O+mDN;=L_J}=L$I3!7;MSR*Af*me(bzh<;NO@Ip1g_T{7L-uKgV zx=l(eLd{XVZX;R@y-v;BzMxTJGL;1%0E>|E%TCPU~@4R2#H7EalacHA<6eT4DR1j=R-voAI4VGLKwf z#kt)zIC--Uo_Kv8T{<1W-z6HQE_O$yhOQOxbF>2cW%b5mmrrwyC3A$Atq|e;niU^w z;@etpDiJnNPY)H;->#2WRmbB1d1buF3o!p`6c)b_@W}ouTCh7?2ug8U+@pyl!DFdDGI7xQ+IO9Wc(;Ax{aeQ%HEI&xnBPs5 zF%r4=ykn5}v}Z@`Q4@9RtcEgvx}5H*$>Q@FIPSGJo@__9$=#0y5BtFn*hBX(?WUdM zI{N=RHCB;s!*@?@aa*Z1MyHH}dq3O3WzqQ6n2V= zN@XFTHX!j|nDM zr0`eX5PJ-^)S5U0S4%f|JCy6~36?`h6H+0p|a{OC#a za@KM-5S&&gi7PFU&dT_p)fG?N-f_3CbjS}cIKLJ?X_S#4YH&oM67QZa4|0X+{O!{t zdh*p6Cm0^&F?F@v$65(Tb}OWxX0oL>+d7rX)oQYf&L@`d82`>MZ?aIe0xI9!VEOmM zg!5${WYcySPk+%Y-Tw>mL|r}E-m=BwL#4vt4(6J-L8f#}WC+I2S1IjGY8Za;I|<9m zh3xmnd?(;!vF&vYK0Heu*G#K`Z%IZN99c`I8;(&+(s6KmkqzH6oI2uj3aQ5vZR{7Q zONr_hyzJ$1+8M3Ki(Ga>qnZ&<(^cmCK?anrcSkySNKaZn(h#kL;bN8DEBe)Dh!axH zSb5fOs%lLlhv9)l&w}A=-GYwwo?&mgA~bK8z@mXJSdij~vv(1NYn!v{t)4jR;$$=r zNyRaG%INQwh|)=A(0QRQu6S1gYnMI{xS<#41dZacV`^YV!*m?C;I)!KOJ?RD|fvFA|N zI~6Q+Hgnx}SAH6;fMLhy+kZ5$;OPx8Dg>% zLS;@qjJNQW4oyu%#Ua;F=UNZ0IbwxZ7bD(%X@PeqsKC1)AHc{VmM$JDz`<=6ESfZd zmVy<&?p}#yi!V|2$`$xw`53XLy@OX|4=0Dp9dJJ%@UFujP_>)Jr3wNTY%vl&V!QBG z*EizhPnmS$uqD4v`v48xE?Is*5ag0_=-60u@y58$qHZ@CT)ycU+#1^n$kFn>4;uo-dvnC3;()6P;f)NDL&Gs8d)pt{(c9Mzp^ZG)L{G z^`B?LzWPYqxy_jCr|-aFCyt9ZGiG41f+kn1W@2qarT8brn(Gf%3Fpt5qtnj>NXj`~g|WJhe+i&~zx&JJgruH$*VD`=ou5L@qb?bxrZ z!Uc(wu=%!HX}J&z;qejl-d+_2UL{P`0iOQbq;!a}4bHc}%PQ>|_&>PCKQmc6xJ5tr z3*;O*pd;d5U(A~nO;I|IysyL^2Vd>PXM|*N;n;CBKXen=L@4kcZv@8$Ct*W~D^`_+ zpxHzjo;LFa%`I?%amgM3c+V2}QAyA~q#LK2s6ta04_I|!kGQF3JLNyk5Z)i_$rrr8 zk=PL_Gi0P8KZ@+QF;A0K{%0)6f6GhhXqT;nTxnyvQEBY^Y%HJulH69PmFixIMZcd< zu*Q(^N?bLTStR0-;%xLXxXnpA`_Z$$iO-kbXBEj6JiV_K<4ZT<_^Y?@?c8*1zcv{! zr@s{DAIZkwA?;}5)3x;Uks$J^PDHnpm6+bWfu}D>A>WDnaKC{@sja@UY)A6Se_x(XhqO{Bq zrJ=1x9S<(OfeDj{&esdC9 zr-@8G8q&Nr|384;+16dinb%Y+Ss&Zu^J_Yut8gzBtzqAMTS4^Na_IZZ>iFJR z1EfHlDcTi*jh1_<%`(J)tH#J85fMl_bBTW2-w(?(TWHI>JXG$uQ6uQS8Z&u@*hOk@ zK+?yJu^PTX)E6B9yLdfdWOB#?-fS>`7KL4Jg}7m*!6fINCr+lS!&sXVW~R?0>sM?9 zLGes5+J}U5RhC_{sDmVZn?WqzjZg@?3M$8b0*5OI5|34|NTvdg89pZM!{w-Ecm)KE zN3rEu1;CtKdOSgybNKr+a@f|bCe`)|Ijoe$B&I*4Nv8euos&KtuA4;b;e*yQ_R4W4 z?GovK+lp#7KjY(E`fW##OK(P3wvIZ-$V0<`06v~RNY!_7Sl^3jc=YBt<(L)0e@Pdh zXXG}Wb})hF^foe$CnUMi!rkQC+Iw)Yju+V0OuXM+iigZb8ISExF+yAuzR!3B>TCO{ z%K10= z9%GyF{l)2=>PRm-jqxT6^t#dTkqtTcpA)gD6y+|zx(deXJn*IaVYp{F1OHvMfLkV` z^!tPcTBPm*#~fGOX`Bklm4>85LC#wnjrOrV|o;g+XEyA}? zqhQEWoU7rOjLzdX;a&MM2vF3d60=T%&$cld@?HhiW)9(O0cG@(D=}L1*=%~PlhT$yvR)Xsvo`bo*_aHuLCTBk5i7{7NYxJ{M zBCfcPfu6ZIS}6~k%np%lF#`D1IULl^{2@;T)o{ThP5hJEk0z)9sseAxe+FXgjIc{J zYj!Lp{X3@iC}4R4uR}t@;Yw)T(pH!N~<62n> z;P4-!LmYMB-(e3w_|2%*mYa<8>1kLsb1U3wRptg9jb+bN+rp4{J!&~bqP}tfT$bl0 zI@f=)BhnG*mmG>Ciu%~wBf&K`ji+hP7~t-cU=Mx%On&rr!@U-kN*K?9-6OA2c=|EFH1Le+PDu+4otfQ?G{_=^f-) zehemP%;S_@=7+G8pXsRrJ~*vs$31&#KS{9pg#Vlj;Z?~Th~2)D=;Q!cZ7Zk3+5DjV zs(@y!sw4)7&9Hn_5Jd%Z=~##_#+-}8lB4k?ls5qswO2A}&o+^pBY?GAme+ zO|PVK>El8>BG4TJ0(zgVD+Q%cUQ>YVD;OZThbwUOC7^u*gEP+Zab>e6>F9TH%*)p& z`#xGgqhTg^T4aOAObK*-dy@>wG=NA-E=aFV!k?ai2UvSJpD-Vnv>(OtZG0&4x)bW! zj^Uc8e#pJK0VQ0@NquEF^^F;()B5B&ofXa0c|aAN8;+uVj{xvQbd$U_Gs(-cEvPuU z9D8@qq`&jMNc00a&a1jMqPIzdO)ETz@!MyBwCjas?e%ulAPZ02HMnd2=>3rrWB4-03_M@D66M2gfIY7jIDCAJvU_ebLD6G0U9^G*9+ROG z8Y1Lr>t)8Xqlx^uuFE}nlf`G}OyNx{W#1+jk<5Q#v}JB2ds^iSQ)8Qn+V`H2K7M)3 z(pU;zQ45jxMh+G)wTGdfd|)pm&3)1)MFmXL>D$2F9Nd~qp|T^8QFcQ3F?CaU~Ekirh4|GLi!jLeR~wy7aUyaJVKs4x&yC@ zL|~sm4UT+VjdlC_iLvJy{6rk#hkZ2I4kw`O-A+`ybC9o5A+ygrJ>-`@h?W5kACpUAP;e0&m6MK%xQvg>C~(%7vp=z}E- z=xye(NM$QMxLcQ_IX3}9It5Uv#t*vsQ*gf1Iq2`x0;3bjaCG@LxO=S? zRizg~5yza-dY*(|3L^1~st(x^9DwVW6he7n6b5x0kf6aRQW_u*8!Ve(v>}|%I&h3! z+`xzGgJt;R-zalBESb6~oFq9jbkVxC7~^X4;LMAB+?~e9efiOZY*RBK(bvr|`G_J0 zOXd(q*;8=+q&Ro($1C(x&~jo?^Nw{|pNO}(Meysi4SAH`Pjy6*6n!g3edOWv{ceHe z_MbE&&={q|`MB%jof-8B19Uh!LM3z4YcA`2!Rz|6Ai9nP0qJU(^{W{j2i~%?3q|M! zw_&od_ZYBeUr@!Vb>*IY4!54^(ASoM_)(Cbdn~Mq{SbAVSvt{#*`Cj+E1MxJ^1>ofIY%-f{2!cf(uCL6OP4Wsy+bz{^&(TtM66c)CH&9*g3I@$Zv4 zu%&ze&zub-KYj_H$+!4C`gChRDcA3hw%7AdMGUo2;M#b$b zV3pi?@<_=Cw$5RoN;IFOTDZb2QA@hvLM+(0^+Cto3Yd}0M+_qNkQ-4U@T2-M`!AUv z-geAo)=1f6)si$^c*=lud9~vH8`tP%>vZ<~;v8Id>JHr0BOE6Is4TDFsy1B6E_@0GC%1ugDE`R0sV9Eq_&Kd{k+UffByjH%HJnXNGK^f z`i4eCo`y1aFF5hf9iE>UqPn{z;nJuX8Ys;nS>_I`O|MC%0|b+&xnX3_sv96U zF3arCh=7PDaR|`Nfx-O6VEA=6nod&m?JQzOwi{q~{2b2JkT&4HUS353$Uc(%>%Y(E-WmMJvDLGwRMU-nStLtw~sQ=RUe26DtYg!xON2mI$^N&L=y@_%qlU?s{-xgBN~*PArKiA=(oTcxGM#HWXN(VreakD~*ua zHyrw>`8>XfKFCNv2)1s?b%2@nMkArTzXJxuk7`_S?;!FyeEaq#dTI8x7r zE{6h|XYGQ@O9r51{T|5e$N{k_{JYlP8%-kv=-yLmwD_R`NzJD8aeE!Jt22-lUb_~Z zW@>VrgHPjMVL>=|W;)e2K1VOLt8%MNBLE{3sQc+^6xbRHV;>9H!-KuV?1&(>E{Z_h z_<|l}cp>p}Clx$zNav;VK>=;Hw$qyhV@Gr_cv6IWrnC_0^fuDdTS9wEZ-8^_J=lA2 z511~|gm3Cua5qyPO1u$Vl!J-OyzBI9urPPaF-eq`5aOs9c@w5IjZXLpW5D9Gr23IN z$i$eE&h#XB7vm4^SG_Q%;4~UVsFHajXW;$LP{!1x0AyxN)SUnB%g$e92PT4t!BYAQ z&3qDxnLmOlhIZ8))hPq}nnK$1o8dTYFoqjv+n7D_Pl-~HG@8mc5an-0$Rqmzr*rQT zkq_q~MOqZAtyY7O_8WHgljk(jD-_G|H609GNW9E@8L?%aU|2f=+l8~>$j9|?(0>|! zIlYud*RXg{?i1ZUKMNKmCgEF?=^**XgqvZwn8;aKpdg<$+})c=Pqp|#w#+x=86Seq z+4k7H{R3X{kc0b%{B*s$D&69`n7Pp|#|{2j2iL7x&7QOkMyE(#0v0PBgIjeGqK3u`We-ATnDZ# z>#fJX-et;h0Zj;y;Ar1#q5}ce@G0pe`}hQx)_0wx5x;`Tqg-7Yp{K{$;yr|C>=Ur% zs5OfG6s~DmkxS1>2On$?%^9^7NH1JnE}~6*kZ5f|ys>}}cdkl-Pe-qi@p(3k`I%KP7$5-M zb}LZ9{Q{2Ws6o*0ov?a-7|NLlU~Tg_<@us9qfitw6XU`6P8|*;x}$to zA}jT@lsuTe2R*(X#5YVF%6xl_$Fo9Vp1&u29X(F#uNRShN&axztbjfAbE;oH2qsRq z2uwE)p$j-~h?nqVxO<}xYG0->OJ_!-)Ba_+lX*c-t(@W^TFR(We+>r4>QQFPPHOjK z3o$(%MqB^<2DNHI?%>n6^!Kb2WU^{0KCzRiUevJ#_@oLj^9VrSBQ7JxF9%iP<>b`G zH(*{7gXg7dNy(vhnzCsLUJhLiHa>iinX(;{l3TE`N&?=#3Zh%@MZx8Nt;~5DC#a-S zaPHJ~GRtHoYijO^#bd!JA0UZ+dz#4*Zz*{z`3S|EM(Hs>dAhqIgqD2@huEtUoX2iv zX#PorQ{jC8WgGT`#_%ah9qO_7VIEbSsfKPjm!Q!zo^ce@fZLwy@x-!dy2P&pR4sqf zH6#fpCSxHk(G0YHw3E)a%J|vefctjOK9ExmqNdxMY1WGZoNkdu|K3c7kaRijvIr+! zeP$zEble7#dLQWbZ<2U;TPW_CsSbUgWuebN8{DO@;g`Jka4CH|ROnHB)cb`xmV`lC z))?*}-yyQq9`jd}5czj;7`>zf4P5@gG+V^-w?eQw?Grp#8AtthGB`4BK{kqAW89rX zA!yq**57PDJnb%}9DVni=b{aGoW5dv$9d6jLl3>!^pZ>%$%1i;5S(#KAYaU5LB1t| z77kkA&K=s^f(5JK%AU2z^*Il_+b?75(>7H6*i*ARyOee*UB%G{GfC^KQo2gBo*JL9 zg1Gi|-0rg()%@FR@caY|7b9J*A+rS3;JZks?)9;^$&aYW=2j z|NN?^;l_=`FDH-Iuh2p1K}rU`MN>?+glH8Xoa+?~xA}Kq;jsyJM(1&0=S4u`w@>D*GsC>9cp>e?uH2u9qS2y&f2Px|ftUe@4Y3WAb&+C))HK zt-TGB(6H_h%t{V}>Gu<8YqubBWy8RAJ|BtgxD#LA1IpKyE;=6I(!6^J&xqwsjUptQ6eu}qu24SMz80hcb26>sP z7;O8IJb3R3V<$Gk*`zplw$2+)ct@}hs zP)Hh0mg0k*-O{+`<6;PSHcWe69O2!8ZW5*}0vli3L3dd^+*UPbNAkJsuL(=!krRjK z8AdRDO1DYt&BVIQK4h22fucz$9iKNJG>Wwrd2L+5N{Y%8yMk00Hc|y!%}g>;ug`gY z#1Of2XVKD(FU+PN*~EMLHM+tWP`KXSY8`;qN)zoOC?xkDE=U)TgIkNwQKm+ zt_YiETY#TsD9ZI((CO#Yxz5&ccxiA7AE!IR$``$K#T5hYx8UzJ{4b~WG2;O_{VE2p z3Mf;X$dANv_F?*?Q-s*fjm0B_k6};!ZXA8~2CqM@f`2n^lC?&87`>c>^6Rg{)XQgDbIitD!Zvb z(mXhnGl}yq{zlFVH8P*O8&o+X==drfGHZhIK-X{TeBKxJ2g=Bafj6X3RTW2+Ea)1C zbBp|nvoT=SGH%7kGGZ)lK~EX)#?Io6G)43>ytJyde)UTW^lrSw!em$MzhQ`p)$wHY zse@d#M*Fm zh%(yOyde!rCs8%slZtP-j^k5jYz)0ZCRe)PmN}2H#fy(qy*vQLeAZE!lioDFeLYG2 zz5-)}V?ldZoVwZv!N;e^aD1&eq#24}#I!K>NbC=y;8{i^%Q{hW$cEE=yb(iE<7l{z z5wu6%fdRojYPrc3_W#lW!#{V3uiw;JJ~xu|%yU7tL|&YIHv*H*UBF=PVmR)_g%i^( zVDhsLZa*#vi>y!Mt}rc*a*z)$bvq5t|INZu_d8_V=Q@fS-vBj>b5Py1fUEa4i8whs zlk3ln@m#ASx;eih!jkU5qocz0NjndN>7tzWhL)E zLXFEO>D^Q5AW?V|%#2un3q7UL@<&-N3Iqld8njgVNX*Xw_haYO+3AH9Why5ljyvXw54UJs7D{?a2? z!yzFw6umNZxcv*x!mflw^1^o=1nB4Ech7vB)$Rg6UXKH}CKmSRErSd5XW-?=a-6oq zgy1)E{3vDy^KVS&d=54xP7m*b?&TAxcBPKYc9*Bml97$;vVsn4lt%1=sHUjxI*1ai8*gmhd`z_&f&!185c_e?856*E#B7f36|X5zAk zLfo{&t6<0c9XMN&m!mb3M+5_xqGLb|mgK~vShg~ZU(01{?OAH2{*iQljlrt0@63+Q zuc)oO01C!bINpbR;eDwTmItXrf~hPB&e8yltF5?n>vh(&po=M)Q3VQ9{lawK@+n5YRSlhY*?=zP98Ug!J*lEfGaWrLc1O^ zJFY%KGm&bdvLu+?*37`F<0{Bsu7RyQGMxV&`kZx7x; z$Q?P@vo;St9OB~0R|)uADgZ{`BQY@X9-Ld?#vaV~!ayko2mHU2$dVxH*PlhAmVc^l z_Me5Bx0iCWJ)qpo}c zGTM(ZJTM*{oeIeKsaWDP;0EVM@}OZ-7!p5ZK>OA^D1IleX3Nq%e0XFLXNTEB>``7! zrLyI5*kB8da|?z?TcxO&-vD*HqJzSBd!SM174^?i=A3erN2!7(Ot7&y@K$HBd79tJ z?7BeuDqa=Mx2(lNw=!J0Dj8nc93YR5UZX;$g81LM&$PEg1~*u#a15{W!JB!3am&yXki8HD62~?{&0cF_Qfo#1 zhBv_48@beQg%q+Sg_!+of(E`nf-`xG@ZslD>bKYg?`Ma=(nB3^IOQktJZOco^Ndh4 zeg+;)*o-q%m*JHeLnQqDAWBW&OKW##!0=NWNLpqGPyW=`NZ1cTosv4ot9U+F`2=DS z4TQ$e*C;ci2Ohajc{e{Br~1Kh3@s7kUTiswiimVePb*Dc8pI?oT!wb-GvH*3CVbrQ zOOnPukn)En%$M!1*m>s=rrNIqyXor*&z^LA<8uLJ>(60R|2g9R^gih39zpXl5u9zp z!wueO2@V%MNc5a2bUT+vdGk#1{JbUzS|E=L(!ZcCuV& ze=&M1r300P#js-04ygLP1{6H6;ILp8N`44I4`C<#-aUyojHY_%?N{W>l8nRPu;$lK#%R7TXfH^|;@zd}fe#<)UcY6i?(I#TNEY*9=rkOf zZwW068X$kaBxmi>0oMC?Ct;!&KMc%KZ$lT@nr4mlt4o{ce*LIH(%)OWdq3ps^B|(bQk?B; zXLI#@FTnnrGB_mi2lE{l6ROKVZ)XRco463_a~`2T)RC&>LQuOY1w7u1(MIn&3Y)J( z5tA&GQf&mMg@eU+iE2^!~F?#x9Vb;t$Y&H z`MhY}+ZHAwW-1r4riDt>U%{BCt@OgL8k%b;4VwxQLDQ<29cj*jf1f!Z@$3ajsg5Q# z+B-4EcOK{9s{6RKjKf*KKnofT7jH}*VG{Y;QX zYh6Kg`3(A|>^v?^yv^AB`VSV)D1?(;$ENVKEC>`G!4*^9L1D8N$o>{2{TD+qtZ+5_ zJSB&3kCYOD;0);E0@c221$>)%F{^Mk>hFI*xAxp8vMoB`T0RUXm=65cn?s%+&12S! z{-ybg?Lp&_04TFN(ci@YK5uVhSB77v+dY&x*GU&W@a;P_-XjVz_e4~(9{s8d5E%3A;$47Orttr~s63|ntngVvvqxxm_g79e%#E^*ma4|9IR zBJ(7eT-c+}dFUibTbiD(q^LK6`_|JR00%-Epx* zI|->jNK#%pLvfND%xg6Rk8_vMs`d@kUua^cAK}86kmaD?uf@{Wc|hNpPkKQeXfTtVs$xRcp2bZWd(JW zI-sGKhW|adL{|4DLQxVwDOwW+4aw3N(_jVRliIYRsF((3&WACF#q2&E)0(wMG)R*A zOx8733Ki!_ar(})R4iMUmG^4^(|6ZkmBAu-?vX&>_C${CIl>DBW3F$xNsJAeNyb8eETNPp4#f27HPSS@3-NgP)HNJM1 zf#rQ=snOLz>KUwNxl0@7!)HuiZ=izAV7`1Q>K{%vJ z9xIu`+^vbA(R&ju*PaB-3&h+9-{r~nT$!NBg*lZ zQ=ivm?Ak0t`cUZ?RSIx~u6bJsbM4C%r*Xt}0+KXIc>WaTD5FRJR4#g)l1x>WX<|u$ z1D33C0V zgK__oSu`TokLe0*hx2{u&>|8IH8lnpL+0Sm4QW`?ej6N~Ta(Qrl9+L&f#lBo1qvT| z@WGu5{FmE-E)v_Oc*$LhB0$>9*2jK+zP4-7Jb(J%fxT zV?`$|bwOR|8d-DXH51x*0iP!F(wl|f;6vxlMN!cQh*(@CRTy7P3>G;-p;0*d!Ep5DnY14TInep79F@{Aa2n^!>2TPLx+n*Ghdi<>cb?` zbD!v{*kABKbvs^Kl?0mqDwt_yH_)f+CD>dQgn{3gHOF4$kt561!8h0d#`p|z!;~+S zZ=Qi(kJ?GxL=l~5W&(!16caW^6X((lFx4;y9qYqnwBZs&ik7eeISn}BuEf1lKE>fC z`A8!72y%s8vGtS}k!Wk7(*>Ve>mAB~>-~=Oed0ru*fK^mK3a34ze^H*TL6b98%Qin zL9dB4Si<{`zBsAQp)+QZ+4eul7cVn%RDBLCcs`4IzU{?luO8#xBM~$!ZV8^X38o2h z?-+HHjf~pa2tpfc$=8API7^|Cc_#If90+_$d>2Zi!8Q+!ysXYC`*ok1`b&`&7Mk2Z zT`5xZsedYG`HW0f=VJa1ANKPKO)_XvfNi1+uuG3lw8;e!VF<>gSBjr$zB=AxnbWJn54uZvWmv75;3(mx7lv0;-*cH{@uLV@-cw5k=GYT6b07BFCozyJt-|GzxV$fs}zb?UY<+&iAk3%Um#r;--af20thugLgqoW^8Z`~aPZb9lSK zo-}{z292CER6OqvO(iwNsdQG2_N&XF6(2yaA1ni>U?CKrJ#$KD-H2Z-VsTFLJXUO) zJzD=N1ox8^zj!B6M^k(9{@Yf76vmIeJcU6_Z*t+8x=GK>c2VtBY5u&S+;SN8%Is>ngdcP=eHF36F4 zexDeYD}rrCGc-!d;r7}6$S>;;&yA#@rPvZRo@jGTc1OU|L{^UiTs3H(O_SO3I;3%_kQvodY7Dcte2C~EP zF}(G#pfj|Nkm5SQsXQsgkiMVf+TA>S?RX8rya^ka%kVCw6U2`-kYUqdNFQvmzIjU< z>t0$Dt9!yA`>K~X-XAApsgfwrbb$80^MJqK%IMhHwWwu2mAm*`Lc5-2g8xBROiBDq zIv-`AWW}pRaa;aFT^bCNvb^-*zD%-uvJ!Y@m*8q%U+cem(L}{Um*zP0V2h6c8|dDH zi>7okU7HuQux%@-8E$2DZ9>T@o@qpL%n3$bYyjujQ+Q<66F9!C8qZ!1#Qjs5h=rzY zq@hL#Ef+3@6!!sI=35FJk3Vd#U=KEgS`meg7F_P_!HA71Kzd_7x&(-V(a9Pb{#t?L z3r$iN$>&hMH4yf`wSrye7SJ8NPB^={5o~U42j)pJuK#)fKI{CY%a++N#R+#Xr_K_* zo=$_&$CqeG@f^IJ+C_e+n_%gy3#eJsftPoC5-sUzFwj1oc4Y~2DjuDoLBAZqe+uvK zx3|Shp)s^BaV;AC9%pRR-V)2D+sM#U5jvu*$(1-1%Dj{Iq0f|bP_yS0N?3O@Vln-+ zkA03GMs@MmXVNi!XOQf*IcxIO<;?p`u4UXop@vBSd?VrBqvlmlN`hr7JCKFyF=$8b%khVybc|oM28R-z^7)E@SRaM`Lo^q7D0N zGua34r*ZGUPp4{!XM*|3AJi&qiWgTUP2~(PFv14&@xE*{dWKD>Jp~KNwTYJv2b?ebWe&(q`Q2$-G3tFH8y@_FIvBjf9y=WfJ(W>Y`FI+cxv(Eo zGBr5MJ_mp{*s*&9xr<_(c7u_)92RF6U|^#)OcsRU;p4Zd+yPgxzrPCq{`*4|Vk!pQIZQSizQL%i zG4xc-4dSMD8TPG7W?OHUQJD`1Kr#`G3w4_l{^2@iCSq+;m_w#xceDaNk-sb zr}enL@FLC@;?ka(=UEZmVl>qZ0PnVD&@);Ft=?+fy$Mb@@%s%%zq`h4+W!o?)UKh- z^-%a)c?q@;-LbyVxCdL_W?|hO2kSHUb7;|(I8qrFjl)wodVC-kjUgUfi{BHsd#22< z+F5YX_#o(bmN9!8)&ggl805`=K+H_|aY@Gy(A!*3GS6vK*=bv?>+NQ6DxTbhIl8|{ z*4z8YE1eD7tbQ`7Id_QYYAM>as+KGe(8TajaT;ZP4!@tgMJ_d|F!%KBs8Qcj^gON3 z+Ra`Hdr$UZ_j5Z`dhLa+w|^5!Gj-N9V*#A;?7)xrr?g%DF*4COO0Fz^#-MO=IunvJUkBi{0TM9j_?m+WS#?#+v5~!nZh>=UOfTuGP4*%|Bg=cBdPSX_len$^b zkB?jbvlFWXq)=8{i>uovgS}EI@H=ig$|ZkB&jtB7rtQNXFXP20X@_9RI}yy9zXCMZ zEysTGL>M&x%I>%DsFC{VfnfvsxbvV5jIJQG?qDH$`IV6RGgs(coBsf6q9J->8?GKw zp-0zA;=!w7xHanGUDc zlk43+@U>2dsP(II&(1&2N_NO`lPV3teP##K=W_srH+L>_zm-V0Sqj3Xs027ID*(My zeO$Kd1^QT5LMPXU82>Y+*URMTzAzW`6$`5wU4H@=?D;{;du~HiRS>k8DM9=sA2acA z7#=!GU{FdFdbHkw{hS9>!BWLq!PXI$bw-ey6B=xV#2no9u9JPTLJb4@mNTkGlms3$ z1xBF*&G$_(>P8++aL*0Ab08SkZF0am`!(o)6HrFE8Yg?6;A#7dcx=ad8qlr)f%i&S zFHQ$-TsMNd?^s|*ei+CO4$djzMRHQogGB{m-A&0ucn)ZM3(pGoeKO!~EYA=zd!gX0n=Pji!GU|nMd{4vw#?&I`A zbmAWPP0r!;fOK*{_z=*zy~wjh6Hf9JqG=fq=lz2aIM$Fy=d|n~t3Eu&BdtSJ_UZ)w zJP`v#b)2=g(WKq96%HKh0e4<4>i}aO;&{yyO6p=s`agTdC+QhkCp#B6XobKvnX{m_ z{RIx65#VkN<^v0TE+J+DT>q+cSlHx`f1F>ycx)IL1nq$d)mP{g*M(<1PBJNE5x~Eh zOj$6Z@W*~o65NO%WES9tx|Lu!aTeaZaYjS#TcVx%3vC6aa_rVsX!|ykNY*Z+qUz`H zK}r;8<&3lK@++`EDiXw}eDTCTeB5qVHecBUdtVut{f_tf^iu?MPWPjf-yMkaTn~f#x6r#g5xotB!Sn12(pU76-q3slx4xZZeXVxE z#fJhMO@TlZP#R#r4n|YA<3(^!=q385xN=rz3F68vJ=Aw)J-yt#0f4cgN^`4e+LT^o zR-cM1mqws>j3h>nXW?k!ReEaoE2tSQqsylQsmjX>n~c3eaJ@Sk)!qt)>g8@0kPiYE>Z?cB-6jJlB~$@}!U<h=&o$(|Hn|G|hq8@&tD*!lj*a}tF zoksaZdr{cu2h(bl3!0*jsj^rM+mXByOb*{;_?Jq+QV%VRP54PhFDY=&{_BAEE2Hs{ z+YZ9U3_;jS8~no@gcw}U8jzRZ(5}sBqzi(P=0^7HfneCxCIxquQ!y(43#%CV8$Je$ zauxU7#Kg{P7*enudhTWrlU*A?*T@_HWK8L0Q+H{alRn~|*lpDH!znzVlE)flB$0J= zoJs{LfbG_Fve@7ajBauVf$uBHUBAa*^5q`g%X6Nb<5yvneV>Dh;36vaV>bKM{xCi- zY=`F`2^9Yg1Is!Ux_eTL{FcmMh7UX?u`fQ6Y?F4nInn~@1ua>_!^GkP(S%MNsCk0e~{_W+|`qwKqoa)iM&oG>$k_^U^ecPh8Ku0#QE?I}R1y%}Wx z&udWM{Fj*b{K4(;7>p!0qq%Sua=f`zGF}axjJII$6(jmwy#n3YHk_u~NXM?;g3fAN zdKR*ATGm>$R;yr2GI+WBL6iLVC>W}pO{Vg%;h>+d0d#6!qs0w$_Pm#5y3$v|*|h+c zt7kLsCr%^Zaala8^qaIDYl3YubHG-{kaJP;DOhP{Lis9rt}oM!3y-V?Vc|L=VXFiK zMICe`Z3P&qE0P&ivYe*c73#|4X#+7PbWL&ctqVx!I*dsfrs9v&ogVHlmg)Esk;8(5boO(W;#=wkalWjH(qmtBH4CLaZ-^x;jfA*gdUk-9uM&z&T_cX?INXKbZc|?I9oXGC7pMClB8x-S2tCe& z9H=DyF&A)P=ok6@tCaXhg|my~^{`ry7Ytl;sN4#El(B0kx(US+!VjPHa)D^n(iZ(HDji8;s zdAO7Q-{|7cQ+J4T_(=Iw?(Cw*C|&1m09&;s;l_%m%=3Q|aCtx*kGWmM*Hc`4U3whd zXS)`fm7cJT+uxCO>bv2^`l;ND;bA=5GD7yA?ZDWu0EpV!3S09fU_U{C@$osOozfaCCuiXs^1Tfr@VCy2w#*6OCB0R zaaF1HyU_x$STmm-oTozXKMMr&BO^@yDIRie)qji3?Sn93OYz;kR14t<`( z9qsgF!zbGaWNCoO@44tO6i7D5>0z(sOMH_ViSXPVqg=Y6;Ccm|EZhq@{Gnu4v=_4} z>m_N}mkpL_0U+Fx3wo-r$j*dZsx5sIKJYGK3R6B(x1yQcwmf-qfB6EGx|M;_`W5v1 zo$F*Grh_hDa}U+6f@oP?8yoQTSB=q^i@-CPkN7_forhaaZy3kh+M+=z4WVSD6pi!T zuhEoMh$0b1l#vnIQ_@~Esf>21&^XWirlQai6%ABovVZpYoqwR~TvzA3&vSpr=WcmT z6hEw_UP~U4cv%5b@cI{PT=<;&Zybk4;V8_!cpkeVO4)Bd-!WMu8Kj@}mShhwZS1*{mBjuy50V@#p=i~86nJ9B<|Yn3@6&*{ z@PxA8d+4(9TBwnj@o!`XP1Yyq29C! zyq`iZk+~HCYVjicj=p9px99+j3N=8Vpc4}i_lQIuxqz0%A<$zzfb*n_aa#0B^1ynS z3=15_UveUNlbdxGdF8=p1v6Zptxi|J%*LY*fe=`2NR~&e!vB(;Vb8r^(8isQm(nGm zcW4#(@s>k%;UqqP@j6`ca2F_yoH@Nw4i@!9X^bh2k!9$Bv)V>$d{jizkN6B_U16?vidBX ze)x%;Ep>#~D1%PL=TPO~Oq?zAmrQ+=f=p{*s%_H&6XXgzwD5G($IHAz}=f{$=FnXk$$cUbGU|xx(DMxCFNs zT%&Y;I>^+l!uB+0GQ}kVzlJ!ny87qH$kf>&TwY1!H*(+egcj^{aAy0(U2*oBQ2e!l zLgf=RSW-5K6{ca>?Z@dAvvhcC=Br`UiUi8z0d2QS(@0^Z%(Fh911NEFpm z^`u7d@fw62w>a}(q5rTdJQahL66p0$YvJNaJ+Q7c$6k-=j8cCbH`~R)s^1x~y)+C% zPqp9+qtlQtt^w|`-S{%|8MVC{g=Q)p*!w{qwmN)&*_ZFa@j`%C^ z61%eH0c7ly;a@1*gjRGF{(W?u)Nc-kl+}Z<;p0D=^|z0{KTr+-);N*pLP4;ls*5PQ z)f3If^89<=H|PrWMR=n^F9&c!WAa}kmBZK-{M7gpaK8;7>W%a{oeEz=Jp?e#D1w_)W7E6e|2r7JSwkzYUM zH}!bWiXR{ z3w{cpXzama>@v87O3x?4K9MXk^dp-!oMDEqt)-ZWyWgRe9}nM@7}ELgL-FyISIpSG zMWk`PBN`g0@M4NPse^(mEMAb0>28u}X_Q_!XZ#;o`tKWg(6$OhCOKnLe>CW8Rp7nW zlhF3E72{-5fd9=L|ICVpP4(QIRa{F9k5V$QQ5@JEaX73c#}v+yVm#B5(d277)i|mI z*XA#z=k`qEm)*F4KeR2m?_v+wi3HGyLSddsP#`%gaiD&2xeT!{I|W4%GAJ4O6z8pw zfG4S2u-kb9Oc6XtU8X&t>!!UU{(gF3vZW5*4eG(-vmVet{1}ckUI876qqzO_HGDtl zgXKw_2X&Fb%n%FI<2&MdaLPkFY+BU>3(rXNznstk(++9gx29ZnPyA_mQq`26nqSGxz*; zc$i4=fe#V*;)jAE*`!`#DdxOB1n&)6u*_h@aizQne_+N95YFN-*tA= z94(kHA;;b0gHgt2GPA!%n{7V73*|avVL{A3{C7K>3OJ4&rjICvg49Jt{Zb6+Sskf-}~aAb<8^Qq=YYYm}>TP)(0DnQ@qk z#|&Z69%m>#NkM7KTNnRFaixEzG* zS1)rN{kQn>?`aIRKLExm6ywJ`&^+$}YuXsYO5fiG2SdAP^Is)Y-kJ)}<8|@s+a_2p zCyP5jTBGsoXd)d{OO7Z+ksq=qJ>UkJeBw@72^-|h+@^0cjjU>l33Go5)NEh1gWA8=<*+z=Nv6ZsoVF6ox1`* z-o*&^ESknI-cf*pO?mJ;HwevL?=Z&5kU6c2+rA*s}tE z@vqX^@fP&x)D%pDUXpj|K4D$Xlh|@Ca3Ai+)?z(0=KP}QvTVBPnk19@F(1=hS}ljQh7vejz8+3*z6@)w9;rWI;!Pvx ztq0LtjaW2Ym3JyF9(z{Jzy(DS@M?4_E!?02Cs)Tp#8-c~Wdd++r8CwDEhaXt%fV&t zQe1P$7JVEn@XuBmo>#2{C5i{?_r6{XTFZLK3FS!m8y?P(zbGQaw*Ijw$hPct^p$Ip(mky?`9l zQ-z~vLs(KCiW_()s7{vfr>{DPjchS^dsuRrilabswn6i?^As1e(7Ub+-5~^(YvD(pLB@vmAFo)gM~h*-xwe@Gen7*fF5RF z%!alB51MyGjjt-51S*Lgq|0S7v$E|uu`a2 zJu8)`&Tmg#fm0kg@50InTD!Z6Ss~)K!{*?%!4aGuJ!iIL^H|j-&u~F~5}K>UGWuf? zu(&xAwL}dY+i6-|pwDc~n1CYNF5~1y z8gL}?80lu$I?6Gp{~q(D_p6c0w( zqMz|Lj88d0GGE+*pbm8`Tk@C&kKUtmT~E>QSyCw4rGS=U&xp?^N(Z8Dz}m^R5SVwA zPGwHiot1G13FpVe<}$}S`x*q#4(*4Te-GomZ&sMyy&e~fW)ZFCYmlG46RtkqkGsFl zr0xe~AvL&%`keVrtfdD@71vRT{%k=%Pq3k~;ff&L=trjilwoZ}7Sc#_EeMtOg3apZ zalXe*9>~UmauDa21ROUH-2I5|_W4J%mD}+jpYyF0GSS;R3H$ZNG5a%*vrRH_1a|oFsS)HB!YbQYdn_94{R?PC||L!O2^`aO~tdT$vIC1IJ9c`B|K{ z{M6)&o%dpEdkx6z*YWVoLk*1lYoOL?KI?hJ9*+%PB8?w!kTj*AU@|s9-*9)Sx7KNp z~BO*tl0{7Cs)vx z?Z4S6*6Zn~!fAA#XDWod$zb3deUxY@ghQi%PM7y#d9euons<`RrW^BUm@^yscoDpq z@D>F&Z!quA{Y`L-T(uP?=}m=2ooa~BoyBHXHq@2Et^zNP&eTjcSM_I4=dy zzo;bn>lE;6ls74T_JF!g7B+vX(gU_S`RuZz7qNB|(jU8r5i|?f&nH}9_fs{L*fk%t z3^Yh$p$OD?8MCz+<-{tt3GF=JV{zjW+`KascaFWq?LS`;`x*D)XOtM*MM7qV?#w%@pOBRuX*aExPwj!7e%OuGh{t)BT$QmJ@!_i@lK~Ye<658Ct;3sFq@C z%zw}V1y5YmWB~RPgYdR)Af_#2VbWAn9M`^@~V2RjTRt~Q(H&G z51~w6H>y6PIPb+PcGy7^|9mt633sj|r4)}p_!EI2A5U{WOX1Bw4;jzeF^F27n`V%(D%R z)eIeu3aS2KY!n29w(j(DY*wo;XGL%Gv~5Yedjh$DPPt zF609}XK6n7+iAVW%_bn;(q-c0QWNY})&JFoeJZ{J*$w_O6(XO&UmxCImQ{xr&m zM37TCtBKncY22zTL=KY4AU~T2OE`u1+J?pGtAP;Zu4mQ=Xsv zE1KAfs6qWKmL|Oq!~EaHBz$NICJ)YmIE8!c`>Bt}B#lWpZQTi`N;eeT#b&}$!%3jH zRUVZxo-pdgK{$z?K-=y8g zK`lxeZ}RL}!$Yq0`1X8U!6F8a1>v{N%V?~i0AvN+r#?$L(xlZ^Ocw|x1Fms6ayboS zYLs{#PcN=g)j0dzEK7zX`n$TEtk?f!Qht@Ass}1kyqyi%9{CU$S@pgyF z@{Bq>;mUP)Quv{X9J_UoRPEGfChXkKdi=OY92ccfcA^XITQ*v^#bGD0PA-FrO%u_% zcn$N!n|!Bu50njxDA1`kfZs$LV8g-C=YrY=pL+%%-2M zHsGfvoCYAPjW-^+VglzY^{%~-B}*hp_DM06V$WmUWLM@^Ry5qn+lf7M=xWl}PDsC91k;$v*WuGAsdTK`>eD8yo!kX0VLoC;+sD>$LI`Q)ZGd7Fi zw1N5(NUKS~yc-Tg!0!S19>y^x2L3@0KOgHhUZ>}tK4o+3auCl=<9lA2iJ!h$k#%N$ z=nyT*ltxN|#1CN}UWp(VpRU475((t(){}Hya2R@LeWI2#qjCPLyCmE<2h}5D7-bWC zVzIFab*L5nQ#(SR9M>d$749f*rbApLn{moW9{KO295ybOr*kdbq3-Sp^8VBj&i~~) z1lnWZt#KGH;}CgzrnG*_%!n&{dCTU>%gEMOgK132UflzLmsD~k?4|(jke%7eIoY_heJni7>$P< zw!Jo&?NpkL-=ogLlB$~+6?2}1Kfgl_7TaUV;3)>=+OW~Qnb=v%^A=7m!_}VdOsD5< zTIeW@0(S%1(3W=cXGEH(VjMx*wG-)YxgzND*pB9t74fCdUNnmP%&;3)vnRJaCF7T^ zQ1Sk{q+Nsjo-gwt0#RbaUefZEw*cdj?ANNbqA1c5hYP#XNbwV zdg3=Y21Z6%G&M7tB*K5p+R-}lB_{=c(T8B4^Msh5E5N_{4J0r+oIaGm`db!{X@Wn& z)EjYB3%f{+UM@YfMwIvcb|K9@97Rsei@~V@Sv2^18Z1mc3zhbB>7;Ogh@{&zZ8QeA z>V#73nVkO7WoB+XTnP6M77%?gN`$`^k-yIbz(vsrCO6;1Gb0P(Q&k2QZHgp2kKDxE z{Xc2YuD$Tz8U|7X-Ef|dFFq6Rra1f>HaJFs?Pe)r_s$AoPd4rG*M);pu24ZrY5tNY zFbxlbfq)AH89c`KsZSZxJGR6r;5Et5t%X^;W!q+7|M0sik8TC@;d=pVRRooBmF5ZQ4 z>+Im-pA;%sS&i=fTsGCW8a~en!j*6CF>^w7AeVbTo=8g3cd15@IT()}(v$c{_ROPm zj@q)9CR|`1?fgc{u6Z#J%BPXz>#oBHilBSQ1w0GS%-@_)hg;>liJw>ZzQO1GZn|m2#%eeNL|?| z==BN&C(XOy|H}`Gu2Q^s=mHHYKLT?P-epD8uQNZK{}QX}c`&vn6(WVkA@?9lUrlKakmyCklmiNfV|77Z8cWB`3M^fPNuLyr_o=jqY`Jtb=GH!Of zj?=jeuv3&G4d~j<<(;E3cF=_>+@Htp)UtvB4@Ek#YAwxP7l9|%MB@6nbFgQ~AC)=2 zhRTR3&Sn_iqxSp6@^T#kbctdk@-HJY6^WDR1AC!+Ts5#G@P6WFNngcfZr zBJ(t?;75M|@VW}1dZQv%yp4fh{X+cqX*WrMbU!g{Nk$*Z_2@nG8s7bHBN!-avU6fx zP=)grt5q-4=wElSNZ}lLzPgu@jFjZtxfWrtWiA{4`UlzI?Mf1FULqc^hRA8#e&Rp= z08eO?!NHd|@!YYC5D-%g(>?0RAna8QU@Gsb*Q-faVk&v`kf)&Mc@KVyAW_%XHy}u{G zo27r)6_aIoB@XGZd~Ou2T{oTQ@Si<+rE@W}zIEV9C(=2w2T(#~0B(3qWyJp+ghC^( zPu(8PSSIgc4)srliRC}p>h*icsgv?}Cgwd}^VRTQCBYwSsMA+P0*q9@18g`N!5(Z{L0x#KV0?TxjBV*5^P{tI=}`-gj~I_S<&Qber<1e3p2kzlz}=E|r2+`W_IJ<6Bi-0d6>;?YAGw@`s!sp*hvCEHCn4p}%o?HDe(7LL`%0l*xtf>HWrXJz>L>2Jv0;zRA)dQdn1481Wk3XJw?llJBN>7}y+?4>D(zY?Uk%Y+xzM{@4~p#pe^hVgxU+_+Qs4~7{uRSU zpM4M#dJUjiobikCz%%oO(Q$JkeY2qfh^Q7CRNFw7wLD7x>w>rzN*xOLXi#qm+bp@P z_jo=IORNLnym6r`F+^Td98Yak#-=TENTJGKCdGY}-STt^J?^g$edeD~-mD#SUur_J zzyLmr`$b&twUd4mC7cNr$UD3Oq*QC+khdr*bK2R3g^8#-Wh!s_*ED=vAk5$I`j<&} zN`%h>%czfTF5TN40`p~ZQBP0>D$K5t*3uQkwsZ;W`*#X{2slo(wWsm6KQZO8Z4x{S zckT{j_ld4A4Fl=rIe6xi5WmZN3M)6SpM2%~^3U7>(2|^mpSZqGKx7+6)$hdj)4B6x zu?lUNI+?LCkVgBQ5cu{j7zTaiP{==?`5_TWO^-HE4Z0Bx2RL3+^-*}f-5MqCgn(T` z1kBem;W}ce)Gv7p+NNovT1guI9_u9o6-7{K_n2(X8lxQ=jm&l}PpGR+hUB&)5SiZ2RE;dNo+gu_IhYsz^-tLi+go8t6JmVW4>hx#WMI**d!fa_-8~ij;V? z$?8Gjo8P#(@GM3Oi}5xcQ-BFuY*BXWC|Sx+z}E@tAhdf2HjMY-HBOVP90RgOu9+%l zaUE8ZkML*R5>#_v2p0$6;=;-?GTt{@KM)g6qy$fJKHLZ3A9eyqyiG(7xse&4gSmdZ zA*S`ZLFTR*m_6PCCSOZ1IiQX4HR`~W3{h&A@(==p!@>7t7fI5Y0q4&TfYNaVylBt8 zbG=KkR`?!x`-h;NQ8Tmb(tPmEYh_Z@55s%k6wKZMDgsN4L86=7uW`;oL zx_4k7;US%KFtHBOD4AOH)u)OoDXIt0(e#CLrJ%L2N6U z(a!fWsG3g2T+SlhuzyPp zxv@$OuC0EHW>R-(OoKEizm}kuQ6`wJkU+biB*4Bg4Ltnc20YB2EywTJ;d8YjtbP=V zyVoe9=(5_H8e%B1P{JDglOYHF8HEZJaqK?zegHgG=gpBacK+vl*_%if6Wg}Q( zaW|j(+Lf~{9VH~OJQsxOW#G(V0rZ;qob;xAB(H1jxVbb2j3$4mJ1r4`EiHG+$*d7n z7<+`mHY)5=y?nayr5z^kGGI5kPvNK39su8|RaA3p5q5pd!K6biVD+8b>u_BFeGVt! z^dphfMYofkEZ9gCZiw*DpPqeO(Ar7edy_}3cN(PkC!qDo=M4}}NR2hNW2xoQ9-=4`M#Zz_K*9Df29=GIUz$gue!V=o!Rc4452+yC z?LiA%?~0zt`Wxl*n9sPLGnmg}dPHq-bmuUrn6{XYev7R&Y7G-LNcyl8rC+ zqr1jcTBvT0nP&4~UWo~;Cb6W{1yAp(T_2SFoXF?xPK`8;O?9zGA3Z5TmT>5rQ`LD{UEhm7~WNK-Nb}S>|E&% z3URey9o$Jj+TDj)KBveH(JEN&KM`5ugT!!IJ=Gb1PlDe)BONOvac$pBY;MUUz7JkA zfBy*8|Gd@2gpAFC7qt%wUW^N!b>#F~D*M6;(h zLTp&&UT!sR&$nvy57<_SH^My*&>tvre0391@0_PvI!`<`l=R4*;9v1TD;G13f$fBNIpQ4 zFXvdSMewbqjZ|@L30{&NS@T7go=eSw(IQnG1Q zlE8bJiZZ_rV@vD>oLVcy`?!RAca7fCZ@&Or`zx5S#ateL(kIw8#S-7{{zT&RH=(BO zFAUvtfKFRj%8b-FV!^ld=-Tsy5#8mCcVpZjKl3nN^?gY##rx>p%&+*hx)49xixTDd zO5n}r6M?0&90x<1CndCm@)wE{OC1%yVW~Qq^d|y>e?O;k?V8LlF>m_(>S{<1?I!ay z^&smF=atl5CauR$Q%~!)=q7j(gMIU0e6u<#%c?>9Y>wS|x}G@e_AweiIPK18A|{n| z!l8HzY|1$UU)ywWWotA>2v~t+Xa|hnY($KVK`YLGyLC&AmM=}B+6$aGKTV(ZcLtE> zH#qKfO8da`TQ7g&)czW4nd@R?&KD##`)OIZeBbOAab0m_Ln#uJoR=z>)R(H7FX+xc& zl&Pp}E=b&&inn`0QPJrgxzDlPO(&U)yWDU8aRXgon@Hn;DUz7 zMeuNODymN8<22`~*tNHV#zZ1}!*_sVI9>w*8IIZe){pUyxpg@mQ!0WW30rx`BQ4WV9YQ4lY1}O)g+8{RwU4VqxW}P8x)&B)NA9m-TR9e$F2N znUYv=bK-d4S1!Zxdz}VoS7+xyH6z zUBdN_3cPo{{TOJTM{DcMsC{P@1SS^I#tCJZ_qTv*MK`nBBOxSEu!d;6Z6Mo2)L}`D z8_akr$opVEg}?2MAkXk_BIqhe;P#+@xOeGg{IBLVoUngSR*4(n&(dx5xIqAVm8>8- z-;BtEJytY$k_c^V`2(p@Ny-&>b_r)+2 zx(uf-E~XFfF2M42vFx671{1fmvfuwrhe_!Ixcm1Av2N%^rK?%ceY&3vFt2yU|mu-`SGKVR7|#kIop5GhxOaYpMVmKd%@-3n*K4L&aMER8wTuz z_+e;C;CK{U*N}|qVR*tqhWC8?8i?QhghX>3$v6M*qe#vq?t4pwDPldOEYpv^ZN0{< zI=hVXXDg|X@qWDasDU2+Es3+F&%%T4JJI{)4=kL14VHiUj}@DJ6=scdI$!ivT5Xie z^g485srq_|6M2Xob>kp4Jqg5?N1@@)7i`^bwK%Y;*C2>BG9PpK>ZOem!`^(tdJ8lwloDF;9 z%D^<-%bq@!4*`9{ zq*y^8(?2CZNwp)KGV(!B|LbH|X&(*xV1Y?_`*3`sAKvx1Az)HDX1P@0r4qZZhwr0_@ zGYabtU%be%kxVPmuuv1hiynG5Y$ zpV?U-(sLfRzGKPS&T%&Ag&S+toR4w(BQW+Wiq*-tfZ}s!IJTi0?y!y^cq9h9Z<|8Z zm@GVzGvvJE9J=Gbi*(_-=XABl6n=qLAE}k+a+6;e8aHNxHxv`hjbCPg&t*~m_xyMq z-f{<=mU3N!!}1t<%ZAL_tcN?!&d1l+xaZ+IKuPA>P2{emFwF9^CNWo{!S#$YS-s>oUB2TM?2qgt;R-Qa|7<=Z_4^lu z=Wl^Go7|!L+-veoMjP*QK7ZCh9$M+0$IUaXz&8-!{iQn6V&{uBo37Bn6YZEYDTzMZ zai3HxC$dYt+(0QI9&Xy*!sn@yRC3Z2)WkZ})4K$zm)gLppW~PdzahjRiT-FS!3%0D z=#R{9GSL5*w%$x7ldZPHvhpYJ`Eo3-S+B+M${fJ`P7GA9o(;{($BFb$Pa+_$2P4xj z;Z@Z|%!dz_j8c6vGd_ewpz16JybVR^FRk>t&3`z5YcV;=&maqzh@kI~AWy$Q1;2jl zq1ne+_6$3pPCGG)eDvSXlQffn8Ew<)R_if(qH`a0jjp8j8`ie(YH{^U0 zOT3-W;MjvR*!k=Z9V^o&hj&VH3_4NX8<_?0Stkm&9ru6@lg}|x&xOIj=>`-IdJ^BP ztvK;^Ab417uqGv6Nbgoz5Zbblr0ZtG&IgmBBdHJ`Se7y86WT$0nmrzUHbi$^mEqC& zButZtBrm(K!`yRgadC_|3TTH=wKKPgjrj)>e1r|DbswYcx| zbaKDU8m4^Fqds}>NrwMZ5Wip#*MiHTw^kJnoWDS3=^*-rDBwfS>$H8$61OJ*#M+Z` z`~*KQygmGv<3pce_Z`1Wwe!S5&r=sw#5-_8+DdRCOVLj90bJX67muGS#TkL_@W!Hn zuov1va7{9sb$bWcujof9<2~qnBM+vn@W8omM#!08UQk)E2VPVzVl(B=gXD-7)$*=p zcLjXK01HoCA5#GDzacrs=^CozJiZy&5;KEp*3l##>LoW2!xwJ&_|$X|nl8xgi~NGE z$t?T(i!+AKmgmW|xZ&JW2gtNeMYIYJrH$A`6Wo`ATeQdsy!kRS3$98<0+^7A=X>`Q!34u`0iPd$1SwRS(IcQS`*$VM$( zG;tx#c+PQ7C(1+3O9ONqIfAp^sNn8pV(?Qu)qJF56Kt;+1QV@2bot|A91U~8`ko1V z*^)Jk=m#J6{5l)LpT?qV*%S0VP(W60lR~prOvG@Sxp}HW@ z97%J%m8fv^U$_#t7)71Gvqf&)j!WAPcGs*cRN5$o-7qs3413PO!{f6d_md=Q>HJ4F z9xuS3N?hOaS158U1)if|J{YSi^K_HdspnL6vP+?fHt#rvZW|1-#PkHmxZMLggkw=< zRtYg0NX3k3ZK}akl6K>Fm>As;VV6qlC#Xe3x@ruz&M?M(F9dOZ$tp@RZ`JR!e$DM6 zJBNlf&&kHZl^|#xg-_iV!GX9ad^97PJgbo9C0AHb8!=7Z2d0jwx5bj1xq~!q?N+9H zrXL#XWFe^8(xcuwCUGTIfC^q+^(U(xHJVm!;_8fIVcP*F{o%p%*)jWkIBj z7i&3i9*U;5(FxgSxqN;jD4LvOqk5HT}-iBvBaW^qPXnJIQ)w zhP1?|5Iq}r+PD8OedM@^dW*dzLZ2qnOA$NKL1P88FWv!i8s*_y#3^FO7MK3s-(XD@7$%Z#h(8JT_yZs4-&}(CCWoQhj!ejbjsv*alC;=}WtP7B* z)MaBddAVXeo5@7bV|h-fWf=pz600CNeFB>>@#b~Ioh9r5$AUf{6GB6qmZPMDF*Zuy zgeNN_uu~)*6hB6=pVS_LjqfWi-_cD%BHQR!W}NgceF=RJHNg7S19J|(ju(Y*L#z5E zSe|kgzKK1pZ`k*OYBf;VGj#f$37^8ZKw7gQc^&f@%h+@- z&oxFDtoTao`<7#c?`&8+>js+t`wlIcLo|QNai}<4!K&yxq13np{%d?d?|T^H3w;UR zJMT`2l1wK%_K0ENqGTGWTaM<+_vpqsT0AS@YOAN=A`LbVT=5L?dkagg?gYrCvTSJ!N; zl7CO)7b);2hbJ@7oWr=?I1Q6+g+cX_lnxd_IW`e}YX3 za{LotiXdlZHjKJ=V9hBnY+ld_6Z|yT!JZBhV7m~O8=68$+(&9OaU)nt#KT>&4Nz%1 zmG@078R{zXNch9cM0`X8HH>mV#Wof_Ru+TQ8ymcrHBQ*aoc7@|0Zys@rEHiwzxiMh zSr#1$-OO>UowW(ZPAcHfPm=V{>T|dydj^Je8{ug`dlZs-M!q=d z`NDilK@ali#2UQwR1h@TcuW~|fW&*5bl5Tuu9$6RPycJeprg~kOD+vhDr~161Eq9k zXeh^^@Z&g!01cl^;MJxqgB(~hZt$+6s zSv?ahQ;Wo(`zA9Fn+Xi9Ux&=U{h(zOh>jt5;Y_b8|IAA-CTSL@y*=CtQ%plh=vf_p zp@;{y8#KYYZSyg4TMb+>{z_L?ouoV3wnM?|F!JV)BGfe;f%~#%;HY&DwyDVR8}l+D zCF>=fx$7#p2Zg|^JYl}JzGjtdMuF5xev_?s+oAb zPBNjpfojS%LR*P3u6h{&7INho|txf8qd^#+izlcm@F!lhC{RG;ze6M zFs8FGztb4LPVYsX`t#H!cOjT><-yFcXXqMlMg~Mz&^wXxRA7oMjy|ZQ6QY+g+x;`y zA?rZe*e!>?ZsClM`Y_qb*C1~O<-lyaG&n}qF-ItcIFf?_Vm+`g;34%0VbPk)bWWIi zjCfDEK@y4=fY07wj$O_%0FrG<Rb01vE$q+t2OUna<~C$Kq_-ur~Y{S)xf-rAaE6 zkaz7HeFZmZao8kguGSH3?#RUNWxxikQ3QXTXu4|3D4jW@nl4NkA>#LBG5gUJcp;iv z|IqF{96#g=U!v_myrL22l@CL}k0e^>5kcc!FM&mkB<(C}!>&YKcx!bRMoxF*BVPwR zo@Gck)!2gc&=%ZqsRnG;dEt?#O4N3x1k~xPLED2I(BH%L@0Lrz+kaY^@}v;Aq?bZz zjUaUvl!57oxc#twoGv9-$ZUI?gklFbqnP~!e%%}y{+`ZwxS%b;-!bD5*yW3W>Q8@6 z_1^=HuhgMIq@L7>7L)ozKUv3qz`h`0)2SQ_zXb0oQ>;}3& zwhv7DMd%R~VYn)E6=puu2hqL@IOoSo2&pWGR@pje={?U5EGYnKX(bHF{D#LBgm{^2 zWngmbQh23gvW2$@o)pjJZ%SPT+OOBp`~~rJ#rRsZZ}~_5 z+cAN6Vr2z9_#K6jiXqf7xer|woneCY95%pt5^tG627VgoVXdY`!Gsr5y!^Ql_~N8K zy}4Exh3~8Kvv&Vuc+-XXx8nOCCG{%h-`EC&nFpb(set%PtwfuFbKGtO1zxc4b?kSn z;dYJPX8xJv(T>d%dES|Z@cO**S}-NP9->W zw+Krlr$enp2E-r8!c!L}b9YWPJh4>-#+4@Fg%!@=?6V!k$|d--*6HK*pL1ZlYcI{= z`YCs3t%C2mmUzZ_Ek3%t13me;EBhcXsCUQt{sx#Wh-%vuHP@n?G|0~#V3=D zT-ifbe@ldA@j)>0!wL}S*XGskyU%`9$U@DQP<*@b37NAZ3~pbL2jS({vHH{uGI@w& z@<$Hh_gz0Zu4N6rl0U|VCk0`K)@}0-?TH|HL>>WBHfUn%iK zOXLfzaI?m5rss&ezcN03G=R{}08S_^P0WaPFPqQ)5m)DDDb>i@IL>)crcJs1fz#*&OKxQ(>t1mYYAsD{&Y;@~U%C1Z)J zxf})?w3ZAsJs?>z#paUN>X=lkB-DNI1hsn~LZ*fvlCdoIFt-=i@n$_l86;!0b0QY< z9jKh-12RJ`7q*UY0LRrQ$#BmBE;Cq3{sliGe!cnRlE-Cq=g!Ifp?=)nvq4(5Z#6r4 zrX+5#52K5PgQ3oL5*b`Dotqa!;qP|={{IY}hhI)#9LKezsY$6wR7y)|JkR+)R3tMZ zWyTLFR47Gh?=7XYgqBJX?R<rIMmZNHPi)GBYyz-M^smy7!!OzvuILzeV|m_;T_S z=BQ~Gv#tFLxya@hf>I>m%ENeg`o@y1t2&D()+fW4t1J__t_p3Vve12xHi#|>rnWhM zsgC_23>4*IaY8&qwWYvWrA;`^rJnO_`${I^qXYC8$fMC_K`>G;hleiPA@hD3vvEr_ z?8A+ibyvl*?eJ-G>?^^~VLwo)tqh)8)RVjQ<&4(>Oa78l9$F<@V0c9uU_sUjLQH?MXFl7_f6l^25 z_gEi2oBzGNI22wT=p3J3@q_+o8ut3}zj;jOu@`!`nqipfRTjGiW!>{qzh! z{Ef$Ddr!jCeIeXCSd9s56S47w0>?XY8yWrUh^J@y!7=MP_FVBBu7zpvcE6cSW<}nm z3k&nWpmr`sMD!A?M^ov^<%N)DBTV9yepBO62d<=|7KZ6R#)F#uL@!`H{3lmKii3}0 zQAZ=xzyCvLL|4ENo9BF7bdEN~3G(7*wBys34^*!)3G~N4LxeRSjWdMEsEt2l9gL*o zZ(B+Dkv7;G`ijPT?xrROvZ+s-2)<`MFJe~>V6SvHS}r|Bk_^1y!TcCB;y1(JJI!!r z%^F;969Ufs!+>|x;OtIOQ2vub{^@D5*<63p9C-mYS*r2R$(ns!XGN3Y2K4 zklYL_Zb4!MIU7-h2RCI>=goDn_T?^ST=@%~daoHnNN1yX~)?D=iu-BZp{0jhZeKn67_Z0$gv*`nEty> zr3|96!k`PvE|qepH<|z%l50;>z?IjvNMUhv3Vlw|_`y_lNA;8c2=Z6dJTWLjU8O&;N=4u{tBAUiREVH1D zOsGZC($2dWZ^gcI4@L1;Uknsj&j0c%3zXf=A?iX6I?W1$ zZ1!GwOPVX*eD03_##E?G&H~)|^B65%8%7nZO31XbFeLfOcEU^ZRCg zq9m$_-aGu1+?-*<1Xbj~SI4td{`g5;>Z?jW>|czydp$_%^uZ;UQv4DI*e8S};QA=U zv7AufEoQugH^S(BHk#YN@E~qyyM}f@SZ?8+H89a5%A1>33~TJ;$mBHvc*|~#YbH2G zE0(e|rJg1HF{=n}*rW^4cqs=Hh#p+O_kuow5!_%W%X2yr#aVys6{uIsfZm!GO!pat z&J+3grvCtG-g6r!1(aZhVJNrLZXQg0lESA>z94BCNfzq6)3JLtKmVc zFAT=3vG<|)Rt9J9ZyUP$ZaaOja24q(av|e(9I{;_0&CA~fbtvD$+L?C@F8U>2`ID2 z=q|QjseYG=nzQ#4Q%}H#)k~?*_f*{WMhi4MB(Qhk4l?}n4C&){;gZwGF~%{PgeION z3nhYC&Ley0EgC}qang`ik_HyvOehP$pJQpRH)LnE4`(+qEGM zN8hZ17vhriqupJacu9w3nQY>nX#Yl5)=Qw2$}Dc8-E?Ae{T;R9my%c6UXbCfM}#?s zmJ%_eEIY52>>e(`B9%D0%8Bg{tW*N03?AMa)BwK=`q;f$44&r)(TD|0;LCz}c;xzC z{H_wp`jKrw@^%m!^-l$x^B)-R>mIms=@kCOs{(lU;veoLz9d!$3(>f%sWhu}Cf2=N z4<3(G!7#f9Tb;MjC;rl8C_xKH`;6d?4~Ok?+0yFi=jr^NO^o=p7hH)!Z+uYghUGQ_ zP;cQzHkn-muao${4VPEUoe#uYH! z`-qD72%=rOs3nm(S@@%!_5={;xRdbNb+-mdJ59$oe60EZ3c{;@};YHc;U!-A)WzW7G=}V z;XjQCO6P0wqEmM>M>ajBo32{G=E7I_tR)n|TnIa5b&m?_}_FHzBVWcrK%Zfd`Bjjq8GN=kYjs;U0t5V=u^ zJg3OzE>nOR&I?fY&60o(H)_0We#91vVMkLaV7D59n8z(wak?)gAF| z*mnRUp@HnV#^%tW6#e< zA%{rTlu~ev+k~FG*2B(@-H_g(jzE*Xi0S#o%<=hUU{{@pPEzI26POF%;wFRWxE_AKX3@~L zr5+V$&8AXcuF*m_b^NbE5084EA#+o8VK>cy1g|!Flz#mPGo~!S|e@pss(TX@~nQ?>d$cy6N4~ghMXBO^TVoLgF zaIh?37iMS*@JjqrQPrRoVlEV-wa9gjX}mI6&o=}i^Y3(zxdLW(hS2W=^0Y)goi;n~ zfIA(&5bi9GaiCV}?@K~ZK3|K@T1JugxFGfCM=ek*MsLmE=2=IDHvR+<=KapV&*)D`fqJ?0s z@_cBrQz7>()2Pt9TO|5T9oe`ske>c&&#gSDfkw?Q=suANa?~pti^pr}bPGvLv&)7j zg0k@A)Ezbtkqg{|?o4>^bVxrbg=Ia>&^A*J8VdVByGnz*NnC(FXY*I(T@j?_o(5-& zdlCjawbNCag8WDQm!T@}H+iK!7w7R}K(sXgSFYxQ;hPiqr_+lb`Em`VGr~}uJ|e%D zZ{Y26e+O0R3^WqE`c=(g)cvs>&+eco?*!{noD0gp<(d-f=B2o7b)_^((yID3m?;r_!!lFgCq2^b;A-faTM+)z4o(3ID3qm$-r%%;E3Yr@;mCp2UZ zf%t4OT=sk)iLPD#mv+icmE;-Be?<+Sox*q{ZHP;dCZ$LF$em5`NEu;p4k#eAauXS~*P*!o zq7anCdC~FRllT_(U#Pyd7%Bul#kq=y>4{H1)MkAY-g(aQavg(3SWWe3oy|V-yq6VnkR@V{!xCT6P+u4Yx(-1^nIcN7YN7wr zIP$b)3H}y#Y1k;;L|z{G+;Aeh7s-P>DD^Jqrb`6DMT-&ItJ_GqS2f|(Ru!C>ww@6S z-hiQ+UKpP12Vc#`QAMtUF;#wqH|49*)ixF-)0|-YJ;#PMzjVORRuBH!zULI(swW2q z^x@d17))+a#|gV8)+wZo!tr_NP#K1QEn9J6^h{#WlFr@bwUrn>RUonYYG|Jt3JUhD z_dp;WQkWGDhBFs|unnKi$Wb8+2X25?(|U-YDe%@>1%i4sFroJvRlXp_+img^X5ZQh zr(};Z*AsKd$@q^n{_zE>nHmMB?uLM+vIM^{REQ6s$YLV>$Yc$C>_$FgFwE4e!j7Qw3(l9(^@f{V`c@usXE zylvmgvH;IwpJFq1=tjVHlim3KSO#gY_5k~iHSp7w?RYH;gU($-{Fm8hVIiA6*(%Tu zYLyRYyOkb2n&OM$I}z=>uaf(JG~k)KH9R#wgA$J=anZNGWPfll_K965tGg{>zQ|H= z>ky%WWd-#7)teX-7y&lnf5@6yQP`PP2R0jS!Hi)+p00Ht`Sar^M@1!>e*Dh@)^0Ci zo*xz=d%l{Wq@oneSBj>VA5~!QpLMAG)`?6!OG2CEN(k%;q=lv>xJo;U26r%^reB349+ansDEVpNR~SJt$XVO*XOhx*raX{$P} z(~{t?%}b!s<-y<|HWQk}|G+$*)39K&BaU1vf~wLM>cO&X>IMcN@VGF@9w)f7x)j|; zQqX9RKCI9C48to&=}oa*qI_>UutrVU-guUnm9JzOkAA`vuf$+dV3Jc?tf|C;_Y# z;m@ug2d$QU^zX?cD4$kMlr~So{*r9^_{|u%Iot}mqkg-psi>(x2>Q3;2ay2!`uAEgZfpvjFIUs4TUTPno9k?!`8Btacle`-HJKZ~za)-G$v3tGOC;Z*z^mB%;H^-^|iQ zE75Y#9*CBRf$h!raDmm82Dvxyz^Bf&VYj#}?C`6E8&*eQq^Ayi!|3vu0(|Nea z@;Oyd5QILvOXPRuAa#&(z?zv?alX zsEhmBN^saM2x{DIn6B^;@S5CAdk^K(3R`}|o~UqGl%Bv;TAc^m>9+XnQZM}K)WMBw z<=}GiZP5LshuRl~kn?wf`=Ih2b{noEuqTq8nl7Syszz!5$K`Z;ZZ!#SJpq62hhU}N zX}CW8k$S1jr|;8`V^dEzwH~_+ORpFZ-%o1DIkF3TOs)C+5-G5=&4heyS*&pRfW2k; z#NVo#&Wo8&z7Th4GGP6iO6Tgy(sr~K{6oui&*C91Q~Yn*X0E!kE9TfAX8aYGV8D(! zRP#OuFMcv^=)Ipude6*a4!p>~OP?#SX1YFJ%uv9I&R=w_brG(~(gMcnCNTmld?$Sc zmW@qj7@{ztcdGUnD!`pWG3n+@HcqCs><1JiXwk=Xb1&?i106_gC1 z=K|{($U8$1E(-!$VBg?7={KnVO~FGzfHVl6Mj-Xj{E$rs@k{8Q`P^;}ac@*^#W{WC9SJqj4Cs7YS zKc9o2?CkADPAj=Gjgm=zLSQbQNlZq%!T;8De6%%#d+T{6*oz6$Un}0?{Hbfu{6ciY z%$7~e0RI%5O};`KYO_Hvz!*NR_(6U$YT%ckijix+QZHL67;v7A_xC>oDTnPi-$WUu z=UE}QKae>^o3US^1{9T@@UcxbnmjiGX}6gAW6dV`!-kzL`20qiFh)+)%0r#|W%k~n z1_ld1lHdD0VfV=-$c(hcfnIh!`%*s;O!g*kdJ>)KcB9`**a?*j6MtU z&IQI|yo@`pR5-{TlPSTqU;BaSJV7r8H4*uSaqdx$Jm2nXHfrZKL*U11xGcH>kJV1+ zXDFS+!?n>2lamPhjK!fKk?q&3UdP6ROBxPdPKF%&yExme9e4(v$&6e)W{uP25QV2sBYl=u__9ZaA5u65+_(qe72pH)%8%h5 zKMd=9OW?qt@4yT$0l7^kX!IeDorB&b=S4+Op3fyaC#op#VJHrn{lV5vD_AaiHs$5o z;WD!Yc$4MG7qRo_s_(Y&+pQ6HoUx#PCq8gaImI(c+(|gai{+iGn&BWf-}x2f9n; zA=e@b77d+bytO)MoZ)TQ)mRHtTr*Mm$~devf542qq7Z8Q2`aZ1!8^?!vhH#cch|W} zI=pl;M7dXU<1H4@_iN&zQA!DierLm6IR#9-`5&qs+)Bz-5ZV6TmE19$R@1u{Q7oJsr3EPasSq6~*z?eR0xF7#flmys>~CU1 z1q;cHu}zk_LmVtg*-V)wh4i216u8$OM#mysFsb4!K8?+xRrk(<*e5@tU;tpTP>nad z-3xu#O#5bCGZ5M81N%&uakrSI&@BH}=>FM@BlkM+{7V_=t_i0{QbpmVh%Y!je+lV) zH@MrOO;0gviCvW?47&J&SltZlKP!N@K6ZlIzj}E8qy$t{w_vVp9d4=cf)5W)(bX5G z!FIoGkkOcg6-)bJd>|HbVufKyeivMEdrt+wgp<)x5i~1(i7pFMF!bwpj{fFwHTNsa`a-HDY~=!aBY$g zSyEX{L)((5cY7$D;)Fu5=RwkJmW($pNt4`WC$ggWJ=3+VjO!vJhTd;?LcPUoNDdsq z$-xv#>LW3Sonv3G_kuf#e=xrKGCY#Vzz538I3t1&cZmkPt_lV1TM3l6^8-P-H{6U| zX|nQH9zD?D17mY9(FE<==<;|u4LByrADSeAv$|E;oc#=XCrA^E)-9@IhW}xMcQzer zF~jC#KtHkPgFBoc-1z(*P*?+M3l5OF?9<3!DuX>$+vp4Vt*|T3mURd1!-ZO^SQ9># zxox5a`31s^aqHTLgLMU(Lp0lUwB6J!6=$cqb%Jo&rZz?Af)+6(0?`GtV+# zz)$!EtEc>hUf*J7(WJ?cz&b<+{tEJx)`w!K!2#m>7|7T<6_)Xo0WXu5L5rXhchgT5 z@P2g*x89ly{u7OC_NbRklGlU_mt$BTV;Svwp99vPrRdzf&mk-&d}Uj~9d}EyEAX+eyGRDc+T5m$^s2tAHZ!Ej(q} zyG!`FWaKLAEBNPv+dV>{jAiSZx|O95(&?ukLU|J~gVu^WGL551YmJboYqlix|x z`~-h+?0i9cOJ8Az#}#s6JOxa)mV!b|B>C`07OToPq61!HTxEixAvlr9I-Vvq=I8Nw zem@E8`ol5K_Car2ja6>%xnD(%i9PEDcy{;)6ZF`K`TgHYJUG(?Q+UF-aNS#GLdX!m z7cK%17dMQGszsk0RwPj*1E{|fZtKf{3RMN1*?R+`d>Bh4>n`?;Cbexzl~ zVMzBi=Xxv%gn;58D(mcnn(MC-qOcv?^@Slaa}VX70b>7>#dw{op$Ak7u;1V*`O;*? zCnqCtlY)tsn>{WG6`X z`&PJ-9|LLnujvi1>G*5x8l57dhC@e>ko7?pBz8Cpx6e+cx;tdixXFq}*G%Pqm-|J1 zo6^wY5jz)YXPt9N3-~J+><0(ejrc3bn(m$YmdcF=G5wBaa9nAE#BuuR`i`}@EO!C? zi^(JP8M-K+mPh97FM->q4XGr1FO9B$838{5Ue^*O(Au$r+-iP|Epw-%m-RV(ELj6@ z&OE0F$9=#day3W_#e<@YFnAh0q;hkUkhgU%tt%LZ_xrrCTf>b=JqsdD=koBA_hGu( z^b`1=3&;0an)G9WC&Z*lf!~}N_?Gq0|CI0KT7BI{q!uN^-#AgQKlO`Sl--B#>s-;D zvlToQ_mMrnB;e&sSNfu)i&F8anjoZJUf*m`Tq0Z}II$C=aKh#}9vMHF}FEk^O1p@TI z+F0`bpEJ~rm15>ES9E3?$kfJ1*y|Wf9TyC;?y`2w`>YKPzD{VkN|1ire~J9`h+@9n z2}MkeU=3oL%m$YaT#I#~5M2>VCYC7kG%f~{)(?h6l%1jLq`$@OE8>a1d>U<>IL>z9 zis}0uf9UUIao$9DAH%zp4f|AI!|!Qw>{)O>F$+w`{8eX(N`C~z%?c;K%TADlZ^CpA zV}ozX9?=6b` zVdU^Rj5PP>J~@|!+KTTiYntSFCd%Kj_M{?=&hUW1zHE}IP(r46e5!uO5fA=;@93!z(z>V$5uVf2=m&nB3z2N@r9&?ote7IkXU##l`wO$!` zQzDaW_$i1pW-Z0%@yoF1AfV?HZ=&J9lEU2(e5G)Ow7A*99qpM!B<%rIyvZYLEWUu> z%k6lVu|n~6Q`qx}9w_@Cor0X}HM)64obaUG`X5i~A_;Bbm4z4%~cY>YChQ=szYH=gg zn5n>P@|2}dwB`9bmMvzza|V3d@y1_4l-v>L;o{{g|)1(5wE zp2mx~)9Gu7Eecf9wc6`JPDLdoG1?OO~6)cITFjkD|XyKaDEejkN9$ zo%+rc-28VCp2ZJF@nb04*Rw(OWgWP&BNIPQpF<)OCPS*9IJmAW2WyuEh@9zywaE{- zQx?p}h_MJtQHea`7QjAEXr%z8xB_wq|m98@1S1GN_I~_0)@*A@kmZLQ(&Tt z9MwrMR#gtB0rz3nu~n=mIRw+EX5lT?C$>hvhiFdbp|biE%vMW+dzw~gSE$Zs^7<#w*Y9qE6G@n0)2s+8dE5|HDi7`tKiDFOIgparT$ zCvfRDBTU!-N}tD>!n5iBNAc1`wkMtpp4VNf#os1lP^G;uNdL74oVsu- zM0)#zVXh~w2)vDt-@21;m&+Qam$qPK>;&nN-VKtM3utX^8h#L3P1+u%!JHj+T=CoG zsLe9$?H;aW824hDUTFxrI`X(4i|DR+isJhv8G)e7L^fI-%q~ZVtuDEDgSEkvxdjEyKVqrA(`;3J#o%#f)CyXEemaipi&-_=Py1=lPIC4cU?8 zdKutcjwG^gPeJ$QLAn#>@D(dIlj%nysnqqOIA3cQn(aS=x9yU!pyLbmVxN(X3DLCR z+WJ7iWCMxPDMOph$)w}STKse_dW-twx1>ZZqZ%B5)N z=RIWq=g}DK8HtgpQow#)W6JC;q|Y}5l4>}NjxmRJJlW4W5gN&5y+SaJWJCK#%RrNB zMsB{*<*S7{!R!J@s`}vviOp?;C%dGGl3_NEjj%4U4n-WV(#Ny+FM_{p4e1=Z1O?s+ z)tBD9Xlh5XG|VlT^~<3>AeB_oXNbyo}ZX6zXIXe?lf3dcM^A>or8f29klz8 zEGlPC!dCBeFnO6z-G3kFJnIl=^tz?tedH|uW0f*cTl^Y(MA-X^oK?{MUo~U;Ooewz z#R2-Fv&gAc(YQC`Hnpj&g^`>Ta0z1B0wZdmBa}+K)3PA(;S=H&+zm~+|Domw)}wf; zmHLi^!sxL(bScl1?45TCLkbg+fp!!6Q;iXFG1t=e0uIz zAe|JOL?`!T;(xK-IOSU~CLN8&U7ZiPP9ssUNl2V`%ej*D&0YXLlj>1p|7_TGq?w-D zy&L@EmeX0YSl?a$G(I=;CsX-pD-4D|qPMdO@o*{S-2JozTW^0O>TV&JTt5dv@Dg~3 z$PnuddDQ#a33|4V?ROCewA%26tFYYe+}2-&O=qmzc5BO933XJ(w$+cM;*mS z0(d~03-7!l>$Ed1z)O5DyN4bmJSSVoU3!xQ{#{8O?aq*b%v$V}>H)d12H0}%FdFw; zpz5qb+%}&NYJm%J#ed=O%WRa!?>qpp+zmJ?v#sI4rf1x;H)}92*bB-E1^98hSng@q zbG$zg#JXJcAmYG1lwf%ZnU+h5u*PrZwq_go*ffVb>q{YgyP-nF(r428S-)vc?gX}+ zo&v`42QapHGSB(76mBqQopiVEg2jQubp5{7@Yaq!>+i}$NtRjukkpZ1tt(0Ii#zlq z%Po7z1W+M}Gs)M{eat#~syoGEt-CH;#Sp$14Ur^4>uk>>t$QftWC| z{I4G299TkMv01!RAI{^)eg8FdSqcZR{Bi ziE(j#vIVRvSVXs^#jrUl7YLJi!nHi$4bgRW$g%kX0eh_QSkD#md$Sn-_qRi2?JIx$ z^VJ2H=FKE?o`gg7^iK`Poj$@3)}=NL747JvtuzEjapHQa++BdEQTAv*3nj{hVT zP_Fng3fqYAQnYo@&1WjFGEy3X>s{##izIlY$bmXZ)}OceFRcigPqqz3;NZSb#NAa6 zwy5TjqT3dfe|o@mFY5vK$3XM87?EjF{@6TVgbwdkfbFtkYB8Nc z4fJBwLq5@euZKPN7DC$gg~+D7h{n|{I>_tBx6ixij1NI{+ae=!u=68yTd)*p@paf*zJIvnjoe{hcOT(%cvit4^@cnj}89I21rmw$@KWUeoOUguh2u|Nif(l4F+Vz8kR^Tl5HKQ7;)+IIAh%av^uOoHMO5$`Qios zz9fRxXKisw+7?u=ngUL|i}XT_b=49qeU1lgHf?0H}5!^YM#D(QnjE&9Uw7gmbJ~M8ByxCdUzj_{5Xu1ea zyLz3PaqrNs`6GDe-3+33MHH_27(m07PWo+ZEiB$G#gEzC#pL`7qnrQ_ruWTLI&FIp zSvhJ3XZ5Yf2KU4Cc*A*AI&_CTOl*TH-z!kA7Xi`&cN$K|PGLRAQB3{s9+GOVOakf) zp<2ftoKH}EdhilAF}I^3M&}7R(3K4?sp|M_+ba^#7)pnB$j|}#0b=>m2VB|iS?Qd~ zFtk{QZ@n}QO|N71E+7}v5S@ZC`e zHy7xF_I4wjJx>;IZ41F`V&bS&*}{GovuvHO?qu9mnj{S$LYvotL~iE=^l>_bFWr@JN3n zG?m1o;~5z+FHb~ionaa+*+LR~qv`#w>!f61G#q-W$=_g;#+ZQ&M!buppQpE@-21;M zshU9&KHp+_b!wO+a~2BSuEN0QXwYBq3R8k3L5X9>d^QPb`2E=)4DO#qv$q+TI5-bW ze#GMC{c%+4wJS_WoTbH6uQYg1X=C~H-CS|mxhz9m9!IL1=&y|$=vW-Y=1uBIY3xa| zG-x&IzbK>U->V_f-~#7cG?jU922$tcP_NTd&}@$^7XCAZ;`DHwmzxK9x&*m*uUXC$ zQiuJG7eHPskVG8sfF_c{=)I`rmft*%oiW02QhzzA$XJGd53a{y*#}J4@{?fD)=Vb# z`~`vMRFt!}aM301dwAFSICcHmMKhKK;v!)6p3A*^#Xd3*vG7Dr2 z0vfhTW?_B%c~Tg9owy7eLhQaPAT^f3xH;^&jKG z>443FO0@HcA@OvdLrppqKy=^)na6U_zg=LnS%p$$&>@fnc}B2~&Q-9yr3bcmS`(*) zB2fIB&rR^UgAWCip<$XB%G^+=ZuaKXp<4}NW!d%krUYMT&jNHv|3s;CFsYb&lLq-W zP~mzd3WB^r^gVDfH`Qq&V@vEXn;g{H8;c13XNau!-)ApTunx? z>-lzgo-q?aaU)p$jo^Zn3yf$*p_FYQ6aIESzCNJBUwu~<>te%^A0>fTF8k0g$Jnf- zbw2swQVjbJO7J}=*Q1P_6mEQY1^$>X0y!Yqz@5rp5tIkw265;isK~W`!38i;$CclX z;ufzfG^$k*sYW;RY1U&%Z(TtTHj2`_XMS-k_IlC5PtWOf0Sol&WOGIvwkU zC%6Bh$-Y9mj-;XFMJ4okl7p{n*i6i@6n~5AO|VuogIOE?GBQ^@+3t`W#x|USjl(S0 zc5N`{#eF|4HNJ^A(!R5Kp|9lRu|lr9<9*H-fy21{7t8H^=Rj&EyTX*977Y8sV~T8^ zGe5%AV8!JOBI{I+ZF8pZZ(KV}SDhashFfAAmcE)oj`S!&gW3Y#hHK|BTv&%bDNn=d zJ?dZ(K8W_?zv!hJ3EsUNO+*@Eo4$$?zY z77#DqOUFIVk?vRpP}sT*7dw9^9?gd#HzAS?B|2k);uOC9x9wOXwhH%;U*V?DDaXU} zgJ44f>z9744hww!fpf|P>8l9tLAjPx%V zB&Bv6&^=fT)KC=z`=-Nx5^y0|9l89WJHnC zh3YUTyA`5e6q6#AFI>0HLA1(lEtJL^W3*KSNH&C^z3e<*wva3sj4US^$BQxb*Kuko zvj*<(y-jv+Jb>R<^uU6HIplqI2e%|E7xvam!Z%SHIA3;&8#xkyD(+%D_Z#InUEP4^ z(K7<-_1ZApI0yY#*}}krB4VbaMNOcC>X&Szo^#HTvS$qp_sey*ugZ%P|5v$62q) zWfAVxAMtg^e@}+yKjQe-xfMMsd_Z|>2zoo`K(OI_v@!Wc&RkH07p(X2ou(rs9=Qm@ zJrN|6Z$`{U&Z5Ana-#Mm43~Ym$6Olq0bJC{?P#gwZs9VtR?!knE~(IAmt3q`)7y}_ zKM^La>qe1!zn6KH;MGq|cq`1~xqt9L8UIOi*}Mdldlrs8dwfZYqYln~^@0``)Y9IG0cLST zKXFxG%Io_vh(F&*qq@ThsxP4f4H?&QB)^>Y>^TMwFD~N;>ll<)uA_#glX*4YcSA4} zNAg}9U|*^t%j~ma>^#<^O@1cR@j@NsYuXvph~@aiNfa+>c{d1-8IW4tT-1~_MeB|r zeAl#?6#4lx8b{+`t63N4={{XpaO6MEoVt7Xq+lV6pOU4q4d=oApB`M9tb_q8B#FpB z0wF%TiOUmqO{#uN;;pp6fg_5psan{Qm4RmtvYt`-BN#Ydh7FgFLWAH^ShlZ+7#*I& zdZ)E;@2XJjIrbh785P0OC*F8xM-jcMeU>{t>lJA)SV5r73BMVYqlz@^rP9{M`Ya7l z-*cT5y!1l3`y04xz9y6KpKYM3Qe1~)Z%Ls`E3^(p!ELe`3oJ8Xnprcg3k`q+r(5W| zZv`az&IT|_?j&#C=s^G25%B6vg)!SYc;;2lJrZ7y-J_GJP=`HEYnp(d?}gmD07Y`} ztSA=bYQVO$RaEJx9QhdWmf6?xgkFk@CMz%Aq*pgOk~>4?jK7^5NZax}EGT+FHg7UM^9FRRbi$&mGH^X^CXecQ(|mTuuH>1FHfKMeQAH=6ST{g*%C4ZxNC=f5>c_>| z8PuUv0~g*sh;A>#zV2_@6BJq%H3q*|zU8U0uM=X-0A=D-**ojZShPem7`eGp3fk*Vy~o*YIThOI}x_cVaKmN?DI}%XUDJ z!xCJ#YJ}Q4HDg0tI0zX9Kz^Yi{@zxBgSj$ryvBn}bJ5|&MMY7GWgD5ZY4JGM=_eem zJ%;b)m%^nvyTFlk@$9PTqw6I^c!N)WK+(p0*eR#T>!=ZiAAc?r6DJGSnJ$cv;{_Xz z8yL{R^akP(JQK&lMltJ{4M(j>6ozbF@ziW9+~;kDhwt2{i>7el%fJ$%@=XKZ2E>tP zN+vMV)DRcMhL9s~f-z-h3I;t7LzyF~B#*SA&dN$s&U!T$r|n0HJ7%H$J~?fqoP?*Yej%W0_WUyrgk5T`fG zqr_WjIG|&JM~k1+^TI0`zUCRco6q*R(pWdBT@AKbr4v8DVCo#`3=1~QB_7GDSiWvL z@0~;=^Fwkw$QwOp+R}Y7HR(3od8J1AwaMtP${XoaHMsZu4E~r;X*Oz-f#Dun;~@zl zgKW3|Vhyx~&p^wvY>=%wj)8S-7xzX6t_}^wWf$`y;9UY)#f^pF^fJ2Mu!VWieF3{H zB~iX?3-`_N1}N>*Mbp&*D6mihdZaRlNn19~y`s(ww`qhGsdB)WiP8QE0nmMDOA0J+ z)4};s#Maslzt0n+(@mtI*ue}WA6LLdBeoCJu>f`#meDhweqb4{3^h+T!WrpdOc9p{ zAIlQRzOx9o?h)X9Ho3_an9S}Y`+vY9Uw3@w90FSsome*GN}PS}8gaO`f-y`n$6A#> zXi3=%^UtNhgRv5F{)+^3PKZI?U>qEl{=&8LY=`2W&G=(S3C30wAseni6Lwz}EB3_2 z96_Fk{BaQe=Z5lLlld2>iusTXg;AZk!+pTp$Q zM*|ddZopR-Qmi-XD-oJG12p%phw*n?a3bgmYKcEb=@0j4%OYdC_LUcWwFKh7wHm(J z`N2ceGSc$a2+yd50vOi7L;fcSbDRr-dI6NKZ)5T)0B%C;v%62FQ7)HT}o~U_Be;pSl zxG5BdZy!eU*ZENOuZ1o(6{Jr;DUtmncgRQfJLSR5By#rLd`9i(MGO~ngfw|WvgUvf zSLWsbZWxsWqdE(0%X0>Y++>V()8fauh@m6q<2J*;gby@b zW<7J=ON$P?;ek-BJfE4T4j+qbV3%_Z`9ZHS8r%NSvSkrO@ZB!93(JF*(=S47U<*o?z!iDKJRx@cwJ%2atNJRgO`@r5mm9XWd8c& z_;!B?{`c)CcnPfq8`nMxCwAkGqI@{JU?~Rvw83j2&I^zb<4;f735#Au!-JCn7<8$@!5Z9#ob->+pGKZLTDaQs9Xedx7m=_*H+Q@jj_bT zGz!c*M)7#Q6fd$Xf}A8_XcjMl?%d2@X_<)i4N{QqKExa?@PyaW(b#ajl5D(dN%T2C z-XpI?u&1kuJ-A^j#uiXa+4&qEzKF-5{Z%A=k}Zbc`bW-Qm*dMcyF;*E0o|x^3LSil zz&o~-oXE1rwIO*>O~in|WFM2|s!dhyMPv2_8J<+aEdF2He`L04FfLy)LQH4O1_$9j zdPC3w>L)LRk$@g{o=;nygsV7g4eF&PhdcT!=r{mts@y4%`~q2^lvO za6*6zz1xul3w`$E&jZ(p$Z!q2N-ENL_kmg*tQCYWcP8;-UatU`{!04w>l9ofqJR$j zKEt(AVV=kSd_1G<3s+q~fOdZf+3KhYby@jzs5uu4xeR_k6GqCUZjo1oXQ1jM=gU4n zPFe$+h`@(~;PxZvEz#QmD9Wo$QigVD6I4>##G1FoaEq0xF z{P09=n;o=mbq%f7D1ktXu1#?l(NPE5pLuDV+@$JLx+cV+t z=)W?y`|)9POgcf%thOU3pU08={yu0`t<8&_T0@SEG~*~!hPAWD>E1R6XgP3?e0X=1 z`h2i~1CyE8SOE?mft@5#hl%ue#-GPjqEd{0KY{0K2u$5VU1gZP|Q z5G|d9!{t}WKVvrttCU1nw=G8YN9Mx`HE}kmtcqcS_k;6+^I#BqlL!d(;KEzRq=J6u zJoT-VoGTy_2cL1SCvn_7lFll%rV_iD*Q{;3EJFSYysCE=gDdXi{+oU<5_K0`?gwJr z8+HCwyQvgzKgT7*UnuXED!l7Zz_+q9FttV#?@5=Tz#dOx&$+O_OU2ObKa23bswREv zq{hD>>_DB4O~e%fXTbb*F`oI6iIRS?pjgNQt;>tih|3W1=Ki9^Uot?h_Y|&4&_bUB z%}jXcVkY73Nm#sAhQ6KG0?rPur0aJz_I&6DGRF1A)0@DXy8(VUlZdax9+O?~xLo3xWL^0Qm@@u{T)%1t@-qj~D|iM-rN?6~*HLnE`iCWdJ>ZFo zA^4ddgmtIgIX4;)2iJ2PD;076^BMQy$8Ds8zbv7@j>W+hHSF|%OSw)F6{#@6D=A5K~ZX~TjCBP=KKUrMl{iGV=@GF$iU%y->A#*3up`&uG{%T3A&4= zaH>cqO&78P^C~CESs@QQmxZF{TVYcExCPpl{Db}BHf-DBhxjX4msI2g!>F$uJh;ZO zOohU5(J1#GV=oUsJbj3)x)d(5yMUT!&jJ;UhmWR-_)cUERy{+J zV)M|mxe4~PpTSF3E7%6reXz}DCV%QuY2M5A?L;MN1zE7WmoNv8(%?Va!MrsA>hBhl z0l7^mzbb=0x%&+}|M(s7l?{Zb8S&JU+bf!vrqYAQiV63Lr?#tNpmEU+`fH^bU$*xu z>0cnt9|%z9`8U@wTF%MzrM`MywJ=4oDp}N2J%w%#r{La)^Ne-SCA@YmAC-izv;3tD z?${+jmkwIv6}{6qvE2nPe&PI)$@Ls>@ihH4{V8ax&VU-LyWqa>06v{&&U6|6U_Qv+ zWple&^7VTK1}=EcaY`r&d@jQ;6yE~j#~-s|Yj03tzrXawR4snOue~5Fe37i{oB)ks6zFfdJ2V;$s|D`bb7fR9NvOe;XUXlrG zcMyjQJ>=Z*8F0Sp2y<3_qG2ijXm(^Yk!x^<`sh#?3Rgh6o#JqQ$6}0|m__Tkj*Ri2 zGkB5XjWrAg5#N)e#w)+sc+0a8P3nU#g&%3{S6QZ_hU-|hjL_7bb9o1=mZ5?Xm(5J5 zV&BS51k<|LM6kvRv^v*Aq-qOw_FqUYW*QNR%3F9`Agk`Y#a4(f2}5m7eGL6(KrSmz z1Hsrmr1hOL?wq@x%or-hQ%>2KY*tG$)(3%vR|!*d(U`mopG&V!<3XyJ73WZp2j%s> zbhR4Sf!eMC&X-d`PgRe5hYo=9@39~>v>H{O@$; zk2|(_aBu~T%?*K(#z6W>c?hkBLvYCyXRve+1BawgtPT=|yMaH*x6?~`ucZfw{-$bd z`qu>(alQC>D3$cR%^^>J8l&NqCLA|=1*M0Nll1Ng4BEMu%#95~6~#F)cyI!rey)eC zsUff^pp{fiy39liq>>9R&oKluf51(`d z5xZWDT2ui?eEsQ}9|UbnrbCyN1W5cnNGjdllMqF5xUy6nBVx@_Xy`c9_*)_0V-=3c zao)K13fR+j5T-WdW3+A^1`j^Q8Lt(1qGPA2LmLwDrgZX>rDR~?Mr@Kwf*Tc|aQW|* zv{70FZ>-;eb^EvC4zpaiP%ep4>iIN4x(F=oCqZ$08&;bouxf&D$o!H;F!5LwtWL0i z3jS2y59>>$o;mxf2Y$@tKreZDPUGL6W$lu(hJV(;d!bPbL9S3 z=GOrqn0};zZjHQ4ub;QTk*(%9c;W!==yv6sc3+`^x4)B+nqay-{HTahu+dmvPJ3JGJ%L4&O0QHLy;MbByp*uH$hdLH^X2 ze^}-95xA$s2nsHEVpQfY%7T$IU~L zFdl0IlI@pJ(JPo9n-K$J9+yx6HUo=k8;QqdPcfU57%QRwzhh zsd|GZU+vOGe4*G&f5h#iW|st^!fOs&oOTXcH3D$Y$S%mY&5(U0(+}9 zKxKvy?D}?$>w&MsZn0Q&;ra{jYoz&RsrM-grm&ff_c6|+0Yk##aN03TkZEwosHSSAc)?DKp(<(yvUrjc(9as&YjwP{j?)ylKwKVnf4T2)R1$^hS1@q=AaT$|r z;&Da*yfT!DYkn|JKGcQx-6p_#%{8L{fNg9=-Ulf>T5*F6?a-l*zggDlQHL3 z1zXs;6g2)u!TJp{xOCz~a8PHl&O`x9w~CSSECt-9F9>N86VT4`FzW0*iqp&9Lg6^a znI1ya*k?qS&Av-_G0q{Q{UCPH~=JiXABNp5NVr0<%V@K2Hs zGkQ^n)-(O=*7wt33D^VA_#CEu`AoNHi1P~LUXc#JaVAaeI7(Mb;ox=)nt3J!bp;Wc zE^RF>&7@`K5D)%LAvg9l!l?kh`U)C&=+P7wF>YujfH#jS_B@ zpA0p3_8+Rn--O$hoxcQMd(G)rh4X~3qyk2vH_&uP zDSM^G1S_Nd;8c={r%TO{_a_ZEBwdANRGg50Ur^)R%|ma4aN!(||9u-~YLZ}Lgck&RLOJQ&BH_~LR17t}OrWE}kvm__u-55c7WjG9poFlwlZf~6RH-&8N z);Aii+=H{_*Iza~V2pkk{mBNRa3iw8pS;NY;gz&Bbd>2RKzQ_4 z5q+l)qrru1p!DPwv3vfVW)|r)<1Sa>=EI9bC)uCayTs!>|2K5EuMp3vD2C2oXows1 zgdleHUn&<}%-rF8KX*nm$V8ER`un&Voo_OeA6fE>@@i(%4JVt~ZoSPYcQ%(@nJx_N zTl8?6;39U4Lo&!bXh-RaWPDq?jPJEqt@gR4F`nk!3fq@+8QjJz?2;f0=-kf(*g1{^ z=3&^{r~~Wv43JXsdmQsBz4q0LU?yZ1VrgM<-HsL^crc|8dV5pADj=JxaL&=3l?yRf zFAC2o=g|MchN)e-v*iG zj;PY$4=?BG@ubdGgNqzssZ#`wnW*qrOu|jh;g!sy3f&q-RK8;uQFA z=q|g)pG!uT-U6f_1SDIA}wPzh%bl@qTRThI*E z1Je(e@Ybg!xO-(Xb3JV?931ET6RY3SWUVG5Y9YXTvOa*sK?F(l)`NhbKdETIJYrFy z0Y41p!|6Snp{vY}+naYlLfU*(8rX;PQsZ$*U7T)u6i*k}Plsu)6S1v`z+Nst*Ky_@ zV=;X{8IJd7MSR8BiJtq(VDn}A&^o|4z}z0^ii?tmN4J5vO&_tWyiHp8@t70$#CYWw zM;JO<2#=FabEiu$vSV<+lH{!Qb z!$e~AIn{Rk0`?ukyo;HV==oBdm^w)DceDwU1dU6WP_hrS{#|0tM!ca{<}E$=bO)@M z8iEgkq{#X5W@;X!M5bw!;mb3I=v>O>ljNlE)3+n2uJVqAF~yw!Zw|-Jn}V-KxM$k$ z-^>crB04%!PUkFeXMk}decdWBY?VW{4?KixXBgNaWsX|H!+6d@jwY`K5Vl@V3dHr8 zZx+w#{+ot){b(KA`$Gdp!WNLaJZ>IuSVPm?W5|Tfws_K|j_7ik-sLNULB#bPe$(oq zpTF8cTjEmOzD58l*gufz!p+$F0vH-U2_wHBMtoZbGn6;Op}Jf2mKuZWv%ixIsq--- z>>cx}c?cA*mqGRIhj`=GPX*RxR?R&)GB(%_jdRgM!gxb%)AFP+qbZXVfe}&lzvg|I=wk>0UJn{L)=JZIv2U>~h7@i-bO3 z5 zsLSoy(8>L+r-YQ@R`FK&PwXfJL@&guE8Fpq`yJBQVULO57ZT;04%n|P%M(>K=S$C< z1Y4{#&{8c4?lne3viwZ^T^C9`iac?P`46xS{YBhMD)1ZEGn0^C1)ugQ5P?s^Xd&o| zn%PgOTemVAw>Sb*I1fV>?}OR91{f8+Y`C`~2i6_7rjyF=V9qBK*5-aBIGYQ=2b0xw zR+<|Q_SHXpw4l*xHH;-I=-6DtGJ)KqE(J%LH~huhbt(1)sZ*9U(j0_74XEUfV5?V;*2|? zoX_Duaya}p9ha4$EykWqxc3O6=p5dGB{p=;wlB=jMTuZwjC9k_R5XMUWF-vAs+%Y9 z9m%BhP8uWm(Go0AT)+kAd~nN$6VO?#&Gneam@ARDnHtwuDElT6y8C~@Kaq+2kfBX5 zi zx2dREJpr6z1z_`&X81gC5uS}j(}8oyUU1>|3Zf73=&&eHTqmD(D$k&5qe0*(7!6}7 zJ#6|RLEhEexiFQx3yXR00)>Aw$oHgcu(CXc>!;e1{<|5N7THVv1ili@S28$OoC1Fj z4$&`RIe0{Ulne}Mf#uq%C|zHHA~U7x9u_@g*52Uw=TZ{e;&ir7HW7q zBOG2G{mtdKco-FR5lr7ahiOSHmcEYld+y_aO)&exl7-&&N5C#)DptG}cE*7}dlK7pL5U0?>=Aa1Y9!ev2zq81qz8cV} z(T0^)hWPSp31r-Ug}Qm%_jg-3{VV@~Zj<>$4c8ySS0suS{9Qu>G8@o(dNn)P_=Y}x z)ri6ea*^ep#jBbs;kkY&Eql5Mz*?3V?A#959xjKt9RCKL~A^^q@gByjlCL$VS6 z(BOG<@O_>p7$w`nuv;eSPtPWc=SRT0%ITy@zlZ$f50PLE0WvYMl)3w^4%RNxV|BUv zqwDEm+TOMh&4;+%-qtIa<5bKtT+c)(`ZRcDUWE;|vxuyxI7GZ2Fgib>1W!hCEb0mJ zsPE*1BUhV<+K4@y!m#Ac=5Bc2#ocl_=2LmN3KbT;OcZlYl2vw5q&ykJr^;?iz}*JBz{V zLNm!;Fh-}x$D*EFA;(d&#eX`Bc;BXn!;B+=a8s)UPbvJR&X*JF)HZyk&YLFV4XreM znO#p*?i+%%S17xFd;)L0SOXXS9U^I5uh)K89*Fdrl7C0$z~<->D4!FEJ6{T5=R!@o zW@$6=IU@#FKc5ADeGO(DIR!@ZH{gRm*QszqB5qv~Pem4+F&Y=|)!qAZBLgBW_G=Rv3|pb)8UhNB2A|h>Kn<6r3x88dXLtSs5gk1;S${U!_H;FuCAvbMjt$_TzdS@Qh)1*Cb@W~B zcE<7k5zvhoqhJ2HqMyJ~rlNTYc)R>$_z|ka&D73F@8fTDTrY>67vmu5TM&HA{f}Nh za1FFuxz5^zH^kL;1sVCt?OCQywY^PrxDoAqPx5s{HqNY;Uo)!)n5i;*TZuxl5PzVOz|z)GHvZ`{JpJua-Tw8v;nSTxxOeLy-R1ugg_9S+ zHRdw4HY&nlu47}(7p9Glh__IT$Lw|ptQ?Bma-TU9~9x3%!{u?arP z7l)8$u5%e7Os1z2oahlwY$F87`50Mx)}Qn22>+s%w#VG6o@tEuDrRY0gz zAkMiN22fncE*pwOTm5!=q+yhuI%fi}$#)I<$5^1w(PcQvVj{0nqXdtyxK-z|TAXfQ z_kyS!@X%184@$2d!xr~oW7X;R$)CDecqP38|66N8!RI`dp1*=~+qpc=6h(*_TLvbo zSIPGJ>(CbM0rP5pQk#|Qp>M%KnoYP})U21RujefsY~O|dE|`M(ySvS;(R1TB1HGxdZvHSggl{0i*I$s-y(WjvZO zfK)IT-wDj%*xv7HdPrt%$i@Gt=g$=oHD1lA1!Qp9ct3dYm#r&JGr{f4%VFArO*Fr| zo0M*OiJ3}E;m8_W7}{MxpUe#+b4DjaxOW3|J-xsxjhVr!^>MJRbDXG6GN&;K2WX|a z3{Ro@J}UjU1LPcq_}aU-V!eMcJMrUoCUX2ay)u)~>$>^iGW9-*R$qgi6?+M{SfZD< zn?lA0z(X@LNv*mIxATasn?KNxqSI`lCd+{HEu<4>V;38+wu9u^%faFo)vUJE5zc$G z0q?FWWxr=f2ay{xIe70JWANcb&WQcIjM7sp?L{k_BckahA<&`8ivkhf8 z8yJ67eu}+<3Z%4P1=O!Cr=!P5@q?@;f9LFN*z!vOZr9Y&UwQ|i)=LWA^agOm{SZFl z?$ZXvCm>8|fP@lTP(SsDI$n2zrHh0i`E?%Qn^j|=%TKVI6HadE2tYxzAbz<&9hz;N zf$Kwnu;e-nQ3=6aW@j*bp%XoMK@{7Ev(R){k5)=^4$i6;6kV#t^)^F6D!YlcK4>5( zqEgVze3*R}r%HtK+sUKd#dtM!HGXO4xSzwrTLvxvXvCZJb3V04%dR*fpOYyw4a-A=QAJTQ8}<1!Rx_RLN8 z0ombLjeB$t!>P^wWW?zj$4}n`Gu0o#%C|biXQD7xrxn4HFV8@5Nk8qJ`4^{nbu+(} zKEjOD0DN&f4<nIQ(D{+N8#CGtF!47YgJ!&&%NUlIhf1P#b+x)p>UM zA)tL?8uphQhLU*)Vcw%>uy3v+4d?a_J2aJ;{*7;FDzAc(kLe)dNBc;15s1+x|zv?ASTaL=pPDo0L#eAJ@=K*BQKT2I8gnjTnHurjaNOd!E0Eb`39U_bG^c>e8vTex zr=>zrw&p19X-WqV$uQ3S6bQVK8Km)0D>-(cjwX4`h4;dWs4(djso)ntihUw2=YORg zdqH-9vmTyCp6HxNe{@mFMn}Q5%+!}^qy`>TgtDKl*NTH zli=?<26vZxz>4}5I-2%{aY*)Ljou;p8U{hqV+O>MwqVPS8kj8?4@T~h*sd2xEUz4A zR6R;)qTpvX{LLuyQ+b$_`rZIJsqc*1if#C6MlQ7;S0rUMF|hl|O|o&;02QlQLl5}a zlQNjYUnhJ57vwLdDK*fJbWq*6NQBew~B zajs{w&E*%m>N-K>UUiaos2Jr|pTsP^m2_L_A!vKL54JX|v-_ugAUjuYN8^#*_&hoq z%HQSTfV?2{+DixGSDht?@B_y!Q{Phr7Hd^O`u;((+lmiR+|kZ1JRd z_*qPt@mJ3PpUNBL`E8C%+A7Sms!4*$W}C1hcQ@lzJVKT^T%mt`MPcQ_>)87>glv+D zpoT48crK>*boWoCwGE!&SIFG0LWa+g43;kU{&--a?&{xysMu0iid5HUzpi`A7GWK zB>$ZK2IRexfq|?&5aN7*q(12YCV=xVq=;i_l`20=E{R@@RzZiDX!hd6`^5fc58eOD z8BhGHhBI3{an+g)l;ft+hdWrUkiIu?t@QehH^C1jtkk0HqDr*v!g) z`0lG$Bk4Dfnt7c9O$|lfuE2e;D*hPhow0}L$F-3-xKex?i1zf-Ta}72wWcv96=n$z|kT!)PsR{6e=){svd;o^(_iSwqtB*#i( znDOjpSQu;sPl^mV_i!*1v`UV@yZJVHx-bmQYrx&1S2$)%3|fC%joEu<;q8KO{4Vo? zdQARE_TzQ>v;01!&R7E5q{is4#kZhdB7l8Xejc(s?vb&`tHeVs6c+Copv9~alx#l^ zO>mtQo?H#-7ni}Lg#9=o7|6Ico}x-Uaaell5uE34vpqgxXj8MD3X5Eay~WQV{B{7P z2fImFiv~Z}!UW~BR)9dd1o8D%;u+MHqJi2Qrv6SFxnI7T_ho$(u9Q^ak8{~jGsk$w zhnJ2KhuhI8JD4?;2se&Rvcl)HRpIUYJ{;sSr)yU42XEy8Drxl)uTGl5YtXm~b0*Cv z@oxJ3*LhOtJZ^>FL3WVs7K68EN5b%~ZZc1+9*k`e0$DD5sGC9d>GqLoZvn2`5`>XM z8ff=)3Z5QGgxRV0Xwc8e{9VHn!6;ppm$!<$E7teW;5qdq{;C0-yT6j_yPIOtYjOPX z^cGGP{sVgdalErRg1jy3#=t682OHi>V?nqi?~>6bIB3o7k_y(t^oPN8TkHXxbT<%{ zHpGIR)h9aLcrI$1T>;6R8SwVVe!ODu1&z-%ctKOs(Ie#KQMV}BYh;`4*6pgmZM@0&N1o^Wxc$49MDH@Sck>#RCq zT?J~Y^@GNSo=DhB);Bq9f<;`|R^Shv~+dc4z(d^c_)5-Su?%0k&#>(*!5?-YZ#fpil4Ylo8vt`{bjlj9PHvrQduf>UTcMGZc3rLd@OV=jDgK(6VUKV z6)h=|1*^QD1l^S(NA4NpA@2vRdxEGLcem}hJQoHWI0lnQ4XpN@0oRQt^Pa6_sLCEu zEc=~CZhW~20=ERQ?t~9++r-j=|C&HF{1JWrNt>VYaS1*5Dw}SyZD$(CeluqUw5i@l zH<w%w=>+{=5Meo;`i z)yHLpU#igjlm=q-Jc1;IoC5* z-X~ZG)g3g%uNg(ks#y`AK(c;{Fx*#mMDfCCa$kEE?OqdsPko=lMTO4{f2;*U7a%z# zEm>!F+X1(P^pb~bwfP>2hBz}wkkzl?JgD0|~nzq1M0o4Q$&UfF%luZef#Pp-~3B_;3wY2-{GbgLy<-Y$^ntaVJ}L zjUW@u!>dc9ps`bkSCoB~`MWL(66~nOU!S`JUQ2@oC*EVZ(+}9zVhE-~?SSeV zAmXAv5z$+VOH89Vw%jl}dfGy#Kr>#R7J_lT32@t7j3`Mz1jFDPG;xOjui>8nY^rpE zFcD2EDJqVwquw~XWSIK&cM#e7CialD2yf}tU?`B{T!Klf;gpCP&tskqdVgF39%Hwe zUr4Bw?iBbModT})(&P*$LMq;Fi(%hB!MSs_tld>V3^(Orw5=Y*ILbqlM;(<9ZX>cv zCpjM9Wr)jc1}lpUI3$}-N*pzf#5+YG!T2i9|9YAG|F2~HP__o?zgO%%G9NYiT;F%Xm2n`64!IiT(xuTjSZYfx%Uop)IJ8Y0`jO#z8xy= zKMWg&f{FdHaaQ0ppA1BI0C3OVX?~`}wzdXSj9wF8fmN7$O9Wo*Zj!lD}Sv5L2m;n+~q(`hHiqW*ETB4`4q%t?Q!?LKDyiX6wTe8MxFi!V9(|% z@aNn@QQb;J(8LR)xvayBr@8-Br4V-r~9_`xZyF*?+~2u|805JOV{F^2Q0qX#XqFCmVb%<+ls1)TG(8|KSR z!1T+zv7gl4wc%=&k|(3zde z`EfUc6LSYf`f}lCSQTtL<^tYtX&zqxrL`TE=yf~LCT5ZMF{i{?--6DJyD;f~|y z%JD{pIV2{(p^qPbp|2k;r0L^TXx$}`o9JBrvwxphr(QsNX>(i}w}1{GzXyS@zZ#z_ z=CW)yZfwfkd{(3?92WkFfzt)XC^*&*0qrlTopu&n?V1e*SpcJaOLUl7PiqxaNm;5f z@4$ge5Ye#-SDMG**P|QZyvZaYaykM7V)RI8h8y1cn+VgO3{G_2VTwL0fXA_qbd_cl zS*m@RG-^AMR`X@3wW^27>h~lcHcOK_uGe&p;}mXCn+C61XYl905Ft4k%RwmJ-gu)8 zki<1Fxjk1AdU$q1&HHk;{si#72X&w}dJP7xe?v4bY{1HSL0Ge;i}Yu`AQdk%nL+0S z`c~&0Ja4!}mV3>BqYAn7dd4aU&;*jIFN+>08sW^LHe3>WAD86#(sRasWOR1~j+8ZH zFXvz{4C4{)2q&CdAO!uo+lasZS@K&vhZ(F{0Oz;nktwryuxEl6TzEGP-;Qv;?|r$r z=+HQ;{COsB2$%&$1LtYTpRXW#D!6vTa3MSj+65;4+cB}(912YG*zZIKXR6Mi;zl}n z&1yINJ68lZ%a%a4T`@hdOPGJA@GrR_V~u*9lOQ_hKUz@i1V8V-A?w~}kdhJwupX-; zYNino#N}Zl*WVy}dL*FhU@uuIOh70^ov&78j(X4Ua65TdaDH?i@=Y((9QS0bD4$I4 zdG3Yp8&8q;Eeq=+0tC?Mq&c~j|Cp$3oe6DakB$F(vJ8SUCHc&kGa#^B94rU?!B2b( z`MviheQ#F+<9(-T$FraGWwIu_%hMH;Z8eys+h@Rg8y7NAe+Yjq*g+ooorJum9JH=I zMRz|BA*+j|c#Fol_lE=HAU|sf`qg&gfP@Qq*R%|m$A78&+4cwI-_D1fo-J%#R1IFK ze@8s7bds7WC-GJ8d^!|7jMF6Sq4Am~$BdNVi+7%2?K-yA^@+C8ly$$0R#)2CAPY_9M9bDaEAl7Nm} zb=doPBYhY=Mmw_;iS2@WkW;O}6Hevw!ahaNaw{5UOz*|IIr((Lihtxyw=b0nSLH|Z z?BS$@FRs+H!l`>clETNyX!13SejHrI&BSl1IA4&w?};J$dr#oSOFQsG$TYk;N1yNT z`#Ltuo*|dFZi7YKy!iLW1{yGZUv10cdJ^WK%t-vGo8?ovuDT3-B@HN+HV+N#mOyBa2EX*FJXn6Pr!&{GZ?t6Lbm(|g3#XrkqyF~&1iGf;q3Mc*q+Y_fG>$FMyNvD~ zk>x*_lF2xwRN`ER1-QUh1yrx^MybD7IZtLKlPV+zw80U?ZstIk#d~V2`j##E#L|H6 zW%N@(mg$lHO-Y(GexH|34Pwo~uv-gPjE_L?t!W@!#c`KRZ-RSyH<{RV4g_y= z&RChV==-FXdf#MW{zhH6e>5K2n`CijxD+1rWN=|8*SBA*3(05Bp;y=gvLr|Z3xaZC z`8o-_+zkW zf*9=OID+L_bYo`-7>((pZ{|r1|CoV?{*xj7CR^aqlec(I<^z42)QJt1U-7kcDDo;- zqx_WsXwA~2w4)s+88u<@_E2p7%4JR^Lb1zAg*WP90cl~jWQjpJO^IKO+RKLFyj3{* zn7<}jT(>1ySsr`MpJ7c9#l3NV*`|+g=_;Kt&_5D^wJpQM%}fN|=ug5T%Jt%x$dSbE zy%?hXiSoEPKj?@ag#XvcW@RoWs*~fG=j-0Wme6c+JIr_Y1Wofq+2jvyRmTkycwb#(6f1(?0_wR>quN-$7<;9X8szqoLwvkZoVa!4@MyK=c%umv)DkTzbnE1ixh*oE%}3 z-89&?;~x3v@PwME^2iEJ?q<6?iazp9!g!rFdg8Yx(|ID8$#33TwRWam_p!*?bD zvm=j~wPiEzyE@ok$)|CtT@J}lZ-u$Rt>o};ReW-1HtFYw($vI*-yrnVe9i>6>cZR)P9U7N5~ZgM zz(Dm3I5m8M=+$2j&?C=eq#)a$Fkf7wHc%ZBs ze7@$vT;D?YnJ^ioN8*@5%{EZlD1gK9{%CUcE847`32QvovEPTPSueMF;CheqxR}i1 zGYj@W(xx@on_NKWrAC8oO)NdBs7>1kCi1t<45OFbI8KXoKHR$fn@-i+ikZTQ!o4Ho+{O@t-%_#DpFq1iMRI`(R!IC_Qb;{bY629 zUa42WXzp$1VSgUF>N`QhJ{=Q;6W~BwBzZP=1$FhlkdxIJbou;PVlO&Eb~{|AZw?z0 z!Td^AeJjVUzEcN=3$MVN=m%7rEX27|llk>`Uo$g%HKDZaI@t*8P}W+NI(xk$^D_g$ zfLy}CtrK}i<0^>noij`^Jit|}J~Gon-(l5*RE~iZO_U^yFk|OR?u?Fr2|hLWMZ}U? z*W1z!ir(aE?>@X%DNB}|tA>FMfnc%8h{o0@qSP%-NRcwfJZBA#g{4WFAPc;TIKKXM zAKaE8#V3)6amAaD)NxGNe3wcW`W))wJVvQ_XS|e#-$1XuJ^#U>gNI1K zA1)`EnNCHxUd%=N3-rtC1laT7IQ`i+fjj4C;hEFnD5VmH>mH`oc9&kjE$nIXBf+!g zey<_MU%O57)2uk(sU^E&)fL>>Hy3?gWXf0-`Spm)o~CbvNsoni{oktDERG9ilj}vFtZ`)Eyf@4f$i~p=8f4AXgW%KO zN=!Y@VnNymCMISQ=NL*S#j!Hv&aKs?X1o^jJ~L3koy(eSeR#Ak9Aoo^@UedhwG+FD z&qW8oZ^ALqx7i0*mFi)JohEpGmgkrXh1jvx2wZFL($g1Xsbs=o5Kq&lv)ivj!rLy~ zZ}uB)RZG~AHd(Ox=z}I9w-^yiZQh2B(Imv0%S)>pgWIP|bXs>O?s_o+GAF-boVRef zt9Kts>GXf}{I@a`<%hv&t^*h(MMAOFQgE1)jGvf~)cn&G5MMimC+vC&R7M%|&qLSg zcQbjOz^QjQdsQ55**uCz42-F+Xgq7RDUk6#7=$h+86+|>i|dt(^UKQu$tx~rL#vm= zihbN3c!VW;9TI6Z&kyF@UjPbrCx~yzE0Uc09oLlA(TOP;IHMCVQCKU}9KB?O)3Ws?xkdFBqjA#`uudeq?)-dry7;j8tqpk|Pq7;_|hdQIqd3ks~TCV%C##h`Ft0(5`kVeHKHX#AU^I=)BDB zVb|tCpuZwtWX)ur?F}XgEOi+D8&RmIw)jo&W)Bf7n;?J?Grf?qRfHubcwGH>S=ek$%5!fCg z01_{wpvBmggvFN8dfPI%P-un$LXV;U-xPA-)jdpD;YFsk{6w9#EU|AbBg6iVc+q_k z-te0UIzKjHz3UgaJ2@WCAMk^3Qu#!}yNjk~ia`jS0wpi+A(tXYF9U04`-4#EA6^3n zPSTKAIfHXb8NrL@bD*((J2+0yg8Umkc+=-Ua4Xe>o0pwv-G3VVHx?6lgD$~Lplvvw zIroI*)a;`>`x~%*I1Lt96wrN3SUPh&4nkvu;BP}PMD*Pw3N=w6ev-?Y)@733aB;G- zGYAV*nn18Sp4={02G3pU=t zv;z$N0wM8g7%ES`h?N{dl%9w}0o6P3e2Wy$UdD&;4YnA-=tJ$?-wprU*c zqCf==SS^9tb2Qkjo67K|s1qk}Y+6CdFL+AfDZbbB#jt^9`tf}y$*$mj7eqk%H8FnT z`;DM19S*Na0Ps|mhLP{#s%#@&@le-y~2NzHW+nr2HukIN4Jf_yz9Ngl&@S(3wpkhYYJVoM7n@f&e;n+Lo((q zSN<`|z9LMG*cYN3`;W#5xl#j(%XoRQH5(E>3X)=*sE<-NQ>Xr(k=^e^t%9`BD$sye zQDe+sGJPU%n=>C+lO8fK_LHdz5+#v;^_be<9dsguvj(SYa8gPIt$JGpel4r%x2d%_ z(TI;lO`16I&^xA9&IT`4m_YesUGmZO0U_MIS$tt84VruaoYur+w|+H!F@BkxUOJyP zuf0N)sG_)?o_5@S|^Nk&BfrcyXf!kh(}c8%x~nR;wF>(ps2qe zgIBu3!kBL6VaG{$q&+}JK!>N%p$HsAkA3po0R|q#qQ<9buzA{kXqS;;GhcPWwtaSN zhO(JZ;vJN*3~dvEr;(-2>Lg@CN>jX$m>U>Cd)_wM&rxSKbs<+RD`4_4j?e& z5uD9<$+-_z`J^_HeY)`-nK5Gr%w15+P@5by{twx5|^o+Sg!cv=n^FH#Cc<(o9$EHqKp}$CwYb z;Dw$eV&mzzH= z0qZUkZ0-~1xBs3={xoi3cX54#cN@2%*b^X1xptVvIn>6qCE-U!QEiTn1bALw2#tT2 zpbK|Ty)w4}YVPO4d8_~E2kA(#Exyb=v#Uev6*IZaa~$0LNjbjWZ??~jhiXn;w68@P zPHRlSS7Qo%@$Hx3_6Ew@cv_N!gN}GT#Rep%m$G{bN5NugK2=T_f;bC5=ogp+K0WK` z?}#}}<JOoTfgi*K-zBfcIOo;)UTla@CUa{QIc9J-^Rg;~`*!N!riC#?`fv@*Io8J> z@igT>h$-nhv21>0cgL66sqlCuTRpw{~+wNA*Tu@iDZ0_Rg1eRol~Ns+Upahw^A9I7((kxWj3IY!c@6lJ+Zz@ZGQI09DBbT-;ACn%uxY z3w>bm`aC**Rt;QFNTAM972=?)2mYrXgKn5Gjg?8mXIxi#=3F)Sc!k?5yUwTAyw5@6 z>@t$>QY|U5(t+P$AD!7fz zUFAoEbH!=9k}1hy-oW1hS@X31Z?KuxQzu6wx*^RQZj`L2S6sTNqId#K$W`U%MHFDk z(T(Ku^uzGUKpH+ePT=h=evWc^h3uWzx%Bz=CLFfU#AOe-d5s&I+BrNlAAI*1i6`d* z7;?p9LJrt`rkoljU53_@7Z@8nndfY}j?R#bfH<)!Y|qsX@EsNlx=vs7B<2s&mww@mk0eHe$hY+FBnap!P}2pS#9Mx_&&`VeDng@&mEu16CFYH zji{zpPU|q;Ooy+$VINfvIK-aFoC`PX!qC_)4+KxVr?Nj;t_QFaOP(Su@YI9Eb~~cq zt%Ppwp|s`q5BO?$7{ilYuvPyUgm1LKWXbE~?yEqEK6b0t;z&1B^+T0iQ_AH5H1@&i zA7M~4)e<8wy`baIobW%zG_rWUBUb6j;u^656nfT%Th2})o4;nzWun47&-9DX_Q4PT zDRKTtc_p+jUdAzVRuP>MGkkYuBJV2ab-e3WM^ABctVvA@m^3&aifiVgqntT-S}5W4 zmNT?+$z{;6Ypj(-9gr};2^Y)(=UB!-j;jioOzuPBdtp?8v{9L^wd^r+#QffVU#4Gw z6ZBSg6RSF9*kt?!g!{UQ`Hm%c_w!V+oEQgALLXsen=;+5bAn#-ErxsYYe9HqBjGO# zL3&x47@rh}Z5{ELQ&vkCuRIOI%~jz3pqgVxrK3c90X8<3zz3;U5T#a%I=c_xH78q) zFVn%g@FH-jj)rB+)9A!6pQxJ{$BE012aP@jnB>97!pX}ZJ2HpXI1WMX-S>EC%8y-HKL>Jmmcjaek@#{)6ln@BWj#kOF}vJv!3w=g5EK)S!z!yFsldFpQMiw^y!_2b z9?5~S>M5{T-qdN8Tb>x3Ct|qIvjD)CZ$p zo@T$C;Guu=Q{s0?0lpVILHGKFsOcO8irgOXS^Y^Ah;>C_3q?9OUmj;=&SN8;6X9dF zE%=c6_^sj$2#y5c1EDmo>#D^68MBPFIR6}fsgKitXP?s9j4;pd0v{)&r(v`Mpy82j z__O9CDta{0ZU;$JP1eTwT~d6}p&C5xejVy>_~DlP-AMbI$>^*rq`9t$%~`PsE%Fty zyQGUQecT2Ia~*NUy=!po+7!Ngyc;-HQgXb0daM>nFaAI`YlYr)65+hK?H9Ju4x0I#G}aGAtr)Xcd>+E$6-ndzMS z+5SAr%NDtxBplV9Br{;zqBJyd>ZMM*66tC)Z)WGZ z9WYgDFC5r#6JE;~a^LfObZ(A@)g=dEhouW1cX|eK`bwlXzY10R_e1c6codl^33peE zp!SAMP+}+wKev@auk=IkTB=0HBxmq$PkO_)Xj;-4<>u%r>J64{f?%_wj9m2?CnAB} zINU0awj)hYF38=dO@3q415ut%`(8%!s|EaQRK)KGHkga_x0CEoUtzDlKBR5)h5JE< zFt7O;Owg-m;&ZZzgTxi~s`hR2`s!?2t(A!-O_O!PBooIm#3n^f)8YUW)3TOTLyeqD4|xPEFRjIPAVPe zo7)8!V8BHWkTQKxTeHCoH)Mw6)&)5vU49RAS!S>^BF@0Bs-@uhXa$YjG0e=jj6nCy z68I>nSJ;xpB z<-tY~QM%)1E%k9V1bK65z`$TAj=hVA=62CU0b8g^TMSD*&tl_6Z>Zt+*+O@7LGS83 zlJO=QM4J-Pa`X#*-)j$ohh=$1dRdS#upVON2~XEf3O^jkW0mc*h+oonTF*ZKv*0oL zBBjRPwQDl`DamEd9-0N)UOptU?(1;yYb^$5Y#_}Beq8Tg2fW<)WYJn}&}gwBr~PKp zP`_62yeGl;(4GtuuN2X$ItNb%S3qX#4zQ>@K>bpT`0rFraLPPsv??i~Z3B9+W6Kb; z?zRAZIA0$g@FU3U=vmlhv4Bq66oK0IJ=AodfIfcL#N__vsDWE^7-BXbmWrIk9?K_8 zy#Y(JzMDXWv>XIsK7D(`7N!!Y(DA!G*o+pXOHsjmeAe%toC8FENVWFMqK|8G6H6^K`A^)7!t<~VtpjE zDiLm{ZUKcG@;G7^hl5#5AchwQA|Ib&^N&iRB$13KQYMpHt53vx@B*$hoXdax`5^UJ zzm2i-*5ucuBlSq;(*hqCOp^P~?p0ccL0X2;K5qhS(3%MA`T{}qw>EZV^pn4gB#I>Y zqEEIBE@qpk;|v-2rk_Ap2i*cUA1Sz4kBqeTF{;F6jS735Aj$O+F5F&?Qon3DM*0=f z+qIEcJU>bl&&$%U7z3+o8{nL?nz?lDboN_g0pv|x08JAV;O40)jI1!H6WxWdWUdT8 zG#7?dTVv6?x)+YrSYy_&LiTrg92z?=<%KCrf}ynunZCUi0&TmgD7PaoUKeccS9gZa zQ7&rZ|1CIe;Fnywlxx4Gg%gRq6_FNybH!4G= zU~{eMu?c)3uYMY;dzF5VOu;vW9vJo^haJ866!q>DVcH3QrvKMsDm$jayQv_=S5&%y zI)*#&z5HdCA5{r2+l}}}&cA4crHI+i+8*M0sen33YvAORIr!`BRG#~OC+vy zBLq2~hYx`^=ypAuJ42-SFYZr36Q3o#POate$Blyhy+yRB)E5uUkbtblx%jzjHz>$& zCVxIkGOIs#;@m)aS~;{8BLs3__(Biob3&MVCW!rT`Z=0BZ=z5w!PBP;`3G|fAwPI8 z&iD9%72I6QTUHnOAL6iR={B@U;R1`7-e8`45H5G^fzbAJv%q(X=(_$cWcN2QvRY@r zSYj=F(kR5dp(${yelg=S!#e=Qtj0W>Ka36{;RNf*aIzA>}|0w7PISoipdiUX>MOi%=EYbi^3* zm&TAY!y=$J6o>7K5pdFcHS??7g8q1|OX8Z^>C{wd>a<`YZ^F|7a^sW;Je{?k<4Eqo zFCR2u=gghprEmu>mA8Yzi!7@6s|Z%?p3WFQIfp+3W{_`2rTFXLBD!SkBt7v_o+t5k z4afC$grt90$;rox@H`ga=~o>%bKM=ZoU6&O$`kfj(L(d=`h7TuC&Je#ybCJ%Gx7S^ z2DFmzqk%%#s1xiXQ)liq|0JjY2Xf~^Y+^I9{Pd0WnxKkj681xP+(~oM+bk41N5T)bbzkncCE?DAMNSh&To2`Pk%q3!@uj0jB?L4 zz-#SnSm3QkPjfxcG0{UfK7At;{z`zGB1zm%@(t5fE|28UQ>J0$G|=$@u;Ox*k2arz zb6IJ0c@dv#wwlmmm`kQbU&iX(a&&PAP;?H*Aa`RtBgVP)%pE|G+c8bnPyp5S{#5Na zAA*&y!HTqSbnM_$vBDs(zbVX9lbcLNRdOJR?1FP;jr7&{WjJLVOmu&Ip~7D-(x=B` zxxM=bQnXJVy=VP`hgwr0?t}&&)@UYiforj&n7fN9a}M68)?{RoE}oW-qE~MawBt5z zAJ<6p|LZ&gRp7?WddKON@l6mCJPrYxGO*7jp2Te10?%YT@K5+dR1Mrp2bKhqxU|zK z!nq7)z1T?(jvC=n4`)2CS!eDyKSKnyYfkt z{#D#MR8C(wE`ZI)9ntyMEdKdR{lJdjp_6ufCx_n-VU|c4+zh-8qAp)J9atH0{O&+6 zmT7TZf&v=C@@RXd6&24_qKYSv!0xUw^W#RZAZ=|m+$zz6En#6qqh&1)`Yb?I?)&pi zih*6j<&Y^@V*biYif=hj2ZYK;h;fK3DuWhytrFnRED3=ak z3F?;*F&oT-;r=;}MeH7o8!dzItkX-P+g8qM1=JE5V-=3q|D)D(mnnYH9KoIEGvLBG zA>P%i@5vFFxj_B$Kzi8&!lp$-@z)aac>5g`aD75|oRK3c2d=|jH6z&iVkMUB8N*ZV z^3b*K62rj`FyUVpyY8V8W*8>n;U+HUzC@ZFlQ>V!$_^8M9|<(ck-~x>0(6D?WvsM& zOn-feBs8o53pu`P^U_3`JS`qOmhVL$!KHY-OaOYv6hV2uEq(o;HHbZ=@KrPc#3Sy!&YlP=MA{d2SCaFzQ5Za^)AFOYovniKfDnU;mbp_?=^1x40Pn$M!<~ z^L$#}XOBDP4U)gYN~jWc6ecgJ!&Pqtcrw4VVdG<#Ho0U|*A@*tuCkEq{&1em=w1i+ zWNvcI@h!!;5&F7dZ!Z>amXD)kiG4A%chw^xn{IxfP2Z!y5Y5fyo?Ej3pkQR?R+i#Ku zC6xXdNQGF>MEdNcI(HttqGv_pYhCUZ;FbS;aMAYw;;4~I=80RP4L_bPtC8UK+^?-w zX&NVKUy9H*;23t!J`TA%KH?|8uZ-x9rzl$)M~tp&9X$_|67jnK$*bdBexEA;7$cxF4~EPd^pulj*B* z?(-vHlss^&Tv8SB&+$y8qDU(Lru&z;-I-1OvPtXj;@5- z*REl+-7pd1X~8$;G-%bACb2hKY`9IG^l2XonMe&w<<1dSX$q z0{SJzF>OXP`#Go*J5)>Q<+YbEea=hr9ZG1cx(Ia~@xiCzSAZLH;sw8RSovu>{CyLH zvPXjP>E!$L#Wr!UJ3E`6c>b9#`!Wq=_0z#au?SCRKc)%nK@`>E*c01zKhhrkcj=ts_AbdoftLrxudRqi z^c+1GSpyID7DMUF3+QRF9b}ST;zP%B?8(%J;qp+}S$7RycpH&;>j4;@8-nn?g1+5# z56;E(;at<_uz3NGoW3kbS}LXRn*CL%Xn70oSIFas86QDMm19d?enE749Z6hE5+U+B z(6!F(|1L|a`HA5jc%6lcy+<&6wI9A(!^f`I$D!%yAQ_yW&wn-K`47~Bm)5rAn;5<5vffzZ`2r9PEh-Ub6`fscSW6dTa zbuU17^%eMJQ7?@jZza_p<}mHXbqX1IG$gQ@ynA>K)+aV!(1l}c z9keAypaJ%guj$&9+hjZMB)rMjBlbRKIC0V;qBfvhyEk$H&0O`yOvyTh%t+iz54QVZ z-vk}#gC53&YowTIy0G@LeyHZHgh?$c$)Vv51HK~$rPD3a2U9?dDD)e?=r0wE6@lA#(?<~g;TjpKK_WJ1& zl@k-_wfj<7HfcWZZ(Ij157tNi!sk@-+ir5ew1B+kvX>*;FX?}aZ_q{jDVSW`W1e_# zJ;oeOfpdEV!M*boLPj7lt`sE(c{appfn$yC2?nKWek1J}hK$`SQTI@R`MjnOs{70z zj!A2Su0S#bsa}D>qFInVzsvm0u`F;scpcSGIHOK+4JI5+K`-%DIC3SKsonUD%SoIh zVrHV$#%CgLf3*aL4{1Vn&qd&WQNlGnIYi~D4()1)qDRUj$nOo8pm|dr{j%Z*=2lJR zN!;H_gS9wscb-04uam&X7yO9RYH@PbGy?ux^B0@kd_kADm2diND=4~%q0+T-xE*ef zW{>U!_Bu|RK0X993d4Dzb%}cp6AQ9 z`mO-wuQP$^n#Mn=riHV;wh$Hjh1^cImA)CQ#fqD9w4$j3n;Wjx`u%Z7^?y>Z#ViE2 zzT@1LPn3B%%ME!K)lE_QY7k`Hn}=GTTN$B$f%t4$C{~WvVtf93T=ni7Vf3|N#U3dT zh$@1=t7k*P#hIW~#Y00AasIT8WhAdr9HmaxK>rIb=*g`oIU7VUc{G&fntY*0t^A?; zXDlR%E+KY~vdHO_VeY2C#Lw;|5x&vMi2H70*-j;<@0bU+sE?B5!dS4$XSp0fB%X9j zMQ~V3Ywo^ax5XDxV!MpI+<6Jo-;C2N8cvcLCu8o=9V&R5LVrgdUW@KzOK+u;Uu}u3 z(TUegjPgAc-141maZrU*8GCWhflydxw~}-FY=vc|m0S*N9Tr};rBb!Jcw1lrgz;4{ zRr>}N;WC7$CmB$Wp-dcIlTL!h>q)=pBhV{zhAnFrV{=R^D%>ijp=y&zih3zt-aW+l z4$Jb41JA=JrGNC-`*3=q%8J$qEQXMme5yPx5~L6QqqR=atm|bZYJ9<0f5jx8 zQCb`v{vJ-h(Mnq1`j`>OlHk`zeqia2A@bJ34;63?I&Ce-I`8|q?5YQ^xaR;slwYlB z&>!eyhgnm>T9SHQf-E14%j&UVN&?O$ zGAQ3#Pu+zI;A;f~^B*6CncLpLYeogFTJLg9e<^;x?>1;ForP75GY&`JHShMdrQa3h zV4(OvG&=9gdFYqY7SBM;ICLFd#S1X>*9hdj3B>H`LcG)U1g<)-z%ctnIC=Fxb+WKT z)6ycUJAQ*HX#Pp%OO|5tt{-N%g(7hJQ&|*rQl%q#_sNMzqCiC76SeV3;^L7&6N5BB zZQD4#bZ{Che|8HCe+E!BpG3F-1GOJ)-ok$O8WJEs2uC%GsCuX^EPHyH@ZY=wRy&NH zdvS=m-(L%bixS!Cm&{0V&l)`V>H|&k(gQZG2=ZMqhc%YNGFIX%Yq1Qg8k{QO{sJcW6!qjVU;+H}?>z@fa-M)uk ze?)=8;)`Tub}_X+@|Z?!m_(#*%EDc@@5Ip0o6b%b1-G5LplNj)A~Mpk>BV!p@SF(# zobwSRhoh)MICtOOB?l!6E8y>|M94jAg%|7R@s}lBU>rjvaIRGr+&tlfOXnYhK#6Na zYj~Iz{wV_yM`^OhSQNv|u3>GYB`mz&KzF;gF#aX&X40?a_?^GL!t*mTi1*x7{OCP^ zZr+mMnP!JuBAyfH$w_cP(~n;JX-lnNH`9ZTorHW+q|u*a@PV;AZZVMJ*>ent_FJ=g z5qIlUbdSQZg~CyY?qv` z&G**k-iW1a4nL1oY(Gl+4y{3n?dGsWbt5|FEJYvHIrvR)m`DxYCf2cf{L_^OLH@cf znR&67G&8ULc>(j2_-xduH4TAq=puryRi3UAzk$&iB6V(&E**O(n&hP zs3V((Psc()NP^?BXtz_ZJwj-cJx(FE9>sWHDgTHay^uz!xZ^2&+7gE=j3wYj*(Thw z_7SV|nWxMy~gNg@y%%^u)SKxNEeGdCMTX zoVnclR{{L&mj*)X*Mo&c1O2fki$=`~2Y+EzFjvZAr_Sz$%b8{1utb*mvx1xZ&YK8q z#|^A;RKOH2&uJMRjmJKukv3x+s$Lg>CyfV*SlS{utG=2T-(ukR0a3cfD2+}D&7!WS z@6fPG)lB;AlW-}u7wVQx!E3J4D6+g1GZvr46SZH+(&HcCSKSScYnw{W>Pw>V+(wvK zD~P8*-KKI&=Wo6bPj38FZIK?`$0V$A~u)k{j-1-(Lc$(uY4k!*FtSgnsE~6w9Nje zLBpbhsMdl!EXg?y-EV7g_kkgliMWo2F~8Z+(sdxc4=MeNfcLy{(3h>B7# zKxBb6=6ZAQNYhp-y{;VAxi^B8Rw=Eim1-WEVNEglcX+vKA-V6yQ8q6bsuXJQDm&rBU1i!9%)-D>#G3)x4L05jwLWjpI zKzo@t{E*%X1y*8oGsm_zORfQ()s^gr1G~^Dc{8*+6BNH$M2EJg!yGyf!}fo}sinT; zQRDwTUhXY{P$no*r zS}=Z@IgAeSP_Ql&9k-S;w{_FmxTtWrgYRI+>~Nwj5DVSE?{Nr!OAPmShiexEXvB~# zNQ!g*t&n1Rh3eB)@$;FpMYnLRD}%vb^0HH<+ET{`fi-& zI*Ip9;|}|0KFe4>p3lF(egTL~dBE-@ZoqaOhivu`ooU}iTW5RKmfCAkFSiYpbCZz+ zyX1H~JEo#fpRalIP92h!I1#qJ4ab1dnJD7tN3x`wp|fNvHB>ji`l>=2aep559ni${ za?=^5psBb(qZO`vc}MonngmDprs9uNg5-Nu2K=*H31{5eFmCZz^2y*VTkv-yF5hYj zRSV0>;QlM<6wari5L?IVHww?o}*rwxrVezes>i4_PB4fbt>N@oMxSUB)@lV%;h0 zrVtPOl-0O2) zl{u$b?<%5FaSMOW1gI^%1QsrNG`(apf06Vzn8K{V_ebi{U1l0c|5}e<9y6%1>I}Zs z%!OW;9PBi=Vjqe76YZsvJfvwUY-$| zRKb)B+JVyTi{_tm55ag!G${UgN5=m7vqtI^d)F&5KIzC7CFJ6w+*;PL;1T&?{SKR@ z-oT+QE;Gn|!_N#x)15Oun|>M-Lc#3AMAVT-_mZ=un9FTi?462no8*wUDioAmdx+T6 zZ{*8;6BwQK7J?Tma15yvba~VtYW#UUoXB1ews+5>O;t1c@FO8AP#Ct~)uBf?uFx~L zDZKbi3gB@&m-~)=;Fvti@reH++<7F6eRx%Y%$jNhBGNm_$<+=}YoG|hITJazqa(b# zJqu?_2Vvml1$c(rL&rPo;$6#Evbo9NX#Spx@D%v_evCqbYCn^dx&!D# zOGT*O~o;1l(-^<&P5h_JAFc zDs;ih0TtfIW0&c~@jlpGaR&YKGa<-cp8V)jg^a=ssP7miL^X{xw7#GdM(a>~-xO5K z5F&|HGf+}fpPnp!PmN;U)CTSi1GCvx_(SzNxcrv|?iZ~%ukj0}tYr~AR^19qCo}Zl z1qJ#%DUW5cUBcMrp>-+ zw%!F7d}$;#*&4hTm*zuA#x+=;*g$j}1Ms1J4Hfxv0Y7)h;eVRP@TaOjT`98=&94=a zo38aJu*ZUo>`cUnu_=6Yi_PTZs}k_%a*z|}6hLipEjmYpQ;l0g(7B?Krlh!;FTU4K z-@7N`SXT^o-%_Inwte_XqzXgzIG?bd1NZK#gr!ZK&yky3AosrYPFhMHyn4*&t0jW8 z)*8Wh(9Zv>GgY80iY|%*NG9}&2k^;)ITCOUMrD_>asYq zwwqn@?<&WN?IFE6FUUph^XRX99h#Nov0_IJ2>Gc1xUPVw;-$E6>rEmxAsig)2AHw? zhe1q0A2L?XL}D+8dl-MB!|kQY%kJUu&KmPaT(89AN)vudnuUCmP?$7lKjt?G^6NgS z!oOM}-cI!#$Xb+Ct2osXAN^N@v64%1Qucl#Y59g7^XAR~M~_;eeRlkSd1>_Egnh8a zU@In=#?W%1JNRln*PrKHC$FzwC2Fec(O7yTSSqTb@wdg~*VRs1c56N}@vAm(ftxB_ zo-+e`Vx;hy(M-~AUj|afh3Mivk;@@5xOCt;B>gVMR`KId@94ya1f59|Co!0zK$=EAItc%^M8$la1bmxN+ev9DxK z(H*d4{$qM(mKVCOI)z0=KVesY3|Y0Z1PeE;hSR@Sz|eLr+-?*OxwZz>&EyHuvJ0R) zoo|Bexi->WlaG%3E&#QfO)M_nq^DL%q4n2HNb9J=X+baOgN8DeF}#OyZ*!TlnjDr< z8-;FVPRG4KoHUforV39TsrFe#K8IeTB}WZGaf>qqHit2j=WXOz^@nL_0OzBwjDcO> zIXB+^9dLfMfvPlc?tr0JEbM3m8}Ab^&^3#-++KkvixLRWK82d~uV>FpPK1C421}2v zLnsR2+pYWyjz#t8tN9Hom)wV?!DV>ANfG9Un}KnHHh+z0BaM5An3dN`KASv&=K|w6 zwr3B%G2H;k;Ap4;@SAyqPU)Z?o?cqCUrqsc2$e{>Z}3wqI# zG+iusGJ|hT#9-{tX%NKUAnd@W53f$;&;NXn&ayPe+RdxrcG+GsD5Fa)U-EE4j0h^U z<&l@DIJ@w47 z>=2s|FX~areI=RJJ`p7kPonnLmFVZF$d+*D&S#S8+&#($#g;Q*URe#>e1&;?b-z)+ zz&28RZxmX(%HZ#Lu!8R_OgK8c$EJ=3J>yiBC)qNiEHT&*$UNeTET^hB#B|y_}p>z75%8 zNAc1~ImDEElOH0vaA1clis>6+Vapflzq%i%rP^Yrl|C=rzmC515#h4@88}1jC$Y=! zrmKBrVJs&XMMCGm=Jd%(n>Nreu5Wkhzf4eY`bU&4OktD#1pZoq9BeQQArm~Gk$UHk zAS3>XJ^MBl#nO=c%yvfoqjhMeDlb@E_Jizo?!{5r)!=Zb0}S+zW7umsLcLpQg!z5c z+G<8O*T#~eX^TNrHyE>RWdv)5`tZx=W5_`pDbK=&wvfS>rhh}$)m?Z)WOvat?5B9QuM;DClzS)iEZ#o@eE9!Spetk ze&VjQZ}h2luFb{$S*Wm)P5le|p>oB3ypdN#JEQ$^e@j0GyW5kDJAZIsk_x+6EuV88m69eQ zF2~gB0-o$wc=PW9kx-dT9W}SnyPJE+#zA@bYdJ`CinFOrbPu_9a2d%P(Sf9wZPbOE zrw#5*hbyxAq%Ow|C;wLo=|XeRfOF~{?jbnw(lz$?ZHC!-R0;Q`<*>O5e=$xNP~vVa zO4rnrUccMav~?=yBbkFrqLGkpQAP4q+v&{DdL;PaA-ofr2Jf|kVaR9=WNtjm`RwxO zoCZh8-R^<-i6#HCBk0JZt!&rLCM*>{LnW)PQ9->9w$jg(=b(T|Szd6&!4V=hMNm^; ze~f6K3HK~Y8K!X&TF1m;lz0f~^hzfGE>0C3{`Z5uU3?sk_5U%}CsH8mPA|N%+klP3 z=}hzy9{7r1qg%U&@$tO3#5<2Oz|5bi8~ zNxH72f_6zJe%KeyRzCNqkM`(O!F4yrKD!!*zt$nE{u>mxRnUr?Pni`J4?(4NHm-Cj z$Bu33+_|lWh-6=(PdwJZhix)~qvaf@otqPQ7P*m)`%QR@pPeJ0`pr?O!h*Dz0qnjc zf!E)Qp^%Xx>ZiY=xvsyc=ed<|a-$BM7d0TFEyrMJR2lDWx4@9!D!h%pv&gI!uRv<) zRr2}r7dCR10iDt{6&LA-Fl96TFbZLdk(t{}14E*r&8-Zu=qvTs(*Uy*pV0coe2~ae zAn)%bf`;}FTAKP8CXPLTSqrN1Wp^@^WN*S{DSxPn;66#?*j|Urjq&B&g=FD8E9|)| zg6`WQi0&_a^uAkYmmxcs{tJ4~c3b9y_xNmb-?a^<{Xa*uNf$L-VqxnGE7Bmm7gd4- zSpDCrSf%MnU6(8K%t9^E*>f`5KOdk?SAW|b3Y>%CUCq!m>n9E7o>TSh05Q8U0OlOK zT1G;Hee*Gds-HRu8oMQUv9DzCVD~hf*JTC?xvhkgjzXD1E(x6R3K#o2qBSZ{PO@N@^>t5d@=|ACGSSO7)m!E^?=5NNod4e0E}OX zC4RpkqE;22Tv?6R@}*HMaUpe0KZM;zS!{v&WWh#}@H%3^kB96D$3S@~3G9R_gC$ z=Ebc?m-a@x12z7jH0wE02>na;aC?dWoX=1TLo+ZI(#Lk`B^cUr0EQPHB3*uunbT%@ zgtdN z2_+uzOKviL*{i|J-ofSd*Or3QPIp}UjfZ7Nym42xC3(2*EW}isV)y&E#Q9Me?HA_L zPgDbweU#C5Y&ZSdG7kbLarY6maVY(|2*Rp^@#;TMGU0HRa&Dyee;snlQQS zeh2Sw*T4+TR@lY)UflX!;Mlu0wD0CEsyM-uI9DZ;u>5e?=1~dTpFU%ASH#fzkJAN# zCO$ZGSX1z_(1J8RLADm<>dd))YaEQIYQlkN zX`IqGM3-8-;(-Sfi2jsqwA?h36&=|^#M>mG;$;<8*Vl)Nkx*i8HbSBo9iiSmo3Qep z3py`)!1d3Lkta4`xN4iI;E%NmijGbrqoqG^>alLrSzklZUkinFS(Jp0u+RP}2JFou zCoAePO!_-&wlQSb;RJktvxus+e!;cAvcxNB3N$v0@pcX++B*7%!OFTMls&9Ug4hG3 zYH-uxi$Tt!JWNB>6G zz#2!Nr{J$!C2ZotAnKXN#{sWszm*$HJKELfiZu$>>?^Xg!gE4Ya4Sl|`i>%yr9bf&P2unw#QR!6?t`t2C zm$JX3KI>29A`9sZLowd0p~pimD zy2(?AY{_e6PgPZ;Q>`S%dd?6u?tH>`p8Am~jxUBH?G$pteksA51?=hjwGTjpbC z2>g+B0W$qGT{ZVJtUMb-Gb*ppqZ77px#ATNBTeAzEPXun!Gk$^!%EPrvw+(pq|s^N zRyb~}HOKv~f}p^&u&P{+H}hjN{B}2`_A+Ml`27R$`}hay7;C^rm`9>kY6)h<&W7*w z5>c052nD6}RAt6&5@vM(4ruR0N#8_F{2D?Zoxe|d%bHNwg-{jyKzRM%Ep%x8OFlKG zQmtJT^hp!X5IwYqP`3@xZOv_MFGs7 z53J0e7wD=w6P~OIft1lzG@|A%l*w&D2hVakSzpsmt<#FqMt3rOvk-Y)eG)e@3Dm4` z2r9ImfJD?xj7h75+=-cxId&3!R!oL{YQNckx$jBzZz1}AMm4@!UB{3x9U|!-HP^F{}EomQ7I zioyyLE?4wCr@y+c9v{;mg(6Sdlt6IX9tT~|ebSK#&%8PcA!6^x44 z;;+0pB<7+ky)OF({Vy+|PbSE~Kb=Gji31$?IY5`ENTW`bA7nhJhpfF5$sY}8;840q zYP!hm(ISp#oJS-)5@1xihxqLZC9c`I#Qx_rUfk*+TQ&jPBF5v5Jx+9z@f_%Mbc4Zt$#{0m z6+0f(V$V8Fh>X=Ct5*J_M~8Y*BUc_Ic2zMmFWMsiGas((wxg3@XX2yiA1wC?^15=5 z0{`wHx#u)S!&U9@tBL`A@!}*|aJU|QKh;vZ_vP@|<_fIMpNPWwon-SZ4<^xP9)#`4 z!zlOjtb%tDEuVFc9{GNS+`hV;?ydF5KiS`jqum#J=Xn8X9jL&oW1;NDs9N}D5DKa( zzEp5K}X)r9k$bnxJVYy@nGEQQ$e<$}5M zL9nAJl@&iDfwuzwuqgkF?$U3@C7GUrd<8kc`+aQmBscgKw~!Wb%$hapV(6@jWgG*u zoIEay@k*#19IZbF9;)*pI*Vh8w=`21gAaDc6OWVG8^2N7R7dxdEx1*g z!i=Z$!PS2w9#nbC2+cKw0~@yC^nLy~LGl?ur5{u5kj!%$$QqPlZF+;};lVc!wzBEk;v2=oaC+)6_|jTP zwe%x>jdtYHS7Uzmu}=s}k7zgNM(S~*#Oj4Q$AX*&SMSOL_S$2G8j#3s3UIl$n=P0# z0s1EgVMBQt#vkW0!u$i+Ja`wJBv+GagWYgU*A=9nd%-#Rg_Ngv49xh6d>cz^2lt9(QB3`9?Uz6>7`}h_Qe|?U*AQ0_jK{^6wGIqM4TZF zej+IO%L$#VqTyUn4|uUl7`r`<4Os|1mD-U*JugxIl-vx7>MRp15~$5n#|?q65>zSzzqMNPW;aA>v2cxjsk$iPJ;SuJ;6$@k5Ju0>B$jg*0IG1567fK-O01a z7l_cRvoopD$O=5R?+E=f_5?O^yTBAzTWZul!q#4k#p>BAkeIcNY_M!%N#0gSdmGID z%}Hm3n*fy#%L}$k3m_x5ms!bi!G^Bp@l_5VKt-8Y%=G5IoBH`^uNaCp-xw;M+d!1s ztEl32?w%MxiDa%jeAHM4Pukwvg*XSH_VRJOFEzec>J){4A96emhu3JkP@nkZT_K%| zFAyai32c)*OWY^*QrGZ#IG7#`>;Fg#HaWYJ2%EjMX_E$4|Ei`%lQ-ddavK8%df;_k z0F2d7Bs%Bv$Z(jVT}5mcJF-qnaK1kSTsP;^H3=pt{q`ZXwp)h1n&uGwV?E}`7jpUG zHSDcF7wOizx1@Y16eemfBsNo0_{T#OQO1YM;vO`GrT$X{BX-ZBuy_r;$`Hfnmpj2@ z=4xpF(ZGy0r{GTYtNiD6Q?QE5X3gKJjK`+8(pK(#*%%-x*m^D>?DwRi{j1fe6IKgp z({tfW$R!jRugnYgvA}t~QZ(4=1e(=OgSAx*T&}OgRDl%M%b$ZIuMUB{Oa+lQdByG- zi@~_S)l5ff4gIRL5)>K=@PS$c+H5nRNtZ67F8>i-aX*BV`zM0alc^wkw}RUIeon^{ zj7jscHvBP7m8U*85vSeGAy$uX(Z##2;Me3~czRY5+IfFz{@>%cz_E$W|8E~Htl<1E zgXf8k?oozr2<7;SL-6290xo|)lh*A@gfr37kTl=~@rt6D(EOFg!KXN6!mUO6wlO7Xia?Pz1S~sZSNarIM7ygz`)H;tQ+Cdc;9;WD*X%Xh=QYlAc;Y7nOR-U6>X z4uZs_CP=v~NqSPBv&ZU-Fv&F-gI`?4loAnvSwJ%WK?U;SUjfEAykiy=#NpP*Cz-I} z@mN^z3G2dM!t*u-@>Vk$&m0bg*{NT-Ug!|JNcknbvay%hw6&A)HwodFr`KV4(p5aK z)XuIo?tm2IU9eW1fdC&aoAZ<7T}Q~k-MWRGv&0&Ye;&enx2@TS3EEtyeI;>Kp|I{o z7VIolMFq{fRAIEA1pe#7r-s}eZlVs1E?WQ!*Du23yFdVxYuJO4Zqa4SibiGIJV zpO=C=4rtM~pi$EP@DBtj*WtJBtGGfuj&6Q()20GNF!=X5F#i;a#^3K^Pgy%vnLEhe zB6gQnNM9vYfekc4><{Z>l7_91+_3Bq4~7Qr<6N-@EJ&RIQ{|(mvV=JAbG8J&+&EQ` z+Fr>2Ibucnk39j~##2OlmKOI;*WjVQ`=I@}3+cK&4l9~=l^TWb$}gTp^J7Oa)S#iRuHM!3|MJ+2LDUv!TB&{ zy!P-gI8Kn^J-RrRUDo@AS=%^)hDB>Iu76KMnprP(Y0$T~k!PHVCiO7s@gIwbrT<(93N{tPZ8}pVD z|62o0S-?!e(WRV^D8UlEUwAXWu5e7V%QkSzC6&4k8DmuADpJd3>v;83*x#fR#!jpU zyK|eNZCD$==C4LYwQjictcl%Upuqg;k%j+U# z=FVeo*>N}{@&eUqvLNAs7uX9;dANP^O-ia8ng4V?p>|U@Ya*0OKi~TglVbmZ%<|a+ zul69;b+r{=F;)hts5m|?=)$7rZFof@0lhCfaNmzt!A`J_$aqztAABJjOxj=<|1I># zkATF^ZfF`3#{ZoBz|FyczFy);>KYxHB3Dbhxz-*`ULcnvQ%xZ*PAUBDg8=iw?v-TzWou`o$4pE_KOQN2h@3E_R_qB%pTnTax%a5(Ph-Lu8&tN{gU4U1pkU&6qP~AE=Vd(sxpOsX zb)OnucQ%5c)Fk{_5d%GgnfSnq%S!*aLiTM6rmrK@L9|$%G46cF#sq$5MeFq#%>%!| zXi_M<{NXlec=;K|7g<25GDAf4C1~_9KU^zy2VLY};<3(7I$NcjtlPU7ih>64b;M!j z&|F7Bhd>%DRSX~_tDer2DS>>SA*g$`0c>y?RL?t24b%zyQC5n1^UaR-2Cb*!n@_@( z#r7yaD+s^ty@>B#&BgPTQuLY-mld*B$DHdCu=KKRi=*E#p&=(u~d)_CF0qm#6WD{7>PPNP8ICuMNdB zRAKhhQq)UPgL|*0LArMiJu+2^zI{}IyCu?Szzk8GZPyL2j5kB&;U?H<@f7p7H?sj- z{!#0v)u3VAM_}h%tWh<_(luQ){`WLgxMvByUXx&fM;y$($P@6Izv2xgA6P9UP7m5k zq4Hi)Uh426+!(isn|syZj}Qqu>Ug8M*E|8<9E`*xsr7X6>;$-Y_!Rl6qeI+2CE18Jg>c(MPTWyIqmztZptKu2`+za&2 z;n&UR5=qM6{3cCmFPPGJ9#(JJMV3F<0UvJeV=AvBE#10?PF}BlOg!;Rw3`+^R}xw6Kl9o1Mh*MeE5I`4oD0dmHDWx(E;6pT;{s zk}>LmH!fMP49g}Y;`+;9=*6`?kZt=I!){E&@E~pu^iv3;@~T+TUE_FRX(2G&GXW2q z6i|`(d(n}b#U#JD59>q!Ba6izQ$fEOX?$yqy^HVC@(=Pb@zxdEx9Tb1!TT2SxX%Q4 zHNm09Qanu57BuLUkt1V$=%`vl?`(cc$1P8S>8Xv_x>J@^KRikLc4ROytA4=7$!@f2 zNR@HD-_2$`-hl580Vq2tq5IaUT$jxoc5wW-;?6uYJ+_}{EZ&9jxu2Nn^7DWV5*Cz6 zRN=~x_knzRfuC7Ep1MPsw;u19iBC1bp#L@57juzxaBLhAUL`ghP(k;)e8RC&$cY=V zuzg7z>z&Qznr5D4+b7-Oyu3H)YQr#Usb>$e+mZ<*Rz<^B#51y;52*z=Pg-naA}~Dn zmRg>;1j%tSP)c^tRY{4sd1g9%y(@&5E8|gSzNFxcl^*K7=Kk+(U5sBfcedWPh<<-R z2d~H{(O2AeOiMZU+|-Tp>BjP3)t$h${9$7H+6TAzYz5((1l+ysJh|xNNpCG^rwc0& zAV&Al4{{RZx0WBQHJC$qOfx)ih)s7x5bCWd7SB2d-THcKd)2{9b5_ z-?^b)@f2=e#Qoo^R{dgHzxhJL3a(e9*+pK>x)6-vN(hE%RU8`^!IyYEH9Kj)n^{M&2DHH!fqJC5z9)x(|hjnUDl zf)@VmLm#dGz(6_+4I>R$jsngXwamoTn|H!lA}+YqAcVZV=IC#jgLVS}G=H5WIy~No z@!N*r+(SDERuaX%Uu*>7FR~fo;e{x^P8+3^SX6$!kMm4FrV)W}P+fi=`&&HHZU!lW zgA)UA%~O4}*?N-9Uw+%Jf7%5!4qe0!Z#Y86&$$klS6qk5FFlxb&+>8618Hyy7a|LJ zQ=sQjDK2WgNleSl;6++A?og`Xyfc0D;cY%Fi}hoUL>=ccvM~_4%^l~7Dr15R4?3@H z#lzxJXlF7RC$?tO*V1BW!E-08+Y{lT^$j+oq7EOw=Q4=8rZn$jI((cK!R@=o5lQ_@ zTC#aLxCpO>5jS0Sv&K7eZf*g@CfvZYxAJhQ)*g&J&Us<;j-%*3Jt#1{hX3OEjA(r+ zdgQ$affARUJj;=St#vVa-*WQ1<^wgW;d9P95mLTy2jO|E&;)x+k`ect`7><=xM|IS z!KSGKqYGkovM#~`jSzF{UnB-uzdGoR@0z?XduQ-?$m4m;#0yyk)fk3gzAV~7V z1EWGbRbV@Ae4zr)#aH3lAfI?AJ)>vOd4inIWO(Se4Suf3fw(>q z!Ht1II;y#p&i0Y!{Vku#TPXAed{m@(m9DSJ6#rE0^H74w;aKv1**d&sk_*m7(mX*? z5)oI@M)L{s%*XA{Ad>Qi#Jg~8`MwNP+hc^^+=Wq!JJazOx8VmxTWqS?35wlrL`F1{ zPPOOqab=ImTfIxLXLdJ6EltEV=VBo>@I2;cMZoEA$3Z736?*)ganr9E*fo9@K%>C%lQQo?QW({0Uq{$iqr~@%Bb?MLAag9E$+P~|*!8H57InU% zhpbgWq;wpw&2l<#URN}lY!}7yJ>qBv?@_+p8spnsi1b(POg$Dz&Xh~TSj0*UNZE(Q zBOP=@|5o~H`X$ns_W@=KMU${nCA?-~j3H;LK(DQnnwgi=YTKoZxY%oay>%3e)4cGQ zZyc(%eqau?TN3q^e)w1;1&iaGz)6aMozAn_yDxQMh4?!9@3l5Q+w+;s?TCgc4>EyR zTXOT7L-h3FX{13%63K^QjIpbvV@diDJ4abC-BklGO@7E?qaQ8dNDR=eC8ZK$0EZRmya!sCwwiO+ z%P)uA-7)YWb`x9m0>_Qo5ayNb(fUYGW(VTEmJ%{0OcN!ZP8UpXcO_3BJ)$yyOhMyKzwO5GxwvG0 z7L-azz=>a>M5kmc7QOj};{KgKD7N0=KT^S8;Hz~nD)g8!^I|r*c$IWz09n^Ut3JLpO*!`LQp1SvI z!aDn_@VGD^q^l0Xjm)>WryvtLYEDz_@Y_`H))o97W)C6{#dsy6hU|7fZ&>ke1s?dd zj_lbxNWAYIgrD}8*!>T0FcS~oWCsro;m?d5>Z7I$E|>^wxxJOK0hbjRf0WwnRzuh7 zTX3<7c&bO{iIfy6s*iE{4^QjLbwi)yd)ld zQo&g7QRR==&W73Fqama11};-?Wfw-rL#b*SZatbqT`jsuZhRHFe>j}E^hBE+&`T(Ea zUAvV2E&L4C#$6;vbcpNYwBeH@Q{icGJI!3N1uJ+t^#0gXNPhbbdhPdO!1xHMKD>%fjBikE;;8w2!@1IC9)EmxV9sag}dd>y7 zE;K^;(x-8;XEWZuR0vY*Q|LnSmSifA;YELQ{5nkp`J0yG!>y6<#;6ERninx+UKZq> zwurzga1CC2xrm8Smj{x+guXj?pUO)WA!LpN>6Be~%=`m8f4vDfP;t9!TV@dFn_=ML z_MS9f2_pL@7DM@!=lFDZ1n&!LqJLN`og`9({|4sawMh>d3r`++6@8%l+zPmUK^+l{ zm|?G#4fCyUHOlB)ld+$!aQ>Vu&xzY_E8fW@by@0y6K_P|Revi{Z4?I+TMeR;)k914 zN3s9CA8e~!LQD@wf!%2*@Q4bdcD14~s{W6thw9;diEntghhw}|%JCK*dknX)wnMV> z1iM#y4!EeP8&vmcf_Fm*4uiv8gRyYM;Ew{!~vy=tT)`gPqX*1wKd@!s$ zxg4X0a?rqZ60bm2o_;uA4@;&_1hp-_xLaJ5KkxfrMyQROfh23=O#4&Rw2$D~lz!5E z3*NvxXh*)MN0*E@U9Q~@n@3VcoJT?43g6CUAXnsTA10X zf-mj;KzGkiGE2AyvJ%%&$GZJIjoNWk=-GIl>Bki7xht%3Md4nY_0t(wAMB+|`|8Ms z6-!X_Z|$WJ({}PcgxeRkh=Za`8{KmH6udI$vJ|VioPhfg9BhdqB8F~s(S|tqmOe_n zT5Vx_)EtmII0;RTR>J7<5Rw>cMKe>%$^Ofi__r?2MBAG-c)!Sn1Y6AJrH^Qk@WU2t zT~{WVUB=Ccxcifm|7?&tm`AfkxLwYMR&wyK2`|b$ijL(g3+lPq>gq{G%@#`4@}FXu_5VOg8MG zTkSQd!vZ~+K7IjO`EeZD<~`uw6@|m!g5a@xKfc>{7&9L7NC5#l>YaoKy|V0zIEU9W z_d#}t!!leK<4eyxbArdK_hZ=&NjQ-x4lz^1@vrxIT*lq=m@yHUG=+g@-0#7(k%uVc zwhWXzXQRoa0%-aY%Vl*%aPHUJ*tYl%bNwM7EB+&t{(;FzI)Mi~cqGLsn?#*oi|e01!K5kea7Q=O?)Je5MkH&5vQGwpub)+RC#qNED0VaukA|?tt*tVdBt}wa;p{MrX7p{Y* zvr7~Is4w8W>JMq^hH!2!9tnHWy}_}r0(xA2qWNMSlB~8Ltad%7?~Yf)DMKgRBE|AG z_h;jl@ESO_RvZH>e}mudlhkn0e!QNO%kfIvkh!VRJg#dyxP5N|=46na)cuth_DKX* zq#E%=#@EsR?E9(kpcXMH&=SPBJFwk8&8(zEGti1vn6;*ge#w`D$;ZsWXki7(+HQ-d zrE)>7Ly7y_6G?zxB;=?k;U852?#^ArGdh=wqA&koT;VD7J93sit+pDUY);~D_n8JS z?4FQ>;-^&e=vyM}TtxFHZGgoARw&QS6F#14VZWRYgj(M#Fn`Das>&9DMAJrkDng!l zG_M#JO*WzlVNpahJpmu=`-Ru~BJfIK2#g$%;6=}0K;O!wGUmhM!EQza8TRO)7ydNi zjc3||F3ECslX5jtph;v$`5SWORwrr<>GCG;asn@lc>ef^UYe3VoxI451NHG!$f2nXIRGL{xaSQKxa%0}1kcYU58}BbC;+<`eiUg2KRyP+(MGOB9=d>KSXmk%%`qemvDKt9sGQK9X|Wr<+Av% z*@r&k zobfO*XTCnECejZK1#34;VQxekF7uTW#5s9^-t7mNeJ2{t_i6!-kLO>_2`8=+;lv@- zgZ$k(kC)>g0vmIE@rL?b_|ExV3W~R2-N!JLnBv76mdiu)NE$t2*h>nmq@hPT0{`k= zBePR;2{ZQ%c_0%2p9GS;kmyvLv$Kj;-4W#_DQ+X1^KFTAMF-clAIF|+7p9GEW7Na= z3;g_>0=H&o!vdjYa6M5PC;bfupCWCL+VY!n=1GVx2?Uj$?pU~A4*5C9nXd61gZ^Hu z&B7~1c$D;$Z^q5UJ}!%z=v?Hy^ZJ6=x3#!0&Y1Yhrh>kbG_Pm!Oj7ss6LWF#Wny0F zKsHC&fMK~fE*8|`)}C~{+OQe>l(=q*ldwS1)XC0EY6@;0ZD)c^ACZq&p(~hwDublBEWzYW|47^$VY+y@4^!Hdc{aDr z$*X=j>aj=l47sQjghbt0fU#9kiEcf|l7A*tmHMR9h`VlbBrW4VSl_ntmTX zkxabdn9Pj7{)NoGm5TQ!_wcv4d*Sqp%Q0x#ZoF#op6fdfqd;DOuT&mUqX9L6fa`O~ zMRHlhg73`!<=<%Qk*DNN-W(`g5P>N#*W$&;<8Uc={0TnPAIj(!p#oSeKY?Z)8Kp_J6Nrhg2?V|7d~!;K z_Y zrDS9TNA^u3uNo4lKS~KM`igPR=Tsn{?}Pk66&}CF`SmO^pi`xQDbdgXx-TD(xepOx z4;RRC$m?$;aS_jLp(6*t5kM!UL=L=fyY%u*oKRxOF3$b3m28 zvWmyS&DF3@WjxPqWE`(~3PH1KIZzvvr=9k$kn^||O7Bj>^6SO;Rga>IfOC@RaC?jc z@5%J1HvI1*HfU*V1+%9wfH23uOn*ooDvw;UwOX=@=IeLT-#Q$-X3KfFFgq0cx2?kg z7BD|Ik2|j>Fs@^gc;KThy%7eOv~Viif0c}vzFtJpVFhxe?-0fdKEvo)6&$-^f{Ut` z;BS|I@Mr%^?u;y;pG#x}i%KTj@~b#@Z9daoc#UIr8pHj>gUogX0^OM+ z^yQssmVG&w3U~T)BTx2bchnUVP5$I}~A@I3QaGXpkO`Y_Q z3~e>$`_=uViyN0f{DN(G^-U&R%6dvJ_$MvCn(jR z#wSC(^=v&|kbjl)&P(%(=Dq@5Vh(F}W)Rz_PiVbHF|_Q6hS>_Q>Fxv@C`$NCG?%u} zl`$ttp(Kkp&BS@Kd#6Ko_aSPn`-t?}PQbLrE;_wt4_>y=W{s3%aB1Qo_rA@;D$9q| z>(_elNQlS9|Lx@%;JUEmWew4Ay99>y%UR80Kk`^Umzn7)iV-LLabkN0osqkkoZaF{ z&S(c?Ajj5nROj}dt`fY!Zx-a}*8tqQE*gu?dyyYjL645VL2`};+p*kTx#Ok~Y>PDm zGxJS&cp!=KpI60vG%QECfhUBgEQ;&kIM*Atron}$VBNSs%v(lCaCjHz6*v%1Y^q9$ zn+8mL`i;pv>1PoTq3rlJT;<=p&l= zL)(JU=q`q@Wk1+J=c~lgUR>rk(;gk<)HqHkdks^|UaJ7}mJB$Sx*j8xXJLAH0;{Fn21>zkIOk|OCc1hT+~idx=D%% znbu?c;W`?1IvbWgP2P9LpsL=Wm!oU0wcS?2U`0 zu=z2ul}?2+Z=_FOWHBEuDUlCZok)(RVZN#{gzu|mS44bew?#dM8OGnhbaX$vqHrQ_ zcmHmzVLhRCK@$0KzK&YQ91vt}If~QqFA*#dr-JFnNnV$@!1T;0omMd$YWMIk-F746 zzs-=C)egg4y>0wkDVgvta4$XD8^bqj52f$4KQLCyZFnsT9qiv-!Eo;WZg!Q)2%K`T zg=-5pz-!KrRWNNfM7%6Pg?(qaJa!|#vEM_MJn#UMbT0F8eg=BJ{zoPqOMv9$Y#i!} zg~oM0Ais&rz_cvKj*c8kYt=y8NKVir%bmkdgmKvt3mo>grTcP^qRl`a-C>-F!QS7P zm%4l?8NG_`zy8pFR@^gEd$!mKsKaOke%bMjxNYEe#-0UWIsXmF*XEGPMmM-_8OQ9f z{!LxtT){D02`{^zBvxGiMKN_337lpCL8j&OjW~@xgs!)z0CC?JJx7r68~mU z5)F@WWnVll2K&|`=3c`_($imxiX3xh7ME+i`n8W;Au_`3u=&BRoET3VO5D(Vw>bT` za4o&NZj5!P52yR2Rk;lKN1U!#g7MvA*!XL_VAlDElNi)<*`)Iqur%5OG##x-OY{#i=yZX+>DmkbcE#e%?^fViC`2@^$LN}MYEZ(T zkNnDbIg$>gYEj4{wzmVY<60Y(JU>na7r6->5!AaVh>1NuUeHIM&{{0xTDri9dZ;!i6Pr zf@h6=^oHpryXg-i_$!^$P+^-Z(LJ`CWhWFeJdaeg9$SQ2{i-;_LJ|JV8V~F1$K#23 z0U4Z{k1oQGNrOrhmA$!;Bvf1@Esv(5m&z)@yE&wD$1_MiuFScbcfjdma#ZnKDMl$u zP@~i$dQk5>y)PQZG5#A-p>8whN);znTW0V~-BpYaRe4`$#ce?7s5kze$JQ4ifdei0g+{y?mKqp88k zaP;(E0i$Bs=y9eH(mWFYo_!?zrB=8~vy7cIJseY)Rf5~9G$=KcfxLC@bbE{#w7PnL zY(OrJiY}vx&3{PWf-ty}UPENAYQaF_M!Im`8hYyH|0p^Se=NT@j@v7V&?LK(lvF70 zb3F=4QOamZNqrM#q$Md^MudnY$;eDd!hNnMm5kER5K&r22~k?|yMO<}^Wr||T-WFG zekWu8L~UMsSu!LX9V=v6?8$_ zM$V(6o?r9#<$H$rOpUxgHHXKF+=Oj1YvAV|LEhJkCDf+q2=Fxfv4t#{ z|I4F&<>u7SXp}K&>LD)|4AeB0^-*h%LB3Np0`*koz|Q*@j0e{dsg7p$Z0{v*NArY4 zZ?J`vmTR%dG7UTTMZ?mW>evx@gDm5^N|Mv<*fw#_jq}_Fo$K4ElcXOkmGq;NcXED{ zgC?wkT?BPoZ_I#w7CmY2j7wd%va$`gz!+|ThtPWnTYMf@ZQac5Flxuxndg|kicmZp z&_vnCE7&ccOrTTp41@-S;Fs%8_}`Vs^kdW$dOe_nF(P63-mTk2Emwkg8rgFEu1O%4 zWJey3h=S?;S~}7(kth9I65kFSpttS!<9huL!kn0nf7D$uVN)@_ZP|+x7j?1LoxV_U zGzzBWJA%H-yPEU-lUN**&K%55#;##8&VMDsJO1`P=h2u(x_j?)o!&qYRPv?nI;WTk z)-CMQz-oNm{}IO2rD3!D04RU(fv#*M!%9ue!(dBPe|L`_w77*mM~a~HTopczlK_#K zz3{Zj3R8PuV(0NBl+OtP=S`PjLC#2Z)1EjoT9Cq8k?`g)Ua5%3z0IhlxX_0=XY$hsUS5!l>h8j#+vd0~8+O-?mBM z^mTVlQ&>4BUB8GKhbqs+Pwaz9;VPM4bK(L;{5TMgrC_(>o)4q z(ZKiMAh;dUUj~vGsi)}1-5pmI*n#&5=ib;c8Cus2(|cYQaP1*UzHXK(J1u@Oqgo_X zz45USuPE#RDO{8e&Vni6xq229Mu@ZGCOdF(`&2sjN;5esvI%BQoPx*ms>vkTpLDNE z4u}MYqy10^yslmXkYc)PD0!uvU%u*Xg84cTB~Z+Db@^!`UC zmMP;JNj)k*#_=bIjD3qMZh9XfTXj_+rB0fo=zo3>}-=D^1lI+1) zA|B3be#1ANhpBz-F_P9^Lk^BdqV$^+xLx!Zyjj1CJkMZh)$vGN1u-x+Vhz3><+}Sr z&rP5`043D*poHsw?^r5?mRAqJxzkgadEQgt=etn)@K8%lv+M-^&G)82uZF?q$uWq7 z>+#DOd$M4_8l9RGXtts>Yg@SsyUKTCbdNSKChRj!Gc2eX?R*Qr74DITmBqMtodG@$ ziAA2!ekN+WHuhR&!B}}X9%QoU@utu0$&UX(d$fS85B&!ZilWHwlbZYy`yKR7N-|h} zyTG_?(*d)I=FBhAUUtU0a56PqjU1Y34>1qs@cz4zOGeWg=yz6)kw~-WnBq(MO)pmR zmTQ+FNqlKCJyrtu?L9*etSrZ;C9=H3nk!i^g^BET_Bs@}IuZLn+cDxB$KYzTBOSt{ zoQI}}d@C3xQ@PF#kLyR5{QE>E{8WMKH~opL(PWsgLI&b@EW;n66L2&&j%;F7G4Ays zc6{*1wgN3A)-&PQvQIQEJpiuO1&}*IpUJkTg1l#qhTz0;72i1)u+e>=p{R8)6kn;L zT2Z_3roKCN#QFk0%cd@6N$AKwK((j++klrEB4z zqz`@g$PzKmyht6&07IT1wG zty_U2(2!>Nn^OZO`3B>9pGFSvAAgcXIh^!0o~Y- zoWIczT8w58Z<|0kbe4h3_8ddmqLUPn+f-|#K~3Y9Anb_zL_XaMz^^B6!rgll@J5Uh z4x0p_p0qCK^c5m4@8jwG5qbWm@U5sZ^%C~DXfnBFH=$|ycHEnJm}&R)rS}vj@(rvw zJD+DgNsYY#rQ8hl_m@sqAgK#GxD3T@!gUD0;g_-2BPbMf&4&t_($&lJ)3>E>w_^D@zW^T8Ef3ZSbZ={Dt zyCH8~VLAs~-n=A-m+7-fiCt95C4}f0EyXlxS^hYh5S2V9a?CA+%%7YBM+0uavge#f zX5U3xXnYkT<1ZqFD9WM(!!lJqcJiqe{dI?9s z`u8tTaIFw;qah2O`;5rMqKn|u(TW>WxGosCM?7+%9~SNMrEZ5d(YE$HYPxb8F4TO$ zcwNlIsmGhBYzzb9E$yVjdOa?}Be1~H78kiS(G?;_j3}ENz&K=hA zR3AwkGy%=K-rzRpHcj~_NykLP@xj7pw2&6lgkl-ewl{-(iAN&tBLcb}8>w&cAw25s zj9Cjm!Jmz4Fyv51VrRAB1)dg-xci8@JQXC7XC9E5Bm);NiN?q!U95tVT+MffAQE}l z6aR4MleW)1oaTLo>?k`A9gC9i)rLB#h*4%NldL$N%4eo>XdGK=iompszvHbqz=DF`+uijZjKPwp=hfFHL#@KK=*qk4KB zI*$}VeAOhzde=m-lbZ&&iUB3BJCYeC#bnz)Nm%^;Zq4$(`OKM+-&Fq34yfL#i(fzt zw72t3WaUTMi6fp|wtgQS!E};zWH~jv&BMzk>R|0SnGMX~{N0bI!EU=6@Z49j2cNQGaF2*W7j#a;9D{Rl^<3K_Ndfb%f_pMyS zJFCz~eunpvmG_sR_INfeO`bv8xvazAu`#-Sw1iIT4<=7PYw;~BGZt8OKBe84fBk#H2;jrmyxLXtn0G+hOsulD~LNGETgWE-gVd&q<5E`(XS_kbS z^*LfNfy)88#~Cv(y*Hwf&??RaCBjIwErWqN&bOT9N2=1kbKSX0OdsZY4G+~wxmGrK z9888+?%!}{bOreJhNA3^IIR6J75q!LkS~v2QBao>kDWKbVU7mO6*p z-mNYNxmyTy$9o#`QHUSBuMsj^{5h{?H@3w$;=W6HxMsZ#xXpFuvdG$4*&IkKqhHb| zJ$Lck3O@T{Niw;baEAC=3!%)#DstnnA?wX`-!!Y=Qtt;9?UC-{$T~)55aPrEwB+IsVdxZRfOt`!2RDv*JwIWe3~f^&XR$6V-3fj zJbSe!Hn*9}D{}XJ&x7Ee@RU4Q9E{ShTA|f=ACL(&%9b3Q4paP^>BSvJ)b~dW z-u0RTVIR9{>~iLU$ao|T*`&+Ob=E=ru^c=;d>Yp=p$1}KIuOn(MT|lDt@Sr$DmhN#7tI2hLN2}co zagXf<48C8ET@l*+=u1NYLvLwc9Jhx)e1ZA%^dEgVdmk*CRzqeUdq#Eg{77Xn*N=@; zA?P(opA2#Co`f5;R(K+BOiTg7rGCBAL=yLf)-e#3cRjCL1lM!$)JTA3ysP7(C=$ zuf9FB;&(b5@W~imx_eRO>;e!x5QLaJ3C?Qp;r*jP$at_gA-?xWt}g&>gm5A*ZGsl=lfkl@{pw=5FDV`e1CBwuD0*Eew7zaes8YpD)e79jVI#yqo!reH;BEWiY?bKOM zlr$wvcI=?j$K-k5hu_nAt#NQsaSkXYDD&qi+=rl;L^NOXn5rz840olH&^fHn=)gemVt|Uim|-I8lziK9w(g1aTki&vE?7VZW*VU1c}-MSET_X&^D%Dfc{=M<85q3VgXcE&!M6`z@#y3R zvV7SZ{2}v>RjJCsk}nU?!NP^CJetMjjLms_f9T=Oi@)jVHa=*!t;h4VpPAM3R@Ugr z6vD?geVi7d#NT%JYmI&W4W>OY7aR(`pmYB>T45hef3*0(#)H{4ZO?y`mA#6%*Om|Y zW~cG>HfQ!o%r=f4?g?H~>P@=-yx}~)%0yLUKRRrC4+NgTU9|}$NBcimsh-bf<2R`0 z?iAC-@?npe5z`YT5A8o&;YHp?6#H-w7Wu8@QT`HIn|BD+ZUjSfy9od8xx3Ju@EIbe zO~5?P>-bWnflfan3kF>?*||Roz&eeaxk)tAxdvgVx3(Ij1P-9Mm<3t2GYkuqLr8yU z1GCxs7}QAqB{PQ9moL%O!4u2GK!!3jcf1SZzC6RUU{#c1qTq#mGVye|4w0bAFP|0* zsxKnx9!FWe!;~f(QmDf@4K;Z0?|Z`7u{``YmCLzDDnWV11TgQqi$eJ$Z1O>ACdR4^ zmHJ2N3F*~jvOyRaE^mU3drKheXeag#2y$NOTBv%Q!Dfm0(}uHWVM#+e2&>kD_xyV5 zqUZ{z^o2psXg$RR6L5UzT;B8a81{z2Tkf712nBwxK+PruCs+JnX7BGOs?m}#BRmjh z6?ziS9!XHH6W}W>|G&2&g#Gs*5z9#_#JpFd9vL@r-NVJK){Hp(n-mVM*GFO9oI<#F zPZ88=<=NqUt18F46ZzZfWbk|CB9pgYDIPvD1aiv-c*o^Ju*3K|YCIb>www?E-~Y>o zQrQDkY3WL$%#+4je=f6|y{|CV%JJkvni4*f4@FPWcsOa%Mh-vq14Ux{sgX*Y`jQ+s;s6>&vP)5 z940d5;&hY$2Hx)Z;jDvq7;AS(7;a37#vDg6(!uwoUb>u%`*Z@8NIHewfDh*8IFg_` z3q~rA2Qhl75TQ61`T?5b z{fAnfDn%#fnP7CUlBn4QgIk;c&+Ehu_Lj#wH0`Ql@1HSXGm`UY!07^baWNSGyd0!I z8voIR)tt7<<3Aiob%5Gm+#YsjBG7ine|y9ZNEC{^es;=O-=1 z{+-Tvsw$p+HdIO~*4>8ZA2)+&tu4KGA(dl_Ek#%PbQ*uclXQxtu`^YVVUX+=;@2Hc zZH6;xsi_e7m^#CO*Kf(*pH|#VRSSFOMYQ@ z-jy;sYsy(X;d};f?s5jRhcY-_s*0s=df;J?81@8SLG!jBsMPWv?C;G6^{4}2-FA^? z=yEekWiilCE@d);9WeA?4nYS#?%Wv51Q))+HK(#^Zqh6qkrTnIT^nKniR{PvMbBL%83?FEDp>e}*gwJ|1-GZl4fnQ5CBgvquLIAY|E9sSm0(}r9aylX z3HK2$gKcmcR*YW70EzF!Dc6fUY3Dfm&klj{RSJ>@;l#l09A+K%$3G`8Lc|2}HtRl0Ju`##G8ydoLPePHl_mXugfVjMOxo6z2v*m7F|phbI&y8G zKAy#)NCTWDk^#fdBgy_RJMok!0r$=w+b5CzV$&fS>8okd`e4TQxq>mNR82O$*`0)YIHqx_(?({Ul_>)Z#t)59z3#=A^K|wu6}He%i4c3^JXJSOFe)@LLIx) zx*=n9tmgc$^;l;#3w<@Dko6Hk6TNIAeD(rHayz~u?-}s^fhKO3JP1yoW$=qV_uW?S zB(CKNbm!nFV(WN>UU@EGb25s@FgcVoT3din&<#AlR|}>;AHXL+Pnl@?^x;QmV+=_k z7!dHCikgJM8ICD3&2SEE{(XVWfkqSF%Ve6Az=*9)bD_tNe^x?ps~0a7hKGG#*7AUS;sbQwkC zu4l_Z`$;zK3wjHSWhL?M_yTg{M-;VP90J}pMm5JwgMp+K!%p`9swa^X55!^b>lP@Uwj54L?uB2=?$egDUf`<3<^8(nz)TN5y7rnNJmcJ{BZrzrp4D5PHKR#rFg@LLlSG@|0=EDK-uy|480e$`{aa3AThkvBmoCO- z5vI}{wXLHd^J?36;+J!R7_#PW%_G@OsK1R6I4cfSgs+hd@W%>Xnb}h##Wm5cw-;cnxDqHC29tH2!GDPq4FVrk(A){3c zu4%nQF3RVFp^hjT-PcAN%ahW9Rugf|ZdcO6o1Z;+`r9C@|rLt>s+r zF4xI|gkE-)^HP4p(kZm#-*r%l-AhtJl=ur3L-Bp-CPu+ojTWAZLfeyj@zxU^m>Bnp z@Y1|#C9ev0do^M12U*y-)q>k=Pr`G9x*)Mj5<}fy(;ERh=%c&i>?!%1AhO?ubGL>w z?;cJfSW7FuDRJGtdvxENFCbxgr{=6z4Bk|$hmLFF{AaF_Q1N0qm8&=p7B45m_i3S2 zM)U*p`RmXtF;z6vt(Z)96@!YX^`tlK6YcxL!!Pd>O*A>7_uVVj7_O?1O;6qg)Z$6ILEM&m5kabMk z26s2$C*uhF8VsNGW}d zTGNe4M35#P@mvYZel5bCZr91^>I%9-$PW4_w-XLOhDIY%v_Y>BZRNk<`ZrJMiHK>C zacCYh=fTiNVQfm{j(i}a+F5M$ zl;`++#R@#MTofH-&%uC4J#3bihJ`&+yei)uxR8_!;*;9I^1lhVd6NMxpP&kT4+=pf zB8Yvb9|!TL#L+?NKB=kti)V3p&0L`_RIMl>8zy-`W2GJK-fTu2-U#x`Cp*wb1AwDP zO{n?QHZrPv3hJh-(XkahV0zYwr>Am*(#{_G&h!)+Q~!p?F3pC-)TMa%V?E4SN5JsR zI^vSM85fnrkg=ZYw0%4ke?AiBA2#u0)t=r3k$7tuOxGrLB87x!_6dJEy7F?v!>tHhO{%sdrJZUqswnR8Ju zzlXTW`=M^#As9RImcQbl64{tiPZnicRj)P*Cg>oL?jW4tC%`I-{F|UY3yd|XU z=ur~9p$2-7UchT{>a4y00311Jhlh(>+1V$if$82XG-~auKByT;zO6e%)Sq-O&%A8M zzc0QA-3+#}VXI8&SEU@-!EvG`4IAlrXPNQLJNLQlCl}PZWJeSymC}*AZ7|io758{r zU{|ar$})#=uU8$Ey_^Wvk5b5w)%mnDcpZ!dd7;f#Km7aG0}V22S;LvS?9oSe;c~@O zsyt7cALn$Ge~in{ZT|3+nE7$eU!J+~Ywt1oVO1kp^PeFk9wl%>>?%2g_3Wme3L<-; z6_fma(Z0)>nDS3T`#8 z!b@6&efHJ~7sr%fZ|qjsX|n_u80S;7=w|xl$Ov=qX*cBh&4aT`b-^X&FcZCzV%|Z2 zvZ|0p9eoGRJ*fd}4W{#54(v9`lna6b$*-9mK1FoH)*0-|dv!!%-+3Ign9obn^@X}k z52zmZes9?pgqIcCQN=U`b8ORzZS8HAEj)*-174ut&sea$H^h{1p0QEC*SJUiHtFkj z#0^5LFz#In@L?zVzZBv16~8vw`M@2vss^EMVgoX&=WyDDCzQvr{%(c|!L020sPaXb zin%5*BiTzpR{tip^{iB71Fn(-VptP!G$aIu(Uv$9MQT7+kNEN zUNtLn+f5W=W3W0$3ohvmQj;0~YBU~5 zVTx-9nbs}Iv8kfb!aWyUCOScgV=kr?T%;O4Qmj!Fbyn3;szVl#8ZVGrIJA!=-9*%=f`NsN82u z?tk#$=AKVY>^3fk;#?oH_9S4z(kygh?&GxIgGBY4BFr1vLzeG9K;z6UVQ+U9W0md+ zDc240MaWWK!n7;UG3h1BZum^@rSo7xj3Tjm`2>T0dxE7}1YF{B)Aw4;IR8{Ab-(Qi z%U31iVsioB$0OdH+bIF}TO5Iib89hW^8h@K)I~ExZFE?Glvy@}pm-HbZ*Wmzw$s z!7$G=9eT#zFa;O0>Ar#az-n+@kl}|=e$0v(c}&BJ`q2<`PLCCjzXK09TEIP{RCq7M zLwE6mMCxPaFX0-4tr!`0m#+MffU8pgWK2@QV;eWG^)53@`qs(Ex7SYa5jgFF3bMT}uH}*nh`R1wXbdwj8FAMf_(CzT|UZ0=`2pP+6-C zJNq@^O85>c;VnzlbHB4s#NEI`YXjbFDIkwq<xfX>%=OjT#BAc6jyVcB~=Ei-qFIj3ve@Vw12V#TXRO2V%}H=W@)`ac1LWY*ou9 zNt=Js_T9p?+Aj=~I6r1~=pWiSUEpIhR7K>mUkKE_h;kiXg0ady9;5Vw&V(%3;+E% zLqC@k;qUp}d9rB>{8kEux2s~ngxkrE35DW^cPXrZ=>P^43gWKKKVX$tCzk%qsd%IE%_vHQ~k$(-z?L~ET>6roA*k6rz-!H+ko%s;`H=0>>zk(QOgyKNw ze@wkw2F(zA=k}jxRe64brvgRE1xx8pn)NH`;v7ZxLn|F zT`IP22mH;-Fs6mC7+0n~WR$}zvU23xtTbL>#}Xm z#kF%w@K0M2u|FL|Yl;={@x^0kf5M)eda4KK-Ye)xIKc2=a(QX=CjM^o}yJzi{3mx_EF1mbdepU{w101;1PNJon@ zd!fmT-WcH+&}jlFICvY&n;3rfk@sM?I15L<&t-hK^plv^`rtpL!Ed?cMh+eMLT_7g zXT?Ju`t-CL49zv6OS}_tIE0&T7?{$t-&)y}9Zy-q$R~I&LQuF zILVdE>;CvjPAc@Wu@Q;n{GNF1FzF=$?_M%v6B;>waSRohD+o6xG(&(x46ciQM63mp z*y>mnFc_PM>Wu}s=wXw05IjpeMmpDwQkfz*)Cnp@<#9_= z`|c19&ar}Sizc$Z*$vL@*h_9@JwT}yzEHnak5~5TGIb1VVOOMz;FB3;HD)$%QL%jo zxoWt9tb7wjCcO%!j+3nLNt!L(SsxA#&<-CT#Bi^M7j&pW4r^78aGmI6wEP!NQwHvU zhauOGXt+&#C+5;|O=Vc<{etOD(;~cp7Q8D`4SFquoQrx7^QT@PcR4Dc^q*8xFwYf+ z6qbNrO%&|iWKJen-iBjijmGoe-GF;jt`XH28E{Q%0l(JyIHaD7MhBsD=wA>BW6V5I ziBjf|=tkk4^U73NV-wCy5@8M682p>4M5>N0$2+eaA%3AWYzwypH8-Fw4z)zn{wryD z!1W7xlXx4Of?x%2CVH=b!?H7#!D_e(r<%`3?~_e5ed-8)pRNyYs+95ULLX!-qft=J z6R#eKp+EFX=!yXeD)`|eh1GY6=|g$``8;2g*waZIR=YsdnNXOy;U!q89)tY-%R#8% zILL-BXC{BIgq!vKU|McN7G^&qlO%Y(@doq|U$OSit_gML$1G>cT zP=B2f5cqTz>Z{jNItUmgc@!-Y=J3pvvWSAqbl9uP?YtjJQt7bvvm8%1HjA!R7y_MHd^D2|Ku@hIcG=59eEy@K zm~kEPpsH7-BVQe+)iG$~wh&``6;Rbf7~Bgc<0756@IhV)2ZgGb9g0Pb5@1Uql&LY!$gv_@&e>4|gr!X37F!s0UB8O(!D@p`m=`iD9Xu0Rc+wPd+i z0eSE|0oFzQW+qoyq7|1l3_2l>+YZFin!j(Vt8F%*V*e_1P)nwM$B)4H+j{EOu^xxi z1fXckEap(JA1dZ9!V|{3G4j*|)IHJz?(#Eviac@LH1``Y%}y8-@|+ztuApUlADDnm z`VeWX2OaG{NPBhwd8*_??fgTjugo8GJ9Qfi`X})Yxx9p3yaHnQ{Tiv$E+O7-&Fs#J zWmtO2pO`8ff|u$m(Cd4W$wrCAB+ur0jpOu4yy|tFoE~i@O78c_m&F zlQ&)a*Z`fr&x15&ZF00%0#Y3>Luj@tea2;yx*iTNKOfD3-wtaqw&^NZM7d+0)R+lw z(9I zL+@Y-h$Y5UuZ(GD>%Oz}j71o3dn(S`c(VpxUwBGxRcc_`iaa=YT>|D}EOcevBKpFr zwEW;*%D=u6`?`hEW6@`F{>)$cd&eGJy1O6jExs~87T%(d5}I)M=mRpNP6b>a38B)X zH+0HXUDP)jfu|1#nUA09FyKcq=?;~nBlGWpX;wDdIk5w0#&Y||_GG$Z+D2Gh`NCLl z+8I>wxq&i|&XKuT17@d7(PK+BofMN_Q#ffh9xm#F+&$xPBEgzmS(!rxCbdJ~S7W&Q zs|*j@i{q7VnYd713>JxA!{OR0_)%tn(<5^rvUCo9$lD9g7ldORJ021r^>HKa!=4wTwmgiY~YjTE! zhV8_UFd!`!1qNBUBph z!|is{L13XFF2AjePLlcLLn_B{Pmx0RwH-k*@=6@Ihz9_n%M`pdMAt%>Q1;;Pv=)1ozxGkax%wH(L*PltSqwEk( zel?03Upg^w{e1pT=x5wLX5+9sm+AWKioWw~VEO&4wDVIq?fR-rm*|whiwlDI&cFlb z|HtK_-mb;$f6rLMtZ3?Gu^G=LI|F>1!Qb9?8l6&OV8JFgEW17hiwpTxPCt{$!7bg` z-L#zME2zO|zpGgD<^lZqD#8E$a4PGk`3BwL3voOVjnz-DL8#7ra(mG1?{B{B zqm5#?AhDWUn3jyo3uXDZeGpFL^^ zd1pDk>%D0F?|UYgtVyG(>3R6L<{F4tRWPlQ^QrL^S+MH3&U`FkAT6%~b!@g0?Z9DL z{`>|g8tUQUvk7p$?lsf!Xa%lYITs#sdAM}T3()18L5#1qu``9%!`nliRN{&}IBmW{ z&T0*@yV_;l*^tootf$Sp$zfu!G*AETX44i8nCgGd(_0szx+P z6874$q%G$yh8>v>6E?kPKB-2S^te&XP3)pA0budDNuesG+avUB*_qKtms zAJ1;>55%&>Y}oTAnsgf~qv@quQt&kp9lU3;z1J$3wesBWT!|z|#2W=oo#4xZLy#?dRmznQmr-SU7Ipah?5f+Y`$?c{EE> z9@?B$QCP1IZ=G}mD4IkDO%*ZmYd*-GUWx@SDx_MUM*r0@|WRxvj8Yj8zS+WxP11fB66X95?^Ai8ntYRMm;@juimkjr=KF5-D^X@LrLSNN|)84APsa8>y?*=eyAx92}#{j434 z?K=hbwsy+M@9eny-~k_;UQkMTl1cR8 znMRuXh|8po2w?ecHxT4;-$5@?zTmSkr0z}R_$D8iw`v!w^Xvi@P)@^^Z!bvhyv0=c z#S=Pz*BW&AU=NvQbK#`?Ro3y+8#-ik2v;vZ#xY*>K;d92re-OkN5pTEF=P#Rd_GBi zo&uGgr(k({Bb|Cs6}rN+Y1sHTP)^?gtGdtA=7lzpZ8VMd=~+E7?i&R6d8Vi({Tkv5 zSiH)G)yDRGV!a<90tzMUYbhT*>wJTBDF>jJ$1fN;dW|HCJX&6TXfaOLd`K7j>)>wu zN+(9T;MF_6xXB_4XHCq+t^UhV*7^jTTf!1E&Mme$H3xpq=fN)BUaBFK$qF9*$~<^j zM)?bb;nMF+GQ)u-TWz_#_-1`*snw(%AI6|L{xgx=-FS702Ze^CA5bsXTR+6nf zhx;C?;goi1Ubl8D`^ z=+FI3pNN=4>B0}_r91_fNyi(H#ToJt0CBkvCK-P96VC!hnKA+pf7zjsBB7v zu?IVFU{43!%yopdb~cQpg&6;>A`j1Tj7GidTx+N>2Nr)=hR?OGG745s%<=vejB;i< zBd}VMcV^FQE-$%;NZM@2wf8mYN|kE*v~DHV-+4vM=jq{TE(0Q+rNhRr{4k*15bK#jTveD@E+B^ ze?qtI>FhmyA^4?d4oNfZQ2Jj9ObQXE{{+owhq5ZJt-nVSX8mD)Wl!eIo)079`s>jx zu8u^nTm|#M8jfp4()4$o)HU)iRWLmY6L)nP_au986k{{{H#VhPv96u|A* zy-58WML6gszzdxG7zHM2^0LeiVcpc-)JAkAxjXj^9DLLSi-sd;eBEuzK}2!F>~Ca~ ze;g_O+=w^LC~9s_2f5kj@!3HaP`KHN!zZ7zT1(EOx~(@1UJ|C|+xqFzIbk?&fMETg zBJT2iPdDG#1k$S=a7aS}&I}t8#l6qKI#vP2!xZqfe;zTtW`kvc-_hA)4NPYXK+0t| zJ&;{a^k$m@Z{Z90nf&V!QMgChe#P-$u^iEqD?K1dBm6 zCjv$EVyNhE;D1un0P5q1)ep^J&TtbnudbyoHoClm3$N(b=q`4!Oa^|edWktoS|$p{ zs_?T>AFhg|qUQn`&LQ83!Ca3-=9v`QEs`Rc$DiPx`J3*zlg?~ht&ANFaxhM& z(JAvUfm_lt+V31s)#tXtMU9Uz@wphP#V?~R3WcEQyaTi)ZBR|(9~rR!MR zpo-KS@G(&&Sy!g<#k%6z+_#Z*8fbwyPYa^^8sQl445Y2#a`4wVr$g2ZsN3g5FP}9B zx@rb7G#Y^obqO$4CX1MGXWXm#Pgu9)7m%g%AAGmse0(2ga(%`o{P<)I))ri$pLw|` zcsc}&N4)8b6crNS&Slu7SJDsP+lYllCfqbkA!e5xar={37%r7XXZ(#IOHLc|4BXY& zJnjy{>^+7K_q~a7?EsKpefavZGh9-NBCUawIcByk#v9HDLyjR?En zNwjqCLE6?IiEFQl@Vrh}W76CzqLMZTuL+zdwVjRsqv*T?v3lPyZe&ylsVE}}g@%&I zd!GAkNusQZh*A;_X{zj*>=iPT%n%acJolT*PMT-v1? znZ&nTw)X>7h*JNYP{2ugDssg%@^$p~)|*C%j6 z(FYmWpA|-p|8ZH9tSlNO(o8~DbL{J)@ie_K30>|VMXe-;Y17jP zn6~sfv9QX5mXr$I;4>Y(U2m|>Tf1P@k``$1(4xM}Wr=N>F_y*|;=WZ|;YC#@Ia+-K zuiBocvS-TZuy7778r=(S;T=q?Y%X;zpqyLkU!(X6YtENf%l@0*%qSk$N2AG+cxAF3 zc=lcf2a|_H`m-VSZgPd=f&bBoO>*!(=o5&KPX&=dH9=1v$M?N=8tjv1V_Y4N4m8Vw zjj<0iK5mWH+sC=7+Mdk#>D%P(7Tqx6Se@{vs!7diL~JF=r2aL zb3Crzagh1>)e0y0AA~DqUgS}nFg*5kC7MDe`1r^rGBkcQT#OT=4i4wh=h+|fTAbr~ z-_4{?);E!x5k+v$EguB$=HR`;JLE;0K6RWTjZX(PP&%oOZnQlNylYvoKPiIuu=qR< zOf{xNR0BLFHk08>ZuZ6w)0t5WWG7hD4~NDSyqI1yp+JFdLupzs)G~FOW3P{ zm9%oLEv&%TB%*c|`86d9-t5Y!UwtAVe8T{*i1!SI%iB@pdlIn!WU)wc8+I92!u1SO zIOW<6=@knEyUm|a!#q`R5Zy#9?r6Z1#3a&b_7blccM*TX7H~hOj_lj*=*^usqgkD> zV%{7Q?9fhRYpwXDR!D3wJ%qpgFJb0%`*Ph?aP#V<3rfOppRFancz+ZVWbe?4{|yo424Sf7$%BM4DU4mBNd{8l zz*oqO%srxjr{;6v=OAO)V{iitx!H4oG1sNPv=4tRHpb(7^1)sKaWYd$o*e&y^Y^Im zxBps0CM%C}8G=FDR!;H1eM#W4U@lC48_c-Q--9_VW>|Vtm&h2^P`8vSQoiL2&(E-w zBz@zYzEjH3&0!&F((WaygHK5A_;sZEH^(dp$>#;mFoRf|Y9=6Kgeva$0!c1Aq?dn` zTHIR$fw2$aaj+@y+~?7OnLgOAAugDiltJX~Ya*+4nWlMrL-EpGc{!Q~5i-TL0qb9n;Ml7#=x=JuJYLj= zewXg!`ZI62x$`1y^>{)YkDlUkE7!@xFTv<_Ig<(|8o>JLvUJi+uFtz)l}>9;r+cR7 zk-WxWydO0OlwO|0(H-q<&uh->BKrdrIVPdf8%K~l^BkqK^uazXmKw9_c;QPDCicWK z>tBh%3~kP}^5!g=HVZIomJ)x#oKtAYEW%Yvv&m4?4U#IdP!J;)iSk1Sq1t5|O4d%` z^-bZt2mB)>P--e#oa;g98>V3S&7J)-y`N6LS3>3uB@u&$yTr!eTce$&8U%)EV~lPc z^GL}EW*6{jp3;63@wGazum<4oZ3+I|_98gn{tElIpTnoDD_OU8B{+YNBi~MJ#-cTH z{K36#Sm&}Dt~WH(811v*lG%gf)@Qms3vmJ6z=L2dw}Z^)n9_wK<6vUgDO~&wA>lEXKg=(IZyWM>XUfi#-H$6E zbXg0tY-u9qSc&r+4*tb8HIjH`<5^11XW+QsRcu{fEe*NIG3*s7{;>E+jXyspW=uHPu5Sok~_cN)(lKuHsxbg094fEZQ>@l6&@nsL(W!v(Z9JHwR9zK7qfdDIdF! z%pyOfg2vtb+W`iX+7(< zI2OOZT|_OpEQdw7DF5UEW%z4y4z9mWB+C0$V8ZUtY}CaikS0A1WW@*Y_ed@rYx@H; z#&6)=>rX%Il`b#8Kl*!L_sNyO^CIrLw1C=oKB zERgoG0?Tdx=!ftmsO{!_Wp@)BGQ($svE6B=_E8YJ{|qB3k9gE@`yFz*<~{ZP@eKZP zUGh!guR-lmHC8)}k+SAq^7P#_xUTBXyF4^P9+ut$kK4mYk>h-MMN^aOcj%ODGN3wQ zWHMnvEq4Ci0}lgOP}gfCPWq?tgqjA#?@@!Y-M6scybLd=q~p)(maWiTqhkL6B|r zjb=wjg5}>L_`9kFGp_ETLa)?lq)IxRwlpC&*5Bx~lrOl?dnMM7cH@^qXL7Yeve6{; zH-w$u$*39T(R9z_c)Qt)B!_2{!suW)V6_>vggQv%ol%^0_y;z|yj($+x z5A)v|a@^zHbmi zLA~o`WCjk?Z}w?$?7tJZ`@Rm^9Q;d8bSL2SAtA{4^o8q7fVC9*|R7@cO>v=oiN1`lQV$j0qt5}eSuSRI3!YJ4EG^G#p|6uE`bO`DR z!=9wc_$190-|rftIX2swf_@z+J;oBRD{6SAQ-^*$dKvWG7tID2FuAV8)#z4 zyT?<&VTCNJ9pz@SbKB{XuXP-s-U4RVZKjGrn`xQlA-tkF4qjjNL8-WX$X=XHIsyl1 zPsU+9{3(F^$a;^G@9)su=b>kpY5bi}gsS_5m7G)d)I&^U-9IKWM%B3O7$h^KxIX)c8sSlKK<)f5+Kro#U7u zMp4y13WMW3=vwP^)Xgx2W&h?u`MurXWHO6XzFg13y<6xJbR68@Plln1hd{?akp1H! zhhoMojvSc9-5V$HHLRb)esgUQ|LF%~FSxsX+i7suOod~i6EJej022=FLi1Z+nejg) z@poPk{E?3abITMqI+N?l%{UB^4*y|6kcwc)wuYY7KMZ%Ao`UoI39xWvjC{Q+#P{Er ziIyju*hI@Bvb0?d)YB56sy>xWnU_U$?hca{4Kes*a0pF*n4n}eFoRJ?h@^8R*&^G3 z9l!lRIp{Qf``i)4b}PZoh~wa#SB7HU_K!~{YZIK8y zA13nKgLDLk`<3CBPXm7Fd&C&{H!}H_Avo(%1fI)ZO@3@lgwcanVQbetwlyn=wn!&| zK5U1d0*-5!ai1!;=0VzVjt{z59{*1JLbEI?iKxX**ni)T>psRqQm8w%YW4%gj4iNk zLNR)^HPg&`t@!=91cq$uDB~vQl(`0=}TI~U>f(7_%O#uZnW&ZpoWiGy> zLvkLP0a4#WB-Ac&JeLwWwm+H~G;BqObC1Y6y_+02WjS4@Wy^lLX2@~j=Fm9HCnVic zk|`7~e7Z=YH-i)i{S%8mVW9tbRxza#Nmn^_g@k0Es-I7%1{NiSTnfP^5 z3P^LA(`|DuGTldY&?l3JT8SCt%Y;>s-m`-?zLpZ)`7;}rY9^t^<|(-0;1~Mb<`5#3 z!;gY9)a>*OLFi%zx^Ab!wd0o=e!@h-=ZSH|(PKN=W2prDyzY>6!(+$~ECe6xdX%gX zBI~~`!kD)^P}gM({QEJ9U(;8`Sf<(&A=Q^uUil&2xmXd`@y~~|Jn|2&b2NJ+^#%ws@I7kl0 z+EJgwvVziGYX#T#PoNT472)#ThxE@O5wPw2O!e%h34Dff@RptiW-m8_VV81l=a++( zbP|LrZ^Ede6kyg(#`F3TWI?Y8O8%TiB63Z^jbFoiELwu8JzO_j{4VX7ERFhd-Q4V* zfq(>GdT<~awv|30S3Ml4`mqNnEF1^Z^NTQa`D9X9(PpyXrVsh*l!d;o2Dmmiv|$hR z!&JTNbZu7&*8eBW&f+EG_Lc6~_-O(Aca{+ZAO6I?{O1bFGt)tM@;CbPX)o>alLrg! zID8)2`c$|QczF8DLCn7h9(!FvyO!552{uu|O!rPU!{_i^Q^PS8$jChA4?hgSKWEje(wZn4G^w)1aD#2YOL|B@xA?O4;jXwb_bsuzV~CCA#-NkphmZB1 zfvMwDSctE%elQGkzf6aDGn~;YU_L7Ab}=GpEPb=Q75%MU8=sjzMxOk4ock6q_0J*l zN^dS)JNcAm9vdUmIOclMX9s%HVGC|66CrOO&co51EZo0F2S#4&z$cMupl6efqko^m z!BH)mIrTEzQ`tvKqn0nF?#|o z;gW<0WQDXi-?{4$UGVWZ?z_yV<9voeb(RuMZgT}yUIN&%j(EotK*#(7iu@CT9kD@h zurq;19Ihk1eV56g(s?+0U?yL&>N3O+F);nded41KNELo+z~PoO^mf03AwOO4hPE#b zrC8DM$$+jw!gzw)_p?8CqVI`hbh@;P%Nbs0JNtQ1KV*pakCfv_wwuum_dvZ4C0trI z51U_=;6dqG^yj||jBQjvgR#*Y*l*Iz=qz>yCh7zntXoLKb#|hvsSxvd`q9R>1#Pgr zK99JrMcOULajfs9VZq=ib)lC?w|6S9^7&6t_MeBYSH3eVF1^F~r@4Jy?MK}9ERba8 z{UQ1*k$w={01qOh$l>3OIP0J=JXrn_T-6x#ObA7V@gn>PsU%pDx(Bq62f&3-L7*p^ z0y9PL(&y2;;PT&#plW^tjrJt41I6}uyKEcHe18tZ&4*w`@=h>P+eXt%{*g+mfwG@Z zk`VD0I-lb+gj5A!E|*7M_^p~eCMMYH8v&ikK>rO?artmAhOw~%PX0a%`N!YjxJ7SJ zulXxXQMwN$TSQ5h`(C=9 z5+KZv+^`9*CQiehH!Mhg##OWploS|VdQSe_PsN2D*BP)kK|RmE@V?L-g!fic_g%KA zqB|9_c@pz0Qwpvu&=XiMlSbRlwYbVpmHa6##Sj z=`yHXcn~`3e2H7L6kobR5jNhj!{46_pya=BwjzfmTgI-FQobG4jtxgEp$z7Wcr8Tm zAMuLsor0qy=g6iL-ylHhDI9PaC4Is{7+wB^_EZ#5^T7G!YU2#XxN9Muqnm+Jf)DJr zQDJiHk2jT{!uc`MKY)&%2RAR*#KKER50hlv*rme%mT?egd%c1cRTGcFPfm+Elg@3B%-q?!~BWE z?YmtDv(U%vZ))aFGE4tU#yvTm)7Vm=lYE5RMLg( z;LIDOu~Tl)Q(Y%v*5YaSAowI{$u%IS<#O@GOGBJwtjqq~7sH;QwXpxc5Inm$2FnV+ zl8;-$NwfQ0Q1i5E7?qpesQXt5_%$5EjXOv5lM6A*rk>0i)1qr`Jf*{zmgC#y$Kd9_ z3v|QUebm%z6?M5UgJ1RxpvI#FqCS=j+n(fOaL8iZe`Oj(Cmf@PyF>&7zU$yEuE&u% zujw3{LooDgA&z&-g1G#dRQumrObPd)ugrhrqq~}rE7?o*Cd=bD<3+IjSvDB;W|01b zV5a-n5tyU3hZbaA#f(+XIL=WKf;Io3NwPfb(|ya@eiYCpGnc@%fg{-a&;nYo>EixL z5#+)CHMp#{5X1G8(B94p%DB7K8_|4R7oo!CC6t-YijPEV$LFUXcY+6%^i)yyR!!w+V#wVP8 z!@X-ApW=o?+J_1Mss@ejbj9?se6AlZfuc2z3lCxtNB^x*{NeQAMN()r+RxEPmr zC1I*YD5K2vqjfkQ9$Rz{o(e^>B`wK-sVOnTdtxNHZ2j^fd89)JR zfzP(fpwy1DXl0#FU#HC=8|#yS^FuXmDdFa_ZiH=XnFo{7bAkL_TEWhxAEY!#zID{5xviMr-%acorq?W(#1jLZqL zGS`wm*L7?dTElvWZ-f*iQ!t_5Z-QYaiBO|{ZKGjx#)vE*yvV9+&=4Q>iE z+De$Jr%#}^P65s|YNCp&G0eCq!25>VU_uNJgBLd7N4w<$=?&fBv$2loJI7*`K?3H@ zFvH3V7m3#q339rpg!=p>@Sl!0)xNWn*!ehvOgQi0IngE{DL0Os@j45>>(XF{v^coFuV#~EO%WSLX@}2aJhEr4;OfZ{ zx-^r^1Z`$8>}nW#gjA56ZMr6*$0E=;Z3@22>?gk`wlro;cOhBo`t$+kP^#!|CTV`c z0=N~AzrJOY%A;a}p@V8zy?;5#XmifF-X%CCIgWGSRiXiKi*Cq zj>}|?7VZYv*ZhmRoa$uK%%(wIuQ2@Qe}kURE++HNM4-WuWB6tAN$T)^m}%$!W)H9Z zNP@Tww}kQ^^5@_p+Hn5{>lEZgHt(*0pQgKU`|l|rBdo*p*G6J&6Oa|R1@K+k4{iRc z;-{bPXm~vhe(^4W?#>@5mt{dF^G?&{eoe4*dN{;Hnc!>ZE%fsjZz|F`%Ko~?oq-Sf zh`qfCUMz`ccKN1Yok9+ccv{8Yz8{J)o~bZ7K$N{ae;#_x76L&)Diz)o3hQrqGNzk^ zskcEC9QADnzrk9(9AJvCHF7p5qTt)e#GuB|U z?SCdi6-#kVr7R|ixRMP$vvFpR1oZS@M3=P5*dTQqZU&ymh^kcbecfePzTrCV&`Txq zS~*~Tou#4N8Ktu=hZ^!(xCA@t(w)Y1#*ZMnwml3~dSn~TGTTwXZ4@qSh=B)7RN%|p zDflc>9%FW_q9>m`gHs_%m_BVfzKzuA z;3bExNMAV6*s)NFjcyUf4d=DEdBqbh3#cymJ?<5ZjSD65HH8=v(F7KpFXQskX>dV3 z2t0HKdHHirV4L+98$C2n zk46_(K(>V%Hp>^lW<@bt6PXBymAv4%SPzNU(h!_AdrSi7uSKozYoTu1UG}lhVXgx# zFF3}#geRA&LG+tb=&q+pR^_dxvpl+KaU=t&=|91ODDfxR-eYG)PDGKRMhyMaf)5<$ zz#-MOm>*n=A9G$q$R97f>|jMpANS)Gj>UI9<{rtd-T~`A4UvwA(Rgca189ia;{tPO zW+Zho*^~5w=4{i)wLM{Eq2F^TQci@ci{_BDz$N^Z6U7A{{Qdkm8_t;tdEl@!z452P z9Ow{eVsC~vwY|^)RotCxrb%x&`}6jbCC$wY=vRYE#{;5mCJ)A;?-q9|NeUbWs#?F=HN4uROwHoPKgNm zm2T0qO9CluT1ZUij!^&WAK4ivCXuRrHC$$Y0gQ;=BC;|6@WG!IP&mtixmq)mr(QG@ zCaj#!cX|067F0Mk=9CtoN{JV~4OSL3%$p=Qp)3P~WmjO5+ab(39fF1%Qm}e%Kl8cf z1ZotGhmAb@)S+*K?*k&j9|Ip2n5#5yal=EnBG@O+vzi zAXh~K$Fi2ufSht%UUMAmz2C7jo?eI7`y%m0`VNkXF~)pLpTs}t`qD&vff?*yI2{k} zOvcwaT_l~$RxdA&qhE^Ga%W8yoE(@7ORtq#;=D^uVU*CDR%FoKhz8L0Pd z6^OXLMjN-MJnN9@ASQ8&dA&;*u2{drwX;{zHL6$O(}pZ`bJfL&{YhZDNK4S|nh4d? zEa;C}ub8Xvcx0ez8lB1YG|yfeCTU^b%J|Ggu>Z0crM35FFO0pr0=psuy><5JJQ0!g+&`-oNX~lTO7suZ+wqy zA|;?=lQt+sg;IyZEwHU{4SD=SLNLVp57r0A(e`v>tg*Sn%v&^-^^3P*j6FPIWyU#B z>UzPbgr<`oUtc(@ScrGb+K78>3{C4eg$W;`(NFphyD`!q5}NIykz>aNUkZljcS>OA zCOvG~?9A=-ZK2%zIOow5A)CL;Q7i4+=$rDD;E8iIhU2>}J|qWcl>U-`n^Q46Fqf?YPo|=okLub)=(sDfCPzZF- z82Kd`j}eguAf;Lc_utLMJ-6e@g??d_I5iXQkAIA5{gZG+r3&WXj=&r@QP3go(9tUf zlKdAC>T3$MN_&VvGz>;kbkQO|4hF#*cNB5EimZ)rJWiRM-d%!IxLjKG-XHkY<X3 z`9a-0?Qmw~Z1jmUrh7u7p{QvSya_%4SrI%m<8q5`*KFZ-!~jj24JKjwo2Yx% zJ=)`R6t?T!C5Es1*y&CYw0!+@Htyni&KLWZPKXwVlWki;RQX!t@P^sY^+o|ZJj5_~ z9=9W(V+kNuPp^pha{J_7682^}JUrO}HL96-STL2pg1yLYm=*+g*tpZjc)rnCi;Kp=rlVm>N4pOP469n z_rWsIx6T0Lb1yd5NB_i&LAh+k?tNtYWg)@L*UktL+#c~l0g1Gxlp z;L~GrG}#uthu6V1jUOasbOE%@kHR@0H*k#gXXKi07Cq{qMc?0AjoP2eA$_U@4c6wN zdqpy?T*_d8qXWHhE|h+$55}4Q36b0B63jfOJks#622~BtqnDKgwp``B@~aK##?ODb zuK5ITnY#(bezx=G)p)XN6vPFgQ+|-CMTTG>x*E1~zOlW2n#AE%JXw7uif&otfyJLa z7^%^E+UwU$BiEjS?wtUGITvV9X+Os*3xxi#Pi#-D1-1!szV|obIJDpro7!N*6trET zdA|$kUA-M>_AVRE12^K2nmX32IstYaT>&owBtR@f1AU)};47z0sPa+7SVjVLqpab_ ziiz<0?-FocUrH;uyXUun2-4~Vxa~$7yLsAk%JSZGEOM6q916k-t_mdZIv2tmE{AD; zjd1>#3^Jv|=v?YV`;EgffqUlW{mv&3E?U8XHx7`nRvL1i1w#JFU)JU3H72_;8&zd1 zv486u(y*WeHu!nt%+5Tz?ek6gBPAYV>wduE;Ck{`*_vdR?PG3leTol`+LLYGq4=td zbE;Ktz!w<643#y|w&k&ye}{$A+g;kdj01$e8W9B<}_k>;Esj&p86b9IBzkd+V| zR=&sb_ZmU#)DS4qO@$)^ubG4wZg{yriX6--gB(XWV)DcVh0jRRq18PNbN6$dBpVm_ z-cbbxY##iSnGC92KG^ko0^O7w388{Itc?yMR=QPG!zcpoc`U{;BY%8a_=()@nrw2f z`~X!M=TEJpPO;0EE8*bj8(?|Mi)sx@qfu=-mROhLf<;SF|ByXQ-DM90^ABRJ?-$jkmj zb>QfvOL$Ey@a|(u41c!HtwP$nfvbVsfQN73C|Rv!BO`>E|{N z>iP9H>KeVJPZs{e0&ezjUF$Oq&WndghRa{i`G&5ltu#2ci7J2Bz!TdW;Kn;Q(yXS5 zuRm2n&9miXIHrd@KfND!iAO?MXe=gxH*H@O4K%Y4j&NVUVby%HnDc_%AmLc?v6*gr zzMaav>41(7GiDsW4kNNs$r2%1NX8Gi^D5A|;;GEsJL^d1%m}pp?S?*^rTAgO|G}$c z7h!yAH)-GRN;3w-;nlD_py65=_-Tf@KUSk+Pb@0N=W_Ry>4KYu!r;E@0QB6f18~x! zho79sae~9}tg(wsEajLepG07>c{qqPzM(t2P54WlyJ*GEQYin^OcEy%( zLyovP|1!7Z-Yq0gSy7g1E?kZCUDkuJ$S$<4oliHpL_=ap0kz+F0S-6>Q;nIW&}0z; zeRr(L{ahC`7Y>A;BT>Lq4r281AiU5X#UtEILodIBEYPcFeP1+@gH_Hj?du#2`|1G8 z=|!-)sDjS!H*l*XH=CQkhiukfMKpS@vnSHDu*F*dm-f%#f7|~FjaL@YR=*M|Z#YU0 zESZOoMF&X!{jIE{wjE<~@Eq2@&VxIfmeMGWqi(g5e={^IY2H(1PL)R`6c#HxzT- z&x#g%9N(}THH;Tg{exv>@JcH4y+DHh?BJTlV5c$WyizFhet$NS8Je)j`~uj|UxR&@ zN3c|AGA4|dMYF(GH2-pnrmUF=^*6Y8Uz0I(JmdxBHthxfnAiJQ+xGhW}V|FprFH9Z|-uy*9+dCm8TAtqW)PRqQN>DNTDmeY8 zh?z2}81`f;kzN~(%~KDs*#UYtAs!CxzDcipCsCn3FGQPNG)vDP&TC$0 zN^gDSrI$R0{8t*NGSLKV7axa{-CB2LkDc)@c2K9>547JZ3O1zW(C?wM`Agij(Pa@s4Ap_`SeZ)V#KY;3 z$N}7y5=<^n41?Z;Pzcp&#Quq1m|W?EJyWmHU;4L+oQNF1RJwt8vbml%49COYKRRH0 z^EsmYefnyj6=ZhYB?{+baL=dWh8es~puSNO?Tcbzjo1}Tcq<@MEJXvnqXqI6G{l@W353eS#f@V=DTnAR1XD*P~?LmE~E5m&4S`dAa20OPZpu)qOkQwukW_+9|NI9s0 z{~h*a80+a|!l5@O*AGigALIeVX7VdlPcPKGHkif^l|F7_1sE z!1Aa(I#_U;TpKW=tKDWWg%7hpL1Pr;inzQ-U^#lulZCN=2vZH-k$c-6@#C*o#N&Gc zCU#_?*Rm9p@>_@xzlMWvLo?%WZan{Q+GEn;x&V8xcfn_lA2O76jtK8PgD1kWVRNY) z`U)Ee#@CBrBx{GSgT8V6t7o{DH%#XID1iw(B1vs9zF zN1p5>8ga*nx&L&Q?hIrf>jc3I{V-g2-+^^Vc0&8CGB9sj2F1lD%<<|ZT=&Tl*?2Wf zTp~)(&C6iN7fYg`pcQ}LJ59^C&Z0p}Z;|gp158cxLUiVRLqS#!{>y*L-gK>j-G{Hh z%g-yYtK=zEXSC4Dk7amFMi2D<=HQb#$Jn;Q-?(zo0+_I79V>Tp7%s+~#ZCA9aUaL= zdQhW6N+P{+?9F*pemn@pvuntkD5Sl`Hc(nDNq*iJ0j@HyPJ_Rid(@{yy21jrB;+}XraNHk>59i46dw1=D%BPPpGrSLnG{3>S zM+x-k69;fAU5B+Ens}Gn-}T#$uoB7A`2P84X6IHuERUBZy)V|%EAI+PRWSGdamyMz z4W=NgGYU^e)^j^4RR}3KMcg%=n0q`)k~tCtu{~w*yp{9R+G|1p_Z(%IePFhvaVCv>rRb@!~EsiD!;B?lV!RXOBwr?H@}B3|_0jk4KHT#Bn8j zaT-UOb{(#B{ctLJH-Y4p&txOCqyl%r7H}k8@zpr7)P#X$z6H#W3p=kJ@lQyOW=^80W2X;n$!n z9b35$ZjSGV08eGSB;$&1gO<3op^#3Eh$8zWr<0O}SD71Q&uJ^ajhwtziyzXz;`NOj z({igY{!SVrSGNf0s{ZkUwl{V#P;ebR&b%hFv3v2|b}^6@%|w5ZaQrhk1y58+fcN5L zke^|~_KRNvwOvVceNb)Vp3C>iWVRAilq~q&chpdw9S4pjLcG988LQxLn;sCmh+X1V%;? z#CgIQoW_UZSoNMp zkc$lh&G3_?BIhKri@Zzn_uQq?KmQ}Y$I0N9_B=+f^)739wv^|Y$db9HJ=ps@8-ji> zXZ##qHtv|xiueAcVW{1Ci2fBuGE~>W>+4k*>aYZwMz}|=F9ug=so}&QBIJF&0vHXy zgXGXD=;9{APw?dWoE@8Rq@)Pg$7^t4TN~Cixxrbvcy#Wb$eW>e6E}v(f@;tw*5I-% zf2)1svVHTcp}E%zUmi;$LcdPny&YwAWE#i2`dWlZBYo_`37kXD=qgUswxfzCrT970 z()^^7E)2S8M9x0TBG<+{&@<;h;RC-|N>^>b2nz>j)$s@2ynK?`IM{epRD}Ptc^Uo* zw`E%`4^goMTiDb-3tLaG#?)$Ah*wP});^Dz`9r7hu|@%FMEYr4-F@6ByaBH_{J;%v z-*KnCj6^CxjVlUN&Ka6FEj*_VWx>k-peWkbfC zN6dK~k7hh&e3ggv+)`Hv`Zte!>pI12x^x{E1|-A&W2Vq_I1k40r_=9`s~S54jp4?d z65uSZ%=JwU)b`L5aE>{Nvn*}Eai|XO_9eoDqsoG)557<+{SW46$1#`V%5c=I8+(^^ zk$KvR0`=J%%o5F4%*I?xL~U$U`5NJ^|?N~c}gdFACUzo?(`9{f=kS&VHNPG zs3wkwKH>E1`{8BO3!>28hSMenLSRKTEYs0qWh9^DtBh}?;EbB!VdrT$ancS-YuX@f zs|8HH>QBN-?~^p4ATl>e4>f;Bg3q-XI456+m-0at+f|?PG_0CnZT=&=p?DC*)u+OP zRj1%U>Kb^R?9DS(okPpnRIcIEyRSl1ehIv#;IYw-`#JUN7J=!?dK zanX1}b(YFX-a>X53sNqLg}Y#WTD5FifxWW5fC;Fg8CQFVE=rdC~nxN*ge<0a;k6IEW|Jo_5z+=%4Nm$Uc*A5w5>XbA|1l(6meHjl^f z@X(hgOgpfI-x_xWCoHI<$(yIb2T4^D*m4TRKTDA4hGU@hx(SSLY7ryDar}!5bLqzS z69rpR|1vdPrs}D&HX1J$7O>SbQEX*98>-&RrrnFc`i9q}+~_;@XvdMCFDs~FS~>er zA)7S$6{V@0bZX}L!GiK91u8&(WS(#+p*Rucp$bu3( z5%Mi}J&HWfr!GfUkhDf8;^Or*nAe;3e+fpNX5-jxq-p=+O@c!$9%r z81G`!W@vKHW5;>Kk-lOH{;b96G|uK6k@LyMpvz6*(Q^rn`zzQYnG_P4x``%b5PFyU zylFG9Ve1lx>k5ZBQSrf%v5c%ZwVL{wRF z`TF;$d`^fgQ4N6;4z@(~cMtqJC5kIEPU5x|2dG0am&;Mqr#-|Iebs+4*}u+{DULzp z(cA@KIj57XbkV@s8GP!sH-z;rJdJt%t1w_#10(*+0e_pXsInmwZvL$xwkK}W*A)Xy z!`v2jw?+rfGT$ti{LLAqz4>IL#u?}nGUR$iYxo+ouM^+X+Th;tfj)5l1P%vhLfUGs zH!!S9xL15wyCM&n)qUjCOf^u;ie^l@)dY%d-6nyzv|(PHIDYsZ4WcezAlccOhIr0| z+uoAQjt}=Cv+E6x%{In^)z6t!j(;Ta6`;%YL2~>~C%Es8g{ykXxFB1fIzN1YogcjL ziu`J_B3Kf;p3ft@roV*ETjtQKvgR;r_g0io$Us}mY;OM@3qscZ?CIUbxUqUAN-itF z@w1ZXi_IP6LZAxj-U*^w<%e);i7%t8G8cxP|AN$=Ik3X62Sj<+%uUt+#6%b1tDQ2i zN<{;ISw15@hlc4}*#j_bg*{VJE{yT-Qc1m$7T#3}!F}hGuteZW7Z;Y0kEh(3LGHQs zKkC<*Wul7W=Vx)eWGUcZ4o6=ZLk!z8iU06KA}dnV2Zd!`yqylJc`&1!%q|h`yiEf@!~&gPYm;#)kpkAY1f`9#7+ZEUF6p zf~1?QV^j-n^_SwDB4W@~VTJcnfHh2d&33mY;Z^lz=)KAif6Lrrlyg3^mA!coU%U_3 zI!A+L!%~Ra$N9R|+@K^Y9WvMvrets{DhOrLLjQLBQ2zor@eV$745e4r$>QrJ2k`5P z5?rbFfH_rp4hphNskMv(?fg5)c+Zr99b5k5NIS!xG_50IhTW{RnkOnp)RSsG1j*Ts zu&k?%S*_|rTQatx*8OBMN0!^w6?%~*c@fObSr4(Y>tR!B2K0qy(v1Fh)F|*GiL}f{ z=b7^t5LF@f;T;~*zhA)6F-yVOvn$#y-xEn>3enOJSb=f34^knG z9#(?$noNmU3|aHfBMqO1KQI1Avi>Q9b!{?nFZO^3B1@sU`ZAsEVunk{PQ&;_X(+qB zjs`hf;a(w0qH=8ya7JewQ9XcJ@2p`@rwHtRcaQ5b{=wt(!?;|}ME(}VGNw-99oc2t zL_H?XMt=PgXgMOo9N82MiPc89?Pn65CZ9oOt?$GsgPQp8StuFyY$SsnQy@Qb3t8gv zn9Tbx4GNyz!$QG4!Tvk4?Be08cumEWC@#{1a<|iT)+>Lw`zC?o5L%fm5NPAF4}MrC zRY0CqUcxPYr8u-(4~5;HH5SjGfC^%M^hjkVG9qudzF#fecX)uagfhufA2}#Xod>59 zhv;W6H*rt=FsTU%qH+%=(+9FE@acm#_#Z{*;ZNoJ#__Vrs)(dyWF%2U;<>L=nkuwJ zDQQP4v}oBQWMo%HMiH4&InR9^86_3sn^Yo2C`zTF((n2G0bZ~3@_L?g?)$nvpZ7cM zKsoHR;FHz-h4`)aA7q^ig`I`Y5INfo6%6Ge#pgU)cPHa_=^VNy?>Ch%BD|M_yRbMo zh8p!3v;Un~fQ-a7=3_x0Id~_UCWPF?hxhYgmePH4`g0Rq(jSN!Lci#mX>vrj;UJEZ z@tFE8i&*`)9#*9_Bi(We=0rx)(uKBgd~*o-#y-G%rYi9D`xSD+;t(S{X)`@xQcg`C zYQdz!TI}ta1znnv=v+7sf>K^F=|^3t*^NH@_jo&WDac z2kAbpAAD-L73^$01(J>eoK@RS^;$hhInQ6S;k#U*eDhjRliUPisTou`@Gjk# z5=Q?%u!Fvf4ba*%i8tukL9~yl&_!7ZFllcik^VMA;QL?|<~^T^d6HH*Yk~nx@t8%L zOXhG)HV4EILjUY5#l%Ie+;52=@%Ij-Do{a-f22Y99chqVeG2^~!#Jjj25DPtkBbf` z;$~Aj)DX|6Ysy2|%lzrgK37?bVyk+1kS2@YeZ+A8jy1UPRw?e*tir_;!tjhc$33$N zhdebi%sXF(_uJ19al4=7a?2HRz$y#;+q#IcTN1uHdH{A_`G6Zw8UV}f7gZiEh2vLz z@Q1`&(qC3gyB)9Krhn_Un)I!wF7jr+r0yq-sIQMb zou;7$De2SL^2UqgOG_#kdt3w4hIf_TY7Jzdte=_uZ4PhV^r^6{q7k=L1fipgCU_m# zf!?7ayhAEe@N&jdZXTV5Lp5`F>d6vl!oP$ocSqrYKN@iD&OGvGPYG;3yaUHXv{7~H z0>SijF_xDZ0qXW#hU~{IcoX)O%)`xCXZ?=a9eYG}@m}C)#(Ivuqm4?zLGZY>joN)p zBi(=NNsCbn`{T@Y=)#eQuuzIEefrX}Sj~@|<90_zx@%EZQB>gD`-Rjm5aLK=UyEA3^#-0c>eGn)XKwj*SIi!jeTVv?X`gEGEeaUWI> z%cJR)B)W5Z0r+jrB#r|Nv6;us`a9&oW??!1+h5&EUG-+mmV-I)A^sUFOcZd_*lliC zvH>>rd_u>BKKhsA;=~>B1>35#@UNo=_pFo!b(s>{ay1y28jq5P+;^tLlgH%MqjuO> z`x@J4XtHYV`KbDP2gcddQp`9+16Eg{^frCkB(wvb6tz=b9an6uE}$}|haj3n(BJAv zSLA=8B@cY@>}3;LB{fQV|HM)LusHA4QU)JC@B~JS+*?A`FD@yzuri6k9}ZYz3--E-$pn|!%6&TDVeW) z7995%KwjT5jx!=dnu$5KZshWvuV!N$FOa|PbuvF-{xLdp-37jtyQk&;j1y!N$7WQF z=psW$SF$HkMDexQZTj87li8agg@F&FL89ym+?;nA-Uqvp@rKXnu8S=se&!JUTDy@h z6-~g+B1>>7?*Z=iUPzkW*b>D(_MH1`G1kpLO=9}wu)NfUpLE5UNlY4=MI|p-axhbHJIaX0IAGm9B0Pu{_>;+RaedM@o*){D|k+~6e4xn zGY)CO71FBc&+HIsVY)}$$xyR};8w$N@_FiC>R~a+tewXP^ZP$=bGSNoR|LaBeh3VF zTZ3}LkD%QAB>g4IL*1@jWT*Q)o~iaJBC0)wSL%Hiho|R4-_asm-7yQ?%tT<>ba`Co zitL;H->6T0IM!Y1;wTvn5PR+{yj$^szH?FmA=ejpF}ng*i=?m;M`}0{>?NW(`##54 zsllJw$yhT_UJ$jgoI3oBr_mCQ*kmO|TmBT#o}Rn-TtWfw6@=k2|7GM>RWQ7)nu#Cs zYvG>$Mcg|36Rf`T8*UEn#L^#wSoiA+tT3(N91B1B10xN@QS}K~JdX3{{`A3IQ$vtd zw!sf}0bu;+Dx-GC584~n)1RVGkaxx##9s7LokUA`E&rPtTDF|a7|ta#>#pEc-%rH9 z^CqJo?ZTd3&`MXI>LA;LfErm@MM?GhfPvNoxWxkR? zdrbitR;F+cgjo=1Ck!_}|3_5vuacI~$G9OPn9SI|8U~jgz(<{~aKS_ZXHT9%?%h9t z69adlu8ajb7wdYq=CC&&YBs5>T}A>N=p%pZR|(g z6g}Aec_(t=XCfzJ51LbM;_m~~m@Fc}`RE%^lV^Yhwys?FQv@7dm5{lkjwHQN2%Gu; zV0x++X?il~&1PfOsV9i<8)>++97MQ!qS-_-fu-$4)UWiX zgL59?fzxa8*Re~`VPXsoF(LS421~m7eHiEY`-s}{OdM!!!rz0rY^F#kP7Tc^#SvUK zRqQ2s_+$}tbzLi2tLwzRcs>u+I||_Ag~j;SC68`hei*(ux)VLCDwsNXA?#Hx2GLIu zKpc*ulA9y!Sf5IIwHyEfvWVYyj)7#UPTyST9Nu$vaoR#ty7j-MG<}&bwsv19lP%)# z%A{*J>GNh3{&S8Edw7NI7kNX~Jk3z(L@YFVZh@6~T(3sQ950&&;Lqx8Mp?#*%+zzi zr_>5d_tcZWOW(lc-a_a;ahUErHAZd=+e5rZE^6iZGP1A6;1|iG>l_(ig=+ANUJzaT zR-6vY4BlPnal-P+LM=hZUv)vhr7&K*$mNk-ImYqBB3N|gB0N&L0E>O(X#JfDxaL$n zx}6=P!uyI~lFZWp?5SA^ z!%aMx{VoF>Qh0POsfUIm<6!NsR4~)J450>Ef(KjF;8s^4E&eq^zO^iY!s|~t7xE_N zxn3O|3RkDiroZX0H(Ov&<^k|8oQq|1O_=Jk39x+hJodiN#sADx>9}{Dq@rLi&RC?3 z5f|;@LEBq;Un-SkJnDu!Y!AmDNWszFJQDj#4xYtd!hNxQBJW=r!gNB?>M1FH7 zirPfrIr5$y;r5hg7Cj|j*bK(>Ocb7r=_8(xmtsZqdphL(iY_#3qR$`b3Rb*SMu`>? z-V1@OpjJDWw#)p(6iFcvxhfzVKPMA~f*ueUm$KVW%*9DZgm`ll%iyp%LA_!(vNQ2F zD}2s{?X4F@x1e@nYPc0OszV{YPE_#V!CjuFKXT{wPkJ>J<|NktS_z&q}4Gt};l zuOqdv__u)Ww~Qs4cZ1l;gDYs7+#OI9mcfi4418Qaf$VgVXOf?l(;Y|l1I|(fB~>X} zv*00|=gt$_4=#{Ti3ds4gROX!F@>7v14Qe&l%V@`6xk zYT#XrdHF6V`Qji+(a}KhJKRpDY%9}krGVO2W8`evO5V%!pCH%Fid;_4hD%3Q^9I&E zArUF}Ebn!A)Gw(;r+KRPp19BC_;dJeBCt zr&G`D!HWsGcLYOTQ!+cv;3c>}s^oXtl=1!*Itg-;66qlmzyruB{?_4mnB=DYdqiU{np`Q7> zaeB>CurIB|od4RvY;rXd(4Pr&H%x>>S}O2gLoulxp3FRXHwpSL9b*;Z4`F%kcwV$p z6&*3MqLHU&lJkjTJWHEcm@~hE9=_WQT6J%jmu}g3mSe=+^>tz#->TwG?@46$lv22S zcB&xYs}8YS^#TK5x57fhWSDgO4!BEXQR}noc?UJ;QdT<%-IV0eaPe(ARyZFPWRHil zrV|BS9e2Q6E)`93B|vK;mx=eZ$1*M#c{S-6F)|*bGO-2t<4rGIoMwj^zH@n9M?TWv zqwyHe{=x8n3)oq0C4A)>X>c$@4K(^XNFJB**igi=D%aGJlY^ApnlcOPKTe>R5;=$S z`aG&;yN~vsf5eYi)`n<}{F{l;>) zMiLWT$=%!Cu7gN&EBsaNXElUx)4Ok*aOx?2-1fYjHtp$P(&-~Q6#CS{a-1FBSQJJd zEEyq{N2X!qk9=qq&BJA!%f9$t4{e(~0h@l`!er|>*f8!c^;j^8dv49F-ph8^x7$QE}SU=GI-|CmmyqUg$ssKgRQg%0KdG2`zs z+)7s6`F{7vtLgB)Bn%%VxkJ5WH@djyQy0BAG%hp{<;}$5E_aUjc&Cvyi`2$}g_Bv) zS8kAgvlSb~@6$btxjEsTd=%=Jq$XBEWb~5_t*!Y@v$VG1oP}TTk9Iz(6#Sqfrfa~y z%@uv(WWhz_8e}{{r{*_^%FEueEA{Z1M?!de9aUd_TAL_pc;mV+y zf`RC8;vn0KYq~y@BNu_ojEAApH+6{KsD@^pIbb;5gpM~DB8Sh-1*v25crR_3mRMbY z)|i)2#diUnZ*8nm9}B_LAJL8B6|kfKD&)@A!ymhMWAas3&K`oWy8fq$KczU$LyC_M>wz^s`!SaVOEFLljf0-#%ba+{ijhzgcOO42(^r{Viw91psB?(xCV9?9R}BsL@jeIF zDKvxiYa56W`HBx5UD5G)Bl)PX7{-=;BUSBZ@Xlx-S+iRb8f^7=vqV(|HJ23yXKgI8 z>C!(U6vM~iRY8y;KvMN#fa_(s;7oQi27jCdRyC67zC#(U`eWeeuO(PiggW7RPcs{-!saYi$s^ZY)EaBpb|c{Y=E%%OU50 zDoVCY6XfKkpa~;QtnQl${%mlvXwW`_J%{?puT`38-zv6M`?ihVTe8I;6XsSiX45JmAhb5m$A9ujv#*vm%x{+f(Id2!B^!JN+{?-SBwy+e3Xi+kT>uaH2EP3k>J}ZICu9uDj8tMeV2SB)opc__v^l~Sy}nu zIIfH9qanFHDH?8fI^$e{H0p3Y?8J?hP;#RfzngJ>n50f_Pj7~=1YIQBcr!MRO(wNb z)=Hf!okX| zkWG?s&*71nC&2YJdpsyx#*^`%gFc6zeHJ;xiDT|G< z?7jxqyZ=GGI@DmM!c6*{7Y6ob-GskzANr|3fy|hD^yFzb@O6mBv5~j*ZS!}O)XTwR z%VQ~ZdJ5N5!Z0PHiKI?+f}iUGa58sS?)JI|`kX&aUZ|GuFEs;}=CV}cp$!&nCW5G1 z39|pJs^HVW4DzN+3y!8)LDLTfW*5hryXII7vVtnKGO~k;-ObdQPsmZKM$YWq&S(ot z(8K*Fy)b?io?c)K%HmeIB;hoAEBb@e1}}y&o`=!@6+uyz6Wy~{jY_Wn&M(;`!K&64 z;#%P;9MjPceM>~Z@?I#ZkIf*{zBOXhk8JSFR>Fs~mqTFS7vlA5vS8ZgYA$!)2&O`h zNaw#|R&n48awK4Qzuz40_fMj0Zl=RhZ#}#hG7q+{2`1C*@6(@KJh@a(H)&6crmGh{ zgRNPau!|8!=Y76#a5xofR(z$2g*8-J2g#Nv1oFdz=y1hV&U>{Q*=IrcRWTdG_wbRg zn1G)iUEn7S9)Z6artqWqBFUZ8LO#Z?gRRLPFrp+2(ccqDZ>uy8Y`co%PMya;*8eE6 z4#$a=7h${JOL}O{Y>V3}dx;Lm^iGZ$M_1(s5l`znoIZUTt&A^)idS(kZlDUoG8<`w zpF0{H(d4=E7Q(a1FX@wzOZbE9wyW4z;l8gXIQ>T*Ui`mDX8J+u(67idI((aa|Jutc zincLIyK+&=`vzO6B0+YM`5%UPoiiZ)!{%Fb4Z(Q13j9w-~3?YiAWc~(1FeTPB)xN9JoxE%@& zZn0h|!5}Yv9$&JAlm&h!5mFpo4wqxgstnScl*Fp7kc0A7>#*(RR(!D_0-rEfF>UA` zQPKZL9md@!3GJ=yiq1W-P_2l(U3-4hV1oc)Y$m>tI$ExHT&Gk#in%0}R^B{q1x zIeI*naN$RMEa=1;tf{qX!|KCDp?;Z<|p z?rKwIMk1W^f4K#d`uVza>AV539!`Yjp+r>sVF;eN%OTjmgtIhfXACtm_6(D zscf7)Z)Sf1%$%VME2=oB)yLD!+NLhcQ-zW6Bf%M7tQb%A(q_|@E3=SUBML*Pfs=kJ z@jfp)2#@Qf=`{yM%FiF7QZtUyV;ti<>3acD4Sfvi;f-*8rZ&{;hohHr2v+~G$81Fr zGIz})*s!mg78o0lm*&2>?Khvx>>ME{=RScnegM8ZQ^iQ|dZ|rA2$a6qf=V2NpjurW z4{-Bh1@A??vgO4r5zC;n?%Klhkb@W!vy5)n3Wh&Uw=7M)3~`!PI#sXjg$MD6$@LW{ zSu-L=Cmnjo<%@%`*E0p$rtM=%`*T)?b5?e^$AjtBmq+emKj-uRD*4_yXP_oL|QJ_DJtE>iuW5-l8W!zyWW$h8i{yL-7_^|htAwJQx~ z%XgE{bMK<9&k5`wG@<`?ea15<(!fN7>*+SBfHIfW?Ah$Y=pMet>@SXlX|W|Zu}d3I zHgI{>kO27pNIV*{haAWkv})9+4Fs zNZmxcR*8U>(lT7o@fy>GR+F76sYLDDQl8IUmiQO$gD-u(3PtT}h zdvXE>q@2gl3-4%1i8Q$~^cQAd6NT+d>+wayBGSiShu6y}{O0jd>+44{=;h1t()_^u zTqV(7Yfl3nIN-O?5o%~4FW9tfl*lYMA>f`&>~_|&H>zGxFks<;(mi6hhlK~XgJ9G3 zM`UEqG@2^6oH$H7$-GehiL-N~8Iy) zzK!Ho+#D8dhy3Z&uZkvrjw0>v%jp$aY%x3h%)URoAgLst3j*-Y~DE z4?>o4Fa)a);LcV9*qA5;mNSoFg#HdRTfG?O?|*7Zvj$kX9xXxP)eE@kOD)y8uT77L zE8{nA@3)v-r29vvvm!4hgGE<6QQ&g_MzVfP?1AN|u~47nUinFsx z-6OI->a9imFJJii&6=vUFzELy2;Ub3VpH1+m}Mmgt#7l)f%(qxb2t;DJ`EG)@2SDKjNyICc526T=0r>Gdjk^!S3MO=$z(`kGb9COy(zvoy*6A z32kIF?JeopFacysGzHa-T-ILr5WM=Z0qVT&Glt#`WDZp%|5mSrXt8K~BO{3~r=5c} zVzQj~nd`|+b%5D^A@J?rW|aOVgEg7Lz)Sp5Y*0!&LfAtn7j=GZ>umSdq1%p2o*$M?Pq#_Ily3+64=R!@(R;$L*l82 z#6tHP?R;0xc~Z4;X}t*yaO~XV{W^F^%MVtWYLkymzIeV&k|%Sa26ohV;S!5LI{6pH zv$^kJa43g9?wil%5hGs6WgBq*b_y*^zY@*g&Tu0!m{}T=P3-TN^5g$H!;nl7`>pgU zeW)@WvR@0KXfel;8p*`o(?J|ZvIGweC6N3Nb!_>bMD%LW!Rcn_L1LN=EWRX5o{P)} z+6=NqAHZQk0{XhDfnlICJd9h7C*K$0>|kZA zH1x*nO)Z#T?TW9`-eA7=c95Ji4%U3J29OWHv@OEy#VCB9PiTC|H)5cB6wXJsVCu$ru(}~hbJc&)#CLMEKYfVo64&6JJwBDU zhVy1hvZ~y9tqSp8INFwn!bIH;SebKy_})ClB+U(lc@Mooopo-D~5dV4et}Nxbx-d8n&% z%koggDO61##igScxm@9O!p>ZRH<}Le|1#<1rEdlrWXaLtLIrxpZal8#*df1z0)gdU z1tE`c);}r?XRw(Zqk4cXZZ?C^2ukyxE8=auK(^bp&~151j9pVEo-R2F&G&sE&*==7 zXv{%P?N|)Z&;hHMsgPH62`>H~W6JhkM^CeHaG4*-EP{C0?puUu9-L#V$(i&ooq=w@ zxQvyA68d+w@x|gOx7T)Q!e7sqj& zH7|aH@;-aI_j?wc3+B8CI>8WQJBxvq9CC@KfJ|H$3PQ)B$rKhFmK;W>EmE-ZlmO-# zX<+Sxm6iU*{?L(d9B=*h0mI`wxNuw-=WiR~?)_hw*J2A%OeBMd-Y>&-iHAYnTONZ0 z&2fds65@1026i3aLFWH5!0T^?sma(7HD99z5?%IS?4$@SNh-YSi7WBh^XE)b!Ef01 zZ3(uid}5SBS3$|{YRsD$M^?rEgXopB;jpPQEMBFI*RS@mdo4f^<@1fc-J!@7DW^eS zV>nurMMG!2A+!Du$2|VD1uI2ln5HwjxbD|F)W6EoX&wb2q_UlNd2J`SH}umtauG*buv%77 zc#K}$X8~LGg%cO{6%3ceqTJVRYCq6`X%Q;C{l`|qng&^7_(2Jyfn1O=_;SuY+Et#dJ{xq zxx2SevnE!S9D(vHscgWO`Dj(yNiqY(xaUg-2HzZGJx>|n_O+7OVKsz@PF}>_hw5;F zh7)S_&7<>@Ps6BNAhUYndt|@O{)QN&$TKDMZ`#u;aH%dCU zmU6k4vm`KZH!;~Sis7=8;a7htx>x7H;d|UYDIpo(xqXCIyW2#Z+jD13--9RQ?csL8 zF?ekw2Ns*FaF(&GAnWuxAU#t+$>$fAb7#PV<#ISLYAJ}ZX;7WE1shKOp)WlnX(i_% zs0$Zj=71}4D|yCo!(QRWu36l9;5f*X%JDp^`iS{NbF^#d=kGI^1s!smAZBPfwhs=G zYOjlM+(!z|#LcFysprVn{2sO^+?Ad@@Qz&Dt^+|bjm%%>Iv%<3$ui_;1DRO2g6kC3 zL(b_N_}NnhyVgeGTJLoHdc6VuD!e61yDHGOb0Qf1H=Fjl#=@YnB>5DvRB*>j9k+PR z48Gt=HZD8AG4lRX)?4KR7 z;8+;Qoe4I<)ZtLPI`=4y%yNQIH#uDMAIBG2sev)w^GN^cdzRWezOdFW)X&QCUNQSx^7kc;^x4)V0i*c{lK*`5Nutonl zN!iPJDo!^-eZ&B@>=~r*FK}}ilT+B&)rjK9`LKDc2g5`Z$&uc>SY0)N{2ZbfE3AtD zWvM}@(pfaii^EVyOI&Vx)N;1uGnD>voz?c5O0TJKY)PR}Sf4t=#&DkUc{!90i#4$( zViUmJ`UTc$7cvj4!om06Tr{@Xh!3j+$jIuq#CgDrs5?(3PlnYo+|V4{T_kwl9i5o= zuw9^AYsrihO7S)*mN1t;ucQmg6j81;kV>&tSm>w?36D3CWhv8fzp)t4TXq_3op%YP z9?QX}_C$K(;7cf*vjLveTi|@PQ{+g80G1Q53!U6!LV+jf2+yRpUufWWpd#w8F^&sj|O^5$`&OFnfk6mF&B(&x@(UCie zb0#)HMyWP_nUK$VG$tTxd5NuZctaF_x6>k<7ub~$$$F_oTihvaCHrkJGg}_KBzoG* zX;P^ziB{VTLg7lJre=cRpMoP!u-GihwBy}2*b$$dR;$NXWm=BWHcgUUNIaEDi9F!hX6vTdt zgYHmAT>9rIIz9G+yV-%|EfP9Zq zZf^1(P7WKP#TGr>?-R@zJd=hETmH~xawAmQd?u=?-^SDB{j}DpfQ(t4fQ{o6O9^ag7snIL?IIphi_pvM94xk(Pqc1uJ2FRm`gYqoP)8`kSU{bn`X$W`eqKB-cwGqN^{3(zmb8z^bi;p8Q)uHcQmAMh2NQ zE8`@M<>uR&3ubamnw?n7ouO(QCgQbeoEJ~M3uauf$K}F^1*W+3TM*M;5`c}b zpQCm1Hkc11XnuSHm04T?8?r-*(VtRs>Fo$(yL%hTx5>bbum0%3Ia=IYqal9cA;L`i z!Zc)NLr(a3-ULAehCF%?`OnX?r^7?QXo?&!GJgi~4CK636BGo-j=MlSJqgR6YLmuQ z7x7X>BzbQh2$`KS0;zRgusbUjQ-&{)DEVJRCUhluCVeF#Iie83@n7>-Ohr5GSeTG% z4<*;lVR5+KYU35B%P4|;wp1sLL?|4rd{Jf5Y>Rx2_7@82zVr$q{nFtG{|o+GKrvDY6XP>{U^W%V(7fOTzPcqC8NT zNSZeOqh6;=NY~X+2<4n!{emm_X^IFb_Ps+?CX~`KuT&^Fw*o#ZJf-VQOi1I;B@jR4 ziSxyS=q0THcEC~$CxmN5z@MjJQsRO&XNFnprMcj6^CmjFRl&KPV>Cv41C9xdaIzKG z#mx?fq>a%u_rJyH7@Po@c8eHJ+6Atk%gMjS)8zAT1&(e!42K_{<}$BaNq?LzdA_QY z9D9~bSF~*hiy2wewOb#zPH$q!vYnXdOmTq>*VEp44yU&MN3SJ#qnJWCs7t+}#m=(e zP}hu)m)#~tWd~^a1aDA`oCKfa;_<<4OKb@k4_$h8crx%Z#MzC(lBO+O9$^k79v#wB`toIFlF_yHCi1A*+bPndqD9HvrJdVk7a_DPNnS_;+BQ@vWeWXAxg1yfy%QGvnT#72odKR7=Ls$J1pU{wX#C&?U6G+A2n|byhsKEngLL7= zW)pnZ?n75_J=pGF6?9~MGU4Z2pz6}uf@v*Wj`d)RrCxFh|^Ei?8?%e{p z*8ygn_G5}V+v$tU82n%;!khhlkZQUtqAO3&fT&d}cyOqX&RA1V8zkGIsx2GF4etTD z(3|9o($*76vb!pne=ho3D?XbfO=DD}Rc#V?7rKt0J~3~9`!PRH-l8ONsZs@u&d zFM~_Ru88h{fnPD*GlfLVX}9f9+mEygkl>H;cnYlGBCY^nA|@G`ssHu z@Q^#TT9t#7yaaglnG<=r{VI4_RN^haIdt{dTU4&)<|@WN*>1J>C?`M4XulhyZ|4S* z(iCr2Vm;^Q80LN}c0_}G8ON6zY#^%-J%WDMGOjmxn`qw-XGRvB0ksHM$kdmG!;Vjh z@}x%E&dON&al8wQj7u>6au}WCu@gp1p3)05HllKvAJgjTPeRV@1ly10a4{d*j9*Q`Xg3^@hAfdu#HRq0l@LDVC zq`V%VoD`6(6`QF3NnwnMF`=Wr7g4Vy52`9Shn#E(;p@F5!WkLp3M0h-b_EfgIUcIM zPQ*_39(Jvshz4Iap@Q3ZDub(Gg_|v^iBA={FPQ`9#wNiCw<}3uV_}oyb|N&Yg=MQw zkibJx`wH zU{=8tcro^dc4I5M;YS!$o!>wV&13|+?`L4a$pOal#zyoXDZ!fPxAfP2O+2^1nN+*X z<2gy6gX1^jXhVA)yt%iS`I7O8^R>pYx~tEQ0B#O2{xLO8hy>&NkKpYV1#AxU zMn``Wh)Hs#E(VCN<#V8Eu>vY9h!U%MHxLW0z*LUDF)pbZ%A#hVNuLqX68%V|b86V- z?bYnwfCyYZ^C0Z}Z$GB3Uks&BS7ZFmIXGNy3unDtaEeSL?0fQ&nsJ%+wNKYl@s<*1 zH0>VTQ|59U52a~)g&w4}o5P|V<3U$76rWk|#(?d1*io}(bfvkwyA|& zu)l~YoOeYvYYv3=loFSyfArzUa#C?Nij;7iw0g@~P#-WrrG6O-#3tN^zV8cp6(8bA z?TJey@}vvi__~F4=vW8$lG>T+T;{*Xe~gv3F>)z}!k4FWd(N8V3-L#scC^UQPi zL!0SBD6l^SL06{Wx~)@iTv{>om~iv#mUu>KOcHDp6(Kl(D;XFygDS6dzN%;y|9W8t zdu+%WCM#Qk)4v@gH2gbhb<|>h+wX!TeFcs;tOZ4nMloVREZ%KckH$Na*~XjI^pf#o zHvP{WC@JUe(rZMp(zu_x?5aZ2P3xI6t$56cCE+z}o*}dDu!CJQzT3^Q!^?Rc< z^6)1>{x;|_zQ}P;^3meYP8hGOgwNZ;rmc<#aR)&ng9?bbn| zV;-^E>S|G9rOF#f?1!+CDZCAp{`lckA!^yRV(Pj;Tq(N?cNW(`$HY3)p|6IUsXOG~ zIDp!w<9VhQzBIOEgh{!?Ifv8?v9zTY9~mT=LNjZ_HAq;!rDT$qALh9D!FuaVreuB=eifYrGhHgkv6^xm zQ%mEz*h0MTis7ibupF+cmEeVUR$wX3IX`2Cc+xS#Xxm{6vSGbs>GaDmJt>KlO<4zz zFZ9w|18;G#>;blBXS0P*{W!nKFOJ2TLzVo3Ee|eRglZv2@=GgHHf+=RVh|Bv$r04W0sBmUL@#qp(X?ze-6RRT| zSydV{>;MO&Uem)n_F&I#ZS=b!jM>|RA?|)MZoC`?9iIx=+_8x~+3Hk`U?m~KIoZ;q zd6aa|+6|L)IM<*~j#8`imwlBlZZc)H>o9MDkX&30Xm|8>|i)0!tS9lfP+cH}&4 z(l)||F=4pYR|9=u0Eb;~(TVG4qG^Q*^SjpxRX6xDXY|FuiR057KC6R6ExBZ6+AsVo znu>?na`w+;Jve)o%c*f(X=&f?3GjtthI^){@p6C)wdX5zptRWsWC+U z%>s}pScr2~7NDoqZ&>NE7~bt(#y9sD=Y2W3gV8%b4*FD*@a**#oF{86xX>O%R=>MU zZ!UgAPHZcJ&D$S?(33~dmMp^Sx@W}AewB&nLIG1y$}(>(_Q3IT^XcAeEBIrJJ<)F@ z4~`@}CPH3!&?n**J7Eu@k{hc@McsFZE70Zj&pnFF1S$BQWWglYb+Z8hZun7I949|m zBv`cJCx*3$U{&M=@;3b{(Q7>hPW@a6i&Wv5!7hyX7|h)fO>sUwI4_5f z$Jy5;OWOp)_`RT5VlEJxF30Qq(?Ntf8%exXA>Umn8)lbsZAX=P7<5n`B}EBX zbZ-jGY}gHlDjXqn5kC6I&qV?RILZu;{%1CU4kZmVHLs_+(h_smxPa zE&*Sz*3sQ5Q^3i|j!FrR;oA2c)BQjYW{zuvCux^4BlQSaZK;AD>Hjd-RiD)N{l~6W z93@g3i^&VJ6=J&|;+n%N=#TlFcQI6yjGh%HDH4-mf3FN4$-9Rp%~~+=@eCpt)rcmQ zGtk+}7xwC9V^B#PU8qomKB;TbnB(JZYcUcyL(S%YIczFB31uJV z!g-r?X6h4DINhv)g8p>45iBXN7ye1`Qvf!18$o147TNT%t&*s?;vJT*nV2V#}zYY&LYyPsD=@b#V5U6dL#2A99~Wg5o1J ztXjnNe8Z=}U}_~9Hq63mi5uk4Pb1X+p+@E?Z=+xPZRvbLJd7Hyp&9#bk*;KaoYvNg zX=m>cGZawaJR?-*x{N;8<}ovbx%sEBDa=-QL!@m|7`B|t4(+w0hq(;67u(4cWew44 zj#u!ZFqNDhy2chgTn@$lZ(--@QesnC#-uF&KiJt_r%ICd1swnQSSjX| zPiIaI-)DN3?8nt{-SE-P172$`=D1P|QApGe*h$j7!{fc!0l@G-X#k>2~j;Z|nMq%!3_hIBzYpUq+kkIlqY>{UkwrB{{C*a4TLX zF|22o4eggHCc#tth=QO1eS>PSTC$ehusZ@xfd|pkQVp(e8;7^t&cjR%d&YN31Rkxs zL%(l+jLdazuB!T)9J^l+>C45jHifJ9yw%4Q@5`~Mrv{IWZzd~4BXH6NbuQhp59WnO zqLkWXTw^T?nIA+1%eIxnTXRu-d#we!{5Da&yglgXIh*viOc&&S@`HUgljt^D%k?A{ z;9;&qc5?L*&@5>pKkuqQrt1T~k((HK(Va>|E+&DF@kQdW#EiBriKo6J!#H!kJn!G@ zND|5pl9=yp_^Mq+aK=}OH?XS&7aLB*RbTwzh1f1K#X6gNey2ij!G88mO&9%z8wnLMfeO|+;)@GP=6WPxB z>u|%iFdFitliYY{j5if4u~}<8tlhdEMW>gMA9ua6uU-r9xV<4?GaPWkv@NV&Upk)H zG6X_}nr!H(B~-q-1qWwY!?Qz|@OIaFvMPMD#RQ!NxJ9@OhpxRw({wprmPm4?mov?!XNd>^th}GbGz8aOM z9D(`A40s3E9i$Sc9z(j$HmC~z4zpeLxxRuVoISdgL zZ&dzA(U}HP^|eu4WtK8z&Mb*2k_u-(x0D7E4N8%usHCFVUm~+iDO09QAw`2Eoc&yx zMWPasQfZV_nw9swpMALQ;qJSi^{n+<#PoR`$|r3b)Rwj($678qGmlQ58(WGJz?Pe>=+5?Ud(nCD&XAG zT$--vkDUjMVZzWo)IBCi)wzE9j;~4BP{ZPmQyN%MHyQl7J|gMhG58sG8)jYEi0?Dh z=uCGBI)8R1K6c8%JF{DuzZsjrWyYUsC5LO|_fMA7=mov-t|$wpbG%wppBXsv>J~kAaxcUvErh`KTb47E z%E;x08GJ*VJM`^>v*Z=EgtO^8;m#2sx;!}rlAB{-$GT-0 z>Y`uZINh~k7Pd&~pm$vz_}!|TG~g#gD)=F_8^vMy}t;#TTat)YenW~yaB&~OVod}T!stR3g8Dp1FU>2LV4R# z;2F0Q*pwte>se=vFPTF(@m>IB=u5afv0_;%?r{CfR0 z@o?7Tv0T2vzxp!uwuomZzm}lpf8t=na2)eAx(L|S&ZI#kk^Yeu;2pZ>Ltj}r(YQ=I zd~x$1QCaexzV_S;v#S^KBCUhy*L#gb$*>$0BM8We#n9}(yQplzG`t`8jWr!NgAwU? zh>bdr2Z;d&=n3)cwh{KkMhouj%V*YyAAs~T;pFF}<!wyt8hM~h-F@9x&F7q>BGO4?Ehn+Fgk@J+yA*Y(QW2RRa z9y;AagSY6hK7oITZO#Hv6-Z{h&U!+>?RhA9vYhks3NEZ z!>58s&J&Kk@M}Ga zaP!V2P8WIt`Qn}sR9nve`n;ADHP1z<>=G0-JcP3gLg*W=E7D}(ELd`9uF$%(_|!51 zGiROx>now4#O1fHI;mjE@=)AZuf%k8SdeG3^KjT#f~Yyqfa64rE{3Fvi7Uyu7C-vcc5@p*y5aGKFu3DH&?TW9G7f#x=-d-GMKh)=6 zO;m&XTBo3FAfMA(vT)}p$MtYo2+Lm1fsL`<%%t`9xOreN&OVk3mD;D-?^CzIWnX{= zF249ssE)D9(ZxrLAA_pcb?VW+5anVw;E@+%bab5a$@^{Q<~m_45RZrawWrb1)&`xW zE~CKQZL0dF2s0!>CQKA_bf_sJEhn$ypTuk$=})zCV@? zw%!?7kDSLlzJ{nt?8Be^$MMa+t$4@b9Zct)!T8(u=<%f1a&*#By0$b32K2*l`}?yH zr=H8sNG`#+r5q=LyE~t(qg1|54wDw7Vb3ISRQ@=^{PKQ5)Ew&Jv91iv;Em=33a=)`G5b1?4{(V!t(q(J8B!l7#RzP`Iv}%Wp+cx5BT~GGZFp zyxf<(jt-+6os41sjP=lgn{eN17jzgAf!(%e>9jlps&gm=eG2t?6K1#I&j;P)N@z9R z`ay-Xo;wDG#=X#;Gk^(I_i*9wk95djIT)#KryI??m=zp9>fPo5{CFcB;8g-fFL%Yz zx8^wbp$()gMfv^rnpoNYj?h{2G{NxRb~tq8EVjoe6AAltSS8y@gA(7;d=nnHbNrIs zM>*fs%6O8(aXW8nsqiGkEFp2zAZ)0zB@T%LmiLA2VdRmTxys}|Vq2?4<8Fq6_kjug z?b`Fmuh0x2MS*mG>}SryQA&KvGQl=JICd|9}JR(9Z+QbmpX2r$aC|_rWLo`x!Gtwn(SMQ zk2p;{r7IW{9B(ptHKFMHE}aVQ5F+0?n!q>1ogVr5t(w0$fxxa7dTjL_G@ej^_rkc$ zXK*fQjdaJR+Yjl7k+Zn5=?r$Kzs60M>gmAR1TNF7O?o)~jNRQ}x~n;$YK^oV1bRJW zeb;#6`BqtUsUHNZm$_(WFbSQ{d*bUm9jx>zDc<#cA+X~xP&}vr(qryKyx}D%?OY4n zOdDtcrvt|A*#haI(|NmQF@vJ>8)D%=_-JlkQ8<`K1A4$enBhCkZ7VbElr4cFebnm9~pfhPI z-`}YS0#}5RJK}{PGJQARF4v`^C&Ix}c0FEDHv-lo9+dyv32S>M@J7~sqS-#W5K%C~ zDvQN{=Ga6qE={4AIsRhw_(IgN{zaX(@k#xd5c<2E#;&K6F=MwKL=|15KaLB~Qq9A# zY%#|f8xDv4AzMIMJB!Vj5`*!l&G7F27IeFH8&!Tjr6-2o;hPg&9$0lEjJm&MO|P8B zhgpT#^_LIo@5ET+9R=j!e{Qg*+npK<_0a=DocD9*a#*I3jArkrz?nbJ#8)bq%~2eM znTx0Kb?*hijah@$&o{QSp*i#5K)Nj6Y+jBov>X;{rBDM^Io?}&?(99!b##|{L)5l$ zCacCAibWSdi{>3pn^}t6V#Ddw7Orb}cPP{@O{cm?WvUk1TxO;$oxt0iL8;ITD|+$d zWvV_yk-X$)DaYVaDj$;1q^^mB&f*yAw?!47#HYe;)8!mT-I8ie;bCW0DiwJdk3}Z` zK-yjmeTG+ZY=c_be0UkYPI}4ZdjFBFMM4mML5i;>eww)R*Ab6|2r?{YK?ClFqmuA_ zfKQ{OX>%qvY6sJbyfMopAANlBM;A3pvTk>tT7$noRYHGZ25qbSiOKQ`knAr`?Ub+L z@v8RfRl*LqfH{Mf{@cJYCWA&Slfa_Wp;-GL4_b#C@LJQp>ie1B!O~g@xN1*cnu`Va zzZfJ#>k(U}C2(C+H0-PC!4Ob}jbmI#%fbXazW59jyb`3kZ`DDnSRR^x=8*iQ4~TY8 zGN$FZJEcJ%}DMPMndZJq$q$2u9-Y&QIY1Z?6N;#^MC(bG$%%B_K9 z!%a1Q-$)<4-IE1RIp)-Kt0F`bAIR$bMpZN?FsA}%)A`oj5Lp$&X)zPw%dAqW*zG_* ztIHEDlYDYm#T9RshLFz9GPLoEF*&xv9!sb9!zFQJ{61TmH}G{OU65f8UFrSIIS)(n z?vNDfmX*VX52di6&yNoOCa`{yIFS_Iif77r%=d>6ptS!iK3nNe{0uJQHSHkq=zjs= zFoEYDTL3|-#q<+5H}z~yg2yl0*!{2K=)&wg5;54y{_=lGcD|HE)$h{iys_N!2*(D6u7UgEk8Y@DRQv0Sqtqs`Lt9FeBH>mev$ zI6&?jIOBKLfZXX{L`u~?Nu!cGM6X z_tE7WO0m5_lDb|QLvc}AlJ{)_PTB2AO5UA<{VPrI)~pg5Ie9XKl~=IWwphbHfvZ#` z_7k`yTw!|*UxSA>mlY}-1d+P)6qKCoPHl-gQ)`T97A_e-}~X7x~OQ~>e9NMSwXvsz$*xcO>LqUj5j#uHIcmn6Kslqe-Y2<$JHL8*40LjYg zXdmsv#%M=!ooV~3Q?9sDE3G)%#bw>(pXZ{CaTGn9wG7k*YvFV6PpTKSoo$rAN-Hkf z!TPCbMEU6;#E-k{-N$XXHaZC88+$Oe>?kdetwLq99Yp!}2daI#3A}>0c~SBnFG+%yV$M2>-Oa}9gx)d@_|SK;Sbt-`zQ z5*WUB0AlCO4i}Ko+sGndlA-7e9y6^4^YLW zhcL7}ojA=8XQ%m<<`fK}W2=##p&aKT~+ z=WA?3pN;i6_ks!5HGHPS-`p|QUK&(mO@ZJ21WQ{2p)qYYe9@YT4?na}-0+EtUCUrk zxSggMw{P@NN)8QNezk?~s7j zJA^^!jwZ^bUdF>*r_Y{(<4kX4EL^u+0$nfOLGQvRbpIq1(2SFTO`AMmZhSw!(XJ#q zzamId|7+@#t%H%xYAALhhmJ)p0+|7Gt|LJY_{qg6WGDmCSVQiw%Lb7^K`@-uLcezW zWG1gT{dk zBvq=D1pUmz!Q)%urLzs4yxxm+w+6wkd7&ihWZbd8=VIq~xlGmX1@%BSeEw+_kjHQ!{C<$}h9(LBI0TgtdyST>I3 z++!>ZGwA-unRMV=3jL4kGCg+v477esL-?SZ^@_}kfdvBs2Rx~vDA3@!ee zAHLMcF$%Eh2`#eI14XVmk+*XLHfAcIovOQCqp?WjZ&b!=fgUQla~(W*@`w=Y5SXje1(PHzE#a0MOy~y9GXKXpw%H{A=9} ze%yDcR#PY=IO0m2}M}Q#L%Qlzg3-fJY8+Oyl=F z>R^8dc~ebzLcjHijl2vD`%QpbTASf?)jcSPGsji7Ww=yt8tj=d2mI$=;{4f)ICu9L zdAIrj)p^K*!HY|5>+#bt#cU;>u=d8NCP5VBslu*vicr)j%rD^b52fFt*mP^hc+ zRX7M%pLD34OFS{^<9aNf%Ye=V6HFFvg4uVIalv3HSmkm%@y@yU{g7kT?Twyv<=x-3 zVnYUe4BFyB2lAyqhBDVulR&2 zVvRI;RRX4j*Z8L@nu!LteM~Q)L z26)$rlbxL}AbLd+T(1vecf7Z!YubR`<|o2>o3G4TsViW=m7-JGUwUPiAhoq=qYw43 z!)v?y*mm|1{LXsM$Z786vaUy=C&-n{#>LUx+==Xj$(c8AU!AR z0$Iz6<>(1<%x&Phsz2pW4@6Vz2FA^)2rr7~(bSUeft zd1Yc^?#?99qJ-`9ed&njS4Kwg47SXQV%IIu!PxO5P}Tq>V8;UZ8@m+8|I8*g=Wu?3NfjjN zcnIkhh{9LmbD8fmuRyOJ=U=+5i{Aca>{Hcf8h2P5`j}kaqNc_8aSeLm*`&p3Pm3eVK(D{!dE%=qGAYI&shaFD#18SI~SiB?SaN! zugOSPE$0W31gkY(jBEa39DMw<+L;}~#Lt^yW6NK*EG3>}Z?EDFag43WzpaQz(l=Ve z?;yjC?PTkv2yDnR!qh-#lp53^;iJc3uIWU+$)-R$+$G6!{u&%uCXO3+*sw?Y*Ad?5 zaQJ?-ht_O;kNCTa95VY(YYwl({X2EAT<|Dyx)P7^5}LHk@-`kl)(+*yLy+Tsl3e)9 z^*`?mV+#H&f|Gkc6WM7mS;IR&@bwiNyoQ}V z_5|nFNYl~44h(SZ1G~H{ke9g_*xCoxsV*+yxV#zTJztUdAFafCaW}kn-Uimw zG9c^ZWn8*m86)?MF?LHvuzb4|U({hbT-8_rSCh19fxr?_KVl8tqY*G6`TF=0wL^_2cbk(?hgsQa-R9E7ljV}XXyIm zFci%OdUL@Bko=JiM_oBC^|u2c84w9aEHc2%Ntb`7J(k)R_QIXkbPP244dWYx_%A*B ziFI5m_$|K=MN+0vc)*(aozzDmfpfTJwK?pX$<5tGwsf-7Ou8a)E&1#ff}fZ1P^~}` zD;%YvN}vk9j%Pvt?kQx4qc~nlw_x@%m)MHKH<+#&Q*l7}1NWKK!;Mv0WUj+%qH{%x zU;EbwrTrE&o!1kwlX*!+J%xF;-Qm!lSV@ZH9C6+v7W0NBVY*vJwQ37%nV-+?0yzJM z)}u8%rwDJ{U9ZYp|7#&BGkwOa_-u>fK@ZWqU;@++^}&;E!8D-E5E?5UQhW9X8>&dD zMCm28xhcq7T6PYK0@b)qvIkthgEe+0t^w<0U-bJt1G|6e(GY$m``7m*uDKaP299LY zu-7HHafs`b4bQ>&?UPB$-bTU;)FERFb@A4Zw=k}q53&#ASU-Fb5-lNVu`gOJ_ z=t(BAMvhZqL}@m3hX|3&M;4Ok3~gwZHbve=C%A0c2q!~1Z7(Ap&y;pDelO-@R9+e^ zaFXC1XUS>M=~@r=JvGE& zQVP7&7{P6}?x4#SK-y$e7|1h$4~Jxs*R%`jOfA8!ya}DErSL&b8K~#W;-NYP>ZEX+ z)Y%kP+vOS3>M=>Wf5Kac*4adL7qyaZ;|R!>3xS8N3?4{33%&AI7<+^BT&I6!qE{4> zn-50ec^ZpGstno+=Mw%_5ww)igX11zXkxgU++i<*ieVUYrhW_A_G*y5C+&y=R$=7J zq;_I)X9*Q}pJ*YnxrY35jlkQy-=v1)g4Mh}g&iLz^1ck_Bj0F0=51#2>-U>f|Mmq` zPi#aj!#i}Q!XR!ARmC6iujssm6*PG%Vz;jW=+BVHJ=PZ3eqRHlx4Q6)-W727ni#ri z7e!a)CXoB8jP^fY!FXX9{#>_+yZ!COGohupoZG=IYUVWPgF1LPA`oI%19ntgAfJ9L zBROXV*a6>L%#%ag(4tM9-WV>z@;lx%FL#I@8dPS|?}gB+zfowVZ_0sqAJF1&&zS?8 zpTOTf12E)WA=k)7V*HS0)(H#q&E+y7(`=;jww*pkY|N~xUaQH^ZVSYD&VukkZ58fl zi$J4;!dPOOiY-gB$?%L$vUu)6OgY2#+$Xn#!HE=dQ9pvXXmqjdZ$fbYT?=rroq>7v z6*Nm?1};mVho@>6LcIG5=yDriOWymS#(Ym)y_54#a+%CGYKPhP>A&g9v;No+A;B`K z66hfEjSh2qdH(u9&@9PCz3faJY||kVK5i!K3_sGDA6}A@ULDA~+6cA|jiflH6_s}E z#B+1TNK4QN3Ua=ed#)DLF{sAUHl+&onMY8S;KfAE<`lggltg~pI_ug=A zWgwL@Zw4!inGnwDUW3!Em~8cDBxH!|!LFl_U-u4fM<&zHJ9nKiy^%1us3Epuc7ZCf$t(Q|-Tu+r)*S({vq5AFZak zPbdhiwg(0O7JTR+2}}F@VRW?-eDrj{oxHUmhH|js{y%E_NdWC;Swh9{)1V@i34#L3 zyd0_R_&0PL_!jR2k;RjFLa-e6+>byfHU^G7&cFce_0_rDb7|PK9gdhi#Oao5kd&0b z%C(nLDUZus#!s2!>|5Y{?fsljF`+7PSQhFPcc9Z2Z=&zS>9!?pOhW!tMCO-y6q?h+zB`=`bG#oDlrdk%NcMbeGVd#Kg66snsbLmxftrp7epr*aAumA%^}sfx#szymcE-V8u>t`hHb5=!bM*-_IY^JYtlF_)j2FTwh`x1{Qy4g_h9^v~9Z^-1+vJyj}8xQM+~l zD^-rd`QxFmh<}}Q$x35b?JWM?qigA}{{PsWR*~eOusHoIXU4JVQplJ5C9wHm3au%7 zMM^k+>|#A%e6|0)xyYF?$h;u}TRZk~-KIQv<{b&w9B&glKVwJ;5`yPblc=7J1VguE z(w!5H>F)g3)vdlmOsBy)h*g`;43qQZt;P_8dpJGsLM<76dx+N6$+1^%ctBWDGjZm8 zbp^)%q3HvbJT<%vGND(A%&}Q?%I-Wmutg2mIp*OG+h^GHwi~qTLTOBCDv_^V1>3*g zCMeKO4j*r!+!mhhUscFPJHLULPqFm9^dYFtv_zYb7E&Dcntfcg1ilH$!uzJXRcR65 z;Np&DL};541_x*0+>{nZqV6xGZ8ayG2Ysufve&?VpD6Bs=N~B_YNLLGxvXGIBChWb z$BJ8z$=S8T#Al&1slQW$qnR(czRnnORHYNwJom)rm9C7)mvXE)JO}UPY@%)sitJB? z`I!3RH7UI+4Nzr3^BgAP$*EKL9eF}@B-jDps$YZsn|vWq=^1Vs%!A8!_0XV6l*<*G z({1`Ik<>l|g;nBw_5bv^UO0C$arU0-P}fk>;~+zBoLoW9`(1&V%k#jm{}o|sIEG93 zbE5UPpQ@N%MeF=L(0`hSCyEX6!Mb}?c4UBju-Zua%qr2xeH&V>+zH2wT(OybPsH0N z;qfWW5TvvdcIF4+?Pt*_9BV-R=Rbpxo9Agz3Fq;iJAe=VtHz?VGek5!AFogRhQ1v8 zWzkA+#^m-y{wJ?!l!~t+zBZ@v-1rzt=`h;OYr+P4UuRg zE6|J%rOO*HpkI1CP2oI0|5+HrkB$oJb&dPmmq2^8 z?r6n5Nl9>ILLR)BdV%dYjc{#(9hZd-qDHE=u-Qf!4P9)p?aUE)e(p3R_~`Sqx&4M+ zo)~`*O(#;PZsMpzJ<&Rnh-Z9O&}Pz0yPh?}3I#XPD4vH(kL3t^Xf8(mi2?z4TL^P{ zL_ZfDCSq;b^p?LkiCo=|_3zr@fJi)^UCCuKt*3&j=S�{Xw70EAkE+S+czW@?`0e z$E18kG4y|G1JC92V8!EgBsFj^Eb`anS*=?$V{f94_?Dh6McFl zSQ$OmtVjNvldxa5pG1@xp{DvM{lk~#yQYVdstak1PFNULe()z+RlBe-uL`#_Gf5U( zjN$ySgfV zmgEoHkfQt&kW4yUW!)>uWA~oJ-+wNku2l!!xw#x;hvR9&t3@zPoXRg@f6~H- z+tEBki@5yd`t0|)Li~fNu%^8TZtF~-UP`S{(&7X4=S(prx0UuC@PUUCjS&7i4a{xK z;N1PqSZI|G_9;KA<$i79nD+fJSuFxw@-86jdXuQ!I|a7%6^)UKCc338-CTVbZ~Q7C zTVr&=-1s4EeZvR6|3t`itI52icNq}vya)%=cabYK#`yBM4FzQ+|D;D#hw%Q>df<2rWSN&495-r$DFH0<$6IM?D8O}#%j9TpF7# zbtc_Z#!3Mm{db1*qm+Wh8z&TaFMpNmn7ZYI z!L5_&Eint8V)1EK=k8M!YSf`G=Y3^I7R0eikN3gbN4n(YF(>e)n4Hcoysbi3!Eylkm^9NS)%jxdMB&ZKe!FfAvAgd{w z{5Wy|wxyP0wV@Jx99syr3V|3hA7F-KD4bmMj!Mlqg+;s)_R*R^3zrS~wBf&3RAyKT z9aFqkqQzUG(RRv|7mGXiF+$+QfDzd2ea~C~#?je|i=HFn|n4SJ<}GwaMBgwi7c%*f-@ zknyJwJ>OOUGcBIX-{}uWFBM@@#Vazhb{@P7o5Al@A$T~&58dS&@SMYZ$c?C_7C}Nh z@!>Gsc7n_4TYX>xPfg@Ezc)i^>Ww=mtO4K4AIZ9<1P#~{@RQM`&AVlIS4aMUdy^zj zqSupLvPxkp=M15N>q+ok(nqf5hT=stPAVTwL{-j5c6Y*UW*{ViUJuV9rtYEW!RZFU zTfd?8gf1GeDG~4A`3eVf&$Ak~uW_bg02XMyWw-$bm$4#i$*UKkoReTs$I2*6-iDI zGsOzc^_K8%zZfCt$MuAS2@{RK>#845L0lE;!G2bq4AFJQ5LMF5X#FlB$GPlwWtu(B zZ#BdXrME$3*I7FKdOm%&SO&b3mVlIrCOIuqPN9~|{fsN4?MF4-y=4Qo>qlT!^GCe) zCK&a74S1uuukfPxETXk6h|w)BrGvQ~KXJY~ZLv`xsb>FC-TGB9Q=jww_#A}4m51Sa z)aMSHW!6AgX8iA(2T0q0Jj%W|s~$>~_PVu6Ra<|BGYmq;UNc zcJ!-hJeLm)fwUrN7;tAY&wGCtQL!7R%6+}$PzA?lpS>8{+UDW>DrIP#lY*m9 zqnHsdC)nWpiOf4Ihe1cg!ERI;Mz$+LjrL3OQ8NTWVtn9f*gx<&Tt>S0azC#V=P?0M zq-CfBuXYJwBj+863Oogt2FGbnn=VL=hH;&ZZ|U}ikMxgo4T!xZQ#_GdcdmPmt3Q#GSUDaJ}q3Sznd~$D;)K)t5TCJ38li_+ri|+O)wvd<#o| zwlnKFR!QEq&qV&ME`2awj!ot(;rqLJu&~1d1+N9dP33Yb^l&=!Z?zI%D83pz%+BJ1 z+so0oF$GtrIIx{rrsP-ee4Mj=BHvc4iLM=IiT`^)V)FPA`5M~@CG{$N)y_TmE$=;i z=Cbzsb1kSuog5u`xC`BtO6kYk$0XG&kLVcmR`--1g_5l|LF?-py7i3#{gzpX8iP^b zd71&eEz9`nNg<%hWzDz9m*FafGT1z{ojfS;#L_1p>4I6&RMsGiAuj36HqqDQh-U#F z^WFeo4>ZD{K@_b#ZiG7GbD)pwEKul7fEfFm;Jx|~)wkO~;x)RNlMmZ0J%`otl;$|+ z87jwP^|x8SM``HxN{(&~NyphCQ#mh}26?+k1w;&{@yIVjFv+3#=fGrWzNLhP#|I#< zxD+#fQmB3*&#T_n32pj1m~o(xtU5Ogf(L)VepgZa_v92FjFlp$fiG~6yCXVF+B32* zGO;v2f!_A!(SK_@aI7l`mpmmn?!_@VcaMYpm=W)LWik$V?|>tdU17BnrGHBGcwcY) zgW<@9D1LYtr?pLh^F~ieMRzd~G;)II)?gg|GKqhQZ$pHmGjVt3ERy`w#`3w>Qee+M z#G_i%_%}Efbcj?X?Y(xMR@G%Po3nDDeK-|QJ>hycM-7SN>_9l!wuS3{t0WtFALy{! zWZs2TH{9CHGLOHh@Smmokc+=m@$hP-xtGE0m%GG{SKzr78GDIW)ZC4!iG zEyw$7hYi;&XxfDlcJ@w9)Y_B*AN|cij#_|R?N&N^t)1(5<@$7QP3E;%a4b`P4%i%B zPpdgThR$d@Oo?@%u)~dG1}p%7k(2mlx*+eC%_cmLFVHM=0w}l?VPZ}RI4r-2JWY;$ z&#`(=&qzh%7*{x4@&&4A86nY}%Ql=}53J~K`e?#Tl0KgsacUoT?>nvRh+DN*>8UPKfE=7MVH4+!ibsBblixBBZ%tcxjyB>B%& zA*L879Tx@VYE3k|Ey>s3zXu4Jf+2#Vxifv>MN(8qQ0w4r+`c7H1-m3z~$%w;ufk_v_0R{@v&&4-V7 z%s^XxF`DG0utHz*E&n-tK*Qq;s5h|+)2zLiK99w;{&E-?`N*R8jsqlFkJ~wOoOy$_ z)_md1g-o=T2v2%*E`$i>v-cKh!~83gFDNcJ%dC!uTziVsdy2`WueXq1 zzm5|dKhv^1+t6XU0~!9)fKHt|q3@;*tbe1zOBI-n=Dy2$T68)cTk(srUabIg%<4$l z9AS)8i-)xNbMW9E4`@5S3x`fj0?&oHxM!XeL`)pQ(tYOCL^-u`7+i@3`2;({(_ogv zX6iNm2y^=lh~jhsj&UvpM&f~}rZ^3I-6G)8J~w{gL2hW%xGKcdDUEuW17B6D+`W89n>BYVh z`3w&dWEj@3i3(Xg#b-&&Aj@P8m_K;HPR{cJv4$Tsd;2oj-1-BQXJmp)kO1$R&l{R^ zN1p%rxemUYdW>{VQzNC?6uu1I#VK9Eu&FYS$xO}01uA?t(J`Ik>st6H*-d@am*diU zQC?T0HQzH!1EoBdV0WzmU(n|eKJ|^odbg$E5ncjeDlG7wS|Isa4fma;MGP7yfU)ib zbi8~Nn!iMVR%ir#Rm$d=em8+t6~Ln0ZYqB4JeUP%aa_!5da}3!FU?rM^-jGe3l$~# zfxEc=*pAnD^T2erUEPV)2BkrR+%a&N-NN3y@R;du7bG9gexp%KyzyM8IwT%B4|ev= zFvqcqNmIE_em>I#;gfe^ORguht!|=WfdiJNUju2hpDR&Oub_jEZxVs1{Wx@;dk-Sl z>CyL9Sf9{HmwvuMEG16keECE;dgK#v>pI5x{))l2tl79y8^HHh6xG;~NPo?0WbT+s zRQKrxVB5!L5R9s!@iuR;|6>%@8~aJ6qy<4)_nqav2YqC=VaIKOxb zT`)TUL{6!b>a3T=%hQ!Fp8KhmWB2VrZ`p754 ziAT54^1v9FIVa$twIoK?>cNXo;n16HLbuyX@tbE%#4pJs)FW4d&%c?3?vc~UNLmHQ zvin7!HLeE}qf)9CEJ*7vF`zhokj--}AYx6y%qXvo)Obq5*xq!EYIR}6HWt!(JxydE zk%OYgV!ZncZqTDQ55m~8>uh{>1?pOgvCnM>@uZg#FEaf*zH3p#kKq&fTYfmgTwfXf z-!+lUct$kUZpdVJI~u|HhXbhRmyRdDiSo>Ud!t$J95Q0yicjq%_&$GQh z^G=0$H>v=tT=Gfo5>Ym0fj9AsJx}(wp2gppr*WpwB%b-)JAOW>a zsM0Tm-pS>#MXedc7OH{9B2j+yjWbxQ$b%!+A(-vkj6e3z;=$)H(E4h$QESMuuvwF1O-k{XYn$Qc6%*;0u@g#eYGgtx zy&<*F7q@KT__a$yDcY|@;qVxGgB=HJGij*5#(>|u3Glu#lu9KwkrQXMQH-0>?whwT zd2NdP!DX{(n}s%<2$JLPc)EapJhv2EgHq6X!#kMu+y%-z%)yqMG4FDAiz4zPj-Gx$|b>JT?Q0_L(NiJ)TDr1-7X{95yMT$Vh$~+E9r3y;F1$YnK)zN5B z7uO2KGiDPCDqlFxpy${;HmqeE9lmUi4YLv;$>1ZTYMYbYQf1`T#$3iRW)5E4uS~BV zMe@mgDH|!;3`0MipfEWdmX--(_lXU7!n>62S`>uee`Ij8wgSBTaT)9SzEO{Z5%|V! z9x8q6K&6;6FqS$47Ou6pjJ?hjgz-th$sYI}Yz*m=+ep@nzk~j zRJs6zmbYMnXa(Bs$YDjr`st(?FB))kGVl)x^WWDnXrp9@UR*!2vi=HkKaR_aL<;a5 z{_)7Z-wibQTPiLIxx_p@bHUQOVOsBa$B1yJi>F4N+5S2#!*Ihm3X9y43R#!H<>lvaX^2;K z#ba$^q;L_p`zT}bA> z2g(v*iT4r|nak~RxY?pE#2XvV1z`D>lkoTI2in~u1dg3PIB#_^&fk6e%q=hgSLFWZ{k6jzNN&XX}Pr4gN1HPgEO6}aO5aq>rTF>?Kp z@Qe3=SVMal<$oe4x&raf)Z-+%CyivLjWKGR->CP+W%~U=JJ~LE4#Q7R!sS8Jz;1W? z|0p^SzZ%~+j<=Kc)|Ql}NJez->&T2!$gC(SB2to2Xz!_|Qc)`Hf!2BM>m*7-30WbE z%t&9G{GQ)m(0SGAdG7nVKA-m+<$U(hDeN8TSo2$UfB1#`d~b;3C+tulR+L-(C;@j* z{X^t~2FRg-3xu88V(Kz+xE{J0PygPB8;S{a;|pTdR{^e#1E2N1O>fm2lfvj^C(vqn zPCm7)2dFzkO2fWWht(~FFLM09 z{LvTmtyLV^dN~75JbryGu3KMFT930Kc{x8s%w&mu)%tHCGF> zWS7CUmSJYZH4UB##gbhr9n4e9aLBcM2wnx}N#pI=WQA5Oov>=e)mPTR!lHWAjeAYP z3-=PSkvh7quZEMM*MgqMMu-hphu`1ihmWrx#9Ll-k=~w98Vid+(NK>cRI?HS1DZEWh=i z?rl`_U5+hm`zHyuy6{WH7 zuVLf|%b?5l6{P>}V;Xn66vW2_VTDB=PUh{w`jks_`qOiyYts@={LeOAtldd1Q_nD_ zKU=7)M?WV=F%(2+T2i&Meq?FKY^(~gfu%yl_#pNsul;)iDk^!QQacapZEk|`oHVe| zHRcvw`Nz(UGjY;{DY=jrkGw=dEbT0UPL(C_^7T>BP7J~ON%A=SFOB6!De&j0PQ}{G zO`MO4I!sss>xEBM!hLOo3cOY1e%84~*0PMV4MiV0#vj#4=M5uxVC_ykA}({%k)k zxfMs!Zi>LZ`W`%9v6cEY?4trxW$}G&e6GRKHT`g~r+ zi(gEH$A9Rts2(ygjS8PSL8p(sB1)pCP-;>d?s5sh?}jFf=R7gc77eE#cL&1WWs|t5 z;7C81|3ZO^)z~|zz@OM7$k$PlAsnMxDz|D8AbanfvoVTg!EOWRns|s%U4l{|0uWt(OsSt+-{V3LLz8<@tXOj@0C?Z$WOg@MT^LyS1&^_{V z*gVSx^xAp|u0Q&+$jr)zLX|14e(a2Tlb0}4(o4w0t4bUb!Nf?EaGEvFG|ti*l&=o)~8JhCNUqS3~10D#>H}X}k%2KJa5aXmI;Y z$nkti=hd*>*wdk8_N$?mtF;dxD10wmB@;w_pr3wkWHb9aG^y{-O=$gWEndC#jTF}V zqQS}$vaYh2JgGF|tLv*l*ibYiY$$-YzLZp@&46BqZgjM8r2kos(pvqSw9zS_e7>v+ zZ(rNstUEhVx9SliQmaF3H|*nyg(<<{{0U}rR1;j;V~#(x52Ll-aTK|;9K$6~Vyayf ztRpSd5RcN*;zd~3{1EaM-=L-@C3N_9Kj(p_3mmNciJvlM(Y-zsr|!B-=UD6~d-p8G z$>Nq^ZIA<6W!2#OD}rY3*28mNznB82%hWGE6SQZ`LdAA{Y?<4RW709~K1ZKQ*B6ph z*$HA6r^@*@I*H#Ye;v1uo=0nQ7ZlZcMz(}ZpzWR=bm#2b@c#C4*6$R7iutQZ)r)@o zr{fB<0w?2whf*-RK@S&y{|c_Tf^?x{Cc3q%qX6qJbh+C{#Z1#U!<#$E##gNn`gJyV ze#r-?EjjGWqXv^xi>wVVrqktnlpwan54>a!upH^%M0%+h$NpC#=lEPDQead~JVgF* zj=Ad7__sXje=i)xKKz8FjWO^o&;}9|+Te4vE$*Ml0jsQ?XgY&O9say0hRG~XPNoF1 zV{d>|e^pj|xJWAF6s(=Z1HIZcRe&3dKBD3F_ z{y47&Eh3(n8)5-5ttS9WDEj>j#-=;9txKk5!ei&vw6rN7W-V*tjabTAtCjiF9kI*l zUCkL}o$occx91s0>pp=^bAG`F_Gd=RkK)e2dqhClgI+hw;4C=7-r-kV!XUGc%>9pV z!J#P|{xdy^b8m>zNpwD$8j{8l`4tBddp%+K(W{tN{*Y1hO&|}r%KV$Ub*Nc1#PmzC z^E{Ki)N}PA8ZkEv(|>58@TegFh(Q7*PCdkEI*Ra9MJw=UYnJu5A9-+hv=`?6lINcZ z+J}$+24KgZ<9Nz{6DD=ELc6RukyzD63MKM+-MeZvF+v!aekEW({N7bca@z zS5uc0zsTq+DOeEs1@!wj;)>~!JfY?8Xld5VcGc?8h2^GP3iZY%EA()+UKeyEE{By} z##CXJId9U<1z@pP9@qNC<6+hjsU^D~uee;`9o_SqwzqVGOP?S8@0KdP`y-utRThw! zD=2PSQ9|r^9^k2{jcW(X@%NTySQsl$56_n6^R)~pcYHCk_*@-^eba=C@20Rb_XK+Q zo*o2vXTZx)QF8TG8GSCK#qr&61a9kIpchP%$zX^N23(Fst#DEPy&V>GNy}BbcDWJp zv^R&)sxt8USV~lK7h-7_+bKGpjq9#BVopgib#YW;zmJ=6MJmE~H37cv(#hy%RR>|O z1#w->3VKMQ4W}nraD#5UL&T<9;xzDwfRQLv8p`6X+$R`xQIamZH<_!~{*lJ5vcd9* zMJT=K9(1eeVD;{7Xlx>6!yFyRo*4}LC7waZ;{uMYgfR)w?nc26E%bVL2`zKb$Bmzs zgT)#NtbM{qud2DwF)9kTEA*^)`o*Af_%5V3Yv|SXR4n;!LrEFyg{=`oQ;D^l@Kv`! z@ydRxaB2xGJZTIXoyja;SB!tTcoJtpqCT8VU5dX&r}AgN+6PK`7s;)uGVp2TYz$r# zNwl}9V((i6x>;@`eSX9lqu*}Ei{(qOVfB4x;ITf6NC|P1N^Wrkq=hi>rYQf=Oi476 zq%6B+01uFUxOvl$%{5B0-Hj#4IE)g(JSn_u(ndTLv(f(aJSd>f%x{hxs#qFR@#i)0 z?x!oPH#3LsN73NFFBH#)*5Q@#XV$;we8y4Zo1Ds=n{Z1|2&z6jBqKq_Fh5q1PU?$8 ziOJ8Y#Qa5^-d|U+yJLbFTQs0cpbIgJxdxSAW8vO^rFhltAn>d@coKES*k!2-b532L zA6mKW?535jSzHTJ5J{)%5GuB%k?pLzgR-nWIig+H2pjb|>R3-hywac0gNbZDK0#-izX^F=(?SZi@OYzHqcbP?VAYzYQjvxlD} z;tTHGXW^8A{#2Ynw@>qwFD&~3CVdsDiJs2?;-@LT|C%G`1KW2bV+FB5OU!Rj+ zRtS=o!=z_mGf8{*5TxX^xSw7}V_l*nI2b$O(S;{rTSq4h4D`_T?oDK+R0hiw)#aD? zN^m1Dvb*T-a%grj7XuZhL-9@t+RZJ2LY9B<`JNw|o(l!#x?ntV&JQIh< z{m43fVeSQKQ_`!Y%>38nfgw9NxbhYk0#4m;JU@APLN_ zqZ%BsWu=fm?LQJSb2;e0E1@I#$3f_xIdqoqfn!05pd@}4g+jDwvn-`bLU*b878RN+ zbdYB-sfXOUD1$PZr@^Agk!SZz8y8&4fhp%}F(%K8FqJ{H;d>-Al5`QawO(daZ3e*I zdMUbok%zR-ujto*ocOqvfX`I+XSZrnmAhhK)ust+6{NxD`z`kVBmv4NtYDhlFZ^ZS zjca#@LVfRg3}*iy|Jy^s{D~A%lahfr)FP9dT+r{iFBTN^lSH^oUl}BljqLmMUw;vb zZ$o@=g7x^R*25%Ewwv}c1?QGiX8-$8Xr`uUs%`|eS60I2S?Nsnjdmu3Wt4rsE{ppr zPWX3V)nfAv1X+Nycd3i37Y`JmQ`K{#46m89LPge|T1Ah4%`xa@Jo{imeqqTEGH)}{HV z?JvZg?-_;bqpe`XFp35nUjVHuSD>h8J7|!6dR|%pwcl!DtXe)SSe!@ug*=h(AOxr6 zO=y<=PW&zMg>k?4m`EOu0oe_#*QW0;S<|BfT`b?EThSA4_qju?Xf#gRo`EKs`!K-e z70zRGsz>Ji#FHOYSWnM=lDqRCUE*E`+LBRN@F0dIFFn^NGH=EzP~ywH$vo&%gtl8aVE=k1{*NBIy_JCKNfC z#1t=-#7SnUC_dX6H>;@Oof_7a=syBB+fR}s_VZ}%n`Syq!htNd5Wp>$9f^VN{T8xf z0B=sD6YGCjbnBL{t+GaKRN;#qbeC{I^XgBSg+p|I?oxQ9eh8;eN`)g^w&9r*?x>u4 zhgk9!fQ{-SGB@fa(b|yBj0D%?57!$I*CPq@4w>RWyE0b)+=<#IL(K57BFa>MC0R$7 zLaf3L&WP$a@@LSDdw<1LXb9$$=~sPm#^FemGdxB7+1huXbjhEFY3@Krel|hn0+vhOYJia}%g$=#8j9*@b7yln z{B2Pb!w$a3o&iJ9`MC@mJXj`OtQ&4I>*p*|9z-{ReK0at2+LQ+LHmJND3;9y@rRjY zkDMSC^f5uDn9Z=vyb=8RC>)TIg|=CKWPz;%*xn6btnwS^T!TDv;>1EcG$;lqE~vr9 zb0Op~`wlXGy^!t=jDP{1<9IS>G08fFFoh9i^JYh&uVEwTd91~CjlH0@a2H6M{2}i0 zqcB~>7Pq7=Ae@OFw7=j3+V&mTbBdk4gpbi&Hjk6ul0suY3=^5JAoTZ^s<6YtjMw^Bk|Pz??y83XD!=ftfeAP z-r=P)LUh1kGUU8zgMzxVbaVe@8q;))_{?o&Hn*-t(t3ciTW1@LPo|3ATtF6|iv7~$K8r@=ZivCfSV?sP`5`_nKWK-ug4Ey(z2(ulxocYRl@|ix( z=(vgp&d1E zzSts?+q)6tN2cNMu@uy>)F6;8LKz7a;>mW}`YdI*pLs&u*X-|Z#wj+}qZSRn)LpT0 z4aYIIh1W%4a$!M;r4rH6klCIohONsRd*ucaJDCTTYCsMKU3l!f3<_`Q!=FI zUWcNezbLFKF~M17Gw`vh9sG$ok3=sDco9j&&tn!2hPZ?EyDFF%o)2O_%y6w&Jso*Z zpoh&|CqJc_>F)=a%taj|AE#pfmm!H{Fwu7n1t6!6uMs1-cxF1>zCi; zH-8h2PnLtVlW(BulPfq`z!KwbyjuAC#A&!ZbOc*GbFp%-2-hwAAvxC;Imn}XO0J{HrzqOArV{f1wE8@txWgNnEn=?G@PL%R*Hdq;tG z)MM;*sX_7Qq0qK+4Ow+tlScN|5&e5N=zqr3@L>E9epvsUdj9c-{dvkj4*G!i6eB3! z#)bJTH?Q)zAyk;^gP+qVk^Rq`#0{~z#1~B`rI#O|md!cXvPGJxjZEg}%oiX(pFV;J-HV*aU2S-E zq7B3=C&3IhiF|X+5mjUq@o{!K-W_Q|{3!Do@SGaA z3G-(?UdQQE$--^Fr?aj}JOG>u7MI#l(dGhpOm8OS#|zFx_f-5kw~N|6soRS9i4O#AJcnl+Tg}BO<1!)03z?V(Ra5D@rkk~Jy1IjY>!)@PgOqNY<$j{{pJbJ z}Qg9)AcOcmtMAdZ<55o}X;-gDh@MC%F^Lpz-Z$v=qDof|D&d=`-gM_fB1~v;(xy}Z*Miu6)XW!I)>*&eBkI;3I0!O4LGHqNH?Bh9eRPQ zVC^L~w-hoJcT_e||EG$eURX+tA6_FG|MpPtf{Q39TR`QGhGO(mVOZ{5!8nP?!W^a5 zaJo{G_;M7W=1DS^t=ES7N7uwS^G#4o##H_cLs z+KxG>spb!oF9Kl-`)q6I&nMfKrO{r|i_lwO3!hzuxkp|}!4;7g%!X_wC@{TGj=x_H z5s^z_(UW=}6Yjzsbzaa+L=^2WEIx@yvdPnFCk{vl)yoD8n(Q-``KxI?LbCPtHB$^R#SOc%ztBzgUH$k+yiwPK|6nRSCHc?0fpw z46ct@JbIs9Orl%P(wUe`{yY7ZxWW!xeJT$^9ZKnqg#abCoggX~3}fu)IQrCr?T^dB ztNYX8mB1XBYtTVULL)%#MH!Zy>LN20Qt+VnAPsg*q`%tOzc1^J6^Pk@KDOQHx+8)d z%WQz5qE6^j4F^Z>lW=(F8P4~qhEP-ft+jDB2dm^PaF6IVxPSRFSl#%K*1H$Nh}dqp z@`61Z?z%=z*8E4#aML;3nclD@$R3Ah?qY-#FM;XlLSEJSADrnQmXm|_%FuAC9~x;i z>yEkx8QQxbeAF3S?#w_n7Y5$^QN&M^%i!hi%VetHRNU>vqgq>DqpCtI`s<7{+HR55 zf9h*ycu0}{SSQB+zHX4Ss^LDJ)vUumGH)3@`c9PpbbAqHmdN1?;dCgmIS%D)C*yIn z8rQWg3Huk1L5Ei(Ub`kiqnn&aOGg~Jo;8IedH6$kdjq^O3nz+_HDtf=e%4bggNt_d zG0^;h?6t1t9A@{6DN9vgu^bGkqNp^uwC!o8mr zM7-Uee(QS1GPk6-_JL0DSuv3c$@g)B?kvRHqaVon&2s$zvdgU9yK_Lh`6h-~zlWs% zn#q#a40-0bn`6CO8!JLHs7p#FdcRiZD$-``oO_LP|!$B)DFRW zA5!yaKAeHK;$-@-LRwHfNcPWuhM{H-{5dO=@!4!e5YIKlrSa_DGN*|gSSiZA7NA3y za+^@}OgbjU)R3uiGW^e%tEtb8SxA40gXr(=co%GNY@(F>yl|exe-zNP(yfJ~$g#F-%zkxacII;qL)v=j0@2c7P8Gy0s9uh3PnSZG;}$ zY=GiTFClE;8F`!AjS_+-Xqe-ViEYJ@a5A5&37;YXtC!*B$p6sn`C0JIEM>BMui!7< zE=*1k#Mt1S@Zs+_!VQ<_3o0Qzck`ip)7N3Z={)M}{gpKTxkC$t-%_CidGPX;2da)Y z!>Xc8&PVwJ5Se?SRWw!u^xX8QZD2e#D%ei1>$42x8D-!zmWS*VkZU!ljv{;5Ec)X> z&K=bw_`Grfy#3RQ4bnCsuagd0$8Ay81*x;F2rL>9Br=Pl&{JX($uqbPcIMBplX9&w zxe&uX6`{s9e*O$bRK4yUx}!)|Nbo+5_}?S8@=aWUBbD2OS3 zGJ5Xva&3}|SgH-yh#pUl$vw%m&6_<@6m+PQG(z9NMl zNv_AkFx2#DV0z}=Vbu36WP73^pfPz3%Q<5G-7RNf;ORNA>uVu@hC}Jur^Td3B^kT+ zdcc|!Cg6DI6_IUHMh}+tC?9ebxallysK6OKe!5{&?n`*cOD4OhAf2AJ3hvz6iTCLc zz0jo!7q^a6ix*#sPx+rk(Ysw}k@*=4$7DI~Y<8B^y1^~pZDKl<3o9!gS?hm&MAd_D zkyWuP!Ty^dm~+EWsagR;Sq|l)W<#LHwM?|x1N;Vupp%`o{o7nmMNG~JaEHb0Ew z1H0DfZ7% zXgm#Nuj_Ed(r@DV1-YPpUKUyeq<{^&;=?#enu$D4!p#G4R=y1%OZqc#TpA;kr!me& zS{ONW4!)H(Qhr>5^}7roPKBNkcu54oC3#19#&!xPg?=R07sP?zyhuO|6|U^2)tEJ3 z2zKV3WO;1j+?c*sROQhbT0UO|roYy}`M)wL$J`i|16pX5+%L9g_?B=NN3*QlItYCz z2huBrP;6~9)7?@{S{*NwvvI3ge?lo$pUpa5l&z?C-Coua&9&Tdi zO&TJ7M5o~fY}~sGy;gP7hub1>P(2R+!dh4u96+ZF<&!h@@woZpJZv4D#7&~l$d{;Z zq$g?$>q%QdTMjNJG0#(=tU8Ni`%7^zSl?wfCzO-FQ^ctbZlr_#ci`&Gx$wu^hKaaL zi105b&X2x_V18veURHoAi#?C#eQdQttf!rJ{g>x=1dlMN^rEs#W4TIY5K@8iTcY#;fCWY zaEpovjr33>jW71m_m_C=ovj*YT)lwF%g;k;MG?qc1$v(A31OGQ@Ob1xq}uh^C+EhW z{F3E8thR^KN$;t{jc8%@$tMue^09*W0jYHMmpjf;ghgdCy{P_ zY(+ebl;HeY9enA18K-C#k{3$t5MPnHNN#KZ4g8Wh3)@(44gm(lF!c0;)IdI9aLRLuAUekzbq*3j8?u{;3!wH&OCC@ggrvC>QL( z9mjh_VItxn?#?}cK1AAe57*4#p#qbGF!+lXhfJ;ArXlI+;7!T+sunfkt% z57?u`pK518Z_VjNIe!n<`5X!PSN@T7+hs6z(~x&0KL-p#Tp{MSIus5#p!1gd4+XfVpFAgP4c^e<*b2Oo-inDS`fxPt1o)^{A%E|7IAXdC*IhjX zPBb3bx8&E(6sYV-xL4` z%g$o~_Z2$l+fx5Yjwr0N5BAyC)9uU({9@ul$4>L;>EmtK`{FXSt(t~)DeK7~yRRw9 z=ivE`88qd$pT|8VrJj{{V z)CzgYHuOxBIJexo9(3#D>0gsJaQ|t8pI@BCvq7(jl&2nFIpZO9J|j-_zBdxLMg>%P zQ4MKHW_&?ESzLDbFMRnTh^H&FaM)IabW1u?{SX&y`PW4hr~e|x4%t{9X#;;2{H1n5 zqIhez48*Wb%m)w3aA^8RBH4X|`ug-UEe$~CyibBnt7WLbwRsT193ix5oENgLht0%A zU<%7X+!tR)s((u2Ri`L4U*d(oIkISbSr#8vPe(HoOZwBZ1b9OsL_lvGBvsgs-$c(=nWYmFiqWjxYJ))JaY(bi(t#{FVvNmvWpXB*8Y0`G2c60a4J@cG5o5;Zi$;&=qr?oKmBRku?9q|!1`Ley(Wo%Brrk}Pa zPR7e`#K|{T3Yi7!R4=p|i>>_NF*6;ecjdI+S{#X+J2b)U{RH!Id#&|ur)6kbbsdG+ zneMCYCwZP*tU>0R7hJY~1dASeL1#lExu&4Qd1f`lpdiZ#7fwSxeK~&Sic?s)!W~j} zpT`sTm8j6?B((m|0&7~+82u@0v3|EbeA=>rYA>uscMnb6_%f30&N~Xfug}D ziv3t}yO&wkor2*Xr|=&(q~nYJN&J1Hf#`AB1it>ufMTz|^w7WWlr2*epN-M*&@2hh zyA}cDSi?S%w>0be4h(cIC4!3<12gW0xweA*DSk7k(=|T1{>u!0%v(r5SGvL)pJDoo zWYEMZ4mf(d0$#1tMBFloL9t8>Jd^}c=HGDW&j+&Y=1P>`-$(P_&u2tyRp9ByWHNt7 z0Xa9c1Y66`upQf6rfED8=UyL#lo!7^!GSO7Be6?3f2k2|VE1ZMC5*W8(bq`Cx_v}k z?l!osmjmDDF4(R96be2hkdAxPaN+rzbnLGL4*r{}OLw4^u{0!7csy(2GAIIR7j0&@;Eb z>nX{<-@Od~@+(2zmF4Gjor5L8rkJ3tj6uugLF3wQa`i9+Q<{fi4A0Qcz0WzD?QWAR z+0(e6_PRh#k{Rosw}#lK&DJ9ce&o?|EvU=?LyYE4p|aK^bY?&--qmnjZ)KVvy|5Kj(u}usgR6oG9!5Um1D}<6s4@mw` zTUe*M8iGBOn3>l-h}gwwylRt=X0mU<%5ye^SyFPnS&{EAmWs8y8I$e$)StRMZ3|pd2VR3{3X2jP=;2@Db!1BS>)^dcbsk5$qMcl6Z!H$n)0w3 z`g}EE#mOSl8MFqYr?WiR$C7l~iYmNzNS4m9{79G7>EjHm0@`&f6n>PNphEW|e7{kP z%=#z@tyQ^j)vEy{R-b@BHJ7MG*k&F*Y6Bl$@MyflN$lQw2vTNT;rM<3LIRp&aa&{z zevg;t^LcTk$|0P-jBtbn&sk3W!A;Pwasy&wBFNf7S+=LIMa6Ra;QX(0-ix*UR6)(0 zDoo3Q67R>HVq+`L1AZ{5#I2+Qe!m$}=Q9{8+e%-LUk2-!>2z=(!(oC4aF_8Na{G}z zdYwK-jpNj4m)~?~NzUN#-(Mi__g%x8W9Oh$a}VB{>q8{X?=z1I)QM5|dm>?AhG#{R z;lD-QlnE|mJqz(rqkRDL)1B#<*Hk>DeV7a!^P_LO%E2yPknbgJMO?#L$sO5rI(PqD zT=3=>;g(aZIWC;jgS&gUXtYmZFoghE<4foob!J@huT&Jfj(?4PX{nv99KFHk11y_PW z>v0!8n_~cp;HUsb#Wsaub;t6Fci%RiUw|^Y8oy@_U3-G(#ia0xnl7~(8z!@5cj0=Y7M3@7pStE< zBDd%5Cexpa!Hw>jkRGcK%T_JpNb}opFCsquQVS)zTrj84s2j`e=vm**x;~w;N%T9F zN^>EqAAi!noHE)laR!&%cjnv_I6|Brw8B}#bWq;3iaEFX6)ZPm9i?6p+|qnkbUPJ_ zU4}{UY&yF?7HlWVt4in_Hy!${^FH1-xsUDF!$@&R7%hzOqApP-Q0ERfPsWSrw~r7_ zO+S3T?Iq08?SUWi#Z2U~HJJMQ7hWGLV7&nu@VBgz-N#?2CN{cspynRL`kjR9vFE|o zeJk}5wxg=$@;G$bE8&*!?5^mv8YI2nOheuq(!;lhN#v{PXyhJAgF`0B`6^MYe8qOKPQS!W4kZ}; z!k8RlITRgAVqB5^6J(k1I5{|C$ekPaha-0DB#A1lfgde~@FzNjbrJAzOj~zR@!A!n zBgBo?ESSml-{L_pAFHRO8Tw#fVU6lf14zP*C3L~=4w$7Q1tnEntEvL|0dd5_*%TLLR}fvf zaI#9r8#nfs&Wt`MN$ zg<|t3=*!&%L>5;v%WCIQ`I9^GvpAs_Sw{YA*3;Dt(m#Q7R=OF525af65l+g z0#;Nn2bZ&2B(C3xTyKeF;#STFVK)KpPMI3wB&kB};{jg1kEbV{E|DmbiZ3hXaE>!` zxmQ}m`OgZE!FdZSbeX;Y!|E*2&{UUYF^7Q6v_<6d@3WjWJ9SY`{}f%U#Km~Qewb|B z&;0rBVC{TsB@S~Rkqeh~phrTVOg&fu2`bM?%(x!*$f!WDc?_{N@dM|(0I56Pkxd73 z@vq}8{F-G-t-aQ8E|2em?aIaw-g+GF3Vo+{S?`Xwbs)Z+w1i|QN1-u$_Lx0O6rR&3 zP|ogEF1-wg%T~=Cm(8q`>E9&CDR08darRLB+W=;6XaCO(WZ~H^6K-wfd6Z;bUVB^8Z^1 z^9o?=Id%qjMH$>n1u#~db-ZLLL+#^OxS&`Ew-p00x31V);qn=@j=Y5LqB^laM+dIX zZ-Kv~&2&S6C3KFTB0NDJeOLV+3{JG6MGxy=Rv)G|>~7CeQUkvXJ;u+Q-<&(nF63 z8xY>{6Q1-QM2$gdwtpQ0V~N>_RrYvpzzDjZZs+_I`$h9MYD3oMM%?N67M;pD=gpu9$w-nphi!6W@uK#BW&| zNVy6SJC9~6l^VNt)yTlZ?(` zU8L+CS*&R{F=IPli{}YK`zsR~b>NZp@gh&MHB}n2_N0Q1r!muf!W%a5Vo_q{ZZI$y z!VSC)@FMtAYgxv6?l$js;_aRWw-WE*YwdZ|(MJ-($CaRSCyb=4kzY>l8%b4VSKz?=Aw6;%rPi6=& zg$1U6p~qkrJX#Te{eK>F+A130;)-~pJIRmb8#s_%yV7C%zF3l#{s=c@cyj6%>Val{ zC%l;y0`FvkQRa*ubYF17Bu+V9ykS2mXXLSYcN@-pgL_Qs4naB?ql;NP6EG)2mYb;iyO%=vZG9RV-`#^P@edgBrtMIBe+ZVY>p;Q}WxLoKT+3tT zP!zBNc8r+9U6z^lPR<|CzkbB>Fg)Nvs5gu}_{8KMolFjND#FyU6R_-F7&rfB6G$~3 z1y?cx;emdHYd(RSqF-`|O(mMW_ehOXOtstzl1eW~V0=(b4$}txV7mk%bT}w;r#4UPN|oVRKBDcIZ5F zhxOA-?l{Y7oVgRV9e8gZ;U%NRG-zoIHmbbmB=Hx*nGg49n%HD8Z4L#6uq!BVl$=iEj0Q4Pg<<) zqjhOh+DB+CJ2D3Q5w}h zVuwp(I-vc&FU-)q1P@mV()!MK)b3C&?}ua`N45VmQ8@RHye(~`d4*jIE9)fqDQDKe zx4%;08O(Ypee9vzBn6yT{6qIveYAJBCBgExVBXO{qE6qV`&~tFkMI;CYPy_Pt(Ob7 z2OB}yES%}rw!xd4x9O|27r1twB)WQt;LOexnqQ&=&P}=a#^WZa|2DuIcW%>VTVrtf z0XN{hC}Bp_U(i)l2vZdZ)=ig1Gv*?!S+|> zz*5s4(6(qD)THi(s@a05*HH+6mM79WtMfQMy%Ap>6+!ry24D8;fr@}3YO>FT`roSX zl_O5e?&6{Vc7JeUi#`axK0x>D3Bm4dU9|DRYc#TWi=kb&AShL%CF%it*PpzEY-mEurmAcqlKQe*ChEa8zEC@S^n^Bt%_6a$YhAX2+g39wan>0w$k#L9IwNs+LNHp*kH&Xlqoh1A0l)4P z=36kEX#4Uevd8Wud9JjE&Iba%m*T&`FswDS zBhsXjb8TP}IhBd{Y+D;D-?YKBX^SxNV;S*qO2wKS6Y}4zOz7As49%_wacr(I_vQ4x z&{X3>+NDEq`#2ZxJtV9^MrrO&2Pmn1H< zpWV(0nODO(wpJKZZ%P59%lafIWup2YKhk)^7G4grEP%sV^pL|;PTQej^6w(L?YDQ==AvQ;#UJXPYT}ZB<$B z_vZ^p?18s6S@Rwzl{C%cMogXN0drncU&LSYH-1t zS-*=)KC6N85C@_@vWD8ueh=E1!Q}DzYOmR7vq;0E~M6{0RNsN z&g{0s$HhmgDm+;m95anOo3=g8c;C~dI z_al|>8^^7z?6OisNwjQ*b6-a(Ng6^WqM<~Q7Ns(?ccE-DDv2Z%&wU+Rs!t*f85vEL zwo>W)eE$G`;GE~V@9TQMU#}~>Wa)dm1GK{N0rD)P$>QiNxPQnNHRJ^$1JC2Mz_SpT zU_p-YO^{1#+;FGSOX5Smp^*CrCTWH)eqo)5(pmoKU0zN0`R;=Ekp*OPLKjsW-$O1p z&*B|FYJ+!`m0+bA7rbZtf~Sl1z&nR4C)U2CvO~Y}xkhG0l+^o;ePxiyelE=hOJ(@;_ZO7Dy?(lnR8${@{pTm_! z^rPq}P6A&Oc*`B3J7qI5Ttk~eOBTAMo5B*sA-dSU6r@eU!E>)PmfD0N^&Vq7%dSF7 z{2TCpc#G6snTwxiU&8R}W+>`P!($a+iEK+2*|U5GdK5H~ZHerRGIl?_-G38*zHp;r zAJZ_wf@RLfgkj~WYKbzreRooqSDCP}^AG)_qyP>2 zL-c8MF|lj>jjJaOXs_IBvT$%J^HF;hHQVu;Uf(f@r>`EtWWke|FyjV2Tyg;~nJdy0 z_XU8jP#rfP9>Q-eGr3zL?lb*qg&Z#UkOjhN@WDf#E5Cgn#$7+lQ5@rd&*oo5buD`a z9c!Zf;_l?gv@wpRmMT=Y93Z9BTrl9XFwd#Q91A%%u;Nn>M^o|*%}_-Uwkl%OePmcy z9qW3{BTW9max8iD!gE} zo%cN^jn4SF7&kJpAiDGoF&t8$+0)eUcVZ}~d(q14A}!8x z0q%jj@Lp~yIpweyM5H+Ic(Fe^ZaN2f%_4N$W?|UzRRC76Tt{n1QmOahdA!rv)5!Dv zYUmWb5bPS~;**gT!0EPy;}PQQ&g?f{I=IWUbs z+$k!@=-^Il>bnb0eED=j`~Y|_)8}w(r(h3L#OSc>@N;v7A$~*=_D6leUyIbq(BCNh zS~?w5ADtyDZjIqc;~`S_?;REJw!yIhQ7ZE^7OtGlhJG1@$;u_P?D!Q%Zs`@gTbTp_ z5{SP(gu?0Pr|4Xlvsl;f$22y_;=7q?K$gdn+Zb23JZC4WpK`>Gk}E9Jwu5fZ&BKQC z%iu#y63Lh8W|+B^PhC^}O1v?%b<2;yW82;6bF2^bFXli`)+*GWc!&ApC#mqM zA6U!YcNJY;P|fWJ;M?w6MtIx}!!`@?tb7@KBzOxinx~^uyE*QE`j%6Cr3H>p>ce>8 zJAAR}7TY0CqQkpRLN1$IO}aFNxA(RO5Q)uLXS){X&uAi@&ip8?JV|Y=gK4lGfhYsE zSE&#QaLN>)?e~CO-VRvXwHwb^eMhaQu2fMik1h?0roJjuVV#8=eEBs(I(C_JlK56( z{fsc;^yUr~4m<~SeMjMQ)+5Fs>pg7k48vxFVk|M5Neqke{}(%kD8 zOJSqt6QcP4u3{38QLnMUS(BP@LE;yU>Q}?hT?ZJ6=%3_YpBJtT+mGeJBDltHHEGLB zgNWQR(idhA(d#0}50>dvZhMt>8J@!`Qx(u^Spq|s7og4M2h=B|0n~V#DDQO?`rtQ6 zE{P`G!G0?DFB@*yp9DFzZW`Mw2#U?qQ2y2y&d!!<5J*qQxb8;2I5Wz+&LFP=ztlRrwQltyud;S9ylnLf@ie}?GKs)n|})O!WQj?TTe=uBXvvQfnF*7dDscH z-gOYKsjtlMBqgFme-TW-u?D&f?Uavaxmbb$kI~mkf zk7klgVrfOeYHSOu#n4vv-r2DdC07!>gga3xFah#j3}Ch91exNUL_;gLgIdxg=rsmH z$@=4PyDpDqw9Y2C?5zPqu3&N87pCUF22fKnWL9UAo zvzhOSjbp?+&z5UrlL9|iZoxG9N5r2h;?ZY!SU0vV`YRO@U_y!h=4<#q zTLvC)8gWr;CdpYmi^R9?Mf+AGwriG&=CS--{patA@G@QAs_+)F%Tf}A)Y;yKdIA2M z$6{%8D$#6_6oj2S3cW6K(Z@iH$31-j6FbU@>17A}ZRbkeUff}B9uEcii5yT`d5BT; zumF@ECHlD&FeO-wd*E+B1m0!m)8{|ahT^k0sd5!XIGds7%`7l6oUS`;;bfT%pFUCKu3}u$7%*y}_k*$%t}S$gI*GII_tJ zJ~hU_ugbGu5K{J zg^P@_)H;lPKeKmqGk5e({7T|lHqkww?$BcMemH%p3cs@~k_7{s;n3^DcuV|9-Gfwb z;-0=5Y@X=iHJ<={GV3MjS^5y=y9u2z`N{O%nhVd_+QMfomFgHL=75I9E% zF&4|}dnf=+ZNRpc8+zS4M@Y;eTK z%y;zKbcaUy>0^%GFtl&IL!b6Z18-OWmK&L(RKZ5LzEB7!Z(If^iPLl~AHvn=G0g4V z(&S?HK<(+mLvT(z2>Qxzpwjpcj(c?u&CAawr{hY=c3)s;;2pH~iw7LFtHr9Anp8=ZG-4c_!w4HpdKiN{(Nb#>tdv3+%s%q-hnw_{lwj4u#|GjC4dZtiY~aE?Y~ zj-a)u1vQqTF!z`wF)JB|xHMxrZIc{SKd&as%Pmof--7kbXu(}s0dBy^DSZ96gx1Zk zz}9{aO>TZf1ZPWN#^X{N)MW^@QI|+(jTs|Yx)Vq16B(}=>&eV1Q+Vo5lhoPPvQF<* zI}X0=rf2u4an1G?p%Hu6ogc{u`gc`8eSRq&E>(hg-?Pxm)evQwYC8Szd@^C&O4e&f zL+z~wY(93F$Z!7yeC+qMqx~$HE3F2_kOUHtrh{#RJh&7zk1El;Nuz^J$S=6u5DxitS-tYM(aAUcamjmHQvFW z%CW@YyBA)N%ZA0P_$fV-0KRSoxZ>(`vV=(CrMzgG)1Y6+7}sM%cPQY1CI(&1;5;>N zCC84W*VRdS!N%1JRC30A=qcRJfzlFiUj2al;3#m{tWv1n@i#bCD2s;7@jVC z4?C-$Vc0e{d&Fkney*B@de_$zmj@20n^H^T58h>5B>kbL@Ee`lx)i0?TX2jntYKZ` z>@Mf%CVc)|9oHF>qheP(g?k{ZZ@4JW(Ch9`#`YrGK_6LjVn8~xHA3* z(3yP>gA4x=sqAZ*?E0LnQnSY1e`~?Rp#XceQ?XI>KYA}=fLxb*O8y>6rcYB<;jb;j zS(L-F2TD$0>ak)J%3ch~{RX%s$A}!E2N;hBfWPKTb3YA)VZ`_*JW8cm_hA)HJDNfj zgi@ffpo4bDb&^!!Ib_8fQSRx9-y~y44c_QuVEJn)Y_2XPafekQWQ`O^XMCbJKJa7i z<8z#L+c*fF+=K;Uws>{+aq1VS#x1^ykTy3P@AEkjiw0qOW<3Xj5;^2(%_y8+6c45b zr8sR)Ihn>XQv9k7aPxz39G;mD-?!eUrGvSs8K_FP)M-FSq9x2S+k^ABZ8sZH^To#@ zJ+)j~?&6$(`BOw06^B`b0*QE5l|DTu%1U$RIX9*0_Z9 zSue&JDG`{wqKZ!bX@>)f2{dyg6^M;A{)y`+A3l47Kg-(y@{hXnI>@ES?XWa9jReac z#~H%xxz5H2Et4au{~uC!9v<~atR1s`!-cKo0L$NVsGpp5Rxxkb|q`|Aosc@WVo zz!h;S!_Bg;RPyXgj>$c7F!=hJ?W`BluQPo?Gq#FQbx#zl^CYiXJ~ppg}GB#K?f@LWf{b3Ga#|8l?)wv+9bs9~PHqaQ@7K6B)4j=&7?UJE&#h2eRUnF8OG$O_IV5 z=&HZwV637F&n%iT$?+k~odu9Fz6wsiU4oTu%3v59%lu`(yH^v0*{oL`VWzg?LA7fj zkfnz|g*QOP>htjLuq{%*IdK0|Iqi8NhTl6E!b+zTxThiv0!5{PepF`O8+vi-zwt=b zyd${O{WCatYeQE5SrmypPhPZC;D&v1WS`k1lDN*IuK)2uu%3*d8vDZW!#z85kG2BZ z5zqGgUx!i$i;HBfc?a~WJfw@iw8O6j3Mj{9E3_Nx z*qQAd!Lx9w#h3o<`bX{EGi`uh z@?3`dJ5z*Lb$2uFd^bifY*K+9vnTNT#}Lfh)q-o4W0*J5YhYqo32yXEgO+(dIB9SZ z_iS_nrz6MES1$y1WiP>$XQj{^U`jftRC9(My3oqE4#+DfvLoD^7@8JP^UgTX(C6}& zmlx0kPnK)GiVu%F89e(mCBE>*F305HOeR zA@|X2&()aK`L^zTlK`ZYDPW(|-n$Cr{@UfK9B zu>b^AT(Mq#8%j6NhTZqxfc-;xj4zHRhQW_OY|Bp0r&aUdg^@7#>(RHwd*cph`yv6O zhsKckb_TSD{K; z@`CA3-Rsoq%op<2TLeE<`C;119J*;mG)_(4LjI1OB~xF`g>LwQ)vE2B<ulh5GMt3^EFKKr)x)DQP?Uo=97QUu@4HMKh z>p1i*+Ki{77ow$O6mepi2%m!1Lh3*Vsy}%~lV8S?->>%)BfGy$X_YAMjJ#`p(q|q1 ztY3piMgDP6x0e_tXF$!RY9g-w)_k3y67SXQ5>DvwMR5C32=5f_vGC~)IDFy&9!t~Y z+Px~nie^n5T(Xb!y4*wZBLl?UBVcN>JWA=RaUTDjiFZUC@owz^uD-AwemZy1uy!dT zZtX$idfw9#-%8Gm=ty{|9FM)X_kw)cb*xyZ4#QT{dG${v(aImN*rynHTD@f4JD)i0 zVx7dp#;hl-oc1nMMSL>{jZNf$;rmM;aBNX|>IO1)cshLEa0R-!C0LwQ4nx;Z(~HXT zP$=n-bFv*F&TWLMPk4g+M>qO*g%r1JX&S&AXWI7R0{(s3NFI60VWG?mVrno2ZGP(D z_?|Q9+;{=3v~N?ZkplK_zZgE13c$k%d8`Op3@v=qIA2~Q;-8G; z?dA3K=vg`3W%`VhZM6a_M22&?oacLs99+8^$S6AN8JgrWEv9- z$uz?lxRb{$DmKKjcz_?fLfnw=%dJK+at@In@`t;G|VB2Ud1?7U_IHo z@CeS4;^O?`dZO|DK8X)zy&SjdQ2M`aJQVqt?5pyJe^(3X!dxZ%DPT;8XEfjsn+oz@ ziX0fTKEWR+640h(K327^#NK8L^iKz;NT6{x-y@|aHZ-r^c)tTj&YLEnDUy*&`AN2ed8!BUq-x^)WhO| zD2_*130^#uXI>x_gMyYP2Z`zL*=*Pi2eIK=V5}>HmkU?5L*Q z{yunOgD_{qGB@~i+lBQ2pP-YndZb)tl={CnV%}eVN1iAcLwP!zKNkN@xXKIgdT|2z z@YjRBoYjhozqQ!A`eR(U)dH0&WVyPl#bDEn8k#bqPQ*K=;LFyn)aCaGrcP{tSI3r9 z5B>vaQ62@BiH}jDU^;bG`~{I^MPxu~C2Hup)_qcJ!R5O(;PKQz{Fb;B1q`a_-ReS| zV$*}u_qf1E#bS7SC<0pCbI^Vw6Mw9TBs$A|p-rZhOcB*1H4_FR(OP|n4V8($KO#T&5r;CE8Sal-7pFnpZmOl$s{ z;El)*=!p~I9<<5E*>gj1<(?q=Y0ex1^WbzC`c|D1|_V&Wj%r;A(9AF8WJE@t^N;=E?* zN3`>NGR$1YG7dj1qNfyP=}@~ZRBzsns+re`qVY_K=`_aTyXW!NBR`M|XhmtqNsiDh zGjJSV2C!0sXIgUxBwcg?dQBKvLoIxFxsOBVIikGfF7TT<1APv=l6U&~W@~KYX=6<$ zmG8R%Z+PBh{=yoXefBp!%6A%BqJAsY{_(?Y9i9p1L5+gO$Yn+Lj`OSVYxGmq0k znOydKPE&U>o=z-}=GlF6LZ=buUKQc`dTO!=v}3pek}$L6Dell(0vo=6N4v~k5}BJw z-qi+!nA~;H_Z49V`{l8I&>0SmO5oZtmT$fBDhkDBLRO6}wfdL~$F^+bG<^%CwNVSq zWz*B@yfz;KpXMeaYAQ$T&u7z!RvQ}96+m~{{KROZ>r|s&00yKhX_sXyeY~DAKWJP= z7qYv)k8BoAo9%qrxolvR&=+RN+LQB-zY)XFtKm42o<{iI zWR7xQbJC})(tNX{v@xk1-$u{IZtHHkQBQ_DJ<=0%*H>|JcGbY>#UApzXccG|+@LCp zUO;)L7W}xiob_$KBZ&vkF~cfqoR;`J)E8!bfYJ)QFSj4zHAN$?uGV=0({}(j7J?-!3|q6C_0b>wFMSXG$Wn!J9v;Vd!_LH zfg*e`+y=f442?880+U~?;Fy3ATx>0d_+4XED6)b~@-K%=8{(i&6=<~08YA=RdqKu-fSWbS$j$HipILQG+kcVpcgF84F*21<#3Q+ z4PP&c#9N>ExT^QAu{)fp%$xAjka{x{-fy`LJ@&dh>zEhE9>B(>!`1jL| zp1Odl)~wHK zvn+FHRSOn{Cy;{67Wnw(3p#845Z(Gj0JSPL&_jTosmX7k$rF#w4cK{}XHN)ecX2|K z(|L5ydI^-8w-v0*_TWdgeo!O;-U6LaQ!|_X5{pC5ViS07nL{eu7N9`xGDh0a5+^*u zX%EXm*Ov^YpDmx!>5WqGW;_GjdvY+d_6|A4Z$bq_eb7F1F5Z`%g8U&*&|XG}8R-=u z&ku2!0d-e=EO`~YCFM{p@B-K`%%yZe3f?*pOcS+r@Lz&7r}Nh^>3a|X3T(f3-MS-i zef2I@X0S8_nsOlLzKAYVh1uKFV3e52iwTcsc75 z?Kp2xx8_$POcQIOIEK2afA>Srh~2gw$e(efV;r=bscWd;nJ7uDgF1KG49YGAXYgw zWXAQW^v;7=__~yRu76J_fp!crIv0$W2Wp6VW-v5e+J)QtX2Z-8#E}2QA-4Am@ zy8HUlm1iuOh3%$zTUm?oeq;ypM^<5h`g@o@_Y)Hn;|<+;U9kUtBK2Ev64w5`NO#|< zgw^XF;<#E9%-C;(f@z~1m$yN9Z>bQ7%hBD?L!|6cC`jH1qCH>* zk0j0`?^XmX)DWiCu3;>r{4U+mQ$v#WNMO2f4(z(Q9=crWalls$Of-|AWcfL^zb=i@ z*E?`k`#%~f)($F5&JvaH%i&Xl0B`y~LF%}PpL^g!0^Ux2fDyI(P*~^-NUkwL`Rbz> z{Z@$UaJd#{pT1xoWH(Ggb;qH|bPKoTqZ)TnW;`(S)`HHLD6qA*1CjhOVz|YI2%M=a(_cR#u||S;+lP1$j8@90)Fx=9$>p&=blYSfYBC*tJB!+=mKK zA{x&+&c^MDNG|c{i%g za^4GZyAvO`Rmu(jxTeFwUk$im`v~-iT*JYc+eua8RLD5F26O&vfGf{5V7W&-e0V*< z(LBd;S#r$SUiYCoe{CI@dLa>nBbK9fZhLb z@ulS-2(FUiZegDVCdGPSx2c0ZSuu~cGY>#kA zFPDBOwjw6~UbC6OP-vI^VwA|G*X8uepWhRUPIS?zbie zCDYKeshPxWzQV*G;RlO*vE-0fDXm=84&UZ4#7pn$@b^%kF5IshJW(Zbq3?gekJR`EcZ4l%;jyem(>ULdyL}i6(|{9iN8PEnU(p0i%ioOS3qLlD0d&U?lBAk8c=&6$fKI)pgQS&nuF=U|FHWj?yG{SUS8 zc#u;>-X=v+L2D7nn7^DVbo^#S9VQ_BuqZ-ICWx$&heov`jI*=nlr2^!#lG1n;mP`7 z|1BW(Gx@0W`^(tKHMw~8hm3mH|72l!d;b%GoIYVE(cv0 zCyp>V!I!KixJyl>_YPVvzdFfBa`E9ak~KAO9Dzw9j0CwT{1Cb1UZ z3?x!3*737g$q^T@&W7xv<)p;;8u+IenA?6xfTh`S7{0I{_xw#Kd>uWcrM(c6*UaF^ z7H?d?kMbD zlR$^>J%gJkuE6QocQjuu9Of=N3SH|C!AV1XOp%|$#xM4w&C-74Ey|<&Mn2G*MN_zn z9jxoBc)NM_#GAS?ziwh@pupX~w+t7v3_KukYtPc?{EW|1CtXW!O&itg!_;NesYICE2zTV&FR^XDqV;J_1_pU2RetH+q4 zZ;BxH_&q$(Re|SMyUB8slc2Btf=n$4qXo0Jk!$r1*um?C znF6n@I}bb^P2ku#KW|phS+qDhv-ay1KhC~DXOs`AryBQo#N_r|dO_tmJet^!%Tgxk zP1gC9Yf?y8`b2`u!rSz+r$XK2rwX{ePZkE&ogz<8S^~M7hu4kgp!iaLtQW~6>-M@K zGzC)2Q@4p((aT!l0ShZi zFGz8p7BU#nkwjYGtc4}1J6LCF9lHy>3>FUep^mAgGh$=wE(&j_d1uu*R~}hIvAY9I zvL4gNx#w`_&?tV)Rbb9OjwYzxM`Vt(@8$ngz)i{mj4uy@zUo8s$*n!?%xw&xqyyHl{COcS3~%AeCDMZfy&Rzq2X%?iGwNh64Cu z#Z}N&eocS(E`o{SJovFVn(p_Fz}-&Tbo`+J3HD6DB^ygif5l?O7TSft_3~oa01$m z)!8{{AsQXfMw^-L#3%m@nQncL%zocYuh7HH0o`^QzH=LqHM&pCf0UC@hpT94PzRjk zZl*_O6ZDtPg)Pq=N$90$GDet38~}3D3aFj;pXh+nBpa zy@NRB2BI6g|FBFvMzw;YK*)O)w#REjshqXeuL(ZTI1+32`00)CFp z0@1HGK>F7tdB^*T*Lq)*qtm4zmGvRcOIe3QY|r$=e@E%m!_H7;l7~aX`Jnw~8q{s7 zAqh37fftqpp-$7WjTt5DvK6@>oQo(;Vi^#JLg7MdHXg9{B1e>_;RM?=-BBI^Lq`nI zdQA~0OIU$i7^)-Fdsac&<*zuFbrW_>6Xnh?2nTPK5$KAUX7+s0kUTbP#o)>ZG|Ea5 z9z@Rsli>$Ydu??w$vdiGeUkRK$5QfQnm9c5-KO(x;8Xibg!j&au=KG!<1V5o% zd|CGzEvG)h8kUz|1B!HCUn0p|6HQNye8QQ3Sw{G`e3Evi6!vIkL6>+Y+vo9OCQFog zcTyi?#ke`vugIm79`0DEX9dqj!ieO%E}ZcxiBy&9qwJL=)M6;6UoF7)vvzRz-*+6S zPsBUx+3)X~P;7YZfsYQTz+~4|JS`TFap8(ovpNHxbF(-zrZMPidmUzF?ZKmP*OC2R z-)P`SGx%7}fU6Iih~zwNT;u5jcA73!H&GPq1#0R0y+ZZ;L6W!jBD2s-1QkKA|Z+)A8hqYmAp~ z2q+BbQ`@acc#$f?%}uQHV6GH>suV--4X@-(yC6fBwms$~jo!su8)Uez(HdUly<{xE zRWo&4_#se79J3--L8WIF_h$}Qq>{SFdQ{aMu{3{ zfc_8;#6g)Xm=v?1_9^zY0!PF!_s%)G36hzmbaMNDtdHh7is~)HSo0W& zX=6aztBI;yoJG?OjkvbbJ20)=A7h>8bFBV!)6swcP&_k_WrNm{4<0HU@l07*74eLY z8#U8sd*t!i!i!jUP5?@d$6`d2GMY~r1iK{31Fi*u76x>^joYdliP*9etg( zBy{h6MmPVG2BuaAf^!<+pR5v@)1m~KFSFp7oE;u*-3LZNMJU7W)2!?66EP1z(5jzH z-B#w&)#BZp4`QRV^6OUko282L%%|Xp4N|u^Ur0(_8KiDXMcKKBz;H1aePiSCsFVw5 zCCfiPeO!T8-+2fsL`;Y?d#5|uoC~dc40#9Lr^9!@HiAz_QT>Q4H=UgoHsvc&Jjk*G z&JRGlo*#N&eFnP1OQFU37U!)*4-~W}fy%`jB;%$UhID*q7KRoR3D(gP_n?9!p_9S6 za#NWn)Gi7SHa#TMq=m?Fof&w2r!ts1U8Gva-w^KYpH$uGP2Crv9=c+@jLj}-)SZ-5 z;FWwXM&GZCFd^FxHf%XcoOC|HgBSP6-`*Oga$6Hzy2N@3kJN$ck50Ur?SRjgXu>m} z0eXDKYY}|k01O(((gr)-v^S=Sb>te&2wQhUj+D0afgTBgUMq7 zVeb6kder%zhNXt}5Mvt*+Xs%r`-nqK!9x=&ox)~4#A z75>_rLHOG%_`u)_$j3GUjnw4M^E`w1*WIS7oqh1rdW8OY@VriXgCm5T`T$&0cbpa( z%ChLLqvZ7s#I?4c##k?gC5c^Ra>sf&t9AjuALfwhCM^3RHi_K7eE~V2^cmfm3&D-` zKM%U>z`G-_NS1#sE~xZHd)fm-RvNs_e-SvY`Ko+H`)V;ytDl=a0}X2U8QYdB*P0InN$(t6DZ z&iWA~tA>g(yGIFj`oCsb`2{e0Vh7f5VL8VU!!&gHQ<}0S0-}v4u=uj2xg`G=h&U#N zD>{$j7wu8ha^dGnmdY_R|FL(>xNSIH^$|H$X-~fF`ak2&lsNwjM7ydQnsw+C=R-gW zy}!+jI3)qhvo}6~Wj6V!JMsgqHpnmv zNoQ#tcR6@JX3vOiBbYpqYkpiMhRzfzhqN=SOA)g;9?_YcIn!T}O&dKRRAL6_Fx$J; zmDvvgSCTPNe2iEL=7Ka#3@dm6z1F9^pD{71jO7ewt_Vl%| zZ&1ak%Zq81B0uk#nm6NCuo67JAZazZNX1n?K&c%i?jDjbUwsw%V*VIh_vE3)gD5zz zs*kQMKImpGPA;eSo9kYmj+WwjSi=)W>oK#p zE_83)LOh#fQRY?zY5iHt`4}U~wG5A@Q-79Ffs;k3X?u#QE$T-nB^kUsSjoCx$MAHb zG1uzL3i^iSK^aGzGi?W_F}_Xlpu44%zWN~qzn^Ph<$ej$RQITE_Ww0N8EZlQVlfKn zy`}rPsigThn_ssSf{4%nEKn%GMc10(MCBs(K4(#PQ*j5Bhw*5>^hFeSnFw4q|M6As zIPNpOh5I+e;%46rx+vrm?Gti^zh(sZUZ~)T;T>edx^+zTy?A(dM2FmFGa(kCf2d#M z1W8^m${TNLrc9d!EITIx;?CFMLg-)W$ht5uPWp1T=vG5QtRnWR_QONl0nXW*)%9wP&ZqIPAfQ#hhBTYyL7}nk%yf74XoS3nPs_sTLfNjDsZ7!1m}5I5S01c zg!5>MsM$}&Bl);bRwa;oGv=eMO#%*R zOA?@w@}`CTIlC-7TUO+oRZbP7r`olF>F4MxmEyz<6O2i@!&xvflLq{8#ovP$ z(9d6v*lx4o9QNN0A^inVKU;!}UaLWCC;Qo`@F)4lZj%_LgP^x23Wh&kr$3Wl)p;cs zu#Ax?Xv)8g){7%aU91inK5><_&mN@u+B!V9+vPZ5Ge#^XEOBcnhiM&|33YdCXwL!x z6w*3H#yj0$B1#VDX=u?6i#eE38;1W*oWMU~Rb-!F2n0S+qxVFeNSe4PzG8p3#g0lc zzfu?HUVKk4iqFT8ks|ud72x0gY0&&77d{EqP?P;?@M51locxPbJidEi(^RB`h{?K zZx-nwrclji44VI1;m?gM+M;8J`#q;3jq4*JJ7dVhGZnNycnvwd`YoL)tV&E5Ud4Zb z=FsHziClW|6y|KIprPOEX`7cn=BP(9ir+tRc6>@=Kle-E*bOJ(U)2cgXqEcUHl(Xh z#L<8&%^()VM|ND+27!5_nCAGN?%iL3$N$sDcV`T+>98M4U2O%WE*`}-z9uQ z`{1I&NAskjm*mu~X|&IN8VV~dLp{e7_;V!~{pZmIy@Lyylw>)BHr1)evJ3($cXU6Lr8W=1C-&cl+#F*02Koamp}4Tg&x>1o;B zG_I@!_k=vcZIhQs?9fY=88wI|D_6t*z?J43=NXe~7bUJ^;1Cu}bA;0&k4a)?E!8-| zf%lRbxYeYay4wArnya-i*Ygn@nK(d32Yf*8cpQW;cnM=QDb(f9V|@Qk9G{FboPdi< zP|u$snR!zHxc`w_Uq$Er{DuryaD6<6 zO`QR8M1qtT8RP1K$IyFe9HT=#VaRtHirtE2jMuaGZ24>Cr1^GuUYkP?Z0{f>T97yH zG=*Z=Q=%}uoEd*u3gQX*)LH5esgN5d%e#i_gbk2^WI5j5dvEC>c30BYtA(GqD)^sw z44Y{Y=ACw4fW0}&xLeNy3_hGE$G;}v9@j}uRa`O49;^*ZM@Ja=FOGBYwNDW=W2WijOdJ>V9js9OFVawEPX!y?vu?ClI7$WD5=Hf=x`y`1uLOmsi@Oq&l9p(PQG%IOxa=#KRa4lrB zcxfP@unDHH&SWy(SK$KNWP{v`0rtTU<{FntEePE2ob0(q7hQfp;D-pli#WW2FX@S!n0(;X!}r(^KtcR?cES5Cx>+#w-~P=)Rg!rz|2`MYsGa)(2x z?oKHXOZ=ZRcAdF^Up9gd>^o8Fb;F42ti}JGE|u#B!Y=Y;ZET@V&hOq&PDFS(1ut>rClCm zS-!$plnpr9aJl57A>(`L0O;>-1QCzpxaUVcir&11wKZbMU*^cm$rm6?Hj1I7- zoQG?RlJQxg6ffBNDIA@6LQ)E{@TRId?7e>;Y_E0GLXS(ZpnMTIv-T`H`Fz6#hO@YV zyB~9+`F^rKyS*^m{R(zXdZPW55%OyLPE;Mf4wg%Pkqxf|uu?9FT8ajM|H{uKqBN9V z`qs-aE>pm+WM`0y3Zad9wM4`J1^H4kM9y`Ez<;xq|3}ez_|^Enal9RAQ4#H|q#~N? zxvxW1A|oP|tYoh=toEK-w3VVkQB+jtxv!(8C}c%_iL6keq3qxD`wx1Z&Uv2uzOK*b z{l*pDYvIizI~oZWNnoBHtxK80Wr{U$@KOXQ-tQsqtKGRS`U(1RH6?HD>v2bu2CVkX zq1oH_ke_-X*sWVk*3JlK?_Mhb8e~tVcCQc=r1 z#+1GlhJW0h&u!QO9k)8c(f0LN|4I^e{Efkf;jhTIY20k%YBEjTtwkUC+v9C{Lpbt4 zgO~Yn4*B|Ail^p0nHuCX3c`0D2t_$F9GOxGPoi>oFmEj*uHQzXY_Tn+H#6b~pbY@{YtBd~ms z1rOT+G8+7e86BU-|Le(R%C~)E-Znn~oH~x)y?zD9d?nz`Ls61A_<(rLbOlYlFRbgw z6taAZ0WZ{ZI@LK1$PcjQ9BT_fBde0poIgl^-gJll18Jm~%XNQPrixj=*3<3R)}ZyX zWnd+U#Ex=*R?Ct=K&l&DEVqQsC4)GBR}K2@P=Mffip&k+ef+`3elV%|OAdBfp-;aS zDX)7?OoMizT)P-=;~z#)RHsSacy6Fl4e9ijh81oxJ%&`Fq60XSz<1X`T4z=y!ZpWFv^y>Wg%%d^ zw@jZY=o^?IXt^W+*}Q62JX{zRMt_iL77TW_`_RP)-eK~dv$)$(7!y@bbGzqaMo`P; zTt7MChc!Rx27VK1Egod=OxZp3|Paq<2l37ewYOCXv{p$G4L;&qi>hbyQsk{r{>fpIYB)J;5f*8Aes(s(OgLr(s zN_EC@0@%l4B!1>FO*QyL?A9E?DsHCWy2A-?p1)0^Eyp1cR54vSlPkSDp`T;jc&4IP+U;?FdwW; zFF?PCGz=f(KJUwtsOfWs*yYXSE7in-bc>8&@~hKm;^sg?|BS;u@+tJxvUymaIhB-& zBv7Nnh&-rxa-&MSC*a{E8H~8L z9Nv%o#W1(?7&vkZ<)bS|`l^G>$n9v{`*RNUu4t^48yO=}g-U{583p)?V=SavYvE$; zWyI#A32M)(B+UYKets~I>n%#r9;bMyT+m4;55*AsMk^ToE+p9U^CnuGg<$<7DIl4e zaMD#;Q1@Gs{(0`h_6_Cm*7{TJ76Ul#9 zIgZ!HPB7KqitXLD9G{&J$@5LA!BG|L`LK~1-<<$wo&}MNmjYs+odL1oKWI_rYIwi% z25#DP0h?7Wz(v(C@=WmtC@HUkg@4Y`=`CkbuktqeToXiHR0~nzb|;LM-eJCM5~J%E zYUA0>O>{v>J(z4fj(4`@(Z3u@>%mwiCT{7&ikYS)G0PJAE+SiM;lmEX=KcOICbJ#}gV(C>vD86ujX)ukVh)7LGxrbfy(bPIB{iRW5VV5R2ZT z#%z(~I^3xIA1i4KAim0*M4TxHAFEKzf29UzCJCr#SU6PL-(|b2$HV1VMU-#5O$D=N zz<;6F$*;AIxJR7DV+oPqaOoq~R;t5)9|PbC*ZU3*jpZ1#-5`2uEk?JWrLDf==$C8& z+2=nq$+iKocH}QT?V3sFw3g#Md0~8dYKCCp$VJ@i(*i*{pEwR>KDxMZ-|v&R7=@ct zS-X)q`o^w}x!c%DJGiXh5uP7eRPlix{Ow6aO%ZJ8G|`N6F%VU>5M|s>P(yw&P80h} z?1GiB{h2Ip%WY9`QCmu*E&}MXGjR8f@9dU>WPCm&mug+PPPHTt!&_G`JWO8Vx!3-D zyPckBBbGu%i+_>`<*PJz(g}2Ua1z()h0uC$Z8W>sM%xRP(qkicu(4(~QH}1UvfF3y zie`w>Cl3E8Z=MU>n<$`CWzo1JKmnt!q~NNVanSrF2(pFuf-=`{(714e`Lb*(eCLbu zuBTan{-6-=q30Mjj%y;14=m+nrvRSc?FK7X*b$YYGbC%~Mtpd+8yY=-ab2LL^hIwI zo!sF{BfX+w-|b;`fm#!^6;&`BmgjQMBmqZTdxyG?o8egT8M6Ih2GmnAF3YwGovT8a z3%uJfk7JM*)lFi}7gdmM$Wf7kgV87kUyC`!-VMD9bCV`oel^43dA>g?Tf?3mMIe1MFNU zJ&?apjXBqB*vXEHf?dl-N$RI)^3ru5eHx!dLubT22}R#q_K#8+mZEr8mO zjWPuv48d+ygyn&ihj6@_4Ea@e2G!-af%8}}Y%@GZGvu$3)cm<vCz>yrxHXns9DWEBv@1he}AEgCmMD^kZ58@*kF> z7WX{vTI-6X&el*VTa6b6I?;QBFYFh(Paie&;g>{+WwOFe*rYOBFlp5(>fWD=77L`Y z`DzlZE-VLktDBhf%Yu2>Swh|}z6^(dQ_OdfBEm+`LE~d4F+Ax67ZQW<&hjZ#Zpl4X zu4FGPpPmZSobFp3(5i;U+$p^4mAQ1rrc#o3Lx;KjbJVg)?lkwi4g}S~;}CBU#m!*k zc~0B3dD)2)X!-d)Y%B=D>n59N)03mz8JKg5_RHg~!ASy@Uf?aTNrDT~>xi_aAyJu- zi?7Os>1wMH_-i(cuH*LO<)>`usaiL1GqHjNA-?Fm)|&_=8Gvy|7&Z#p;X1RNto!#C zcHYv3*S{i}>cEp&_OXv1YbwA%$@wVxZ5o*(w2|z+R1DU_&RDL-c?AW3@wN9sa?)cV zN>y|q**}T=XIg{b?-oN=i7_*6TPDmYK4h6w=TIU-0Ong_@M*3q8hEBtcGwJl zKmCO@L(8!6_Cc^Xyp7y1DkqldWkiUZWg4ytg?~0Bq+iv6)oXOdLN&HeA@$I zis?f##C?9n<148`tT59*&I)CoO^5ibe!TUP>zc7Ob9%u$T z@_#apE54Cp>8U(x>r&MF%VmkhGja9QgXD+2I_kZ;%d{6aK$n>bQRsgG(*BXGuhkt` zXsJ(D9$f>4_Q$v^odEtklERvVoA9=GJU`lX8rV5Z0^KRKmc5)y)9d&HniQ4F%>-(h zz}^~IZSe%+essg+tuoj+YZ=4OXkk96usC-@C4XM3IPZY{39{2Hn*C=h0lRkE3FIdK zqpMacVa``2beT~~liD3&^ygmk_LBzaIb@N_dK(xlHNlYG&E(I0uE+W6Jc`t>Mg>+> zuy4FLIhL44uf_~eav+#2eiKXHH*AS3q)_cXWC zW!m%T#(Njx6vyV>-X{uc-YYVfZf9fJ3NgWow~=s<+m*_f$^!4ZqSz}NmlyDwa&G>7y1y_d$YZWU-DZ%%)Q4oa@N0A+y)M_Q|>!X`zC3`u}_ z-`sF@_b5ACSir7WDo1w(Ny7m(KlnN4QSIcKRI0JN6+Jhss@1c5L<9UdzAazC3g;wY zhV)XVaAqR;dgKrYB?Mz~t`WqzuI4-sXP`!39TUROVqu*t{*m`3*(YD1SJ`tYXFXx* zb`3c3Pr&)Sbr89Juh+3LwrWufnbqDwmOl5#fvN!r&5MMT_a%5_ehHU9o`qUR>mkv* zffOu};cb}}25W{>V6iwK4tPF*qtP=hOC9Hu#`)7=-*ItRso_d4JqaKOs{4t>!#KLE z&7GTRzM;rTjCS#=Xr2BeFq-CwdGS*qo_pRsRlZ}nu>% zG>$gUc}cR`j=_%aN65gY8L(rA3Q|20zq`#~mktS)@G5$6v&$r|r3Wnlgw~ zB@p+!b7AGuUg&6kN&Ex8lC4{QK=j*Y63qLBGMbMdcD|#(FZ+c?hn|S^+OkHR4>q zAkgLZoLdY}K^&KRe{g(+?7sPo@Vhsonn)m*$u)%1d)hqhrzxPnQVibs1>lPiE!y&> z14@dx?`Gt5?73KhTSRQg-Va_-;Bf|aUsZu>k3(R^{a!6f*WmaN=e-B8uEt!%c!p$X1;6~U0 z2<^X$m#!I-&z1xO60FJCN;@?3xdgXlW(Xedoj`wNaICRyo3UoCF>mdit9Wx^4%WY~ z1d+CWv*eB$I*m+nsS|TsRoUI+K-W+Oa=wBk5T`ips$gAhDQR z(%U{~^kr(`y<@a2_UAN{}v-8j>zhfOX5lP}6<_vEmpB&U2fnM*ecJ zcpd@!{$#>&;zyfio<_yDfoPMNLKB-@=`KkwXYcO-HXR=H>6?K{0ZE%jX|c2a+%IK zc=>@pmR(&5u45A1UN4MY8>RxkMylpFR z{wVhhyI~CK+onL#6BktKTTZ|{2nO6)S{OGGeYlMHYfPUZu|c?M!_ zei0f;QYxg|j8C18!Lo*E;yZjAZ*$##E}9HY=8MrV+yj?&7{HPfSFm?RH#8SZp^>Ev zJrXfOce#{-lerpNuQH*dGj?&#vmZp^Wd<=MdhGlOMI-||*!QMY?0^BsxtZPwG7j1_ z)7}EJ&au!buaBpc6k%D?DUdgP$3Eb?hFg6#FkkyT>2zXCAf)O@snpSE}PA3La1S%v$B^=sMMn*_@x5_IWB3OuzSIm=&+LNc@*S{D_#KLGSc3{x`A|+8X_V4^E_*&e?&BWX6D3780&}S6 zw%5e}Mman_^pHAjsirS~^^vL@hj3c$X*_P9#nuZ&L-LD_u;;Z0dGJCJvev&~m!xEa z`nUw-Yfa@H?N>ps?SDwWtqp1i?4-31R-nSVEzFFsLR4QQl^Lgy1{=JN(0g;o3HEbm z=6SAp%sNwZIz1tq{Ft3eX70>DgTj7hG~pH)Dy2izjwraeJe4d|IEPPi*3v~1g>Y`v zoLoyDCKERa@kaNo#@lu`Nw!l2jy+xi-BQa0-M_8j{;#!o<@p+%JM##8xlEf(J$@dh zF8zf2f}QDM?LkUcwc-5TDfCD73&!e#K2HCr3x*|5Z~}~Yx?N}4#5La3SlkC|%yQ^# z+Y07^{8B;2!ZujB=QZ7_Hiajhd>pU53dCRSBlO$zNHRV80yurLqk=nI;ITKi!+57o zGji5i#+8PnmrtZQt4R0LoC^*#Qx!g$Jj^+EHkcTn$J5iFA{WrhMSlNYaM z;kJ`&P|i#TguN&j_U@xE?V}-gBpv58@1YIEPM~#CkvxdmgBwdzAmWxTHdbj9O%o4{ zE0Cv0R|a9uqL(!DsA)iA;B>7K={cfLx+RsUP3c2+ z@m^stGkiL!Uj%|q@$13VDp?f z^z&%IA1VQ0aJ~aN<+5l=WCAU}JxiduJrq3;q|-0uKS)Gb8LkMM44dwLtIj*63X@7_ zqTBSJyyz_|2aeWN7>gif72UjQUX&h$JcX^a{Df06jwS- z)^N-+@v~Vp*Kz`$rIExtE{6JlJ`F0FPqCHf0+VbCK{xFQ?Yff2=92Z?Iq?NicHM`I z$H$Wo*XH5T?uEcoM`-!^l9k?bmH9S@j}H=O!gup>TDrms9b{C&USfL(#C>gsg_{D=aAzv+DBp_W9G|FdLJ+FY zP$uUW+raSG>9A8>LNKr?4SYifU}3@%Y7x+eVxn7deNjH>pq=3GzG}w7K9DYX+ks`< z^@xc57qU+99+nI!W5>bg)aNyqd(fLLP?*DUZ||;$4>vDE?@wE-wVOc`5)b0cqkBp4 z6%$Mv9f!To)Ig1^@odlBYsow)BHgDIp~j~U-#r|HqKmGKtl*pN70;IXQnOl)ukzqzGk#`hVxD*Zdo zUwjYqFR7q+PX@V_K)C)zH-6UdA~8%j&N&iIFJIu8=*F`kTQ>(!v&L9=KY%}FcsZ{4 zXb-b|B(d?!TU^;3kHOmML^?1S-~Id#ZypSSn&MpOkFBB^>K7pHb}~x)2@~?0v$y6i%FjbqU{V=dPUv&zWp`*7`h% z<_3|gpIDsu_BdnpZIEo9X@o&L4Y+sD8a$CTmDe#=!S-%nNXP4arr*achR1j3S+40; z=FUVJIMd)dKPIGr$bKDSeonbWQzoc_XZUQoQJ@a$$7(PjFP3SSkmi}M=kgX2!Z@6J zii{jC1*h)U%z6g}!Lsi?RK$D}4l2$i>w@m#i$--cuNou){a-QkK{_Zq#6U$*9IDr) zV-H)wjOae$#@n3l$AaFZ9}6jqJ>{A(_*Dwzb` zgwo&RbeIKMJ&f!hM_jKN3-^bF1V-mRg6|Rs$kJ7!g~>gY6sxomN@vj zY9_2L(xU6%^kKf~M$F4o5ZI1!xw&W2pjwg-!x-tRn?gT;ghd_AUZIAzE z@1pU|+Bi{00grSTLHzMOXge|q-wzhiyhbTz>$@&cyWa$?XD_+5e3Ueo+vBY>X7JB@?ggRZ<`x-@?^vP1Mk~ zu$_4x^2cOo%g0GHAapC*F)tR&Jh>e38)r6e(HZoeREe>#E;3WLO@n`O?Kn-h0<7Lj!TjAE%dh1TlTi`| zeY;AiRAL-Ekgf>H-X?7*$H zE<=y7B5E?cPk#XKS4=~Hhf+46AqbXp=Xl3FH?BL`L@$VZAPXY3sL1X!P!-P26ecuK zMr}3YW?~0PH;%Ad6!akV;~%pA{1~mVy+S`^>x0^?6kKc8f~x)>>5=vlGFfkg%zi8i zwUr8j-+MbOm7;dgYK=AcN+OA!p5_RXhdKZ5lGh~2aSD8z9!V`Fo{=5TY{)Zr0rBSg z?kKe#?UwN2!|hdg?Dl=Evx-Nx@J2GATuk$d3aB-+0sh3TM#%cap0J|;}ka67(@29`LtGv(uoyAFn4DZku#DZF`W1EV-44TFPRVPJ73V)$0~14B8vS<*kllh`Q95a#$qwoQ+{oEwVeB#iysq9`J>R* z`JLVQV~7aN(O^ioJgR#c;JDkf1=7o#n8T|^QJpsx^m9g8zk7daU|J42=urg|axHoL zHb{WEtP@;5=toB$2jQgs*|6$RAexEZW9pY4g)Ka;-x(B1czb&2QMH>$cT3}=fz34H zZVA_OEhn3<&SG9(m%)Ed6_}l6f{zaQf~QRt(>tGIdU~H@*Qu>Vo%gAhyYj|C$m8ks z^X(7xQI|h{pAiT50~2A#kTEpa%z&@HIpp7=B*N$aWL1xO(p|1*xK=71=h^SXv@L&O zX|q4=(G9_auRYOX&n)zRjFb9`XXYG%>E8P>dwK@>Z(}Uv zN2}t$POjJStqVF+6u_jdk2-f=pv(5eac%@<^w_0~U#^zWK$Asay>tQyWgoyyJt??R zh|KfCOwfogN3m{Sm?Xa)HX2`~KkQFHP8c7m!~tiYxky|JC!unb3htjJ0*z~B1xjKs zneeKgL~*GKwJg{Vmp)J6EsdLo2RkSX%;< zk;yNoLedHiJhV!J_tZ;Nu!jl8C8FaYBl{g)C|(RL>RB*ibc=MqcgNz>*>tw!bN)N- zEVNd(k*_78EVw>-k^m}ZgJ@$dw7rw%T^EbMuWC|)wC(&+h(<-nQd|innJ3WZwr!bPyejR%%Jka)d7_RW%jpO3-+3<23 zEZCb!=|va(-cmxJf3RjRpViVj68k zBk#mf+b_Ay*H;|#N@Wjf8g@|nRFpT?zn5wER6&KO9`xP>8J^Pniqd6a=zh8%t&@?PcTExGaej>c z-4EbhY7dbsk`yRTk;G3Qr{ZFn*Ua5@UeLAKh>V{npdMnYzpXcUzp^VA77BEjc zi_WXn;H%YqT3{YYD60r`*-v`Q-T*dDyNZ)IPNmS^55RM_#wnHsup&Ykms~Fd|1=vI zIfS6~Xd!m)oy@yE9Km$&tfY@;XJhi-GRyT@Zs>gT9CvTO#_=f2A@foTkvYWiw$43a zN~haFcPY0Uvk}Gr|JF+wm2-#H7d~UWVi#+Zm<|c? zMP%Cem&EPi60lLrq-RqR_PfThUZtC$Y|Cj7Rsec-p}OryjdO?OwOT6sJsn0G9`?4^qS99pho;hy^a!P2$_RGtqV=(ySsi?7DRvy=UNd5J+#Su>mO8?}huxERXzrdg9Iat_!%aXpun z=Q{GABXOaMs^HgleeBr17ohHd_mQsu?;i19zp)9{l%`BjI^ zf+^fCGn$dJ2*iJ>d^)x(3QEJj)Bg2lr10J+&PTk7e#@1>@pdurvh4#axyKwVj?Uz5 zWF>f?N7X?mDHb&hykW~R2~=(2oJ~A=CUSu$Pfd_WFT*9!d;AQ1?&aW0?_3&EqXvIx zX2WHz6QEIj2jq_!f$EcXTF7P1{AA{V^LuBqCCUK=T;}q>E?@jTF_i2yJS+%}kVhML z!g20u*@)H87<;}D20c^3hH1)}Hdh4~uZst#c80u2n}-HNt{|_&K-|kzFjkvM7IbQ$ zPW1tro3)ufQ%%I7W0q9$+!)H-*Mx=|d)Ta9Ox{S!z)7L4bRYMe@-a`LuH5EyuGj>@ zJ&p%;XSEV_E=&Wl6Pj4GRTos|=TmT7&E+0Gvp4fH2%BMmvrqmeW{z9HX1@^cMffuM zZ|M*;hS}oNo0D)~UoXA+K!yh6RPO#4LW6xCQmNNYSmq#%(pgCyKY9oxbrN8$u{>JG zeS_ImS$M5zfO(Xz4{d3>%(sk9q+k0Yu4DE7XRI&zM9c;NyPI6xn?Rss(m% zWWh%KgDXMsx`-ZczfVZQTe4$PF5R{&AG7BUVRM8Wuj_0he}$e5{S*F@Toca6BB5a> zJLwt8SC0cNH*LYZd%wsZ#tPjEl?C-2d*jFp4U8(iioNZ7QFieVc~<9!2~jU0)Zzuq zkKBQFU;odAh$m5kCSv(n4Djz1uuXnPW~&Xt=;~@LTolCWN0$@N;VAe!L7v8~djh{_ zWTOAIT)aOcAC4|K4NbF$Nu7c)hCQz&W0lv4Cu0kl!&}LJwWVasm_Lx-nat}f8%!uH zg?t)@2Ng4M>*6SK#Xtq>43#jwO$v3NdfqE*5m=;rjrcqi$F2oBG}>?h z^J22Jz`WHQDxj#W6UPr;Tm1TGOn^}SJe@!>4oZgHI+Oub(mekSPbT2c=X61;HwP`0YV$&KX@%kpX<=;um^(G2_95WO+EU3qm{-InZLk{Bi zb7#x1->7*19xA#e7^Lw$bujT05@)SVGjzttCewq1jnbLYXSu|SwmvIUo4-Ui9HMff^fCUV?~ z>nPbJ2g|j^c+cE3(O{`Rv#DS*4LIBj6`Sj6^geOH9nCn{*30E>dLKX`+yfIYalEOu z0akNNxJ7ZQ%=j-XGiE5nw24UI1;hU^zGgO*2%m-Og$r?g~;i2tMX;E+t?WhmHRSL(b^9gYXFF6C}_R9+dtAXjfWWe1)&VtyrbjbB=fxf;x zSb5Hiqz#C&2WGr{6Q`y%`z6~e5M+=q8l=TOTI9U7@K zMk0;cK&WOu3ddc90ED(dvNvoHe75K&2(&E zORW0F(Ew3LXxJ<&@O_X^-`)#>kn?%q;9h{dP(zr*-PiwkorZM3Msl|yncfX_h4Vuz zaZ5%#d|tJe?Eln9*DK8tcqc}(mF?mJ9@iJx?k_5^Zji>wnH5m}q771>&%_UXN~lr4 zf#K)uhr@~vsQvs37FV5uhYQ;9k(fHqB~%yQo4evK!v^q>Y=LIN!{`Wa`loLulnwr_ z70dJ{pFOzQj_q2=yR!)LJFFpM#GAI9$%l(hy0BD2oNU?6WuHfe$={8OVRnTSsP(mz zj4pq&f24<&2FIXA`ZFx~t%%>_=3D;T!1ZPJpCSLW_328lWOPU|Cz?{V%%$s7@uI{f z{I^dWO$_%#`^t3s+dmw3CDucwoCK~)Yk{7bKFD_yru!GDb5DO^NEo(R-K{ z^C!N;s^SxHaNlLnT>O;&{g#a#C)07dVF=3mKH~O~9Bq0V=fUjDCPl6pC?#yiv7pDO z|873GPv?Baf!m?jgJXe|e!>LLyYy3I2ZU(;;cF#v95vrmy!@}0jPuKf%sKbTmp8TK zsp}(}HtQbs(0GhXP7OoQ;TFjD=pc5{jd*A0Gg6=Kh$@FU2jd+%m@xQ=5D^J^Rf6o*qh`99;%(&FA453K;Wp zV85P{rAM9?fo?VDL)+F3LszQ6c(4xE;(bQaI0vN*F41#yFT>p>52##V2-#ftmM$-r z=7o+sNgiFOh9QadXkotyY7@IiJ?1eHo)aKuh7BZ4c@7WT%jmB3D*Q7IY0TT_WdFwT zs4#mY?0b<7w;E@F{`oJM^6ecaPTqlUlUeYV5EY!@Tn8qy1Hj{W+bctrc(bMy(p_J! z(%rSnyh9#IsKDjp9t^(3){AG+SltCwwz|?Ac}p!f%v}WT!fhnYx1au6lLbNMEX%AK zf_>r5u;P6pj712;MQ=gPe$hz2%ijT3Eh8T5J}x8r@2to{Xd}NQ#PA-+gjXL7hoeDB z+)l5Xc2=~~7o{5XATvaL=9Gi{-8OvT_Z=E$DG|xC4ECs&C~xEJc)I9R8C~96Mk8FN z0rNwO)i@GDbc|Nwxws6h6rYXHGq1ye1Ig$Y)r-1*`_QJ}hdLD;!scZ=(E7X@$IF|9 zIbBf@h`M;m^99MBCd|9@dLwuDS%>%7=&H4-r_&~37A##cNZyW4CXU(y)z{hF<*MQ|_wl%?Q(ylXf z?>J}9)fx|`uO$R~a=iJ`QxC#4fe;a%bCjaq*Qx9zDjCN5_amXi+gPt*#R~9YJt4GK>Ih05^2e)+5hO8wfc3m{6atEOpu(%gU^K-7A6>|yogyV*yzw8$ zbUDtM$A9CGAHVR|Wf_4Ga}IbxVc=HWS=+1CN^_?#hm%vvf zJWO{FfWr16x<{>*_TO3zlA=1eGg1=eJcX#k(i4o$p(J+Gms~7S@1b6O9q>Wy7uUTAQYkmhja^U(N+vf_L#_z?D>6Da* z#W8*^%dpR!UK!>GA{e`*jN4gB1a_mbGzq4W za!{jj&boNgxdCdfmP6(GTSVmAE?o2CA*_TfCRW*y3uMjZIKp4KXU}80?R_8fQ}qcQ zoz}-zr0#{y8bx@FhEi=Yd47G}419Of8h_u4f%3aqu;*tp_pGm?_3vG9dU7=W&YDgu zjwX^dq>lrYNS&pIY zR`^dg425J{$x4ejTrHA^PgeJl+yBnsP^<(XmY~D66udVz90w0v1&J%U(3&1ZXN;V| z*_{TMl2b%FkB38Ir!idk#K+1OJv@B6nOGazVYAd*C|f3p+u9Gn;jQl=Rs9BN3Uc|~ zerricuO)=a4KlhHKQb#P=HX_Q0tnJBAi~6yo!Z!doBd@#_W5rJIaotzd?&SBqs?(v zSCN(9uEUIiK(Zlhlq8%kq!n5syw+0=xbq9=w_Gejp1oc~Ywx9klqEOw-X?+dE5q2t z(ru(*>NE24Mm>GH7qFmgE+tcI>C*-svbMa8il=L$S9%cf?tVxI+m;C?7MNr7GbOzL z=_kZq5=AMpoyP6^kDvO$8r&@l@bOE1OMRPSvP%K~1)d`+cWz+ce^nqa;4>L5$>iZY zW9SO3qua8=iO7Dvr^=--QIhJT_U`NEkmSe2sLFe>LoPGWTRC+C? zHhaR^B@@i>M~(-%V*Z8e4Ky(c_b0>tCOdS=l3+S<8@%mE!=M$burj854GzK6Z?=K@p*qW{krQeDj%fma+c55& z?F$>8a~!S5XPFt2M>sdqWAdskmRw3WOwVwgrm2Val4QzpW~G2yb<7p4NmC^@(~Niv zYSW?COOmKJY0{%Rm%@ojcQ9p65p4ET08O)Cytn->wb?nBRD|Dv-DT5hV~Ylz^lS&* zYrKTU>^gK9{f+^rIW}i;FomOcKyY;d1{y|ThDbe}GhYabldI^b6nF6BcJP{lE2QYz zTUK~iEck@yz`b(|$a1eoY~Zb{U?SO%iw;ag)%KV4X{|6h2W3$BF$r{B??7jmIo$KT zfb)*(LYczx>OHqVqWkiVFnPQr&*S<9{IT#1{`E8BEsD*vd@&G4zwK!t?pKp=M;w>+ zDZNf_?-m8y8HmCg!>H;q3J(SkqZ3~j!*uQOhx10}ZTScGo?IFEX6FrhpNGlIE*(fy zx{nGH1Jt4HHnem`pkuB!jD8&lyM8Cr&|}f`f+tJ6G~HoNkRq52su9($C3M@#A7IqD ziF_Eiht`_|At>T5xqIR`RCtzi-eihm2bU4eYImyYC`LzGe-WJ^Z`dfg0{S{AiC%r2 z{QX)DyEwMu&BwD~?RW>gG58Yxna_k(8PdFuT^b;^Tb=mJ%7Ml1ENU>lo4CE(0QIVc zjMp<8xL*>-HeYUJ#d6)q^Qo~Yb4s3eO4g8;ZE{rSKn=a0u>s5bZs1)D1wo|D5cS`v zPgneD0GAAPD9qf7E2Iy=n$|*E`p$xkQ`REQ>M59JfARla2wX1xo~XO6tWt9}zN8#XGyyco)pH;kfZ4 zta@(>_eUvfF;alR-TM58X)o~0^q1xj1ljb;93$-a2}7q9ZDhqRM-2JdN43`L-~pZ# z`nEZPn%WX{tXc_wx5dEC5fifJn}EcQn&HT-*UaD9|ETt>08Dz4jf-D%|L+%b>2f1q zT$k*`b&wlz=A;sA&~YZ|c7*5;go4~?6>N*V%<5Z2v4vA*alG4RxHat-%LushHm`Xev?%xIxivp(cFl9 z?)~@d40-I!<<>nU1-0#Z;61zq6FB~*yPq^vK_XZlD8|snJUSz>jUG#K#^OW~p7|p= z-nZ+b0%8k%A-R3HhyD6E*3p8K}*XTdM)hZ|2R4if2`jB zkDHm%B3Yq<6jDiXuJ>)Us8Cd-L=i56An>~qLe45$pnt_glLP%+|3aWG zms3zg=XaqXI%f~H{w_*nLfr97;yTQ%y+=K1ABZ>Kr@}(qy zd-g_p@!(a~pnelPc05ija_(a9Yeff1xcU?{ zbcMhZ-BGHZdx@R+(FBW?U%;|^(;*;InAR1gQSFtXK&GF^;4PYs;T(@JFoKY$8&dI> z`X91h#+Uv*8v?WPiox>hA?AsbEU#oi2EeK*ythi(obIR$or=L^{*}8_@C(Nt7RsZE zHwYPY-i?VIOZj(}8m!UdxXc~F^xIt?T6S)ykLRApW{YreGh2pQYksqN?~l=(JbCbq zISCaT?vRd2#?W%k0a`6aP&nEZFaC>X%6tNd#i%65gZe~e4?m~h--N@J=lWnBEx-@8 zsV3KyjfwEjF|x~VFQzZfXJ%wJKyrgAy|5x02QEy<5uHSnrOlgQ|3rB(FCC>tZU}~7 zT1dt}EvEFD8EY&)&f0w#C%vmqk-NJlf$ENrG&wMdoZ37ETxJJjyO|2aY)~hf*>*50 z6#!|90_fUP43CLAsJ4|7hr~&sWt#}nJ0giA_x$`fXpIGqaX59H+q3PQ&)n?Ofm4E9 zF2JCcq=md_tmm?RI=dx!yZT&+s^5Bea%Bg14!wpTKNnZYtinVOqB_jy`?C-VE}Bmhw?( zDVYbW%#Nc_`va_rctH&Ao51>oCox2Y*4=$wPSA@aQiyS&X_p>Rby({EBlmM>NL}(xnr#EEeYN^y?=~p z%_*4bwgBc7590Q;QBpEx1x?cvY$!XGjnD0sL2VwV!}d?-o6bjL7(v3R^(g!uftLna;EIySj2m>SgHY`mXShwA@{lZf7(^mTbHt5;A+ zGXiIl2TSKd5ht zrH?)xh>pi(c!GYPKA9L__JxEqTAUWPgbXj=f&HS(!C&wS#4oMD&sFW>se95>=89*m!7*wf`#w9zOKprZ06+_1C;VU8=Z zY;H8X+bE6yxS8Ra@=GSU`t#XODie8mf`Nx=n+dT7{LL-l?r z;e)$naHcC49Bm8GRVs>Fk;x%%iBL-jQoyNzZ@8Gzy2;asenwWWW z{yB}U_|~1GkfAH3DvM#x6HWBq=L1Kda9yG73@FJB$EBic@b~9N=2>bh3DP)B{~FCg z8J8{+Z@dqWSaE&gomTX_!XVRS*U1e2wuQAXlVIjRB>CyE3$i0Vl5-ZiSeq&eBf1$> zs>m6GgA3lj#;`N2XJe!eQYpi{jB*RUHbKM!|!go$xV2jHlH$04w{qfcv(Wq%on4K3w}1b4Jr?PQM`> zb?b(lJD*@!Nq{fwTnWv}pXl~Ig80(GgYs4{B8vj9GX9oZ!8Do&2j?z8uQXZ64bn!J z^f0pW>UKz(mWM@Ihd`4t!@?MMx_?F_tz8lhx&Ii<4oab-8mSPTb{%vkFC}-jH#8ov zttUeQVRUKuLmWMn%iP-O1X@#OlH{sWtdBzq4s-WPW8W5d{f_IBHMU~6SIO|6Zf}6$ z+0C5Jy8$9HWwCpfF#nd8Id0IgrNL9wv1#gh>bXLP_Em02cS!@BD=3bO1D&y2sSlr) z{-R0AS4iPHL*9-rQ!({O1kJj#h4PbnSh>7unBlh&_Aa{!&#Q%Coq7~pV^ zgBgv$Gvp~d58oiSj&tw*3ZntyEB%x6oUMS`FOAG}o$2^R<_aG4>&L4tW+d=^1C(VR zzyQNsSX1GNASi(4b|-OM{xoLZd{0_Wi9$`6CT*Oc4R)!UVQ5_o^J`Bu_0Xw=*2^mJ zqU0bROZ|({Q3<$1UINCgOJMinHF(H&gk33H$j$KH(v8BaNak)IY{(vlol9%*d0jcC zJ`jSs2b0m@Y!d98c?rszU$HTd6lg&I2$X#r$IqKx0F}MyS;g~ou8}wKi&3H1Eu*=< zo{i-0%)Q_!)J;yU`^tVgc^Qv~bYN@JGBlj1imXo^{X6`No%CY{+*mD7g`-8lFR7=o z=&Cj>oZ|>-r5!lRIU9TvKI8hGNpN}{Z6AzhmK!;w!quYagDp{LDJND*zjh zk6_(@%A~wZi+3S@B8gi~(QLUr1i!h*&AJrOG-nF=JW@;RG%ILj^#$txaEvNvo*{b= zwnOEnXp|lIg)^RAG+$&Hy1WXUUV8d~L2-~ofCwrO)^Y^@=XR6b2MqejvuW*GW_cxa{mj^JDB8Ziug8b0{TGZjXiRWS7zx29>Y!%bEUigVY!?b&?Vv~NH3e!M_- zI|%U|W<`KS?kgIn`~$+KaUBik^GHwNV-$)x3W|$^VBLZ%#Bg*EY;WTFFYk!)A72sV zyTm;q-%bTXdQ2a_S>Z`yhOQBx;4N_a=yfbOl88HNCUL&+AX;#Q!4+?lsqFm;Y<_Sr zt&-&YvIF}_T4WYXQxXEqO=Yha*U?pm+d*QEA+pKEf*}DUJb7=XTp~@?i_2VgE!_!;VqAa*dZ~QPN`L<)8`V*bzsoY<|@`y zjnI_UGJMVtMpxR+M?3y$GPyVrn$XlN59^z!`xGdWILN4O=N^(f%<FLX-hG8{gu zhSu>KyuOGM=)0IseqT5X&PK^_u(Sc9GoFC9{)>hY4_{)c=}Uc+9zc|U4HfRv0j-@f z{Ntwwh;X7HMxLr?LcP)(cil_kIwA+?fxUA4uE>{k)_xt>tPl&#jjQD7c~PGD+oPcC z!Ohyo6VQQq2X5A#T<2^xestz?rYqaPC`5=Cw<;E|N?1d)T{uywQ)emHIbbBQgD^kJ z0DIRl^ZGLI0C(MgE=#h?PUmQ9c@#dDPY1VTMc!J+Ox7%0ls?KWCMkCEka~D8jw)7g z49a+XJV6%Zj4#3@-3<)Cd>fj^-iO4hG?ZNEimuK{SY;k;GIPdnCMxnB*J(D571GS7 z;}Yv2#kdg`F24$HVrMXQsu6bntl@mnz0}t7JsmMufx92V@j1sz+BKZPtQb8+hbvh4 zy)KWN-vr_jZsxpOy&I-4*2dl8nfR9P3^THGP*zkI>pa!SMVm@|{4k6t&pQAo3QVb) zz9et*p&{mEUJj0>Swq(GGZ?QJ4429}*gvBi(dSk%H#e=LmI9xslic;jBOT>XrIb&1 z?!JsQ_a2kof98PC&m_3_LyaGD;vb#=X$YMSkD6rsPb$$XatXzO|yp}_Y^io)Va{=zCrckQ5fH&{M8>(`F>uPDtL(j0o=qjiR z_2&>H9~>n9HfeZmd<%s;$+)*a6t3PKSnj)c8&sV-0E=(v@YV^G5V448X5q7Va(SK^ z`fDihtDCuuk3%n6dU^+Z>8)dQ_WxiW1*nm4KW)*P>r>pSW{(D3KV76qFeEKN1Z?ar{i^S3CpJ&P0I0P$2yJ^yqbMyDaDJ~M)00WIxZICK}DS) zn7vsKqJqjK;iv>$|7HXWv&V4lz73dr=L9C+Y()K`Zu&aflNhZK4{)jI}<|k{==YAeeAe%xv_Dw5*i(NN z?rzVc13M<-?!mSEn>m{?K4})wOO?VsFW<0t`Z~$;jAp26t_6{Vqu8~zojA5QkzW^1 zLA&fVR{U2Yc<;SS6ED7?<3<&g>^cqRynX4%TVUHF zCuA3&CXB{4vO-0ecQ9HOCI_kVCdwC~ZKW8}a-`WInNZOBowSzyVe=sr#W}A>w6qP( zXrId4Sa^+!MFoSah7^CMhb%tYt&TUA)sS;333%jW5w2L$N_HRJLn>fLa?tt9H}nZt?bX^2a8NdNpnx_6|I z9JRlOKNa4ZEa~z>tH7DC^Xx`QKbQ{Rd$XZ^*B7#8;SAg)5Qx8G+|ju}2y6C#p*t0p z8}AqS2cB~@NoC!0Dy@7AYWj9!_}oyK?YS9Zm1i-vxAV!5ZIsqs*2IUt=c!Hk8&Ke8 z&sWq%KxgGk@*^ylaWtI;tF8&6(24_~W_60Dbv>h<5J<;nW|Q^{QPjy=oVW7#1pKq_ z3|hJfL8j9MX5>9~qO%EKL2RkRY;JM-vP*t1+n`4B@ z*f(W<_)Bp(Cd4sJYS+SeiyZ72zQ*O7QZPU{ABQ=fjSuGoDY3A?8=Q8zsNWwKoA=Sc z`a%%0s03~ID4KRSgySp8@nPT}{X1tiF|J8voth#rBH=0{w#)@BfhxUuD?% z?DZ%B4fMZif9yH64-&Zf&Bd&YByXPgs)PuJPe`Fon#@;cx@gc}$`0W7OB=Q8+ zu7qK2gaTwed&c%!ixGC>Nw~Y~2OKw_YNB~Q943vK(C*h3aOZ#t)^rp@(1D4d*CmXm z^c`*g`-)VVUnXA+g}Jl#dRE|i3aip}8#}@;LZZL4$$$IZaN*Gm@JWwCU!nWJZSP2N z7@*=k?*85l1245)x~jASJA8^UM0|k8+l!)G>LbuknM6|K+sP5#985g4hdjIY1T?NG zq34Pc$aW<#_D2s}WYbZ!t(8QuRw(-698S|-1YWc(jBurj7EX|=W& zj$8ro=_-Vu3iHTfnQ^-8VG+5q?lCBs3&QY^t?2ys1vJ^jqtIJ>sB5c($D!_2WM4Su z1TZ`k_4`zc=K|L_4Q}wA6R74F;)`An_E1SMGc0BRjr*mL?>(73m}Ekv-Unm#tR_^x zY>1{ms+d#09mq@<$LmThxN6i0=%^s?<;rmQc_^D0Pq0G8?8DHyb{h%mi3Oc)8^QWw z3%%EO3NBg%;|#4%($Ta59Od(fe*a4HWOE8mtmHC6!*}Uz?+Y-)y&bhXydZsxEu?up z2Eq3!MEipY$7U;IXMHfE2kob0qDB`UJX}Jbl=tEA1zmo5!5QMMcLH7?R{_3L3&_5- zrYY0qv2l$w+ASC&r zE^CO4ipNgRHj_0T>b!!c2#$l3h?5;>!2wwdGQs0pYXZ2qG~1a9#_RA!*9%iI{_f*&NA;+g5h_PfQeiE1~f`DCE(ZxcbYEHpct;h zBkUJj&iNH?c5+^pkD(~MtDEF1?S}9tzlpqSBx(Bdh*;*&hJd&Ly#C-nSa>BHJSP7n z-?zjP_ib}&;!6*x{3FjdlT2j9&MV?xpWCoG$eeZ*T9S;n3|4L^f*So-WG=_%UT0^^ z+D$q|G+JYjIaf%cqbSEom4I4S1&d2`Sf>TAne#sju=Ao6G>;duYt6&J^H3h~FTR9@ zGXrsKmmWNFlW6QaDi0nHuaN8E{y3?PCCB`V7@5z8+-&#@3cuM(4_P>H%(I`=ZRt{w zOmU?byP|L|=No^0)D$a!I$@5L1Qr?PlUkcAWXpXKteh_e9|ntXZ^>-_-QANZZ^BvJ z%lRFqU0=$sjMboh)2E<-@E0cT^aA{N@HrXu@B)#A1$5_CCkTDH4ko5=rIYlW7#RVU zO}_S(Oj>w>KGljR3KkqIP^Xb&Vs(-{=M!*znG~KIn~yU=4w53i!jtu{U_qxXsSFS! zL5ICaM~)CaO$vfd!}750sR|xi@relZ`=DXXBNRVei92}TOqS_fBjJKmvF4N_EV1mQ zZ>CDonpN7UeApB%x<4~|y~oL|$Ty^uOlJOGKa4eV1>s9j68eAC;5QZ5pP z%IeaXhBodwDL~% zy9`?eZb7}h9@gbfr|)8Cqsz&D)+>A)n2W@~hR{pPh5X~l%uml~k@R0`8EJ|$lDWR( z+Z;FA$pi8wvtWKj2X$2lhc7q6*gM?&tlL~dyCUS_n__t5mV1-o)}AFe+k&BDzOh8J zw2OHr$K#ownNL29{(e6fizjeM3DnlTCyc2KDz4c>hZY>f7w_JI$?;IGKlglN zC%=!r85iYMewIP$!SV*F@3+`ZD*_?F!x~RZxOY317Mm}Gx|7m8 z-5E{fR%s>_r_BbtOKrH=c8tiJjfYRmmaupyiagx0p8iVTkJM!{Z7P;-TyT0J2F#G- zWA!TFCz#=qq)0eZn@u|dXJLliZt_qq1ciHr>15X`dP7;5- zOaIHXNn#l&TvVc=F)C632>6eMvsSZ z?S(zu*VuzNaTk_k_rUw`Y_6|H6y_XQ1ZV%`VfFegG{e0Bjw!n1k_B&oe%TDUy~Q}Z z`aTH#TLH&+ej$l!!t@bQp?VWX(d&KPu zY|fA>%-MSZ!VBB5>x%&2(fv8R8oB~9%CWFh#thee4+TG-5WU?r3FOs|f_>y9G}kr4 zk+91Uc20|IDy^lrhYDdNh0^w~MO3mz1*eqA@#nAMfE#g&v}AlWp1J&w4U*YGtbDUgJwc5&b8&vfO~x$MS`g1lIVY`ByygDu>5Y0_gc{#@sC;CD2h z);cLcyv|uNCuBD=LwOcQ+icN|X0)<1Q#- z=QEbZbIGLcJPd1~gzxFb{MY>wE%!#zzT6#bx0Vo14a_6!{0KH2oPg8v{;<|x22t{e zGrk(wglFDy>$Z6z4YybV_3g5Jvsxi0h_>L$PDa();tw^!2po(9^@ zgVMyUu>H6-*;`e_u^(@;ZL5D`#sy^FeXJw&ej&IeMM3?Vsj$yH3k~vrFrHPucr-tX zjzuisrBw9M_qJ=0J_VQ+9SuiyYiaq*R#2Q9LPAL&)eqzNd!GTuFMY?5^>vWG;U;X+ z|ApF09FJkQ8=Z=6ntCbq)enWmgsdV zpPuDy0Sl)jY*}f6TR{ZY&bv)!Z<&O@3to_M69agm?nU+(=D^gfn{-pAC8`!>kppiO zFj?paUcUDc1{yi``M^EW@nJ3bmp2m{junxthg_$LMkaYZtpR0i&cf#&eLBI%8?Kh0 zL8HzAy1w=;J)gA{0&`NZ>!UiDu9|}CXZ|oFOPyFfZst+&?Kkcie}f`tlz1H~#_%Vr z20vYi!J03ISgdl9P%+LQy0acP3VlcWpWdWbe>G`6(`quT6A8~aZ?a6wMs``B87}3Q zQ68t=S4qD{yQn2_dR&R8JbMN>sjK5y>t(v>p(IgnGU7ept-wK}L|8Q85n8p+#?J9A z-2Kjh^PG>VFf);nHf&*E?dF(72S-Uk!(7ZZZ>I52r}4yWrtl~48sap3Azr`yUy3<( z3}nZmj*mPq+<6EElZ$cnYY|>^UNZ`jQ1&r*t~$l0W2??2bhqH6wC;Z{|WlO9@dDIe|T|4rk%e-hZU zoKJ4Ez5pJ7W`#2v_mWvVOQCytBCJwexm69vcRRt|lGAwd zQ5;cken4dYL}Q#)AlcW)-S7R2cwanM(Z(1b&^29x>F#dqO+^EUpsD0LeToI5c|i7X zxyqe>c;amejB?)E$37_-vta@1ri>8o`kCn^H$l3m9y(ovp!8D*ol&|7{=^pG;_gS7 z_BouGH_5`ZM_*7@ZWZi)sZM02InD8tC@_OgU(ctL z{iJYWD8~etE5`HgosDzkrZpP8ibNDM1s6eM@OG}m!qg^{Ig*_wF}8W&KGTTT#`S1w zEnkes7dxAr2u`OFzx3d^fi5pASQIlkF30@l%O)>ZoP*}kD`eHpd+7T!0fweaz;d&C zdQ&f-oDfV0t7Ylv{l3KHFGj%xMuD%;96_vv-ory}guN>+(N&c){D(T1XuVMcCi{7# z{lY)&$7h-_*jR|DnOwlYz_SpJdgMlbDLa=is97I` z%)=D0-h2|nofkly;XIrYcY~PNT|>(RXW|vkl3DV{(Dr~b7+%^2i&s|Q0iWB{z^;aL zFtsG|rWicYA0Regc7PA_mbpK1i1nQ|4OM$$8Qg%%-rKC*YVqyY5t4@d2t1CgVyNE<`nc`E~>*31$ z=j5&T!v^olbC6{x1B;6FK*{$xneYAKER zW)CUZ^N=5;jE9=d`F-++xasF4!rnNEoARYO=JNm*-98soqU-R$7kSezK=Ox{e+AKIWSZ0HBgLa343I?PUYqebhlJB?VR@${wu$TYvSjT zAOFe0JA=r^VDC)$p(Vz+N?{5KFfwG{EU{CZsFBLMxkG|7_H{?sK@n8&QH#=(b|$SvFLw7B$Q zV~l(!EwNMu&-bk;FU2vZXT4^Rg~e0zKt)D8XcGjw+M)Ovf*(bu;uQ50FlKE}9F#lY z!9{a?;G6|bN7|s`&_ycn;S}snHi!FF$>1hcKva^rwWmlPO3b((t@#`e^1vEwHaN#= z0e8{0{4urLQp;tS$B6U54lI!{#jt=Kbd63jd1j)4eVz6Y8=^?NO+PR->o$X}&IDeS ztuh?iZ2=GIarUEa5uGt#4i~&?fxWlI(Q4~1xUkC_mGoPdIUF~EQ?e83t@l~@vwah~ zN4LX!?o495VGY*@o`_TN5$T(A9x5}j0=Isj#LGy!3@6>mJh$8lkG$E7 z_qU#e^_|z5$J2t)T1N`+-Uy-PTMco)oE)jVlwo2YFUPC%3#88W@7UBm&*9@aF`mS# zgXs6Pl3C#Xk{yraa-VK)7@Ik0qWD4s<(x;UXIeWdkBOn)nlg0D4yGb=xpyFRBJ_A# zQYHU<_;8y~RxFIe0b?87c;y8xeP;$aYKb6fkbq(aY1q~mg8Y@q7&A1Hf8qRNu-<)| zng7rMgJ(QI&6{UH^>G#%UKI*X90w`1XMla4ncAE}xW@UVpQcSA zE>~FCbnGx>a#=OMojLTnzaAaRY$J(LTDbA&1@g}Q5bgIA;j653$G0o}@wRU^k+UA7 zb{<8f&nbt>Xs*CNVUkcVb_ymxSI3}NA5s5w7MNDbf~x#22=hHmtJ{W1`>X{J&4|If ziPykzss!IM$ddVhIRyHSg8Yj-8d%p(zmLSC=iw9Rrs;^9#{%Hl=Q8@O%Cs@k{to$i zoX{$|Q)2P!3uAW(w{qo5U-| zrzGU@OL`)+likSi0X?>JT8rTy!gGoP-tYwes%<&6y*Lz}xl~hu;c%E{e*zc3LS|Ln zAGrTB9{%eT`F|f_a7!v0(k_$RsWP+P8&W+|4UDUn`7zh zT2T*heeXf5#!RV7cr|mS#E!OFAwAcwLuzh$(05-IVP>TrzOdCGG(KF9dV=GOwogB~V0D6>BOnLE z2G7uooWov8Q+Rix93Kkyv*S_`4GQK`yf39!VA>%)h<2!e#iPrxs%inYFSuU+_h%ca z-4f1D_^b|(geIbdx*&acK?+(_Rk8H_P5j<-g!^t5V-%AC2Lm6%iXJ|cS**ZiO6Q4` zg)KdCV>!YJWg@xzFqxA9=1=upXm1^9U18ZqtTzu~meQSIZ1+**S&7NX-rE85ZR)o=yo&uQv zEQM+1@~;{WlX+s>UQ@fa?Chju+VWg-r-!B%*2+XB2%V2VbM zQ-NrxkiLIecpv7`_5M$2?nsx(DY;9aFrl12<@Cn>6NKK0{SS_Le5YFNlDxJQ9kAa2 zk#6RA3Hi^t-^$zrH)B`A-F+!A+suyc)fDHQEswd3YG~H9 z3Q%6sO*RXL;p8}RDB{`SE1w`V3@l{To|eEY(UT_24>)7{=@!^fHb6gzZ$$fTi!fav z0z4~C@vy5Hca|Tctu0gVaBwP$j@gs5CFbzW(irrrQ?R;043f{zgK6z55Vf%bb>3To zQ~w%}F8&8C-_O9X+7Dvks6d{4t0V>`@g^IgjO$Fj2;2Q5sot_sxHq!`mfSiEQZ3y$ ztv;K|o{d9KS!0vcVnRG$B>|@TYz;&{w}b_&%g6zZuN*R2Uw9#N3G%g~VYbL6Qs%&|E1mr`@cT77H(#A6lyidoo23pT ztzi)HrI{}DUWB7-Svt6=3%h=vrB~DasZ+!|wnAfs{qKS*jz8z}x<}3Nr{NTO%z7Dw zJs2?w|L-7N7t_X<4QJuhMp3@;v`esg*#R;qGl3kHPip9HI0y&ld}mwNPND~ghhWQX zYkcA;NmB1_0h3pAA?~vZqz1&{8jju8J*S#{d!dY;cZ%VRjUYMWIRaav!-@O@H?VqP z4oU|n@wPsljq2Ldxo>?ABQf8ZO5tIM z*HJwGE~|7il77&b0A>MQm}PE;>7mkW(SJ%f-|Qqrr%UoI+AfgCZWQ>lxSYia7jQpV z2XDDEp?~O2kiBOPb)nH<*k2C15>s$2L>A^>`$wOCctC_%GtNtW8V+yvg&w6rdhOy5 z@+JR03SSPSP5La=Z!N^?t?P*16FWNZ>uz|ycq$bx`%5>cP3QI7xsu5pjwIpoEBI#O zhC=o6niFh^B;~Bj*|ddKr4C% z$mG%u5No^)Y{t$rksmJNN7k9VH!q-N1*Wj_vjnegFpEtsjU&TVEIoZA6Q}IUB;Fx< zIBV(&JoUf|eIn|J(4wjIY*#kaJS!xJ&P}3ItnQJ`Z%r71lYDY&QZs1g-T~`hG9diF zz0_w_2#hfz8P$#uVlth+P@jO8wq&8i>klT!G;Bb&e-b?@y${r5t{^T>g2PJ%`Q?W; zU?|t6?Y(g^WJtZlC4VS!sJu?AI=zXD`!t@Z++|d~xtx4zsDZTW=fP-CKE%6K(7$WF z$zdl&&j02Is%M8tJTrr=OLHV;4|-|lqDpxC^aLBoO>kmw7DM9JW5{=&jSg4`Ly^7Y zjSH6{^611Q_gADj`vzTk?I`-@=irx~m1JsG65V;~BI(sCML~xpSXyO>z7h@K_g0(# z=SVc9-FVG(cU6&w#K{=H{yx(qnvQE-0d}4Dhi5H?WboiZdg7NB=3851gXaLo-@O2K z&$)b7V>M+!8sBVUWJ4H z^HgAG%VET_sW?kNi`{>ypUz?~!-;WC-X8;fI_;_mP7-jTg&T74@c3>_+pNz2y?!FS zq$>mTk~8sD&_jig3LHGPkHkN6At%&B$(z_$5VA9ZkErr>!-{5O*9c^=ZPVR;E6Xv}Eu`$%g;pq~*XPX@he?I5YKZD_y=IidA^`hSoiNhEXr`&_oT1K=Ws`)sQ46T<_6m^||n} zF%*Y4grbV|aS#s)2S+!3lyots2L~;2tORn$RREjb&*e2W2SdfQJhE_?D>Pck;E_$g$mZ{Y zyyRR1blOvjWEQ>uP9nL4@&@x+SyS?#89j1rGR zlSy0A?1nQgDSd#`Uzejy`a-z3v;_kNqJdW}!m}PPBkyZ1Kqtfz7aWhq<)5Duq4C=g zv{Mj5x(6}aH-p?1-Aey86<~k(M-pgIOr_Jyh~wsL&^dGq4p;Qk6hBjltv*Y3Mzu4; zmAg#u}sev>Rvt}&u<4aeJ4U8(6|Jgv&&J>{5SJ<#u=iyLJn4TH!?OgrNoHb$4TDY z|8zPW&K>argPcdindzWL2P~lR;36~)RVEu2>*A;P+DxFtR=O_QoK@dCh-af^`Lzp` zh_;yoqY^inRc&vlf9G7kD{LfumhYMBus?mA{`)b^f-~E64lL;lneyeNGh=v)kcO$1o=6E+$sn9-#Zx zI23J21(z$y_-k)04Gi#r{sCn;Y>Z&G{x2lnv4UlTB9Okkj&73SdN?-)LS0oT*?DUl zK6B3iqmshK?=t|wQJt^Er0urv=& zo9jXSyM0trmq7mVF6zB(7bq`R$HM02Wb>nEbf(P~I5y}F-PNY-xAxE~1$X{hp44s(9SVB*{J-+8%Ma`PhN^* zoQ8A$aU`_F%)x!C0iJ3R2vg?&bcNU;q!TQ_Ke^X6Y+!j>mEbXEx`cfCOQ zcmdwc!w)febtvN}at5U)4G_EBy=b&c1%m!oa5>%8CT|OpxNdYsRM?XN9$P8?dJqDh zdkpYjSU0FnkHA&$w=k6wMzC#pF0(-@77gD5+H9H(X?_uqFH*&(+}?q$?Om8-Y)>7{ z!!SoNn|!>c#UDH&i9=Fb;oPR{cvQ2JY8dNdqofgc&J^WxE!@m0o`=)oCh}gDsF1YB zUEoZM=!8obU`u^^U2+;~J%}XgFJ6)fLwC`JI|Iy5 zk>T&OS&EVFxn!+;2b5V$U|IGbdS#0g9`3w>=eI0?p@1UtAX0}4N$tQnYJGI~EISaC ze~p%HvuU?+4m_ML%o}_=%%})ef$Q7PH1%3C*f|oStnZ2fAN#dyD)ec{0 z_CoHSW8h|qG-=r+-tN;b2yyywL$I7`|JH$u^j=!|BnnV+1Xp~oM2Em^N~ham?euK& zRNEZH()OYI(hgd+(wJ55C}m{l?Lysq7QEM%d#T2`YjEzo9B;r%l$RrUl-wL z1+A*Nz*t!j9S2){|L_x=HFS$AC3dppkqef$B*Nl{Vt93T2C?1#gucG93+kqYfw;n6 zyi{=*9D=rDXQ2}K-#!hi4@KiK|4gX*kISp8U8ZtZ1;BPyFcAY67~WV!SHw)@_f<}Z z^1eLW9I40?_d89y_2;tLDhe2~@hCYh)=8qfxty4-CR*zn!n6ljXly7#L-M$M)vI+7 z;1v&?bOT>sTENrQj)dHdhLr04kT=T^JREDFZf6e&!*lEoO&~YRd}*yiCY|G4OSjkQ z;*<>=!T5YK8V=9LWi7M#r)|gBAv05Y&MU96Gd7EcG_1zC<5PG~P42;|H$^b%gEDLu z-OIx0W9(3~pua5`_~`nYx+{-BtNTS9jJ(MTS{)%K<#Fh5LD1W!n(O;LNB)nZ^A4x_ zedD;jS4tr}vneCwe4hK1sO*x4w6v8}w3oeQhwPP+5{1Hf?vn^9Dh(QxQfa7fOEUU> zewTk;T#j?jbDsNtzh5s^f%(+!t1$k~GUo0T;4>TL9l-JEQ+oV>3i;h$NA!A!P;lo~ zG$pxMEdCpU?H^(9mJ3v8yAJR8LQS%6G0TwOnv6D+zB0+J=HPB7$)6p50@eR@VGH45Gf+1+1LQ|jK|S#TF1dS-rf9{$!1XGs zx84d?m=*n)5BeyqY&~Y8jdcTh1oj4c(I~$iMYdiVw`6Q8|5=# zJ^TK#;+raFc}T(fgHbSUJO}nKO@P${N-(?NKce>6i!Sjkh1%x?4{7LerRJ@twR3(D zJ@NwzZ9mer;`LM;))TYXnKa}82Z~QTAns2W!Qe+Nj)~Y`X7vUudhsO715U{yiJ7C& zE>wZ#-IDwQHs3UN`CC}^B9+ZZ%W;=qtAsRfQF^LmHvf|AY&;XZmJ)L?NJ) zMo5&<{hPvYGgC`cR)<4ZkX`-OIybyup$~b=sp#vQi1t&usX~_%ecY!+ccuQMc7^7k zKDe7~?vVk>4jbZ=*FrD6G6eJIKS|`;>v+(>pV}EROu1Vm2^9^bQzoZ??ZYto{Hiak zJ=#W(Dmc;+Rdd{ZbtBuyS_Dy+ZD7r$5$mI~IT02LtPeFB&ZQZl*_N5$pPx#ZQ|08> zjLW=HrC0c0JHdmqWWe3Ih|YgM3)Ow&p=fqH7;RVJYtN8@n2okTPp%^W-BTt{-4EjQ z3vzJ$0fG6iBp^2{6XZXIcM8PwLYQqdsBEB_8e{k-N@ zMn!cK^n0ZO!3%=`v>WNuv=FjlVK;pkF-X@u%>(+YjP3niff+h0XzyGnbdil=XM5}L z#O7l(`iC-6*%tz_0qdbx_93JEy^B-?q!W!JjbLaVO|N)m)7Y1aj80@S6&Wyw|IBk? zq@xN~IAmdpp(rX@P2vB$1lWvS^Z|39Q7)>3M)*NDi(iD`@FrR_rb=7#ZE=OGF1tv

zJ`Y$y=AX~t@>m0V>xyxjK|F?J1~V&}#V9w-@?a+CLUBKIbv1qK^ipaES4*^6R!#>j8!^-KU*Ym;(G!3UJz&fu@)#Zc{NcGzX6=@gl50q z!Udm3#_io|o_g^%GJP|jI*XlU*ot(8o&=%%QeKw-O*!pP;tXf>e?^^V2Bb=ShVlSR4j?&reB?x`54@)=Cl2GQY)Bg{I%LiV$40ki6Q zDx}N(qq8L5^8$T^=|d7i(E1U|9ydb4-8-0bb$@Zzx;Xe=myQyXf|!C0p{(z539L*R zgZbLK@y>rDGEYO=`ff-%q@GEv5y!ge3qnn$_zYi>nI<$(0DY?Q?si&~2FoD*dvjMB# z6FAA0^{-6{=T)$*^iOaC5{6BA$6E~{@L~tHuxE45w`BHnNeVsnK^>)+h~w%pfI$96EzdycS=OE3h?+>4zq8r)&U zeZ1z*KlICOMQpLlB%Ow~A>6YBehIO>@-jW{nZ%EHeV!n@2{?~^MRVDlS00ll8bPv) z9%0f}JJ6c`7fI0oDa>_5{%Zk#TKx?C^JyV62PN?DJ#|?8f^}ZKSb|>)He*eu4wmV8 zkqkOSx1Br!Hm7evyv19T9+BqfcM8M2TT)!D4WXp9@c?h_a0yiHX<+>xZu}4P8)?Cc zG}_a64;;ixNV@DCW^j!+q!fRKs%@@d@<)b93K^2BsF^shNgF%a$L8_*FZ4iaEQ-}Q zlYMXB!j8-qa>}EE21_}Ed_+C`cfXObN$Q}_o>aouXQ431YYT0U7Uw?Q*nomBQ{mkA z49qRDCL{Itu=d(!dTZllQoyh?lFkyIU5N`U%nF6~tEySz55dq0M7Bc~uHmeUrykmKQL1#S&(3feQ+|)<9|19Ng2)@(GOD z_c$X>*nF%Eb=ueQ#X29;*E$`Lb^RF)lQ_*|c5RaM^|h$WnLE#j$-!=-GZ6!%qK##4bDHP<0Uhomz|C3v#We*Pch)V}Hpe zB}Fu-nvceJele>JSdR!CfX69(5~e9iS~F5fIup1!hyldcpxQDtb6FBnkI-MmM z2s_lJX$jwvCwH6OeXDQA7wTC!N1=(x-rB)^;ww+9ho+PBuF>${l84xPy#-DRo6v1) zOX;S-PsFm(9=-_Z!3n1{xGiQ4uTO#i@#36ay8#3E0We-&Lhpa;VEPlzLj0Q<=#&siWan~0 z@uMqzxNioc@$ST9-&_!UBgVB*8AiYJ_4Ly|9sbt388Gy$fL=NLh{oSsNz^=>iS#}J zh!w1W`d3wS(Yr>RKHNiVvor9+fnp*{eb|hc9CXGkK#TJUyiO%C{@J{F_ zPFcJK_Vn|xDDFPqk~V>>Vqx^!=U>572wAQL#1B|He9SFZCA|k)Ah?#GPIxmTIL6JhXSzbwFp;ven02w zb`!E?C+qSA4HQ3ON$$J3GAA;OaA4JGI&mT!ucq&Ysh!bOu_+DR$FIW0&>5sFhxN)W z;j#Iy*-Y~t9@610@SxS5`8a2cG%w#s7YyE{;;uoAZgI$QN9WGX6UJT`pq$w71G+ELDRaHXeuuqr<4b_XW7U)&rZ-E2O4;E_8l; z1nV7#XiVjw2KRrV&?HevItvAG?b*}#_P-3~#gcV+IQSxL$m5`<-3z=>F9V!Q3|g^) z#qN}SJcn0x;IvN{?h6Tm`dNM0enAE^-)zUPGk4?Dj4WFG;T|y$D#jew5zt$s11&R` z!o+J2eDk8r`(PZ>T4T!2(VCtWfkUlwt#S3{?$d)yet*mP=SoT zHYX*YV{q0qOPbko3m#AYh-tLK`rD`N@M)_8Nt1}=EnIU2&NAYB&Dl?}zulbmnCt~+ zOcmvIRAAbRdU7b`HJscUN8{hi!4>{PaJ>+TwXb8*aM1$zIaQ5&Q<~jtEY*bXV*kK@ z^-nl(^dWhDjm@)oh7&jSNDy9`fD1oGvK-B5684}E@}l05w%{4qJP^ZrGE=F!OES5= zH-Wc3)|cM#H>Z`^9iTq?jqJ+$5A5v97%7V=vRkzj7EQZ@FP`3JQdk`I`DcC1w4*st zcjYzu+}7l-5PU+0{0(r1tQ%fAau#ExW$pSnu6qfi2dq&-G;F=KL z`p^k#ToD@TrONz?Z-bVDEc+|P1s>Y@q3pYj+^XIC>it|<(y^R4zj7(d6T0PqQPwl@ z+FVE87Kxo8Y;zR1sMkS&+%MMSxRmRm1l(0W55kVUL3sCd1ctissQx))DwiQb+((*G zY)&z()K-Fk-!~d!hvIlIL;)sk-U8+n@x#$*V!Y-%-IBeE^b88%K)Vtegv-O8{B@MK zU4i?gtef6kT!70*YDh@)JCKRYgYE0Lz|V+n5V-ab$Fe+@DXzJSW|8;cpVk6&WM?8T zPIrA{is$D5Cmi&zKw&S)@?@&@OC1vsJX zi;sT`fgH<4i|uuzwpY%=IoUc`IwZ@Tdh#Tjg>eAmUz%9&n}Q*U8|YURCHT5L1@yv} zG6l}gz;SMa#T}<y)EwI#V({hlyMwF>mbnt=Im6zujY(KV}#(cRY-#RpTs zp8aY*aDn~ZHZY5Qe+l1#7^6mT#_Y%7Z%_uyeo{Jp?LKDW(E{8t=K(sf{51C(ZnzR;&A!i{eHXoG}D^T;r--f#IC!DoyIk@m> z5>;Nbf;hd^hRF6HyvfdpY)m)c%IGBQT&c_txGaq2{d0NCQYR6wG6OufGLbahZ9`Sr zwbln?_2H9}JYP2VGCkG#1|w#qQqyy4WRG1dDq5BRG4CVB(|myYERFP(+`)yRwfJgV z0-CbCw|m9W;HbY7KK?rc+2Y3NdP)Ve&or`m{9L>ktxvSREUTN^VXD z+~b@G+Vk?M*{}|Hw_ZidkYjkDuY^-l`IEVs8HoXdjrjED14x^9gFchLhhw1ywDHLa zIJbEdismfirY9S~kL8D9fzo`^GR+?=W9Gn-_|@>|s1V9qRO9i69MrtwfV)k{spS1X z@UY+`4Jx8wxh)@aXS$G&_ZN_lacl;QuY#9ts=@osRVwl4Ja4~=3Q3om$~Bnn13L!3 z5!2C+B=LMCb9-qyj^AGh{&hEar^Z=FpWgy%qb-TA$E@IA(qrnKWK67EdZD<*6>zZy zv`HnSd$=cNCUsIpSx=PiTtpwleS=4|mR`ADOGDVa6i05DPW9~q>7$b{|9}|ASBIjF zd^QRS*;<)S+Dr{&(N|X%2_D%;%5GB@cEP#s(jp50aY$|0x8%j-r z$bV|@IbkjpXrZWr0ZoOR{V}CPt@sR$90{V6<~EX5$x;nPyK9N7BZml_O~hB)+i^Ug z1&=+yi^@I`?Q3C?m&(6;{zJ^4ire^=#_`cyesEwB;# zj(9@st8@@;Qw3}B4BY6G3I7JgxtTc<+}fTX*uOIl;?}ZWvjbr;Xc-E&F8|0vw%6hD zX){OSS_gf-pcyx83Bu87Y!=FUERIW))83{Ul>QO5xb66mopx8Sle(0gl&qgiSL) z;^*?&csy|%evC??azn#Bviu;#%@N=~*lz?M2X(2;`#^kD@P=hUhM@GTa#&PM;knhEyM)Y#azewX;u~aVux!lU9j*KF=9LU zJrQ>FWzh?mIQ*L^?`Z}Zt}*wpYCjh1_2H@_6R!S}eC)1zh2Gx1=w<(%*I7BtjKm@+ zQgw3RWH1!Wj-=if&1mqdP{`i!iTtq-L~X_h{jJ2fZ@=B9_LgnLl%2Ug*d>W;|723% z^EY6ha|~Py5eiriVS3m^1#!Pcz3SYGXc8TOg5$f*PZ?dM=8ivTIRCox)g;sj%ek4%8UGL;EL&=>$8EQtQ=(B{Us>hd*VO z6{a)u*RtNz^|HZuCA}islaYz*aMt!JH84{HPlEuQ#tmm2egatTdO~cj+$5f& zY~E?U+vqIj!Oe-dx@ytfgGfCT?&a^qYLZfdwscYU5pqn0m* zE&KEE)5>kU^eaK=b|(QeE7j2OX&y;9)JxvhIG`8XN!xh#89DZKA?#RyQtOh33db?Pn2-}IJ_0P!`(H4wxSq^@qg8Wa7>!@qK5I@lCGF|5Jku1e3 z$a^6G^ISWz?(bE67rKWmyC=?nRhI}ppIQF?q=y)IoQv1Xv(ZOz8s3=P03$Wo*x8^A z_6Mi3uLTwE60>5`7^cRb%w}kc6FLYtLW2H%nO7g>YfIjxuw20cC5XGI3|eb4;X|+( zX{X7s`l1^?jqM$^AeB}5*2m2SCDlCqQK0~irH)`xR)}{@ zBzVVP8N-|9>v`p2axl%e9L`4lC3&AHY^o9_LT(hAMAqxE~zANWV8^bKjM`Q)B(aC;Sk;FxpA{M`a;U zYah=sGMmJlSwK(i`9Sv@T;%Y4p71h`4KZI=S3&r-I~?uc47AW!!S@U8P;o{ohAxW) zNr5al^tKzz&pxFsEr(Gxav!X7TSu3xEyuGv^vUj1XVA$o49;p16ufgBKCOyC-<^Zh zY-oydWdf)uh$;WswBEesn5*yJE zcbb*)d{GF#JZOUXPkTrPe=pH$Sxk2iy~NgOT=dfY3qN^E2;E!Zyud=(>k`Cj<=)}h z{0nSn>MQYd7lM4%HLT0o4#h8bAWawHt4_Iqmdcg*L9G@oV@;sxj0^OrsDX}K9(YS> zpqVd)oxQfq4GT9MzVe*t!en^)tC*DQPeyIN6}Dx06O}hgBtz*r$v*WMb4ovw6_VoQ z>8lcSvoWOM9^>_g(phJXa3kKWu_8StW+2mFO$=@+;ouHIuq|QtdCgXIRhI(@v)R>4 zyX~37I@hc%HUY7p5aBL;d;<6PnPcIeSX{RFE&kk_h-K~7G$nmLo{6+(RyfwdHum$y zSxyS+Y&A#6xByxc|CDGwWPMPlx8gX#`+iz5HKCGTyzKxrQ)9{Q%RiVDZWE3C z{hi#9eMQHfBw7$N(DP+5)vu9|)f|+&_t6@B)$sbcD)QjaU6NM(p5vx+ zp58y)4B9)-;n*!_H2N|CQ>|U#&(Kd={=AHc%?Y5dzrCiv>%@Wo_XaP;#hdWb!{Ci_ z7MZio4+`0teh?E&W7;EeV0i-Fv?i3e9=4@36d&W(A~QHNq5;V|5y-;Q;CGM)iIrar zCL*UnY-Te3)K^9P|NN!Kmh~99V?DY*vjvWnJB6#P{b`y3c)k%8y#W&KM#d8@XfFo8fB(3HU`dk-V<&VffeP z8I9e00pI1c!s4T85UQ~sUQS&DIU-#kfZ52l(CMH17C7!D%k6D{#Pjsl!qu-#(U!f7 z%+sl(t7mNEboZuU;95~EwQ?aH8}b+-t4>&+UXA~KW5|1hQZnmvBFmS2iw`etfrdlB z8A@d^Ko zuckk{f=NNY0tCE}~~7rxjC5?_i)%F^XH&-XS* z@8Jg6<1-sg27^E)u>-wi+lb#;W2*bFmfqLP;t9>Oz_MOHbgCaEt(P{_lE2^S_irQQ z(76P33oVBgYMU@~i6Buwmky`XGGVgHEb41m1$8wZ=pkT(JI(f!0xJ%6a#@J`S&pEg zRV9Qu^w1w04ETBezv~s6*e<2!EckPZ&v>Q&LWvlAcs)?SgxTD}>2mw1bX+UOTxv#7 zvq5H(q7+!0oTs)#oNi@VY37@R__IDhuVBN5 z-}}Ec`1%}z=n@CE^UwBr)Cv5Yo(pG63ZS>b3`+DD)AomVu<(aA)qm&-p1x0@=CC+7 zQZ@xg=9- zqF~K7)X^4)HLbe*ezWORne$KjhdJVQ%pKcMxf3POmP?#6xMNU>BoCgV8u%U@q|D4bov%}xh53$|7u0g zb3xQ@TP(S|{xU3B`-=oUa)(RDCX-56bqMR$fz=Y7oRFFb{N{LqUb^>%ZhLM9FFi-; zOz+Laz0`ziuD{Eu|M^7j$92N`w(CSLZ;+lSQwLcE4OD2B2EV>_X!u7DP@*0!Qin*% zr^{fn&IzN=y`}n*veY=j0QzPgN4?w2XyLdSy{U5&N4DRFjPcu4RI`!JH)J!iS02Ec zyw7m>;WIjZo=*fLhUsVD=e!EzNVHlh$*ml{Mr|K-F|(6DHT3_M-t5>9 z^wp0)5I-dc4_w#c&vF>V-Cx=XXVYCG@p=WlSA30=6PAolCHgdbtcI>yeUm744dV@q zW>CGkjpsdc8dWwchVk1kNaf~x;9Ho=OZN4Hr?%}-`Ktg5Fwv|M0(RVwYC+m5$y%tptg{VYPwS?#e1<4w2G%%|bRbKW5`ULwuAX5B^nqf_B{bvvzlv>2THH&G!oE`25t zgUfxSz+GM+=a@1eUDb-`**?HEEqSg;UOb%59HElOm!ifKN%WF8L#44HJR$A_>9Mb& z*+LH$d@~ZhVidcle?^UxnNd?b& z<%vfya^xND3Y7=`yfJ!GQww-0VlW(S3}5Qs5REboc(>{*O6Kh#(=O&CG4H}%GgxoN zkt#BjEQSI92GDeT9XDzAS#ls}oKw;miK6WPDUX>ARt0j<9wkqHTyY2Yn-}r(R~Lwl ztiS_3%ki=Cez0s8p} z@gz(txXEn&7R0y77W;mMK;1zf5ZGMFs2nQ>xw|(|sQM^~Zi<7eRzK?p%i8gl zrZ73Y@D0iPrNBK~J{>Cs#tE`Kz@Y|hjE`G~$`k9*$?h8JRf^&8>m`^v-4+z(El@!E z9C25nlvFy~}VYLIWWp4)hi^v2M{t zvSFGV95=0o+uP@2jpZl}f0sr)lMa(t`ZvfK#Rqik^d;aJ%|n}S=@=Hh3DWMzVb4kl zMx`bL(;g^5={dGz8#q7@TG~Lwf_7Y3Yy#^hb8*WcpFD3Eq2eZGkTGFT8m6S;+M?U! zjgKe1%+11pR5{+LMkN}^rop7qKxNOY34u}(d;=&r@dOrUFf}rWix%WKD<1P zPW5r6>Tbr^Q}08aa*yyH9$w7Ec6X6=IWK6_`y`&{xm9p|^*q+;odEYtHsYwvU1nhk z!s{s)$lZ|Tpsjd?*5GW+4!KL6qL)&;Ib8gH-vd>3*Wk2+i(zKDI=UPw0cW!`%rlxn zwpAIxtVjVW_upR1}8?FZVfD8|+K#PTK^b>VaFBHo)IHS}#okkRace+7=H z`pXmk1$L4RcXz;2dm}o$(VZ-tZHh)D66z0K<%LPF;$7C2Wnu8XgMj(b3&eBTJtC|W2}2!o(92~eh@~Ba zyKx#^nKgHCb+DHJ2CCI+YXtR|HMNo|EeGwKVpR3cT2!i-JSGJWF7CXk{VviHkD? z6iabOE4?6cyEdctu##+;A<7RuG7o;lusbr~S18E=mK{w4~{o68QcrwEnPHuN5 zih_B(B$FRxkj*cBJy?m0m7_5un2R|_Mj%`qxv6Y?^@;p$Nn*sUZ&=9}7)IowUqqo9b& zCp_?y@n;A=Uq-4Pnm`ioEIp*Gh;rZ8;BS{)2t6o*>dkIcOjv>#EPRX)^w|ExsXCb4 zZiH6S6CiHh22Yo$!ro+ndn@L_tm2Qv)@KB*mdk@}E$idfVqK%jn_#k}0{_97; z$mELx@OWK4Il8@=N*!eP|H=nBZ4YN*intb*Y1Gra&Nfs!7;UXTg~h_nTnriSMDSB~ zCRp*!XhYUVdhlKh+?LS>Dc>kO^+1|CzEqbs`DVi5fLAE_umgC@Wr@p#A#?MKA^f(= z$DYO0X!(i~x*+QW+&=P|Dt@~|e+vC074caBxyi=H#R)~ty;))&Lg)77x$L=>#l zH6el**!;2OVz4isfiEs=fR()--%BMJg1=jkwb^52Nt6^fia!SrIdh=HErv-7(S&9H z4Z)NYCsGpOO1<>*X}U%LI_=9RKZKu9F$0#n!HY!is$LLP@BnG~xAb1LBs;?|!y7Dj z?{?)$Sk*U;m$>j0GM>Q)@%Pk=0!Z*m*%CjiAaVgjSJDaPpNd`&Dd$2re0sqLQaz&v<{I3g0@eg6h}^-TOKi?#-;m`e$5_bIyP} zPPd>~qlWC5vL9os{AoaT3+y=cnOaG0hR|*mni42UbN7uh8}{Df)tm33-lzKE}_XC~cYxV_LfqFP!FG`0PUTcCNwfhKo?T z=MPP;y@;);?@89H*D$*6C7duVgx+)QTENfKkHPIFHv;iQP^py?>U{S=-^1^gM@m#s+;%`t|;L#H_^6(V?bK^jK= z34|NEM;Sjh+q3+_M<|bf$=G=PB(E1s@HM_JBN`tINVY) z0#jXq`UcB)Fk(yQeciN-qW9T{OhFiA^~5{Zu%~X5xhom0~r!zclk;XXD$m zFs<|z=dtS<8fyBJ?!Bvyvo^e>6_ZEk;0zi3x2FSjj-JKE+YC{7$raks_Kw`+Tq0BM zxAXSC?xMct`^l+FKMZHHQc*X3@aVu2u)ZM7x8-({iS92fJJylzFuskmR*qAq^$JAk z>7sGmR8V`?0ms$?IhPeyus)d*`TN*kCOxp!KEvnNW9vF?A(>n zj5n}`{laLwa|`U1NhZnP_?#8_#MEgr55bFs^BbeDCv6^K~&cUb&C4D^}7% zyJVDJl|mXVQpl}#e<+(H!BuO|fEV>^z;~%M1kO+bk8uvXoqihF+6HJy9V8>g)yQGb z>D`Caak=D4Xvkr3Y-vx!7bPLIyPZfC<+2Eq!>0+pK@HQUG(grxZ{D{TlX3jc7wF8J zg^qV-z%Ad$kh?^Pl-=J&`UVT(nu!?d=skfr^=zI?>^t7ih&${o(iJ#wE?|~%2xots z0cxH}!#9WWc*ou?g6l!~yeTjK;<`sVSiQoU7Z~FNJxxX+=EHh7mab%`sGE}EebX_) zPn}=;;WNEkok87?PvX{aLuk&vZk$+t4|ZqlZJ00d3G|z?VAOjlI174`sJE9G{hwSg z|K)_Ut~_CGvagftY)8AhWGj(7vWZvPdK#nL*~d;{6AeqRW8G1wLC%|p-=3$_h@QKY zbNLN9GT)AB`4i~=oyPWY9^=0kt;AWo1xM_zfP~3s65bO*HRkH^49hiOU{wX#lu%10 zo`%rFyB?v*A$eH++rh^R{&24K95yW6g~p4vV8%K{QW|%c?0Nf+q+4DkLx0oY0cSC$ zt9@rg>c62J>yvTipQ0x%j4_U7Tzg#h$L59x*lW85c6RL~yVUP7#Sa=V`^z47F6fF% zQ!kMIxzo8t-hBAba}JDVlt7~XL0F;^!wc+c$Bs%VnEgN(=gWO3!9i!)yIOX`7vs%% z>&klk^L8z?Ey*U!v>0j+ajO)lvTiBsLk@=; zWevK_KoRxYO6g8l*3;p01_SFq;w_#BXZ%tKyb)0$yRuJH3#l?H=5mr&4F3n(aZ(sQ zk;=0e*ov1mCh@0sWRuUU3|Ky15!tjhfc|opMTZ-=h`^bCY&}ql4z>w+QAd!@<-Q^} z?GtE&g%*g$9)M(RSsaqc<83ldrOp+nK|!(;hv&^>Rva@R-4mHad`}~}!{$e~{RWim z+eX{UO>xen%j9=3fu+-;;A_r%Gzu^Wy+7G-xGEN|z7xm3Sw;{Yu8X=Bt<=-~C@*N; z2YO7yntL;050$)X2zxFahQ^rhyqRUC#L>r>XSP|M9Ipw3l$wi}F&Y9+!rxf$oG0-; z`2`1JqSb-Ll^kZNS{Q`HLLepF&$N6W$Dy{ zEyVTDa-7J`f)gbXydNEsP}6arSmwMT{_b`#;Cd38)GNuLr57=qVTQ}uy_()KE&d*{ zR>nM2g*y?i1?x|rh9~be!2d%9r#35rn$7vtaLTzJOXLrMdctM8`fvd7oSN|NyH87;Ig>}6q_3ISCz_eU-d7i(bEHIlEP%z(GZK99FFj=#zfOxLwl+>a~>SbuchM$ zquE{(%Zlq$!r-wEmM1fc#(P>YGbIUot`)+U?OWltaUO1WC}w9AOW<2q3lyZW^N1-X zV3gvBzpt>oueMt3y{QhRKkkBIuL$hD`Ej?h8-xgT&pB|1x#G;V-6MFqa7+o8rKxX+qhwP7O zEF)EWmeKl|;AA9{;MeLy&$jm8Oei!4U5Ah~%sdgcBJ8m?(1-d*fe_|B`NyWR42@ z`4FmU!{*HEE|a>G6rI&Ih(z~vtl(zhiMBV?czYm>$7Mm}&vPW$x(WD-$HCmR?RPsoQ;0WmGU2>s05t1F;pbPi^x5@%#yB+q-nnF8lc5+mYh1z<$9p)pG!e|Q z*yocWQnt-H@QbmlG1@pMKDVu>(6`*J&>dDsd#?|1_+F_BLWf z`&)?LYX^JR|G>ZXAvosz3Kq^1;ruF}g>%*-Z@Al%%J+5AF!dfP=YI!BUL}K2bT#z< zy~r+~*xAw}X`Ec>NO&E5T6w^VCS2WvzhWw2_nuBF>2!;noGppAN(DrEpTe2X$7%;?K-^VO#_9M^l{#1P9asn-FgHi5t7c7)Yp~V{w=_r3TEErZN+@UyV zn7xni_4i@>J_oWsriTt6DWY2vM#%c*dT2a&4D6TbVz^5tvHrB4I4o<0Gx0xpZO6`| zP~luucYFk@5z(NQT8rkjxgfySYnN@2=ez4Xg|Ah5cxhlInZ2tNZ7k{`ZT=F}e-eP_ zCi+Qua5c@|^c3CZ+JSfEV%W555>D7JgBJhi=t2YVn)N8O>TSX%+vAX2&30{{_j7zN zN|SCjxBX{=-Jv;uWR8ujpr`*!WL6JN;P%l4_kH;wcBn6bcY{W_RIirU z>r12c@Exjf?JCEOF=PbRcM}iIA~yCrPWM3tbG0l9ZIGa|&JADRfUd)cL(n+1Su_+9tF#UC?SN-L=}V z@BY94zVrO`{`Y*p&-e2URJg!v;dy9#-vWnNRz;){1r`6q!Fua?nj=sncb_#VmzSOd z!QW?v{j8TwpYe6Clxo3XmOXX3_K?;vAFni<^+gp9lY_~8)SRlsLYAp};KJ@im%k^! z-$=s769KTcVz`Pox4O38} z)l2FkqQI^<8^4wnQQ7e@deUH4Ijs|o)_xDU$kre*=Js>5nr^h;wF*^J?l@{TCyXs{ zZ@#RziP~5eVQPynmA@>38P7F1=aC4P5h(HCoW2bP&UXL>yzD=KhSAz|k@4>MFUEGqRC0rKl z1>Wy0P)lI}hV~QaEeJz#&Q=(5O(iasL8!4HgUnm%uw66{3gXkjIlmEh*p<^U^IL2e zB`nu-hTh!4Zdub_U{#46-7W+QLp(MKM*{4yhH-IS))b(a{gd{GEIMO*8)h1kVa4u1 zND7tEBLX?jbxP5E%P$z|?*Y5*mBc=}4O_LGu&`gL3=i&s0mnp4>zqZ=_qDL|*I4?< zcsrbq@xk(WYtfkMFD$s&OOqzlHk6GUfJfR-*x@w{K?~b)l5rVw`wek{JedZ#_0aQn zb)dL;0W>6;u+|Bwle!;NobJHH5nayix;OXk9OCM2h9G60T}$?JJn-@pJ&`{|xU#Do z)_UoSq_@U#ub>IC6pI;~>k^n5WI%%HD7AIpM~YV5fuC0eV~!*bi+lK@h$mfGM<+qA zwhD;TELE${gRJ|_(5-s~b>8VTS2|14X15I2B^u$Kn>l3Y%sOi47Xq667i4S1Ihv-3 zfL`$z;-Ye-@l2>2YNz%AsU6# zV_brz0&m2{U`c%n?Yq}W>dS43X#NK2kx6h^Wkq)V!ADPa6W8;^q6U6tuAIdOg|1CET;Na>fr2if=uZ7i(DluVfo|5koHw3*8Y_i}(~S*-V@cwDF+_5j_c2O;#3Bk?a(upX}l?zk-v!x98!tA#0k5O>pw zdvYQ1mO{_d_4s*vJ?1HtWXkU@{a~#L9ktb5Xs#g`RbGVIZgtq>tBE$*u~54C2W+*B zr24sCG`&?0ADaYFKX(;!%Jb2;*Aj2#&k++Xiw%wI&%PM8bVZSi^5I!qAXSat=c=-YU}K*{bLq7T@V80@-*G2Q zpISznPR*d|PZ_kA8&UaV)>Ffmq5G#fDBXFSJF?>qPJZk|eV=$#v5fqS;Y=uN$slhe z=|caGv(W3C01b)Tns?dobPq@F4+#q2Ck+XY;hXYIv={M>cm{mE|DI!l4j+w>hD7t! fwbi66c^Z*NWBHamb8R)g$^VV8v2Q%pm+=1uro*jO literal 0 HcmV?d00001 diff --git a/ksmt-neurosmt/src/main/resources/encoder-ordinals b/ksmt-neurosmt/src/main/resources/encoder-ordinals new file mode 100644 index 000000000..c051615a6 --- /dev/null +++ b/ksmt-neurosmt/src/main/resources/encoder-ordinals @@ -0,0 +1,39 @@ += +SYMBOLIC;Address +SYMBOLIC;Array +SYMBOLIC;BitVec +SYMBOLIC;Bool +SYMBOLIC;FP +VALUE;Address +VALUE;BitVec +VALUE;Bool +VALUE;FP +VALUE;FP_RM +and +bvadd +bvand +bvashr +bvlshr +bvmul +bvneg +bvnot +bvor +bvsdiv +bvshl +bvsle +bvsrem +extract +fp.add +fp.eq +fp.isNaN +fp.leq +fp.lt +fp.mul +fp.to_fp +ite +not +or +select +sign_extend +store +zero_extend From 63740f79ee936cf2354f93346aca4d4720769bb5 Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Fri, 25 Aug 2023 17:16:24 +0300 Subject: [PATCH 51/52] update code for usvm-formulas export --- sandbox/src/main/kotlin/Main.kt | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/sandbox/src/main/kotlin/Main.kt b/sandbox/src/main/kotlin/Main.kt index 61d8807c1..fb9557a83 100644 --- a/sandbox/src/main/kotlin/Main.kt +++ b/sandbox/src/main/kotlin/Main.kt @@ -34,7 +34,7 @@ fun serialize(ctx: KContext, expressions: List>, outputStream: val marshaller = AstSerializationCtx.marshaller(serializationCtx) val emptyRdSerializationCtx = SerializationCtx(Serializers()) - val buffer = UnsafeBuffer(ByteArray(100_000)) + val buffer = UnsafeBuffer(ByteArray(20_000)) // ??? expressions.forEach { expr -> marshaller.write(emptyRdSerializationCtx, buffer, expr) @@ -63,26 +63,33 @@ fun deserialize(ctx: KContext, inputStream: InputStream): List> return expressions } +object MethodNameStorage { + val methodName = ThreadLocal() +} + class LogSolver( private val ctx: KContext, private val baseSolver: KSolver ) : KSolver by baseSolver { companion object { - val counter = AtomicLong(0) + val counter = ThreadLocal() } init { File("formulas").mkdirs() + File("formulas/${MethodNameStorage.methodName.get()}").mkdirs() + + counter.set(0) } private fun getNewFileCounter(): Long { - return synchronized(counter) { - counter.getAndIncrement() - } + val result = counter.get() + counter.set(result + 1) + return result } - private fun getNewFileName(): String { - return "formulas/f-${getNewFileCounter()}.bin" + private fun getNewFileName(suffix: String): String { + return "formulas/${MethodNameStorage.methodName.get()}/f-${getNewFileCounter()}-$suffix" } val stack = mutableListOf>>(mutableListOf()) @@ -110,13 +117,15 @@ class LogSolver( } override fun check(timeout: Duration): KSolverStatus { - serialize(ctx, stack.flatten(), FileOutputStream(getNewFileName())) - return baseSolver.check(timeout) + val result = baseSolver.check(timeout) + serialize(ctx, stack.flatten(), FileOutputStream(getNewFileName(result.toString().lowercase()))) + return result } override fun checkWithAssumptions(assumptions: List>, timeout: Duration): KSolverStatus { - serialize(ctx, stack.flatten() + assumptions, FileOutputStream(getNewFileName())) - return baseSolver.checkWithAssumptions(assumptions, timeout) + val result = baseSolver.checkWithAssumptions(assumptions, timeout) + serialize(ctx, stack.flatten() + assumptions, FileOutputStream(getNewFileName(result.toString().lowercase()))) + return result } } From 9fd38c49f1fa7f985d2bdd93c3e7d32bf1fd54d3 Mon Sep 17 00:00:00 2001 From: Stephen Ostapenko Date: Thu, 31 Aug 2023 03:45:16 +0300 Subject: [PATCH 52/52] remove sandboxes --- ksmt-neurosmt/src/main/python/sandbox.py | 106 ------ sandbox/build.gradle.kts | 59 --- sandbox/src/main/kotlin/Main.kt | 463 ----------------------- settings.gradle.kts | 1 - 4 files changed, 629 deletions(-) delete mode 100755 ksmt-neurosmt/src/main/python/sandbox.py delete mode 100644 sandbox/build.gradle.kts delete mode 100644 sandbox/src/main/kotlin/Main.kt diff --git a/ksmt-neurosmt/src/main/python/sandbox.py b/ksmt-neurosmt/src/main/python/sandbox.py deleted file mode 100755 index 381836b9d..000000000 --- a/ksmt-neurosmt/src/main/python/sandbox.py +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/python3 - -import sys -# import os; os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"; os.environ["CUDA_VISIBLE_DEVICES"] = os.environ["GPU"] - -import torch - -from pytorch_lightning import Trainer -from pytorch_lightning.loggers import TensorBoardLogger -from pytorch_lightning.callbacks import ModelCheckpoint - -from GraphDataloader import get_dataloader - -from LightningModel import LightningModel - -from GlobalConstants import EMBEDDING_DIM - - -if __name__ == "__main__": - - pl_model = LightningModel().load_from_checkpoint(sys.argv[1], map_location=torch.device("cpu")) - pl_model.eval() - - """ - trainer = Trainer( - accelerator="auto", - # precision="bf16-mixed", - logger=TensorBoardLogger("../logs", name="neuro-smt"), - callbacks=[ModelCheckpoint( - filename="epoch_{epoch:03d}_roc-auc_{val/roc-auc:.3f}", - monitor="val/roc-auc", - verbose=True, - save_last=True, save_top_k=3, mode="max", - auto_insert_metric_name=False, save_on_train_epoch_end=False - )], - max_epochs=200, - log_every_n_steps=1, - enable_checkpointing=True, - barebones=False, - default_root_dir=".." - ) - """ - - MAX_SIZE = 3 - - node_labels = torch.tensor([[i] for i in range(MAX_SIZE)], dtype=torch.int32) - edges = torch.tensor([ - [i for i in range(MAX_SIZE - 1)], - [i for i in range(1, MAX_SIZE)] - ], dtype=torch.int64) - depths = torch.tensor([MAX_SIZE - 1], dtype=torch.int64) - root_ptrs = torch.tensor([0, MAX_SIZE], dtype=torch.int64) - - torch.onnx.export( - pl_model.model.encoder.embedding, - (node_labels,), - "embeddings.onnx", - opset_version=18, - input_names=["node_labels"], - output_names=["out"], - dynamic_axes={ - "node_labels": {0: "nodes_number"} - } - ) - - torch.onnx.export( - pl_model.model.encoder.conv, - (torch.rand((node_labels.shape[0], EMBEDDING_DIM)), edges), - "conv.onnx", - opset_version=18, - input_names=["node_features", "edges"], - output_names=["out"], - dynamic_axes={ - "node_features": {0: "nodes_number"}, - "edges": {1: "edges_number"} - } - ) - - torch.onnx.export( - pl_model.model.decoder, - (torch.rand((1, EMBEDDING_DIM)),), - "decoder.onnx", - opset_version=18, - input_names=["expr_features"], - output_names=["out"], - dynamic_axes={ - "expr_features": {0: "batch_size"} - } - ) - - """ - pl_model.to_onnx( - "kek.onnx", - (node_labels, edges, depths, root_ptrs), - opset_version=18, - input_names=["node_labels", "edges", "depths", "root_ptrs"], - output_names=["output"], - dynamic_axes={ - "node_labels": {0: "nodes_number"}, - "edges": {1: "edges_number"}, - "depths": {0: "batch_size"}, - "root_ptrs": {0: "batch_size_+_1"}, - }, - # verbose=True - ) - """ diff --git a/sandbox/build.gradle.kts b/sandbox/build.gradle.kts deleted file mode 100644 index 3839eeb1a..000000000 --- a/sandbox/build.gradle.kts +++ /dev/null @@ -1,59 +0,0 @@ -plugins { - kotlin("jvm") - application -} - -group = "org.example" -version = "unspecified" - -repositories { - mavenCentral() -} - -dependencies { - testImplementation("org.junit.jupiter:junit-jupiter-api:5.8.1") - testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.8.1") - - implementation(project(":ksmt-core")) - implementation(project(":ksmt-z3")) - implementation(project(":ksmt-neurosmt")) - implementation(project(":ksmt-neurosmt:utils")) - implementation(project(":ksmt-runner")) - - implementation("com.microsoft.onnxruntime:onnxruntime:1.15.1") - // implementation("com.microsoft.onnxruntime:onnxruntime_gpu:1.15.1") - - implementation("me.tongfei:progressbar:0.9.4") -} - -tasks.getByName("test") { - useJUnitPlatform() -} - -application { - mainClass.set("MainKt") -} - -tasks { - val fatJar = register("fatJar") { - dependsOn.addAll(listOf("compileJava", "compileKotlin", "processResources")) - - archiveFileName.set("sandbox.jar") - destinationDirectory.set(File(".")) - duplicatesStrategy = DuplicatesStrategy.EXCLUDE - - manifest { - attributes(mapOf("Main-Class" to application.mainClass)) - } - - val sourcesMain = sourceSets.main.get() - val contents = configurations.runtimeClasspath.get() - .map { if (it.isDirectory) it else zipTree(it) } + sourcesMain.output - - from(contents) - } - - build { - dependsOn(fatJar) - } -} \ No newline at end of file diff --git a/sandbox/src/main/kotlin/Main.kt b/sandbox/src/main/kotlin/Main.kt deleted file mode 100644 index fb9557a83..000000000 --- a/sandbox/src/main/kotlin/Main.kt +++ /dev/null @@ -1,463 +0,0 @@ -import ai.onnxruntime.OnnxTensor -import ai.onnxruntime.OrtEnvironment -import com.jetbrains.rd.framework.SerializationCtx -import com.jetbrains.rd.framework.Serializers -import com.jetbrains.rd.framework.SocketWire.Companion.timeout -import com.jetbrains.rd.framework.UnsafeBuffer -import io.ksmt.KContext -import io.ksmt.expr.* -import io.ksmt.runner.serializer.AstSerializationCtx -import io.ksmt.solver.KSolver -import io.ksmt.solver.KSolverConfiguration -import io.ksmt.solver.KSolverStatus -import io.ksmt.solver.neurosmt.getAnswerForTest -import io.ksmt.solver.neurosmt.runtime.NeuroSMTModelRunner -import io.ksmt.solver.z3.* -import io.ksmt.sort.* -import io.ksmt.utils.getValue -import io.ksmt.utils.uncheckedCast -import me.tongfei.progressbar.ProgressBar -import java.io.* -import java.nio.FloatBuffer -import java.nio.LongBuffer -import java.nio.file.Files -import java.nio.file.Path -import java.util.concurrent.atomic.AtomicLong -import kotlin.io.path.isRegularFile -import kotlin.io.path.name -import kotlin.io.path.pathString -import kotlin.time.Duration -import kotlin.time.Duration.Companion.seconds - -fun serialize(ctx: KContext, expressions: List>, outputStream: OutputStream) { - val serializationCtx = AstSerializationCtx().apply { initCtx(ctx) } - val marshaller = AstSerializationCtx.marshaller(serializationCtx) - val emptyRdSerializationCtx = SerializationCtx(Serializers()) - - val buffer = UnsafeBuffer(ByteArray(20_000)) // ??? - - expressions.forEach { expr -> - marshaller.write(emptyRdSerializationCtx, buffer, expr) - } - - outputStream.write(buffer.getArray()) - outputStream.flush() -} - -fun deserialize(ctx: KContext, inputStream: InputStream): List> { - val srcSerializationCtx = AstSerializationCtx().apply { initCtx(ctx) } - val srcMarshaller = AstSerializationCtx.marshaller(srcSerializationCtx) - val emptyRdSerializationCtx = SerializationCtx(Serializers()) - - val buffer = UnsafeBuffer(inputStream.readBytes()) - val expressions: MutableList> = mutableListOf() - - while (true) { - try { - expressions.add(srcMarshaller.read(emptyRdSerializationCtx, buffer).uncheckedCast()) - } catch (e : IllegalStateException) { - break - } - } - - return expressions -} - -object MethodNameStorage { - val methodName = ThreadLocal() -} - -class LogSolver( - private val ctx: KContext, private val baseSolver: KSolver -) : KSolver by baseSolver { - - companion object { - val counter = ThreadLocal() - } - - init { - File("formulas").mkdirs() - File("formulas/${MethodNameStorage.methodName.get()}").mkdirs() - - counter.set(0) - } - - private fun getNewFileCounter(): Long { - val result = counter.get() - counter.set(result + 1) - return result - } - - private fun getNewFileName(suffix: String): String { - return "formulas/${MethodNameStorage.methodName.get()}/f-${getNewFileCounter()}-$suffix" - } - - val stack = mutableListOf>>(mutableListOf()) - - override fun assert(expr: KExpr) { - stack.last().add(expr) - baseSolver.assert(expr) - } - - override fun assertAndTrack(expr: KExpr) { - stack.last().add(expr) - baseSolver.assertAndTrack(expr) - } - - override fun push() { - stack.add(mutableListOf()) - baseSolver.push() - } - - override fun pop(n: UInt) { - repeat(n.toInt()) { - stack.removeLast() - } - baseSolver.pop(n) - } - - override fun check(timeout: Duration): KSolverStatus { - val result = baseSolver.check(timeout) - serialize(ctx, stack.flatten(), FileOutputStream(getNewFileName(result.toString().lowercase()))) - return result - } - - override fun checkWithAssumptions(assumptions: List>, timeout: Duration): KSolverStatus { - val result = baseSolver.checkWithAssumptions(assumptions, timeout) - serialize(ctx, stack.flatten() + assumptions, FileOutputStream(getNewFileName(result.toString().lowercase()))) - return result - } -} - -const val THRESHOLD = 0.5 - -fun main() { - - val ctx = KContext( - astManagementMode = KContext.AstManagementMode.NO_GC, - simplificationMode = KContext.SimplificationMode.NO_SIMPLIFY - ) - - val pathToDataset = "formulas" - val files = Files.walk(Path.of(pathToDataset)).filter { it.isRegularFile() }.toList() - - val runner = NeuroSMTModelRunner( - ctx, - ordinalsPath = "usvm-enc-2.cats", - embeddingPath = "embeddings.onnx", - convPath = "conv.onnx", - decoderPath = "decoder.onnx" - ) - - var sat = 0; var unsat = 0; var skipped = 0 - var ok = 0; var wa = 0 - - val confusionMatrix = mutableMapOf, Int>() - - files.forEachIndexed sample@{ ind, it -> - if (ind % 100 == 0) { - println("#$ind: $ok / $wa [$sat / $unsat / $skipped]") - println(confusionMatrix) - println() - } - - val sampleFile = File(it.pathString) - - val assertList = try { - deserialize(ctx, FileInputStream(sampleFile)) - } catch (e: Exception) { - skipped++ - return@sample - } - - val answer = when { - it.name.endsWith("-sat") -> KSolverStatus.SAT - it.name.endsWith("-unsat") -> KSolverStatus.UNSAT - else -> KSolverStatus.UNKNOWN - } - - if (answer == KSolverStatus.UNKNOWN) { - skipped++ - return@sample - } - - val prob = with(ctx) { - val formula = when (assertList.size) { - 0 -> { - skipped++ - return@sample - } - 1 -> { - assertList[0] - } - else -> { - mkAnd(assertList) - } - } - - runner.run(formula) - } - - val output = if (prob < THRESHOLD) { - KSolverStatus.UNSAT - } else { - KSolverStatus.SAT - } - - when (answer) { - KSolverStatus.SAT -> sat++ - KSolverStatus.UNSAT -> unsat++ - else -> { /* can't happen */ } - } - - if (output == answer) { - ok++ - } else { - wa++ - } - - confusionMatrix.compute(answer to output) { _, v -> - if (v == null) { - 1 - } else { - v + 1 - } - } - } - - println() - println("sat: $sat; unsat: $unsat; skipped: $skipped") - println("ok: $ok; wa: $wa") - - return - - /* - with(ctx) { - val a by boolSort - val b by intSort - val c by intSort - val d by realSort - val e by bv8Sort - val f by fp64Sort - val g by mkArraySort(realSort, boolSort) - val h by fp64Sort - - val expr = mkBvXorExpr(mkBvShiftLeftExpr(e, mkBv(1.toByte())), mkBvNotExpr(e)) eq - mkBvLogicalShiftRightExpr(e, mkBv(1.toByte())) - - val runner = NeuroSMTModelRunner(ctx, "usvm-enc-2.cats", "embeddings.onnx", "conv.onnx", "decoder.onnx") - println(runner.run(expr)) - } - - return - - */ - - val env = OrtEnvironment.getEnvironment() - //val session = env.createSession("kek.onnx") - val session = env.createSession("conv.onnx") - - println(session.inputNames) - for (info in session.inputInfo) { - println(info) - } - - println() - - println(session.outputNames) - for (info in session.outputInfo) { - println(info) - } - - println() - - println(session.metadata) - println() - - //val nodeLabels = listOf(listOf(0L), listOf(1L), listOf(2L), listOf(3L), listOf(4L), listOf(5L), listOf(6L)) - val nodeLabels = listOf(listOf(0L), listOf(1L), listOf(2L)) - //val nodeFeatures = (1..7).map { (0..31).map { it / 31.toFloat() } } - val nodeFeatures = (1..3).map { (0..31).map { it / 31.toFloat() } } - val edges = listOf( - listOf(0L, 1L), - listOf(2L, 2L) - //listOf(0L, 1L, 2L, 3L, 4L, 5L), - //listOf(1L, 2L, 3L, 4L, 5L, 6L) - ) - val depths = listOf(1L) - val rootPtrs = listOf(0L, 3L) - - /* - val nodeLabels = listOf(listOf(0L), listOf(1L), listOf(1L), listOf(0L)) - val edges = listOf( - listOf(0L, 0L), - listOf(1L, 1L) - ) - val depths = listOf(1L, 1L) - val rootPtrs = listOf(0L, 1L, 2L) - */ - - val nodeLabelsBuffer = LongBuffer.allocate(nodeLabels.sumOf { it.size }) - nodeLabels.forEach { features -> - features.forEach { feature -> - nodeLabelsBuffer.put(feature) - } - } - nodeLabelsBuffer.rewind() - - val nodeFeaturesBuffer = FloatBuffer.allocate(nodeFeatures.sumOf { it.size }) - nodeFeatures.forEach { features -> - features.forEach { feature -> - nodeFeaturesBuffer.put(feature) - } - } - nodeFeaturesBuffer.rewind() - - val edgesBuffer = LongBuffer.allocate(edges.sumOf { it.size }) - edges.forEach { row -> - row.forEach { node -> - edgesBuffer.put(node) - } - } - edgesBuffer.rewind() - - val depthsBuffer = LongBuffer.allocate(depths.size) - depths.forEach { d -> - depthsBuffer.put(d) - } - depthsBuffer.rewind() - - val rootPtrsBuffer = LongBuffer.allocate(rootPtrs.size) - rootPtrs.forEach { r -> - rootPtrsBuffer.put(r) - } - rootPtrsBuffer.rewind() - - val nodeLabelsData = OnnxTensor.createTensor(env, nodeLabelsBuffer, listOf(nodeLabels.size.toLong(), nodeLabels[0].size.toLong()).toLongArray()) - val nodeFeaturesData = OnnxTensor.createTensor(env, nodeFeaturesBuffer, listOf(nodeFeatures.size.toLong(), nodeFeatures[0].size.toLong()).toLongArray()) - val edgesData = OnnxTensor.createTensor(env, edgesBuffer, listOf(edges.size.toLong(), edges[0].size.toLong()).toLongArray()) - val depthsData = OnnxTensor.createTensor(env, depthsBuffer, listOf(depths.size.toLong()).toLongArray()) - val rootPtrsData = OnnxTensor.createTensor(env, rootPtrsBuffer, listOf(rootPtrs.size.toLong()).toLongArray()) - - /* - val result = session.run(mapOf("node_labels" to nodeLabelsData, "edges" to edgesData, "depths" to depthsData, "root_ptrs" to rootPtrsData)) - val output = (result.get("output").get().value as Array<*>).map { - (it as FloatArray).toList() - } - */ - - var curFeatures = nodeFeaturesData - repeat(10) { - val result = session.run(mapOf("node_features" to curFeatures, "edges" to edgesData)) - curFeatures = result.get(0) as OnnxTensor - println(curFeatures.info.shape.toList()) - curFeatures.info.shape - println(curFeatures.floatBuffer.array().toList().subList(64, 96)) - } - - /* - val output = (result.get("out").get().value as Array<*>).map { - (it as FloatArray).toList() - } - - println(output) - */ - - - /* - val ctx = KContext() - - with(ctx) { - val files = Files.walk(Path.of("/home/stephen/Desktop/formulas")).filter { it.isRegularFile() } - - var ok = 0; var fail = 0 - files.forEach { - try { - println(deserialize(ctx, FileInputStream(it.toFile())).size) - ok++ - } catch (e: Exception) { - fail++ - } - } - println("$ok / $fail") - }*/ - - /* - with(ctx) { - // create symbolic variables - val a by boolSort - val b by intSort - val c by intSort - val d by realSort - val e by bv8Sort - val f by fp64Sort - val g by mkArraySort(realSort, boolSort) - val h by fp64Sort - - val x by intSort - - // create an expression - val constraint = a and (b ge c + 3.expr) and (b * 2.expr gt 8.expr) and - ((e eq mkBv(3.toByte())) or (d neq mkRealNum("271/100"))) and - (f eq 2.48.expr) and - (d + b.toRealExpr() neq mkRealNum("12/10")) and - (mkArraySelect(g, d) eq a) and - (mkArraySelect(g, d + mkRealNum(1)) neq a) and - (mkArraySelect(g, d - mkRealNum(1)) eq a) and - (d * d eq mkRealNum(4)) and - (mkBvMulExpr(e, e) eq mkBv(9.toByte())) - - val formula = """ - (declare-fun x () Real) - (declare-fun y () Real) - (declare-fun z () Real) - (declare-fun a () Int) - (assert (or (and (= y (+ x z)) (= x (+ y z))) (= 2.71 x))) - (assert (= a 2)) - (check-sat) - """ - - val assertions = mkAnd(KZ3SMTLibParser(this).parse(formula)) - - val bvExpr = mkBvXorExpr(mkBvShiftLeftExpr(e, mkBv(1.toByte())), mkBvNotExpr(e)) eq - mkBvLogicalShiftRightExpr(e, mkBv(1.toByte())) - - val buf = ByteArrayOutputStream() - serialize(ctx, listOf(constraint, assertions, bvExpr), buf) - deserialize(ctx, ByteArrayInputStream(buf.toByteArray())).forEach { - println("nxt: $it") - } - - /* - KExprUninterpretedDeclCollector.collectUninterpretedDeclarations(mkAnd(assertions)).forEach { - println("${it.name} | ${it.argSorts} | ${it.sort}") - } - */ - - KZ3Solver(this).use { solver -> - LogSolver(this, solver).use { solver -> - solver.assert(constraint) - - val satisfiability = solver.check(timeout = 180.seconds) - println(satisfiability) - - if (satisfiability == KSolverStatus.SAT) { - val model = solver.model() - println(model) - } - } - } - - /* - KNeuroSMTSolver(this).use { solver -> // create a Stub SMT solver instance - // assert expression - solver.assert(constraint) - - // check assertions satisfiability with timeout - val satisfiability = solver.check(timeout = 1.seconds) - // println(satisfiability) // SAT - } - */ - } - */ -} \ No newline at end of file diff --git a/settings.gradle.kts b/settings.gradle.kts index 850a5c3e8..1a9014436 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -20,4 +20,3 @@ pluginManagement { include("ksmt-neurosmt") include("ksmt-neurosmt:utils") // findProject(":ksmt-neurosmt:utils")?.name = "utils" -include("sandbox")