From d009a99d6070134d0436605f309f3960ad69f758 Mon Sep 17 00:00:00 2001 From: Tim Poterba Date: Sun, 9 Feb 2020 17:22:29 -0500 Subject: [PATCH 1/8] [hail/ptypes] Support ptype inference for lowered aggs --- .../main/scala/is/hail/expr/ir/AggOp.scala | 10 +- .../main/scala/is/hail/expr/ir/BaseIR.scala | 2 + .../main/scala/is/hail/expr/ir/Compile.scala | 15 +- hail/src/main/scala/is/hail/expr/ir/IR.scala | 1 + .../scala/is/hail/expr/ir/InferPType.scala | 181 +++++++++++++++++- .../scala/is/hail/expr/ir/Interpret.scala | 18 +- .../main/scala/is/hail/expr/ir/TableIR.scala | 49 +++-- .../scala/is/hail/expr/ir/agg/Extract.scala | 32 +++- .../is/hail/expr/types/physical/PType.scala | 1 + .../is/hail/expr/ir/Aggregators2Suite.scala | 11 +- .../test/scala/is/hail/expr/ir/IRSuite.scala | 2 +- 11 files changed, 264 insertions(+), 58 deletions(-) diff --git a/hail/src/main/scala/is/hail/expr/ir/AggOp.scala b/hail/src/main/scala/is/hail/expr/ir/AggOp.scala index 6437476f6a6..f18d0f69894 100644 --- a/hail/src/main/scala/is/hail/expr/ir/AggOp.scala +++ b/hail/src/main/scala/is/hail/expr/ir/AggOp.scala @@ -49,14 +49,6 @@ case class AggStatePhysicalSignature(m: Map[AggOp, PhysicalAggSignature], defaul def lookup(op: AggOp): PhysicalAggSignature = m(op) } -object PhysicalAggSignature { - def apply(op: AggOp, - physicalInitOpArgs: Seq[PType], - physicalSeqOpArgs: Seq[PType], - nested: Option[Seq[AggStatePhysicalSignature]] - ): PhysicalAggSignature = PhysicalAggSignature(op, physicalInitOpArgs, physicalSeqOpArgs, nested) -} - object AggStatePhysicalSignature { def apply(sig: PhysicalAggSignature): AggStatePhysicalSignature = AggStatePhysicalSignature(Map(sig.op -> sig), sig.op) } @@ -70,6 +62,8 @@ case class PhysicalAggSignature( def seqOpArgs: Seq[Type] = physicalSeqOpArgs.map(_.virtualType) lazy val virtual: AggSignature = AggSignature(op, physicalInitOpArgs.map(_.virtualType), physicalSeqOpArgs.map(_.virtualType)) + lazy val singletonContainer: AggStatePhysicalSignature = AggStatePhysicalSignature(Map(op -> this), op, None) + } sealed trait AggOp {} diff --git a/hail/src/main/scala/is/hail/expr/ir/BaseIR.scala b/hail/src/main/scala/is/hail/expr/ir/BaseIR.scala index 6afc59fbd62..89f9a1eaa74 100644 --- a/hail/src/main/scala/is/hail/expr/ir/BaseIR.scala +++ b/hail/src/main/scala/is/hail/expr/ir/BaseIR.scala @@ -11,6 +11,8 @@ abstract class BaseIR { def deepCopy(): this.type = copy(newChildren = children.map(_.deepCopy())).asInstanceOf[this.type] + lazy val noSharing: this.type = if (HasIRSharing(this)) this.deepCopy() else this + def mapChildren(f: (BaseIR) => BaseIR): BaseIR = { val newChildren = children.map(f) if ((children, newChildren).zipped.forall(_ eq _)) diff --git a/hail/src/main/scala/is/hail/expr/ir/Compile.scala b/hail/src/main/scala/is/hail/expr/ir/Compile.scala index 673fea02a8b..dc17d164bc4 100644 --- a/hail/src/main/scala/is/hail/expr/ir/Compile.scala +++ b/hail/src/main/scala/is/hail/expr/ir/Compile.scala @@ -212,7 +212,7 @@ object CompileWithAggregators2 { private def apply[F >: Null : TypeInfo, R: TypeInfo : ClassTag]( ctx: ExecuteContext, - aggSigs: Array[AggStateSignature], + aggSigs: Array[AggStatePhysicalSignature], args: Seq[(String, PType, ClassTag[_])], argTypeInfo: Array[MaybeGenericTypeInfo[_]], body: IR, @@ -221,8 +221,7 @@ object CompileWithAggregators2 { val normalizeNames = new NormalizeNames(_.toString) val normalizedBody = normalizeNames(body, Env(args.map { case (n, _, _) => n -> n }: _*)) - val pAggSigs = aggSigs.map(_.toCanonicalPhysical) - val k = CodeCacheKey(pAggSigs.toFastIndexedSeq, args.map { case (n, pt, _) => (n, pt) }, normalizedBody) + val k = CodeCacheKey(aggSigs, args.map { case (n, pt, _) => (n, pt) }, normalizedBody) codeCache.get(k) match { case Some(v) => return (v.typ, v.f.asInstanceOf[(Int, Region) => (F with FunctionWithAggRegion)]) @@ -243,7 +242,7 @@ object CompileWithAggregators2 { assert(TypeToIRIntermediateClassTag(ir.typ) == classTag[R]) - Emit(ctx, ir, fb, Some(pAggSigs)) + Emit(ctx, ir, fb, Some(aggSigs)) val f = fb.resultWithIndex() codeCache += k -> CodeCacheValue(ir.pType, f) @@ -252,7 +251,7 @@ object CompileWithAggregators2 { def apply[F >: Null : TypeInfo, R: TypeInfo : ClassTag]( ctx: ExecuteContext, - aggSigs: Array[AggStateSignature], + aggSigs: Array[AggStatePhysicalSignature], args: Seq[(String, PType, ClassTag[_])], body: IR, optimize: Boolean @@ -273,7 +272,7 @@ object CompileWithAggregators2 { def apply[R: TypeInfo : ClassTag]( ctx: ExecuteContext, - aggSigs: Array[AggStateSignature], + aggSigs: Array[AggStatePhysicalSignature], body: IR): (PType, (Int, Region) => AsmFunction1[Region, R] with FunctionWithAggRegion) = { apply[AsmFunction1[Region, R], R](ctx, aggSigs, FastSeq[(String, PType, ClassTag[_])](), body, optimize = true) @@ -281,7 +280,7 @@ object CompileWithAggregators2 { def apply[T0: ClassTag, R: TypeInfo : ClassTag]( ctx: ExecuteContext, - aggSigs: Array[AggStateSignature], + aggSigs: Array[AggStatePhysicalSignature], name0: String, typ0: PType, body: IR): (PType, (Int, Region) => AsmFunction3[Region, T0, Boolean, R] with FunctionWithAggRegion) = { @@ -290,7 +289,7 @@ object CompileWithAggregators2 { def apply[T0: ClassTag, T1: ClassTag, R: TypeInfo : ClassTag]( ctx: ExecuteContext, - aggSigs: Array[AggStateSignature], + aggSigs: Array[AggStatePhysicalSignature], name0: String, typ0: PType, name1: String, typ1: PType, body: IR): (PType, (Int, Region) => (AsmFunction5[Region, T0, Boolean, T1, Boolean, R] with FunctionWithAggRegion)) = { diff --git a/hail/src/main/scala/is/hail/expr/ir/IR.scala b/hail/src/main/scala/is/hail/expr/ir/IR.scala index c5fa1aed63c..75836ebc506 100644 --- a/hail/src/main/scala/is/hail/expr/ir/IR.scala +++ b/hail/src/main/scala/is/hail/expr/ir/IR.scala @@ -244,6 +244,7 @@ final case class ArrayAggScan(a: IR, name: String, query: IR) extends IR trait InferredPhysicalAggSignature { // will be filled in by InferPType in subsequent PR def signature: IndexedSeq[AggStateSignature] + var physicalSignatures2: Array[AggStatePhysicalSignature] = _ val physicalSignatures: Array[AggStatePhysicalSignature] = signature.map(_.toCanonicalPhysical).toArray } final case class RunAgg(body: IR, result: IR, signature: IndexedSeq[AggStateSignature]) extends IR with InferredPhysicalAggSignature diff --git a/hail/src/main/scala/is/hail/expr/ir/InferPType.scala b/hail/src/main/scala/is/hail/expr/ir/InferPType.scala index bef0a72ab2a..a45cc8ee31c 100644 --- a/hail/src/main/scala/is/hail/expr/ir/InferPType.scala +++ b/hail/src/main/scala/is/hail/expr/ir/InferPType.scala @@ -5,6 +5,66 @@ import is.hail.expr.types.virtual.{TNDArray, TVoid} import is.hail.utils._ object InferPType { + + def clearPTypes(x: IR): Unit = { + x._pType2 = null + x.children.foreach { c => clearPTypes(c.asInstanceOf[IR]) } + } + + // does not unify physical arg types if multiple nested seq/init ops appear; instead takes the first. The emitter checks equality. + def computePhysicalAgg(virt: AggStateSignature, initsAB: ArrayBuilder[RecursiveArrayBuilderElement[InitOp]], + seqAB: ArrayBuilder[RecursiveArrayBuilderElement[SeqOp]]): AggStatePhysicalSignature = { + val inits = initsAB.result() + val seqs = seqAB.result() + assert(inits.nonEmpty) + assert(seqs.nonEmpty) + virt.default match { + case AggElementsLengthCheck() => + + val iHead = inits.find(_.value.op == AggElementsLengthCheck()).get + val iNested = iHead.nested.get + val iHeadArgTypes = iHead.value.args.map(_.pType2) + + val sLCHead = seqs.find(_.value.op == AggElementsLengthCheck()).get + val sLCArgTypes = sLCHead.value.args.map(_.pType2) + val sAEHead = seqs.find(_.value.op == AggElements()).get + val sNested = sAEHead.nested.get + val sHeadArgTypes = sAEHead.value.args.map(_.pType2) + + val vNested = virt.nested.get.toArray + + val nested = vNested.indices.map { i => computePhysicalAgg(vNested(i), iNested(i), sNested(i)) } + AggStatePhysicalSignature(Map( + AggElementsLengthCheck() -> PhysicalAggSignature(AggElementsLengthCheck(), iHeadArgTypes, sLCArgTypes), + AggElements() -> PhysicalAggSignature(AggElements(), FastIndexedSeq(), sHeadArgTypes) + ), AggElementsLengthCheck(), Some(nested)) + + case Group() => + val iHead = inits.head + val iNested = iHead.nested.get + val iHeadArgTypes = iHead.value.args.map(_.pType2) + + val sHead = seqs.head + val sNested = sHead.nested.get + val sHeadArgTypes = sHead.value.args.map(_.pType2) + + val vNested = virt.nested.get.toArray + + val nested = vNested.indices.map { i => computePhysicalAgg(vNested(i), iNested(i), sNested(i)) } + val psig = PhysicalAggSignature(Group(), iHeadArgTypes, sHeadArgTypes) + AggStatePhysicalSignature(Map(Group() -> psig), Group(), Some(nested)) + + case _ => + assert(inits.forall(_.nested.isEmpty)) + assert(seqs.forall(_.nested.isEmpty)) + val iHead = inits.head.value + val iHeadArgTypes = iHead.args.map(_.pType2) + val sHead = seqs.head.value + val sHeadArgTypes = sHead.args.map(_.pType2) + virt.defaultSignature.toPhysical(iHeadArgTypes, sHeadArgTypes).singletonContainer + } + } + def getNestedElementPTypes(ptypes: Seq[PType]): PType = { assert(ptypes.forall(_.virtualType.isOfType(ptypes.head.virtualType))) getNestedElementPTypesOfSameType(ptypes: Seq[PType]) @@ -38,10 +98,20 @@ object InferPType { } } - def apply(ir: IR, env: Env[PType]): Unit = { - assert(ir._pType2 == null) + def apply(ir: IR, env: Env[PType]): Unit = apply(ir, env, null, null, null) + + private type AAB[T] = Array[ArrayBuilder[RecursiveArrayBuilderElement[T]]] - def infer(ir: IR, env: Env[PType] = env): Unit = apply(ir, env) + case class RecursiveArrayBuilderElement[T](value: T, nested: Option[AAB[T]]) + + def newBuilder[T](n: Int): AAB[T] = Array.fill(n)(new ArrayBuilder[RecursiveArrayBuilderElement[T]]) + + def apply(ir: IR, env: Env[PType], aggs: Array[AggStatePhysicalSignature], inits: AAB[InitOp], seqs: AAB[SeqOp]): Unit = { + if (ir._pType2 != null) + throw new RuntimeException(ir.toString) + + def infer(ir: IR, env: Env[PType] = env, aggs: Array[AggStatePhysicalSignature] = aggs, + inits: AAB[InitOp] = inits, seqs: AAB[SeqOp] = seqs): Unit = apply(ir, env, aggs, inits, seqs) ir._pType2 = ir match { case I32(_) => PInt32(true) @@ -214,6 +284,11 @@ object InferPType { assert(body.pType2 isOfType zero.pType2) zero.pType2.setRequired(body.pType2.required) + case ArrayFor(a, value, body) => + infer(a) + + infer(body, env.bind(value -> a.pType2.asInstanceOf[PArray].elementType)) + PVoid case ArrayFold2(a, acc, valueName, seq, res) => infer(a) acc.foreach { case (_, accIR) => infer(accIR) } @@ -400,15 +475,103 @@ object InferPType { infer(theIR) theIR._pType2 })), true) - case ResultOp(_, aggSigs) => - val rPTypes = aggSigs.toIterator.zipWithIndex.map{ case (sig, i) => PTupleField(i, sig.toCanonicalPhysical.resultType)}.toIndexedSeq - val allReq = rPTypes.forall(f => f.typ.required) - PCanonicalTuple(rPTypes, allReq) - case _: AggLet | _: RunAgg | _: RunAggScan | _: NDArrayAgg | _: AggFilter | _: AggExplode | - _: AggGroupBy | _: AggArrayPerElement | _: ApplyAggOp | _: ApplyScanOp | _: AggStateValue => PType.canonical(ir.typ) + case x@InitOp(i, args, sig, op) => + op match { + case Group() => + val nested = sig.nested.get + val newInits = newBuilder[InitOp](nested.length) + val IndexedSeq(initArg) = args + infer(initArg, env, null, inits = newInits, seqs = null) + if (inits != null) + inits(i) += RecursiveArrayBuilderElement(x, Some(newInits)) + case AggElementsLengthCheck() => + val nested = sig.nested.get + val newInits = newBuilder[InitOp](nested.length) + val initArg = args match { + case Seq(len, initArg) => + infer(len, env, null, null, null) + initArg + case Seq(initArg) => initArg + } + infer(initArg, env, null, inits = newInits, seqs = null) + if (inits != null) + inits(i) += RecursiveArrayBuilderElement(x, Some(newInits)) + case _ => + assert(sig.nested.isEmpty) + args.foreach(infer(_, env, null, null, null)) + if (inits != null) + inits(i) += RecursiveArrayBuilderElement(x, None) + } + PVoid + + case x@SeqOp(i, args, sig, op) => + op match { + case Group() => + val nested = sig.nested.get + val newSeqs = newBuilder[SeqOp](nested.length) + val IndexedSeq(k, seqArg) = args + infer(k, env, null, inits = null, seqs = null) + infer(seqArg, env, null, inits = null, seqs = newSeqs) + if (seqs != null) + seqs(i) += RecursiveArrayBuilderElement(x, Some(newSeqs)) + case AggElements() => + val nested = sig.nested.get + val newSeqs = newBuilder[SeqOp](nested.length) + val IndexedSeq(idx, seqArg) = args + infer(idx, env, null, inits = null, seqs = null) + infer(seqArg, env, null, inits = null, seqs = newSeqs) + if (seqs != null) + seqs(i) += RecursiveArrayBuilderElement(x, Some(newSeqs)) + case AggElementsLengthCheck() => + val nested = sig.nested.get + val IndexedSeq(idx) = args + infer(idx, env, null, inits = null, seqs = null) + if (seqs != null) + seqs(i) += RecursiveArrayBuilderElement(x, None) + case _ => + assert(sig.nested.isEmpty) + args.foreach(infer(_, env, null, null, null)) + if (seqs != null) + seqs(i) += RecursiveArrayBuilderElement(x, None) + } + PVoid + + case x@ResultOp(resultIdx, sigs) => + PCanonicalTuple(true, (resultIdx until resultIdx + sigs.length).map(i => aggs(i).resultType): _*) + + case x@RunAgg(body, result, signature) => + val inits = newBuilder[InitOp](signature.length) + val seqs = newBuilder[SeqOp](signature.length) + infer(body, env, inits = inits, seqs = seqs, aggs = null) + val sigs = signature.indices.map { i => computePhysicalAgg(signature(i), inits(i), seqs(i)) }.toArray + infer(result, env, aggs = sigs, inits = null, seqs = null) + x.physicalSignatures2 = sigs + result.pType2 + + case x@RunAggScan(array, name, init, seq, result, signature) => + infer(array) + val e2 = env.bind(name, coerce[PStreamable](array.pType2).elementType) + val inits = newBuilder[InitOp](signature.length) + val seqs = newBuilder[SeqOp](signature.length) + infer(init, env = e2, inits = inits, seqs = null, aggs = null) + infer(seq, env = e2, inits = null, seqs = seqs, aggs = null) + val sigs = signature.indices.map { i => computePhysicalAgg(signature(i), inits(i), seqs(i)) }.toArray + infer(result, env = e2, aggs = sigs, inits = null, seqs = null) + x.physicalSignatures2 = sigs + coerce[PStreamable](array.pType2).copyStreamable(result.pType2) + + case AggStateValue(i, sig) => PCanonicalBinary(true) + case x if x.typ == TVoid => + x.children.foreach(c => infer(c.asInstanceOf[IR])) + PVoid + + case NDArrayAgg(nd, _) => + infer(nd) + PType.canonical(ir.typ) } + // Allow only requiredeness to diverge assert(ir.pType2.virtualType isOfType ir.typ) } diff --git a/hail/src/main/scala/is/hail/expr/ir/Interpret.scala b/hail/src/main/scala/is/hail/expr/ir/Interpret.scala index 66526cbb798..fee113744cb 100644 --- a/hail/src/main/scala/is/hail/expr/ir/Interpret.scala +++ b/hail/src/main/scala/is/hail/expr/ir/Interpret.scala @@ -610,23 +610,29 @@ object Interpret { } else { val spec = BufferSpec.defaultUncompressed + val physicalAggs = extracted.getPhysicalAggs( + ctx, + Env("global" -> value.globals.t), + Env("global" -> value.globals.t, "row" -> value.rvd.rowPType) + ) + val (_, initOp) = CompileWithAggregators2[Long, Unit](ctx, - extracted.aggs, + physicalAggs, "global", value.globals.t, extracted.init) val (_, partitionOpSeq) = CompileWithAggregators2[Long, Long, Unit](ctx, - extracted.aggs, + physicalAggs, "global", value.globals.t, "row", value.rvd.rowPType, extracted.seqPerElt) - val read = extracted.deserialize(ctx, spec) - val write = extracted.serialize(ctx, spec) - val combOpF = extracted.combOpF(ctx, spec) + val read = extracted.deserialize(ctx, spec, physicalAggs) + val write = extracted.serialize(ctx, spec, physicalAggs) + val combOpF = extracted.combOpF(ctx, spec, physicalAggs) val (rTyp: PTuple, f) = CompileWithAggregators2[Long, Long](ctx, - extracted.aggs, + physicalAggs, "global", value.globals.t, Let(res, extracted.results, MakeTuple.ordered(FastSeq(extracted.postAggIR)))) assert(rTyp.types(0).virtualType == query.typ) diff --git a/hail/src/main/scala/is/hail/expr/ir/TableIR.scala b/hail/src/main/scala/is/hail/expr/ir/TableIR.scala index 98452e3c524..cc5f4c8aaad 100644 --- a/hail/src/main/scala/is/hail/expr/ir/TableIR.scala +++ b/hail/src/main/scala/is/hail/expr/ir/TableIR.scala @@ -987,6 +987,7 @@ case class TableMapRows(child: TableIR, newRow: IR) extends TableIR { val extracted = agg.Extract.apply(agg.Extract.liftScan(newRow), scanRef) val nAggs = extracted.nAggs + if (extracted.aggs.isEmpty) { val (rTyp, f) = ir.Compile[Long, Long, Long]( ctx, @@ -1021,6 +1022,12 @@ case class TableMapRows(child: TableIR, newRow: IR) extends TableIR { rvd = tv.rvd.mapPartitionsWithIndex(RVDType(rTyp.asInstanceOf[PStruct], typ.key), itF)) } + val physicalAggs = extracted.getPhysicalAggs( + ctx, + Env("global" -> tv.globals.t), + Env("global" -> tv.globals.t, "row" -> tv.rvd.rowPType) + ) + val scanInitNeedsGlobals = Mentions(extracted.init, "global") val scanSeqNeedsGlobals = Mentions(extracted.seqPerElt, "global") val rowIterationNeedsGlobals = Mentions(extracted.postAggIR, "global") @@ -1040,22 +1047,22 @@ case class TableMapRows(child: TableIR, newRow: IR) extends TableIR { // 4. load in partStarts, calculate newRow based on those results. val (_, initF) = ir.CompileWithAggregators2[Long, Unit](ctx, - extracted.aggs, + physicalAggs, "global", tv.globals.t, Begin(FastIndexedSeq(extracted.init, extracted.serializeSet(0, 0, spec)))) val (_, eltSeqF) = ir.CompileWithAggregators2[Long, Long, Unit](ctx, - extracted.aggs, + physicalAggs, "global", Option(globalsBc).map(_.value.t).getOrElse(PStruct()), "row", tv.rvd.rowPType, extracted.eltOp(ctx)) - val read = extracted.deserialize(ctx, spec) - val write = extracted.serialize(ctx, spec) - val combOpF = extracted.combOpF(ctx, spec) + val read = extracted.deserialize(ctx, spec, physicalAggs) + val write = extracted.serialize(ctx, spec, physicalAggs) + val combOpF = extracted.combOpF(ctx, spec, physicalAggs) val (rTyp, f) = ir.CompileWithAggregators2[Long, Long, Long](ctx, - extracted.aggs, + physicalAggs, "global", Option(globalsBc).map(_.value.t).getOrElse(PStruct()), "row", tv.rvd.rowPType, Let(scanRef, extracted.results, extracted.postAggIR)) @@ -1387,26 +1394,32 @@ case class TableKeyByAndAggregate( val res = genUID() val extracted = agg.Extract(expr, res) + val physicalAggs = extracted.getPhysicalAggs( + ctx, + Env("global" -> prev.globals.t), + Env("global" -> prev.globals.t, "row" -> prev.rvd.rowPType) + ) + val (_, makeInit) = ir.CompileWithAggregators2[Long, Unit](ctx, - extracted.aggs, + physicalAggs, "global", prev.globals.t, extracted.init) val (_, makeSeq) = ir.CompileWithAggregators2[Long, Long, Unit](ctx, - extracted.aggs, + physicalAggs, "global", prev.globals.t, "row", prev.rvd.rowPType, extracted.seqPerElt) val (rTyp: PStruct, makeAnnotate) = ir.CompileWithAggregators2[Long, Long](ctx, - extracted.aggs, + physicalAggs, "global", prev.globals.t, Let(res, extracted.results, extracted.postAggIR)) assert(rTyp.virtualType == typ.valueType, s"$rTyp, ${ typ.valueType }") - val serialize = extracted.serialize(ctx, spec) - val deserialize = extracted.deserialize(ctx, spec) - val combOp = extracted.combOpF(ctx, spec) + val serialize = extracted.serialize(ctx, spec, physicalAggs) + val deserialize = extracted.deserialize(ctx, spec, physicalAggs) + val combOp = extracted.combOpF(ctx, spec, physicalAggs) val initF = makeInit(0, ctx.r) val globalsOffset = prev.globals.value.offset @@ -1517,13 +1530,19 @@ case class TableAggregateByKey(child: TableIR, expr: IR) extends TableIR { val res = genUID() val extracted = agg.Extract(expr, res) + val physicalAggs = extracted.getPhysicalAggs( + ctx, + Env("global" -> prev.globals.t), + Env("global" -> prev.globals.t, "row" -> prev.rvd.rowPType) + ) + val (_, makeInit) = ir.CompileWithAggregators2[Long, Unit](ctx, - extracted.aggs, + physicalAggs, "global", prev.globals.t, extracted.init) val (_, makeSeq) = ir.CompileWithAggregators2[Long, Long, Unit](ctx, - extracted.aggs, + physicalAggs, "global", prev.globals.t, "row", prev.rvd.rowPType, extracted.seqPerElt) @@ -1533,7 +1552,7 @@ case class TableAggregateByKey(child: TableIR, expr: IR) extends TableIR { val key = Ref(genUID(), keyType.virtualType) val value = Ref(genUID(), valueIR.typ) val (rowType: PStruct, makeRow) = ir.CompileWithAggregators2[Long, Long, Long](ctx, - extracted.aggs, + physicalAggs, "global", prev.globals.t, key.name, keyType, Let(value.name, valueIR, diff --git a/hail/src/main/scala/is/hail/expr/ir/agg/Extract.scala b/hail/src/main/scala/is/hail/expr/ir/agg/Extract.scala index be1e18e7a69..33004ee9c44 100644 --- a/hail/src/main/scala/is/hail/expr/ir/agg/Extract.scala +++ b/hail/src/main/scala/is/hail/expr/ir/agg/Extract.scala @@ -4,6 +4,7 @@ import is.hail.HailContext import is.hail.annotations.{Region, RegionValue} import is.hail.expr.ir import is.hail.expr.ir._ +import is.hail.expr.ir.lowering.LoweringPipeline import is.hail.expr.types.physical._ import is.hail.expr.types.virtual._ import is.hail.io.BufferSpec @@ -43,9 +44,9 @@ case class Aggs(postAggIR: IR, init: IR, seqPerElt: IR, aggs: Array[AggStateSign def eltOp(ctx: ExecuteContext): IR = seqPerElt - def deserialize(ctx: ExecuteContext, spec: BufferSpec): ((Region, Array[Byte]) => Long) = { + def deserialize(ctx: ExecuteContext, spec: BufferSpec, physicalAggs: Array[AggStatePhysicalSignature]): ((Region, Array[Byte]) => Long) = { val (_, f) = ir.CompileWithAggregators2[Unit](ctx, - aggs, ir.DeserializeAggs(0, 0, spec, aggs)) + physicalAggs, ir.DeserializeAggs(0, 0, spec, aggs)) { (aggRegion: Region, bytes: Array[Byte]) => val f2 = f(0, aggRegion); @@ -56,9 +57,9 @@ case class Aggs(postAggIR: IR, init: IR, seqPerElt: IR, aggs: Array[AggStateSign } } - def serialize(ctx: ExecuteContext, spec: BufferSpec): (Region, Long) => Array[Byte] = { + def serialize(ctx: ExecuteContext, spec: BufferSpec, physicalAggs: Array[AggStatePhysicalSignature]): (Region, Long) => Array[Byte] = { val (_, f) = ir.CompileWithAggregators2[Unit](ctx, - aggs, ir.SerializeAggs(0, 0, spec, aggs)) + physicalAggs, ir.SerializeAggs(0, 0, spec, aggs)) { (aggRegion: Region, off: Long) => val f2 = f(0, aggRegion); @@ -68,9 +69,9 @@ case class Aggs(postAggIR: IR, init: IR, seqPerElt: IR, aggs: Array[AggStateSign } } - def combOpF(ctx: ExecuteContext, spec: BufferSpec): (Array[Byte], Array[Byte]) => Array[Byte] = { + def combOpF(ctx: ExecuteContext, spec: BufferSpec, physicalAggs: Array[AggStatePhysicalSignature]): (Array[Byte], Array[Byte]) => Array[Byte] = { val (_, f) = ir.CompileWithAggregators2[Unit](ctx, - aggs ++ aggs, + physicalAggs ++ physicalAggs, Begin( deserializeSet(0, 0, spec) +: deserializeSet(1, 1, spec) +: @@ -90,6 +91,25 @@ case class Aggs(postAggIR: IR, init: IR, seqPerElt: IR, aggs: Array[AggStateSign } def results: IR = ResultOp(0, aggs) + + def getPhysicalAggs(ctx: ExecuteContext, initBindings: Env[PType], seqBindings: Env[PType]): Array[AggStatePhysicalSignature] = { + val initsAB = InferPType.newBuilder[InitOp](aggs.length) + val seqsAB = InferPType.newBuilder[SeqOp](aggs.length) + val init2 = LoweringPipeline.compileLowerer.apply(ctx, init, false).asInstanceOf[IR] + val seq2 = LoweringPipeline.compileLowerer.apply(ctx, seqPerElt, false).asInstanceOf[IR] + InferPType(init2.noSharing, initBindings, null, inits = initsAB, null) + InferPType(seq2.noSharing, seqBindings, null, null, seqsAB) + + val pSigs = aggs.indices.map { i => InferPType.computePhysicalAgg(aggs(i), initsAB(i), seqsAB(i)) }.toArray + + if (init2 eq init) + InferPType.clearPTypes(init2) + if (seq2 eq seqPerElt) + InferPType.clearPTypes(seq2) + + // should return pSigs, but cannot until we use the inferred ptype to generate code + aggs.map(_.toCanonicalPhysical) + } } object Extract { diff --git a/hail/src/main/scala/is/hail/expr/types/physical/PType.scala b/hail/src/main/scala/is/hail/expr/types/physical/PType.scala index a3c9fe0f6a2..49a5ef35665 100644 --- a/hail/src/main/scala/is/hail/expr/types/physical/PType.scala +++ b/hail/src/main/scala/is/hail/expr/types/physical/PType.scala @@ -234,6 +234,7 @@ abstract class PType extends Serializable with Requiredness { case t: PStruct => t.copy(required = required) case t: PTuple => t.copy(required = required) case t: PNDArray => t.copy(required = required) + case PVoid => PVoid } } diff --git a/hail/src/test/scala/is/hail/expr/ir/Aggregators2Suite.scala b/hail/src/test/scala/is/hail/expr/ir/Aggregators2Suite.scala index eca90052e51..a399fb4ee9e 100644 --- a/hail/src/test/scala/is/hail/expr/ir/Aggregators2Suite.scala +++ b/hail/src/test/scala/is/hail/expr/ir/Aggregators2Suite.scala @@ -32,8 +32,9 @@ class Aggregators2Suite extends HailSuite { val argRef = Ref(genUID(), argT.virtualType) val spec = BufferSpec.defaultUncompressed + val psig = aggSig.toCanonicalPhysical val (_, combAndDuplicate) = CompileWithAggregators2[Unit](ctx, - Array.fill(nPartitions)(aggSig), + Array.fill(nPartitions)(psig), Begin( Array.tabulate(nPartitions)(i => DeserializeAggs(i, i, spec, Array(aggSig))) ++ Array.range(1, nPartitions).map(i => CombOp(0, i, aggSig)) :+ @@ -41,7 +42,7 @@ class Aggregators2Suite extends HailSuite { DeserializeAggs(1, 0, spec, Array(aggSig)))) val (rt: PTuple, resF) = CompileWithAggregators2[Long](ctx, - Array.fill(nPartitions)(aggSig), + Array.fill(nPartitions)(psig), ResultOp(0, Array(aggSig, aggSig))) assert(rt.types(0) == rt.types(1)) @@ -54,7 +55,7 @@ class Aggregators2Suite extends HailSuite { def withArgs(foo: IR) = { CompileWithAggregators2[Long, Unit](ctx, - Array(aggSig), + Array(psig), argRef.name, argRef.pType, args.map(_._1).foldLeft[IR](foo) { case (op, name) => Let(name, GetField(argRef, name), op) @@ -63,14 +64,14 @@ class Aggregators2Suite extends HailSuite { val serialize = SerializeAggs(0, 0, spec, Array(aggSig)) val (_, writeF) = CompileWithAggregators2[Unit](ctx, - Array(aggSig), + Array(psig), serialize) val initF = withArgs(initOp) expectedInit.foreach { v => val (rt: PBaseStruct, resOneF) = CompileWithAggregators2[Long](ctx, - Array(aggSig), ResultOp(0, Array(aggSig))) + Array(psig), ResultOp(0, Array(aggSig))) val init = initF(0, region) val res = resOneF(0, region) diff --git a/hail/src/test/scala/is/hail/expr/ir/IRSuite.scala b/hail/src/test/scala/is/hail/expr/ir/IRSuite.scala index eedad5aee73..4c661a8d85f 100644 --- a/hail/src/test/scala/is/hail/expr/ir/IRSuite.scala +++ b/hail/src/test/scala/is/hail/expr/ir/IRSuite.scala @@ -339,7 +339,7 @@ class IRSuite extends HailSuite { // should not be able to infer physical type twice on one IR (i32na) node = ApplyUnaryPrimOp(Negate(), i32na) - intercept[AssertionError](InferPType(node, Env.empty)) + intercept[RuntimeException](InferPType(node, Env.empty)) node = ApplyUnaryPrimOp(Negate(), I64(5)) assertPType(node, PInt64(true)) From cd9c0404307aabf6f977f5aeb01ec0e98441bc9b Mon Sep 17 00:00:00 2001 From: Tim Poterba Date: Wed, 19 Feb 2020 14:52:56 -0500 Subject: [PATCH 2/8] fix --- hail/src/main/scala/is/hail/expr/ir/Compile.scala | 2 +- hail/src/main/scala/is/hail/expr/ir/InferPType.scala | 10 +++------- hail/src/main/scala/is/hail/expr/ir/agg/Extract.scala | 2 +- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/hail/src/main/scala/is/hail/expr/ir/Compile.scala b/hail/src/main/scala/is/hail/expr/ir/Compile.scala index dc17d164bc4..562e7f3110f 100644 --- a/hail/src/main/scala/is/hail/expr/ir/Compile.scala +++ b/hail/src/main/scala/is/hail/expr/ir/Compile.scala @@ -238,7 +238,7 @@ object CompileWithAggregators2 { TypeCheck(ir, BindingEnv(Env.fromSeq[Type](args.map { case (name, t, _) => name -> t.virtualType }))) - InferPType(if(HasIRSharing(ir)) ir.deepCopy() else ir, Env(args.map { case (n, pt, _) => n -> pt}: _*)) + InferPType(ir.noSharing, Env(args.map { case (n, pt, _) => n -> pt}: _*), aggSigs, null, null) assert(TypeToIRIntermediateClassTag(ir.typ) == classTag[R]) diff --git a/hail/src/main/scala/is/hail/expr/ir/InferPType.scala b/hail/src/main/scala/is/hail/expr/ir/InferPType.scala index a45cc8ee31c..59664a1f8b6 100644 --- a/hail/src/main/scala/is/hail/expr/ir/InferPType.scala +++ b/hail/src/main/scala/is/hail/expr/ir/InferPType.scala @@ -447,13 +447,6 @@ object InferPType { theIR._pType2 })) case In(_, pType: PType) => pType - case ArrayFor(a, valueName, body) => - infer(a) - infer(body, env.bind(valueName -> a._pType2.asInstanceOf[PStream].elementType)) - PVoid - case x if x.typ == TVoid => - x.children.foreach(c => infer(c.asInstanceOf[IR])) - PVoid case CollectDistributedArray(contextsIR, globalsIR, contextsName, globalsName, bodyIR) => infer(contextsIR) infer(globalsIR) @@ -569,6 +562,9 @@ object InferPType { case NDArrayAgg(nd, _) => infer(nd) PType.canonical(ir.typ) + case x if x.typ == TVoid => + x.children.foreach(c => infer(c.asInstanceOf[IR])) + PVoid } diff --git a/hail/src/main/scala/is/hail/expr/ir/agg/Extract.scala b/hail/src/main/scala/is/hail/expr/ir/agg/Extract.scala index 33004ee9c44..0eeb6e350a3 100644 --- a/hail/src/main/scala/is/hail/expr/ir/agg/Extract.scala +++ b/hail/src/main/scala/is/hail/expr/ir/agg/Extract.scala @@ -98,7 +98,7 @@ case class Aggs(postAggIR: IR, init: IR, seqPerElt: IR, aggs: Array[AggStateSign val init2 = LoweringPipeline.compileLowerer.apply(ctx, init, false).asInstanceOf[IR] val seq2 = LoweringPipeline.compileLowerer.apply(ctx, seqPerElt, false).asInstanceOf[IR] InferPType(init2.noSharing, initBindings, null, inits = initsAB, null) - InferPType(seq2.noSharing, seqBindings, null, null, seqsAB) + InferPType(seq2.noSharing, seqBindings, null, null, seqs = seqsAB) val pSigs = aggs.indices.map { i => InferPType.computePhysicalAgg(aggs(i), initsAB(i), seqsAB(i)) }.toArray From 5da069ab0842bc078d4f1dcde4c1c07875a40848 Mon Sep 17 00:00:00 2001 From: Alex Kotlar Date: Tue, 25 Feb 2020 21:52:57 -0500 Subject: [PATCH 3/8] merge --- .gitignore | 3 +- batch/batch/front_end/front_end.py | 12 +- batch/test-sa.yaml | 4 + batch/test/test_batch.py | 4 +- benchmark/scripts/benchmark_in_pipeline.py | 8 +- build.yaml | 118 ++-- ci/ci/build.py | 2 +- ci/ci/github.py | 12 +- docker/requirements.txt | 2 +- hail/Dockerfile.hail-run | 2 +- hail/Makefile | 31 +- .../hail/docs/experimental/vcf_combiner.rst | 10 +- hail/python/hail/experimental/vcf_combiner.py | 4 +- hail/python/hail/expr/functions.py | 2 +- hail/python/hail/expr/types.py | 56 ++ hail/python/hail/ir/base_ir.py | 3 + hail/python/hail/ir/blockmatrix_ir.py | 3 + hail/python/hail/ir/blockmatrix_reader.py | 21 + hail/python/hail/ir/blockmatrix_writer.py | 19 + hail/python/hail/ir/ir.py | 37 ++ hail/python/hail/linalg/blockmatrix.py | 34 +- hail/python/hail/methods/statgen.py | 5 +- hail/python/hailtop/pipeline/Makefile | 8 + hail/python/hailtop/pipeline/backend.py | 82 ++- hail/python/hailtop/pipeline/conftest.py | 37 ++ hail/python/hailtop/pipeline/docs/Makefile | 36 ++ .../docs/_static/images/dags/dags.001.png | Bin 0 -> 18806 bytes .../docs/_static/images/dags/dags.002.png | Bin 0 -> 23003 bytes .../docs/_static/images/dags/dags.003.png | Bin 0 -> 24853 bytes .../docs/_static/images/dags/dags.004.png | Bin 0 -> 24655 bytes .../docs/_static/images/dags/dags.005.png | Bin 0 -> 38358 bytes .../docs/_static/images/dags/dags.006.png | Bin 0 -> 43766 bytes .../docs/_static/images/dags/dags.007.png | Bin 0 -> 63005 bytes .../docs/_static/images/dags/dags.008.png | Bin 0 -> 83960 bytes .../docs/_templates/_autosummary/class.rst | 37 ++ hail/python/hailtop/pipeline/docs/api.rst | 78 +++ .../hailtop/pipeline/docs/batch_service.rst | 163 +++++ hail/python/hailtop/pipeline/docs/conf.py | 158 +++++ .../hailtop/pipeline/docs/data/example.bed | Bin 0 -> 33 bytes .../hailtop/pipeline/docs/data/example.bim | 10 + .../hailtop/pipeline/docs/data/example.fam | 10 + .../hailtop/pipeline/docs/data/hello.txt | 1 + .../pipeline/docs/docker_resources.rst | 99 +++ .../hailtop/pipeline/docs/getting_started.rst | 38 ++ hail/python/hailtop/pipeline/docs/index.rst | 30 + .../python/hailtop/pipeline/docs/tutorial.rst | 435 +++++++++++++ hail/python/hailtop/pipeline/pipeline.py | 58 +- hail/python/hailtop/pipeline/resource.py | 43 +- hail/python/hailtop/pipeline/task.py | 88 +-- hail/python/hailtop/utils/utils.py | 5 + hail/python/scripts/drive_combiner.py | 10 +- .../hail/experimental/test_experimental.py | 8 +- hail/python/test/hail/methods/test_impex.py | 3 + hail/python/test/hail/test_ir.py | 11 +- hail/src/main/scala/is/hail/HailContext.scala | 28 - .../hail/annotations/RegionValueBuilder.scala | 117 +--- .../StagedRegionValueBuilder.scala | 112 +--- .../scala/is/hail/asm4s/FunctionBuilder.scala | 15 + .../main/scala/is/hail/backend/Backend.scala | 11 +- .../distributed/DistributedBackend.scala | 29 - .../is/hail/backend/spark/SparkBackend.scala | 2 + .../hail/backend/spark/SparkValueCache.scala | 18 + .../is/hail/expr/LowerArrayToStream.scala | 1 - .../scala/is/hail/expr/ir/BlockMatrixIR.scala | 27 +- .../is/hail/expr/ir/BlockMatrixWriter.scala | 8 +- .../main/scala/is/hail/expr/ir/Children.scala | 5 +- .../scala/is/hail/expr/ir/Compilable.scala | 2 + .../main/scala/is/hail/expr/ir/Compile.scala | 12 + .../src/main/scala/is/hail/expr/ir/Copy.scala | 12 +- .../src/main/scala/is/hail/expr/ir/Emit.scala | 165 ++--- .../is/hail/expr/ir/EmitFunctionBuilder.scala | 14 - .../scala/is/hail/expr/ir/EmitStream.scala | 588 ++++++++++++++---- .../scala/is/hail/expr/ir/FoldConstants.scala | 1 - hail/src/main/scala/is/hail/expr/ir/IR.scala | 10 +- .../scala/is/hail/expr/ir/IRBuilder.scala | 2 +- .../scala/is/hail/expr/ir/InferPType.scala | 63 +- .../scala/is/hail/expr/ir/InferType.scala | 4 +- .../scala/is/hail/expr/ir/Interpret.scala | 7 +- .../scala/is/hail/expr/ir/Interpretable.scala | 3 + .../main/scala/is/hail/expr/ir/MatrixIR.scala | 1 - .../main/scala/is/hail/expr/ir/Parser.scala | 19 +- .../main/scala/is/hail/expr/ir/Pretty.scala | 5 +- .../is/hail/expr/ir/PruneDeadFields.scala | 8 +- .../main/scala/is/hail/expr/ir/Simplify.scala | 7 +- .../scala/is/hail/expr/ir/Streamify.scala | 24 - .../main/scala/is/hail/expr/ir/TableIR.scala | 6 +- .../scala/is/hail/expr/ir/TypeCheck.scala | 14 +- .../is/hail/expr/ir/agg/AppendOnlyBTree.scala | 2 +- .../expr/ir/agg/DownsampleAggregator.scala | 2 +- .../scala/is/hail/expr/ir/agg/Extract.scala | 8 +- .../expr/ir/functions/ArrayFunctions.scala | 22 +- .../is/hail/expr/ir/functions/Functions.scala | 43 +- .../hail/expr/ir/lowering/LowerTableIR.scala | 4 +- .../scala/is/hail/expr/ir/lowering/Rule.scala | 1 - .../is/hail/expr/types/BlockMatrixType.scala | 9 +- .../is/hail/expr/types/encoded/EArray.scala | 9 +- .../hail/expr/types/encoded/EBaseStruct.scala | 11 +- .../is/hail/expr/types/encoded/EBinary.scala | 7 +- .../is/hail/expr/types/encoded/EBoolean.scala | 7 +- .../is/hail/expr/types/encoded/EFloat32.scala | 7 +- .../is/hail/expr/types/encoded/EFloat64.scala | 7 +- .../is/hail/expr/types/encoded/EInt32.scala | 7 +- .../is/hail/expr/types/encoded/EInt64.scala | 7 +- .../is/hail/expr/types/encoded/EType.scala | 28 +- .../expr/types/physical/ComplexPType.scala | 18 +- .../is/hail/expr/types/physical/PArray.scala | 2 - .../physical/PArrayBackedContainer.scala | 13 +- .../expr/types/physical/PBaseStruct.scala | 271 +------- .../hail/expr/types/physical/PBoolean.scala | 5 +- .../is/hail/expr/types/physical/PCall.scala | 2 - .../expr/types/physical/PCanonicalArray.scala | 195 +++--- .../types/physical/PCanonicalBaseStruct.scala | 223 +++++++ .../types/physical/PCanonicalBinary.scala | 93 ++- .../expr/types/physical/PCanonicalCall.scala | 2 +- .../expr/types/physical/PCanonicalDict.scala | 3 +- .../types/physical/PCanonicalInterval.scala | 2 +- .../expr/types/physical/PCanonicalLocus.scala | 2 +- .../types/physical/PCanonicalNDArray.scala | 20 +- .../expr/types/physical/PCanonicalSet.scala | 2 +- .../types/physical/PCanonicalString.scala | 17 +- .../types/physical/PCanonicalStruct.scala | 21 +- .../expr/types/physical/PCanonicalTuple.scala | 13 +- .../is/hail/expr/types/physical/PDict.scala | 2 - .../hail/expr/types/physical/PFloat32.scala | 10 +- .../hail/expr/types/physical/PFloat64.scala | 11 +- .../is/hail/expr/types/physical/PInt32.scala | 10 +- .../is/hail/expr/types/physical/PInt64.scala | 10 +- .../hail/expr/types/physical/PInterval.scala | 2 - .../is/hail/expr/types/physical/PLocus.scala | 2 - .../hail/expr/types/physical/PNDArray.scala | 2 - .../hail/expr/types/physical/PPrimitive.scala | 65 +- .../is/hail/expr/types/physical/PSet.scala | 2 - .../is/hail/expr/types/physical/PStream.scala | 22 +- .../is/hail/expr/types/physical/PStruct.scala | 4 +- .../is/hail/expr/types/physical/PTuple.scala | 8 +- .../is/hail/expr/types/physical/PType.scala | 40 +- .../is/hail/expr/types/physical/PVoid.scala | 10 +- .../is/hail/expr/types/virtual/TStream.scala | 10 +- .../scala/is/hail/io/TextMatrixReader.scala | 2 +- .../scala/is/hail/io/plink/LoadPlink.scala | 8 +- .../main/scala/is/hail/methods/Nirvana.scala | 4 +- hail/src/main/scala/is/hail/methods/VEP.scala | 14 +- hail/src/main/scala/is/hail/rvd/RVD.scala | 19 +- .../hail/scheduler/BreakRetryException.scala | 4 - .../main/scala/is/hail/scheduler/Client.scala | 218 ------- .../main/scala/is/hail/scheduler/DArray.scala | 10 - .../scala/is/hail/scheduler/Executor.scala | 160 ----- .../scala/is/hail/scheduler/package.scala | 60 -- .../hail/utils/richUtils/RichIterator.scala | 15 +- .../scala/is/hail/variant/HardCallView.scala | 5 +- hail/src/test/scala/is/hail/HailSuite.scala | 9 +- hail/src/test/scala/is/hail/TestUtils.scala | 21 +- .../annotations/StagedRegionValueSuite.scala | 75 +++ .../is/hail/annotations/UnsafeSuite.scala | 4 +- .../is/hail/expr/ir/Aggregators2Suite.scala | 6 +- .../expr/ir/ArrayDeforestationSuite.scala | 8 +- .../is/hail/expr/ir/EmitStreamSuite.scala | 257 +++++++- .../is/hail/expr/ir/ForwardLetsSuite.scala | 10 +- .../test/scala/is/hail/expr/ir/IRSuite.scala | 65 +- .../hail/expr/ir/RandomFunctionsSuite.scala | 10 +- .../scala/is/hail/expr/ir/TableIRSuite.scala | 4 +- .../types/physical/PhysicalTestUtils.scala | 7 +- .../is/hail/scheduler/SchedulerSuite.scala | 61 -- scheduler/Dockerfile | 10 - scheduler/MANIFEST.in | 2 - scheduler/Makefile | 18 - scheduler/deployment.yaml | 79 --- scheduler/executors.yaml | 36 -- scheduler/scheduler/__init__.py | 5 - scheduler/scheduler/__main__.py | 4 - scheduler/scheduler/scheduler.py | 498 --------------- scheduler/setup.py | 11 - scheduler/templates/index.html | 100 --- scheduler/testng.xml | 7 - web_common/web_common/templates/header.html | 19 +- 175 files changed, 3503 insertions(+), 2817 deletions(-) create mode 100644 batch/test-sa.yaml create mode 100644 hail/python/hailtop/pipeline/Makefile create mode 100644 hail/python/hailtop/pipeline/conftest.py create mode 100644 hail/python/hailtop/pipeline/docs/Makefile create mode 100644 hail/python/hailtop/pipeline/docs/_static/images/dags/dags.001.png create mode 100644 hail/python/hailtop/pipeline/docs/_static/images/dags/dags.002.png create mode 100644 hail/python/hailtop/pipeline/docs/_static/images/dags/dags.003.png create mode 100644 hail/python/hailtop/pipeline/docs/_static/images/dags/dags.004.png create mode 100644 hail/python/hailtop/pipeline/docs/_static/images/dags/dags.005.png create mode 100644 hail/python/hailtop/pipeline/docs/_static/images/dags/dags.006.png create mode 100644 hail/python/hailtop/pipeline/docs/_static/images/dags/dags.007.png create mode 100644 hail/python/hailtop/pipeline/docs/_static/images/dags/dags.008.png create mode 100644 hail/python/hailtop/pipeline/docs/_templates/_autosummary/class.rst create mode 100644 hail/python/hailtop/pipeline/docs/api.rst create mode 100644 hail/python/hailtop/pipeline/docs/batch_service.rst create mode 100644 hail/python/hailtop/pipeline/docs/conf.py create mode 100644 hail/python/hailtop/pipeline/docs/data/example.bed create mode 100644 hail/python/hailtop/pipeline/docs/data/example.bim create mode 100644 hail/python/hailtop/pipeline/docs/data/example.fam create mode 100644 hail/python/hailtop/pipeline/docs/data/hello.txt create mode 100644 hail/python/hailtop/pipeline/docs/docker_resources.rst create mode 100644 hail/python/hailtop/pipeline/docs/getting_started.rst create mode 100644 hail/python/hailtop/pipeline/docs/index.rst create mode 100644 hail/python/hailtop/pipeline/docs/tutorial.rst delete mode 100644 hail/src/main/scala/is/hail/backend/distributed/DistributedBackend.scala create mode 100644 hail/src/main/scala/is/hail/backend/spark/SparkValueCache.scala delete mode 100644 hail/src/main/scala/is/hail/expr/ir/Streamify.scala create mode 100644 hail/src/main/scala/is/hail/expr/types/physical/PCanonicalBaseStruct.scala delete mode 100644 hail/src/main/scala/is/hail/scheduler/BreakRetryException.scala delete mode 100644 hail/src/main/scala/is/hail/scheduler/Client.scala delete mode 100644 hail/src/main/scala/is/hail/scheduler/DArray.scala delete mode 100644 hail/src/main/scala/is/hail/scheduler/Executor.scala delete mode 100644 hail/src/main/scala/is/hail/scheduler/package.scala delete mode 100644 hail/src/test/scala/is/hail/scheduler/SchedulerSuite.scala delete mode 100644 scheduler/Dockerfile delete mode 100644 scheduler/MANIFEST.in delete mode 100644 scheduler/Makefile delete mode 100644 scheduler/deployment.yaml delete mode 100644 scheduler/executors.yaml delete mode 100644 scheduler/scheduler/__init__.py delete mode 100644 scheduler/scheduler/__main__.py delete mode 100644 scheduler/scheduler/scheduler.py delete mode 100644 scheduler/setup.py delete mode 100644 scheduler/templates/index.html delete mode 100644 scheduler/testng.xml diff --git a/.gitignore b/.gitignore index 453a5623024..debfd977750 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,5 @@ target __pycache__/ *.pyc *-checkpoint.ipynb -*hail/python/hail/docs/tutorials/data* \ No newline at end of file +*hail/python/hail/docs/tutorials/data* +*hail/python/hailtop/pipeline/docs/output* diff --git a/batch/batch/front_end/front_end.py b/batch/batch/front_end/front_end.py index 35c9a450b3e..2a01655018e 100644 --- a/batch/batch/front_end/front_end.py +++ b/batch/batch/front_end/front_end.py @@ -481,6 +481,7 @@ async def create_jobs(request, userdata): batch_id = int(request.match_info['batch_id']) user = userdata['username'] + # restrict to what's necessary; in particular, drop the session # which is sensitive userdata = { @@ -579,7 +580,12 @@ async def create_jobs(request, userdata): secrets = spec.get('secrets') if not secrets: secrets = [] - spec['secrets'] = secrets + + for secret in secrets: + if user != 'ci': + raise web.HTTPBadRequest(reason=f'unauthorized secret {(secret["namespace"], secret["name"])}') + + spec['secrets'] = secrets secrets.append({ 'namespace': BATCH_PODS_NAMESPACE, 'name': userdata['gsa_key_secret_name'], @@ -587,6 +593,10 @@ async def create_jobs(request, userdata): 'mount_in_copy': True }) + sa = spec.get('service_account') + if sa and user != 'ci' and not (user == 'test' and sa['name'] == 'test-batch-sa' and sa['namespace'] == BATCH_PODS_NAMESPACE): + raise web.HTTPBadRequest(reason=f'unauthorized service account {(sa["namespace"], sa["name"])}') + env = spec.get('env') if not env: env = [] diff --git a/batch/test-sa.yaml b/batch/test-sa.yaml new file mode 100644 index 00000000000..9312a7dac92 --- /dev/null +++ b/batch/test-sa.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: test-batch-sa diff --git a/batch/test/test_batch.py b/batch/test/test_batch.py index ec510e2cd6d..83d4b22c653 100644 --- a/batch/test/test_batch.py +++ b/batch/test/test_batch.py @@ -414,10 +414,10 @@ def test_service_account(self): b = self.client.create_batch() j = b.create_job( os.environ['CI_UTILS_IMAGE'], - ['/bin/sh', '-c', 'kubectl get pods -l app=batch-driver'], + ['/bin/sh', '-c', 'kubectl version'], service_account={ 'namespace': os.environ['HAIL_BATCH_PODS_NAMESPACE'], - 'name': 'ci-agent' + 'name': 'test-batch-sa' }) b.submit() status = j.wait() diff --git a/benchmark/scripts/benchmark_in_pipeline.py b/benchmark/scripts/benchmark_in_pipeline.py index 152790d6b1e..55b9390d53d 100644 --- a/benchmark/scripts/benchmark_in_pipeline.py +++ b/benchmark/scripts/benchmark_in_pipeline.py @@ -15,7 +15,11 @@ N_REPLICATES = int(sys.argv[4]) N_ITERS = int(sys.argv[5]) - p = pl.Pipeline(name=f'benchmark-{SHA}', + labeled_sha = SHA + label = os.environ.get('BENCHMARK_LABEL') + if label: + labeled_sha = f'{labeled_sha}-{label}' + p = pl.Pipeline(name=f'benchmark-{labeled_sha}', backend=pl.BatchBackend(billing_project='hail'), default_image=BENCHMARK_IMAGE, default_storage='100G', @@ -64,7 +68,7 @@ combine = p.new_task('combine_output') combine.command(f'hail-bench combine -o {combine.ofile} ' + ' '.join(all_output)) - output_file = os.path.join(BUCKET_BASE, f'{SHA}.json') + output_file = os.path.join(BUCKET_BASE, f'{labeled_sha}.json') print(f'writing output to {output_file}') p.write_output(combine.ofile, output_file) diff --git a/build.yaml b/build.yaml index eace9249f29..ea1464b1bdb 100644 --- a/build.yaml +++ b/build.yaml @@ -61,6 +61,13 @@ steps: dependsOn: - default_ns - batch_pods_ns + - kind: deploy + name: deploy_test_batch_sa + namespace: + valueFrom: batch_pods_ns.name + config: batch/test-sa.yaml + dependsOn: + - batch_pods_ns - kind: buildImage name: base_image dockerFile: docker/Dockerfile.base @@ -96,6 +103,8 @@ steps: to: /repo/pipeline/ - from: /io/repo/notebook/sql to: /repo/notebook/ + - from: /io/repo/hail/python/hailtop + to: /repo/hailtop/ dependsOn: - base_image - kind: runImage @@ -327,6 +336,7 @@ steps: dependsOn: - default_ns - batch_pods_ns + - deploy_test_batch_sa - deploy_default_admin_admin - auth_database - service_base_image @@ -344,7 +354,7 @@ steps: contextPath: hail publishAs: hail-run dependsOn: - - base_image + - service_base_image - kind: buildImage name: scorecard_image dockerFile: scorecard/Dockerfile @@ -487,8 +497,6 @@ steps: to: /testng-cpp-codegen.xml - from: /io/repo/hail/testng-distributed-backend.xml to: /testng-distributed-backend.xml - - from: /io/repo/scheduler/testng.xml - to: /testng-scheduler.xml - from: /io/repo/hail/python/hail.zip to: /hail.zip - from: /io/repo/hail/test.tar.gz @@ -565,13 +573,6 @@ steps: python3 -m pylint --rcfile pylintrc hailtop dependsOn: - hail_base_image - - kind: buildImage - name: scheduler_image - dockerFile: scheduler/Dockerfile - contextPath: . - publishAs: scheduler - dependsOn: - - service_base_image - kind: buildImage name: hail_jupyter_image dockerFile: apiserver/Dockerfile.hail-jupyter @@ -660,9 +661,11 @@ steps: | pandoc -o python/hail/docs/change_log.rst make -C www make -C python/hail/docs BUILDDIR=_build clean html + make -C python/hailtop/pipeline/docs BUILDDIR=_build clean html mkdir -p www/docs rm -f www/*.md mv python/hail/docs/_build/html www/docs/0.2 + mv python/hailtop/pipeline/docs/_build/html www/docs/batch HAIL_CACHE_VERSION=$(cat python/hail/hail_version) find www -iname *.html -type f -exec sed -i -e "s/\.css/\.css\?v\=$HAIL_CACHE_VERSION/" {} +; tar czf /io/www.tar.gz www @@ -1065,6 +1068,36 @@ steps: - service_base_image - deploy_batch - setup_pipeline + - kind: runImage + name: test_pipeline_docs + image: + valueFrom: base_image.image + script: | + set -ex + export HAIL_DEPLOY_CONFIG_FILE=/deploy-config/deploy-config.json + cd /io/hailtop/hailtop/pipeline + python3 -m pytest --instafail \ + --doctest-modules \ + --doctest-glob='*.rst' \ + --ignore=docs/conf.py + secrets: + - name: gce-deploy-config + namespace: + valueFrom: default_ns.name + mountPath: /deploy-config + - name: test-tokens + namespace: + valueFrom: batch_pods_ns.name + mountPath: /user-tokens + dependsOn: + - default_ns + - batch_pods_ns + - base_image + - copy_files + - deploy_batch + inputs: + - from: /repo/hailtop + to: /io/ - kind: runImage name: cleanup_pipeline image: @@ -1085,28 +1118,6 @@ steps: - base_image - setup_pipeline - test_pipeline - - kind: deploy - name: deploy_scheduler - namespace: - valueFrom: default_ns.name - config: scheduler/deployment.yaml - wait: - - kind: Service - name: scheduler - for: alive - dependsOn: - - default_ns - - deploy_router - - scheduler_image - - kind: deploy - name: deploy_executors - namespace: - valueFrom: default_ns.name - config: scheduler/executors.yaml - dependsOn: - - default_ns - - deploy_router - - hail_test_base_image - kind: createDatabase name: notebook_database databaseName: notebook @@ -1147,49 +1158,6 @@ steps: - default_ns - image_fetcher_image - deploy_notebook - - kind: runImage - name: test_scheduler - image: - valueFrom: base_image.image - script: | - set -ex - HAIL_TEST_SCHEDULER_HOST=scheduler.{{ default_ns.name }}.svc.cluster.local CLASSPATH="$SPARK_HOME/jars/*:/io/hail-test.jar" java org.testng.TestNG -listener is.hail.LogTestListener /io/testng-scheduler.xml - inputs: - - from: /hail-test.jar - to: /io/hail-test.jar - - from: /testng-scheduler.xml - to: /io/testng-scheduler.xml - scopes: [] - dependsOn: - - default_ns - - base_image - - build_hail - - deploy_scheduler - - deploy_executors - - kind: runImage - name: test_hail_java_distributed_backend - image: - valueFrom: hail_run_image.image - script: | - set -ex - cd /io - mkdir -p src/test - HAIL_TEST_SCHEDULER_HOST=scheduler.{{ default_ns.name }}.svc.cluster.local HAIL_TEST_SKIP_R=1 java -cp hail-test.jar:$SPARK_HOME/jars/* org.testng.TestNG -listener is.hail.LogTestListener testng-distributed-backend.xml - inputs: - - from: /hail-test.jar - to: /io/hail-test.jar - - from: /testng-distributed-backend.xml - to: /io/testng-distributed-backend.xml - outputs: - - from: /io/test-output - to: /test-distributed-backend-output - scopes: [] - dependsOn: - - default_ns - - hail_run_image - - build_hail - - deploy_scheduler - - deploy_executors - kind: runImage name: cleanup_ci_test_repo image: diff --git a/ci/ci/build.py b/ci/ci/build.py index aa394774003..c8bc75a5936 100644 --- a/ci/ci/build.py +++ b/ci/ci/build.py @@ -283,7 +283,7 @@ def build(self, batch, code, scope): cache_from_published_latest = '' push_image = f''' -time docker push {self.image} +time retry docker push {self.image} ''' if scope == 'deploy' and self.publish_as and not is_test_deployment: push_image = f''' diff --git a/ci/ci/github.py b/ci/ci/github.py index ef45bf0299d..975ea2e792e 100644 --- a/ci/ci/github.py +++ b/ci/ci/github.py @@ -9,6 +9,7 @@ import zulip from hailtop.config import get_deploy_config +from hailtop.batch_client.aioclient import Batch from hailtop.utils import check_shell, check_shell_output, RETRY_FUNCTION_SCRIPT from .constants import GITHUB_CLONE_URL, AUTHORIZED_USERS from .build import BuildConfiguration, Code @@ -19,7 +20,9 @@ log = logging.getLogger('ci') -CALLBACK_URL = get_deploy_config().url('ci', '/api/v1alpha/batch_callback') +deploy_config = get_deploy_config() + +CALLBACK_URL = deploy_config.url('ci', '/api/v1alpha/batch_callback') zulip_client = zulip.Client(config_file="/zulip-config/.zuliprc") @@ -657,11 +660,14 @@ async def _update_deploy(self, batch_client): 'to': 'team', 'topic': 'CI Deploy Failure', 'content': f''' +@dev state: {self.deploy_state} branch: {self.branch.short_str()} sha: {self.sha} +url: {deploy_config.url('ci', f'/batches/{self.deploy_batch.id}')} '''} - zulip_client.send_message(request) + result = zulip_client.send_message(request) + log.info(result) self.state_changed = True @@ -828,7 +834,7 @@ async def deploy(self, batch_client, steps): self.deploy_batch = deploy_batch return deploy_batch.id finally: - if deploy_batch and not self.deploy_batch: + if deploy_batch and not self.deploy_batch and isinstance(deploy_batch, Batch): log.info(f'cancelling partial deploy batch {deploy_batch.id}') await deploy_batch.cancel() diff --git a/docker/requirements.txt b/docker/requirements.txt index bf4e4aa9cd5..7444c3dba44 100644 --- a/docker/requirements.txt +++ b/docker/requirements.txt @@ -1,4 +1,4 @@ -aiodocker==0.14.0 +aiodocker==0.17.0 aiodns==2.0.0 aiohttp==3.6.0 aiohttp-jinja2==1.1.1 diff --git a/hail/Dockerfile.hail-run b/hail/Dockerfile.hail-run index a0c30138cf4..cb6dee60200 100644 --- a/hail/Dockerfile.hail-run +++ b/hail/Dockerfile.hail-run @@ -1,4 +1,4 @@ -FROM {{ base_image.image }} +FROM {{ service_base_image.image }} COPY python/requirements.txt . COPY python/dev-requirements.txt . diff --git a/hail/Makefile b/hail/Makefile index 9461e230986..9c329baf74d 100644 --- a/hail/Makefile +++ b/hail/Makefile @@ -291,31 +291,44 @@ install-benchmark: clean-docs: $(MAKE) -C www clean -HAIL_CACHE_VERSION = $(shell cat python/hail/hail_version) - -.PHONY: docs -docs: install $(PYTHON_VERSION_INFO) install-dev-deps +.PHONY: base-docs +base-docs: install-dev-deps mkdir -p build rm -rf build/www cp -R www build/www - rm -rf build/docs + $(MAKE) -C build/www + rm build/www/*.md + +.PHONY: pipeline-docs +pipeline-docs: base-docs + mkdir -p build/pipeline/docs + rm -rf build/pipeline/docs + cp -R python/hailtop/pipeline/docs build/pipeline/ + $(MAKE) -C build/pipeline/docs BUILDDIR=_build clean html + mkdir -p build/www/docs + rm -rf build/www/docs/pipeline + mv build/pipeline/docs/_build/html build/www/docs/pipeline + @echo Built Pipeline docs: build/www/docs/pipeline/index.html + +HAIL_CACHE_VERSION = $(shell cat python/hail/hail_version) +.PHONY: hail-docs +hail-docs: install $(PYTHON_VERSION_INFO) base-docs mkdir -p build/docs + rm -rf build/docs cp -R python/hail/docs build/ cp python/hail/hail_version build/ sed -E "s/\(hail\#([0-9]+)\)/(\[#\1](https:\/\/github.com\/hail-is\/hail\/pull\/\1))/g" \ < build/docs/change_log.md \ | pandoc -o build/docs/change_log.rst - $(MAKE) -C build/www - rm build/www/*.md $(MAKE) SPHINXOPTS='-tchecktutorial' -C build/docs BUILDDIR=_build clean html mkdir -p build/www/docs rm -rf build/www/docs/$(HAIL_MAJOR_MINOR_VERSION) mv build/docs/_build/html build/www/docs/$(HAIL_MAJOR_MINOR_VERSION) find ./build/www -iname *.html -type f -exec sed -i -e "s/\.css/\.css\?v\=$(HAIL_CACHE_VERSION)/" {} +; - @echo Built docs: build/www/docs/$(HAIL_MAJOR_MINOR_VERSION)/index.html + @echo Built Hail docs: build/www/docs/$(HAIL_MAJOR_MINOR_VERSION)/index.html .PHONY: upload-docs -upload-docs: docs +upload-docs: hail-docs pipeline-docs cd build && tar czf www.tar.gz www gsutil -m cp build/www.tar.gz $(docs_location) gsutil -m retention temp set $(docs_location) diff --git a/hail/python/hail/docs/experimental/vcf_combiner.rst b/hail/python/hail/docs/experimental/vcf_combiner.rst index d6819161829..37c13fd6ecf 100644 --- a/hail/python/hail/docs/experimental/vcf_combiner.rst +++ b/hail/python/hail/docs/experimental/vcf_combiner.rst @@ -1,11 +1,11 @@ VCF Combiner ============ -Library functions for combining gVCFS and sparse matrix tables into +Library functions for combining GVCFS and sparse matrix tables into larger sparse matrix tables. What this module provides: - - A Sensible way to transform input gVCFS. + - A Sensible way to transform input GVCFS. - The combining function. What this module does not provide: @@ -27,7 +27,7 @@ Sparse Matrix Tables -------------------- Sparse matrix tables are a new method of representing VCF style data in a space efficient way. They -are produced them using :func:`transform_gvcf` on an imported gVCF, or by using +are produced them using :func:`transform_gvcf` on an imported GVCF, or by using :func:`combine_gvcfs` on smaller sparse matrix tables. They have two components that differentiate them from matrix tables produced by importing VCFs. @@ -46,7 +46,7 @@ example: :: This record indicates that S01 is homozygous reference until position 15,000 with approximate ``GQ`` of 40 across the few hundred base pair block. -A sparse matrix table has an entry field ``END`` that corresponds to the gVCF ``INFO`` field, +A sparse matrix table has an entry field ``END`` that corresponds to the GVCF ``INFO`` field, ``END``. It has the same meaning, but only for the single column where the END resides. In a sparse matrix table, there should be no present entries for this sample between ``chr1:14524`` and ``chr1:15000``, inclusive. @@ -64,7 +64,7 @@ over 3 terabytes. Even if we only had the minimum required ``PL`` arrays materia still be looking at gigabytes for a single row. A sparse matrix table solves this issue by creating new fields that are 'local'. It only stores -information that was present in the imported gVCFs. The :func:`transform_gvcf` does this initial +information that was present in the imported GVCFs. The :func:`transform_gvcf` does this initial conversion. The fields ``GT``, ``AD``, ``PGT``, ``PL``, are converted to their local versions, ``LGT``, ``LAD``, ``LPGT``, ``LPL``, and a ``LA`` (local alleles) array is added. The ``LA`` field serves as the map between the ``alleles`` field and the local fields. For example (using VCF-like diff --git a/hail/python/hail/experimental/vcf_combiner.py b/hail/python/hail/experimental/vcf_combiner.py index f5cc0182863..8acd76c7e24 100644 --- a/hail/python/hail/experimental/vcf_combiner.py +++ b/hail/python/hail/experimental/vcf_combiner.py @@ -402,12 +402,12 @@ def drive_combiner(sample_map_path, intervals, out_file, tmp_path, header, overw run_combiner(sample_names, sample_paths, intervals, out_file, tmp_path, header, overwrite) def main(): - parser = argparse.ArgumentParser(description="Driver for hail's gVCF combiner") + parser = argparse.ArgumentParser(description="Driver for hail's GVCF combiner") parser.add_argument('sample_map', help='path to the sample map (must be readable by this script). ' 'The sample map should be tab separated with two columns. ' 'The first column is the sample ID, and the second column ' - 'is the gVCF path.') + 'is the GVCF path.') parser.add_argument('out_file', help='path to final combiner output') parser.add_argument('--tmp-path', help='path to folder for intermediate output (can be a cloud bucket)', default='/tmp') diff --git a/hail/python/hail/expr/functions.py b/hail/python/hail/expr/functions.py index 37465a974f7..5da2defba7b 100644 --- a/hail/python/hail/expr/functions.py +++ b/hail/python/hail/expr/functions.py @@ -2059,7 +2059,7 @@ def range(start, stop=None, step=1) -> ArrayNumericExpression: if stop is None: stop = start start = hl.literal(0) - return apply_expr(lambda sta, sto, ste: ArrayRange(sta, sto, ste), tarray(tint32), start, stop, step) + return apply_expr(lambda sta, sto, ste: ToArray(StreamRange(sta, sto, ste)), tarray(tint32), start, stop, step) @typecheck(p=expr_float64, seed=nullable(int)) diff --git a/hail/python/hail/expr/types.py b/hail/python/hail/expr/types.py index 7799f44d8dc..12466209871 100644 --- a/hail/python/hail/expr/types.py +++ b/hail/python/hail/expr/types.py @@ -31,6 +31,7 @@ 'tstr', 'tbool', 'tarray', + 'tstream', 'tndarray', 'tset', 'tdict', @@ -100,6 +101,7 @@ def dtype(type_str): str = "tstr" / "str" locus = ("tlocus" / "locus") _ "[" identifier "]" array = ("tarray" / "array") _ "<" type ">" + array = ("tstream" / "stream") _ "<" type ">" ndarray = ("tndarray" / "ndarray") _ "<" type, identifier ">" set = ("tset" / "set") _ "<" type ">" dict = ("tdict" / "dict") _ "<" type "," type ">" @@ -768,6 +770,60 @@ def _get_context(self): return self.element_type.get_context() +class tstream(HailType): + @typecheck_method(element_type=hail_type) + def __init__(self, element_type): + self._element_type = element_type + super(tstream, self).__init__() + + @property + def element_type(self): + return self._element_type + + def _traverse(self, obj, f): + if f(self, obj): + for elt in obj: + self.element_type._traverse(elt, f) + + def _typecheck_one_level(self, annotation): + raise TypeError("type 'stream' is not realizable in Python") + + def __str__(self): + return "stream<{}>".format(self.element_type) + + def _eq(self, other): + return isinstance(other, tstream) and self.element_type == other.element_type + + def _pretty(self, l, indent, increment): + l.append('stream<') + self.element_type._pretty(l, indent, increment) + l.append('>') + + def _parsable_string(self): + return "Stream[" + self.element_type._parsable_string() + "]" + + def _convert_from_json(self, x): + return [self.element_type._convert_from_json_na(elt) for elt in x] + + def _convert_to_json(self, x): + return [self.element_type._convert_to_json_na(elt) for elt in x] + + def _propagate_jtypes(self, jtype): + self._element_type._add_jtype(jtype.elementType()) + + def unify(self, t): + return isinstance(t, tstream) and self.element_type.unify(t.element_type) + + def subst(self): + return tstream(self.element_type.subst()) + + def clear(self): + self.element_type.clear() + + def _get_context(self): + return self.element_type.get_context() + + class tset(HailType): """Hail type for collections of distinct elements. diff --git a/hail/python/hail/ir/base_ir.py b/hail/python/hail/ir/base_ir.py index fa044665882..462a667b447 100644 --- a/hail/python/hail/ir/base_ir.py +++ b/hail/python/hail/ir/base_ir.py @@ -362,6 +362,9 @@ def renderable_new_block(self, i: int) -> bool: def parse(self, code, ref_map={}, ir_map={}): return Env.hail().expr.ir.IRParser.parse_blockmatrix_ir(code, ref_map, ir_map) + def unpersisted(self): + return self + class JIRVectorReference(object): def __init__(self, jid, length, item_type): diff --git a/hail/python/hail/ir/blockmatrix_ir.py b/hail/python/hail/ir/blockmatrix_ir.py index 8dbcebfb2cf..1626089d00d 100644 --- a/hail/python/hail/ir/blockmatrix_ir.py +++ b/hail/python/hail/ir/blockmatrix_ir.py @@ -27,6 +27,9 @@ def _eq(self, other): def _compute_type(self): self._type = Env.backend().blockmatrix_type(self) + def unpersisted(self): + return self.reader.unpersisted(self) + class BlockMatrixMap(BlockMatrixIR): @typecheck_method(child=BlockMatrixIR, name=str, f=IR, needs_dense=bool) diff --git a/hail/python/hail/ir/blockmatrix_reader.py b/hail/python/hail/ir/blockmatrix_reader.py index e423cb0e2b8..6d1d8b29a5c 100644 --- a/hail/python/hail/ir/blockmatrix_reader.py +++ b/hail/python/hail/ir/blockmatrix_reader.py @@ -14,6 +14,9 @@ def render(self): def __eq__(self, other): pass + def unpersisted(self, ir): + return ir + class BlockMatrixNativeReader(BlockMatrixReader): @typecheck_method(path=str) @@ -49,3 +52,21 @@ def __eq__(self, other): self.path == other.path and \ self.shape == other.shape and \ self.block_size == other.block_size + + +class BlockMatrixPersistReader(BlockMatrixReader): + def __init__(self, id, original): + self.id = id + self.original = original + + def render(self): + reader = {'name': 'BlockMatrixPersistReader', + 'id': self.id} + return escape_str(json.dumps(reader)) + + def __eq__(self, other): + return isinstance(other, BlockMatrixPersistReader) and \ + self.id == other.id + + def unpersisted(self, ir): + return self.original \ No newline at end of file diff --git a/hail/python/hail/ir/blockmatrix_writer.py b/hail/python/hail/ir/blockmatrix_writer.py index 310e956270b..333b9b39600 100644 --- a/hail/python/hail/ir/blockmatrix_writer.py +++ b/hail/python/hail/ir/blockmatrix_writer.py @@ -141,3 +141,22 @@ def __eq__(self, other): self.add_index == other.add_index and \ self.compression == other.compression and \ self.custom_filenames == other.custom_filenames + + +class BlockMatrixPersistWriter(BlockMatrixWriter): + @typecheck_method(id=str, storage_level=str) + def __init__(self, id, storage_level): + self.id = id + self.storage_level = storage_level + + def render(self): + writer = {'name': 'BlockMatrixPersistWriter', + 'id': self.id, + 'storageLevel': self.storage_level} + return escape_str(json.dumps(writer)) + + def __eq__(self, other): + return isinstance(other, BlockMatrixPersistWriter) and \ + self.id == other.id and \ + self.storage_level == other.storage_level + diff --git a/hail/python/hail/ir/ir.py b/hail/python/hail/ir/ir.py index 4651b6e6284..23f514d8735 100644 --- a/hail/python/hail/ir/ir.py +++ b/hail/python/hail/ir/ir.py @@ -606,6 +606,25 @@ def _compute_type(self, env, agg_env): self._type = tarray(tint32) +class StreamRange(IR): + @typecheck_method(start=IR, stop=IR, step=IR) + def __init__(self, start, stop, step): + super().__init__(start, stop, step) + self.start = start + self.stop = stop + self.step = step + + @typecheck_method(start=IR, stop=IR, step=IR) + def copy(self, start, stop, step): + return StreamRange(start, stop, step) + + def _compute_type(self, env, agg_env): + self.start._compute_type(env, agg_env) + self.stop._compute_type(env, agg_env) + self.step._compute_type(env, agg_env) + self._type = tstream(tint32) + + class MakeNDArray(IR): @typecheck_method(data=IR, shape=IR, row_major=IR) def __init__(self, data, shape, row_major): @@ -2183,6 +2202,24 @@ def is_effectful() -> bool: return True +class UnpersistBlockMatrix(IR): + @typecheck_method(child=BlockMatrixIR) + def __init__(self, child): + super().__init__(child) + self.child = child + + def copy(self, child): + return UnpersistBlockMatrix(child) + + def _compute_type(self, env, agg_env): + self.child._compute_type() + self._type = tvoid + + @staticmethod + def is_effectful() -> bool: + return True + + class TableToValueApply(IR): def __init__(self, child, config): super().__init__(child) diff --git a/hail/python/hail/linalg/blockmatrix.py b/hail/python/hail/linalg/blockmatrix.py index 1fd48b0eaac..52649888771 100644 --- a/hail/python/hail/linalg/blockmatrix.py +++ b/hail/python/hail/linalg/blockmatrix.py @@ -8,15 +8,16 @@ import hail as hl import hail.expr.aggregators as agg from hail.expr import construct_expr, construct_variable -from hail.expr.expressions import expr_float64, matrix_table_source, check_entry_indexed +from hail.expr.expressions import expr_float64, matrix_table_source, check_entry_indexed, \ + expr_tuple, expr_array, expr_int64 from hail.ir import BlockMatrixWrite, BlockMatrixMap2, ApplyBinaryPrimOp, Ref, F64, \ BlockMatrixBroadcast, ValueToBlockMatrix, BlockMatrixRead, JavaBlockMatrix, BlockMatrixMap, \ ApplyUnaryPrimOp, IR, BlockMatrixDot, tensor_shape_to_matrix_shape, BlockMatrixAgg, BlockMatrixRandom, \ BlockMatrixToValueApply, BlockMatrixToTable, BlockMatrixFilter, TableFromBlockMatrixNativeReader, TableRead, \ BlockMatrixSlice, BlockMatrixSparsify, BlockMatrixDensify, RectangleSparsifier, \ - RowIntervalSparsifier, BandSparsifier -from hail.ir.blockmatrix_reader import BlockMatrixNativeReader, BlockMatrixBinaryReader -from hail.ir.blockmatrix_writer import BlockMatrixBinaryWriter, BlockMatrixNativeWriter, BlockMatrixRectanglesWriter + RowIntervalSparsifier, BandSparsifier, UnpersistBlockMatrix +from hail.ir.blockmatrix_reader import BlockMatrixNativeReader, BlockMatrixBinaryReader, BlockMatrixPersistReader +from hail.ir.blockmatrix_writer import BlockMatrixBinaryWriter, BlockMatrixNativeWriter, BlockMatrixRectanglesWriter, BlockMatrixPersistWriter from hail.table import Table from hail.typecheck import * from hail.utils import new_temp_file, new_local_temp_file, local_path_uri, storage_level @@ -212,6 +213,7 @@ class BlockMatrix(object): - Natural logarithm, :meth:`log`. """ + @staticmethod def _from_java(jbm): return BlockMatrix(JavaBlockMatrix(jbm)) @@ -999,6 +1001,13 @@ def sparsify_triangle(self, lower=False, blocks_only=False): return self.sparsify_band(lower_band, upper_band, blocks_only) + @typecheck_method(intervals=expr_tuple([expr_array(expr_int64), expr_array(expr_int64)]), + blocks_only=bool) + def _sparsify_row_intervals_expr(self, intervals, blocks_only=False): + return BlockMatrix( + BlockMatrixSparsify(self._bmir, intervals._ir, + RowIntervalSparsifier(blocks_only))) + @typecheck_method(starts=oneof(sequenceof(int), np.ndarray), stops=oneof(sequenceof(int), np.ndarray), blocks_only=bool) @@ -1090,13 +1099,7 @@ def sparsify_row_intervals(self, starts, stops, blocks_only=False): if any([starts[i] > stops[i] for i in range(0, n_rows)]): raise ValueError('every start value must be less than or equal to the corresponding stop value') - starts_and_stops = hl.literal( - (starts, stops), - hl.ttuple(hl.tarray(hl.tint64), hl.tarray(hl.tint64))) - return BlockMatrix( - BlockMatrixSparsify(self._bmir, - starts_and_stops._ir, - RowIntervalSparsifier(blocks_only))) + return self._sparsify_row_intervals_expr((starts, stops), blocks_only) @typecheck_method(uri=str) def tofile(self, uri): @@ -1186,7 +1189,7 @@ def is_sparse(self): ------- :obj:`bool` """ - return self._jbm.gp().maybeBlocks().isDefined() + return Env.backend()._to_java_ir(self._bmir).typ().isSparse() @property def T(self): @@ -1261,7 +1264,9 @@ def persist(self, storage_level='MEMORY_AND_DISK'): :class:`.BlockMatrix` Persisted block matrix. """ - return BlockMatrix._from_java(self._jbm.persist(storage_level)) + id = Env.get_uid() + Env.backend().execute(BlockMatrixWrite(self._bmir, BlockMatrixPersistWriter(id, storage_level))) + return BlockMatrix(BlockMatrixRead(BlockMatrixPersistReader(id, self._bmir))) def unpersist(self): """Unpersists this block matrix from memory/disk. @@ -1276,7 +1281,8 @@ def unpersist(self): :class:`.BlockMatrix` Unpersisted block matrix. """ - return BlockMatrix._from_java(self._jbm.unpersist()) + Env.backend().execute(UnpersistBlockMatrix(self._bmir)) + return BlockMatrix(self._bmir.unpersisted()) def __pos__(self): return self diff --git a/hail/python/hail/methods/statgen.py b/hail/python/hail/methods/statgen.py index 782ae3cddd5..bd628297ecf 100644 --- a/hail/python/hail/methods/statgen.py +++ b/hail/python/hail/methods/statgen.py @@ -2647,10 +2647,7 @@ def ld_matrix(entry_expr, locus_expr, radius, coord_expr=None, block_size=None) starts_and_stops = hl.linalg.utils.locus_windows(locus_expr, radius, coord_expr, _localize=False) starts_and_stops = hl.tuple([starts_and_stops[0].map(lambda i: hl.int64(i)), starts_and_stops[1].map(lambda i: hl.int64(i))]) ld = hl.row_correlation(entry_expr, block_size) - return BlockMatrix( - BlockMatrixSparsify(ld._bmir, - starts_and_stops._ir, - RowIntervalSparsifier(blocks_only=False))) + return ld._sparsify_row_intervals_expr(starts_and_stops, blocks_only=False) @typecheck(n_populations=int, diff --git a/hail/python/hailtop/pipeline/Makefile b/hail/python/hailtop/pipeline/Makefile new file mode 100644 index 00000000000..394c73f438f --- /dev/null +++ b/hail/python/hailtop/pipeline/Makefile @@ -0,0 +1,8 @@ +.PHONY: doctest +doctest: + pytest \ + -v \ + -r A \ + --doctest-modules \ + --ignore=docs/conf.py \ + --doctest-glob='*.rst' diff --git a/hail/python/hailtop/pipeline/backend.py b/hail/python/hailtop/pipeline/backend.py index fec96e116ab..1952d59beb4 100644 --- a/hail/python/hailtop/pipeline/backend.py +++ b/hail/python/hailtop/pipeline/backend.py @@ -12,8 +12,19 @@ class Backend: + """ + Abstract class for backends. + """ + @abc.abstractmethod - def _run(self, pipeline, dry_run, verbose, delete_scratch_on_exit): + def _run(self, pipeline, dry_run, verbose, delete_scratch_on_exit, **backend_kwargs): + """ + Execute a pipeline. + + Warning + ------- + This method should not be called directly. Instead, use :meth:`.Pipeline.run`. + """ return @@ -31,11 +42,11 @@ class LocalBackend(Backend): ---------- tmp_dir: :obj:`str`, optional Temporary directory to use. - gsa_key_file :obj:`str`, optional + gsa_key_file: :obj:`str`, optional Mount a file with a gsa key to `/gsa-key/key.json`. Only used if a task specifies a docker image. This option will override the value set by the environment variable `HAIL_PIPELINE_GSA_KEY_FILE`. - extra_docker_run_flags :obj:`str`, optional + extra_docker_run_flags: :obj:`str`, optional Additional flags to pass to `docker run`. Only used if a task specifies a docker image. This option will override the value set by the environment variable `HAIL_PIPELINE_EXTRA_DOCKER_RUN_FLAGS`. @@ -59,6 +70,24 @@ def __init__(self, tmp_dir='/tmp/', gsa_key_file=None, extra_docker_run_flags=No self._extra_docker_run_flags = flags def _run(self, pipeline, dry_run, verbose, delete_scratch_on_exit): # pylint: disable=R0915 + """ + Execute a pipeline. + + Warning + ------- + This method should not be called directly. Instead, use :meth:`.Pipeline.run`. + + Parameters + ---------- + pipeline: :class:`.Pipeline` + Pipeline to execute. + dry_run: :obj:`bool` + If `True`, don't execute code. + verbose: :obj:`bool` + If `True`, print debugging output. + delete_scratch_on_exit: :obj:`bool` + If `True`, delete temporary directories with intermediate files. + """ tmpdir = self._get_scratch_dir() script = ['#!/bin/bash', @@ -174,24 +203,34 @@ def _get_random_name(): class BatchBackend(Backend): """ - Backend that executes pipelines on a Kubernetes cluster using `batch`. + Backend that executes pipelines on Hail's Batch Service on Google Cloud. Examples -------- - >>> batch_backend = BatchBackend(tmp_dir='http://localhost:5000') + >>> batch_backend = BatchBackend('test') >>> p = Pipeline(backend=batch_backend) + >>> p.run() # doctest: +SKIP + >>> batch_backend.close() Parameters ---------- - url: :obj:`str` - URL to batch server. + billing_project: :obj:`str` + Name of billing project to use. """ def __init__(self, billing_project): self._batch_client = BatchClient(billing_project) def close(self): + """ + Close the connection with the Batch Service. + + Notes + ----- + This method should be called after executing your pipelines at the + end of your script. + """ self._batch_client.close() def _run(self, @@ -201,7 +240,32 @@ def _run(self, delete_scratch_on_exit, wait=True, open=False, - batch_submit_args=None): # pylint: disable-msg=too-many-statements + disable_progress_bar=False): # pylint: disable-msg=too-many-statements + """ + Execute a pipeline. + + Warning + ------- + This method should not be called directly. Instead, use :meth:`.Pipeline.run` + and pass :class:`.BatchBackend` specific arguments as key-word arguments. + + Parameters + ---------- + pipeline: :class:`.Pipeline` + Pipeline to execute. + dry_run: :obj:`bool` + If `True`, don't execute code. + verbose: :obj:`bool` + If `True`, print debugging output. + delete_scratch_on_exit: :obj:`bool` + If `True`, delete temporary directories with intermediate files. + wait: :obj:`bool`, optional + If `True`, wait for the pipeline to finish executing before returning. + open: :obj:`bool`, optional + If `True`, open the UI page for the batch. + disable_progress_bar: :obj:`bool`, optional + If `True`, disable the progress bar. + """ build_dag_start = time.time() bucket = self._batch_client.bucket @@ -333,7 +397,7 @@ def _cp(src, dst): print(f'Built DAG with {n_jobs_submitted} jobs in {round(time.time() - build_dag_start, 3)} seconds.') submit_batch_start = time.time() - batch = batch.submit(**(batch_submit_args or {})) + batch = batch.submit(disable_progress_bar=disable_progress_bar) jobs_to_command = {j.id: cmd for j, cmd in jobs_to_command.items()} diff --git a/hail/python/hailtop/pipeline/conftest.py b/hail/python/hailtop/pipeline/conftest.py new file mode 100644 index 00000000000..310f176b700 --- /dev/null +++ b/hail/python/hailtop/pipeline/conftest.py @@ -0,0 +1,37 @@ +import doctest +import os +import pytest + +import hailtop.pipeline as pipeline + + +@pytest.fixture(autouse=True) +def patch_doctest_check_output(monkeypatch): + # FIXME: remove once test output matches docs + base_check_output = doctest.OutputChecker.check_output + + def patched_check_output(self, want, got, optionflags): + return ((not want) or + (want.strip() == 'None') or + base_check_output(self, want, got, optionflags | doctest.NORMALIZE_WHITESPACE)) + + monkeypatch.setattr('doctest.OutputChecker.check_output', patched_check_output) + yield + monkeypatch.undo() + + +@pytest.fixture(scope="session", autouse=True) +def init(doctest_namespace): + # This gets run once per process -- must avoid race conditions + print("setting up doctest...") + + doctest_namespace['Pipeline'] = pipeline.Pipeline + + olddir = os.getcwd() + os.chdir(os.path.join(os.path.dirname(os.path.realpath(__file__)), + "docs")) + try: + print("finished setting up doctest...") + yield + finally: + os.chdir(olddir) diff --git a/hail/python/hailtop/pipeline/docs/Makefile b/hail/python/hailtop/pipeline/docs/Makefile new file mode 100644 index 00000000000..ca93635a14d --- /dev/null +++ b/hail/python/hailtop/pipeline/docs/Makefile @@ -0,0 +1,36 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +#SPHINXBUILD = sphinx-build +SPHINXBUILD = python3 -msphinx -T +SOURCEDIR = . +BUILDDIR = _build +ALLSPHINXOPTS = -W -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: clean +clean: + rm -rf $(BUILDDIR) + +.PHONY: check-python +check-python: + @echo "Ensuring that Pipeline can be imported..." + python3 -c "import hailtop.pipeline" + +.PHONY: html +html: check-python + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." diff --git a/hail/python/hailtop/pipeline/docs/_static/images/dags/dags.001.png b/hail/python/hailtop/pipeline/docs/_static/images/dags/dags.001.png new file mode 100644 index 0000000000000000000000000000000000000000..7029b1f956318bb7717cde4fd06ef100082472e2 GIT binary patch literal 18806 zcmeHPd010t_P(Hklqy=Qv_^;(TiQX5mI^44U<=~X!nAfoWeJKxZDbQM0kRNnYb!#u z)Iw#6)e+jV1j?`z5~U^rLRupP3?x8=5Vo*{gq!WoWz+=x|2NO`d*=R?@A>Ziw)36y zp7*?geE0VN{}r#Se+2-575n%3eg*(b;pP%xxfT3TSGRTr{9%#sng54CY5N8R0Qdm= zefNA$vKa5R`C;tT@u7akC(HbNHk|s$N!_l6}{FW$)gI4O!tK7a~5-zWD9& z3%vOIo4?$!al-c2t$0!2y&9Cgub-cvH8JGxuLs2)j~cLssQh{fUvfq}ug?1>uax(QiYYbUh0oL^AFFk*@ zbm^-A{d>R7i(j<@qFzjfytlx*h+PnWnGX!a4-h{<{9qm!5kElu0P%wt>5A`0wFCj?+e;n=WxJ?-gE&$k-LRGEP%3rS$Z#Bo3e07chSj|Itap zuiL%>98S7Bu)1ao9YdW;la)2C%#ohbjaRz^A?tn=tbXCWtGXU6^I8_}g!z*ZXd3>X zMyEg53F!21rYtZ1knWV>aiRe-DDnHxkCpQ?gaMw>Ug~m&gXdUTQ_HyIEwi{1p1AzX z*}ul36Me5;0O)xCaDlgGgL6J<;7;Syr_)oFkDhMR2L0~s}F-B^X7#h{U- zDC7I+))|WX0NZ@Rjb&O`0xDu1`tAj2J=2o7ci`@LwyM z1~n=tMq30OiZg<8yPUqYw^V-S5v3|iM~k>StOs`c!kcV!De7+771#DR?88X?T}kph zzG@#3Ykqrq@wWNRj~rTWHtqb=vH*)fizCiyD4PY6n@=it=ynZxd{b@^Dza}(M<`oCJ^_9syjZBx9GzO}mN>{z8JtjOug z>!x8r>tO@eJo>!f`lU0W0$}HN%Pmoo-?Yq$E?0ELGorW29I<{ko#Tw%_E&^~yDIEM zPP{k-{p;nVrQ8p*L%(_vfzNJ#OAzd+A+#W8b9~G{uwA_!*z#4k-%E!*3m>+1mYDcr z)@O$Nsl6!*mkYrC`wgG7NZy_0i`od zE+Pqw7!DZh^l)XV=ApC0j*O&cf~ao%GI>%T#W|TT@9<~Gf~A4Qzdz{enyu+`Ey`#qLk>U%a8v<>P@4HgQamj+4=@^ zI_t`YVFHzqLRGK$+vFvp?qa!Yo97iKq_;<@cBk}A7szS_r2aXch_f>$c}D6WppM|u zl@yK6MWi2fcJOFa-(j)HWIt)r(+Eemd{r?&os~OXnMh6ZR>tLka#p=Ck%ys8wI)iU zB5J073u3p-i8-DpH>j$uuL#J~Pg@Pwh))vvQ&;^bJOg%t2MN8JHv_%MAw>S@729n! z;d;ZGeDCwMl6C?`Vr_!jrEYF8j`2FPnZyp8o<8DvlwH!}C43Yksg(@WNH0QX8o-^g zp^3ZP3w+Sf184ruYW7f#@}lt^d5(g15Y^w^7Qxi@L`Piii+bAxr*ZGm+c1e~sO814 z(eV$BXK8anrh5QxJVo}m3eGF|bAS2CXxPD<@p%r$rUUm5akS|mo>(o394(gxh#cxC zs#-lD{S1GJqgHx&B0+DBb;E+?fdZW5MA}x~&w-w#kl8PTLP33eY?5Y9ejATFpG`5m zSqO!DX{Q+Ls~g7Kg0LK{a>Jy^4$VmJix?v28P8HCo6zpOJ&^jkJdbE3wU2#jiJLRp z@dK%PVY~=e+gihCL#-_i1^%yo9JsdYWacNlT~SQe0?q&~QcbDq`Lgx8PG6u5A@zSP zd)vQ}oIBYccxAk_a@+&XYVfYSNn>PoUgu~qNB=d;QHbfZOP*W8qtyw=hw!f6<9&k| zaz{;DSoL~ZErfp5v96E{q*?_jFq+_y zQ)yGf{Bc}q`D{nf1o5bnJjq7C$=an&kI3o9fSkE*46o6yUV4m3(}<<+DIfgtbA<2{ zcL!w>-Ov42>n1)^F;!7P#irC45_rOu5MK5~)J>%_X%x*RzKWT~p^Tym)0;9#1Ns6xwsLN& z%H7DEiqc7Df;2S2CB8gBt0yT&nVtpl;o5u#Zt^|ZjzEv_5Xh=NOfL=@(glZ7il%;d zH;jNoLd+cubsSfc2tm5qLy@=tpzzva;xobVOI{;3GR6co3tSa1Ns_03WJh{L>P%aS z5)i7*wc`Ax<)7A=Kr za4Br39>>KPnGEA7pVz@E@0fdJ-}pXc#TQ^3XVAT;2eTjrPa!kdCCwePhvQVqo}Sto zJA>xCH>%sNGo;fus?#~iDttTbT5e3P(bCZ8uLnKUiLv-vfws}rFiUL{-hF)$ z5vU*&D>1y|zCMbqv=-LO5}3w&U6J*K3|SP%i`Bf_`}h7Ly!MIyuUBV04eN{qEy3We z)H%CIPbY7_FCnsJbHmoS#(sp7vy3<_2d%t?ifgy(jJ@NBO)^nN|fKDIkGn84R1L_cgg#N z5m_UrD9by0w@uKdG@`mpJ2OQ)a}ws(W!KgY@k4QRMOIZwq#aEPzJ}>h_jNm2p3i3w zi&bvke&O}q_#ZJ~m7$WA*qmeRWfYEvM7I`|%z_G5M-()jTNxw1iat}Pcqd|%5JUO3 z_>vcR6BIg9gGG{&ku2)&yOpKv-nl`HAWV5MM0P{;2F^=%se(y(h^~Teqm869Njo?@ z*`az=j1FSP#7&6{G}XN^IR_>G3PT58Z0gJ+{l-1F(rw5d8^Y?FFqI)!?I)L~>Q^7h z1B94CaTp;7OlXK@%eV4wKs_fJz5K`uj&ERWN^gHvgx1!0Us@1EW;%Y~*a&w`RmnzsQm^1%C&pQxoxOryTrY3qbX0qs;j4L4pZ8>(*XZVCFBwIZ&WqvlNe zL0+`Frj|Tv6LI2PmYCt*FoN&YZet~NqdK)+3o^>#+uV05AZMz&+9P)9I;ZXIxE>ql z-0g1S6T)(tW6&6;W>lA%4`#aZg^_|aoIbrRZ=9;^l;u>&^O&|D0TaZy$x^#?92>&Quyze1tuZ?*g=3FSQjuB-Qa802 z+9Q+DjCi@=8q;{R4%ItnC9|Q?9%P5%!jm6BLNvsP$Qo68)l;fL+$^!Oq|}Y}7FSVe z6QCb1jAau z+hW;#{SBdgfPhcCwCTsqeS)*lO`wU z*34|3a0`P++-beZRVbE6niBwh+>k&i#va`2Vp6`j;(&!PBSf3-<%WNB3m)W6G4>)BwGO<$? zvcnbj1byEYJ{w&oAoaRPK5$H$Y~tl9lM6x0k5hO#lXBO?@PliVIyi%^)^>o$c!K^L zCSFl=2+lIQQ`nCRjy(m+td1vNt5(u9SGv<4gvOQxDe*I);;nw+jFy;Zm?k%0nqk!n z1`W!iIBx=3E-r-XMKuj9D4Jrdub@qZCx6+b5y$xWp)bq!2&VsFU_v}pRNR{@UNPGP%z z%YJy1n@L$Oh>P#i1}mR`ezC(eY93~8ZVd;%&|thx8Faqk3)jWhZrxw>*DLg@REwLv z9F@ri^Nm$-P3e1iU8V`}=NnJpI{7^GXpOme@=5jab9_a8&;3rt=#IeC}K~-~j}I>;X7pKNJ7Ha8Ic|hq=JjlCZc%5&JGIuN{bPJz$m@;dze25nGjfcQLGN4o5cm z+;oT~&CZ7S=zZ2n3l^9RlX&uWyiyVgX)*<(U@R~Nf|KTyu^cJHkU|XMlV$}N;ge=?K=|Z; zn@{dJ{ZtG9Hk;>v3t$zZ3n97?0!IiOAu^yz`$A+ulPM4x&}<5%y!~G*Z|RWr+RLZG V9h#I<_=l8${eA(yr5}c0_&0$9Z1w;E literal 0 HcmV?d00001 diff --git a/hail/python/hailtop/pipeline/docs/_static/images/dags/dags.002.png b/hail/python/hailtop/pipeline/docs/_static/images/dags/dags.002.png new file mode 100644 index 0000000000000000000000000000000000000000..acd0acc549e888a5027ade5fc4e3c1a414c56106 GIT binary patch literal 23003 zcmeHvcTiJX`|bu&=~yXJ1qG!AMT!s-)T3A_Dov>(qN0S}N#KB>0-{Gg0co}a0@4Jf z1%gCC2m(q8NHZYa5C|m+X?MdpQk?nCH+SybnLBgCKWrAOyz6<(df#X53j~f8qP2jKc@|{B9FCLd8=Z*kCVVnTNaU=M*oYTqk=Kvs35dcCX z0ALYp3LOIg9}NJQZ~y?kL;w(TPboKp0sxowjf+;^R_D&@IU#PVJ6=UxaaQ-g?GBCr z0I0tn_;}md+fmN{_ANIrJ%0oFKgJe zemOZgsOQyddguQ*@$Yc(F9Z4O-rnwd5Qv|jpSqu>I>OTha^Udc!w`*wkb?)+z#eK| z0dC%o{%USs3ja*v#p><82@>zqHWG&p&c{-?+BAlAG7R z+X6QTS^5v;fVu``Wo~dNbg5O(#PfzTxbo8cumjNLp6lBFT?YzTT6~Q#|LkWge3NyeDWw}U%!C-_v7(` z-25jW?%TFkL2mHjp3#^euiP6vKFJ+1d?>Z&r0@ppwB=6rc#v^R%Bi`jA`@aJp1V9J z$B#&&EqHb#EIY<$oo9GuczCzS8HNH}-2Xfn=5AcZ8B4N$~Ol z2d}@~#^xKC4yjU=(aJh5!NVI9UVTi&V8@foZe`OAO6f6gDId3+Naa;bl^1+HtrAO4 z6I#Ox&AD;80CQ8UB!jRS(Jr?NVm5?5oWOi+thc|{9V_HWCzgnWEY9&^qeMi4rpiYB zU%C`o(4eVcLAWz&9SE2G9AerSqMau}@v|B>GWordh@zyp7i#%@(Mp zDuzLRiFVnrGKh=2^0Fb&T0wVN;88sJJyxfmcQhS^EFHYBQe*4aXKqU$IMDj+NO(<~ z30yiO*{%G+Io6pKSwjtlcx--L4p&qTt1jh-vKg}-v21m4je@3wxCpTKel&fmg7D2X zUud>T(?qLDpM>5XDI}?#ckEl-3J(W_*~x!b46S@+mT~IYPHx%Y=_6e?b8@qvi`sgB zF|`xc=(u>{PKU*1HPyn8+XzW6X<7Zrj*$|VJanDbuyGF`hMobo-HgoIa#zi?zYp~v za|=G5fnYD6*A0rvs!fMSQwZ7mSMpZDk$D#uY;o-6beB_{Ruv<^YH@<_7Ng){x4c0Z-k@d}i zsiD9#{JW-zHG+VdSv6IF7pNO(Qu>W3&ELKSm@9vhq_JMc|5MLNofBnONc*ZHnjgQ)@+mtpMFtR3r{ zTbJP0m*`oy#ZbdoP-h&W*C>&L^csu&|K2<*7r1p;^G^x}@}eR{`ss2vb1m)3c=z6| zN|yDdb7&Q8gHigIVSiK?>9u?Felg!}?cIfEQY`Y0nYLoGsL zsXjO63m-?O$*5zdCz_A>8~+MghI0EsM?X*0Nb=U3??Puin5XPmU@_esGb>93;-J%AR=C7~>9d4U%r^}vt*gzcuf!>e zjb<9_4u-zOhThv0ktu`9aPmrEKcP!i4YX>oOGX6pE6F5{xrakFe;!8}tOmzd%5uQT zcbX&Sl3|BOlN1=i$+0ziM{Vpj0+ViwkKMaBT6Sbq>4_fvTpV5li(+ZNVGO85vY7PP z_a9Z%(Ta?gKsbN2>P)WtIL$J0UWa`n5%D3ws3Jdh^&~yNWh}jBPg7WR1aGhn!xKLD z5FUk(wzpSOM($*haS$v*q-%eZ#?V|WAq}-~$9)WD6uCd+seIF+S)H2ihp!1KoQUC_ zL@cz`!TkC>P5i&7MP=hRY!c)Rk}5uv=>OL}NfLXW%oZ9=pGnYgnxW~~qf~yD;=koE!_`gR(?36POpf)<^HoEY^*iGXj&+T*VKmyPLQP?M-x)beK#p_bC@+%RfRUI_Z zidWK^!`yUHJc%7VDReC0ehAH=#P~&!)0^WG?ELpi1-DmqWb`1)Rct~U25L;SZ6y=>AN~ zD_>feSD#d5n;6pkAvH>3X!ev+@}U&u_)G_EIN|GK2s8u7V*AfE$j4hLsg{0XbS(6} z?^a@bk5aKb@)68AWfr>JFzz2uZH^)*jFV@&uqF}V6zGfBx6>zlR9HTPt!*_;Ma?^m zE2u3U?g%vNn)&3HOj(%m5p3jThANsonA4HKPDn!?j#1@aLY{0VAj$NsyxF9WkPwwY zQyxW*8r3GuR{BAoy>^|@sP8SZq~9v+QzR4*HLFk1k9E06`{Wd4cteh*c?2++AL?tQ zb^4?q=n)IDT}X`4z5-&MeiVE03#EQ|Xb9ahGb)@5uCrlyaUWJfWc9e;FHp)OH$0Vi zxrBRI>n8+8@${cc8jXhhMCG!|=$%0|S09_~N<$9jxV|ST+vU%N4@=s!a331+JyY+U z6Fl&J-o(1egG?4Lhh!f zC!|A66i8IEeN~zts|=gWM&<TzTMpfMV_a996NYDq?@#`vV5!0~nOJSxSQ`tXY@P>we8698I3-c1e%Aci(;* zXB4pgW3KNayydNZ^3DV!8w*_>`1@YRqJla8Gv>)>C=U)irMMzPh|YL1JOj}_W*wl7 zy#%M5&AQpseMV(`k*s%{wO5mDze5jsAAxBw6>dV9F~TeK@}(9eEIqd!GndTp(`GNc6mKcC+s` zJ2e|8bIIrDDJ98y&p&TuNbZ2qezbYOKK{61EYpviC0?<6u6G-QHtVl8MoNcFLch1O z*`DFeJ&mImM4n~`wCPr=Z;0CeRDI_@{1HUe3_*9No6@RkowmAfs7N-WCIT@IKec*F_A{PcNZ7b)V}U&ogBX`S=f#Otd3aNLi-+ zV|hcC)H+Eg|MShRW4r6cTrR1KHDcaQXNErz5P(!U3~e8-fGuld1^fEwP{_Mje)sKP zVd(ZM5qlHt(4{j2SVFL6or;2AfH)5kA_B9`%Q+7EmUz#h3|FJ))4P=tvame3SPGg0Ru01bd14y%+16qCIFnm7^aY)ctnI+Tuloy;!l4Z?3Qr<$?8Rh{xE ztC8~hY|-0FDw?hzZ21;q$I!FuhdWFmn$4L9G1G$r=Jl#@E4w&Mi}7X){M!~SlGJmw zY>jcyZG7qTfzln9`m5dF{f!qcfg^2q8ow$2bdvzVr|+yf&?Xogmw?(dGb4X}U-%s| zQqc9yEVk7{;slMh{Q*=f9zWA^(0^p&uV`{)Oa&3+G}c4OL6fti*}G&^11X?m?F_5E z;8=)&*hIjX%F$o5gHwXrB?!;yY#C7%w@&}dX7%4D^S7$&O@5uuu|;mRxa3T$vndZc zpH}Anlc0155-q9V>u(HoADr$Pd=vdutJCRYrhNoHJs30gxmx4uC(B7lXJO1rz5Xyg z6p+KDQLfkK3Iclk`w`@YI#-0K&ZFd9TpwM(A7a7>$)A+92@Y_BrDB}Y3z?K~3XC_u ziV;{+FWxhHc^*lljNr~lHrlgsu2W7>J=V4(;}VnNvS_xW|57%kr&z4ylvBxY<;&ku zpXim={StXYwg*}_8Uoh1*SiybxxYj3Mjl{D_Mq2%m6>zi9u<$&v=Gf#i5`%nJX zZVDC}*bL=+>zfaO#qZPO=rv5zaKii$1x`4;z}{IS}aZL}W+qgu4K@}Jcae9-y!3xx0C zB6%<lJKxeD;$bIq@lRu5|WH97-xuSb74_j=Zb1y0eMB za=>d#xcM=Ml#QL25tRjXyIc<~*n8CO#0u;G+|roL6}ZA7fhPeRGzMm1w){E;$!XjV z1sry4lUzk%`7mLE<}g2*b$@+x^*&I^W+!j0RXiyFJV=AK=VvO*YaHQjfwO~`Qar`6 zBG@0RS~px=w%l=(HbK5EcXv;>pjetsAPpV+oK7Lh${?wPg4Uw~TJHmnt{G#%lfq>h z2h0Amnul+RB6o~dd(Rs}N6y1HE)~HCKk&Vn+r7G-1_9j-ax~E-xiBhsI&$%b|LkWS z?59TxSkc5h({+1I1;WBMY>wQ0EVBKD=<$^W zUMYa=x_6+X;;L~h?N#n67!_W8+2wzhJA`hsBO?_tethAMK_Ww~m7D{? zSP_FrsUX^L{cNl6j zpWt<(yMEKE#982{3eJy51Qgpk472SNnsjHsPtO?1o-kPWGg@`)1NKlDp>{M)XcpyK z=-$3N=#GvZ>YYbZ>H;eSB$bpT^Ofcrp6E|)U4YGXP5*2&iSfM)*5Jh*?`TvF9;p@)SDS@p6KlU36C!E=Nt0wsvJ<$d?O zpJU6h2f_-`5OzW;DzLt%12qsp&vpM1RuM%Lwb>Kz_~AelGj9TskcwVCAi?WbK<@dq z#M1^FM-qdo3K}f*E$rn+*_|&OQ)5vQd8N~D0!1%B&bAAp-BET(419XySUM~OBwyJk z)cThqK3P-Q(Jg$gEbpO$>A^EQnNZC)72}R`}lyi2`9WTdCBS zaUhkOpEsPJKb&71UC@`RpMx>Sv@;#66{?S2HXhOFJpBDf>t ze@5`eGOOGlkY0ej=4x5-1Y<+0`dnuVvuMtOp7~DHVN3T1jlu~*Ik`J6=riT9AxnI+ zeKHR65({S*zjl5yJ!v4!zd)7)b5*a&Klj>h)xH5XO~QGMfYwUsI1F4HIvX_wS;|jyFi_*ov_{7fu}?*lh0N z{{}!t3Y643vlwQn{BB&_Ogwx3v3r=x@A5BGlL+^7)^;NrJ!!l?DaO(8mZDp zSclNkZa0_di)hx3J~r02)*Y|GnA1T*{Mq!VSN_CX_TxAMDdhVG3-G!{EayFGe)4k- z#lkj+U%t`ZAziwJH%i9OSNsssyMrDFCmZTA{Pd$j;>4NrEZa@8^rh6Am|IWO2PX@nhqn&W#^+Av9A77%T1i-tDkeNG- zKS5HNHA<8Zrk$EsVd)CB7PsBqs!;R1)<(O>7q3o|@)~hAm%HEwB`{ofwN(84v4+5l z;IZL;XsUVExpyOsMje{|I@=MBgmN*27He znnYK89QjtH7P9=yn6LEYd-AR5aAd`8D-5rE})fA?XBHj@U4PZ?LuA> zE@5N=eDx_Mfg-`QmpGS3$f$JYn*<%oB()YjYut8LCuB6iDcDZh#R1>;6L%mAwb&pI z$2aQG7l`21jAu4THxK(onI{+&Pgo6G^M8)}I?-kVbL;OVPBvsFW|zhgVsF|F-=Gi2 zlJ)E}TECPuZ#Zjk{hJ25wD%d!%q{()k%aD9SZ%y5Oz-;B@j7~4)Uf^Oee5eW`M%vs z-YKcA&ZcXE-rT{Z_mbX5CQVs#KNge{Z-m^vAez|*StR}~$&UBCO2U(JY+M_hEu@AM zhRG&n|3R@3HnPLwpdRfK`dSQzyC&us>XHgoiI~!l1e!~F-jw!0S!ef1N@jWcbMq*5 z1NO`qHkQyBq#Tdxbo=C~qVqnMcn$OIb0byI_5j8ylURwM;(~;bw`xRexhT=lW5kz6re<@*(Qps-j?j+8xv#kpUFsRz(jh zwvC08Z2k}aF?em#5$*a3vv@qP)*NZ|6Jz5sgsG_6N@r*2^;Mw>HtfOp8FKk72JUXs z^>=B~ckn)LBAV11PEln=^bX{-eZgV$F(WP7e6EOsZv4}rQIT|H@79ilLGXU%FvGv{ z92QIIv;FMa955mhBo%ydA1Vm~(#{OOiU{j#$(l`CBt}R2V)83p&FAQ?4guNp4cIAO zRK;M;%`Px>s|8o^syEvA=rC&Yqag*eJ9OqGWg7Lsv!`e|^mIYynA*{tn0W=TBs1XX#tx7s~-XVp%(U}~*$3iZQZUXinT zl`7!9X;ZAp6N>Xb(rf$0eqZc6$NBVITROYP#5(B1qb9Hk3ZZaNk3~NFfSka%rcLrL`yaOJWkkx6!H@>Bk`UhrMF17m!!#>O+QTsHU z=4QnoGe@?nq8G@~eKJl*>rx)J#G_`Op^Kk0<}v6ISlvs;JiXG7eW7F5r1)_Dcz)H~ zQSOEaNuyYmxs2jFP~3Zf-I05 zqA6x=jRIDXXTb`RGxaKj96&f0@^66nzv?p2P~{gii}VCyq-L!BXYch)*u40*YK0)} z8xQhEf8C5~SRt5}AHsB(IXsU@ZRPd;E%)#~IAEz@`)$BoFeAQO8mYMQ?#1%Muj-D| zc}dy-4GoLhkMZCaFU&7}7r?L6g99VSsQz)-Fpi6@aS9F^Ya=ZO4GtRr{d`&DKAg^L z4TggT2MrJbP83~h1{^dvXmA4KS{vY?!9fE=fD;(kngIt54jP=mxYhW@Zq=|^2l+Y7e0;7P6z&I9^s-mER z1Vy9;fVFCfGSX}ay(9Y(;lU>LGZ{o5^}zNXt8txl?&!-Kul z^|beE>uiIHtE;P<1fRcPbn5WY<>|mT(``PXp@Bva2m*o7M(Aq8gS{bp4;(lE(b)&t zw@(YWLn|aAAk+h?6%eBFeUTsQ9QF$F489l`dJ!I=E?C#&96T)4blWz8pnre9?=$q` zh5sZ4ge)HmI3Psu8^~U59mv0H15-@|qeeEt7rg+9!6qfsP+OXonP<7!WPr*tujdoqP5Aqi$n|{kz^As~=3p zC^_G_rZs;!St{zSTW4157wPMR!ZMWN|GxuqG zC$jGkWRYkFe)s6NvqlroiSG~=5tCIniv>hmlm8)2}$}4~7u&ccCUu~>X z^dI4Jl~=CJ-d1_#Dz5}!;(rEK-ctWj_W1If&e_`7z#gx(JKJBBv>16*Q13Q9Gb#O6 zM~DkjkOke)%~D*h#K=FYO@h>HPB}1gFSs)=2Nfw)d!_rg@xub*h`Snl0)&za2mi5> zMxxw=glY-nlkH-bvceb~FS0wl&%f?ml9qxIVoF7S5J?IA`Yb;I+3^WjChn+$;o_HD zb@7UBj*#Jz47fCBqQqwNexH*upVoXso|E=**HFLbJy*a42-|NQCE?JHs6}es-&@U( zYNB6#+#!J&2s`Zj!ab^#?-RTOXPy2J#kZY4(M$KmYO2 zB2`Vw<);14Z#(wisSffnK0!bHV5TkhcIr{?=BU*P#DQU z^QNckWDLUNy0ZVB3W|Fi3sV1h?r*aiVa7T%1jh}Iu$p*YP^3hz23>0R#Z)VFxuch+ z+bqDH3EboV8GzXipzda=^h{FNQ)BwQGW@myuZrBoP<6&&NOuF z^ONYLx0gi|YhpMl&3#S@KM7yAAy6iE-KD$s+ox^U*09J)#gr7|yYLd=Dp6DosUG^Z zL$k1YzSra9_B2Thmifm4c;5XB>}O!=9Pq^Msf(PO-U2~fA{GsZyo5UZ^=)PDKf>}$ zCoOSb#94mAA}DAa62aH&a@6QYkYcEb)(5D!J+dQ4*3+vO+*mA1YSYW@cOAFQjuSCl zz1SRhbMyuP?Ru@%-9}#53QKg50{=O_W4#S@@%(6SsqeqGdPFDl=!Gg$ymCPK+ZMD}nQXlL2Xpno)=##|{$*0>zGKHqwx!82GI&n#*{l8;@WvFfZE#8@XKN-`p4?}1zQqHNH08^>` zAZ(r}t%YXp&tBRO_)sEpN9?t42e)$qYNphwf~@}(3>;;b#?Xe?#@A;a{>%LZ7bm@d zI^&%xO!$hR|MUh>`P}!y;rA7Q|9@H+C}~<2Y+4Bl_5iHdRd6nSx9Me(agBnqZ~LYA z6=h$uJ8MNi_ER;zE2^gt_6gsz%R3csEh}7?ASDsQna@)1!v7(A@8nve&{5e+1ReUY zD0}YG!LuL!-uloThtm3_XBlg@akTrT=V;dpWl6z-Sn&dNO1$Mbt~j3hbcnd&nju=p zyi-_Q_~cIMlJH48^pud0@K#AM+jaB)p^%MI+N}Yfr9CROPv{{xIpmo2i37j#+RW{h z_vhpdCJR)1Mhk!zhg%WAyq|UIV*;p@GWSVI{s#hoM_LC7G0+*Sk+nbHt8Vsy0o0C@ ziMqa)|M}GS#Q_))xbn(H-pk8^_PY@Z_x>WY6nJ2r=iOEk77-;IGf*qiwB-FSGCKjE zUETQqMdM}x0_T#lF`ct>>S}`ERG>i2>i@$|W?Aj(Kv7KuQ0fP}A?!?x#LjFaS;TeV zkc8I$gN3B6i7FcaWVs*gN^ar1^V!7p=c)@umS;ZD0<7Kt*;ut8;dn4Ck_|+xz;-Kx z_-6gSLYIXnX>LZ*Knxq6?}Nk3&lKt`Pdgh5JhHa2dlmvK>5<@4z-d|9c2Ei#3x5?pLXNx(;xB$n??zDY3jAz%62Hb<{#&t&r#y!>-V;g3#)9 z$Nn8T38Hq_IiRAo3{GNC3|2=aMom@hoG7QHP^0R-QVhvgfp`JyNLyq!LF(T01vuBB zu5G_7wlg#EP|P5(HrN`rG#fbmRy2)1@Z?0AEaae%(cCx@QdmyD?(q_@AmLk7ap9`P zV1_kghUg5hmL)~qveMt;g=+W8fbzIAgt{(gxT=C{XdAQi??D0AWXMcNek__-ssuMU=D0o z@9&vx%e7$DfqAlk;EL%F8mYIN=Dv0o1gIbyDJel%r$NtDQwMFtaJ=H(8BrNE6>`|@ zUdPe)e4BFM1cxLysiC=yU=){YWiaqsa(y<6Jr>BoD<%#gNT!lo_C3x7UrrG4jLfhu zCoid>K^}+h&Yx&A?e@t{Zp`AaYHUmxhwmZD*P1lg3_T$BhE=Gk8ik6XE14;`_bKqujF0{xKLDAr1HdRZyd@Fs0%Pf<15<%Cu9?INt|NK9JpO5d1FCjpyqSg32)jZt2v%(r_LHUtOb3{np zRYTd!i2~YX!UpFMjK&M8J;IDBPSEv(hfS1VeSRf(2$V`LBNZ(Pfc);9vRPPAA}o8X zcp$tl{@RZ8r~QjC<_C=?Up5YigG{fPW&`(bk3`kR2Hj6O<`LlsC6ALg5)S!%wUg%2 z(F#uIZ?cnxGE$;`c(#vS+;OEa{0jqt7u}FWj=v~-XEO6iJ})!4@xpIGiCBHSD)r;e z9Bhv?rvOu<1m&?Pz1yf`i*q~im&;335VMp&*(^4zd`uecGM|MQe1`jS8ppO@3X0Yt zx`cEhhA3@|OY?o09xn2L*<>Z>{)Kc+?uY?yebb&2%L-jEWNr6_4F7bkoX8C%MEoWP zAN}1$-rTS_NB7KArr1c zv*+EaWNH=NqvZhJldjWgXWOZmMC>K(ZBUt`?N#3#aXr7iL06JI?pfP9I~UYLNrlz$ zwv=Wq=F$JGN;JEqN=b=qC)PJ@P>URW7)Z-PBmfk$0Gl~MwO&>pPE?C+^ICgd1=f7PJx$9Sio>CPNvjbRg03Yzt z8>i6i?R}JHI$3j)p~UN*WshSLSs4qZnrf3k7Le>~V{Q@;ohk=lwS(F`v8_8(so@ zv8=`t3$6udtSg#FK=VqlnrBt6A|`PF1e;4rU9!bu%qA~HmcG2pyI}yj0V^ZU=AjuA zalNJz_sClLAAv?u$Q$(K^YJi%s$GoJC79@P$86BzjQwn6J(RcK4SNgaf8BtVgqTXT zXr~aUc;mSSDy4M3RC5py<9%|+%DH3Tz_O>%0CB!uLZOQ6RqqB;nkFI+P&{>vL3E4q zpQPq4>GHTkmE%Rqch6jUCX1+~j8ml6QX=a$)n1T72&PR7DFz@xCeu_RP3{5ZW{@?x z48^r%D-b0{mi$C~ujSJvqqn1O$~3L2iC12uvM~6_eeWWO8c8GjNb!8HFR{Vo2anlb zS$i1&=L4`(wb(exP?+K4#+_s=ldulM-2qnXxst+li73T&QY@4DtjQo#rV9L$qD@~; zl_3?Thga=GZiG#JK}>YNtoh&;HEFL#O(hOCWbu*JXsox{rR&D@pa$c?kGN7>E*Ci7 z3+|@B9BChHu~SetaakfHTc|Z%vqrnd8yn%_n+IUHel&Y5pSop3DG_k+h~eQxu(YCU z=%4;RH_-1(6v-Hg*BP(P?-ezj@^l##7J2@gPV{J+$F#4_ZIbaoWmv(fu}j+lTN2q0 zkiqrORUHIr4a`d2tsPpanH`J+a)%oMH+2Alo*ItAR`JV$7-&(m8{NKe{o!HkWpw++ z%ToGJNj}A6$mnPQ-_l@wMQ}sZ)*DS53AeAtH9H}+L}2#}e$nA6<|pg-{+{~C?%*;l-F&3#1nt}fIJl*-NK=2>SJh}m>D*QjZ8FqXqHx()lWC2UZ>BIOR-n`z*45afg zte!21`n3>k*)_3VX|DQ$O!2Q+cdW9*@-BUL15wcVcw|n3d^s(JQ#Z^}QLbT4Brz(d z%G?kZhD%pMnh52*>lt+w`7Rw3GrSC?MWPMmjr5v1w+cT}BG34}SBAFEm+`H`q_Tc_Z5WO^ z=n^_Y9Zo+{1aS|yrc2Z%pCrx_fx&H3I5T^CxIWifijG@27&9CrLzi=l8k=L!1jS63Rr5AT z!ybAgy=r|wmN{Ok*xt0LOVGEyYFw6VWToHpcz6g$SDZIN5~n*Tgp`*zdW;B4Zx;I$ zJZeiI$w586!a30yD7DS(?R+1r`;o=iwXmGonB_b+V_+wlHl&!(=J|l zdwgfc9+kk^;~r&eQp5=+ZSWffCVeU)30jEJKcj6-_1=5=N<`05(}wAyb7IEy5TDVA z3C&^7IT}A9vxo3FxUSm61mRb>?%kuPQI{<2Mly`F%V#G45G9SB_a9&aV~{vq$9 z9v4yhN%c+7B4N0)DL^tx^_6q?`El44<)X^%6R&CWQg%m59l|Qd@}&}0%Fs@;<_ojO z70~USW$`3CH-iF&z#$Y>$MbFts?%cOd zh=Bt*!fq85YuE#b#{u}Nk+RYL875~Rcu^#DTd63;_C$c{*j4Og{ zSIHJ&>-SJ~Sz{9cv+x?`dO-jh>{$a1w73V5rPNZ$GyV z9GV{p)CZHBAKhKNFLF*ymfzwM+IGaF%gNh+#%8HfOx=2SO6B`p4C84a7u82AcJp|D zAI&W^aF42Bxpf;@4qQyvnJEm}NP^EKz;R1&#I_&`F$4{nepU7Y$7T_`^!Qdnj124{ z3zdO|@{2MsTH2lms#qLD3l@-*zANH@ovxf27@uhvd21b|x*^i9>dy)vli--z77bnz z(M_4CnZ13Qkw{M@tV3^baE`iMJf8sLW!>^45m5#=H_!fPsSI)&X_0{ed&jYrZCKBGTa{?Fw77&w=w(VeH>jikm!!3wK8* zhEMixBF3TYxw*{OfuVHsI!WDR!_SObzNrcsZIxd(7`;;^gy}(THD8!t55zNXR-J_YQBpn<~V7~BmN*% zvpX&{;if;uZoA!WxCh()7nOxF25&%BwLJHdmE$v_hl&yv`5@yR^&0I#k@FvEb$d;1 ze!pb=IC@Zl9}-TNc;^@LNlCjryne0{EXmfXKVwr-Omti;?cQJ z_+rj}xZ_*8>fPS|?d_MBpM%sliKc4|f00OmMCr3`+0DvznpN@M6{0jadd^w3+%_+&Nw(AlBdI1eBlwEnc-qgju*CzaQbA(Da_8(kBw zX*22O)8?KAW_+{&CtQy5Q?#!B@}9|=EP&vCw=RQ~mlx6|0D~I|zMnW%p;6CIx28)* zo2mqn4^UgtQyj!q;shU6;@?P<83^%4PCaXQ=X83|jpure9?|V)eA}25(xU6$UbbbD zqM4VuFEeB=i@Gl>@4kL!(4toJUGVh8tzjGINk- zfm3$#T5OD$B&l3lcBILyQ79&`d%d?_R89`_3)VITqP>`|n;FwNN$smI*63mrLNICt z4?Noa`3i$E;M zq!p9P#q8MU)u4G@V;AN&v3DV2sI8a@KmC->ZSW$`96?nF`ZJgWt2b&vUv@#Au!dP2 ztSzL7fMYy&ZWQj+M+hJlT&jKdhIH8nvFF_Cjmh>ay%cHDpL&r@p1oh!}av$irUuixo% zgqkqV;>h4}jTB@j%!Hv{)G$r9OfYl}?V=7R($Qjx^G50bcQ5n!Ocq0BlC5NApaD20 zxQG!kL41R!${B2I0bGx}u|qY|&p8uiL_XHpt+tp-kKjo0wTl+{AxpaC9!X{H#Q}XB z{b0Kquw~-I7x{QaK11@_8x)+Lrmz_(Z~d-|N>Syr9-7zijIeX>3-E}(Br3Xe(>05$ z?x^7&@|g*|IZ2ehes@t#cnl4uEuVvMjjr!8C?(e0GJV@0;cqVG8H`m?-O~~-UmySK zOM)|&_*~*ZlI$##$)4^Ve?LOVLyM+EQ!vT-H;Lgq%8WMd>zT~S{Z9?6e5R)RZ>4VX z{%UzTn$WN6-I)&%p@&vD61k1t+k)o)jog73FL<#@ErvB%+?YPz1uJ_ZN?quimb3Us zX=VTIs+Ll>8y}u0X(%QR5@NAy?UUC6My9H=C*>D zrcAIqnr213O?Mi1GRg3$hVddYr=oQHMq8O~AVy&VQq!XS*+rQQXh7wS20>2HdQaHD zgw}em)OmcJ!d1Td{YQpNG4lt!;STsN{1%}SMT3|mTkPSpz2pzf5ACzMmIlm|NdyYb za?pc$VTRK7^`Uu!p0aX|&5ms3Bc09;+vHQ@D2oZFY=wS@98>)5TsLc-0m=4+t>y_S za~7eQm7YQA^agxnJoEJuG8Zdt#k~~c%T8$L(t|JOc`_LL+ryP2dW6<8=<){2y8RfKbW*Vf~9s^;B$#nR5G(zoMnLA3bh7dB*YI#cc$xcXFXL19In zk0HMXEgn!!&$$35g><)WeTHvn>IT@Fijnoe#oK2VptCG}9x5aF_FSrid1AC_Bl5%= z)%>q^(YTnCMW;>tVS~S8#=OK!YxH84_P1NRyIAXeMfj+I(=^vf1Kgpq=R|&1K!SrP zh>^B|4B5f;r<)&|oVaoU{&l?|*E_#^=rrh+zyDx7K8txVZ`{s}vNZlgL86qeFkyuD zS-@$K2hzqB$a35~Tkf-5wz@719!IVs_juU%)?k&(zk<9Df0fP`V?zn ztbJ#H7}1q#Bb3tY+k5W*N8`}ng^S1PI`Ct&N&E}<*=R<}#AI*shsi3fs#6i?E5>Me zdKqg5?P-i(D4(m`ovwvijL5%}Pcu+fNwrbfqaB2#bfz^{H5`NC? zO^kWfynLclRZ5@SU~f8#cQHsLx7?arHB5>p?ct2;6qiqNlL$3?0u;(ZmR|FbbrljV z<@*cSe1}~&O)4%bH{HCNH<_1tpe#l_0t+^$-U3ZT|A~DXnRu3V&r*F+`@q}d>aGFk6lFR!GRJqK#X9z){IE5`Mk;V z=mG2uL6y8DJ8$AcWJ@hI#=oD2G1evcLQ4Kbj#gAPqOllkcki$}zvglIGot0(>GwGY zql?6p&K1kj8;S?tk8dQtO(~busNogL6J!`hydGnMgIlm1{6jfOF6s*Mv5s|_OOyrj z(dd8y{$?>P2eo$(BjZR?YrgA6xfkTnsgH{Byi`-wyT1o?vCjV4a5Uq^>Hg6PHra>h z{;tut@T9hGCI6bYc4pUjCV5{1-Ml&EmGrInrKA$=N8_yY1eqmmNu}wCwhZZqQ44Z? zBo=!Qy>DLFJVA2(5_?W!a>t}>-uS-xw@#O=p!|t;rV#^~Jpa!LjO-w4rgWN0%ZYcL zH-3cbMPr9uu-Va660gXRa-m*s3n$dnvjf0zLxbZCQ#p(=n$57N z8^kw@q`7(2r@-l+kqWS_3!yJHuzxTX7;lPsIpl@bH_z;q@?Ro99_-f~jLffomyi*l zn$4waP$Qe$z1{{6?cE6^y|xKR-8#3#_sS_P@*`R3x6FLr4oHjTRRiH0BzWpVOu6CQ z(pTmTIO@w=G6(g9NgoEXk?QWa$y5EUaL21*(VR&Gys4BLWYF^=^FuUNWgF^3CfSCh zS^*F0abPSUvbAyI43tEHunS%Tc{rTe#8HQhC??RtH z(cNkDoTl&f1#8E`zRzBx@>-AVGqnvT!FZbo^IF=f&XrFoMa6G-I4Al-j3E=n7KCk? zjs-3kLir)}P~&(^QqCVY3{nfN+O)d%LGy>ragvwaqqc=kbnguv@A8jE-Cp=c^4oy6 z_tDFq4t)}@y+9|OtB|+KWCl6KP!2gZ_|x20tCw0 zhULsZP=I+_V&DafjD-=(6Dh_h+S!!@V^(&euBk_+UfQW z1J|=?$dTG|OZSa$-$lV?{203LsiqB7J%*wTSs<6sVozPOz`;1Pw_bHI=C&|=*j4aq zr*C3!xnoiM0WytCF6B^kZ>zLNzP?ZcdZ2gauAnl?0$SELV9xYR+09@fGqR_ooX;_! z9B2OKMa^f52IwiSLR3V_1<3!0q+YXuc%w0-2I_fzYU_X1#BZ%q!HWt_`XfL3dx9|_ ze*(I&t2YZm#i}+|ZR5KgJox{`Q{s-F$@Y?tW9*zBC?h47<*1nOptZ(=MzrDv)>OvU z;W`?a8xI|5`0+Z>8HK=g|63p5!G3t})w83&Ktin9soE`|;|iBoB855xKnk}dJE@hO zieutppnvwPTk(=nteG3o0>QGMDgAJv?{og%C=2?{N%F_Omc0uLU`@oFt==!{-y6zf9%BU6woa>zxS6vS!4puk}sV$oxeca@(X!_^5}o7eAx~( zhYNC^|L6fN&p8ep`5(Ocid$troB*VqhS{4hcR`jraQ^^D{w?X?&#TI+|D6WB{)a(5 z|LLVX!D7DwFRQF77npds)r=SAB z|7yVd9wb)--YTP5!NXPq-v6Y4w;JSDlMdkJ_tmWKdnUM=)d4j^fhtzh=H;NaT9;U6 z{;LqN3K8EmunG~&8d!yhmCSz?B32<{6(Ux<8{ceTwSBbOK3dkmayxLfKli((sBRDf{%^JHI4NB?CpH7WSLNfjUz)h?!qTxA)6!pht(rriV+vbW1O{2whCopC zWzLJ0Rcb9k60frI&}80{{=V-3VZ+n literal 0 HcmV?d00001 diff --git a/hail/python/hailtop/pipeline/docs/_static/images/dags/dags.004.png b/hail/python/hailtop/pipeline/docs/_static/images/dags/dags.004.png new file mode 100644 index 0000000000000000000000000000000000000000..0faa49f8d9c6780df9000badf3220af6a0d3bc48 GIT binary patch literal 24655 zcmeHvXH-*J^lvD35wIbh(GgIDfCxwnI2Kf@N^hY^krqOcPGGEnfDQ^uS4NReq}PCu zptMk=mtZ0worDrX+PlHA;9Bp)TkF05mxq4f=G>E0_SyTl_x|mB6JemI#kE^tHwXmc zx^nr#br6UXxa0us+z$MytUSa8{NZ#ur+W?rLdNc)*=_?qi`iekt_uPMii1F5cR`?a zpegJt2;?gZ0!`b1Kq}8cAU@CJ%4=#M5S!U8LsK79T^$vBn7fSaO_-g7jK8}l@C^`1 z&0ht$ba(Kv74vs@^YB*jS3kPZLj|~IHG_|eZFKQ*RX=K~Yan(W=H(!!Aah1W_9$ex zn3$NF*G)&2>lZHl_jTZt`cY>eA5Rr9*w4>T#!p@b=H&#IQ&v_6%bo_GJ}nLOkoFGn z@Uium_V7OTZIJKdTyXHV_qyfja|`An#v0ew4(98le)K47q96aht<&e0e`7W`vu;Hzq^Ruz4(TMod?tnndoY8yQ_wf#4b8kjZtFN^uM(~VYOtB~Dl z;2&ax?ACcNVFm(eg05URXXwv1JF=_Xicqm*(d6jmyFB)nThB)*e=*kOtC3bHi|opN z`C0*M-}d(_7q=(@3zxUX(PeL6T@_f;;XTLmx7Z(d?pF1BVQ=Zx{@H#7_)qDDsq4=V zP={O#YDRHYsn%3f-+J$DzUUkN%iu zb0^591G-$xw3h) zvz*iBO(E6bd)V~NNy^%9;{?g~4rTH>^RX9HldjHI&g3JTPv%@mgM`=(xtB#RwX1@g zp>3ytK>X_m_Q_;#JBQ&c|ga1DAanXI|vaKG&X;a9GG`S6d& z6AX=VQH*sYlR@~|AG&etD#Iu1bBY97{Ex($- zr(4B%JaN8iL2)ld`#a2OhG~ieER~Kx)?W5D-Xo`PUREIDqLoIE094`a;Sll&v3`eW zqkGxN;g(ZHIwcXj$1XM2g>xhoDbM_Z4;|s0t!y3Pi4G>abPDP|JUM2MRVU{v7ENd8 zU~0&tg6e_G4|SDB&h(nvNBd8`PRQ;%Z0T7)MVDNV02WZiBY6i)^2%&aQ7wz2BM)q( zdNz)`TS)=P)>h%g0nVMRXT!a2GYbHORtaqGkIlrV2T`!v4wld zku%N)zDi;IWloR?z%cS846G-LhrL=9JUjvYi*6D{sI9Ng4V|4}+c>GW8DZL>>3&~@ zKI8P{&&)*E{0L#cVPs_+4(KVQuR-@ABrw|)hfXnQlNwYKbm-WH&zvM4Nr^KffMf(= zPAZ|awZ$MUks70ItO0e>!sJ13u-K|yfCkz&+();7QI_;|R2}=C#_QVBheFnurwUj4 z;jFGFWgT0ePseAD59mk9R60L(V*kd)YNh~&=sRCK_){wYL8mLS%doxG%{g@S=fMFR z^gRqvuoYAnK#JcZ8o-(cSxjcP%e_rZ=KpqF_OQWX^Orx;Yu1H<6QIE*zK&k@zzZhM8!7Fx|lrHFM{ zZ>zjLCqSo?VFguAs5fG=fy-ZkL+dzbt)LU8d7`5xi2Sv$Ky)u_;^$Y`+dHJ&+8e|;c2L+deaX!#WAe1C@8Zc> zNO`>eQ2IrQH-Jb&4D{7sy{KuEJXqb?!w$yFE0*Z(v~ovD<%@9 z6o)+HmfsR529^BphG69jQ--tV0mo7BOZy>VEE5gE!4srd9`$U zo!;Nw#d1!Kv7yTu{?l2Rc4KwnJ-8w)L^Y5+4}c?F>lBJcf&-ov{g1YcnJ%a4cJ?@# zaWi&c(Pp%@CXE=0RR5#%C?=zPzH$cNuDX!yV}$*IWwjSHL5s&Vt;Jaq26K)EOs1@T ziPL{&QefRv2ON|jj?(UDH`|Zg)!C+m6Ur&54O|(o2}Wus$XpAhpvL~*^N(+)q9*|Q z#zT||C6rlSZ=OZj*_qrTJZ%Y^HaJv$>ANY)@&hhjZ#$ZYWwZaX?*UhP`c5+kT?>Q( zNm;nnQYMfsGFAUd9X_THq1=4RR^QlCz0@vfRdqWWaJOQPtW$s{;t5)!A;r6oG#Bj? zh$rdX<%0$zKWCwSz(dFLO7hmTCQ1@?aZmk6hCVPC(VYX37WJM6Gdv4%BVh| z+2Hklmqa5FPjxSTf_`@(;f}k3Ant{M@iD;Pk8Ep(u8;wpH@)@xS@sOI#e)1yeoe`8 z2CBDMwuX;t0zwX<@fF5oYLEvjW#6iHPx+yVtP2BIz;WjdK^O4M+f;Qp?T^gH58PU% zCmN-dhDG?2DdaG&*rsMcCGJwjT~BJ;^jFq0tM-AyYCa%DP2QB?pse5a-Jb?6!ipXQA>j9t^_V zwRJPr>b|Y~3*ZJnR8+YnW!GVn7=FvsHJ-sVqQng=SRiz^YI--keu@v1Q&24;m0KO0 z?AsMUT(wv@6-nZ6J=O^3jmnAYJ zs`RvuB!{dH;GHr>4)0*NrTOEZWTJc$_@=t0Zd0c8h_r5|$GiEdrWD~lE9SDSP&!lC z+j!a9ty8=}f9~Vs`T{g+9^vnjkRwFf;E*CN`z08xtNgD3jAlTVrt*RHemVOL+977TovKxpue+sF zpy-B!9P%!B7*5tWxU`xq(QKh863|tTbS65KDYqQrvlbrbn@J(BS=0I|U5#NUz^di= z_c?aaj2yH_5Hnq%iZFpfFMumbJH2toLa$fXx)8K1y%^7k{pa-!QKW>-wFK0#--|qL zAq4DXL8(J7y1iw~f$SMyGwh!xS64ysjol zw)<p!{^Gazvo`VQNe^kSOe;e9)*g|UoYnHW##iYibq__&T$UkT>D>Lw_aPkQD;E3Ao$ zU{ZST8}SgQGSe_ms6wJn)@Q^5+m&D|>)o*$f*dg_MQ`Vxix0g>mEcPaxpk_m%EWTL z32KxS;MBjE-RlijtuZ^-pjQRsgH^9Ou8cER{y0x4I{Fwm8_7xYHAbx8+foWPXgen3 zPO9@Vt$fXkQN*ElYC2IO-ZYTtk}^UbP?o6~^Sc=0)Ci0My{GUxh@W)EDS*_OnFC$H zRxD2%VJ}rP&&|^B+`$Vb!G_mXGqwv|zjtE1F2YH8WUU;2&k-US6tK2y2Sy6a6>%d5 zQnd5FapVHSCTV)jbwd1`FWs<3oBO)}w&MOZYI^TM$pTx%+_RTGd<9c*eO9|iOWi;XH-_kSr;(uh;<03iTda81J32RNWG%Qy!8p<+YNkpsUa+9z`0A<9 z0cjL1L7TE?nH%xWe%NOuk1OX46-Jo}X*f+Jc)2~pdkX>-12%!)+$Bew^J{rP{A7>4 z6dN!Zvm+9W(`-1!Q@FMSn0WcZm=4*T3uY&rFZGH z=X2M8w6`@A(xt^Ar@&2XBj`RuC&*JLnPuy0%7I;qIY*@0&V-X8>wj7YxL(Mus7`G! z7GJaZW|-+ctQ}4Nt9bSpXES(wX?gaV6`^L`Jm*3Ks$khlQH2k6Awip2uo6Y$qeUcm zVaTMX%M&v6z9~K>D8eGHr=a?4;PjWZYRj5y)d`{-M2ga=qhX#JhwWQDecXk|v&6+y0d+?eg3=Se4}dBYn3z^3dV2KA6k)=SVIMY$hGr+~c+Ra* zO+H>PwT13e@NPD)H+)kVmF3Zh-0)}aq zEqiHWR~i!v8`|95mOnDW5Vy2{ZjC3XIMo>rWaa~?uqKOH)H*N0ktUkM3W9pX{C7|Kj z(322(9#!VCY~j)D#QA7bI7z{io?*EBJM6-}-uG%)h1_y0{%DM5z5##2lgoMqQ$uN= z%c-jigb>L@`1M}VI+)-RVs5Fok~a;F zJCsQIQjsYFmI(+tQCl%y+7_%uwRERFzb0q5n9=!-gMm_quLNBLvb1l{#eG>dv~%4K z;_`hKS6#EzyJ)w+0-e}8idzf}xauEZg-F9M&tB1=gqcpZFk)z)H>;I>`VVRlT1Ii^ z1t_CoVtRG)OMuO+o~V_Mzpu*sG}V2D|ColrAy_TL*r=&^^-GHjSRaeXSbKCl!7MmY zf-b(y1e^Nc5 z%SH*wqCuj8mJJ3`OlzOjEaX)05TD;WvuyN&I~WGUfeCv3-0YmwG98LIvFZG52LDL| zJ(4jasq4C)h>2ddlDc+)VEuR$LOy7!#CeFh+jvy&<>&le?nbbHs_OA5ZGl0r6IdZh z?9C6h%8Yj?bR3ww<>k|THyR}DHBx448FXa*HAa#-kL-X_ue0Z;58M{72kXHQlVtZ5 zg7H9avB#Ho3y+6sLOwch@!qGfbCuNE2f8y8N%Y!h#uoC{fuCRO69%2BGTn$THIsJl z1PwmzV`Y-Ebas8NP*nzH88NRXN4F@xH+4wzd>F|%VREi4;MAug>as`vNHr6SD6cMW zoggDyNg)}Nv^^J;acD){kV2sJI@(Lk8QiiC{ov+Yt~R^QFnWR zwE(5_0lHc=Q?GK+FijM(D1rpew#mfUyu^Iro>S^0du|$>_NqKbtKwtwhUzO1MwiC`Q>40hp@#ub-%`f{K|?-ovh@4rY4(& z40SwJ5xGRD?j+v{4Q9KmA6yk7P?4>e%AITQB&)q*Lhf%Yl zk9aFyn^dJ07|{1BO4=;>-lBOjoc6Nk#qJQ3Ildjj-sVNX8a^)=q9&ArDPUZAFwQ7a zepe-CvFyPi6b3vz>#mE9RI(yhnIiiEmY}*6F0Tzn*~z_}>A3V?bqSfH||Vr4d>O6+L~nSsMg|;`2Rs zee5qGt|w2UaE&=-g9NYlRn!Fr?ue?R4rwMRV0S2G1`%M<$CnnQ!Z4#n0$~`_Ku690 zY7qo6`@9qVh`#5~nRt5#U3r%iFtaLyf{RhgZi%DnHf<$+`nPQtqp*B_g#*J0%nO)} zq_WsB{f0KqCS*hC20Ow@QB3Jfx>&wy=!023a+)Qx*_E;&?O2`WAK?Af6+NdYns?u$ z;&b-2z*M~wx{IW zSo0g18Iwfxr1_jcw1Fp&`cFh?B~}+-PD2v_eUkJDQF#ZiPCk&H>wT&H9kWduWzi;A zIGqTMj$pQp*pFi9F8&7AiOAu|oHJ@G2^(t%DvLl#<3+pRE*+F+0E$W~4H9&JSHBE% z6DV68ty*YeGRE7FtPdt3>;7J9@!QK4#mxO{LreLRj?__=i5CEWU#c!9E59v%P93^}MmWwlJYvjRks)XFAi)de z#Kac+xfPU1MijI`t6Hf#aT{tlp{=&YKG|R`=!FSW&*B8qJi^>PvN!|-;#F7I1u zHIEr4udNY7s}7P&tz}1r8KI}Tf=^)>7w z3lIZ)Sbmu{#jS*fJeZ?6QQ?j5NYCGN|QPunW zebCwvOn1eEYO(`mSYw@fO}4_}Yfgb-!?@M%W_dnG%xRC|a{s*YjB`LLmV~EqASzay zB`0@C!glmuOCZ``LO-mJy+ocL$$+BXgzISKeGCPvqv^+X(zZ3xlXj#Ixi792M z+yQ#<4cZtGn}kLdC{!EVf7lZI8rIN#YrX2D@sN>|(5b?Xqj5E8Yvo(Ggp_xvwd*Z9 zD@8%z+AoICG+uDLw}4LDG!Sx0iCYA#mkx$b@*`CtcUTs7nvVx`bJS+sS0_AEBuG4; zITDT5=_$HY=3$isorqPQE5sk{Uf#3*^kg+1qe|dTfI{!?-Gg4V9Y0YXgwCcLdsxyd zrI-irFesPhl2!}--9L1=1TQVP?RQ;f;!92Kn;u0M=|NCuLww!fb5p001dkzw+Oi7n z(a@O`yEpUR+0b5qbTmwM@a({@M&w`AWjO~O-E=iwM&SX3T#1nj(L7*#ibo^btqQa8CK??e>rP zsulGcOo5tB8Kz=eGnug{p^DM%Q%Lz7f+OxBFISBPZOof3m_Zs>H zybFvt%%TqE`1>Bf*G8KzDqA4D6TD96C|9<3-dsstYP&H^2y}}>mIilab)Q0yuuVi> zCA44hy9_{VtjD?PboQ@Z#W~~*tVqz;NG2p#8K%8L z_1OD8jT5-hKNV>cfkeDFSijN*o|K(~Cqbdl9+9D$(BWXZyDo;6IOA8Z3o7!vwn#|j zzx%72M0zYaV;PcXB*2ueHB6h1H;)TlwebA|6Z z0EtrdsI6Kr3604wYMMDe`Vb3Yo@y>)7q@PPc7o3m@GfuFY7lq})49TyQaF%(Vq!~& zGAonEQdJqeL$ie!SB_WP3ZmB0Fv9CeXuddG@HqO99;)BK3>DeGzb411xqDTVf6X7B zHlt5?A(2`z!Twdvb@J@u*O17eKwqhO4#p+R!1{%3=+X}L#mmW_U$c6swMaAut=>rG zTN3n%h-+Urn?((M?hGPcw4WM}mxRDPTRYujT#nYA7>UoZWztw>!T)5N?z_$LMD2w$ z2uEZu-OVEBWnDkg-X5rxb^ce8=Gw-%;iW9a7|3Lt+y~Z+Nb9*t(cTI0vsVKf|)L#`sS&NfI(7`Zn#i`S`XUo?d;7#Gj z^i-U?5?oe-Y0r1b%~E1txx#~r04VOuUKNTIwi-?d?MOe@8Bm(v>HVU_OY#aAr@5}p zZ&_WAd*A(y`hf%c6e6N*mfkC_X(O<$8T_RsuTY2R%EY~*^#psh0Qe25_>4iG2qD5u zZSzIE(IhnLMa{Cg&&#a&_Z688mD~NkLs|KRiyKkRH_5!w2guWq&{18RO3vnPk~HrF z5tPREUpfq$*8LggzSnK(7t3IUYwExg3xntRW`C=%JKF>03_*?|wVxWz8qAp+h+9`X zwo!kc$Hu`7nrebNFm;OhKaS~J^ButS)P3q9-}#kq6^d|0;Ngj4xx%)eI?F0tgs%dx z;W%?!6>|T!#%KWawwAsn_5WuI6K*!-PgT^ndB=VLLO*pM0)1K4n=x)JYYxm^Zk+ZD07P3&a3@oH{~_nZ1}Y43)uY=OfDr2HR)gVID!1PCN@i}hOoTR6Oh!y67$bIZ+b z#3Nf##769~6{Ky1dt2bJ1rEQ&?OWim1r9$m(JgS;0*5Vd*rIo~=$#E3ZHqnIA}4?0 zNVmA(4d1W@4qM>xlg;0%Tzo6~Y=OfTIQ%UCZAA|M7b6E|UySg6HsH6Hd1i2lAY51y zT22U&2c0O?6RX474_6cx-bk58Dn+z%g#B)Q;)bu`QNmUE16##=b8MD79|L=ldG! z(!w~#U^k*Sw`y_z08pO%b5!Nlt1XoMZF98 Jf7$%~zW{irVqE|L literal 0 HcmV?d00001 diff --git a/hail/python/hailtop/pipeline/docs/_static/images/dags/dags.005.png b/hail/python/hailtop/pipeline/docs/_static/images/dags/dags.005.png new file mode 100644 index 0000000000000000000000000000000000000000..ba71d312345099a32ad7174bef0e2ed1991908cd GIT binary patch literal 38358 zcmeEuWmMGN7w*uAk}4=7pc2vwDAJ>dG>9M}Fr(6f(j5a9C@9^Hq;yF$hzio}NDkd4 zHNe2kJ%jQ3f4|?g?ppW#f(SG7J7=FA&-3hkcy(J{ftre$3Ic&pD=FT*1A&l&Z%H7O zfi>itf)j@p%J*h=MWe6kNa>(|f4KmK_?#j7 z_fLtdp|AG!O#`-@G1JAA}CL)F~iVEMo z`S&WoFMJ>n%->5s{4hpJI!Ix1NcQ=EZ-^99LGiyEA*Cw>pW#FHM)f59zaGjCsh9pA zbAZqGAtNS*@LQO?_y6}pi6ATg-Zd#d9XsUy^$S|^|FRx1lP&H4nwpaRzAuCba+EoS z_x!&e3QX=;l8J@-HCyAK3d}^!i@{`7eQh5c!Y7{BNH4k1hU-+y6(*|2I$kr-A%0(D=Wa zK!UrTO56cbjDJRb!;7QS!<7!997+t?cXM<^gUnq1rblZEzOLm7U)YY`ilKv!y{*qT zsy1l}=b2G?!+a&pu+kou$#W;o-+H9#uCQ6##ik%O%-&YT|LnvS?!yWh6`P@=lnSq3 z8L3W3;Bhl7mc1EoA|Um0C!?SJEjHB)k$oi^i)z1-@^5X*U!TYw@;R5=T#o$~nasha zmi`Cp7;L=5Y)1qVk28>PTHuHgHWeP4?MxEt%}_hO%ws#*NX?^_5g0CDpaXLqCsT?N zOu*_H>6e(jImRri=)5w_Be9qH@XPD<&0YB=Xw4a)QN@f)48={D%PV!O#(;$C~D;etl@c<;XF05*%@PINs#9JBNK?|s+Nu~Ya6 z^;L2aYo55MU!3NaMT5kCcs!JL|KrmcUqiN<3-{KinlJF_<~veQajC~&w5i)QyIHOL z!FoW|mfd4}X$IqO4=J@msmoxOL^i6)PcsU^_?tsHMJkCmvD2+ls}e4^ZwFiW+;~RD z|95T~Gaz~i+4+`~bHP@__w89p<&By45v;{69Q}n`(SCb5x&Ncd_pAVCm(>N{89V(;{a6+gz@@(^2goE-1W>-iQ^V-%m$5 zO7AmWyj?HVxA+Mu$b72H4}bF&Lv!?HV-v8vo;oiNrC@en!)j->DyK!pkEaKo|818d zfqFWh?~JqVlsDy)AAEYrz{0{0_ufqxSFY|~{8T0g))!ghwh_BUE3*S#Ko!`0D=}$4 z@3hc&d2vt+8YS(qU4iaeG{7;zJ-g#|tLKoIQIFIStdSg58xbqQ3x9w~^`$i0{4O8B z*nA;KYGQP<9PKT#2)r~n_KMYO7}9>aCH4A|v&_l9kh8JDm(JmbE4uZr|05aIQxNYn zRnm4HSns+V`hj{P* zXu4EF?h4C#JoF(|j@W&BiRViIQ`F86JO+VKW7I2vNq3HnEDkQipB<~3+nsHXo5iW= zmU1_Y)_RyueGOS3cIZ?Bsb(0-CgJ#X>h#WR5@LAhZ|g&d(3JbDN9x5;v+pU{*L;YX zA+`d?xm1ILWN~_l2%MX^$9Aq&3D)6YZ%f#DNl(A%;ftCdjc3f}>a(>nO?uLlxR`ua zDjs|y?|z&ZFvHemjVGa?akTjsPn439GE+^?wpmkR-Wi#h=-#sAu`(>KUtyav)tQ7| zau{zQ3oEGiBG&nzbKnLlw`W6#JiY5BBCmJ(BTG)=b`#IB9L-_ezya2lV^2fHhU>hB ztkK1$0i}=nTlVL>-}x!EL~*GxcXK?GAWIf}b)4JTWm@b^Jn~M4D$nlvv4s^LUxW3%^4<7%CY z&~;wbN)xk#`8K%L2tIS+)dKyJ#00`VCWSqof0pQthvr5bK!ceaf4oPCbjKM ztzG%j&jCytFR9PW;FLT_S!6uFlTtWx5gCaRrNG|VjyG^MC&Lt?fH%K5LLU8Q|Hc2r zuOJk{{;j!^ow)mWS2iwI%w~nPq(Ez7KF&+6E|Ye<%CN>&^IN`=6~o?q8$^OLNf7v9 z35%q&YEQZf_r`3e*ubt+H5b)zl~ay|aD8$zdzcD)Dk-9>z^c0l=d}3k3Vf>_34@c6 zXl1HLWT?f(uFpHSi;OWV->!9c>Q12=ThR36qw91FuIN?xUOqjxK*?^)M%4Zl zu|B0F!kJWOiE6`DEsfS1(6CBf{POywh|Tt@19P|FzF&L_X)GVRN+kb`fcIdj<*Lhf zF9hl)0=vEJK*3@^+p(<9>WY3g@IQ^68nE~J#m|QSB#U3)p^si7t;alu9{JmK2)|pa z0-cP4;|_l>`0zEJLg1tRKYd4V7e+pr^y-o4FJHER5@G#b*h!v!koe=rk9Xc{XIro} z5&c!qhE)K3!1RoZ{hg)=dgX*aL>e-oC`W$p%2S5Mu$~}tvU^25Ld39>6`soJ0D-ZfhE9G^(1#Jqa* zhD0{Wy5+LF{pV@Nh!mwbnEgcGX%y-dK7MObsK@-ftHUVX8y}X%cJ#00;>!;zXs|(0 zvB%xy83eEL2maI3XqBj1_k@WT8?6c#xW3}#6w>s3J$t;mPKmqkr(t7Bs>&iqusBBq zqATLbV|Y~Oo=a$g2aUYz>(}W4G3XK#WuYiR_$%8azmaQC7FaA%UePDqR_6 zpvu~bvo%?I0MLb}o zSIa6HiRme)S`*|(t-7IgdovW>euL%ANd7RS5#7f`U*vqAE<6%;)bs*Xh^5b#^S$!|JX#l7m)j0gYxfhzt5UWdH`1q>#{-hV9c^(x%K;tIsNR{=+LgGh>T?67;59r@n~yg3==Qjxn{Qa` zIDaQwJ3G{Kt6#S#`TDc(g5;=wri1)R0DPE$Ubyq3noJO0$&1BHov+5QT}2prq$wt9 ze^{2dUo|MCU#9t9S8&mL2j!LMQYU+N(+;&VQzAbZHH=m{$;bXlDs)qA_eqMIRxT~R z3cOEsP{O7(vjfHjm3gu+boJuxKi{Gn-;)d;q=c$t?d&DG&{1xJp zKTbrP+2qC)_nwwR06hdFSBUkaFw`%@xnMpa6K{Cj(f0Td$Zu4u3Cq26PT9O)*iz_q z`;c?oK^XVZNP!_t?6!Z;k<`N8HaC@cA2rmaEmkp(&u7!3C9F~^N5$0Ho64jTF`6+~ ztwN6Q&t26(bULAMMY}P8{OFfjgaAII%%vL5PJQNTxEFR?Nl{TT+P={&rh@SwKfF&x z2cc|pe%FVg?^3Yz%E4Jwtjs?~=L(6JmfqU8#j&tlI)hMQya`)2S*NhOWZ zSDFoD`*_k+U)@-dm8Y7r8$HH-$>T1smTHD)^`*cVGf&h!KDEkXX5X@dBzIPLAtz@2 zPQP+uTul|uB*nA1so6ByX(QJv(QqI>h2}B_VYjvKg=KXyDqY;~TV-r6nO3eSAF+~m zpp2LB7!52(=|R8}8!!qx%; zPl*{llF}WWk4?O-E=ijuaSV!qm;9ie?^$)=iI$D>?3wjanyPuxzj09IR;|RfW4$X$ zTgOJjB7j&L=GKT35R?_v7f@Rqbt`<6y?4y%C5!6r6W>1N;J zU$+{UNE6#u7$)*oTK8>RsBGuE4s8U5S|_O*Y1XIXj>FhMJbK#eOF>-voq29G%#RN? z-@>FSi&e71x-=H;+TG|-`$QVoj3&y=<&KzNdk)iLN}&0XkpC=U^5(j9q?Why`ZdZ^!>cqq{yg*sn>Oeg79jAI2yQ!&6Y|8?@3Ed zf~I-J>Rb2$7NhblS6|J(o`Ex!(f`$E+Wwz2Wv813Qo`q5uI&P!$|f$(z1F{^x1C0efgUKAF!j(5HX48>DxKzx-XB8GWJ7! zA}2TJd+%hbpVhDPtmIIO?K?)1Wq?t)m$!)u+qm#&nWPyIBAE90tJUbT| zgSZH9;2AM>a%yH1VhGT|!W!S4kyN_)E;0HtSL>(Bf0+Be0hoeBl+&gYviS;MSGo#Q zjF?SgnqtTe92Sjl>D7=G075r%)o$s}YT55c0p?RZ7&my{2g-6XOY_~!T%&3opfz$X zq6(rw^=(;2;#n1}RLcH&5bI4JNd2=cyJH|3j`KW4hCSz-@7AGdWcp< zIpm+$d!7RnlWD>5H~>j&q0lPF`9r}qS6`&kaXvJL=l<8=bFeDrO!+@%pyU8@o^Rj1 z@RQ&z{!gJmFjkGfrn!e%wFxD8^wtN+E3Z2mKK?WAdcOc{!(%8h_5#=j6XlU3Az<0C zP;L!Dtqj!=^CT}F_4sS_O;&1K|19hLaR4M@jtv1F-}f5(?GiIJg2X=67Sqddq;JXx zvHwGXCF}Ive`bvQ^TE;1y)Kdk`=+~3{NgCn)n+1~Uvc}x3+?Q1W7LA38p__ zaO?qaS*cw1CW3zuu?L@%BiFBT%x)%I1Itrh9xOB6SsN$1ZU6e8SyVbLP=FsQhi(v5 zG;hJ+YjzJ~q}+-(mxpwppz8$AN;->{kO7gpy;DW~$3L5Sc*6$*RlR6W0}?yC52^d+ z+_@67_E2ExW{~&Oz=7HW*&gzqNd6xyyV3zG)223F26_V2=V;&iyDB<)_c#Yi%%l2q zbXBtNq?-(VcuWN2dmZ#gORyCJ@A)Nbu+tnp!Kc&4LBVfSRmvjas3@v52DIZz+c(+F zzw6}va)N&iB@JHi@tGzDosLjYn&g?qp71jX!NQ|p?n<}bvRGD9^(*{7{0Tw;RSe|! zQ4!9*7z)6z$wFUtPmZoo7)v3j#yvG|4wc>q?%|yu-~OYC;0VGLe#oDJpl=kN@2YmO zi4k|m0D-DasN_IN4(ZL*Xlf~|r2M@$A0G@j9#?cUAHmi{7j6@RyV5@0q&@boPOjdA zP?DAJMHlY3Xk7oN)pMQ+I87xrlJE{4suMym!>iWc**R4r!+~6@I8bbAI^OU?=iY70 z-vyRk5a=}iDtSsq;IMp8F?@{|FX4yR88VAnurmo727`Wp=B+o(C*{m%MgQ3i=)yq8 zjymPP0){8`f!f*@swT-8Rk^GRmOmK@z4~}iF-Ip)k>(WSu>$??#rgV!q2!xs4Ob5z zT@FQhAE>fJNr;V`-bOs?f8QJUF)qzVLi9e^ z#hofg%YibhD2>+7Uw(f+5oszn;8pIhr8i(zdVFhrS{gA^KCv9OK+fe7AM2FGTjx=n z{Bw}RYQQCv%T7%bJkIlCaxck(sIP?480gSU?an4eT{dpqDZFXO{3lx2--89E@_z9? zw8JfS%bql0hZ)uR!Lpp;8aJ&`x7o9nBUNR?6?P=&ew!CUB+Cd))$~TLn-C0$LvG^< z^nT&TgQ7sPe-+3oJ-A8U1ta>a*@Of9uwHiP#T^1-AUfBTrFjALir^d|R}{Pcco@cF zyEkbk1xF!U<9k$FkV2jeQh$TQ^aP zzgFZ4E(qLtcZ*safvaH)e|!FLBZM!`M}cx!eUL3pc-Ki^kW5>OOl7iYS1gs=$t%?;6|KWk&B zlLHobNEzCD_}pQEU=ee5{aW|3zFd8+j(G74paID#J~pCi{ilcZP#!o;?j;9$V1jy> z?^JVGD+Y}aIxFEA46Gf%MX}VEeY}}yxt23a{MWhcD1w?7G*-zE{(nCV3L0Ie-wPiI z1G;^Lib{BKkZ%n1iLYM1t^3NCNrJX<;sy}FtdwAQ$QmL391d+SrOqc=B5S)^RM z1W6wS{65M5c|s2K1=YzZ6dVx&Ufcp&YpVA35q!FXE!S&*M~SZLKsLLU#vs;^J8j_oCA{9Y2oH=2-CHZtN5FI})x5fhqzAWD>hs6Rbe zX0<*OXQPbDH=6#IXP|j4mXJt(`AsSiZ`rFJE*<7)I%9$ZeDq=KN=OKS+s*83l_&cB zEZ+(ay44RQpVO%{nedF1)YK86<+1B>j^`|~3?n5C&DB>wJPnT}8RF`FN!BLmetjdkfakY1OCX)cB)v$VulDIk4#P&#@FAIK3JTO8`~ zdW~<-@)gu>$I6OQn?ixiF0q`>w>APowK+l3rL;4JVt;2c$R0Gmx)E%S(Z_x%RkY`! zkg@B1*A64AK8IM$W>mD9NZ}2$Y5s8rhVyJ*n<1cw)R}{to}&C*3Z3mpFrDj4nJEom z6bOBPH%F$6lj_99`|gc|8t|4yd>uMF#{To5z7qYqOETs{e1w4`7+B1(G4S+-BgeVI ziAcy5K{cNSts-w-9jZT#W75|^>nBc6zsODS&+K{-W@$iWu?Kxro_nQPF%p*)-m*kK zy1fiN-j&=B+OcXyTItH1*RNlv)h2U?NYS04I6SoH#}!YlNWy>m*czWN05snMQ}Xpo zGOP!Rzi=q-xEW$zbR1aALHAaJeN z7y(B);8N+JYc^AE)*c(v2}FtY0Nhmm-LtV-x4liwWRF&Sp~ zUgK60lQ7W|!RK;Ubye-`V!=m<(fL-Qu@-a9lpAE?fX2-oJC&%iy)!xfp_SRw$Km@cli%cm()fHe`F@yk#&n56AN6WX7up;9cq6h@DU54o znY(0Q@2-fxW;7_6{xbX3X4!f*u0^GfhrV%S_lfQ7ZOgoG1&7{7=PPJbr3fl3hy3dk zGhImBo=S#l48y#oXi@UwIP!MN4Pl#6DOTKt{t1dhXNZExK@1r@4}E&4)`yF<&yRKa z)(99Om`4>PS)=QW>Tr8jO0lBZ=%^&(MrxvEqGT1&(UkYg08=PRi5@HBN=Y)c&0R?s zHP2T%Ek=G0Vk^V&OMF3xX~{|7Yvxh0bw;GgHY6nNQ6Ec2s})bJ_V$p1@P%UF0pQRIK88Gjh7lRg3ZPuitu{rg=y<`IlpKzH)vv&EJcRn0aK_8fzF)21_W;lK6z7KXkmUa>(@RilvpT(Ch22|1CE=zZbkgwaKFRwRq zCn-K9J(A|KI!aLFE*YhhWC)+=D-VT8h|^8jkb@S z7Op@U+sm7Oa$PEFx@rFLqGsxI@v`k(r>sZ@cvBOWkE zWIb5QW6O85`9K*evTv363b5EXOB{$h4!|+O05p%YuLC7g*kjweI~!74YvBVB?!6h| zdo*h5Uw?wu`dZQn@!qe)btszsmxpc#Wv7}W(NB8VHE=E+HaNr zKrPJ^z`x`#Iv)zGl-5KE8ETo7HMZkV!+Gy2&$Pu{0AP!1Nm(&fN#iOFTV3L?me*m=!{2$L(MXWxDjeh~dPE z6J3yf?3u21g>hpg9|XeqNNg*(b=pZfM>HnlE_j$P#|ntgojWIN^^IRCl0QZ@R#b^s zH=mzdBXJWoaYyiQk%XNo6CZh3$|QM??i2AV2GZuqnAc5f`r>DaeFf;;4_+^c7jYKy z(fcXGb94j<$XbaeD`}8VE{uecSNmeJ)oIxW!D;QfT*-^jae9AC-!myu(@VHFm%Z+F z5O~T?0>CEw`AQe*2v+hzQWErH0+}^!2@e68?iJS&kaFsfCxYz%xN|oHaAyHQ=R?7& zP>tQVF|gQrK-hA)qKGg)19P9}JT2o{@%UKZZw+L^2V}f!UH;mKdRBckG2mn&pi(OX z&ZzdV{@0w}G~k}2wU^BQaL0$YLHa-T(4LoI3*QQXu6GKw|EBYxQ}kt7SxfSlx%@>y zh)-67Bii0^m0EUnD9xrUW@ip|=2`E+uF0XS;~&wDu8V@PsXJi!;&i@-5xzchV3 zbd;+ELO%@j43Mu_uGvBX$>BT7$X^)pRN|Mw2ZaDOPnq19nsxXF+=XCq2Qz{oC~DCK z6*R24pK+8$+&&F|A~^Jq(Z)fEeoivZ_tP0QPZC^czBf~)09HE%T%9m@Xxab%WHZq_ zs;@t(FOZjV2;?auF5qVhm-hvOXmL;N14S-1k8T`^WFHZYAiq^nTAB6RF(67l?O1`mJ)N;d(g5oFT(um_9j5&|=KmqR$Z;5^-WS%5hg8rvVmW514R;k=>);jvAJq<3+IcMa9RO z*bw}1#6cN>8xclpMeZi6pZ--DK{&f=fh;QKBrZlcEoeMlh5Ocf0y*e21zH9m95H_R zx}Z9m0C6W2$dJ49nMtZJA0VemZ@P&S{XXKq0m^4PU!YaDu8n^>sEX8(`tckdUZ4=f zTI{r_qeW;VEasIzOfxiXjf?#q&5>fH9{DFoe8x53~-s6los*B+&vLk z4@l9$yc{)HF~HL)otrV}uCVlu)}cenv`(Qk88$gEpK6Nu{UQRK*R<)+*8L z6c)1lmN6d?B656;P!{RL$bbiP?seTOynmhN?)&iO8wEpBQDC^BRisnu0+_~)0ARHl zjC&2s2A#Ux&}M*sf0xgJW+YQ1NrsU4F*0}@wtz6oIpdnJ>J7-`3cAz)3zT$1X_y&y zTc4so`*f@_7mkGS>f~BfeOffI_1aje1WD%Rqi?x9?khmrAKt2-txH@a2!cReFd`JR zxBbWV3GB6+Fa}7NSWWWY%a%IW>|LLG7u2G_<~1c_xF`xrh}&`*YIw+z0IKbG)*kde zmTjY@{lV$Iza#9rc0cBtT`DOV*{fJ9ud@9=8W>a+U?2Y@@*Tc}N>J$|GnIJF?uIx1 zz@?jF8H_Q%bnB9KUWyidJlN+bjWTfo_>9TxyR6%KT6T=g(gzzf3`nvU!0dGtD4wxt zu6y+h_cl`u*#ftA-kcGOJ=lDp69u^c>$mSv^$kE|QC*kxsviLZ$-71cXh8Cha{%%# zOBj1KbFTrj z4V3no>yp>5=|u{`pS0w?+I4&{V?;pXK)4ng*lQ;bLOKrP0L+*A@VV^~5Ybc%IRXBg z*^lKO1M0+O2m6lSBB%~Q39H)8mLjJ7&Yo=4;?yGU)tUe}nbtHQGAh2rWW)Wa?77Vp zJra_txr$9Nv#kNtNzidUfx1p7H3FQz-#&-E6=W0yk^{8l-S*_oEiuUPhvHDK?W}T>N33mHtA1 z4mV-q*TV!cPlDWbblqqdlg~u=`r$!Zj=g*kC1@nP?mE9Ggj@L5$i_hxn(ovI6BOAVO z%mH>4IpRl{Laxo{15?+Ivh|BYy8seD-!<|hsc0No)OOR5-U^T5)nJ<`v#P1%L84P< z`~56Cv`?@pTHIPRVCveQzHRX61N zjyyWcik3tgMsh5%X>kuQthO_*(j8luR^4cHX^56zv(*LeC9k*z(x`OR5KFM607|^R zJ!Dhex$LIi>9l-~(1ht!Gczg&X}Uev1NN!xU5=pyqiivZeDK2c*12|hY9J)>0ip8J zbbsQRL9Gj(IC7gvUx(kQfuOh1xeSKJ60#XRm4yoyB;Jv&IP*|xuc6$L_A_EnmcW8L zr9G&hBZl&f=C(dc0~nUtw3s>jV`~v5M!Ut#QGrgCqXF8;akKJT4@?zX&sGXNv!xeC z1WrlJjocBTt9x*X!rim`9JQp&%I0@_^Sc|TEKqo?z~DxGZ?gkpIW~z$i>p%&rmu?x zd?hCBmBe$oXY4}A{o*54&ymUIL2uTOzk~%)hAQxghzEyK7F`!HkS!V7t7L(w?jpT8 z?lxNEwz67KxA(poT41x`)GanvwLI7TZpBNKtv=*}!Z3O(MLUdmA+?f-aQlOTmemoT z??z_Wyg7X4f_^utkHko#d5`R}^jM32@^sH-d_LMq3i*w%xO^kO)dGW1Q=VJsDHt+u zi+*4xtQf}i#;ROV76A6M^Vbm6khwsrqO!Rm#1-|pt9gK<(Q>6sq{#UVN1^Tqet1T% z{7=#LC#@NBxtz*Z{FAYLBXeA;s0%?o+1$&3Wk&X@?&ZseZCK9WgiTwb=hq#VDpv}O zmyH6N7RXnxEd~w8(xs1!LM{VheEisLq4qDy zC45=*e<&Rtj6HE9a@Kk0ji*Ar0rB1k^;t%ZaCSx3PVHfXsPP&~(lqGY7GLq~XU1+n zOPgB#Z)whZ8G6-A3Z-+;^6I{z{Puy+UhUF-d2Y{HziAZQjiI6LiYKBM*jvk|9gS_4 ztqOp_Gi!R!5haRK8P0awVPgAkw`NL1QcLFS+||`!rxHx-eTeLV@YgHX;4}>QvnfER z>;1a$%Bx=%W`Ua6Ui#`Nlc4A~@M8zeTvt4~pj_vTO5DMlq)UoDGklY~#}_H}glsQt z7RQeuyL0fkdqKwtb?F6xp`@O^NJ=QwlEe}j2e1I=%FZYubggp;(6>f6T-sOg$pJ7~ zPaW@02g}WI|7GuMAITEmkBE^0*keEDP;VEl`0vab*drQ~BJHA1c_I5KD8(D~d4 zR^3Gs?+S1Xt*v$+ZFllBnKg|BiT-19U4DW)i$E}a>7-7iez;{+g0u9ZD6ffL?@vvY88>`{D4x!}0-4A(s!p$q#F5IY}C-d;(E z8;2it*i5-~c5}Ivu{|uP>O)8XQ-?Os_N`cjmeoFM5Z>SVD9X6-+ zh{duZy;`|;t?z4`zEV-npD2s1!m!X_aETWh%G#)|}j{H$5$PVg6~YYsi=j!;&6u zoJMY2xLQb1sALv(__=>vL-rY$`-7;dsMF~b}Y3 z|4aR+y-r!r**;_(hf3o49;+X}EC^`FTBmc0_OByTftqV{$qL*nz0?HS`)!-Yy7Z4a zI+Gy`1SLP6l40x9ddnt#{8+mnMr&`(Edg}>~v3cNt$ZJJz=vBt^v08 z$7T=KJfGD%?>-!;@h;1Toj2cW?x^)so&Xn@`i(>}1~Hd+k2Z&M#>MZ?Mwjj;7o6aM zwH$b8>x`~+$qgdyxG{1=`3GrMiy4~Y{L0v+kxI2&o3tUR8WB$J)Q%!e&tbKXJ{%mN z&dp1D#;196tH!&p>nmzhyfDhz9{)%_?wG8H5K_)$yqre2C(vlC|`{=CF|F zXoJu2yt?bLQJ6&=mqarsSnKCKzM@<(g4zn2UB`?+>oMQaD^X&zcWU!qzSZ7_iBMr| z*%mls+J-=g(x$n0z|P)Nu71L%^wu!QEpaAsQ7McwtO(67icM2axP`M>sPR}~OJLiO z(Ntt@4}-^+tys|X`rI1G%(>>|D)+(qo;fRS%Z~|WX@aZh7(p5s%=%Wo$9}x`j5Ps! z>XWpMoEs_M+pUjRj#EV)+f4V~1U7%t$ zFK$wLys%QlXs$tA#H6jzd;Lk&s2Lg@W=M_3iAb}u#1IE_^=3a3S>$p4YYq}^l{<&1 zc4eXQHTGsO@(w*_u`q= zr}9bm%THv=RO+mFmTCqto;wL*8&(*SjFoq@HtRZ)8{}0^nTh$IQa4Wv!K5(WiHj?5 zXp2(a$enRN;^xUxXt%Xy3a--Pzr@3c^5s2Lt@Snn_O};#MH=z-qekj3xUOo}1|uwX z!lAb+VkWO{aU9-y`(S#nr}+9-_3L*|&SYep3d_^=Klebm4S0BU_h*h8JC{t`^-(EI z>Dy9}yYLB9)xkJKOp-uKye2Q+tO=o|N53>5j z@%*uvym)b^h8k&nv%c({)ELUGtdge_@SZe{+46L=5w6rv^re!um7|!kGLG@y2IXOYX_gN(a0wwlpGAR*%p>4CLe$~49{Af{7n(BD1WtQ2&B{>>a z8TPXVE)n>Sy?g4ClUzfk6=AyuyL&Xtg8Cw+VKZtCTwbpQm${9}TZ<+YreArIO}%Lp zSe5WxetB2e16OY5fqCndAdy*P#Rh-U<@XHstV(=Z=wiV)uY*MS-Q}P+!VVpxnag|4 z-_9aiYvfDP%`hZy#N3S3;9fw4vV3Re4vzoA}v?X6ubT84?<0W2A zk1qE#x!qS4)GwY6W-dFU+#4Hl^Y~}W_3DyL4W)G6E2{lh3*eKaBr~xoq(-vL=Z+-Y z^ou234N^lY$0&Cy%~N$MAE55Vnz*ludUtcQ8{jrP-Li33Tw`=$GKsyAg}7ac81l86 zuuH45LZeEe(cMaVbGvkT&FeCIV@Behwn>K9Ty${a+vW4uFPCbVhjl_X7%Wyu)Ly!d z%nKAcEShmM)r%ij?M{2Kkgc*87}HkhD~8di|Ylta8S&7~?7T{4jp~BE^?Mto-67(!4yOnWD5;uXi9etqL#5 zRb_Q%dLh>rw??f|dSHZ}@6(<)7xldw{@o3!b6(XnjwioxC#9eYz9t?w;P~z)dGEdU zW^p(+t=_ONllYS#Z4|hSs@$0NnNyKaJloDLf~0!lBCCA%-!+JE({+j zq;+pLSzgjHw^a-gE;}M|vor|#xSHNwzhedc6^X=syA)0VggHad?!K(va)jRB%~z^& zy=ACotFp|?jYTTGJ#f%=UK-_^R8W0YwBc}`^r6aq&>Oph^!DP> z&bl2=LR573Ll#US3^~F_6i-gHF(pLT-z#4Me;uC9A@AEjk7HCkW3)@p`k@UYg805+ z_4eto8yn(#uPu-_K!7s=)ytJALW}&ZPn{ax^(Gs*qhq7QGSw!xykFoFMf70QdR4Ym z!<~AGH#=VJ^j@T_`L>~YRR3a6ZZ}?7LkM2dCw2Q#`!fbICG-;#&Q}|ejb&~6ms;KY z&)U3|^`5A3^eD&-zsOv>@-|_VIs}D&d|&**8k7^;{c zjkxtlJDaBro;Az+)U9h-GOgN8#r4YOd#y-*hIMZ?C1%0vkysxJy>7vrq#*oFor5Yv zR=Y;bO9@nW`NBZSsM5j~Bs1JGC{4^1Bem_9H1qX&HoQVw)&i3f?kT)M$)lpP zKcy;k(qu3V??N;l!`Q*`Nt?T4#+gmmsXA2(-$J2BKEHrMp}tyr zKj9&7idu5x~o zWu9JnTIafZzFRi2b^L-~yj=#~{Hmoo@Rw$CDhDoWfqSjn6I zgz-?xkJn7{lZXks>@o9Pvbo@nq_G%Y()oiqy~$9AKu_+2qdbPmSp%ap&&sFVxG9r; z{lPGlc*v6w`ifHw`X|=gw3N)!#l2IihEB) zXD>!2slU-lzRuRwo7sM!__Woxf^eEuM<=GmsD2fb0~@Q6M7%^ygXE?w>d3XUuqM`a z<87?l?v3WzK&u4Ng)FOu*$qOZ#0>#!Kc^IkNTmpamX8XbMAVy_SDjbOEngjD-DtYIJEa~umEiF~ zC58L-*VkpOX)V3|;66xIkhhS|MQhyCj4ia|*(ak*Kr%(Gw_|!uYzydT#^L4jaWgsv zNIo#B9wqo#)^V9JT>~YTv$9l~k51y>566zfk+O;9KJnMw4DGK?&R>CqG%7KqOP`dA zlE%8kkQ&B*<>6vfW0(We4jYzleP(&@MnqU4LLRA?@NSDMoC7*Wem$+(&%rnrg2{ssulD6)q$$h}%aMfD&b zG1ASCLYB70)m540p36q&1~d-MDjh@9r5$n8YbsTgGvYMJpv7t!_cf*OZdqH8xB9!j z$NAk8xwFVmHeNk}Sr|sEUYX`y{N%mx%w-2C4)98M@%D?+MQZh8)C?LQ71RL`N(b1sJ}!cgj)SM4+A!o0C??^wacSrN>c74&a6~y z^vmi6HvUEd+sACxklOj$Wb@*M2GQlH*pZ!CIHU5`EX?+-eP^dVC*RldjSRn-$R{-- zEBg^ltAVq~*^ieujYEjPwB3IBG>D0&_ar>?c%O;d!2ZNduQL;hCKFZcGpnnMf^@P4kCi&vGBgf_svOT{eONJ}l{GA0lBRTyxl3{R|#b&&=*Nj`(?Ul9xtr@k_hx2-yWGdd5Tiet4>K-U?SBFzNw_!cPf7o?)2P!r7`1ws@x3)p61g;5-m2=dnG!~M z2}u9NPbMG5NaGE(GOnD_X`%2M6| z7Xfir3%;>y1;(9MXDUEliz(ker(Rr}wH@8%d1QmJJyQ0A&DCB=m5;roF}79e?x^SE zI|VUE%;x&M7IHpDR|#DS?or2D+Gb$J=-fU$JQ;HZk@4eetax0n`l3l;?>F4&2>LEu zh8_&j)8e{M8Y50#+wHX5od^p`Yc^Qs_^R|=@6!k-8>x45)tQPacW zW@)^%%Pn=u$2m^@?0!h0Uyvl)ZxDT^MWY4CUR)Z9{I%{|p;?YwE(h2%hfQ`S=WoYk zvl+hloBG-%(S)ZTo&8$MhWctw;?c5t=G)ij`YI1APvGn-f3@KEv5Rc&*LgsfG5|zM zjel$QDx>WIMHZ3_vLeW1K=AszxthY%qin}t#P%vgcrH(uN}D9O}o`jVjEgg;H!h25=)hEDcPm$ zigI-OWuUuEd_CS7==2SJv2OpN_V}!phKlYw?i)`Px0a_|jH@W>TX7 z3P!Jj%1;o?M6R@K95v75KWW9p_08j~tzHdg`;6Vd0C&?&-e8&+*HiqdP0Xo2{DC5Xe+^I?>4MZ6)txJoP(bqOASbt;=t*Fr61S3w547;#%*JfYiB`Y>#TRYweVp-$W>%x}r z*H*2vM&YL4IzCL?{IS>C*(B|2LTi8S|AFtv(padX3mWgMZdI&fQoJ56Z^PfWPaL7?9B(8DO5|nfBgo4}*PS zvlS+%XpXxZF%^du!R}YOKTW8~cVF^Y)3j@5Q_mG-KE{87Zu@(ILVweIITlg7_;CT% zO!Z=UaGBx4OBqJwo|@n00kzn!>DH92oAC&>*xQ08<wDIHQZoqJsNU3x%1ic`Fy( z)^Up%E#KKjjIzF_IU!F{pH4J^i9bJiTREyQoz?pl? zy1~j}wR#>S)Dv%X;r!NoJMuNVgDOR<=M?V3b&%JeEzI84h!rUJOkw|F+}~P>OZYrc zs5!a&+9Ch!ZZKE}X^T3PD0%75BPbz2q3jR)vNV5yzZj+fI*6(250D<|vY|>YVHvHr z-ZgyoYm@}9cDT=SIc>6qEXh(N zw9v6u_H3yTLZ;}Hh=goo+H6^(>|4oFDYV!bl}wXLq9|*Oec!UqJn!{9gVXtae*eDz zyr0fL;mq^g_kG>pYrF67btff+2A`HdNQcb^|&sw*{0j8&KE{GXWMuWz3g92 z7&k!RwvF(u7Dy0w1ugD}+j8bcoW+Eg%~|gVyo>R9Lz=VLDH<2J>z(NSAwoy^CRlO9 z!`7oIFH`6R^u6DtvT==$k1yCaxec1xWRUbCDo?z0Cgk&c>iS6KS?`x9>vyjs@$mGS ziQ9*8)#7(o9`vtJX356Iu8IgpH_KoB;c9(p0Yi^>A_0}awcSJh{OyE8Qsd=@lRYz6 z#0U`r?-VBP`LNs7mcD(&{w2HEz|{mvxUm~oEa;{&<&DgrF5)tkUZ`&iM&{i!8l;|- z>SYX^gt`DG=B}N_k4pm_+%U|hF0k-%t_$K<_Pn^qxz#7^no@TxzT1 zeUPuN1d4(3gk^F}p!%~!weOiD2hj0Z&K(4^9B=Gl#PDko%#;E$T>FpIh$FfMlwbZ( ztF@E5Wsd~*YL@o~pkvMQ=akX;8vkNemImkLkT`V`%0oX7t_(XzBY_t`p!jh~EpP3$bjbVXLiRUQ;^L5pvt7N$p&HmX6`9kzcoS+0?hg-X*|F>AbdH zH4q=J;?CsVa0^`^+XhJwCRTsfn%uFCsH{$~ZEKr>yD62(E<|ZC_fRo&D%3%K#Xkmq z-CQ-!0Pa}*YBn+E?O*YMIUn~S`zJ7l&=(yb9+oNE)-sWzPxqee-59PsuJb{8HfPQL zE4%iur~tAl_L!^^lN8QAvqJ^@&6t@PZivF(tXETZDjCYpf;u2L*^@9-nX2aq)Mvv) z;;$|U=3qQ_uiPedir?9RW3dnFMmd0&7z+KrsNYUk3ZPI8(+DI3lt}5pJ^J>p;-GJIIX8kk#xG zF68SO$uSPn2sd&d0{eG`Td5;pD}Ay%JQ_RJAN_&0Y1G*vHK98k?QQ} z>=$43y7ua%i7>EnP9HY8s<$N;6T^bGEl_4$CTq2|f%s-!91!6l1 zAJp7*aiw~#623th9VP`ZgN^(XA%u^Bbo8x+LE_>G?Z4>Rq6 z%@uZfoJ0{%co0B~TWHXiX}*h1{8+F*vwHgJABxz*PqR3c&vv86*asM|eQW$3lCOxH zT3&b=8Du2P5Ul!@HIleH4%VT;=<|BUw$NhGWv(w1xMM1IP=x9N)p=wfVcZ7}f*9|U z2*%QhuIS2(&eVINzo%|v1)j|Y6#vCFgue8cu=6=(`_=(IBX1oy{{N1n)Be0hH3SHr zRn@yl0sz}k@8Kc|_huI|N3bzt5B=4}*d%es4n~JUGb~Vb{q~}sEphp%6GR#4d@{^9t$xRl6n>t zL^G{^BRbx(`xVE{PI7_2_J-nh86eY;JYn1MH@Hl*G}ho5U}e#>0u8C$v<(XeB@Kwc z3$EIzEwB+dfj&?dA^K^5PvT@d33{a7v9GwlG%IU#R)`S}L+5F(8F)D!HMwL#Z)rH3 z;3;T!Jv<0lI8oxF!dSbyZ5}{U+SExw6cuqPd+GdeN#7~LNa=(jI2L8p)B=Q$w+4v$ zEiQ(c#gQq?e;fZ+^bMTa!r$B4fCH!_j(~wU;nkQzA-FhBL1>TuZ)qzlhiq8?rdz5k1Na7u2FxFt3KNOT}=1 z*BwQG2ObKAT%aa|3}m&m1J_ov*QaPPpaROu0ZL*(F`Mzt)K3t|3FWi?x#p%3_+$24 zLTaioLvEhXE=LGZ{un0{B+Xb59#)u+R`pw-HQW{KMlTs1E=L z`oMs-d3drXbcMlxKNDj>nsRQ+A3FH!5yRAjN_MkEOD?#N-HI;tU=QrHNP8Y-8{Eq%z&?R)ZL+(KxlYTt22e(|)ZMD%!8)UuIg|T;cVg zq3;ewS3)f)_{L+xaic;$qpxa71jO8xBG&m{D(4pJ0B^S*?OBrJ+nuSn+`_jl;pleg z>DdCutj`YtldqMg75Zgsg-pqyiHY2rn~dEOHvS8wWhdE>c{JDUqwvZH5;OuBwY={5 z^BDoO3xz*#772u;F6rk4>{CVQbss}?N@uoT1S?KuZX z^x+c-moaYC2*bt>!}egQr-4ACs{|%Z8gNL*8u8(8q35ZM%es|Vj0EZx4L12qSJPP- z&$yNu>Xw7vtW%OURGp>j7}7}XSJ^xAo18)b2WBrqUSXRBLA6NxU;gjkJwF0_k=nru zW;~(sQjJPykVJAIP-V5jrh}M%WqI=87-CZILan9Y)6hFGMJ}mPP&94m%Gyyj-5LVy zl>L!3#L_WO^Q{-JB4aCz<516H-x>a7VotgTjg-&JoE%0VSILyZy+0SR@(@GMHpHT!oxSe5 z86A$QZHqe5*Q+Hek7#egRiod!GYa)}el~EZM?yO}; z4HD|I*Q-n#dle&V6`D7!67;lPtKIc$2_Er0#<;9mfk15B;BBL|+CrMZr0fKYheM&b z>M)3)5%x2rzQl>)zL=P2xs7q}Jm?xmj1BdK>gzP%C(8ljz479kwRmW9fr4dxF*;z| zFq7lI8@F&2MF6pxz1eSms1~|ZyN#y^%qciF%0?m04qtJMw-fEyC2^dV#|Y>Ia~oOU zof{pJE}ObR*oL|bh%WU=v(EKdwRRz|LYHQAWn(U@@UA!x2~=Dkw#tGFn!HG&x*_S& z7CWAn&-G8%BPV3Wufk?w3s1EGa~%H~i}l`xQ~+^J5Y8Qf;a^o~*dPEcO9FolHT8)3 zU{(u#>YN5Vq1D)e0)&P8SbKVmwVu(O(B5;$vdez`=!--^V1xrDbzzG`7|Wr~H1-%j zlwGil?}}K+ZUayWQK%$n%qwSC87K>cPwUXd?KW2iYY;;IeiB`ftq7}35j7G;EdKTN z)h=l20ezA=szFtMw^FXC;ZC&T-wq?VZkGl6HR^Voy#}6Oz7zuGGzfNo*VVn-!IwtO z4MV#ECgw|Ayh`qlP);X^<;vGquI&g@8rE5lB%Bu&S`5EIn5GbPW-6W=<>;nC zXgUTdMdgmi*&Nqatq%hinVs>*m9T_DCqQ9DQ~l~h7^O>Uw1z<6o&Vh)b*5iUE#G3! zv|6D>ON+61X>ox1Tb7}H=+4Lzm>aCUFANqBJRO7ZjtuF@7Bt{QlOj4_2^MEzG71BM zU3xHvSw?9?A{2CMLa^BZgz+%yY@9>#W}u3O1G{zSC)wtZr9H`qJ&|mx=i^29L^Q&u ziBzLHukmb>1c8xDoq@tzKxRw|Qy{jn3}*g-9VU{qyv!}L89i8sGmt(rsg7$ zwU@ggg#F>)8_@}&YuPJe%dT#aRf3`L0W+m${sJT?eN;kcsDk18`NQjw3C0J}*Oh}^ z8#9=~Ect}UMF>i(43&mBfyNx#;@-3~`AT;j8m#oBr5|2;KMS`;1-C!zkve-K7CJ#h zX4C$Qk0XnPNc~&_HH3GEjDbkH(IL@A3r#e(bdHgQgv_j5H0vLwi=WKbRghhT|29Y= z^tP_O3e!PJaG zuQ?IQXVyO&no!}$nnB;QQ;Rh9#!*cwKmp3nxq#Vq3)-D*kKcEJ&!f%P13IKX6}V~t z7p8e&0j$8*Qnwmh1xTrvvqHz{+1~nxM}6qCJ+)Bx$18tYC%a_0cvkRE?~;$%zbW^p zyu<2EGU&dENBGpWL2(7ZsWVI|qsKpXp=W1EhZwd6EbkA1o(eoWi;gCFgoq}mV4hN9 zf!8eH-gs&ZawKga%Iod1IJM}JSl(NLRL4K%)WRQ9wOAr;7IefHpe4CCn9MUVC%Gk( z+)P3%J2DpM4?b)o_^@4=4-4KQ3l^#u+TkeA4?eHWJ@p2z!E7Mfyq7io88hA}Hq_lg zc!Ty9=+6}bEY{Gjd-4Git~w`)<`cY-^gBMIh+iuF(X>29$Wb)$J=#OF|x*_3CgQF)N! zY0J;Hxz!oey`k&V9aePhTrxI6t04t5KX@HHC((p^rm2n2J=9bX4!!!t-P?SpdH;4h zn_b-hC|%&$vYxWgrBZS2Dex_wVBd}k?{6QubJP$EE6`eaC~0tQhSFhlGk=S{{MG+q zgSGD&hprVYY5iIu9E1 zUWR1arTK@-@)+Di@UGlUkD&dIaD)euPMgnco0Nz`lH7l6zqww>X*^~6L2wER6SfUm zo`A$)L;{Dx=J}dLkJc--kTcIEc=(ESio$~4gL&rAzk%SMq_8a0_i)ybsyu#{6W$Q(OoT6~sQeiqB#4>< zG+mP^Y`@wI7sa4A4(0O~MHW}#uTEzH*SDyM?9!u<7FAk%4{@O?SPyNT&|{#975B!JeVhXdSF+*v%sp*@a1AE$+qELC|pPt>E2aG zLDJ4lq0`}@Br^(u(UesNYp8@*Dlbugjn0aCM4d@E{KTf+@QXZ~6q&QZ&sWVVCED+LGEhw_rQ2HpU?Ol&Be{Bju0 zjRK+B%L8u&y2#X;I~8?x*S;q&J^(Y=3i~F7?%O&RxFZ%&@tO>Ts$Ff@enN_>Z%4GH z6PlOz2z!Yvhmm)#0yy+U*zHgcXvG3OLk1=-5%`LS9nc+Y1C$7bLK^rX{YLn*2C^-{ zkx6@xcq{k;bqX7nwL{Y22_`oUY8B-?N81~HCO#e(S0!TDKv6~221~U-)q5Z&IhqrY z9C*pFT(Zhn&FQw50yR0HBfi>MXGCuF(|qU6$IfwGelbaxjyL<^krSf{hvzx)Pj*GDrOAC*r7zPbyFKgphcjUg_t`eLdeH9st~aM#pg zJAA`}prupMnYl{@aYQU;E7Q`pxr*Adsyms8eKz5onQ|s@jg-+2{8Y~u!pBurOF!G7 z@Wbtz;#Ai2`)`sSfac{;J^8YL>Ta=UhN9QQF(oAY1I@0&h}HSrwzj9+(a?>CDJ z-G_PRu9#^2^W}wZy|Xb%HkqVZJ=GBQ_hmh2HQ#4swf$2h>oTvtlpBJD>z1Xz1H%4P zyQ9wSS2)ZsCGG8KZ`-nAy4lYr(es;J?Nq8tXKTqy@@)x$_{>uuc7OZvGA3-rx&c0X zIUj$CJKeyGc705(YBM>RKL(k)YVOsS-t(QjU^P0V(Ds)7jZN69fR;@*eq+Z=X(JvT zj}l~(QZn|s8bDZfj#|z*=tHq)Yz=zSb=N43A{il#&^t97s(O#uXbrtCAW{1I%W9NF zOwIq9?N3(qLOOddeY@MIu<+5$TF&$&{%$N>BD*P_9_c@y>}#h{D%+7hd0Adpg?2US zzG2JXUsKfF+C*zdqM7nc?$Kgv@tremh|IngyUzZao*vztqh3`b>lEe6Ca>@Ieygie z6R7wTu`rC2-%I!Dv%$kX64|58Pe^GfAMe(PkLhQR403vQ+%`xoI#5^5AehU{TEtL# z@@S46=vXB*d z4NOU*C&VuHeMfek+??xn&ldXhCAr;De|{z*W7>0e)~s{xu~Sfz68&#$fiYJv-)ANr zGrRrFw!BGOuPi9BK<|CM#=pv?X^PhIV#4Uw&FVc_^ya1e4AAZ_NeKGFPVCK?)H;wad(h>gtz0R*YEB2OuFz9Dud2~SqAgE-pNy9f+RR5u zi4#ikQ{Ev{+CMn?BvZ(H-_7QUaNrpjB1QDtdt*PUkucKQLUGzkrM`)2a+2AMILR!# zy!+8^kKGMD6qBihRdlhk43{V^mqUo!M^{=xPT6ruy+JO1r)Pe1^oJxRsUIW7c#FVT z`7?TntpkaL3v=vg=k71eRXbPSaImuGZ94IM^119T`W!jIuIqR_*}%gl!F}RuuUwyb zPWkMY8$0$mww|k-A=AP$e8nHLd@7x(@0X3bN*+>Z<{~G%ADe5nw{$XzZB)=JkWBP` zJ5o38RGi^lI{y0nSIw9Q5fPjRBej77@x(8={3mu#$*}Qv9zwDfC;lzBduPPpDTMl_ zrsK`DsvdIerB1nvUr*@tm$F|qs{6OPpCz4yTlFIGE3}Z03faZVw32}X-p5F-lQE<~ ze#Gtk)BunaIC`x7w5G6&ky2kWgF2wvVnDQLis3f_56qr@5ku#d}(TkYqShx68!UmrDjv! z2~I^W+ce`J9p~9StuQgszY>kB7X6##A!5e@i7tO#?!h>Zj=0|xdv%*-_BXreSAxgP zH-5JJ+TG_(;U;EO?~Ci2puT-p{SiqXq^e#X_?na+Zhu3SpuC=l$KkjDw5T2Fzr!Xa!gd0_x3&-72_~)BqWY=VZ^Gn^SlqrRC zn-9~c`NvEzs)hNgobdc|+KA@d`d?-(l@^FrFx~%_p;=_o+7UwdW>2c>XbfpsyYYTEK5rr*3 z(l0~}@lXx~4mdT+pJW}I%}f*h-qFsuXT|1P(aZ|B%IGYu+M8d0Sz!NED4k)D#BJ&H z{ECl>xrOTR{8%5+g<|S|mtBe}(m8&bHuNg5^V5a8GYf5gv# z;>_ppi61^}bFMGF<2^fFV3!(V;rRT`w>*O;(>{~Q>YB2@*EMCDIhM6V-}brIa;7b; zPovbllw$NF)U$h**RzL~NA@crhltt)X|Gw;;yXI5?P8j8v@WGs332Ir!6!^7F3RhU%NNMs0=QmbS5l|PQr8<8w(fj?|K9>24*X}o{5;Jf*Ct#N7 zaoKCe+SRXuEaYP`bE3F7Z>*a?eOjMSJU}S<^6S}uN?uw#-AK8@Cb3Z7;FA-pvu>t@ zRz>ft!KqGvVE;#r;!Pp?9#?xlGjNh{T(PA6nK^#wYe$Phn^6G{^%iMuA6I<2wi($^ z&8y!^Ipq58l!-vErDE?$zzF%RyOL9s;DVX72kl?b5iFW>Lwj@{>{11!9$=~0MX6FLdcIo`@_dyzp}XyuCbwOSJSy^b`XqDKWb@ zE?%cC>KX9^n2b2ep8rQ7e>>s(T_xqg2AmeJTS}4_XF@dsqes}kX_0cUG{1Ln9QkL` z%i{YZ>19!1Km38me{}8-JpRA~^%8&V^s>vse)!{D{u4X=fyWTeDDS^f=%3JZ mIa@$~_ydps!|-TFRvc3O`>5sA%1j*mr>kkG@${hW)&BwcQ5=i_ literal 0 HcmV?d00001 diff --git a/hail/python/hailtop/pipeline/docs/_static/images/dags/dags.006.png b/hail/python/hailtop/pipeline/docs/_static/images/dags/dags.006.png new file mode 100644 index 0000000000000000000000000000000000000000..0c71333930e0608872f8ddbe7b05d33e730d42f1 GIT binary patch literal 43766 zcmeFZWmr_}`vy87NGJlLAR!>~oaX=$;0G31Q|=L{tcPY51d;?P$Vh2>V{J_1`QL1tXy1|GAo1NYj!@LI&Cblk zW@XU4k-+wNwFyt%7n&_ESEzY|L`G8bEob<>Y?4|L)59rx&Xq@a}Ajct*1Y!O85;>7v$yO>Ma zN$9c6q*%xp{`c#QpxVp-d-HP1V4PswH0ALBxi$V9&>YqO3`DX88|ylV+@1Fh!@q`s zPcMlbZ2#{;1D9UGkq7Db{dgJv=a4avApn*AYl8m``oER_!$$ryk3SXz`#(qgpD^~H zdHiP{|0S3INzMPI@qYuG|FFlu5Y~U@@t=A8l{Nk=0{(?3{;OR6T~hvM9{-uge-y(1 za2fyc9{+}$|1TLP`Yc?Y>i`=iqi#0-$x_4%k@}8NAzRNhfd9Fi7R{ePrBQ=^#?gA` zTWCKs4oU6Z?UjrmyKhmHh%gFMe+@P9qi2t7Gl3>F48eloJT@#j4-AyI>^ z1wXg|nJa*ktv&aJSYik@8?;pj?$3TK;t=`t><_9*tnRN4vKrS{cD(0;QkZy4#ft9A zho>C=i&o>g(?SE{$?6d%#m|@g@s4R4vi7FUYhx_y(JR6_!6o>&hb2s=&HjcBo~0}r z+pGB*MNhwkKE0`PurWTpBpE{^JazW88>_G0!#-fYUk(u%_e{UeE$L}hpX)MP?wvu3pEkdO@1i5^V1_l{r$;LmP%e?&NGcE553B|Y@V+T7dK2^ zqZ8j4Sr5ctQmcnh*iUczP*pTpXP5 zy}evfRO2uIERTr|gXKW9l|n1h^w`+WjXRb%A$LE_Wqs!9U{i)krRPBv@W2W5#NE{^v^ZaJAe8@yy~?f5Rj#)RC64xck;XN0hMDMiI zg|TNJY)-Q26lo(4AcD?zgCA6=MBEEbreMioX2`YSteB^l{&Y4`Z-7)Lo=N3nAYsBb z4L~NO2sjo2hE(?j(MH1 z_s=L`E;~2-FgQtA!ul~Vd%`3f$vGgr+#sS6QU`W%c)Y^;lAvFp(@diQ8MAU<3cvkO zrqm@=e~4DTs_oiPA>s+>!(1YV(X(n?!7ngkmAi|eU&(5ZMnUBoYjLXeJ z(!Cc5YBuV?E0qr8yf@kP`#8Y!t+-yz*lM_c8bq*U6u|j~lX|K~NG{=Y39eZsH5sU; z3e6WX`s^+t>g$fK`SS&jyPO<2dYwRF=S9paDYZc}LFfQ4&E^NT1K*W-ZA`Fi62HDC z^9$W=piYHpeo!vN#wDy>R(#L7=`>U0oDq7Ja-j^;ehGU$05$Dp+2g#%(~E}Gfv_p~ z?$BTPb9cU$fC&C_Z@m`fIf~k(rw3NynDB&9U7)C)_TNJ<#i`ZvZ8?vX-*Kr}IXa`_ zx4UOl?SMF-n*E!w*Q z(OXM)KImTZgMXW0r{FRf`W8(+O>ou)!Rlf=>iT5fMtEDyS9^6Ze-sF4Q{zY7o_rc} z&ag9PKFZA4*w}~fCH2WplRt$-`i#JIOAQfnd)+W$n8MYrRADW^V8N5t)exV{*mO(< znt2gjIfPc)p#NDTNzoPAg`QjUNsf5da-fhC{3OaFH6Wljzw7lMbh0`S>?|{Gs0=~} z-sCcwIV8a5(`QL`k*G5n*=&L_nANKz?gtCo;aU?6r!2jn1a}(f$E)*8WztcO3 zB>e=gJV@pt56l1g01kZc|Cfhc|K?9-6QY8BettetX&a|s3riL>r4$zMf0msT_~7f?m?+1mhm2AA19buBMisVP*!`?7Omo%Mv{%DN!0N)k zT`ITa`rnxpW{xafU#zAie z1-ff~C+Xiw|GjhU*MXj(hBX=M?}1!g2aJXhw6W>OAooW~2OO?k9_R`_PE#rPpV=M| zzWHzP|E>K0$7VzI#B3*`%5FFysmHYThQHj6pp>&=LL}W;veEjGkJ1B&sV+0?p90%t z0ldggvq^@%r*m>z)85}yGU(v*_Ccp@)6VA2E<0{Xmb72LNzti+CwV*#$2PAze3|6% zTl-qb7ebeFx^n8QTa4OPaB9lnI@SFa`v+hMsb!bMrsqU)ew^0IoY+%i^BYmFcR%V7 z>pUAxi@kZjATZ~{Pw1sEozs5oH6HKOb79W`txXUNK2e2o?p^s_c39;rM~B=hNNXT| zY0x&97uj%9Vbzn_saF=2yZT9#mFRi1A+SJw(|Wt0=zLDBroA`Dm=A(|Bn(OSP%de+o2|z zM6QOJk~67xCy4c9hn`Vh!K$q$1LMJFeJU1s>e8lBQ(fw$y85?f2kuJ^hbGtuIF;(cT(`<9;=V>EWr4Re*7O0JfOa2Wf zZ#DCkvo;KP@5O?1yv~k^_zU&EXjv`Sb4-+5Hk^#JXfO~3z!HEOa$_9bHactD8ubdX zBd3UGwBH$3(uC#Tis(%PcHe0O=u1kSgROYu#hq$Cl8! zj?_!vi|4s}`7~7yccCpZ-+mN3zWa5a^9r-#8sw`M99A|}@6j0``q{t2(P;*HgjY*m z_bChwaUHz8_gy;8uD8xHt!mDw%1N8$k&02xkzT+KlJ&N$`FVkPsu=@ALw_wI6MQcjXggN$l*-1yN77P9<<*$yWKk_b<0 zk3!C)Ov{?5c4tnv6Yf3fD@81=R@eOK=FT|6IuH^bPv%9Zfx@lgyjz(aC=1 zl#>_$;~85erV%aLkv^SroIT&pvEni;JoYeglYo3Os@WQPw|Y1Jp2nk^9>6B+1ZRnt zduU72{Q~3@hxC-JW85Nh71A^##i_|l4!MFhbQgW0Mc!H1h=b70D+4;d@;K{h|CsOQ$8PE<@`E>E=(5%Vg{MsdBbo4o;n1Xbk9rz@AH zI#*hA>X#8u@bA)4q~8SdQPtuY3r;GM`wfBjoW&&!xuG$_Iapoldl{)XGKMR%;N=Mp ztM}I9tD@3Di{FTPRm!d981+UR&)b0S7jfUc&njUgJ6noY;F3UT*V^$r^5)#D-W*#_ zR7>yDgPk|GZ^pXlkid>pcN6?* zR*j1!uBX)(&z9g5dY#PPBmXKOh|U!rUrXf}%PCW79RCMqpf3PGX(m-kI~5j~&9Pah z_qMrrJ!N7Vo1c~7HSpa;amONc9!q&I>+jwLx1Tz23x?eZ8_!pZo1hHcqLW?yIetUJ z#6Qn1G%+m7L%oK)r)E;aOLdXb?Z+F=D&XDWEU6d18of53!QBHkW7Jc;(Yyx_&W?p&NsQtq z+hCF>@!0dG>NO3V-K2bN7TVK$>Q={0v_)Q)NR866?k#qH|KW2-&4$2Gn%5nd8;{G= z-X5CKkPG*2jusq8t^E9gPIZx0PW6=a63w0%X07W*60eVctC%g3S!TmkDH()4DdMND zDSZ6G(6>Z>TZ8GdY8T!gXt7a-K7B9t^xo;Uex>YeN)xuZtu3euMBoR@Q_+N;+owiT z;R44~N0D~3_VG?ag2-h>^1<^L=%1%;N^8UOV!h>PTqQmGp|9hc;)0F4(ZJRV>2Q@Iu415x3M z0<-3+p5pU^h94k4(#*P#0=-SWXXray_JLC$cJq{Kzf2kNmz4VZxP6~Z?(6+ZI@iTw z1}FCZN_pK`RzMUXdhKbCE%~0vS1aQ%p>_q_)Z7k2YjGUDr&+I00we401okH9n?d)A zz8G@pxwCxdYft(?w!&$$s`^dPR$mb18Y*l$O_@ShnZ`jqq4E5Q_O4uv;mfJK3*)1|1zA7XdVxh5^6F zoqwcu^w&X8aEP_(S;;^lM1q1ytHjgOo@l4O@smp^Je((9&dPg2y@Qaqm);rxz$o{H zT+B3*=4~M#Z+nt+J(qFa!pto^)nJNbUxY{34myZT>RfdEg_v~^)WY06W7Ug~kl`o( zo6Jq_vu11b9j{z@edp`=h9gEJ3LtMKwZ3bA+Z4SMPz8iT&ep0l9?N03*uTho3Q)Uq zP6WXcXpxeo9vY(YikE>*<~V9nu76(x!ivPrMqrsok`mko>WgsB5G`xYpKV?z#oFCH z6Q2T{-P1yS|EXF2Aeg_JVDa@^q;F7q>@Nu?KqsZai6xOupZoqPh(d1Nq{kx3IZefs zRJApdDcV1(P6vsH%5@(bgW>$%Ih*oG^wO1;n{T>ieVv65{;9BD-T*GteAx>imEiXT zj6hzE!>jHK6Qg34Iz3$D*#$SvSk;!9?4dp-4$%BF6=!mKthw$c=}92F znP%<|GF7_HNQ%Yt#RX36yyp`(qj2;H{PbtdDgnR&6q&gMc0Z#FkS|n{oAeXd;t4!L zfk@uJ)JySdg7!9bJrPZD#4_#Z$tG2i@A9r<;sb^9!DosYx^>`Z1l1hdUncpBf99Ou z_ShLnXi<54Y;dw1LB)scl)r7iC>6WZSw;uDRYI6YJ4|Yg6P!%5eZ^YlU-SqHC_P=c zZXetsiN*$im-*E@)vb5DnEe{ybR>3u+RkGQ=r@ECr*8R36JCLW4bu$Aw)KJB?+EE1>wiQ!;@vR$% zlG=FuyOu3Cq(6l1mwur4#%Yu`TlJ#mKPh(h_?fXdQ2}K80`_Fy0;Nb_QoA_{)1Z6; zK5|yS#H%8{3zfcJ{%U!OH+F}f%HAp&uk04 z-_N|(gES+Ls~&9<_l8=|IuDI(KCF6njPLdGbDPrp;oLoX6#OJ>YJ8fNYB=CA5X^@= zs*Hv{No}eRGb_B)mzJ>E>>=RxpXyRS*{Yd8MpFC!fypBO#AF`cEy(q(>W_E)vD2tW z)#tH=)AVp+`c_P#gQj?~tHrLy3BuAT+d7i>{~0$MjyFzAgcGy8GntU*wGlSePdHyYj#fjwzIL{WpgL;erP?LD2|AzEGdoBe~ z7er0UQ6?;J`jhpY)lqQ6Dn!1GErimI&Mhh6_(5UX(?eoJ)6ULfn_kh4AMp?U63v1x zd)Q}9)<}mnp|^rkU*l81n)#gQa^*%xh86@yQ<6^`q#qWW*01I4KKk4_=qhUGhRp9L zhED<{%7Q2QdK$%*8v%?NszT7YEA|SjjVsfl3@$Ls{iV8Nmw{CEwscP8Cl%y@)D9;Y zypTZAfj3E>MyGg#GjsanTH()}BW?IjYr4eI?lZmTI9Wl+T=#dlcjCzV$D3;ny^bsI zSJ@eI4F_!+7&os$Q#e9xetv!B%Kv2H(LW;z27y5`pvY`|(R(2d2E1##3B^?0$9#E; z$xj*%&P(M8mozHSXAend9arr4&*=i!1PZJ88fX32>1rPYp6GbcJgmftd-1e*JM- zR?7I^m8EVW10M9h6Wa%lWYBOMprOh6-2U%^}y% zHpw*1dpE)D73^l&of!?=(EUM`R&~~UNh65C?jK{__66ul?*NLn(#yQq^iD{Yewk89 z_b3p)!~E;P6`h=MGsm%Nr5|z3{avJv$cQ+aKe~P8a>1magMI$D48g%|*j8g`uR*_J z6C@#G5yVW%ZQj8U`4}IQ-7l!eay7sY0|HdP;?tWI1}B4DCdxJMGObD4D*LzjZeTb{y&y6U0HJ51j7COI)kN5moy}E@%MEX?BL@^#NU{Do9T8H? z0f2WW6jwh6Aw4B*JodLOg-n|Y^X3m8XP~tUQ;aJuP@lOwKau`DBuRQFOlO7qO#TLt zm?dB0Mo6T{K~{)$SIv(+=DsV48MLb@HP2Gb`wIO%CV&d}#9VXxOok1xwh$ZA@2Pj9 zb$d5*1dGK=Bi}<;6PPdY`>wsS5vEA^2Uz^mH36Ozbp>E57Msb-t&2qvOTmuEDjIiT zJoSNRvQahBm;cGvGHE~&RZx)&w7w@RHeAoBKBR`5T zQ^o!!eYXL=H#mqx9~1vdC<+YnN=8KDE?ITP>)KbJ&Z}NKY)_XSX#oAkkY8~DNZ^`9 z@fr(Ql<-F(?S#w2iAU1fpT0{^c>S#H050Ym(^sY7G#>8UBI-+`RIFEbn$_%TDj*;v6 zO7A&s+ahij8P${-jRIuLaGuh8`O(ghzaf&O0~r9uCVitW@ENgui=cve_j!4f?*r&^ zpx?|V6eIsC4f55;+?pC0!r*VVTOKOuq6YW1K4=52`Qd8EN%`S6sef)t zfw`&Qn}fT+67;k8)fgKKlFqF+a+tNI-YlBRgaQoL?YK+nA|8rA;#g4lmL2g5z`zdt zt$y=;(M<0V%;N z_8#lsW6~w$#-`Wxq>jdTu(>R~5!@YNk8Q8ZnFFSP7TDzkQVH7obUpYcygpQ@rIW9E z8wHqTSo7rH zw(hZ5_oS}%Wqp~x_VnT3lOS2gxrIftULSQCV}vtz)oUgOJ0N%COdJLD*#sTDnDI8U zp)8(S=TioB@^`yy&K`OphAMi+*M(;Ro17K~DZgWA1bUMaKt~br>_tdBjYME7DJbOD z>0eF)dk#2pL4UDL5ww?bV6&*cgzyvu7(#eNj=A@_gGKN z7vbBOol1qYHQqK``1)ET=)BPymb?rDy4LBv^9DS~q`FI8j3FUP>m#L>J#D{b0$*=p zf=3?9>r0s6Ar(SnFknBuaqf$((-c`3y{w-1)Kn$4fiI#$$JHPb*I`VW|A6v`&G#6kw)WKwqE1-D!{T^9L13(UUs2B2!p1AqD4#IALFj zK?VM>^wYmBt_6a-Alxe-ioT=Z9P_fejSy?E5dmDa6vv&exN`9NrsIdV805o-Ra1pD z3kIYfKr$}G^O(3N7JCGX=al|n|1%dlvl@i$z5Kt>CS-PX3^NjoYk2iN-XaeS&E`9IC-+}il6|fW^KNK&>N-KcAMh7xR z{3H#(LJ%yOyn01%y$NBlOy?raC90zgmh(~vKqj`=9@#%>=rVwk=`SQ7`1B;BxNA+v zQOz0sa+HG~hHBOWb&?GO0g}@0UmSzl^4zr49@vU&JzB&%bc5st) zj)7GF<``z~0aPco-zx#kJFfEdKD?$JG?R}vskH}@)Plpc?LrQ{~!6$|FI$O{b@L+V|Cjtw(T%-`!LY6Gkvv0>^S z=0W6FhM)!-#UGHGm#y7e^6aY%9d1i(O1Mv%lRo-QGSa33nGVmoJNW`$(1hftd&@`! z^2op1UGAgvBL=Wx+4L@+z~8gSM0)@qq=XdGV%D9$QZk(;8B;Ge?RQy-cpPM>`{mb& zpDi@>;g0_{aC#C;0Lo`ay^FYb2m`|#hB_#DG)fENWun=Fq_+DhC~F+1I$CN}!=OoI z`8z4%wB84vm8$-$3<>we;uD@LpauFCL3~V#Ns=#E*tL-#UbycJ3>QdH^8bbjiC2Jb z>M1t8KW62qaj1rURN2JW3*8Ij*4MXcil)}5_fc=jX8ImX0E_f%VqYmkylc{!dSs;O zO}PFd@_Ygw+R|h5+@D)tFV@UcYbKspJ((j1>OoA+=+qUJd|s%PZwCnac)`o@97ej+ z^&VOn%^oitT7De_*GMK7Ka|W@%kb>fDSKkTZ#T3}^M&x&_TB*~gqJtw`7nP4>k+0l zxEOdC5)VcoCxO7hLTr;&Q}iCyQ9V#l%Tew)DUiUXr>X}vA-ZV;qy)=!q~H%w#gyZ< z;Fcrybl~K@Xf~o&Y*b4;18}ME_%J8e`vd zi|H?8eTfkzKFusAz^quy2@cTK`{2xsR3EJ8`YtgSI)gaO62FD4^M;h7kUCGCH98YH zrXfM#3W=w!IHgbOG)oPu+*k@9%+5iax4tx#7}fLy$YcUY1d~JnrO=c#@okY@KxH&l zMaB2gY_24;?|3a0F#^71K5JbdUt!dCUZLkQudlBHa?J=j z3IExOB)!uFfX6E3vmYsO;{EJDR%QRbfn&+kH1H@-xwVD8&TZLEjt0>1F+bxG|K)Tf zih#%6yDo5n`Mnz?_iBm@B-7c8rtOm8K_@u}uzusnZ%6n265VHJ+$SjTaJssieag^F zKjIMJV_+BqfEu(EBOR4SfAL#%zSl(*G}jF~q7jk{Hh{UzKR+UM5C+4e00rd=%$>Gk6-=e8_pM|sPmK2;G?R8#VC*Cq|{!CmywZ(yDH4~BvCiXww zX_q3l{#E^j_yGiQEer9(ivZbZkZCsKaJqp9+&GXiw2ID=c-l;NljHt0_PIfg(GM|>{4Eq>&L9|swUuxCliZTcBF zjO!oB5QzVZr6kpWH{-c0L)A#FtWh z7Ow#zAl}q#G&$#!pUq`k4O`8FfGY1<`eN^Ut_nF%~`6gxj$bQ-pL&6=E~S(_ ziVIPM|BxaM7}7$^8BmS^Jjcft!Qz^FD1M{MYLVj2?tJJs4;}bbTXyn0?wGU%gu!wZ z0LD3rsWxJQavL^mOSr%;8qW?rPu7txTuL6jr}g@g562(h|D*T>VT*XcpPSh8$k+!> z6h3Qy%^#poJ?x$JfcICp1yWin#TErbs;%7D{fL zsng8I`oD42LN1_}UwBq?{XzjR>5UVrtWOVn$_FHg*)kK8xMM-Rg|GeS^Mmcm=zkuh z4M3N0JB_sqd>XEU9jWKo580Gk9Z0ZeHEIkdkJ`*uUe#(Zx@GgbmLs7BAm6-3_Ek(m zl1wE(bR3-p=#U^ZSW~zncXWXukoDLFA#|JX^Iy9W9Dot0t%DgZ0v5jX+}G8;qJD>R z0f))FC51Q}czCsMp5_m3#ZMOr2D7~3Y1qoxw;rBE>s|z0nU9j++za=o_h*Xqnx3Bs z1uI;>npVpl(fi2b_mXuH0OcQiqg1p427>`k{>*X%CtA`Q_pj<_!?1N^JEPV#)C zlJX{1r?-Z@WdTH#5UaM4PV;6K*6)&~Zkb9phjLC`*TjAV3Lyb720m7++^-$4=Pxci z%5c}^o&B0pZ&t-RvYr>$a~GTa_wyadVlG!-ozCwEz9|b{%F_I7P4^3f3 z(=36}p(%yDyx352Q^WO@t1irVG?_>VbfG4HgHYThPc49!zpIrtN<$`*6+3YA-Xi~I z)*XEB7cE3+Hj$H2wVA#f7wA!@7;`Kl;eE$NOg68O{3z#U-}(FwtClG28jWxv%le`1 z50sGwSibUa3XdIVbD6oWkcnV6tmO=sH&3l}%X+^bGMqlr);YSb7#BjtW7&U3oQs3` z1|}lSVeoTfiL}i%%sZHZ{TK741`%`z^vWK$gmj`$zx@Qd4l|?rhQF2iQ;&mjT8z!c zqArlBqJKEcMDs?E7=Qc@h+C^X0@#H z^bE56#eHQBRMoG=iiI^+)m8qM5N~5NQ0nEs@V?|B4lP@}o)Mf_;|Bw=?QGT}>$>u_ z+WoMMFJ9bvzuXmonzsN4Cz0MxCBN_}&nwSeE*S>-u^AOS5fiZ(l9e`y>6ASur4V3*is|^L1!#NM* zN5AThg?EzEc53#>F7b|3@hV-%)9z%7%wo?iGz;e|`M>=R013cKyoAxP>9@I9yD{cBca?lGgfo9SS6&4L zXG83zC@}4ne=-AYJf>h}>h`%Xr4kDH#0RgNX$B39Awzx9keZ>cJn7=?Fl9*buCC1tQxNihgjM1r>t~Il6pSE>|JU)DV;V;08 zOeF9uo2Pl2zZMd~v3*c{bN?(#ydy~MPmCyp4&rgoQv!n@S<2nGVuwz=Zr2p&2!1&65ix7%m{!#;Eo6=j4 zTs4%)hnW-}zQoi~6lF|I{J!C=F% z$JsX(egu!|=CRsSfQwkaH}Q*Nk^C2Vf&Z20f>)4#k6s;=;#i8imn9oeEi+6-JoGA3 zZ$UO#219&dLJ$upV0Ygdg8!n`x1o|}DK{~FGfN2Ybd2l_Hos^;=V+n&%kk(* z6aaU9`*M-{V!?lsjs4=iZw9bz@1|t<0b0F8ulr}RXoj=GB zklBnaJuaGYf5T}~PZhCP$)_3onRs(6?fziFgbEH7pViab5#T7NWhw z=?1T2x8)92&D?OsME2GDz%eQT%vmMF8mBbiT-%m7c6q>wCvt06qRY1{=Q7+i#Jnw{ z4#{rtiLuMg)zTtIq}O(NC|&WrNL}-JU~xJSLPT8-U$Qi^>75}Toy{1+Jo2rn+5SpH z!IJ90mP+yeLOF+O*uZfj9iSe2U_ZjnZ_`JwZTvSSKGWp4UNajw2$W3xUWg~%5J5F2 z#rQtpXdiI4O3fBHT7{4Y@ev2t^%K`UY4X(p$^k9Ru{;2C+5u-75sg?dG;Iw_2b*cp zn}@=4cZn?rW5mL=i>xS6p=jV3RMF^;^x0{9;0iLtG;C?%q8P_)?Ho`c%{{tsd2cgp z;2e<-ux8pHlqpfEK(Rjv99!Zy{~C^8^?7Iyz_$&vr;tm7c}gqHAn~K6IQ(HU>MJYV z$p!!f*G|ZfTwhAQQlt-keKx?$#bA5W>h|f%Gl#1Hiv!pVuqu)dt}sTykAIP2 zAgtJ6$^rC*J=8}Wk^oe6>)+qr{aii0^`%z=T>@0Bb^yDbo(;hG1fRy1P$r#-XUD$g zf|%@0r)r$tHhZ>2aPndA_-kxF*YF!o^xhxGskj03Gbi%*#DJ3(fXLUiU@!SKqKo@r zn-i|_9MCo0e;jcmT>>y+?XYXNTa6xhP@HOoWXes_wzIw|o`GHqnS1Rc5V2#;M zSrE>!*)RJt#O?ZnBk#p<3ZILVEp0I?eIxyQ`Jh^2oorF_2K8S2DD<^;7u0pEb zbz@;=n8TM2$os1Yt@>;ej36`J-GLdjqug=)p zGkTrPDbxrHY4THWJCtgo!s2^3>G4!K98Ag@RMR>NH62}Z{*r#x?aSKDwb3fe+(v|F z^LGdI*=XspO&=Z=U+#NdP6Lvn2+r=Akspigj~t2d@%u7qH>YkP$Sn$9j9u8^94U~c zv&?Wq=CE~Gn{pitKCKTd2RU;;e;=Nr_V`LdUpddEO7s>?d+DwDn&85v+hXUj`Mbm- z2ZQMW=(9Swr^-Wnd8h`Up;uFzmeT7;^zWF?U;^t}|J+{AY{-5VFr2dNTj<2ZzcnOp z>|3KXD!z&QoLMH~%6cfVw&lqS)*6RSlAIm1L%rA*#?Nd6U1&J;x#V zLW9wqeD6po)~hu8(J17)c!RM7um1S2UIt?09R9V*7wKkwBDfCJnF$#g82}ZN8%ji7 z1a!vgJvUPftL!X2!~A%lBn}gmg(u%afEZZpGT(~G!JN^&;&uCDof)9yNg0FH$sP{t zMXxB`m!62vYCQsSEr;oeQo$y%-Is#%>6|6)S&lQsRX!`v^Jj#*vpJ9H;_7TAYSCO` zPikw$Y5I&=Vt_-HtprHqA_rPCLh3&5qtxpRNpf_36-fH?>5p{dd_sSXE z->w)2oeNOxPC5O^X4u`w_t-?bkVFCWiu8GZk3#U`f-A!;t1H zrTD8Jii7#$PenPG78i5M&SZ;JYR)H`k?rd3A20Iy+zhrTwu>{}0Rzb<_R;Q=-r#ZR zgY{0^wVCFcbG`hxaQ39aYL$nNPwMEF2v`%K8;raM1&VDH2@kDJP(yHd(Avh2wV_W& z@%+nO$0UrpN3U(Pe0m+F-ND6B97yZnRT*d#O(&MnrDhfm4Tys$)(~`H`UdF zP^q~&Q8k@>MAQyMuNQ?$HK3s)9VMV2rvJ4V;LxxxaAt7Im-pHCOn_U$LqzOJbc4}Mk#I1&(%!rM*?aI8*g<}p;mRow&{z>E zBTm!9*1d1aAS=rGxc_HifS+g4$mG_mh&w20+CV<6v>Ni&~bG$M}%5HXQZQdIa& zsY;ALne4=rrgAy8BoR<<7Xwl!V7VmJu)Q!1BwRGTzaoT8(1m+Jv4lWHi!cc=~eGh;_zQ zy*vt2omeS4`}vLD)-=$zmn<}jcI?Y-DKl0783oA^=NYNk8%Khx=7C-*?PLN_)6yZ^ z!TT*o)8mDs+?L%tTU$L$dlI8i3C97-6|d8Bf(*RK`NRIZRHuP{Vn)k19j9x*oryUU zx+C}=HTOWcA8|?uTSkbRd|Rc?c+-G58o%0xcHQ)nMPZ&i6crWlQ(o;mx_*WiHF)FIt;l?QkeX!IkRO8~5Ed|42Zhyh#FG zI}>nYF?SDbA9yCAExUs5WvDMPIEog}*nQ3WtYaaDu<%Itu3%#Lh_nv?Md>*SWj0K4 zTs4A|7D^;nOdN-n(h^w>3u>ts3kO+-JD?w}qRv{64n)+<2!p@=Mes_Mw8B~_Grs?-R|W5ggxR78K<;O-v@dq z&!%{mQE92-x|Mp$B5~&q^BUzfZQK2$mvg<+h_R#MHP}vBgs5Q`$Ka<T`=fT7NLfdZf%mQ1TF^81K^_ z-Wp%t6J+Y;$lUsal*?!?ZCG&n2IHpCMfGVYCMg7}f0cP0|82|J{o zJnWv5Bh%_DW1BwN~^7!|n-NS;P#}h?_hX-e8XNSk*<0~q7-31PxfJ0Fx zHNn%%J!uW7bNJat1DHkCzqfwls8`sUfpY-0x)IA?g0gAYbLQk#OED7gG05mnF6(hz z6hQMY5dI`ZsnmPL*H2C!31&pzd z)Y^%hVBh^*Mcj(`4vr1pwdzW@k{A~BNizEZMy8{z^lHsW{bZg~;yxXSwxMvH^^>OYQf`-U zM)*(I5J`)Bp5kK#3mv=qw2~bebqFte2W z;gv7F#(T7;8TeO}a@}nQ<%;dMMlXN!<0($+uo0Qj5afG_o;s?~AT~vT6^bji3@&Aw zAK~p!w;mbR9{P#x&RGyvoWEyP6i+V6IDsC)P@B5z+2XY|#>|J?a!OR8X&pxr3U-HR&HfFq4hgTRnvo*-5Kb+HpOx9 zH5DgRVsU|5DN84*;9imN-Z5;<31!H((wq8FY+zZd#Ah*9ude={UM|_SfP2OZo{rP! zHOkWrBUPFFxSvn&EERKDgm%5vofEvhL}?nxKKL4z?AY6EJR^LJN`Nx%RZ1)fSguVT zq~N7nK&l=3pq&&x;)Yn^;N!*K;R74|t7s-FQ50 zr&nwbOX*=hw9*X}ly2;fv#Cg~`?(jCWUa<@YiN)vF`con%1V;%?7W>&r@rPB0=k`< zqQAF$68kgnm}{i+VJqC$v&k@eBj!*=Zw^h6bRQ2^rk|^36_9(mDr4BW`2A`zF1X|- zQqWet_wB)(ebTd;p)1e_y5jbVlJn0WZ|a@pRl;2k%E_@G1W9rk9Tg+(8j?LDlmbuM z0*Nw?78@VOjj$#`-&AFq)*i=hSVCFMMu@qgj1TToJ8^mBr{&l6vmcJGA_Wdeh92QY zjHzul7)5Tjd47Fq!1WYSV`{T$T3mV!J&c>BT#s8?cXkue+`Cb6+8LAlnX{^h`U zdr+`wPg^XZW}+9z`uOU4SqDm@#9+k!I4aCXWidjhvt*PaX6IUT6_v@I)1++exjedyTfhUtYRhLk+`+R5i-k)x%Bb$ zlIcFeDC8&8rW)%N0l%WRKF9Vmt25R163!Y$q9%ig2Nd@;3{Ofyt*-WB(R#jDKaJvV zo5V|cHlytlKdAV0ra(+IKD?lOzo}#Y85TG)<-PtszDs@Ilp@f&)M#MVao(&v{UKW;c|+*51?Hq@J>p@2D`eZd+DpWkfR-?pmP(*C^S-F7Cp~m zSNOQ8edu_2bWv0JWMYB{V*d=rS$~G`bKm_cG0|Y(uqJ5lSQ_c#!EqbhUq8&o9x1q4 zXqcDX*bLqJuHaksIG{MaEq*K|uzD3L(YQ{Ybu9j5KC+Ue!zHMPQ$+EbL|Tlnf=YUC`W-?X4-Txp_b^IVhJ>?t|NgGk8_ zs&TSjI<=;Lo=&4zdbNiGh;^&bo7~RtxYLOx3_4X`pfQccM+Wp=5i+w+yv5^@1%9cx z%5q?UgHucqN{+CM6&)4tdEK^qdCyR&!YpmoM)pb5EV#jEWk;a+3*`T4?<=FC+}^i` z0Tcs45IKT`B2r3m;dE{Rg#dQlXm5(3>2FXN&%uY0VsK!%aFVNO; z=@uCBYrPP=!O6ccChFxd6Yf=?zoGx7b-CJ%x&yr{qW<H^FjB~|SI>Vw zD}s;&P}}c$XpNCU#3*r14d{wbp7wuHtnA@J?I5hoGETpvoUOV?*iOY&&gU7F82jE2 zVLLbW;KNKd`n56BwVkopF!vP)EiLr=_$!92UBeLT=&nRjmE63i$I$wL&n*!=Cx#1- zQ$r>>0P1i{CV&f)($_aYtfgeNCdci0S1wdAW`8Gx(l7*h4*Aadv5}U1w^B_;#N3q3 zo7&aQ(;Z2#FnKz+-jDq*9b*F(aK&}j4=!FSX*7R#YKVAhNtO&IF$veS1?Q1BFBR<+ z+U=NsTw~2St-TuAR3h96CUs*P!fCp_ zs_DiKasjry01Xp~rg0B`8^A%`Qzgsp%b#c0llShlqMf|ae#xNYtpG>>ZB82D{ANpj zvg}Ut^Q9o#BTjW;&pZ3XRbqA?yYTgef&SK!h9}G zj?at6_P1Cg$lK1eN3)9IlCHZ_P~oL8 zD2NRye%76?)7j}X_8VL%W2r{W-(%{I**?45d9-@%Y|*H2M*La)R_Jxyww!WiU2Bd` zVXE3V6AIh0e=6<_RVH!lii$X0RnW9b+Zw@UpvtZicWuOVik_Eq@6)z*tFN!=XxCM4 z}XA>Mg0--E`^=+uo60PVjVHzl^8eaE+Cx3Y4bzD42P?(39uYB!vwB z<^Vzn+n%~|-?qB(?ogylgu3eL)n*>aBP|%jy|LL`rxtFZqkK7|Y3UiN%(*1@$Ez)Q zXY7sxOV%*ssY=v`Ema_$s4EJ`NRlah@aW1NpXJPvTL170`$n-#g5HP(+ljTo8|nDz zJ-r>%oio!NNlHggG2N}yH6AjuX!GjlXFC~{xAmpVYIfOfb9H2JA!mt+ZV8K?niY1` za(EgXY;ZG(#uicZ6f^#Pc?njfhJA)96X(2KpD^s4;~m_9;3Rd&t%TcESDy-e!s@7$ z;c_e`#_e3C{rG#eiRAYa(O7fJa}EwH?XeumaSa2#2EpaII@Ujm#%tK&ZpevRgN?Yd zzErir)g=5*$UI^_&_OHrrck1|w=Ap+uc7dcz2Et&_&v5`ACB_Xi{SjN+FKg0{c2Wt z(`E`SLe@~;T|2||!Pu{}8!d&u2lqEA_=Q-{o}z_?xr_z+dS6@GQ1zNyWzetwZROyY zp@bKSxM-rKxpAXA3u=(`dJyNv)s#_l%XPBP!SPZU(ue@k!fikJ`xG*Ac3OEr*iXZxERl-_6 zuOxMU_=MLW$IVo{<8J5!oT*4WYX;LAf7@%Asbx1;!rlaTCx53cLNw%ZD03PITtqAR z4Xs6nMk37px~-FdAFo#4Z`XF}Un&CU^EO>>$MOkbBp0G2t?b9Nr~Ke{kJA?<&4b2n znlyN%%=m1+S&*WyWwlNDz;1u9%g_^@JJf?mSPCq1U)i$^WAa#u_;xZIg=u-ah6 zWb6UL^m)y>W;n}6j)7ZijG~MIEKdWUArzLWW31^qcWs5byw6NO`1KeaD;B>j>=MaMrq7t zyUEZe*6X(n99l>7=!$-4q9ndxzG3|FSlxJ>er`I9uV?D?gGOO;Q$(8I#|cnVBiZUR>kfRX*p?yS?JPs=#ZEdGeH91Hf&_-jk znugwsxit%W$;3kN_9iE&%{R(9=Luo}GT-?k;oxf-of5?&&6^HoDto30m+M%3Z(!>! zcx-2W%)hZuUhDOcG*0(QyRYg;$a}phNj1Am+#@i4bY&T=mjDr@KEkFy?Y>=rHTQXG zRvgru>ipokXGp%Q1aAxMG;D0U=f3bYySDf@PC}{B<(f>5wEGQP)S8e!`uQEP(8Wmf z`*PV1q3(vPglhCi;%Zi5kB}y}MDBvswSyJAB=h4X zF3L=`#PuW*_>~w}ijJttru>|-qH3q#T~_(*)aXWoamM@KhwJa>zKps;+Z>BO1=_iK zPg(A{Kam8g03qQ39Z-``xNR;vzJpp$H3%VkI@BGg%9B21_D>jy+)z2IjDhrM8aco}MxDa9O;yuzG`zfEn+`+HQPrPaPZ5&2QJ{prbhHT(0 zoc(h%>>}`Z2WX72!?P%SgO;T|u4V)7^7HpGxV@!0T;QXW;ybZWH{@{2Dk`izd4_Kr zN$)y8`cbuJBvshO|LM`l4*dl9oko$3PduTKeC~dWDx@;`d)K+s%c*s!;1{Hyd3fmu zp(wUwCs9nhW!7K3uq959&tfZ#M=|I7*f5bgSlgi_cVv8mLqeMaBFJN+Gtov;#yBl- zY-B_vCS`Rn*~Z5eU$7>4qE|RyPmfAsElp={=NNo1D2P?d?=N14ci~07&BTP- zzDlHYC9vntzP;zmXHIFsB3>0;VxZRLuHR@tt4H!pwU8=wuzq)Qsc-j}-1kPzMMQer zFJX&kKk(I-=?#hKSlrJ2bnF93yPEYK$DfRdGXdm?8j{eZF1I|v5?*d|%25&5E;H9P z718EAv(>^)X3GiVENg{&;j&6eTl0Pse@>-}^E-Cm)VNKt;*b0zTbJlI=>5c7^+TqE zgqL8Eo695iYV}_E-Hr~Ms8AN0t6h~JaSxZflZ3+K20WUMUn#qbwd9-s^whC)XUoWY z*-ryr&(QxuWa!J_?j){W@xEw`G#MkIEFoYQYNo?+ap#^Prd{S(kXBP3XCoj-Wmid1 z{!%h=F0K5PQF%ws*6o&c&EzgTHjr^c6>VGhiz6&IV2~?(K6|_PmmJbS(4wx!+<9PE zjgZ{k@2tHN)A+jMgP*BL;VpHXm?_UC}FB6~kY?WER& znJ#hZc5M&w9l4Xvb*GN3Bgkg)ILHm-K2i6FBhskGx0&G^eX|YKC|+MJxSL zywjyKTdnJ;#~do7Lb}H6O3c2!dfmc9yY^>KT}#B*ksmh_a)bvR)6SK=T6)85pp#6X zn=DPKSA!KfHioxKr**{rQa8dEZ9d0~u-E}~A!c#MG@GtWp})Ad1BR1nYdY)A_1g&O zhc_nV49JThXxXoy`Fe~RLQ#D=^V9xEj3$?=9S3cM3?EL0^_;`VmRtcCVe}k6vvsd}7j0U)*s*o>;qXIn@Y? z0WJr`-MzCv7wVNthy%$=<&EwAnrLdZKy81!e$A^#;e~W${JDGG@jr}YPRx{O#tRme zS0Kf*+`6%qD<%@9Q`Zl^fkFd>gEvQtw1`!(tW{)|zO~BT4PvrZ`MMCPy9G}yLl^Uu z&-Hon#7cm^fy4oobN!jC%=eETB#+e3Ih)P|JZD%sXkiI}-sbQ#>)sYg#Lu`g_XT0R z*ojCuA&$tFKZuHVcd9`l%sK-%vP0d ze?_I6xWM#OzrE!pm=g$e^~^my#$FpPAP)?)M$%F&00YOgGdyq}87+3i-vIwKX@k$~ zUE1GTYH8@i75#iKq};;qd1r3jn%4~pJ=(*i4^F()dBfs2qn{6DY#mvu!@Ahv~)#zm0+J8qCM-v4Ev-IsC12>rLOK-P@O)`ZhZ{}mG zhuU2SDt&!*)@Nt;``JjydLj+-Z)Iiz)}xkd`Kc zYyseasDk8yGCVj9WOVTS94u42###!Vj<;`%7YaGSenIJv@X(Q*B{>-xm61ezDGA%k zcn8ade7_%75EUzmJn z_fDUG$1t3qO~ZbXnZ;B^=F$GZDHI73I-I9od9=#h1ttF*+f_qa)wi;nIM1i= zPZtvS`ZOzqT2MtrhA25dsJHBpX9ul?@9k3IQ!j2iTaW)<=1zcRGRj;~`!oATB$t_< zZBao6iQN*8s*|PLO6$_i#%{Ctl;f2)!yOk~aEZSzTO9m;RUi;wM#v)uqUr>0QJg#v z)7%&d9jIP8VQbd8f#0kjiWKmT+q(2c^3Ns)sFB4$CuwDA_6xfwbEG}CWF#ki6EZrt zWnO&xbnU~U^TPK%E{{WtGDJ=RsTwCcL)=Bydt|4WBvaz@pIOhxbt=Y-J(coIO-;)4 z(Jj&n_xiJ1Py%+>#Q6H>zVtq8N{)BW9tn9|<~(6G-5x9B%;Bn;{b3uL>KO4y6{(AW zaIM_mw|-43$9>9I+=b`(36!8il9lwmOUH~Ng`Icel_N(FG!a5FalDA)y@R9zC2=1= zleN^r)YQt)KXk`2>NJ|1$lOdtvzWt3we~BJ1D{+3UF%Q!G$QuVy!T{)1RyVZR~eC} zM$g&7t(RQkI#_E|AHdD_bboQcj4uKx@OjopseNrProQ3#vuvU4TmefoHap_myLqgc zofhWBuK6R5=!3~apv%B?J9Z}}5XVXLzG*^yb2tQRH(qQaTJoSLnzknU;ry**MVHU~ z!OL!`1VE&-%5Q8CnWUaqHhr>K%;UhbxpXj)IC`({#fcZ!0KNCeanF^)w5zpNGbY9S$z8%hfA^M;pMmhi>VHB^QQIwk-*7@9A+~hq`ZmVfR zMlEj&&FoIle0<8mdi>KwJO?!TSl$rB){ zudOY)i52A3x`9T!r*geQKI+%nWC~izr|SI@%hjlU`$*;4;-B3O3^+}(J2V2#xOV2i zq(i9hHCUF~mN9&IET@)r@1tRyNOrA#XS5_lQTFs7aShx$!5=7qbNjLXrYF$;*{g`h zp~jWs`0yXP`bscH=9wlzD&>8 ze;v%9b+Q8Zv8X>25e~RY^|xe;K)|7=1Ziim6{AT)U=uQ1HzmbN3`zG6;v3 z!oINHq}VGq3oh@&vm&J&waA^U27f4wX_YvRlc~EMCr*{1IiIzGAQ^tC+HXPUB%9e= ziDz&c9-nFE`6W3!t}61(PzE&IuruJ}foCDgibBwU9gc1KhFz^q9J zKp?RF<-W;RWHV~4?fvL=#gf?j1U(_!uFv*0ixmcs8^*QvJ6C}2Kj#G4^oJw6#2vcH zvC{muJg48X3McBS;~vFnn!i3_1gPAO*LZ1M`16WMS>Rwj*e-Mt_wdraz+)`_xvg;+ zASUIxQA2$FNBg$bit-gc9*E70#?qw?C+*-+_ z@b*1Wz;c8`odrR>Z!v8AOdqiJ9mwwPk& zJ(`GbU2Zf|vO1Z8_E)*OkIw?+CyPM6qGgceMOdzztSjI;hJ}sZeqqx+<=X3Z46uS; z$#||uKR@5RE1t7&!htky=>Xj2k9lrV1L1z~V)amq?Tko!#HB|UL+7{a^hNwS)T-Z6 z{+Y|`72pLf)m={oEKUK^Ur#KFdngO*Q}(vOD!F~*w04#{bvNX{2wp%z31*D=CPzsy z{h|J;^MoHrHyz8v!Q*o(&*#Vnyi@hx~i;pu*IMk7gz@k6@{P8GZg#)GO{t{a~0c4mzQIx*G zQcV{TgfY>bd1XUdUKMfJ`bBYrBm+Wb`0g$%@k7r6t)8J)ND}S(M|v2S`n{MV?zU;X zSK%^sPr%8>!HWX#0ekzLoMzvZRIxe!{;n`sk+g#cxj*P-FR)29Ru2tsuh#s_>5y}f zhLBs3jR%}z0g6DO6G&;WnWZ0cbEIPTy^~dUS~YF$;wBAwPZXRpYNjJaV*>z+kYfn?!Ty3{L@pcBGC@BkGYkpX2tLMftpt9v>z#uU*Hg z&T3I?$3>YVUHQ#k&5M|>-KQt$3%olwRHZ=wF`*AyHCZYjuoh$is+i4b&^)Xi&1+QP z&NF+sWMu3>_iqZ*KP9Y4kme*{Q{U9YKk9E#Uwfc%&F+Es2C7He43UEu83A+xg@E#4 zY_8<5ZWeJjWkOGeII1ocG|Xg!zh2Xky5@CJj}PREZ~|#c8SrF0D1xxG@{`a}H>M)` z-O(b~!V*ix%wgJ^g(UHpJLx)dVdvH3uG!RiTC_*0%!Cl55yGNT$P&5;?&oT7fE{KH zq}W><^@=R)*UmtjAejoVmQe6WnnG8gI^RI2}+CfH9D8fvsYVXZJWk$IM&+7K_}0Jo_9}idsHa zFdfLv?BfWi&CDo8ou^#BSf z9#a+rH>Z>)@5OryT6I0-JAcD(5AcnP@K$_tWcp(&CUbyG)!qI^;BBS28Yl$3;;?S( zTnIT2(H?=YUw@!*;jf@`eSoM6DL)TX_MSIpIn-rImId6?k`6jaYyersxDsge;1#M5 zINWJ@PAD0KMp21ePn4EJCG9+W|53gCtW!`C4#)XD3#RWLYcn**CTlUwH*yw0Dh=?C zwiw=W%Z80O=9^-!JLWvHGB|wx)>F(9Unw9|M-H$9Y1W$* zTYjG`g0ikZC3=aK=%$WsCGG5+e6?tk>ODLC>&``L;`kS|90rK06(sq?c~OTMg+89SZA0*YaWv+DEE5^3hCc{`&&u=ES!3GcFeh@k79}NYH=Gzo|SUTTUN?zjJfHLc>J zw(Jaig}o#%qNF%}^TMR(2?!iQCgZGejOBy}0z&bu67nS2WZ;Bk3w$9uB!csba&wJD zu4?ztO7VC9Z^-Y(&Bn2?pXagot~TZi1a=c|=0;CiX`f1PKHZ)M&g0Gkwtx!_-y8}1 zM6|n;FToqCKB;C9yw=L`^HN^7giRQ7*mFg=RC3Pj{0=>pv)MBirt`g{w{hI^ES5(r zt7n7v#h|FrSW_NP(|+C#hJNV|aDT^&2RR8~6ZLzjzx{an!qp=-t>evY@xRUz@RjtC zX{w$+4FYMyzAY~UPWFk8x26I3Ns~a}Hmp#iev{30p~F+Vs=mmwaZG#I-Yll>{^X^@ z75c0WFt1+}E&FUwz2q%^E0G(&{fox79q#6TIID`3n}vzd%Kb6RB)TA6n&gJX65@qp z`*6gl!^_J)Yl)xoa@K7Obn8b4mA*%QSU2|>go z`~mfVcgctT;kO657-8KcK!Ej`HjK!NeE01466eO8`Wy1`6(~(NcoH6DS~-U2bXuvV zlaDR@VS)1-;g&wL5Jl=JvA;oXJ}eVnxq9I{fhAOvs=HVCtl` zzMKR*LVXvC>ew<4zlManNHHy|p(E{Y#Nu4% zr+!`tT>n+ajbw2{7Byw&;_AahfuRJHG{Xp3Ed(gN?BB z98ezImH{EW0+@4_bznlANOkS;>Gr92WeSG_lWxL8mB-f`MMa4V#mR{(^d-b&m?;7e1GN(rJD!koq_8dpP0yR7{WvEMuc~GtA1(I#d={}g z7f3KG^+Q4qR&h36YJX3=f=%wC3x2JCtNWt(Fn<|VVnAE&pno+f`w;>neR03Jww0u3u<^PKza{ZT_@BCiy zTsL&^fPplyhZ+FT)@4HF1p8VC1@VZ+;`qC3)1@qDESZ>kQUblo3Vum;6RSA(XYEK( zk}kJ&9l{d4g0|vB)5Yv$t*6#tk%FFrzBlR;7bN`DUJW@N=0?&0fdzZ6>#e~^kj6M@ zd?iY=oc`!MDseYb2rdhvz1(G9?YX!i=R+vw-=ATX4P=QfC{5a<8d`m4=$iJeqt>lU3BVnKZMuwzM;J35;?#jXr@`Fk%7lf% z4S3YFRp7MSm}xC=+`}Qs2+e!d)^;T~lhAj>191zfOM!~S7~LQc^?)QV(VHrpfI3uqM%#aj z(E-^M0&Agk>HE%I$G`V)0tDfNqXL2ukX$Ho*|%!h6jeM;7Xv^88G(XvQYbKd_KL=IOCd<6vAz95#ABOpRK2mj(wl9s6!zU6?|3Wri z)w4*hT=ZFUc{Ow?;xSwoo9$=xY+)(3RKMCK@`wv##>&cxW9eIuxM!=;Q#nI^X=B@J z37l`n7qCI&3+Ri9_#0eK0t5{t2qZNFfqNv>EhB1`I%{p~@kohHeAm6iuVwT7Eo@J} zk7<;E+bsODqjMy&F)yRJb4I9y2Q<4SQXYzyj2x>&WY+JvGSk$4E^r=HVlpY}Hs;AM z!HjdEEuiGHT}hfFE;a@4^na)sRXM5`mM_>G)mo$LNafEh_39Pd^ckIvXfz$K;_yfm zSEgpa-EvYfQY=zC*OWTn(4)Kwgq=2E z<-T_RA%}K7eiP5#%=~9UU{ppx!h43$g`bwo>|v(br{lRyL{?yUz&pNTg>Wx{2aytzP{1d z?dpiXV7CJ=ZK)H#g0RU{j_mWYGuCkRDUMIF>y>to;-Ovi#(GZ0ZH#KyPqQLL9hJ$^ z4@&8Gd5k$LVIk7Se;cJ^$^hko!uek^dK0?v*59v+6=B}L)l>8Dl$htmigCg>uZ-Vu zdpMSTppmiE{fNy$xlzyRR<{e5f6mY7__8UZzz(QyF-3 zTW+I!mDX$PTU(CKt^kTWBNd);{K?a&Tt66SlCE0mxR&(^BD(e7GJw!roYfx~ica=R7Yl{Ay`uRGu}kUwKw5*x^tFVillK$p zjE-|i2}GV@uEPECq13@#>%leQ%?-=6&pcxW-t8+`-4lZ;t?dYk0B{D~$FVTdnA(@7 zf|O*5Pt4KTp_0;9U{T$>XW}EM%jfE2EQ=`yr`Kr1W#4BmaeMReZwe-g>{IC?vQlZ%Lxx+$ zE0@#dlCmBiy~t}^mt}9aWBoZTaLpwGZ&+$fS{q)pfZ)*(6G5B(oO6#9az9ls?p&AH zkwO}btO@gV_^v|Euw0Z>Yur38pmpF||6swQ(q{1}ir3C{sef4W z%ud*n8H>0354oLR7}xRlA;&JoNIxG=@-1#~pOB&s#tT_^rr6OkiF+B+=Gl;4er_ri zU*Y7;czynaE2DslZeD=SJAEqKz5pFT3yZ%EximoE$RPHYK5vt8na(@{Wx{f!l}km9 z5|R9B2_l6P=H2BcYiK9Q6?qwl*$Kn@QWI`+$GWFCFv^w>C#mLEU;o*#Z8*|4)DPN`IukaVkA`!)HyG+y zde!$CB>rTFT@}i#8%1=EF7=*Nc)K+Nb1s#CpNV!56aW_+N%|y5@#?1pGkZ71I~clu zK)7PUBzFdL+QIKf&PTlxD?*v3UD8aGPuWpf^auwX6A{X%;dQ7>g$%wTp@<-)kN#sY}Y57P1vOxQ9!rYhlR4j`9kg{|k%N#<_tKB@)ou$zJ%>GPs7aH7-l>_jut2=h}1RN?{ zwoEraXWEE|>Kq}y-?vPclu3+9WB5lyH72B+T-t;lDDnv<;KEAM>rK8Wp4It@hn?ng zM;QsKt$=zoK9eXrCFc5M0E`mviqo$}bDWl6W z-Pfey=y+a^gVst3VrjUM;Hnc5PnDbf#)T1^p`_Ch^j&c+WmV{Axzd z!>e1CXSc-dJ28Gm3s|?HMuU(J?8=53LMv!&vu^zo;0_xIs4TOb78^rLdyZ*lCrF<0 zy)#klm155rS?E#OLA#{Ckvqz?8lyyqNu7ywYxg(G)??{bM?O=t8GH#VYto;yADs{% zfXiEX%X{I~B?lWk>Ae-3@@3ZG?JahSZ@8b^eld#S=;y2uO=;RUlJ^qUS`})wt z&hPr}rKjf_<1wjrW+NZ3I9Uggvgn<_CQ3X~q~n>m&$mbKKK0c-=FJ7mk&ivASRV%I z7YRyBFpEa^$4(EvWxYbc&&?_?Wt`V5(;H(E&h1BKP_zQP!o?XZNyMgBw=jy3*Neq~3t!VTGHF(W*-oN`>;*QLYhR_ zIq1j|=&N)fP1_0I{*V#@bmI)!JyO;)`mth!4CHTrPVuA;M5(Xwry=|EJ)AlC07%qt z@n{}!yB#jbYjD0wy#|9q{tiKj|9cVyzUZ#C#lk->VkE|CYfyz-#t?u@{5+?)G5GwtLfrSRmE2lX=D*9x*h?B>;u#ty-3P|LoyeI+TF7%QvIOmx{y zWw|1}KDD8L5h)S;YtOeWo+T?R(&5=FoTu=EgcDyW|1l%7KoH>uy+56O>(iMkm-)Ar zK3h+h3!NOwjxUWAw~C4Ij6C8v^=eLB2cyMZSdnP3FvY>cW3dqNrlgu3etEhpUZ_u? zJ2-A5hhX2uoOkkO@76WI)JtL6Z$)aa#os$JyhC!SlEttr#KwpptrCqA=C3L=4?VzyS_#4Ga$ zr=I&;N(OR`)BN9M0IEjt2mPM>f8tsN}*4tUkV#Xw_{EpwandIzh6^oc+MNP=Tjw1>7g{^{f6f0K#`}> zu7@EGHHo{;FmCng)2B~ME+u*-;bD@aoEAA>Q&0YUdwJ-ek|Mna#Nl&yXf1V>4Y&%t zixW`W_x5>CzIoYZL0N55WBn93X1_9YvFHjK6H;#9Y2r0o`e>8lalekSQM&H=^7q2^_wQ5_Z=`=ioX1w#gIrXr^1AOe^FC>d zwuIgWY`5^DaM3OWv zcX0y;FMU^%R5!E0WUTS)%XYTHHiLJaEq&;sAJ{L9xaJP=__ny{o0I`I$t^cg|KoV*4rNH1cZbI5GXduy}tBV+K^1p>Ko`75x1W2ZjU5n13wsY z;hnE{&v0qpskyE>b5hKoSE5gV19nOAjAcC&Yrb&Y1>LlV%Jq0GWFR_#PTsA$sEDo>Nci8*Nh0t8THd5hPdz_9ZOSXFmJ!6*iA)HZa829iQ zCO#9Fl<(IsCRm8^KJX!aJInNgIVNMEShYaF#$mW>W5N8fKdK1jFj7;zHq&iqV*U(? zZ+IBmVTfzQeYY+oe|_&^LwxcTkQ9db zEwSt(Ud0N2j(MWjy$+U5^8j+Mdw7}-QgW|dlXwMk&r-d^(h(Ob8+Lb(`9YP9R_ zlKY(G4eG#Prz^)24EBlVg6b51yO7D#&vDTnD+8f%tkgZwjN8U(9v6H?Fh^rEX9Vml zH+;H}?+7F|5(*dXM%Hb#2X>si+u!hvx`yz(t&HYOmtRKB8O~J&sZ-0X^2V#Z?)kG@?klH`2ChMd-usVJm9Jdb# zejxXI0JUtw&;0%$yB-iQ0X)xEV#3M4ae{p(vC;^RI`HxD*#0?Yf{*Y6u4B^ON$_vr zX5Y&Kn2!vy=b#$)&+!o4)RqxAE&Oj1e+(!~p9h}cK^NmY(*M<{0-(%}^~X4gal=3V=^{KBk2(K;hDFe*MHOk1|MS2Y zJV3Yn{?|akBm)+KZW=m3kN=+sXz(CGP zeQ<+d-2U5&KXUcoR{T?%K)L;YTJcXq@t=YEr;P=P$A1RupEmnHU-3^r2u`{GbKC!l zWd3v8{|Y|>{)?dg6;%Eg%>L8Y{uj*tdo}*cEB<9XAkhC(ynh8_RlF;(sY{~Bl5B2k Q2>7EQqk5;{mTA!c1MaLT2LJ#7 literal 0 HcmV?d00001 diff --git a/hail/python/hailtop/pipeline/docs/_static/images/dags/dags.007.png b/hail/python/hailtop/pipeline/docs/_static/images/dags/dags.007.png new file mode 100644 index 0000000000000000000000000000000000000000..eca0b9ea47fc0ad7fb7880c4482fa931d595a79c GIT binary patch literal 63005 zcmeFZXIN8h+cg+KK~RcaIx0oFbm>i`g8~*xK&AHwI-rPo0*Ai3S7$(Q0bk zHUNQ60Dn?|PEi8C>gq1i0>8*S4b)XYB|R({5J(B6d0W}YmuzXmC%{Mz-Mk*B!m&9C zJ)5l|@tEvl00Z-l8#K=hPGZz3$j{Y5qPaP)^sHr*Y@wCVf^*cRjvbZ#&ZBH6}i$+RQ;}l0Qi2Ue}>F?x|Y!fm};m1D)u6hs* z0$H3m!Jq^pJMtqo7(^D#+1D<9_UNaF7m=FptN=izZ#q1YVk92Lw7s2K;!Cxj62kHLk-4Nfi z8W+SsEQ0H}t=QLiA`82p8i?$=o3wyg6${O0@?)33j01^*etl{EdUP5d&dQo_5E*D^ zk;mcHA9GPD^~5RAmse2{e@+WGdSKSAeAgd5ckB+p+_*)}0D5_z{qprcr#l-kA;+_j z@41ex0^ln5a~vQB9tEL0N8b3b)n<9%<~8by_q6|0a1$#xhxK+|l*+k$_-A<$W5!S!pzwHXupvB3AKD7SSX=^5`m zEM8IT;GUt=Rf`_;8>+3nFHJ*U?f#|+k1{p%A_k($$9iPxvN!dG za@d5Hc9S2MzSvlZ?PBcb879_>9P4)mc3?Pusfw_5_`ZK^j_F&Xg^N0d#0%ZKoq{gq zGDLw{=(!Y!ul48X3SRg3iKnrC-C49woDH{jT&*QH7PF1_*2PZZw${CVVukOQ2cEN- zyGM8I2GI=UP1NoKRDQkMNu?Tn(XTvackMOP<>1f5!#gupoH^C2x_TUzCVd`}6W%6n z+7D=~uo!16HJN!Vk9ygEU%PLcK4(L-yQe$0Y>TpLw`x;&=2Ck-+k=}Ddc=D^ueTh{ zuDBJo2}9MMh!^}R#~vpoFC!i!(_J@O6sX|)aMERJ%+DBk7TG!Cd3kBtgdQeppI9{2 z01n%B;_>S`b5d8zWjxx2J%}%<)GWb>m&jFXAtmoGh_Mcg!eR=qEjwdb9P?;L821Vg{}#%FD5rt#&7O-)Sh*Ajvu z{?+NgI9y%m3D?+z*spkNKF5W&>XwUf4Elfg7@1np)eN(o4QNy2jpoUl?MhcGF{4co zDrO!na_DnhwDnkA)Ev-TUcS7gFp(I}09LN_qEoL@`4>{F_0M8w~o03fnD1??wq4-MYdp z*e%Z=4;xNDSXlD2AAA?pMGT1E-TSuWh!pmrV|Fuc{Q2;_fb^9csp)vJVs8!A*SvZQg_Jg?G6)y57+ zfnCy|A-P^e??tW4o>tYR-M`Tg7~Rc*vG7`_eko!-muQ`Zt?VvZ$KD_>w#w%B-kVx& zHL8`5n&#%_Jxe94X;fK$IHR1hb5`m8rF(u6@#2mP{0aO|sMaF2F9})jyq)ybgnU+3 z=QGa^4sgxINCg!p&O`^&I!!2bB-B}FEWgrCu}Kt?+ktS6)XfN=w_2CfZwq?m1>&e9 zq}>z~sGw&J4eBlo-Ga~T4R@BLz(J!6&@LFw-hzhAXUj2mX!ic~vxFgcg{soY$;nvs zc(rQpPlLe|*R(G3b6Qe1u#Y`$i#oPB%Q28e(Sa84nji}nq(a5fgH@VfvBzC5^xy`F zny9VLg8yg}ns-A)SLQA{f2BlI>g_mF*@(^#SipZU7HgcSgle$KElA zMI)GVbu~K3RPnv;%8lHwwjFNunmVH%#67R_@n7+a9N+_Sd*h(}MvK>qMjYH~;}TIW zZOPKF_oVl9sOIZJ9?!yDCl)1mlcg5Gol&}BE6YqHOLcxTzo&~upNyKpW%x9nGB>>H zmT!+Z_8#LA%|X{&ptUkF7BjAtb27VK(F$*k_$>PIc>@E1fq_oTg+_eW2TcIsi3Ln056HriGRDyn_jACsqpxIW{d&>G`9 z#T+#8vtfB4Dk&*cUGEm_AZLb?d;OhBIB%qDVlwz(ssOAy2_v3dus{h3TIa%P&syYX zjAT7Y8;Mfhv3Jvrcu;g1Jd_adA?PWukT0^!aU8}KNYF(w&R^kh)LgtB|)%BCT_Pv$_K;0RRLtd(1mTfFi3+efs zdA+F##PBH1@Of7)_&NF!Z+aGDj?-vWr%O@Zz!FjS=99-eLn8i}#a@mh*3riPcMVb+ z?+*xFy?ZeEnWx>O{GK^l(sgXQIpgEN?0Yzu+wQRDm~h=IiwETgDpA{SontKAld6JmVmM>J)7An;$hW!?Q)kn-QE|JavIZi1BvC` zwqY{$IlG&;3B>Ui4Aj2|1wK(mOX65b%npfuETUv_S@R25N>f&WhH9Oac;9=9LBC2V z3M6sapk9jFoL8H0bDwI-l3eTi?gYDn#CSZaagx5kO(K!bF)%P7)eJ8kJ%Tx}l10&j zZtWRRZMAkOm1u;>o%LSzNk#Lt)|od1rd6Mn2sM^Kod%w0M?9A+I@L!A#wx<^UyGW*1nkM=pMPxzyAC7A)=&y9`)zvzL@~d z)nB?l`uvZkfLlbJ2I(BQysB7M7$nUHW>-yaSx&{oS?jJ1)z53cJaam+w0v0xx}u^o zP70Yc40^CioJ6+8yIz{CCeR^w6E@itc+ES}Nb@D9-|}ADnWyGlCabhZgT&2*%nFmw(t{nmO!Fkj*fuQNVEo^_z!6Rv6ukm-@;W=%^08@5inOEK> zlcZ*uZTpSzsI=Hiv=#|y^Mc=9)vMF@7`N~O(mp>k#fu*r@;wxok;`)p3y=YqB=e44 zy_(b&H}1G?zKm$(P3TC1%$qiCUuG0_she$wck665UE4A*7Mc0D zNea5no=CqJ`8S5-44O>!=&mTW{yf@a#ve2(fR%E-ER4G_@(>xn<8=9_{YI)~HWCWYiCblQX~lf%70S6XTwDpDwy(zD z(k2&+BIxARHAF4gP4g^y^t9)+tksBy_`DDLTU(5XA9?(oQT@tuu0W+|7BW|v~lhml4165aDc6^RXM z3h>QJD~+TFB$3dqtMX%7-&OFbBkKC3I}eH`IE_WEKM*Gm5$eN&WZXUyO4p_y$}^1J zk{!-cJ-;9wh$BW}b=bw@bQt^Fhk2(`WU3M?>RVph1A=aNwLsj z;Fy)%*|uS@D02wJuVe4U0wAisAwwC#VGEjG4Gh1%!}jQ=SIwgMn&*P*Y`)B@=2Y_~ z5w?Bo-r5Qf?CdIoxe(Mwb|xDGl;2xQ9Y=kp-|Jl2eyCQ8 zVh8DQ+^mFl0Q+|J)375J^bYLrX5vjNt%i5u8(-U^W(n4g*4<$ia|0pZ*$Cs)R@g{X zwo`kijGW@WK35|w5 z{1zZ$y5DU?^s_kg5ijAk{s56KS~o?Am3QQ3zPTv-v@AfOD4cZ=TSxM?DMqb4nbufs zAU&fUT3mRrI_Tcyle%#6ScL0)m)wOF)W<`*Y7>p!yS<&jq^~wS?)+^$ryl@dvKfX( ztQk~Q4mX^;l+b5EFR8tYe8^RXDgAl6f@BAZ z1P*Y@phrtrhFdlKv9f|FF~Cd2mqNl`u6h|9bjxTKPgK9U#9sVu3-ngCgOPd|0u*r_R3qcfh(n6Sow6URIUp}aNm7SFy zDSDVhI%scqakIa8&ce`5ll6=?o z0_kl^Z$~y9H@CQMxZOYHO>#z~TXR$SE_S9^pNP0_mTA}?wCnEt-12+i`=>)=`p9xl zl}lhBb6jo5*~kfusz>=HMaTIwHk7Ik^$p59xB5`(uP;ju{LVs>g7QSOWJg(R!0>kkt_*A{-vU5ZK&=5qZ}@+3%R6$sQMBUYF*4A;DDo@GVunT!q&h@8Ie zVSF!eD1>BHF5V&$Xq{*zXU!rZkGt*n9ZplSbLxia_=I@@-3eSJH29AhcfSF}Bw)&D+_U2F?l4el*Q%*s1RP7FMp5Z6QR zU~PQUpi%JyHu8klEK~(z5F-j|6$nnw?dV)N&R&z`?^ZvY|nc{v3}RDfWl?(~G4Rd}IkPf5JY(q@79 zeN{b)dp%?*MfS!_pY}?&W+0&`x^0V1WSt-HxueYcw1Y|9rhr8pL0ypKa%Z>g^9$!~ ztvRifw=6-iF~~q?TVDZD;yY*G8Vy)`j6`{HM2~^pfb)CW+nCAoduhf_ufk&kiWWr*0r=#3B$rPR+jf|=#vm%@&=-9>T zTM#_MJ}IK5+GLQxdb->JcqL{>7SjZhVfX=3bBc8l&vP6*ot!^MvXLP*a)KR$*4o^8 zpGuPI(NhTYBclWh3WRhf?Z(!LM|LpzS{6Ko$giY@# z+CMz&rWt{emdj$dKl$eve+MG5mxdih+E*o!_lLDT-iQAd|lKzr$+)e2~pwd&@t3&_8_8KYY+Xe9%99 z&_8_8KYY+Xe9%99&_8_8KYY;N!Nvbed=UNLe2~#5&&*TAUSz6(M5itti-HTbfVf5|k$30in=JW(bR66(GC6Q;NWk1!jMFTi#g#Tyvr|PO zxc68&kA`O;^1q}RlwJ-mqoS$-X-w;=Nf27za(UP6fR6=e`i7V%+cV0*$W#wvg zhLviyKYj5lAM#rHSWa*S+&n`LLYG=32@zX6FhM;Xm5%6~08F^+0Nxs_hmmlKwd#$% zM~BF(5bWYuu1qSB!7a=B8r=z0(Cz`(+|b4Ti0`=Ad8-$LxQYaHPY+eOIA>#@hk?s? z9Hy3Z^NpIa#ilkYQEk!g16=w&P9E$17xQ-r(KVj?L`=tz_%-%9*w~r=Xw#=#N6dyb8}ox3JRhXwz8Tr*J^S5|0Dmi5U!id z+hPh!x)&(hxo_=g)mqZ{YavNzFJP@CTQ9*t$fA}{S(V!)?*i}slELec2@~y}CF4}< zFI&3FB1+Rz1m+m*nRVFE&1xRc=gVhiv1x*xzF$(?QIHav#k=SWDcd5jXhha zZJXb|?FCzNt)=RkP2_uH~{G;5jLXtBQ?T#%efYxPWb zj$4SFq`s|K-w=mVYetNXlG$JU&pFmzhkmb(VrJC*<{A>6LQ~kyP;udU)3+?n;}5?=P7ekt^mAE!#%7q zwxH!C?X+Eg9^SNSxhyn@ShN!eh#$_~t$NP7kNqf(q%R{o69K&^|w6fq~6`Fl z+|7NyB=7bG?+4I0IojB;9{GcHWv0(j`(jL=R<(} zfjXf&V3}JJ!JXg~wa~r4%`U?Y2=(H68Wm&OEe<%;qmbT4g5OHtPWg_|k+#T}>0MV3 znkLJG9$j5VpfD9}@mJFttkXBu6C6;x6>0c)3y^4}kh23qw-fqq^!&!>D95yj5p$AGM+r9B-dZWGNcZQbMM$XCmTa>!k=?MW6= zNsK4){@T)Xem&c+mX}e_opzD|k*$vfhtfnkmRog9`$EwV19ywWn9bsKpU0gZSTEA+!Dp*-N7Bqitr4y>-MV*GrCi?hUhrN{$(9JA zMPEoLO^@pMzT{dCfSfJE216O?;&tma2GryL@uzpUQi4>2v!uhCD( zqxEWijOI!ksjL%>la>!jmKYI>F7JSfQW<`Z4Vi;ej8E}D|IM>>8!j1ccl^z>Bm+Fl zq$Iy`URJ$AV3VoeeLlG8{Ox^enKTm%fMM~eA57PG8Tw`0o!Zj%Si?C4hnj<9~w^7NJ z94eR6+!@BQw+gLo61pUmK@(!rm6r>@Rh`;6E&!&!Ha*apeWJF!Q`hV9e{m->k{|x& zP6#*`nTr-u?a)D|y1Y_-xxJU*sZzvGuN8R%J%raU_9&7;EWD*ih{8>-omJdBB>+eC zFn*F-@A@D5#IesyDG)8_3uWEIy)cmi)4Ver4A)QPZPiPUVYAT>aY$fKFqvQ2701Z1a|YH-(j%)R#XyKpR1t!dcIKc4;E#ox~E{Rt|n2xMuJQCE;7uC`_fv-UC0 zTM93$3rFXG>YBh*1CbJw?v1t(wBt<`s#vCsm%9BtKcQW9Bw&rbM8PA2gf)?=?Bb!5 z*KB4K(H5s(>KZc%-17caK`S)wD4$u6qzoaALN#bon!l#~et(HT$6C2u1CU6SQ}ii} zluQh6C!{kA(P=#r%Y(fu+pBfG&1oCowsz+}xQ%$+=-Nb~PMkRLrSevZ`Vn{&!pS%N z5lEX4BJ;Y+o=e(D5!*jQzwGAwJuOi5dUYd1jAE1Lnpb{nsKyBp+jG&KBTA-b+({Q_ z!&UsUPV^;jjMO)mnmlFrKt0b2KR6&L!kFSJy*S^kE(fr4W(8Y5M$G7`w zI`S@%Tex{oi0{`v(B(+~jg|jSyWi#l zQu=%sBaeemAf^B0Fr{ClYJXh34GiYJLmj`?3(G%S@=XR(`t;blDWQM2{qGG}$>=bp zZ}9WVkw^X4KeKTIDgDnf@$<)b>QfvH2|!B!z0s|(KUxa@UwI9HV3GfaVEIqq|3k3+ zIjL+<{qIY#)SRB8S1d`Wnh5W}pT4{rE9R1iNo3Zf;E%)095! zQPp-jf3$?>9eZ@eio6B8#pNcFfy5Zk`hB*!JCC%Y2;iFPQ-5j(B$P06cu)lE`<8b zEpIobC2cXT*QUz*w9u7oP4mRrwC{C~S=;q`$F_Fe?SssGqs45m=Xu~e?rSFWTf`E$ z;-0E)Xg5AA7{iTj-$Mpn2Lyt~WmX*=wj_BTd0B~sGOmbG+}_j&YB4U~&Q;L@S6evB z>Fv^R<->g)X%pmNzJUyGV!0bxPp^^ZgVoeJxZtr8@G@IJY2GF2_AN!LSPKYvXFGn}#q`&`>n@=DTd_!-wRQTBU@<15Ze}6W#`{xSZ&OvL zBGaIsIe?aaRyE)fixMw7N*QOx&lB9*$9;TD>8j^ujxW;aK5{m8(4k19!tnuGx=kJs zmD-Qyqq@@MGy(zwIA?!x;qBH=Fe<(;wCargzTNsOf!V5EpZVHmcZh1M^`~k2<&V~d zQXS|i;iNlW78OW|)x_m`tDhA=V+G))g5_${N*Ly06`HwT)_19k3X2(m>u+KtwZk^; z=Lo#2X(Lc?C%$Dxrj`Bi$*xOl+uN~)4?m~2+Q4P*2i0X7K{xI}od671_7gpowxF(` zE(k~=;}4a3t)D)XXcC9m+lR?Qda8U}?oCmW$xyzH-~WMy{=<7hIG!XIKM@ReqvI^65SWDyQz z3(5dTJ?ZZsFIT~YFXMxH(v$upFLdo{+Qxe~8u!3>jViN%#vSFgHGHt z^P@GD;&C^w0dMvWBBJh17%hV7E-Zk=8UvF%xdR7Krq{9vUWb8ZJhvC0y<=`kZ*sj6 zWSR{zJ2yAiwX!$qcQZvw|2`kT@_hbI0kT>{v)sGMp!S?MXtl=b=Z07R;@hO#QY?_{ zUuraMJ2ojWT(yHV-3S_pF|*C`O({pyhqLW%FrPg2AI28Tl0=m+x#FJhCq1G9W!6to zi+xrh^oi`E8)P0KrAwk3mkQw28G%i$OL-YVk=bH$Rm6|#9g62Zy0QYIwBLZD&aXRA zc9TU&O_w9n=%b3LO*7%6VA%dKgwsrMc5M+9Oqa8j?++8x@YHoi3`$UZTiWjXs{N4D z^BbfjZyB0)jeQ`S!Yl3fN(yA)7VArew(Nm19$Ku+YP}x;DeBVUW6*y_i@9mLp*T!$ z4($zv605gDM`t6SngryYsYZOs76cM!Qk{Z&uju->^y3n@TBis(yQ(Lj8hioc7Q^1a zv)&`KdLfTXm`~kV*%^9k9{0@8bFy-O*lKaIB&NFrAyq;*$#*`tJ0U;o8D~`sVt72R z;rM$rp4yi>*N{|r9xp+%xmyz2dj>P0k+Z>-79=A-0qF6DnzV96MTuivXH7C4FenRut4-%DjAXoAo-^>q{T7EphOz8I#Fx#j?vb&wAzVO+LsX zH0xW}<8=vnZ2UYx3eG9L4#n}d8$#IC@w`+VYta?x6%-7v--$@V=jqqA!dUEFao+Xj z24D75lJ^L4%mEd+-lLd-YH~hDNtXm5F)&Iv)Um$4 z)M1Nrk}|~0ZZlo2W4W$gCE(GrG?$^j&pep_q!-(J)KEuyg#CItdFnkHPO%dU>GNwlQ@09M8@sw=p+o~z+V=yupRT-9)CgQ zq3H7L@8|mK)ut!^b1B&vptqN*C=sT$b)wTC3>^XfR z1!5Il^=fTK(SUi>wgkw5CT=0BB9fAlcK3GM`qGy<|79>fT-ED_3&YqbFxTNA3*81(&NDEO6uBHCaWm6k4R{(%VyEKFBwQ_A2h25ME7c`pk?_RA%U$L^KUS-$nu4pnffiOSz8P3P?=usw9CjCK+oRZw=U*HtgrAh z-Gk>`R>&1pJ3SZY!`$#RLh!~y#|1p*yQ0=3OrzGf&LaHz#`yEZu~3Y#|GlO)xtUec?-PyZJlOnX;*K`6R) z9y6ZVWcMF?v1AQ{$9n8y^TW9R!8da4Oq28c4O0D6@>4QyNQJuUY;Amc*fnu(p~g#Sea4!+^U_D&(0k zCt&p>-T-U|G1s}HKlpw-wZ7H3Rar8YiAaTRQ+rB=7ZI!xKqMtFrO)UP%2mfXOJP{r z#W>IzN0#Hv$aKO0+jO0~lesdzvP7m04T{W7Np(wba?;ZnU-F<5UCF@AH!ZHpLCsy8&c3Dw*#{b>Odqya9Md>?oHaSdE`< zniTq51Gnkg+XPd^Mgle+z=`tF8vAh*)31?LCoH4ax_F-|_TQe7h(pgvz~D#C=U7DJ z{Z71eh7R=zbowQ(T;VJ_Z+7&K?%r(+TaK>e@*l#GRwRSYd3_Wqj>x#HDCs zK%BqahXz8e&aN8*|E44^|K^yw=i}i2I(!Wq6dm8zyFO~LpCUD6I%Ou7UZ z*q^)vmz$$i@?D>(1wvTZ3r!}mwzsLGw!5xwCHiq*e07dm8006qU*M>(bAbRYF<0dG zdo=}eNy#Zzq1cC(?^a27EM{oXyLDmFT-WSVkCR(>mNRt<^3YpzuX~bMZau(W(GnZO|gZs%4L6ytL#A)7Hes! zlknQw6_JoJ@Ewd)m@01IOVDAQZ?AS_lASZn$D3KBReb`G<7eKNr&MlY26C!8nITaW zL}<3}6I}MhLL|C2-M6Cab6Iyq5P`kg<_vJK7&Ptrhd3tIHRVf*QQ zQ-y=0fCWVU3f$aT^dgeDI`h@DP(@Xh>(;GD`vHyFPf0KJ-7TT~V5!L(Rg3zwsS0Xn zWV^OYZW;mcb{#_7q>w{=Iz$fNUcM%q2Cbx~i|2cwvF8)mi|6#|8@BgeD+&Dpxc^3V zXUkG5f69+aRfYB`m5WU2+uX#vS3fWy=D+q1^h2zuc2yjpQxQWL1rQL0+vGLaN*azJ z{Tot8_?5DAjb#`HvbCx2=hpEh*onK+Y8GySKjK9a>-AmFrvg@f~+1)2%)25i81eU}YlT7Y;P9 z6sKKU=)Uvv(9n9byg_BP{-95!Uig^{cQRm8Gwqom$wF72*~wQ3L$W^g2u{V8>h$H5 z8e>Z}cO)OVY@0rMA}L4Qa>jLHgFOPme#^YzKFkO*MUl3=j%gO^!Qkug zmSb$6X9wZs8$}}-`nI&GBFxI(Qh~W4S9O8nD|UJN52dvE0g!PuXhngeRPG>t(7)uh zd?|5jAw3kpvmqIZWSHe;>%_YataI})iQm=LP~Am?=Y+K5cyx#09e=D?0UIB1sDacK zG1K6@=Tr6_4MHdTwwC=ot$paOv7#rV9H2(6|tRAH!yce1O#ktA@thZ)@qI4$_vwvI-6 z8RTrtAPt=fXorBdou&5xQm!E!XAgN=Fcn(V$t89=LmMTsop)JV-fUNSQf6``2gRMC zcs5^Kp6&KE&>Ba4E!7e1nl3zj4*}w?TImld`J1?i&Zas54qpo!|3*NfBJMxK<=)zY zA{BhfogzgS&M%N4#+onGUu#}1Hwr#ZNYsP^^@hgZCB3(dz4Vw;DRu?Mbflk^8mN5- zOF>@{G$)TQuKydXMBicx?q~4F4a}hQck5Txo}an9;(4b8D&}afNT@ZXR0~XYJb zt0BXH3FT!Ye9U~^y?SV4VTEF+k;J1%ye?1qHc6E0bsI^wH5${2`~$nV?5U> z4mVrCs^U}ivrC>vhv*$6UifeTpoC(Dy9I*1b$=h)bXPu#3mq-y1Yqfx^kh@JH=c{0 zi)w!P%Ay?rzFdHEL=%Hl>Ce>_Ofwzyd9XHGiS4)jvTzt80>JNN{{5`igMoPC1w3HY zoBd2SGdl|p;*ygy13cE8>^_cMhm}I61!b+eE|=#_^8Zky!si?AX}9Hc8qC03v)d8E zvYn5r-4OwJO?`sj%ut#r^cQGB`j1oU6z3Zt-TT}fCG11=>?Nyp$K}3RIcO3*06-U` zoQuhGsPhCq>5c2~eJunApTgv`$PXwDVrID?XXGCy*GA`A{eV50;0L71wuigi19J@U zu^zj-nul_vGLP*??WrP-wCB#?A$48mA5_eP@>3dZNby$ziH`M@aEihw{3P47;vWNa z;_PE;rDq?ekcdldafWos5<5V}jLxL;%N;*0d90hmi?59f05D=<+CpD$|1OMBVKIdS z#2mYMKt-W_tO)IeT|8Huq0OD(>y2a67Y`AUOrPu6u9tC@L)HxBIT`9Y!?avG^MAi9 z3Q=?;(77|ti$(gz8T)q1QE={(@O2XsRdo~ql3 z#uMV0A`m(oLBt4C7?nKwwsmuLHLLkAdRAwjNaW(+m{yp4m3dSw(HwU&nQS_wu~6~g za#9N__BPutl;gba@$Qa|2j(*8j|q3jr7%HMXNMQ!>)6shYBrCyA5f0C^G6pRZ4wwM zHh|g;rn<9ClJYMK;Qg+6oRtj@CQz^OgI3VCgMU^L3wd<7he$A9{-ZBAreOdlI$qlQ z{&?{%U=R6tNy7*VGOIr;G!|(V;dQ+RM5ceK2EK+G#I!e5(Eiu73E{jR{hk~o(^KoV zWfR+y_wMNCf4CU|#^X}AIJUv=6$la`<*KEj(Uzv?$wGGG&A&}yOs9`c?dkLr=hQ&c z=?*<=e>TOzU(~1 zDCC7s^PcsXhbI_rlD|Wjv`ytzJvnaA1B&G_g4}=2JUu?tWGBW|fyyn=&GVIi)_vsA z1LpbvuPY1QPAqzKBcxHL?L{=sU4VD5b3tJDPS{&!JF!3s+mttO@oRgh@2BaN180|jJm*bijayWWRo-!`_N z$bk>S*1dSQ-0c0Tdd?dx%t-;Y^ip@0(|59_$*%ji|GId3c3qu+>{nxBlGW*Hi9N?r*WCUzNfjqhJSj=$1{0KJB;|0Ak;LFUQ?_>^wK zN}7M?m=~g>s8%dnwUd54`($;&NU^1`nCY))XO^VLjaS{{6k{mDw>3!(C(~Nl1`!)M z?16Wye<>W>VU(WHaNG!>_Vp$AxPHna26uf~|T}K97h%>6cVU)nChr3b|Un!o6An`Z4x}Kj@|}BuLtaWWm^HnJiyAFdBsSYTP8c zMbCD`w6fK^&xCC9aaKL?Y9##5n-f*b zSEJ%@SoUIrEEU;#5ml53vi(!W4f|C1nbovkNdjU7Y7B_co?>p9jGPs*Qtv%Wa6HA9dYn5oioJB~75@`P}*gHA2| zFgIx!_y(}3xL$jE!q0k9;(+Vdf0g?S(>VC4EZQ*js69m*VQtampTCH7l!43!zgbf3 zVKb@SR{t(R*upkAE0#VgH-e3QRI|}d5pM5#Y}~mtoRt8P(*JyVmVeenr()GGM~ADN z=io68V2_oXQf{H)iXWVPFr#^KIM2>I&bRZ>77`R@>M8YZRYt}8ihjS>3%2_r!DGvh zC*5Jc2Hm|G6fHmbY~OwKC|P`U_-@!H+nf^=o6G9NkYS42$-$TkE)51c1Jwzirs$pGWwegO$@alN^+X|QdmJyzy86o`YAjN+=OC-Q2c zH#m>2$!6A*u@KPoMnY%Vl#cvV`GG58VX=8IzrRfgXH`G@IjW5bC%!+Dm+nayED;40 zGAv8IULWZEsQML<*BbGe$aWoSfQ#O;%Vyno%oWccNX}4ogcFJAd1|Q7f|rP2mVN_k z`&Q%3BS>i!l~G~{ax=!Or7*|$du%2a14d5@6YOJXP3+b_#k{@*8+L-!C{4n$@_~YW z&Qrevd7Zbp`m%wFejEBR1~kNjy}${npFmZnjhWwE8wOh@-a{ePN!6ZfrE?D8O8rQr z-26B$V+iN9G;Prfg+jIY&)@9J(9)p*`AYHHQLx$f;o{KoG(0>v(!aIQ{b43>=@UG$1E z{Io&ELt0?=Ptwd#DgaEcboT=?d$FMXik6sr#3`f9d%kF7Es6Zq0}%D&-lM5*gO zEX~U}gp?-ZX&~uj!n0go84sJF)UFa30lRiXkG(*%;0Y2lrdG{sraD%}t}FR!)*@t2 zz4HNtlyKught2^?A~`^AS`#kedgnPaR@{wd7sI=sRhZ;$u?cd zb2DB*616@Rx>DPVMdB1Xi|?+8;61)~*^ThPL>?5}4-^^w_+|XN`PN%gmCTDHokf-z zxP33@XXC@+FEA0BWXds&} zde(ioz`u*V&6wj=1Rc|+ZU+~;! z{;@UbM7+V=`7o5{DHZ5Pz@pRdyRP%>cZ?f8|UPp*3K`qLW zejj)q>)l=DvLA2Q`uy-R0*c@|NM;20H!qoIFv_*ow~pDKNbiy{t;Nc__7mBS<%;7) z)qQJ?G6qF-}3(VpG5qu_@VHHdio%>uQMn{Atk1Za3J} zSiubb^up|#MYxM!)EM7pSVT|S!B_z8bP>^@g*Go09QeR_chccT{dNu377vjL?)k1? zlHIsg*NlXD{d_Mke;$16rnG_G@Wm0Yr-j{@66_A+|>@CyPhQ19Xh*gjO=#wr2DozHQ29*=Vlzc6iH zCI^)-=qKqZJ1;NH{ECnTK!9=Nt0BcTCg;mP0UO=7)HUS+X>4g(%bZqp$G$#)sol*l~NZ z>eHaXfSf8{e}4@tE31ZOn#(m0_sdUwA=G8pOpov&lLGuUI&(TN$Nlnd_ovs_*QZ{Q z)PRlPajb>v5n;}=OH;PzDg%Utg$-9*%B;t7?dBrWZ<(o-zIbgxsmLhuT=1+Xhokp?cEy^n&fQNV@Y5|5wD#O)6+XC zk@xy2JOc{83Y_GH-?$}(x`wv8<-T~5N{Ol0oO%vCR%sJc~kO>*XMy1{225@71I6Dob zgdK-Dg|~Nnd1gt%ED=_(0h_2aY|!oXLRTqQg#*R>Iz=DfX2)hX4xWuU+h1>~*q(5` z+VESkBxFzxO+Q(^)$dr@QkeJ~IVsh+`~&Uy<3sPXi$`~ZaAS2;`&ii%({`q4!Ec!l z15i45U~;!RlKv0&-ZCu8c5NG`1eFpD1f-;-m6S%MQ$a*xK&7OS?h#P|>6Vs|0TG67 z7!(ERW@u14hlXJW<~>>KS>Fq;_2+xH=hw46|L*O^xvq)xI?w$$_BcErd$}Dn&i8sz z+!)cLz`6dC!R-dERHQ9mOD_|h^`^y*C`GfEB2v(pvYoZ9j4OOT={i4JbeexHGa+JZ z^-)neFfCQszL#+C41GQQ*EPs>4Jx8A`)o;OgA9)Z7;rvL<?G)4WSNfqz~ zB0M(~M#p8%KQX<7hJn~`9Xq9cRPd#*-Z%ZO*s-gh$3~B_Ag@QLS$BNXljn0ub6_Jk z@BNb1EtEO+)LT7(!h}D?D&2(Ar_$}6?V)EtM^aVe(1UhcWY#;PgZ=35=ROSE7>HW) z&=D#@BQ=DgE-+^ zk5HY8nGYsg$dX>tBV_HLvNp+F=j(4su?^gjefo!Gb8j!+ZzAyS&y>5a$UrS zwa}ddD)b@n^`RFNht%i~_$0;RgIRM%p*-n6cb@q4qua_Zn2ccQs$PyZ6#5K%zrDaQ zccp$|y7$Oz;B|xo{jLNY0gf1pqu`aUqe-td2KukPgkj6V?iHzIwtxMUY>{6G@kVE$ zy~4&>d%vDQ57|qr#$@Y%Sl?|Cj!Yi1v!ZvYfs^luYDg-#d=k~flZ(LVec2lzt=NR4 z%QBJ&aZ{7a>>P_di)-^0w$gCvgL0dJ{?5CcYr%dceL|WuCIVj%4@&1p+T?k%>cz$Oy?lfV1AJPD5gssIsG%`oRIoV4U8xG$gcluz)X%mm4-yy^X}t` zk`^N2o$ffWo}yC`4iQ6m?x`V5l|Ca%Yri{ag(R~m1rzNU?)_FJp;)_2{tR2aPuaf> zjR1+gK`XJ_fHgAq%=#T7=!#9K__ijILB1xtzaH&NvNtS?eC0Z|i4oX(*eYn@htmgT zUp;t*jdAK_JaBM>vh-D5Spu$_GrygZ{)t-o7l9!Dx?J*u>JUpwx z8n7V$MYmhFG^mlDD_tz{_Glcvn7=($#Z*9WXVplse)kT!KT^@kC`R;lKgJ>`UOJ!F ztB@Q+uOmkm!r=As=aNfqL%35=Qc_+2{fo0a*LNSjsc=k#W6lG%htcEMT6YlII!$&GaM}497Cyo^Lb}0KH5H2FwtqQSRV{<`~@-vrf{Zb z1d>SgXL2#b+AhBpAZ6Wg1I;3x(|bW$Sbt@m21!A&kWeOjx zLsY!qmy;mlzDVz8`UuXYvuqcdI$!IGyeQVO?VehjI;EG7;McLxaPlgy#MNlTe?;$Y zsz~VhRV;mrNZN7sXn*3zu>0`q*i}NF*41$zO8A)|@^wMTr>~+R3KUP*FYKUc5q( z%8?P~uE(%jX|_qyc}R#0^{dJ&x=!&DPX)#yuVKQM#C0_^D@Qd#iUq8y2)Brm&u!v1 z{8IW$OUZ>epXMLAKa0c8h@G;p<99R1MAwRr_DK626upda5Jebme(9Qtka@n~x3at} zfrEC|=vqu3HCepGa&wtfytyK{jfqH?;*dUcjqbP9YAt%jG^(&XDb$Z=^q@pZ^ZL#0 zhBVfEq%WtTyKQ`Wg>9srWaa4lPofe5#IB|H&ZWv4|MKB}sOO4v870y&p_Ck#df+A7 zf56MmM7Cx{zgqKgY7CC7?51+PS3S(Az7Xl1$Y_;X3U}_U#FnTuK&Q{C%ltQ|`cOr&tonNBGc;#{9j?c>DAcgk%5thZ4PJ-9p9(b$GUDT2Q0hgtd zQ@k6MqGsigFyjK@r`n%i=(gN%A5y@h;XP{Ow-%eiTHk&UxNFnMUXdMw_vPNN99QkR z;xBtb-V&9#rMB+YZ%rg-hk804moPLPnNXRnJdoEXTcu^uk!nrIX-#K!?S{`U-aolG zMla~_peAk4^gDjJ-0e-*)N|yTSGXPzTz^~~f~T0j@Iit4={%KAH#A51#}j06jACoL z1wO)CL!Y&ovLfUHG35&Tp{E?V$IHH-mt9#}UZsaoLqg<~_&aWQf9l6`GV8Wl$;4T> zN(2{*f3nef%!{k?bZEzUUK6^?a;I-M$Wh#Psfr;gl2Z$NOq)=vBjaR+#>%a5I#7!O#f^za?&h%WjmoB7Ix&eZOXk**s$@$%9+ z-_G0Jwjq>~QK3m1vzf10iS&lNUU%yQ_*x@i@6#3jr$3eo!X3ovczblQB2|vHyFqJd zqrZ^7bmz8dYStvI(ZP?jQg3G1P%qvOmZ+I-2AO^AwI8#O$9nO8gsG$}AT@iINbW$O zrpE{Cf$t$v5YCE|j94P#V~j%JLHvU=nSE8Y=f*kkBBQ|xKIno2(!G4?D^K6q8V0(K$IlSp1(UiY$ z-e`9+HKsFv_TJyuswxNcj^UUi@n1LpfBGQIcB)>{8$q{6oSiIHzfim0V;(tola)2P zp5ifhbN~0&+6cK17xCm372D=)XtWXBT7U7!ujMXYRyL)Zh-!tvf`ZOhJNmzeIw45b z(@V}tNlX9i?iMiDiyl4Qpq2Y@RhbI^^_BbwKmYG-{`p^ym+`ORyTs76nx4fHz-pHa zKKroC-M@!Ty-q$U;Qsd&rwgI~*6%7?<3cwTXy&r8uyAE?58(S>;hA1p;qmwP=c@2~ zOm=3eRN%4VY42e&+)lrnll8*(LGHtK{An+00WAlv_|j0D1e1TTH_!FVkim9O&;HmK zsGkwZz4`Z1NB#pRPAb>B|1jn04R~bOg8qHVlQIK%khNR9|9zekn=&Z5|0mr)DnNcB_h*nG2){S9T-{Ru;Lx&eDS*qAi-;U{3RdmPRv zdsE@!)v$h%V)GhKL$_dIT-4c(z?}x1462hwqHf7Z2fltp(#?sn{iqrd`qrWg z;d!$cj`J)k-R@bG{qUq6w)v=o0_Stvae(*dXUE^WinB*=8GbR+p7?+{yR=%l9g!q> z{E!AVLdBgZL^uN#sLjUcZw%xiKpK9#=N-Uo#ZMB}D$lFyv$=OkX)~L!T%e?kIdl+t zwe{6_Kh<`)ZgSn*+f!|-kdQr9OJV{0=CiZ;R!8vOp($s$3bLY}ts$(Ng0EJkKEB*G zabk2gex8Bu;~JCzFFp zw2ZG8rfw8d$a+Co0c^W(TTRW@4t1mXR-}QJhHGh{CQu5Lgx;on4fF2X-a9(c8F#wgj{(I;GwnqkdzfqCyQkj;Q+F)AR5GRSFFp4jVX#EgR4)#g3@2fEnEXsR%N5yKQJWrir z|1;<(wY>|W$(|~j{8yn}0Gp2(61Dg5Zg4~#3S?OmhBX5s)2g~zwekN zrgNo|n!y-4DC`KannXI!wyJ8KK+Mqxl75(Zr6`&VbbtKL>+|7i5!s7Ahq=xugns!C zVm@Jy->UmGT=v%_jCfsbM4T8(5Oztfk=hmz0*yndO_heFzV>>YEYt+4ah{Sg;c9+D zD`|I~trCMr_NF5p&Y*BecdE5_1}bjDSZ7r5-VVFsNl8g*^_+m1?s2dUo*Wzle++=W zCqmDry7Cznm!HW6tEtS8?0pTKNy4kM$W1xwYPDVz|S~&bvp193L^pap{!~@xeMRW6iDw zddtJ!D~m+F>%uws>#EjS$cc{xB}C&Mx4JO2M}jJeBus8wdijTMT!dNg2;+U?&?mIt z8#2`^G#>BI1k@iUv2+T-|9*tlDJuW65QTBarUr%eTIj>}$(4LzSeL|0Yi zT%2Q!5uoV|jraXSWBS=(kS2>kt%G4rcdCk=%^<42LxNbM+XTtsG5h<2#l`zfz1_>z z+Mkua{K{Y@VIUFd?InAJ6m*=^=$I-hy4@=m!U2KHDVZ$Zfa>+R(p4On=wF<}x!1fc zgk5sId;Hzpc`b+-DRSXxkc&a$oLr3NQ4bZ#V7i}*+i`<*hJv&7=}Lt;l9wQO%sh!V zBC;r%f`X1~0^=IHvGRLMdE{aVu199!!V z>Z<{Fxp9Fh{pf{!-<^D(c&O1EV%N*K4*b3Tw-N%-amy^Tpu{G)ig!t1C@u4`K6dWU z&k(1BMHb13Dc!bO^2t)tn?v?{NskM5LiKmnvv1rjGuf%yYfKmSHh3Sw#AevJh@YAO zZ`$hrtj1aM@`VwMbpV;Hkjh8>j#p#h zd~T5UWmeLk)siROR0DW^UA}hBUNQKf4gd0`9=Uh?UzAVPM*x~@XMxI)HD#6Q9x`tD z!TtL`+WQSZ%<;zXhL2VaqS|Rhx9d%pH;2w`?lBIVHsC$%AzRq5FOWz>IBJ5v?qd9Y zRl$9Ss>N|Mq7#YNH7am*+_7zB&~orSs^EQzO+>9L@9k6uYEHV}JG8 za6L-v^2SEQt5LZZgWf3riD*tsncjDVF|6cw@}Z9yB#f{Os^OX&|mV{v0sOfd> z%wa;O*~RCyw@V_VHW1puUX&Nl3U%$3nAg^fTq=!pW_}vDw`Xmo1hd`t{zjIER%PJY zo^BF!S(UTecybx+x54txRcLm zj~Q%l&%Lcyk`+hW2aTP#)?};nIMf;W7P``7+^t0;mD;7}Q~JXqWN10Sei=nd)7=g- z_B!e{RksfdJ{v775xgQVs%={v2}rC3x8hRy3KZ%^)H>c^tcl= z2a~b;{Nsq-?r88nOp(ZSnG;Y(&R_N?MjgzyI<(l;ET9-oKoOjLKj+jI+CHEXM%}N1 z7UCpQ3QXmQ;&3%oj=HZf*Pl0I-g&5-i8D^~SmpAb0&}dO$h0YQdBm#|ff2e_o$LAt zw2sfcY2H=v)eoB9^>1ehwMZiZdm6k&I&4GTk)kh`Zf2Rtk=M`rW0&d2?TRLl%|{2_ zve5vus3;;rMU{J?@{wTd^M{>yzvL=CNFdAVylkN+6 ztCRb%)P3pg81x@=y0V2X9N!6r4iI~=^NdLVF=*|z7ERxYmNumD-jB11Zx_ptSI7$v zlIafeGhMqwF5y${nB(gwy#a7v$IF=bL$1%Y9zM+c-sse zNqUKkiD~$a>YmXzBxRIZJr(m?fVAiYR#h(d$>)S)`qT?H%FNkzpkS(F9~^TD_7QMR z?ehJ@>sVt-@2_rQHCydn>8aZSQrjhVwkXM>7RG+BmuU?F-oH+WkH&XH+f9$Yxh?x! zjBd4a%xo4|MQW8_v+PQX_bQl-UDiUZ#^&Gqv^r7}Qh2msy({)LpP0s|pLbw>fx!>} zEP+0Z%#((aSiY+-SF&0fu8Xx0o4OcIh+Mz-*B<`nwP6ANC+TAvjhVs01HEZ*WwV90 z+M zCNFJzbfTa|1-h&bmj75Ni>kq`(6G!Wt!`&Lzsqlc7&>6bNl1UDbs#5r0XueIHRmGR zyxjvp9;aK?O@7Q2?SdyM+6}$_+FNpvqXwvr^8Y|>6tZkQ(z0y#(i)g}T3L>tG%5C= zOiEv)NsoR{Ea&z0Ymz(4GtUdP90(ccRvnK-JYnyy{LUF)B70_mFvJd7Vp7XpZELZJ6E}$gb zB@yr^P)JTk((KL)2b^m%@15UJX*TApOzLZ<~=($*uJ)N9>S|d-w)&^f;vKl3i&T`LKA1P z@3y;&5B$=KRe!=de!b!9+O63{hX(iV&6H&AdQXOO_qv|H1{&@=$GBQId80y3%F~~T z8SE-)_c+**osj*(_iMkK0)O^5>6D=7oERDkrdpVM?;E&So5`T|LYDF%w`_(A1h?0j z;>#^IE4Ivelv#ujxPw;HzAO_?ao@dIGWz`th!jd%h`ezyM$+l~rNsT+8@<2RPjM5f z=lVYl{rn9;9$+URvG7Rr<7dBenCAMeE~_6&&w`q#!;8RAyTpSF{p_XI*fZo$?4)0P zgKt{qJ5)%&vsD1Hy?v}}MAr2%JGJR}ZP|Pafl&+c`>si|o_yOg6kCjO@3kD7^nygL zw~*V|rUJ6d40fMakaQ$0QTTp%@;SVKz`gjkt7$Gk$WGaS(4Tg&`(?fI`3W5_7@qmX zG{b!5=Iha6X}fEZDGq)jA0IWWW_3AYa9*Es(+n#d1YL1R9r)l+QdtrI;3Ja#*3fDl zyJ>k1*3LHplt;Lm;nf{CgK9Xw|3*-ZQ%oPRqR%H;uRcte7N??q#cgWdSJQ zB_z6D7HzZGhJ_Hv^B#Zl>{`rH-C=M;A-{dSyx|r7bptAW(@4`q!X737VuO^JQqQ`e zei$-qWUFj8a7PK8Kg~t7v#BmAsSA`uxJ!&(56x@L&(Jf6Cr+h_xyx9FyQ`zAS3VN6 zy1acO+6dO~J8Zp0u_S{ND!EHcCPBipd;z;*Lh{-1+s)pFOVegSP49_G!&C2C2ar}8 zMhx3$-D74bcyt?@VL0@NZ!fu=U|pp9RW0eIWp5%KTH*5iBPEFC z^b^m^UQ)uVA66xbHGF)1*>7)53EX8XGdi1!SF6@&M~KI@B$6iOdml8*q9Ta>s27a4 zV;@78sUdPo#EYCach&G!q5xcccQzFd%(Zd2S&1?HiMikn9o8*0uDABK-xYc*LGU`( z>)cNMMZ*d^Y6gYrnKe_G_u`em67eGGh_2&D(oIsk*5#kOj_;_4@MAnx)%wd{cFc$4 zu_gTnE13moVUDl9+q!#ZX9P{vA16cPIsQW=-v69eSn2d_zg2qMnieCeqkS@TsFnW` zEp61Q>!-83(cwAm$9T+wf*|C$Hu6x{li(DA@efR^{Sg#Ri{a|)5(@J2zuu7wV`DJV zj<{I9>7^wuKR-XN+UNm_(|s)MuW}#G<7Gc(;QqI8zvS~p=RYOaDKq|+DuL%e49bij zIZfh!-+f#j&_@6JiqqBCf9rPzxGazGla?izCv`zLgW`PlQ+yK>lM7^I>czWRFHS!= zktN{qi!7Onl~f;NbT=MKz;JK-t#dE1^?=FS6XiQ66khIO#TN`%02Mt zQ~%&Bv7JNqape}IARPJq*?;<>$A6ZZFAn>Q_J>xR{8|idGDS3MblV?#X6;Vaqq^li z@B2b$WSLTLnLj#z=3bG}=d=g(@U#U73YPbb2A5BG`Po08L!!A%=MBx(Ha1DrV&mge z>Dz4vry@x{Bt*lkAIH)O3`HA>J8@R6X7hf+Z7}QCI2Av>rPn$+)WjM#?(T5M>S3)_ z%9qWJ2mVg0KOTvDVS@mR4h9GQh+j&x9bxao{^cSZhXCq@+p}qO*v=SHu8y<*;Nmc) zv^-?XRywZ{Q_(4z6U})5OPUxxm)J|aVEzpLT(iDKlaA{|tM>*?D%lckon@Qnq~gl6 zepa{0mfr|r!mDyYN0VLeRXUb%llQ6fwRLcO2s=Khl@@-kjvx%ZC|CbgO)C5QyknD+ zQeQ?4{%9@rns6_9zU@_uTASko^Y3C^WR_C4N#ncXT7@`A|0&KJ+lxLBm60r4<=8bC zDlvDnK^i8xzkF7KyMRY_n46CET@MBN;DI#c^ELX)uQ8N0ez!o?SK4xtBjX#m6Un1j zsC>Ayw6<-HstsrBxk5|Jb(td~s(eLHrQ+-UuamM~XN7U=S#aR0=h4(hEu~*th7Z&U z{i=9@GZ$N90|R4_6BjN}Bgp(T8xCxN2oJG}H4OxsqTVY^WJ_EvT8!mC&B&{iK>cc| z_|(kl!yxnhcFiz$jU?f{iF#{A#X21w=|Ax&hJTNz0=-k1kSj5Y4q}L?#}od5 zy%gx(BK?}Y+W)KKYX@L!E))8w|63CV)U3sOJ?=JoL0uzoNLJy5Nwq~aw$8K^=;-9? z)J^sOp`>#WtN;D(;y8Wz$ETG-+1<>}9DF67C1!gGv`pR3Tkh5qkeVGk)T$VKO4Y^V zNVmVy^|Pl*cN8a&G+!K#lojRL!5YojNB2~mN88h}pmfpSJl>0=#9D_`;i66DJvFmKqCs=4zbQ?Oud0h`+K`o z_pSU36lH6ZMjT;D>$6LeRka!1sjelici9ik5F0R^i`wb`@0<9Z=fLBBjD8XWT1+p$k zl*NjZS|%~>t#`S;8tGwl(E9*X9oNPj-ZiQUI+~fFSi;Rldr}cCj~Tv z>vU9elNHvc+Ca~jQz-%mQa?7F%IScBZ$Vm0D*N4q>n|U&7vkg9uOqbMKIqOJ85hh= zzEQ{qGOBFf5dP>%u_Nfe88Z}4z7RY+xs{%EAJ2>O-r^c76~ZlBW5AfH6_O|rfiMMdFpYy$k{4R%K^jEzvNmM{h@|ld6_lAhAMW# z#0Mln6fgmx9NyLg?K+DCoNQR@MgNG_uju~FiKl`7D?rtq#2xFc-e4pwWUv=(5*lZ% zYZB_ld&j*@&vzz1@}=h^HNtV*Qrf47$a4j>m4^3ivAU^}Zh z4@efk*v^!yC8xQ`_)LzY=gMiURd*}Dv7WOIlu1p!V8}s zzq&v-pn(-48P>YIT!#fht7sMr3IxYmE2E&r>jTTpgb*5mw%=KSt5SY?TqseGZvLRB zwIep#<$XdHy|pn;(>;E@Goysi`Bf`g8NWzdkt`Ay;|7X$tpxtm#mwMWso|f}@v*OF z=$6J|sVf!KUVM)Y#H>)OlwvD*t!JJf1WblI_QUg93~7Fh zc|VIl6Uf~YN}1ZPSoS#nE-sEmW*AP+R9vk8o$0)Q6Ez{cYlC>~*u-V2m#$0SP)XNT zD7}8DFOy2~$$2M+$4a1%e* zfS85qkGnKD`g{MBRcq8}(fRIgneJn}HPLd7=Rc@V_tv{oq5&_*btE^E{8iKeqe2{xm2(=8c@Ta>ns;}~`lf)McV}xQ zZrV8Uwr5$2Bvf-UZ$yrfZ~vpSrkKp3*+-*lF9m)hDFKk^E3_}c1*q?Mr_b$@=my-Y z&z%XJdd493IJ54!FS*2V#K`IiTsPF!i9fWYc)!tpN#s+=i8It`p@S8@eJyrl<2C?v zc`-JfhbZ%=`Q2y^4LZPY7Wnk+K2TWzdTPW=v=7?$u;rhMhXhe+;Iku0-8PkBV3D;W zkMgg7-`?jy(+aP3`%syX;9^J5=0}U2{r?;bbBgkc%Bb)oHofq1O4hE7Yp>{!#;(W?j%_ai>_2|z*_T;%U++j?P z)@_t@!VSsI1aRCbe~~?kUUAYdaw+@@uIj5tMMX95+j$vCL#(tb-vKcDSSD4%c<8H% zIj&x6b6Cge=+JdZsH%2#bh|cmQB2dxqI)j(=Nb7Xb7UXUI6q#G$bqjXChSoAi@cT1 z(e4M3W4$T|tTMyvHZy+M1*X>QNBwMBCW`cIi?TPB1p(aA()Amf=lX~!B3l$!e{!@@ zKu$1z&{sxfl3`$#C}ti^``)BcQvj&*(=OwkkaE4{Bo=;xQ=(71Q*nl05W2cGm8i=L z%?M)hH-UT6&%c?nc~{)1xX6R#6$=~wZ!j|yk(+e(RBIVpVY_O>7pyKnwHo{!45S_A zWzCg*yzg;VpRyNTYi?~8wSuC;g8sJENOkcQ1pF;v{MqTo7pF{f>qcg;0>%oyb6Zni z5Ht47KMg_Wr7J;P&-vic$83AK7_HV&=KrP(wL1XOfANNfPWht{aF4!SEG90PigwnL zf+tg#EGg+%&(kMvZc5Sf_Hwb%J^ZztK1h67ID*)}#Il{gX0IuKaH^2nXw?7Fnd;y)ky?DF(Y&)gaw}85kg8 zP=x_T<5V7$s<@7`qw9;+UmE^mShlhli_ z$>;9XD}n!$MtM8TSz`ryc#m6p>uvu5(iEgk)zDTKJrc9)d;PjWkKx>JRaQ(y!V3kQ zrMqRaCI6GMst$}-xG%*~Jwo|TFO*U&hbGhG&hch3PaNC3^b#4`*S<#Hnf`h z;=08kUSmP-Yn(gC7P2^5cbUUwtk^mZDb1fadH59{M|{-?4S6sI!4w}^Kf77)@cyNK zS&QM>c0P(o!U!q+&t`s4IsFP8G$+xR@u}Xe@hX%=B6|$N&a8G1f-sn}z4O#;XWQAl z81V=uvxu+P1gR`%N#{{YOIB}BD9D9nvFlBLz08}zgPXin7a)GN!au? zssQPkTk>C-YFrwP1&Sw){#Kn9jTy`oF|11#_q@rvr5>z~Hcp*mT0xY=UCZBOd5%p| zP-OKO+PD00MbLbRO_B`)pE_O^i{{ISG*gAb7!teasxiMWTw)tE6AT!iL$j!KsKU%EmbNa34QF0NMl&j2}j`2 z`Lw+o4n;7tViNPEwx3-hH5n_!wi68_jNYFO$T6tKU9v(G;J)ZBM8g^ER%IO&<9GqL zsV_9m(~rTSMyYVFd<$oyRs3hKD4WjTZ^22U}M>IspB6)$V;+M@9V5l-oZ< zC(Dbyr2o{+cIyCRv2znBxvl9l>WSTok-N*6Y~L@*+ASowlUbVvR@7cA=y$~sb2z zusBBg(7mLoJ3wMT0q_>zCzX&?!}fI30A0VJ`R3ik`vFEuZ>^pGdxm60(Fwn_@snt7 zKP~K&<`F{(P+F?&`!Og4R@yx9!DaZmL5Lvgw}HpdP8_$3Ky;_ZO+T++tL0$9E}!@yB-`%MSo$Co^dxuhwI>2uXRYF!2BA)P$i6Og-kg&;6J>);lkNBu zPSZhmVLz$&yj6Bs;(e_Em4Kl1dev729N!uC4#Qq63 zdA+jxat0EZK4tx@j-R&@+2PQgEScUPL^eevuXT4H}PoJJ`0@RoDyTD{H<-n{sTcXSX4`0N3S1ki_OP4^5>{TH5yJgsGIR$jbq=tQ`K3Ru7gPZM zsJcq=d}|4|zEehL|u0 zbkvnF+|w`LxXv{7n2ibS>G6URzB9CqW10$`c9(S}T?}O(=Z^4;G}MiG`;dJ2>!m>v0FEEL{Jd3HFS!>3 z@jkAa(RziQy>lWPWxQpqN^xV96KH8-KufVHY~$0HEYtbgNEQ$&}&gaB^wKx zckqIT%$584HAapTr7cV|J+^eAOL-S&60Mg8od8i~f6gsQW(TH7xf%hV13N}q{Ig!* zv)09gM}DI<_his7zSbT8Nd_yECYq%_-Kfsx-WQqUuP34jSm zMB48Z)4!Vk%$_F_qX-N`bv^|uBB+W#;Em$&)(I#B{#v%t`U>1(sqn$G&{#mgNlyaB zZQhez1^U}Pglk@9k#3!mD!^(synP}~WlX;NhxEj143KZpq5%OZ%#T%{uw;b!n^HPi z%sDkgbVB}M5mG(l6Td8KIh!lwOGQyeKoj0 zWVHta4u`QxOs+c*&;$T9(t;AuHquO!u`f-zU4IPP9jVO-#sRsL$c!Lj9SrZI$X_VS4?^-_OD`$S^R|Oy0PrOm?p6bb?i~D_B*P<9=1@7 z-AV_n?$Ts~R?51^T$?%UITIM5w}f;qblF3d7HVMf7YdE)sn@Pc{|8&bVdPAHLE!w{I2e%Hi>zgGm`g-*^-sHQRQ8ecjm z&?%X@yC!{m9pyU*?A=B%w89y4%Oz!hf>1nVe}Or-d>3>ZJMHe~!%{~R3L!C**`BDQ zt&Ypxl0TP>j+n#>L8_5PoS2EPApkf6Y7G#!6R9}9bl|80VW97AgXKyuoex)!((Bjv zWubAuznP{8x@2iB9J_Vleh4%NPnH5a__>c&H*@a!Ur_?g<`=T>@YE;gyx7Y5Cx`K` znt$s_Gx@sT2wAJ_tF4b$I!q2K(&rm9eD@3(u3i#ZdI23NojZ|OUE94q=KhfRH2HaZ zgW~VNxm3V=Ouu)~lf@-OHY1|Phd10Ga4??5XjS=O5s&M_C7)AbfUaD>-aZaQtN*IZ zPf+WoU}F0!SE%^c*Cz)oqz;f5ba=n1o|i1lc6rDW(J)VU5ky5%tz$i;=uhv2 zY*D8yOVNwH)#RuDdb0btNBkEZ*b){L6iG_+@_=*Sd-|Bmu(7{i2aCmOyRqk;&Vc}g zlK&MdJmq4EvyPXO*0>neGb%YT&nT(01t7Rf@yP@-jX>wWKYLX$9?(So?<-E9QU9Ia z6|k>t+{nntpr)pdrv7rD;gWJOK|@0WHE3YHSW!A-I%75`y-mPuj$pDr)wlY``;yB% zpB+SnSID`!vlVH?E?Db!vUjjv^4qPsfjN>o+n$npbF%F*%kcm8e8@4o0(+V=11i|y zbn3{-4I-JrqjTXuWYlQ~f`fHnGcA|>?^Sz&8uYpTC*33}P7RUpiQ;JPdo}3=Uaq}% z8ff|BW-{~-izG?k3?ISe?O}mw+Z3XmBVQSNgOsi4v)h$&CxMlY^PJB8iTo$|H*0aj zJPG`e_|Qb5$ztxfTe?OP^_@WZ&tQ)8d{Bx!iJj3?Gb zA7K$dWJVfU3hP!~>FQKaVyf!<5y8=v{Iw^`Mlc?mRNs)1BBnW%8c$q7Q2VM zK=1EANl)Mh-;Nc0NdhTuMES9Lk3FzK7;=Un3^+uV>umS+pqKz#oF~jj!ZLL2=b&J1 zPmI{+$JZF>O1bawc@D?+;h6^v6O8c4)w~gg9}26Bc{m0cPe9wsh?3j>s_T{_7E-M z^OwfWBM0t;&5x4X7_d4p3BXduMtYc=8YW1xpSEIFbKmE((#z1It8a`#hl!2LY<(-f zm>WlOZOc})2s`#xJWB95hv%bSq!b{0$j?EB6K>{)Znml~$$r_F)w!x3p5d$1x*HU9 zecS2M*(Ye}AO0QhNXm6dTe>vNTKbB)mDXONiThaf-pUh)qNV}RZtKi?fK-b=PUopT z3LAX@`J%2ertD_^QP#PbhJ{Uj2Iub~*BBADVw8&7^UX%f$&*BBFx zjix*j5|m5l%xGuMCuLFEeyicBKK7}Qva0>lW z^vZ8}jjQP%@?I6B@o)C$6EI}NHCP>PonJ&zE&wOhAr$F6a%3^w9Wznk%i=|Ns<(hret^L_wZnX+dXMhiLq%8ri zVkf%bVynBUU99CDa(xC(lbZ2>E|WX`V=HJJPv465?q~p$$dZxQk&ls4%wC;9QYo9k z9v4%0EbNOc)!Ne+>x*>f48h>vI+OMxS2CBXxC->WBNB~P-|8t?w4`6sN`+AuS z(qSeZ+I~~jJbC%wkN%P>P*vqC;pIMK5)ziJ8hcjF=rnI;5|{kW`Y% z*!#zBzqiAad;u*fOcg^jJY+`KGbh>?KUoaR%L9Iq01Smn1b!-8ft~^FH7NtX$ubCI zfsrwzuT&3NIurszH`&8%Dx$X{qHr9*o)HiVLQV&adUqkc#Klr3F7_J!*Y^r&%! zC35=)x>i<-dtzflX|cs-q6-3$Ix{*>=y`ZGMT#gz1E?6w;dHTD?Yp`138ftbMc$1Ie(B z@W*ImsNl=d3jg~1g!sRo{}!|#mfzgjh?T_+n<`oP$oM6B7OHEPc~H?_5xy^qEov=n z`qY1Chv8*nzf3B-E8Sqt&(tg?V=4DUTVuqay4d*y23ednl3+tPgrZ_?s86%Er~3HN zxSYRXsXtNVeE9G?Cf5nPvuV?jbm#g$GX6HhUG&P;&*cX}a8mVBYo=^ut+?}uAge>wL%`st_>F1`&j z;jYr{16HLqU{zWvBZ1K0gWWR|ey|b4y7U2)AYdF{*DP7x@3%eaJlD!qf6aM`^LShu zg7AYJx-;E0(J*vuVchNS)iE1rd=+fq2Ru7FE22;5yPjSz7+*7VR|RLsTr~RY@A=3A z>T%b|x5ngQ@qr9y5XH16X%z%-I?g~KC=dn%*$>TQg>t{KU_I;9!PP#Ho@{@!tqEhI zH)HCM%JpfMfQ@_bY6?P@3c3c_}hs$e`gi{r%o!xmuT zMJwwQ4GPO3Es6A(p2KI|=j&y1qmsS>1CNhG;CgBrgP#lY748?kMK}7$nC62P&mEd4dqEOV#6PRV?ojB6+g60*oXz!@Y{; zQ+guqHT!YE{-Ms7`mM4U)bw%F7P`vH={FcspuaD|rI!(ULaTy<x@7aZnn$oudS z??H#nidg>rPcf(#)Pva>s&tjYI`O#@+U27^^xf?x=XkP@Yh7pk``A-_itSdXydM(p zUFW71vIJIoXXQx(Px*AXTT@b(Ig5>G+dR49i6-mMGFQZCa-GlbNM4bpqw@2%Fdh*4OM{RIt>zUzle_IGcl0Z@#_YGaN##@YaWB`c}NFF{!)6k zdM0LwzPr0cBTijkaD)o~B&+Sa0;z<8dfIKsP@+41Bwu6QzEEGycCGeS#-A>2GhcyK)wnF}1hs<0=+P=W)o8nNn zoz#B|W3lNj=(Z>?bfq?6x4-$w(iGN~+f5Q;(oB}QT^6`cZ%UsXm+sdw-)eFjzH?Yq ziY>CPPmy)szJc+|p)x7J<_nsCYl-HB+Dq1dg3R!1;ys&MKN5t_zev1mS+;n4W7{I_ zIJsLaa!=ZzrFs*--4wXT!2XTZOQm8}->&`%Kx#e+AG$_G^{Z=68 z`WSV189wOON^Dd1x-zVi?yRuFfA*5{iL$H1hE>cuVpT{$iq`H`<-1ZuyjSPSFmWn@ z4Pm|h*!w=rk>8(-ct6e^=Phdi?s9%m(P&_D`D%z>>zIc1NXc&99tXmB;p#-Fb6?tG zOa2oVI-J+@IT)i*_cdSn2L)atDS}MAI@&|<$;H*~v z7d}~KZ%t-o4>(nq-8S73#KI@^m;-M0-nOhX4igUPrJF8vzH?-)RiC47!S*08kf~^F zbfv(bSWm)?K!~ofhjHh%?XlA;e5OzbMC&=w)$J8py^bg&w?|&Edx<)7M%=i~$^nAq zcKi&qL@=0DPa*9;mdHFPR0A`16{t0D)LeVr&Xd112RA4vdHeN9CC#hhgHNSP`9SV= z3GcCWp0jThhIB0IvV(}ngP}qrPJw~k`JKfT<V55I@C$}(?<1+s~xuV2sRj!)%hieGW-T`4qY0a}C$gyHy*${a40a zrQDa}i?8<_(}Ie-o9@=H ztpc3&KiHQ^m7gM% zXI#Akl0@zdt3IQTV3!sz`7k6-f5|$&Cg7RAxPhzUV8Fk<5aQWeB7k)Yj>X}i^bsZ z1i#~+Gfzd0vTcI9hCz!{!~2Bg>0<-K5`lB&rjek7jF_ zjO$l7{PK6j-#oaZf~7hw|C4{(NXqT;El~f<2e-)pa^)QH<%w3=KVb2as_;8Q*5Z zy_QeQ&Fqu1m6p}+ue?9}g%q>XTP&r%oxQWdaa8Kj`-`|RKGeKe_mO@@G>-rR2OsK? zuX)whv^TUcFVt|ggR!L0o8R61+f@ENh4dxp8@meH!OA?BZResup1Sx0Zo)zFMa$X@ zc~R#zJbk;5@g#&xORrjmQ&I~y!fttPEFF@^+38YOmkJO&79)9K8-INCRI6wwd|FgH zi6a1Gq77riHl%n)(4z*aiyG0?{&>l1Qk(Go!Db+=T{3Gd`WHWl^4&};A8?$B8mrV8 zP?VX~xu&+a>`vG3F%NKw#oy7CFErOD5<9gu$KyF?I%n?;kXvon@~pHJkK)QKqzE=D z2{??W+*AE_IA#4Jb7zhV`%hHa19;Ab8|X%QU<(GpIb2ZC>R|me+nFc(I~C~(0eJ9E zPh{UY#u4Jb0GpaDS327ty%kBd#GI|Et$eLp<~KS-Uk{lPz;c!v?NF4aZM zyJu^3FrStwF`)3w3-}2;-&8s^c;4jA`yM zRl;k1!uHY`A6~;B#)!T9iA1-5}1WC{gXLKtwBy&Om;Id#*|af~u`DwtJg@$;ee z`MK4@I1mP3vpC<3q_<~h4Lc8uLz@PP=;`Jvtbvf?S|+I{x(v!fUR-bkvgmwaa-Vaytx z00wn+;v?@g{z3LIg85{hnKk~$v|$b;axJ3F_{V`na^StwXlQIcyNod)v>Tv@ARFdv z%I9_^Kbc_6TmLpJ>ipwwVoK5`y49m8&h^lr3J{cvSLoTW?TNtgg&5nss23}jVp$3?MQj2d5LPl3EjKcp~t{P7*_ZWeD|1U1Jx={NRAV^aBOOUvjj6I z_q#Ko3V4I;t~E`yzuABNUHbSw_i(up@Bwms{N1$EqcvEo#EWKm(?!@~b8J>)Ays4B zIwfvSnrx+c4TYTeZZ#47elT}};W^-KbaNvQP^fn(VAQb~9=kRXevzi<#HB4txRjZB zEI2U|%Z7$wN(UE5vTb3V5oE+FT3XQuLl>XCJA!o^0wii-cnBJ$l*5=T1j6!Dg zA92s6WUg8Fl;}TgKGk))X5qQqHjO9WI?1N4TzUds!+{Ru&7(fLda^&b1((e9kCl}S zQf?c*PSQ-vC@U+g@wpGWJ%@x1O%|`iKcsSG$0;c1NtDku6j1h57^!r36NTCwZ+05b zjBvz|F9Udkg>oRt3H|sSV$6!Cc3t5e>f^borqyo9)`j_P2hJ zO2exAcPz$Mb<9#J6})&rtkhawxVS3X7?v8YJ>l_?W8APlVL{&Tg;8f~_@$sxB+wOU zCJNbk{bC;#vWv5bGEF=*30F5+b&L=Y#D!))#(5%KcI zP`WtvjlkxmtZ9nNml+Ugo$;YFY(3&tl}q+7$78v5Elw#)>rbkQ(mT6udz$|W%Qjwb zd$X+7Wd+Q`*y1Hew@#!zt^h`baUM5nSlvZ9;@ZY8mLhFbI4 z9@?9EtbN)4(=}Os`e#(Fp>q<}(fC^|tUYNrr7uTQ|7-tO%P@3>NziiJ>FTlfc6?R| zrSwAU^SvPyk~+QpCNyJEK|k5qZlWRq_@KBV{CgUl5)D17BLqoF$()U)!eZrA z5MrTvI9Xe&)A{fWo0AbE{hoT4pN zNZaG$y0!SJ2ban~TTjls3g2S3`yNRplfHx0S{2Hdg7qXd+3{Ys_V+X>;xnr?B@3KJPUV42Ki4$Qwg#;N9V_T3cj}tcb4M=xKuF!jy%5eSO6b z?;^|Xr(ap?R{jdTFJd*aGz1Q|t=X3BfcRx*ArJ0#csn6qzT zF9+dj_21#?f0HFBYM9* zb!=+5QSQ{K`i)@rKr2zMr(#^qMypNfV)-zaD7GBTKB=x&QnK`Fy*KrMx;uGqME~^8 z)vseK$DB5)dxXb@jMaIM^I@Bmc4Bb`hV*e`YVaywaU-9|xu{0(!DYqSiQsf+j9@Td!TM0 z@ulOuFZij~cloJQBj~T{A3O+Pt026mYxmGpIY=F+OHIMo+?~$ifkKHLnmae-$1c{G z`)ggk-`-8q2^=Y2E-& z@~DLmx=Q^wVoa9zIaf7Z*PbC3GTko`A3brHYvU0w-4F`7LD~g`5hBQmnv@cDaWw7TUGdNl%bGKp%o5(yd2S}lh5GaEY3)tZfv+JP5TXeqSYT^ z6}Av8B^ATe2V9?pci%=QU(s1z`x^4UQ$iujp`3fDiA~PAZxKttZ zN`Rc+FWW2md09&1FGI4{5^6Khg+{#oW3rPToJVv>TWf@r-$} zkCg=3LM4VJsw60rIv@I3U`*-A!prf`|6~1{;y#YwaPOjW*q-`^wn#->F0 zK&6LnV*2UJX}+ht-t%yT-9yhyL*FgMSBDNO6{z;7PVSYcUf~?b^l%47$AH*LH9#P3;WGdASG~!~XIH8STG7ICF9Y4=2a8)l>CdKd9Ts+m zi^}wZ^B#UNmtDS7R@j$o`1JE0RPmufma~KZ57c~rX;Rakp^r)~myJ|oSW39R;TF&33a6GWw)BaMyTRIAm3G(UT+EIkR#iPO@@e z=`@dCX<$`{#n*pS4xVm>HUOViWYODsD`OB}Z6T3S5dNM|j_^H-M=K|j-9r@CcRSrpl(!`BR=nKsSU z2!~sLzO`#3G@O=(s1EdpOCP%ll{pP2I+YA%e$#QT67-ifenp~Nz=#ZHqHIW%wzEnZ{DhZSn4XwrMi6* zYJ<8Y-CU%^&jO;462LUstsXS{@WfXu#(LH2rCMrDs?&7uk&==|VnVCnxLd4-@8LY) zyM5y{P$O**Z?7W2<6;az)Kn&)KgWcjV0+2kflmBfCf~hwAc@&{iY)-=bk2e>#U&60MvOB(A zQ?>le2*TQZ9H_~BWWU3oxfsXQq=ZkiFfDI+R$uJebu{#2Nf40mT; z;6c&7oO%K>YkUHtkx9?0Ntl5TQVxM2f4Om>?n)ymI7Uco{h8|I0$<8W9$HMqQS@-| z4(i320Ll(gx>m55D8cg7_oN?nlm?UHaz{?Gf<3adQDAe+s~2%!eAcLj))bFgx8pfl zemILL(Ht3QXgnX5ZL-vWGcePzRXaZ(H`<&=M(47SEz3|$CuLtG;I{X2MHjSzO&Mv32pscB0XRL@I8=GTP@ zeKA^SS#;IyL|LW^SPeJ0oa}yd>6)}eR(MISJvB?5^Xf*ej-UqB#N9&?{S~$5>l7A; zrbbDn>t_70WVqdSk&oJ5$%w>oKFZ$NY|#+r^~ILVqJJxw*J96u^ub4@CM`3Mlh4|vQKe_ zx%i~etRvCE;07*S1U}~Oz%o(2?z-j|XiTaUDUa-1z7S_8>S$7X9Da~MZuho;{Ym<> z$Xf8?DUl{E6K553qkVwV?$|2py6%Mf^RAD9wZ1ZzCU(ef3ql>WDaI`ql=svDcMu7= zo|R`CK9Q5wJ06bmxypH$K4K`^%HkzfZLx$BgFXRdhh)E-sc|jblW2Q=a$j*S*f~_c zkiwfgL1IB=vBszSA1efV6yo4$smkkweP3u_ zlJMa-{3@ySXu@vzkq_D=h$i?nEytkbcTSVR0M|r`JV+m9gQQqc+dX?ywsbV>{IKRTYprv( za1%9UX43S`iRoeexT&PS>R!Jc2p;3;sk!)nZtpAIvwYFkAHq3 zW_CK%J^S4q&HKGgf}PjoarB_}kj-tALI8j~_AvMD z6#W$K{1gLyu5TLsM|ol+GnTu`tL5%L2bfb6rS3f=M2)qf1}z|knH>fGQJ_yFh}BR* zN}#^s3}{az4*E>q95?b(SQC_s-nnZzkZX(ZHkZUi%qFdI1+o`XzTY0jcX7H>H^UM< zB6mEHJHkLIuNsSYM4l-2AsTUGFwRajaddmW02I5u2(qlQSi2TAXsUDalY`yJU^-4z zr6&S!B(TEtF4~MfsVAco!EJ&Eh-E8T5Hl6-7SsXr-NU)#_qG*oSw;$kF0}rTAHI{z0X6Q88 z+wi!7-_w1g0%9~%dYxHHfm=ZF<8D_uO1RLXlK%F!8&vlAqPDE?&<&S3AJgK~hLwwB zs#49OY&@(61e7)vH&loKR;{;&ahF8K`#I41E0gy%cvHF$8Yw!*M3`mHA4N zo?Qw_Pb8)J2cyt|!4?Y%GsLjMH-Km9sR2fzUrPi#BxUfE5u*f1VaJW=Kj))vdqEI& zYimwlKOZSCtAdgcF{=k(v0a@Q{ObC_-SyPo7@fa)IzMP?IY%}N+wvB`q}t%__JaTP z#tZ+y=}6O*(r!6G^L){CYf5sTxroaQr(V1|YLy{5GdD7ID+Cf8knr$uhQ-bXcS(+` z=c$ysUi@QK#OPWWAT6qHb$3gk_KT7Nw%uZ7WMuq|n7BHbV?D6ixu$>UA^zqcqTf17OoCH$z|i1MCe3fU zve^@uHgta2%K%waca^*6Jjy+lfq{*Srf*^SpJL`e59Cw!av??MTXM?+_x_*I4H-h% zEmx4ntd117QU>3pxxD$r|Kmpnc7v+2#@Mh|XWu73vkPwbAq-Md2ITZotHT=6=Zr5b0(EhK7Px0BK+M$B?+&U7}oPR|*6``EFW}Ishy3 zEt0E0d?P~Rzr%W&ni2+BJwcA$Gcisb87>#@5~0(1R?|;L7Z|iFTD{D}qJ+VW`Q5~D zPB_Uazs-U6X$w{$BmL`k=J;dK&@Xi%pRv z^MO85ph-<%f!F0XQ*`Ym30NwB?}^0z)Nh*Ngn3->6TT~To+SAC;9rv9Uy|TolHk9Q zB>0yY_?H;?ml*h$82FbM_&a|2e~B2N$IjtEG%ZYlp%f4|g)g7_L6j5~%OZYVt);F` zdpp;ZuogU3Y8sbDqRPvbdm^e+dU{~R5xbrg zl;mruFpLvGW@cuLW%k4754tHST>nyUc~o{pnhhVcivX2w<+v-K8vpe8zL_Mq4fyf7R8o_a<-C29&UNpe+7*RJLC?p{hEp!=T16kt z|9ilB49kK~NT58*lgsQFtLm`6DoU~;u0bvbEkXU!3MQ8vZ32=LvHX{5yp_I_Oz29o zh~#p|rpd{-`&5KjrZ}$dcd9 zOHZC2Pjxz*2aPW)xc(Q2zeAggDA9iKAiQdOAVdRbCr~dy zDQz)V&w+6Cjbi%ng9A63KBF(IV+BtS$uIDuCKlR4c+ai^KR*NsAYbZ&phSv4zq;j6(#B}=P~mVldUozq41X{#{@upq+On^N-~NX)D#fu&h=HWnD`R&dY^{< zbY(*K`(hI-o9Z{2(xLb#BuAhEPcS`1ExU2S8=XQ%RkdF~TEQ8zz+K?~8fF&y-5+0a z;Op>d`ks;CZg$SoYS~!c6Q$p3^tJ&kVl3%!lDs)SSrRhVrA#bjKgA0R*|)08u>8=1 z(eRWY>8)PL-Uu)EYPHRbU?#87I}ty^no${D;`D4mND+U2ik`xvArI;OR$0Di^e<5B1MBq`?mHXl@^H}gc@oT%xPRGob zUKY_nG)`PfvT7$aHhH3tyKISHhZ?%zGITf} zFSavQ09H@&y>95yVZt5ew6z8{so2PkdYXW_?S2&RWbMNfv(99}`D5WPQ;Kl$&WulM zq39KF;9bY8sTj$eh)_2y^J-1tJem(`=yYFUTB!F*WZ|+CJ`^Xy ztupovA*3kE8|AWY{eG^+F+*-4Cn1tqO=4Hp*WPJtw^E(G%i=;g4>J2$h5#+v96JlI zyB8U7x+-D%3NJCOL@$> zMUn&ex-h+hjs)2FVX^UcJa5X;p_~H2>&N-gb5E3A{v=J$_DUz21f^R^0sU1eaz($n zP`Ox{&(ovlA&D5RhJC&(!xsLI~#r2omcwR9aeS4>+5A*c)W#>_1 zn&TLrry96zUg$aN{l{FKK#p%8dR@ePfFgeD*XmsIAJsIg+ifZ3x0|zReUF?GBzwj# zne3W*(Z{!avm6a;mnv|?j15aYOdDk8TAVw?ME=bDc}iDQc&dmN+eVj4Hysa8wo;ms z;hiJGZsH%e>6s5shQ+lAhJ5%C1CFjHA>N5QHp%y*SZkSgns5s$tyFr~<2qD63TCko zIUK-mS_CuOWnVi@Apb&+$~Osgv_gFA8)wPz9`a53PYU5@Ci2S#{NIg+(RWQ#Ydgq% z_tDo%D8F79k;{3)xv}pCqs!53@8AWh^g(hPS1w(|3gmUt8$#!%%21iE>0J)}QiEJ$ zZG}PJc##+1QyTN`Z9C&gLe=vf2ZVD_!#tP2?5>XF2V@zA@2p7-(M~c$JxgMU=)PL! zr>zj16CIxrWe>3pRMe&LSbs^<{qUbslV3d1alf3t1l)VIh5{@>#pM?p9Rc@&XPtuH z;Daus4e!m+F&9s|q|Of~%ao^K9%R(e{>$Ni)#ZtA=IUgB)Gr;_)6yTELa(kySs6`wN^R{a+}n;v()c4`j-m z*Sja@-d4V#u>o~#FKUgTjRXBIC|JA$wgy{kvm&b0%~gRS=N9rq7S;sq@Vfrm7ke~q zTC${xmq(5#0q-C=9qz^O~+Lg>Bb!@et*M@+>S zr*-RzxEJ3U`L;q?Li7b;yJLl*w9;@&*IjS7p_SaltF&_G=EUq*&1*a=i8J~D6Juq@ zJNwADahq>8uD#J3DX5*uME+7q__S(K02Oy=%juAYfx3mR`POCPSCJXK+{TOB&8NxX zf=^?q1|Lz+IdZkQ7S4Aeov7-X>SrUXT5gx4mS1@tTGp-D2y-Aj{@f(i_|APg%_bf^ zWw~-So!PJhTa|8`fgvKsl#AvT#%qr*zY`20H*JgVt_u7X;CRc!p%VsMmZBGKC(qv( z4%w2N+*zg~Vjg>*)+;ZRe=6tHq6`8L?+l~vnK$DPsq(O;$YosNwg^ZWS3L=_+ZTu? zM55^Vt!rljPy^QRyb0VXlRYZz;x$k>AwUU81Jgb|Y$Om&@jg&0+ruTb&x`aK+A+ig zP4N-0z&C|5)r8xUYNUKV2WEVZEy8=cOwu=aCr>6{36%3tpMgLzo>OHNKq{17nckWP z^(cza7e6VRcNX7Xd`PoBZO`rqV;`<%Q)4v#N^oYKVOCrou)Cfb5ju8= z!UeTuxdI;ZI^{7h?C@oBX+S_*;SG!~`ftx8XqAG7#s;tht~H#6C3c7%Ki?o4-S1Aa8;ppmC?* zNIHL%5w3dDgt}>Xu0vx?0MnOTFP-~Z>2q{%W_C8CajHY;6eHQXGYw}g9{}akAl9!O zo<<%1JhF3*so1qtv9}Lb8=S8;*mkKDP6tY%)sy_EE^p~>ioNvH`r?pJD`x)$Qv|KK z5E=&}j1h+VvQJUQfZffQ=nWfj{H=bI6&$^@chr!;lrp)7b8>F9%H)9gq3!cpL27Og z_TOKVWWWUBI;KmQfI1Pezrj-e1f@KmoZ^fAWSk{FKV;ZwotU=3Z>0DQ6DK&+`965? zk>-v)`v*GDt>Q(SMZ;Pvh0~Z)gXpW$IX(K7qGG2NQ3ppEhNv?n*x%_NIgUJ}>>m1> zz9xKT|M*J?l;E4u%EtHOAh_{F&YPaRrz^(#TVunHk({s&&YnL7j%G&5jj+TlymzIJ zi!Yde9~YeKid!ByE=Ho%MvrIst*jBtp>YvDKK>BF!Nl=Kw!<78an#(LV_$Ih`|hG) zd(}x{$6N(#T?zN7oC38-$r~Ux>Rc%u(oZqH43vfXJDpqNpqv3Kr%Po6bY7=bR1#13 zF#n?QY8Q;9bIrX$wIl9}H&F(NKT}Y^#jXiHFidcq>b)iXUCQBadG_Aw1n2mnYjn8k zUl`SM)scgE`N}%QUe#pDTC~)?pULh^aiP^CEw6?{=8y%wb2&7En-0j|+cNE4%ogVa zK$kbUaHI=+JYQ9YP~E)A2~4ypE-&$-T1{PDUsjzQ0*x#DnEDQE$IfN6maQfAYZ@+v zUB%F(jXP2?#_}p9{lXD9)!UGFaX&|4Ag6u%I*`J={7s+n8G=iu0u+46h*=wWN5UgJ zY9sX~wqQZ3TG+)*o9uvlkm?^}&t44_ly3c+5izr|!P|N*H+t{+shUn;@|gxPlO2vZ zEvx7K$PusBouaCx^VzoFKFQ`a$c7ca@_9cxzg7&>#yX3Cc0e$T=Z3Z@Z2^fQ)hq*Ua<(lU4H5(2kCV^(hxuYpgDkiv;D{8 zd&z*3luH79kIt8*_(351*gyhUgHk~ESVwdtf#WDEg4@&)fiJ;<3=_5?1m-GYi2Wt( z*99K7DKVPPDh|7X+WEnrp84%;#C}J0%4vG-N_#5D0`OBQ5>{1cC#8!h(0@|4lbi@$`@Yj? zZWb;U`P?8A1WzXngRBGrW2gW8*;)$!XvX=6CeOeA`}Yl1Ku2#Re%<-cL!GcN2%r@a zs>=7jU-SDS0_cM8f0_WU1tWq{L7f4TW;pc!xrhonaQ*%G?_OY_!{LL15yJ+by!g*W ze9-d#e@8-2io^=SKqpL8%l^+rR#5BGzsG(5Qel1!0}wgsGxj3s|6DWxL0Vw{Gdw>x z@>Y;AzlDi<)ITQt*}qm0$QSW{%?kk^3PWHZZeMzfDgR@82n^`%;h&T7_nTJy0J|Ti zM`=m?w#ILx0TV*<|J#HRmt=&)z`z_FA6EuumZM^qzl6cV!>em((6+hMWc^oXTkGMu z!L9XkK{;9fnT!O9)+dr6e9M)!O~~s~g+O^9V2YnFYZ+iX)PI-_|8Fz-;Xkm46Ary_ zWBd32feCBn1YD#%%*BZQzfQ&W2$)$(uf~M`Op^f-;EFSne4k?fJG=M~01u)P)RX&< zJCvgVE^nQcqWSz^!~DyOLhn5llZ++zKO>z91ZJ{kRDqG^KbSM#pix8O0cyuU0+~eO1+g@;EaOM)$jyG@zp$*X2A! zO?!r}eNJ38@xzVme=P8c5?B&&#T-IV{MibVH_&nOzuunn{(H~zVWs`uidrmPEV{R( zT*v80SEIOB#5zoVj}+4!EGC*+kku?9VSG1>3~#N@1{1im`7G6nflefey9YoTvfin8NuQ(BOY+vgh-C#IL`ywPFn zYXd#|+PhbyF30k$P$q09RnmA5dN?nd2SBT)usuVa0YMSY8!uh%_>I}lNO=QCZ+WR2 z2s6mHPIsm^Ijo736q zeaKP^ufu;Voi-iU1^nCZg>_J7yg}*4Wuu$(h0UL0JU13$t9Ciz7|g?$rM12>fyFUp>?{z_rYGMB8LHK74!eBZA379V*+zv_UH9f^4-B zA(1H_&Z~&N{|$fbhu?hZtp7UJVxZI4X%^x#MVXW*@OKGMzq)w{jl^w^xfVM#gb*@C zlKa8xH6lIR(=P5;J&k#NagtTxu{4Th{%OC=H?)U#_@)0^U$nvcuCj-|!LsbL?JsyS zK^{aqd+Vdph%o%#-B}Ch&R=R?k?4|{)6yCXVtQX*cM@PToVaZI+`dfvrcvE2(y+fH z?xa<|y7*(sWles^+Cx;T+DAC8bV!*zqG*vR!eyL;ybEY@+R#^cp2|qiTadAXqGA2UX+A5On3y;;@1qab9_of{ z&JoSGtW=Nvi(}UJ`Ja#1WpSoR+&XBiYIj8{_nZA0xh%%jBdENXo8FylyyE5boY9+;9ba5UK#^SAQAtvrvVBl~|GDFI#gbKp; zfL?wSXzcsOga`7Gu{3-8>SmuKoi3x>9AUWB=~EGZxZaGtx#}g&H4VSuO!(KvV_)`^ zhXObLCmxpvLv#^A*$hmIle%iOL(iNomGzzRpKj^iDW*~BCdK9kow^cGlFd?+I1_B! zCYg+KZw0?(?>jgnF!tEFCgpV@RQb**hB--Zb7h03i%QVXa;mU=V7rC*sSmfm{GzI0 ziR+!5u3nQzPOkQRpwrm8RP>tj(V#C~>=D6NIo|IT5QAQq{V%)P zYyD9)?KfGkMfZn?PPQuCRux*P4pLgc$B!?_3)?>C&&O>1Pu3J1-rA6A;8a?3WTm5; zr^B08ph6>cIVKS*#LmAk%9g?G-E~A$Zm<<-z!CNNhq^b48!};mPrt{#uQL@{QvNPq&uUDXO8)O=?+A=VG+U{`M^pYs(woJ1{jOFaC zE03&Q#u>x3xlRr?^4K&(U|%FgiM!=vDtfB)5Lx`q85vIo9#-*<;Va$H#g4p{1dlxL z9ORsG+nu!Sf@?gB(!jZoPnq;h6ET+E`g0 zRq-$sATZm{^ncLd=^8wNWkat$M@ls@5kP4%Sr?ZoN`431J=}_0t)w>+zO=L48r*m6 zARqRq?)7b)EuK~xP%O-vGBJT^y@t18wi*x?%2i{I|993TO=Y7FZ$A!l%=aE~8-xUHutyxkLN*SW? z*@|h4^R)&(|I;&!V6OW#KCU=31z8$ou+YYEaB?D^^z2o+%^W8J%x+bQ7So@JQD-jX zl1!u9>{TKc{7M0&va*u!b2NTW1ew%_Cu;T^!zpf0+ARMaYesQ^oVEUSV>lQWTl&Kj zoLsp?e@cn>tVjkTu8Z&g+!N2Jfm~0Z`tgp@G`K(|Z>j@I09U?aX14xk@`7P?pJ5U* zJq3CKZvrE%1q%@Y9nY~#{n^c#1|T&&TWl{*TX+C8ThC_goGyPe4w+|RwxcYY+!0(hAH4LoDaJA0Woa>L zogpwA(${O+%;^`f>#H}oB<1x$*1ZaU3nYmb$IHr2J3^!^dT?-jw1q^dU~#q3|B1dp z+ZHcsc^^HNgQTaY)76$X_6W!qVkAAo!j(JNV+55)G_O1H)%F5>(@@p-*v4@w$yqV_q93;f%N$3wwuzK%?i24O zZ(6e=VUTd>iT#l~`&}?BH=$_uD%!~QqBhC9BAT^*9ca1Wk*so6M-gqCO z;>reDD-+^_#J8m9-`cSv-Rw|6_egd0ohc zW$#%Gt>R?NeEgN3?n@5g`$)cFGwTh?%4HNjVb6A$Jf>88zt$VaBcS7G(k83cQX{{EKl1f#gJM^}4fWwReMJ^O{r)u1%=p^+ zj*G$9#`UW!qT2n877eRd2DKTtTl^)71HK#d$EZi`h4KydHxusMJT&dCWOJLe6CFP&!UPN4!>qgz#d?AR}3<=Noq$1RGAcOhc%hnt66}+OJgq2^`KIlzks!Y}UlARU7 z2D1O&Zrb}5*6MrMTU1!zi_Hg5CT)U57D=P_ZU$}+7O8R#zXqrF=7VvfIjxrvw{DVh zQSi#zT-Ovm;^9beOXlC;qFHYLkmox$hU)ixa=Dv(EuBib*uh5bY_;YvP&#t>g2d(l z*zGD*s)1~KbGaQ0nrH9wdaTFv_RYhg77mX^05CA|BN}%VV_I5Dh~A7+B8hir`ra?%FoUl&3nYxPFAM#<`TEtB&!HclhcG}u6s`rY+A~UPd*HLR+hqkGWw3h&C9!Tws6huwR=ksXUoH$B}}T3 zGF1JsmZF74)`uW>PM-H*NC5HMG=0Fyet-6?>~dz{c3HB(^E4K}kZzJ|!Qy*vifz~0n z$TLnL;nWKui!+z;JB1phkg={s0h>7Q5>>>x z$az`pK>jw)>`U3{)eXPY?$^D^js$+?_Lq_u`FUS*`+#gW>m!^bwSb1=oykpqF$S>b&ketRLy$haL>4ICIGjq zaHOz-$g_Cr-Jd=4=u^2X6NDnoo!`BES=cn4pLZ^l?{Z@^X|{R_I?Cx$T;&wsHt7v7AvdmG{; zQp;Sz|E)PE(Bq>^&HB5eha9`}k(O#DNCf>hNhYeoz3z9pJy}<`4KLFSyEE15TvltE zPxi6QPj0UA*Hk{~$Z&)ic$dqX&|Pj1H3a-XvCGO#vTHH83^d5=Z9INCDfqC4r5Mee z15Ze-e!>z?j9s8fWQsd7&FJk>zTxRNbSOH*$d|K$i9iv7ubI+*^Sl_2mtW}lf2RdyreGrP7jsA=1C9_+k63^=+M*})Emt$ePFWQLe| zZ`X$YU8o8{q24Wv?%F3@2^<@3@;H6+=&`+UjjHmFh0m3;*0UXUCA3??q7BCZN_W%A zOwS|wknX-4A1%}ot5&yp<>T2P`Q#$Jf|ysJKxGiskMX5Lq^Z6($+26l7lyaCEAY~C}%5?JMmvo|-x z#TqaV3uJq@h$sdyW`D$6%J>mQ=oKwRi1#LOSYn@K5U>84UGsml z4=Z_)fbA*(vrWSOf!Vwp@O4!OoYw1KqUi~=J+$g=l!K4F{4tiOQ1Css$0gbsCY$OB z%=>(5EenP3H4OzB>&%wiK2Fht)h$k^2uFo)p_$js?hkN#pWHpt3AW{FXJ~LbG9_(6 zosF<1pz(E#RhbfuG>RaJqQg@W=;7UaoK=6z z$chxif>`Kc4}cZVSmtcf<*e9l`sBG(AXFaUE)oBVK6&*2B6lf zxnP=X;bN3Gs9@Dn9Rkk=I%!&>UP9O(OOI9Z zdVjHrXu4E)e0r)#108@cq3VK-!M|-V6Ab})I|tom2n2L-dD+$mBfU2$^1xw%w6Vqc zL8CGh>l7WyKO_WYeqn)0om&gG6$rsPb*%Vs<@imrb`wP!EYjxsV>LE&PD)bEw9jLa z%JiB<+x+1jj@D#mO;2B4oorc*=Y}zFbiTtH{bbx@@!@i(tfjk9t>~m$F-=e+np!6F z13Jq;ep^Frphg9v#y-60BGNuG)}lA41rt9osFfdh|A%9Bs=yQ>fqGT)AJIJT9Di(3 zO^?CjSBIZDNG`ngwOf6p-siTS*+b{XR${vN)p%}X-g$I1jV$hhwHwKA{|Jr;t7C$s zBgtW2!FKA+Nx;!!s#b*;mUy2|08qJhY25`IA-Q@pk`17M9 z034J>aagH`67#UnTx!?acFliwbl9C?6OeBm8bTYpEY+^v^E_UUXVMOQi-Nb3=6fen zs9K;@W36>_(1A{;QL0nU-!T_SPaA-U-kmDoX}{Wy7R#vh2mtTxd1*fH$8uzeSQcvS z$jk;4GHTaJ)r!<7mpo54{eZS8JkNH4;y~x_Y~|@om|DA~DfioxaYZ`S{7AO6^5#$y zK3wzRiT===GXxi_fk9ec^RF z;rl|iOpL(>r`;m|#PK6q1>$TlY}+uJ)hZC8MF`kUkT}Tbdm7!3L*ws`23Wd}CkR2^ zx1Sw1rml~_4-sYilXp9!nrdXh`wyoH+3~ly{P^1K2;AcVt#eony{2vfe4KD(1Mi=4 zpo4WB6mO>~fT3V~x{qVX;AOzYJp6>+>-({}zyNyf1_~|{D+LcNX38nNyuC*&Ol5WP zwPqAkO{8dlEVYoqB0hX`DukoTQVtuR(k$n^VQVm<4UFH|5~4V(tKIdD>?Ag~SEs!n7~;3+Zxnt<(Opp>1UFbns{4cE zunaI-Ur!k>$V|ycO|w6*ciktAW7d7IlBZy}+8tJZ%yO;Y>_tzetiZmZA3ZOYdPDRg z00GUZRiwuBP=LVH{_5nFH@9|;^-Q%@{9P1@pP!#c%MDc)Jn*7YeGZHHODM~ax!UP0 z1e4U3Vlr~mIA_%Ncr4+7|87^Knuzm)7oIZh>K zoLO-+Ywj{Wydjh^G8PCk+%k5bM<=*&&RgnCofGo zyxc9B6S-Ch|jXcGvciMV1 zY3(NXJII!M0EBXJ^E*vpWX&>A0LWpS)5pgLm}=K}cj!?+pBOuT8<(}zx(Y9}w6xTN zqA3WSY&-9a{gH4q>LC{+J6)Q9=h+*6n-Xp0h>-84hW|y*OicFCN%Yn$NJjJW*PPZpL-UKv?Dn~~| zL_FlRw+RVe`zPJd#*#L$f;V#q@{65EtZ*CM+-!X>{Zp5bxtH$up##08W4?ZlPjDt=%#pKzQK$2n$1eviw5f zB2U^7$u?iXM0vv-Xr2cT`Xt^NBwVwUvPX+diTxo#I-^`pzoVk{SQY;v0XYRkErU@?aaUapNBJRqnZeQ%`t^Y!gO)FNjj*vkV3O$p447JlM` zKd&1x8sLP0`mNG#Zf=Zp2+V|GJaK=LCPV6@(I~J-cJGJolFFBHvCB_e=y9`GN`6-U zU8q_yK`!I5#YphQ+7K$3;s$R`-hX-qbB$+TmHUL3hc`XBD24;)ooR+}G7A6Sw2Xt^ zLq}{Nfy{rA^WMk2(aUzOM!uzIk@zxbFu+$u^u7H@Fht;1-q729e18TM@5J6mk#B@AvZjofeT=^L4|(QKj8vK29SPZK01U$ zRw4(qJ#fKjYTy3}p+HVl4xkmJxj-r4rzS3d7vlU91^#S74wzQfwq*@K8+3FZWEYcN z|ImT~hzKA=MqJEG=nd5FFaL$0{!vr8JOR)~3MXTc1dR6mRyg%CbUo*vDq%Uty#%PF zOH&Vd<`b~cUFPeG3IAywup=NI+K_Jr1B$L(;PnZX2Ymlr24N_~A`m(Z9{cqo z!0atHS>^Jv6a6tg$TKZ~3C@`?0*`-3RO= z{+zs+9O860S8Ww!Ggsr61)`Mbi}{lZ0wI7XFK9HI+_?Z6Crm2hmWunSS&*-GHG$aU z!O_G)-3r{SJf={sKy|OvE(Wfug1S4$ZJh*~563TE$q2tY$dcd-z9OyJ=9f%mBI&JcQ4n>Ll8DKC`6}DN85(O3s3N4#*@jl&_oVR107@_YOSw!%JtM8m~m| z<*SuYGV3)RE;@9CQ!@|Ky=09rd^#-X*kC46KT)uOy{hx3AIj5G`2QVp4oCysH+HWnR4$$ zHTi8=)$P8L{J43VEgLY*v?F!x;i#**U!>^#F@ZtoJC=ao`at~Z)#=W@Zlm|Livp~h z;#Q&ji=M(fH*w-?xg%1W)mN!W4BZH`_v|T_{9L)UzE)J zYqPF___F2E6QWff?Fo#7Cd7y>n>_J%tc7;8)C#-S{LM~7oHrE)In8(7KRofjG;z$! z;&PHk48{xUYNy*4rCv?=3DvZn>muN@5OLc+Cgt}k=9{iKz9%Klzp#bwO%u#cBEid) z>9y3nK<1w;nw8?Ku-dr||wdQRfMzxN^377cFtL zvwAf0e;|Jfyt4TLN{uZQZ!~_~oF!BHW1#d(G(I|+fJk9%A5}f|%wd$u_Z>_OBY_uM zL2|(UyDZ~&(n)4JTS+dPd*XIS2@>FaHktQc1UfqIaQT&KQbD2rJ6hnmWkhk}(>H@p z27ky}I<;SHwQOyX$!|{ye~9RBblvMmUbwM z(8Y?D?z}UVLGoSjUtYNk1-p=eHfOA!@zxgF?XE3;J_j*1ptZf%qKafS6a@;r@0+Q- z*=)oWP2|!fD~p$E(1~jHQO;yvA0DAygx|>Q&s8}J`CJ>LBuzqJ*bRIRib!eydTAU* z7nLoO;O-7$^LqG(Sm*G&9T57NU;J-nA^eENSARp0DrmceE+HIjftd$dn(R;a>avd%=mVO`hi` z{Jxh*e@=f@R#8P?|G+kH?x^Dx}R(Xgcqr(J|MPz&KWGFpD5Zyqm)Zef{Fd zL}pY5O;^9ph$?%P^uyZnvH}0XNrQtqxBin4EDy*|Cp>%7f|SLFQh=gn`Ccm#vsQyC zdXo1!j!Bz{*pt|4G~ax~s}iDV&fT%JMUtd+)zMmn7isy~mzLY2)r$SAy@$SPku-iD zqCVvPFpz^iSCz2xNX}kv@c*fE^mC&%mBeq=WH3? z>IU{oamq;P2G)Okto2NRH#1I-z0tcnVezUtC_JS*mS;do-1+%OA8j!_u|Ob`!{Tn# zz7nync+1WxXtRATPpN{p6|39iKr|*3PbL#R-S-rlp(I$_Yl8WWr-IzLUa8;Q;^|hx z0N9^mu1T!YE{Z`a4(T|e|3MQU*LdE*N*Ld5c&qCj^5DnifP4hI%-16q9;I;Lmo?S} zzM?1yy>lRLWb)=am-5Bm=_v#ph3Php6*@S}=@K5Y9V@%>)VXktPQO0*P%1EwkN5FJ zb%ACx94_MmMzI-tyq+EuZ}!CD^CQiw*DB@Xv?E&nFVz?j=FErDwk-DfPHf#;)_w?B zWa_O7Pd&cqZK_N4FNc<`cF*}7iM-OO_ZZ(Kf4oP7vc+@M9b_jD8gl?7O9w|X^EFgPb< z>v2t&K;0T&q+Fp7;P&{&m+ib=z4EM{3i-O;oAh95@(%UGc2yT%nhz4~q)6=L3Okx` zG-DMl!>qoN$^#SUYjMlFx?+NIXzzn+Ivp+5yYgpWpIs6*1u04lku@q6h|(9&UFCUf z-@V=QN{W#<+T$^)K(@$kEvSSRNZch)P&&659kJSZfBfjWXLvAD86`>Ly8=bkC|yuJ zzStoQS)Q%;)5G}WQc0#<^4fj=^EC*n`#HSS(zpIuffLY5a!7!b;#L>bB%vt;`7!GL z0>ul(Ds;LSA1YsE74-M{%J5sK*kG+%>+d|S&77lzTotO%M8G%v~=EeswFJUM%0O8`)?H)SLxawMb!qim^R(G9f?3TJ1nG5=+J;@Ad&K4% z!4ct>l8};+?Kf1@`1<~F2X|!Bm(g5Q$t`a3LM3t2XS0^k5*PoEGNt7=pT{d($%&hZ z^ZWO~uCA_*=5|jn1$^&_g`hW20Vo@6($@}cs_tCE{^(~cLXRN|0kU?8eazo67$X91 zMIJcqP)%QCX%4=LIITt7-rtW{>x&V?y*fTRLgvD_Pk(PCAd_rG!o4l@K;^IFYqixU zw?m`K#h}Dr)m;N`@ONLGSKiJh_t_QSqdL!XW}mAq+Pcr54KEJ9MYi6TFEIrCq#{T& zJkR&8eh^l~A^e8RXG}mN-nA@NQZNlT2IQ@_ALDQ;(}jUrIGe+YxOWdAg#31QdyU)3 zXD9KqMg{OByFLI<5^&^$9W3wnvf9!RkWYPbM=2WOD_(z&>IVS|b@+J*)Xz^`*rba} zP*Egd7#>~hHS90%?1TUXr;hP)EXR%E>ddiEbiZadK?Nv#Ia;!_Aa_=@j^`;J0`(Ei z<`5e*C{U}@Y4T9J?)?ap0N{*X6uA^$oUZpc9XFT9qd>LI>*8BbR7-;SFA!w#^d1Y! zseDSu2C0|ng=v)Ob;%+p?xb;8O`UJ=0Zg`f$;&Ng7}Oq9OyS1>vN-}@9&ZFiM4)?| z@1X&uB4VD`aKtiL?n0?Qu}(P-OanTIQL_f-8p<1O5{ay(AT- z-nF|Ac|=^KCOX~lIc-9YH%5%dvSl*Dk%RXoi-q7bsmP&oN$jTYn^hOf9S&EzHcdLL ze)X%|=-z^zoy~w0UO8KMFTteEYBdoO%KfU)U%^xsyL~=q=S$5N*9{5#_&y{QWYDuD zmoOLzHbn9|n>DvsY~)E5@-2m)<95HAm;dF4VaPKJ*q}tR1>;_mc_3+I(Qg*rT7h4( zN`jSANQWj6^u`LJmFDMFkru|B{m=l!Y`S$$Xjo><-5ou59{A@y>(B8c`r?&U@Rlz_ zKtKr(8^(|2w5h(z5^|Iif

aSb6gN)`AECxr(tsdiOV&z-1j{J+s|J$z^{mw=IV z?JT*{Io!QEF#V&o%6;JUl*@KW%(rZcPNT5=@Z$fF!Sm3Jsh~vD$ord*=(n0ruzaO= zFro$c06@U+EF34d73SV~BAd>>%j3kprm~kZGy7nHpO8W~NtJhpr=_fGTfX6wb+>&} zlhN(N;@L*2>e)O%?*!Is@d;CC>PffA;4i${sOkbX4Aw?h#NZYRoB3}+TPPZo&i&Yh z@~IhsDjqH9WMhnD?p{=Dp_-ReXdY?H0%+u&td-F(U$3_UOnAA?h{);n3&b=Hr|I+C zSDV<$ysnrnp*If_pXPQ916ri3O+DTg04^Dgu8FkmwS{e>^qwHPuh=D-n2Z1g7bMi1T>W@5F?E~6*~zgC=IWtjGiV=ONt8q&v{$_2 zzV-PTs}hCJ+P^P7CO5Ac^zaag2^Z`JwI-s(Qz-gR4-ysOD!=X&nC*qPg`H@wI*MgHG3sx_&b?V&EnhTHVW&9t8sN0H$AaIhyQ;Gy!tWDaZ>|6 z8BHN^R8wXM(&j>?^L(wLrQO!H3!fwU9OK&av>BSpv~F1(*3x{#dRht&-=%8!?ycZ7 zDfW-_vdFn6&`DAQ8cPNx^V6{Sa^I+%HLjF;@dtw$*{G0 z6anYg>G7pb3y=z#e^v8GdDI$3t#}LoL?Dx|U^*{R&A{N>HY^~RKBoavE{@C54 z1Vi>Rh||^W%}u#U-=o{{ItOFvpttY9*sKx4029D!JI^ifXt{^XS9~amBZ}Ys^|8sY z&aV&=V=1(zKjUUd;hu;z!I;_i&4jcGGM4lT&L<#*gY@ zRz$T6ve@01AsS(G$ML075fA0V)+56|4SMR=d1W0Eci`$uc2Y3H1#b_y-QQe z!Wh=6^IuZ8nG(`=ArlAm?hbMSuei*w_K%zaWKYwGa1*N4Qqo;bOXBMU{ivqE7CO`9oQSm&eqv{{l;}`%kQV3Nkg7;=Y4&a0nn7aQB;V67t62BX8gMb7|x|H`bg-ZQ|T~PGE9iHW;PyI9cfRyM? znonA36OYwV==(67#;LPFnbB*nnqDcj+1p%I6`<);-NXisuvfG_lUXpm678iXS&5Y4 zZTwrjb?0QQhYr*O-tY^2ontRFoGSBO4q%r!#SWZ$l23ucmGl(8+3OwllXAl((c=%R z8Cv+IMaNyVnxB30NKZex45a0k=8~6Z*?nQ#&&wF!-`cv=%UiO~)A{^5!)a^m?<0n0 zk0M*Fnwy*Lot!#Pw#FSU4h)~J4_g2B7+IL(vx=39$J>iXZ?2~nu0492GU%YGEpJFH z^=c?43twW!n6a=~Zz~euV7yNCEyv2NJJc;1VKp9oa*BOj(|q((yk!E2xBAW{X?pFQ z#`8un*J;aGbjrQN%EwJCyh|*}5a;lw0xOe3(kOH-Is7RSQy0mth;IHngzNBKLT z>x=BwhDs4N9-sIxBNW^v(uxrcl3o-lBSuUnp#)Zq@9TafJbQb7nV3{5;unA!4J8vIJ<1lU>db>?6xfr&4^k;rSty@u zj*6Atu6+BJAv6^8)bW=blDrh|(U}H)sjY2YnUnRp&;iz83N5xB&0h+wmPPr%m0tp_ zOT$?s51$sszzq%q{=Iznm5#^55|opP9{`CK?dP8otu}Rz`P(m}<1=}X-2mc?U8UxLvGPC=ZI3aUBJoD1U~Tirq3 zFE1T8o~iQ9D-xKobR3ut-~DJnKOsK*snY6Y*v*#1GG}fpsKirV&tjKO=gk$E^mlr+ z8##^leAeX28&qYo>j6rg7H2%~i6n5Ej}C72R!?P>SN3~{ zVoBJ_U%kAfhQp1iq7^(+ZYBzxe2d+9hC^JJlZ#vo0nUVIWo3A*AGxhS?^9 zx0Ce|nN0fIMR9H-B|xSyLYK4N%XXnExJOS!-39blc_%+f!}|j2tDhyjNm1 z-=1j~>o(xJ?k}QE9&V)hO2#qnM@X`5EX69>4W?k18~Wwn$kXdL^W!q>1Ot4|mNUrY z3hj5=Uc<1Q9yp88<$MT;wN5hi6v7|UZCtq=eolzn>Oa-?;J7Rg92`&uB`z_iyy_s{ zPgs1~xI=?aK_c@&Nt~uH=D^WXuSNT8&|bOYFe_%Q?~-P6Yp4kK?y0w!uW{Kj_ftuY^ z(e205rx;V8*%qKb#ad^s_hK!%%F53>c8iVgR=Pq~&wkWWS)$@ZeuHj|q;XjzT!)6W z(6`yPq9MoC&cz!hdgmL1jRqd)8HhuW>Bxu68j9R5qRqGOf~8QMyV zHh$c#nUJk>9~RAY)G_36^ItAK)(5P>|oGg^@`aE7U`(XEe)lNb8hck7F+ zKnFvX=6$(sBup+*j8Ez)p7w%m>t>)AQfTQXDeH76RwU*%SL=4Ta?T4iYW)U1{Bhmf z&I}WzF9H%({!s%Cx2Z?bp9B2*y*t+JT84Msz)rK;ekC zm=k}`ipWLHw9+V2b}~tqd6FLRkvhD4%5n%q_%x|goGvTZvlkm9cLpnx;!d~0WfF$S zeR2uVt2o?K_+r2jZAH65ER&d+$bip!vjpjTRFshozE37so7Zvzd>JcD8Q8aA(&yV%8}Rm zs;v12k?=Y}ii#K%lDYk;W&Fz`9+~5;g#CJZa1~@h0VcQ#qzF(r#R}wA!!6kCD!Fp^ zax0J1ZBu7cGe0_meels^2p}C=0cLu0JTIb2BmVT~iSwBpfJ$+hEiT4}x7Hvp9&X8y zd<17PTajZJ0|?~+xqwLtuD-s0(_J5cZX9kd98eY#Yk!5oRviG_i#?HDrF#3Cs6!yXbh`r`&?j2be1{{NprRPL} zU75m2Fl1z8Cd0`D03Lo@jM6Ip3sU?52nho|o37{=t+wzpm$h@brd_*A0aD6f;YY`qGVq}XIX)R3r$lNbgM)jH>wvQIEloUuaqQI}{EXyk?CC9fB z!v}{93qf-evepxVnjUm^FU3-b1sZIP<;>JOGnkE}hJ7FUI7dXZ^YdI#ISNb>GDuoz z<1mEF1!my+Uq$VeOh7FG1X5R`CN8|0j1B0G=w`J*X|cJD$IQ^ltaq%;s^W$h-o@bcQy$j5_ymzlw?6~LSS}# z2I5&hnU9cq9kd~G>eH(KJmF=a2-bo7^0#^h=Rx_gokUn$rc$dY7#ufpNLZ>gZI_=d>ud4>u^k$<&>yNm?VwiSp=0jpC3 zE#Yt=gowg0VAMJ9;F`!o0R76!w7#zdAY3X=S7ikMjrquZP=K#Y+}Qd~rGm{cfrZN$ zI`{_cLIY^PBtuCBUQQPPOV?|mOt+eu;cs84xhIQDYKRBqVHoZocuwfwl@7r{uCf>f z1EgRNARGEpX|Bk?ND=aL^#<#Mlff7*Uq4poK(v6>&c#6cGz+4OOf2AmZQ36@T86-H zzYKc*;fZPdsY%Yyth9z4;PY#~JZGeWFIIx2_%c{y&R4YeGh+kv5G!Xs!K%w0w^A<uY16XTZ)Ai1r~>;wJ$=(+mu0#)qf^0Xx_k7?N>H z0#`cFOmC_HUmzM$?#|Z?F+_gfJAS>1j~{+|k7fuEC#J?8F(SejtKQqr1|K&B%=zR? zsV-kEoiZ4Ddro$B_QNWx+ThdAU8jZ~(0$a*+A?y$Gg1Q1Bar|ouRHnt?p1W6Z;_=F zSSiAZdHeyrP!Auh`p=JxVBr0={(v%bTAIGjMA~^aWC*FQuC8@qVZmtKbYsbAq7f5* zB}W+0`BM0r3tnwf?S8FBdHC?*;h5itpI;5pd;w7bF%}GYDz{3)0b4;-pb;CrK?fKX z0aX|{2wD!T-ixcNulDkqPJiz;kRM2zlSrVyPC7eRkO;)Zf^jk2aPi-C#KRgEV1jmU zw9}Q{QIM5{Q5f}`i2*890ywRDuhGj^OiTT>E|EJk?|p@mxiS}0avuV$n7WT%wAR1_ zF)P4wvhcnbyE&EbQun3eO3G_dP4QUErhiFHQ@HpcAMSbCL6ZU>{A ztDQ1nllJWGnE&-fl-AJu-4ruMOSx73exI9+GyY6(a1H+adfmSmkz7U(6CWSn;`)OT z+E2Gvzh7M+I8j3+U}W!mMgM9s2#k#++X|F&;Zblq0UcgQTL5B^GvWEKGqf0Dz~)E^ z()s^HRMvVfkEcAzq#oT(&uLcoSLZMXPNE7!rk`add%GVFg>NdNCu z4xgNj0r!`Uy9AQIQOtbJPn2J)UIhU9Z~+{0UiSmSzr6$KHlU3s_aX}+8H#5>tgpFx z=!jb;a@g)ZTCZ#~s9nEkbj~#$_99n0&;gcTIai#5F_ja)i0enV%M`&_j>tybX_j}> zXXfJb$-H{`!4GL1YaPQa997@}zysc3=>jjw{{@Po$ zk1pRX3XbhxTuQqG6;mUAmQ0qs5k2u*PPC{_8a7fv4}Vqq7%3;2@>Qz5!D%Y;PLJ%0 zhl$jR`)x4?o9~3?E6hcJit6Ew1jxh5qEHbh;4Uj~^{86AFeN#5hzNCml2q%dIZ~op z!d(2s=9|ND-|=ywq=^#0CuASo_vb$lxflVbS-;Y~yYhA`+~ce9ojxC~%q0QbRmG~D z@vqYfPIo}mjQ~706Ch#obsIQe*iTR7ROEzR^bIqt)} zbRqJ`sgxu)Y<~j)RBUglE2H}}rU`;!HTo?!WWYI7ykZh&y=IuDsQ-()w+@Q(4ckDM z6qE)DX;``yDT$SC1d);s5kW}_B^K%KP-z4u1q4A_>5!0??(U9-^Q^${JLj8oX3os{ z>&)!R?Zjy1rSRca@nf9_UYwxUy5H@MD}D=| zE&iPIZ&J?GErCycwVubC!SsQTdMzF>Y64P@{-pc0AN(o>cLwgh zs?3{SU%aR?9Ww+i`JWotlf4w$VwhxD?mKx*2ZuE~GMqO3{Q1*0ciCmO%dS?F-H52J zaM-u@1p z>4gCK?qL)SPvM{ZrSU)f2h)}6`C-Q?R%YCFn)wr*6;$VQE~ek7$kgg~BqS^5rYK-{ zQ+j%Ceq|!=J9E3D^wHJcl-^MP61F0*t(THpm8<%}YNbD;c7^xZpQ^J|aXA1ws^G6~ z-jx;K(hC}9Z`F$robJn{42cVg= zB4XJ5mW7V{-~P>y(%IY~yw&LAdL<>Yyu+xlBjUCoLDIfR`ks?PRF2eKnN0B*=Or_> zqb`isyvpVI0hhykE0M{$x^TM7;;KlY~cIM&543?gRrwoOmoyBS^WMJ}xBRqVf#R+&1oWCLsoU6w1F zZR)j7gtHMiYA#O7UmWwUI1rqc@0EDIUWC}3ROn-K>iu9FUXEGhd{~fs+h_gWqQ6t= zA@v)r@os~Q+ zIGF@ppzcPSw`RMiXkHex-pQo@#jeO*Cfh($M&py;XZ&IC<@l}lgLK~E$`pSXWOhFN zT+!1nd#fpXH*tab@8J_l7n8&Dx_$chE4>*HRj7@lte6L5fsfA~GzLrEyY*)49SGfA3btz);+2CQI4-?QS_ewkfl4vKNh0c(m6^0Zy6E- zc&A3`{=Yg*ZH?Sd=#%<+J>T??SYbXZDud}uxRYIFx4UCRF|62IlOfA z$TAPNa)h!ctaut0hF&5bi@m-;{ats;epe=RPZ(JzLwRkaOFPv+asgP$YAH@?Gz$g0 zN41V#b5Eo;`@YPALc(-=*~b?&UIt<}VXBeBt0iA3gnI+<~5f+%6*Pf1DS#s-z$>g ziXHkkdnh8rYr5mE|2Pu_!gX1rGNjubX@f>fee>(3As#OXCEvHxSq+^te$grXRO@v` zji|Xuywkop;zeaHBD)Z}ysc9(4@w>%`wI#qI##i=l2QM!%!r?)@P3wn?d~P3TmRp* zTi(W&JX@4d)JJwf7Txt$`6!SM>hhpfhfOs_Gze6*b#$?Hsje3IOKc$DRo_GPC#SAZ z#K8@ipQkmGcKRt)9ZB|r`{8dWgh9tca#u%gAN8PEMvP^vH8gD-hN$qn7i+M>uGTWo z=K5S=A`~de-A9z_8ID@!DWKlbh~Uq8ycF*Cd>?STY4&}^E7$Mn)n=cXH@3PB94EYO z-9vNyyo3n>d6tV3uLKBCxzXpsAfST?2C78>De!8PA#_!{dc<+8#*YyAcqust(I$EZ zF~oRLVgm7@=od-RQ*V%>XTZEsUH6s%U+*~8UsPK6!zrRZ{1X&s^pgcn$Ugl$Ra$hAtZ;;K1Dc_A1SP9IKxr}lv4sZ$_ zBuH?|5N#f$O-IY0Q}c^_If<_R;(wG50Z015#U)3&UKYdL<2Hb>prGapfSsl&Mmd{O zW=@bSBgF*iUIDgr;8U-ONn6Jx8h&IK2Kz3 z?te1wIno`Tw@hxpxmJgn6*?-L<|>c@DVVhtc0B0aDe{E@lp~b-0(bzakfs(c^; zU`Y$eDQ!OWbAp%uHTJulj%0Pj5z+9 z(pxQR$pJBs8F0>uasM|9KrGJG^($U$G+PZ{L|`q@mLRIWp&|*Kul?pvM%V9FfgnPa zimAN+=&++5Xc!2{sy+SEF_}4gNS>c<*s9nKiZ;?8nT=N0zUDfwh})qe^>`2g5nCoB zGfTbcIpW=R7eENZY$EzKkUYa(?#mji`q-a?H?;PC|FG=od1L8uN0hLX2q@d!03lE_ zs3%i!xBS0n3fxjh@H^d9O|A+wl0EnBr0ApA4gSpoaEfg1U67vIJfxEgry>U(kd@Bs z!FbdnQ-BBrS}ltGGQNdgfs{hG?H)QUX%OS>sc* zC!g!&<79`=M$zX0*H(?^`Jdh!w^;D|7n6V3QAn|L{@hcqaSGU<&Sd#=$r4D@8klm! z=omI{xLCedx_2kz!2-pTG|4|e^-%N1YaV8tgFmB0fm?@O{wTTMxB*`PNap;<7eI2R zfeM>eg4E957EL(H_*2G+r_LVXXN(WwK60uVfQ*4;4%jsy-l#?5q3xcfABZ}7}LLLp|e zeQ|$Z@*RfN{kNs?iqLTJ+mcawy6tI2>kkx?8lJI31$Q$cI}=ejsHb9Y28GMLp|9dm!|_?Ox>$vOeKK$mliR8jSVSCb^S;tK?m zh;q9B<}&ht(0yl53{YsASD{Ni=K+->wj=KW*ML9oB5T{#yjEVWs{AoQsU{5AHUUy^iWe zbu$1gyATm0&kRr#OAnRijWw8%LI90D)BS~U5w&B>iQw75}&@5%@(9&IvylAP8PGdvv=nGW(BZ-NMM@Lq6LDu?Bs;5=BMFZNnDRb+j}%w2lj!?= z;cqSVjh70^R(PHz`>d5A^zNO^$~*TNQ0Nqhgebnqx-~k#7!^2b4H~Yc-9=(-#s#SJ z?Qwk|IgEqvl!F>ja&ukVpx(3ikjnxBK?17teNliif5d9S&QF0vb<7-G8c#Bm_7#2E7l6KKlIQ1DDDx? zS{FS3gl46!-0Fjb$6Ls$fq*jOq%0=ZZ>%QN4Qeuiwt$9{oeqpxBIrXiuR^mLJ;`h=kayl-w z6^hI!3fBT?$kPdFC3Q*ATVg>kSWA;^yqGs@=< z6gvCaf=c|Mpeb`QgYY(`Ph(x*bKNAf8Zfe{7ipVQ}?y^zSF7!y+;)Hg2ym$CM z`GZ}#g6fcxlaqbQWNGf2;0D^^f;cV7)7(b)x3PDHE4^6y^Yvw05?m;F4SyiKte^!k zqL5nR1?z;v()PF-hab`G+7O0hG?-@+G}d^h+Cs0(H~LKVcR~D3v`B+hG`@yb;`QC1q04B~tJqL=Ae4Uxdwe{ZD#vHsNu~K!=MEXSfp#02 zk?BFAr~lj9#&^!tLWC|GV$!&AX=>G&8#&UEX#F38Oyt{tJ^5JH;iYmrnt){Q&gU*e z4PglTY@c4--do+Tj~BUTgWA365rfLhf^CCXH}#@CZC2{64JNAz;oC#0N){z9k2j8b+0^>2Fo2_!7LPmX zKJ$ft*<)&cM_In{qDiRIA+b5elQh=`0eY+wX=GNaTw$#r+H$a_DQKlt6*O5{uza8R ze%_;hbw0gMs8!X|4Hcd_!$vgV=wcZNW*_oH8e{23V<6ZSsNO)oz_cZt`uX2dHc}qL zwVRI@s1)Gn^58?)j-E6KA+R9Qvayr%V{)S4MVZW5+s+<;&Vgq+WFYa_5Q32H?2+Rk zWlJ0?w7^xXNcGz8lap5b62Ns67xF!=lMnn1RdPK@w;z3yCw%SDdFu?wIUsBBnQB2h z;Qq+P4sXJ4EO(7q>!G6yE`*N7NWY2kcXF|sH*&yF(%RSuR9cx1S=t7|I4|KR)7^|0 zUoz>9Rw6?VL*CFu+`Mf}j~ry8Ko3;6-G!z|da0?|*-T>8_H8&evA;4N<*9}S>g7Zb z1sb64K>`iVsgltr6yP+FP3gv)$gP-$U9EX+0y+k>cOij%^+Q7m=E&Fo_vyqE8eoGw z+nr&$+L)2>5W-063qV3%(v1Mh8Gys#!iSo~ef71pG8_{gc)Z2+-{ok8k;4}PFaIv- zLy|>aJJ7YUv0r_slM2Cw3|e*){&zPUdE?GRz7zR+z!Kk;hEO^zKDx9O2WoIk$f*%y zh{Dpn7tGIi^sI}|8CIUpjAb2}%s(@*=r5%$vOydK=I&Bo|9B`2;P8A_f1u+=pme%* zv~3o$|2zEZR`pTXJ&>u^JTjRPxx>xS$Cf-Y#ZUV4CMz9j(OS9yEPreV_}e(CJJgD# zYy|t^!|$%fBSRrg_uDfh%KvJ-5`z!5=+az)egJ&1PUS3*w0ht(O&Cbb60VAsNcRxc{dB$j2yr;skk`I=`7LznF8WMWr?P>0 zM$95tgI~u`l#se4)Lto+ge^KfL7Uj@;Hx2q$?%-5{o#a^sOuhj>R_R-e`-JP#HoY_ zW~%x(yVD6Y^cy!A%_{lH$LolrS(I+pNwx6Jo<4bgrmGSdoyr+Rd6tTi<~)GkbCRbP zD(-oTU?o+h&foO9SVhD7hjQaI+)U3Qk@3hG}P?A;>f8>Qs}Olb%hmW{>S=m9B_W zr4Hu%O1B%+HRWxRySi9;XZY=`oB~a*6-y;cT)7dA&FPijqgxbWx2m*%OY0qK{*~wT zJTOJ(MY!UHg`+k{CZgtxGVSqAF=5sSxY6~T`lGwMyCbQo$OfrxTd@%m31IlQ zL+DUW8R`ijr+N|O3Nai{pXqDx9eM<)wE$aazu){abO+lFJ=${nm;Vlr&+X3C$D0Qo z`+(3k6)zN(e%5i1j|61(5SLgvcuwiX$-RQJjvCezgQxfF4~)(o3;5YCvJbPJ&s49b z_&f8|!h%02p0AiR5l;aC;XA4KY}D0$A16pdsecvEu<)*x`#e1Yc~Q!;cZdHA=kCx+1*G=pR+#4x)atWf|R z3DXD$4d$zzUWAQu2V(PXM8GFB@kES9!YvrEnU>MrYaZ}7ah2uKjzn|3eMTkjd`Inx zeqD~dDx8rW2hgFFv*mO^OWygTVuS{?)427>?RjSOdGXZ}hu07#@|wJB4~apQKK+VG1Z(c3 z)tY^O=iB2D9`!Q!9D>)bQda|gAWa82NaPQkIgt%FG-7YkKzE0HJm-6{AbH*Eg|UJ@ z^F?`6@k!`75ji=QgoFe%l=g;&!}-ThZf7Qkwce(jT?k7hqZ#+8)){Jf})*> zrYaE{jj%)GYZJSLOIyIL0L)&3{=t&UNqxH6FkT61T(A&ZU7)escRr%4A9 ziJ(sbdhpiH&O29(Yoi3uBQVB-=jHP+E*F8vm?~0|`w4v8S}d}|MjeP<06h-ezHAjK zPzYbw)Cj)*#dszdKBQyE-1f>M!BCx5VbK>sGdvbOX%7K$BR*M4dHZ^F@1j446yeAn zNM1b;bP9&PGW0c~Tl^F!f>t8?1{|$`Qikc@cmz^_g=yLxQE=sAfqRVoSEuH;(#SWg zxieaB>(Zn9$mZH(iK0}sqY$v*MT!!>B#JT;A9TV%we43(Rzve~!%5@d6j%gC)z}d! zUnC1^7Evr}G(ZH5Du|fn`wtwd$Sb35BU4|91Nr`ezl9z}4hnWArka|X97*zBqz8tN z6MCI>#c#0>TCij!Y7Hn_9F82Pq3Xf#wY-H zNHR+?4ynJQ-Bj;(84w5wgE2~S_ZAkr`g=NW7h)vZDJm*zcY5%$GgX2Xkoi1Z7zh6a zSm3G~j4(8~4;OjUGInyNf2Fhl|(1qxc+TeDE3@O0zM%5{6RP^^`L+fPq2OJ+I5t%<_98ygA$@x zpQ{;@ujj@Bl<*sX_*CB(!@~Dw?|D^ZkmR)`V#4A@uO}iI`S17Pj|qprS>oqD$-@MM zIOGxlF_&}hvHji4Cn`k7JB2?slJ5WaMCj;9o)fSZ)1dupsXym2`u*eJtJ_lrK*S(a zmIbVA*O$X_vBzrOl7Whm>(&%XJg4qEW|a@ac20Qz?HMpYkdAv65&ttlFgk-uC#1rz59UR>>t{yD&M;X%C)t-nuH}I(7pFdmXw>KbnIPxO8O7tZZ0s{vdx&QVh z5&BIs?$BC~(;0C71TYnU{2>X6LM#gZE|_vcK2aspyzhUv;8ZcLhobaeneGiblGh>|c4vCI@ADFHXnP_i9M>yU2Gw~Qh2WpXuG49(Pgkfv#|B-EeA}@I z9%3da3Xz=ys^F_J7?97~2An-4pBk7zmmLbnHE%r5}>6on|Eb2{w}V33m9P znbYP3HLw}qR$SMTt{#X}p}tTQLe!{)6%oqBB`k3ugNmm{w>99H^uv*44`O$eHHW-1 z$rql4`3Q*MAr%J2-t5(by_IDCED7`9YJqdZ?hr>Zm~T3#oZW$YUsq ztU&`JK-u$)wa}{@=gR;?ZyjY(_upWuDy0GGjH88*wge<3z5ocN6A~J>T5J!xj3R{a zkw(IjOPKg-wCGSMiI6N1fa7NnQ*?j0-{H^b$a+1LG~nutmau~VrdB@*2AF}MQ4>eS zWUAh~!sGON2=VmZ<=R5vjqJqxhvkdC@ddH(NI??G_0#);%g=H^sn-%^mu)w-;&MEVPEql4>M2|6{0?l9Mi(sqL`3sg-& zq~aITo!EEjK}HY-@GguhX zc_pW{;a-_%zpbMGMQ=hxU?i!m%2+x6yV`)59Q3RG`K%UUo_eyyg?NAA3*y4yh1no!*Wv2e!Qz~E7Q zy(S1yRX*@PEPF9vw>crUw=xj&=xYA!Utu+tQz}qtM*baGo!R>l<9f2j2GrH9%5+J% z^ih*f%bFHnQ>MxPH2tzI|K7*`GtC97^0B;86SxQK4{lQQS>Dx}KFe|>zuQd7_V|xDJ9-@i1#c#9iom*L@*oi4;@w=AVL{i@J*WD;mW7QS}cESsaGXW zb8c{)Z#A1x@f}i#fi~rC3ap6aXk<5+xUGhscvRuR4~tToJp$ql=r;CwV&7S_uL^vX zEDNU1aQN}!q<1^U%i3aM&vPO}S1-uuu1lCFw1vEEwHIeLK=t0j#19phm*`6p8G?WL z_x}{0-lgSBk^VmR!{t_<0B6I;DZw|37bm~1$u}STem-GsN9Ocud|W5SE7m^k6pnZF z>aRv-SJ5_Fk@K{c#aL@FQ2I6=+ao-(8Wri!Gno8B+zXDt3M#diHcu6CaZh|g#+gzu z==5FSuDl;1@eQd^*6WQ#K!Lhr8|I6u`PGmb6hHcZpg7F8-+rH+e;uxA(NF!MhoE$$ z*{T0XNv(2~g`s%K#VeoZWs_SCL2E|KmlvKdMMI}Dk}PHNM=B&K`5Vb2 z@5;0j6d#Tr+IT-T74&J!USA#9aD8a=nC~=QN$~3gt5J5eC+99G=_fkEDq7{fw+dzKeIRogSPZFO zT!_f`Tuzb8Q-F z=Dmku+)S8t5j%}X6$_s~DBL+LckB4J$b%XxKtyV6*l0&6QETCq#FP&cci9+k_LlIH zDnF;@&A1SHS;R~Gap=lg5)gU=2}Ls?Uo!mtC1pI1Q8;KiV$WD{zYZ(37`IZhEz9WF z%FWd8j6bfD8e3LMU^R2U`Iba|)P(2owsk1EfS{dR}e&w*8K9K2uB z)fAnM6p1T(J5^9wBB>CPHf_2IGNM^q_UO;kU+eJcwGS?SF+o$`{djP$HaaM&idCc; z6^Vz963LKGJ7fRou;Go6{Yqk%zp=#S8*)q1d7SJKFv4Sjpzlmu%*A<_BO!Vc^W`)q z69i_G{L+y?Qz0roK0c;@Sna#js`DrNIe;^KQ5a9@mR=LI$Fm10Y4g08@2eW6YgqT{ zJHSmuAo%<9HROI&vVddy4!2C={<3DST-YB$FeA3T{4RygnGhR^aO))jg36z@e7sL$ma7H2EIaUo0iYAD&FHJK)#Zu(;4 zkk744=q#)HhJ*Rw?P-Cu*L(Azy_WY?z*Db%i?@=S_~yASdK(I_ z=aS}JSClp-XLC1s&av%@8ydUkBa9X`nt>Mj@FfYu0JNpH>e&!q*b((ugf{#8+1}ZJ zxb3vi9lF39o3Est0{wmM0<|Ug70Y510x7vU-Zws?SGfJs0M$sW<^2P>>Fl|!z0#zp zhQZw_SLB?4tYog6OZcYU#W^DK5gg!Gm3Q&FlK**Ux_GRLYLxm$gV?9SM&d4A)g~LR zN|M#L6xlkXxITx+2$qGkAF`ZB*snJa*tG;I8ch$}_5o*G;_oY9ObRVcFm^Fv?`on=Z%i~lb^Y9a&Hx!D=#{_>g9uFh zrbotG^*|jIM8XvJo-GxE@~Qp$-}}}$4dT~nnlcD*eS`e)L3V@H8mZ6|ijqsV=Hh5&K? zdX1nuV85}?Qf>P*9YM}pB4~(Si>1!OF3686?MJ$bjJ)9pB9NmKgKiu@KnF70n)-Rb z!OC=%Biun%wS+uB(j|WVNmU;p-*RqI4vVcm{W)WqV_iTE>d7e&n?ML9lvh0H7{1q;Oqr-qBo4p^W}m!YQ}JMyUl+g6a_ zGfAhB5VJuM7q%}Qoh)XZ&qYI;VT0XjmkO1{vEYd_HF<`rlkIc#KyKmh|K918e(U@c z96Sks`0yb})Pn~x{Slr1(UXub9!tJ`P#j@;X5W z>M5FL<1!#DQm>Sz$eiH07|mL>Xp4PP^~OtgljE%PC0Dhc6ZT`~%=D2ausL~MO9er< zf?p@aH@dfjNmw`E_<=Nlvh0BRNajZr?W3wzdjbhhcM#K3nhLtjcCT(D{XA-^c zXQVh|6r0qu%!n=S#?@bzt~^``rMLaGb#7ijUn%Sr)|TG(mufj|Q7SXM>&)<^+;Me7 zel|s*v(V+N1o!cx3haF}t?{GH>B{g{;b$wn`c?K%x-a%W4(X{*lg91jP4Xn}BAfsr z`wl+Mcnj}lacjbZy56R0nR~^DUR6;7Dk5W5USuA16oICGO5~f}NfKe|F#%uhD}|k| zEM~YQoHsxCF5?=P#b+%4;fibP5*4YmrvpRAwU%RVJ2iJCGJtC zLji%=`teH^>3ERx`MYx8wEiK&RP?HsblyKz>gggfayyQ>2ve7E+2&;aea9y!a{)G1 zPYZl)WmujWtULY7W5g_hho1}M_VnaY=}@rSNhyU>nZc=N^MTkXuXC%p2akW?=kBw( z5^?Ftklu$R#_#K5$LlugClFP9MT@*#0ed56#w<^~c){MxTWuxdLk-lJ$j%{aPssU% zLwT+jL^@LSrd)3H88vRoVVp@;h|2@clp4KVei7`&nNN3F13OMT?QfOtoYqYqS~YpK zt@yEO6p;s$a-VB;JUD#Ck=f+F_qe^xEvmwe&lrbh7W7HD;Pf6)((=3{=c90^5v`6; z^%#9F+{Ei}cuq|vcAntoa-tW?e!NT4GsGn#sNo;R`8SZkkqo2GvF^JLuSM~>?)!^b z&fi(O4TF`nXH=z|Bl-m`Bo@l;SQOtTi)a#sY%ryB5-QF;rQUuyBmTY zBClU0iM+lEB8zgbbT<0SXsChE?m0eWWZNq@R(qFvq>^^HU3rFd=TVhB=RC%|M;%t@_fRPv^GfZ$j>hlpa_5Y&@Ol!1IjZ(QCE% z6tX6*ou@<0Ja2SI>#L^HZ>~=dKe=Uk#i_d)l{~jP%N0dcwrDzGBER=Mnm(X0Ddfhl zT-=UW2eDR76|c>Ofzc^ue?qpIdAsvJ!b#egxW|c#JLYV5GyTN`H!GG6bS67uPhb+N z??tnwJt8FyLT>ow=-0j>G_4nUJ6-zsSS^6EyzrwT#UzhG4IHyT;^&NTUfHjgZ|H*$ zMs8W!ZSS}t_{7LITGS9>Gn;Xm+X(#amT9zX?PizWCKXIbP{-z2_Q#(}&d1i;c_EXY zlXegq9*xl5#NJW63+W7C1?DkNPkL( zEzE`x<6h9yKLi7PtV53KYf@x~ywTLnPui9npBPnBbt+XoO9n`T>aHALC24}EdY6pb ztBEgR6AENJVLeQSW3J@)?NGCG8{MMy)j(u{t){w71P-f9C}dRhyr8K)5vk_ECWdPI zr$=raZL}BdYEWQ)3M`f<;!*q~{+Li8n-^`nxi#fJG_OBL!}?fx|K)MAEad5*1s!C# zW9)`{&$3H;`j=m^&B$FOMgyd1%KwY8>a(!VK92~|VUA4K=8d&9%};2WTk1I5sbr~! zsquRCa`qLR*RFiFU?pSkl92Xeo#_91%0yal)HU2Be$cIMqrSbb&PK#hr?EcukxUrR z7NA@U5e(Jbo4SM?g1%#XkrlfUBtEUNprW;`<1(UDQ89zU(?De{V?bZ7l98dX0j7i!}u#I(cl0!#xo>qQK zOJZ7#04>^=IWgfrWkP1s3~QF@NN$DuzD6LQ;%$xsmy>x9fC1DFoCjFkntPIeJBBa~ zkj!OO;h^B%_(7B^@iGYBL3dc_45F7h$K72WF3!YL3U#+01W+^*`~+p1z{R}wsyy9t zf7VJ+Q}bCn%2tVIQ}lUql9n5eh+kVFx&Y@Q(!@D!dd%OcyJ4XiCsnxi!~S%G3x{fi z?2vY%;nRrO>*z{bO>%e})7zCXY@KqqxbW1A)g$^dw<8xSvF!vcbeZKn>LDY_+fYx@ z1QGT}C6!)P=7rIMh0cO%AE56w#6wJu8XxMJB$#^|d5vgvH;AU|OB2VC@1Vbs(=$C_ zB1LRX-Y9n56d+-LL=IZphJsEPxRZkV7F8cEa8^x8wcx`WheMQw`@710C2hvOt5&5= zYGFgPU27qKG;Nga)-?l%TeFQ(CYpD!F7AHRZH!Nq+S1 zWTbzfdGh`^f6^1|vz;%s*_{+$9oa$?`NeiF^^f214pa(?<2YTmZ90Sy49k0JX{ z9|d0i0vS1IoWbLN-UC}CDDo9uyR2B%zd?r%h0LKvib8f=g6CXJuEAv`Lc1dT1c$X1d*!cGs?`Ml2E*HMbz9==*k;k3NOD}XK|KSYP1tfHXKbz8DZ zPidwafSr0pF3$70=%%I+f@l1w6)S3Zi@+aFwW344LIMJA07>=`r1}+?p7gpCyp7WGUjO_c)o^>oL z$40h!1A6&S#OYx#P%lH)5Mr2BegQ2*`XE;z!G&;Uyd@F!I~$OS)ek>_gF5>2b^9_H zIrS?ZEiOc>p6)FZ7i`H2=e`9kD|&^%L!`SO?~uktC4*6vMzWcDEnDKD#>A-cypX^& ze$-*}A3|722d)zf128K#V$+drW)EDP}D{k0KdPzn%$lA)=9LVT3KOTCu_ z;Oq$~DEOFHmk%}p$tb|-!W{Pihw~MgG+>6{F}0t79`M(x=jZ2u=P(29F~L1w>SY8) z1--gx=|&FiLY%xi$Z2gvMF|kc3ik!Binz z2M1a$9hjL?0eA#MFGxp;E8LIo0yJr_9*iF!==bmkJxI%^U@m6BwizzBOvP+|ySUGeAO)T>9Cvbf0ecS5ciHazsXu zQ=7i&FLI3xr4+3Ekpw4s6$38P+Hf%%FduRNiln0_+Pq6Hub~85gjxyPHIR%I)285MYfrG(ig3Y- zwrO~u83{KaOe3r>jQ9D4oC26HIuzX)kZq2@Iap;x%x#On0a2Jx)|OIv1J7&*cAIQK z$te%5LOsj&j)S5P6ax?hB$Pn}$kVp>De$m4Lnb8jAVa^SHylWz;+k|4%IK54*fqU* zimcRb^{GkF8&Y)WEknZN#wfV7!j~SHWIV;0i2Te5tRSBD_ zz>#j1Rw<%xPLkq$%z<~gWpD^fR%u5VQgFc;J$=pLpeyO;TvO5ORV~D!v<@PWP+GN= zm+fLkhWF*)(xTiQ3b69yuzS$z+uvIe0>gvNk_`P)TeUna6FZ5*s@Q$495zXV3~ z3!pYat;!1u#jP8au6Gg@rLpPeW3jeLu@HoQ`l_6~Y)1~C%mM$bRPG%)X=5ay+ydo! z0o>iLO+cEz7weUy52&x1VOHYJXs_voaiK1LByDCzz;2Q+-s?%;1|S@)6sre9z-412 zjoKvx2fjp4t;-+aW^UN>ROL{=m}E1@a;L@Q;@NIx2aLpoqUd7<)Ovu4EoWjdHNx}{ z$T)S<8JCXPtRL2?8${<0)rz5K+Lc+6o*gKV6}t-$&ifay%{30(?H>U`ZZ#3 zx3#mSI?By;8sJF)6h)H7Op(>v;+7P~nE$m06Vc6Cbe(Nva$*|cCDxs$ljzD-7laCI zH!B+(CEo1GbIb#5@#ojN^A{Bta{XEp3Ji5XtD|P0&tm=pRPdd!ex^P2ar)x@jTweKd{OHgqKhOrS=o2l&bw)`0NExdNb>Ww);P_gbD| z!oez86?#IU6K$>_u)aicY}V&A7f)3#g9Zbfi!d=l0zX9(ueL`#uf_ojAj{r7Nvusl zmq8BZd(-c%?Q$vel3sn#+^J81F;2vDw)I!~^%n|>#* zrEwC=$V?2CCrI949A^d zTh&BS8^18F>g8xKab>b@0KmfL zxO00q`4~d%4*}vS65-bw1&DC<&VXfUaJY0#JfbQaE{zH$2tkLQKl+8DYt(dCM z#T7KuxCr1|C}|+zEV}6@AqS4f4c}x!NGog3wwdi)zOzRE)QdJ%h;qzRDJ)i}eoAEX z{w)NHhm^<;6P1XWk$aFxn=2g#q;p>TTm6`$f`0Z{JCSfdB8cYizQcW7vg!qnp%Rk? zC;gQPY6Ec{{b)fJ71Tn-dTvpO{%vs(S%$;lDUk3YMS2VO?HLSawUqoA?&H%_saOLA z);Mr-TIB74aS*sTIk1id{Q#U8EvRAwcNowhlYYJwkW&%z9AE-rnsry-8c~LtW>D~_zLur2>p3quV^9yO3X`0eH!s4&eGyifmf?< zXg~wuoFou2R~h#`^#DeExXR_LCy&JHjN;1_>IY-~?hjF&CF!gO^CR;m|Gbp3ef=zS$@g@U31O2NaIhV8EXZ|p?|h} zY#FE=P57ht(}uly+23pY69f%ILnT_-5e_K0;;vi9SV5W-aO2lWG3qUv+O>I%S?`|M zWUCu7W!zn1lM&VN`$P_i&5FwcD{@Zhu@}a#5B8p}t02CRt7raj(7wNtLj`GeynUFT znkSLy12Uu+{e1iMyMJrmma7|`@Yg&RhU_`c6IpjM0!gi(?fZjYx^EHLK~Vv&eR5cG z778dZLs8P7^O12oXbEDjH-ik+#Uo#8li7F&Ju+F(UOjeP$C{dA@0FDfqbRB(AzZ`# zb>|o8>u6~W4UivDGB|6_z?kgQP6Xwu205}GranYW2hgsZfM{o+YJDd9PNh6~;F+kr zz;cO2$DOYhau*R2=e!#YRgvEcnK{I`1^r(&4-;={doBhMhLxotXg z@%mjBCIat+l&A5|8$4yya5>=ET6DglpkY2QKR`Auz8Ip-K2il_Ix3UJxn0L6j6_(QhPmBk1}gxnIj0sv1eeik_;_6ht(DXoVSKy z3QrV!EAY46_UmaC;O6GEKd9@_pVQ$8+D(ww&Sr4q_{DBI7Aa&E^q#ayAzULwxdNu+ zu}y(JdK?#8nIY~MH;c1uSrztpgDrz1J)++ISf!3Bh1%zZ6?bt{&^67Jm++_ z130R-fcy;X7Aehrjsci~WYBuQbLsm#o)i_mgGxB2s9vvNGvDwU*WB_kykx zu1#rmprdH0GjJ4&P8AM!{n@&?1dNds&RgUPL4F8609TnFw3PzZelq%VbXS%Pojkhs zuWjp-P+#csz1wZ%+*PX%gQc-RBaD&;HCtq%YTB|p<-V9e?Z-nF{N4GE8+=sqQRi8$ zvv0J9}3jx0vrPHJ0NiDuep0*QS^wC{b>T@_fLU*Fv%M_^7Xn{NT}lNeyh5} z&IQqR#WRr@q;*M{U%>=tNF#mUbcuq6LNJR05$^;8#n&^iz(*LuG(CD2 z&7l-J<*%rtnN@NF2sV8{qNr;vbfAPPz`|=i$eBFm2%4L05#mKZc(1tH^@Si5G}i8T z2tuE2WWwoX&8)E9ukP~3<>}b6T9Qz@U+mElr~)oqD}1+PFG1t3ItQ*qr|iXT5ZQb( z3VN3xK78-Vy;>#rV)LB@Wkj?&s26WHKX{x}w@ugYw>B7Y=e}ag1n|k^(P2g@IhjmS zeKvNmI;7^pE=%0@(RjzSXiR=B2;DRQPJV=mPm_N`yTx+lqTEc5 zC|vR1@-wzYk{Qcyr;9?T;OLT+DA2Z#Zb@Zms%<7Rs^Fw#Zo}#?_l}hgyd24l`EXj_ zvPD;@g*(NT0K#{mCH&(@$~yu!Ut+e~cENG84^4NZ$oO!aRtIr`;H8f_jfBfaQlSP2 zf(R7dmz=ZT3crK}vrc4$Dye!SO@BC7EQ5o53dEYsgfZx5R_ijBLoUiKY@u5y$H&~7 zUkE^55^cNoKO~ACh6eG%Lk2*S^lC?CwjA!G>tb*a`64t2*vLgE4tPBI6`m96CJ(62#I!G|Bhl4@w@m0iINH z0!)$wx(G85#bTy70_+AlQ&fkwdE{}uVI}FiF4AMswqR#~p-PGZ)}^Z=H;EX#@h-|8 z6jL`8AyH&tbtZv=-9prTr)^|=HbUY~x&S%|cz`*Y`hY&Mp11r#UkETz0S(qz2!=CN z{EtPh6A5=+1X>;HhpcWWbFMJbf0!}K5EI}JtQLw{s-kK!!5HCBwo4rUAszhQKx25M1HI2n9kt7ONMaZO>ZxJ`*6iu9D^zZJoKV> z<4b3-);^$9MZl)=7KI!azszFOD4GHoC>%=^*-R0w*cBGhwQZkXiFX)Z_xP7g0~xn+ zy{_lyiY+49w<~}J-wf8-BM{sI?NJgW_5w&08RMxrCSym~QOsVteOz@Qt71ic=#Ap) zdQPVcdOipNMSGkJ8-oF5kqO`utx=lHjBqOTrc2{(Oo<9mM|yD3V<-^v1DzV6lgfeQ za>A;>MuLxXVqnmt4AFrJv(CR~38*Q?b?6|R3&byqy;2IM38RjZkYFe2G-vD`~MxsTKo`Z=9xRMIO7U6SNi%o)P|l-tF-(n zSbRO;H8i{lx*>=sP^AG5y2%e-MT_3D2dDCxu6SQe+r8@rwGjnY`B61pLI|pUBF$2D$~V-cV$AZ3O@1 zHsQ|U;eYBh{b>Rt0U{MQpngvV@PH!4x6J8~NI_)bcbii%UkTje>!{$42nQ`sD*<#| zN&u&@?Wk8GMhu=jsI7xD_MYAm^oKIY024w$_7%)$tREUC5dXo;^gg-^iaps<$o&ma z4xJ(D{rW2O21SPp_^2=pLt)S`7?1}c*eiXlfiRG)Ok*jf^5J}c$Jh=yBTyv>I1`|m zk`)-Wh&CZ_c<-`MBl0H*MWpW6`2`GbH7$5v8-6FmGT78BuF4Ft-`i2EKI}4uMeg`f0gkiQ2mCg(`wtqB<)4)+1HH{Q{^FIlIvB3^Gt``kuD)?+3{T7=BzMK*}S0MjE*`7gX+YDNpN@c5|;D}RQ#nDkYII1CoajoLU2=N$x*>s z9R#wRY*2gzC8tik_)i$Edh#cT;ih~q?iG?ase)Q!(8Q^dDHN7Jm;qGA613yLSj-Ww zPL^R;_6+9P$j2*8r-idg@yVtaQ4l>^j~t7x^bznI+dfCb6C z9fanrl?SV~5u>%j?-iMdIg9ONeY7+d(Z4CXw85v567$kwQOBU4*4egn`^~wYtL?Y^bK}sx zyNN#=oO~-+C*>din=!)_Jh06xK3_d|7?GPFUcdm2;p}G8njchl2 z($0k40qQbwKv6}+Xxy7r6`2*N5+*m;Gx>;=VpxvkwqIf#57n%T1Mt6oOA^z4y-fGw@n@=rk6=?5DX>#f)iwXMd*%kGtP7mrmRJ^~dxLiCO zV+b5oTtre#E`}_-CWUJL6-U7?%IPUm(K_Gn{)C@B*in@t%!wBF|7cVn`O9p zCYv702{gxYqv+|*f7LCt7uEyBQq`x5syGR@$Yg#pQ)!bhD&)3lK!pTNCOa&e;;+~! zZntk!L)Vgc0%opa67FqMPH8~)a#ryJ?W?sTpesNqtBrsaHZcAF0X>f*TUz}7?tYHc ztHLwOI(6o9Qq#Q0clNDDDngr+_6tKFQRr9%29u!x2-;n`BcJmO4tvFom!*Fyy_+Ve zxI%5%UW&c>w>(O*)Tj9~##~fAti?q!4b@#!K5*kg(KZnKAFpZmv`szNu3#tyb1@)z zw@ac;k61bGAK@6tHG4f@xebsrRkd6$8t8#Idhkd3Jr5KU*p|NLjLjBL%+kuHXQVgA z2_`l$*1x1yXE1(p^2+fXZz|2?CKm;HmFhojI< zC59i%(ELO?hhjAT`p;(fSBLBkK1WJ*Ir$3dd>x*31f-`uBzNT!4EN`Y&%6juCNJVf zOBlUOCRXcSPm=OGZs3DfdQRfHZ04g;vj{@2{pcKGdw}iu$@#`1#-A=t_}Q%l_Os^U zypGvp^$arVu?x)y3JX_!cJ4B?pL%qyaVAHj;HhLdYVFOgT+fL%aNC>Pd_!>0NQ322*6!ao<0Jm**d|wQ& z-NLy`_Bs-Ak(w1#9<-G%Pp8R#SfB3_Qah5wsor%$e8iGH35RPj^|@=eh9A<#Z+^GU zrKBbu*C>92Rya-OSmXOGrb2VCPL=MjOu{-E8X71t`Or?_XuD%G3>Y~QC3v?q-jX1@ zC7?a1$@ab|;Okfk8D&tI+k@m4UhdJVMOXEWjv;Vi-3dK@+i{W_zG!)}zAe?0c-dl) z^*!HsD;hphT@_1&%&x_dDfw^O9AghWZxDMp$nU3kqoVG1*!S-?Kjkrg zL5<&QWdCnXrpU{~l_Gax4z&HF2gPhd0j@6RWujp3s0b~t3;MDy@ zf2&Ah$>IRl_FWOA|}fgOI;b{pds2>m8#=R+44C)x~O>= ziy+m_J9_Po)t6jXSBKLz#vZ3}f8U)QQKrGWAuS2VejQ7S1=8~!VYCWF^c2xF-LR|e z81L1`l_8e8sLj~YV=nMbP9^PAlIOl7#P?2xD<20$KCOHcf+Pz z8isVXwT0No+4@g3WZwUwrztKl^4fiSCC}Rjxc0`Nm=AmkDjdjR@km|MXZ9~ZWZun8 z)1tTJ%5h!d@N*&z&2b4byTpI(X-c7Nv1z3cZ@8uH~K9rEa$mB0pR9{^NydJ@5xPCa~0S> zGU8~6TgF3b6=lhb9y%x6h_W%~8eX;B>!U#P3H3yX@e^u#gE!zcLLE5}+hYQ5a*TcX zrT>z@?QPh;vpXJaguE_aC8rJPXR__g8E2XO zz0+IGhr(=l(jPcATkCICQEIVdhCY$rPjGRZ@C%aYaQU9`Xhle-=j%JH=M8{nFg77o z3MV7vc1Q0hrQk!P;YrLi!M`9k5|C{OUWxGnzHo>v3=wXLqv{J{0A@cC#Ozl?g_M7w z4!f%_@-=V-AT;7>KdNO(TKq1&3rZ1Y|C`wSzyz(;mjwd&;5JdhLPbS220o=gH-W9* zc|91!rZmoLzj!v^1EPH%7RdUau8i1QVx~1ph+`KmcfP>Vi=)4G`~V)2y#FrO z?;X)caNwCVuUzNVNaxHTr@1U4U}W&eM`gIT4;Y5D8FgpM7EnC<(7cXtYQZP7HX`E8z+FsM{7G-up~?= zW=0IcR%wC5vYNg9Esj4`c0y;zWC5CMWWt`Dj%hQIXiqCcIVfthm}whH*XIoz9MDFV zl_i@557o1j%p-aqtc2V>V|e=#NAEKa6P?+a5`>hGh=L6Ia^>UNOfLmALfDuG)G$7SImu*LSW(V8jDetlg(@n0qQI8QBNd##2ME#CSMBu&qPxI`_9<@XZUii^Ll$jWC-lp|KWjzj_-;jQ$0Ez5xpouuD>pGleu9E;R!-C^JxXV9}XT0W}5ck2nsbmM-O*^lL8I?Xn2EQaF z<;-lK9iEo|KzE1tRX_5$0^a`t*`u{k3;OXp%+vT>oRDojAPFO4lrV|`M1r1ZW%S++iaUe;C zOEp8yHR|#GPuMU1bUjx&^5e?-^*k#QlfT8vx(}`LnrAA#QGBMGo)wLg7O2;ebZ|6o z4{fm&O5T?CG1*<+l{z`13IiaoyYrh@&O#)daBWYeo|H<xSjB?L0w2Gp74MwkrMxEe1ToPF&?1^)pnjJG@l?EI58B3Xo@eXM5qgaBZJ#J? zcR=K+nFvR*m5_gcl}QK`LG>xHWR-VEdH!ZAJ^B7Yn((9GXMh^sMH_hOKaByj{GfE9%?iBIk3lNU+9N9E~IrBK!Q56wSG#<6H7WQ>LH! zKR(L*0jY!aOa(^}KW`>0{i}bkOyB5UjyQ_=x|aebD;xaVwr-hm;CrdF4W3e>$)|X- zTa@n3sRK3%c}91U&SF-J?wIm$0#fW)c!}m}KJj|>4?%L_&d#Q5c>IKE0}3!H>gYit zsubQ^8ff~f`!QkDX9oM%V2al;B~TbNZG;0B;`8xlx6?)+rUqh-SAS0Ybpf%JPd_xx zEI(=}3q#`{K!o0jNq<6XRQOET&5fF)-s^2Wwa@E=`?4PoEJvgq#eS9i6?YotSz+s! z3@;UMyeI{{OvA3k3#7bo_t)-=<1rOFmMB5`w_SN;OLe2gzi58uwwo4iADwX&sbIdY zKbB(3%Z~z|n;}5{DK*>&ARDNL6G#yHTw7QuGi~EUOsn^+!YQf4x{t0Z>%VcF?>ek# zb>8~}&rg#=2)QPUyzL5gTRIAeK12vU68+uV^6L&Gm-3pg>(?cp|ADq|nPaqkC*@e= z{kf#mVx(4Nsw{Py6g+x=R5FwEQtDv1QQc~x`Ni;0Zr08{wdG$^Ut?%PHdZ5cO%olT z;u+MtAIhHcCLWmaL}HTeI`aGeqa3&&*fpVi|M1oJ)l|5_tu%gT_BW(F^(|h9*%RqT ze*QUU#(;@X>#**TaBg*G4$=lisoi_rs%N!6eXrkGkB%D8WeDPs`;Z^;$Y%aJQ}XnY z7t3DEm(S4PboQmz{+!cDz_;#sTE4sDT%gm?>|j^ClM1lCZUpqmK*B^8cojrCCk)&Z zcJl+G6- zLys*J=I@7lug>3-EpM-NnE!I5P?;qqOr@0R@%Ro(1P}PwbqLsx)b`U_OF+r=r^W z85>sK?yXZ76Vcc7IjzYaz2$SQ^;5ulAnG?F{`s!y28KSPDJD3xK6XhMq3!i*!<71b zg{8XS zoVAx1WO{P!dFM7hVZi$qTx9iY3+`J-FZpJGB7z9B=%5B5>;dX-q$^uhG;DIPvh%Ge zR3>Jal}2TNM=`(>gkg$FSI5VJWY2H(tkMGgmMZy$=6Ebml-{gP-9pNWJ1Ul?`RA(?5(ua@&4mfGL0O-cn3F4CrC&`mleR z)^6JaqOrb|AMg@o`~~_uJL6xiSb{Hof4wle0Q6h7 zq>|t#)S6nh8%56d{AF##WID^WG_U-MEIv+7o97#ZcYIULc%VWR?}iCa6D^)xiKhpZ zBR2PGmUep3zGelyz?dgP$Ar|4M)#5uzBVt)#J&hHwGUI`S-E>4$bzbBY1H6h&?NdU z377`NPwY!90xwjrbbHk``Ip|Ab0X@ROD_CAdj&7qu<+lu6vi#`;J2#|w{_0NmBixe z#6F4>=o9QK3wzgCCbJ?d_|oY2*4M$7u9*8Ra?O*cZ`?YQKeua2wu}g3v|Wk$JmrKt z3@N`=d!dx6AkFkvnrprV;jzF=BGi zbhWziy2Evn)S4v~0>PPpip&lN!$Vl(+aNr|7B<1^z|v-{&Pzgl;N#;Xf72uh<7tu;udkrCf+2IzVHK;Kt{fzW&9gr|kU(?`SG*bJ%)v+|qQKpwLk;2wD za0>F8SdjGKGovO?%PlvM-3GSOVfq4l(t6RMT?E_cyT5xri0IVv-nz{rzNo$*y0t4^ zzNJ%QH2mz+EW6S5(6*%FGIix{sD%%YJL$=qz`&%yEuggI=Hbzmdq2d=6OdXVD8kAh zJavbs5UYn%YoUvFUqE-k^@*+-IP3Z8kaVun=R?=cZ9Xi`%mBjonzV2EcLMF*!$U~5 za@xcE%z)ytk7Vw5qobl@pd(SNM8m=j!yBZ2^Sxz8-+hY8WJ~1^vN5iHv~L89p<#4o ze7f(`fXDmQ@0*)F;nBSVElS(Ma#eQUW4Gf1+_E$+kV0uM5|Tz%Tx=L{U*9vVgy5f7s1vA|8Dwx0|F_t9`j(8Yak z5zA>}zD)8M&ON+?pkqmg`(2!$FC`hhu7U56524Tq^LioFY@+2_|4G6&EU^G_;Tr$~ zRW7MU(?%BdHH!T3FTVhoX)S)Q1DcKq8W8NTTp22C66_k`7(RoloGx3FVG#EE?Sb%Z z9%;Y@`*%ehfN2M<2bwihHtoVtfGsr!y==0F7O}Xl54Yc?+Sj2b!n4KykEtS$6R_{pKGm!B0$;7SL(cuz>X z_=i*R4l#_`;Ef7UVrgZ>;OX0K z4lG_s2uYIx+qSRfv1ZyyTt>w)&k`{}J#aK{4x#q}w2uy8r7S8gvY<)Eiky`Q2komI zU;@oz*jTHAt(z-PP!HF;#di+!Mo?Uzw{!*?XPRF$v4D$a)iSM#HUNBJh`{u(x&vWg zG}0h!q)KIS+AYxO_TNeH{gBDZGYjzS>E`vn3J2l@<3{aRC}-j%<}15Vn;2B~|&g zY?z+`kuff;!SLQKIpks)8s6R(D}p&|L`|jkJjSzR*p1AHEl)R)vr(`>0Raa> zsj6lOe*;}aHPEpqQyiSB;*ssa)%Sz>j@7r)3H>47Y=g}j_p^XztT-{&AC1$PY>Ps~ zLBlEW5mmzyp$OrX?&OeN$} z>FU6W=%_-#dZ68h0M^&ifyG)}yOstT-hNF4{5}`8+82z^GBY!&0qWAmj!xqqV(B4K z5CIv>D1Rq@LVFjmT+_YZ3W$N<$kEV*2)nzxOZYJR?*DkKc|~P1%t8-MAFxCefmy>o zNW}sP%#VB9Z1;b-1IR2O&Vz#@2k3wGB(M@eB`bhz_|wpHqjT1Z4mFyQUf@a~7mI_f zcfXJql9!ytLv^U<0;=j2u7rZfD>|{ z;(ZzxXfeYQeN*R`jiEQ%Q9%aVt{aSidab>QJQ?giG*}Q*E56^5w3{m8Z#1uF!a^aS zw0!F)R8Y7J1pE=;2ABl+JJf8N1;!)00@uozerEUpL}6A)lm}4a(1~FCJ|F-QyGwio zNUvCY7#9~88yx-4&8ZW_1SBM{KL(!@eEYfVx!chE0+b{;I7Ob<*hYa{VGbxU+Pk{O zJbK2@eSL~}LtwS$v5=o}0Fm%<&+f6Jgv9mn22>zW+}zw|0V6`U{tYXDYgqrUa`FHD zRwUr#>}cmuppPrPjSRrIKYM*a)A{qo-_CF~OvFFiL283EJXRp$I#Le*87Iz86#F0@ zh`riyqJcrAU-1Kj2=AVL)@ttU>~O)1iOJ-$o;I*m{A04vX28X^>y7Ea7L2%z0OgMJ zK(v6+E9G>Rje&pcnj6TBhnJb_=f0>YmKi5g$0(rqpLX6fC?o(B@j0=i!&<=&y$itL z?Z1DM0G?6Dt4wc_JeR_2M3HeF=xZC-Sq*-OZo5MK*((d zp6iF3gTNy&gB3J82mhY-l=*5fsicL1yDO!$ksk}1S{nZ#ppPof{>*N?jNsD1WBy+S zbJ+&#-tt$zi)hfrE_kA-!_iSG5Km`k{K*cFTCBj$sD95btzYf8-EyO)!_OpESIMQB z1f7_2uv>L+`2!H|EP!ehXnua@2(L+Scgnc__6~Bb1;8S`!IDv5)%o@B?zf67v(&RO znygQ||F(ACN|GK!q8cjjc}mQoOIby(BfC(NO0W2-aF_3;*P9K|?`Wy6*5Y~PT3W~2 z+6p`4`DjC1=h5oq#Y$n0I}M%`HQSTY)2e$ipOs%E^5zbP>P*1d>%bYmcr7BT^cKn| zAY3KXt(%^nmVd_(_Bm8V;QEk{gyJUIO0u{xx?bD->Z!69OUT&&Hp74am@qqC>s%fG zUKxFLcd}1r|Gir{PrC^ANV=#iZ?Pd9ug+{3I0LKN$+{a;VyZ;(vB@i|e3w9jvj%#n zE_U=_ppO)=pX@K6&V~aj!@>Fjq7oJsm=o0(<7%&t#e^JYdwiVjtjr<^0=sBF81S~A zgZ7olRk^^IrWpe<>@@+(VJ*>f7ph1;gVe907{`vX`OTY^>tpDC4o~IgcH=|rsRh-W z030Gv4*so2!YqzM5%*B(%8!i4q@b5_V|MoA^8JSGc?Fs!fQjG9*nZn`41as@k9YQ4 z`}yjOH%s^(&aCM^t`Z{=ht=emPpztL%bV*CNAq|8O#HFfcaR8Qq|2t5|4$2m75gyT zDUM6kC#}wY!Wf2c7am6xJO>JLUsB&#K4`6*VfoQcmzbr;L-%HLMq<;iXTm$*UH0OS zlACl9DjM3iA7=cE4}7PQE#Ol5uH+elp2+)2@Gr|N#+j7IkM7Z zEf8=b^r!v#8MuMMJ*AKfHYgT#fjf7h^p(rCB(1R}D*H5SXeJLMZQ{)Xmbzo=NvUHA zsxpf%>eA}!t_Sc}WJ`0{c#pZD9M)T`di#oKkEmJyxaroQ`p9Tby02Ye$A<4rZB_kD z*@`Nqm^W(QO5xk6Pwa}BO~EeT@NFJn#Or4UZ!`T|U&Q@hX1e8}#jN~Vjy=60ev?<1=yPZ=-!{QTNqyUfX!7e-8F|b2h1v*T25~iX*u=;iPl={%FqZ ztntO&Hjn{H7CN&=y{w*h;`E@#= z#$0||;K8#<0DgF}y}Ml`_-Yv2M)?)lB>NN-ZU~urf0o+m;}ESyN^sP$`}>uYaOoQu zjA}MDHqxMb-2~}`p{z&|d@y0(T}qvYWBZCaoL{J^Oo8N!gUa7=PM}J1WYI8J)qK+X z5vE90$EBW|U})fd^=E^*y=h}3MD=Hl*AK!-rz6eY7R@Ig=<;5^Sq%p$B}<_VDeTQ` zM*`7ht_x?)32io=J7!EVt%jLXED0@`3>l(c8<)RZ_*iq^l5rip6SI@B9yUGFdu&%C zrjaRhpINX|^aG0MrSC=aqB6euo4;Ee7CTEdz3Z)PVTlr_Nj@^#&*J~O z+TL}%=cQLurg62@l$dy@-U1zk=Zo|xjAm0*`-@g_m}q;$!pNk%`c6gL>XbPR<%TjG z4_xQedhX4L0>%PDh84Jj=Ky?Jflf)rE~c!)^$D%?&QFHoT4;}Q+)+liF}kSX^`PUI zS4b&B9)jHm1lK})RglnLFqdVOC`zDVkT3D}7-2KFd1VcsElqT$V1PixOSYwY_jc{q z*1IaP?##o)_Dz@+md7rjRJr*#Cfh3$;HVPVw7b+=Y=5Ay=DVAQB8eyGDxPx!lmp#P zT~<^OS`gRK_YE!6+?_#FPYx5bJtv_2vLRMUDuV7B@ORdsCxO<o8-=dwIld;y1x_P%RN0y}mctT^RaU!BQktxAwCby>yFXg2 z@`yKU^_O~=#6tRNzu7CLYf~w+IGjH4Ne#eyjt%%Z-!o<(kw;rnCJJS3Z&I*D8zp5q zH5h41c5J7UXT2Qds>*dew%_d1Lf{upR*{%~P==VLK=_3SJs*&V(I^n*kVE)|EumTr z6n)Qs?Q|^sm{Y!6Hno|#OL`~&lC)W3OZ@g^W6OjmKYixuFfQ9Ut~>dxCHK%)k6P=# zcKbgkW=#vfzq7~PdlZ7WuC%B4Ubyf;e2xzbVIy){oPN0qMLeY;6}K#hqJ(K_vr>{m zMT#xak$5!ar(sXoHLsK%Mp>XFhmi=9Xf8HtiRLx$Eg*uie?VKgmO;jMMSW)>)E4H0 zS^vWHcz4%7s*2;*3(h;P6%n$k*^|Aed+wEP+sB)bJiKuUyW`W35b4 zPvjZB$D;gQsVO`y9(xC6F66EB&KloBJRsSi@9{Y}Q(&{Z!#&-CJsRu|J&dyH}m5yN(0JKF!HKyiOPE;fgw7+T8 zv49+!BjY!dNCpcz182!W78g^XtOi zjS}iG5w|Cbct!9``^3a8$M?2x6f^WQ+jx{GR{0cT&x}S3LcsA%H(Tc$5ZNsenFMmh z1p|sFpiU=_OigdUH7Nj+D;mJsNQT6`nUv4Un7lxah&Uy9m|A_!=_JNZ)|cZNm+lkF zdx21#$QOqNQTpi!7SC5n$ET!>1WhUkW?c3!+~ztCm7mQ)?H_>A+=}a~m8ga9h)POI zs+C*(bETmaaKHrDo%AQ{5BB@jTp$d*JM-H+-sX7i5I!s7OP(R@9YCF0B?a0BbO$`n z)fhk}u=N2ve~o(c9B(`8>EA6Dz4FGa>0)mon!|GQ@3?T7@59qiXuoZqq2=>UP2Z2t zGO3LKo2PAO7$r_ni~PlpG#r776pj)T=dI-?`qM$IpI z-WNR&x^TZu=UtDz=1^g$$7btTaai&rxuHW(@0cAV54Vd_Yve8`W>$Zcx$Lu9|E+pf zGz!27Rfo@Ug!F_iChKqS`Mkqx;Y^F?JV&7IpL8RKfA!Qd{^ zv>~ec{QXgmu+iImmbNE#nr;#Rw26)Gr>JuUHQ)I>qcn6gg}o!sIIK+N?Z#N{!%ZLB zr}9(Yvo-8V7L6w~>CO;)#OR`k>hMA|Bj?js(0e~Io8o;0dhh$&HvC=}u=j({zAna! zdg5#bR7SVQG3ywPezH2xz^89Du4%TI=~cK(nsF-j6!hN7W+#ZKd^W6SPsJY;zM(jI zK=R{+K47m%&~uQBCjpycin>C0nI7POOI!ue--v%3ymf!cC?(XP-FsEK@}!PdR`s5L z7J2@|&s2$J;?o%DXxW*4T*A>?B4O<+$ zxgH{i(4+65STT0HKpWba5_M~) zB?c5a*-SnueF%7H`I9Wf_{OiKwGb;=3m1sb9gijG7@Q_0##0%FC64voQkK~IQ~wnN zkRLW!`c%p{?#boCML|q9!$;_7jU2v8M z0#VsVV(?p+^*GT(&CdIo*vEO{iJd?h3g?)Yd0ah7;7J^nH;-YVZo@A%xt7(rU!;kB zzQwkk+;le5?x6in$Z3ThrA6VRBh~;P|6Hc{_Q8tkKV|w$VaNH8Ovg!2PLsl`tE*Q( zmbn`d$hfCf0-^OnTaeC=uG; z9gS_@v1NT4BZE9;1J^b;(1zi zH8_ecC2KFwjeBnjJ}@Zo<0~EK?L>HO?)PTIwdsI)ur);i^!~*`cV`$>U^@r`a&BrZ zlt>kOFw9%HlDCxIy&Hmb22dX5@fl2vh36R#ihc;xm_I!K<1=Adyr-HW+dlw5y6o=4 z9CBG7$@4bT99my5-aPYuk!hN2U#fa9D}2K(!W8bJaGvoZL|dV-@(34QDr9wT?ta zRPmGbHaCY+eR*TF+M9N%pb&Tq!9punU>c1AS`W`R77D{& zgq%-b3Z?8_%FZR6>^EV!P%s(Q{2ohgqL~+~zWvogPd{YF`%2e0u}`aW+bT)AfDdGOzedQU^ZtyFf7bQbHIBVN zKU1AUmU;A6@O5>LMrZHrK^+l#jDKp4o$)U9el~AomWm9=wC`xw3_azN+to}(id;K| znQ9zzXQ_KPx)eoSY$H;)3YGdG`EQ%TW{fk+1J&+_(F7A-8?p(P9w|ECL9f3m4e5T> z6#Q`=z36Yg;_1g0Loh=_!;gE^I@05KP_)EAiKKAC90t$DrWU1i-*3kKuE>wYPlyPl zK(r{I-6<&t_iE&4Vk#RE6%{SLF6o($HZXix|I(SPNLLGrE3xTs^f=;Dn_ikzPxfhT z{^Yze6MbJ|7|Hc@OzVkB)`bjr;h=mVY3qw^tS9cI=8rba$sAwyMPWa;cAF^K7libu)VWL%D^Ip}8?r5pQXtm>Dw0G^=!T(WB5?>g6n(FR6r&7LC*N?#ExC0kOEQ zq?EsPQNIVlZImF?#Lybw;oV+|>d&isoJFMBf2DSO)q6Qnc$Lv(@@+gA{#TQ6d1{$hfD#2mjbby`yBhHhYgw zp7$F-qTN!dA|+^+!s=ef4=H2JGaST$;G1j zdkEMsQb+z;-jm5Ak8Q-5-<;!dOfg9^nw&qkI=!_eRvYm;(Ta@=Lt)@gejm-9Gnz|w zgDSNp`PfJCXYU$Iu99ZRvFXYUNx6)M-U*40I5K#Bai?dHz~cy-_x^sFQfc+kkwczE z(ktm>ENQLVG3v(VraB1aPVPltLKS@=&q0r*qoY$>TMNku0-RpZt*D6Zxj+B;4@uJ) zyP(6oU9uY^&QC-1ml?|@eV==I9u*2;Kgpfu6BU*DK#GT^Zk+Sj^*Er1oA^IR9)TO6R63xY|#Wzi2UYu(;PWq_-690X zQs1&{_~~n5g8sC-Kl;f2-9y!PW1XLXwl3Rmr^MQm9@B*!woEs3w|=q+mx z_;~&)z0DDifYH8JIvMxRXgc@0{q3UOeR|pixO~U#4SA4=ekf0oH1DFuZYHC*C9H_* z+V0AU*0>rSW~NKCQWp42>|nm0qZ>aQ!!M8IyU5P=QpEEu*!@lEpx`3JkL76LA%R6hPc`Nr}N} zY-+;E-|(uWz5idRB@887_db>@BHjS;HnH@t($mstm^twPejM#~7w2cLuW>^6^sk2G!`3vVW>9&kOk97e5aLwIe#C zjKO(?aKQDr%F2BNVttsHnBrfxi*`nJjP9m;5hHqE@~ymB3|0M>soRAB{shP&@WAwR zD#{J2ST^(5Z)|2pxaVSr@FPvOt~ZYoRF7w>)TJiUnmjjZ#%|{) zRn|V`<7i}HtZ;pxZHA(FNzo*;a&e&Y^SGVffd{b}il3VAbbFkIn zyOx{X+Zu@lRquel#_M1}UX=o}IX~R0(-7d0SijE%py&FzNS6Tsax*B#yh=<@lKp!R zua<3$sywNi6+WAqCw7e3=||xdbN^MS`z8OE5CHGl~WbwB&rMY{n&6 z;g0EmdI)71E>}&;wpwwoJg|I=#1@zl_Z$o>`My6WB(#r>k%B&>TnH{zzjMkpM>f_k zj1V*#!#F3;J4kBayPWX^BqYILI_~Fr9d_Lz`}z0KdvwPAXui`a(3JWrZ{wg(s0naN zS~r8 zj+^lD?dy#X6~XUPT)pjd}e&ae*jJ*hxQP{rK_2xHK3cdhFAE&Y`|DF`b2; zFsU`|Hf$)s2l?rU(SH0*K*Jv03X-h3#l_sn9Z>C%0Js2-Xe4I9V}?TR3#hSoVMT0? zmDz&Xn(&M$jz@}#gO9q_VV|wJ&~R|b(Dd+_YBGm$M=ywE=F%@v?$~VP=n!bu63XJG z7Q93Eh)fHvo_hXKyjk+g)T!^F>ATdVEWA2O&&5nb$JqBf!4^v91hCnqPZT@xf|yh%^29h7`7R#FtN zraXOm(jTZst~hRX-5J$xL~bKhI7$N4R!w$2#tSpUZhHU!?{(-#8D?;E9R{@)>D8zK z&q-nLGh)z0?|eMtMHFxQ2Z0v=D{?*^wCv}p9NO?`y5XI&r=(#s+80DVB$WSoJ17N; zgWxf7Zw3f=vcW%;C1Zp!9F5sI2g`6$ublDb2(_l_Idbd%GXqPCT@5;h~s}2 zJ0=8#0q2EeI58{KGyQcm6Se3jY=5RK4?47GP1A zHxK{*Uk@_!156c(dwDd=`uaoyKV=nLJ3G;IiuEM`Zgi;1N+%~V9zwf7$Dbc00FaI< zc2fd@ifG}eDw(1#xBD-as^NG zaiQ+~^R}nYuh07k=2+>!&c5cm#F*Ey1mdTtq%q(=6+Ek80`U~B0&IWr`+gjW7eXN& zeAg}+KLuEnl?V&Y&A}1tM}RQz1TRW_;dnuv=u5B95Psb_-%$rwlov6^rD6f{C?1Dv=KuKleXpN~h!MCM5*hG^|Myu8vD~QAyIep3As{fY3<$S^ZUCbR z`ZCMSgyVOE?Tg*_HnMnUu0NzmlD2_DQ;EESP z!XXKka-;-%FJg6o&jAHFd_S@rvl~(p_*PC#LqK>k?*bg~>AfuiTn|?el|2&36p^Iz zPuhPr+_B=nFgO0z;6dUlQBgP$JP16UToJ9+f2)@OGBhOM!j-vMpGbuAZpgH>wDyjU zZ2-*A{h|k-4UW}nXJpdqhxADR>&bsZy$zfs z{f@=w=&z>&{On_}2RbdpKu43yu(1`aTXi7r(TDo&+4T@6GXiKr@>02@#?54auTKkY zIS?D#PNnqvD#6OHmx!M_^yunbcsGh%V0X#0AcI3$4h-S!>S`#U#nS`r8LUT;H!R}w z{ZVNFP6hQ_KLTWOJ@_9W!`xxOFlWqOgaHUj|5usu>$x(00>B8#@5~ScE<|ipK?`k| z$$ID8K=rUqRqXNw-~PPx&CJpW13lEB|Lac}L2ti0L<_loYHMrrdmONUaQn3bHowXB zVD!<%{v!KymzWq8=-!&)zvYef8zwBRsgbL5T#D2dy@i<4Px3UNSf`?*;&!^1C%6~W z+rnB*XMP=XhO&WAlujgGznM&M+fw-hLD~f%-Pe;Y^f(xo>b1-MLeT#{V9pL~2(7%j zO!9r(=R-Xoa64L{N}{ySx?Zt9yN=~Lc#wdEEx(Y^SHpbeRCElC5dM4qQ~#|N2S{7R zp$H@IWw;~~z-||f_~y`i#P*;9O9mvu4mx$G&;@P+#!w9b#tD|7|MUdld}d}Vc=p-F#h{@fwL8c(_hA32 zl^H8Q2(#SLW&N9*pfvys7~Z3R)Shj5W^{BkK*4_PdR*Rvg#}YIa34s$85Z`?Rv-oq z?X^b>mLs1&Z@@_&daD5O6~q!H28hlRnd{&j{N(mThI~nZ7?L^=0tF1_p^cS3teY&L z^&ITa9Z@!~{~1DKKGbU4sc;Z|wSW7D1WFVyA8@H$^GN{SMiIi>5Sn@4j1dhWb^=${ zP)>t-7I(QwGKdNNA##Hl(cJ(~GY?6!IVdMw{?Cdd!|?~=Y4ZSL!~zr={tuLg{R8;Z zGW|=cRM&5LB|;e@?j(H~bau@qOE2A&lZ)7jo|ofRClr2`A5`=U?WvI_9Y#-^@?@%KnXV9CkC6sM! zw*Rg)^aRUT+z%T@~L4f6ulqg_~TLeE?Ot@JFz}mLZppvkO$-zhn zY)&97T3W6WkZ?D?^Aff@xj>L<#-eueAQzDPrez(KQf2SnSG3fSxniZ4BUjgoiw>|b9mv@q$`c@bEK zmC$j<*2tXeKCbnqb=B0Y4jFegDeubxW8HUOlS)kbx$&ynGi9EVIV}!EgJoT5J@!j& zCDQ=IezBFunJat(RQpR6U%ql{asR=`o?e5TZ9l!=5R)NQ|IzJVcJQ%!-RkcW`ud9@ zJ*~}0l@w=hD6tPbrX1J)d~my1#R`tP`|)0=UPI4=o+7UDmxF~FVOi#l>n~76@q)Y^ zJ^xHh(AixHpVTWD92zc*`Ysc0%s9a;NAf#!Fk*(h#hF8o{SB9u(qqg*RNCAp#jR%! zNIBC^N%^YzRv+b3yvw&A`YN+tAvs$9r`&}i06CEO!7ich`8m@M`qt-DzV9ACPN}hc zvFcPjCMoQ>7DGB{RKSO~EhcG||DMGlD>rMPn-j>Ig9_4@*O@h~`6pRY`>rWzBJTJ4 zmWzHZCu#Ok+QfvA$N1BGbbrn77Lwe##y^W!VgY6c4B*#wHibn*;+T|EERMgS-<*5K z(GWg|W^2_(sEU`R4sT1WqqTS-xvql<%1?YrKC*_7%q|O1&U12_A6GZ6F<+3*cU7g5 zJsN)|a^|@r_NUbGHhImN_SE)tGiiQw=C_jjjlVdH4+L_%F%Rzj#;FtiPSBhxUqbax zXtqY{foA$IOAV8wz0bjC*#$AHVqNw#)lT;r%AN$wexzxyi28cfvGRy5N26+QZM^O& z=|l6Ypt(!rZN?{I$A*G3BDZmIB?gaBxJ~Er+b4zI*rD(s*WyryXs7(VxK#Z?Os^+I zAtlCBhfm+0s5qwfvFBCqSK7u`?Hk|s?H9h3@HWcN@YbZ{ksE(YdA9a}jMJloc-lXNNP%4o_p-Gq;Oi`RW#2V_Blu|qn> z+y6#KU!72`&F6))Fg`0OeL0R^t3*}^lGgWh=8l#B4n%%mHmh%zt{-Oav4%w+5MDj^ zB^iHJ*PEAE6F4}7vHNV!i{;s;WR2NFKB1ZoiFj)A*If}m&ghqDOT9-pD=q&1*-_#Y zJi5S=dD3`<9edhQt^4d5>-Oz`%nS^u-FWF58JL4rk38(RX~UCPUvtCc)5)EixhHRj zy!l79{ zS+9ANpp?uGL6Sn}S&FB|9bZO>H$^QI#41N~(N={m~+xgMy-`WKFVq4Tiy~gIPMvH3q6eF8i7CeBm$kyh3 zGT-6f7*l88-AP8Tzm3gn?cO`638RsB#9XYyG@qMxGCJvWeA!OsvmGkHU6kA7puAYY zM0a-mhJWnS6U*wY)bQduO!9YQpuvKJY$8_{!b1R&IEt8@GOL^E5IrJ-3BvE^ozqC=N+Ulc@8iGoBW%Hu;yiCc0srEzS$o{(;=AhV`M^Gr!fC zMLCbQ1qwE2^R@C}@|eM7`fHCa2;ZHkMKZq&q|J&bA^8bmYC6uP7C%pYL?6hkOCHL$ z7e+jXDH{5OV^zBQ zuT(FFo$N{P;Bg7fJ02KD)Z3_EejQAtXkqC7dStG3$RXf3%kN^rxi?la**Y|Q8TX5c z6@XLbPUfB#oXEIqUNojHi_V}D>F0lxs<34-%ENZCxzyXNj|iYjk~zD#E{tVK`E=pu zDZDZ_m@j#~3X@(qxWHshd;b^?9D~cb=bU}^*?X?F=9<%E zzHEOeiQnN9U+JSRt-G8ZwXO@zGp(#Q{R2$8$mG&Pee@$Vw^DqJmVzSrizZ&tUqo8Y zqhttpT^9A+Fe+xG+!U<&fH4^L#7A2zx_yTBZG!(?uEOPQH>6)pJ7JN{0KCj^ZX+A-ZrQ2>dh9YaN`yev4!+-`p*?ZLfzZ#X;>poWAoJPIV97iF5;cT~ zcwds(Z>GnAe({HAtBJ3EK}R|u8Ysa4H_Z$S_oc9gE63k^K8%Jh^j{t%LB$9d#P1#) zNCLwAV6HN`VrR?QuTWMZ8{pX>!kdOMzIJ#3PFGA!Pt@4agHT^d6|$&9r2vS)QO&^1 zqRIDv{Gvcw6DG-FKG*p_~U&q}_zT;Wa%H(Omy6{Ia{kZbqPHM{X$|e|@W; zy?&%&DM;Lb^Xb#4FR8f^*oaHRs|&7$tPyg<-s0t51qK8d4bSa8sdUs2@K&W1_Q$*G zq>&K*z8f^aRfA0y_`Z3pedik*E&><4iSJHW^U=oW{Er`n=AuAI#7U{}HSL_^uPA^< z22P3j`VmZLZQVscMjk|oVN+Gfwo|&b<%~Y~UM7_@id|MSM~bPTHvt8)`|j273hX=bz8yH}0f8sfwD%YYr4^b|kk{IxHIg>Pkk)x}TDMZX zPn-04hOM7W(zG0XXiDNY;R4aY8}nlFj2AiTZ(QuTJoZJ9QBXd^v>~tjUyVGmuD|q+ zF$*RAu{F8uubl>^RXpxZtkj^m0u4IF z?@C>1Y~-fiv#{a5uYDE+_bCKlDwzp=cj;-}VS?4#2onulQNFD_KDpe^>YVGpt+o>W z&LOwI^OGd=yoe1=mF+nuasMSIxi0x3*${S_m@I;y4WPy;6PDoTU1@vn=MmhfgY~pp)YYn#~NdT=1qHnBRF& zc&{K%7S6osish`%2tAuo+>NZN0!Vq}`&CA6gNsZw{tfIMPQwPoo7I+bsZ2TN296s$ zrmoDy!hBl~BJbf4BPciuw1QM_UW$~)I5fs)PNvC*qyOmw&VEFxcUq)@D-( z;Qv6C4FgyXI_RZ)mZAi<Wu@Q0Lz6nGxFs4KT z>AgA}6-vPBZakXqBYTRYUvH0;#RM$BfdWUhaV#HfXZUJX7}oNsm3eOigOf23VCO(K zcN}^DDIHh;IGpK8;KO2r)S2-(aHXe)<_$IgxkocOv(GM19TbrU0&h}1l}Mb2BX;AL zK}B7AitH179WRPudpuM3kXkv|^FBYdR%zYs%;fFKfxg^(m2CXnKO^gcyspmY1cAu1 zXO9y}m^^KrIOf18)OnQiCWy5P2Mk!s)Y%bh#f5y6UGjnJ5lzfqn*2Nc(L0cIZ|Zhc);-+Ou= zU@uYAz5){ql^UlLZGiaAa}ndX5_z&*i6KJE+w~JwS5x)>{~&+`WkXZ zo2Bs4C3?S?%zb`{)D7$6^O5*{PABPLOsH<7b*tg&{OU@fts7^3x3UgzT<>k-@!;zQ zb2I;aPHbrYW9bxKsmM1g)@0=8cQgypj(5bK)fhH9ioUTDzTlQ+Da+!dLbFP=9d^#3 zXC0;FMG@lX3X?~BySr@mtG7&k{z9#*VV&bw?xT=(pP8imtE4FDtLbIK(ZHgA{>qD%iTm z(BOR>zy(aB;~D5+jaO|SGtCzH9W8DmD@{OfDF`#;rmAU$a}`Zo2~X+J=69B*xi3uZ z>Ike7`-p5UR(&pJXuhIyq@jw8E_sgLFo+wBTE$Lckp~3L6y{LQ| z^RlG<_1Gsy`)4y0>Qdc1q7^B1*Izl&#pU)AgquusWaB@oPIO%q36@En(rb*ZJw-vd zMnK7m5|bh89LD>xF+lJmR>n7g?u#9mO!zUr0a=2Af|gX8PuC`^IThk=M@O|f-1&vv z?vf)ir6NCcuGyLOK(nhI4c)||KE3$p9jbHm;i=yVP?psT(vkqt8NEp87v(D%r;nc3 ztgRf!!fwCG&X$4irT)+n(jwXXJ&3}D6N@VtXgJVZ z-Q4EE%h(l7kNPgvFtDXv8v36F8yLMi;l7ojn)n_=nQID4>G4kZ*?mcObQT_+XEuaf ziMgD=3RxveJT=b2Sft)6cYGbwbtaE9RqtS!I=~T(Xz}a}K}sI53R7D!CGh*K^=hRV zJ%pV$??h0FWGsxd%hA;1EWQ(s*R5%bTpcVu}b4KKIZuYPEL;7_u zNi&m}K;mI9Sr7G?Qk}cIzq;m7apAt*{lu^XD*C)k`$~-b;@6yV(W%a=>@YtNciSBg z-(dHggLocOsXIBI94hF{#!fYUbo-(ycL`(K(ulc{z+t%1I72kQ${-J6e&&XSV`VX| zRbZgz(CU+EeDT3c`!UN9fN;{`bt8|4#N>mRpF%z)6luJPZM}#rn8Jv;tJZWgW^8X< z!-uh+fhNw?v%5S>L9yyYer77fZ*^@QZ+Yr8k(-1;bx$apw&8`X;lPnj{%h*XKvl6X zH*CMO&L!+v$NS%ZfQUhOt@M~PgY*`kl?Y>CfU9HB>gS&J4VtGAQXY$k8C3g}FSms; zVmgh^1IBM_lYutX4An6 z+haEB;tYntcOB!T z-%N51+*>CacJ%eSSZZ%F%u3jNA)%SMVM~lYP~+XM<>kTvZJm@}p_^sr`rJ@@HF=cf zLH%EFRJn`g5r4`mv;}l3u>~pTKvOJSjoVd7!g1{r=z#a-!hJSTQNV5PeKPIx1NyXs zhQRE1zFULQh!0SJbRgXlJ5!^SCXF)Qj-yjaOSyMCl)|NbicQ(V7%R3)E$R55;B5}qlf`#M7{ra0Xd*-lplm*ic$ zbX@3bf(UH{Mu0QX6Q7U<39hB107-fpR%BpI;0Flrg@r(>BYU6v*0ZADjgZvvno=ZW z!3tmQ1>&b82w;9Q`>MQ32djIo1U$Y1IfSB~Qy-vrw{n;3wRBe4f9t-CYI8OyG`rUk zMj?j<`EH{(e?tg+rgj?xG>v~|Kxas@*qGJekmy_XR1j&zbnVcK+RnO@$G6_Umv|U19{yVPHh(yv7{D`}bklrW9yjI_TGO2Pw-~4wb zo^?={GXae!3NygFjRI6A!=pFhM!Y_-tGo$s(CiNu$sjfH7lcq`_mk~#AfUeRp|t4% z*gO|q?jHZwQX_!~!ofVqletWXo?K~~5*38guSIS`&4FD>qX)nZ=7Ymo?8SLgiVX?4wDbMNi{#yvg+=Fnl4h8SuQ|waBr_Ny+Q_ zIMA(qN**TqD;oBFjSo08uQ^2Wlo5pxrU`uWkk+J5fj2s@vV)0@05mhnfG{}+G4D&tuvvaq1xnH;z~N@B(u%mvLu%^K z40W`lL&g?=eHExZ&uwo!`u)04^O9-5xKrSA(4s160+58sk1NpejuL#|LjDP2=4m7; z4+`ixX#tX7S(lUwPgn&1V45O;VG<*=$!Q}y2+4yY_7!cEE^ z?^ghkG%^(Y8}6MEl&;5{Z|C27(+Y2ZLacfS^jazIlQ`^BV>*Pbs%=^k6i_kXsnin2 zz-Hj_y|=%t=)@U(-Zu9V_-h6|xcWpVKLgx1UIVTQY2gf5WdY^v@kz|pRidG{+9=RF z=>!b6VW;cMGi;!Tc?>K^1C-u!&jV)`l(#vVQ;I{9ZiYoqxJHh>pr&F#;Y__0C<+xu zqz8 zHN@_cy4%)u7C|+;JCoI_jYKyQ)mG@(>UtY%JV!rhsn%YGUO;c}c${oMR=&Eti~`45Ym#;eeBCI?h0%=EFz7lab}}Q~F(Ct-Iz% zDIy{g1q_D%dD-E9A~I$Wz_Ypa3Q1|zmDM{?uK680uq@K~0PGfF4;ElKK>Xx0=rZsE zeg==IAi*a$4Rck~OKcd^fVrNayEX8qR5%;m2#&dQE_nPQ0dz#T;5(F(Lx6!rVc}{| z{6^{`if+ZT4Cm$LeE#vuMA(WV+AGh|Ld{|f;IDYKOj5HJ2huF$g`s!c2h9d|jzEK) zF5GJOhx6C>_oGD6@1U!`_kmrr)|vqG;ay-76v<*P;lF&&k^HZlX1`qR4z z<0Hyz59lQx5JaB2;%Aj6R^QSAnIq5h^*lI#*+AN>2j>NFyLhzO*k-gL?R?xiyD@O2 zs%YUslofDkl)-@DD976|%AQF=wQu}WY2M6>Ji zNsgR!8@rytFuU)+CythM&d9QI{5pBSV~*Tr#IR}B$goHKTbv{cu#n+2qf0whpTkc8 z(~wa6Rfj_Wa(?kCfo?}qOyRiXy0ads&DId zyME7I)GlSV!HuVlgl*ZY1M!HC3|eJ9c|0?LZ}ailAl;D)_f9C6Jk|}c8wM~S$2~+= z!(wDf4?t~`jn=p!wH~L3VxQRp+&ADl8&1f^o?)G0^!QOrVGDb-GE{;kaHqy!Ilg)L zJ{(~3C0edI8+}a%(#=syjzH%?z`x#|{C+o63E0>^k@w!25ru0fw`=v{c>xZin}v+J zETJ_^8GQ*G40v^D$)2xc9}1m&y%3LORLsT47ZMb_Z;vRq_9~1$4`aAkcl4_l@SDLN zwex}-od)s0N)xzy_bx8*S`6EGItIczy+-$2_80+#-S=r{QGfT>|B~$Z{Aopw0oVWn zY%#+~$mk9%ct|H|HOOX_yv~{4y(`!*g#f#4P4b%d6HQRu(ro3HkJ&B1eCS1 za_lfFCWO&^WEcRh$ZuQFXt>T?wx`s%#*O8^_5z!j!(M?43f||0C|D#AR8&+a!gN=b zwWp|bc@({g05uAb??_Sx^Y;QJ2kcUxvt?pHEg;`^!>dNtEP9gB5ZKmC|MU-=L;?|A zl_q#)=4=A>-hAgu5V%4nB?)6!tKobEJI2KFskbCwU5PZ}#y|$AVC-ByM?pIIqnfh` zSdI!sP{H-UY0u~uM?Ce;EE!bK$;Vs z4I=x8|Md`H#W`X>fOCgL_;m@Gt4Ws{^0&TR-6p-N7)T|iaaa(F?xn~F&Rt50e=;0^;M zbO0(D{Gi1dKYJfvSz$r;pL4*OmJ6R&yBLg|;nS?1aXA2fQ=VR4YWWXoMLdqdZu&w3 zP`FK|h{=BuXh3%dHegk?Vapza_xe4akpKh=FY5~-2x_Xu!9bn=;n`@$?r$0}{#$TW z_&=?~lIRiK5t9}R?Nu0Xoe+U_ZzrI^`hza11+bR_OCjB1@^Yr%O|GReah5EZ@XY;zQsiHMO@ar&QP?vs5E>*A0IRY-OI|@@{2(C&Vq#*w zdS~n$#Y7n(N1izo#rq3t{N&t90?sWLDq!3(HNS-P8u;|jl%c&7aZ_ixw%?our0kJO zGrYN<5_e$LXs&^qV%PkXV(KP%&+OaH3Gx+)YIJaq2mwJs+I#mNy@mv#{!x~S0%f7Q z7&m?Y;}eq*lY)Yfj7;;)3>CokJlqAD94Zgo5dlh>2aU^W#1Xvm&4}o5`5IDg)1dOV z1MdV0KuJiG3l?*#iVrFKFQSF-0mOdU9gH||hS^;$#5!gPupPin$>DV?P<(g=M}?}| z&NP5MNV0lWCW(KS8PWJr7#q@03KgK|y|w_%GN2u_9{RcVzi&;tg^{;HYt&ZCh%-MJnOz+xQ(J3gb!;QBIUm=IRZtnf4<+>*zpwhlR z@`0F2+&c{H$83uR@0|OA+tja>b|K7HjTEx*LK7~})%*;dAser`M+lw5L4brE6A<`< z?@NS**jYlxs;tQqIP@{VvH)HpcCOquu0KyIQw}g+v(Q3Dz_%_$_}U=jo|uSINZ9+q zi)nyo3;J_f)qLzP4S`abvK3<7@0adQ7$epki-vk`F3W) zfpU}a;o2aP(kqSsgi>h`zR`go$+DO@hY_J1?vXdn!Om=Mjtmz?29i|;+RDlI&?6!u z%=si$e!883^YkUcPf}m?jubtK0)7&k?%N-4;Nva>&~*XlkTk)rJun%XS$@ZE>u{|? z$DdXL+VF_^4rp`bCO`y{pI;4Za(w(LkRUMUW>fi(^#^iqaFw-$^6&3h(37y>ConI# zoqq!lv=_vj?tl^=3AZU4z!4Vr%U2)$#`Yp`S})Z`7cih3#{=isOo$`kwnGG|old}H zixT(o#v-}f+-4!U_xlKLxHi_?pL%}sxIrO!G@xDw1j?AWs4?Uba~0;m8GyNr#=-y~ z5rS^lT%lV%zge|J%7C`_gaa1wrG9Q`lyV?VB-Bhbnt%4AJx>EupJu!!yLRBX#)|Ze zuWgo;#CMPrX;&-kRS1@(;imV!4Fiy<73`7;0-x<3;eW*asXhsG3jk5sALu*lb`=RK z|4!11!#R`o%2wZt-`pAJp{JR1yWAc%7m8RER+H_Ys(mW#whOnK%TKcc0JwAP`_qm z83+0XnKPtTU=5ZEZ3b4paMw0qm`e0N=5HN*{zg|`Fxc8uZ0#dya_D4XLB34h>-b@| zp)7UIZ=6u@1Y?;k(eL2&K8ctO&6`L{h+D!FD(rSeV(DV{kr}iuHqdcVg+~MQBypDw zdw_W){6)!%Pkj~bTY&Eh)L=pGD5s5hSsYZV{~d6!5y(AXC;XpGy2s9o2=wl>CM*0t zOL~o-*d}k>XaEVZD~6ZAeyP92xk$Il>Us0YKnaq7{ajLA^`}oQ|C>qotN6GY#euub zBb1{Hjab{4v>nMF{c5xJ&F=2*`3|=dr7%GxwP?#nu2KS?;NLvD=TsQ#pFe-TJLANS zn2!2cthO;^zAr`mCMfD^H754;A6Rg-T#KSc@pt!mFg=$)7 znHyW>8RKhze^ILEA}A4S#Z9O?k5>GVX=!A9vB~+~gD4zm%&G-&wPPBAzy>dE4^Sa! zoRODc8Zjo4*W#lf=Qifl@%>MRDOja>0Z1U~TzXHc6k{oD`(NDT`uKszoMSJbvXb{Q z{l>G9g?lw_IDs{xiXg>bdX5ft=P#LV%kS?0F$k{+zKPfUS~F7p!~5j8QNZaE%Ugf` z1{aQ8x}S~5Ja;#T1<&5Kz zTY$+m{R_P#X0cK=()hx7t%3g3y7w#x%vd|buAFKL4JSh^KeS)IL<4jgsbBMX@6oQAeT1FWq^N zCY5{izv*^bzvy;)aJn7DaJaIya6fujoTxp3|>&`=B&c{+L-R`e>d-t=|+A9yH;(*zVq(c1&tWnyPJ%?gc=wPGtR3C zqR;jvCBL|S()K~#!yRj~B-W|0e&NbQOzCkQK*wm{fzst^tXou66nFCakiKVnPcko# z`Lly{IMtL#(1KYAs2Z#10VW03$!A)9cUd=zyd3)@43h7?*dU`Fd`VWy;pOwDaRt3O zMgaLRz^85!HmSb$6oO9%-|KLN5n~eD;W~G`S*y){NB|qQ`5YLJ3hVybKLAUG=pX)_ zj1uCg%mG?g0)40s0Xp7GrNougBl*1AE?FRY)x8-jou8(;!L!?*GJS@|xwtO#i-33V zzX*7|nKk4$h2EfS1ad4O&_@RU`2!`vrmN(WKW`M1!d!T+MJ7Q|_*iCK-XNj2T^^@)RnmN( zZ`Gt#>LF0R(#si&qP*OdHeO#ot;r`EcQb?a)>djfDDCy}=Q$*!wc=q`_z4zBQ_NG0{OfA^s~` zltg!IXn@JYy`JSq_IDW^_eLHFF(6BX1b7FDRFV%jpYaeA=nzJn_#ob5ebn3$PHuOy zt?#s%P61=4<(JYzNcXtgo>d{m!s zv*ymw$&*!ixy2;1#C}*FBX32EUa2(6JeL%;sQtY^f0b5rXN5{bPqToVQ7FTL+g21dJc+Hi5n@Iy$c9E_~Z!hJaPG!4iO%sIZV z#ABRcth9s9REjZ=C(BQxUSeq@tvl?M_6`<$IkbAx)sKCAjv28jp8J|SVBv7DoR!$O zGQsiV>Rbt{l%(;MnZ7iUN1&+=&Z`vGy{si{BY8bU%c1Q1!0%vW0HB^v=3mGv51>FN zqSX>2XfGvNEwj-7CM(u=4{vRzvEI4D8j{R!8v%%-B1kwb^7_u}#qX6Og>)hI9w*15w{|GSJfJfM_zU_MJ3{u)EIOcD{iUO4W2@!N z1220=cl=E}i@YAqh*y=NUyJ&Alyt;uEn+ zw$sL;Rp|sg{CF7bjEv*HHs3u>^(l0o2XS;F30+cKQXLnKMgO~e&<&9Z<8NnpuO=7D zfIIG^(f0lrWW(V(FaeisiZW9-N*s&m_X3u`;gcjH(GJ4<(t?_l-hsf7Vj}8vRbdVe zKIypi%e|t9c|)H`mzc}&$3=2d-|^9mpAwV@b^2YBNOq?(=WzwyClz}jae@`~-1w0x zh=ziw5I~N;oJNR!u zeRGo<2lVv><*5PjCgV}kziS%_BRsgY*S<{%>P~@ZQ&1mjkidB~-ff9?ipwa*3|yF^ zOzr{mZ$v9AD_0K>SpjS&chOILzav>+F1Utol2%|DNSt^^H3J2Ba3l?76OCU!folPy zsFS_Dy1cf>;hG;Xrj%;rf}^}&aTy`P3gm9Va^^>d-DG$h7CTWpylJaXpC1`s%RoZG zdKeP+?rsZwmF7j{G4F%Rpwi@7)YR3<6yyEwVFOVTh&3QTV+}@vC}fwIj1)#Xzuh;! zVqWL(pL{a@SS?TV34aHK4s^ItZE#6tje%U1)I!RgWV+8DGyh!-JlG;eDChr490Y9c zwp}}J*6;Xx9&=<}#qrfWo~wYZN!i&4~o3{at97MIJ?@d97EOER!{?x?ephd_ND5&iw``*3;a=9H~M(%;q75<+T zs_$F45SZ_61LnxI7SNiM*%((_u{b*ID>Z`WHHISNHQ6um6w-fa8!QM>r?=*j##l1WOc7>t&c@_8tFi#{&{K2 z_f`Obhf5p=V_st)(bdRe=Y|6KX8vrYe3d4GLI6e0bLlj< zCF}e%$J^9-AO~G#vNtswt!?Equp{PYzzAzA;<~op}6c<#}RLmfCwcG39GTGR(Q|#k}NqjZu#mOA|$fkDrb= z#P^sy_?``zZ|7`BwVdmvCu^gvIn})O**;YB}!G zyxXpc-8GK$;&7zd(MS5yXot2&)k*D&a!>G0H#8#rE{g3!1mqU1#-uy#zhuCFKkxYa zj*~**Uj=qD%elAiguP8t2^Zry|=ciP)?|uPJ*%0@5+^}b}CXO7sv;a zYOExkk5*4&LJjGxG)rRlG-M_&zH--BY%^NdEhS`{yZUo{(c%5HcyQrJ_6*5bnd6x; z{hiCH)AcmU^IjfKv31wy#3vD@&WUR2*p=w0x^iSNbe{V@qt~_qITV){QJ&n+Wewlg zZ`JR7>bTi!3%%U-^x5&R+^jiIopc`eNB?q?Ne5BIybzwCQVnH zdy9rkbX!qi?nx+aY1JcjqPE{}?jMGNX3)CSZ%RX>@Av5Ik~hr?+{z7$HE%JWjuPyY z=#LDu+eN;SQKoX}8Mawf^tl`#zkWWXBz%@(!F|swt8{UuGR1v^f8fSwn!h183KhR{ z*c!eUtB&)Ft8bTEAAQnHjxNHXHVH#!1B}`t7vJ`4dl}5{`F(OaaP9`=zs=bTt&mWw z-9vglaK1`W&_1SEY5vi}k0iU>KH@3ycjr`VY;LZkJK_QD@u3^b@ahrhPGw7#h!dlkH_Rj zu+9F&@}$Nn6KY}Tt$^`%9&Nqy&Z%>T(%C6nZKJWaaKQaovX&Ow^~VkDvVo*ar~IGh zSxzrko6E(k<#8A)rz~QSCO$|p$6wB;q`w#nk`QNM-`O604;V1>>OdI>*z0sgFkajC z*K6fD{+=3Oeq5^3yvK)}zqy$6Yt7W(7VWJPckbGM%_f?nUT@>{RzPU4Xmf&%M%3p= zY00|S^_i?jYV{cTck7D9jnovm4u#h}%cc{nJOq>l(}PD_4)(-X=&kE37Ptf))RAu) zDmRAZYS_Auc6aY~yGR>!Th91yT%x2w)mJPoACzSqu0~U@vd1`2>x_5r-%FaQw}Mh8 zNAGr&t@d$rMy9@WzG`?DU!L|ZzL0l24)(Sd8J{Co{L=9A7t`X6t7(nu+%*wohg>hV zn@Q<-rriPv*yn-R&p`ziOLI%x*+t$w zl&nB+nu{;0HS5TJc6Jt9V|J9}GnuW5&h|Aejc>ZkUb9Jcj?F$UFDufk(@Eeouw+WQ zNEaEb@Y|}IZSr7SpLGg)E5$KiH9`G}re9!!@PJTXolwq}2w+ z&bK_$KlofruY9>Ss7l#B6Y>`8L*Cdqbkov=HE0$-f=aSZdj;NmzMr2eJNNnx`=;vj zdWv)ET*(w=^#r#g#brM2;@0V@GWm5BnAq~JrZtZDVprW}o}W-0<{qw#bg6vJi>3WR zfgvPlyuokS_{BS}4(pUoNuD+M{QhFd*iiR##A%a`o>!-?y4%I!`dT zIG%0$%jP*ZMSF zj%E=9n+k1hZDX^WIOy03h&h~_A!*4MlU5eR{I(xs*SM((C`~%6H}34Eam^Ies@M+< z+ANY8Y4NqVDG+VdHGR7z^L4Hr&A=$oKKybvqkr+jH+7d7QC&TO3G;f!MIuB`>-NZr zUBQOs4$4+>2Ekn>r^x3S4mX}_d6H;#Ab7j<4*y{%Xw%nxK2pdC_*W`r2A~64ZWyVV9HuJ2zu@WT zn~q*{1!M94t~bT;-?FsMiu+I!g5UGZDE-lHX{TkB_K>65K+qiQtDmH~^YyrT)#Vp- zTQ2R53IF%51smT89ary8uU3}G-}v5Lif5CqAtFOU^TY<-;&<#aGMW@}; zllRMpS`E*VU-?`K-rsVI!fSyoPfhJ!6<5!&f>3L-XPKw;a$Pflp4JvPMw{V_q;q`upzsG2D=YFv!Oqa|#{VBpyGw z-C}|KLwxW38nt2v<{B1^QpoXR3-q`F&O>w{V1Q6yX#Uy;;d^0CwnQ@yLXBzh89rF~ zWF-7y7u)p%{REliSD-ZZjTA@X#0P3AV?+4ogw3ePa*$IA`j?n!G%xM^4*X%ja~N>& zoZbl{xL)|!@gLLu-6LLdfg+}!(rYg?ny7U|VoZ2IFGu+%AaU5;BbRCAl8rpKF8o z`4Z3|;J#E=c#XSji15k>W-Yu;@#o{_6EIY#-sE{aUsDeZ*N1bYI7|?Ayq=a&I9m zK0OTrWPi>I5F%O>L@tzSmCje9@CMqohH_`{zXvPOC>Hkb56N4YEH3YeM|c!ChDbN7 z-stbRSPy6Rnj|Hls6 z4?@)Sac-8RqdhGi%14E1q+Iys@0KmGf5Vdw0hf%7wR*?GnDtOYKyF~c#lxHbvcN+A>rOl$(sVUA-E6pS7eo8PPCMu97lv^?y{&Zh zcL;Ii6hIL%Hs+}~nD!}PAnMfAasPRK1w=FmQO|``q`=hI z4mXhX*d%s$CUWx@{tlDaZ3)C?RPxul4`^_~13=2SRe<#8n5K(CSEcdVf}a2gyPa5 z%n^TSQ>L4{KtzT$&B{#C{yD90J(v|W^;ZR=i0gC`yXhQZgs=Xd)mIBXXW|n-O~nCpmHlm#~s-@*@N+0^^{ib4Dyf%}oWMb1p?3uzG__;VDPk=Kl1 z9H0}S_0rAEU>v+MNo&ymUM@0Ru7*Zhj{?jNWS9uj-|<5`%>%ayd%?d==BI>5b2>oi z%n|bUizVvdbCg&H<37zUn4(C~@PfbhDPe?Zy7&4=dq+nPkRtb**Q`NB_S6J|FiN0c zmcVi%`+u(A@z!oIvV@-AyUkgsxIUZAQ#9mc2~$(kTm}gE@-6vHvAm>Z|eKoh=NJ8FS;MkOI9swg#V&lyk`R7yI_wa{M@w9*lu>d@J z)nbZOFyOx*@7yH#)U)2JMi`m`Tu#Q{phE|rI1sTEI3wwFI$$Z`(6{ z@~G0sVxXgUv$QE1*Q!?O*K@rj^_w?Pt`pqwby-&y@4 zwzZsWRa$j%Z;(5G^lorONJ7_(XnZ0fCjPhv6{;7zzE=ht*Ms>3rt7M`jJDSe_Df&! zEJr@oKHEVW;I?U#Ei-QyTIc#0>&d^0I(|4wjZa3zYF%Fu_hP)<9Y=9+HPz*DRPDD_ zc_qg%-fsqo-=Ik*?#KaSTgLo^@gT%>>bW#p>a@QcLl?Z|)RX3LI)~?Wyv&?4m0B&> zsb|B;v{Hd}2E{wRBv;VqROPpFY+(UGOI$ZW)4ii!I_+|qo~Mh01unv+$A>j@Obs>u zbF-l}g#WxszBe&y#1P(352oC^hFMVC-hpcQX=85e!bUXn1L<78_& zTAV}3qIvr%SEAlPGW!Qlfg>*urIIN@v&~idcZ<9acdj}h7WM0F;f%Kh~$Vi;g*qg=v^PGabjMa9K?ih z?nrh!T>0ze@Ij<**2mq4dGCgcp2a-9NJv#LR6?KpK>Kvy&8JqTt`f?6>jmlF?{y<} zjcH2n_&T<~wMnNC3+3EKz_p1HP0k*yBDv~64V~HVG+|bR&WrYAKfWv5Fzr#(mMZEo zXmVxE+yaGGlAMJ8Sy6l?CT{4ZLca2M`Pj@_BMwj=540B8mhsz;8Fx-_p4ruIH&KPu{bf+QZDyK1uqPkSIK(~9-=s}mxbzI-6ag|r=wjhUK zy7SZ}6qQL&wZHb(hq&@a$^`aHZG|`vj_P=5JIZdF12p|2_MYoGUTnkZM0xKq?x{^b zqFd&=NxzuBTfvtD^_A}qqDU2Dmii!vV7-IMz6!YUWzOyHlLWLq6@wW_fjKYEQc{t9 zsWc9*>r<_rx)Q}Bw^c@S6vBmbbSK5s56?fV54ko@TuTSv6Ggc6qCGu1AzHfr7aFAc zejtF52|OR&e)XCl@`H`BJ2K%tlM$yuT12-#2B%fdGbQ4)B@Xx0psqFt^0hXX!Mtk^ z;q%(f*5 zE;*JS2Rph;^D<7>q*p>W6EK*x_xco76;yaWEm%-1$!m-kh}OU;lPC18qD|EXWFs&Y@O(K^~q+VBHR15gz z^i9r0`B-F+j{i8q_Wms0IkYj*$>=i3zr&N)C+~RL#uc06JWUpSoU{6-U+ts&na}h^ z<%`r#iTd7+DddUvCjC*(73B+|)iNO>pPdq~2>;|bcC8Ky>J^H`@c10@s~0nKtCdj1 z_mX$-&8x3Qdq^{j4KD@vw?qy0+VEscf*3TjQT)(*yFg2W@48Ab%a(Dv1nkH*afAVX zXP&PiMok^6Z`BjdZ@f^|)%*Cw7fh`8GUWv3A5_Edo9=AJZIC3{p=#M3#6nIF4+x-ZC2_a>YqF^K$>4Q@S3=MWJo1w ze@actYuZqxd-dVP#BQMJK$`Mr`qmti(yD#|x4e%Gnuo*$AjiPcD0Eo8`iy&7=UeYw z4&T5y&pkX`4<^kSYHpPWuOycum36H-!_l+Ec#nEm+HD)|qa0JP_7WFY9Nb-diDiw% ze(-HCU2&0^jL~kntsa;yvVBM>;cCS;ULLI^-_(Q9nh&+`rRt8+k_mk`oP4~)$ZZwy z*54>H^;B&Yd$;rT+$^e7hmeLzHSK{zEEG&#HmPWBlnvqeROGX=a?h zFh)Khm1A2Grs(LOQ8wW$VvA1*NEQ7Pu=o;!C@SiULJp!l4LdsI6->e%KmXrA%#z^c z5e39DYw-oLr!I)3swV^MCI7k6EC78?D+ER*X^;R&<$=6a{3p(vvjEuWvZ$p-ADIIL zT0VlbO|pNFZB7NBPK|F53rWC%=udR^7c8#R0Mtc^%Yukth)Z*N@>+9_>R%|Y@8_!u zQM&LhT8#DJEg5J&YGM~PT&>p=6;e5M4g$5q993zw}J zZr_%Bprjhu?=`9DDphlE#n00et15o_poDLgdd%mcj*GXn4LfYsD}sD4x=8zziaXXO z!|a!{`+Qf&fdOw`D@<~*GwL1}*|YwL!`eDY+nj8iP52Tl$di+di#dJ0`**6#nf+#hX18p`iBvA1nZPW!gJVR^6=XzPP7D!Wmzbqheafu_rB z!upJSPws68?fpgj#o332-}4#l?CiMnSMQFOS*9G4&$PyQyw=c^DFP8o0QcyhyaK2< zyhH|%d)J3@Hs;NM2&o(57jF(<^-Ka{JVqkpuX5;l2NQ-4^en`tr(s-tOzLx>VUsNZdR?hm>}XDh*NEd|O%8o_aj)tp z9o}M}nG_x6O#1yWG2TgJSb^Fi#XphA6g%!i2gtL#rT0rf#V2=! zzu^Yvt(Q7(9mM51@&_HDDw9g~9y+z+Hdx)sqWyfd_$jzm+1aMmFNvLkUNc41OY6ks z<7G}3Dg=Fe&n|-wyzDg-k|oM?idj{5OIN2rqi@CLseR_RBlaqb+T~;JGKWcyfm}~s z%iMdm7q1+s&t(ifXp~@{RSsbm%Zw)PTZ%Z6Nz{n3XIy?(kfO8=y{%yhNeuqp(7n+w zX&Rhowa&}#1)B9C*w;tRcfZxoy(LbsSra5XJfXPE1;6tZadVhl)J&RdRIdyH*C4#{ z9jf|0ISn(-EJ7abp4%A~NIegoT^Z_R%cZItKrR~~&G*ih7C20_Up%jdHO zSjI%vYT)+QpT#D83W__t+l`7VK8?*b8{!ViH$yL|ORTF0l9oEa^1^vK+3+;f!PC7s zj9T0PG$=3ux5s#RQZbrNdg#Nmp8J0{2aw*ng_chR$usF6v)Nd!j&~M4xigq&{6fod zpvqfDN@k2kY`&?YC{Q+_`>gyR9qLuPpQR%6_TrMWU`|lX@cRF?cjaGAU0XO{6c7|K z%nG6m5fspX(1e7cAPUk@K#f!aTA7@XqND;z38?UZB?ekSL_;gH(l7}jgh^2bm6;kz zAfOOr9t6S^NKb6nGw}X`x0dX{`L$9SY-I5s~buTV35!bJo}Vaf1<;@TS|T> zRxxm7VTLD*LzO(n)61Q9Vj8>NDvc|BU+BoNTW&9MVO`rL6r|^t9xG2OKHFn&QDv#? zb27&Rn?<6Q;UWn~oEP|uLA(=ruH1_8w7WwEcj}!I*})fd*vXytU(Eo>s`$hVW6m96<0k-9m?KmF06W~6IO(* z?>?24@g}Xf1~loRxixca)U|$A^;DIArkj*A-gRl8y!u!#Ag72HSF7Hn@c%@g89dOC zmkGX_gbGt`wFHZW2~)NaS|-XUUiF>iW~JLPF@-%Tw%^{!v>ul2waXp|LIntJLPRxt8j8GLj`!;1-`9QPl(ChS@U9 zvEK8t$T5C5I)s)~C-;WnHZIrw059u^*%IvX#KpGmb@^Y_?3hnQsbmxy(M_EUdGaT0 zH-8v?6l7p-`hbd#oVd6$_$27%iN(vvMz3%DAbL+MR!XWw3p!Yl*No@}p+uoZ1cz__DdCN#(&knc!1zZn+#!i~%59lVzbE zk~JNkOIhyB5AeTyJmEIV%EMe`~A^dcOr^sT9WAYt6s6j*%;~AWR!UV zKGu14{^m|(aW;d7+MVX-S_Rrg&8~!Scc9JMoWW^p=!X>NaP=wI*SrZg%$$mJdy&92 zym0(*VBzBCc2^@n3W(RuRg``QC7ZV;p5~J$Y<23KP^t-so(tsO@4(__2dKM_)|1eL zqH)h%m-PBY#Q%BhQSYeRTX(qAM{sSs_koS%_;8Il#eKi2(Vrg@n^xA42t`@w*fz4_ zu|P>lO9QYK{jSTZ0kU75ehDj>bdHzAlu1~Cir8JzY8N2~SQX0IP7mqz58j9#gz!Do z)<5|7A1UE~L|~#{W%paPP& zWCBXf_f>z3rGbcctrA>{LURQA4BGX!<4JYr#XNRV0mHi~x|q#>$n`2L01WLDNSFbj zppim?LZY@GLFDROz2VpV;+6D_{c3AO?WdN6^dRTT`>ym=Qu91KKJ7Of-<%qo84{6L z;qjA?J9wJWe#~Xqp zH%d6wc^<}-SR!XJUP;Wm5J~=x0uV5A!jm!bwuz%_WP}AN!k1oK@mmh6VPIrWZbnCU^_M@LFwB1%X5)BQ|13S+tJ>jOZ-G2Fkz>%kM-)|n zQEALBu5;iLLoOwr87q)M__66CbYUEMPES4=K*h|}XUUVuQG%3kort|-!RbVZTRj#k zm8C~*x+<^U5jpNz_i6LXe1Q*_OUGU-EaJWT<{8+iv|?Z`XlT-wfA7Yi$sRMVK=xvs55JSqPCNa?8$W%#Qy-+pv}~*&sd95^YhF$F9kPDT z>&*3!F#|#0+O7cLULMkWIaRT@3!L3Pu+Y}Oef#zw;eBD#C0Rirs9kQ<&-=dZB9HF% zi+TT2`h(hFEEz0^v3#Sc{a5{na1g3pF%Q1?tQDkkWrR~F7wyD66`;#g7;uQdf@f;R zQv4$km(s!;O0bAv@WBcZZewA+o-Vl$8v}%aIY>~+! z$8S$Le)kV0I{sy# zhl<*=R)r*bw@V{TD(n6DhA&^dX!tMa(0sRx-apS`(F99_)deBxX>1=>d=LNlxDq9> zt#OrBhNwQCLkxS~lNaWqB@P+viBUFobu$(BR1Cq^GmuN3q+ipL`n(*P@O1vO#E$F(A4B1;JRAiik zR_qNlGwTS?sf{a1_ZJsi!(NFrN63TQamm`uM~KH09#ZQJoCHgH!A??3N5_s4aXJM_hCC^4MX_3m@;cJlq{>B zVz72yk_Na7*>A0SgNjrDYZz#w?RHpPc%lXzg-oqMS40!IPej8S`FlV9LWBN0<}wiK z`nEwmK#esUZ#yZiS;fF>S`sxTYjp*vgDj}A?DvG+J?q7$1G&CQ3?cd~oWKY`g{qh$ zPOO`+pzDw|Wb0>9uM_-NVSg3&7ajQL>Z@k|W0bzg<*ON4Z-~B{5%D+&=*$1D8R-*B YFn=*VH4&P90Rnywhp~3FZ_h;j4Kv9ixBvhE literal 0 HcmV?d00001 diff --git a/hail/python/hailtop/pipeline/docs/_templates/_autosummary/class.rst b/hail/python/hailtop/pipeline/docs/_templates/_autosummary/class.rst new file mode 100644 index 00000000000..c72f9cf3767 --- /dev/null +++ b/hail/python/hailtop/pipeline/docs/_templates/_autosummary/class.rst @@ -0,0 +1,37 @@ +{{ objname | escape | underline }} + +.. currentmodule:: {{ module }} + +.. autoclass:: {{ objname }}() + :show-inheritance: + :members: + :no-inherited-members: + + {% block attributes %} + {% if attributes %} + .. rubric:: Attributes + + .. autosummary:: + :nosignatures: + + {% for item in attributes %} + ~{{ name }}.{{ item }} + {%- endfor %} + {% endif %} + {% endblock %} + + {% block methods %} + {% if (methods | reject('in', inherited_members) | list | count) != 0 %} + + .. rubric:: Methods + + .. autosummary:: + :nosignatures: + + {% for item in methods %} + {% if item not in inherited_members %} + ~{{ name }}.{{ item }} + {% endif %} + {%- endfor %} + {% endif %} + {% endblock %} diff --git a/hail/python/hailtop/pipeline/docs/api.rst b/hail/python/hailtop/pipeline/docs/api.rst new file mode 100644 index 00000000000..0b98f7fa8ef --- /dev/null +++ b/hail/python/hailtop/pipeline/docs/api.rst @@ -0,0 +1,78 @@ +.. _sec-api: + +========== +Python API +========== + +This is the API documentation for ``Pipeline``, and provides detailed information +on the Python programming interface. + +Use ``import hailtop.pipeline`` to access this functionality. + + +.. currentmodule:: hailtop.pipeline + + +Pipelines +~~~~~~~~~ + +A :class:`.Pipeline` is an object that represents the set of tasks to run +and the order or dependencies between the tasks. Each :class:`.Task` has bash commands +to execute, a docker container in which to execute them, and settings for storage, +memory, and CPU. + +.. autosummary:: + :toctree: api/pipelines/ + :nosignatures: + :template: class.rst + + pipeline.Pipeline + task.Task + + +Resources +~~~~~~~~~ + +A :class:`.Resource` is an abstract class that represents files in a :class:`.Pipeline` and +has two subtypes: :class:`.ResourceFile` and :class:`.ResourceGroup`. + +A single file is represented by a :class:`.ResourceFile` which has two subtypes: +:class:`.InputResourceFile` and :class:`.TaskResourceFile`. An InputResourceFile is used +to specify files that are inputs to a :class:`.Pipeline`. These files are not generated as outputs from a +:class:`.Task`. Likewise, a TaskResourceFile is a file that is produced by a task. TaskResourceFiles +generated by one task can be used in subsequent tasks, creating a dependency between the tasks. + +A :class:`.ResourceGroup` represents a collection of files that should be treated as one unit. All files +share a common root, but each file has its own extension. + + +.. autosummary:: + :toctree: api/resources/ + :nosignatures: + :template: class.rst + + resource.Resource + resource.ResourceFile + resource.InputResourceFile + resource.TaskResourceFile + resource.ResourceGroup + + +Backends +~~~~~~~~ + +A :class:`.Backend` is an abstract class that can execute a :class:`.Pipeline`. Currently, +there are two types of backends: :class:`.LocalBackend` and :class:`.BatchBackend`. The +local backend executes a pipeline on your local computer by running a shell script. The Batch +backend executes a pipeline on Google Compute Engine VMs operated by the Hail team +(:ref:`Batch Service `). You can access the UI for the Batch Service +at ``_. + +.. autosummary:: + :toctree: api/backend/ + :nosignatures: + :template: class.rst + + backend.Backend + backend.LocalBackend + backend.BatchBackend \ No newline at end of file diff --git a/hail/python/hailtop/pipeline/docs/batch_service.rst b/hail/python/hailtop/pipeline/docs/batch_service.rst new file mode 100644 index 00000000000..30bf3076a6a --- /dev/null +++ b/hail/python/hailtop/pipeline/docs/batch_service.rst @@ -0,0 +1,163 @@ +.. _sec-batch-service: + +============= +Batch Service +============= + + +What is the Batch Service? +-------------------------- + +Instead of executing jobs on your local computer (the default in Pipeline), you can execute +your jobs on a multi-tenant compute cluster in Google Cloud that is managed by the Hail team +and is called Batch. Batch consists of a scheduler that receives job submission requests +from users and then executes jobs in Docker containers on Google Compute Engine VMs (workers) +that are shared amongst all Batch users. A UI is available at ``_ that allows a +user to see job progress and access logs. + + +.. _file-localization: + +File Localization +----------------- + +A job is executed in three separate Docker containers: input, main, output. The input container +downloads files from google storage to the input container. These input files are either inputs +to the pipeline or are output files that have been generated by a dependent job. The downloaded +files are then passed on to the main container via a shared volume where the user's code is +executed. Finally, the output container runs and uploads any files from the shared volume that +have been specified to be uploaded by the user. These files can either be specified with +:meth:`.Pipeline.write_output` or are file dependencies for downstream jobs. + + +Service Accounts +---------------- + +A service account is automatically created for a new Batch user that is used by Batch to download data +on your behalf. This service account needs to be added to Google Storage buckets with your data and Docker +images under Permissions. + + +Billing +------- + +The cost for executing a job depends on the underlying machine type and how much CPU and +memory is being requested. Currently, Batch runs all jobs on 16 core, preemptible, n1-standard +machines with 100 GB of disk total. The costs are as follows: + +- Compute cost + = $0.01 per core per hour + +- Disk cost + Average number of days per month = 365.25 / 12 = 30.4375 + Cost per GB per hour = $0.17 * 100 / 30.4375 / 24 + = $0.001454 per core per hour + +- IP network cost + = $0.00025 per core per hour + +- Service cost + = $0.01 per core per hour + +The sum of these costs is $0.02170 per core per hour. + +.. note:: + + The amount of CPU reserved for a job can be rounded up if the equivalent memory request + requires a larger fraction of the worker. Currently, each 1 core requested + gets 3.75 GB of memory. Therefore, if a user requests 1 CPU and 7 GB of memory, the user + will get 2 cores for their job and will be billed for 2 cores. + +.. note:: + + If a worker is preempted by google in the middle of running a job, you will be billed for + the time the job was running up until the preemption time. The job will be rescheduled on + a different worker and run again. Therefore, if a job takes 5 minutes to run, but was preempted + after running for 2 minutes and then runs successfully the next time it is scheduled, the + total cost for that job will be 7 minutes. + +.. note:: + + You are billed only for the time in which your job is running. We do not bill for the time it + takes to download input files, pull Docker images, upload log files, or upload output files. + +Setup +----- + +We assume you've already installed Pipeline as described in the +:ref:`Getting Started ` section and we have +created a user account for you and given you a billing project. + +To authenticate your computer with the Batch service, run the following +command in a terminal window: + +.. code-block:: sh + + hailctl auth login + +Executing this command will take you to a login page in your browser window where +you can select your google account to authenticate with. If everything works successfully, +you should see a message "hailctl is now authenticated." in your browser window and no +error messages in the terminal window. + +Submitting a Pipeline to Batch +------------------------------ + +To execute a pipeline on the Batch service rather than locally, first construct a +:class:`.BatchBackend` object with a valid billing project name. Next, pass the :class:`.BatchBackend` +object to the :class:`.Pipeline` constructor with the parameter name `backend`. + +An example of running "Hello World" on the Batch service rather than locally is shown below. +You can open iPython or a Jupyter notebook and run the following pipeline: + +.. code-block:: python + + >>> import hailtop.pipeline as hp + >>> backend = hp.BatchBackend('test') # replace 'test' with your own billing project + >>> p = hp.Pipeline(backend=backend, name='test') + >>> t = p.new_task(name='hello') + >>> t.command('echo "hello world"') + >>> p.run(open=True) + + +Using the UI +------------ + +If you have submitted the pipeline above successfully, then it should open a page in your +browser with a UI page for the pipeline you submitted. This will show a list of all the jobs +in the batch with the current state, exit code, duration, and cost. The possible job states +are as follows: + +- Pending - A job is waiting for its dependencies to complete +- Ready - All of a job's dependencies have completed, but the job has not been scheduled to run +- Running - A job has been scheduled to run on a worker +- Success - A job finished with exit code 0 +- Failure - A job finished with exit code not equal to 0 +- Error - The Docker container had an error (ex: out of memory) + +Clicking on a specific job will take you to a page with the logs for each of the three containers +run per job (:ref:`see above `) as well as a copy of the job spec and detailed +information about the job such as where the job was run, how long it took to pull the image for +each container, and any error messages. + +To see all batches you've submitted, go to ``_. Each batch will have a current state, +number of jobs total, and the number of pending, succeeded, failed, and cancelled jobs as well as the +running cost of the batch (computed from completed jobs only). The possible batch states are as follows: + +- open - Not all jobs in the batch have been successfully submitted. +- running - All jobs in the batch have been successfully submitted. +- success - All jobs in the batch have completed with state "Success" +- failure - Any job has completed with state "Failure" or "Error" +- cancelled - Any job has been cancelled and no jobs have completed with state "Failure" or "Error" + +.. note:: + Jobs can still be running even if the batch has been marked as failure or cancelled. In the case of + 'failure', other jobs that do not depend on the failed job will still run. In the case of cancelled, + it takes time to cancel a batch, especially for larger batches. + +Individual jobs cannot be cancelled or deleted. Instead, you can cancel the entire batch with the "Cancel" +button next to the row for that batch. You can also delete a batch with the "Delete" button. + +.. warning:: + + Deleting a batch only removes it from the UI. You will still be billed for a deleted batch. diff --git a/hail/python/hailtop/pipeline/docs/conf.py b/hail/python/hailtop/pipeline/docs/conf.py new file mode 100644 index 00000000000..7cd1f65c4a3 --- /dev/null +++ b/hail/python/hailtop/pipeline/docs/conf.py @@ -0,0 +1,158 @@ +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/master/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) +import inspect + +# -- Project information ----------------------------------------------------- + +project = 'Pipeline' +copyright = '2020, Hail Team' +author = 'Hail Team' + +# The short X.Y version +version = '' +# The full version, including alpha/beta/rc tags +release = '' + + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +needs_sphinx = '1.5.4' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autosummary', + 'sphinx.ext.autodoc', + 'IPython.sphinxext.ipython_console_highlighting' +] + +automodapi_inheritance_diagram = False + +numpydoc_show_class_members = False + +autosummary_generate = ['api.rst'] +autosummary_generate_overwrite = True + +napoleon_use_rtype = False +napoleon_use_param = False +# napoleon_include_private_with_doc = True + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates', '_templates/_autosummary'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = None + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Extension configuration ------------------------------------------------- + +def get_class_that_defined_method(meth): + if inspect.ismethod(meth): + for cls in inspect.getmro(meth.__self__.__class__): + if cls.__dict__.get(meth.__name__) is meth: + return cls + meth = meth.__func__ # fallback to __qualname__ parsing + if inspect.isfunction(meth): + cls = getattr(inspect.getmodule(meth), + meth.__qualname__.split('.', 1)[0].rsplit('.', 1)[0]) + if isinstance(cls, type): + return cls + return getattr(meth, '__objclass__', None) # handle special descriptor objects + + +def has_docstring(obj): + if inspect.ismethod(obj) or inspect.isfunction(obj): + doc = obj.__doc__ + return doc is not None and len(doc) != 0 + return False + + +def autodoc_skip_member(app, what, name, obj, skip, options): + exclusions = ('__delattr__', '__dict__', '__dir__', '__doc__', '__format__', + '__getattribute__', '__hash__', '__init__', + '__init_subclass__', '__new__', '__reduce__', '__reduce_ex__', + '__repr__', '__setattr__', '__sizeof__', '__str__', + '__subclasshook__', '__weakref__', 'maketrans') + + excluded_classes = ('str',) + + cls = get_class_that_defined_method(obj) + + exclude = (name in exclusions or + (name.startswith('_') and not has_docstring(obj)) or + (cls and cls.__name__ in excluded_classes)) + + return exclude + + +def setup(app): + app.connect('autodoc-skip-member', autodoc_skip_member) + app.setup_extension('sphinx.ext.napoleon') diff --git a/hail/python/hailtop/pipeline/docs/data/example.bed b/hail/python/hailtop/pipeline/docs/data/example.bed new file mode 100644 index 0000000000000000000000000000000000000000..b533853b5c2eac6dd518c9619107f4efa6eafa28 GIT binary patch literal 33 pcmd0iW?aP2)VG?cLxag+71zJt+#8e_I9~8EbulzBalQY~0|2lo3iAK} literal 0 HcmV?d00001 diff --git a/hail/python/hailtop/pipeline/docs/data/example.bim b/hail/python/hailtop/pipeline/docs/data/example.bim new file mode 100644 index 00000000000..bbd205e11ae --- /dev/null +++ b/hail/python/hailtop/pipeline/docs/data/example.bim @@ -0,0 +1,10 @@ +1 1:1:A:C 0.0 1 C A +1 1:2:A:C 0.0 2 C A +1 1:3:A:C 0.0 3 C A +1 1:4:A:C 0.0 4 C A +1 1:5:A:C 0.0 5 C A +1 1:6:A:C 0.0 6 C A +1 1:7:A:C 0.0 7 C A +1 1:8:A:C 0.0 8 C A +1 1:9:A:C 0.0 9 C A +1 1:10:A:C 0.0 10 C A diff --git a/hail/python/hailtop/pipeline/docs/data/example.fam b/hail/python/hailtop/pipeline/docs/data/example.fam new file mode 100644 index 00000000000..2e34893f7d2 --- /dev/null +++ b/hail/python/hailtop/pipeline/docs/data/example.fam @@ -0,0 +1,10 @@ +0 s0 0 0 0 NA +0 s1 0 0 0 NA +0 s2 0 0 0 NA +0 s3 0 0 0 NA +0 s4 0 0 0 NA +0 s5 0 0 0 NA +0 s6 0 0 0 NA +0 s7 0 0 0 NA +0 s8 0 0 0 NA +0 s9 0 0 0 NA diff --git a/hail/python/hailtop/pipeline/docs/data/hello.txt b/hail/python/hailtop/pipeline/docs/data/hello.txt new file mode 100644 index 00000000000..b6fc4c620b6 --- /dev/null +++ b/hail/python/hailtop/pipeline/docs/data/hello.txt @@ -0,0 +1 @@ +hello \ No newline at end of file diff --git a/hail/python/hailtop/pipeline/docs/docker_resources.rst b/hail/python/hailtop/pipeline/docs/docker_resources.rst new file mode 100644 index 00000000000..884dd6b7961 --- /dev/null +++ b/hail/python/hailtop/pipeline/docs/docker_resources.rst @@ -0,0 +1,99 @@ +.. _sec-docker-resources: + +================ +Docker Resources +================ + +What is Docker? +--------------- +Docker is a tool for packaging up operating systems, scripts, and environments in order to +be able to run the same code regardless of what machine the code is executing on. This packaged +code is called an image. There are three parts to Docker: a mechanism for building images, +an image repository called DockerHub, and a way to execute code in an image +called a container. For using Pipeline effectively, we're only going to focus on building images. + +Installation +------------ + +You can install Docker by following the instructions for either `Macs `_ +or for `Linux `_. + + +Creating a Dockerfile +--------------------- + +A Dockerfile contains the instructions for creating an image and is typically called `Dockerfile`. +The first directive at the top of each Dockerfile is `FROM` which states what image to create this +image on top of. For example, we can build off of `ubuntu:18.04` which contains a complete Ubuntu +operating system, but does not have Python installed by default. You can use any image that already +exists to base your image on. An image that has Python preinstalled is `python:3.6-slim-stretch` and +one that has gsutil installed is `google/cloud-sdk:slim`. Be careful when choosing images from unknown +sources! + +In the example below, we create a Dockerfile that is based on `ubuntu:18.04`. In this file, we show an +example of installing PLINK in the image with the `RUN` directive, which is an arbitrary bash command. +First, we download a bunch of utilities that do not come with Ubuntu using `apt-get`. Next, we +download and install PLINK from source. Finally, we can copy files from your local computer to the +docker image using the `COPY` directive. + + +.. code-block:: text + + FROM 'ubuntu:18.04' + + RUN apt-get update && apt-get install -y \ + python3 \ + python3-pip \ + tar \ + wget \ + unzip \ + && \ + rm -rf /var/lib/apt/lists/* + + RUN mkdir plink && \ + (cd plink && \ + wget http://s3.amazonaws.com/plink1-assets/plink_linux_x86_64_20200217.zip && \ + unzip plink_linux_x86_64_20200217.zip && \ + rm -rf plink_linux_x86_64_20200217.zip) + + # copy single script + COPY my_script.py /scripts/ + + # copy entire directory recursively + COPY . /scripts/ + +For more information about Dockerfiles and directives that can be used see the following sources: +- https://docs.docker.com/engine/reference/builder/ +- https://docs.docker.com/develop/develop-images/dockerfile_best-practices/ + + +Building Images +--------------- + +To create a Docker image, we use a series of commands to build the image from a Dockerfile by specifying +the context directory (in this case the current directory `.`). The `-f` option +specifies what Dockerfile to read from. The `-t` option is the name of the image. +More in depth information can be found `here `_. + +.. code-block:: sh + + docker build -t -f Dockerfile . + + +Pushing Images +-------------- + +To use an image with Pipeline, you need to upload your image to a place where Pipeline can access it. +You can store images inside the `Google Container Registry `_ in +addition to Dockerhub. Below is an example of pushing the image to the Google Container Registry. +It's good practice to specify a tag that is unique for your image. If you don't tag your image, the default is +`latest`. + +.. code-block:: sh + + docker tag + docker push gcr.io//: + + +Now you can use your Docker image with Pipeline to run your code with the method :meth:`.Task.image` +specifying the image as `gcr.io//:`! diff --git a/hail/python/hailtop/pipeline/docs/getting_started.rst b/hail/python/hailtop/pipeline/docs/getting_started.rst new file mode 100644 index 00000000000..9a971ebd995 --- /dev/null +++ b/hail/python/hailtop/pipeline/docs/getting_started.rst @@ -0,0 +1,38 @@ +.. _sec-getting_started: + +=============== +Getting Started +=============== + +Installation +------------ + +Pipeline is a Python module available inside the Hail Python package located +at `hailtop.pipeline`. + + +Installing Pipeline on Mac OS X or GNU/Linux with pip +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Create a `conda enviroment +`__ named +``hail`` and install the Hail python library in that environment. If ``conda activate`` doesn't work, `please read these instructions `_ + +.. code-block:: sh + + conda create -n hail python'>=3.6,<3.8' + conda activate hail + pip install hail + + +To try Pipeline out, open iPython or a Jupyter notebook and run: + +.. code-block:: python + + >>> import hailtop.pipeline as hp + >>> p = hp.Pipeline() + >>> t = p.new_task(name='hello') + >>> t.command('echo "hello world"') + >>> p.run() + +You're now all set to run the :ref:`tutorial `! diff --git a/hail/python/hailtop/pipeline/docs/index.rst b/hail/python/hailtop/pipeline/docs/index.rst new file mode 100644 index 00000000000..bac7dcfe0c2 --- /dev/null +++ b/hail/python/hailtop/pipeline/docs/index.rst @@ -0,0 +1,30 @@ +Pipeline +======== + +Pipeline is a Python module for creating and executing tasks. A task consists of a bash +command to run as well as a specification of the resources required and some metadata. +Pipeline allows you to easily build complicated pipelines with many tasks and numerous +dependencies. Tasks can either be executed locally or with the :ref:`Batch Service `. + + +.. image:: _static/images/dags/dags.008.png + + +Contents +======== + +.. toctree:: + :maxdepth: 2 + + Getting Started + Tutorial + Docker Resources + Batch Service + Reference (Python API) + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`search` diff --git a/hail/python/hailtop/pipeline/docs/tutorial.rst b/hail/python/hailtop/pipeline/docs/tutorial.rst new file mode 100644 index 00000000000..2ca3f6a0bb1 --- /dev/null +++ b/hail/python/hailtop/pipeline/docs/tutorial.rst @@ -0,0 +1,435 @@ +.. _sec-tutorial: + +======== +Tutorial +======== + +This tutorial goes through the basic concepts of Pipeline with examples. + + +Import +------ + +Pipeline is located inside the `hailtop` module, which can be installed +as described in the :ref:`Getting Started ` section. + +.. code-block:: python + + >>> import hailtop.pipeline as hp + + +.. _f-strings: + +f-strings +--------- + +f-strings were added to Python in version 3.6 and are denoted by the 'f' character +before a string literal. When creating the string, Python evaluates any expressions +in single curly braces `{...}` using the current variable scope. When Python compiles +the example below, the string 'Alice' is substituted for `{name}` because the variable +`name` is set to 'Alice' in the line above. + +.. code-block:: python + + >>> name = 'Alice' + >>> print(f'hello {name}') + hello Alice + +You can put any arbitrary Python code inside the curly braces and Python will evaluate +the expression correctly. For example, below we evaluate `x + 1` first before compiling +the string. Therefore, we get 'x = 6' as the resulting string. + +.. code-block:: python + + >>> x = 5 + >>> print(f'x = {x + 1}') + x = 6 + +To use an f-string and output a single curly brace in the output string, escape the curly +brace by duplicating the character. For example, `{` becomes `{{` in the string definition, +but will print as `{`. Likewise, `}` becomes `}}`, but will print as `}`. + +.. code-block:: python + + >>> x = 5 + >>> print(f'x = {{x + 1}} plus {x}') + x = {x + 1} plus 5 + +To learn more about f-strings, check out this `tutorial `_. + +Hello World +----------- + +A :class:`.Pipeline` consists of a set of :class:`.Task` to execute. There can be +an arbitrary number of tasks in the pipeline that are executed in order of their dependencies. +A dependency between two tasks states that the dependent task should not run until +the previous task completes. Thus, under the covers a pipeline is a directed acyclic graph (DAG) +of tasks. + +In the example below, we have defined a :class:`.Pipeline` `p` with the name 'hello'. +We use the method :meth:`.Pipeline.new_task` to create a task object which we call `t` and then +use the method :meth:`.Task.command` to tell Pipeline that we want to execute `echo "hello world"`. +However, at this point, Pipeline hasn't actually run the task to print "hello world". All we have +done is specified the tasks and the order in which they should be run. To actually execute the +Pipeline, we call :meth:`.Pipeline.run`. The `name` arguments to both :class:`.Pipeline` and +:class:`.Task` are used in the :ref:`Batch Service UI `. + + +.. code-block:: python + + >>> p = hp.Pipeline(name='hello') + >>> t = p.new_task(name='t1') + >>> t.command('echo "hello world"') + >>> p.run() + + +Now that we know how to create a pipeline with a single task, we call :meth:`.Pipeline.new_task` +twice to create two tasks `s` and `t` which both will print a variant of hello world to stdout. +Calling `p.run()` executes the pipeline. By default, pipelines are executed by the :class:`.LocalBackend` +which runs tasks on your local computer. Therefore, even though these tasks can be run in parallel, +they are still run sequentially. However, if pipelines are executed by the :class:`.BatchBackend` +using the :ref:`Batch Service `, then `s` and `t` can be run in parallel as +there exist no dependencies between them. + +.. code-block:: python + + >>> p = hp.Pipeline(name='hello-parallel') + >>> s = p.new_task(name='t1') + >>> s.command('echo "hello world 1"') + >>> t = p.new_task(name='t2') + >>> t.command('echo "hello world 2"') + >>> p.run() + +To create a dependency between `s` and `t`, we use the method :class:`.Task.depends_on` to +explicitly state that `t` depends on `s`. In both the :class:`.LocalBackend` and +:class:`.BatchBackend`, `s` will always run before `t`. + + +.. code-block:: python + + >>> p = hp.Pipeline(name='hello-serial') + >>> s = p.new_task(name='t1') + >>> s.command('echo "hello world 1"') + >>> t = p.new_task(name='t2') + >>> t.command('echo "hello world 2"') + >>> t.depends_on(s) + >>> p.run() + + +.. _file-dependencies: + +File Dependencies +----------------- + +So far we have created pipelines with two tasks where the dependencies between +them were declared explicitly. However, in many pipelines, we want to have a file +generated by one task be the input to a downstream task. Pipeline has a mechanism +for tracking file outputs and then inferring task dependencies from the usage of +those files. + +In the example below, we have specified two tasks: `s` and `t`. `s` prints +"hello world" as in previous examples. However, instead of printing to stdout, +this time `s` redirects the output to a temporary file defined by `s.ofile`. +`s.ofile` is a Python object of type :class:`.TaskResourceFile` that was created +on the fly when we accessed an attribute of a :class:`.Task` that does not already +exist. Any time we access the attribute again (in this example `ofile`), we get the +same :class:`.TaskResourceFile` that was previously created. However, be aware that +you cannot use an existing method or property name of :class:`.Task` objects such +as :meth:`.Task.command` or :meth:`.Task.image`. + +Note the 'f' character before the string in the command for `s`! We placed `s.ofile` in curly braces so +when Python interpolates the :ref:`f-string `, it replaced the +:class:`.TaskResourceFile` object with an actual file path into the command for `s`. +We use another f-string in `t`'s command where we print the contents of `s.ofile` to stdout. +`s.ofile` is the same temporary file that was created in the command for `t`. Therefore, +pipeline deduces that `t` must depend on `s` and thus creates an implicit dependency for `t` on `s`. +In both the :class:`.LocalBackend` and :class:`.BatchBackend`, `s` will always run before `t`. + +.. code-block:: python + + >>> p = hp.Pipeline(name='hello-serial') + >>> s = p.new_task(name='t1') + >>> s.command(f'echo "hello world" > {s.ofile}') + >>> t = p.new_task(name='t2') + >>> t.command(f'cat {s.ofile}') + >>> p.run() + + +Scatter / Gather +---------------- + +Pipeline is implemented in Python making it easy to use for loops +to create more complicated dependency graphs between tasks. We define a scatter +to be a pipeline that runs the same command with varying input parameters and a gather +is a final task or "sink" that waits for all of the tasks in the scatter to be complete +before executing. + +In the example below, we use a for loop to create a task for each one of +'Alice', 'Bob', and 'Dan' that prints the name of the user programatically +thereby scattering the echo command over users. + + + +.. code-block:: python + + >>> p = hp.Pipeline(name='scatter') + >>> for name in ['Alice', 'Bob', 'Dan']: + ... t = p.new_task(name=name) + ... t.command(f'echo "hello {name}"') + >>> p.run() + +In the previous example, we did not assign the tasks we created for each +user to a unique variable name and instead named it `t` each time in the +for loop. However, if we want to add a final gather task (`sink`) that depends on the +completion of all user tasks, then we need to keep track of all of the user +tasks so we can use the :meth:`.Task.depends_on` method to explicitly link +the `sink` task to be dependent on the user tasks, which are stored in the +`tasks` array. The single asterisk before `tasks` is used in Python to have +all elements in the array be treated as separate input arguments to the function, +in this case :meth:`.Task.depends_on`. + +.. image:: _static/images/dags/dags.005.png + +.. code-block:: python + + >>> p = hp.Pipeline(name='scatter-gather-1') + >>> tasks = [] + >>> for name in ['Alice', 'Bob', 'Dan']: + ... t = p.new_task(name=name) + ... t.command(f'echo "hello {name}"') + ... tasks.append(t) + >>> sink = p.new_task(name='sink') + >>> sink.command(f'echo "I wait for everyone"') + >>> sink.depends_on(*tasks) + >>> p.run() + +Now that we know how to create a `sink` task that depends on an arbitrary +number of tasks, we want to have the outputs of each of the per-user tasks +be implicit file dependencies in the `sink` task (see the section on +:ref:`file dependencies `). The changes from the previous +example to make this happen are each task `t` uses an :ref:`f-string ` +to create a temporary output file `t.ofile` where the output to echo is redirected. +We then use all of the output files in the `sink` command by creating a string +with the temporary output file names for each task. A :class:`.TaskResourceFile` +is a Pipeline-specific object that inherits from `str`. Therefore, you can use +:class:`.TaskResourceFile` as if they were strings, which we do with the `join` +command for strings. + +.. image:: _static/images/dags/dags.006.png + +.. code-block:: python + + >>> p = hp.Pipeline(name='scatter-gather-2') + >>> tasks = [] + >>> for name in ['Alice', 'Bob', 'Dan']: + ... t = p.new_task(name=name) + ... t.command(f'echo "hello {name}" > {t.ofile}') + ... tasks.append(t) + >>> sink = p.new_task(name='sink') + >>> sink.command('cat {}'.format(' '.join([t.ofile for t in tasks]))) + >>> p.run() + + +Nested Scatters +--------------- + +We can also create a nested scatter where we do a series of tasks per user. +This is equivalent to a nested for loop. In the example below, we instantiate a +new :class:`.Pipeline` object `p`. Then for each user in 'Alice', 'Bob', and 'Dan' +we create new tasks for making the bed, doing laundry, and grocery shopping. In total, +we will have created 9 tasks that run in parallel as we did not define any dependencies +between the tasks. + +.. image:: _static/images/dags/dags.007.png + +.. code-block:: python + + >>> p = hp.Pipeline(name='nested-scatter-1') + >>> for user in ['Alice', 'Bob', 'Dan']: + ... for chore in ['make-bed', 'laundry', 'grocery-shop']: + ... t = p.new_task(name=f'{user}-{chore}') + ... t.command(f'echo "user {user} is doing chore {chore}"') + >>> p.run() + + +We can implement the same example as above with a function that implements the inner +for loop. The `do_chores` function takes a :class:`.Pipeline` object to add new tasks +to and a user name for whom to create chore tasks for. Like above, we create 9 independent +tasks. However, by structuring the code into smaller functions that take pipeline objects, +we can create more complicated dependency graphs and reuse components across various pipelines. + + +.. code-block:: python + + >>> def do_chores(p, user): + ... for chore in ['make-bed', 'laundry', 'grocery-shop']: + ... t = p.new_task(name=f'{user}-{chore}') + ... t.command(f'echo "user {user} is doing chore {chore}"') + + >>> p = hp.Pipeline(name='nested-scatter-2') + >>> for user in ['Alice', 'Bob', 'Dan']: + ... do_chores(p, user) + >>> p.run() + +Lastly, we provide an example of a more complicated pipeline that has an initial +task, then scatters tasks per user, then has a series of gather / sink tasks +to wait for the per user tasks to be done before completing the pipeline. + +.. image:: _static/images/dags/dags.008.png + +.. code-block:: python + + >>> def do_chores(p, head, user): + ... chores = [] + ... for chore in ['make-bed', 'laundry', 'grocery-shop']: + ... t = p.new_task(name=f'{user}-{chore}') + ... t.command(f'echo "user {user} is doing chore {chore}"') + ... t.depends_on(head) + ... chores.append(t) + ... sink = p.new_task(name=f'{user}-sink') + ... sink.depends_on(*chores) + ... return sink + + >>> p = hp.Pipeline(name='nested-scatter-3') + >>> head = p.new_task(name='head') + >>> user_sinks = [] + >>> for user in ['Alice', 'Bob', 'Dan']: + ... user_sink = do_chores(p, head, user) + ... user_sinks.append(user_sink) + >>> final_sink = p.new_task(name='final-sink') + >>> final_sink.depends_on(*user_sinks) + >>> p.run() + +.. _input-files: + +Input Files +----------- + +Previously, we discussed that :class:`.TaskResourceFile` are temporary files +and are created from :class:`.Task` objects. However, in order to read a file +that was not generated by the pipeline (input file), we use the method +:class:`.Pipeline.read_input` to create an :class:`.InputResourceFile`. An +input resource file can be used exactly in the same way as :class:`.TaskResourceFile`. +We can refer to an input resource file in a command using an f-string. In the example +below, we add the file `data/hello.txt` as an input resource file called `input`. We then +print the contents of `input` to stdout in :class:`.Task` `t`. + +.. code-block:: python + + >>> p = hp.Pipeline(name='hello-input') + >>> input = p.read_input('data/hello.txt') + >>> t = p.new_task(name='hello') + >>> t.command(f'cat {input}') + >>> p.run() + +Why do we need to explicitly add input files to pipelines rather than referring +directly to the path in the command? You could refer directly to the path when using the +:class:`.LocalBackend`, but only if you are not specifying a docker image to use when running +the command with :meth:`.Task.image`. This is because Pipeline copies any input files to a special +temporary directory which gets mounted to the Docker container. When using the :class:`.BatchBackend`, +input files would primarily be files in Google Storage. Many commands do not know how to handle file +paths in Google Storage. Therefore, we suggest explicitly adding all input files as input resource +files to the pipeline so to make sure the same code can run in all scenarios. + + +Output Files +------------ + +All files generated by Pipeline are temporary files! They are copied as appropriate between tasks +for downstream tasks' use, but will be removed when the pipeline has terminated. In order to save +files generated by a pipeline for future use, you need to explicitly call :meth:`.Pipeline.write_output`. +The first argument to :meth:`.Pipeline.write_output` can be any type of :class:`.ResourceFile` which includes input resource +files and task resource files as well as resource groups as described below. The second argument to write_output +should be a local file path when using the :class:`.LocalBackend` and a google storage file path when using +the :class:`.BatchBackend`. + + +.. code-block:: python + + >>> p = hp.Pipeline(name='hello-input') + >>> t = p.new_task(name='hello') + >>> t.command(f'echo "hello" > {t.ofile}') + >>> p.write_output(t.ofile, 'output/hello.txt') + >>> p.run() + + +Resource Groups +--------------- + +Many bioinformatics tools treat files as a group with a common file +path and specific file extensions. For example, `PLINK `_ +stores genetic data in three files: `*.bed` has the genotype data, +`*.bim` has the variant information, and `*.fam` has the sample information. +PLINK can take as an input the path to the files expecting there will be three +files with the appropriate extensions. It also writes files with a common file root and +specific file extensions including when writing out a new dataset or outputting summary statistics. + +To enable Pipeline to work with file groups, we added a :class:`.ResourceGroup` object +that is essentially a dictionary from file extension name to file path. When creating +a :class:`.ResourceGroup` in a :class:`.Task` (equivalent to a :class:`.TaskResourceFile`), +you first need to use the method :meth:`.Task.declare_resource_group` to declare the files +in the resource group explicitly before referring to the resource group in a command. +This is because the default when referring to an attribute on a task that has not been defined +before is to create a :class:`.TaskResourceFile` and not a resource group. + +In the example below, we first declare that `create.bfile` will be a resource group with three files. +The attribute name comes from the name of the key word argument `bfile`. The constructor expects +a dictionary as the value for the key word argument. The dictionary defines the names of each of the files +and the file path where they should be located. In this example, the file paths contain +`{root}` which is the common temporary file path that will get substituted in to create the +final file path. Do not use f-strings here as we substitute a value for `{root}` when creating +the resource group! + +We can then refer to `create.bfile` in commands which gets interpolated with the common temporary file root path +(equivalent to `{root}`) or we can refer to a specific file in the resource group such as `create.bfile.fam`. + +.. code-block:: python + + >>> p = hp.Pipeline(name='resource-groups') + >>> create = p.new_task(name='create-dummy') + >>> create.declare_resource_group(bfile={'bed': '{root}.bed', + ... 'bim': '{root}.bim', + ... 'fam': '{root}.fam'}) + >>> create.command(f'plink --dummy 10 100 --make-bed --out {create.bfile}') + >>> p.run() # doctest: +SKIP + + +As described previously for :ref:`input files `, we need a +separate mechanism for creating a resource group from a set of input files +using the method :meth:`.Pipeline.read_input_group`. The constructor takes +key word arguments that define the name of the file such as `bed` to the path +where that file is located. The resource group is then a dictionary of the name +of the attribute to an :class:`.InputResourceFile`. + +In the example below, we created an input resource group `bfile` with three files. +The group's common root file path can be referred to with `bfile` in a command or +you can reference a specific input file such as `bfile.fam`. + +.. code-block:: python + + >>> p = hp.Pipeline(name='resource-groups') + >>> bfile = p.read_input_group(bed='data/example.bed', + ... bim='data/example.bim', + ... fam='data/example.fam') + >>> wc_bim = p.new_task(name='wc-bim') + >>> wc_bim.command(f'wc -l {bfile.bim}') + >>> wc_fam = p.new_task(name='wc-fam') + >>> wc_fam.command(f'wc -l {bfile.fam}') + >>> p.run() + + +If your tool requires a specific extension for the input files to have such +as the file is gzipped, then you'd need to create the resource group as follows: + +.. code-block:: python + + >>> rg = p.read_input_group(**{'txt.gz': 'data/hello.txt.gz'}) + >>> rg['txt.gz'] + +Backends +-------- + +There are two backends that execute pipelines: the :class:`.LocalBackend` and the +:class:`.BatchBackend`. The local backend is used by default and executes tasks +on your local computer. The Batch backend executes tasks in a shared compute cluster +managed by the Hail team. To use the Batch Service, follow the directions :ref:`here `. diff --git a/hail/python/hailtop/pipeline/pipeline.py b/hail/python/hailtop/pipeline/pipeline.py index 02292d78bd0..deab42a4025 100644 --- a/hail/python/hailtop/pipeline/pipeline.py +++ b/hail/python/hailtop/pipeline/pipeline.py @@ -2,7 +2,7 @@ import re import uuid -from .backend import LocalBackend, BatchBackend +from .backend import LocalBackend from .task import Task from .resource import Resource, InputResourceFile, TaskResourceFile, ResourceGroup from .utils import PipelineException @@ -14,30 +14,35 @@ class Pipeline: Examples -------- - Create a pipeline object: >>> p = Pipeline() - Create a new pipeline task that prints hello to a temporary file `t.ofile`: + Create a new pipeline task that prints "hello": >>> t = p.new_task() - >>> t.command(f'echo "hello" > {t.ofile}') - - Write the temporary file `t.ofile` to a permanent location - - >>> p.write_output(t.ofile, 'output/hello.txt') + >>> t.command(f'echo "hello" ') Execute the DAG: >>> p.run() + Notes + ----- + + The methods :meth:`.Pipeline.read_input` and :meth:`.Pipeline_read_input_group` + are for adding input files to a pipeline. An input file is a file that already + exists before executing a pipeline. + + Files generated by executing a task are temporary files and must be written + to a permanent location using the method :meth:`.Pipeline.write_output`. + Parameters ---------- name: :obj:`str`, optional Name of the pipeline. - backend: :func:`.Backend`, optional - Backend used to execute the jobs. Default is :class:`.LocalBackend` + backend: :class:`.Backend`, optional + Backend used to execute the jobs. Default is :class:`.LocalBackend`. attributes: :obj:`dict` of :obj:`str` to :obj:`str`, optional Key-value pairs of additional attributes. 'name' is not a valid keyword. Use the name argument instead. @@ -90,20 +95,22 @@ def __init__(self, name=None, backend=None, attributes=None, if backend: self._backend = backend - elif os.environ.get('BATCH_URL') is not None: - self._backend = BatchBackend(os.environ.get('BATCH_URL')) else: self._backend = LocalBackend() def new_task(self, name=None, attributes=None): """ Initialize a new task object with default memory, docker image, - and CPU settings if specified upon pipeline creation. + and CPU settings (defined in :class:`.Pipeline`) upon pipeline creation. Examples -------- + Create and execute a pipeline `p` with one task `t` that prints "hello world": - >>> t = p.new_task() + >>> p = Pipeline() + >>> t = p.new_task(name='hello', attributes={'language': 'english'}) + >>> t.command('echo "hello world"') + >>> p.run() Parameters ---------- @@ -187,9 +194,9 @@ def read_input(self, path, extension=None): Read the file `hello.txt`: >>> p = Pipeline() - >>> input = p.read_input('hello.txt') + >>> input = p.read_input('data/hello.txt') >>> t = p.new_task() - >>> t.command(f"cat {input}") + >>> t.command(f'cat {input}') >>> p.run() Parameters @@ -224,15 +231,15 @@ def read_input_group(self, **kwargs): ... bim="data/example.bim", ... fam="data/example.fam") >>> t = p.new_task() - >>> t.command(f"plink --bfile {bfile} --geno --out {t.geno}") + >>> t.command(f"plink --bfile {bfile} --geno --make-bed --out {t.geno}") >>> t.command(f"wc -l {bfile.fam}") >>> t.command(f"wc -l {bfile.bim}") >>> p.run() Read a FASTA file and it's index (file extensions matter!): - >>> fasta = p.read_input_group({'fasta': 'data/example.fasta', - ... 'fasta.idx': 'data/example.fasta.idx'}) + >>> fasta = p.read_input_group(**{'fasta': 'data/example.fasta', + ... 'fasta.idx': 'data/example.fasta.idx'}) Create a resource group where the identifiers don't match the file extensions: @@ -254,7 +261,7 @@ def read_input_group(self, **kwargs): This is equivalent to `"{root}.identifier"` from :meth:`.Task.declare_resource_group`. We are planning on adding flexibility to incorporate more complicated extensions in the future such as `.vcf.bgz`. - For now, use :func:`ResourceFile.add_extension` to add an extension to a + For now, use :meth:`ResourceFile.add_extension` to add an extension to a resource file. Parameters @@ -314,6 +321,10 @@ def write_output(self, resource, dest): # pylint: disable=R0201 raise PipelineException(f"undefined resource '{name}'\n" f"Hint: resources must be defined within the " "task methods 'command' or 'declare_resource_group'") + + if isinstance(self._backend, LocalBackend): + dest = os.path.abspath(dest) + resource._add_output_path(dest) def select_tasks(self, pattern): @@ -326,7 +337,7 @@ def select_tasks(self, pattern): Select tasks in pipeline matching `qc`: >>> p = Pipeline() - >>> t = p.new_task().name('qc') + >>> t = p.new_task(name='qc') >>> qc_tasks = p.select_tasks('qc') >>> assert qc_tasks == [t] @@ -353,9 +364,10 @@ def run(self, dry_run=False, verbose=False, delete_scratch_on_exit=True, **backe >>> p = Pipeline() >>> t = p.new_task() - >>> t.command('echo "hello"') + >>> t.command('echo "hello world"') >>> p.run() + Parameters ---------- dry_run: :obj:`bool`, optional @@ -364,6 +376,8 @@ def run(self, dry_run=False, verbose=False, delete_scratch_on_exit=True, **backe If `True`, print debugging output. delete_scratch_on_exit: :obj:`bool`, optional If `True`, delete temporary directories with intermediate files. + backend_kwargs: key-word arguments, optional + See :meth:`.Backend._run` for backend-specific arguments. """ seen = set() diff --git a/hail/python/hailtop/pipeline/resource.py b/hail/python/hailtop/pipeline/resource.py index db2a4506217..9e5a1e47313 100644 --- a/hail/python/hailtop/pipeline/resource.py +++ b/hail/python/hailtop/pipeline/resource.py @@ -6,6 +6,10 @@ class Resource: + """ + Abstract class for resources. + """ + _uid: str @abc.abstractmethod @@ -84,6 +88,7 @@ def add_extension(self, extension): >>> t = p.new_task() >>> t.command(f'echo "hello" > {t.ofile}') >>> t.ofile.add_extension('.txt') + >>> p.run() Notes ----- @@ -116,6 +121,17 @@ def __repr__(self): class InputResourceFile(ResourceFile): """ Class representing a resource from an input file. + + Examples + -------- + `input` is an :class:`.InputResourceFile` of the pipeline `p` + and is used in task `t`: + + >>> p = Pipeline() + >>> input = p.read_input('data/hello.txt') + >>> t = p.new_task(name='hello') + >>> t.command(f'cat {input}') + >>> p.run() """ def __init__(self, value): @@ -135,10 +151,19 @@ class TaskResourceFile(ResourceFile): """ Class representing an intermediate file from a task. + Examples + -------- + `t.ofile` is a :class:`.TaskResourceFile` on the task `t`: + + >>> p = Pipeline() + >>> t = p.new_task(name='hello-tmp') + >>> t.command(f'echo "hello world" > {t.ofile}') + >>> p.run() + Notes ----- All :class:`.TaskResourceFile` are temporary files and must be written - to a permanent location using :func:`.Pipeline.write_output` if the output needs + to a permanent location using :meth:`.Pipeline.write_output` if the output needs to be saved. """ @@ -166,20 +191,24 @@ class ResourceGroup(Resource): ... bim="data/example.bim", ... fam="data/example.fam") + Create a resource group from a task intermediate: + + >>> t.declare_resource_group(ofile={'bed': '{root}.bed', + ... 'bim': '{root}.bim', + ... 'fam': '{root}.fam'}) + >>> t.command(f"plink --bfile {bfile} --make-bed --out {t.ofile}") + Reference the entire file group: - >>> t.command(f"plink --bfile {bfile} --geno 0.2 --out {t.ofile}") + >>> t.command(f"plink --bfile {bfile} --geno 0.2 --make-bed --out {t.ofile}") Reference a single file: >>> t.command(f"wc -l {bfile.fam}") - Create a resource group from a task intermediate: + Execute the pipeline: - >>> t.declare_resource_group(ofile={'bed': '{root}.bed', - ... 'bim': '{root}.bim', - ... 'fam': '{root}.fam'}) - >>> t.command(f"plink --bfile {bfile} --make-bed --out {t.ofile}") + >>> p.run() # doctest: +SKIP Notes ----- diff --git a/hail/python/hailtop/pipeline/task.py b/hail/python/hailtop/pipeline/task.py index 7a372082807..e8075e87bcf 100644 --- a/hail/python/hailtop/pipeline/task.py +++ b/hail/python/hailtop/pipeline/task.py @@ -111,18 +111,19 @@ def declare_resource_group(self, **mappings): Declare a resource group: + >>> p = Pipeline() >>> input = p.read_input_group(bed='data/example.bed', ... bim='data/example.bim', ... fam='data/example.fam') - >>> t = p.new_task() >>> t.declare_resource_group(tmp1={'bed': '{root}.bed', ... 'bim': '{root}.bim', ... 'fam': '{root}.fam', ... 'log': '{root}.log'}) - >>> t.command(f"plink --bfile {input} --make-bed --out {t.tmp1}") + >>> t.command(f'plink --bfile {input} --make-bed --out {t.tmp1}') + >>> p.run() # doctest: +SKIP - Caution + Warning ------- Be careful when specifying the expressions for each file as this is Python code that is executed with `eval`! @@ -157,6 +158,10 @@ def depends_on(self, *tasks): Examples -------- + Initialize the pipeline: + + >>> p = Pipeline() + Create the first task: >>> t1 = p.new_task() @@ -168,6 +173,10 @@ def depends_on(self, *tasks): >>> t2.depends_on(t1) >>> t2.command(f'echo "world"') + Execute the pipeline: + + >>> p.run() + Notes ----- Dependencies between tasks are automatically created when resources from @@ -200,38 +209,38 @@ def command(self, command): Simple task with no output files: >>> p = Pipeline() - >>> t1 = p.new_task() - >>> t1.command(f'echo "hello"') + >>> t = p.new_task() + >>> t.command(f'echo "hello"') >>> p.run() Simple task with one temporary file `t2.ofile` that is written to a permanent location: >>> p = Pipeline() - >>> t2 = p.new_task() - >>> t2.command(f'echo "hello world" > {t2.ofile}') - >>> p.write_output(t2.ofile, 'output/hello.txt') + >>> t = p.new_task() + >>> t.command(f'echo "hello world" > {t.ofile}') + >>> p.write_output(t.ofile, 'output/hello.txt') >>> p.run() Two tasks with a file interdependency: >>> p = Pipeline() - >>> t3 = p.new_task() - >>> t3.command(f'echo "hello" > {t3.ofile}') - >>> t4 = p.new_task() - >>> t4.command(f'cat {t3.ofile} > {t4.ofile}') - >>> p.write_output(t4.ofile, 'output/cat_output.txt') + >>> t1 = p.new_task() + >>> t1.command(f'echo "hello" > {t1.ofile}') + >>> t2 = p.new_task() + >>> t2.command(f'cat {t1.ofile} > {t2.ofile}') + >>> p.write_output(t2.ofile, 'output/cat_output.txt') >>> p.run() Specify multiple commands in the same task: >>> p = Pipeline() - >>> t5 = p.new_task() - >>> t5.command(f'echo "hello" > {t5.tmp1}') - >>> t5.command(f'echo "world" > {t5.tmp2}') - >>> t5.command(f'echo "!" > {t5.tmp3}') - >>> t5.command(f'cat {t5.tmp1} {t5.tmp2} {t5.tmp3} > {t5.ofile}') - >>> p.write_output(t5.ofile, 'output/concatenated.txt') + >>> t = p.new_task() + >>> t.command(f'echo "hello" > {t.tmp1}') + >>> t.command(f'echo "world" > {t.tmp2}') + >>> t.command(f'echo "!" > {t.tmp3}') + >>> t.command(f'cat {t.tmp1} {t.tmp2} {t.tmp3} > {t.ofile}') + >>> p.write_output(t.ofile, 'output/concatenated.txt') >>> p.run() Notes @@ -310,9 +319,11 @@ def storage(self, storage): Set the task's disk requirements to 1 Gi: - >>> t1 = p.new_task() - >>> (t1.storage('1Gi') - ... .command(f'echo "hello"')) + >>> p = Pipeline() + >>> t = p.new_task() + >>> (t.storage('1Gi') + ... .command(f'echo "hello"')) + >>> p.run() Parameters ---------- @@ -335,9 +346,11 @@ def memory(self, memory): Set the task's memory requirement to 5GB: - >>> t1 = p.new_task() - >>> (t1.memory(5) - ... .command(f'echo "hello"')) + >>> p = Pipeline() + >>> t = p.new_task() + >>> (t.memory(5) + ... .command(f'echo "hello"')) + >>> p.run() Parameters ---------- @@ -359,11 +372,13 @@ def cpu(self, cores): Examples -------- - Set the task's CPU requirement to 2 cores: + Set the task's CPU requirement to 0.1 cores: - >>> t1 = p.new_task() - >>> (t1.cpu(2) - ... .command(f'echo "hello"')) + >>> p = Pipeline() + >>> t = p.new_task() + >>> (t.cpu(0.1) + ... .command(f'echo "hello"')) + >>> p.run() Parameters ---------- @@ -387,9 +402,11 @@ def image(self, image): Set the task's docker image to `alpine`: - >>> t1 = p.new_task() - >>> (t1.image('alpine:latest') - ... .command(f'echo "hello"')) + >>> p = Pipeline() + >>> t = p.new_task() + >>> (t.image('ubuntu:18.04') + ... .command(f'echo "hello"')) + >>> p.run() # doctest: +SKIP Parameters ---------- @@ -412,9 +429,10 @@ def always_run(self, always_run=True): Examples -------- - >>> t1 = p.new_task() - >>> (t1.always_run() - ... .command(f'echo "hello"')) + >>> p = Pipeline() + >>> t = p.new_task() + >>> (t.always_run() + ... .command(f'echo "hello"')) Parameters ---------- diff --git a/hail/python/hailtop/utils/utils.py b/hail/python/hailtop/utils/utils.py index e30c0959c11..ae60b7ae018 100644 --- a/hail/python/hailtop/utils/utils.py +++ b/hail/python/hailtop/utils/utils.py @@ -7,6 +7,7 @@ import urllib3 import socket import requests +import google.auth.exceptions from .time import time_msecs @@ -199,6 +200,8 @@ def is_transient_error(e): # socket.timeout: The read operation timed out # # ConnectionResetError: [Errno 104] Connection reset by peer + # + # google.auth.exceptions.TransportError: ('Connection aborted.', ConnectionResetError(104, 'Connection reset by peer')) if isinstance(e, aiohttp.ClientResponseError) and ( e.status in (408, 500, 502, 503, 504)): # nginx returns 502 if it cannot connect to the upstream server @@ -233,6 +236,8 @@ def is_transient_error(e): return True if isinstance(e, ConnectionResetError): return True + if isinstance(e, google.auth.exceptions.TransportError): + return is_transient_error(e.__cause__) return False diff --git a/hail/python/scripts/drive_combiner.py b/hail/python/scripts/drive_combiner.py index 3d620530d0e..dd39c267978 100644 --- a/hail/python/scripts/drive_combiner.py +++ b/hail/python/scripts/drive_combiner.py @@ -1,4 +1,4 @@ -"""A high level script for running the hail gVCF combiner/joint caller""" +"""A high level script for running the hail GVCF combiner/joint caller""" import argparse import time import uuid @@ -74,18 +74,18 @@ def run_combiner(samples, intervals, out_file, tmp_path, header, overwrite=True) def main(): - parser = argparse.ArgumentParser(description="Driver for hail's gVCF combiner") + parser = argparse.ArgumentParser(description="Driver for hail's GVCF combiner") parser.add_argument('--sample-map', help='path to the sample map (must be filesystem local). ' 'The sample map should be tab separated with two columns. ' 'The first column is the sample ID, and the second column ' - 'is the gVCF path.\n' - 'WARNING: the sample names in the gVCFs will be overwritten', + 'is the GVCF path.\n' + 'WARNING: the sample names in the GVCFs will be overwritten', required=True) parser.add_argument('--tmp-path', help='path to folder for temp output (can be a cloud bucket)', default='/tmp') parser.add_argument('--out-file', '-o', help='path to final combiner output', required=True) - parser.add_argument('--json', help='json to use for the import of the gVCFs' + parser.add_argument('--json', help='json to use for the import of the GVCFs' '(must be filesystem local)', required=True) parser.add_argument('--header', help='external header, must be cloud based', required=False) args = parser.parse_args() diff --git a/hail/python/test/hail/experimental/test_experimental.py b/hail/python/test/hail/experimental/test_experimental.py index b533e5bcaba..45bf6f1975e 100644 --- a/hail/python/test/hail/experimental/test_experimental.py +++ b/hail/python/test/hail/experimental/test_experimental.py @@ -268,9 +268,13 @@ def test_sparse(self): assert mt._same(expected_split_mt) def test_define_function(self): - f = hl.experimental.define_function( + f1 = hl.experimental.define_function( lambda a, b: (a + 7) * b, hl.tint32, hl.tint32) - self.assertEqual(hl.eval(f(1, 3)), 24) + self.assertEqual(hl.eval(f1(1, 3)), 24) + f2 = hl.experimental.define_function( + lambda a, b: (a + 7) * b, hl.tint32, hl.tint32) + self.assertEqual(hl.eval(f1(1, 3)), 24) # idempotent + self.assertEqual(hl.eval(f2(1, 3)), 24) # idempotent def test_mt_full_outer_join(self): mt1 = hl.utils.range_matrix_table(10, 10) diff --git a/hail/python/test/hail/methods/test_impex.py b/hail/python/test/hail/methods/test_impex.py index c0e8dc66d2d..a708644a740 100644 --- a/hail/python/test/hail/methods/test_impex.py +++ b/hail/python/test/hail/methods/test_impex.py @@ -1519,6 +1519,7 @@ def test_import_matrix_table_no_cols(self): t = hl.import_table(file, types=fields, key=['Chromosome', 'Position']) self.assertEqual(mt.count_cols(), 0) + self.assertEqual(mt.count_rows(), 231) self.assertTrue(t._same(mt.rows())) @skip_unless_spark_backend() @@ -1662,6 +1663,8 @@ def test_devlish_nine_separated_eight_missing_file(self): 31, None, None, 34] assert actual == expected + assert mt.count_rows() == len(mt.rows().collect()) + actual = mt.chr.collect() assert actual == ['chr1', 'chr1', 'chr1', None] actual = mt[''].collect() diff --git a/hail/python/test/hail/test_ir.py b/hail/python/test/hail/test_ir.py index 3586e7ef7be..ccf306ae502 100644 --- a/hail/python/test/hail/test_ir.py +++ b/hail/python/test/hail/test_ir.py @@ -51,7 +51,6 @@ def value_irs(self): ir.MakeArray([i, ir.NA(hl.tint32), ir.I32(-3)], hl.tarray(hl.tint32)), ir.ArrayRef(a, i, ir.Str('foo')), ir.ArrayLen(a), - ir.ArrayRange(ir.I32(0), ir.I32(5), ir.I32(1)), ir.ArraySort(a, 'l', 'r', ir.ApplyComparisonOp("LT", ir.Ref('l'), ir.Ref('r'))), ir.ToSet(a), ir.ToDict(da), @@ -75,9 +74,9 @@ def value_irs(self): ir.ArrayLeftJoinDistinct(a, a, 'l', 'r', ir.I32(0), ir.I32(1)), ir.ArrayFor(a, 'v', ir.Void()), ir.AggFilter(ir.TrueIR(), ir.I32(0), False), - ir.AggExplode(ir.ArrayRange(ir.I32(0), ir.I32(2), ir.I32(1)), 'x', ir.I32(0), False), + ir.AggExplode(ir.ToArray(ir.StreamRange(ir.I32(0), ir.I32(2), ir.I32(1))), 'x', ir.I32(0), False), ir.AggGroupBy(ir.TrueIR(), ir.I32(0), False), - ir.AggArrayPerElement(ir.ArrayRange(ir.I32(0), ir.I32(2), ir.I32(1)), 'x', 'y', ir.I32(0), False), + ir.AggArrayPerElement(ir.ToArray(ir.StreamRange(ir.I32(0), ir.I32(2), ir.I32(1))), 'x', 'y', ir.I32(0), False), ir.ApplyAggOp('Collect', [], [ir.I32(0)]), ir.ApplyScanOp('Collect', [], [ir.I32(0)]), ir.ApplyAggOp('CallStats', [ir.I32(2)], [call]), @@ -111,7 +110,9 @@ def value_irs(self): ir.MatrixWrite(matrix_read, ir.MatrixPLINKWriter(new_temp_file())), ir.MatrixMultiWrite([matrix_read, matrix_read], ir.MatrixNativeMultiWriter(new_temp_file(), False, False)), ir.BlockMatrixWrite(block_matrix_read, ir.BlockMatrixNativeWriter('fake_file_path', False, False, False)), - ir.LiftMeOut(ir.I32(1)) + ir.LiftMeOut(ir.I32(1)), + ir.BlockMatrixWrite(block_matrix_read, ir.BlockMatrixPersistWriter('x', 'MEMORY_ONLY')), + ir.UnpersistBlockMatrix(block_matrix_read), ] return value_irs @@ -292,6 +293,7 @@ def blockmatrix_irs(self): add_two_bms = ir.BlockMatrixMap2(read, read, 'l', 'r', ir.ApplyBinaryPrimOp('+', ir.Ref('l'), ir.Ref('r')), "Union") negate_bm = ir.BlockMatrixMap(read, 'element', ir.ApplyUnaryPrimOp('-', ir.Ref('element')), False) sqrt_bm = ir.BlockMatrixMap(read, 'element', hl.sqrt(construct_expr(ir.Ref('element'), hl.tfloat64))._ir, False) + persisted = ir.BlockMatrixRead(ir.BlockMatrixPersistReader('x', read)) scalar_to_bm = ir.ValueToBlockMatrix(scalar_ir, [1, 1], 1) col_vector_to_bm = ir.ValueToBlockMatrix(vector_ir, [2, 1], 1) @@ -318,6 +320,7 @@ def blockmatrix_irs(self): return [ read, + persisted, add_two_bms, negate_bm, sqrt_bm, diff --git a/hail/src/main/scala/is/hail/HailContext.scala b/hail/src/main/scala/is/hail/HailContext.scala index d687794a880..5a1004fad84 100644 --- a/hail/src/main/scala/is/hail/HailContext.scala +++ b/hail/src/main/scala/is/hail/HailContext.scala @@ -5,7 +5,6 @@ import java.util.Properties import is.hail.annotations._ import is.hail.backend.Backend -import is.hail.backend.distributed.DistributedBackend import is.hail.backend.spark.SparkBackend import is.hail.expr.ir import is.hail.expr.ir.functions.IRFunctionRegistry @@ -290,33 +289,6 @@ object HailContext { hc } - def createDistributed(hostname: String, - logFile: String = "hail.log", - quiet: Boolean = false, - append: Boolean = false, - minBlockSize: Long = 1L, - branchingFactor: Int = 50, - tmpDir: String = "/tmp", - optimizerIterations: Int = 3): HailContext = contextLock.synchronized { - require(theContext == null) - checkJavaVersion() - val hConf = new hadoop.conf.Configuration() - - configureLogging(logFile, quiet, append) - - hConf.set("io.compression.codecs", hailCompressionCodecs.mkString(",")) - val fs = new HadoopFS(new SerializableHadoopConfiguration(hConf)) - val hc = new HailContext(new DistributedBackend(hostname, hConf), fs, logFile, tmpDir, branchingFactor, optimizerIterations) - - info(s"Running Hail version ${ hc.version }") - theContext = hc - - // needs to be after `theContext` is set, since this creates broadcasts - ReferenceGenome.addDefaultReferences() - hc - } - - def clear() { ReferenceGenome.reset() IRFunctionRegistry.clearUserFunctions() diff --git a/hail/src/main/scala/is/hail/annotations/RegionValueBuilder.scala b/hail/src/main/scala/is/hail/annotations/RegionValueBuilder.scala index 77f312c36b9..0c137883748 100644 --- a/hail/src/main/scala/is/hail/annotations/RegionValueBuilder.scala +++ b/hail/src/main/scala/is/hail/annotations/RegionValueBuilder.scala @@ -38,7 +38,7 @@ class RegionValueBuilder(var region: Region) { else { val i = indexstk.top typestk.top match { - case t: PBaseStruct => + case t: PCanonicalBaseStruct => offsetstk.top + t.byteOffsets(i) case t: PArray => elementsOffsetstk.top + i * t.elementByteSize @@ -51,7 +51,7 @@ class RegionValueBuilder(var region: Region) { root else { typestk.top match { - case t: PBaseStruct => + case t: PCanonicalBaseStruct => val i = indexstk.top t.types(i) case t: PArray => @@ -300,76 +300,6 @@ class RegionValueBuilder(var region: Region) { endBaseStruct() } - def fixupBinary(pt: PBinary, fromRegion: Region, fromAddress: Long): Long = { - val length = pt.loadLength(fromAddress) - val dstAddress = pt.allocate(region, length) - Region.copyFrom(fromAddress, dstAddress, pt.contentByteSize(length)) - dstAddress - } - - def requiresFixup(t: PType): Boolean = { - t match { - case t: PBaseStruct => t.types.exists(requiresFixup) - case _: PArray | _: PBinary => true - case _ => false - } - } - - def fixupArray(t: PArray, fromRegion: Region, fromAOff: Long): Long = { - val length = t.loadLength(fromAOff) - val toAOff = t.copyFrom(region, fromAOff) - - if (region.ne(fromRegion) && requiresFixup(t.elementType)) { - var i = 0 - while (i < length) { - if (t.isElementDefined(fromAOff, i)) { - t.elementType match { - case t2: PBaseStruct => - fixupStruct(t2, t.elementOffset(toAOff, length, i), fromRegion, t.elementOffset(fromAOff, length, i)) - - case t2: PArray => - val toAOff2 = fixupArray(t2, fromRegion, t.loadElement(fromAOff, length, i)) - Region.storeAddress(t.elementOffset(toAOff, length, i), toAOff2) - - case t2: PBinary => - val toBOff = fixupBinary(t2, fromRegion, t.loadElement(fromAOff, length, i)) - Region.storeAddress(t.elementOffset(toAOff, length, i), toBOff) - - case _ => - } - } - i += 1 - } - } - - toAOff - } - - def fixupStruct(t: PBaseStruct, toOff: Long, fromRegion: Region, fromOff: Long) { - assert(region.ne(fromRegion)) - - var i = 0 - while (i < t.size) { - if (t.isFieldDefined(fromOff, i)) { - t.types(i) match { - case t2: PBaseStruct => - fixupStruct(t2, t.fieldOffset(toOff, i), fromRegion, t.fieldOffset(fromOff, i)) - - case t2: PBinary => - val toBOff = fixupBinary(t2, fromRegion, t.loadField(fromOff, i)) - Region.storeAddress(t.fieldOffset(toOff, i), toBOff) - - case t2: PArray => - val toAOff = fixupArray(t2, fromRegion, t.loadField(fromOff, i)) - Region.storeAddress(t.fieldOffset(toOff, i), toAOff) - - case _ => - } - } - i += 1 - } - } - def addField(t: PBaseStruct, fromRegion: Region, fromOff: Long, i: Int) { if (t.isFieldDefined(fromOff, i)) addRegionValue(t.types(i), fromRegion, t.loadField(fromOff, i)) @@ -444,51 +374,18 @@ class RegionValueBuilder(var region: Region) { def addRegionValue(t: PType, fromRegion: Region, fromOff: Long) { val toT = currentType() - assert(toT == t.fundamentalType) if (typestk.isEmpty) { - if (region.eq(fromRegion)) { - start = fromOff - advance() - return - } - - allocateRoot() + val r = toT.copyFromType(region, t.fundamentalType, fromOff, region.ne(fromRegion)) + start = r + return } val toOff = currentOffset() assert(typestk.nonEmpty || toOff == start) - t.fundamentalType match { - case t: PBaseStruct => - Region.copyFrom(fromOff, toOff, t.byteSize) - if (region.ne(fromRegion)) - fixupStruct(t, toOff, fromRegion, fromOff) - case t: PArray => - if (region.eq(fromRegion)) { - assert(!typestk.isEmpty) - Region.storeAddress(toOff, fromOff) - } else { - val toAOff = fixupArray(t, fromRegion, fromOff) - if (typestk.nonEmpty) - Region.storeAddress(toOff, toAOff) - else - start = toAOff - } - case t2: PBinary => - if (region.eq(fromRegion)) { - assert(!typestk.isEmpty) - Region.storeAddress(toOff, fromOff) - } else { - val toBOff = fixupBinary(t2, fromRegion, fromOff) - if (typestk.nonEmpty) - Region.storeAddress(toOff, toBOff) - else - start = toBOff - } - case _ => - Region.copyFrom(fromOff, toOff, t.byteSize) - } + toT.constructAtAddress(toOff, region, t.fundamentalType, fromOff, region.ne(fromRegion)) + advance() } diff --git a/hail/src/main/scala/is/hail/annotations/StagedRegionValueBuilder.scala b/hail/src/main/scala/is/hail/annotations/StagedRegionValueBuilder.scala index ed2adf15945..f28116a76f8 100644 --- a/hail/src/main/scala/is/hail/annotations/StagedRegionValueBuilder.scala +++ b/hail/src/main/scala/is/hail/annotations/StagedRegionValueBuilder.scala @@ -9,85 +9,28 @@ import is.hail.expr.types.virtual.{TBoolean, TFloat32, TFloat64, TInt32, TInt64, import is.hail.utils._ object StagedRegionValueBuilder { - def fixupStruct(fb: EmitFunctionBuilder[_], region: Code[Region], typ: PBaseStruct, value: Code[Long]): Code[Unit] = { - coerce[Unit](Code(typ.fields.map { f => - if (f.typ.isPrimitive) - Code._empty - else { - val fix = f.typ.fundamentalType match { - case t@(_: PBinary | _: PArray) => - val off = fb.newField[Long] - Code( - off := typ.fieldOffset(value, f.index), - Region.storeAddress(off, deepCopyFromOffset(fb, region, t, coerce[Long](Region.loadIRIntermediate(t)(off)))) - ) - case t: PBaseStruct => - val off = fb.newField[Long] - Code(off := typ.fieldOffset(value, f.index), - fixupStruct(fb, region, t, off)) - } - typ.isFieldDefined(value, f.index).mux(fix, Code._empty) - } - }: _*)) - } - - def fixupArray(fb: EmitFunctionBuilder[_], region: Code[Region], typ: PArray, value: Code[Long]): Code[Unit] = { - if (typ.elementType.isPrimitive) - return Code._empty - - val i = fb.newField[Int] - val len = fb.newField[Int] - - val perElt = typ.elementType.fundamentalType match { - case t@(_: PBinary | _: PArray) => - val off = fb.newField[Long] - Code( - off := typ.elementOffset(value, len, i), - Region.storeAddress(off, deepCopyFromOffset(fb, region, t, coerce[Long](Region.loadIRIntermediate(t)(off))))) - case t: PBaseStruct => - val off = fb.newField[Long] - Code(off := typ.elementOffset(value, len, i), - fixupStruct(fb, region, t, off)) - } - Code( - i := 0, - len := typ.loadLength(value), - Code.whileLoop(i < len, - typ.isElementDefined(value, i).mux(perElt, Code._empty), - i := i + 1)) - } - def deepCopy(fb: EmitFunctionBuilder[_], region: Code[Region], typ: PType, value: Code[_], dest: Code[Long]): Code[Unit] = { - typ.fundamentalType match { - case t if t.isPrimitive => Region.storePrimitive(t, dest)(value) - case t@(_: PBinary | _: PArray) => - Region.storeAddress(dest, deepCopyFromOffset(fb, region, t, coerce[Long](value))) - case t: PBaseStruct => - Code(Region.copyFrom(coerce[Long](value), dest, t.byteSize), - fixupStruct(fb, region, t, dest)) - case t => fatal(s"unknown type $t") + val t = typ.fundamentalType + val valueTI = ir.typeToTypeInfo(t) + val mb = fb.getOrDefineMethod("deepCopy", ("deepCopy", typ), + Array[TypeInfo[_]](classInfo[Region], valueTI, LongInfo), UnitInfo) { mb => + val r = mb.getArg[Region](1) + val value = mb.getArg(2)(valueTI) + val dest = mb.getArg[Long](3) + mb.emit(t.constructAtAddressFromValue(mb, dest, r, t, value, true)) } + mb.invoke(region, value, dest) } def deepCopyFromOffset(fb: EmitFunctionBuilder[_], region: Code[Region], typ: PType, value: Code[Long]): Code[Long] = { - val offset = fb.newField[Long] - - val copy = typ.fundamentalType match { - case t: PBinary => - Code( - offset := t.allocate(region, t.loadLength(value)), - Region.copyFrom(value, offset, t.contentByteSize(t.loadLength(value)))) - case t: PArray => - Code( - offset := t.copyFrom(fb.apply_method, region, value), - fixupArray(fb, region, t, offset)) - case t => - Code( - offset := region.allocate(t.alignment, t.byteSize), - deepCopy(fb, region, t, Region.getIRIntermediate(t)(value), offset)) + val t = typ.fundamentalType + val mb = fb.getOrDefineMethod("deepCopyFromOffset", ("deepCopyFromOffset", typ), + Array[TypeInfo[_]](classInfo[Region], LongInfo), LongInfo) { mb => + val r = mb.getArg[Region](1) + val value = mb.getArg[Long](2) + mb.emit(t.copyFromType(mb, r, t, value, true)) } - - Code(copy, offset) + mb.invoke(region, value) } def deepCopyFromOffset(er: EmitRegion, typ: PType, value: Code[Long]): Code[Long] = @@ -180,7 +123,7 @@ class StagedRegionValueBuilder private(val mb: MethodBuilder, val typ: PType, va } def start(init: Boolean): Code[Unit] = { - val t = ftype.asInstanceOf[PBaseStruct] + val t = ftype.asInstanceOf[PCanonicalBaseStruct] var c = if (pOffset == null) startOffset.store(region.allocate(t.alignment, t.byteSize)) else @@ -196,7 +139,7 @@ class StagedRegionValueBuilder private(val mb: MethodBuilder, val typ: PType, va def setMissing(): Code[Unit] = { ftype match { case t: PArray => t.setElementMissing(startOffset, idx) - case t: PBaseStruct => + case t: PCanonicalBaseStruct => if (t.fieldRequired(staticIdx)) Code._fatal("Required field cannot be missing.") else @@ -207,7 +150,7 @@ class StagedRegionValueBuilder private(val mb: MethodBuilder, val typ: PType, va def currentPType(): PType = { ftype match { case t: PArray => t.elementType - case t: PBaseStruct => + case t: PCanonicalBaseStruct => t.types(staticIdx) case t => t } @@ -275,10 +218,17 @@ class StagedRegionValueBuilder private(val mb: MethodBuilder, val typ: PType, va case _: PInt64 => v => addLong(v.asInstanceOf[Code[Long]]) case _: PFloat32 => v => addFloat(v.asInstanceOf[Code[Float]]) case _: PFloat64 => v => addDouble(v.asInstanceOf[Code[Double]]) - case _: PBaseStruct => v => Region.copyFrom(v.asInstanceOf[Code[Long]], currentOffset, t.byteSize) - case _: PArray => v => addAddress(v.asInstanceOf[Code[Long]]) - case _: PBinary => v => addAddress(v.asInstanceOf[Code[Long]]) - case ft => throw new UnsupportedOperationException("Unknown fundamental type: " + ft) + case t => + val current = currentPType() + val valueTI = ir.typeToTypeInfo(t) + val m = mb.fb.getOrDefineMethod("addIRIntermediate", ("addIRIntermediate", current, t), + Array[TypeInfo[_]](classInfo[Region], valueTI, LongInfo), UnitInfo) { mb => + val r = mb.getArg[Region](1) + val value = mb.getArg(2)(valueTI) + val dest = mb.getArg[Long](3) + mb.emit(current.constructAtAddressFromValue(mb, dest, r, t, value, false)) + } + v => coerce[Unit](m.invoke(region, v, currentOffset)) } def addWithDeepCopy(t: PType, v: Code[_]): Code[Unit] = @@ -292,7 +242,7 @@ class StagedRegionValueBuilder private(val mb: MethodBuilder, val typ: PType, va elementsOffset := elementsOffset + t.elementByteSize, idx := idx + 1 ) - case t: PBaseStruct => + case t: PCanonicalBaseStruct => staticIdx += 1 if (staticIdx < t.size) elementsOffset := elementsOffset + (t.byteOffsets(staticIdx) - t.byteOffsets(staticIdx - 1)) diff --git a/hail/src/main/scala/is/hail/asm4s/FunctionBuilder.scala b/hail/src/main/scala/is/hail/asm4s/FunctionBuilder.scala index 04d1bd89ab4..09086769da2 100644 --- a/hail/src/main/scala/is/hail/asm4s/FunctionBuilder.scala +++ b/hail/src/main/scala/is/hail/asm4s/FunctionBuilder.scala @@ -3,6 +3,7 @@ package is.hail.asm4s import java.io._ import java.util +import is.hail.expr.ir.EmitMethodBuilder import is.hail.utils._ import org.apache.spark.TaskContext import org.objectweb.asm.Opcodes._ @@ -219,6 +220,20 @@ class FunctionBuilder[F >: Null](val parameterTypeInfo: Array[MaybeGenericTypeIn l.foreach(init.instructions.add _) } + private[this] val methodMemo: mutable.Map[Any, MethodBuilder] = mutable.HashMap.empty + + def getOrDefineMethod(suffix: String, key: Any, argsInfo: Array[TypeInfo[_]], returnInfo: TypeInfo[_]) + (f: MethodBuilder => Unit): MethodBuilder = { + methodMemo.get(key) match { + case Some(mb) => mb + case None => + val mb = newMethod(suffix, argsInfo, returnInfo) + f(mb) + methodMemo(key) = mb + mb + } + } + protected[this] val children: mutable.ArrayBuffer[DependentFunction[_]] = new mutable.ArrayBuffer[DependentFunction[_]](16) private[this] lazy val _apply_method: MethodBuilder = { diff --git a/hail/src/main/scala/is/hail/backend/Backend.scala b/hail/src/main/scala/is/hail/backend/Backend.scala index c9acd9c3d9e..474e12d8683 100644 --- a/hail/src/main/scala/is/hail/backend/Backend.scala +++ b/hail/src/main/scala/is/hail/backend/Backend.scala @@ -7,9 +7,10 @@ import is.hail.annotations.{Region, SafeRow} import is.hail.backend.spark.SparkBackend import is.hail.expr.JSONAnnotationImpex import is.hail.expr.ir.lowering.{LowererUnsupportedOperation, LoweringPipeline} -import is.hail.expr.ir.{Compilable, Compile, CompileAndEvaluate, ExecuteContext, IR, MakeTuple, Pretty, TypeCheck} +import is.hail.expr.ir.{BlockMatrixIR, Compilable, Compile, CompileAndEvaluate, ExecuteContext, IR, MakeTuple, Pretty, TypeCheck} import is.hail.expr.types.physical.PTuple import is.hail.expr.types.virtual.TVoid +import is.hail.linalg.BlockMatrix import is.hail.utils._ import org.json4s.DefaultFormats import org.json4s.jackson.{JsonMethods, Serialization} @@ -18,7 +19,15 @@ import scala.reflect.ClassTag abstract class BroadcastValue[T] { def value: T } +abstract class ValueCache { + def persistBlockMatrix(id: String, value: BlockMatrix, storageLevel: String): Unit + def getPersistedBlockMatrix(id: String): BlockMatrix + def unpersistBlockMatrix(id: String): Unit +} + abstract class Backend { + def cache: ValueCache = asSpark().cache + def broadcast[T: ClassTag](value: T): BroadcastValue[T] def parallelizeAndComputeWithIndex[T: ClassTag, U : ClassTag](collection: Array[T])(f: (T, Int) => U): Array[U] diff --git a/hail/src/main/scala/is/hail/backend/distributed/DistributedBackend.scala b/hail/src/main/scala/is/hail/backend/distributed/DistributedBackend.scala deleted file mode 100644 index c8ccea292b0..00000000000 --- a/hail/src/main/scala/is/hail/backend/distributed/DistributedBackend.scala +++ /dev/null @@ -1,29 +0,0 @@ -package is.hail.backend.distributed - -import is.hail.backend.{Backend, BroadcastValue} -import is.hail.scheduler._ -import org.apache.hadoop.conf.Configuration - -import scala.reflect.ClassTag - -class DistributedBroadcastValue[T](val value: T) extends BroadcastValue[T] with Serializable - -class DistributedBackend(hostname: String, hconf: Configuration) extends Backend { - - lazy val scheduler: SchedulerAppClient = new SchedulerAppClient(hostname) - - def broadcast[T: ClassTag](value: T): DistributedBroadcastValue[T] = new DistributedBroadcastValue[T](value) - - def parallelizeAndComputeWithIndex[T: ClassTag, U : ClassTag](collection: Array[T])(f: (T, Int) => U): Array[U] = { - val da = new DArray[U] with Serializable { - type Context = (T, Int) - - val contexts: Array[Context] = collection.zipWithIndex - val body: Context => U = { case (t, i) => f(t, i) } - } - - val result = new Array[U](collection.length) - scheduler.submit(da, { (i: Int, u: U) => result.update(i, u) }) - result - } -} diff --git a/hail/src/main/scala/is/hail/backend/spark/SparkBackend.scala b/hail/src/main/scala/is/hail/backend/spark/SparkBackend.scala index 1c2de5eefdd..8605a70be13 100644 --- a/hail/src/main/scala/is/hail/backend/spark/SparkBackend.scala +++ b/hail/src/main/scala/is/hail/backend/spark/SparkBackend.scala @@ -18,6 +18,8 @@ class SparkBroadcastValue[T](bc: Broadcast[T]) extends BroadcastValue[T] with Se case class SparkBackend(sc: SparkContext) extends Backend { + override val cache: SparkValueCache = SparkValueCache() + def broadcast[T : ClassTag](value: T): BroadcastValue[T] = new SparkBroadcastValue[T](sc.broadcast(value)) def parallelizeAndComputeWithIndex[T : ClassTag, U : ClassTag](collection: Array[T])(f: (T, Int) => U): Array[U] = { diff --git a/hail/src/main/scala/is/hail/backend/spark/SparkValueCache.scala b/hail/src/main/scala/is/hail/backend/spark/SparkValueCache.scala new file mode 100644 index 00000000000..a51ac8c6c73 --- /dev/null +++ b/hail/src/main/scala/is/hail/backend/spark/SparkValueCache.scala @@ -0,0 +1,18 @@ +package is.hail.backend.spark + +import is.hail.backend.ValueCache +import is.hail.linalg.BlockMatrix + +import scala.collection.mutable + +case class SparkValueCache() extends ValueCache { + private[this] val blockmatrices: mutable.Map[String, BlockMatrix] = new mutable.HashMap() + + def persistBlockMatrix(id: String, value: BlockMatrix, storageLevel: String): Unit = + blockmatrices.update(id, value.persist(storageLevel)) + + def getPersistedBlockMatrix(id: String): BlockMatrix = blockmatrices(id) + + def unpersistBlockMatrix(id: String): Unit = + blockmatrices(id).unpersist() +} diff --git a/hail/src/main/scala/is/hail/expr/LowerArrayToStream.scala b/hail/src/main/scala/is/hail/expr/LowerArrayToStream.scala index bd5f70b10fa..d2f2cb4976c 100644 --- a/hail/src/main/scala/is/hail/expr/LowerArrayToStream.scala +++ b/hail/src/main/scala/is/hail/expr/LowerArrayToStream.scala @@ -33,7 +33,6 @@ object LowerArrayToStream { valueName, seq.map(boundary), boundary(result)) case RunAggScan(a, name, init, seq, res, sig) => RunAggScan(toStream(a), name, boundary(init), boundary(seq), boundary(res), sig) case MakeArray(args, t) => MakeStream(args.map(boundary), TStream(t.elementType, t.required)) - case ArrayRange(start, stop, step) => StreamRange(boundary(start), boundary(stop), boundary(step)) case ArrayZip(childIRs, names, body, behavior) => ArrayZip(childIRs.map(toStream), names, boundary(body), behavior) case ArrayMap(a, n, b) => ArrayMap(toStream(a), n, boundary(b)) case ArrayFilter(a, n, b) => ArrayFilter(toStream(a), n, boundary(b)) diff --git a/hail/src/main/scala/is/hail/expr/ir/BlockMatrixIR.scala b/hail/src/main/scala/is/hail/expr/ir/BlockMatrixIR.scala index 4d8dd269b9e..b7429355fec 100644 --- a/hail/src/main/scala/is/hail/expr/ir/BlockMatrixIR.scala +++ b/hail/src/main/scala/is/hail/expr/ir/BlockMatrixIR.scala @@ -85,7 +85,7 @@ case class BlockMatrixRead(reader: BlockMatrixReader) extends BlockMatrixIR { } override protected[ir] def execute(ctx: ExecuteContext): BlockMatrix = { - reader(HailContext.get) + reader(ctx, HailContext.get) } val blockCostIsLinear: Boolean = true @@ -94,13 +94,13 @@ case class BlockMatrixRead(reader: BlockMatrixReader) extends BlockMatrixIR { object BlockMatrixReader { implicit val formats: Formats = new DefaultFormats() { override val typeHints = ShortTypeHints( - List(classOf[BlockMatrixNativeReader], classOf[BlockMatrixBinaryReader])) + List(classOf[BlockMatrixNativeReader], classOf[BlockMatrixBinaryReader], classOf[BlockMatrixPersistReader])) override val typeHintFieldName: String = "name" } } abstract class BlockMatrixReader { - def apply(hc: HailContext): BlockMatrix + def apply(ctx: ExecuteContext, hc: HailContext): BlockMatrix def fullType: BlockMatrixType } @@ -113,7 +113,7 @@ case class BlockMatrixNativeReader(path: String) extends BlockMatrixReader { BlockMatrixType(TFloat64(), tensorShape, isRowVector, metadata.blockSize, sparsity) } - override def apply(hc: HailContext): BlockMatrix = BlockMatrix.read(hc, path) + override def apply(ctx: ExecuteContext, hc: HailContext): BlockMatrix = BlockMatrix.read(hc, path) } case class BlockMatrixBinaryReader(path: String, shape: IndexedSeq[Long], blockSize: Int) extends BlockMatrixReader { @@ -124,18 +124,21 @@ case class BlockMatrixBinaryReader(path: String, shape: IndexedSeq[Long], blockS BlockMatrixType.dense(TFloat64(), nRows, nCols, blockSize) } - override def apply(hc: HailContext): BlockMatrix = { + override def apply(ctx: ExecuteContext, hc: HailContext): BlockMatrix = { val breezeMatrix = RichDenseMatrixDouble.importFromDoubles(hc, path, nRows.toInt, nCols.toInt, rowMajor = true) BlockMatrix.fromBreezeMatrix(hc.sc, breezeMatrix, blockSize) } } +case class BlockMatrixPersistReader(id: String) extends BlockMatrixReader { + lazy val bm: BlockMatrix = HailContext.backend.cache.getPersistedBlockMatrix(id) + lazy val fullType: BlockMatrixType = BlockMatrixType.fromBlockMatrix(bm) + def apply(ctx: ExecuteContext, hc: HailContext): BlockMatrix = bm +} + class BlockMatrixLiteral(value: BlockMatrix) extends BlockMatrixIR { - override lazy val typ: BlockMatrixType = { - val sparsity = BlockMatrixSparsity.fromLinearBlocks(value.nRows, value.nCols, value.blockSize, value.gp.maybeBlocks) - val (shape, isRowVector) = BlockMatrixType.matrixToTensorShape(value.nRows, value.nCols) - BlockMatrixType(TFloat64(), shape, isRowVector, value.blockSize, sparsity) - } + override lazy val typ: BlockMatrixType = + BlockMatrixType.fromBlockMatrix(value) lazy val children: IndexedSeq[BaseIR] = Array.empty[BlockMatrixIR] @@ -392,14 +395,14 @@ case class BlockMatrixDot(left: BlockMatrixIR, right: BlockMatrixIR) extends Blo info(s"BlockMatrix multiply: writing left input with ${ leftBM.nRows } rows and ${ leftBM.nCols } cols " + s"(${ leftBM.gp.nBlocks } blocks of size ${ leftBM.blockSize }) to temporary file $path") leftBM.write(hc.sFS, path) - leftBM = BlockMatrixNativeReader(path).apply(hc) + leftBM = BlockMatrixNativeReader(path).apply(ctx, hc) } if (!right.blockCostIsLinear) { val path = hc.getTemporaryFile(suffix = Some("bm")) info(s"BlockMatrix multiply: writing right input with ${ rightBM.nRows } rows and ${ rightBM.nCols } cols " + s"(${ rightBM.gp.nBlocks } blocks of size ${ rightBM.blockSize }) to temporary file $path") rightBM.write(hc.sFS, path) - rightBM = BlockMatrixNativeReader(path).apply(hc) + rightBM = BlockMatrixNativeReader(path).apply(ctx, hc) } leftBM.dot(rightBM) } diff --git a/hail/src/main/scala/is/hail/expr/ir/BlockMatrixWriter.scala b/hail/src/main/scala/is/hail/expr/ir/BlockMatrixWriter.scala index 670226f978c..383859ab7d7 100644 --- a/hail/src/main/scala/is/hail/expr/ir/BlockMatrixWriter.scala +++ b/hail/src/main/scala/is/hail/expr/ir/BlockMatrixWriter.scala @@ -9,7 +9,8 @@ object BlockMatrixWriter { implicit val formats: Formats = new DefaultFormats() { override val typeHints = ShortTypeHints( List(classOf[BlockMatrixNativeWriter], classOf[BlockMatrixBinaryWriter], classOf[BlockMatrixRectanglesWriter], - classOf[BlockMatrixBinaryMultiWriter], classOf[BlockMatrixTextMultiWriter])) + classOf[BlockMatrixBinaryMultiWriter], classOf[BlockMatrixTextMultiWriter], + classOf[BlockMatrixPersistWriter])) override val typeHintFieldName: String = "name" } } @@ -34,6 +35,11 @@ case class BlockMatrixBinaryWriter(path: String) extends BlockMatrixWriter { } } +case class BlockMatrixPersistWriter(id: String, storageLevel: String) extends BlockMatrixWriter { + def apply(hc: HailContext, bm: BlockMatrix): Unit = + HailContext.backend.cache.persistBlockMatrix(id, bm, storageLevel) +} + case class BlockMatrixRectanglesWriter( path: String, rectangles: Array[Array[Long]], diff --git a/hail/src/main/scala/is/hail/expr/ir/Children.scala b/hail/src/main/scala/is/hail/expr/ir/Children.scala index f25ec8c9e22..03a71f4ba84 100644 --- a/hail/src/main/scala/is/hail/expr/ir/Children.scala +++ b/hail/src/main/scala/is/hail/expr/ir/Children.scala @@ -53,8 +53,6 @@ object Children { Array(a, i, s) case ArrayLen(a) => Array(a) - case ArrayRange(start, stop, step) => - Array(start, stop, step) case StreamRange(start, stop, step) => Array(start, stop, step) case MakeNDArray(data, shape, rowMajor) => @@ -186,9 +184,12 @@ object Children { case BlockMatrixToValueApply(child, _) => Array(child) case BlockMatrixCollect(child) => Array(child) case BlockMatrixWrite(child, _) => Array(child) + case UnpersistBlockMatrix(child) => Array(child) case BlockMatrixMultiWrite(blockMatrices, _) => blockMatrices case CollectDistributedArray(ctxs, globals, _, _, body) => Array(ctxs, globals, body) case ReadPartition(path, _, _) => Array(path) + case ReadValue(path, _, _) => Array(path) + case WriteValue(value, pathPrefix, spec) => Array(value, pathPrefix) case LiftMeOut(child) => Array(child) } } \ No newline at end of file diff --git a/hail/src/main/scala/is/hail/expr/ir/Compilable.scala b/hail/src/main/scala/is/hail/expr/ir/Compilable.scala index e16f6498c87..eef119be51d 100644 --- a/hail/src/main/scala/is/hail/expr/ir/Compilable.scala +++ b/hail/src/main/scala/is/hail/expr/ir/Compilable.scala @@ -13,6 +13,7 @@ object InterpretableButNotCompilable { case _: MatrixMultiWrite => true case _: TableMultiWrite => true case _: BlockMatrixWrite => true + case _: UnpersistBlockMatrix => true case _: BlockMatrixMultiWrite => true case _: TableToValueApply => true case _: MatrixToValueApply => true @@ -37,6 +38,7 @@ object Compilable { case _: TableMultiWrite => false case _: BlockMatrixCollect => false case _: BlockMatrixWrite => false + case _: UnpersistBlockMatrix => false case _: BlockMatrixMultiWrite => false case _: TableToValueApply => false case _: MatrixToValueApply => false diff --git a/hail/src/main/scala/is/hail/expr/ir/Compile.scala b/hail/src/main/scala/is/hail/expr/ir/Compile.scala index 562e7f3110f..192e769b93f 100644 --- a/hail/src/main/scala/is/hail/expr/ir/Compile.scala +++ b/hail/src/main/scala/is/hail/expr/ir/Compile.scala @@ -124,6 +124,18 @@ object Compile { apply[AsmFunction5[Region, T0, Boolean, T1, Boolean, R], R](ctx, print, FastSeq((name0, typ0, classTag[T0]), (name1, typ1, classTag[T1])), body, optimize = true) } + def apply[T0: ClassTag, T1: ClassTag, R: TypeInfo : ClassTag]( + ctx: ExecuteContext, + name0: String, + typ0: PType, + name1: String, + typ1: PType, + body: IR, + print: Option[PrintWriter], + optimize: Boolean): (PType, (Int, Region) => AsmFunction5[Region, T0, Boolean, T1, Boolean, R]) = { + apply[AsmFunction5[Region, T0, Boolean, T1, Boolean, R], R](ctx, print, FastSeq((name0, typ0, classTag[T0]), (name1, typ1, classTag[T1])), body, optimize = optimize) + } + def apply[T0: ClassTag, T1: ClassTag, R: TypeInfo : ClassTag]( ctx: ExecuteContext, name0: String, diff --git a/hail/src/main/scala/is/hail/expr/ir/Copy.scala b/hail/src/main/scala/is/hail/expr/ir/Copy.scala index d48bb509cd8..c7eb6179398 100644 --- a/hail/src/main/scala/is/hail/expr/ir/Copy.scala +++ b/hail/src/main/scala/is/hail/expr/ir/Copy.scala @@ -65,9 +65,6 @@ object Copy { case ArrayLen(_) => assert(newChildren.length == 1) ArrayLen(newChildren(0).asInstanceOf[IR]) - case ArrayRange(_, _, _) => - assert(newChildren.length == 3) - ArrayRange(newChildren(0).asInstanceOf[IR], newChildren(1).asInstanceOf[IR], newChildren(2).asInstanceOf[IR]) case StreamRange(_, _, _) => assert(newChildren.length == 3) StreamRange(newChildren(0).asInstanceOf[IR], newChildren(1).asInstanceOf[IR], newChildren(2).asInstanceOf[IR]) @@ -297,12 +294,21 @@ object Copy { BlockMatrixWrite(newChildren(0).asInstanceOf[BlockMatrixIR], writer) case BlockMatrixMultiWrite(_, writer) => BlockMatrixMultiWrite(newChildren.map(_.asInstanceOf[BlockMatrixIR]), writer) + case UnpersistBlockMatrix(child) => + assert(newChildren.length == 1) + UnpersistBlockMatrix(newChildren(0).asInstanceOf[BlockMatrixIR]) case CollectDistributedArray(_, _, cname, gname, _) => assert(newChildren.length == 3) CollectDistributedArray(newChildren(0).asInstanceOf[IR], newChildren(1).asInstanceOf[IR], cname, gname, newChildren(2).asInstanceOf[IR]) case ReadPartition(path, spec, rowType) => assert(newChildren.length == 1) ReadPartition(newChildren(0).asInstanceOf[IR], spec, rowType) + case ReadValue(path, spec, requestedType) => + assert(newChildren.length == 1) + ReadValue(newChildren(0).asInstanceOf[IR], spec, requestedType) + case WriteValue(value, pathPrefix, spec) => + assert(newChildren.length == 2) + WriteValue(newChildren(0).asInstanceOf[IR], newChildren(1).asInstanceOf[IR], spec) case LiftMeOut(_) => LiftMeOut(newChildren(0).asInstanceOf[IR]) } diff --git a/hail/src/main/scala/is/hail/expr/ir/Emit.scala b/hail/src/main/scala/is/hail/expr/ir/Emit.scala index 4986447c050..e20567f0953 100644 --- a/hail/src/main/scala/is/hail/expr/ir/Emit.scala +++ b/hail/src/main/scala/is/hail/expr/ir/Emit.scala @@ -4,6 +4,7 @@ import java.io.{ByteArrayInputStream, ByteArrayOutputStream} import is.hail.HailContext import is.hail.annotations._ +import is.hail.asm4s.joinpoint.{Ctrl, JoinPoint, ParameterPack, TypedTriplet} import is.hail.asm4s.{Code, _} import is.hail.expr.ir.functions.{MathFunctions, StringFunctions} import is.hail.expr.types.physical._ @@ -302,6 +303,7 @@ private class Emit( emit(ir, env, er, container, None) private def emit(ir: IR, env: E, er: EmitRegion, container: Option[AggContainer], loopEnv: Option[Env[Array[LoopRef]]]): EmitTriplet = { + import CodeStream.Stream def emit(ir: IR, env: E = env, er: EmitRegion = er, container: Option[AggContainer] = container, loopEnv: Option[Env[Array[LoopRef]]] = loopEnv): EmitTriplet = this.emit(ir, env, er, container, loopEnv) @@ -312,6 +314,9 @@ private class Emit( def emitArrayIterator(ir: IR, env: E = env, container: Option[AggContainer] = container) = this.emitArrayIterator(ir, env, er, container) + def emitStream2(ir: IR, env: E = env, container: Option[AggContainer] = container): COption[Stream[EmitTriplet]] = + EmitStream2(this, ir, env, er, container) + def emitDeforestedNDArray(ir: IR) = deforestNDArray(resultRegion, ir, env).emit(coerce[PNDArray](ir.pType)) @@ -548,13 +553,15 @@ private class Emit( val vab = new StagedArrayBuilder(atyp.elementType, mb, 16) val sorter = new ArraySorter(er, vab) - val (array, compare, distinct) = (x: @unchecked) match { - case ArraySort(a, l, r, comp) => (a, Subst(comp, BindingEnv(Env[IR](l -> In(0, eltType), r -> In(1, eltType)))), Code._empty[Unit]) + val (array, compare, distinct, leftRightComparatorNames: Array[String]) = (x: @unchecked) match { + case ArraySort(a, l, r, comp) => (a, comp, Code._empty[Unit], Array(l, r)) case ToSet(a) => val discardNext = mb.fb.newMethod(Array[TypeInfo[_]](typeInfo[Region], sorter.ti, typeInfo[Boolean], sorter.ti, typeInfo[Boolean]), typeInfo[Boolean]) val EmitTriplet(s, m, v) = new Emit(ctx, discardNext).emit(ApplyComparisonOp(EQWithNA(eltVType), In(0, eltType), In(1, eltType)), Env.empty, er, container) discardNext.emit(Code(s, m || coerce[Boolean](v))) - (a, ApplyComparisonOp(Compare(eltVType), In(0, eltType), In(1, eltType)) < 0, sorter.distinctFromSorted(discardNext.invoke(_, _, _, _, _))) + val compare = ApplyComparisonOp(Compare(eltVType), In(0, eltType), In(1, eltType)) < 0 + InferPType(compare, Env.empty) + (a, compare, sorter.distinctFromSorted(discardNext.invoke(_, _, _, _, _)), Array.empty[String]) case ToDict(a) => val elementType = a.pType.asInstanceOf[PStreamable].elementType val (k0, k1, keyType) = elementType match { @@ -564,15 +571,17 @@ private class Emit( val discardNext = mb.fb.newMethod(Array[TypeInfo[_]](typeInfo[Region], sorter.ti, typeInfo[Boolean], sorter.ti, typeInfo[Boolean]), typeInfo[Boolean]) val EmitTriplet(s, m, v) = new Emit(ctx, discardNext).emit(ApplyComparisonOp(EQWithNA(keyType.virtualType), k0, k1), Env.empty, er, container) discardNext.emit(Code(s, m || coerce[Boolean](v))) - (a, ApplyComparisonOp(Compare(keyType.virtualType), k0, k1) < 0, Code(sorter.pruneMissing, sorter.distinctFromSorted(discardNext.invoke(_, _, _, _, _)))) + val compare = ApplyComparisonOp(Compare(keyType.virtualType), k0, k1) < 0 + InferPType(compare, Env.empty) + (a, compare, Code(sorter.pruneMissing, sorter.distinctFromSorted(discardNext.invoke(_, _, _, _, _))), Array.empty[String]) } val compF = vab.ti match { - case BooleanInfo => sorter.sort(makeDependentSortingFunction[Boolean](compare, env)) - case IntInfo => sorter.sort(makeDependentSortingFunction[Int](compare, env)) - case LongInfo => sorter.sort(makeDependentSortingFunction[Long](compare, env)) - case FloatInfo => sorter.sort(makeDependentSortingFunction[Float](compare, env)) - case DoubleInfo => sorter.sort(makeDependentSortingFunction[Double](compare, env)) + case BooleanInfo => sorter.sort(makeDependentSortingFunction[Boolean](compare, env, leftRightComparatorNames)) + case IntInfo => sorter.sort(makeDependentSortingFunction[Int](compare, env, leftRightComparatorNames)) + case LongInfo => sorter.sort(makeDependentSortingFunction[Long](compare, env, leftRightComparatorNames)) + case FloatInfo => sorter.sort(makeDependentSortingFunction[Float](compare, env, leftRightComparatorNames)) + case DoubleInfo => sorter.sort(makeDependentSortingFunction[Double](compare, env, leftRightComparatorNames)) } val aout = emitArrayIterator(array) @@ -633,12 +642,14 @@ private class Emit( } val compare = ApplyComparisonOp(Compare(etyp.types(0).virtualType), k1, k2) < 0 + InferPType(compare, Env.empty) + val leftRightComparatorNames = Array.empty[String] val compF = eab.ti match { - case BooleanInfo => sorter.sort(makeDependentSortingFunction[Boolean](compare, env)) - case IntInfo => sorter.sort(makeDependentSortingFunction[Int](compare, env)) - case LongInfo => sorter.sort(makeDependentSortingFunction[Long](compare, env)) - case FloatInfo => sorter.sort(makeDependentSortingFunction[Float](compare, env)) - case DoubleInfo => sorter.sort(makeDependentSortingFunction[Double](compare, env)) + case BooleanInfo => sorter.sort(makeDependentSortingFunction[Boolean](compare, env, leftRightComparatorNames)) + case IntInfo => sorter.sort(makeDependentSortingFunction[Int](compare, env, leftRightComparatorNames)) + case LongInfo => sorter.sort(makeDependentSortingFunction[Long](compare, env, leftRightComparatorNames)) + case FloatInfo => sorter.sort(makeDependentSortingFunction[Float](compare, env, leftRightComparatorNames)) + case DoubleInfo => sorter.sort(makeDependentSortingFunction[Double](compare, env, leftRightComparatorNames)) } val nab = new StagedArrayBuilder(PInt32(), mb, 16) @@ -661,10 +672,10 @@ private class Emit( GetTupleElement(Ref("i-1", tt), tt.fields(0).index) -> GetTupleElement(Ref("i", tt), tt.fields(0).index) } + val compare2 = ApplyComparisonOp(EQWithNA(ktyp.virtualType), lastKey, currKey) + InferPType(compare2, Env("i-1" -> etyp, "i" -> etyp)) val isSame = emit( - ApplyComparisonOp(EQWithNA(ktyp.virtualType), - lastKey, - currKey), + compare2, Env( ("i-1", (typeInfo[Long], eab.isMissing(i-1), eab.apply(i-1))), ("i", (typeInfo[Long], eab.isMissing(i), eab.apply(i))))) @@ -713,53 +724,39 @@ private class Emit( srvb.offset )))) - case _: ArrayMap | _: ArrayZip | _: ArrayFilter | _: ArrayRange | _: ArrayFlatMap | _: ArrayScan | _: ArrayLeftJoinDistinct | _: RunAggScan | _: ReadPartition | _: MakeStream | _: StreamRange => + case _: ArrayMap | _: ArrayZip | _: ArrayFilter | _: ArrayFlatMap | _: ArrayScan | _: ArrayLeftJoinDistinct | _: RunAggScan | _: ReadPartition | _: MakeStream | _: StreamRange => emitArrayIterator(ir).toEmitTriplet(mb, PArray(coerce[PStreamable](ir.pType).elementType)) - case ArrayFold(a, zero, name1, name2, body) => - val typ = ir.typ - val tarray = coerce[TStreamable](a.typ) - val tti = typeToTypeInfo(typ) - val eti = typeToTypeInfo(tarray.elementType) - val xmv = mb.newField[Boolean](name2 + "_missing") - val xvv = coerce[Any](mb.newField(name2)(eti)) - val xmbody = mb.newField[Boolean](name1 + "_missing_tmp") - val xmaccum = mb.newField[Boolean](name1 + "_missing") - val xvaccum = coerce[Any](mb.newField(name1)(tti)) - val bodyenv = env.bind( - (name1, (tti, xmaccum.load(), xvaccum.load())), - (name2, (eti, xmv.load(), xvv.load()))) - - val codeZ = emit(zero) - val codeB = emit(body, env = bodyenv) - val aBase = emitArrayIterator(a) + case ArrayFold(a, zero, accumName, valueName, body) => + val eltType = a.pType.asInstanceOf[PStreamable].elementType + val accType = ir.pType + implicit val eltPack = TypedTriplet.pack(eltType) + implicit val accPack = TypedTriplet.pack(accType) + val accTI = typeToTypeInfo(accType) + val eltTI = typeToTypeInfo(eltType) + + val streamOpt = emitStream2(a) + val resOpt: COption[Code[_]] = streamOpt.flatMapCPS { (stream, _ctx, ret) => + implicit val c = _ctx + def foldBody(elt: TypedTriplet[eltType.type], acc: TypedTriplet[accType.type]): TypedTriplet[accType.type] = { + val xElt = eltPack.newFields(mb.fb, valueName) + val xAcc = accPack.newFields(mb.fb, accumName) + val bodyenv = env.bind( + (accumName, (accTI, xAcc.load.m, xAcc.load.v)), + (valueName, (eltTI, xElt.load.m, xElt.load.v))) + + val codeB = emit(body, env = bodyenv) + TypedTriplet(accType, EmitTriplet(Code(xElt := elt, xAcc := acc, codeB.setup), codeB.m, codeB.v)) + } + val codeZ = emit(zero) + def retTT(acc: TypedTriplet[accType.type]): Code[Ctrl] = + ret(COption.fromEmitTriplet(acc.untyped)) - val cont = { (m: Code[Boolean], v: Code[_]) => - Code( - xmv := m, - xvv := xmv.mux(defaultValue(tarray.elementType), v), - codeB.setup, - xmbody := codeB.m, - xvaccum := xmbody.mux(defaultValue(typ), codeB.v), - xmaccum := xmbody) + stream.map(TypedTriplet(eltType, _)) + .fold(TypedTriplet(accType, codeZ), foldBody, retTT) } - val processAElts = aBase.arrayEmitter(cont) - val marray = processAElts.m.getOrElse(const(false)) - - EmitTriplet(Code( - codeZ.setup, - xmaccum := codeZ.m, - xvaccum := xmaccum.mux(defaultValue(typ), codeZ.v), - processAElts.setup, - marray.mux( - Code( - xmaccum := true, - xvaccum := defaultValue(typ)), - Code( - aBase.calcLength, - processAElts.addElements))), - xmaccum, xvaccum) + COption.toEmitTriplet(resOpt, accTI, mb) case ArrayFold2(a, acc, valueName, seq, res) => val typ = ir.typ @@ -822,29 +819,24 @@ private class Emit( xresm, xresv) case ArrayFor(a, valueName, body) => - val tarray = coerce[TStreamable](a.typ) - val eti = typeToTypeInfo(tarray.elementType) - val xmv = mb.newField[Boolean]() - val xvv = coerce[Any](mb.newField(valueName)(eti)) - val bodyenv = env.bind( - (valueName, (eti, xmv.load(), xvv.load()))) - val codeB = emit(body, env = bodyenv) - val aBase = emitArrayIterator(a) - val cont = { (m: Code[Boolean], v: Code[_]) => - Code( - xmv := m, - xvv := xmv.mux(defaultValue(tarray.elementType), v), - codeB.setup) + val eltType = a.pType.asInstanceOf[PStream].elementType + implicit val eltPack = TypedTriplet.pack(eltType) + val eltTI = typeToTypeInfo(eltType) + + val streamOpt = emitStream2(a) + def forBody(elt: TypedTriplet[eltType.type]): Code[Unit] = { + val xElt = eltPack.newFields(mb.fb, valueName) + val bodyenv = env.bind( + (valueName, (eltTI, xElt.load.m, xElt.load.v))) + val codeB = emit(body, env = bodyenv) + Code(xElt := elt, codeB.setup) } - val processAElts = aBase.arrayEmitter(cont) - val ma = processAElts.m.getOrElse(const(false)) EmitTriplet( - Code( - processAElts.setup, - ma.mux( - Code._empty, - Code(aBase.calcLength, processAElts.addElements))), + streamOpt.cases[Unit](mb)( + Code._empty, + stream => + stream.map(TypedTriplet(eltType, _)).forEach(mb)(forBody)), const(false), Code._empty) @@ -1894,13 +1886,24 @@ private class Emit( } private def makeDependentSortingFunction[T: TypeInfo]( - ir: IR, env: Emit.E): DependentEmitFunction[AsmFunction2[T, T, Boolean]] = { + ir: IR, env: Emit.E, leftRightComparatorNames: Array[String]): DependentEmitFunction[AsmFunction2[T, T, Boolean]] = { val (newIR, getEnv) = capturedReferences(ir) val f = mb.fb.newDependentFunction[T, T, Boolean] val fregion = f.addField[Region](region) - val newEnv = getEnv(env, f) + var newEnv = getEnv(env, f) val sort = f.newMethod[Region, T, Boolean, T, Boolean, Boolean] + + if(leftRightComparatorNames.nonEmpty) { + assert(leftRightComparatorNames.length == 2) + newEnv = newEnv.bindIterable( + IndexedSeq( + (leftRightComparatorNames(0), (implicitly[TypeInfo[T]], sort.getArg[Boolean](3), sort.getArg[T](2))), + (leftRightComparatorNames(1), (implicitly[TypeInfo[T]], sort.getArg[Boolean](5), sort.getArg[T](4))) + ) + ) + } + val EmitTriplet(setup, m, v) = new Emit(ctx, sort).emit(newIR, newEnv, EmitRegion.default(sort), None) sort.emit(Code(setup, m.mux(Code._fatal("Result of sorting function cannot be missing."), v))) diff --git a/hail/src/main/scala/is/hail/expr/ir/EmitFunctionBuilder.scala b/hail/src/main/scala/is/hail/expr/ir/EmitFunctionBuilder.scala index e3310e021a3..bd88d650ac6 100644 --- a/hail/src/main/scala/is/hail/expr/ir/EmitFunctionBuilder.scala +++ b/hail/src/main/scala/is/hail/expr/ir/EmitFunctionBuilder.scala @@ -191,8 +191,6 @@ class EmitFunctionBuilder[F >: Null]( private[this] val compareMap: mutable.Map[CompareMapKey, CodeOrdering.F[_]] = mutable.Map[CompareMapKey, CodeOrdering.F[_]]() - private[this] val methodMemo: mutable.Map[Any, EmitMethodBuilder] = mutable.HashMap.empty - def numReferenceGenomes: Int = rgMap.size def getReferenceGenome(rg: ReferenceGenome): Code[ReferenceGenome] = @@ -508,18 +506,6 @@ class EmitFunctionBuilder[F >: Null]( }.toArray: _*)) } - def getOrDefineMethod(suffix: String, key: Any, argsInfo: Array[TypeInfo[_]], returnInfo: TypeInfo[_]) - (f: EmitMethodBuilder => Unit): EmitMethodBuilder = { - methodMemo.get(key) match { - case Some(mb) => mb - case None => - val mb = newMethod(suffix, argsInfo, returnInfo) - f(mb) - methodMemo(key) = mb - mb - } - } - override def newMethod(suffix: String, argsInfo: Array[TypeInfo[_]], returnInfo: TypeInfo[_]): EmitMethodBuilder = { val mb = new EmitMethodBuilder(this, s"m${ methods.size }_${suffix}", argsInfo, returnInfo) methods.append(mb) diff --git a/hail/src/main/scala/is/hail/expr/ir/EmitStream.scala b/hail/src/main/scala/is/hail/expr/ir/EmitStream.scala index ac844716acc..c95f6764cf4 100644 --- a/hail/src/main/scala/is/hail/expr/ir/EmitStream.scala +++ b/hail/src/main/scala/is/hail/expr/ir/EmitStream.scala @@ -8,9 +8,396 @@ import is.hail.expr.types.physical._ import is.hail.io.{AbstractTypedCodecSpec, InputBuffer} import is.hail.utils._ -import scala.language.existentials +import scala.language.{existentials, higherKinds} import scala.reflect.ClassTag +case class EmitStreamContext(mb: MethodBuilder, jb: JoinPointBuilder) + +abstract class COption[+A] { self => + def apply(none: Code[Ctrl], some: A => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] + + def cases[B: TypeInfo](mb: MethodBuilder)(none: Code[B], some: A => Code[B]): Code[B] = + JoinPoint.CallCC[Code[B]]((jb, ret) => apply(ret(none), a => ret(some(a)))(EmitStreamContext(mb, jb))) + + def map[B](f: A => B): COption[B] = new COption[B] { + def apply(none: Code[Ctrl], some: B => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = + self.apply(none, a => some(f(a))) + } + + def mapCPS[B](f: (A, B => Code[Ctrl]) => Code[Ctrl]): COption[B] = new COption[B] { + def apply(none: Code[Ctrl], some: B => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = + self.apply(none, a => f(a, some)) + } + + def flatMap[B](f: A => COption[B]): COption[B] = new COption[B] { + def apply(none: Code[Ctrl], some: B => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = { + val noneJP = ctx.jb.joinPoint() + noneJP.define(_ => none) + self.apply(noneJP(()), f(_)(noneJP(()), some)) + } + } + + def flatMapCPS[B](f: (A, EmitStreamContext, COption[B] => Code[Ctrl]) => Code[Ctrl]): COption[B] = new COption[B] { + def apply(none: Code[Ctrl], some: B => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = { + val noneJP = ctx.jb.joinPoint() + noneJP.define(_ => none) + self.apply(noneJP(()), a => f(a, ctx, optB => optB(noneJP(()), some))) + } + } +} + +object COption { + def apply[A](missing: Code[Boolean], value: A): COption[A] = new COption[A] { + def apply(none: Code[Ctrl], some: A => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = missing.mux(none, some(value)) + } + def fromEmitTriplet[A](et: EmitTriplet): COption[Code[A]] = new COption[Code[A]] { + def apply(none: Code[Ctrl], some: Code[A] => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = { + Code(et.setup, et.m.mux(none, some(coerce[A](et.v)))) + } + } + def fromTypedTriplet(et: EmitTriplet, ti: TypeInfo[_]): COption[Code[_]] = fromEmitTriplet(et) + def toEmitTriplet(opt: COption[Code[_]], ti: TypeInfo[_], mb: MethodBuilder): EmitTriplet = { + val m = mb.newLocal[Boolean] + val v = mb.newLocal(ti) + val setup = JoinPoint.CallCC[Unit] { (jb, ret) => + opt(Code(m := true, v.storeAny(defaultValue(ti)), ret(())), + a => Code(m := false, v.storeAny(a), ret(())))(EmitStreamContext(mb, jb)) + } + EmitTriplet(setup, m, v.load()) + } + def toTypedTriplet[A](t: PType, mb: MethodBuilder, opt: COption[Code[A]]): TypedTriplet[t.type] = + TypedTriplet(t, toEmitTriplet(opt, typeToTypeInfo(t), mb)) +} + +object CodeStream { self => + import is.hail.asm4s.joinpoint.JoinPoint.CallCC + import is.hail.asm4s.joinpoint._ + def newLocal[T: ParameterPack](implicit ctx: EmitStreamContext): ParameterStore[T] = implicitly[ParameterPack[T]].newLocals(ctx.mb) + def joinPoint()(implicit ctx: EmitStreamContext): DefinableJoinPoint[Unit] = ctx.jb.joinPoint() + def joinPoint[T: ParameterPack](implicit ctx: EmitStreamContext): DefinableJoinPoint[T] = ctx.jb.joinPoint[T](ctx.mb) + + private case class Source[+A](setup0: Code[Unit], close0: Code[Unit], setup: Code[Unit], close: Code[Unit], pull: Code[Ctrl]) + + abstract class Stream[+A] { + private[CodeStream] def apply(eos: Code[Ctrl], push: A => Code[Ctrl])(implicit ctx: EmitStreamContext): Source[A] + + def fold[S: ParameterPack](s0: S, f: (A, S) => S, ret: S => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = + CodeStream.fold(this, s0, f, ret) + def forEach(f: A => Code[Unit], ret: Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = + CodeStream.forEach(this, f, ret) + def forEach(mb: MethodBuilder)(f: A => Code[Unit]): Code[Unit] = + CallCC[Unit]((jb, ret) => CodeStream.forEach(this, f, ret(()))(EmitStreamContext(mb, jb))) + def mapCPS[B]( + f: (EmitStreamContext, A, B => Code[Ctrl]) => Code[Ctrl], + setup0: Option[Code[Unit]] = None, + setup: Option[Code[Unit]] = None, + close0: Option[Code[Unit]] = None, + close: Option[Code[Unit]] = None + ): Stream[B] = CodeStream.mapCPS(this)(f, setup0, setup, close0, close) + def map[B]( + f: A => B, + setup0: Option[Code[Unit]] = None, + setup: Option[Code[Unit]] = None, + close0: Option[Code[Unit]] = None, + close: Option[Code[Unit]] = None + ): Stream[B] = CodeStream.map(this)(f, setup0, setup, close0, close) + def flatMap[B](f: A => Stream[B]): Stream[B] = + CodeStream.flatMap(map(f)) + } + + implicit class StreamPP[A](val stream: Stream[A]) extends AnyVal { + def filter(cond: A => Code[Boolean])(implicit pp: ParameterPack[A]): Stream[A] = + CodeStream.filter(stream, cond) + } + + def unfold[A, S: ParameterPack]( + s0: S, + f: (S, EmitStreamContext, COption[(A, S)] => Code[Ctrl]) => Code[Ctrl] + ): Stream[A] = new Stream[A] { + def apply(eos: Code[Ctrl], push: A => Code[Ctrl])(implicit ctx: EmitStreamContext): Source[A] = { + val s = newLocal[S] + Source[A]( + setup0 = s.init, + close0 = Code._empty, + setup = s := s0, + close = Code._empty, + pull = f(s.load, ctx, _.apply( + none = eos, + // Warning: `a` should not depend on `s` + some = { case (a, s1) => Code(s := s1, push(a)) }))) + } + } + + def range(start: Code[Int], step: Code[Int], len: Code[Int]): Stream[Code[Int]] = + unfold[Code[Int], (Code[Int], Code[Int])]( + s0 = (start, len), + f = { case ((cur, rem), _ctx, k) => + implicit val ctx = _ctx + val xCur = newLocal[Code[Int]] + val xRem = newLocal[Code[Int]] + Code( + xCur := cur, + xRem := rem - 1, + k(COption(xRem.load < 0, + (xCur.load, (xCur.load + step, xRem.load))))) + }) + + def fold[A, S: ParameterPack](stream: Stream[A], s0: S, f: (A, S) => S, ret: S => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = { + val s = newLocal[S] + val pullJP = joinPoint() + val eosJP = joinPoint() + val source = stream( + eos = eosJP(()), + push = a => Code(s := f(a, s.load), pullJP(()))) + eosJP.define(_ => Code(source.close0, ret(s.load))) + pullJP.define(_ => source.pull) + Code(s := s0, source.setup0, source.setup, pullJP(())) + } + + def forEach[A](stream: Stream[A], f: A => Code[Unit], ret: Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = { + val pullJP = joinPoint() + val eosJP = joinPoint() + val source = stream( + eos = eosJP(()), + push = a => Code(f(a), pullJP(()))) + eosJP.define(_ => Code(source.close0, ret)) + pullJP.define(_ => source.pull) + Code(source.setup0, source.setup, pullJP(())) + } + + def mapCPS[A, B](stream: Stream[A])( + f: (EmitStreamContext, A, B => Code[Ctrl]) => Code[Ctrl], + setup0: Option[Code[Unit]] = None, + setup: Option[Code[Unit]] = None, + close0: Option[Code[Unit]] = None, + close: Option[Code[Unit]] = None + ): Stream[B] = new Stream[B] { + def apply(eos: Code[Ctrl], push: B => Code[Ctrl])(implicit ctx: EmitStreamContext): Source[B] = { + val source = stream( + eos = close.map(Code(_, eos)).getOrElse(eos), + push = f(ctx, _, b => push(b))) + Source[B]( + setup0 = setup0.map(Code(_, source.setup0)).getOrElse(source.setup0), + close0 = close0.map(Code(_, source.close0)).getOrElse(source.close0), + setup = setup.map(Code(_, source.setup)).getOrElse(source.setup), + close = close.map(Code(_, source.close)).getOrElse(source.close), + pull = source.pull) + } + } + + def map[A, B](stream: Stream[A])( + f: A => B, + setup0: Option[Code[Unit]] = None, + setup: Option[Code[Unit]] = None, + close0: Option[Code[Unit]] = None, + close: Option[Code[Unit]] = None + ): Stream[B] = mapCPS(stream)((_, a, k) => k(f(a)), setup0, setup, close0, close) + + def flatMap[A](outer: Stream[Stream[A]]): Stream[A] = new Stream[A] { + def apply(eos: Code[Ctrl], push: A => Code[Ctrl])(implicit ctx: EmitStreamContext): Source[A] = { + val outerPullJP = joinPoint() + var innerSource: Source[A] = null + val innerPullJP = joinPoint() + val hasBeenPulled = newLocal[Code[Boolean]] + val outerSource = outer( + eos = eos, + push = inner => { + innerSource = inner( + eos = outerPullJP(()), + push = push) + innerPullJP.define(_ => innerSource.pull) + Code(innerSource.setup, innerPullJP(())) + }) + outerPullJP.define(_ => outerSource.pull) + Source[A]( + setup0 = Code(hasBeenPulled := false, outerSource.setup0, innerSource.setup0), + close0 = Code(innerSource.close0, outerSource.close0), + setup = Code(hasBeenPulled := false, outerSource.setup), + close = Code(hasBeenPulled.load.mux(innerSource.close, Code._empty), + outerSource.close), + pull = hasBeenPulled.load.mux(innerPullJP(()), + Code(hasBeenPulled := true, outerPullJP(())))) + } + } + + def filter[A](stream: Stream[COption[A]]): Stream[A] = new Stream[A] { + def apply(eos: Code[Ctrl], push: A => Code[Ctrl])(implicit ctx: EmitStreamContext): Source[A] = { + val pullJP = joinPoint() + val source = stream( + eos = eos, + push = _.apply(none = pullJP(()), some = push)) + pullJP.define(_ => source.pull) + source.copy(pull = pullJP(())) + } + } + + def filter[A: ParameterPack](stream: Stream[A], cond: A => Code[Boolean]): Stream[A] = + filter(mapCPS[A, COption[A]](stream)((_ctx, a, k) => { + implicit val ctx = _ctx + val as = newLocal[A] + Code(as := a, k(COption(!cond(as.load), as.load))) + })) + + def zip[A, B](left: Stream[A], right: Stream[B]): Stream[(A, B)] = new Stream[(A, B)] { + def apply(eos: Code[Ctrl], push: ((A, B)) => Code[Ctrl])(implicit ctx: EmitStreamContext): Source[(A, B)] = { + val eosJP = joinPoint() + val leftEOSJP = joinPoint() + val rightEOSJP = joinPoint() + var rightSource: Source[B] = null + val leftSource = left( + eos = leftEOSJP(()), + push = a => { + rightSource = right( + eos = rightEOSJP(()), + push = b => push((a, b))) + rightSource.pull + }) + leftEOSJP.define(_ => Code(rightSource.close, eosJP(()))) + rightEOSJP.define(_ => Code(leftSource.close, eosJP(()))) + eosJP.define(_ => eos) + + Source[(A, B)]( + setup0 = Code(leftSource.setup0, rightSource.setup0), + close0 = Code(leftSource.close0, rightSource.close0), + setup = Code(leftSource.setup, rightSource.setup), + close = Code(leftSource.close, rightSource.close), + pull = leftSource.pull) + } + } + + def multiZip(streams: IndexedSeq[Stream[Code[_]]]): Stream[IndexedSeq[Code[_]]] = new Stream[IndexedSeq[Code[_]]] { + def apply(eos: Code[Ctrl], push: IndexedSeq[Code[_]] => Code[Ctrl])(implicit ctx: EmitStreamContext): Source[IndexedSeq[Code[_]]] = { + val closeJPs = streams.map(_ => joinPoint()) + val eosJP = closeJPs(0) + val terminatingStream = newLocal[Code[Int]] + + def nthSource(n: Int, acc: IndexedSeq[Code[_]]): (Source[Code[_]], IndexedSeq[Code[Unit]]) = { + if (n == streams.length - 1) { + val src = streams(n)( + Code(terminatingStream := n, eosJP(())), + c => push(acc :+ c)) + (src, IndexedSeq(src.close)) + } else { + var rest: Source[Code[_]] = null + var closes: IndexedSeq[Code[Unit]] = null + val src = streams(n)( + Code(terminatingStream := n, eosJP(())), + c => { + val t = nthSource(n+1, acc :+ c) + rest = t._1 + closes = t._2 + rest.pull + }) + (Source[Code[_]]( + setup0 = Code(src.setup0, rest.setup0), + close0 = Code(rest.close0, src.close0), + setup = Code(src.setup, rest.setup), + close = Code(rest.close, src.close), + pull = src.pull), + src.close +: closes) + } + } + + val (source, closes) = nthSource(0, IndexedSeq.empty) + + closeJPs.zipWithIndex.foreach { case (jp, i) => + val next = if (i == streams.length - 1) eos else closeJPs(i + 1)(()) + jp.define(_ => terminatingStream.load.ceq(i).mux( + next, + Code(closes(i), next))) + } + + source.asInstanceOf[Source[IndexedSeq[Code[_]]]] + } + } + + def mux[A: ParameterPack](cond: Code[Boolean], left: Stream[A], right: Stream[A]): Stream[A] = new Stream[A] { + def apply(eos: Code[Ctrl], push: A => Code[Ctrl])(implicit ctx: EmitStreamContext): Source[A] = { + val b = newLocal[Code[Boolean]] + val eosJP = joinPoint() + val pushJP = joinPoint[A] + + eosJP.define(_ => eos) + pushJP.define(push) + + val l = left(eosJP(()), pushJP(_)) + val r = right(eosJP(()), pushJP(_)) + + val lPullJP = joinPoint() + val rPullJP = joinPoint() + + lPullJP.define(_ => l.pull) + rPullJP.define(_ => r.pull) + Source[A]( + setup0 = Code(b := cond, l.setup0, r.setup0), + close0 = Code(l.close0, r.close0), + setup = b.load.mux(l.setup, r.setup), + close = b.load.mux(l.close, r.close), + pull = JoinPoint.mux(b.load, lPullJP, rPullJP) + ) + } + } + + def fromParameterized[P, A]( + stream: EmitStream.Parameterized[P, A] + ): P => COption[Stream[A]] = p => new COption[Stream[A]] { + def apply(none: Code[Ctrl], some: Stream[A] => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = { + import EmitStream.{Missing, Start, EOS, Yield} + implicit val sP = stream.stateP + val s = newLocal[stream.S] + val sNew = newLocal[stream.S] + + def src(s0: stream.S): Stream[A] = new Stream[A] { + def apply(eos: Code[Ctrl], push: A => Code[Ctrl])(implicit ctx: EmitStreamContext): Source[A] = { + Source[A]( + setup0 = Code(s.init, sNew.init), + close0 = Code._empty, + setup = sNew := s0, + close = Code._empty, + pull = Code(s := sNew.load, stream.step(s.load) { + case EOS => eos + case Yield(elt, s1) => Code(sNew := s1, push(elt)) + })) + } + } + + stream.init(p) { + case Missing => none + case Start(s0) => some(src(s0)) + } + } + } +} + +object EmitStream2 { + import CodeStream._ + + private[ir] def apply( + emitter: Emit, + streamIR0: IR, + env0: Emit.E, + er: EmitRegion, + container: Option[AggContainer] + ): COption[Stream[EmitTriplet]] = { + val fb = emitter.mb.fb + + def emitIR(ir: IR, env: Emit.E): EmitTriplet = + emitter.emit(ir, env, er, container) + + def emitStream(streamIR: IR, env: Emit.E): COption[Stream[EmitTriplet]] = + streamIR match { + + case _ => + val EmitStream(parameterized, eltType) = + EmitStream.apply(emitter, streamIR, env, er, container) + fromParameterized(parameterized)(()) + } + + emitStream(streamIR0, env0) + } +} + + object EmitStream { sealed trait Init[+S] object Missing extends Init[Nothing] @@ -62,17 +449,17 @@ object EmitStream { .headOption } - def init(mb: MethodBuilder, jb: JoinPointBuilder, param: P)( + def init(param: P)( k: Init[S] => Code[Ctrl] - ): Code[Ctrl] = { - val missing = jb.joinPoint() + )(implicit ctx: EmitStreamContext): Code[Ctrl] = { + val missing = ctx.jb.joinPoint() missing.define { _ => k(Missing) } def loop(i: Int, ab: ArrayBuilder[Any]): Code[Ctrl] = { if (i == streams.length) k(Start(ab.result(): IndexedSeq[_])) else - streams(i).init(mb, jb, param) { + streams(i).init(param) { case Missing => missing(()) case Start(s) => ab += s @@ -83,14 +470,14 @@ object EmitStream { loop(0, new ArrayBuilder) } - def step(mb: MethodBuilder, jb: JoinPointBuilder, state: IndexedSeq[_])(k: Step[EmitTriplet, S] => Code[Ctrl]): Code[Ctrl] = { - val eos = jb.joinPoint() + def step(state: IndexedSeq[_])(k: Step[EmitTriplet, S] => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = { + val eos = ctx.jb.joinPoint() eos.define(_ => k(EOS)) behavior match { case ArrayZipBehavior.AssertSameLength => - val anyEOS = mb.newLocal[Boolean] - val allEOS = mb.newLocal[Boolean] - val labels = (0 to streams.size).map(_ => jb.joinPoint()) + val anyEOS = ctx.mb.newLocal[Boolean] + val allEOS = ctx.mb.newLocal[Boolean] + val labels = (0 to streams.size).map(_ => ctx.jb.joinPoint()) val ab = new ArrayBuilder[(EmitTriplet, Any)] labels.indices.foreach { i => @@ -105,7 +492,7 @@ object EmitStream { f(elts, { b => k(Yield(b, ss)) }))) } else { val streamI = streams(i) - labels(i).define(_ => streamI.step(mb, jb, state(i).asInstanceOf[streamI.S]) { + labels(i).define(_ => streamI.step(state(i).asInstanceOf[streamI.S]) { case EOS => Code(anyEOS := true, labels(i + 1)(())) case Yield(elt, s) => @@ -124,7 +511,7 @@ object EmitStream { f(elts, { b => k(Yield(b, ss)) }) } else { val streamI = streams(i) - streamI.step(mb, jb, state(i).asInstanceOf[streamI.S]) { + streamI.step(state(i).asInstanceOf[streamI.S]) { case Yield(elt, s) => ab += ((elt, s)) loop(i + 1, ab) @@ -136,9 +523,9 @@ object EmitStream { loop(0, new ArrayBuilder) case ArrayZipBehavior.ExtendNA => - val allEOS = mb.newLocal[Boolean] - val missing = streams.map(_ => mb.newLocal[Boolean]) - val labels = (0 to streams.size).map(_ => jb.joinPoint()) + val allEOS = ctx.mb.newLocal[Boolean] + val missing = streams.map(_ => ctx.mb.newLocal[Boolean]) + val labels = (0 to streams.size).map(_ => ctx.jb.joinPoint()) val ab = new ArrayBuilder[(Code[_], Any)] labels.indices.foreach { i => @@ -151,7 +538,7 @@ object EmitStream { f(elts, { b => k(Yield(b, ss)) }))) } else { val streamI = streams(i) - labels(i).define(_ => streamI.step(mb, jb, state(i).asInstanceOf[streamI.S]) { + labels(i).define(_ => streamI.step(state(i).asInstanceOf[streamI.S]) { case EOS => Code(missing(i) := true, labels(i + 1)(())) case Yield(elt, s) => @@ -177,17 +564,12 @@ object EmitStream { def length(s0: S): Option[Code[Int]] - def init( - mb: MethodBuilder, - jb: JoinPointBuilder, - param: P - )(k: Init[S] => Code[Ctrl]): Code[Ctrl] + def init(param: P)( + k: Init[S] => Code[Ctrl] + )(implicit ctx: EmitStreamContext + ): Code[Ctrl] - def step( - mb: MethodBuilder, - jb: JoinPointBuilder, - state: S - )(k: Step[A, S] => Code[Ctrl]): Code[Ctrl] + def step(state: S)(k: Step[A, S] => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] def addSetup[Q <: P](setup: Q => Code[Unit], cleanup: Code[Unit] = Code._empty): Parameterized[Q, A] = guardParam({ (param, k) => Code(setup(param), k(Some(param))) }, cleanup) @@ -200,19 +582,19 @@ object EmitStream { val stateP: ParameterPack[S] = self.stateP def emptyState: S = self.emptyState def length(s0: S): Option[Code[Int]] = self.length(s0) - def init(mb: MethodBuilder, jb: JoinPointBuilder, param: Q)(k: Init[S] => Code[Ctrl]): Code[Ctrl] = { - val missing = jb.joinPoint() + def init(param: Q)(k: Init[S] => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = { + val missing = ctx.jb.joinPoint() missing.define { _ => k(Missing) } f(param, { - case Some(newParam) => self.init(mb, jb, newParam) { + case Some(newParam) => self.init(newParam) { case Missing => missing(()) case Start(s) => k(Start(s)) } case None => missing(()) }) } - def step(mb: MethodBuilder, jb: JoinPointBuilder, state: S)(k: Step[A, S] => Code[Ctrl]): Code[Ctrl] = - self.step(mb, jb, state) { + def step(state: S)(k: Step[A, S] => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = + self.step(state) { case EOS => Code(cleanup, k(EOS)) case v => k(v) } @@ -228,13 +610,13 @@ object EmitStream { val stateP: ParameterPack[S] = self.stateP def emptyState: S = self.emptyState def length(s0: S): Option[Code[Int]] = self.length(s0) - def init(mb: MethodBuilder, jb: JoinPointBuilder, param: P)(k: Init[S] => Code[Ctrl]): Code[Ctrl] = - self.init(mb, jb, param) { + def init(param: P)(k: Init[S] => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = + self.init(param) { case Missing => k(Missing) case Start(s) => k(Start(s)) } - def step(mb: MethodBuilder, jb: JoinPointBuilder, state: S)(k: Step[B, S] => Code[Ctrl]): Code[Ctrl] = - self.step(mb, jb, state) { + def step(state: S)(k: Step[B, S] => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = + self.step(state) { case EOS => k(EOS) case Yield(a, s) => f(a, b => k(Yield(b, s))) } @@ -245,11 +627,11 @@ object EmitStream { implicit val stateP: ParameterPack[S] = self.stateP def emptyState: S = self.emptyState def length(s0: S): Option[Code[Int]] = None - def init(mb: MethodBuilder, jb: JoinPointBuilder, param: P)(k: Init[S] => Code[Ctrl]): Code[Ctrl] = - self.init(mb, jb, param)(k) - def step(mb: MethodBuilder, jb: JoinPointBuilder, s0: S)(k: Step[B, S] => Code[Ctrl]): Code[Ctrl] = { - val pull = jb.joinPoint[S](mb) - pull.define(self.step(mb, jb, _) { + def init(param: P)(k: Init[S] => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = + self.init(param)(k) + def step(s0: S)(k: Step[B, S] => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = { + val pull = ctx.jb.joinPoint[S](ctx.mb) + pull.define(self.step(_) { case EOS => k(EOS) case Yield(a, s) => f(a, { case None => pull(s) @@ -270,19 +652,19 @@ object EmitStream { def emptyState: S = (self.emptyState, dummy, false) def length(s0: S): Option[Code[Int]] = self.length(s0._1).map(_ + 1) - def init(mb: MethodBuilder, jb: JoinPointBuilder, param: P)(k: Init[S] => Code[Ctrl]): Code[Ctrl] = - self.init(mb, jb, param) { + def init(param: P)(k: Init[S] => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = + self.init(param) { case Missing => k(Missing) case Start(s0) => k(Start((s0, zero, true))) } - def step(mb: MethodBuilder, jb: JoinPointBuilder, state: S)(k: Step[B, S] => Code[Ctrl]): Code[Ctrl] = { - val yield_ = jb.joinPoint[(B, self.S)](mb) + def step(state: S)(k: Step[B, S] => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = { + val yield_ = ctx.jb.joinPoint[(B, self.S)](ctx.mb) yield_.define { case (b, s1) => k(Yield(b, (s1, b, false))) } val (s, b, isFirstStep) = state isFirstStep.mux( yield_((b, s)), - self.step(mb, jb, s) { + self.step(s) { case EOS => k(EOS) case Yield(a, s1) => op(a, b, b1 => yield_((b1, s1))) }) @@ -295,9 +677,9 @@ object EmitStream { val stateP: ParameterPack[S] = implicitly def emptyState: S = () def length(s0: S): Option[Code[Int]] = Some(0) - def init(mb: MethodBuilder, jb: JoinPointBuilder, param: Any)(k: Init[S] => Code[Ctrl]): Code[Ctrl] = + def init(param: Any)(k: Init[S] => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = k(Missing) - def step(mb: MethodBuilder, jb: JoinPointBuilder, s: S)(k: Step[Nothing, S] => Code[Ctrl]): Code[Ctrl] = + def step(s: S)(k: Step[Nothing, S] => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = k(EOS) } @@ -309,18 +691,10 @@ object EmitStream { def length(s0: S): Option[Code[Int]] = None - def init( - mb: MethodBuilder, - jb: JoinPointBuilder, - buf: Code[InputBuffer] - )(k: Init[S] => Code[Ctrl]): Code[Ctrl] = + def init(buf: Code[InputBuffer])(k: Init[S] => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = k(Start(buf)) - def step( - mb: MethodBuilder, - jb: JoinPointBuilder, - state: S - )(k: Step[Code[Long], S] => Code[Ctrl]): Code[Ctrl] = { + def step(state: S)(k: Step[Code[Long], S] => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = { stepIf(k, state.readByte().toZ, dec(state), state) } } @@ -336,10 +710,10 @@ object EmitStream { def emptyState: S = (0, 0) def length(s0: S): Option[Code[Int]] = Some(s0._1) - def init(mb: MethodBuilder, jb: JoinPointBuilder, len: Code[Int])(k: Init[S] => Code[Ctrl]): Code[Ctrl] = + def init(len: Code[Int])(k: Init[S] => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = k(Start((len, start))) - def step(mb: MethodBuilder, jb: JoinPointBuilder, state: S)(k: Step[Code[Int], S] => Code[Ctrl]): Code[Ctrl] = { + def step(state: S)(k: Step[Code[Int], S] => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = { val (nLeft, acc) = state stepIf(k, nLeft > 0, acc, (nLeft - 1, acc + incr)) } @@ -351,16 +725,16 @@ object EmitStream { def emptyState: S = elements.length def length(s0: S): Option[Code[Int]] = Some(const(elements.length) - s0) - def init(mb: MethodBuilder, jb: JoinPointBuilder, param: Any)(k: Init[S] => Code[Ctrl]): Code[Ctrl] = + def init(param: Any)(k: Init[S] => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = k(Start(0)) - def step(mb: MethodBuilder, jb: JoinPointBuilder, idx: S)(k: Step[A, S] => Code[Ctrl]): Code[Ctrl] = { - val eos = jb.joinPoint() - val yld = jb.joinPoint[A](mb) + def step(idx: S)(k: Step[A, S] => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = { + val eos = ctx.jb.joinPoint() + val yld = ctx.jb.joinPoint[A](ctx.mb) eos.define { _ => k(EOS) } yld.define { a => k(Yield(a, idx + 1)) } JoinPoint.switch(idx, eos, elements.map { elt => - val j = jb.joinPoint() + val j = ctx.jb.joinPoint() j.define { _ => yld(elt) } j }) @@ -378,24 +752,24 @@ object EmitStream { def emptyState: S = (outer.emptyState, inner.emptyState) def length(s0: S): Option[Code[Int]] = None - def init(mb: MethodBuilder, jb: JoinPointBuilder, param: A)(k: Init[S] => Code[Ctrl]): Code[Ctrl] = - outer.init(mb, jb, param) { + def init(param: A)(k: Init[S] => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = + outer.init(param) { case Missing => k(Missing) case Start(outS0) => k(Start((outS0, inner.emptyState))) } - def step(mb: MethodBuilder, jb: JoinPointBuilder, state: S)(k: Step[C, S] => Code[Ctrl]): Code[Ctrl] = { - val stepInner = jb.joinPoint[S](mb) - val stepOuter = jb.joinPoint[outer.S](mb) + def step(state: S)(k: Step[C, S] => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = { + val stepInner = ctx.jb.joinPoint[S](ctx.mb) + val stepOuter = ctx.jb.joinPoint[outer.S](ctx.mb) stepInner.define { case (outS, innS) => - inner.step(mb, jb, innS) { + inner.step(innS) { case EOS => stepOuter(outS) case Yield(innElt, innS1) => k(Yield(innElt, (outS, innS1))) } } - stepOuter.define(outer.step(mb, jb, _) { + stepOuter.define(outer.step(_) { case EOS => k(EOS) - case Yield(outElt, outS) => inner.init(mb, jb, outElt) { + case Yield(outElt, outS) => inner.init(outElt) { case Missing => stepOuter(outS) case Start(innS) => stepInner((outS, innS)) } @@ -417,39 +791,35 @@ object EmitStream { def emptyState: S = (left.emptyState, right.emptyState, (rNil, false)) def length(s0: S): Option[Code[Int]] = left.length(s0._1) - def init(mb: MethodBuilder, jb: JoinPointBuilder, param: P)( - k: Init[S] => Code[Ctrl] - ): Code[Ctrl] = { - val missing = jb.joinPoint() + def init(param: P)(k: Init[S] => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = { + val missing = ctx.jb.joinPoint() missing.define { _ => k(Missing) } - left.init(mb, jb, param) { + left.init(param) { case Missing => missing(()) - case Start(lS) => right.init(mb, jb, param) { + case Start(lS) => right.init(param) { case Missing => missing(()) case Start(rS) => k(Start((lS, rS, (rNil, false)))) } } } - def step(mb: MethodBuilder, jb: JoinPointBuilder, state: S)( - k: Step[(A, B), S] => Code[Ctrl] - ): Code[Ctrl] = { + def step(state: S)(k: Step[(A, B), S] => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = { val (lS0, rS0, (rPrev, somePrev)) = state - left.step(mb, jb, lS0) { + left.step(lS0) { case EOS => k(EOS) case Yield(lElt, lS) => - val push = jb.joinPoint[(B, right.S, (B, Code[Boolean]))](mb) - val pull = jb.joinPoint[right.S](mb) - val compare = jb.joinPoint[(B, right.S)](mb) + val push = ctx.jb.joinPoint[(B, right.S, (B, Code[Boolean]))](ctx.mb) + val pull = ctx.jb.joinPoint[right.S](ctx.mb) + val compare = ctx.jb.joinPoint[(B, right.S)](ctx.mb) push.define { case (rElt, rS, rPrevOpt) => k(Yield((lElt, rElt), (lS, rS, rPrevOpt))) } - pull.define(right.step(mb, jb, _) { + pull.define(right.step(_) { case EOS => push((rNil, right.emptyState, (rNil, false))) case Yield(rElt, rS) => compare((rElt, rS)) }) compare.define { case (rElt, rS) => - ParameterPack.let(mb, comp(lElt, rElt)) { c => + ParameterPack.let(ctx.mb, comp(lElt, rElt)) { c => (c > 0).mux( pull(rS), (c < 0).mux( @@ -471,16 +841,12 @@ object EmitStream { def emptyState: S = Code.invokeScalaObject[Iterator[Nothing]](Iterator.getClass, "empty") def length(s0: S): Option[Code[Int]] = None - def init(mb: MethodBuilder, jb: JoinPointBuilder, iter: S)( - k: Init[S] => Code[Ctrl] - ): Code[Ctrl] = + def init(iter: S)(k: Init[S] => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = k(Start(iter)) - def step(mb: MethodBuilder, jb: JoinPointBuilder, iter: S)( - k: Step[Code[A], S] => Code[Ctrl] - ): Code[Ctrl] = + def step(iter: S)(k: Step[Code[A], S] => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = iter.hasNext.mux( - ParameterPack.let(mb, iter.next()) { elt => k(Yield(elt, iter)) }, + ParameterPack.let(ctx.mb, iter.next()) { elt => k(Yield(elt, iter)) }, k(EOS)) } @@ -499,34 +865,34 @@ object EmitStream { (left.length(s0._2) liftedZip right.length(s0._3)) .map { case (lLen, rLen) => cond.mux(lLen, rLen) } - def init(mb: MethodBuilder, jb: JoinPointBuilder, param: P)(k: Init[S] => Code[Ctrl]): Code[Ctrl] = { - val missing = jb.joinPoint() - val start = jb.joinPoint[S](mb) + def init(param: P)(k: Init[S] => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = { + val missing = ctx.jb.joinPoint() + val start = ctx.jb.joinPoint[S](ctx.mb) missing.define { _ => k(Missing) } start.define { s => k(Start(s)) } cond.mux( - left.init(mb, jb, param) { + left.init(param) { case Start(s0) => start((true, s0, right.emptyState)) case Missing => missing(()) }, - right.init(mb, jb, param) { + right.init(param) { case Start(s0) => start((false, left.emptyState, s0)) case Missing => missing(()) }) } - def step(mb: MethodBuilder, jb: JoinPointBuilder, state: S)(k: Step[A, S] => Code[Ctrl]): Code[Ctrl] = { + def step(state: S)(k: Step[A, S] => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = { val (useLeft, lS, rS) = state - val eos = jb.joinPoint() - val push = jb.joinPoint[(A, left.S, right.S)](mb) + val eos = ctx.jb.joinPoint() + val push = ctx.jb.joinPoint[(A, left.S, right.S)](ctx.mb) eos.define { _ => k(EOS) } push.define { case (elt, lS, rS) => k(Yield(elt, (useLeft, lS, rS))) } useLeft.mux( - left.step(mb, jb, lS) { + left.step(lS) { case Yield(a, lS1) => push((a, lS1, rS)) case EOS => eos(()) }, - right.step(mb, jb, rS) { + right.step(rS) { case Yield(a, rS1) => push((a, lS, rS1)) case EOS => eos(()) }) @@ -541,12 +907,12 @@ object EmitStream { def emptyState: S = Code._null def length(s0: S): Option[Code[Int]] = None - def init(mb: MethodBuilder, jb: JoinPointBuilder, ib: Code[InputBuffer])(k: Init[S] => Code[Ctrl]): Code[Ctrl] = + def init(ib: Code[InputBuffer])(k: Init[S] => Code[Ctrl])(implicit ctx: EmitStreamContext): Code[Ctrl] = k(Start(ib)) - def step(mb: MethodBuilder, jb: JoinPointBuilder, ib: Code[InputBuffer])( + def step(ib: Code[InputBuffer])( k: Step[Code[T], S] => Code[Ctrl] - ): Code[Ctrl] = + )(implicit ctx: EmitStreamContext): Code[Ctrl] = (ib.isNull || !ib.readByte().toZ).mux(k(EOS), k(Yield(dec(region, ib), ib))) } @@ -875,23 +1241,23 @@ case class EmitStream( val setup = state := JoinPoint.CallCC[stream.S] { (jb, ret) => - stream.init(mb, jb, ()) { + stream.init(()) { case Missing => Code(m := true, ret(stream.emptyState)) case Start(s0) => Code(m := false, ret(s0)) - } + }(EmitStreamContext(mb, jb)) } val addElements = JoinPoint.CallCC[Unit] { (jb, ret) => val loop = jb.joinPoint() - loop.define { _ => stream.step(mb, jb, state.load) { + loop.define { _ => stream.step(state.load) { case EOS => ret(()) case Yield(elt, s1) => Code( elt.setup, cont(elt.m, elt.value), state := s1, loop(())) - } } + }(EmitStreamContext(mb, jb)) } loop(()) } diff --git a/hail/src/main/scala/is/hail/expr/ir/FoldConstants.scala b/hail/src/main/scala/is/hail/expr/ir/FoldConstants.scala index f47a1b160a6..1d7fd564780 100644 --- a/hail/src/main/scala/is/hail/expr/ir/FoldConstants.scala +++ b/hail/src/main/scala/is/hail/expr/ir/FoldConstants.scala @@ -15,7 +15,6 @@ object FoldConstants { _: ApplyAggOp | _: ApplyScanOp | _: Begin | - _: ArrayRange | _: MakeNDArray | _: NDArrayShape | _: NDArrayReshape | diff --git a/hail/src/main/scala/is/hail/expr/ir/IR.scala b/hail/src/main/scala/is/hail/expr/ir/IR.scala index 75836ebc506..c0e89d7a437 100644 --- a/hail/src/main/scala/is/hail/expr/ir/IR.scala +++ b/hail/src/main/scala/is/hail/expr/ir/IR.scala @@ -34,7 +34,7 @@ sealed trait IR extends BaseIR { try { _typ = InferType(this) } catch { - case e: Throwable => throw new RuntimeException(s"typ: inference failure: \n${ Pretty(this) }", e) + case e: Throwable => throw new RuntimeException(s"typ: inference failure", e) } _typ } @@ -163,10 +163,8 @@ object ArrayRef { final case class ArrayRef(a: IR, i: IR, msg: IR) extends IR final case class ArrayLen(a: IR) extends IR -final case class ArrayRange(start: IR, stop: IR, step: IR) extends IR final case class StreamRange(start: IR, stop: IR, step: IR) extends IR - object ArraySort { def apply(a: IR, ascending: IR = True(), onKey: Boolean = false): ArraySort = { val l = genUID() @@ -190,6 +188,7 @@ object ArraySort { ArraySort(a, l, r, If(ascending, compare < 0, compare > 0)) } } + final case class ArraySort(a: IR, left: String, right: String, compare: IR) extends IR final case class ToSet(a: IR) extends IR final case class ToDict(a: IR) extends IR @@ -432,6 +431,11 @@ final case class CollectDistributedArray(contexts: IR, globals: IR, cname: Strin final case class ReadPartition(path: IR, spec: AbstractTypedCodecSpec, rowType: TStruct) extends IR +final case class ReadValue(path: IR, spec: AbstractTypedCodecSpec, requestedType: Type) extends IR +final case class WriteValue(value: IR, pathPrefix: IR, spec: AbstractTypedCodecSpec) extends IR + +final case class UnpersistBlockMatrix(child: BlockMatrixIR) extends IR + class PrimitiveIR(val self: IR) extends AnyVal { def +(other: IR): IR = ApplyBinaryPrimOp(Add(), self, other) def -(other: IR): IR = ApplyBinaryPrimOp(Subtract(), self, other) diff --git a/hail/src/main/scala/is/hail/expr/ir/IRBuilder.scala b/hail/src/main/scala/is/hail/expr/ir/IRBuilder.scala index ffb9d162e91..e1ca522fdb6 100644 --- a/hail/src/main/scala/is/hail/expr/ir/IRBuilder.scala +++ b/hail/src/main/scala/is/hail/expr/ir/IRBuilder.scala @@ -36,7 +36,7 @@ object IRBuilder { implicit def arrayIRToProxy(seq: Seq[IR]): IRProxy = arrayToProxy(seq.map(irToProxy)) def irRange(start: IRProxy, end: IRProxy, step: IRProxy = 1): IRProxy = (env: E) => - ArrayRange(start(env), end(env), step(env)) + ToArray(StreamRange(start(env), end(env), step(env))) def irArrayLen(a: IRProxy): IRProxy = (env: E) => ArrayLen(a(env)) diff --git a/hail/src/main/scala/is/hail/expr/ir/InferPType.scala b/hail/src/main/scala/is/hail/expr/ir/InferPType.scala index 59664a1f8b6..1d2af2e636e 100644 --- a/hail/src/main/scala/is/hail/expr/ir/InferPType.scala +++ b/hail/src/main/scala/is/hail/expr/ir/InferPType.scala @@ -72,28 +72,30 @@ object InferPType { def getNestedElementPTypesOfSameType(ptypes: Seq[PType]): PType = { ptypes.head match { - case x: PStreamable => - val elementType = getNestedElementPTypesOfSameType(ptypes.map(_.asInstanceOf[PStreamable].elementType)) - x.copyStreamable(elementType, ptypes.forall(_.required)) - case _: PSet => + case _: PStream => + val elementType = getNestedElementPTypesOfSameType(ptypes.map(_.asInstanceOf[PStream].elementType)) + PStream(elementType, ptypes.forall(_.required)) + case _: PCanonicalArray => + val elementType = getNestedElementPTypesOfSameType(ptypes.map(_.asInstanceOf[PArray].elementType)) + PCanonicalArray(elementType, ptypes.forall(_.required)) + case _: PCanonicalSet => val elementType = getNestedElementPTypesOfSameType(ptypes.map(_.asInstanceOf[PSet].elementType)) - PSet(elementType, ptypes.forall(_.required)) - case x: PStruct => - PStruct(ptypes.forall(_.required), x.fieldNames.map(fieldName => + PCanonicalSet(elementType, ptypes.forall(_.required)) + case x: PCanonicalStruct => + PCanonicalStruct(ptypes.forall(_.required), x.fieldNames.map(fieldName => fieldName -> getNestedElementPTypesOfSameType(ptypes.map(_.asInstanceOf[PStruct].field(fieldName).typ)) ): _*) - case x: PTuple => - PTuple(ptypes.forall(_.required), x._types.map(pTupleField => + case x: PCanonicalTuple => + PCanonicalTuple(ptypes.forall(_.required), x._types.map(pTupleField => getNestedElementPTypesOfSameType(ptypes.map(_.asInstanceOf[PTuple]._types(pTupleField.index).typ)) ): _*) - case _: PDict => + case _: PCanonicalDict => val keyType = getNestedElementPTypesOfSameType(ptypes.map(_.asInstanceOf[PDict].keyType)) val valueType = getNestedElementPTypesOfSameType(ptypes.map(_.asInstanceOf[PDict].valueType)) - - PDict(keyType, valueType, ptypes.forall(_.required)) - case _: PInterval => + PCanonicalDict(keyType, valueType, ptypes.forall(_.required)) + case _: PCanonicalInterval => val pointType = getNestedElementPTypesOfSameType(ptypes.map(_.asInstanceOf[PInterval].pointType)) - PInterval(pointType, ptypes.forall(_.required)) + PCanonicalInterval(pointType, ptypes.forall(_.required)) case _ => ptypes.head.setRequired(ptypes.forall(_.required)) } } @@ -143,16 +145,6 @@ object InferPType { val nElem = shape.pType2.asInstanceOf[PTuple].size PNDArray(coerce[PArray](data.pType2).elementType.setRequired(true), nElem, data.pType2.required && shape.pType2.required) - case ArrayRange(start: IR, stop: IR, step: IR) => - infer(start) - infer(stop) - infer(step) - - assert(start.pType2 isOfType stop.pType2) - assert(start.pType2 isOfType step.pType2) - - val allRequired = start.pType2.required && stop.pType2.required && step.pType2.required - PArray(start.pType2.setRequired(true), allRequired) case StreamRange(start: IR, stop: IR, step: IR) => infer(start) infer(stop) @@ -455,19 +447,28 @@ object InferPType { PCanonicalArray(bodyIR._pType2, contextsIR._pType2.required) case ReadPartition(rowIR, codecSpec, rowType) => infer(rowIR) - val child = codecSpec.buildDecoder(rowType)._1 - PStream(child, child.required) + case ReadValue(path, spec, requestedType) => + infer(path) + spec.buildDecoder(requestedType)._1 + case WriteValue(value, pathPrefix, spec) => + infer(value) + infer(pathPrefix) + PCanonicalString(pathPrefix.pType2.required) case MakeStream(irs, t) => if (irs.isEmpty) { PType.canonical(t, true).deepInnerRequired(true) + } else { + PStream(getNestedElementPTypes(irs.map(theIR => { + infer(theIR) + theIR._pType2 + })), true) } - - PStream(getNestedElementPTypes(irs.map(theIR => { - infer(theIR) - theIR._pType2 - })), true) + case ResultOp(_, aggSigs) => + val rPTypes = aggSigs.toIterator.zipWithIndex.map{ case (sig, i) => PTupleField(i, sig.toCanonicalPhysical.resultType)}.toIndexedSeq + val allReq = rPTypes.forall(f => f.typ.required) + PCanonicalTuple(rPTypes, allReq) case x@InitOp(i, args, sig, op) => op match { diff --git a/hail/src/main/scala/is/hail/expr/ir/InferType.scala b/hail/src/main/scala/is/hail/expr/ir/InferType.scala index 510e8ee0c35..ef0a1fb9f08 100644 --- a/hail/src/main/scala/is/hail/expr/ir/InferType.scala +++ b/hail/src/main/scala/is/hail/expr/ir/InferType.scala @@ -30,7 +30,6 @@ object InferType { case MakeNDArray(data, shape, _) => TNDArray(coerce[TArray](data.typ).elementType.setRequired(true), Nat(shape.typ.asInstanceOf[TTuple].size)) case _: ArrayLen => TInt32() - case _: ArrayRange => TArray(TInt32()) case _: StreamRange => TStream(TInt32()) case _: LowerBoundOnOrderedCollection => TInt32() case _: ArrayFor => TVoid @@ -211,6 +210,7 @@ object InferType { case _: BlockMatrixCollect => TNDArray(TFloat64(), Nat(2)) case _: BlockMatrixWrite => TVoid case _: BlockMatrixMultiWrite => TVoid + case _: UnpersistBlockMatrix => TVoid case TableGetGlobals(child) => child.typ.globalType case TableCollect(child) => TStruct("rows" -> TArray(child.typ.rowType), "global" -> child.typ.globalType) case TableToValueApply(child, function) => function.typ(child.typ) @@ -218,6 +218,8 @@ object InferType { case BlockMatrixToValueApply(child, function) => function.typ(child.typ) case CollectDistributedArray(_, _, _, _, body) => TArray(body.typ) case ReadPartition(_, _, rowType) => TStream(rowType) + case ReadValue(_, _, typ) => typ + case WriteValue(value, pathPrefix, spec) => TString() case LiftMeOut(child) => child.typ } } diff --git a/hail/src/main/scala/is/hail/expr/ir/Interpret.scala b/hail/src/main/scala/is/hail/expr/ir/Interpret.scala index fee113744cb..4a8a3fd5143 100644 --- a/hail/src/main/scala/is/hail/expr/ir/Interpret.scala +++ b/hail/src/main/scala/is/hail/expr/ir/Interpret.scala @@ -251,7 +251,7 @@ object Interpret { null else aValue.asInstanceOf[IndexedSeq[Any]].length - case ArrayRange(start, stop, step) => + case StreamRange(start, stop, step) => val startValue = interpret(start, env, args) val stopValue = interpret(stop, env, args) val stepValue = interpret(step, env, args) @@ -586,6 +586,9 @@ object Interpret { writer(hc, child.execute(ctx)) case BlockMatrixMultiWrite(blockMatrices, writer) => writer(blockMatrices.map(_.execute(ctx))) + case UnpersistBlockMatrix(BlockMatrixRead(BlockMatrixPersistReader(id))) => + HailContext.backend.cache.unpersistBlockMatrix(id) + case _: UnpersistBlockMatrix => case TableToValueApply(child, function) => function.execute(ctx, child.execute(ctx)) case BlockMatrixToValueApply(child, function) => @@ -678,8 +681,6 @@ object Interpret { } } wrapped.get(0) - case x: ReadPartition => - fatal(s"cannot interpret ${ Pretty(x) }") case LiftMeOut(child) => val (rt, makeFunction) = Compile[Long](ctx, MakeTuple.ordered(FastSeq(child)), None, false) Region.scoped { r => diff --git a/hail/src/main/scala/is/hail/expr/ir/Interpretable.scala b/hail/src/main/scala/is/hail/expr/ir/Interpretable.scala index d27ee4cdbd9..97c3929753f 100644 --- a/hail/src/main/scala/is/hail/expr/ir/Interpretable.scala +++ b/hail/src/main/scala/is/hail/expr/ir/Interpretable.scala @@ -31,6 +31,9 @@ object Interpretable { _: NDArrayMatMul | _: TailLoop | _: Recur | + _: ReadPartition | + _: ReadValue | + _: WriteValue | _: NDArrayWrite => false case x: ApplyIR => !Exists(x.body, { diff --git a/hail/src/main/scala/is/hail/expr/ir/MatrixIR.scala b/hail/src/main/scala/is/hail/expr/ir/MatrixIR.scala index 9444fe0a5c8..cbf14148157 100644 --- a/hail/src/main/scala/is/hail/expr/ir/MatrixIR.scala +++ b/hail/src/main/scala/is/hail/expr/ir/MatrixIR.scala @@ -228,7 +228,6 @@ case class MatrixNativeReader( val cols = if (partFiles.length == 1) { ReadPartition(Str(partFiles.head), colsRVDSpec.typedCodecSpec, mr.typ.colType) } else { - val partitionReads = partFiles.map(f => ToArray(ReadPartition(Str(f), colsRVDSpec.typedCodecSpec, mr.typ.colType))) val partNames = MakeArray(partFiles.map(Str), TArray(TString())) val elt = Ref(genUID(), TString()) ArrayFlatMap( diff --git a/hail/src/main/scala/is/hail/expr/ir/Parser.scala b/hail/src/main/scala/is/hail/expr/ir/Parser.scala index 6ad01787a3b..166d7f344cd 100644 --- a/hail/src/main/scala/is/hail/expr/ir/Parser.scala +++ b/hail/src/main/scala/is/hail/expr/ir/Parser.scala @@ -780,11 +780,6 @@ object IRParser { val s = ir_value_expr(env)(it) ArrayRef(a, i, s) case "ArrayLen" => ArrayLen(ir_value_expr(env)(it)) - case "ArrayRange" => - val start = ir_value_expr(env)(it) - val stop = ir_value_expr(env)(it) - val step = ir_value_expr(env)(it) - ArrayRange(start, stop, step) case "StreamRange" => val start = ir_value_expr(env)(it) val stop = ir_value_expr(env)(it) @@ -1155,6 +1150,8 @@ object IRParser { val writer = deserialize[BlockMatrixMultiWriter](writerStr) val blockMatrices = repUntil(it, blockmatrix_ir(env), PunctuationToken(")")) BlockMatrixMultiWrite(blockMatrices.toFastIndexedSeq, writer) + case "UnpersistBlockMatrix" => + UnpersistBlockMatrix(blockmatrix_ir(env)(it)) case "CollectDistributedArray" => val cname = identifier(it) val gname = identifier(it) @@ -1171,6 +1168,18 @@ object IRParser { val rowType = coerce[TStruct](type_expr(env.typEnv)(it)) val path = ir_value_expr(env)(it) ReadPartition(path, spec, rowType) + case "ReadValue" => + import AbstractRVDSpec.formats + val spec = JsonMethods.parse(string_literal(it)).extract[AbstractTypedCodecSpec] + val typ = type_expr(env.typEnv)(it) + val path = ir_value_expr(env)(it) + ReadValue(path, spec, typ) + case "WriteValue" => + import AbstractRVDSpec.formats + val spec = JsonMethods.parse(string_literal(it)).extract[AbstractTypedCodecSpec] + val value = ir_value_expr(env)(it) + val path = ir_value_expr(env)(it) + WriteValue(value, path, spec) case "LiftMeOut" => LiftMeOut(ir_value_expr(env)(it)) } diff --git a/hail/src/main/scala/is/hail/expr/ir/Pretty.scala b/hail/src/main/scala/is/hail/expr/ir/Pretty.scala index 7f9da32da68..2b909fbe4cc 100644 --- a/hail/src/main/scala/is/hail/expr/ir/Pretty.scala +++ b/hail/src/main/scala/is/hail/expr/ir/Pretty.scala @@ -416,8 +416,11 @@ object Pretty { case RelationalLetTable(name, _, _) => prettyIdentifier(name) case RelationalLetMatrixTable(name, _, _) => prettyIdentifier(name) case RelationalLetBlockMatrix(name, _, _) => prettyIdentifier(name) - case ReadPartition(path, spec, rowType) => + case ReadPartition(_, spec, rowType) => s"${ prettyStringLiteral(spec.toString) } ${ rowType.parsableString() }" + case ReadValue(_, spec, reqType) => + s"${ prettyStringLiteral(spec.toString) } ${ reqType.parsableString() }" + case WriteValue(_, _, spec) => prettyStringLiteral(spec.toString) case _ => "" } diff --git a/hail/src/main/scala/is/hail/expr/ir/PruneDeadFields.scala b/hail/src/main/scala/is/hail/expr/ir/PruneDeadFields.scala index 8553dc2f3a6..97ef79146bd 100644 --- a/hail/src/main/scala/is/hail/expr/ir/PruneDeadFields.scala +++ b/hail/src/main/scala/is/hail/expr/ir/PruneDeadFields.scala @@ -1637,9 +1637,9 @@ object PruneDeadFields { RelationalLet(name, value2, rebuildIR(body, env, memo)) case RelationalRef(name, _) => RelationalRef(name, memo.relationalRefs(name)) case MakeArray(args, _) => - val depArray = requestedType.asInstanceOf[TArray] + val dep = requestedType.asInstanceOf[TStreamable] val args2 = args.map(a => rebuildIR(a, env, memo)) - MakeArray.unify(args2, depArray) + MakeArray.unify(args2, TArray(dep.elementType, dep.elementType.required)) case ArrayMap(a, name, body) => val a2 = rebuildIR(a, env, memo) ArrayMap(a2, name, rebuildIR(body, env.bindEval(name, -a2.typ.asInstanceOf[TStreamable].elementType), memo)) @@ -1786,11 +1786,11 @@ object PruneDeadFields { AggArrayPerElement(a2, elementName, indexName, aggBody2, knownLength.map(rebuildIR(_, aEnv, memo)), isScan) case ArrayAgg(a, name, query) => val a2 = rebuildIR(a, env, memo) - val query2 = rebuildIR(query, env.copy(agg = Some(env.eval.bind(name -> a2.typ.asInstanceOf[TArray].elementType))), memo) + val query2 = rebuildIR(query, env.copy(agg = Some(env.eval.bind(name -> a2.typ.asInstanceOf[TStreamable].elementType))), memo) ArrayAgg(a2, name, query2) case ArrayAggScan(a, name, query) => val a2 = rebuildIR(a, env, memo) - val query2 = rebuildIR(query, env.copy(scan = Some(env.eval.bind(name -> a2.typ.asInstanceOf[TArray].elementType))), memo) + val query2 = rebuildIR(query, env.copy(scan = Some(env.eval.bind(name -> a2.typ.asInstanceOf[TStreamable].elementType))), memo) ArrayAggScan(a2, name, query2) case RunAgg(body, result, signatures) => val body2 = rebuildIR(body, env, memo) diff --git a/hail/src/main/scala/is/hail/expr/ir/Simplify.scala b/hail/src/main/scala/is/hail/expr/ir/Simplify.scala index d9c89f1f8f1..3a816a87af4 100644 --- a/hail/src/main/scala/is/hail/expr/ir/Simplify.scala +++ b/hail/src/main/scala/is/hail/expr/ir/Simplify.scala @@ -77,7 +77,6 @@ object Simplify { case _: Apply | _: ApplyUnaryPrimOp | _: ApplyBinaryPrimOp | - _: ArrayRange | _: ArrayRef | _: ArrayLen | _: GetField | @@ -195,8 +194,6 @@ object Simplify { case ArrayLen(MakeArray(args, _)) => I32(args.length) - case ArrayLen(ArrayRange(start, end, I32(1))) => ApplyBinaryPrimOp(Subtract(), end, start) - case ArrayLen(ArrayMap(a, _, _)) => ArrayLen(a) case ArrayLen(ArrayFlatMap(a, _, MakeArray(args, _))) => ApplyBinaryPrimOp(Multiply(), I32(args.length), ArrayLen(a)) @@ -214,7 +211,7 @@ object Simplify { case ArrayFlatMap(ArrayMap(a, n1, b1), n2, b2) => ArrayFlatMap(a, n1, Let(n2, b1, b2)) - case ArrayMap(a, elt, r: Ref) if r.name == elt && r.typ == a.typ.asInstanceOf[TArray].elementType => a + case ArrayMap(a, elt, r: Ref) if r.name == elt && r.typ == a.typ.asInstanceOf[TIterable].elementType => a case ArrayMap(ArrayMap(a, n1, b1), n2, b2) => ArrayMap(a, n1, Let(n2, b1, b2)) @@ -556,6 +553,8 @@ object Simplify { if (child.typ.isInstanceOf[TArray]) ArrayRef(child, I32((i * ncols + j).toInt)) else child case LiftMeOut(child) if IsConstant(child) => child + case x@UnpersistBlockMatrix(BlockMatrixRead(BlockMatrixPersistReader(_))) => x + case _: UnpersistBlockMatrix => Begin(FastIndexedSeq()) } private[this] def tableRules(canRepartition: Boolean): PartialFunction[TableIR, TableIR] = { diff --git a/hail/src/main/scala/is/hail/expr/ir/Streamify.scala b/hail/src/main/scala/is/hail/expr/ir/Streamify.scala deleted file mode 100644 index e3306423769..00000000000 --- a/hail/src/main/scala/is/hail/expr/ir/Streamify.scala +++ /dev/null @@ -1,24 +0,0 @@ -package is.hail.expr.ir - -import is.hail.expr.types.virtual._ - -object Streamify { - def apply(ir: IR): IR = ir match { - // If and NA are currently broken - // case NA(t: TStreamable) => NA(TStream(t.elementType, t.required)) - // case If(c, t, e) => If(c, apply(t), apply(e)) - case MakeArray(xs, t) => MakeStream(xs, TStream(t.elementType, t.required)) - case ArrayRange(x, y, z) => StreamRange(x, y, z) - case ArrayMap(a, n, b) => ArrayMap(apply(a), n, b) - case ArrayZip(as, ns, b, behavior) => ArrayZip(as.map(apply), ns, b, behavior) - case ArrayFilter(a, n, b) => ArrayFilter(apply(a), n, b) - case ArrayFlatMap(a, n, b) => ArrayFlatMap(apply(a), n, apply(b)) - case ArrayLeftJoinDistinct(l, r, ln, rn, c, j) => ArrayLeftJoinDistinct(apply(l), apply(r), ln, rn, c, j) - case ArrayScan(a, z, an, en, b) => ArrayScan(apply(a), z, an, en, b) - case ArrayAggScan(a, n, q) => ArrayAggScan(apply(a), n, q) - case RunAggScan(a, name, init, seq, res, sig) => RunAggScan(apply(a), name, init, seq, res, sig) - case Let(n, v, b) => Let(n, v, apply(b)) - case x: ReadPartition => x - case ir => ToStream(ir) - } -} diff --git a/hail/src/main/scala/is/hail/expr/ir/TableIR.scala b/hail/src/main/scala/is/hail/expr/ir/TableIR.scala index cc5f4c8aaad..2eec7d2c08b 100644 --- a/hail/src/main/scala/is/hail/expr/ir/TableIR.scala +++ b/hail/src/main/scala/is/hail/expr/ir/TableIR.scala @@ -870,7 +870,8 @@ case class TableMultiWayZipJoin(children: IndexedSeq[TableIR], fieldName: String assert(childValues.map(_.rvd.typ).toSet.size == 1) // same physical types - val childRVDs = childValues.map(_.rvd) + val childRVDs = RVD.unify(childValues.map(_.rvd)).toFastIndexedSeq + val repartitionedRVDs = if (childRVDs(0).partitioner.satisfiesAllowedOverlap(typ.key.length - 1) && childRVDs.forall(rvd => rvd.partitioner == childRVDs(0).partitioner)) @@ -985,7 +986,6 @@ case class TableMapRows(child: TableIR, newRow: IR) extends TableIR { val scanRef = genUID() val extracted = agg.Extract.apply(agg.Extract.liftScan(newRow), scanRef) - val nAggs = extracted.nAggs if (extracted.aggs.isEmpty) { @@ -1290,7 +1290,7 @@ case class TableUnion(children: IndexedSeq[TableIR]) extends TableIR { protected[ir] override def execute(ctx: ExecuteContext): TableValue = { val tvs = children.map(_.execute(ctx)) tvs(0).copy( - rvd = RVD.union(tvs.map(_.rvd), tvs(0).typ.key.length, ctx)) + rvd = RVD.union(RVD.unify(tvs.map(_.rvd)), tvs(0).typ.key.length, ctx)) } } diff --git a/hail/src/main/scala/is/hail/expr/ir/TypeCheck.scala b/hail/src/main/scala/is/hail/expr/ir/TypeCheck.scala index ffdcb65fa5e..319f6a15b00 100644 --- a/hail/src/main/scala/is/hail/expr/ir/TypeCheck.scala +++ b/hail/src/main/scala/is/hail/expr/ir/TypeCheck.scala @@ -157,10 +157,6 @@ object TypeCheck { assert(x.typ == -coerce[TStreamable](a.typ).elementType) case ArrayLen(a) => assert(a.typ.isInstanceOf[TStreamable]) - case x@ArrayRange(a, b, c) => - assert(a.typ.isOfType(TInt32())) - assert(b.typ.isOfType(TInt32())) - assert(c.typ.isOfType(TInt32())) case x@StreamRange(a, b, c) => assert(a.typ.isOfType(TInt32())) assert(b.typ.isOfType(TInt32())) @@ -386,11 +382,19 @@ object TypeCheck { case BlockMatrixCollect(_) => case BlockMatrixWrite(_, _) => case BlockMatrixMultiWrite(_, _) => + case UnpersistBlockMatrix(_) => case CollectDistributedArray(ctxs, globals, cname, gname, body) => assert(ctxs.typ.isInstanceOf[TStreamable]) - case x@ReadPartition(path, _, rowType) => + case x@ReadPartition(path, spec, rowType) => assert(path.typ == TString()) assert(x.typ == TStream(rowType)) + assert(spec.encodedType.decodedPType(rowType).virtualType == rowType) + case x@ReadValue(path, spec, requestedType) => + assert(path.typ == TString()) + assert(spec.encodedType.decodedPType(requestedType).virtualType == requestedType) + case x@WriteValue(value, pathPrefix, spec) => + assert(pathPrefix.typ == TString()) + spec.encodedType.encodeCompatible(value.pType2) case LiftMeOut(_) => } } diff --git a/hail/src/main/scala/is/hail/expr/ir/agg/AppendOnlyBTree.scala b/hail/src/main/scala/is/hail/expr/ir/agg/AppendOnlyBTree.scala index 04db4faed6d..c5bedadccbb 100644 --- a/hail/src/main/scala/is/hail/expr/ir/agg/AppendOnlyBTree.scala +++ b/hail/src/main/scala/is/hail/expr/ir/agg/AppendOnlyBTree.scala @@ -29,7 +29,7 @@ trait BTreeKey { class AppendOnlyBTree(fb: EmitFunctionBuilder[_], key: BTreeKey, region: Code[Region], root: ClassFieldRef[Long], maxElements: Int = 2) { private val splitIdx = maxElements / 2 private val eltType: PTuple = PTuple(key.storageType, PInt64(true)) - private val elementsType: PTuple = PTuple(required = true, Array.fill(maxElements)(eltType): _*) + private val elementsType: PTuple = PCanonicalTuple(required = true, Array.fill[PType](maxElements)(eltType): _*) private val storageType: PStruct = PStruct(required = true, "parent" -> PInt64(), "child0" -> PInt64(), diff --git a/hail/src/main/scala/is/hail/expr/ir/agg/DownsampleAggregator.scala b/hail/src/main/scala/is/hail/expr/ir/agg/DownsampleAggregator.scala index c4b2b55cf1d..4fd41980191 100644 --- a/hail/src/main/scala/is/hail/expr/ir/agg/DownsampleAggregator.scala +++ b/hail/src/main/scala/is/hail/expr/ir/agg/DownsampleAggregator.scala @@ -513,7 +513,7 @@ class DownsampleState(val fb: EmitFunctionBuilder[_], labelType: PArray, maxBuff srvb.addDouble(Region.loadDouble(pointType.loadField(point, "y"))), srvb.advance(), pointType.isFieldDefined(point, "label").mux( - srvb.addIRIntermediate(resultType)(pointType.loadField(point, "label")), + srvb.addIRIntermediate(labelType)(pointType.loadField(point, "label")), srvb.setMissing() ) ) diff --git a/hail/src/main/scala/is/hail/expr/ir/agg/Extract.scala b/hail/src/main/scala/is/hail/expr/ir/agg/Extract.scala index 0eeb6e350a3..6ace16a238b 100644 --- a/hail/src/main/scala/is/hail/expr/ir/agg/Extract.scala +++ b/hail/src/main/scala/is/hail/expr/ir/agg/Extract.scala @@ -304,7 +304,7 @@ object Extract { Begin(FastIndexedSeq( SeqOp(i, FastIndexedSeq(ArrayLen(aRef)), state, AggElementsLengthCheck()), ArrayFor( - ArrayRange(I32(0), ArrayLen(aRef), I32(1)), + StreamRange(I32(0), ArrayLen(aRef), I32(1)), iRef.name, Let( elementName, @@ -317,13 +317,13 @@ object Extract { Let( rUID.name, GetTupleElement(result, i), - ArrayMap( - ArrayRange(0, ArrayLen(rUID), 1), + ToArray(ArrayMap( + StreamRange(0, ArrayLen(rUID), 1), indexName, Let( newRef.name, ArrayRef(rUID, Ref(indexName, TInt32())), - transformed))) + transformed)))) case x: ArrayAgg => assert(!ContainsScan(x)) diff --git a/hail/src/main/scala/is/hail/expr/ir/functions/ArrayFunctions.scala b/hail/src/main/scala/is/hail/expr/ir/functions/ArrayFunctions.scala index c0b289af28c..db5a8e5966d 100644 --- a/hail/src/main/scala/is/hail/expr/ir/functions/ArrayFunctions.scala +++ b/hail/src/main/scala/is/hail/expr/ir/functions/ArrayFunctions.scala @@ -186,7 +186,7 @@ object ArrayFunctions extends RegistryFunctions { updateAccum(Ref(value, t), Ref(idx, TInt32())), Ref(accum, tAccum)))))) GetField(ArrayFold( - ArrayRange(I32(0), ArrayLen(a), I32(1)), + StreamRange(I32(0), ArrayLen(a), I32(1)), NA(tAccum), accum, idx, @@ -227,7 +227,7 @@ object ArrayFunctions extends RegistryFunctions { Ref(accum, tAccum))))))) Let(result, ArrayFold( - ArrayRange(I32(0), ArrayLen(a), I32(1)), + StreamRange(I32(0), ArrayLen(a), I32(1)), NA(tAccum), accum, idx, @@ -253,8 +253,8 @@ object ArrayFunctions extends RegistryFunctions { registerIR("[*:]", TArray(tv("T")), TInt32(), TArray(tv("T"))) { (a, i) => val idx = genUID() - ArrayMap( - ArrayRange( + ToArray(ArrayMap( + StreamRange( If(ApplyComparisonOp(LT(TInt32()), i, I32(0)), UtilFunctions.intMax( ApplyBinaryPrimOp(Add(), ArrayLen(a), i), @@ -263,27 +263,27 @@ object ArrayFunctions extends RegistryFunctions { ArrayLen(a), I32(1)), idx, - ArrayRef(a, Ref(idx, TInt32()))) + ArrayRef(a, Ref(idx, TInt32())))) } registerIR("[:*]", TArray(tv("T")), TInt32(), TArray(tv("T"))) { (a, i) => val idx = genUID() If(IsNA(a), a, - ArrayMap( - ArrayRange( + ToArray(ArrayMap( + StreamRange( I32(0), If(ApplyComparisonOp(LT(TInt32()), i, I32(0)), ApplyBinaryPrimOp(Add(), ArrayLen(a), i), UtilFunctions.intMin(i, ArrayLen(a))), I32(1)), idx, - ArrayRef(a, Ref(idx, TInt32())))) + ArrayRef(a, Ref(idx, TInt32()))))) } registerIR("[*:*]", TArray(tv("T")), TInt32(), TInt32(), TArray(tv("T"))) { (a, i, j) => val idx = genUID() - ArrayMap( - ArrayRange( + ToArray(ArrayMap( + StreamRange( If(ApplyComparisonOp(LT(TInt32()), i, I32(0)), UtilFunctions.intMax( ApplyBinaryPrimOp(Add(), ArrayLen(a), i), @@ -294,7 +294,7 @@ object ArrayFunctions extends RegistryFunctions { UtilFunctions.intMin(j, ArrayLen(a))), I32(1)), idx, - ArrayRef(a, Ref(idx, TInt32()))) + ArrayRef(a, Ref(idx, TInt32())))) } registerIR("flatten", TArray(TArray(tv("T"))), TArray(tv("T"))) { a => diff --git a/hail/src/main/scala/is/hail/expr/ir/functions/Functions.scala b/hail/src/main/scala/is/hail/expr/ir/functions/Functions.scala index a2c3be6d027..76ec7d05f4c 100644 --- a/hail/src/main/scala/is/hail/expr/ir/functions/Functions.scala +++ b/hail/src/main/scala/is/hail/expr/ir/functions/Functions.scala @@ -21,8 +21,8 @@ object IRFunctionRegistry { userAddedFunctions.clear() } - val irRegistry: mutable.MultiMap[String, (Seq[Type], Type, Boolean, Seq[IR] => IR)] = - new mutable.HashMap[String, mutable.Set[(Seq[Type], Type, Boolean, Seq[IR] => IR)]] with mutable.MultiMap[String, (Seq[Type], Type, Boolean, Seq[IR] => IR)] + val irRegistry: mutable.Map[String, mutable.Map[(Seq[Type], Type, Boolean), Seq[IR] => IR]] = + new mutable.HashMap[String, mutable.Map[(Seq[Type], Type, Boolean), Seq[IR] => IR]]() val codeRegistry: mutable.MultiMap[String, IRFunction] = new mutable.HashMap[String, mutable.Set[IRFunction]] with mutable.MultiMap[String, IRFunction] @@ -30,8 +30,10 @@ object IRFunctionRegistry { def addIRFunction(f: IRFunction): Unit = codeRegistry.addBinding(f.name, f) - def addIR(name: String, argTypes: Seq[Type], retType: Type, alwaysInline: Boolean, f: Seq[IR] => IR): Unit = - irRegistry.addBinding(name, (argTypes, retType, alwaysInline, f)) + def addIR(name: String, argTypes: Seq[Type], retType: Type, alwaysInline: Boolean, f: Seq[IR] => IR): Unit = { + val m = irRegistry.getOrElseUpdate(name, new mutable.HashMap[(Seq[Type], Type, Boolean), Seq[IR] => IR]()) + m.update((argTypes, retType, alwaysInline), f) + } def pyRegisterIR(mname: String, argNames: java.util.ArrayList[String], @@ -56,28 +58,29 @@ object IRFunctionRegistry { def removeIRFunction(name: String): Unit = codeRegistry.remove(name) - private def lookupInRegistry[T](reg: mutable.MultiMap[String, T], name: String, rt: Type, args: Seq[Type], cond: (T, Seq[Type]) => Boolean): Option[T] = { - reg.lift(name).map { fs => fs.filter(t => cond(t, args :+ rt)).toSeq }.getOrElse(FastSeq()) match { + def lookupFunction(name: String, rt: Type, args: Seq[Type]): Option[IRFunction] = + codeRegistry.lift(name).map { fs => fs.filter(t => t.unify(args :+ rt)).toSeq }.getOrElse(FastSeq()) match { case Seq() => None case Seq(f) => Some(f) case _ => fatal(s"Multiple functions found that satisfy $name(${ args.mkString(",") }).") } - } - def lookupFunction(name: String, rt: Type, args: Seq[Type]): Option[IRFunction] = - lookupInRegistry(codeRegistry, name, rt, args, (f: IRFunction, ts: Seq[Type]) => f.unify(ts)) + def lookupIR(name: String, rt: Type, args: Seq[Type]): Option[((Seq[Type], Type, Boolean), Seq[IR] => IR)] = { + irRegistry.getOrElse(name, Map.empty).filter { case ((argTypes, _, _), _) => + argTypes.length == args.length && { + argTypes.foreach(_.clear()) + (argTypes, args).zipped.forall(_.unify(_)) + } + }.toSeq match { + case Seq() => None + case Seq(kv) => Some(kv) + case _ => fatal(s"Multiple functions found that satisfy $name(${args.mkString(",")}).") + } + } def lookupConversion(name: String, rt: Type, args: Seq[Type]): Option[Seq[IR] => IR] = { - type Conversion = (Seq[Type], Type, Boolean, Seq[IR] => IR) - val findIR: (Conversion, Seq[Type]) => Boolean = { - case ((ts, _, _, _), t2s) => - ts.length == args.length && { - ts.foreach(_.clear()) - (ts, t2s).zipped.forall(_.unify(_)) - } - } - val validIR: Option[Seq[IR] => IR] = lookupInRegistry[Conversion](irRegistry, name, rt, args, findIR).map { - case (_, _, inline, conversion) => args => + val validIR: Option[Seq[IR] => IR] = lookupIR(name, rt, args).map { + case ((_, _, inline), conversion) => args => val x = ApplyIR(name, args) x.conversion = conversion x.inline = inline @@ -123,7 +126,7 @@ object IRFunctionRegistry { def dtype(t: Type): String = s"""dtype("${ StringEscapeUtils.escapeString(t.toString) }\")""" irRegistry.foreach { case (name, fns) => - fns.foreach { case (argTypes, retType, _, f) => + fns.foreach { case ((argTypes, retType, _), f) => println(s"""register_function("${ StringEscapeUtils.escapeString(name) }", (${ argTypes.map(dtype).mkString(",") }), ${ dtype(retType) })""") } } diff --git a/hail/src/main/scala/is/hail/expr/ir/lowering/LowerTableIR.scala b/hail/src/main/scala/is/hail/expr/ir/lowering/LowerTableIR.scala index d2a27d90574..ed17b982bf4 100644 --- a/hail/src/main/scala/is/hail/expr/ir/lowering/LowerTableIR.scala +++ b/hail/src/main/scala/is/hail/expr/ir/lowering/LowerTableIR.scala @@ -137,10 +137,10 @@ object LowerTableIR { MakeStruct(FastIndexedSeq("start" -> start, "end" -> end)) }, TArray(contextType) ), - ArrayMap(ArrayRange( + ToArray(ArrayMap(StreamRange( GetField(Ref("context", contextType), "start"), GetField(Ref("context", contextType), "end"), - I32(1)), i.name, MakeStruct(FastSeq("idx" -> i)))) + I32(1)), i.name, MakeStruct(FastSeq("idx" -> i))))) case TableMapGlobals(child, newGlobals) => val loweredChild = lower(child) diff --git a/hail/src/main/scala/is/hail/expr/ir/lowering/Rule.scala b/hail/src/main/scala/is/hail/expr/ir/lowering/Rule.scala index 1043758f775..25e34298b76 100644 --- a/hail/src/main/scala/is/hail/expr/ir/lowering/Rule.scala +++ b/hail/src/main/scala/is/hail/expr/ir/lowering/Rule.scala @@ -68,7 +68,6 @@ case object StreamableIRs extends Rule { case ArraySort(a, _, _, _) => a.typ.isInstanceOf[TStream] case GroupByKey(collection) => collection.typ.isInstanceOf[TStream] case _: MakeArray => false - case _: ArrayRange => false case _ => true } } diff --git a/hail/src/main/scala/is/hail/expr/types/BlockMatrixType.scala b/hail/src/main/scala/is/hail/expr/types/BlockMatrixType.scala index ee58983b945..0ef92855c5f 100644 --- a/hail/src/main/scala/is/hail/expr/types/BlockMatrixType.scala +++ b/hail/src/main/scala/is/hail/expr/types/BlockMatrixType.scala @@ -1,7 +1,8 @@ package is.hail.expr.types import is.hail.utils._ -import is.hail.expr.types.virtual.Type +import is.hail.expr.types.virtual.{TFloat64, Type} +import is.hail.linalg.BlockMatrix object BlockMatrixSparsity { private val builder: ArrayBuilder[(Int, Int)] = new ArrayBuilder[(Int, Int)] @@ -79,6 +80,12 @@ object BlockMatrixType { val (shape, isRowVector) = matrixToTensorShape(nRows, nCols) BlockMatrixType(elementType, shape, isRowVector, blockSize, BlockMatrixSparsity.dense) } + + def fromBlockMatrix(value: BlockMatrix): BlockMatrixType = { + val sparsity = BlockMatrixSparsity.fromLinearBlocks(value.nRows, value.nCols, value.blockSize, value.gp.maybeBlocks) + val (shape, isRowVector) = matrixToTensorShape(value.nRows, value.nCols) + BlockMatrixType(TFloat64(), shape, isRowVector, value.blockSize, sparsity) + } } case class BlockMatrixType( diff --git a/hail/src/main/scala/is/hail/expr/types/encoded/EArray.scala b/hail/src/main/scala/is/hail/expr/types/encoded/EArray.scala index f807543f378..abaf771785c 100644 --- a/hail/src/main/scala/is/hail/expr/types/encoded/EArray.scala +++ b/hail/src/main/scala/is/hail/expr/types/encoded/EArray.scala @@ -2,7 +2,6 @@ package is.hail.expr.types.encoded import is.hail.annotations.{Region, UnsafeUtils} import is.hail.asm4s._ -import is.hail.expr.ir.EmitMethodBuilder import is.hail.expr.types.BaseType import is.hail.expr.types.physical._ import is.hail.expr.types.virtual._ @@ -35,7 +34,7 @@ final case class EArray(elementType: EType, override val required: Boolean = fal } } - def buildPrefixEncoder(pt: PArray, mb: EmitMethodBuilder, array: Code[Long], + def buildPrefixEncoder(pt: PArray, mb: MethodBuilder, array: Code[Long], out: Code[OutputBuffer], prefixLength: Code[Int] ): Code[Unit] = { val len = mb.newLocal[Int]("len") @@ -80,7 +79,7 @@ final case class EArray(elementType: EType, override val required: Boolean = fal writeElems) } - def _buildEncoder(pt: PType, mb: EmitMethodBuilder, v: Code[_], out: Code[OutputBuffer]): Code[Unit] = { + def _buildEncoder(pt: PType, mb: MethodBuilder, v: Code[_], out: Code[OutputBuffer]): Code[Unit] = { val pa = pt.asInstanceOf[PArray] val array = coerce[Long](v) buildPrefixEncoder(pa, mb, array, out, pa.loadLength(array)) @@ -88,7 +87,7 @@ final case class EArray(elementType: EType, override val required: Boolean = fal def _buildDecoder( pt: PType, - mb: EmitMethodBuilder, + mb: MethodBuilder, region: Code[Region], in: Code[InputBuffer] ): Code[Long] = { @@ -121,7 +120,7 @@ final case class EArray(elementType: EType, override val required: Boolean = fal array.load()) } - def _buildSkip(mb: EmitMethodBuilder, r: Code[Region], in: Code[InputBuffer]): Code[Unit] = { + def _buildSkip(mb: MethodBuilder, r: Code[Region], in: Code[InputBuffer]): Code[Unit] = { val len = mb.newLocal[Int]("len") val i = mb.newLocal[Int]("i") val skip = elementType.buildSkip(mb) diff --git a/hail/src/main/scala/is/hail/expr/types/encoded/EBaseStruct.scala b/hail/src/main/scala/is/hail/expr/types/encoded/EBaseStruct.scala index 85b729e3b27..f5b414049b4 100644 --- a/hail/src/main/scala/is/hail/expr/types/encoded/EBaseStruct.scala +++ b/hail/src/main/scala/is/hail/expr/types/encoded/EBaseStruct.scala @@ -2,7 +2,6 @@ package is.hail.expr.types.encoded import is.hail.annotations.{Region, UnsafeUtils} import is.hail.asm4s._ -import is.hail.expr.ir.EmitMethodBuilder import is.hail.expr.types.{BaseStruct, BaseType} import is.hail.expr.types.physical._ import is.hail.expr.types.virtual._ @@ -90,10 +89,10 @@ final case class EBaseStruct(fields: IndexedSeq[EField], override val required: PNDArray(elementType, t.nDims, required) } - def _buildEncoder(pt: PType, mb: EmitMethodBuilder, v: Code[_], out: Code[OutputBuffer]): Code[Unit] = { + def _buildEncoder(pt: PType, mb: MethodBuilder, v: Code[_], out: Code[OutputBuffer]): Code[Unit] = { val ft = pt.asInstanceOf[PBaseStruct] val writeMissingBytes = if (ft.size == size) { - val missingBytes = ft.nMissingBytes + val missingBytes = UnsafeUtils.packBitsToBytes(ft.nMissing) var c = Code._empty[Unit] if (nMissingBytes > 1) c = Code(c, out.writeBytes(coerce[Long](v), missingBytes - 1)) @@ -166,7 +165,7 @@ final case class EBaseStruct(fields: IndexedSeq[EField], override val required: def _buildDecoder( pt: PType, - mb: EmitMethodBuilder, + mb: MethodBuilder, region: Code[Region], in: Code[InputBuffer] ): Code[Long] = { @@ -181,7 +180,7 @@ final case class EBaseStruct(fields: IndexedSeq[EField], override val required: override def _buildInplaceDecoder( pt: PType, - mb: EmitMethodBuilder, + mb: MethodBuilder, region: Code[Region], addr: Code[Long], in: Code[InputBuffer] @@ -228,7 +227,7 @@ final case class EBaseStruct(fields: IndexedSeq[EField], override val required: readFields, Code._empty[Unit]) } - def _buildSkip(mb: EmitMethodBuilder, r: Code[Region], in: Code[InputBuffer]): Code[Unit] = { + def _buildSkip(mb: MethodBuilder, r: Code[Region], in: Code[InputBuffer]): Code[Unit] = { val mbytes = mb.newLocal[Long]("mbytes") val skipFields = fields.map { f => val skip = f.typ.buildSkip(mb) diff --git a/hail/src/main/scala/is/hail/expr/types/encoded/EBinary.scala b/hail/src/main/scala/is/hail/expr/types/encoded/EBinary.scala index cb5eeed152a..75cf721e663 100644 --- a/hail/src/main/scala/is/hail/expr/types/encoded/EBinary.scala +++ b/hail/src/main/scala/is/hail/expr/types/encoded/EBinary.scala @@ -2,7 +2,6 @@ package is.hail.expr.types.encoded import is.hail.annotations.Region import is.hail.asm4s._ -import is.hail.expr.ir.EmitMethodBuilder import is.hail.expr.types.BaseType import is.hail.expr.types.physical._ import is.hail.expr.types.virtual._ @@ -13,7 +12,7 @@ case object EBinaryOptional extends EBinary(false) case object EBinaryRequired extends EBinary(true) class EBinary(override val required: Boolean) extends EType { - def _buildEncoder(pt: PType, mb: EmitMethodBuilder, v: Code[_], out: Code[OutputBuffer]): Code[Unit] = { + def _buildEncoder(pt: PType, mb: MethodBuilder, v: Code[_], out: Code[OutputBuffer]): Code[Unit] = { val addr = coerce[Long](v) val len = mb.newLocal[Int]("len") val bT = pt.asInstanceOf[PBinary] @@ -25,7 +24,7 @@ class EBinary(override val required: Boolean) extends EType { def _buildDecoder( pt: PType, - mb: EmitMethodBuilder, + mb: MethodBuilder, region: Code[Region], in: Code[InputBuffer] ): Code[_] = { @@ -40,7 +39,7 @@ class EBinary(override val required: Boolean) extends EType { barray.load()) } - def _buildSkip(mb: EmitMethodBuilder, r: Code[Region], in: Code[InputBuffer]): Code[Unit] = { + def _buildSkip(mb: MethodBuilder, r: Code[Region], in: Code[InputBuffer]): Code[Unit] = { val len = mb.newLocal[Int]("len") Code( len := in.readInt(), diff --git a/hail/src/main/scala/is/hail/expr/types/encoded/EBoolean.scala b/hail/src/main/scala/is/hail/expr/types/encoded/EBoolean.scala index e2e2f3033ef..ae62f5a6855 100644 --- a/hail/src/main/scala/is/hail/expr/types/encoded/EBoolean.scala +++ b/hail/src/main/scala/is/hail/expr/types/encoded/EBoolean.scala @@ -2,7 +2,6 @@ package is.hail.expr.types.encoded import is.hail.annotations.Region import is.hail.asm4s._ -import is.hail.expr.ir.EmitMethodBuilder import is.hail.expr.types.BaseType import is.hail.expr.types.physical._ import is.hail.expr.types.virtual._ @@ -13,18 +12,18 @@ case object EBooleanOptional extends EBoolean(false) case object EBooleanRequired extends EBoolean(true) class EBoolean(override val required: Boolean) extends EType { - def _buildEncoder(pt: PType, mb: EmitMethodBuilder, v: Code[_], out: Code[OutputBuffer]): Code[Unit] = { + def _buildEncoder(pt: PType, mb: MethodBuilder, v: Code[_], out: Code[OutputBuffer]): Code[Unit] = { out.writeBoolean(coerce[Boolean](v)) } def _buildDecoder( pt: PType, - mb: EmitMethodBuilder, + mb: MethodBuilder, region: Code[Region], in: Code[InputBuffer] ): Code[Boolean] = in.readBoolean() - def _buildSkip(mb: EmitMethodBuilder, r: Code[Region], in: Code[InputBuffer]): Code[Unit] = in.skipBoolean() + def _buildSkip(mb: MethodBuilder, r: Code[Region], in: Code[InputBuffer]): Code[Unit] = in.skipBoolean() override def _compatible(pt: PType): Boolean = pt.isInstanceOf[PBoolean] diff --git a/hail/src/main/scala/is/hail/expr/types/encoded/EFloat32.scala b/hail/src/main/scala/is/hail/expr/types/encoded/EFloat32.scala index 6e272810f1a..44ed2b5564d 100644 --- a/hail/src/main/scala/is/hail/expr/types/encoded/EFloat32.scala +++ b/hail/src/main/scala/is/hail/expr/types/encoded/EFloat32.scala @@ -2,7 +2,6 @@ package is.hail.expr.types.encoded import is.hail.annotations.Region import is.hail.asm4s._ -import is.hail.expr.ir.EmitMethodBuilder import is.hail.expr.types.BaseType import is.hail.expr.types.physical._ import is.hail.expr.types.virtual._ @@ -13,18 +12,18 @@ case object EFloat32Optional extends EFloat32(false) case object EFloat32Required extends EFloat32(true) class EFloat32(override val required: Boolean) extends EType { - def _buildEncoder(pt: PType, mb: EmitMethodBuilder, v: Code[_], out: Code[OutputBuffer]): Code[Unit] = { + def _buildEncoder(pt: PType, mb: MethodBuilder, v: Code[_], out: Code[OutputBuffer]): Code[Unit] = { out.writeFloat(coerce[Float](v)) } def _buildDecoder( pt: PType, - mb: EmitMethodBuilder, + mb: MethodBuilder, region: Code[Region], in: Code[InputBuffer] ): Code[Float] = in.readFloat() - def _buildSkip(mb: EmitMethodBuilder, r: Code[Region], in: Code[InputBuffer]): Code[Unit] = in.skipFloat() + def _buildSkip(mb: MethodBuilder, r: Code[Region], in: Code[InputBuffer]): Code[Unit] = in.skipFloat() override def _compatible(pt: PType): Boolean = pt.isInstanceOf[PFloat32] diff --git a/hail/src/main/scala/is/hail/expr/types/encoded/EFloat64.scala b/hail/src/main/scala/is/hail/expr/types/encoded/EFloat64.scala index c182b76e8fe..fb1bb84e133 100644 --- a/hail/src/main/scala/is/hail/expr/types/encoded/EFloat64.scala +++ b/hail/src/main/scala/is/hail/expr/types/encoded/EFloat64.scala @@ -2,7 +2,6 @@ package is.hail.expr.types.encoded import is.hail.annotations.Region import is.hail.asm4s._ -import is.hail.expr.ir.EmitMethodBuilder import is.hail.expr.types.BaseType import is.hail.expr.types.physical._ import is.hail.expr.types.virtual._ @@ -13,18 +12,18 @@ case object EFloat64Optional extends EFloat64(false) case object EFloat64Required extends EFloat64(true) class EFloat64(override val required: Boolean) extends EType { - def _buildEncoder(pt: PType, mb: EmitMethodBuilder, v: Code[_], out: Code[OutputBuffer]): Code[Unit] = { + def _buildEncoder(pt: PType, mb: MethodBuilder, v: Code[_], out: Code[OutputBuffer]): Code[Unit] = { out.writeDouble(coerce[Double](v)) } def _buildDecoder( pt: PType, - mb: EmitMethodBuilder, + mb: MethodBuilder, region: Code[Region], in: Code[InputBuffer] ): Code[Double] = in.readDouble() - def _buildSkip(mb: EmitMethodBuilder, r: Code[Region], in: Code[InputBuffer]): Code[Unit] = in.skipDouble() + def _buildSkip(mb: MethodBuilder, r: Code[Region], in: Code[InputBuffer]): Code[Unit] = in.skipDouble() override def _compatible(pt: PType): Boolean = pt.isInstanceOf[PFloat64] diff --git a/hail/src/main/scala/is/hail/expr/types/encoded/EInt32.scala b/hail/src/main/scala/is/hail/expr/types/encoded/EInt32.scala index b061393891a..942503ebea9 100644 --- a/hail/src/main/scala/is/hail/expr/types/encoded/EInt32.scala +++ b/hail/src/main/scala/is/hail/expr/types/encoded/EInt32.scala @@ -2,7 +2,6 @@ package is.hail.expr.types.encoded import is.hail.annotations.Region import is.hail.asm4s._ -import is.hail.expr.ir.EmitMethodBuilder import is.hail.expr.types.BaseType import is.hail.expr.types.physical._ import is.hail.expr.types.virtual._ @@ -13,18 +12,18 @@ case object EInt32Optional extends EInt32(false) case object EInt32Required extends EInt32(true) class EInt32(override val required: Boolean) extends EType { - def _buildEncoder(pt: PType, mb: EmitMethodBuilder, v: Code[_], out: Code[OutputBuffer]): Code[Unit] = { + def _buildEncoder(pt: PType, mb: MethodBuilder, v: Code[_], out: Code[OutputBuffer]): Code[Unit] = { out.writeInt(coerce[Int](v)) } def _buildDecoder( pt: PType, - mb: EmitMethodBuilder, + mb: MethodBuilder, region: Code[Region], in: Code[InputBuffer] ): Code[Int] = in.readInt() - def _buildSkip(mb: EmitMethodBuilder, r: Code[Region], in: Code[InputBuffer]): Code[Unit] = in.skipInt() + def _buildSkip(mb: MethodBuilder, r: Code[Region], in: Code[InputBuffer]): Code[Unit] = in.skipInt() override def _compatible(pt: PType): Boolean = pt.isInstanceOf[PInt32] diff --git a/hail/src/main/scala/is/hail/expr/types/encoded/EInt64.scala b/hail/src/main/scala/is/hail/expr/types/encoded/EInt64.scala index 8df0202ec8d..f73640c253a 100644 --- a/hail/src/main/scala/is/hail/expr/types/encoded/EInt64.scala +++ b/hail/src/main/scala/is/hail/expr/types/encoded/EInt64.scala @@ -2,7 +2,6 @@ package is.hail.expr.types.encoded import is.hail.annotations.Region import is.hail.asm4s._ -import is.hail.expr.ir.EmitMethodBuilder import is.hail.expr.types.BaseType import is.hail.expr.types.physical._ import is.hail.expr.types.virtual._ @@ -13,18 +12,18 @@ case object EInt64Optional extends EInt64(false) case object EInt64Required extends EInt64(true) class EInt64(override val required: Boolean) extends EType { - def _buildEncoder(pt: PType, mb: EmitMethodBuilder, v: Code[_], out: Code[OutputBuffer]): Code[Unit] = { + def _buildEncoder(pt: PType, mb: MethodBuilder, v: Code[_], out: Code[OutputBuffer]): Code[Unit] = { out.writeLong(coerce[Long](v)) } def _buildDecoder( pt: PType, - mb: EmitMethodBuilder, + mb: MethodBuilder, region: Code[Region], in: Code[InputBuffer] ): Code[Long] = in.readLong() - def _buildSkip(mb: EmitMethodBuilder, r: Code[Region], in: Code[InputBuffer]): Code[Unit] = in.skipLong() + def _buildSkip(mb: MethodBuilder, r: Code[Region], in: Code[InputBuffer]): Code[Unit] = in.skipLong() override def _compatible(pt: PType): Boolean = pt.isInstanceOf[PInt64] diff --git a/hail/src/main/scala/is/hail/expr/types/encoded/EType.scala b/hail/src/main/scala/is/hail/expr/types/encoded/EType.scala index 8eddb62f29b..9da1a5db790 100644 --- a/hail/src/main/scala/is/hail/expr/types/encoded/EType.scala +++ b/hail/src/main/scala/is/hail/expr/types/encoded/EType.scala @@ -4,7 +4,7 @@ import java.util.Map.Entry import is.hail.annotations.{Region, StagedRegionValueBuilder} import is.hail.asm4s._ -import is.hail.expr.ir.{EmitFunctionBuilder, EmitMethodBuilder, IRParser, PunctuationToken, TokenIterator, typeToTypeInfo} +import is.hail.expr.ir.{IRParser, PunctuationToken, TokenIterator, typeToTypeInfo} import is.hail.expr.types.physical._ import is.hail.expr.types.virtual.Type import is.hail.expr.types.{BaseType, Requiredness} @@ -26,11 +26,11 @@ abstract class EType extends BaseType with Serializable with Requiredness { type StagedDecoder[T] = (Code[Region], Code[InputBuffer]) => Code[T] type StagedInplaceDecoder = (Code[Region], Code[Long], Code[InputBuffer]) => Code[Unit] - final def buildEncoder(pt: PType, mb: EmitMethodBuilder): StagedEncoder = { + final def buildEncoder(pt: PType, mb: MethodBuilder): StagedEncoder = { buildEncoderMethod(pt, mb.fb).invoke(_, _) } - final def buildEncoderMethod(pt: PType, fb: EmitFunctionBuilder[_]): EmitMethodBuilder = { + final def buildEncoderMethod(pt: PType, fb: FunctionBuilder[_]): MethodBuilder = { if (!encodeCompatible(pt)) throw new RuntimeException(s"encode incompatible:\n PT: $pt\n ET: ${ parsableString() }") val ptti = typeToTypeInfo(pt) @@ -45,11 +45,11 @@ abstract class EType extends BaseType with Serializable with Requiredness { } } - final def buildDecoder[T](pt: PType, mb: EmitMethodBuilder): StagedDecoder[T] = { + final def buildDecoder[T](pt: PType, mb: MethodBuilder): StagedDecoder[T] = { buildDecoderMethod(pt, mb.fb).invoke(_, _) } - final def buildDecoderMethod[T](pt: PType, fb: EmitFunctionBuilder[_]): EmitMethodBuilder = { + final def buildDecoderMethod[T](pt: PType, fb: FunctionBuilder[_]): MethodBuilder = { if (!decodeCompatible(pt)) throw new RuntimeException(s"decode incompatible:\n PT: $pt }\n ET: ${ parsableString() }") fb.getOrDefineMethod(s"DECODE_${ asIdent }_TO_${ pt.asIdent }", @@ -64,11 +64,11 @@ abstract class EType extends BaseType with Serializable with Requiredness { } } - final def buildInplaceDecoder(pt: PType, mb: EmitMethodBuilder): StagedInplaceDecoder = { + final def buildInplaceDecoder(pt: PType, mb: MethodBuilder): StagedInplaceDecoder = { buildInplaceDecoderMethod(pt, mb.fb).invoke(_, _, _) } - final def buildInplaceDecoderMethod(pt: PType, fb: EmitFunctionBuilder[_]): EmitMethodBuilder = { + final def buildInplaceDecoderMethod(pt: PType, fb: FunctionBuilder[_]): MethodBuilder = { if (!decodeCompatible(pt)) throw new RuntimeException(s"decode incompatible:\n PT: $pt\n ET: ${ parsableString() }") fb.getOrDefineMethod(s"INPLACE_DECODE_${ asIdent }_TO_${ pt.asIdent }", @@ -84,7 +84,7 @@ abstract class EType extends BaseType with Serializable with Requiredness { }) } - final def buildSkip(mb: EmitMethodBuilder): (Code[Region], Code[InputBuffer]) => Code[Unit] = { + final def buildSkip(mb: MethodBuilder): (Code[Region], Code[InputBuffer]) => Code[Unit] = { mb.fb.getOrDefineMethod(s"SKIP_${ asIdent }", (this, "SKIP"), Array[TypeInfo[_]](classInfo[Region], classInfo[InputBuffer]), @@ -97,13 +97,13 @@ abstract class EType extends BaseType with Serializable with Requiredness { }).invoke(_, _) } - def _buildEncoder(pt: PType, mb: EmitMethodBuilder, v: Code[_], out: Code[OutputBuffer]): Code[Unit] + def _buildEncoder(pt: PType, mb: MethodBuilder, v: Code[_], out: Code[OutputBuffer]): Code[Unit] - def _buildDecoder(pt: PType, mb: EmitMethodBuilder, region: Code[Region], in: Code[InputBuffer]): Code[_] + def _buildDecoder(pt: PType, mb: MethodBuilder, region: Code[Region], in: Code[InputBuffer]): Code[_] def _buildInplaceDecoder( pt: PType, - mb: EmitMethodBuilder, + mb: MethodBuilder, region: Code[Region], addr: Code[Long], in: Code[InputBuffer] @@ -113,7 +113,7 @@ abstract class EType extends BaseType with Serializable with Requiredness { Region.storeIRIntermediate(pt)(addr, decoded) } - def _buildSkip(mb: EmitMethodBuilder, r: Code[Region], in: Code[InputBuffer]): Code[Unit] + def _buildSkip(mb: MethodBuilder, r: Code[Region], in: Code[InputBuffer]): Code[Unit] def _compatible(pt: PType): Boolean = fatal("EType subclasses must override either `_compatible` or both `_encodeCompatible` and `_decodeCompatible`") @@ -183,7 +183,7 @@ object EType { encoderCacheMisses += 1 log.info(s"encoder cache miss ($encoderCacheHits hits, $encoderCacheMisses misses, " + s"${ formatDouble(encoderCacheHits.toDouble / (encoderCacheHits + encoderCacheMisses), 3) })") - val fb = new EmitFunctionBuilder[EncoderAsmFunction]( + val fb = new FunctionBuilder[EncoderAsmFunction]( Array(NotGenericTypeInfo[Long], NotGenericTypeInfo[OutputBuffer]), NotGenericTypeInfo[Unit], namePrefix = "etypeEncode") @@ -217,7 +217,7 @@ object EType { decoderCacheMisses += 1 log.info(s"decoder cache miss ($decoderCacheHits hits, $decoderCacheMisses misses, " + s"${ formatDouble(decoderCacheHits.toDouble / (decoderCacheHits + decoderCacheMisses), 3) }") - val fb = new EmitFunctionBuilder[DecoderAsmFunction]( + val fb = new FunctionBuilder[DecoderAsmFunction]( Array(NotGenericTypeInfo[Region], NotGenericTypeInfo[InputBuffer]), NotGenericTypeInfo[Long], namePrefix = "etypeDecode") diff --git a/hail/src/main/scala/is/hail/expr/types/physical/ComplexPType.scala b/hail/src/main/scala/is/hail/expr/types/physical/ComplexPType.scala index 42553f082cf..82c39ef8159 100644 --- a/hail/src/main/scala/is/hail/expr/types/physical/ComplexPType.scala +++ b/hail/src/main/scala/is/hail/expr/types/physical/ComplexPType.scala @@ -16,13 +16,6 @@ abstract class ComplexPType extends PType { override def containsPointers: Boolean = representation.containsPointers - def storeShallowAtOffset(dstAddress: Code[Long], valueAddress: Code[Long]): Code[Unit] = - this.representation.storeShallowAtOffset(dstAddress, valueAddress) - - def storeShallowAtOffset(dstAddress: Long, valueAddress: Long) { - this.representation.storeShallowAtOffset(dstAddress, valueAddress) - } - def copyFromType(mb: MethodBuilder, region: Code[Region], srcPType: PType, srcAddress: Code[Long], forceDeep: Boolean): Code[Long] = { assert(this isOfType srcPType) @@ -41,4 +34,13 @@ abstract class ComplexPType extends PType { this.representation.copyFromType(region, srcRepPType, srcAddress, forceDeep) } -} + + def constructAtAddress(mb: MethodBuilder, addr: Code[Long], region: Code[Region], srcPType: PType, srcAddress: Code[Long], forceDeep: Boolean): Code[Unit] = + this.representation.constructAtAddress(mb, addr, region, srcPType.fundamentalType, srcAddress, forceDeep) + + def constructAtAddress(addr: Long, region: Region, srcPType: PType, srcAddress: Long, forceDeep: Boolean): Unit = + this.representation.constructAtAddress(addr, region, srcPType.fundamentalType, srcAddress, forceDeep) + + override def constructAtAddressFromValue(mb: MethodBuilder, addr: Code[Long], region: Code[Region], srcPType: PType, src: Code[_], forceDeep: Boolean): Code[Unit] = + this.representation.constructAtAddressFromValue(mb, addr, region, srcPType.fundamentalType, src, forceDeep) +} \ No newline at end of file diff --git a/hail/src/main/scala/is/hail/expr/types/physical/PArray.scala b/hail/src/main/scala/is/hail/expr/types/physical/PArray.scala index 3dc1e449b14..b81136cbf9d 100644 --- a/hail/src/main/scala/is/hail/expr/types/physical/PArray.scala +++ b/hail/src/main/scala/is/hail/expr/types/physical/PArray.scala @@ -17,8 +17,6 @@ trait PArrayIterator { abstract class PArray extends PContainer with PStreamable { lazy val virtualType: TArray = TArray(elementType.virtualType, required) - def copy(elementType: PType = this.elementType, required: Boolean = this.required): PArray - def codeOrdering(mb: EmitMethodBuilder, other: PType): CodeOrdering = { assert(this isOfType other) CodeOrdering.iterableOrdering(this, other.asInstanceOf[PArray], mb) diff --git a/hail/src/main/scala/is/hail/expr/types/physical/PArrayBackedContainer.scala b/hail/src/main/scala/is/hail/expr/types/physical/PArrayBackedContainer.scala index d54f319425e..4fd27c6e123 100644 --- a/hail/src/main/scala/is/hail/expr/types/physical/PArrayBackedContainer.scala +++ b/hail/src/main/scala/is/hail/expr/types/physical/PArrayBackedContainer.scala @@ -150,16 +150,15 @@ trait PArrayBackedContainer extends PContainer { def copyFromType(region: Region, srcPType: PType, srcAddress: Long, forceDeep: Boolean): Long = this.arrayRep.copyFromType(region, srcPType.asInstanceOf[PArrayBackedContainer].arrayRep, srcAddress, forceDeep) - def storeShallowAtOffset(dstAddress: Code[Long], valueAddress: Code[Long]): Code[Unit] = - this.arrayRep.storeShallowAtOffset(dstAddress, valueAddress) - - def storeShallowAtOffset(dstAddress: Long, valueAddress: Long) { - this.arrayRep.storeShallowAtOffset(dstAddress, valueAddress) - } - def nextElementAddress(currentOffset: Long) = arrayRep.nextElementAddress(currentOffset) def nextElementAddress(currentOffset: Code[Long]) = arrayRep.nextElementAddress(currentOffset) + + def constructAtAddress(mb: MethodBuilder, addr: Code[Long], region: Code[Region], srcPType: PType, srcAddress: Code[Long], forceDeep: Boolean): Code[Unit] = + arrayRep.constructAtAddress(mb, addr, region, srcPType.asInstanceOf[PArrayBackedContainer].arrayRep, srcAddress, forceDeep) + + def constructAtAddress(addr: Long, region: Region, srcPType: PType, srcAddress: Long, forceDeep: Boolean): Unit = + arrayRep.constructAtAddress(addr, region, srcPType.asInstanceOf[PArrayBackedContainer].arrayRep, srcAddress, forceDeep) } diff --git a/hail/src/main/scala/is/hail/expr/types/physical/PBaseStruct.scala b/hail/src/main/scala/is/hail/expr/types/physical/PBaseStruct.scala index b2b50bff9a1..7453cedf133 100644 --- a/hail/src/main/scala/is/hail/expr/types/physical/PBaseStruct.scala +++ b/hail/src/main/scala/is/hail/expr/types/physical/PBaseStruct.scala @@ -138,283 +138,40 @@ abstract class PBaseStruct extends PType { def nMissing: Int - def nMissingBytes: Int - def missingIdx: Array[Int] - def byteOffsets: Array[Long] - - def allocate(region: Region): Long = { - region.allocate(alignment, byteSize) - } - - def allocate(region: Code[Region]): Code[Long] = - region.allocate(alignment, byteSize) - - def copyFrom(region: Region, srcAddress: Long): Long = { - val destAddress = this.allocate(region) - this.storeShallowAtOffset(destAddress, srcAddress) - destAddress - } - - def copyFrom(mb: MethodBuilder, region: Code[Region], srcAddress: Code[Long]): Code[Long] = { - val destAddress = mb.newField[Long] - Code( - destAddress := this.allocate(region), - this.storeShallowAtOffset(destAddress, srcAddress), - destAddress - ) - } - - def storeShallowAtOffset(destAddress: Code[Long], srcAddress: Code[Long]): Code[Unit] = - Region.copyFrom(srcAddress, destAddress, this.byteSize) - - def storeShallowAtOffset(destAddress: Long, srcAddress: Long) { - Region.copyFrom(srcAddress, destAddress, this.byteSize) - } - - def initialize(structAddress: Long, setMissing: Boolean = false): Unit = { - if (allFieldsRequired) { - return - } + def allocate(region: Region): Long - Region.setMemory(structAddress, nMissingBytes.toLong, if (setMissing) 0xFF.toByte else 0.toByte) - } + def allocate(region: Code[Region]): Code[Long] - def stagedInitialize(structAddress: Code[Long], setMissing: Boolean = false): Code[Unit] = { - if (allFieldsRequired) { - return Code._empty - } + def initialize(structAddress: Long, setMissing: Boolean = false): Unit - Region.setMemory(structAddress, const(nMissingBytes.toLong), const(if (setMissing) 0xFF.toByte else 0.toByte)) - } + def stagedInitialize(structAddress: Code[Long], setMissing: Boolean = false): Code[Unit] - def isFieldDefined(offset: Long, fieldIdx: Int): Boolean = - fieldRequired(fieldIdx) || !Region.loadBit(offset, missingIdx(fieldIdx)) + def isFieldDefined(offset: Long, fieldIdx: Int): Boolean def isFieldMissing(off: Long, fieldIdx: Int): Boolean = !isFieldDefined(off, fieldIdx) - def isFieldMissing(offset: Code[Long], fieldIdx: Int): Code[Boolean] = - if (fieldRequired(fieldIdx)) - false - else - Region.loadBit(offset, missingIdx(fieldIdx).toLong) + def isFieldMissing(offset: Code[Long], fieldIdx: Int): Code[Boolean] def isFieldDefined(offset: Code[Long], fieldIdx: Int): Code[Boolean] = !isFieldMissing(offset, fieldIdx) - def setFieldMissing(offset: Long, fieldIdx: Int) { - assert(!fieldRequired(fieldIdx)) - Region.setBit(offset, missingIdx(fieldIdx)) - } - - def setFieldMissing(offset: Code[Long], fieldIdx: Int): Code[Unit] = { - assert(!fieldRequired(fieldIdx)) - Region.setBit(offset, missingIdx(fieldIdx).toLong) - } - - def setFieldPresent(offset: Long, fieldIdx: Int) { - assert(!fieldRequired(fieldIdx)) - Region.clearBit(offset, missingIdx(fieldIdx)) - } - - def setFieldPresent(offset: Code[Long], fieldIdx: Int): Code[Unit] = { - assert(!fieldRequired(fieldIdx)) - Region.clearBit(offset, missingIdx(fieldIdx).toLong) - } - - def fieldOffset(structAddress: Long, fieldIdx: Int): Long = - structAddress + byteOffsets(fieldIdx) + def setFieldMissing(offset: Long, fieldIdx: Int): Unit - def fieldOffset(structAddress: Code[Long], fieldIdx: Int): Code[Long] = - structAddress + byteOffsets(fieldIdx) + def setFieldMissing(offset: Code[Long], fieldIdx: Int): Code[Unit] - def loadField(offset: Long, fieldIdx: Int): Long = { - val off = fieldOffset(offset, fieldIdx) - types(fieldIdx).fundamentalType match { - case _: PArray | _: PBinary => Region.loadAddress(off) - case _ => off - } - } + def setFieldPresent(offset: Long, fieldIdx: Int): Unit - def loadField(offset: Code[Long], fieldIdx: Int): Code[Long] = - loadField(fieldOffset(offset, fieldIdx), types(fieldIdx)) + def setFieldPresent(offset: Code[Long], fieldIdx: Int): Code[Unit] - private def loadField(fieldOffset: Code[Long], fieldType: PType): Code[Long] = { - fieldType.fundamentalType match { - case _: PArray | _: PBinary => Region.loadAddress(fieldOffset) - case _ => fieldOffset - } - } + def fieldOffset(structAddress: Long, fieldIdx: Int): Long - def deepCopyFromAddress(mb: MethodBuilder, region: Code[Region], srcStructAddress: Code[Long]): Code[Long] = { - val dstAddress = mb.newField[Long] - Code( - dstAddress := this.copyFrom(mb, region, srcStructAddress), - this.deepPointerCopy(mb, region, dstAddress), - dstAddress - ) - } + def fieldOffset(structAddress: Code[Long], fieldIdx: Int): Code[Long] - def deepCopyFromAddress(region: Region, srcStructAddress: Long): Long = { - val dstAddress = this.copyFrom(region, srcStructAddress) - this.deepPointerCopy(region, dstAddress) - dstAddress - } + def loadField(offset: Long, fieldIdx: Int): Long - def deepPointerCopy(mb: MethodBuilder, region: Code[Region], dstStructAddress: Code[Long]): Code[Unit] = { - var c: Code[Unit] = Code._empty - - var i = 0 - while(i < this.size) { - val dstFieldType = this.fields(i).typ.fundamentalType - if(dstFieldType.containsPointers) { - val dstFieldAddress = mb.newField[Long] - c = Code( - c, - this.isFieldDefined(dstStructAddress, i).orEmpty( - Code( - dstFieldAddress := this.fieldOffset(dstStructAddress, i), - dstFieldType match { - case t@(_: PBinary | _: PArray) => - t.storeShallowAtOffset(dstFieldAddress, t.copyFromType(mb, region, dstFieldType, Region.loadAddress(dstFieldAddress))) - case t: PBaseStruct => - t.deepPointerCopy(mb, region, dstFieldAddress) - case t: PType => - fatal(s"Field type isn't supported ${t}") - } - ) - ) - ) - } - i += 1 - } - - c - } - - def deepPointerCopy(region: Region, dstStructAddress: Long) { - var i = 0 - while(i < this.size) { - val dstFieldType = this.fields(i).typ.fundamentalType - if(dstFieldType.containsPointers && this.isFieldDefined(dstStructAddress, i)) { - val dstFieldAddress = this.fieldOffset(dstStructAddress, i) - dstFieldType match { - case t@(_: PBinary | _: PArray) => - t.storeShallowAtOffset(dstFieldAddress, t.copyFromType(region, dstFieldType, Region.loadAddress(dstFieldAddress))) - case t: PBaseStruct => - t.deepPointerCopy(region, dstFieldAddress) - case t: PType => - fatal(s"Field type isn't supported ${t}") - } - } - i += 1 - } - } - - def copyFromType(mb: MethodBuilder, region: Code[Region], srcPType: PType, srcStructAddress: Code[Long], forceDeep: Boolean): Code[Long] = { - val sourceType = srcPType.asInstanceOf[PBaseStruct] - - assert(sourceType.size == this.size) - - if(this.fields.map(_.typ.fundamentalType) == sourceType.fields.map(_.typ.fundamentalType)) { - if(!forceDeep) { - return srcStructAddress - } - - return this.deepCopyFromAddress(mb, region, srcStructAddress) - } - - val dstStructAddress = mb.newField[Long] - var loop: Code[_] = Code() - var i = 0 - while(i < this.size) { - val dstField = this.fields(i) - val srcField = sourceType.fields(i) - - assert((dstField.typ.required <= srcField.typ.required) && (dstField.typ isOfType srcField.typ) && (dstField.name == srcField.name) && (dstField.index == srcField.index)) - - val srcFieldType = srcField.typ.fundamentalType - val dstFieldType = dstField.typ.fundamentalType - - val body = dstFieldType.storeShallowAtOffset( - this.fieldOffset(dstStructAddress, dstField.index), - dstFieldType.copyFromType( - mb, - region, - srcFieldType, - sourceType.loadField(srcStructAddress, srcField.index), - forceDeep - ) - ) - - if(!srcFieldType.required) { - loop = Code(loop, sourceType.isFieldMissing(srcStructAddress, srcField.index).mux( - this.setFieldMissing(dstStructAddress, dstField.index), - body - )) - } else { - loop = Code(loop, body) - } - - i+=1 - } - - Code( - dstStructAddress := this.allocate(region), - this.stagedInitialize(dstStructAddress), - loop, - dstStructAddress - ) - } - - def copyFromTypeAndStackValue(mb: MethodBuilder, region: Code[Region], srcPType: PType, stackValue: Code[_], forceDeep: Boolean): Code[_] = - this.copyFromType(mb, region, srcPType, stackValue.asInstanceOf[Code[Long]], forceDeep) - - def copyFromType(region: Region, srcPType: PType, srcStructAddress: Long, forceDeep: Boolean): Long = { - val sourceType = srcPType.asInstanceOf[PBaseStruct] - if(this.fields.map(_.typ.fundamentalType) == sourceType.fields.map(_.typ.fundamentalType)) { - if(!forceDeep) { - return srcStructAddress - } - - return this.deepCopyFromAddress(region, srcStructAddress) - } - - assert(sourceType.size == this.size) - - val dstStructAddress = this.allocate(region) - this.initialize(dstStructAddress) - - var i = 0 - while(i < this.size) { - val dstField = this.fields(i) - val srcField = sourceType.fields(i) - - assert((dstField.typ.required <= srcField.typ.required) && (dstField.typ isOfType srcField.typ) && (dstField.name == srcField.name) && (dstField.index == srcField.index)) - - val srcType = srcField.typ.fundamentalType - val dstType = dstField.typ.fundamentalType - - if(!srcType.required && sourceType.isFieldMissing(srcStructAddress, srcField.index)) { - this.setFieldMissing(dstStructAddress, dstField.index) - } else { - dstType.storeShallowAtOffset( - this.fieldOffset(dstStructAddress, dstField.index), - dstType.copyFromType( - region, - srcType, - sourceType.loadField(srcStructAddress, srcField.index), - forceDeep - ) - ) - } - - i+=1 - } - - dstStructAddress - } + def loadField(offset: Code[Long], fieldIdx: Int): Code[Long] override def containsPointers: Boolean = types.exists(_.containsPointers) } diff --git a/hail/src/main/scala/is/hail/expr/types/physical/PBoolean.scala b/hail/src/main/scala/is/hail/expr/types/physical/PBoolean.scala index fad46652367..f0040325249 100644 --- a/hail/src/main/scala/is/hail/expr/types/physical/PBoolean.scala +++ b/hail/src/main/scala/is/hail/expr/types/physical/PBoolean.scala @@ -1,7 +1,7 @@ package is.hail.expr.types.physical import is.hail.annotations.{Region, UnsafeOrdering, _} -import is.hail.asm4s.{Code, MethodBuilder} +import is.hail.asm4s.{Code, _} import is.hail.expr.ir.EmitMethodBuilder import is.hail.expr.types.virtual.TBoolean @@ -32,6 +32,9 @@ class PBoolean(override val required: Boolean) extends PType with PPrimitive { } override def byteSize: Long = 1 + + def storePrimitiveAtAddress(addr: Code[Long], srcPType: PType, value: Code[_]): Code[Unit] = + Region.storeBoolean(addr, coerce[Boolean](value)) } object PBoolean { diff --git a/hail/src/main/scala/is/hail/expr/types/physical/PCall.scala b/hail/src/main/scala/is/hail/expr/types/physical/PCall.scala index e01c28b45d5..c3215742d2f 100644 --- a/hail/src/main/scala/is/hail/expr/types/physical/PCall.scala +++ b/hail/src/main/scala/is/hail/expr/types/physical/PCall.scala @@ -18,8 +18,6 @@ abstract class PCall extends ComplexPType { PInt32().codeOrdering(mb) } - def copy(required: Boolean): PCall - def ploidy(c: Code[Int]): Code[Int] def isPhased(c: Code[Int]): Code[Boolean] diff --git a/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalArray.scala b/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalArray.scala index 2392cb40efb..b6ff905463b 100644 --- a/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalArray.scala +++ b/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalArray.scala @@ -35,7 +35,7 @@ final case class PCanonicalArray(elementType: PType, required: Boolean = false) } } - def copy(elementType: PType = this.elementType, required: Boolean = this.required): PCanonicalArray = PCanonicalArray(elementType, required) + def setRequired(required: Boolean) = if(required == this.required) this else PCanonicalArray(elementType, required) def loadLength(aoff: Long): Int = Region.loadInt(aoff) @@ -353,28 +353,6 @@ final case class PCanonicalArray(elementType: PType, required: Boolean = false) ) } - def storeShallowAtOffset(dstAddress: Code[Long], valueAddress: Code[Long]): Code[Unit] = - Region.storeAddress(dstAddress, valueAddress) - - def storeShallowAtOffset(dstAddress: Long, valueAddress: Long) { - Region.storeAddress(dstAddress, valueAddress) - } - - def deepCopyFromAddress(mb: MethodBuilder, region: Code[Region], srcArrayAddress: Code[Long]): Code[Long] = { - val dstAddress = mb.newField[Long] - Code( - dstAddress := this.copyFrom(mb, region, srcArrayAddress), - this.deepPointerCopy(mb, region, dstAddress), - dstAddress - ) - } - - def deepCopyFromAddress(region: Region, srcArrayAddress: Long): Long = { - val dstAddress = this.copyFrom(region, srcArrayAddress) - this.deepPointerCopy(region, dstAddress) - dstAddress - } - def deepPointerCopy(mb: MethodBuilder, region: Code[Region], dstAddress: Code[Long]): Code[Unit] = { if(!this.elementType.fundamentalType.containsPointers) { return Code._empty @@ -392,8 +370,8 @@ final case class PCanonicalArray(elementType: PType, required: Boolean = false) currentElementAddress := this.elementOffset(dstAddress, numberOfElements, currentIdx), this.elementType.fundamentalType match { case t@(_: PBinary | _: PArray) => - t.storeShallowAtOffset(currentElementAddress, t.copyFromType(mb, region, t, Region.loadAddress(currentElementAddress))) - case t: PBaseStruct => + Region.storeAddress(currentElementAddress, t.copyFromType(mb, region, t, Region.loadAddress(currentElementAddress), forceDeep = true)) + case t: PCanonicalBaseStruct => t.deepPointerCopy(mb, region, currentElementAddress) case t: PType => fatal(s"Type isn't supported ${t}") } @@ -416,8 +394,8 @@ final case class PCanonicalArray(elementType: PType, required: Boolean = false) val currentElementAddress = this.elementOffset(dstAddress, numberOfElements, currentIdx) this.elementType.fundamentalType match { case t@(_: PBinary | _: PArray) => - t.storeShallowAtOffset(currentElementAddress, t.copyFromType(region, t, Region.loadAddress(currentElementAddress))) - case t: PBaseStruct => + Region.storeAddress(currentElementAddress, t.copyFromType(region, t, Region.loadAddress(currentElementAddress), forceDeep = true)) + case t: PCanonicalBaseStruct => t.deepPointerCopy(region, currentElementAddress) case t: PType => fatal(s"Type isn't supported ${t}") } @@ -429,108 +407,105 @@ final case class PCanonicalArray(elementType: PType, required: Boolean = false) def copyFromType(mb: MethodBuilder, region: Code[Region], srcPType: PType, srcAddress: Code[Long], forceDeep: Boolean): Code[Long] = { val sourceType = srcPType.asInstanceOf[PArray] - val sourceElementType = sourceType.elementType.fundamentalType - val destElementType = this.elementType.fundamentalType - - if (sourceElementType != destElementType) { - assert(destElementType.required <= sourceElementType.required && sourceElementType.isOfType(destElementType)) - } else { - if(!forceDeep) { - return srcAddress - } - - return this.deepCopyFromAddress(mb, region, srcAddress) - } + val srcAddrRef = mb.newLocal[Long] + val len = mb.newLocal[Int] - val dstAddress = mb.newField[Long] - val numberOfElements = mb.newLocal[Int] - val currentElementAddress = mb.newLocal[Long] - val currentIdx = mb.newLocal[Int] + Code( + srcAddrRef := srcAddress, + len := sourceType.loadLength(srcAddrRef), + constructOrCopyWithLen(mb, region, sourceType, srcAddrRef, len, forceDeep)) + } - val init = Code( - numberOfElements := sourceType.loadLength(srcAddress), - dstAddress := this.allocate(region, numberOfElements), - this.stagedInitialize(dstAddress, numberOfElements), - currentElementAddress := this.firstElementOffset(dstAddress, numberOfElements), - currentIdx := const(0) - ) + def copyFromType(region: Region, srcPType: PType, srcAddress: Long, forceDeep: Boolean): Long = { + val sourceType = srcPType.asInstanceOf[PArray] + val len = sourceType.loadLength(srcAddress) + constructOrCopyWithLen(region, sourceType, srcAddress, len, forceDeep) + } - var loop: Code[Unit] = - destElementType.storeShallowAtOffset( - currentElementAddress, - destElementType.copyFromType( - mb, - region, - sourceElementType, - sourceType.loadElement(srcAddress, numberOfElements, currentIdx), - forceDeep - ) - ) + def copyFromTypeAndStackValue(mb: MethodBuilder, region: Code[Region], srcPType: PType, stackValue: Code[_], forceDeep: Boolean): Code[_] = + this.copyFromType(mb, region, srcPType, stackValue.asInstanceOf[Code[Long]], forceDeep) - if(!sourceElementType.required) { - loop = sourceType.isElementMissing(srcAddress, currentIdx).mux( - this.setElementMissing(dstAddress, currentIdx), - loop - ) - } + def constructAtAddress(mb: MethodBuilder, addr: Code[Long], region: Code[Region], srcPType: PType, srcAddress: Code[Long], forceDeep: Boolean): Code[Unit] = { + val srcArray = srcPType.asInstanceOf[PArray] + val srcAddrVar = mb.newLocal[Long] + val len = mb.newLocal[Int] Code( - init, - Code.whileLoop(currentIdx < numberOfElements, - loop, - currentElementAddress := this.nextElementAddress(currentElementAddress), - currentIdx := currentIdx + const(1) - ), - dstAddress + srcAddrVar := srcAddress, + len := srcArray.loadLength(srcAddrVar), + Region.storeAddress(addr, constructOrCopyWithLen(mb, region, srcArray, srcAddrVar, len, forceDeep)) ) } - def copyFromType(region: Region, srcPType: PType, srcAddress: Long, forceDeep: Boolean): Long = { - val sourceType = srcPType.asInstanceOf[PArray] - val sourceElementType = sourceType.elementType.fundamentalType - val destElementType = this.elementType.fundamentalType + private def constructOrCopyWithLen(mb: MethodBuilder, region: Code[Region], srcArray: PArray, srcAddress: LocalRef[Long], len: LocalRef[Int], forceDeep: Boolean): Code[Long] = { + if (srcArray == this) { + if (forceDeep) { + val newAddr = mb.newLocal[Long] + Code( + newAddr := allocate(region, len), + Region.copyFrom(srcAddress, newAddr, contentsByteSize(len)), + deepPointerCopy(mb, region, newAddr), + newAddr) + } else + srcAddress + } else { - if (sourceElementType == destElementType) { - if(!forceDeep) { - return srcAddress - } + assert(elementType.required <= srcArray.elementType.required) + + val newAddr = mb.newLocal[Long] + val i = mb.newLocal[Int] - return this.deepCopyFromAddress(region, srcAddress) + Code( + newAddr := allocate(region, len), + stagedInitialize(newAddr, len, setMissing = true), + i := 0, + Code.whileLoop(i < len, + srcArray.isElementDefined(srcAddress, i).orEmpty( + Code( + setElementPresent(newAddr, i), + elementType.constructAtAddress(mb, elementOffset(newAddr, len, i), region, srcArray.elementType, srcArray.loadElement(srcAddress, len, i), forceDeep) + ) + ), + i := i + 1 + ), + newAddr + ) } + } - assert(destElementType.required <= sourceElementType.required && sourceElementType.isOfType(destElementType)) + def constructAtAddress(addr: Long, region: Region, srcPType: PType, srcAddress: Long, forceDeep: Boolean): Unit = { + val srcArray = srcPType.asInstanceOf[PArray] + Region.storeAddress(addr, constructOrCopyWithLen(region, srcArray, srcAddress, srcArray.loadLength(srcAddress), forceDeep)) + } - val numberOfElements = sourceType.loadLength(srcAddress) - val dstAddress = this.allocate(region, numberOfElements) - this.initialize(dstAddress, numberOfElements) + private def constructOrCopyWithLen(region: Region, srcArray: PArray, srcAddress: Long, len: Int, forceDeep: Boolean): Long = { + if (srcArray == this) { + if (forceDeep) { + val newAddr = allocate(region, len) + Region.copyFrom(srcAddress, newAddr, contentsByteSize(len)) + deepPointerCopy(region, newAddr) + newAddr + } else + srcAddress + } else { + val newAddr = allocate(region, len) - var currentElementAddress = this.firstElementOffset(dstAddress, numberOfElements) - var currentIdx = 0 - while(currentIdx < numberOfElements) { - if(!sourceElementType.required && sourceType.isElementMissing(srcAddress, currentIdx)) { - this.setElementMissing(dstAddress, currentIdx) - } else { - destElementType.storeShallowAtOffset( - currentElementAddress, - destElementType.copyFromType( - region, - sourceElementType, - sourceType.loadElement(srcAddress, numberOfElements, currentIdx), - forceDeep - ) - ) - } + assert(elementType.required <= srcArray.elementType.required) - currentElementAddress = this.nextElementAddress(currentElementAddress) - currentIdx += 1 + initialize(newAddr, len, setMissing = true) + var i = 0 + val srcElement = srcArray.elementType + while (i < len) { + if (srcArray.isElementDefined(srcAddress, i)) { + setElementPresent(newAddr, i) + elementType.constructAtAddress(elementOffset(newAddr, len, i), region, srcElement, srcArray.loadElement(srcAddress, len, i), forceDeep) + } + i += 1 + } + newAddr } - - dstAddress } - def copyFromTypeAndStackValue(mb: MethodBuilder, region: Code[Region], srcPType: PType, stackValue: Code[_], forceDeep: Boolean): Code[_] = - this.copyFromType(mb, region, srcPType, stackValue.asInstanceOf[Code[Long]], forceDeep) - override def deepRename(t: Type): PType = deepRenameArray(t.asInstanceOf[TArray]) private def deepRenameArray(t: TArray): PArray = diff --git a/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalBaseStruct.scala b/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalBaseStruct.scala new file mode 100644 index 00000000000..1a1b268963a --- /dev/null +++ b/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalBaseStruct.scala @@ -0,0 +1,223 @@ +package is.hail.expr.types.physical + +import is.hail.annotations.{Region, UnsafeUtils} +import is.hail.asm4s._ +import is.hail.expr.types.BaseStruct +import is.hail.utils._ + +abstract class PCanonicalBaseStruct(val types: Array[PType]) extends PBaseStruct { + val (missingIdx: Array[Int], nMissing: Int) = BaseStruct.getMissingIndexAndCount(types.map(_.required)) + val nMissingBytes: Int = UnsafeUtils.packBitsToBytes(nMissing) + val byteOffsets: Array[Long] = new Array[Long](size) + override val byteSize: Long = PBaseStruct.getByteSizeAndOffsets(types, nMissingBytes, byteOffsets) + override val alignment: Long = PBaseStruct.alignment(types) + + + def allocate(region: Region): Long = { + region.allocate(alignment, byteSize) + } + + def allocate(region: Code[Region]): Code[Long] = + region.allocate(alignment, byteSize) + + def initialize(structAddress: Long, setMissing: Boolean = false): Unit = { + if (allFieldsRequired) { + return + } + + Region.setMemory(structAddress, nMissingBytes.toLong, if (setMissing) 0xFF.toByte else 0.toByte) + } + + def stagedInitialize(structAddress: Code[Long], setMissing: Boolean = false): Code[Unit] = { + if (allFieldsRequired) { + return Code._empty + } + + Region.setMemory(structAddress, const(nMissingBytes.toLong), const(if (setMissing) 0xFF.toByte else 0.toByte)) + } + + def isFieldDefined(offset: Long, fieldIdx: Int): Boolean = + fieldRequired(fieldIdx) || !Region.loadBit(offset, missingIdx(fieldIdx)) + + def isFieldMissing(offset: Code[Long], fieldIdx: Int): Code[Boolean] = + if (fieldRequired(fieldIdx)) + false + else + Region.loadBit(offset, missingIdx(fieldIdx).toLong) + + def setFieldMissing(offset: Long, fieldIdx: Int) { + assert(!fieldRequired(fieldIdx)) + Region.setBit(offset, missingIdx(fieldIdx)) + } + + def setFieldMissing(offset: Code[Long], fieldIdx: Int): Code[Unit] = { + assert(!fieldRequired(fieldIdx)) + Region.setBit(offset, missingIdx(fieldIdx).toLong) + } + + def setFieldPresent(offset: Long, fieldIdx: Int) { + if (!fieldRequired(fieldIdx)) + Region.clearBit(offset, missingIdx(fieldIdx)) + } + + def setFieldPresent(offset: Code[Long], fieldIdx: Int): Code[Unit] = { + if (!fieldRequired(fieldIdx)) + Region.clearBit(offset, missingIdx(fieldIdx).toLong) + else + Code._empty + } + + def fieldOffset(structAddress: Long, fieldIdx: Int): Long = + structAddress + byteOffsets(fieldIdx) + + def fieldOffset(structAddress: Code[Long], fieldIdx: Int): Code[Long] = + structAddress + byteOffsets(fieldIdx) + + def loadField(offset: Long, fieldIdx: Int): Long = { + val off = fieldOffset(offset, fieldIdx) + types(fieldIdx).fundamentalType match { + case _: PArray | _: PBinary => Region.loadAddress(off) + case _ => off + } + } + + def loadField(offset: Code[Long], fieldIdx: Int): Code[Long] = + loadField(fieldOffset(offset, fieldIdx), types(fieldIdx)) + + private def loadField(fieldOffset: Code[Long], fieldType: PType): Code[Long] = { + fieldType.fundamentalType match { + case _: PArray | _: PBinary => Region.loadAddress(fieldOffset) + case _ => fieldOffset + } + } + + def deepPointerCopy(mb: MethodBuilder, region: Code[Region], dstStructAddress: Code[Long]): Code[Unit] = { + var c: Code[Unit] = Code._empty + + var i = 0 + while(i < this.size) { + val dstFieldType = this.fields(i).typ.fundamentalType + if(dstFieldType.containsPointers) { + val dstFieldAddress = mb.newField[Long] + c = Code( + c, + this.isFieldDefined(dstStructAddress, i).orEmpty( + Code( + dstFieldAddress := this.fieldOffset(dstStructAddress, i), + dstFieldType match { + case t@(_: PBinary | _: PArray) => + Region.storeAddress(dstFieldAddress, t.copyFromType(mb, region, dstFieldType, Region.loadAddress(dstFieldAddress), forceDeep = true)) + case t: PCanonicalBaseStruct => + t.deepPointerCopy(mb, region, dstFieldAddress) + } + ) + ) + ) + } + i += 1 + } + + c + } + + def deepPointerCopy(region: Region, dstStructAddress: Long) { + var i = 0 + while(i < this.size) { + val dstFieldType = this.fields(i).typ.fundamentalType + if(dstFieldType.containsPointers && this.isFieldDefined(dstStructAddress, i)) { + val dstFieldAddress = this.fieldOffset(dstStructAddress, i) + dstFieldType match { + case t@(_: PBinary | _: PArray) => + Region.storeAddress(dstFieldAddress, t.copyFromType(region, dstFieldType, Region.loadAddress(dstFieldAddress), forceDeep = true)) + case t: PCanonicalBaseStruct => + t.deepPointerCopy(region, dstFieldAddress) + } + } + i += 1 + } + } + + def copyFromType(mb: MethodBuilder, region: Code[Region], srcPType: PType, srcStructAddress: Code[Long], forceDeep: Boolean): Code[Long] = { + val sourceType = srcPType.asInstanceOf[PBaseStruct] + assert(sourceType.size == this.size) + + if (this == sourceType && !forceDeep) + srcStructAddress + else { + val addr = mb.newLocal[Long] + Code( + addr := allocate(region), + constructAtAddress(mb, addr, region, sourceType, srcStructAddress, forceDeep), + addr + ) + } + } + + def copyFromTypeAndStackValue(mb: MethodBuilder, region: Code[Region], srcPType: PType, stackValue: Code[_], forceDeep: Boolean): Code[_] = + this.copyFromType(mb, region, srcPType, stackValue.asInstanceOf[Code[Long]], forceDeep) + + def copyFromType(region: Region, srcPType: PType, srcStructAddress: Long, forceDeep: Boolean): Long = { + val sourceType = srcPType.asInstanceOf[PBaseStruct] + if (this == sourceType && !forceDeep) + srcStructAddress + else { + val newAddr = allocate(region) + constructAtAddress(newAddr, region, sourceType, srcStructAddress, forceDeep) + newAddr + } + } + + def constructAtAddress(mb: MethodBuilder, addr: Code[Long], region: Code[Region], srcPType: PType, srcAddress: Code[Long], forceDeep: Boolean): Code[Unit] = { + val srcStruct = srcPType.asInstanceOf[PBaseStruct] + val addrVar = mb.newLocal[Long] + + if (srcStruct == this) { + var c: Code[Unit] = Code( + addrVar := addr, + Region.copyFrom(srcAddress, addrVar, byteSize)) + if (forceDeep) { + c = Code(c, deepPointerCopy(mb, region, addrVar)) + } + c + } else { + val srcAddrVar = mb.newLocal[Long] + Code( + srcAddrVar := srcAddress, + addrVar := addr, + stagedInitialize(addrVar, setMissing = true), + Code(fields.zip(srcStruct.fields).map { case (dest, src) => + assert(dest.typ.required <= src.typ.required) + val idx = dest.index + assert(idx == src.index) + srcStruct.isFieldDefined(srcAddrVar, idx).orEmpty(Code( + setFieldPresent(addrVar, idx), + dest.typ.constructAtAddress(mb, fieldOffset(addrVar, idx), region, src.typ, srcStruct.loadField(srcAddrVar, idx), forceDeep)) + ) + }: _* + ).asInstanceOf[Code[Unit]]) + } + } + + def constructAtAddress(addr: Long, region: Region, srcPType: PType, srcAddress: Long, forceDeep: Boolean): Unit = { + val srcStruct = srcPType.asInstanceOf[PBaseStruct] + if (srcStruct == this) { + Region.copyFrom(srcAddress, addr, byteSize) + if (forceDeep) + deepPointerCopy(region, addr) + } else { + initialize(addr, setMissing = true) + var idx = 0 + while (idx < types.length) { + val dest = types(idx) + val src = srcStruct.types(idx) + assert(dest.required <= src.required) + + if (srcStruct.isFieldDefined(srcAddress, idx)) { + setFieldPresent(addr, idx) + dest.constructAtAddress(fieldOffset(addr, idx), region, src, srcStruct.loadField(srcAddress, idx), forceDeep) + } + idx += 1 + } + } + } +} diff --git a/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalBinary.scala b/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalBinary.scala index 13907f0e8d1..3e8d2eb69c8 100644 --- a/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalBinary.scala +++ b/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalBinary.scala @@ -13,20 +13,13 @@ class PCanonicalBinary(val required: Boolean) extends PBinary { override def byteSize: Long = 8 def copyFromType(mb: MethodBuilder, region: Code[Region], srcPType: PType, srcAddress: Code[Long], forceDeep: Boolean): Code[Long] = { - if(this == srcPType && !forceDeep) { - return srcAddress - } - - assert(srcPType.isInstanceOf[PBinary]) - - val dstAddress = mb.newField[Long] - val length = mb.newLocal[Int] + val len = mb.newLocal[Int] + val srcVar = mb.newLocal[Long] Code( - length := this.loadLength(srcAddress), - dstAddress := this.allocate(region, length), - Region.copyFrom(srcAddress, dstAddress, this.contentByteSize(length)), - dstAddress + srcVar := srcAddress, + len := this.loadLength(srcVar), + constructOrCopyWithLen(mb, region, srcPType.asInstanceOf[PBinary], srcVar, len, forceDeep) ) } @@ -34,27 +27,12 @@ class PCanonicalBinary(val required: Boolean) extends PBinary { this.copyFromType(mb, region, srcPType, stackValue.asInstanceOf[Code[Long]], forceDeep) def copyFromType(region: Region, srcPType: PType, srcAddress: Long, forceDeep: Boolean): Long = { - if(this == srcPType && !forceDeep) { - return srcAddress - } - - assert(srcPType.isInstanceOf[PBinary]) - - val length = this.loadLength(srcAddress) - val dstAddress = this.allocate(region, length) - Region.copyFrom(srcAddress, dstAddress, this.contentByteSize(length)) - dstAddress + val srcBinary = srcPType.asInstanceOf[PBinary] + constructOrCopyWithLen(region, srcBinary, srcAddress, srcBinary.loadLength(srcAddress), forceDeep) } override def containsPointers: Boolean = true - def storeShallowAtOffset(dstAddress: Code[Long], valueAddress: Code[Long]): Code[Unit] = - Region.storeAddress(dstAddress, valueAddress) - - def storeShallowAtOffset(dstAddress: Long, srcAddress: Long) { - Region.storeAddress(dstAddress, srcAddress) - } - override def _pretty(sb: StringBuilder, indent: Int, compact: Boolean): Unit = sb.append("PCBinary") def contentAlignment: Long = 4 @@ -105,6 +83,63 @@ class PCanonicalBinary(val required: Boolean) extends PBinary { Region.storeInt(addr, bytes.length), Region.storeBytes(bytesOffset(addr), bytes) ) + + def constructAtAddress(mb: MethodBuilder, addr: Code[Long], region: Code[Region], srcPType: PType, srcAddress: Code[Long], forceDeep: Boolean): Code[Unit] = { + val srcBinary = srcPType.asInstanceOf[PBinary] + + val srcAddrVar = mb.newLocal[Long] + val len = mb.newLocal[Int] + Code( + srcAddrVar := srcAddress, + len := srcBinary.loadLength(srcAddrVar), + Region.storeAddress(addr, constructOrCopyWithLen(mb, region, srcBinary, srcAddrVar, len, forceDeep)) + ) + } + + private def constructOrCopyWithLen(mb: MethodBuilder, region: Code[Region], srcBinary: PBinary, srcAddress: LocalRef[Long], len: LocalRef[Int], forceDeep: Boolean): Code[Long] = { + if (srcBinary == this) { + if (forceDeep) { + val newAddr = mb.newLocal[Long] + Code( + newAddr := allocate(region, len), + Region.copyFrom(srcAddress, newAddr, contentByteSize(len)), + newAddr) + } else + srcAddress + } else { + + val newAddr = mb.newLocal[Long] + Code( + newAddr := allocate(region, len), + storeLength(newAddr, len), + Region.copyFrom(srcAddress + srcBinary.lengthHeaderBytes, newAddr + lengthHeaderBytes, len.toL), + newAddr + ) + } + } + + def constructAtAddress(addr: Long, region: Region, srcPType: PType, srcAddress: Long, forceDeep: Boolean): Unit = { + val srcArray = srcPType.asInstanceOf[PBinary] + Region.storeAddress(addr, constructOrCopyWithLen(region, srcArray, srcAddress, srcArray.loadLength(srcAddress), forceDeep)) + } + + private def constructOrCopyWithLen(region: Region, srcBinary: PBinary, srcAddress: Long, len: Int, forceDeep: Boolean): Long = { + if (srcBinary == this) { + if (forceDeep) { + val newAddr = allocate(region, len) + Region.copyFrom(srcAddress, newAddr, contentByteSize(len)) + newAddr + } else + srcAddress + } else { + val newAddr = allocate(region, len) + storeLength(newAddr, len) + Region.copyFrom(srcAddress + srcBinary.lengthHeaderBytes, newAddr + lengthHeaderBytes, len) + newAddr + } + } + + def setRequired(required: Boolean) = if(required == this.required) this else PCanonicalBinary(required) } object PCanonicalBinary { diff --git a/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalCall.scala b/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalCall.scala index 23c4a109b67..729fb50ae25 100644 --- a/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalCall.scala +++ b/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalCall.scala @@ -11,7 +11,7 @@ final case class PCanonicalCall(required: Boolean = false) extends PCall { val representation: PType = PInt32(required) - def copy(required: Boolean = this.required): PCall = PCanonicalCall(required) + def setRequired(required: Boolean) = if(required == this.required) this else PCanonicalCall(required) def ploidy(c: Code[Int]): Code[Int] = (c >>> 1) & 0x3 diff --git a/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalDict.scala b/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalDict.scala index 4f8cfb19347..40707d3f6a2 100644 --- a/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalDict.scala +++ b/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalDict.scala @@ -7,8 +7,7 @@ final case class PCanonicalDict(keyType: PType, valueType: PType, required: Bool val arrayRep: PArray = PCanonicalArray(elementType, required) - def copy(keyType: PType = this.keyType, valueType: PType = this.valueType, required: Boolean = this.required): PDict = - PCanonicalDict(keyType, valueType, required) + def setRequired(required: Boolean) = if(required == this.required) this else PCanonicalDict(keyType, valueType, required) def _asIdent = s"dict_of_${keyType.asIdent}AND${valueType.asIdent}" diff --git a/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalInterval.scala b/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalInterval.scala index 2e986cb573c..05d3ac62d05 100644 --- a/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalInterval.scala +++ b/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalInterval.scala @@ -20,7 +20,7 @@ final case class PCanonicalInterval(pointType: PType, override val required: Boo "includesStart" -> PBooleanRequired, "includesEnd" -> PBooleanRequired) - def copy(required: Boolean = this.required): PInterval = PCanonicalInterval(this.pointType, required) + def setRequired(required: Boolean) = if(required == this.required) this else PCanonicalInterval(this.pointType, required) def startOffset(off: Code[Long]): Code[Long] = representation.fieldOffset(off, 0) diff --git a/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalLocus.scala b/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalLocus.scala index e9b36128da9..a08e4ff6dd0 100644 --- a/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalLocus.scala +++ b/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalLocus.scala @@ -30,7 +30,7 @@ final case class PCanonicalLocus(rgBc: BroadcastRG, required: Boolean = false) e override def _pretty(sb: StringBuilder, indent: Call, compact: Boolean): Unit = sb.append(s"PCLocus($rg)") - def copy(required: Boolean = this.required) = PCanonicalLocus(this.rgBc, required) + def setRequired(required: Boolean) = if(required == this.required) this else PCanonicalLocus(this.rgBc, required) val representation: PStruct = PCanonicalLocus.representation(required) diff --git a/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalNDArray.scala b/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalNDArray.scala index d181fa9cb7d..cb4eea7183f 100644 --- a/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalNDArray.scala +++ b/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalNDArray.scala @@ -21,11 +21,11 @@ final case class PCanonicalNDArray(elementType: PType, nDims: Int, required: Boo off => Region.loadInt(representation.loadField(off, "offset")) ) @transient lazy val shape = new StaticallyKnownField( - PTuple(true, Array.tabulate(nDims)(_ => PInt64Required):_*), + PCanonicalTuple(true, Array.tabulate(nDims)(_ => PInt64Required):_*): PTuple, off => representation.loadField(off, "shape") ) @transient lazy val strides = new StaticallyKnownField( - PTuple(true, Array.tabulate(nDims)(_ => PInt64Required):_*), + PCanonicalTuple(true, Array.tabulate(nDims)(_ => PInt64Required):_*): PTuple, (off) => representation.loadField(off, "strides") ) @@ -208,13 +208,6 @@ final case class PCanonicalNDArray(elementType: PType, nDims: Int, required: Boo )) } - def storeShallowAtOffset(dstAddress: Code[Long], valueAddress: Code[Long]): Code[Unit] = - this.representation.storeShallowAtOffset(dstAddress, valueAddress) - - def storeShallowAtOffset(dstAddress: Long, valueAddress: Long) { - this.representation.storeShallowAtOffset(dstAddress, valueAddress) - } - def copyFromType(mb: MethodBuilder, region: Code[Region], srcPType: PType, srcAddress: Code[Long], forceDeep: Boolean): Code[Long] = { val sourceNDPType = srcPType.asInstanceOf[PNDArray] @@ -239,6 +232,11 @@ final case class PCanonicalNDArray(elementType: PType, nDims: Int, required: Boo private def deepRenameNDArray(t: TNDArray) = PCanonicalNDArray(this.elementType.deepRename(t.elementType), this.nDims, this.required) - def copy(elementType: PType = this.elementType, nDims: Int = this.nDims, required: Boolean = this.required): PCanonicalNDArray = - PCanonicalNDArray(elementType, nDims, required) + def setRequired(required: Boolean) = if(required == this.required) this else PCanonicalNDArray(elementType, nDims, required) + + def constructAtAddress(mb: MethodBuilder, addr: Code[Long], region: Code[Region], srcPType: PType, srcAddress: Code[Long], forceDeep: Boolean): Code[Unit] = + throw new NotImplementedError("constructAtAddress should only be called on fundamental types; PCanonicalNDarray is not fundamental") + + def constructAtAddress(addr: Long, region: Region, srcPType: PType, srcAddress: Long, forceDeep: Boolean): Unit = + throw new NotImplementedError("constructAtAddress should only be called on fundamental types; PCanonicalNDarray is not fundamental") } diff --git a/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalSet.scala b/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalSet.scala index 63bb1b58736..bd1e39234fa 100644 --- a/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalSet.scala +++ b/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalSet.scala @@ -5,7 +5,7 @@ import is.hail.expr.types.virtual.{TSet, Type} final case class PCanonicalSet(elementType: PType, required: Boolean = false) extends PSet with PArrayBackedContainer { val arrayRep = PCanonicalArray(elementType, required) - def copy(elementType: PType = this.elementType, required: Boolean = this.required): PSet = PCanonicalSet(elementType, required) + def setRequired(required: Boolean) = if(required == this.required) this else PCanonicalSet(elementType, required) def _asIdent = s"set_of_${elementType.asIdent}" diff --git a/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalString.scala b/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalString.scala index e935d819a75..75e919f36d7 100644 --- a/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalString.scala +++ b/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalString.scala @@ -5,7 +5,7 @@ import is.hail.asm4s.{Code, MethodBuilder} case object PCanonicalStringOptional extends PCanonicalString(false) case object PCanonicalStringRequired extends PCanonicalString(true) -abstract class PCanonicalString(val required: Boolean) extends PString { +class PCanonicalString(val required: Boolean) extends PString { def _asIdent = "string" override def _pretty(sb: StringBuilder, indent: Int, compact: Boolean): Unit = sb.append("PCString") @@ -31,13 +31,6 @@ abstract class PCanonicalString(val required: Boolean) extends PString { override def containsPointers: Boolean = true - def storeShallowAtOffset(dstAddress: Code[Long], valueAddress: Code[Long]): Code[Unit] = - this.fundamentalType.storeShallowAtOffset(dstAddress, valueAddress) - - def storeShallowAtOffset(dstAddress: Long, valueAddress: Long) { - this.fundamentalType.storeShallowAtOffset(dstAddress, valueAddress) - } - def bytesOffset(boff: Long): Long = this.fundamentalType.bytesOffset(boff) @@ -73,6 +66,14 @@ abstract class PCanonicalString(val required: Boolean) extends PString { dstAddress ) } + + def constructAtAddress(mb: MethodBuilder, addr: Code[Long], region: Code[Region], srcPType: PType, srcAddress: Code[Long], forceDeep: Boolean): Code[Unit] = + fundamentalType.constructAtAddress(mb, addr, region, srcPType.fundamentalType, srcAddress, forceDeep) + + def constructAtAddress(addr: Long, region: Region, srcPType: PType, srcAddress: Long, forceDeep: Boolean): Unit = + fundamentalType.constructAtAddress(addr, region, srcPType.fundamentalType, srcAddress, forceDeep) + + def setRequired(required: Boolean) = if(required == this.required) this else PCanonicalString(required) } object PCanonicalString { diff --git a/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalStruct.scala b/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalStruct.scala index 35219b29804..d08b7d7c951 100644 --- a/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalStruct.scala +++ b/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalStruct.scala @@ -15,7 +15,7 @@ object PCanonicalStruct { def empty(required: Boolean = false): PStruct = if (required) requiredEmpty else optionalEmpty - def apply(required: Boolean, args: (String, PType)*): PStruct = + def apply(required: Boolean, args: (String, PType)*): PCanonicalStruct = PCanonicalStruct(args .iterator .zipWithIndex @@ -23,7 +23,7 @@ object PCanonicalStruct { .toFastIndexedSeq, required) - def apply(names: java.util.List[String], types: java.util.List[PType], required: Boolean): PStruct = { + def apply(names: java.util.List[String], types: java.util.List[PType], required: Boolean): PCanonicalStruct = { val sNames = names.asScala.toArray val sTypes = types.asScala.toArray if (sNames.length != sTypes.length) @@ -32,31 +32,24 @@ object PCanonicalStruct { PCanonicalStruct(required, sNames.zip(sTypes): _*) } - def apply(args: (String, PType)*): PStruct = + def apply(args: (String, PType)*): PCanonicalStruct = PCanonicalStruct(false, args:_*) - def canonical(t: Type): PStruct = PType.canonical(t).asInstanceOf[PStruct] - def canonical(t: PType): PStruct = PType.canonical(t).asInstanceOf[PStruct] + def canonical(t: Type): PCanonicalStruct = PType.canonical(t).asInstanceOf[PCanonicalStruct] + def canonical(t: PType): PCanonicalStruct = PType.canonical(t).asInstanceOf[PCanonicalStruct] } -final case class PCanonicalStruct(fields: IndexedSeq[PField], required: Boolean = false) extends PStruct { +final case class PCanonicalStruct(fields: IndexedSeq[PField], required: Boolean = false) extends PCanonicalBaseStruct(fields.map(_.typ).toArray) with PStruct { assert(fields.zipWithIndex.forall { case (f, i) => f.index == i }) - val types: Array[PType] = fields.map(_.typ).toArray - if (!fieldNames.areDistinct()) { val duplicates = fieldNames.duplicates() fatal(s"cannot create struct with duplicate ${plural(duplicates.size, "field")}: " + s"${fieldNames.map(prettyIdentifier).mkString(", ")}", fieldNames.duplicates()) } - val (missingIdx: Array[Int], nMissing: Int) = BaseStruct.getMissingIndexAndCount(types.map(_.required)) - val nMissingBytes = (nMissing + 7) >>> 3 - val byteOffsets = new Array[Long](size) - override val byteSize: Long = PBaseStruct.getByteSizeAndOffsets(types, nMissingBytes, byteOffsets) - override val alignment: Long = PBaseStruct.alignment(types) - def copy(fields: IndexedSeq[PField] = this.fields, required: Boolean = this.required): PStruct = PCanonicalStruct(fields, required) + def setRequired(required: Boolean) = if(required == this.required) this else PCanonicalStruct(fields, required) override def truncate(newSize: Int): PStruct = PCanonicalStruct(fields.take(newSize), required) diff --git a/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalTuple.scala b/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalTuple.scala index e9b25517ac2..ae79c309122 100644 --- a/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalTuple.scala +++ b/hail/src/main/scala/is/hail/expr/types/physical/PCanonicalTuple.scala @@ -5,20 +5,13 @@ import is.hail.expr.types.virtual.{TTuple, Type} import is.hail.utils._ object PCanonicalTuple { - def apply(required: Boolean, args: PType*): PTuple = PCanonicalTuple(args.iterator.zipWithIndex.map { case (t, i) => PTupleField(i, t)}.toIndexedSeq, required) + def apply(required: Boolean, args: PType*): PCanonicalTuple = PCanonicalTuple(args.iterator.zipWithIndex.map { case (t, i) => PTupleField(i, t)}.toIndexedSeq, required) } -final case class PCanonicalTuple(_types: IndexedSeq[PTupleField], override val required: Boolean = false) extends PTuple { - val types = _types.map(_.typ).toArray +final case class PCanonicalTuple(_types: IndexedSeq[PTupleField], override val required: Boolean = false) extends PCanonicalBaseStruct(_types.map(_.typ).toArray) with PTuple { lazy val fieldIndex: Map[Int, Int] = _types.zipWithIndex.map { case (tf, idx) => tf.index -> idx }.toMap - val (missingIdx: Array[Int], nMissing: Int) = BaseStruct.getMissingIndexAndCount(types.map(_.required)) - val nMissingBytes = UnsafeUtils.packBitsToBytes(nMissing) - val byteOffsets = new Array[Long](size) - override val byteSize: Long = PBaseStruct.getByteSizeAndOffsets(types, nMissingBytes, byteOffsets) - override val alignment: Long = PBaseStruct.alignment(types) - - def copy(required: Boolean = this.required): PTuple = PCanonicalTuple(_types, required) + def setRequired(required: Boolean) = if(required == this.required) this else PCanonicalTuple(_types, required) override def truncate(newSize: Int): PTuple = PCanonicalTuple(_types.take(newSize), required) diff --git a/hail/src/main/scala/is/hail/expr/types/physical/PDict.scala b/hail/src/main/scala/is/hail/expr/types/physical/PDict.scala index 3cd216e21c8..13fc3e40aca 100644 --- a/hail/src/main/scala/is/hail/expr/types/physical/PDict.scala +++ b/hail/src/main/scala/is/hail/expr/types/physical/PDict.scala @@ -14,8 +14,6 @@ abstract class PDict extends PContainer { val keyType: PType val valueType: PType - def copy(keyType: PType = this.keyType, valueType: PType = this.valueType, required: Boolean = this.required): PDict - def elementType: PStruct def arrayFundamentalType: PArray = fundamentalType.asInstanceOf[PArray] diff --git a/hail/src/main/scala/is/hail/expr/types/physical/PFloat32.scala b/hail/src/main/scala/is/hail/expr/types/physical/PFloat32.scala index 115b42ef1db..3e233eb675f 100644 --- a/hail/src/main/scala/is/hail/expr/types/physical/PFloat32.scala +++ b/hail/src/main/scala/is/hail/expr/types/physical/PFloat32.scala @@ -1,14 +1,9 @@ package is.hail.expr.types.physical import is.hail.annotations._ -import is.hail.asm4s.{Code, TypeInfo, _} -import is.hail.check.Arbitrary._ -import is.hail.check.Gen +import is.hail.asm4s.{Code, _} import is.hail.expr.ir.EmitMethodBuilder import is.hail.expr.types.virtual.TFloat32 -import is.hail.utils._ - -import scala.reflect.{ClassTag, _} case object PFloat32Optional extends PFloat32(false) case object PFloat32Required extends PFloat32(true) @@ -59,6 +54,9 @@ class PFloat32(override val required: Boolean) extends PNumeric with PPrimitive override def multiply(a: Code[_], b: Code[_]): Code[PFloat32] = { coerce[PFloat32](coerce[Float](a) * coerce[Float](b)) } + + def storePrimitiveAtAddress(addr: Code[Long], srcPType: PType, value: Code[_]): Code[Unit] = + Region.storeFloat(addr, coerce[Float](value)) } object PFloat32 { diff --git a/hail/src/main/scala/is/hail/expr/types/physical/PFloat64.scala b/hail/src/main/scala/is/hail/expr/types/physical/PFloat64.scala index 06ba1fd8596..7f74e56c446 100644 --- a/hail/src/main/scala/is/hail/expr/types/physical/PFloat64.scala +++ b/hail/src/main/scala/is/hail/expr/types/physical/PFloat64.scala @@ -1,15 +1,9 @@ package is.hail.expr.types.physical import is.hail.annotations._ -import is.hail.asm4s.Code -import is.hail.asm4s._ -import is.hail.check.Arbitrary._ -import is.hail.check.Gen +import is.hail.asm4s.{Code, _} import is.hail.expr.ir.EmitMethodBuilder import is.hail.expr.types.virtual.TFloat64 -import is.hail.utils._ - -import scala.reflect.{ClassTag, _} case object PFloat64Optional extends PFloat64(false) case object PFloat64Required extends PFloat64(true) @@ -60,6 +54,9 @@ class PFloat64(override val required: Boolean) extends PNumeric with PPrimitive override def multiply(a: Code[_], b: Code[_]): Code[PFloat64] = { coerce[PFloat64](coerce[Double](a) * coerce[Double](b)) } + + def storePrimitiveAtAddress(addr: Code[Long], srcPType: PType, value: Code[_]): Code[Unit] = + Region.storeDouble(addr, coerce[Double](value)) } object PFloat64 { diff --git a/hail/src/main/scala/is/hail/expr/types/physical/PInt32.scala b/hail/src/main/scala/is/hail/expr/types/physical/PInt32.scala index d47c1f3d9fc..4149727d688 100644 --- a/hail/src/main/scala/is/hail/expr/types/physical/PInt32.scala +++ b/hail/src/main/scala/is/hail/expr/types/physical/PInt32.scala @@ -1,14 +1,9 @@ package is.hail.expr.types.physical import is.hail.annotations.{Region, UnsafeOrdering, _} -import is.hail.asm4s.{Code, TypeInfo, coerce, const, _} -import is.hail.check.Arbitrary._ -import is.hail.check.Gen +import is.hail.asm4s.{Code, coerce, const, _} import is.hail.expr.ir.EmitMethodBuilder import is.hail.expr.types.virtual.TInt32 -import is.hail.utils._ - -import scala.reflect.{ClassTag, _} case object PInt32Optional extends PInt32(false) case object PInt32Required extends PInt32(true) @@ -56,6 +51,9 @@ class PInt32(override val required: Boolean) extends PNumeric with PPrimitive { override def multiply(a: Code[_], b: Code[_]): Code[PInt32] = { coerce[PInt32](coerce[Int](a) * coerce[Int](b)) } + + def storePrimitiveAtAddress(addr: Code[Long], srcPType: PType, value: Code[_]): Code[Unit] = + Region.storeInt(addr, coerce[Int](value)) } object PInt32 { diff --git a/hail/src/main/scala/is/hail/expr/types/physical/PInt64.scala b/hail/src/main/scala/is/hail/expr/types/physical/PInt64.scala index f544198642e..f65e9b64169 100644 --- a/hail/src/main/scala/is/hail/expr/types/physical/PInt64.scala +++ b/hail/src/main/scala/is/hail/expr/types/physical/PInt64.scala @@ -1,14 +1,9 @@ package is.hail.expr.types.physical import is.hail.annotations.{Region, UnsafeOrdering, _} -import is.hail.asm4s.{Code, TypeInfo, coerce, const, _} -import is.hail.check.Arbitrary._ -import is.hail.check.Gen +import is.hail.asm4s.{Code, coerce, const, _} import is.hail.expr.ir.EmitMethodBuilder import is.hail.expr.types.virtual.TInt64 -import is.hail.utils._ - -import scala.reflect.{ClassTag, _} case object PInt64Optional extends PInt64(false) case object PInt64Required extends PInt64(true) @@ -57,6 +52,9 @@ class PInt64(override val required: Boolean) extends PNumeric with PPrimitive { override def multiply(a: Code[_], b: Code[_]): Code[PInt64] = { coerce[PInt64](coerce[Long](a) * coerce[Long](b)) } + + def storePrimitiveAtAddress(addr: Code[Long], srcPType: PType, value: Code[_]): Code[Unit] = + Region.storeLong(addr, coerce[Long](value)) } object PInt64 { diff --git a/hail/src/main/scala/is/hail/expr/types/physical/PInterval.scala b/hail/src/main/scala/is/hail/expr/types/physical/PInterval.scala index 050c097e1f8..2cb357f68c0 100644 --- a/hail/src/main/scala/is/hail/expr/types/physical/PInterval.scala +++ b/hail/src/main/scala/is/hail/expr/types/physical/PInterval.scala @@ -23,8 +23,6 @@ abstract class PInterval extends ComplexPType { CodeOrdering.intervalOrdering(this, other.asInstanceOf[PInterval], mb) } - def copy(required: Boolean): PInterval - override def unsafeOrdering(): UnsafeOrdering = new UnsafeOrdering { private val pOrd = pointType.unsafeOrdering() diff --git a/hail/src/main/scala/is/hail/expr/types/physical/PLocus.scala b/hail/src/main/scala/is/hail/expr/types/physical/PLocus.scala index b71e74e3e67..a1fadd7d579 100644 --- a/hail/src/main/scala/is/hail/expr/types/physical/PLocus.scala +++ b/hail/src/main/scala/is/hail/expr/types/physical/PLocus.scala @@ -32,6 +32,4 @@ abstract class PLocus extends ComplexPType { def position(address: Code[Long]): Code[Int] def positionType: PInt32 - - def copy(required: Boolean): PLocus } diff --git a/hail/src/main/scala/is/hail/expr/types/physical/PNDArray.scala b/hail/src/main/scala/is/hail/expr/types/physical/PNDArray.scala index 3aa45573611..79ddb133c36 100644 --- a/hail/src/main/scala/is/hail/expr/types/physical/PNDArray.scala +++ b/hail/src/main/scala/is/hail/expr/types/physical/PNDArray.scala @@ -32,8 +32,6 @@ abstract class PNDArray extends PType { val representation: PStruct - def copy(elementType: PType = this.elementType, nDims: Int = this.nDims, required: Boolean = this.required): PNDArray - def dimensionLength(off: Code[Long], idx: Int): Code[Long] = { Region.loadLong(shape.pType.fieldOffset(shape.load(off), idx)) } diff --git a/hail/src/main/scala/is/hail/expr/types/physical/PPrimitive.scala b/hail/src/main/scala/is/hail/expr/types/physical/PPrimitive.scala index 2e0f474e15b..bcc97603b9c 100644 --- a/hail/src/main/scala/is/hail/expr/types/physical/PPrimitive.scala +++ b/hail/src/main/scala/is/hail/expr/types/physical/PPrimitive.scala @@ -1,23 +1,66 @@ package is.hail.expr.types.physical import is.hail.annotations.Region +import is.hail.asm4s._ import is.hail.asm4s.{Code, MethodBuilder} +import is.hail.utils._ -trait PPrimitive { +trait PPrimitive extends PType { def byteSize: Long - def storeShallowAtOffset(dstAddress: Code[Long], srcAddress: Code[Long]): Code[Unit] = - Region.copyFrom(srcAddress, dstAddress, byteSize) + def copyFromType(region: Region, srcPType: PType, srcAddress: Long, forceDeep: Boolean): Long = { + assert(this.isOfType(srcPType)) + if (forceDeep) { + val addr = region.allocate(byteSize, byteSize) + constructAtAddress(addr, region, srcPType, srcAddress, forceDeep) + addr + } else srcAddress + } - def storeShallowAtOffset(dstAddress: Long, srcAddress: Long): Unit = - Region.copyFrom(srcAddress, dstAddress, byteSize) + def copyFromType(mb: MethodBuilder, region: Code[Region], srcPType: PType, srcAddress: Code[Long], forceDeep: Boolean): Code[Long] = { + assert(this.isOfType(srcPType)) + if (forceDeep) { + val addr = mb.newLocal[Long] + Code( + addr := region.allocate(byteSize, byteSize), + constructAtAddress(mb, addr, region, srcPType, srcAddress, forceDeep), + addr + ) + } else srcAddress + } - def copyFromType(region: Region, srcPType: PType, srcAddress: Long, forceDeep: Boolean): Long = - srcAddress + def copyFromTypeAndStackValue(mb: MethodBuilder, region: Code[Region], srcPType: PType, stackValue: Code[_], forceDeep: Boolean): Code[_] = { + assert(!forceDeep) + stackValue + } - def copyFromType(mb: MethodBuilder, region: Code[Region], srcPType: PType, srcAddress: Code[Long], forceDeep: Boolean): Code[Long] = - srcAddress + def constructAtAddress(mb: MethodBuilder, addr: Code[Long], region: Code[Region], srcPType: PType, srcAddress: Code[Long], forceDeep: Boolean): Code[Unit] = { + assert(srcPType.isOfType(this)) + Region.copyFrom(srcAddress, addr, byteSize) + } - def copyFromTypeAndStackValue(mb: MethodBuilder, region: Code[Region], srcPType: PType, stackValue: Code[_], forceDeep: Boolean): Code[_] = - stackValue + def constructAtAddress(addr: Long, region: Region, srcPType: PType, srcAddress: Long, forceDeep: Boolean): Unit = { + assert(srcPType.isOfType(this)) + Region.copyFrom(srcAddress, addr, byteSize) + } + + override def constructAtAddressFromValue(mb: MethodBuilder, addr: Code[Long], region: Code[Region], srcPType: PType, src: Code[_], forceDeep: Boolean): Code[Unit] = { + assert(this.isOfType(srcPType)) + storePrimitiveAtAddress(addr, srcPType, src) + } + + def storePrimitiveAtAddress(addr: Code[Long], srcPType: PType, value: Code[_]): Code[Unit] + + def setRequired(required: Boolean): PPrimitive = { + if (required == this.required) + this + else + this match { + case _: PBoolean => PBoolean(required) + case _: PInt32 => PInt32(required) + case _: PInt64 => PInt64(required) + case _: PFloat32 => PFloat32(required) + case _: PFloat64 => PFloat64(required) + } + } } diff --git a/hail/src/main/scala/is/hail/expr/types/physical/PSet.scala b/hail/src/main/scala/is/hail/expr/types/physical/PSet.scala index fdca473975d..6966237b738 100644 --- a/hail/src/main/scala/is/hail/expr/types/physical/PSet.scala +++ b/hail/src/main/scala/is/hail/expr/types/physical/PSet.scala @@ -11,8 +11,6 @@ object PSet { abstract class PSet extends PContainer { lazy val virtualType: TSet = TSet(elementType.virtualType, required) - def copy(elementType: PType = this.elementType, required: Boolean = this.required): PSet - def arrayFundamentalType: PArray = fundamentalType.asInstanceOf[PArray] def codeOrdering(mb: EmitMethodBuilder, other: PType): CodeOrdering = { diff --git a/hail/src/main/scala/is/hail/expr/types/physical/PStream.scala b/hail/src/main/scala/is/hail/expr/types/physical/PStream.scala index 4f8e5a11cfe..d7c228e21b8 100644 --- a/hail/src/main/scala/is/hail/expr/types/physical/PStream.scala +++ b/hail/src/main/scala/is/hail/expr/types/physical/PStream.scala @@ -7,15 +7,9 @@ import is.hail.expr.types.virtual.{TStream, Type} trait PStreamable extends PIterable { def asPArray: PArray = PArray(this.elementType, this.required) - def copyStreamable(elt: PType, req: Boolean = required): PStreamable = { - this match { - case _: PArray => PArray(elt, req) - case _: PStream => PStream(elt, req) - } - } } -final case class PStream(elementType: PType, override val required: Boolean = false) extends PStreamable { +final case class PStream(elementType: PType, required: Boolean = false) extends PStreamable { lazy val virtualType: TStream = TStream(elementType.virtualType, required) override val fundamentalType: PStream = { @@ -45,15 +39,17 @@ final case class PStream(elementType: PType, override val required: Boolean = fa def copyFromTypeAndStackValue(mb: MethodBuilder, region: Code[Region], srcPType: PType, stackValue: Code[_], forceDeep: Boolean): Code[_] = throw new UnsupportedOperationException("PStream copyFromTypeAndStackValue is currently undefined") - def storeShallowAtOffset(dstAddress: Code[Long], srcAddress: Code[Long]): Code[Unit] = - throw new UnsupportedOperationException("PStream storeShallowAtOffset is currently undefined") - - def storeShallowAtOffset(dstAddress: Long, srcAddress: Long) = - throw new UnsupportedOperationException("PStream storeShallowAtOffset is currently undefined") - override def deepRename(t: Type) = deepRenameStream(t.asInstanceOf[TStream]) private def deepRenameStream(t: TStream): PStream = PStream(this.elementType.deepRename(t.elementType), this.required) + + def constructAtAddress(mb: MethodBuilder, addr: Code[Long], region: Code[Region], srcPType: PType, srcAddress: Code[Long], forceDeep: Boolean): Code[Unit] = + throw new NotImplementedError(s"$this is not constructable") + + def constructAtAddress(addr: Long, region: Region, srcPType: PType, srcAddress: Long, forceDeep: Boolean): Unit = + throw new NotImplementedError(s"$this is not constructable") + + def setRequired(required: Boolean) = if(required == this.required) this else this.copy(required = required) } diff --git a/hail/src/main/scala/is/hail/expr/types/physical/PStruct.scala b/hail/src/main/scala/is/hail/expr/types/physical/PStruct.scala index 5cba78158b2..ed97c792eb1 100644 --- a/hail/src/main/scala/is/hail/expr/types/physical/PStruct.scala +++ b/hail/src/main/scala/is/hail/expr/types/physical/PStruct.scala @@ -28,11 +28,9 @@ object PStruct { def canonical(t: PType): PStruct = PCanonicalStruct.canonical(t) } -abstract class PStruct extends PBaseStruct { +trait PStruct extends PBaseStruct { lazy val virtualType: TStruct = TStruct(fields.map(f => Field(f.name, f.typ.virtualType, f.index)), required) - def copy(fields: IndexedSeq[PField] = this.fields, required: Boolean = this.required): PStruct - final def codeOrdering(mb: EmitMethodBuilder, other: PType): CodeOrdering = codeOrdering(mb, other, null) diff --git a/hail/src/main/scala/is/hail/expr/types/physical/PTuple.scala b/hail/src/main/scala/is/hail/expr/types/physical/PTuple.scala index 6263ef10551..2ac44dff976 100644 --- a/hail/src/main/scala/is/hail/expr/types/physical/PTuple.scala +++ b/hail/src/main/scala/is/hail/expr/types/physical/PTuple.scala @@ -12,12 +12,12 @@ case class PTupleField(index: Int, typ: PType) object PTuple { def apply(args: IndexedSeq[PTupleField], required: Boolean = false): PTuple = PCanonicalTuple(args, required) - def apply(required: Boolean, args: PType*): PTuple = PCanonicalTuple(required, args:_*) + def apply(required: Boolean, args: PType*): PCanonicalTuple = PCanonicalTuple(required, args:_*) - def apply(args: PType*): PTuple = PCanonicalTuple(false, args:_*) + def apply(args: PType*): PCanonicalTuple = PCanonicalTuple(false, args:_*) } -abstract class PTuple extends PBaseStruct { +trait PTuple extends PBaseStruct { val _types: IndexedSeq[PTupleField] val fieldIndex: Map[Int, Int] @@ -29,8 +29,6 @@ abstract class PTuple extends PBaseStruct { protected val tupleFundamentalType: PTuple override lazy val fundamentalType: PTuple = tupleFundamentalType - def copy(required: Boolean): PTuple - final def codeOrdering(mb: EmitMethodBuilder, other: PType): CodeOrdering = codeOrdering(mb, other, null) diff --git a/hail/src/main/scala/is/hail/expr/types/physical/PType.scala b/hail/src/main/scala/is/hail/expr/types/physical/PType.scala index 49a5ef35665..9bfcea3687d 100644 --- a/hail/src/main/scala/is/hail/expr/types/physical/PType.scala +++ b/hail/src/main/scala/is/hail/expr/types/physical/PType.scala @@ -212,31 +212,7 @@ abstract class PType extends Serializable with Requiredness { final def unary_-(): PType = setRequired(false) - final def setRequired(required: Boolean): PType = { - if (required == this.required) - this - else - this match { - case PVoid => PVoid - case PBinary(_) => PBinary(required) - case PBoolean(_) => PBoolean(required) - case PInt32(_) => PInt32(required) - case PInt64(_) => PInt64(required) - case PFloat32(_) => PFloat32(required) - case PFloat64(_) => PFloat64(required) - case PString(_) => PString(required) - case t: PCall => t.copy(required) - case t: PArray => t.copy(required = required) - case t: PSet => t.copy(required = required) - case t: PDict => t.copy(required = required) - case t: PLocus => t.copy(required = required) - case t: PInterval => t.copy(required = required) - case t: PStruct => t.copy(required = required) - case t: PTuple => t.copy(required = required) - case t: PNDArray => t.copy(required = required) - case PVoid => PVoid - } - } + def setRequired(required: Boolean): PType final def isOfType(t: PType): Boolean = { this match { @@ -321,20 +297,16 @@ abstract class PType extends Serializable with Requiredness { def copyFromTypeAndStackValue(mb: MethodBuilder, region: Code[Region], srcPType: PType, stackValue: Code[_], forceDeep: Boolean): Code[_] - def copyFromType(mb: MethodBuilder, region: Code[Region], srcPType: PType, srcAddress: Code[Long]): Code[Long] = - this.copyFromType(mb, region, srcPType, srcAddress, false) - def copyFromTypeAndStackValue(mb: MethodBuilder, region: Code[Region], srcPType: PType, stackValue: Code[_]): Code[_] = this.copyFromTypeAndStackValue(mb, region, srcPType, stackValue, false) def copyFromType(region: Region, srcPType: PType, srcAddress: Long, forceDeep: Boolean): Long - def copyFromType(region: Region, srcPType: PType, srcAddress: Long): Long = - this.copyFromType(region, srcPType, srcAddress, false) + def constructAtAddress(mb: MethodBuilder, addr: Code[Long], region: Code[Region], srcPType: PType, srcAddress: Code[Long], forceDeep: Boolean): Code[Unit] + def constructAtAddressFromValue(mb: MethodBuilder, addr: Code[Long], region: Code[Region], srcPType: PType, src: Code[_], forceDeep: Boolean): Code[Unit] + = constructAtAddress(mb, addr, region, srcPType, coerce[Long](src), forceDeep) - def deepRename(t: Type) = this - - def storeShallowAtOffset(dstAddress: Code[Long], srcAddress: Code[Long]): Code[Unit] + def constructAtAddress(addr: Long, region: Region, srcPType: PType, srcAddress: Long, forceDeep: Boolean): Unit - def storeShallowAtOffset(dstAddress: Long, srcAddress: Long) + def deepRename(t: Type) = this } diff --git a/hail/src/main/scala/is/hail/expr/types/physical/PVoid.scala b/hail/src/main/scala/is/hail/expr/types/physical/PVoid.scala index a74174b3cc1..6edf6fb0f5e 100644 --- a/hail/src/main/scala/is/hail/expr/types/physical/PVoid.scala +++ b/hail/src/main/scala/is/hail/expr/types/physical/PVoid.scala @@ -24,9 +24,11 @@ case object PVoid extends PType { def copyFromTypeAndStackValue(mb: MethodBuilder, region: Code[Region], srcPType: PType, stackValue: Code[_], forceDeep: Boolean): Code[_] = throw new UnsupportedOperationException("PVoid copyFromTypeAndStackValue is currently undefined") - def storeShallowAtOffset(dstAddress: Code[Long], srcAddress: Code[Long]): Code[Unit] = - throw new UnsupportedOperationException("PVoid storeShallowAtOffset is currently undefined") + def constructAtAddress(mb: MethodBuilder, addr: Code[Long], region: Code[Region], srcPType: PType, srcAddress: Code[Long], forceDeep: Boolean): Code[Unit] = + throw new NotImplementedError(s"$this is not constructable") - def storeShallowAtOffset(dstAddress: Long, srcAddress: Long) = - throw new UnsupportedOperationException("PVoid storeShallowAtOffset is currently undefined") + def constructAtAddress(addr: Long, region: Region, srcPType: PType, srcAddress: Long, forceDeep: Boolean): Unit = + throw new NotImplementedError(s"$this is not constructable") + + def setRequired(required: Boolean) = PVoid } diff --git a/hail/src/main/scala/is/hail/expr/types/virtual/TStream.scala b/hail/src/main/scala/is/hail/expr/types/virtual/TStream.scala index 6480c7a14ec..ec1ef777a96 100644 --- a/hail/src/main/scala/is/hail/expr/types/virtual/TStream.scala +++ b/hail/src/main/scala/is/hail/expr/types/virtual/TStream.scala @@ -8,9 +8,17 @@ import org.json4s.jackson.JsonMethods import scala.reflect.{ClassTag, classTag} trait TStreamable extends TIterable { + def makeRealizable(t: Type): Type = t match { + case t: TStream => + TArray(makeRealizable(t.elementType), t.required) + case _ => + assert(t.isRealizable) + t + } + def copyStreamable(elt: Type, req: Boolean = required): TStreamable = { this match { - case _: TArray => TArray(elt, req) + case _: TArray => TArray(makeRealizable(elt), req) case _: TStream => TStream(elt, req) } } diff --git a/hail/src/main/scala/is/hail/io/TextMatrixReader.scala b/hail/src/main/scala/is/hail/io/TextMatrixReader.scala index 134b09121f0..1527ecc0c65 100644 --- a/hail/src/main/scala/is/hail/io/TextMatrixReader.scala +++ b/hail/src/main/scala/is/hail/io/TextMatrixReader.scala @@ -184,7 +184,7 @@ object TextMatrixReader { } } it - }.countPerPartition().scanLeft(0L)(_ + _) + }.countPerPartition() } } diff --git a/hail/src/main/scala/is/hail/io/plink/LoadPlink.scala b/hail/src/main/scala/is/hail/io/plink/LoadPlink.scala index a1eac16ed39..0d6ed5fc41f 100644 --- a/hail/src/main/scala/is/hail/io/plink/LoadPlink.scala +++ b/hail/src/main/scala/is/hail/io/plink/LoadPlink.scala @@ -4,7 +4,7 @@ import is.hail.HailContext import is.hail.annotations._ import is.hail.expr.ir.{ExecuteContext, LowerMatrixIR, MatrixHybridReader, MatrixRead, MatrixReader, MatrixValue, PruneDeadFields, TableRead, TableValue} import is.hail.expr.types._ -import is.hail.expr.types.physical.{PBoolean, PFloat64, PString, PStruct} +import is.hail.expr.types.physical.{PBoolean, PCanonicalString, PCanonicalStruct, PFloat64, PString, PStruct} import is.hail.expr.types.virtual._ import is.hail.io.vcf.LoadVCF import is.hail.rvd.{RVD, RVDContext, RVDType} @@ -45,14 +45,14 @@ object LoadPlink { """^-?(?:\d+|\d*\.\d+)(?:[eE]-?\d+)?$""".r def parseFam(filename: String, ffConfig: FamFileConfig, - fs: FS): (IndexedSeq[Row], PStruct) = { + fs: FS): (IndexedSeq[Row], PCanonicalStruct) = { val delimiter = unescapeString(ffConfig.delimiter) val phenoSig = if (ffConfig.isQuantPheno) ("quant_pheno", PFloat64()) else ("is_case", PBoolean()) - val signature = PStruct(("id", PString()), ("fam_id", PString()), ("pat_id", PString()), - ("mat_id", PString()), ("is_female", PBoolean()), phenoSig) + val signature = PCanonicalStruct(("id", PString()), ("fam_id", PCanonicalString()), ("pat_id", PCanonicalString()), + ("mat_id", PCanonicalString()), ("is_female", PBoolean()), phenoSig) val idBuilder = new ArrayBuilder[String] val structBuilder = new ArrayBuilder[Row] diff --git a/hail/src/main/scala/is/hail/methods/Nirvana.scala b/hail/src/main/scala/is/hail/methods/Nirvana.scala index 9c24dab658f..8703c1c7ccd 100644 --- a/hail/src/main/scala/is/hail/methods/Nirvana.scala +++ b/hail/src/main/scala/is/hail/methods/Nirvana.scala @@ -428,7 +428,7 @@ object Nirvana { } .grouped(localBlockSize) .flatMap { block => - val (jt, proc) = block.iterator.pipe(pb, + val (jt, err, proc) = block.iterator.pipe(pb, printContext, printElement(localRowType), _ => ()) @@ -446,7 +446,7 @@ object Nirvana { val rc = proc.waitFor() if (rc != 0) - fatal(s"nirvana command failed with non-zero exit status $rc") + fatal(s"nirvana command failed with non-zero exit status $rc\n\tError:\n${err.toString}") r } diff --git a/hail/src/main/scala/is/hail/methods/VEP.scala b/hail/src/main/scala/is/hail/methods/VEP.scala index 00789a70e89..3dcd0069cc1 100644 --- a/hail/src/main/scala/is/hail/methods/VEP.scala +++ b/hail/src/main/scala/is/hail/methods/VEP.scala @@ -67,14 +67,12 @@ object VEP { } } - def waitFor(proc: Process, cmd: Array[String]): Unit = { + def waitFor(proc: Process, err: StringBuilder, cmd: Array[String]): Unit = { val rc = proc.waitFor() if (rc != 0) { - val errorLines = Source.fromInputStream(new BufferedInputStream(proc.getErrorStream)).getLines().mkString("\n") - fatal(s"VEP command '${ cmd.mkString(" ") }' failed with non-zero exit status $rc\n" + - " VEP Error output:\n" + errorLines) + " VEP Error output:\n" + err.toString) } } @@ -84,13 +82,13 @@ object VEP { val env = pb.environment() confEnv.foreach { case (key, value) => env.put(key, value) } - val (jt, proc) = List((Locus("1", 13372), FastIndexedSeq("G", "C"))).iterator.pipe(pb, + val (jt, err, proc) = List((Locus("1", 13372), FastIndexedSeq("G", "C"))).iterator.pipe(pb, printContext, printElement, _ => ()) val csqHeader = jt.flatMap(s => csqHeaderRegex.findFirstMatchIn(s).map(m => m.group(1))) - waitFor(proc, cmd) + waitFor(proc, err, cmd) if (csqHeader.hasNext) Some(csqHeader.next()) @@ -155,7 +153,7 @@ case class VEP(config: String, csq: Boolean, blockSize: Int) extends TableToTabl } .grouped(localBlockSize) .flatMap { block => - val (jt, proc) = block.iterator.pipe(pb, + val (jt, err, proc) = block.iterator.pipe(pb, printContext, printElement, _ => ()) @@ -211,7 +209,7 @@ case class VEP(config: String, csq: Boolean, blockSize: Int) extends TableToTabl val r = kt.toArray .sortBy(_._1)(rowKeyOrd.toOrdering) - waitFor(proc, cmd) + waitFor(proc, err, cmd) r } diff --git a/hail/src/main/scala/is/hail/rvd/RVD.scala b/hail/src/main/scala/is/hail/rvd/RVD.scala index 88fdc84b3b4..ec55f56caa1 100644 --- a/hail/src/main/scala/is/hail/rvd/RVD.scala +++ b/hail/src/main/scala/is/hail/rvd/RVD.scala @@ -13,7 +13,7 @@ import is.hail.io.index.IndexWriter import is.hail.io.{AbstractTypedCodecSpec, BufferSpec, RichContextRDDRegionValue, TypedCodecSpec} import is.hail.sparkextras._ import is.hail.utils._ -import is.hail.expr.ir.ExecuteContext +import is.hail.expr.ir.{ExecuteContext, InferPType} import is.hail.utils.PartitionCounts.{PCSubsetOffset, getPCSubsetOffset, incrementalPCSubsetOffset} import org.apache.commons.lang3.StringUtils import org.apache.spark.TaskContext @@ -1413,6 +1413,22 @@ object RVD { new RVD(typ, partitioner, crdd).checkKeyOrdering() } + private def copyFromType(destPType: PType, srcPType: PType, srcRegionValue: RegionValue): RegionValue = + RegionValue(srcRegionValue.region, destPType.copyFromType(srcRegionValue.region, srcPType, srcRegionValue.offset, false)) + + def unify(rvds: Seq[RVD]): Seq[RVD] = { + if (rvds.length == 1 || rvds.forall(_.rowPType == rvds.head.rowPType)) + return rvds + + val unifiedRowPType = InferPType.getNestedElementPTypesOfSameType(rvds.map(_.rowPType)).asInstanceOf[PStruct] + + rvds.map(rvd => { + val srcRowPType = rvd.rowPType + val newRVDType = rvd.typ.copy(rowType = unifiedRowPType) + rvd.map(newRVDType)(copyFromType(unifiedRowPType, srcRowPType, _)) + }) + } + def union( rvds: Seq[RVD], joinKey: Int, @@ -1421,6 +1437,7 @@ object RVD { case Seq(x) => x case first +: _ => assert(rvds.forall(_.rowPType == first.rowPType)) + if (joinKey == 0) { val sc = first.sparkContext RVD.unkeyed(first.rowPType, ContextRDD.union(sc, rvds.map(_.crdd))) diff --git a/hail/src/main/scala/is/hail/scheduler/BreakRetryException.scala b/hail/src/main/scala/is/hail/scheduler/BreakRetryException.scala deleted file mode 100644 index b3df31b5eb3..00000000000 --- a/hail/src/main/scala/is/hail/scheduler/BreakRetryException.scala +++ /dev/null @@ -1,4 +0,0 @@ -package is.hail.scheduler - -class BreakRetryException(cause: Throwable) extends Exception(cause) - diff --git a/hail/src/main/scala/is/hail/scheduler/Client.scala b/hail/src/main/scala/is/hail/scheduler/Client.scala deleted file mode 100644 index 3e120efb9ba..00000000000 --- a/hail/src/main/scala/is/hail/scheduler/Client.scala +++ /dev/null @@ -1,218 +0,0 @@ -package is.hail.scheduler - -import java.net.Socket -import java.util.concurrent.LinkedBlockingQueue - -import scala.util.Random -import scala.collection.mutable -import org.apache.spark.ExposedUtils -import is.hail.utils._ - -object ClientMessage { - // Client => Scheduler - val SUBMIT = 5 - - // Scheduler => Client - val APPTASKRESULT = 6 - val ACKTASKRESULT = 7 -} - -class SubmitterThread(host: String, token: Token, q: LinkedBlockingQueue[Option[(Token, DArray[_])]]) extends Runnable { - private var socket: DataSocket = _ - - private def connect(): DataSocket = { - var s = socket - if (s == null) { - s = new DataSocket(new Socket(host, 5052)) - info(s"SubmitterThread.getConnection: connected to $host:5052") - writeByteArray(token, s.out) - s.out.flush() - socket = s - } - s - } - - private def closeConnection(): Unit = { - val s = socket - socket = null - if (s != null) - s.socket.close() - } - - private def withConnection(f: DataSocket => Unit): Unit = { - try { - val s = retry(connect) - f(s) - } catch { - case e: Exception => - closeConnection() - throw e - } - } - - def submit(taskID: Token, da: DArray[_]): Unit = { - withConnection { s => - log.info(s"SubmitterThread.submit: submitting task with ${ da.nTasks } tasks") - s.out.writeInt(ClientMessage.SUBMIT) - writeByteArray(taskID, s.out) - val n = da.nTasks - s.out.writeInt(n) - s.out.flush() - - var i = s.in.readInt() - while (i < n) { - val context = da.contexts(i) - val localBody = da.body - val f = () => localBody(context) - ExposedUtils.clean(f, checkSerializable = true) - writeObject(f, s.out) - i += 1 - } - s.out.flush() - - log.info(s"SubmitterThread.submit: finished submitting task") - val ack = s.in.readInt() - assert(ack == 0) - val returnToken = readByteArray(s.in) - assert(returnToken sameElements taskID) - log.info(s"SubmitterThread.submit: task was acknowledged by the scheduler.") - } - } - - def run(): Unit = { - while (true) { - q.take() match { - case Some((taskID, da)) => - retry(() => submit(taskID, da)) - case None => - return - } - } - } -} - -class SchedulerAppClient(host: String) { - private val token = newToken() - private val q = new LinkedBlockingQueue[Option[(Token, DArray[_])]]() - - private val submitter = new SubmitterThread(host, token, q) - private val st = new Thread(submitter) - st.start() - - private var socket: DataSocket = _ - - var nTasks: Int = 0 - private var receivedTasks: mutable.BitSet = _ - var nComplete: Int = 0 - var callback: (Int, Any) => Unit = _ - var callbackExc: Exception = _ - - private def newToken(): Token = { - val t = new Array[Byte](16) - Random.nextBytes(t) - t - } - - private def connect(): DataSocket = { - var s = socket - if (s == null) { - s = new DataSocket(new Socket(host, 5053)) - info(s"SchedulerAppClient.getConnection: connected to $host:5053") - writeByteArray(token, s.out) - s.out.flush() - socket = s - } - s - } - - private def closeConnection(): Unit = { - val s = socket - socket = null - if (s != null) - s.socket.close() - } - - private def withConnection(f: DataSocket => Unit): Unit = { - try { - val s = retry(connect) - f(s) - } catch { - case e: Exception => - closeConnection() - throw e - } - } - - private def sendAckTask(s: DataSocket, index: Int, taskID: Token): Unit = { - s.out.writeInt(ClientMessage.ACKTASKRESULT) - writeByteArray(taskID, s.out) - s.out.writeInt(index) - s.out.flush() - } - - private def clear(): Unit = { - callback = null - receivedTasks = null - callbackExc = null - } - - private def receive(taskID: Token): Unit = { - withConnection { s => - while (nComplete < nTasks) { - val msg = s.in.readInt() - assert(msg == ClientMessage.APPTASKRESULT) - val taskToken = readByteArray(s.in) - assert(taskID sameElements taskToken) - - val index = s.in.readInt() - val res = readObject[Any](s.in) - log.info(s"SchedulerAppClient.receive: received task result for index $index / $nTasks.") - - res match { - case re: RemoteException => - // we are done here - clear() - throw new BreakRetryException(re.getCause) - case _ => - if (!receivedTasks.contains(index)) { - try { - callback(index, res) - } catch { - case e: Exception => - log.info(s"SchedulerAppClient.receive: got callback exception for task $index / $nTasks:\n$e") - callbackExc = e - } - receivedTasks += index - nComplete += 1 - } - log.info(s"SchedulerAppClient.receive: tasks completed: $nComplete / $nTasks.") - sendAckTask(s, index, taskID) - } - } - } - } - - def submit[T](da: DArray[T], cb: (Int, T) => Unit): Unit = synchronized { - val taskID = newToken() - log.info(s"SchedulerAppClient.submit: submitting task with ${ da.nTasks } tasks") - q.put(Some(taskID -> da)) - - assert(callback == null) - nTasks = da.nTasks - nComplete = 0 - callback = (i: Int, x: Any) => cb(i, x.asInstanceOf[T]) - receivedTasks = mutable.BitSet(nTasks) - - log.info(s"SchedulerAppClient.submit: finished submitting task with ${ da.nTasks } tasks") - retry(() => receive(taskID)) - val ce = callbackExc - clear() - - if (ce != null) - throw ce - } - - def close(): Unit = { - q.put(None) - } -} diff --git a/hail/src/main/scala/is/hail/scheduler/DArray.scala b/hail/src/main/scala/is/hail/scheduler/DArray.scala deleted file mode 100644 index f4a0c0e00bd..00000000000 --- a/hail/src/main/scala/is/hail/scheduler/DArray.scala +++ /dev/null @@ -1,10 +0,0 @@ -package is.hail.scheduler - -abstract class DArray[T] { - type Context - - val contexts: Array[Context] - val body: Context => T - - def nTasks: Int = contexts.length -} diff --git a/hail/src/main/scala/is/hail/scheduler/Executor.scala b/hail/src/main/scala/is/hail/scheduler/Executor.scala deleted file mode 100644 index 2aee6e4ca74..00000000000 --- a/hail/src/main/scala/is/hail/scheduler/Executor.scala +++ /dev/null @@ -1,160 +0,0 @@ -package is.hail.scheduler - -import java.io._ -import java.net.Socket -import java.util.concurrent.Executors -import scala.collection.mutable - -import is.hail.utils._ - -class RemoteException(cause: Throwable) extends Exception(cause) - -object ExecutorMessage { - // Scheduler => Executor - val EXECUTE = 2 - - // Executor => Scheduler - val PING = 1 - val TASKRESULT = 4 -} - -class TaskThread[T](ex: Executor, taskId: Int, f: () => T) extends Runnable { - def run(): Unit = { - val v = try { - f() - } catch { - case e: Exception => - val re = new RemoteException(e) - ex.sendTaskResult(taskId, re) - return - } - ex.sendTaskResult(taskId, v) - } -} - -class PingThread(ex: Executor) extends Runnable { - def run(): Unit = { - while (true) { - try { - ex.sendPing() - } catch { - case e: Exception => - log.info(s"PingThread.run: sendPing failed due to exception $e") - } - Thread.sleep(5000) - } - } -} - -class TaskResult[T](val taskId: Int, val result: T) - - -class DataSocket(val socket: Socket) { - val in = new DataInputStream(new BufferedInputStream(socket.getInputStream)) - val out = new DataOutputStream(new BufferedOutputStream(socket.getOutputStream)) -} - -class Executor(host: String, nCores: Int) extends Runnable { - private var pool = Executors.newFixedThreadPool(nCores) - - @volatile private var socket: DataSocket = _ - private val outLock = new Object - - private var pendingResults = new mutable.ArrayBuffer[TaskResult[_]]() - - def sendPing(): Unit = outLock.synchronized { - val s = socket - if (s != null) { - try { - s.out.writeInt(ExecutorMessage.PING) - s.out.flush() - } catch { - case e: Exception => - log.error(s"Client.sendPing: failed due to exception $e") - closeConnection() - throw e - } - } - } - - def sendTaskResult[T](taskId: Int, result: T): Unit = outLock.synchronized { - val s = socket - if (s != null) { - try { - s.out.writeInt(ExecutorMessage.TASKRESULT) - s.out.writeInt(taskId) - writeObject(result, s.out) - s.out.flush() - log.info(s"sent task $taskId result") - } catch { - case e: Exception => - log.error(s"Client.sendTaskResult: queuing result, send failed due to exception: $e") - pendingResults += new TaskResult(taskId, result) - closeConnection() - } - } else { - pendingResults += new TaskResult(taskId, result) - } - } - - def handleExecute[T](): Unit = { - val s = socket - val taskId = s.in.readInt() - val f = readObject[() => _](s.in) - log.info(s"received task $taskId") - pool.execute(new TaskThread(this, taskId, f)) - } - - def run1(): Unit = { - try { - val s = new DataSocket(new Socket(host, 5051)) - - s.out.writeInt(nCores) - s.out.flush() - - socket = s - log.info(s"Client.run1: connected to $host:5051") - - while (pendingResults.nonEmpty) { - val tr = pendingResults.last - sendTaskResult(tr.taskId, tr.result) - pendingResults.reduceToSize(pendingResults.size - 1) - } - - while (true) { - val msg = s.in.readInt() - msg match { - case ExecutorMessage.EXECUTE => - handleExecute() - } - } - } finally { - closeConnection() - } - } - - def run(): Unit = { - retry(run1) - } - - - def closeConnection(): Unit = { - val s = socket - socket = null - if (s != null) - s.socket.close() - } -} - -object Executor { - def main(args: Array[String]): Unit = { - val host = args(0) - val nCores = args(1).toInt - - val ex = new Executor(host, nCores) - val pt = new Thread(new PingThread(ex)) - pt.start() - - ex.run() - } -} diff --git a/hail/src/main/scala/is/hail/scheduler/package.scala b/hail/src/main/scala/is/hail/scheduler/package.scala deleted file mode 100644 index 113efefe0ae..00000000000 --- a/hail/src/main/scala/is/hail/scheduler/package.scala +++ /dev/null @@ -1,60 +0,0 @@ -package is.hail - -import java.io._ - -import scala.util.Random -import is.hail.utils._ - -package object scheduler { - def writeByteArray(b: Array[Byte], out: DataOutputStream): Unit = { - out.writeInt(b.length) - out.write(b) - } - - def writeObject[T](v: T, out: DataOutputStream): Unit = { - val bos = new ByteArrayOutputStream() - val boos = new ObjectOutputStream(bos) - boos.writeObject(v) - writeByteArray(bos.toByteArray, out) - } - - def readByteArray(in: DataInputStream): Array[Byte] = { - val n = in.readInt() - val b = new Array[Byte](n) - in.readFully(b) - b - } - - def readObject[T](in: DataInputStream): T = { - val b = readByteArray(in) - val bis = new ByteArrayInputStream(b) - val bois = new ObjectInputStream(bis) - bois.readObject().asInstanceOf[T] - } - - def retry[T](f: () => T, exp: Double = 2.0, maxWait: Double = 60.0): T = { - var minWait = 1.0 - var w = minWait - while (true) { - val startTime = System.nanoTime() - try { - return f() - } catch { - case b: BreakRetryException => - throw b.getCause - case e: Exception => - log.warn(s"retry: restarting due to exception: $e") - e.printStackTrace() - } - val endTime = System.nanoTime() - val duration = (endTime - startTime) / 1e-9 - w = math.min(maxWait, math.max(minWait, w * exp - duration)) - val t = (1000 * w * Random.nextDouble).toLong - log.info(s"retry: waiting ${ formatTime(t * 1000000) }") - Thread.sleep(t) - } - null.asInstanceOf[T] - } - - type Token = Array[Byte] -} diff --git a/hail/src/main/scala/is/hail/utils/richUtils/RichIterator.scala b/hail/src/main/scala/is/hail/utils/richUtils/RichIterator.scala index 6aa554e8c20..c4aaef7480a 100644 --- a/hail/src/main/scala/is/hail/utils/richUtils/RichIterator.scala +++ b/hail/src/main/scala/is/hail/utils/richUtils/RichIterator.scala @@ -40,18 +40,17 @@ class RichIterator[T](val it: Iterator[T]) extends AnyVal { def pipe(pb: ProcessBuilder, printHeader: (String => Unit) => Unit, printElement: (String => Unit, T) => Unit, - printFooter: (String => Unit) => Unit): (Iterator[String], Process) = { + printFooter: (String => Unit) => Unit): (Iterator[String], StringBuilder, Process) = { val command = pb.command().asScala.mkString(" ") val proc = pb.start() - // Start a thread to print the process's stderr to ours + val error = new StringBuilder() + // Start a thread capture the process stderr new Thread("stderr reader for " + command) { override def run() { - for (line <- Source.fromInputStream(proc.getErrorStream).getLines) { - System.err.println(line) - } + Source.fromInputStream(proc.getErrorStream).addString(error) } }.start() @@ -67,8 +66,10 @@ class RichIterator[T](val it: Iterator[T]) extends AnyVal { } }.start() - // Return an iterator that read lines from the process's stdout - (Source.fromInputStream(proc.getInputStream).getLines(), proc) + // Return an iterator that reads lines from the process's stdout, + // a StringBuilder that captures standard error that should not be read + // from or written to until waiting for the process, and the process itself + (Source.fromInputStream(proc.getInputStream).getLines(), error, proc) } def trueGroupedIterator(groupSize: Int): Iterator[Iterator[T]] = diff --git a/hail/src/main/scala/is/hail/variant/HardCallView.scala b/hail/src/main/scala/is/hail/variant/HardCallView.scala index 3fded60d733..a4cd9bcb6ca 100644 --- a/hail/src/main/scala/is/hail/variant/HardCallView.scala +++ b/hail/src/main/scala/is/hail/variant/HardCallView.scala @@ -22,8 +22,11 @@ final class ArrayGenotypeView(rvType: PStruct) { } private val (gtExists, gtIndex, gtType) = lookupField("GT", _ == PCall()) - private val (gpExists, gpIndex, gpType: PArray) = lookupField("GP", + private val (gpExists, gpIndex, _gpType) = lookupField("GP", pt => pt.isInstanceOf[PArray] && pt.asInstanceOf[PArray].elementType.isInstanceOf[PFloat64]) + // Do not try to move this cast into the destructuring above + // https://stackoverflow.com/questions/27789412/scala-exception-in-for-comprehension-with-type-annotation + private[this] val gpType = _gpType.asInstanceOf[PArray] private var gsOffset: Long = _ private var gsLength: Int = _ diff --git a/hail/src/test/scala/is/hail/HailSuite.scala b/hail/src/test/scala/is/hail/HailSuite.scala index 462a105b64f..5f9b15273c1 100644 --- a/hail/src/test/scala/is/hail/HailSuite.scala +++ b/hail/src/test/scala/is/hail/HailSuite.scala @@ -9,9 +9,6 @@ import org.scalatest.testng.TestNGSuite import org.testng.annotations.BeforeClass object HailSuite { - def withDistributedBackend(host: String): HailContext = - HailContext.createDistributed(host, logFile = "/tmp/hail.log") - def withSparkBackend(): HailContext = HailContext( sc = new SparkContext( @@ -25,11 +22,7 @@ object HailSuite { lazy val hc: HailContext = { - val schedulerHost = System.getenv("HAIL_TEST_SCHEDULER_HOST") - val hc = if (schedulerHost == null) - withSparkBackend() - else - withDistributedBackend(schedulerHost) + val hc = withSparkBackend() hc.flags.set("lower", "1") hc.checkRVDKeys = true hc diff --git a/hail/src/test/scala/is/hail/TestUtils.scala b/hail/src/test/scala/is/hail/TestUtils.scala index 63cf41c9686..b858eff58a9 100644 --- a/hail/src/test/scala/is/hail/TestUtils.scala +++ b/hail/src/test/scala/is/hail/TestUtils.scala @@ -21,7 +21,7 @@ import org.apache.spark.sql.Row object ExecStrategy extends Enumeration { type ExecStrategy = Value - val Interpret, InterpretUnoptimized, JvmCompile, LoweredJVMCompile = Value + val Interpret, InterpretUnoptimized, JvmCompile, LoweredJVMCompile, JvmCompileUnoptimized = Value val compileOnly: Set[ExecStrategy] = Set(JvmCompile) val javaOnly: Set[ExecStrategy] = Set(Interpret, InterpretUnoptimized, JvmCompile) @@ -146,7 +146,6 @@ object TestUtils { None } - def loweredExecute(x: IR, env: Env[(Any, Type)], args: IndexedSeq[(Any, Type)], agg: Option[(IndexedSeq[Row], TStruct)], @@ -163,7 +162,8 @@ object TestUtils { env: Env[(Any, Type)], args: IndexedSeq[(Any, Type)], agg: Option[(IndexedSeq[Row], TStruct)], - bytecodePrinter: Option[PrintWriter] = None + bytecodePrinter: Option[PrintWriter] = None, + optimize: Boolean = true ): Any = { ExecuteContext.scoped { ctx => val inputTypesB = new ArrayBuilder[Type]() @@ -215,7 +215,8 @@ object TestUtils { argsVar, argsPType, aggArrayVar, aggArrayPType, aggIR, - print = bytecodePrinter) + print = bytecodePrinter, + optimize = optimize) assert(resultType2.virtualType.isOfType(resultType)) Region.scoped { region => @@ -246,7 +247,7 @@ object TestUtils { val (resultType2, f) = Compile[Long, Long](ctx, argsVar, argsPType, MakeTuple.ordered(FastSeq(rewrite(Subst(x, BindingEnv(substEnv))))), - optimize = true, + optimize = optimize, print = bytecodePrinter) assert(resultType2.virtualType.isOfType(resultType)) @@ -352,6 +353,16 @@ object TestUtils { pw.print(s"/* JVM bytecode dump for IR:\n${Pretty(x)}\n */\n\n") pw }) + case ExecStrategy.JvmCompileUnoptimized => + assert(Forall(x, node => node.isInstanceOf[IR] && Compilable(node.asInstanceOf[IR]))) + eval(x, env, args, agg, bytecodePrinter = + Option(HailContext.getFlag("jvm_bytecode_dump")) + .map { path => + val pw = new PrintWriter(new File(path)) + pw.print(s"/* JVM bytecode dump for IR:\n${Pretty(x)}\n */\n\n") + pw + }, + optimize = false) case ExecStrategy.LoweredJVMCompile => loweredExecute(x, env, args, agg) } assert(t.typeCheck(res)) diff --git a/hail/src/test/scala/is/hail/annotations/StagedRegionValueSuite.scala b/hail/src/test/scala/is/hail/annotations/StagedRegionValueSuite.scala index d70b36f1761..70ac541e7f6 100644 --- a/hail/src/test/scala/is/hail/annotations/StagedRegionValueSuite.scala +++ b/hail/src/test/scala/is/hail/annotations/StagedRegionValueSuite.scala @@ -495,4 +495,79 @@ class StagedRegionValueSuite extends HailSuite { } p.check() } + + @Test def testUnstagedCopy() { + val t1 = PCanonicalArray(PCanonicalStruct( + true, + "x1" -> PInt32(), + "x2" -> PArray(PInt32(), required = true), + "x3" -> PArray(PInt32(true), required = true), + "x4" -> PSet(PCanonicalStruct(true, "y" -> PString(true)), required = false) + ), required = false) + val t2 = t1.deepInnerRequired(false) + + val value = IndexedSeq( + Row(1, IndexedSeq(1,2,3), IndexedSeq(0, -1), Set(Row("asdasdasd"), Row(""))), + Row(1, IndexedSeq(), IndexedSeq(-1), Set(Row("aa"))) + ) + + Region.scoped { r => + val rvb = new RegionValueBuilder(r) + rvb.start(t2) + rvb.addAnnotation(t2.virtualType, value) + val v1 = rvb.end() + assert(SafeRow.read(t2, r, v1) == value) + + rvb.clear() + rvb.start(t1) + rvb.addRegionValue(t2, r, v1) + val v2 = rvb.end() + assert(SafeRow.read(t1, r, v2) == value) + } + } + + @Test def testStagedCopy() { + val t1 = PCanonicalStruct(false, "a" -> PCanonicalArray(PCanonicalStruct( + true, + "x1" -> PInt32(), + "x2" -> PArray(PInt32(), required = true), + "x3" -> PArray(PInt32(true), required = true), + "x4" -> PSet(PCanonicalStruct(true, "y" -> PString(true)), required = false) + ), required = false)) + val t2 = t1.deepInnerRequired(false).asInstanceOf[PStruct] + + val value = IndexedSeq( + Row(1, IndexedSeq(1,2,3), IndexedSeq(0, -1), Set(Row("asdasdasd"), Row(""))), + Row(1, IndexedSeq(), IndexedSeq(-1), Set(Row("aa"))) + ) + + val valueT2 = t2.types(0) + Region.scoped { r => + val rvb = new RegionValueBuilder(r) + rvb.start(valueT2) + rvb.addAnnotation(valueT2.virtualType, value) + val v1 = rvb.end() + assert(SafeRow.read(valueT2, r, v1) == value) + + val f1 = EmitFunctionBuilder[Long]("stagedCopy1") + val srvb = new StagedRegionValueBuilder(f1.apply_method, t2, f1.partitionRegion) + f1.emit(Code( + srvb.start(), + srvb.addIRIntermediate(t2.types(0))(v1), + srvb.end() + )) + val cp1 = f1.resultWithIndex()(0, r)() + assert(SafeRow.read(t2, r, cp1) == Row(value)) + + val f2 = EmitFunctionBuilder[Long]("stagedCopy2") + val srvb2 = new StagedRegionValueBuilder(f2.apply_method, t1, f2.partitionRegion) + f2.emit(Code( + srvb2.start(), + srvb2.addIRIntermediate(t2.types(0))(v1), + srvb2.end() + )) + val cp2 = f2.resultWithIndex()(0, r)() + assert(SafeRow.read(t1, r, cp2) == Row(value)) + } + } } diff --git a/hail/src/test/scala/is/hail/annotations/UnsafeSuite.scala b/hail/src/test/scala/is/hail/annotations/UnsafeSuite.scala index 4f2a43d4bbd..635f7e70602 100644 --- a/hail/src/test/scala/is/hail/annotations/UnsafeSuite.scala +++ b/hail/src/test/scala/is/hail/annotations/UnsafeSuite.scala @@ -283,8 +283,8 @@ class UnsafeSuite extends HailSuite { @Test def testPacking() { - def makeStruct(types: PType*): PStruct = { - PStruct(types.zipWithIndex.map { case (t, i) => (s"f$i", t) }: _*) + def makeStruct(types: PType*): PCanonicalStruct = { + PCanonicalStruct(types.zipWithIndex.map { case (t, i) => (s"f$i", t) }: _*) } val t1 = makeStruct( // missing byte is 0 diff --git a/hail/src/test/scala/is/hail/expr/ir/Aggregators2Suite.scala b/hail/src/test/scala/is/hail/expr/ir/Aggregators2Suite.scala index a399fb4ee9e..d00a6a122ef 100644 --- a/hail/src/test/scala/is/hail/expr/ir/Aggregators2Suite.scala +++ b/hail/src/test/scala/is/hail/expr/ir/Aggregators2Suite.scala @@ -412,7 +412,7 @@ class Aggregators2Suite extends HailSuite { Begin(FastIndexedSeq( SeqOp(aggIdx, FastIndexedSeq(ArrayLen(a)), state, AggElementsLengthCheck()), - ArrayFor(ArrayRange(0, ArrayLen(a), 1), idx.name, + ArrayFor(StreamRange(0, ArrayLen(a), 1), idx.name, Let(elt.name, ArrayRef(a, idx), SeqOp(aggIdx, FastIndexedSeq(idx, seqOps(elt)), state, AggElements()))))) } @@ -710,7 +710,7 @@ class Aggregators2Suite extends HailSuite { implicit val execStrats = ExecStrategy.compileOnly val sig = AggSignature(Sum(), FastSeq(), FastSeq(TFloat64())) val x = RunAggScan( - ArrayRange(I32(0), I32(5), I32(1)), + StreamRange(I32(0), I32(5), I32(1)), "foo", InitOp(0, FastSeq(), sig), SeqOp(0, FastIndexedSeq(Ref("foo", TInt32()).toD), sig), @@ -740,7 +740,7 @@ class Aggregators2Suite extends HailSuite { Begin(FastSeq( InitOp(0, FastSeq(I32(5)), takeSig), ArrayFor( - ArrayRange(I32(0), I32(10), I32(1)), + StreamRange(I32(0), I32(10), I32(1)), "foo", SeqOp(0, FastSeq( RunAgg( diff --git a/hail/src/test/scala/is/hail/expr/ir/ArrayDeforestationSuite.scala b/hail/src/test/scala/is/hail/expr/ir/ArrayDeforestationSuite.scala index ef1284e7b11..9612714a94b 100644 --- a/hail/src/test/scala/is/hail/expr/ir/ArrayDeforestationSuite.scala +++ b/hail/src/test/scala/is/hail/expr/ir/ArrayDeforestationSuite.scala @@ -12,15 +12,15 @@ class ArrayDeforestationSuite extends HailSuite { def primitiveArrayNoRegion(len: IR): IR = ArrayMap( - ArrayRange(0, len, 1), + StreamRange(0, len, 1), "x1", Ref("x1", TInt32()) + 5) def arrayWithRegion(len: IR): IR = - ArrayMap( - ArrayRange(0, len, 1), + ToArray(ArrayMap( + StreamRange(0, len, 1), "x2", - MakeStruct(FastSeq[(String, IR)]("f1" -> (Ref("x2", TInt32()) + 1), "f2" -> 0))) + MakeStruct(FastSeq[(String, IR)]("f1" -> (Ref("x2", TInt32()) + 1), "f2" -> 0)))) def primitiveArrayWithRegion(len: IR): IR = { val array = arrayWithRegion(len) diff --git a/hail/src/test/scala/is/hail/expr/ir/EmitStreamSuite.scala b/hail/src/test/scala/is/hail/expr/ir/EmitStreamSuite.scala index 9e90a50ecba..fdfa90141d8 100644 --- a/hail/src/test/scala/is/hail/expr/ir/EmitStreamSuite.scala +++ b/hail/src/test/scala/is/hail/expr/ir/EmitStreamSuite.scala @@ -1,7 +1,7 @@ package is.hail.expr.ir import is.hail.annotations.{Region, RegionValue, RegionValueBuilder, SafeRow, ScalaToRegionValue} -import is.hail.asm4s.{AsmFunction1, AsmFunction3, Code, GenericTypeInfo, MaybeGenericTypeInfo, TypeInfo} +import is.hail.asm4s._ import is.hail.asm4s.joinpoint._ import is.hail.expr.types.physical._ import is.hail.expr.types.virtual._ @@ -13,11 +13,257 @@ import org.apache.spark.sql.Row import org.testng.annotations.Test class EmitStreamSuite extends HailSuite { + private def compile1[T: TypeInfo, R: TypeInfo](f: (EmitMethodBuilder, Code[T]) => Code[R]): T => R = { + val fb = EmitFunctionBuilder[T, R]("stream_test") + val mb = fb.apply_method + mb.emit(f(mb, mb.getArg[T](1))) + val asmFn = fb.result()() + asmFn.apply + } + + private def compile2[T: TypeInfo, U: TypeInfo, R: TypeInfo](f: (MethodBuilder, Code[T], Code[U]) => Code[R]): (T, U) => R = { + val fb = FunctionBuilder.functionBuilder[T, U, R] + val mb = fb.apply_method + mb.emit(f(mb, mb.getArg[T](1), mb.getArg[U](2))) + val asmFn = fb.result()() + asmFn.apply + } + + private def compile3[T: TypeInfo, U: TypeInfo, V: TypeInfo, R: TypeInfo](f: (MethodBuilder, Code[T], Code[U], Code[V]) => Code[R]): (T, U, V) => R = { + val fb = FunctionBuilder.functionBuilder[T, U, V, R] + val mb = fb.apply_method + mb.emit(f(mb, mb.getArg[T](1), mb.getArg[U](2), mb.getArg[V](3))) + val asmFn = fb.result()() + asmFn.apply + } + + def facStaged(n: Code[Int], ret: Code[Int] => Code[Ctrl] + )(implicit ctx: EmitStreamContext + ): Code[Ctrl] = { + val r = CodeStream.range(1, 1, n) + CodeStream.fold[Code[Int], Code[Int]](r, 1, (i, prod) => prod * i, ret) + } + + def range(start: Code[Int], stop: Code[Int], name: String)(implicit ctx: EmitStreamContext): CodeStream.Stream[Code[Int]] = + CodeStream.map(CodeStream.range(start, 1, stop - start))( + a => a, + setup0 = Some(Code._println(const(s"$name setup0"))), + setup = Some(Code._println(const(s"$name setup"))), + close0 = Some(Code._println(const(s"$name close0"))), + close = Some(Code._println(const(s"$name close")))) + + class CheckedStream[T](_stream: CodeStream.Stream[T], name: String, mb: MethodBuilder) { + val outerBit = mb.newLocal[Boolean] + val innerBit = mb.newLocal[Boolean] + val innerCount = mb.newLocal[Int] + + def init: Code[Unit] = Code(outerBit := false, innerBit := false, innerCount := 0) + + val stream: CodeStream.Stream[T] = _stream.mapCPS( + (ctx, a, k) => (outerBit & innerBit).mux( + k(a), + Code._fatal(s"$name: pulled from when not setup")), + setup0 = Some((!outerBit & !innerBit).mux( + Code(outerBit := true, + Code._println(const(s"$name setup0"))), + Code._fatal(s"$name: setup0 run out of order"))), + setup = Some((outerBit & !innerBit).mux( + Code(innerBit := true, + innerCount := innerCount.load + 1, + Code._println(const(s"$name setup"))), + Code._fatal(s"$name: setup run out of order"))), + close0 = Some((outerBit & !innerBit).mux( + Code(outerBit := false, + Code._println(const(s"$name close0"))), + Code._fatal(s"$name: close0 run out of order"))), + close = Some((outerBit & innerBit).mux( + Code(innerBit := false, + Code._println(const(s"$name close"))), + Code._fatal(s"$name: close run out of order")))) + + def assertClosed(expectedRuns: Code[Int]): Code[Unit] = + (outerBit | innerBit).mux( + Code._fatal(s"$name: not closed"), + innerCount.cne(expectedRuns).mux( + Code._fatal(const(s"$name: expected ").concat(expectedRuns.toS).concat(" runs, found ").concat(innerCount.toS)), + Code._empty)) + + def assertClosed: Code[Unit] = + (outerBit | innerBit).mux( + Code._fatal(s"$name: not closed"), + Code._empty) + } + + def checkedRange(start: Code[Int], stop: Code[Int], name: String, mb: MethodBuilder): CheckedStream[Code[Int]] = + new CheckedStream(CodeStream.range(start, 1, stop - start), name, mb) + + @Test def testES2Range() { + val f = compile1[Int, Unit] { (mb, n) => + val r = checkedRange(0, n, "range", mb) + + Code( + r.init, + r.stream.forEach(mb)(i => Code._println(i.toS)), + r.assertClosed(1)) + } + for (i <- 0 to 2) { f(i) } + } + + @Test def testES2Zip() { + val f = compile2[Int, Int, Unit] { (mb, m, n) => + val l = checkedRange(0, m, "left", mb) + val r = checkedRange(0, n, "right", mb) + val z = CodeStream.zip(l.stream, r.stream) + + Code( + l.init, r.init, + z.forEach(mb)(x => Code._println(const("(").concat(x._1.toS).concat(", ").concat(x._2.toS).concat(")"))), + l.assertClosed(1), r.assertClosed(1)) + } + for { + i <- 0 to 2 + j <- 0 to 2 + } { + f(i, j) + } + } + + @Test def testES2FlatMap() { + val f = compile1[Int, Unit] { (mb, n) => + val outer = checkedRange(1, n, "outer", mb) + var inner: CheckedStream[Code[Int]] = null + def f(i: Code[Int]) = { + inner = checkedRange(0, i, "inner", mb) + inner.stream + } + val run = outer.stream.flatMap(f).forEach(mb)(i => Code._println(i.toS)) + + Code( + outer.init, inner.init, + run, + outer.assertClosed(1), + inner.assertClosed(n - 1)) + } + for (n <- 1 to 5) { f(n) } + } + + @Test def testES2ZipNested() { + val f = compile2[Int, Int, Unit] { (mb, m, n) => + val l = checkedRange(1, m, "left", mb) + + val rOuter = checkedRange(1, n, "right outer", mb) + var rInner: CheckedStream[Code[Int]] = null + + def f(i: Code[Int]) = { + rInner = checkedRange(0, i, "right inner", mb) + rInner.stream + } + val run = CodeStream.zip(l.stream, rOuter.stream.flatMap(f)) + .forEach(mb)(x => Code._println(const("(").concat(x._1.toS).concat(", ").concat(x._2.toS).concat(")"))) + + Code( + l.init, rOuter.init, rInner.init, + run, + l.assertClosed(1), + rOuter.assertClosed(1), + rInner.assertClosed) + } + f(1, 1) + f(1, 2) + f(2, 1) + f(2, 2) + f(2, 3) + f(10, 3) + } + + @Test def testES2Filter() { + val f = compile1[Int, Unit] { (mb, n) => + val r = checkedRange(0, n, "source", mb) + def cond(i: Code[Int]): Code[Boolean] = (i % 2).ceq(0) + + Code( + r.init, + r.stream.filter(cond).forEach(mb)(i => Code._println(i.toS)), + r.assertClosed(1)) + } + f(0) + f(10) + } + + @Test def testES2Mux() { + val f = compile2[Boolean, Int, Unit] { (mb, cond, n) => + val l = checkedRange(0, n, "left", mb) + val r = checkedRange(0, n, "right", mb) + + Code( + r.init, + l.init, + CodeStream.mux(cond, l.stream, r.stream).forEach(mb)(i => Code._println(i.toS)), + l.assertClosed(cond.toI), + r.assertClosed(const(1) - cond.toI)) + } + f(false, 2) + f(true, 2) + } + + @Test def testES2ZipMux() { + val f = compile2[Boolean, Int, Unit] { (mb, cond, n) => + val l1 = checkedRange(0, n, "left1", mb) + val l2 = checkedRange(0, n, "left2", mb) + val mux = CodeStream.mux(cond, l1.stream, l2.stream) + val r = checkedRange(0, n, "right", mb) + + Code( + l1.init, l2.init, r.init, + CodeStream.zip(mux, r.stream).forEach(mb)(x => Code._println(const("(").concat(x._1.toS).concat(", ").concat(x._2.toS).concat(")"))), + l1.assertClosed(cond.toI), + l2.assertClosed(const(1) - cond.toI), + r.assertClosed(1)) + } + f(false, 2) + f(true, 2) + } + + @Test def testES2MultiZip() { + import scala.collection.IndexedSeq + val f = compile3[Int, Int, Int, Unit] { (mb, n1, n2, n3) => + val s1 = checkedRange(0, n1, "s1", mb) + val s2 = checkedRange(0, n2, "s2", mb) + val s3 = checkedRange(0, n3, "s3", mb) + val z = CodeStream.multiZip(IndexedSeq(s1.stream, s2.stream, s3.stream)).asInstanceOf[CodeStream.Stream[IndexedSeq[Code[Int]]]] + + Code( + s1.init, s2.init, s3.init, + z.forEach(mb)(x => Code._println(const("(").concat(x(0).toS).concat(", ").concat(x(1).toS).concat(", ").concat(x(2).toS).concat(")"))), + s1.assertClosed(1), + s2.assertClosed(1), + s3.assertClosed(1)) + } + for { + n1 <- 0 to 2 + n2 <- 0 to 2 + n3 <- 0 to 2 + } { + f(n1, n2, n3) + } + } + + @Test def testES2Fac() { + def fac(n: Int): Int = (1 to n).fold(1)(_ * _) + val facS = compile1[Int, Int] { (mb, n) => + JoinPoint.CallCC[Code[Int]] { (jpb, ret) => + implicit val ctx = EmitStreamContext(mb, jpb) + facStaged(n, ret) + } + } + for (i <- 0 to 12) + assert(facS(i) == fac(i), s"compute: $i!") + } - private def compileStream[F >: Null : TypeInfo, A]( + private def compileStream[F >: Null : TypeInfo, T]( streamIR: IR, inputTypes: Seq[PType] - )(call: (F, Region, A) => Long): A => IndexedSeq[Any] = { + )(call: (F, Region, T) => Long): T => IndexedSeq[Any] = { val argTypeInfos = new ArrayBuilder[MaybeGenericTypeInfo[_]] argTypeInfos += GenericTypeInfo[Region]() inputTypes.foreach { t => @@ -35,7 +281,7 @@ class EmitStreamSuite extends HailSuite { Code(arrayt.setup, arrayt.m.mux(0L, arrayt.v)) } val f = fb.resultWithIndex() - (arg: A) => Region.scoped { r => + (arg: T) => Region.scoped { r => val off = call(f(0, r), r, arg) if (off == 0L) null @@ -83,7 +329,8 @@ class EmitStreamSuite extends HailSuite { JoinPoint.CallCC[Code[Int]] { (jb, ret) => val str = stream.stream val mb = fb.apply_method - str.init(mb, jb, ()) { + implicit val ctx = EmitStreamContext(mb, jb) + str.init(()) { case EmitStream.Missing => ret(0) case EmitStream.Start(s0) => str.length(s0) match { diff --git a/hail/src/test/scala/is/hail/expr/ir/ForwardLetsSuite.scala b/hail/src/test/scala/is/hail/expr/ir/ForwardLetsSuite.scala index 91738428ace..c96b7df599b 100644 --- a/hail/src/test/scala/is/hail/expr/ir/ForwardLetsSuite.scala +++ b/hail/src/test/scala/is/hail/expr/ir/ForwardLetsSuite.scala @@ -11,13 +11,13 @@ import org.testng.annotations.{DataProvider, Test} class ForwardLetsSuite extends HailSuite { @DataProvider(name = "nonForwardingOps") def nonForwardingOps(): Array[Array[IR]] = { - val a = ArrayRange(I32(0), I32(10), I32(1)) + val a = ToArray(StreamRange(I32(0), I32(10), I32(1))) val x = Ref("x", TInt32()) val y = Ref("y", TInt32()) Array( ArrayMap(a, "y", ApplyBinaryPrimOp(Add(), x, y)), ArrayFilter(a, "y", ApplyComparisonOp(LT(TInt32()), x, y)), - ArrayFlatMap(a, "y", ArrayRange(x, y, I32(1))), + ToArray(ArrayFlatMap(a, "y", StreamRange(x, y, I32(1)))), ArrayFold(a, I32(0), "acc", "y", ApplyBinaryPrimOp(Add(), ApplyBinaryPrimOp(Add(), x, y), Ref("acc", TInt32()))), ArrayFold2(a, FastSeq(("acc", I32(0))), "y", FastSeq(x + y + Ref("acc", TInt32())), Ref("acc", TInt32())), ArrayScan(a, I32(0), "acc", "y", ApplyBinaryPrimOp(Add(), ApplyBinaryPrimOp(Add(), x, y), Ref("acc", TInt32()))), @@ -42,7 +42,7 @@ class ForwardLetsSuite extends HailSuite { @DataProvider(name = "nonForwardingAggOps") def nonForwardingAggOps(): Array[Array[IR]] = { - val a = ArrayRange(I32(0), I32(10), I32(1)) + val a = StreamRange(I32(0), I32(10), I32(1)) val x = Ref("x", TInt32()) val y = Ref("y", TInt32()) Array( @@ -60,8 +60,8 @@ class ForwardLetsSuite extends HailSuite { If(True(), x, I32(0)), ApplyBinaryPrimOp(Add(), ApplyBinaryPrimOp(Add(), I32(2), x), I32(1)), ApplyUnaryPrimOp(Negate(), x), - ArrayMap(ArrayRange(I32(0), x, I32(1)), "foo", Ref("foo", TInt32())), - ArrayFilter(ArrayRange(I32(0), x, I32(1)), "foo", Ref("foo", TInt32()) <= I32(0)) + ToArray(ArrayMap(StreamRange(I32(0), x, I32(1)), "foo", Ref("foo", TInt32()))), + ToArray(ArrayFilter(StreamRange(I32(0), x, I32(1)), "foo", Ref("foo", TInt32()) <= I32(0))) ).map(ir => Array[IR](Let("x", In(0, TInt32()) + In(0, TInt32()), ir))) } diff --git a/hail/src/test/scala/is/hail/expr/ir/IRSuite.scala b/hail/src/test/scala/is/hail/expr/ir/IRSuite.scala index 4c661a8d85f..59d933ca21e 100644 --- a/hail/src/test/scala/is/hail/expr/ir/IRSuite.scala +++ b/hail/src/test/scala/is/hail/expr/ir/IRSuite.scala @@ -825,7 +825,7 @@ class IRSuite extends HailSuite { If( In(0, TBoolean()), In(1, t), - MakeStruct(Seq("foo" -> MakeStruct(Seq("bar" -> ArrayRange(I32(0), I32(1), I32(1)))))) + MakeStruct(Seq("foo" -> MakeStruct(Seq("bar" -> ToArray(StreamRange(I32(0), I32(1), I32(1))))))) ), FastIndexedSeq((true, TBoolean()), (value, t)), value @@ -836,14 +836,14 @@ class IRSuite extends HailSuite { assertEvalsTo(Let("v", I32(5), Ref("v", TInt32())), 5) assertEvalsTo(Let("v", NA(TInt32()), Ref("v", TInt32())), null) assertEvalsTo(Let("v", I32(5), NA(TInt32())), null) - assertEvalsTo(ArrayMap(Let("v", I32(5), ArrayRange(0, Ref("v", TInt32()), 1)), "x", Ref("x", TInt32()) + I32(2)), + assertEvalsTo(ToArray(ArrayMap(Let("v", I32(5), StreamRange(0, Ref("v", TInt32()), 1)), "x", Ref("x", TInt32()) + I32(2))), FastIndexedSeq(2, 3, 4, 5, 6)) assertEvalsTo( - ArrayMap(Let("q", I32(2), + ToArray(ArrayMap(Let("q", I32(2), ArrayMap(Let("v", Ref("q", TInt32()) + I32(3), - ArrayRange(0, Ref("v", TInt32()), 1)), + StreamRange(0, Ref("v", TInt32()), 1)), "x", Ref("x", TInt32()) + Ref("q", TInt32()))), - "y", Ref("y", TInt32()) + I32(3)), + "y", Ref("y", TInt32()) + I32(3))), FastIndexedSeq(5, 6, 7, 8, 9)) } @@ -1559,18 +1559,18 @@ class IRSuite extends HailSuite { } @Test def testArrayZip() { - val range12 = ArrayRange(0, 12, 1) - val range6 = ArrayRange(0, 12, 2) - val range8 = ArrayRange(0, 24, 3) - val empty = ArrayRange(0, 0, 1) + val range12 = StreamRange(0, 12, 1) + val range6 = StreamRange(0, 12, 2) + val range8 = StreamRange(0, 24, 3) + val empty = StreamRange(0, 0, 1) val lit6 = Literal(TArray(TFloat64()), FastIndexedSeq(0d, -1d, 2.5d, -3d, 4d, null)) - val range6dup = ArrayRange(0, 6, 1) + val range6dup = StreamRange(0, 6, 1) - def zipToTuple(behavior: ArrayZipBehavior, irs: IR*): ArrayZip = ArrayZip( + def zipToTuple(behavior: ArrayZipBehavior, irs: IR*): IR = ToArray(ArrayZip( irs.toFastIndexedSeq, irs.indices.map(_.toString), MakeTuple.ordered(irs.zipWithIndex.map { case (ir, i) => Ref(i.toString, ir.typ.asInstanceOf[TStreamable].elementType) }), - behavior) + behavior)) for (b <- Array(ArrayZipBehavior.TakeMinLength, ArrayZipBehavior.ExtendNA)) { assertEvalSame(zipToTuple(b, range12), FastIndexedSeq()) @@ -1802,7 +1802,7 @@ class IRSuite extends HailSuite { assertEvalsTo(ArrayFlatMap(a, "a", Ref("a", ta)), FastIndexedSeq(7, null, 2)) - assertEvalsTo(ArrayFlatMap(ArrayRange(I32(0), I32(3), I32(1)), "i", ArrayRef(a, Ref("i", TInt32()))), FastIndexedSeq(7, null, 2)) + assertEvalsTo(ToArray(ArrayFlatMap(StreamRange(I32(0), I32(3), I32(1)), "i", ArrayRef(a, Ref("i", TInt32())))), FastIndexedSeq(7, null, 2)) assertEvalsTo(Let("a", I32(5), ArrayFlatMap(a, "a", Ref("a", ta))), FastIndexedSeq(7, null, 2)) @@ -1815,14 +1815,14 @@ class IRSuite extends HailSuite { val arr = MakeArray(List(I32(1), I32(5), I32(2), NA(TInt32())), TArray(TInt32())) val expected = FastIndexedSeq(-1, 0, -1, 0, 1, 2, 3, 4, -1, 0, 1) - assertEvalsTo(ArrayFlatMap(arr, "foo", ArrayRange(I32(-1), Ref("foo", TInt32()), I32(1))), expected) + assertEvalsTo(ArrayFlatMap(arr, "foo", StreamRange(I32(-1), Ref("foo", TInt32()), I32(1))), expected) } @Test def testArrayFold() { def fold(array: IR, zero: IR, f: (IR, IR) => IR): IR = ArrayFold(array, zero, "_accum", "_elt", f(Ref("_accum", zero.typ), Ref("_elt", zero.typ))) - assertEvalsTo(fold(ArrayRange(1, 2, 1), NA(TBoolean()), (accum, elt) => IsNA(accum)), true) + assertEvalsTo(fold(StreamRange(1, 2, 1), NA(TBoolean()), (accum, elt) => IsNA(accum)), true) assertEvalsTo(fold(TestUtils.IRArray(1, 2, 3), 0, (accum, elt) => accum + elt), 6) assertEvalsTo(fold(TestUtils.IRArray(1, 2, 3), NA(TInt32()), (accum, elt) => accum + elt), null) assertEvalsTo(fold(TestUtils.IRArray(1, null, 3), NA(TInt32()), (accum, elt) => accum + elt), null) @@ -1847,15 +1847,15 @@ class IRSuite extends HailSuite { implicit val execStrats = ExecStrategy.javaOnly def scan(array: IR, zero: IR, f: (IR, IR) => IR): IR = - ArrayScan(array, zero, "_accum", "_elt", f(Ref("_accum", zero.typ), Ref("_elt", zero.typ))) + ToArray(ArrayScan(array, zero, "_accum", "_elt", f(Ref("_accum", zero.typ), Ref("_elt", zero.typ)))) - assertEvalsTo(scan(ArrayRange(1, 4, 1), NA(TBoolean()), (accum, elt) => IsNA(accum)), FastIndexedSeq(null, true, false, false)) + assertEvalsTo(scan(StreamRange(1, 4, 1), NA(TBoolean()), (accum, elt) => IsNA(accum)), FastIndexedSeq(null, true, false, false)) assertEvalsTo(scan(TestUtils.IRArray(1, 2, 3), 0, (accum, elt) => accum + elt), FastIndexedSeq(0, 1, 3, 6)) assertEvalsTo(scan(TestUtils.IRArray(1, 2, 3), NA(TInt32()), (accum, elt) => accum + elt), FastIndexedSeq(null, null, null, null)) assertEvalsTo(scan(TestUtils.IRArray(1, null, 3), NA(TInt32()), (accum, elt) => accum + elt), FastIndexedSeq(null, null, null, null)) assertEvalsTo(scan(NA(TArray(TInt32())), 0, (accum, elt) => accum + elt), null) assertEvalsTo(scan(MakeArray(Seq(), TArray(TInt32())), 99, (accum, elt) => accum + elt), FastIndexedSeq(99)) - assertEvalsTo(scan(ArrayFlatMap(ArrayRange(0, 5, 1), "z", MakeArray(Seq(), TArray(TInt32()))), 99, (accum, elt) => accum + elt), FastIndexedSeq(99)) + assertEvalsTo(scan(ArrayFlatMap(StreamRange(0, 5, 1), "z", MakeArray(Seq(), TArray(TInt32()))), 99, (accum, elt) => accum + elt), FastIndexedSeq(99)) } def makeNDArray(data: Seq[Double], shape: Seq[Long], rowMajor: IR): MakeNDArray = { @@ -2192,9 +2192,9 @@ class IRSuite extends HailSuite { assertPType(Die("mumblefoo", TArray(TFloat64())), PArray(PFloat64(true), true)) } - @Test def testArrayRange() { + @Test def testStreamRange() { def assertEquals(start: Integer, stop: Integer, step: Integer, expected: IndexedSeq[Int]) { - assertEvalsTo(ArrayRange(In(0, TInt32()), In(1, TInt32()), In(2, TInt32())), + assertEvalsTo(ToArray(StreamRange(In(0, TInt32()), In(1, TInt32()), In(2, TInt32()))), args = FastIndexedSeq(start -> TInt32(), stop -> TInt32(), step -> TInt32()), expected = expected) } @@ -2202,7 +2202,7 @@ class IRSuite extends HailSuite { assertEquals(0, null, 1, null) assertEquals(null, 5, 1, null) - assertFatal(ArrayRange(I32(0), I32(5), I32(0)), "step size") + assertFatal(ToArray(StreamRange(I32(0), I32(5), I32(0))), "step size") for { start <- -2 to 2 @@ -2223,7 +2223,7 @@ class IRSuite extends HailSuite { val sumSig = AggSignature(Sum(), Seq(), Seq(TInt64())) assertEvalsTo( ArrayAgg( - ArrayMap(ArrayRange(I32(0), I32(4), I32(1)), "x", Cast(Ref("x", TInt32()), TInt64())), + ArrayMap(StreamRange(I32(0), I32(4), I32(1)), "x", Cast(Ref("x", TInt32()), TInt64())), "x", ApplyAggOp(FastIndexedSeq.empty, FastIndexedSeq(Ref("x", TInt64())), sumSig)), 6L) @@ -2236,7 +2236,7 @@ class IRSuite extends HailSuite { "x", In(0, TInt32()) * In(0, TInt32()), // multiply to prevent forwarding ArrayAgg( - ArrayRange(I32(0), I32(10), I32(1)), + StreamRange(I32(0), I32(10), I32(1)), "elt", AggLet("y", Cast(Ref("x", TInt32()) * Ref("x", TInt32()) * Ref("elt", TInt32()), TInt64()), // different type to trigger validation errors @@ -2451,14 +2451,13 @@ class IRSuite extends HailSuite { } @Test def testGroupByKey() { - implicit val execStrats = ExecStrategy.javaOnly + implicit val execStrats = Set(ExecStrategy.Interpret, ExecStrategy.InterpretUnoptimized, ExecStrategy.JvmCompile, ExecStrategy.JvmCompileUnoptimized) def tuple(k: String, v: Int): IR = MakeTuple.ordered(Seq(Str(k), I32(v))) def groupby(tuples: IR*): IR = GroupByKey(MakeArray(tuples, TArray(TTuple(TString(), TInt32())))) val collection1 = groupby(tuple("foo", 0), tuple("bar", 4), tuple("foo", -1), tuple("bar", 0), tuple("foo", 10), tuple("", 0)) - assertEvalsTo(collection1, Map("" -> FastIndexedSeq(0), "bar" -> FastIndexedSeq(4, 0), "foo" -> FastIndexedSeq(0, -1, 10))) } @@ -2572,7 +2571,7 @@ class IRSuite extends HailSuite { MakeTuple.ordered(FastSeq(F64(0), F64(2), F64(1)))))), ArrayRef(a, i), ArrayLen(a), - ArrayRange(I32(0), I32(5), I32(1)), + StreamRange(I32(0), I32(5), I32(1)), StreamRange(I32(0), I32(5), I32(1)), ArraySort(a, b), ToSet(a), @@ -2588,7 +2587,7 @@ class IRSuite extends HailSuite { ArrayFold(a, I32(0), "x", "v", v), ArrayFold2(ArrayFold(a, I32(0), "x", "v", v)), ArrayScan(a, I32(0), "x", "v", v), - ArrayLeftJoinDistinct(ArrayRange(0, 2, 1), ArrayRange(0, 3, 1), "l", "r", I32(0), I32(1)), + ArrayLeftJoinDistinct(StreamRange(0, 2, 1), StreamRange(0, 3, 1), "l", "r", I32(0), I32(1)), ArrayFor(a, "v", Void()), ArrayAgg(a, "x", ApplyAggOp(FastIndexedSeq.empty, FastIndexedSeq(Cast(Ref("x", TInt32()), TInt64())), sumSig)), ArrayAggScan(a, "x", ApplyScanOp(FastIndexedSeq.empty, FastIndexedSeq(Cast(Ref("x", TInt32()), TInt64())), sumSig)), @@ -2596,7 +2595,7 @@ class IRSuite extends HailSuite { InitOp(0, FastIndexedSeq(Begin(FastIndexedSeq(InitOp(0, FastSeq(), sumSig)))), groupSignature, Group()), SeqOp(0, FastSeq(I32(1), SeqOp(0, FastSeq(), sumSig)), groupSignature, Group()))), AggStateValue(0, groupSignature), FastIndexedSeq(groupSignature)), - RunAggScan(ArrayRange(I32(0), I32(1), I32(1)), + RunAggScan(StreamRange(I32(0), I32(1), I32(1)), "foo", InitOp(0, FastIndexedSeq(Begin(FastIndexedSeq(InitOp(0, FastSeq(), sumSig)))), groupSignature, Group()), SeqOp(0, FastSeq(Ref("foo", TInt32()), SeqOp(0, FastSeq(), sumSig)), groupSignature, Group()), @@ -2646,8 +2645,12 @@ class IRSuite extends HailSuite { BlockMatrixCollect(blockMatrix), BlockMatrixWrite(blockMatrix, blockMatrixWriter), BlockMatrixMultiWrite(IndexedSeq(blockMatrix, blockMatrix), blockMatrixMultiWriter), - CollectDistributedArray(ArrayRange(0, 3, 1), 1, "x", "y", Ref("x", TInt32())), + BlockMatrixWrite(blockMatrix, BlockMatrixPersistWriter("x", "MEMORY_ONLY")), + UnpersistBlockMatrix(blockMatrix), + CollectDistributedArray(StreamRange(0, 3, 1), 1, "x", "y", Ref("x", TInt32())), ReadPartition(Str("foo"), TypedCodecSpec(PStruct("foo" -> PInt32(), "bar" -> PString()), BufferSpec.default), TStruct("foo" -> TInt32())), + ReadValue(Str("foo"), TypedCodecSpec(PStruct("foo" -> PInt32(), "bar" -> PString()), BufferSpec.default), TStruct("foo" -> TInt32())), + WriteValue(I32(1), Str("foo"), TypedCodecSpec(PInt32(), BufferSpec.default)), LiftMeOut(I32(1)), RelationalLet("x", I32(0), I32(0)), TailLoop("y", IndexedSeq("x" -> I32(0)), Recur("y", FastSeq(I32(4)), TInt32())) @@ -2821,7 +2824,9 @@ class IRSuite extends HailSuite { sparsify3, densify, RelationalLetBlockMatrix("x", I32(0), read), - slice) + slice, + BlockMatrixRead(BlockMatrixPersistReader("x")) + ) blockMatrixIRs.map(ir => Array(ir)) } diff --git a/hail/src/test/scala/is/hail/expr/ir/RandomFunctionsSuite.scala b/hail/src/test/scala/is/hail/expr/ir/RandomFunctionsSuite.scala index 0c06fff8fe6..52971210582 100644 --- a/hail/src/test/scala/is/hail/expr/ir/RandomFunctionsSuite.scala +++ b/hail/src/test/scala/is/hail/expr/ir/RandomFunctionsSuite.scala @@ -101,21 +101,21 @@ class RandomFunctionsSuite extends HailSuite { @Test def testInterpretIncrementsCorrectly() { assertEvalsTo( - ArrayMap(ArrayRange(0, 3, 1), "i", counter * counter), + ToArray(ArrayMap(StreamRange(0, 3, 1), "i", counter * counter)), FastIndexedSeq(0, 1, 4)) assertEvalsTo( - ArrayFold(ArrayRange(0, 3, 1), -1, "j", "i", counter + counter), + ArrayFold(StreamRange(0, 3, 1), -1, "j", "i", counter + counter), 4) assertEvalsTo( - ArrayFilter(ArrayRange(0, 3, 1), "i", Ref("i", TInt32()).ceq(counter) && counter.ceq(counter)), + ToArray(ArrayFilter(StreamRange(0, 3, 1), "i", Ref("i", TInt32()).ceq(counter) && counter.ceq(counter))), FastIndexedSeq(0, 1, 2)) assertEvalsTo( - ArrayFlatMap(ArrayRange(0, 3, 1), + ToArray(ArrayFlatMap(StreamRange(0, 3, 1), "i", - MakeArray(FastSeq(counter, counter, counter), TArray(TInt32()))), + MakeArray(FastSeq(counter, counter, counter), TArray(TInt32())))), FastIndexedSeq(0, 0, 0, 1, 1, 1, 2, 2, 2)) } diff --git a/hail/src/test/scala/is/hail/expr/ir/TableIRSuite.scala b/hail/src/test/scala/is/hail/expr/ir/TableIRSuite.scala index 96629f19825..776c375555b 100644 --- a/hail/src/test/scala/is/hail/expr/ir/TableIRSuite.scala +++ b/hail/src/test/scala/is/hail/expr/ir/TableIRSuite.scala @@ -88,14 +88,14 @@ class TableIRSuite extends HailSuite { val t = TableRange(10, 2) val row = Ref("row", t.typ.rowType) - val t2 = TableMapRows(t, InsertFields(row, FastIndexedSeq("x" -> ArrayRange(0, GetField(row, "idx"), 1)))) + val t2 = TableMapRows(t, InsertFields(row, FastIndexedSeq("x" -> ToArray(StreamRange(0, GetField(row, "idx"), 1))))) val node = TableExplode(t2, FastIndexedSeq("x")) val expected = Array.range(0, 10).flatMap(i => Array.range(0, i).map(Row(i, _))).toFastIndexedSeq assertEvalsTo(collect(node), Row(expected, Row())) val t3 = TableMapRows(t, InsertFields(row, FastIndexedSeq("x" -> - MakeStruct(FastSeq("y" -> ArrayRange(0, GetField(row, "idx"), 1)))))) + MakeStruct(FastSeq("y" -> ToArray(StreamRange(0, GetField(row, "idx"), 1))))))) val node2 = TableExplode(t3, FastIndexedSeq("x", "y")) val expected2 = Array.range(0, 10).flatMap(i => Array.range(0, i).map(j => Row(i, Row(j)))).toFastIndexedSeq assertEvalsTo(collect(node2), Row(expected2, Row())) diff --git a/hail/src/test/scala/is/hail/expr/types/physical/PhysicalTestUtils.scala b/hail/src/test/scala/is/hail/expr/types/physical/PhysicalTestUtils.scala index 6bcfb7a3669..9e5b0230931 100644 --- a/hail/src/test/scala/is/hail/expr/types/physical/PhysicalTestUtils.scala +++ b/hail/src/test/scala/is/hail/expr/types/physical/PhysicalTestUtils.scala @@ -2,8 +2,9 @@ package is.hail.expr.types.physical import is.hail.utils.log import is.hail.annotations.{Region, SafeIndexedSeq, SafeRow, ScalaToRegionValue, UnsafeRow} import is.hail.expr.ir.EmitFunctionBuilder +import org.scalatest.testng.TestNGSuite -object PhysicalTestUtils { +object PhysicalTestUtils extends TestNGSuite { def copyTestExecutor(sourceType: PType, destType: PType, sourceValue: Any, expectCompileErr: Boolean = false, forceDeep: Boolean = false, interpret: Boolean = false) { @@ -14,7 +15,7 @@ object PhysicalTestUtils { if(interpret) { try { - val copyOff = destType.copyFromType(region, sourceType, srcAddress, forceDeep = forceDeep) + val copyOff = destType.fundamentalType.copyFromType(region, sourceType.fundamentalType, srcAddress, forceDeep = forceDeep) val copy = UnsafeRow.read(destType, region, copyOff) log.info(s"Copied value: ${copy}, Source value: ${sourceValue}") @@ -44,7 +45,7 @@ object PhysicalTestUtils { val value = fb.getArg[Long](2) try { - fb.emit(destType.copyFromType(fb.apply_method, codeRegion, sourceType, value, forceDeep = forceDeep)) + fb.emit(destType.fundamentalType.copyFromType(fb.apply_method, codeRegion, sourceType.fundamentalType, value, forceDeep = forceDeep)) compileSuccess = true } catch { case e: AssertionError => { diff --git a/hail/src/test/scala/is/hail/scheduler/SchedulerSuite.scala b/hail/src/test/scala/is/hail/scheduler/SchedulerSuite.scala deleted file mode 100644 index 1eb4f239732..00000000000 --- a/hail/src/test/scala/is/hail/scheduler/SchedulerSuite.scala +++ /dev/null @@ -1,61 +0,0 @@ -package is.hail.scheduler - -import org.scalatest.testng.TestNGSuite -import org.testng.annotations.Test -import is.hail.TestUtils._ - -class SchedulerTestException(message: String) extends Exception(message) - -class SchedulerSuite extends TestNGSuite { - private var schedulerHost = System.getenv("HAIL_TEST_SCHEDULER_HOST") - if (schedulerHost == null) - schedulerHost = "localhost" - - @Test def testSimpleJob(): Unit = { - val a = Array(-5, 6, 11) - val da: DArray[Int] = new DArray[Int] { - type Context = Int - val contexts: Array[Int] = a - val body: Int => Int = c => c - } - - val conn = new SchedulerAppClient(schedulerHost) - - var n = 0 - val r = new Array[Int](3) - conn.submit(da, - (i: Int, x: Int) => { - n += 1 - r(i) = x - }) - assert(n == 3) - assert(r sameElements a) - - conn.close() - } - - @Test def testException(): Unit = { - val da: DArray[Int] = new DArray[Int] { - type Context = Int - val contexts: Array[Int] = Array(1, 2, 3) - val body: Int => Int = c => { - if (c == 2) - throw new SchedulerTestException("test message") - else - c - } - } - - val conn = new SchedulerAppClient(schedulerHost) - - interceptException[SchedulerTestException]("test message") { - conn.submit(da, - (i: Int, x: Int) => { - assert(i == x - 1) - assert(i == 0 || i == 2) - }) - } - - conn.close() - } -} diff --git a/scheduler/Dockerfile b/scheduler/Dockerfile deleted file mode 100644 index 1802273bc5c..00000000000 --- a/scheduler/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM {{ service_base_image.image }} - -COPY scheduler/setup.py /scheduler/ -COPY scheduler/scheduler/ /scheduler/scheduler/ -RUN python3 -m pip install --no-cache-dir /scheduler \ - && rm -rf /scheduler - -EXPOSE 5000 - -CMD ["python3", "-m", "scheduler"] diff --git a/scheduler/MANIFEST.in b/scheduler/MANIFEST.in deleted file mode 100644 index e4b5caeb8c5..00000000000 --- a/scheduler/MANIFEST.in +++ /dev/null @@ -1,2 +0,0 @@ -recursive-include scheduler/templates * -recursive-include scheduler/static * diff --git a/scheduler/Makefile b/scheduler/Makefile deleted file mode 100644 index 60edf8ef7d0..00000000000 --- a/scheduler/Makefile +++ /dev/null @@ -1,18 +0,0 @@ -.PHONY: check clean - -PYTHON := python3 - -PY_FILES := $(shell find scheduler -iname \*.py -not -exec git check-ignore -q {} \; -print) - -flake8-stmp: $(PY_FILES) - python3 -m flake8 scheduler - touch $@ - -pylint-stmp: $(PY_FILES) - $(PYTHON) -m pylint --rcfile ../pylintrc scheduler --score=n - touch $@ - -check: flake8-stmp pylint-stmp - -clean: - rm -f flake8-stmp pylint-stmp diff --git a/scheduler/deployment.yaml b/scheduler/deployment.yaml deleted file mode 100644 index 03389c5b3fb..00000000000 --- a/scheduler/deployment.yaml +++ /dev/null @@ -1,79 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: scheduler - labels: - app: scheduler - hail.is/sha: "{{ code.sha }}" -spec: - selector: - matchLabels: - app: scheduler - replicas: 1 - template: - metadata: - labels: - app: scheduler - hail.is/sha: "{{ code.sha }}" - spec: -{% if deploy %} - priorityClassName: production -{% endif %} - nodeSelector: - preemptible: "false" - containers: - - name: scheduler - image: "{{ scheduler_image.image }}" - env: - - name: HAIL_DEPLOY_CONFIG_FILE - value: /deploy-config/deploy-config.json - resources: - requests: - memory: "1G" - cpu: "1" - ports: - - containerPort: 5000 - - containerPort: 5051 - - containerPort: 5052 - - containerPort: 5053 - readinessProbe: - httpGet: - path: /healthcheck - port: 5000 - initialDelaySeconds: 5 - periodSeconds: 5 - volumeMounts: - - mountPath: /deploy-config - name: deploy-config - readOnly: true - volumes: - - name: deploy-config - secret: - secretName: deploy-config ---- -apiVersion: v1 -kind: Service -metadata: - name: scheduler - labels: - app: scheduler -spec: - ports: - - name: http - port: 80 - protocol: TCP - targetPort: 5000 - - name: executor - port: 5051 - protocol: TCP - targetPort: 5051 - - name: client-submit - port: 5052 - protocol: TCP - targetPort: 5052 - - name: client-result - port: 5053 - protocol: TCP - targetPort: 5053 - selector: - app: scheduler diff --git a/scheduler/executors.yaml b/scheduler/executors.yaml deleted file mode 100644 index 8fdaa2f3812..00000000000 --- a/scheduler/executors.yaml +++ /dev/null @@ -1,36 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: executors - labels: - app: executors - hail.is/sha: "{{ code.sha }}" -spec: - selector: - matchLabels: - app: executors - replicas: 2 - template: - metadata: - labels: - app: executors - hail.is/sha: "{{ code.sha }}" - spec: -{% if deploy %} - priorityClassName: production -{% endif %} - tolerations: - - key: preemptible - value: "true" - containers: - - name: executor - image: "{{ hail_test_base_image.image }}" - command: - - /bin/bash - - -c - - | - CLASSPATH="$SPARK_HOME/jars/*" java is.hail.scheduler.Executor scheduler.{{ default_ns.name }}.svc.cluster.local 1 - resources: - requests: - memory: "1G" - cpu: "1" diff --git a/scheduler/scheduler/__init__.py b/scheduler/scheduler/__init__.py deleted file mode 100644 index cd91fb050ff..00000000000 --- a/scheduler/scheduler/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from aiohttp import web -from .scheduler import app - -if __name__ == '__main__': - web.run_app(app, host='0.0.0.0', port=5000) diff --git a/scheduler/scheduler/__main__.py b/scheduler/scheduler/__main__.py deleted file mode 100644 index 575dfe76396..00000000000 --- a/scheduler/scheduler/__main__.py +++ /dev/null @@ -1,4 +0,0 @@ -from aiohttp import web -from .scheduler import app - -web.run_app(app, host='0.0.0.0', port=5000) diff --git a/scheduler/scheduler/scheduler.py b/scheduler/scheduler/scheduler.py deleted file mode 100644 index 556f97e0cd5..00000000000 --- a/scheduler/scheduler/scheduler.py +++ /dev/null @@ -1,498 +0,0 @@ -import struct -import datetime -import logging -from base64 import b64encode -import asyncio -from aiohttp import web -import jinja2 -import aiohttp_jinja2 -import uvloop -from hailtop.config import get_deploy_config -from gear import configure_logging - -uvloop.install() - -configure_logging() -log = logging.getLogger('scheduler') - - -async def read_int(reader): - try: - b = await reader.readexactly(4) - return struct.unpack('>I', b)[0] - except asyncio.IncompleteReadError as e: - if not e.partial: - return - raise e - - -def write_int(writer, i): - writer.write(struct.pack('>I', i)) - - -async def read_bytes(reader): - n = await read_int(reader) - return await reader.readexactly(n) - - -def write_bytes(writer, b): - write_int(writer, len(b)) - writer.write(b) - - -counter = 0 - - -def create_id(): - global counter - - counter = counter + 1 - return counter - - -executors = set() -available_executors = set() - -clients = set() - -jobs = {} -pending_jobs = set() - -task_index = {} - - -scheduling = False - - -async def schedule(): - global scheduling - - if scheduling: - return - - scheduling = True - while pending_jobs and available_executors: - e = next(iter(available_executors)) - await e.schedule() - - scheduling = False - - -# executor messages -# scheduler => executor -EXECUTE = 2 - -# executor => scheduler -PING = 1 -TASKRESULT = 4 - - -class ExecutorConnection: - def __init__(self, reader, writer, n_cores): - self.id = create_id() - - log.info(f'executor {self.id} connected: {n_cores} cores') - - self.reader = reader - self.writer = writer - self.n_cores = n_cores - self.running = set() - self.last_message_time = datetime.datetime.now() - executors.add(self) - available_executors.add(self) - - async def handle_result(self): - task_id = await read_int(self.reader) - res = await read_bytes(self.reader) - - if task_id in task_index: - t = task_index[task_id] - - duration = datetime.datetime.now() - t.start_time - log.info(f'executor {self.id}: ' - f'task {t.id} for job {t.job.id} complete: {duration}') - - t.set_result(res) - - if len(self.running) == self.n_cores: - assert self not in available_executors - available_executors.add(self) - self.running.remove(t) - - await self.schedule() - - async def handle_ping(self): - log.info(f'executor {self.id}: received ping') - - async def execute(self, t): - log.info(f'schedule task {t.id} for job {t.job.id} on executor {self.id}') - - # FIXME time this attempt - t.start_time = datetime.datetime.now() - - # don't need to lock becuase only called by schedule and - # schedule is serial - write_int(self.writer, EXECUTE) - write_int(self.writer, t.id) - write_bytes(self.writer, t.f) - await self.writer.drain() - - async def handler_loop(self): - try: - while True: - cmd = await read_int(self.reader) - if cmd is None: - return - if cmd == PING: - await self.handle_ping() - elif cmd == TASKRESULT: - await self.handle_result() - else: - raise ValueError(f'unknown command {cmd}') - self.last_message_time = datetime.datetime.now() - except Exception: # pylint: disable=broad-except - log.exception(f'executor {self.id}: ' - f'error in handler loop, closing due to exception') - finally: - self.close() - - def close(self): - executors.remove(self) - available_executors.remove(self) - for t in self.running: - t.job.pending_tasks.append(t) - self.running = set() - - async def schedule(self): - while len(self.running) < self.n_cores and pending_jobs: - j = next(iter(pending_jobs)) - t = j.pending_tasks.pop() - if not j.pending_tasks: - pending_jobs.remove(j) - - self.running.add(t) - if len(self.running) == self.n_cores: - available_executors.remove(self) - - await self.execute(t) - - def to_dict(self): - return { - 'id': self.id, - 'n_cores': self.n_cores, - 'n_running': len(self.running) - } - - -async def executor_connected_cb(reader, writer): - log.info('executor connected') - n_cores = await read_int(reader) - conn = ExecutorConnection(reader, writer, n_cores) - asyncio.ensure_future(conn.handler_loop()) - await conn.schedule() - - -class Job: - def __init__(self, client, token, n_tasks): - self.id = create_id() - self.token = token - self.client = client - self.start_time = datetime.datetime.utcnow() - self.end_time = None - - self.n_tasks = n_tasks - self.n_submitted = 0 - self.index_task = {} - self.pending_tasks = [] - self.complete_tasks = set() - - jobs[token] = self - - async def add_task(self, f, index): - assert index == self.n_submitted - t = Task(self, f, index) - self.index_task[index] = t - self.pending_tasks.append(t) - - self.n_submitted += 1 - - if len(self.pending_tasks) == 1: - pending_jobs.add(self) - await schedule() - - def ack_task(self, index): - t = self.index_task.get(index) - if t is None: - return - del self.index_task[index] - self.complete_tasks.remove(t) - t.ack() - - if self.is_complete(): - self.client.end_job(self.token) - self.end_time = datetime.datetime.utcnow() - - def is_complete(self): - return (self.n_submitted == self.n_tasks) and (not self.index_task) - - def to_dict(self): - n_acknowleged = self.n_submitted - len(self.index_task) - n_complete = n_acknowleged + len(self.complete_tasks) - n_running = (self.n_submitted - n_complete) - len(self.pending_tasks) - timef = '%Y-%m-%dT%H:%M:%S.%fZ' - start_string = self.start_time.strftime(timef) - end_string = '--' if self.end_time is None else self.end_time.strftime(timef) - return { - 'client': self.client.id, - 'id': self.id, - 'n_tasks': self.n_tasks, - 'n_submitted': self.n_submitted, - 'n_complete': n_complete, - 'n_running': n_running, - 'start_time': start_string, - 'end_time': end_string - } - - -class Task: - def __init__(self, job, f, index): - self.id = create_id() - self.job = job - self.f = f - self.index = index - self.start_time = None - self.result = None - - task_index[self.id] = self - - def set_result(self, result): - if self.result is not None: - return - self.result = result - self.job.complete_tasks.add(self) - - result_conn = self.job.client.result_conn - if result_conn: - asyncio.ensure_future( - result_conn.task_result(self.job.token, self.index, self.result)) - - def ack(self): - self.result = None - del task_index[self.id] - - -# client messages -# client => scheduler -SUBMIT = 5 - -# scheduler => client -APPTASKRESULT = 6 -ACKTASKRESULT = 7 - - -class ClientSubmitConnection: - def __init__(self, client, reader, writer): - self.client = client - self.reader = reader - self.writer = writer - - async def handle_submit(self): - job_token = await read_bytes(self.reader) - log.info(f'received job') - n = await read_int(self.reader) - - j = self.client.start_job(job_token, n) - write_int(self.writer, j.n_submitted) - await self.writer.drain() - - i = j.n_submitted - while i < n: - b = await read_bytes(self.reader) - await j.add_task(b, i) - i += 1 - - # ack - write_int(self.writer, 0) - write_bytes(self.writer, job_token) - await self.writer.drain() - - log.info(f'job {j.id}, {n} tasks submitted') - - await schedule() - - async def handler_loop(self): - try: - while True: - cmd = await read_int(self.reader) - if cmd is None: - return - if cmd == SUBMIT: - await self.handle_submit() - else: - raise ValueError(f'unknown command {cmd}') - except Exception: # pylint: disable=broad-except - log.exception(f'error in handler loop') - finally: - self.close() - - def close(self): - self.writer.close() - self.client.submit_conn = None - # new in 3.7 - # await self.writer.wait_closed() - - -class ClientResultConnection: - def __init__(self, client, reader, writer): - self.client = client - self.reader = reader - self.writer = writer - - async def handle_ack_task(self): - job_token = await read_bytes(self.reader) - index = await read_int(self.reader) - - j = jobs.get(job_token) - if j is not None: - j.ack_task(index) - - async def task_result(self, job_token, index, result): - write_int(self.writer, APPTASKRESULT) - write_bytes(self.writer, job_token) - write_int(self.writer, index) - write_bytes(self.writer, result) - await self.writer.drain() - - async def handler_loop(self): - try: - while True: - cmd = await read_int(self.reader) - if cmd is None: - return - if cmd == ACKTASKRESULT: - await self.handle_ack_task() - else: - raise ValueError(f'unknown command {cmd}') - except Exception: # pylint: disable=broad-except - log.exception(f'error in handler loop') - finally: - self.close() - - def close(self): - self.writer.close() - self.client.result_conn = None - # new in 3.7 - # await self.writer.wait_closed() - - -token_client = {} - - -class Client: - def __init__(self, token): - self.id = create_id() - self.token = token - self.submit_conn = None - self.result_conn = None - - self.job = None - - clients.add(self) - log.info(f'client {self.id} created') - - def start_job(self, job_token, n): - self.job = jobs.get(job_token) - if self.job is None: - self.job = Job(self, job_token, n) - return self.job - - def end_job(self, job_token): - if self.job is not None and self.job.token == job_token: - self.job = None - - def set_submit_conn(self, reader, writer): - if self.submit_conn: - self.submit_conn.close() - self.submit_conn = ClientSubmitConnection(self, reader, writer) - log.info(f'client {self.id} submit connected') - asyncio.ensure_future(self.submit_conn.handler_loop()) - - def set_result_conn(self, reader, writer): - if self.result_conn: - self.result_conn.close() - self.result_conn = ClientResultConnection(self, reader, writer) - log.info(f'client {self.id} result connected') - asyncio.ensure_future(self.result_conn.handler_loop()) - - def to_dict(self): - is_disconnected = ((self.submit_conn is None) and - (self.result_conn is None)) - return { - 'id': self.id, - 'token': f'{b64encode(self.token)[:4].decode("ascii")}...', - 'is_disconnected': is_disconnected, - 'job_id': self.job.id if self.job else None - } - - -async def client_submit_cb(reader, writer): - token = await read_bytes(reader) - - client = token_client.get(token) - if client is None: - client = Client(token) - token_client[token] = client - - client.set_submit_conn(reader, writer) - - -async def client_result_cb(reader, writer): - log.info('here') - token = await read_bytes(reader) - - client = token_client.get(token) - if client is None: - client = Client(token) - token_client[token] = client - - client.set_result_conn(reader, writer) - -app = web.Application() -routes = web.RouteTableDef() - - -@routes.get('/healthcheck') -async def healthcheck(request): # pylint: disable=unused-argument - return web.Response(status=200) - - -@routes.get('') -@routes.get('/') -@aiohttp_jinja2.template('index.html') -async def index(request): # pylint: disable=unused-argument - return { - 'executors': [e.to_dict() for e in executors], - 'clients': [client.to_dict() for client in clients], - 'jobs': [j.to_dict() for j in reversed(list(jobs.values()))] - } - - -app.add_routes(routes) - -aiohttp_jinja2.setup(app, loader=jinja2.FileSystemLoader('templates')) - - -async def on_startup(app): # pylint: disable=unused-argument - await asyncio.start_server(executor_connected_cb, host=None, port=5051) - log.info(f'listening on port {5051} for executors') - - await asyncio.start_server(client_submit_cb, host=None, port=5052) - log.info(f'listening on port {5052} for clients, submit') - - await asyncio.start_server(client_result_cb, host=None, port=5053) - log.info(f'listening on port {5053} for clients, result') - -app.on_startup.append(on_startup) - -deploy_config = get_deploy_config() -web.run_app(deploy_config.prefix_application(app, 'scheduler'), host='0.0.0.0', port=5000) diff --git a/scheduler/setup.py b/scheduler/setup.py deleted file mode 100644 index 9e4237a837c..00000000000 --- a/scheduler/setup.py +++ /dev/null @@ -1,11 +0,0 @@ -from setuptools import setup, find_packages - -setup( - name = 'scheduler', - version = '0.0.1', - url = 'https://github.com/hail-is/hail.git', - author = 'Hail Team', - author_email = 'hail@broadinstitute.org', - description = 'Scheduler', - packages = find_packages() -) diff --git a/scheduler/templates/index.html b/scheduler/templates/index.html deleted file mode 100644 index b1eaae8bad7..00000000000 --- a/scheduler/templates/index.html +++ /dev/null @@ -1,100 +0,0 @@ - - - Scheduler - - - -

Executors

- {% if executors %} - - - - - - - - - - {% for e in executors %} - - - - - - {% endfor %} - -
idcoresn_running
{{ e['id'] }}{{ e['n_cores'] }}{{ e['n_running'] }}
- {% else %} - No executors. - {% endif %} - - -

Jobs

- {% if jobs %} - - - - - - - - - - - - - - - {% for j in jobs %} - - - - - - - - - - - {% endfor %} - -
clientidstart_timeend_timen_tasksn_submittedn_completen_running
{{ j['client'] }}{{ j['id'] }}{{ j['start_time'] }}{{ j['end_time'] }}{{ j['n_tasks'] }}{{ j['n_submitted'] }}{{ j['n_complete'] }}{{ j['n_running'] }}
- {% else %} - No jobs. - {% endif %} - -

Applications

- {% if apps %} - - - - - - - - - - - {% for app in apps %} - - - - - {% if app['job_id'] %} - - {% endif %} - - {% endfor %} - -
idtokenis_disconnectedjob_id
{{ app['id'] }}{{ app['token'] }}{{ app['is_disconnected'] }}{{ app['job_id'] }}
- {% else %} - No applications. - {% endif %} - - - diff --git a/scheduler/testng.xml b/scheduler/testng.xml deleted file mode 100644 index 24b9f1465dd..00000000000 --- a/scheduler/testng.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - - diff --git a/web_common/web_common/templates/header.html b/web_common/web_common/templates/header.html index b414fd93c32..9c644c0615e 100644 --- a/web_common/web_common/templates/header.html +++ b/web_common/web_common/templates/header.html @@ -15,21 +15,20 @@ {% endif %} - {% if userdata['is_developer'] == 1 %} -
-
- Batch -
-
+
+
+ Batch +
+
+ {% if userdata['is_developer'] == 1 %} Driver Billing Projects User Resources -
+ {% endif %} + Docsopen_in_new
- {% else %} - Batch - {% endif %} +
{% if userdata['is_developer'] == 1 %} From 8827fe314a73de27bc884a5bfffb4c3c873737bd Mon Sep 17 00:00:00 2001 From: Alex Kotlar Date: Tue, 25 Feb 2020 21:58:29 -0500 Subject: [PATCH 4/8] pstream --- hail/src/main/scala/is/hail/expr/ir/InferPType.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hail/src/main/scala/is/hail/expr/ir/InferPType.scala b/hail/src/main/scala/is/hail/expr/ir/InferPType.scala index 1d2af2e636e..824ccbb805d 100644 --- a/hail/src/main/scala/is/hail/expr/ir/InferPType.scala +++ b/hail/src/main/scala/is/hail/expr/ir/InferPType.scala @@ -553,7 +553,7 @@ object InferPType { val sigs = signature.indices.map { i => computePhysicalAgg(signature(i), inits(i), seqs(i)) }.toArray infer(result, env = e2, aggs = sigs, inits = null, seqs = null) x.physicalSignatures2 = sigs - coerce[PStreamable](array.pType2).copyStreamable(result.pType2) + coerce[PStream](array.pType2).copy(result.pType2) case AggStateValue(i, sig) => PCanonicalBinary(true) case x if x.typ == TVoid => From 27f4e308287399e042c818fbdcd3412051b0e87f Mon Sep 17 00:00:00 2001 From: Alex Kotlar Date: Tue, 25 Feb 2020 22:51:17 -0500 Subject: [PATCH 5/8] backport leftJoinDistinct fix --- hail/src/main/scala/is/hail/expr/ir/InferPType.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hail/src/main/scala/is/hail/expr/ir/InferPType.scala b/hail/src/main/scala/is/hail/expr/ir/InferPType.scala index 824ccbb805d..6b1830b1c31 100644 --- a/hail/src/main/scala/is/hail/expr/ir/InferPType.scala +++ b/hail/src/main/scala/is/hail/expr/ir/InferPType.scala @@ -301,7 +301,7 @@ object InferPType { case ArrayLeftJoinDistinct(lIR, rIR, lName, rName, compare, join) => infer(lIR) infer(rIR) - val e = env.bind(lName -> lIR.pType2.asInstanceOf[PStream].elementType, rName -> rIR.pType2.asInstanceOf[PStream].elementType) + val e = val e = env.bind(lName -> lIR.pType2.asInstanceOf[PStream].elementType, rName -> rIR.pType2.asInstanceOf[PStream].elementType.setRequired(false)) infer(compare, e) infer(join, e) From 1185e3d8515cfbd279b2bc3490ceb3f9bcab4851 Mon Sep 17 00:00:00 2001 From: Alex Kotlar Date: Tue, 25 Feb 2020 23:14:54 -0500 Subject: [PATCH 6/8] arrayFor use PStream --- hail/src/main/scala/is/hail/expr/ir/InferPType.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hail/src/main/scala/is/hail/expr/ir/InferPType.scala b/hail/src/main/scala/is/hail/expr/ir/InferPType.scala index 6b1830b1c31..f8a7cb679bc 100644 --- a/hail/src/main/scala/is/hail/expr/ir/InferPType.scala +++ b/hail/src/main/scala/is/hail/expr/ir/InferPType.scala @@ -279,7 +279,7 @@ object InferPType { case ArrayFor(a, value, body) => infer(a) - infer(body, env.bind(value -> a.pType2.asInstanceOf[PArray].elementType)) + infer(body, env.bind(value -> a.pType2.asInstanceOf[PStream].elementType)) PVoid case ArrayFold2(a, acc, valueName, seq, res) => infer(a) @@ -301,7 +301,7 @@ object InferPType { case ArrayLeftJoinDistinct(lIR, rIR, lName, rName, compare, join) => infer(lIR) infer(rIR) - val e = val e = env.bind(lName -> lIR.pType2.asInstanceOf[PStream].elementType, rName -> rIR.pType2.asInstanceOf[PStream].elementType.setRequired(false)) + val e = env.bind(lName -> lIR.pType2.asInstanceOf[PStream].elementType, rName -> rIR.pType2.asInstanceOf[PStream].elementType.setRequired(false)) infer(compare, e) infer(join, e) From be73a6df7aa92228e02d61f6282466ec08223b6a Mon Sep 17 00:00:00 2001 From: Alex Kotlar Date: Wed, 26 Feb 2020 12:51:29 -0500 Subject: [PATCH 7/8] fix run agg scan --- hail/src/main/scala/is/hail/expr/ir/InferPType.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hail/src/main/scala/is/hail/expr/ir/InferPType.scala b/hail/src/main/scala/is/hail/expr/ir/InferPType.scala index f8a7cb679bc..1ea7aa705c4 100644 --- a/hail/src/main/scala/is/hail/expr/ir/InferPType.scala +++ b/hail/src/main/scala/is/hail/expr/ir/InferPType.scala @@ -553,7 +553,7 @@ object InferPType { val sigs = signature.indices.map { i => computePhysicalAgg(signature(i), inits(i), seqs(i)) }.toArray infer(result, env = e2, aggs = sigs, inits = null, seqs = null) x.physicalSignatures2 = sigs - coerce[PStream](array.pType2).copy(result.pType2) + PCanonicalArray(result.pType2, array._pType2.required) case AggStateValue(i, sig) => PCanonicalBinary(true) case x if x.typ == TVoid => From 3cd483b568df03fc1a0bc9af1718dcca78c7c9d5 Mon Sep 17 00:00:00 2001 From: Alex Kotlar Date: Wed, 26 Feb 2020 14:57:54 -0500 Subject: [PATCH 8/8] remove debug message --- .../main/scala/is/hail/expr/ir/agg/DownsampleAggregator.scala | 1 - hail/src/main/scala/is/hail/expr/ir/agg/StagedArrayBuilder.scala | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/hail/src/main/scala/is/hail/expr/ir/agg/DownsampleAggregator.scala b/hail/src/main/scala/is/hail/expr/ir/agg/DownsampleAggregator.scala index 4fd41980191..94247216ebc 100644 --- a/hail/src/main/scala/is/hail/expr/ir/agg/DownsampleAggregator.scala +++ b/hail/src/main/scala/is/hail/expr/ir/agg/DownsampleAggregator.scala @@ -51,7 +51,6 @@ class DownsampleState(val fb: EmitFunctionBuilder[_], labelType: PArray, maxBuff def createState: Code[Unit] = region.isNull.mux(r := Region.stagedCreate(regionSize), Code._empty) - val binType = PStruct(required = true, "x" -> PInt32Required, "y" -> PInt32Required) val pointType = PStruct(required = true, "x" -> PFloat64Required, "y" -> PFloat64Required, "label" -> labelType) diff --git a/hail/src/main/scala/is/hail/expr/ir/agg/StagedArrayBuilder.scala b/hail/src/main/scala/is/hail/expr/ir/agg/StagedArrayBuilder.scala index ad274ff249c..491ab934a99 100644 --- a/hail/src/main/scala/is/hail/expr/ir/agg/StagedArrayBuilder.scala +++ b/hail/src/main/scala/is/hail/expr/ir/agg/StagedArrayBuilder.scala @@ -14,6 +14,7 @@ object StagedArrayBuilder { class StagedArrayBuilder(eltType: PType, fb: EmitFunctionBuilder[_], region: Code[Region], var initialCapacity: Int = 8) { val eltArray = PArray(eltType.setRequired(false), required = true) // element type must be optional for serialization to work val stateType = PTuple(true, PInt32Required, PInt32Required, eltArray) + val size: ClassFieldRef[Int] = fb.newField[Int]("size") private val capacity = fb.newField[Int]("capacity") val data = fb.newField[Long]("data")