From e3e38f4a60b8bc5e9e7d4e4f84c4a3af0841d6e6 Mon Sep 17 00:00:00 2001 From: manoflearning <77jwk0724@gmail.com> Date: Wed, 17 Sep 2025 16:47:03 +0900 Subject: [PATCH 1/7] feat: add more backend features and tests --- Cargo.lock | 27 ++++ Cargo.toml | 3 + src/backend/kernels_simd.rs | 16 +++ src/backend/mod.rs | 22 +--- src/core/mod.rs | 1 + src/core/reduce.rs | 151 ++++++++++++++++++++++ src/core/view.rs | 74 +++++++++-- src/lib.rs | 4 + src/py/mod.rs | 241 +++++++++++++++++++++++++++++++++++- src/tests/backend.rs | 17 +++ src/tests/mod.rs | 1 + src/tests/reduce.rs | 108 ++++++++++++++++ src/tests/view.rs | 32 +++++ 13 files changed, 671 insertions(+), 26 deletions(-) create mode 100644 src/core/reduce.rs create mode 100644 src/tests/reduce.rs diff --git a/Cargo.lock b/Cargo.lock index 5c95aa0..602f351 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -30,6 +30,7 @@ version = "0.1.3" dependencies = [ "pyo3", "rand", + "rand_distr", "thiserror", ] @@ -68,6 +69,12 @@ version = "0.2.175" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" +[[package]] +name = "libm" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" + [[package]] name = "memoffset" version = "0.9.1" @@ -77,6 +84,16 @@ dependencies = [ "autocfg", ] +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -219,6 +236,16 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rand_distr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +dependencies = [ + "num-traits", + "rand", +] + [[package]] name = "shlex" version = "1.3.0" diff --git a/Cargo.toml b/Cargo.toml index ada02ee..ba51973 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,9 +10,12 @@ crate-type = ["cdylib", "rlib"] [dependencies] pyo3 = { version = "0.24.1", features = ["extension-module"] } rand = { version = "0.8.5" } +rand_distr = "0.4" thiserror = "1" [features] +default = ["python"] +python = [] abi3 = ["pyo3/abi3-py37", "generate-import-lib"] generate-import-lib = ["pyo3/generate-import-lib"] diff --git a/src/backend/kernels_simd.rs b/src/backend/kernels_simd.rs index c63a07e..439cd3d 100644 --- a/src/backend/kernels_simd.rs +++ b/src/backend/kernels_simd.rs @@ -1,3 +1,4 @@ +use std::simd::num::SimdFloat; use std::simd::{f32x64, StdFloat}; const CHUNK_SIZE: usize = 64; @@ -34,6 +35,21 @@ pub mod unary_ops { .for_each(|(a, b)| *b = a.sqrt()); } + pub fn relu(a: &[f32], b: &mut [f32]) { + assert!(a.len() == b.len()); + let (a_main, a_rem) = a.split_at(a.len() - a.len() % CHUNK_SIZE); + let (b_main, b_rem) = b.split_at_mut(b.len() - b.len() % CHUNK_SIZE); + let zero = f32x64::splat(0.0); + a_main + .chunks_exact(CHUNK_SIZE) + .zip(b_main.chunks_exact_mut(CHUNK_SIZE)) + .for_each(|(a, b)| f32x64::from_slice(a).simd_max(zero).copy_to_slice(b)); + a_rem + .iter() + .zip(b_rem.iter_mut()) + .for_each(|(a, b)| *b = a.max(0.0)); + } + pub fn exp(a: &[f32], b: &mut [f32]) { assert!(a.len() == b.len()); let (a_main, a_rem) = a.split_at(a.len() - a.len() % CHUNK_SIZE); diff --git a/src/backend/mod.rs b/src/backend/mod.rs index 4eb19ee..18ad865 100644 --- a/src/backend/mod.rs +++ b/src/backend/mod.rs @@ -1,6 +1,6 @@ use thiserror::Error; -use crate::core::view::View; +use crate::core::view::{contiguous_strides, View}; #[derive(Debug, Clone, Copy)] pub enum UnaryOp { @@ -8,6 +8,7 @@ pub enum UnaryOp { Sqrt, Exp, Log, + Relu, } #[derive(Debug, Clone, Copy)] @@ -58,20 +59,14 @@ impl Backend for CpuBackend { UnaryOp::Sqrt => kernels_simd::unary_ops::sqrt(a_slice, out_slice), UnaryOp::Exp => kernels_simd::unary_ops::exp(a_slice, out_slice), UnaryOp::Log => kernels_simd::unary_ops::log(a_slice, out_slice), + UnaryOp::Relu => kernels_simd::unary_ops::relu(a_slice, out_slice), } } - let mut stride = 1isize; - let mut strides = vec![0isize; a.shape.len()]; - for (i, dim) in a.shape.iter().rev().enumerate() { - let idx = a.shape.len() - 1 - i; - strides[idx] = stride; - stride *= *dim as isize; - } Ok(View { inner: Arc::new(out_inner), offset: 0, shape: a.shape.clone(), - strides, + strides: contiguous_strides(&a.shape), }) } @@ -94,18 +89,11 @@ impl Backend for CpuBackend { BinaryOp::Div => kernels_simd::binary_ops::div(a_slice, b_slice, out_slice), } } - let mut stride = 1isize; - let mut strides = vec![0isize; a.shape.len()]; - for (i, dim) in a.shape.iter().rev().enumerate() { - let idx = a.shape.len() - 1 - i; - strides[idx] = stride; - stride *= *dim as isize; - } Ok(View { inner: Arc::new(out_inner), offset: 0, shape: a.shape.clone(), - strides, + strides: contiguous_strides(&a.shape), }) } } diff --git a/src/core/mod.rs b/src/core/mod.rs index c65f10b..167a54d 100644 --- a/src/core/mod.rs +++ b/src/core/mod.rs @@ -1,2 +1,3 @@ +pub mod reduce; pub mod storage; pub mod view; diff --git a/src/core/reduce.rs b/src/core/reduce.rs new file mode 100644 index 0000000..511af68 --- /dev/null +++ b/src/core/reduce.rs @@ -0,0 +1,151 @@ +use std::sync::Arc; + +use crate::core::{storage::StorageInner, view::View}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AxisError { + ScalarHasNoDim, + OutOfRange, +} + +pub fn numel(shape: &[usize]) -> usize { + if shape.is_empty() { + 1 + } else { + shape.iter().product() + } +} + +pub fn ensure_contiguous(view: &View) -> View { + if view.is_contiguous() { + view.clone() + } else { + view.to_contiguous() + } +} + +pub fn normalize_axis(dim: Option, ndim: usize) -> Result, AxisError> { + match dim { + None => Ok(None), + Some(axis) => { + if ndim == 0 { + return Err(AxisError::ScalarHasNoDim); + } + let mut axis = axis; + if axis < 0 { + axis += ndim as isize; + } + if axis < 0 || axis >= ndim as isize { + Err(AxisError::OutOfRange) + } else { + Ok(Some(axis as usize)) + } + } + } +} + +pub fn reduce_sum(view: &View, axis: Option, keepdim: bool) -> View { + let base = ensure_contiguous(view); + let device = base.inner.device(); + let shape = base.shape.clone(); + let data = base.inner.as_slice(base.offset, base.numel()); + + match axis { + None => { + let total: f32 = data.iter().copied().sum(); + let out_shape = if keepdim { + vec![1; shape.len()] + } else { + Vec::new() + }; + let inner = StorageInner::from_vec(vec![total], device); + View::from_inner_contiguous(Arc::new(inner), &out_shape) + } + Some(axis) => { + let ndim = shape.len(); + assert!(axis < ndim, "axis out of bounds"); + let axis_size = shape[axis]; + let outer = shape[..axis].iter().product::(); + let inner = shape[axis + 1..].iter().product::(); + let mut out_data = vec![0.0f32; outer * inner]; + for outer_idx in 0..outer { + for inner_idx in 0..inner { + let mut acc = 0.0f32; + for axis_idx in 0..axis_size { + let idx = (outer_idx * axis_size + axis_idx) * inner + inner_idx; + acc += data[idx]; + } + out_data[outer_idx * inner + inner_idx] = acc; + } + } + let out_shape = if keepdim { + let mut s = shape.clone(); + s[axis] = 1; + s + } else { + let mut s = shape.clone(); + s.remove(axis); + s + }; + let inner = StorageInner::from_vec(out_data, device); + View::from_inner_contiguous(Arc::new(inner), &out_shape) + } + } +} + +pub fn reduce_max(view: &View, axis: Option, keepdim: bool) -> View { + let base = ensure_contiguous(view); + let device = base.inner.device(); + let shape = base.shape.clone(); + let data = base.inner.as_slice(base.offset, base.numel()); + + match axis { + None => { + let mut current = f32::NEG_INFINITY; + for &v in data.iter() { + if v > current { + current = v; + } + } + let out_shape = if keepdim { + vec![1; shape.len()] + } else { + Vec::new() + }; + let inner = StorageInner::from_vec(vec![current], device); + View::from_inner_contiguous(Arc::new(inner), &out_shape) + } + Some(axis) => { + let ndim = shape.len(); + assert!(axis < ndim, "axis out of bounds"); + let axis_size = shape[axis]; + let outer = shape[..axis].iter().product::(); + let inner = shape[axis + 1..].iter().product::(); + let mut out_data = vec![f32::NEG_INFINITY; outer * inner]; + for outer_idx in 0..outer { + for inner_idx in 0..inner { + let mut best = f32::NEG_INFINITY; + for axis_idx in 0..axis_size { + let idx = (outer_idx * axis_size + axis_idx) * inner + inner_idx; + let val = data[idx]; + if val > best { + best = val; + } + } + out_data[outer_idx * inner + inner_idx] = best; + } + } + let out_shape = if keepdim { + let mut s = shape.clone(); + s[axis] = 1; + s + } else { + let mut s = shape.clone(); + s.remove(axis); + s + }; + let inner = StorageInner::from_vec(out_data, device); + View::from_inner_contiguous(Arc::new(inner), &out_shape) + } + } +} diff --git a/src/core/view.rs b/src/core/view.rs index e2a19ab..1c30876 100644 --- a/src/core/view.rs +++ b/src/core/view.rs @@ -2,6 +2,20 @@ use std::sync::Arc; use super::storage::StorageInner; +pub(crate) fn contiguous_strides(shape: &[usize]) -> Vec { + if shape.is_empty() { + return Vec::new(); + } + let mut stride = 1isize; + let mut strides = vec![0isize; shape.len()]; + for (i, dim) in shape.iter().rev().enumerate() { + let idx = shape.len() - 1 - i; + strides[idx] = stride; + stride *= *dim as isize; + } + strides +} + #[derive(Clone, Debug)] pub struct View { pub(crate) inner: Arc, @@ -21,6 +35,25 @@ impl View { } } + pub fn from_inner_contiguous(inner: Arc, shape: &[usize]) -> Self { + let expected = if shape.is_empty() { + 1 + } else { + shape.iter().product() + }; + debug_assert_eq!( + inner.len(), + expected, + "inner length must match shape product" + ); + Self { + inner, + offset: 0, + shape: shape.to_vec(), + strides: contiguous_strides(shape), + } + } + pub fn numel(&self) -> usize { self.shape.iter().product() } @@ -57,18 +90,11 @@ impl View { new_shape.iter().copied().product::(), "reshape size mismatch" ); - let mut stride = 1isize; - let mut strides = vec![0isize; new_shape.len()]; - for (i, dim) in new_shape.iter().rev().enumerate() { - let idx = new_shape.len() - 1 - i; - strides[idx] = stride; - stride *= *dim as isize; - } Self { inner: self.inner.clone(), offset: self.offset, shape: new_shape.to_vec(), - strides, + strides: contiguous_strides(new_shape), } } @@ -120,4 +146,36 @@ impl View { } // TODO: general strided iteration, overlap/alias checks. + + pub fn to_contiguous(&self) -> Self { + if self.is_contiguous() { + return self.clone(); + } + let numel = self.numel(); + let mut out_inner = StorageInner::new_full(0.0, numel, self.inner.device()); + { + let out_slice = out_inner.as_mut_slice(0, numel); + for (i, slot) in out_slice.iter_mut().enumerate() { + let mut linear = self.offset as isize; + if !self.shape.is_empty() { + let mut idx = i; + for axis in (0..self.shape.len()).rev() { + let dim = self.shape[axis]; + let stride = self.strides[axis]; + let coord = idx % dim; + idx /= dim; + linear += coord as isize * stride; + } + } + let linear = linear as usize; + *slot = self.inner.as_slice(linear, 1)[0]; + } + } + Self { + inner: Arc::new(out_inner), + offset: 0, + shape: self.shape.clone(), + strides: contiguous_strides(&self.shape), + } + } } diff --git a/src/lib.rs b/src/lib.rs index 7891b73..827f1ea 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,15 +1,19 @@ #![feature(portable_simd)] +#![cfg_attr(not(feature = "python"), allow(dead_code))] mod backend; mod core; mod device; +#[cfg(feature = "python")] mod py; +#[cfg(feature = "python")] use pyo3::prelude::*; #[cfg(test)] mod tests; +#[cfg(feature = "python")] #[pyo3::pymodule] fn cranberry(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; diff --git a/src/py/mod.rs b/src/py/mod.rs index 9f554ab..1c9a234 100644 --- a/src/py/mod.rs +++ b/src/py/mod.rs @@ -3,8 +3,123 @@ use std::sync::Arc; use pyo3::prelude::*; use crate::backend::{Backend, BinaryOp, CpuBackend, UnaryOp}; -use crate::core::{storage::StorageInner, view::View}; +use crate::core::{ + reduce::{self, AxisError}, + storage::StorageInner, + view::View, +}; use crate::device::Device; +use rand::{distributions::Distribution, rngs::StdRng, Rng, SeedableRng}; +use rand_distr::StandardNormal; + +fn axis_error_to_py(err: AxisError) -> PyErr { + match err { + AxisError::ScalarHasNoDim => { + pyo3::exceptions::PyValueError::new_err("dim specified for scalar tensor") + } + AxisError::OutOfRange => pyo3::exceptions::PyValueError::new_err("dim out of range"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + + fn make_view(shape: &[usize], data: Vec) -> View { + let inner = Arc::new(StorageInner::from_vec(data, Device::Cpu)); + View::from_inner_contiguous(inner, shape) + } + + #[test] + fn numel_handles_scalar_and_shape() { + assert_eq!(numel(&[]), 1); + assert_eq!(numel(&[3, 4, 5]), 60); + } + + #[test] + fn ensure_contiguous_clones_or_copies() { + let base = make_view(&[2, 3], vec![0.0; 6]); + let cloned = ensure_contiguous(&base); + assert!(Arc::ptr_eq(&cloned.inner, &base.inner)); + + let permuted = base.permute(&[1, 0]); + let contiguous = ensure_contiguous(&permuted); + assert!(contiguous.is_contiguous()); + assert_eq!(contiguous.shape, vec![3, 2]); + } + + #[test] + fn normalize_axis_accepts_negative() { + assert_eq!(normalize_axis(Some(-1), 3).unwrap(), Some(2)); + assert!(normalize_axis(Some(0), 0).is_err()); + assert!(normalize_axis(Some(4), 3).is_err()); + } + + #[test] + fn reduce_sum_none_matches_total() { + let data: Vec = (1..=6).map(|x| x as f32).collect(); + let view = make_view(&[2, 3], data); + let permuted = view.permute(&[1, 0]); + let out = reduce_sum_view(&permuted, None, false); + assert_eq!(out.shape, Vec::::new()); + let slice = out.inner.as_slice(out.offset, out.numel()); + assert_eq!(slice, &[21.0]); + + let out_keep = reduce_sum_view(&permuted, None, true); + assert_eq!(out_keep.shape, vec![1, 1]); + let slice = out_keep.inner.as_slice(out_keep.offset, out_keep.numel()); + assert_eq!(slice, &[21.0]); + } + + #[test] + fn reduce_sum_axis_matches_manual() { + let data: Vec = (0..12).map(|x| x as f32).collect(); + let view = make_view(&[2, 3, 2], data.clone()); + let out = reduce_sum_view(&view, Some(1), false); + assert_eq!(out.shape, vec![2, 2]); + let slice = out.inner.as_slice(out.offset, out.numel()); + let mut expected = Vec::new(); + for i in 0..2 { + for k in 0..2 { + let mut acc = 0.0f32; + for j in 0..3 { + let idx = (i * 3 + j) * 2 + k; + acc += data[idx]; + } + expected.push(acc); + } + } + assert_eq!(slice, expected.as_slice()); + + let out_keep = reduce_sum_view(&view, Some(1), true); + assert_eq!(out_keep.shape, vec![2, 1, 2]); + } + + #[test] + fn reduce_max_axis_matches_manual() { + let data: Vec = vec![1.0, 5.0, -2.0, 3.0, 4.0, 0.5, 7.0, -1.0]; + let view = make_view(&[2, 2, 2], data.clone()); + let out = reduce_max_view(&view, Some(0), false); + assert_eq!(out.shape, vec![2, 2]); + let slice = out.inner.as_slice(out.offset, out.numel()); + let mut expected = Vec::new(); + for j in 0..2 { + for k in 0..2 { + let mut best = f32::NEG_INFINITY; + for i in 0..2 { + let idx = (i * 2 + j) * 2 + k; + best = best.max(data[idx]); + } + expected.push(best); + } + } + assert_eq!(slice, expected.as_slice()); + + let out_keep = reduce_max_view(&view, Some(0), true); + assert_eq!(out_keep.shape, vec![1, 2, 2]); + } +} #[pyo3::pyclass] #[derive(Clone)] @@ -30,10 +145,84 @@ impl StorageView { }) } + #[staticmethod] + #[pyo3(signature = (shape, device="cpu"))] + pub fn zeros(shape: Vec, device: &str) -> PyResult { + let device = Device::from_str(device); + let inner = StorageInner::new_full(0.0, reduce::numel(&shape), device); + Ok(Self { + view: View::from_inner_contiguous(Arc::new(inner), &shape), + }) + } + + #[staticmethod] + #[pyo3(signature = (shape, device="cpu"))] + pub fn ones(shape: Vec, device: &str) -> PyResult { + let device = Device::from_str(device); + let inner = StorageInner::new_full(1.0, reduce::numel(&shape), device); + Ok(Self { + view: View::from_inner_contiguous(Arc::new(inner), &shape), + }) + } + + #[staticmethod] + #[pyo3(signature = (shape, device="cpu", seed=None))] + pub fn randn(shape: Vec, device: &str, seed: Option) -> PyResult { + let device = Device::from_str(device); + let mut rng = match seed { + Some(s) => StdRng::seed_from_u64(s), + None => StdRng::from_entropy(), + }; + let total = reduce::numel(&shape); + let mut data = Vec::with_capacity(total); + for _ in 0..total { + let sample: f64 = StandardNormal.sample(&mut rng); + data.push(sample as f32); + } + let inner = StorageInner::from_vec(data, device); + Ok(Self { + view: View::from_inner_contiguous(Arc::new(inner), &shape), + }) + } + + #[staticmethod] + #[pyo3(signature = (shape, low, high, device="cpu", seed=None))] + pub fn uniform( + shape: Vec, + low: f32, + high: f32, + device: &str, + seed: Option, + ) -> PyResult { + if !(low < high) { + return Err(pyo3::exceptions::PyValueError::new_err( + "uniform requires low < high", + )); + } + let device = Device::from_str(device); + let mut rng = match seed { + Some(s) => StdRng::seed_from_u64(s), + None => StdRng::from_entropy(), + }; + let total = reduce::numel(&shape); + let mut data = Vec::with_capacity(total); + for _ in 0..total { + data.push(rng.gen_range(low..high)); + } + let inner = StorageInner::from_vec(data, device); + Ok(Self { + view: View::from_inner_contiguous(Arc::new(inner), &shape), + }) + } + pub fn len(&self) -> usize { self.view.numel() } + pub fn shape(&self) -> Vec { + self.view.shape.clone() + } + pub fn to_vec(&self) -> PyResult> { if !self.view.is_contiguous() { return Err(pyo3::exceptions::PyValueError::new_err( @@ -43,6 +232,12 @@ impl StorageView { Ok(self.view.inner.to_vec(self.view.offset, self.view.numel())) } + pub fn contiguous(&self) -> PyResult { + Ok(StorageView { + view: self.view.to_contiguous(), + }) + } + pub fn slice(&self, offset: usize, size: usize) -> PyResult { if self.view.shape.len() != 1 { return Err(pyo3::exceptions::PyValueError::new_err( @@ -100,6 +295,20 @@ impl StorageView { )), } } + pub fn relu(&self) -> PyResult { + match self.view.inner.device() { + crate::device::Device::Cpu => { + let backend = CpuBackend; + let out = backend + .unary(UnaryOp::Relu, &self.view) + .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string()))?; + Ok(StorageView { view: out }) + } + _ => Err(pyo3::exceptions::PyNotImplementedError::new_err( + "unary ops not implemented for this device", + )), + } + } pub fn exp(&self) -> PyResult { match self.view.inner.device() { crate::device::Device::Cpu => { @@ -197,4 +406,34 @@ impl StorageView { )), } } + + #[pyo3(signature = (dim=None, keepdim=false))] + pub fn sum(&self, dim: Option, keepdim: bool) -> PyResult { + match self.view.inner.device() { + crate::device::Device::Cpu => { + let axis = + reduce::normalize_axis(dim, self.view.shape.len()).map_err(axis_error_to_py)?; + let reduced = reduce::reduce_sum(&self.view, axis, keepdim); + Ok(StorageView { view: reduced }) + } + _ => Err(pyo3::exceptions::PyNotImplementedError::new_err( + "sum not implemented for this device", + )), + } + } + + #[pyo3(signature = (dim=None, keepdim=false))] + pub fn max(&self, dim: Option, keepdim: bool) -> PyResult { + match self.view.inner.device() { + crate::device::Device::Cpu => { + let axis = + reduce::normalize_axis(dim, self.view.shape.len()).map_err(axis_error_to_py)?; + let reduced = reduce::reduce_max(&self.view, axis, keepdim); + Ok(StorageView { view: reduced }) + } + _ => Err(pyo3::exceptions::PyNotImplementedError::new_err( + "max not implemented for this device", + )), + } + } } diff --git a/src/tests/backend.rs b/src/tests/backend.rs index a115ff8..7d3200d 100644 --- a/src/tests/backend.rs +++ b/src/tests/backend.rs @@ -41,6 +41,23 @@ fn unary_sqrt_matches_scalar() { } } +#[test] +fn unary_relu_matches_scalar() { + let a = vec_view(vec![-3.0, -0.5, 0.0, 0.5, 2.0, 5.0]); + let be = CpuBackend; + let out = be.unary(UnaryOp::Relu, &a).unwrap(); + let actual = out.inner.as_slice(out.offset, out.numel()); + let expect: Vec = a + .inner + .as_slice(a.offset, a.numel()) + .iter() + .map(|x| x.max(0.0)) + .collect(); + for (got, exp) in actual.iter().zip(expect.iter()) { + assert!((got - exp).abs() < 1e-6); + } +} + #[test] fn binary_add_matches_scalar() { let a = vec_view((0..130).map(|i| i as f32 * 0.5).collect()); // span SIMD + remainder diff --git a/src/tests/mod.rs b/src/tests/mod.rs index d970109..e116269 100644 --- a/src/tests/mod.rs +++ b/src/tests/mod.rs @@ -1,3 +1,4 @@ mod backend; mod device; +mod reduce; mod view; diff --git a/src/tests/reduce.rs b/src/tests/reduce.rs new file mode 100644 index 0000000..7b71d7f --- /dev/null +++ b/src/tests/reduce.rs @@ -0,0 +1,108 @@ +use std::sync::Arc; + +use crate::core::{ + reduce::{self, AxisError}, + storage::StorageInner, + view::View, +}; + +fn make_view(shape: &[usize], data: Vec) -> View { + let inner = Arc::new(StorageInner::from_vec(data, crate::device::Device::Cpu)); + View::from_inner_contiguous(inner, shape) +} + +#[test] +fn numel_handles_scalar_and_shape() { + assert_eq!(reduce::numel(&[]), 1); + assert_eq!(reduce::numel(&[3, 4, 5]), 60); +} + +#[test] +fn ensure_contiguous_preserves_or_clones() { + let base = make_view(&[2, 3], vec![0.0; 6]); + let cloned = reduce::ensure_contiguous(&base); + assert!(Arc::ptr_eq(&cloned.inner, &base.inner)); + + let permuted = base.permute(&[1, 0]); + let contiguous = reduce::ensure_contiguous(&permuted); + assert!(contiguous.is_contiguous()); + assert_eq!(contiguous.shape, vec![3, 2]); +} + +#[test] +fn normalize_axis_reports_errors() { + assert_eq!(reduce::normalize_axis(Some(-1), 3).unwrap(), Some(2)); + assert!(matches!( + reduce::normalize_axis(Some(0), 0), + Err(AxisError::ScalarHasNoDim) + )); + assert!(matches!( + reduce::normalize_axis(Some(5), 3), + Err(AxisError::OutOfRange) + )); +} + +#[test] +fn reduce_sum_none_matches_total() { + let data: Vec = (1..=6).map(|x| x as f32).collect(); + let view = make_view(&[2, 3], data); + let permuted = view.permute(&[1, 0]); + let out = reduce::reduce_sum(&permuted, None, false); + assert_eq!(out.shape, Vec::::new()); + assert_eq!(out.inner.as_slice(out.offset, out.numel()), &[21.0]); + + let out_keep = reduce::reduce_sum(&permuted, None, true); + assert_eq!(out_keep.shape, vec![1, 1]); + assert_eq!( + out_keep.inner.as_slice(out_keep.offset, out_keep.numel()), + &[21.0] + ); +} + +#[test] +fn reduce_sum_axis_matches_manual() { + let data: Vec = (0..12).map(|x| x as f32).collect(); + let view = make_view(&[2, 3, 2], data.clone()); + let out = reduce::reduce_sum(&view, Some(1), false); + assert_eq!(out.shape, vec![2, 2]); + let slice = out.inner.as_slice(out.offset, out.numel()); + let mut expected = Vec::new(); + for i in 0..2 { + for k in 0..2 { + let mut acc = 0.0f32; + for j in 0..3 { + let idx = (i * 3 + j) * 2 + k; + acc += data[idx]; + } + expected.push(acc); + } + } + assert_eq!(slice, expected.as_slice()); + + let out_keep = reduce::reduce_sum(&view, Some(1), true); + assert_eq!(out_keep.shape, vec![2, 1, 2]); +} + +#[test] +fn reduce_max_axis_matches_manual() { + let data: Vec = vec![1.0, 5.0, -2.0, 3.0, 4.0, 0.5, 7.0, -1.0]; + let view = make_view(&[2, 2, 2], data.clone()); + let out = reduce::reduce_max(&view, Some(0), false); + assert_eq!(out.shape, vec![2, 2]); + let slice = out.inner.as_slice(out.offset, out.numel()); + let mut expected = Vec::new(); + for j in 0..2 { + for k in 0..2 { + let mut best = f32::NEG_INFINITY; + for i in 0..2 { + let idx = (i * 2 + j) * 2 + k; + best = best.max(data[idx]); + } + expected.push(best); + } + } + assert_eq!(slice, expected.as_slice()); + + let out_keep = reduce::reduce_max(&view, Some(0), true); + assert_eq!(out_keep.shape, vec![1, 2, 2]); +} diff --git a/src/tests/view.rs b/src/tests/view.rs index a1fe64c..4d432f7 100644 --- a/src/tests/view.rs +++ b/src/tests/view.rs @@ -35,6 +35,16 @@ fn view_reshape_contiguous() { assert_eq!(r.offset, 0); } +#[test] +fn view_from_inner_contiguous_matches_shape() { + let inner = Arc::new(StorageInner::new_full(1.0, 12, Device::Cpu)); + let v = View::from_inner_contiguous(inner.clone(), &[3, 4]); + assert!(v.is_contiguous()); + assert_eq!(v.shape, vec![3, 4]); + assert_eq!(v.strides, vec![4, 1]); + assert!(Arc::ptr_eq(&v.inner, &inner)); +} + #[test] fn view_permute_non_contiguous() { let inner = Arc::new(StorageInner::new_full(0.0, 12, Device::Cpu)); @@ -65,6 +75,28 @@ fn view_expand_incompatible_panics() { assert!(res.is_err()); } +#[test] +fn view_to_contiguous_copies_data_in_row_major_order() { + let data: Vec = (0..12).map(|x| x as f32).collect(); + let inner = Arc::new(StorageInner::from_vec(data.clone(), Device::Cpu)); + let base = View::from_inner_contiguous(inner, &[3, 4]); + let permuted = base.permute(&[1, 0]); + assert!(!permuted.is_contiguous()); + let contiguous = permuted.to_contiguous(); + assert!(contiguous.is_contiguous()); + assert_eq!(contiguous.shape, vec![4, 3]); + let slice = contiguous + .inner + .as_slice(contiguous.offset, contiguous.numel()); + let mut expected = vec![0.0f32; 12]; + for i in 0..3 { + for j in 0..4 { + expected[j * 3 + i] = data[i * 4 + j]; + } + } + assert_eq!(slice, expected.as_slice()); +} + #[test] fn view_reshape_requires_contiguous_panics() { let inner = Arc::new(StorageInner::new_full(0.0, 12, Device::Cpu)); From 82c21953f85c6a247e05b51691e01321d689aa3d Mon Sep 17 00:00:00 2001 From: manoflearning <77jwk0724@gmail.com> Date: Wed, 17 Sep 2025 19:07:17 +0900 Subject: [PATCH 2/7] feat: remove NumPy --- .github/workflows/main.yaml | 4 +- cranberry/cranberry.pyi | 13 + cranberry/features/datasets.py | 3 + cranberry/nn/__init__.py | 15 +- cranberry/ops.py | 38 +- cranberry/optim/__init__.py | 23 +- cranberry/tensor.py | 933 +++++++++++++++++++++------------ cranberry/view.py | 9 +- examples/mnist.py | 12 +- tests/test_storageview.py | 630 +++++++++++----------- 10 files changed, 996 insertions(+), 684 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 27e6519..ca1a00c 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -55,7 +55,9 @@ jobs: run: uv sync --locked --dev - name: Ruff - run: uv run ruff check . + run: | + uv run ruff check . + uv run ruff format --check . - name: Pyright run: uv run pyright diff --git a/cranberry/cranberry.pyi b/cranberry/cranberry.pyi index 1a87693..5325d94 100644 --- a/cranberry/cranberry.pyi +++ b/cranberry/cranberry.pyi @@ -6,17 +6,30 @@ class StorageView: def full(value: float, size: int, device: str) -> StorageView: ... @staticmethod def from_vec(vec: Union[list[float], np.ndarray], device: str) -> StorageView: ... + @staticmethod + def zeros(shape: list[int], device: str = ...) -> StorageView: ... + @staticmethod + def ones(shape: list[int], device: str = ...) -> StorageView: ... + @staticmethod + def randn(shape: list[int], device: str = ..., seed: int | None = ...) -> StorageView: ... + @staticmethod + def uniform(shape: list[int], low: float, high: float, device: str = ..., seed: int | None = ...) -> StorageView: ... def len(self) -> int: ... + def shape(self) -> list[int]: ... def to_vec(self) -> list[float]: ... def slice(self, offset: int, size: int) -> StorageView: ... def reshape(self, shape: list[int]) -> StorageView: ... def expand(self, shape: list[int]) -> StorageView: ... def permute(self, dims: list[int]) -> StorageView: ... + def contiguous(self) -> StorageView: ... def neg(self) -> StorageView: ... def sqrt(self) -> StorageView: ... + def relu(self) -> StorageView: ... def exp(self) -> StorageView: ... def log(self) -> StorageView: ... def add(self, other: StorageView) -> StorageView: ... def sub(self, other: StorageView) -> StorageView: ... def mul(self, other: StorageView) -> StorageView: ... def div(self, other: StorageView) -> StorageView: ... + def sum(self, dim: int | None, keepdim: bool = ...) -> StorageView: ... + def max(self, dim: int | None, keepdim: bool = ...) -> StorageView: ... diff --git a/cranberry/features/datasets.py b/cranberry/features/datasets.py index 013bae3..960522d 100644 --- a/cranberry/features/datasets.py +++ b/cranberry/features/datasets.py @@ -17,11 +17,14 @@ # Optional tqdm progress bar try: from tqdm import tqdm as _tqdm # type: ignore + tqdm = _tqdm # noqa: N802 - keep name compatibility except Exception: # pragma: no cover - fallback path + class _NoopTqdm: def __init__(self, *a, **kw): pass + def update(self, n: int): pass diff --git a/cranberry/nn/__init__.py b/cranberry/nn/__init__.py index 48e5b93..aa51cbe 100644 --- a/cranberry/nn/__init__.py +++ b/cranberry/nn/__init__.py @@ -7,16 +7,20 @@ class Module: @abstractmethod - def __call__(self, x: Tensor) -> Tensor: pass + def __call__(self, x: Tensor) -> Tensor: + pass @abstractmethod - def parameters(self) -> List[Tensor]: pass + def parameters(self) -> List[Tensor]: + pass class ReLU(Module): - def __call__(self, x: Tensor) -> Tensor: return x.relu() + def __call__(self, x: Tensor) -> Tensor: + return x.relu() - def parameters(self) -> List[Tensor]: return [] + def parameters(self) -> List[Tensor]: + return [] class Linear(Module): @@ -38,7 +42,8 @@ def __init__(self, *layers: Module): self.layers = layers def __call__(self, x: Tensor) -> Tensor: - for layer in self.layers: x = layer(x) + for layer in self.layers: + x = layer(x) return x def parameters(self) -> List[Tensor]: diff --git a/cranberry/ops.py b/cranberry/ops.py index 01fb513..db311ac 100644 --- a/cranberry/ops.py +++ b/cranberry/ops.py @@ -3,19 +3,41 @@ class UnaryOps(Enum): - NEG = auto(); SQRT = auto(); RELU = auto(); EXP = auto(); LOG = auto() # noqa: E702 - def __repr__(self): return f"{self.name.lower()}" + NEG = auto() + SQRT = auto() + RELU = auto() + EXP = auto() + LOG = auto() # noqa: E702 + + def __repr__(self): + return f"{self.name.lower()}" + class BinaryOps(Enum): - ADD = auto(); SUB = auto(); MUL = auto(); DIV = auto() # noqa: E702 - def __repr__(self): return f"{self.name.lower()}" + ADD = auto() + SUB = auto() + MUL = auto() + DIV = auto() # noqa: E702 + + def __repr__(self): + return f"{self.name.lower()}" + class ReduceOps(Enum): - SUM = auto(); MAX = auto() # noqa: E702 - def __repr__(self): return f"{self.name.lower()}" + SUM = auto() + MAX = auto() # noqa: E702 + + def __repr__(self): + return f"{self.name.lower()}" + class MovementOps(Enum): - RESHAPE = auto(); EXPAND = auto(); PERMUTE = auto() # noqa: E702 - def __repr__(self): return f"{self.name.lower()}" + RESHAPE = auto() + EXPAND = auto() + PERMUTE = auto() # noqa: E702 + + def __repr__(self): + return f"{self.name.lower()}" + Op = Union[UnaryOps, BinaryOps, ReduceOps, MovementOps, None] diff --git a/cranberry/optim/__init__.py b/cranberry/optim/__init__.py index 026e1e3..b134d35 100644 --- a/cranberry/optim/__init__.py +++ b/cranberry/optim/__init__.py @@ -1,4 +1,4 @@ -from cranberry import Tensor +from cranberry import StorageView, Tensor from typing import List @@ -11,16 +11,29 @@ def __init__(self, params: List[Tensor], lr: float): self._params, self._lr = params, lr def zero_grad(self): - for p in self._params: p._grad.fill(0.0) + for p in self._params: + p.zero_grad() def step(self): - for p in self._params: p._data -= self._lr * p._grad + for p in self._params: + grad_storage = p.grad_storage() + if grad_storage is None: + continue + grad_view = grad_storage.contiguous() + shape_list = list(p.shape) + scale = StorageView.full(float(self._lr), max(p.num_elements(), 1), p.device) + scale = scale.reshape(shape_list) if shape_list else scale.reshape([]) + scaled_grad = grad_view.mul(scale) + new_data = p.data_storage().contiguous().sub(scaled_grad) + p.set_data_storage(new_data) @property - def lr(self): return self._lr + def lr(self): + return self._lr @lr.setter - def lr(self, lr: float): self._lr = lr + def lr(self, lr: float): + self._lr = lr # https://pytorch.org/docs/stable/generated/torch.optim.Adam.html diff --git a/cranberry/tensor.py b/cranberry/tensor.py index 5ccfc1d..ea1057f 100644 --- a/cranberry/tensor.py +++ b/cranberry/tensor.py @@ -1,155 +1,297 @@ from __future__ import annotations import math -from cranberry.ops import Op, UnaryOps, BinaryOps, ReduceOps, MovementOps -import time -from typing import List, Optional, Tuple, Union +from typing import Iterable, List, Optional, Tuple, Union + import numpy as np -from math import prod + +from .cranberry import StorageView +from cranberry.ops import BinaryOps, MovementOps, Op, ReduceOps, UnaryOps from cranberry.shape import Shape -def shape_for_list(x: List) -> Shape: - shape = [] - while isinstance(x, List): - shape.append(len(x)) - x = x[0] - return Shape(tuple(shape)) +def shape_for_list(x: List) -> Tuple[int, ...]: + shape: List[int] = [] + current = x + while isinstance(current, list): + shape.append(len(current)) + if len(current) == 0: + break + current = current[0] + return tuple(shape) + + +def _flatten_list(x: List) -> List[float]: + flattened: List[float] = [] + + def _recurse(value): + if isinstance(value, list): + for item in value: + _recurse(item) + else: + flattened.append(float(value)) + + _recurse(x) + return flattened + + +def _ensure_shape_tuple(shape: Optional[Union[Tuple[int, ...], Shape]]) -> Optional[Tuple[int, ...]]: + if shape is None: + return None + if isinstance(shape, Shape): + return tuple(shape.dims) + return tuple(int(dim) for dim in shape) + + +def _prod(values: Iterable[int]) -> int: + result = 1 + for value in values: + result *= int(value) + return result + + +def _shape_to_list(shape: Tuple[int, ...]) -> List[int]: + return list(shape) if len(shape) > 0 else [] + + +def _storage_shape(storage: StorageView) -> Tuple[int, ...]: + shape = tuple(storage.shape()) + return shape + + +def _contiguous(storage: StorageView) -> StorageView: + return storage.contiguous() + + +def _coerce_to_storage( + data: Union["Tensor", StorageView, float, int, List, np.ndarray], + shape: Optional[Union[Tuple[int, ...], Shape]], + device: str, +) -> Tuple[StorageView, Tuple[int, ...]]: + target_shape = _ensure_shape_tuple(shape) + + if isinstance(data, Tensor): + storage = data._storage + inferred_shape = data.shape + if target_shape is not None and target_shape != inferred_shape: + storage = storage.reshape(_shape_to_list(target_shape)) + inferred_shape = target_shape + return storage, inferred_shape + + if isinstance(data, StorageView): + inferred_shape = _storage_shape(data) + storage = data + if target_shape is not None and target_shape != inferred_shape: + storage = storage.reshape(_shape_to_list(target_shape)) + inferred_shape = target_shape + return storage, inferred_shape + + if isinstance(data, (float, int)): + value = float(data) + storage = StorageView.full(value, 1, device) + if target_shape is None: + inferred_shape: Tuple[int, ...] = () + storage = storage.reshape([]) + else: + inferred_shape = target_shape + storage = storage.reshape(_shape_to_list(inferred_shape)) + return storage, inferred_shape + + if isinstance(data, list): + inferred = shape_for_list(data) + flat = _flatten_list(data) + storage = StorageView.from_vec(flat, device) + if target_shape is None: + target_shape = inferred + if _prod(inferred) != _prod(target_shape): + raise ValueError("provided shape is incompatible with data") + storage = storage.reshape(_shape_to_list(target_shape)) + return storage, target_shape + + if isinstance(data, np.ndarray): + arr = np.asarray(data, dtype=np.float32) + inferred = tuple(int(dim) for dim in arr.shape) + flat = arr.reshape(-1).tolist() + storage = StorageView.from_vec(flat, device) + if target_shape is None: + target_shape = inferred + if _prod(inferred) != _prod(target_shape): + raise ValueError("provided shape is incompatible with data") + storage = storage.reshape(_shape_to_list(target_shape)) + return storage, target_shape + + raise ValueError(f"invalid data type {type(data)}") class Tensor: def __init__( self, - data: Union[float, int, List, np.ndarray, np.float32], - grad: Optional[np.ndarray] = None, - shape: Optional[Union[Tuple, Shape]] = None, + data: Union[float, int, List, np.ndarray, StorageView, "Tensor"], + grad: Optional[Union[List, np.ndarray, StorageView, "Tensor"]] = None, + shape: Optional[Union[Tuple[int, ...], Shape]] = None, requires_grad: bool = False, prev: Optional[Tuple[Tensor, ...]] = None, op: Optional[Op] = None, + device: str = "cpu", ): - # self._data - if isinstance(data, (float, int)): - self._data = np.array(data, dtype=np.float32) - elif isinstance(data, list): - self._data = np.array(data, dtype=np.float32) - elif isinstance(data, np.ndarray): - self._data = data.astype(np.float32) - elif type(data) is np.float32: - self._data = np.array(data, dtype=np.float32) - else: - raise ValueError(f"invalid data type {type(data)}") - - # self._grad - self._grad: np.ndarray = grad if grad is not None else np.zeros_like(self._data) - - # self._shape - if isinstance(shape, Shape): - self._shape = shape - elif isinstance(shape, tuple): - self._shape = Shape(shape) - elif shape is None: - if isinstance(data, (float, int)): - self._shape = Shape(()) - elif isinstance(data, list): - self._shape = shape_for_list(data) - elif isinstance(data, np.ndarray): - self._shape = Shape(data.shape) - elif type(data) is np.float32: - self._shape = Shape(()) - else: - raise ValueError(f"invalid data type {type(data)}") - - assert self._shape == Shape(self._data.shape), f"shape {self._shape} must match data shape {self._data.shape}" - assert self._shape == Shape(self._grad.shape), f"shape {self._shape} must match grad shape {self._grad.shape}" - - # self._requires_grad + storage, inferred_shape = _coerce_to_storage(data, shape, device) + self._storage = storage + self._shape = inferred_shape + self._device = device + self._requires_grad: bool = requires_grad - # self._backward - self._backward = lambda: None - # self._prev self._prev: Optional[Tuple[Tensor, ...]] = prev - # self._op self._op = op + self._backward = lambda: None + + if grad is not None: + grad_storage, _ = _coerce_to_storage(grad, inferred_shape, device) + self._grad_storage: Optional[StorageView] = grad_storage + else: + self._grad_storage = StorageView.zeros(_shape_to_list(self.shape), device) if requires_grad else None + + # ******************************************************** + # *************** internal utilities *************** + # ******************************************************** + + @classmethod + def _from_storage( + cls, + storage: StorageView, + *, + requires_grad: bool, + prev: Optional[Tuple[Tensor, ...]], + op: Optional[Op], + device: str, + ) -> Tensor: + obj = object.__new__(cls) + obj._storage = storage + obj._shape = _storage_shape(storage) + obj._device = device + obj._requires_grad = requires_grad + obj._prev = prev + obj._op = op + obj._backward = lambda: None + obj._grad_storage = StorageView.zeros(_shape_to_list(obj.shape), device) if requires_grad else None + return obj + + def _shape_list(self) -> List[int]: + return _shape_to_list(self._shape) + + def _numel(self) -> int: + return _prod(self._shape) if len(self._shape) else 1 + + def _ensure_grad(self) -> StorageView: + if self._grad_storage is None: + self._grad_storage = StorageView.zeros(self._shape_list(), self._device) + return self._grad_storage + + def _add_grad(self, grad: StorageView): + if not self._requires_grad: + return + grad_buf = self._ensure_grad() + grad_c = _contiguous(grad) + self._grad_storage = _contiguous(grad_buf).add(grad_c) + + def _constant_like(self, value: float) -> StorageView: + storage = StorageView.full(float(value), max(self._numel(), 1), self._device) + return storage.reshape(self._shape_list()) # ******************************************************** # *************** backward prop *************** # ******************************************************** def backward(self): - assert self._requires_grad, "cannot call backward on a tensor that doesn't require gradients" - assert self.shape == tuple(), f"backward can only be called for scalar tensors, but it has shape {self.shape})" + if not self._requires_grad: + raise AssertionError("cannot call backward on a tensor that doesn't require gradients") + if self.shape != (): + raise AssertionError(f"backward can only be called for scalar tensors, but it has shape {self.shape}") - topo = [] + topo: List[Tensor] = [] visited = set() def dfs(t: Tensor): + if t in visited: + return visited.add(t) if t._prev is not None: for v in t._prev: - if v not in visited and v._requires_grad: - dfs(v) + dfs(v) topo.append(t) dfs(self) + self._grad_storage = StorageView.ones([], self._device) - self._grad.fill(1.0) for v in reversed(topo): v._backward() - # ******************************************************** - # *************** broadcasting *************** - # ******************************************************** - - def _broadcasted(self, other: Tensor) -> Tuple[Tensor, Tensor]: - shape1, shape2 = self.shape, other.shape - while len(shape1) < len(shape2): - shape1 = (1,) + shape1 - while len(shape2) < len(shape1): - shape2 = (1,) + shape2 - shape = tuple(max(s1, s2) for s1, s2 in zip(shape1, shape2)) - if self.shape != shape: - self = self.expand(*shape) - if other.shape != shape: - other = other.expand(*shape) - return self, other - # ******************************************************** # *************** unary ops *************** # ******************************************************** def _unary_op(self, op: UnaryOps) -> Tensor: - out = Tensor._dummy(shape=self._shape, requires_grad=self.requires_grad, prev=(self,), op=op) - match op: - case UnaryOps.NEG: - out._data -= self._data - out._backward = lambda: self._grad.__isub__(out._grad) - case UnaryOps.SQRT: - out._data = np.sqrt(self._data) - out._backward = lambda: self._grad.__iadd__(0.5 * out._grad / out._data) - case UnaryOps.RELU: - out._data = self._data * (self._data > 0) - out._backward = lambda: self._grad.__iadd__(out._grad * (self._data > 0)) - case UnaryOps.EXP: - out._data = np.exp(self._data) - out._backward = lambda: self._grad.__iadd__(out._grad * out._data) - case UnaryOps.LOG: - out._data = np.log(self._data) - out._backward = lambda: self._grad.__iadd__(out._grad / self._data) - case _: - raise RuntimeError(f"invalid unary op {op}") + method = { + UnaryOps.NEG: "neg", + UnaryOps.SQRT: "sqrt", + UnaryOps.RELU: "relu", + UnaryOps.EXP: "exp", + UnaryOps.LOG: "log", + }[op] + + input_c = _contiguous(self._storage) + storage = getattr(input_c, method)() + out = Tensor._from_storage(storage, requires_grad=self._requires_grad, prev=(self,), op=op, device=self._device) + + if self._requires_grad: + + def backward(): + out_grad = out._grad_storage + if out_grad is None: + return + out_grad_c = _contiguous(out_grad) + if op is UnaryOps.NEG: + self._add_grad(out_grad_c.neg()) + elif op is UnaryOps.SQRT: + half = out._constant_like(0.5) + contrib = out_grad_c.mul(half).div(_contiguous(out._storage)) + self._add_grad(contrib) + elif op is UnaryOps.RELU: + values = input_c.to_vec() + mask = [1.0 if v > 0.0 else 0.0 for v in values] + mask_storage = StorageView.from_vec(mask, self._device).reshape(self._shape_list()) + self._add_grad(out_grad_c.mul(mask_storage)) + elif op is UnaryOps.EXP: + self._add_grad(out_grad_c.mul(_contiguous(out._storage))) + elif op is UnaryOps.LOG: + self._add_grad(out_grad_c.div(input_c)) + + out._backward = backward + return out - def neg(self) -> Tensor: return self._unary_op(UnaryOps.NEG) - def __neg__(self) -> Tensor: return self.neg() - def sqrt(self) -> Tensor: return self._unary_op(UnaryOps.SQRT) - def relu(self) -> Tensor: return self._unary_op(UnaryOps.RELU) - def exp(self) -> Tensor: return self._unary_op(UnaryOps.EXP) - def log(self) -> Tensor: return self._unary_op(UnaryOps.LOG) + def neg(self) -> Tensor: + return self._unary_op(UnaryOps.NEG) + + def __neg__(self) -> Tensor: + return self.neg() + + def sqrt(self) -> Tensor: + return self._unary_op(UnaryOps.SQRT) + + def relu(self) -> Tensor: + return self._unary_op(UnaryOps.RELU) + + def exp(self) -> Tensor: + return self._unary_op(UnaryOps.EXP) + + def log(self) -> Tensor: + return self._unary_op(UnaryOps.LOG) def sigmoid(self) -> Tensor: - # 1 / (1 + exp(-x)) or exp(x) / (1 + exp(x)) return 1 / (1 + self.neg().exp()) + def tanh(self) -> Tensor: return (self.exp() - self.neg().exp()) / (self.exp() + self.neg().exp()) + def gelu(self) -> Tensor: return 0.5 * self * (1 + (self * 0.7978845608 * (1 + 0.044715 * self * self)).tanh()) @@ -158,123 +300,188 @@ def gelu(self) -> Tensor: # ******************************************************** def _binary_op(self, other: Union[Tensor, int, float], reverse: bool, op: BinaryOps) -> Tensor: - if isinstance(other, (int, float)): other = Tensor(other) - self, other = self._broadcasted(other) - if reverse: self, other = other, self - - out = Tensor._dummy( - shape=self._shape, - requires_grad=self.requires_grad or other.requires_grad, - prev=(self, other), - op=op, - ) - - match op: - case BinaryOps.ADD: - out._data = self._data + other._data - out._backward = lambda: ( - self._grad.__iadd__(out._grad), - other._grad.__iadd__(out._grad), - ) - case BinaryOps.SUB: - out._data = self._data - other._data - out._backward = lambda: ( - self._grad.__iadd__(out._grad), - other._grad.__isub__(out._grad), - ) - case BinaryOps.MUL: - out._data = self._data * other._data - out._backward = lambda: ( - self._grad.__iadd__(out._grad * other._data), - other._grad.__iadd__(out._grad * self._data), - ) - case BinaryOps.DIV: - out._data = self._data / other._data - out._backward = lambda: ( - self._grad.__iadd__(out._grad / other._data), - other._grad.__isub__(out._grad * self._data / other._data**2), - ) - case _: - raise RuntimeError(f"invalid binary op {op}") + if not isinstance(other, Tensor): + other = Tensor(other) + + left, right = self, other + if reverse: + left, right = right, left + + left_b, right_b = left._broadcasted(right) + + method = { + BinaryOps.ADD: "add", + BinaryOps.SUB: "sub", + BinaryOps.MUL: "mul", + BinaryOps.DIV: "div", + }[op] + + left_c = _contiguous(left_b._storage) + right_c = _contiguous(right_b._storage) + storage = getattr(left_c, method)(right_c) + requires_grad = left_b._requires_grad or right_b._requires_grad + out = Tensor._from_storage(storage, requires_grad=requires_grad, prev=(left_b, right_b), op=op, device=self._device) + + if out._requires_grad: + + def backward(): + out_grad = out._grad_storage + if out_grad is None: + return + if left_b._requires_grad: + if op in (BinaryOps.ADD, BinaryOps.SUB): + left_b._add_grad(_contiguous(out_grad)) + elif op is BinaryOps.MUL: + left_b._add_grad(_contiguous(out_grad).mul(right_c)) + elif op is BinaryOps.DIV: + left_b._add_grad(_contiguous(out_grad).div(right_c)) + + if right_b._requires_grad: + if op is BinaryOps.ADD: + right_b._add_grad(_contiguous(out_grad)) + elif op is BinaryOps.SUB: + right_b._add_grad(_contiguous(out_grad).neg()) + elif op is BinaryOps.MUL: + right_b._add_grad(_contiguous(out_grad).mul(left_c)) + elif op is BinaryOps.DIV: + num = _contiguous(out_grad).mul(left_c) + denom = right_c.mul(right_c) + right_b._add_grad(num.div(denom).neg()) + + out._backward = backward + return out def add(self, other: Union[Tensor, int, float], reverse: bool = False) -> Tensor: return self._binary_op(other, reverse, BinaryOps.ADD) + def sub(self, other: Union[Tensor, int, float], reverse: bool = False) -> Tensor: return self._binary_op(other, reverse, BinaryOps.SUB) + def mul(self, other: Union[Tensor, int, float], reverse: bool = False) -> Tensor: return self._binary_op(other, reverse, BinaryOps.MUL) + def div(self, other: Union[Tensor, int, float], reverse: bool = False) -> Tensor: return self._binary_op(other, reverse, BinaryOps.DIV) - def __add__(self, other) -> Tensor: return self.add(other) - def __sub__(self, other) -> Tensor: return self.sub(other) - def __mul__(self, other) -> Tensor: return self.mul(other) - def __truediv__(self, other) -> Tensor: return self.div(other) + def __add__(self, other) -> Tensor: + return self.add(other) + + def __sub__(self, other) -> Tensor: + return self.sub(other) + + def __mul__(self, other) -> Tensor: + return self.mul(other) - def __radd__(self, x) -> Tensor: return self.add(x, True) - def __rsub__(self, x) -> Tensor: return self.sub(x, True) - def __rmul__(self, other) -> Tensor: return self.mul(other, True) - def __rtruediv__(self, other) -> Tensor: return self.div(other, True) + def __truediv__(self, other) -> Tensor: + return self.div(other) - # TODO: __lt__, __gt__, __ge__, __le__, __eq__, __ne__ + def __radd__(self, x) -> Tensor: + return self.add(x, True) + + def __rsub__(self, x) -> Tensor: + return self.sub(x, True) + + def __rmul__(self, other) -> Tensor: + return self.mul(other, True) + + def __rtruediv__(self, other) -> Tensor: + return self.div(other, True) # ******************************************************** # *************** reduce ops *************** # ******************************************************** - def _reduce_op(self, *args, op: ReduceOps) -> Tensor: - out = Tensor._dummy(shape=Shape(()), requires_grad=self.requires_grad, prev=(self,), op=op) - match op: - case ReduceOps.SUM: - dim, keepdim = args - out._data = self._data.sum(axis=dim, keepdims=keepdim) - out._grad = np.zeros_like(out._data) - out._shape = Shape(out._data.shape) if isinstance(out._data, np.ndarray) else Shape(()) - - def backward(): - if dim is None or keepdim: - self._grad += out._grad - else: - o_new_shape = tuple(1 if i == dim else s for i, s in enumerate(self.shape)) - self._grad += out._grad.reshape(o_new_shape) - - out._backward = backward - case ReduceOps.MAX: - dim, keepdim = args - out._data = self._data.max(axis=dim, keepdims=keepdim) - out._grad = np.zeros_like(out._data) - out._shape = Shape(out._data.shape) if isinstance(out._data, np.ndarray) else Shape(()) - - def backward(): - if dim is None or keepdim: - self._grad += (self._data == out._data) * out._grad - else: - o_new_shape = tuple(1 if i == dim else s for i, s in enumerate(self.shape)) - self._grad += (self._data == out._data.reshape(o_new_shape)) * out._grad.reshape(o_new_shape) - - out._backward = backward - case _: - raise RuntimeError(f"invalid reduce op {op}") + def _normalize_dim(self, dim: Optional[int]) -> Optional[int]: + if dim is None: + return None + if dim < 0: + dim += len(self.shape) + if dim < 0 or dim >= len(self.shape): + raise ValueError("dim out of range") + return dim + + def sum(self, dim: Optional[int] = None, keepdim: bool = False) -> Tensor: + axis = self._normalize_dim(dim) + storage = self._storage.sum(axis if axis is not None else None, keepdim) + out = Tensor._from_storage(storage, requires_grad=self._requires_grad, prev=(self,), op=ReduceOps.SUM, device=self._device) + + if self._requires_grad: + + def backward(): + out_grad = out._grad_storage + if out_grad is None: + return + grad = _contiguous(out_grad) + dims = list(self.shape) + if axis is None: + if not keepdim and dims: + grad = grad.reshape([1] * len(dims)) + if dims: + grad = grad.expand(dims) + else: + if not keepdim: + reshape_shape = dims.copy() + reshape_shape[axis] = 1 + grad = grad.reshape(reshape_shape) + grad = grad.expand(dims) + self._add_grad(grad) + + out._backward = backward + return out - def sum(self, dim: Optional[int] = None, keepdim=False) -> Tensor: - # TODO: add keepdim - return self._reduce_op(dim, keepdim, op=ReduceOps.SUM) + def max(self, dim: Optional[int] = None, keepdim: bool = False) -> Tensor: + axis = self._normalize_dim(dim) + storage = self._storage.max(axis if axis is not None else None, keepdim) + out = Tensor._from_storage(storage, requires_grad=self._requires_grad, prev=(self,), op=ReduceOps.MAX, device=self._device) + + if self._requires_grad: + + def backward(): + out_grad = out._grad_storage + if out_grad is None: + return + grad = _contiguous(out_grad) + input_shape = self.shape or (1,) + input_vals = np.array(_contiguous(self._storage).to_vec(), dtype=np.float32).reshape(input_shape) + + out_shape = out.shape or (1,) + max_vals = np.array(_contiguous(out._storage).to_vec(), dtype=np.float32).reshape(out_shape) + grad_vals = np.array(grad.to_vec(), dtype=np.float32).reshape(out_shape) + + if axis is None: + if not keepdim and len(input_shape) > 0: + grad_vals = grad_vals.reshape((1,) * len(input_shape)) + max_vals = max_vals.reshape((1,) * len(input_shape)) + grad_broadcast = np.broadcast_to(grad_vals, input_shape) + max_broadcast = np.broadcast_to(max_vals, input_shape) + else: + axis_idx = axis + if not keepdim: + grad_vals = np.expand_dims(grad_vals, axis_idx) + max_vals = np.expand_dims(max_vals, axis_idx) + grad_broadcast = np.broadcast_to(grad_vals, input_shape) + max_broadcast = np.broadcast_to(max_vals, input_shape) + + mask = np.isclose(input_vals, max_broadcast, rtol=1e-6, atol=1e-6).astype(np.float32) + grad_result = (mask * grad_broadcast).astype(np.float32) + grad_storage = StorageView.from_vec(grad_result.reshape(-1).tolist(), self._device).reshape(self._shape_list()) + self._add_grad(grad_storage) + + out._backward = backward - def max(self, dim: Optional[int] = None, keepdim=False) -> Tensor: - # different return types than pytorch: https://pytorch.org/docs/stable/generated/torch.max.html - return self._reduce_op(dim, keepdim, op=ReduceOps.MAX) + return out - def mean(self, dim: Optional[int] = None, keepdim=False): + def mean(self, dim: Optional[int] = None, keepdim: bool = False): out = self.sum(dim=dim, keepdim=keepdim) - return out.div(prod(self.shape) / prod(out.shape)) + factor = _prod(self.shape) / _prod(out.shape or (1,)) + return out / factor def _softmax(self, dim: Optional[int]) -> Tuple[Tensor, Tensor, Tensor]: if len(self.shape) == 0: assert dim in [-1, 0], f"invalid dim {dim} for tensor {self}" dim = None - # it makes the softmax more numerically stable m = self - self.max(dim=dim, keepdim=True) e = m.exp() return m, e, e.sum(dim=dim, keepdim=True) @@ -284,10 +491,6 @@ def softmax(self, dim: int = -1) -> Tensor: return e.div(ss) def log_softmax(self, dim: int = -1) -> Tensor: - # x.log_softmax() - # = x.softmax().log() - # = log(exp(x_i) / sum(exp(x_k))) - # = x_i - log(sum(exp(x_k))) m, _, ss = self._softmax(dim) return m - ss.log() @@ -295,69 +498,94 @@ def log_softmax(self, dim: int = -1) -> Tensor: # *************** movement ops *************** # ******************************************************** - def _movement_op(self, *args, op: MovementOps) -> Tensor: - out = Tensor._dummy(shape=Shape(args[0]), requires_grad=self.requires_grad, prev=(self,), op=op) - match op: - case MovementOps.RESHAPE: - out._data = self._data.reshape(args[0]) - out._grad = self._grad.reshape(args[0]) - - def backward(): - if out._grad.base is None: # TODO: initially, out._grad is a view of self._grad, but some reason it becomes a copy - self._grad += out._grad.reshape(self.shape) - - out._backward = backward - case MovementOps.EXPAND: - out._data = np.copy(np.broadcast_to(self._data, args[0])) - out._grad = np.copy(np.broadcast_to(self._grad, args[0])) - - def backward(): - s_shape, o_shape = self.shape, out.shape - while len(s_shape) < len(o_shape): - s_shape = (1,) + s_shape - axis = tuple(i for i in range(len(o_shape)) if s_shape[i] == 1) - self._grad += out._grad.sum(axis=axis, keepdims=True).reshape(self.shape) - - out._backward = backward - case MovementOps.PERMUTE: - out._data = self._data.transpose(args[1]) - out._grad = self._grad.transpose(args[1]) - case _: - raise RuntimeError(f"invalid movement op {op}") - return out - def reshape(self, *shape: int) -> Tensor: - assert shape.count(-1) <= 1, "can only specify one unknown dimension" - assert all(s > 0 or s == -1 for s in shape), "shape dimensions must be positive or -1" - - if shape.count(-1) == 1: - assert prod(self._shape) % -prod(shape) == 0, f"cannot reshape tensor of size {prod(self._shape)} into shape {shape}" - shape = tuple(s if s != -1 else prod(self._shape) // -prod(shape) for s in shape) - assert prod(shape) == prod(self._shape), f"cannot reshape tensor of size {prod(self._shape)} into shape {shape}" + shape_tuple = tuple(shape) + if shape_tuple.count(-1) > 1: + raise ValueError("can only specify one unknown dimension") + if any(s == 0 for s in shape_tuple): + raise ValueError("shape dimensions must be positive") + total = self._numel() + if -1 in shape_tuple: + known = _prod(s for s in shape_tuple if s != -1) + missing = total // known + shape_tuple = tuple(missing if s == -1 else s for s in shape_tuple) + if _prod(shape_tuple) != total: + raise ValueError(f"cannot reshape tensor of size {total} into shape {shape_tuple}") + storage = _contiguous(self._storage).reshape(list(shape_tuple)) + out = Tensor._from_storage(storage, requires_grad=self._requires_grad, prev=(self,), op=MovementOps.RESHAPE, device=self._device) + + if self._requires_grad: + + def backward(): + grad = out._grad_storage + if grad is None: + return + self._add_grad(_contiguous(grad).reshape(self._shape_list())) + + out._backward = backward - return self._movement_op(shape, op=MovementOps.RESHAPE) + return out - # https://pytorch.org/docs/stable/generated/torch.Tensor.expand.html def expand(self, *shape: int) -> Tensor: - assert len(shape) >= len(self.shape), f"the expanded shape {shape} must have at least as many dimensions as the original shape {self.shape}" - assert all( - s == 1 or s == e for s, e in zip(self.shape, shape[-len(self.shape) :]) - ), "the expanded shape must be compatible with the original shape" - return self._movement_op(shape, op=MovementOps.EXPAND) + if len(shape) < len(self.shape): + raise ValueError("expanded shape must have at least the same number of dimensions") + target = tuple(shape) + storage = self._storage.expand(list(target)) + out = Tensor._from_storage(storage, requires_grad=self._requires_grad, prev=(self,), op=MovementOps.EXPAND, device=self._device) + + if self._requires_grad: + + def backward(): + grad = out._grad_storage + if grad is None: + return + grad_reduced = _contiguous(grad) + in_shape = self.shape + out_shape = target + padded_in = in_shape + while len(padded_in) < len(out_shape): + padded_in = (1,) + padded_in + axes = [i for i, (s, t) in enumerate(zip(padded_in, out_shape)) if s == 1 and t > 1] + for axis in axes: + grad_reduced = grad_reduced.sum(axis, True) + grad_reduced = grad_reduced.reshape(list(in_shape)) + self._add_grad(grad_reduced) + + out._backward = backward + + return out def permute(self, *dims: int) -> Tensor: - assert set(dims) == set(range(len(self.shape))), "invalid permutation {dims}" - return self._movement_op( - tuple(self.shape[dims[i]] for i in range(len(dims))), - dims, - op=MovementOps.PERMUTE, - ) + if set(dims) != set(range(len(self.shape))): + raise ValueError("invalid permutation") + storage = self._storage.permute(list(dims)) + out = Tensor._from_storage(storage, requires_grad=self._requires_grad, prev=(self,), op=MovementOps.PERMUTE, device=self._device) + + if self._requires_grad: + inv = [0] * len(dims) + for i, dim in enumerate(dims): + inv[dim] = i + + def backward(): + grad = out._grad_storage + if grad is None: + return + self._add_grad(_contiguous(grad).permute(inv)) + + out._backward = backward + + return out def flatten(self, start_dim: int = 0, end_dim: int = -1) -> Tensor: if end_dim == -1: end_dim = len(self.shape) - assert 0 <= start_dim < end_dim <= len(self.shape), "invalid start_dim or end_dim" - return self.reshape(*(self.shape[:start_dim] + (prod(self.shape[start_dim:end_dim]),) + self.shape[end_dim:])) + if not (0 <= start_dim < end_dim <= len(self.shape)): + raise ValueError("invalid start_dim or end_dim") + front = self.shape[:start_dim] + middle = self.shape[start_dim:end_dim] + back = self.shape[end_dim:] + new_shape = front + (math.prod(middle),) + back + return self.reshape(*new_shape) def transpose(self, dim1: int, dim2: int) -> Tensor: dims = list(range(len(self.shape))) @@ -367,73 +595,76 @@ def transpose(self, dim1: int, dim2: int) -> Tensor: def view(self, *shape: int) -> Tensor: return self.reshape(*shape) + def _broadcasted(self, other: Tensor) -> Tuple[Tensor, Tensor]: + shape1, shape2 = self.shape, other.shape + while len(shape1) < len(shape2): + shape1 = (1,) + shape1 + while len(shape2) < len(shape1): + shape2 = (1,) + shape2 + target = tuple(max(s1, s2) for s1, s2 in zip(shape1, shape2)) + left = self if self.shape == target else self.expand(*target) + right = other if other.shape == target else other.expand(*target) + return left, right + # ******************************************************** # *************** processing ops *************** # ******************************************************** def matmul_2d(self, other: Tensor) -> Tensor: - assert len(self.shape) == 2 and len(other.shape) == 2, "matmul_2d only supports 2D tensors, but got shapes {self.shape} and {other.shape}" - assert self.shape[1] == other.shape[0], f"matmul_2d shape mismatch: {self.shape} and {other.shape}" - N, M, K = self.shape[0], self.shape[1], other.shape[1] - return (self.reshape(N, 1, M) * other.permute(1, 0).reshape(1, K, M)).sum(dim=2) + if len(self.shape) != 2 or len(other.shape) != 2: + raise ValueError("matmul_2d only supports 2D tensors") + if self.shape[1] != other.shape[0]: + raise ValueError("matmul_2d shape mismatch") + n, m, k = self.shape[0], self.shape[1], other.shape[1] + return (self.reshape(n, 1, m) * other.permute(1, 0).reshape(1, k, m)).sum(dim=2) def matmul(self, other: Tensor) -> Tensor: - # https://pytorch.org/docs/stable/generated/torch.matmul.html - # if both tensors are 1-dimensional, the dot product (scalar) is returned if len(self.shape) == 1 and len(other.shape) == 1: - assert self.shape[0] == other.shape[0], f"matmul shape mismatch: {self.shape} and {other.shape}" + if self.shape[0] != other.shape[0]: + raise ValueError("matmul shape mismatch") return self.mul(other).sum() - # if both arguments are 2-dimensional, the matrix-matrix product is returned - elif len(self.shape) == 2 and len(other.shape) == 2: + if len(self.shape) == 2 and len(other.shape) == 2: return self.matmul_2d(other) - # if the first argument is 1-dimensional and the second argument is 2-dimensional, - # a 1 is prepended to its dimension for the purpose of the matrix multiply - # after the matrix multiply, the prepended dimension is removed - elif len(self.shape) == 1 and len(other.shape) == 2: + if len(self.shape) == 1 and len(other.shape) == 2: return self.reshape(1, *self.shape).matmul_2d(other) - # if the first argument is 2-dimensional and the second argument is 1-dimensional, the matrix-vector product is returned - elif len(self.shape) == 2 and len(other.shape) == 1: - return self.matmul_2d(other.reshape(*other.shape, 1)).reshape(-1) - # if both arguments are at least 1-dimensional and at least one argument is N-dimensional (where N>2), then a batched matrix multiply is returned - # if the first argument is 1-dimensional, a 1 is prepended to its dimension for the purpose of the batched matrix multiply and removed after - # if the second argument is 1-dimensional, a 1 is appended to its dimension for the purpose of the batched matrix multiple and removed after - # the non-matrix (i.e. batch) dimensions are broadcasted (and thus must be broadcastable) - elif len(self.shape) >= 1 and len(other.shape) >= 1 and (len(self.shape) > 2 or len(other.shape) > 2): - raise NotImplementedError("batched matrix multiply is not implemented yet: {self.shape} and {other.shape}") - else: - raise RuntimeError(f"Invalid matmul shapes {self.shape} and {other.shape}") + if len(self.shape) == 2 and len(other.shape) == 1: + return self.matmul_2d(other.reshape(other.shape[0], 1)).reshape(self.shape[0]) + raise NotImplementedError("batched matmul not implemented") - def dot(self, other: Tensor) -> Tensor: return self.matmul(other) - def __matmul__(self, other: Tensor) -> Tensor: return self.matmul(other) + def dot(self, other: Tensor) -> Tensor: + return self.matmul(other) - # ******************************************************** - # *************** functional nn ops *************** - # ******************************************************** + def __matmul__(self, other: Tensor) -> Tensor: + return self.matmul(other) def linear(self, weight: Tensor, bias: Optional[Tensor] = None) -> Tensor: x = self.matmul(weight) return x.add(bias) if bias is not None else x def sparse_categorical_crossentropy(self, Y: Tensor) -> Tensor: - assert ( - len(self.shape) == 2 and len(Y.shape) == 1 - ), f"sparse_categorical_crossentropy only supports 2D tensor and 1D tensor, but got shapes {self.shape} and {Y.shape}" - assert self.shape[0] == Y.shape[0], f"sparse_categorical_crossentropy shape mismatch: {self.shape} and {Y.shape}" - - Y_pred = self.log_softmax() - # TODO: need more efficient implementation. currently, it's not possible to use Y as a tensor of indices - Y_onehot_data = np.zeros_like(Y_pred._data) - Y_onehot_data[np.arange(prod(Y._data.shape)), (Y._data + 1e-5).astype(np.int32)] = 1 - Y_onehot = Tensor(Y_onehot_data) - return -(Y_onehot * Y_pred).sum() / prod(Y._data.shape) # reduction="mean" + if not (len(self.shape) == 2 and len(Y.shape) == 1): + raise ValueError("sparse_categorical_crossentropy requires 2D logits and 1D labels") + if self.shape[0] != Y.shape[0]: + raise ValueError("shape mismatch between predictions and labels") + y_pred = self.log_softmax() + num_classes = self.shape[1] + onehot = [[0.0] * num_classes for _ in range(Y.shape[0])] + labels = Y.numpy().astype(np.int32) + for i, label in enumerate(labels): + onehot[i][int(label)] = 1.0 + y_onehot = Tensor(onehot) + return -(y_onehot * y_pred).sum() / Y.shape[0] # ******************************************************** # *************** random *************** # ******************************************************** - _seed: int = int(time.time()) + _seed: int = 1337 + @staticmethod - def manual_seed(seed: int = 0): Tensor._seed = seed + def manual_seed(seed: int = 0): + Tensor._seed = seed + @staticmethod def _nxt_seed() -> int: Tensor._seed = (Tensor._seed * 1103515245 + 12345) & 0x7FFFFFFF @@ -441,33 +672,21 @@ def _nxt_seed() -> int: @staticmethod def randn(shape: Tuple[int, ...], requires_grad: bool = False) -> Tensor: - np.random.seed(Tensor._nxt_seed()) - return Tensor( - data=np.random.randn(*shape), - shape=shape, - requires_grad=requires_grad, - prev=None, - op=None, - ) + storage = StorageView.randn(list(shape), "cpu", seed=Tensor._nxt_seed()) + return Tensor._from_storage(storage, requires_grad=requires_grad, prev=None, op=None, device="cpu") @staticmethod def uniform(shape: Union[Tuple[int, ...], int], low=0.0, high=1.0) -> Tensor: if isinstance(shape, int): shape = (shape,) - np.random.seed(Tensor._nxt_seed()) - return Tensor( - data=np.random.uniform(low, high, prod(shape)).reshape(shape), - shape=shape, - requires_grad=True, - prev=None, - op=None, - ) + storage = StorageView.uniform(list(shape), float(low), float(high), "cpu", seed=Tensor._nxt_seed()) + return Tensor._from_storage(storage, requires_grad=True, prev=None, op=None, device="cpu") @staticmethod def kaiming_uniform(shape: Union[Tuple[int, ...], int], a: float = 0.01) -> Tensor: if isinstance(shape, int): shape = (shape,) - bound = math.sqrt(3.0) * math.sqrt(2.0 / (1 + a**2)) / math.sqrt(prod(shape[1:])) + bound = math.sqrt(3.0) * math.sqrt(2.0 / (1 + a**2)) / math.sqrt(math.prod(shape[1:]) if len(shape) > 1 else 1.0) return Tensor.uniform(shape, low=-bound, high=bound) # ******************************************************** @@ -475,55 +694,83 @@ def kaiming_uniform(shape: Union[Tuple[int, ...], int], a: float = 0.01) -> Tens # ******************************************************** @staticmethod - def _dummy(shape: Shape, requires_grad: bool, prev: Optional[Tuple[Tensor, ...]], op: Op) -> Tensor: - return Tensor( - data=np.zeros(shape.dims), - shape=shape, - requires_grad=requires_grad, - prev=prev, - op=op, - ) + def _dummy(shape: Tuple[int, ...], requires_grad: bool, prev: Optional[Tuple[Tensor, ...]], op: Op) -> Tensor: + storage = StorageView.zeros(list(shape), "cpu") + return Tensor._from_storage(storage, requires_grad=requires_grad, prev=prev, op=op, device="cpu") @staticmethod - def zeros(shape: Tuple[int], requires_grad: bool = False) -> Tensor: - return Tensor( - data=np.zeros(shape), - shape=shape, - requires_grad=requires_grad, - prev=None, - op=None, - ) + def zeros(shape: Tuple[int, ...], requires_grad: bool = False) -> Tensor: + storage = StorageView.zeros(list(shape), "cpu") + return Tensor._from_storage(storage, requires_grad=requires_grad, prev=None, op=None, device="cpu") @staticmethod - def ones(shape: Tuple[int], requires_grad: bool = False) -> Tensor: - return Tensor( - data=np.ones(shape), - shape=shape, - requires_grad=requires_grad, - prev=None, - op=None, - ) - - def detach(self) -> Tensor: return Tensor(data=self._data, shape=self._shape, requires_grad=False, prev=None, op=None) - def numpy(self) -> np.ndarray: return self._data + def ones(shape: Tuple[int, ...], requires_grad: bool = False) -> Tensor: + storage = StorageView.ones(list(shape), "cpu") + return Tensor._from_storage(storage, requires_grad=requires_grad, prev=None, op=None, device="cpu") + + def detach(self) -> Tensor: + return Tensor._from_storage(self._storage, requires_grad=False, prev=None, op=None, device=self._device) + + def numpy(self) -> np.ndarray: + contig = _contiguous(self._storage) + arr = np.array(contig.to_vec(), dtype=np.float32) + return arr.reshape(self.shape) if self.shape else arr.reshape(()) + @property - def data(self) -> np.ndarray: return self._data + def data(self) -> np.ndarray: + return self.numpy() + + @property + def grad(self) -> np.ndarray: + grad_storage = self._ensure_grad() + contig = _contiguous(grad_storage) + arr = np.array(contig.to_vec(), dtype=np.float32) + return arr.reshape(self.shape) if self.shape else arr.reshape(()) + @property - def grad(self) -> np.ndarray: return self._grad + def shape(self) -> Tuple[int, ...]: + return self._shape + @property - def shape(self) -> Tuple: return self._shape.dims + def requires_grad(self) -> bool: + return self._requires_grad + @property - def requires_grad(self) -> bool: return self._requires_grad + def device(self) -> str: + return self._device + + def zero_grad(self): + if self._requires_grad: + self._grad_storage = StorageView.zeros(self._shape_list(), self._device) + else: + self._grad_storage = None + + def grad_storage(self) -> Optional[StorageView]: + return self._grad_storage + + def data_storage(self) -> StorageView: + return self._storage + + def set_data_storage(self, storage: StorageView): + self._storage = storage + + def num_elements(self) -> int: + return self._numel() + @property - def op(self): return self._op + def op(self): + return self._op - def size(self, dim: Optional[int] = None): return self.shape if dim is None else self.shape[dim] + def size(self, dim: Optional[int] = None): + return self.shape if dim is None else self.shape[dim] def item(self) -> float: - assert self.shape == (), f"item() only supports tensors with a single element, but got shape {self.shape}" - return self._data.item() + if self.shape != (): + raise ValueError("item() only supports tensors with a single element") + return float(self.numpy().item()) - def __hash__(self): return id(self) + def __hash__(self): + return id(self) def __repr__(self): out = f"Tensor({self.numpy().round(4) if self.shape != () else self.item()}" diff --git a/cranberry/view.py b/cranberry/view.py index f71d447..3732018 100644 --- a/cranberry/view.py +++ b/cranberry/view.py @@ -35,7 +35,8 @@ def create( return View(shape, stride, offset, contiguous) def reshape(self, shape: Tuple[int, ...]) -> Optional[View]: - if self.shape == shape: return self + if self.shape == shape: + return self assert all(x >= 0 for x in shape), f"{shape=} can't contain negative numbers" assert prod(self.shape) == prod(shape), f"size mismatched, can't reshape {self.shape=} -> {shape=}" @@ -86,7 +87,8 @@ def expand(self, sizes: Tuple[int, ...]) -> View: for i in range(len(sizes)): if i < len(self.shape): - if sizes[i] == self.shape[i]: continue + if sizes[i] == self.shape[i]: + continue if sizes[i] != self.shape[i] and self.shape[i] == 1: n_stride[i] = 0 # means this dimension is "broadcasted" else: @@ -100,7 +102,8 @@ def expand(self, sizes: Tuple[int, ...]) -> View: ) def permute(self, dims: Tuple[int, ...]) -> View: - if dims == tuple(range(len(self.shape))): return self + if dims == tuple(range(len(self.shape))): + return self if not self.contiguous: raise NotImplementedError("permuting non-contiguous views is not supported yet") diff --git a/examples/mnist.py b/examples/mnist.py index 9175f96..8b7b0db 100644 --- a/examples/mnist.py +++ b/examples/mnist.py @@ -12,8 +12,10 @@ X_test_np, Y_test_np = X_test.numpy(), Y_test.numpy() model = nn.Sequential( - nn.Linear(784, 128), nn.ReLU(), - nn.Linear(128, 64), nn.ReLU(), + nn.Linear(784, 128), + nn.ReLU(), + nn.Linear(128, 64), + nn.ReLU(), nn.Linear(64, 10), ) @@ -43,7 +45,8 @@ running_loss += loss.item() if (step + 1) % 50 == 0 or step + 1 == steps_per_epoch: avg = running_loss / (step + 1) - print(f"epoch {epoch+1}/{epochs} step {step+1}/{steps_per_epoch} - loss {avg:.4f}") + print(f"epoch {epoch + 1}/{epochs} step {step + 1}/{steps_per_epoch} - loss {avg:.4f}") + # Simple test accuracy def accuracy(X_np: np.ndarray, Y_np: np.ndarray) -> float: @@ -56,5 +59,6 @@ def accuracy(X_np: np.ndarray, Y_np: np.ndarray) -> float: total += e - s return correct / total + test_acc = accuracy(X_test_np, Y_test_np) -print(f"test accuracy: {test_acc*100:.2f}%") +print(f"test accuracy: {test_acc * 100:.2f}%") diff --git a/tests/test_storageview.py b/tests/test_storageview.py index 46541c4..bed09cc 100644 --- a/tests/test_storageview.py +++ b/tests/test_storageview.py @@ -14,340 +14,340 @@ class TestStorageView(unittest.TestCase): - def test_from_vec_len_to_vec(self): - v = StorageView.from_vec([1.0, 2.0, 3.5], "cpu") - self.assertEqual(v.len(), 3) - self.assertEqual(v.to_vec(), [1.0, 2.0, 3.5]) + def test_from_vec_len_to_vec(self): + v = StorageView.from_vec([1.0, 2.0, 3.5], "cpu") + self.assertEqual(v.len(), 3) + self.assertEqual(v.to_vec(), [1.0, 2.0, 3.5]) - def test_full(self): - v = StorageView.full(2.5, 4, "cpu") - self.assertEqual(v.len(), 4) - self.assertEqual(v.to_vec(), [2.5, 2.5, 2.5, 2.5]) + def test_full(self): + v = StorageView.full(2.5, 4, "cpu") + self.assertEqual(v.len(), 4) + self.assertEqual(v.to_vec(), [2.5, 2.5, 2.5, 2.5]) - # unary ops: neg, sqrt, exp, log + # unary ops: neg, sqrt, exp, log - def test_neg_1d(self): - def test_cranberry(): - a = StorageView.from_vec(A_np.tolist(), "cpu") - out = a.neg() - return np.array(out.to_vec(), dtype=np.float32) + def test_neg_1d(self): + def test_cranberry(): + a = StorageView.from_vec(A_np.tolist(), "cpu") + out = a.neg() + return np.array(out.to_vec(), dtype=np.float32) - def test_numpy(): - return -A_np + def test_numpy(): + return -A_np - np.testing.assert_allclose(test_cranberry(), test_numpy(), rtol, atol) + np.testing.assert_allclose(test_cranberry(), test_numpy(), rtol, atol) - def test_sqrt_1d(self): - # ensure positivity - data = (np.abs(A_np) + 1e-3).astype(np.float32) + def test_sqrt_1d(self): + # ensure positivity + data = (np.abs(A_np) + 1e-3).astype(np.float32) - def test_cranberry(): - a = StorageView.from_vec(data.tolist(), "cpu") - out = a.sqrt() - return np.array(out.to_vec(), dtype=np.float32) + def test_cranberry(): + a = StorageView.from_vec(data.tolist(), "cpu") + out = a.sqrt() + return np.array(out.to_vec(), dtype=np.float32) - def test_numpy(): - return np.sqrt(data) + def test_numpy(): + return np.sqrt(data) - np.testing.assert_allclose(test_cranberry(), test_numpy(), rtol, atol) + np.testing.assert_allclose(test_cranberry(), test_numpy(), rtol, atol) - def test_exp_1d(self): - # clamp to avoid inf in exp - data = np.clip(A_np, -10, 10) + def test_exp_1d(self): + # clamp to avoid inf in exp + data = np.clip(A_np, -10, 10) - def test_cranberry(): - a = StorageView.from_vec(data.tolist(), "cpu") - out = a.exp() - return np.array(out.to_vec(), dtype=np.float32) + def test_cranberry(): + a = StorageView.from_vec(data.tolist(), "cpu") + out = a.exp() + return np.array(out.to_vec(), dtype=np.float32) - def test_numpy(): - return np.exp(data) + def test_numpy(): + return np.exp(data) - np.testing.assert_allclose(test_cranberry(), test_numpy(), rtol, atol) + np.testing.assert_allclose(test_cranberry(), test_numpy(), rtol, atol) - def test_log_1d(self): - data = (np.abs(A_np) + 1e-3).astype(np.float32) + def test_log_1d(self): + data = (np.abs(A_np) + 1e-3).astype(np.float32) - def test_cranberry(): - a = StorageView.from_vec(data.tolist(), "cpu") - out = a.log() - return np.array(out.to_vec(), dtype=np.float32) + def test_cranberry(): + a = StorageView.from_vec(data.tolist(), "cpu") + out = a.log() + return np.array(out.to_vec(), dtype=np.float32) - def test_numpy(): - return np.log(data) + def test_numpy(): + return np.log(data) - np.testing.assert_allclose(test_cranberry(), test_numpy(), rtol, atol) + np.testing.assert_allclose(test_cranberry(), test_numpy(), rtol, atol) - # binary ops: add, sub, mul, div (1D and 2D contiguous) + # binary ops: add, sub, mul, div (1D and 2D contiguous) - def test_add_1d(self): - def test_cranberry(): - a = StorageView.from_vec(A_np.tolist(), "cpu") - b = StorageView.from_vec(B_np.tolist(), "cpu") - out = a.add(b) - return np.array(out.to_vec(), dtype=np.float32) - - def test_numpy(): - return A_np + B_np - - np.testing.assert_allclose(test_cranberry(), test_numpy(), rtol, atol) - - def test_sub_1d(self): - def test_cranberry(): - a = StorageView.from_vec(A_np.tolist(), "cpu") - b = StorageView.from_vec(B_np.tolist(), "cpu") - out = a.sub(b) - return np.array(out.to_vec(), dtype=np.float32) - - def test_numpy(): - return A_np - B_np - - np.testing.assert_allclose(test_cranberry(), test_numpy(), rtol, atol) - - def test_mul_1d(self): - def test_cranberry(): - a = StorageView.from_vec(A_np.tolist(), "cpu") - b = StorageView.from_vec(B_np.tolist(), "cpu") - out = a.mul(b) - return np.array(out.to_vec(), dtype=np.float32) - - def test_numpy(): - return A_np * B_np - - np.testing.assert_allclose(test_cranberry(), test_numpy(), rtol, atol) - - def test_div_1d(self): - bnp = B_np.copy() - bnp[bnp == 0] = 1.0 - - def test_cranberry(): - a = StorageView.from_vec(A_np.tolist(), "cpu") - b = StorageView.from_vec(bnp.tolist(), "cpu") - out = a.div(b) - return np.array(out.to_vec(), dtype=np.float32) - - def test_numpy(): - return A_np / bnp - - np.testing.assert_allclose(test_cranberry(), test_numpy(), rtol, atol) - - def test_unary_2d_then_flatten_compare(self): - data = np.arange(N * M, dtype=np.float32) - - def test_cranberry(): - a = StorageView.from_vec(data.tolist(), "cpu").reshape([N, M]) - out = a.neg().exp().log() # identity-ish - return np.array(out.to_vec(), dtype=np.float32) - - def test_numpy(): - with np.errstate(divide="ignore", invalid="ignore"): - out = np.log(np.exp(-data.reshape(N, M))).reshape(-1) - return out - - np.testing.assert_allclose(test_cranberry(), test_numpy(), rtol, atol) - - def test_unary_random_shapes(self): - rng = np.random.default_rng(1337) - shapes = [(N * M,), (10, 10), (3, 4, 5)] - - for shape in shapes: - data = rng.standard_normal(np.prod(shape)).astype(np.float32) - # prepare variants for stability per op - pos = (np.abs(data) + 1e-3).astype(np.float32) # for sqrt/log - clip = np.clip(data, -10, 10) # for exp - - # neg - a = StorageView.from_vec(data.tolist(), "cpu") - if len(shape) > 1: - a = a.reshape(list(shape)) - out = np.array(a.neg().to_vec(), dtype=np.float32) - np.testing.assert_allclose(out, -data.reshape(-1), rtol, atol) - - # sqrt - a = StorageView.from_vec(pos.tolist(), "cpu") - if len(shape) > 1: - a = a.reshape(list(shape)) - out = np.array(a.sqrt().to_vec(), dtype=np.float32) - np.testing.assert_allclose(out, np.sqrt(pos).reshape(-1), rtol, atol) - - # exp - a = StorageView.from_vec(clip.tolist(), "cpu") - if len(shape) > 1: - a = a.reshape(list(shape)) - out = np.array(a.exp().to_vec(), dtype=np.float32) - np.testing.assert_allclose(out, np.exp(clip).reshape(-1), rtol, atol) - - # log - a = StorageView.from_vec(pos.tolist(), "cpu") - if len(shape) > 1: - a = a.reshape(list(shape)) - out = np.array(a.log().to_vec(), dtype=np.float32) - np.testing.assert_allclose(out, np.log(pos).reshape(-1), rtol, atol) - - def test_binary_random_shapes(self): - rng = np.random.default_rng(2024) - shapes = [(N * M,), (17, 11), (2, 3, 5)] - - for shape in shapes: - a_np = rng.standard_normal(np.prod(shape)).astype(np.float32) - b_np = rng.standard_normal(np.prod(shape)).astype(np.float32) - b_np_div = b_np.copy() - b_np_div[np.isclose(b_np_div, 0.0)] = 1.0 - - def make_sv(arr): - sv = StorageView.from_vec(arr.reshape(-1).tolist(), "cpu") - return sv.reshape(list(shape)) if len(shape) > 1 else sv - - # add - a = make_sv(a_np) - b = make_sv(b_np) - out = np.array(a.add(b).to_vec(), dtype=np.float32) - np.testing.assert_allclose(out, (a_np + b_np).reshape(-1), rtol, atol) - - # sub - a = make_sv(a_np) - b = make_sv(b_np) - out = np.array(a.sub(b).to_vec(), dtype=np.float32) - np.testing.assert_allclose(out, (a_np - b_np).reshape(-1), rtol, atol) - - # mul - a = make_sv(a_np) - b = make_sv(b_np) - out = np.array(a.mul(b).to_vec(), dtype=np.float32) - np.testing.assert_allclose(out, (a_np * b_np).reshape(-1), rtol, atol) - - # div - a = make_sv(a_np) - b = make_sv(b_np_div) - out = np.array(a.div(b).to_vec(), dtype=np.float32) - np.testing.assert_allclose(out, (a_np / b_np_div).reshape(-1), rtol, atol) - - def test_slice_then_unary(self): - base = np.linspace(-3, 3, 101, dtype=np.float32) - v = StorageView.from_vec(base.tolist(), "cpu") - s = v.slice(5, 77) - out = np.array(s.exp().to_vec(), dtype=np.float32) - np.testing.assert_allclose(out, np.exp(base[5:82]), rtol, atol) - - def test_slice_then_binary(self): - a_base = np.linspace(-2, 2, 111, dtype=np.float32) - b_base = np.linspace(3, -3, 111, dtype=np.float32) - a = StorageView.from_vec(a_base.tolist(), "cpu").slice(7, 55) - b = StorageView.from_vec(b_base.tolist(), "cpu").slice(9, 55) - out = np.array(a.mul(b).to_vec(), dtype=np.float32) - np.testing.assert_allclose(out, (a_base[7:62] * b_base[9:64]), rtol, atol) - - def test_numpy_input(self): - arr = np.array([1.0, 2.0, 3.0], dtype=np.float32) - v = StorageView.from_vec(arr, "cpu") - self.assertEqual(v.to_vec(), [1.0, 2.0, 3.0]) - - def test_unary_simd_remainder(self): - # length intentionally not divisible by SIMD width (64) - data = np.arange(65, dtype=np.float32) - v = StorageView.from_vec(data.tolist(), "cpu") - out = np.array(v.neg().to_vec(), dtype=np.float32) - np.testing.assert_allclose(out, -data, rtol, atol) - - def test_binary_simd_remainder(self): - a = np.arange(129, dtype=np.float32) - b = np.arange(129, dtype=np.float32) * -0.25 - va = StorageView.from_vec(a.tolist(), "cpu") - vb = StorageView.from_vec(b.tolist(), "cpu") - out = np.array(va.add(vb).to_vec(), dtype=np.float32) - np.testing.assert_allclose(out, a + b, rtol, atol) - - def test_binary_2d(self): - a_np = A_np.reshape(N, M) - b_np = B_np.reshape(N, M) - - def test_cranberry(): - a = StorageView.from_vec(a_np.reshape(-1).tolist(), "cpu").reshape([N, M]) - b = StorageView.from_vec(b_np.reshape(-1).tolist(), "cpu").reshape([N, M]) - out = a.mul(b).add(a).sub(b).div(a) - return np.array(out.to_vec(), dtype=np.float32) - - def test_numpy(): - out = (a_np * b_np + a_np - b_np) / a_np - return out.reshape(-1) - - np.testing.assert_allclose(test_cranberry(), test_numpy(), rtol, atol) - - # movement semantics and error paths - - def test_slice_1d(self): - base = np.arange(100, dtype=np.float32) - off, size = 7, 23 - - def test_cranberry(): - v = StorageView.from_vec(base.tolist(), "cpu") - s = v.slice(off, size) - return np.array(s.to_vec(), dtype=np.float32) - - def test_numpy(): - return base[off : off + size] - - np.testing.assert_allclose(test_cranberry(), test_numpy(), rtol, atol) - - def test_reshape_contiguous_to_vec(self): - v = StorageView.from_vec([1.0, 2.0, 3.0, 4.0], "cpu") - r = v.reshape([2, 2]) - self.assertEqual(r.to_vec(), [1.0, 2.0, 3.0, 4.0]) - - def test_expand_non_contiguous_to_vec_raises(self): - v = StorageView.from_vec([1.0, 2.0, 3.0], "cpu").reshape([1, 3]) - e = v.expand([2, 3]) - with self.assertRaises(ValueError): - _ = e.to_vec() - - def test_permute_non_contiguous_to_vec_raises(self): - v = StorageView.from_vec([1.0, 2.0, 3.0, 4.0], "cpu").reshape([2, 2]) - p = v.permute([1, 0]) - with self.assertRaises(ValueError): - _ = p.to_vec() - - def test_unary_on_non_contiguous_raises(self): - v = StorageView.from_vec([1.0, 4.0, 9.0, 16.0], "cpu").reshape([2, 2]).permute([1, 0]) - with self.assertRaises(RuntimeError): - _ = v.neg() - with self.assertRaises(RuntimeError): - _ = v.sqrt() - with self.assertRaises(RuntimeError): - _ = v.exp() - with self.assertRaises(RuntimeError): - _ = v.log() - - def test_binary_on_non_contiguous_raises(self): - a = StorageView.from_vec([1.0, 2.0, 3.0, 4.0], "cpu").reshape([2, 2]).permute([1, 0]) - b = StorageView.from_vec([1.0, 2.0, 3.0, 4.0], "cpu").reshape([2, 2]) - with self.assertRaises(RuntimeError): - _ = a.add(b) - with self.assertRaises(RuntimeError): - _ = a.sub(b) - with self.assertRaises(RuntimeError): - _ = a.mul(b) - with self.assertRaises(RuntimeError): - _ = a.div(b) - - def test_binary_shape_mismatch_raises(self): - a = StorageView.from_vec([1.0, 2.0, 3.0], "cpu") - b = StorageView.from_vec([4.0, 5.0], "cpu") - with self.assertRaises(RuntimeError): - _ = a.add(b) - - def test_add_device_mismatch_raises(self): - a = StorageView.from_vec([1.0, 2.0], "cpu") - b = StorageView.from_vec([1.0, 2.0], "metal") - with self.assertRaises(ValueError): - _ = a.add(b) - - def test_ops_not_implemented_other_device(self): - v = StorageView.from_vec([1.0, 2.0], "metal") - with self.assertRaises(NotImplementedError): - _ = v.neg() - with self.assertRaises(NotImplementedError): - _ = v.exp() + def test_add_1d(self): + def test_cranberry(): + a = StorageView.from_vec(A_np.tolist(), "cpu") + b = StorageView.from_vec(B_np.tolist(), "cpu") + out = a.add(b) + return np.array(out.to_vec(), dtype=np.float32) + + def test_numpy(): + return A_np + B_np + + np.testing.assert_allclose(test_cranberry(), test_numpy(), rtol, atol) + + def test_sub_1d(self): + def test_cranberry(): + a = StorageView.from_vec(A_np.tolist(), "cpu") + b = StorageView.from_vec(B_np.tolist(), "cpu") + out = a.sub(b) + return np.array(out.to_vec(), dtype=np.float32) + + def test_numpy(): + return A_np - B_np + + np.testing.assert_allclose(test_cranberry(), test_numpy(), rtol, atol) + + def test_mul_1d(self): + def test_cranberry(): + a = StorageView.from_vec(A_np.tolist(), "cpu") + b = StorageView.from_vec(B_np.tolist(), "cpu") + out = a.mul(b) + return np.array(out.to_vec(), dtype=np.float32) + + def test_numpy(): + return A_np * B_np + + np.testing.assert_allclose(test_cranberry(), test_numpy(), rtol, atol) + + def test_div_1d(self): + bnp = B_np.copy() + bnp[bnp == 0] = 1.0 + + def test_cranberry(): + a = StorageView.from_vec(A_np.tolist(), "cpu") + b = StorageView.from_vec(bnp.tolist(), "cpu") + out = a.div(b) + return np.array(out.to_vec(), dtype=np.float32) + + def test_numpy(): + return A_np / bnp + + np.testing.assert_allclose(test_cranberry(), test_numpy(), rtol, atol) + + def test_unary_2d_then_flatten_compare(self): + data = np.arange(N * M, dtype=np.float32) + + def test_cranberry(): + a = StorageView.from_vec(data.tolist(), "cpu").reshape([N, M]) + out = a.neg().exp().log() # identity-ish + return np.array(out.to_vec(), dtype=np.float32) + + def test_numpy(): + with np.errstate(divide="ignore", invalid="ignore"): + out = np.log(np.exp(-data.reshape(N, M))).reshape(-1) + return out + + np.testing.assert_allclose(test_cranberry(), test_numpy(), rtol, atol) + + def test_unary_random_shapes(self): + rng = np.random.default_rng(1337) + shapes = [(N * M,), (10, 10), (3, 4, 5)] + + for shape in shapes: + data = rng.standard_normal(np.prod(shape)).astype(np.float32) + # prepare variants for stability per op + pos = (np.abs(data) + 1e-3).astype(np.float32) # for sqrt/log + clip = np.clip(data, -10, 10) # for exp + + # neg + a = StorageView.from_vec(data.tolist(), "cpu") + if len(shape) > 1: + a = a.reshape(list(shape)) + out = np.array(a.neg().to_vec(), dtype=np.float32) + np.testing.assert_allclose(out, -data.reshape(-1), rtol, atol) + + # sqrt + a = StorageView.from_vec(pos.tolist(), "cpu") + if len(shape) > 1: + a = a.reshape(list(shape)) + out = np.array(a.sqrt().to_vec(), dtype=np.float32) + np.testing.assert_allclose(out, np.sqrt(pos).reshape(-1), rtol, atol) + + # exp + a = StorageView.from_vec(clip.tolist(), "cpu") + if len(shape) > 1: + a = a.reshape(list(shape)) + out = np.array(a.exp().to_vec(), dtype=np.float32) + np.testing.assert_allclose(out, np.exp(clip).reshape(-1), rtol, atol) + + # log + a = StorageView.from_vec(pos.tolist(), "cpu") + if len(shape) > 1: + a = a.reshape(list(shape)) + out = np.array(a.log().to_vec(), dtype=np.float32) + np.testing.assert_allclose(out, np.log(pos).reshape(-1), rtol, atol) + + def test_binary_random_shapes(self): + rng = np.random.default_rng(2024) + shapes = [(N * M,), (17, 11), (2, 3, 5)] + + for shape in shapes: + a_np = rng.standard_normal(np.prod(shape)).astype(np.float32) + b_np = rng.standard_normal(np.prod(shape)).astype(np.float32) + b_np_div = b_np.copy() + b_np_div[np.isclose(b_np_div, 0.0)] = 1.0 + + def make_sv(arr): + sv = StorageView.from_vec(arr.reshape(-1).tolist(), "cpu") + return sv.reshape(list(shape)) if len(shape) > 1 else sv + + # add + a = make_sv(a_np) + b = make_sv(b_np) + out = np.array(a.add(b).to_vec(), dtype=np.float32) + np.testing.assert_allclose(out, (a_np + b_np).reshape(-1), rtol, atol) + + # sub + a = make_sv(a_np) + b = make_sv(b_np) + out = np.array(a.sub(b).to_vec(), dtype=np.float32) + np.testing.assert_allclose(out, (a_np - b_np).reshape(-1), rtol, atol) + + # mul + a = make_sv(a_np) + b = make_sv(b_np) + out = np.array(a.mul(b).to_vec(), dtype=np.float32) + np.testing.assert_allclose(out, (a_np * b_np).reshape(-1), rtol, atol) + + # div + a = make_sv(a_np) + b = make_sv(b_np_div) + out = np.array(a.div(b).to_vec(), dtype=np.float32) + np.testing.assert_allclose(out, (a_np / b_np_div).reshape(-1), rtol, atol) + + def test_slice_then_unary(self): + base = np.linspace(-3, 3, 101, dtype=np.float32) + v = StorageView.from_vec(base.tolist(), "cpu") + s = v.slice(5, 77) + out = np.array(s.exp().to_vec(), dtype=np.float32) + np.testing.assert_allclose(out, np.exp(base[5:82]), rtol, atol) + + def test_slice_then_binary(self): + a_base = np.linspace(-2, 2, 111, dtype=np.float32) + b_base = np.linspace(3, -3, 111, dtype=np.float32) + a = StorageView.from_vec(a_base.tolist(), "cpu").slice(7, 55) + b = StorageView.from_vec(b_base.tolist(), "cpu").slice(9, 55) + out = np.array(a.mul(b).to_vec(), dtype=np.float32) + np.testing.assert_allclose(out, (a_base[7:62] * b_base[9:64]), rtol, atol) + + def test_numpy_input(self): + arr = np.array([1.0, 2.0, 3.0], dtype=np.float32) + v = StorageView.from_vec(arr, "cpu") + self.assertEqual(v.to_vec(), [1.0, 2.0, 3.0]) + + def test_unary_simd_remainder(self): + # length intentionally not divisible by SIMD width (64) + data = np.arange(65, dtype=np.float32) + v = StorageView.from_vec(data.tolist(), "cpu") + out = np.array(v.neg().to_vec(), dtype=np.float32) + np.testing.assert_allclose(out, -data, rtol, atol) + + def test_binary_simd_remainder(self): + a = np.arange(129, dtype=np.float32) + b = np.arange(129, dtype=np.float32) * -0.25 + va = StorageView.from_vec(a.tolist(), "cpu") + vb = StorageView.from_vec(b.tolist(), "cpu") + out = np.array(va.add(vb).to_vec(), dtype=np.float32) + np.testing.assert_allclose(out, a + b, rtol, atol) + + def test_binary_2d(self): + a_np = A_np.reshape(N, M) + b_np = B_np.reshape(N, M) + + def test_cranberry(): + a = StorageView.from_vec(a_np.reshape(-1).tolist(), "cpu").reshape([N, M]) + b = StorageView.from_vec(b_np.reshape(-1).tolist(), "cpu").reshape([N, M]) + out = a.mul(b).add(a).sub(b).div(a) + return np.array(out.to_vec(), dtype=np.float32) + + def test_numpy(): + out = (a_np * b_np + a_np - b_np) / a_np + return out.reshape(-1) + + np.testing.assert_allclose(test_cranberry(), test_numpy(), rtol, atol) + + # movement semantics and error paths + + def test_slice_1d(self): + base = np.arange(100, dtype=np.float32) + off, size = 7, 23 + + def test_cranberry(): + v = StorageView.from_vec(base.tolist(), "cpu") + s = v.slice(off, size) + return np.array(s.to_vec(), dtype=np.float32) + + def test_numpy(): + return base[off : off + size] + + np.testing.assert_allclose(test_cranberry(), test_numpy(), rtol, atol) + + def test_reshape_contiguous_to_vec(self): + v = StorageView.from_vec([1.0, 2.0, 3.0, 4.0], "cpu") + r = v.reshape([2, 2]) + self.assertEqual(r.to_vec(), [1.0, 2.0, 3.0, 4.0]) + + def test_expand_non_contiguous_to_vec_raises(self): + v = StorageView.from_vec([1.0, 2.0, 3.0], "cpu").reshape([1, 3]) + e = v.expand([2, 3]) + with self.assertRaises(ValueError): + _ = e.to_vec() + + def test_permute_non_contiguous_to_vec_raises(self): + v = StorageView.from_vec([1.0, 2.0, 3.0, 4.0], "cpu").reshape([2, 2]) + p = v.permute([1, 0]) + with self.assertRaises(ValueError): + _ = p.to_vec() + + def test_unary_on_non_contiguous_raises(self): + v = StorageView.from_vec([1.0, 4.0, 9.0, 16.0], "cpu").reshape([2, 2]).permute([1, 0]) + with self.assertRaises(RuntimeError): + _ = v.neg() + with self.assertRaises(RuntimeError): + _ = v.sqrt() + with self.assertRaises(RuntimeError): + _ = v.exp() + with self.assertRaises(RuntimeError): + _ = v.log() + + def test_binary_on_non_contiguous_raises(self): + a = StorageView.from_vec([1.0, 2.0, 3.0, 4.0], "cpu").reshape([2, 2]).permute([1, 0]) + b = StorageView.from_vec([1.0, 2.0, 3.0, 4.0], "cpu").reshape([2, 2]) + with self.assertRaises(RuntimeError): + _ = a.add(b) + with self.assertRaises(RuntimeError): + _ = a.sub(b) + with self.assertRaises(RuntimeError): + _ = a.mul(b) + with self.assertRaises(RuntimeError): + _ = a.div(b) + + def test_binary_shape_mismatch_raises(self): + a = StorageView.from_vec([1.0, 2.0, 3.0], "cpu") + b = StorageView.from_vec([4.0, 5.0], "cpu") + with self.assertRaises(RuntimeError): + _ = a.add(b) + + def test_add_device_mismatch_raises(self): + a = StorageView.from_vec([1.0, 2.0], "cpu") + b = StorageView.from_vec([1.0, 2.0], "metal") + with self.assertRaises(ValueError): + _ = a.add(b) + + def test_ops_not_implemented_other_device(self): + v = StorageView.from_vec([1.0, 2.0], "metal") + with self.assertRaises(NotImplementedError): + _ = v.neg() + with self.assertRaises(NotImplementedError): + _ = v.exp() if __name__ == "__main__": - unittest.main() + unittest.main() From 5309cdb873a7d657c61b4827c358bd49b4f27ee1 Mon Sep 17 00:00:00 2001 From: manoflearning <77jwk0724@gmail.com> Date: Wed, 17 Sep 2025 19:12:12 +0900 Subject: [PATCH 3/7] fix Rust test error --- src/py/mod.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/py/mod.rs b/src/py/mod.rs index 1c9a234..8550f94 100644 --- a/src/py/mod.rs +++ b/src/py/mod.rs @@ -26,6 +26,16 @@ mod tests { use super::*; use std::sync::Arc; + use crate::core::reduce::{ensure_contiguous, normalize_axis, numel, reduce_max, reduce_sum}; + + fn reduce_sum_view(view: &View, axis: Option, keepdim: bool) -> View { + reduce_sum(view, axis, keepdim) + } + + fn reduce_max_view(view: &View, axis: Option, keepdim: bool) -> View { + reduce_max(view, axis, keepdim) + } + fn make_view(shape: &[usize], data: Vec) -> View { let inner = Arc::new(StorageInner::from_vec(data, Device::Cpu)); View::from_inner_contiguous(inner, shape) @@ -194,7 +204,7 @@ impl StorageView { device: &str, seed: Option, ) -> PyResult { - if !(low < high) { + if low >= high { return Err(pyo3::exceptions::PyValueError::new_err( "uniform requires low < high", )); From 2a1ac31ab609d7f7edf9ff79f683703c3d3991a3 Mon Sep 17 00:00:00 2001 From: manoflearning <77jwk0724@gmail.com> Date: Wed, 17 Sep 2025 19:13:42 +0900 Subject: [PATCH 4/7] minor fix --- cranberry/ops.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cranberry/ops.py b/cranberry/ops.py index db311ac..c005e90 100644 --- a/cranberry/ops.py +++ b/cranberry/ops.py @@ -7,7 +7,7 @@ class UnaryOps(Enum): SQRT = auto() RELU = auto() EXP = auto() - LOG = auto() # noqa: E702 + LOG = auto() def __repr__(self): return f"{self.name.lower()}" @@ -17,7 +17,7 @@ class BinaryOps(Enum): ADD = auto() SUB = auto() MUL = auto() - DIV = auto() # noqa: E702 + DIV = auto() def __repr__(self): return f"{self.name.lower()}" @@ -25,7 +25,7 @@ def __repr__(self): class ReduceOps(Enum): SUM = auto() - MAX = auto() # noqa: E702 + MAX = auto() def __repr__(self): return f"{self.name.lower()}" @@ -34,7 +34,7 @@ def __repr__(self): class MovementOps(Enum): RESHAPE = auto() EXPAND = auto() - PERMUTE = auto() # noqa: E702 + PERMUTE = auto() def __repr__(self): return f"{self.name.lower()}" From 51881ef861d9ff2d35d34b66af1c4b245b96a8fc Mon Sep 17 00:00:00 2001 From: manoflearning <77jwk0724@gmail.com> Date: Wed, 17 Sep 2025 19:22:44 +0900 Subject: [PATCH 5/7] update examples --- examples/demo.ipynb | 206 ++++++++++++++++++++++---------------------- 1 file changed, 103 insertions(+), 103 deletions(-) diff --git a/examples/demo.ipynb b/examples/demo.ipynb index 535c2d9..fe9342b 100644 --- a/examples/demo.ipynb +++ b/examples/demo.ipynb @@ -41,7 +41,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 4, @@ -96,7 +96,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Tensor(1.0106998682022095, op=add) 0.5\n" + "Tensor(1.0274624824523926, op=add) 0.47\n" ] } ], @@ -134,106 +134,106 @@ "name": "stdout", "output_type": "stream", "text": [ - "step 0 loss 1.0107 accuracy 50.0%\n", - "step 1 loss 0.9412 accuracy 50.0%\n", - "step 2 loss 0.8435 accuracy 50.0%\n", - "step 3 loss 0.7663 accuracy 50.0%\n", - "step 4 loss 0.6874 accuracy 79.0%\n", - "step 5 loss 0.5799 accuracy 79.0%\n", - "step 6 loss 0.7027 accuracy 71.0%\n", - "step 7 loss 0.8839 accuracy 77.0%\n", - "step 8 loss 0.4291 accuracy 79.0%\n", - "step 9 loss 0.3768 accuracy 81.0%\n", - "step 10 loss 0.3398 accuracy 84.0%\n", - "step 11 loss 0.3166 accuracy 86.0%\n", - "step 12 loss 0.3030 accuracy 86.0%\n", - "step 13 loss 0.2927 accuracy 87.0%\n", - "step 14 loss 0.2797 accuracy 87.0%\n", - "step 15 loss 0.2741 accuracy 88.0%\n", - "step 16 loss 0.2760 accuracy 87.0%\n", - "step 17 loss 0.2790 accuracy 88.0%\n", - "step 18 loss 0.2653 accuracy 87.0%\n", - "step 19 loss 0.2601 accuracy 91.0%\n", - "step 20 loss 0.2567 accuracy 88.0%\n", - "step 21 loss 0.2550 accuracy 90.0%\n", - "step 22 loss 0.2594 accuracy 89.0%\n", - "step 23 loss 0.2664 accuracy 88.0%\n", - "step 24 loss 0.2464 accuracy 88.0%\n", - "step 25 loss 0.2421 accuracy 91.0%\n", - "step 26 loss 0.2408 accuracy 90.0%\n", - "step 27 loss 0.2364 accuracy 90.0%\n", - "step 28 loss 0.2399 accuracy 91.0%\n", - "step 29 loss 0.2477 accuracy 88.0%\n", - "step 30 loss 0.2321 accuracy 91.0%\n", - "step 31 loss 0.2328 accuracy 91.0%\n", - "step 32 loss 0.2227 accuracy 92.0%\n", - "step 33 loss 0.2158 accuracy 91.0%\n", - "step 34 loss 0.2172 accuracy 93.0%\n", - "step 35 loss 0.2232 accuracy 92.0%\n", - "step 36 loss 0.2090 accuracy 93.0%\n", - "step 37 loss 0.2066 accuracy 91.0%\n", - "step 38 loss 0.2006 accuracy 93.0%\n", - "step 39 loss 0.1987 accuracy 92.0%\n", - "step 40 loss 0.1947 accuracy 93.0%\n", - "step 41 loss 0.2032 accuracy 92.0%\n", - "step 42 loss 0.1811 accuracy 93.0%\n", - "step 43 loss 0.1779 accuracy 92.0%\n", - "step 44 loss 0.1759 accuracy 93.0%\n", - "step 45 loss 0.1881 accuracy 92.0%\n", - "step 46 loss 0.1602 accuracy 93.0%\n", - "step 47 loss 0.1597 accuracy 93.0%\n", - "step 48 loss 0.1572 accuracy 94.0%\n", - "step 49 loss 0.1745 accuracy 94.0%\n", - "step 50 loss 0.1392 accuracy 94.0%\n", - "step 51 loss 0.1484 accuracy 95.0%\n", - "step 52 loss 0.1338 accuracy 96.0%\n", - "step 53 loss 0.1515 accuracy 94.0%\n", - "step 54 loss 0.1135 accuracy 96.0%\n", - "step 55 loss 0.1292 accuracy 96.0%\n", - "step 56 loss 0.1192 accuracy 97.0%\n", - "step 57 loss 0.1974 accuracy 93.0%\n", - "step 58 loss 0.1121 accuracy 96.0%\n", - "step 59 loss 0.0784 accuracy 96.0%\n", - "step 60 loss 0.0734 accuracy 98.0%\n", - "step 61 loss 0.1141 accuracy 97.0%\n", - "step 62 loss 0.1714 accuracy 95.0%\n", - "step 63 loss 0.0861 accuracy 96.0%\n", - "step 64 loss 0.0582 accuracy 99.0%\n", - "step 65 loss 0.0522 accuracy 99.0%\n", - "step 66 loss 0.0720 accuracy 98.0%\n", - "step 67 loss 0.0606 accuracy 99.0%\n", - "step 68 loss 0.0935 accuracy 97.0%\n", - "step 69 loss 0.0478 accuracy 99.0%\n", - "step 70 loss 0.0371 accuracy 100.0%\n", - "step 71 loss 0.0360 accuracy 100.0%\n", - "step 72 loss 0.0368 accuracy 100.0%\n", - "step 73 loss 0.0621 accuracy 100.0%\n", - "step 74 loss 0.1439 accuracy 95.0%\n", - "step 75 loss 0.0583 accuracy 99.0%\n", - "step 76 loss 0.0389 accuracy 100.0%\n", - "step 77 loss 0.0318 accuracy 100.0%\n", - "step 78 loss 0.0278 accuracy 100.0%\n", - "step 79 loss 0.0266 accuracy 100.0%\n", - "step 80 loss 0.0263 accuracy 100.0%\n", - "step 81 loss 0.0220 accuracy 100.0%\n", - "step 82 loss 0.0210 accuracy 100.0%\n", - "step 83 loss 0.0202 accuracy 100.0%\n", - "step 84 loss 0.0189 accuracy 100.0%\n", - "step 85 loss 0.0201 accuracy 100.0%\n", - "step 86 loss 0.0175 accuracy 100.0%\n", - "step 87 loss 0.0190 accuracy 100.0%\n", - "step 88 loss 0.0163 accuracy 100.0%\n", - "step 89 loss 0.0183 accuracy 100.0%\n", - "step 90 loss 0.0287 accuracy 100.0%\n", - "step 91 loss 0.0183 accuracy 100.0%\n", - "step 92 loss 0.0161 accuracy 100.0%\n", - "step 93 loss 0.0148 accuracy 100.0%\n", - "step 94 loss 0.0144 accuracy 100.0%\n", - "step 95 loss 0.0166 accuracy 100.0%\n", - "step 96 loss 0.0137 accuracy 100.0%\n", - "step 97 loss 0.0132 accuracy 100.0%\n", - "step 98 loss 0.0131 accuracy 100.0%\n", - "step 99 loss 0.0135 accuracy 100.0%\n" + "step 0 loss 1.0275 accuracy 47.0%\n", + "step 1 loss 0.9822 accuracy 64.0%\n", + "step 2 loss 0.9547 accuracy 65.0%\n", + "step 3 loss 0.9150 accuracy 71.0%\n", + "step 4 loss 0.8393 accuracy 71.0%\n", + "step 5 loss 0.7278 accuracy 70.0%\n", + "step 6 loss 0.6400 accuracy 79.0%\n", + "step 7 loss 0.6066 accuracy 82.0%\n", + "step 8 loss 0.5284 accuracy 80.0%\n", + "step 9 loss 0.6220 accuracy 84.0%\n", + "step 10 loss 0.3714 accuracy 80.0%\n", + "step 11 loss 0.3343 accuracy 85.0%\n", + "step 12 loss 0.2893 accuracy 86.0%\n", + "step 13 loss 0.2804 accuracy 89.0%\n", + "step 14 loss 0.3013 accuracy 86.0%\n", + "step 15 loss 0.3691 accuracy 89.0%\n", + "step 16 loss 0.3944 accuracy 81.0%\n", + "step 17 loss 0.3160 accuracy 87.0%\n", + "step 18 loss 0.2278 accuracy 89.0%\n", + "step 19 loss 0.2113 accuracy 91.0%\n", + "step 20 loss 0.2004 accuracy 92.0%\n", + "step 21 loss 0.1935 accuracy 93.0%\n", + "step 22 loss 0.1880 accuracy 92.0%\n", + "step 23 loss 0.2160 accuracy 91.0%\n", + "step 24 loss 0.3084 accuracy 90.0%\n", + "step 25 loss 0.3075 accuracy 86.0%\n", + "step 26 loss 0.2319 accuracy 93.0%\n", + "step 27 loss 0.1753 accuracy 93.0%\n", + "step 28 loss 0.2142 accuracy 93.0%\n", + "step 29 loss 0.1971 accuracy 92.0%\n", + "step 30 loss 0.2414 accuracy 92.0%\n", + "step 31 loss 0.1964 accuracy 93.0%\n", + "step 32 loss 0.1888 accuracy 94.0%\n", + "step 33 loss 0.1307 accuracy 95.0%\n", + "step 34 loss 0.1366 accuracy 95.0%\n", + "step 35 loss 0.1282 accuracy 95.0%\n", + "step 36 loss 0.1683 accuracy 95.0%\n", + "step 37 loss 0.1618 accuracy 96.0%\n", + "step 38 loss 0.1710 accuracy 95.0%\n", + "step 39 loss 0.0761 accuracy 97.0%\n", + "step 40 loss 0.0875 accuracy 100.0%\n", + "step 41 loss 0.1280 accuracy 96.0%\n", + "step 42 loss 0.0624 accuracy 99.0%\n", + "step 43 loss 0.0499 accuracy 99.0%\n", + "step 44 loss 0.0961 accuracy 98.0%\n", + "step 45 loss 0.2148 accuracy 95.0%\n", + "step 46 loss 0.1859 accuracy 93.0%\n", + "step 47 loss 0.1219 accuracy 95.0%\n", + "step 48 loss 0.0603 accuracy 98.0%\n", + "step 49 loss 0.0463 accuracy 100.0%\n", + "step 50 loss 0.0495 accuracy 98.0%\n", + "step 51 loss 0.0347 accuracy 100.0%\n", + "step 52 loss 0.0566 accuracy 98.0%\n", + "step 53 loss 0.0243 accuracy 100.0%\n", + "step 54 loss 0.0279 accuracy 99.0%\n", + "step 55 loss 0.0157 accuracy 100.0%\n", + "step 56 loss 0.0246 accuracy 100.0%\n", + "step 57 loss 0.0242 accuracy 100.0%\n", + "step 58 loss 0.0501 accuracy 98.0%\n", + "step 59 loss 0.0187 accuracy 100.0%\n", + "step 60 loss 0.0119 accuracy 100.0%\n", + "step 61 loss 0.0102 accuracy 100.0%\n", + "step 62 loss 0.0095 accuracy 100.0%\n", + "step 63 loss 0.0089 accuracy 100.0%\n", + "step 64 loss 0.0090 accuracy 100.0%\n", + "step 65 loss 0.0221 accuracy 100.0%\n", + "step 66 loss 0.0404 accuracy 98.0%\n", + "step 67 loss 0.0135 accuracy 100.0%\n", + "step 68 loss 0.0099 accuracy 100.0%\n", + "step 69 loss 0.0089 accuracy 100.0%\n", + "step 70 loss 0.0082 accuracy 100.0%\n", + "step 71 loss 0.0073 accuracy 100.0%\n", + "step 72 loss 0.0074 accuracy 100.0%\n", + "step 73 loss 0.0071 accuracy 100.0%\n", + "step 74 loss 0.0069 accuracy 100.0%\n", + "step 75 loss 0.0068 accuracy 100.0%\n", + "step 76 loss 0.0068 accuracy 100.0%\n", + "step 77 loss 0.0066 accuracy 100.0%\n", + "step 78 loss 0.0065 accuracy 100.0%\n", + "step 79 loss 0.0064 accuracy 100.0%\n", + "step 80 loss 0.0064 accuracy 100.0%\n", + "step 81 loss 0.0069 accuracy 100.0%\n", + "step 82 loss 0.0064 accuracy 100.0%\n", + "step 83 loss 0.0063 accuracy 100.0%\n", + "step 84 loss 0.0062 accuracy 100.0%\n", + "step 85 loss 0.0062 accuracy 100.0%\n", + "step 86 loss 0.0061 accuracy 100.0%\n", + "step 87 loss 0.0060 accuracy 100.0%\n", + "step 88 loss 0.0061 accuracy 100.0%\n", + "step 89 loss 0.0061 accuracy 100.0%\n", + "step 90 loss 0.0060 accuracy 100.0%\n", + "step 91 loss 0.0059 accuracy 100.0%\n", + "step 92 loss 0.0059 accuracy 100.0%\n", + "step 93 loss 0.0058 accuracy 100.0%\n", + "step 94 loss 0.0058 accuracy 100.0%\n", + "step 95 loss 0.0057 accuracy 100.0%\n", + "step 96 loss 0.0058 accuracy 100.0%\n", + "step 97 loss 0.0058 accuracy 100.0%\n", + "step 98 loss 0.0057 accuracy 100.0%\n", + "step 99 loss 0.0057 accuracy 100.0%\n" ] } ], @@ -274,7 +274,7 @@ }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi8AAAGdCAYAAADaPpOnAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjYsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvq6yFwwAAAAlwSFlzAAAPYQAAD2EBqD+naQAAZdNJREFUeJzt3Xd4XOWdNv77nOkjaUa9F0u25F5wt2k2GEwJCSkEEhJKCEn4QcqafbOQNxtCyvrNJtmQEBJCCgQSUlhCT0wxLmAbGzeMm2wVW7KkUR+Nppdzfn+MJFuWpsmackb357rmAktnZh5pRufc85TvI8iyLIOIiIhIIcRkN4CIiIgoFgwvREREpCgML0RERKQoDC9ERESkKAwvREREpCgML0RERKQoDC9ERESkKAwvREREpCjqZDdgskmShPb2dmRlZUEQhGQ3h4iIiKIgyzIGBwdRWloKUQzft5J24aW9vR0VFRXJbgYRERFNQGtrK8rLy8Mek3bhJSsrCwDQ/PMvI8ugS3JriIiIYvfX6hkQckqT3YyEctmd+Pc1t41cx8NJu/AyPFSUZdDBZGR4ISIi5TFkGCBkZiS7GUkRzZQPTtglIiIiRWF4ISIiIkVheCEiIiJFYXghIiIiRWF4ISIiIkVheCEiIiJFYXghIiIiRWF4ISIiIkVheCEiIiJFYXghIiIiRWF4ISIiIkVheCEiIiJFYXghIiIiRWF4ISIiIkVheCEiIiJFYXghIiIiRWF4ISIiIkVheCEiIiJFYXghIiIiRWF4ISIiIkVheCEiIiJFYXghIiIiRWF4ISIiIkVheCEiIkohz9TUJbsJKY/hhYiIKEUMBxchtzzJLUltDC9EREQpgMElegwvREREScbgEhuGFyIioiRicIkdwwsREVGSMLhMDMMLERFREjC4TBzDCxERUYIxuFwYhhciIqIkYHCZOIYXIiKiBHqmpo7B5QIxvBARESUIg8vkYHghIiJKAAaXycPwQkREFGfcr2hyMbwQERHFEVcWTT6GFyIiojhhcIkPhhciIqI4YHCJH4YXIiKiScbgEl9xDS/bt2/HDTfcgNLSUgiCgBdffDHs8Vu3boUgCGNuFoslns0kIiKaNAwu8RfX8OJwOLBw4UI89thjMd2vvr4eHR0dI7fCwsI4tZCIiGjyMLgkhjqeD37ttdfi2muvjfl+hYWFyM7OnvwGERERxQmDS+Kk5JyXRYsWoaSkBFdddRV27NgR9liPxwObzTbqRkRElEgMLomVUuGlpKQEjz/+OJ5//nk8//zzqKiowJo1a7B///6Q99m4cSPMZvPIraKiIoEtJiKiqY7BJfEEWZblhDyRIOCFF17AjTfeGNP9Lr/8clRWVuKZZ54Z9/sejwcej2fk3zabDRUVFeh54mswGXUX0mQiIqKwGFwmj8vuwL1LP4WBgQGYTKawx8Z1zstkWL58Od59992Q39fpdNDpGFKIiCixGFySJ6WGjcZz8OBBlJSUJLsZREREYzC4JEdce17sdjsaGhpG/t3c3IyDBw8iNzcXlZWVePDBB9HW1oann34aAPDII4+guroac+fOhdvtxu9+9zu8/fbbeOONN+LZTCIiophwh+jkimt42bt3L9auXTvy7w0bNgAAbr/9djz11FPo6OhAS0vLyPe9Xi/uv/9+tLW1wWg0YsGCBXjrrbdGPQYREVEyMbgkX8Im7CaKzWaD2WzmhF0iIooLhpf4iGXCbsrPeSEiIiI6F8MLERERKQrDCxERESkKwwsREREpCsMLERERKQrDCxERESkKwwsREREpCsMLERERKQrDCxERESkKwwsREREpCsMLERERKQrDCxERESkKwwsREREpCsMLERERKQrDCxERESkKwwsREREpCsMLERERKQrDCxERESkKwwsREREpCsMLERERKQrDCxERESkKwwsREREpCsMLERERKQrDCxERESkKwwsREREpCsMLERFRlJ6pqUt2EwgML0RERFEZDi5CbnmSW0IML0RERBEwuKQWdbIbQESxkyUZHUcG0XV8EJIkQ6URkZGrgc6kQek8E1Rafi4hmiwMLqmH4YVIYaytLmz7eSMGOz0QBECWR39fY1Rh/keLMfu6IgiCkJxGEqUJBpfUxPBCpCCOXi/e+GE9fK4AgLHBBQB8zgD2/7UNXmcAi24qS3ALidIHg0vqYt8ykYIc+1cnfK4AZCnysYdfscDR441/o4jSEINLamN4IVIIWZbRuL0nquACAIIANG7viW+jiNIYg0vq4rARkQIEvBK6TgzC54oyuSA4pDTY5Yljq4jS0zM1dQwuKY7hhSiFSQEZH77UgeOvd8HnDMR2ZxnoPmGH3yNBrYu9k7WnyYETb3Whq94BQQCK52Zh5rpCZFcYYn4sIqVgcFEGhheiFCVLMt79VTNa9vRP+DHsPV7seLwZl399ekz3+/DFDnzwfPuorw12enDy7R4s+Ww5Zl9bNOE2EaUqBhflYHghSlFtHwxcUHABAMhA614r+k47kVtljOouLe/3jwku59r37BlkFulQsTj7wtoGwOPwo+mdXnSfdAAACusyUHNJHrQZPDVRYrHsv7LwDEGUok5s7oYgIuoJuqEIItC8sy/q8PLhix0Rj9nz5OkLDi9n9lvxzmPNCPjO/oAt7/fjwN/bcem91SifhHBEFA2uLFIerjYiSlHWFtcFB5dhHpsvuuMG/ehvcUU8zmX1w9E78cnAvU0ObPtFIwJeCZAx6hbwStj2i0b0Njsm/PhE0WJwUSb2vBClqKhL/AsIXvjDMGRronoovy/6tPT2TxrgcwSg0omoWp6D2isKkJGnjeq+R161hG+zHDzmsq/GNleHKBYMLsrFnheiFFWxJBtChL9QjVFEpB0AZAmoviQvquc0mDQRH2/YQJsbzn4fBi0eHHnVgpe/eRgdR2wR7xfwSWjdZw3bqyRLwbk6gRjCFFEsGFyUjeGFKEXVrSuAIIZPErOvLcac64tDHyAA1RfnIrssuuXNolpAdmV0c2PO7TmRJSDgk7H1fxrgtI4eonLbfDjxVjcOvdCOk1u64ezzRjUcJkuA383wQpOPwUX54hpetm/fjhtuuAGlpaUQBAEvvvhixPts3boVixcvhk6nw4wZM/DUU0/Fs4lEKSuzQIfLvj4doloY1QMz/P9VK3Mw76PFWPSpUsz9SHHw6wIgqs4eP/2yPKz8YlVMz7v89oqJNVgGJJ+Mhq3Bqr6SJGPfs614/muHsOfpFnz4Ugd2/6EFrzxwFKIqcveOWidCY1RNrC1EITC4pIe4znlxOBxYuHAhvvCFL+ATn/hExOObm5tx/fXX4ytf+Qr+/Oc/Y/PmzfjiF7+IkpISrF+/Pp5NJUpJ5YvM+Oh/z8XJt7vRstcKyScju8KAunUFKF1gGtk1+qKbyzBrfSFO7eqDs98HXZYa01bmILNAF/NzFtRmYt7HinH4JUvM95VloHWfFQtuLMHeZ1px4q3us98bqrEn+SNM0EEwoE2/LC+qkEMULQaX9BHX8HLttdfi2muvjfr4xx9/HNXV1fjpT38KAJg9ezbeffdd/OxnP2N4oSkrs0CHi24ux0U3hz/hGrI1k1Y8btGnypBTYcShF9ox0OYOfjGKicFAcLXQYJdnVHAJaZzHFERAa1SFHw4jihGDS3pJqdVGu3btwrp160Z9bf369fjGN74R8j4ejwcez9klmzZb5AmDRIng6PGiu8EOyED+jIwJ9YIkU9WKHFStyIGjxwufJwC/V8Km7xwPex9BBHIqDWh6tzdyjRohGLhc/b5giAEAGcguN+CS+2pGrVySZRm9zU70NjkgCAKK5mTBXKK/8B+SpgQGl/STUuHFYrGgqGj0J8eioiLYbDa4XC4YDGMnHW7cuBEPP/xwoppIFJHL6sPuJ0/jzIGBUb0KZYvMWPGFShhzoltOnCoy8s+2t3BmJrpP2kOGElkC6q4oQNOO3oiPK4oCpq3KQdWyXHSdtAcfvzYTedONI8NhAGBtdWHH481j6s+UzMvCiruqMHDGDbfNB0O2BsVzTRxqolEYXNJTSoWXiXjwwQexYcOGkX/bbDZUVExwwiHRBfLY/Xj9+8fh6PGOGQ5pPzSA179Xj2sfngW9Kbq6K6lm+e2V2PS94wh4pXEDzPTL81A4KxNtHwxEfCxZkqHP0iB/RgbyZ2SMe8xgpxuvf78efs/YTSk7jgzixQ2HR/2e9SY1Fn26DDMuz4/6Z6L0xeCSvlIqvBQXF6Ozs3PU1zo7O2EymcbtdQEAnU4HnU5Z3fGUvo79qxOOnvGXAssS4Ozz4uhrnVj8mdhPpgGvhNN7+nHqvT54HQFkFuowY00+imZljuqpiKfsCgOueWgW9v65FZbDgyNf12UG56jMua4IgiBg2qpcHH2tM8wjBSf3Vq3ICXvMoX90wO8JjN/TM878G7fNj/d+dxoBr4SZVxVG8yNRmmJwSW8pFV5WrVqFf/7zn6O+9uabb2LVqlVJahFR9GRJxonN3RGLr53c0oNFny6LaXjD3uXBm//vBBzd3pFJrr1NDpza2YeKJdm45N5qqDSJKduUXW7Auv+og73bA5vFA7VORF6NESr12efPrTKibJEZ7YcGxv99CEDNJXlh5wH5XAGc2t03oS0S9v/lDKovzoOWS62nNAaX9BXXs53dbsfBgwdx8OBBAMGl0AcPHkRLSwuA4JDPbbfdNnL8V77yFTQ1NeGb3/wmjh8/jl/96lf4+9//jn/7t3+LZzNJAWRZRuexQex5qgXv/qoJB/7WBluHO9nNGsXnluB1jB3eGHOcKwCvwx/14wb8Et760Qk4e73BLwz1OAxf1Fv3W7H3T62xNveCZRboUDrfhMK6zFHBZdgl91ajZJ4JACCoAEE4p0bN8hysuLMy7OO7rL6R5dWxCvhlnNrVN7E7k+I9U1PH4JLm4trzsnfvXqxdu3bk38NzU26//XY89dRT6OjoGAkyAFBdXY3XXnsN//Zv/4af//znKC8vx+9+9zsuk57iPIN+bPlZA3pOOoIrWOTghfDIqxbUrSvA0s9XQIxQiTaeJL8MiIBKI0S9nFgd7b5FAM7ss8Le5Q19gAw0bOvBwk+UQm9Onbk0Gr0Ka/99BnobnWje2Qu3zQ9DjgY1l+RFtcP1hRSoE0QBg50T3ziSlIvBZWqIa3hZs2YNZDn0mXy86rlr1qzBgQMH4tgqUhJZkvH2TxvQN7TD8HBvw/Db6sRb3dDoVbjo5rKEtksKyGjc1oPjb3SN1EHJn5EBg1kDlzX0Ds6CCBTPyYJaH/2FueV9KwTh7M88HjkAtH0wgOmXpc5EVVmSIfll5E03hpyQG47BrEFBbQa6GxxRBcLRTy5DrefuJ0TpKqXmvBCdr+OwDb2NjrDHHNvUiTkfKYIuIzFvZ8kvY9vPG9F2cOBsfRIAPQ3h2wkEw9ecj8RWfM3rCoQNLgAAAfC5UmMfoN5TThx9zYKW962QAzJ0WWrUXpGP2euLoMuK7TWaf2MJ3v5xQ8xtkCWgcml2zPcjImXgRxNKaad29UfcWVnyyzizz5qQ9gDA0X91nl0KHGOPwIovVKJkrimm+5iKdBF/B5CBzKLkr7pr2duPTQ8dQ8uefsiB4C/HM+jHkVcs+Od3jsHZH2b4axylC8xYcVcVBBGRfwdDBBEomWdCTrQbTBKR4jC8UErz2H0RV5sIIuCxT3BmZ4wkSUb9G12xD2MAmLW+ELVrC2K+34zL8yP+DgzZGpTOjy0UTTb3gA/vPtYMWR5bWXd4mfiuJ07F/Li1a/Lx8Z/Nx7yPlaB0oQlli8xY8MkSmMuDFXaHV4kPh5vc6gxccl/1BfwkRJTqOGxEKc2Yq41YZl6WAGPu+BNVHT1eNGzrgfWMCyqNgLJF2ahclj3hZcWObk/YOS3h9J1yTuh+OVVG1F6Rj5Nv94Q8ZtltFUmvLNuwrQdSQA4Z7GQJ6Dg8CFuHG6YYS/sbc7VY+InSUV+b95ESnDlgRdM7vXBZfTDmalFzWR7KFpqT/rsgovhieKGUNv3S8BdtANAYRJQvzh7z9SOvWnDg723Bya5S8JP5qV392P9XDa78P7XIrhi/8GE4EeeehLuvNPE7L7+9EnqTBsf+1Qm/52ySM+ZpsOzzlahYkj3xhk0Sy7HBqHqkuurtMYeX8YhqAZXLclC5LHyhOyJKPwwvlNLyphtRuSwbLXutIS+Mi24qG7P0uHF7Dw78rQ3A2cAx3HvjHvDhzY0n8NEfzYWoFuBzB6DLVEfVG5ORr4U2QxVVPZdzCSKQX5sZ031G31/Awk+WYs71Rej40AavM4DMfC2KZmdBSOIy8VGizGbhViASEUWD4YVSmiAIuPieamieakHjO8HN/gRRgByQodKKuOjTZWPKwMuSjEMvdIR8TFkKTiLd9L3jGLQEa4GoNAJqLs3DvBtKRm1EeD6VWkTdugIcedkSUy+MLAc3LLxQGr0qZXsaCmoz0Xl0MOLvpWDG+CFOCshoOzCAjsM2SAEZudOMqF6dC42BVXKJaDSGF0p5Ko2IVXdPw4JPlKJ1bz+8jgAy8rWoXJ4DzTj1UnpPOYMbI0YwHFwAIOCT0bC1By17+rH+O7PCDmvM+2gJLEcG0dMYRf2RoaJ1yz5fgawUWA0UTzPW5uPwyx0hfyeCCORNzxh3uM7a6sKW/2mAo8cLYeglbdgK7Hv2DFZ/aVrEPZCIaGrhaiNSjIw8LWatL8KCT5Ri+mX54wYXADEP6QyTJcDrDODdXzeHPU6tFbHuwTos+HgJ9Kaz+T+jQIuCugyImrPDOPnTM7Bmw/QpsUlgRq4WK++qAjB2WbMgAtoMNS7+8thVQC6rD2/+Vz2cfcHAKQcwsi1AwCvhnceaYDlii2vbiUhZ2PNCaScjL/SwTySyBPQ1O9F7yom8aaHrhKi1IhZ8vBTzPlYCt9UHQRSgN6shCAL87gBcA35o9GJKletPhOmX5cOYp8WRVyywHAnuOq3SCph+aT7m3lA87mtzYnM3vM4QO0cj2Hn1wT/aURxjfZzJMtjpwendffDYA8jI02DaqlzoTVPrdSVKNQwvlHbMpXrk1RjR2+ycUD0WCEDPSXvY8DJMFAUYc0dfkNV6FbJiKP+fbkrmmlAy1wSvww+fR4I+Uw1VmL2cGrf3hF8KLwPdJxxw9HovKJjGKuCVsOt3p3FqV1+wJ0kQIEsy9v3lDObdUIIFnyiBIKTIZGmiKYbhhdLSks+W483/OhHMLjHvi4NRZf/PFfBKOL2nH807euEe9COzQIfpl+WhdKE5qZtDpiJthhraKLY08tij22HbPehPaHh599fNI5Wbg+Eq+EaSA8CHL3ZAVAmYf2NJwtpDRGcxvFBaKpyZhSv+vRY7f3sKrn7fyG7U0QaZoplZY77m6PXirY0ngrsVD03Etba60LrXiqLZmVizYUbIeTgUmt6sgaM78gRrgzlxp6veJgda91rDHvPhyx2YeVUBtAnaU4uIzuKEXUpbJfNN+Pgj87H2/hlY9OkyLP1cBa59eCZUmtA9JIIIFNZljlkRI0ky3v7xSdi7h1YonVc7pqvejl2/OxWHnyL9zbg8P2RPFzC0E/fcLBhzEtfr0rSjL/KeWj4ZLe9bE9IeIhqN4YXSmigKKFtkxtzrizHr6kLk1WTi0vtqIKjGXxGjN2tw8T1jV8S0H7JhoM0dcm6GLAEtu62wd3nGP4BCqruiAIZszfhhQQjeFn6ydJxvxo/LGnlPLQCw9/D1JkoGhheacsoXZ+Pa785G1YocCEN74GgzVJhzXTGu//7scYvUte6NvLs1BKB1v3XyG5zmdFlqXP3tmTCXBXu7BBEjr4suU421G2ag4AKqE0+E3qQO2xs0zNriin9jiGgMDtbSlJQ7zYhL/r8aXPwVGQGfDJVWCLtyxO+WIlaOFYTgcRS7rEIdrv/hbHTV22E5MgjJLyN3mgHlS7KhUif+M9a01Tk48VZ3xOO6GxyQJTl1tmggmiIYXmhKE0QBal3kC09WsS64wWOYACNLweNoYgRBQNGsLBTNGjtZOtGyy6LbtNMz6IfXEYAui6dSokTisBGlHdeADyfe6saHL3Wg6d1e+NwTq7h7rhmX5UecA6E1qlJid2e6cLH09ohhJoATUXzw4wKlBVmW4ez14eD/tqF5Zx8gB+dOyBKgelLEoptKMWt94YSLimUW6jDvo8U4/LIl5DHLbq+MamdqSn0qrYjCmZnoPmkPXfl3aK8mLo8nSjyGF1K8U7v6cORVC/rPmzw5fNEJeCXs+/MZAMDsa4om/DwLP1UKbYYah1/qgNd5tjfHmKvBklsrULWcmwemkznXFWHrz+whvy9LwWOIKPEYXkjRDv2jHYde6IhqZcjB59oxY03oDR0jEQQBc64rwsyrCmA5MgiP3Q9jnhZFMzM5YVNhZFmG5eggmt7phbPPC71Zg+rVuaMqJZcvzsbCT5Xig/9tH+nFA8726M3/eAkqlzKwEiUDwwspVm+TIxhcgKgq5wa8Elret2L6pXkX9LwqjYiyReYLegxKHp87gG2PNMJyZHAkiAgicPq9fuRWG3HF/6mFfmgC7vyPlaB4ThaOv9mFrmN2yLKMollZmHlVIQpnJnb5NhGdxfBCilX/VveoT8SRCCrA2Re5DD2lt52/OYXOo8Edr4ffO8P/7T/txLafNeDq/5w5Mj+qoDYz4XVmiCg8zi4kxQo3mXI8sgTouA/NlGbrcKN1rzXkkndZArpPOtB90pHYhhFRTBheSLEiFY07nyAAFcuy49IWUoaWvdaIlZIFFdCypz8xDaJJ9UxNXbKbQAnCj6GkOI4+L/Y+0wp7Zwz7yghA7RUFMJg18WsYxZXH4UfT9l6c3t0Hr0uCqViH2rUFKF1ginrCtM8ZCKbYcJOkZMDnuvDaQJRYw8FFyC1PcksoERheSFGcVh82ffc43AO+mO5XfXEult5aEadWUbz1nXZi8/87AY8jMJI7Bi1unNk/gLJFJlz2telR1djJLNRCDkTussso0ME96Iej1wuNThyqsMwVZamKwWXqYXghRTn0fBvcA9Ht+KvNUKFqVQ7qrihETkV05d4p9fhcAWz+0clgbZ1zcsfwe6DtAxv2PXsGy2+vjPhYVStzsfeZVgR8oQOMLAM9DXZ8+EL7yHOYSnSY97ES1Fx8YSvVaPIxuExNDC+U0mRZxqDFA79Xgi5ThaYdfRGDi6AWsPjTZai9sgBqLad1KV3zzj54Bv2hD5CBhq09WPiJ0oh7DGkNKlx0Szn2PtMa8hhRJaDjQ9uo95mtw4Odj5+Co9uL+TeWxPojUJwwuExdDC+UkmRZRuO2Xhx51YLBobkt0S6LFmRg9rWsfJouWvdFnjwr+WV0HLZh2qrciMfOuroQolrAwb+3wes4O7dFrROhzVTB2ecLOSXmg+fbUbEsO+qNGyl+GFymNoYXSkkH/taGo691jvpatMuiuVFeevG7o3vh/Z7o183XXVGA6Zfmof2QDa5+H3RZamTka7Hpu8fD3k8QgZObu7HstshDVBQ/DC7E8EIpp7vBPia4REsQgYrF2ZPbIEoqc7kBPY2OiOHVVKqP6XFVGnHULuBN7/ZGvI8sAb3Nzpieh+KDwWVq44QASjknN3dHrMURiiwDs64pnNwGUVLVrs0PH1yE4ITagtqMC3oeURVdj120x1F8PFNTx+BCDC+UeqL5lA1g1GaMghi8rf7yNORVX9hFjFJLXnUGaq/IH/+bQvB1X3Fn1QUvZS6anRVVaNZlquCyxrZUnyYHgwsN47ARpRxRHV2mLpyVCWevD6IKKF1gRt2VBTCVxDZ0QMqw/PZKZORpceS1zmChuSE5FQYsu61yUjZJNGRrULUyF6ffC7+irXXfAM4cOISaS/Ow/LZKqLiiLSEYXOhcDC+UckoXmDDQ5gp7ARE1AtZ8fTq03KtoShBEAfM+WoLZ1xShs94OvzuAzEIdcquMk/o8y2+vhK3djb5T4ee1yBLQuL0XLqsPazfMiLrCL00My/7T+Xjmp5RTd0UBjm3qQqj1qoIATL8sL62CSzqdnD/fdCJuj63Siiidb4rb42uNKlz9nzPR9E4v6t/sxEBbmC0oZKD9AxssRwdRMi9+bZrquLKIxpM+Z39KG5mFOlx6bzXe+WUTgHOWSA9tSVNQl4kln0mfUv/pdHKW+87gmZq6uAaYeFNrRdRdWQBzmR5v/jD8zyGIwMmtPQwvcZJOfxs0uThYSympclkOrv/BnKEeFhVUWhE5FQasuKsKVz5QC7Uuvd666XJyHv450qEnydHjjXiMLAH2rhg2CKWoMbhQOOx5oZSVXWHAyrumYeVdyW5J/KTjJEQhtzwtemC0RlXkg4TgHlo0uRhcKJL0+vhKpCDpGFyGpUMPTPFcE9T6CKdIGaiOYksCip4Sg4u7P4DW7U40bbKjbZcLPkf01Z5pYhISXh577DFMmzYNer0eK1aswJ49e0Ie+9RTT0EQhFE3vZ7LXym9KCG4+N0S7B1+OHv8kOXQuzCHovQAo9aJmHt9ccjvCyKQkadF1QqGl8mitOAi+WXU/68N7/2oD43/dKB1mwsnX7Jj5w97ceotx4T+big6cR82+tvf/oYNGzbg8ccfx4oVK/DII49g/fr1qK+vR2Hh+JVQTSYT6uvrR/59ocWniFJJql/MPQMBNL/uQOdBD+ShkiqGPBGVa4woXqaP6e9R6UNI8z5aDPegD/VvdI9sDDr8X2OeFuv+oy7t5l8li9KCCwAcf24QXR94RhZGDmcVOQCcetMJCMC0K1k0Mx7iHl7+53/+B3fffTfuvPNOAMDjjz+O1157DX/4wx/wwAMPjHsfQRBQXBz6Ew9RNGRZRudxO3obHRBEAUVzspA3bXLrgsQq1U/Q7v4A9j9mhdchAef0fLt6JdQ/b4ezO4Dp18dWEE7JAUYQBSz7fCVq1xagYWsPbBY3NHoVKpdlo3xJNlTjFFSUZRk+ZwCiSoBaz/kw0Uj1v4vx2Dv86DoYfrL26c1OlK02QGNgwJ1scQ0vXq8X+/btw4MPPjjyNVEUsW7dOuzatSvk/ex2O6qqqiBJEhYvXoz/+q//wty5c8c91uPxwOM5+way2WyT9wOQYvWecuLdx5owaPEES77LwU9F+dONuOS+GmTm6xLeJiWcoE++ZB8TXM7Vut2F/Lk6mKdpYnpcpQQYvzuA5p19aNrRC7fNj4w8LaZfno/KZdlY+rnwy/P9Xgn1b3Th+BtdcPUHtw/In5GBOdcVoXJZTiKar0jJ+rvwDEpo2+FCx/su+Bwy1AYBxUv0KL/YAH1O5NBp2ece6YULRQ4A3Yc8KF1hmMSWExDnOS89PT0IBAIoKioa9fWioiJYLJZx7zNz5kz84Q9/wEsvvYQ//elPkCQJq1evxpkzZ8Y9fuPGjTCbzSO3ior0qf9BEzPQ7sabP6gfWcIqS2e7c3ubnXj9e/Vw2xK7N40SgovbGkDvcW/I4AIAEIG2Xa4JPX6qz4Gx93jwyreOYveTLeg+6cCgxQPL0UHs+FUzXn+4Hh67P+R9/e4A3vqvEzjw97aR4AIAvY0ObP9FEw4+15aIH0FxkvV34ez2Y+8jfWjZ6oTPLgMy4HfKOLPDhfcf6cdgW+Tzg3dQQqQpLYIIeGycvBsPKdeXtWrVKtx2221YtGgRLr/8cvzjH/9AQUEBfvOb34x7/IMPPoiBgYGRW2tra4JbTKnm0AvtCPikcT8RyRLgHvCh/s3uhLVHCcEFABwd/lBFjc+SAFvrxINfqv4OZEnGlh83wNk7VNtl+Pcw9N/+Fid2/Lo55P0/+EcHepsdY35/wxe3wy9bYDnCXuFzJevvQpZlHH7aBp9THvt+l4CAR8aHT9kgBcL/MWiMIiJN/5JlQJuRcpfZtBDX32p+fj5UKhU6OztHfb2zszPqOS0ajQYXXXQRGhoaxv2+TqeDyWQadaOpy+sKoGVPf/iuXAk4uSUx4SXVg4ssyfC5JEh+GYIquom44gXu4yPklqdc70vHYRsG2t0h3zeyBLQfsmGgbWyvk98j4eSW7rDvOUFEQgNzqkvm34W1yQdnVyB0D6MMeG0Seo+GL1JYdJEu7GsOBLcyKZif+CHqqSCu4UWr1WLJkiXYvHnzyNckScLmzZuxatWqqB4jEAjgww8/RElJSbyaSWnEPeCLeEIJHueHLMV3GWMqBxevXULjP+149+Fe7PhuL7Z/uwdndjghRBjqF0Qgd6b2gp8/1QJM6z5rVD976/6BMV8faHfB7w7/ppMloPP44IU0Me0k6+/C2uALzoMLQxCB/sbw4SWrQo2cOk1w25IQSlfpoc1iz0s8xP23umHDBvz2t7/FH//4Rxw7dgz33HMPHA7HyOqj2267bdSE3u9973t444030NTUhP379+Nzn/scTp8+jS9+8YvxbiqlgWg3a1TrxbjuBJzKwcXdH8Den/ejdbsLAffw2k6g/4RvZGl0OKWrJmfyYSoFmIBXijxkJgjB484XbQZmyQ8Aya9xFG3tlUh/C4IgYO7nTMitC05eF0QEr6hDV9WSFfqYV+ZR9OK+VPrmm29Gd3c3vvOd78BisWDRokXYtGnTyCTelpYWiOLZDNXf34+7774bFosFOTk5WLJkCXbu3Ik5c+bEu6k0iWRZRm+zE65+H/RZauTPyIhrWBimz1KjaHYWuuoHQ/bACCJQc3Fe3NqQysEFAI7/fRBe+9iL9Zjf19BGmCP/D6DiMgP02ZP7mScVViCZSvQRe+zkgAxTydiCmeYyA9R6MWzviyAChbN4IUt2cAGArHINZCn8pHNZCvasRKLWiVjwhWzYzvjQddADn1OCzqxC8RIdjPncfSeeBDnNSgDabDaYzWb0PPE1mIwca0yG1v1W7P/LGQxazi5hN+ZqsOimMtRcEr/QMMxybBBvbTwx/iddAVBpBFz/gznjXoguVKoHF0enH+//T3/E4/LmaOG2BuC0BIKTTs/5XaqNAqaty0DZ6tgK1oUi9wVXEiYzwAx2efDS/YfDHyQAtzyxaNzaLfuebcXx17vCBqB1D9SieO7UnpOXCuFFCsjY9V+98DnGmbA7RKUFVv9nPlRaFkhNJJfdgXuXfgoDAwMR569yMI4mVfOuPmz7WeOo4AIAzj4fdv7mFI6/3hninpOneHYWLrmnGqJaCPYYCBgZ49YYVLji/9ROyeACAAOno1spJPlkVF5uDF6MzzvB+50yGl624/RbzklpUyosoe6qt0c+SAa6TzrG/dbCT5QitzpjzPyH4Ww394biKR9cUoWoEjDns6bgOeH8bDJ0vpj9GRODS4pjvxZNGr9Xwp4nT4c9Zt9fzmDaqlzoTbEVOYvVtFW5KJlnQuM7vehtckAQgaLZWahelRuXqqdKCC6xkAMyTr4Y/oJ+arMTxcv00Gdf+O8z2UXsuk/YIxYcE0Sg64QdJfPHhhC1XoWrvlWH45s6Uf9mN1zWYEjMq8nA7OuKULWcRepSSc50LRbfm41TbzqDtY2GAnrODA2mrcuIuQgjJR7DC02alj398Lkir7poeqcXc64vhmvAh5Nvd6N5Zx+8jgAy8rSoXZuP6kvyoNZeeKegLkuNOdcVRT5wkighuJgrozgpC4A6Q4TfFboo27CO992ovmpy9m5JZoCJavQ8wgdxtVbEvI+WYO5HiuF1BiCqBWi4PUDKyirTYP4dZvgcErx2CZoMEdpMDkYoBV8pmjQD7e6ItUIEIXhc3yknXvmPI/jwhQ4MWjzwDPrRd9qJ3U+2YNN3j8MzGPnCmSpSYRw/WhnF6uCnyjB/+YIA6HNUEZeTAoCrJ4rlSTFI1hBSQW1mFBN2gYLayEFNEAXoMtUMLgqhyRCRUaRmcFEYvlo0adRaEZHrZQOiWsDbPz4Jnysw+vCh/x9oc2HHb0JXM00lSgouw2bdnBWs+nn+X//QHIBZn86CLkuM+FIOT36ebMkIMNNW5kJjVIXsXRFEIKNAi5J5Jjj7vPjg+Xa88YN6vPGDehz4exvs3eE36KP4kyUZvcc8OL3FidZ3nHB0KucDEMWOw0Y0acouMuOD59vDHiMHAI1eBbct9IlFloD2D2ywWdwwFU/+xNrJosTgAgCGXBWWfC0brdtc6NjjQmCoFldunRaVa4zIrtbA2RNA42vjT04dIQH58y68YN14Ej2EpNaJuOyrNdjy0wbIkjyqF0YQg99ffEsZNv/3SViOjC42133SjiOvWrDyC1WYsSY/7m2lsfpOeoMlAGxSMITLQOOrDuTUajD7FhN7VdIQwwtNmtwqI4rmZKHr+Pg1VgQRMORo0byrN/KDCcFy7KkaXlKluNpE6UwqzLghEzXXZcDvkqHSCqNWVxjzVcifq0XPUe+4y0kFETDkq5BbF5/wAiQ+wJTMM+Ha787C4VctI1tMiGoB1RfnIqtIh3ceHb83cPi9/t7vTwd7Z7iqKKEGTvnw4R8GzvYUnnPu6W/04YPfWrH43hyuHkozjKM0qS69rwbmsqEKrMPniqH/iioBzl4vXP2Ru3MFAZB8qbkbazqtLBJVArSZ4rgn9lmfzoKpcujzzXmvpS5bxIIvmONeeDDRQ0g5VUZcem8NbvntRfjULxfg5t8uQu3aAhx8LnyPIhAMdEdesSSglXSuxn/ax9QiGiEBDksAnQfciW4WxRl7XmhS6bPUuPbhWTi9ux8N23rg7PNBb1ZD9gcr7kZLloDsiskpQz+Z0iW4OHv86NjtxmC7H6IKyJ2lQ/FFOqgNZz/PqPUiFn05G33Hveh43w13fwCaDBFFi/UoXKiLy3yX8SRjFZJKK0I1tOLt2KZOCELk6VyyBFiODMLrCkBr4GTdRHD1BmA7HcWquD1ulK5IvfMJTRzDC006lUZEzSV5I9V07d0evLghQvXScwnBirwl81Kr+z1dgsvpLU40b3KMKv/fV+9D8+sOLLjTPKrGhagSkD9Xh/y5ya1Wncxl1Gf2D0S12ecwv0dieEkQtzW61W6OTj+8gxI3SUwjfCUp7s7st0askTFMEABRFLD6y9UJ2QspWukSXCz73MHgAozpZg94ZHzwe2vUF4RES9YyaskffXJR60ToMhlcEkVjjO4SJvmA93/WB7slci+NLMnob/SifbcLnQfd8DlTc/h6qmPPC8Wdzy0Fq5dGcU0smJmJxTeXI3/G5BQ+mwzpElxkWcapzWFWEMnBk3z7Lhdqrk3NTQSHe2ASyVymh/WMO4pdp4Hpl+dDpeZnwkTJKFbBkC/C1RM5YPhcMj58cgArvpkLMUQ9qr56L068MAh3/9nHE1RA6dAO0aI6dT5QTXX8K6O4yyrURbG9PFC3rgBX/9+ZDC5x4ugMwN0bqRIb0HkwtWuWCLnlCe19qVtXGDm4ADCYNZh7Q3H8G0QjBEHAtGgrPEuAxyqh95h33G/3nfDi0JMDcFtH/43IAaBtlxtHn7VFV4mZEoLhheKuYkl2sABYGLIMzL524qX8A34J7kE/JP/knVzSKbgAQMAd3e8m4En9E3QiA8z0y/JQOCsz7NBnQV0G1j80E8Zs7omTaEWL9JhxQ3QBRhCDvSvnk2UZJ18aqt8z3ttfBnqOeGFtim5jU4o/DhtR3Km0IpZ9vgI7f3Mq5DFzri9CVmHsk0L7W1048koHTu+xQg7II3U55t1QjKyiideISbfgAgD6nCg+qwxtDaAUiZjAq1KLuOL/1OLgc21o2NIDv2fok7kA5FUbsfjWchTVZcW1DRRe+SVGuPoCaNsZeXhPDow9wHbaH3noSQQ6druRMz1+tY0oegwvlBA1l+RBVAnY95czcPWf/fSiMQQ3s5tzfey9Lpajg3j7xydHVUSV/DKa3unF6d39uOpbdcirjn0IKh2DCwDozCrk1mnQd9IX+gQvA6UrU7Mw4PkSuQJJrRWx9NYKLPxkKXqbnZADMnIqDNCb2dOSKrKrtWjbEb6eiywDmaVjL3uuvigm5EmAc5L38qKJY3ihhJm2KheVK3LQeXQQzj4vtJlqlMwzTWgH6YBXwvZfNEIKyGMuxLI0/P0m3PjTeTGtWkrX4DKs5rpMWB/rh+TH2AAjBE/sRYuVEV6AxC+h1uhVKJ7NXpZUlDdHC02GAJ9z7DlhmKDCuO9vtS66c4Razwm7qYJzXiihRFFAyTwTpl+Wj4rF2RMKLgBwek8/vI5AyJOULAGOHi/aD9mifsx0Dy4AkFmixqKvZMNYeN7QkAAUzNNi4d3mhBWfmyzJWkJ9oWRZhrXVha56Oxw9408iBQCfO4Cmd3tx+OUOnNjcDfcA512MR1QJmH2zCYKAsfOThv498xNZ4y6vzqnVQoxiNKhwYXLrHdFZ7HkhReo6YYegCr/8WlAFN80rW2SO+HhTIbgMM5VrsOzfcmBr8cPeEaywm1OrhT5bOXNdzpfMInYT0byrD4f+0Y5By9mVXUWzs7D4ljLk1Zwd6jz+RhcO/r0Nfs9QuQEJeP/pFtRdWYAln63g0t3z5M7UYtGXs9H0ugMD50yuzSxVo/pqI/JmjR8+VFoBFZcacXpziCrgIqDNEFG4SDm9kumO4YUUx+Pww9bhjqnqaTSmQnAZJggCzFUamKsmNmfDbQ3A75ShzRJTpmqpUgLMsX91Yt+zY2vVdNUP4vXv12PdA3UonJmJ4693Ye+fWke+P/x+lyWg/q1u+NwSVn9pWoJarRzmaRpc9OVsuPsD8NgkaDJEGPMjB/Np64zw2AKwvO8ZCYrDVai1mSIWftEc9fASxR/DCymGLMn44Pl2HP1nZ1RLouUAUDgzcrG1Z2rqplRwuRB9J7049aZj1H4yuTM1qL46A1nlyZ+8muoBxtHjxb6/jF9kT5aCQ0k7n2jGdd+fjYPPtYV+IBloeqcXc64tSsk9wFKBPkcV08o5QRQw61MmlK70oWO3G87uAFQ6AQXzdQndy4uiw/BCirHv2TM4/npXVMcKImDM00bcH4nBJXqdB9049tfBMV/vO+FDf4MVC+4yp8Qy0lQOMA3besJv8igD9i4vDr9sObskOwRBBBrf6cGSz1ZMfkOnMFO5BqYwQdzvkdB33AufS4beLCKnThuyYi/FD8MLJY3kl+FzBaA2iBFLqtu7PDEFF7VOxOVfmx52pdFUCS5ehwRrow+SX0ZmiRqZJbH/2ftcEuqfGwxZwEuWgGN/GcTKB0OXXk+kVA0w/S3OiMOdggBYW11nhy5CkGWEnehLk0uWZJza7ETrNiekc+ZMazIETP9IJooVtEovHTC8UMINdrpx5FULmnb0QfLJEFQCqlbkYN5HikN2gTe+2xvxZA4Eg8uMNfmYc31x2KJ3SluZMhEBr4yTLw+ic59n1O8tq1yNmZ/KiinEdO73BJdXhyID3sHgJ9Jk70A9LBUDjKgWR+3mPR4ZgMagCt07M0QQAG1m+p/CnT0BdOxxwWHxQ6UVkDdbh4IFiR/GaXjFHiyCdx6fQ8bxvw1CloCSpQwwiZIaM+1oyug95cRr3z6Gxu29kHzBs7MckHH6vT7886FjsBwbOywBRPcJU1AJmH1tEVbcWRVVcEnnXhcpIOPQHwZg2esZE/gG2/w48Kt+OKLYYffsfXwRzxaCGHzsVJJqy6hLF5gi75MkA7PWFwSX/IY7TArWTkpXsiyj+U0H9vy4D63vuNBX70P3YS+O/30Qu3/UB3tH4t5rji7/uMHlXA2v2BHwpf7WGumC4YUSRpJkbP95IwJeacwFVZaCw0jD3z+fNiOKiXeSDF2Un0TTObgAQNcHHgw0h6ikKwMBP9D4T3vUjyeKQritfUYIKXhGSaUAM21lLnRZ6pDBRBCBskVmFNRmofaKgpD7KQkikF+bgaJZqbn793hi/f137Hbj9FtDS5eHTwlD72evXcIHv7XC55zkJYchWPa6I14tA24ZvUdTe1PTdJKCpxpKVx2HbHD0eEMP/ciA1xHA6T39Y741bUVuxCEjWQaqludceEPTQPsuV9iNBCEBffU+uK3RlTvPqdVE/v1LwXoxqShVAoxaJ+KKf58BtUE1OsAM/b+5zDCy/HnprRWouTjYsyKIwWOGw2He9Ays3TADQqTumRQRa2/n8PyS0AcAPqeMjvfD94ZMFldv4GyACkEQAVdfYsIUcc4LJVD3yegLy9Vckjfq63nTjSiek4XO44PjX0QFoHpVLjInsLljOnL1hq4+fC53XyCq4nT5c3XQZjngtUvjP64YrN5rqkzdU8rwHJhky6vJwA3/bw5Ovt2D5h298DoCyMjXovaKAtRckge1LphQRLWA1V+uxuxri9C4vTe4pUaGGtNW5aJodmbaBhcAsLX44bVFSstA5wE3Ki83XkjzoqLWi5EnUEvRbzNAFy51zzRE5xAEAZd9rQZbf9aIrnr7yIlk+L/li81YeVdVspuZdLIso+eoF35PdGPvqihPtqJawNzPm/DBb62jVloAAARAlyVi7udMKX9BFXLL8QyQ9Am8xhwtFn6yFAs/WRrx2JxKI5Z+Lv4X6HiY6Pwyvzu6Hgy/KzFzTArma4NDR+EIQP7c1Ox5TEcML5QwBXWZkF8Of4wcAApqxx/H12aocdX/rUPncTtO7eyD2+aDMUeLmkvzkD899t2j040syzjxgh0du6PrSteZxahXHDk6/TjyJ9vY4ALAVKHGvNtM0GYpY3uBVAkw6e5CJsZHVVxOAAy5iXnP5dZpkVGsgrMrELLnt3iJDjqzMv4G0gHDCyVM6XwTMvK1cPaFmPciBCfmhpu3IggCimdncWffcVj2uqMOLgBQudYY1Y7bfndwcqTXPv6nXFuLHx17Pahaq5zeAQaY+LrQFX0ZRWpklqlhb/eHHv6UgZIViVmaLIgCFtxlxge/HYCzKzCy3H245zdvlha1N/KclEicsEsJI4gCLvv6dKh14phVKYIYHJq47GvToZrgTtNTmSzLaN3uivr4issNKF0Z3Ym/c78H3kE57Bya1q1ORS4TTfYE3nQ0WaUIZtyQMf4O0Qh+zVSpRsH8xM1x05lUWPqNHMz9vAn587Qw12hQdJEOi75ixrzbTdw+IMHY80IJlTfNiOu+PwdHXrOg6d1grRdxqEjd3DBF6ig8n0MOfiKMwuL7smGqiH4fos4PIvfm+N0yBpp8yJ2pnDH/VCxiF8pAuxv1b3Sh5f1++L0STMV61K0rQPXFuVCpRdi7PKjf3I2WPf3weySYSnSou7IQVStyklLxeDJKEWRXa7HgLjOOPzcIj1U6W9xPCM5BmfmprIT/bKJKQME8HQrmcWFAsjG8UMJlFemw8gtVWH5bJXzuANT6yNsDUHhyIPpej1i3B4h2UmS0kyxTiRICzJn9Vmz/RRNkWR4Zbu077cR7vzuNxu09mHt9Mbb/sgly4Oz3e+x+dJ9oRsO2Hqy9fwbUCerNnOwtN3JmaLHyP3LR3+CDs8sPUS0gd6Y2pg0XKT3xikFJI6oF6DLVUQUXnyuAk1t7sP+vZ3DohXb0t0Y/RDIVaLNEaDIifwo1FKggqmP7tGrIU0V1ptAnaPLkZEuVGjDjcfR5sf3RJkjnBBMAI0N43Scc2PbzRkj+0d8f3lqg89gg9ofYxXqyxWuvMEEUkFunRfklRpSuNDC4EACGF1KAk1t78L/3HcLu35/G8U1d+PDFDrz2raPY/KMT8NhTqxx9sgiigLJVhvCF6QCUXxz7sFzpCn3EAl3GQhWyypXbkZuqAaZhS0/EXjVZQthJrQ1be+B1xPfvZKpsckqpg+GFUlrzjl7s/v3pkS0Dzv0Eajk6iLf/+ySkGIZM0lnF5cZggAgxwTGnToOS5bGvzhC1CN2rMzShsvZjyimaFkoqBpi2gwMRN2iMRPLL6KyPfiuIWKXS74umDuV+VKK0J0ky9v+tLeT3ZQnobXbizH4rKpdxWwCVVsCiL2Xj1GYH2t9zI+AOXvU0GQLKVhtQucYY8wTH9vdcOPGCPWSPjs4kYuYns5AzQzkTdcM5dw5MIkSaZxPwT848IilOK8ESscmpLMkYOOWDzyFDaxJhqggG9O4PvWjb4YKt1QdBAMw1GpRfbEDeLE6mnQoYXihldR23w9U/TlW0cwgi0Li9h+FliEorYPq1mai+KgOunmA9CkO+akKrMhwWP068OPSJPcS1L2emRlErjKKRqOGPaCYK59dkwNbujrivVCTxWMWXiODSsdeN5tcdo7YK0OeI0OeKsDb6R1YgyQD6G3zoP+FD5VoDaq5RzoaVNDEcNqKU5bKGDy5AsPfF2Rf5uKlGVAvIKFYjo0g94eWkbZE2dwTQuc8Dn0t5q4xSQTTDVLVXFoQPLhFeH0EECuoyYC6d3GJuiQgure84Uf/c4Jg9jtz9UjC4AKND9dBhLVtc6OHuzmmP4YVSlj4rio5BAdCbo69ZQtHrP+mLOFFXDgDWRobHiYp08c+vycCc64vGv68IaI0q1FySG/L7ap2IFXdO7p5fiQguXruEpn86JnZnAWh9h6sR0x2HjShlFc3Ogt6khtsWZqWEDNRcmhf6+zRhcpQzRY8+a0PpCgNqrstgldEJiLRVwUU3lyGrSIfDr1jg6PYG7yMCFctycNGny5BZoEVeTQYOv2I5O8wqAKULTFj8mYpJ7XVJRHABAMs+98QnKsvAQJMPsiRHtf0FKVNCwstjjz2GH//4x7BYLFi4cCEeffRRLF++POTxzz33HP7zP/8Tp06dQm1tLX70ox/huuuuS0RTKYWIagELP1mK3U+2jPt9QQRMJXpULs1ObMOmCHOVBh6rJ+J8CzkQHGKyW/xYeJc55joyFD7ACIKA2rUFmHF5PgY63PB7JGQW6Eb1TM68qhC1Vxag/7QTfo+ErEIdjLmTOxcpUcEFAJzdAQgCLmillSxHHFUjBYv7sNHf/vY3bNiwAQ899BD279+PhQsXYv369ejq6hr3+J07d+Izn/kM7rrrLhw4cAA33ngjbrzxRhw+fDjeTaUUVHtFAS66pSy4F5IACKrgDQByqoxY90AdVBqOfsZD2cWG6CeKDn3ateyPfmNIGk3ILQ87/0UQBWSXGZBfkzESXAa7PGjZ248zB6zwuQLIq85A0awsRQcXABfcg2csmtgkdVIOQY62b3iCVqxYgWXLluGXv/wlAECSJFRUVOCrX/0qHnjggTHH33zzzXA4HHj11VdHvrZy5UosWrQIjz/+eMTns9lsMJvN6HniazAZuWQuXbgHfGh6txc2iwdqvYjKpdkoqJtYbREW1Ipe85sOnH7LGd3BApBZosLSr48/B4Mik/uC1XAjLaEe7PRgz1On0XF4cORrolrA9MvysOSzFVDrJi/QJzq4AEB/gxcf/HZgwvev+3gmSldynzSlcdkduHfppzAwMACTyRT22Lh+ZPV6vdi3bx/WrVt39glFEevWrcOuXbvGvc+uXbtGHQ8A69evD3m8x+OBzWYbdaP0ozdrMOf6Yqy8qwpLb61A4cwsxRdFS1V+twTLfjdatzthLFBh1qczYSiI4lQhB7v7aeKiWYFk7/Zg03ePwXJ0cNTXJb+Mhi09ePvHJyetPkwyggsAZE/XIKMkum0pRhGA3FlaFC+b3NVVlHriGl56enoQCARQVDR6tnxRUREsFsu497FYLDEdv3HjRpjN5pFbRUXF5DSeaIqRZRmn3nJg5w96cfxvg2j8pwPH/jKIEy/akTc7ul5Mzne5cJECzMHn2uB1BsYd0pNloKvejuZ3+y64HckKLkBwns/8O8ww5A5doobfVkP/zShRoexiA9SGs+83bZaA6vUZmHebiUNGU4DiVxs9+OCD2LBhw8i/bTYbAwzRBDS/7kDLlnOWmA4NKEte4Mx2F9QGIewO04II5M9Nr4J1yRJqt2uP3Y/Tu/sj1n45sbkbM9bkT/j5kxlchumzVVj6jVx0HXTDss8Dn12C1iyiZKkeBfN1wWGy6zLg7g8WY9TncJ7LVBLX8JKfnw+VSoXOzs5RX+/s7ERxcfG49ykuLo7peJ1OB52Oc1uILoTbGkDL1vC1MfzuCBsEykD5JcbJbNaUNl6AsXdHXv0FGbB1THzidCoEl2EqjYCSZQaULBt//oqoFmAsUPxncJqAuA4babVaLFmyBJs3bx75miRJ2Lx5M1atWjXufVatWjXqeAB48803Qx5PRBeu80AUFUnl4FwEINjLMkIM/nvOZ7KQWcILyWQ6fwhJrY3ulK2K8rhIz0uUquJ+ptmwYQNuv/12LF26FMuXL8cjjzwCh8OBO++8EwBw2223oaysDBs3bgQAfP3rX8fll1+On/70p7j++uvx17/+FXv37sUTTzwR76ZSEkkBGV5nABqdeMEnXoqduy9yXQ1BBWSWqlF9dQbadgU3xBNVAnJnalG60gBjvmrc+8myjIFTfthahjbQq9bAVMGqyNE6twfmc1I9Mgu1sHd5Qx8vApXLsif0XFNhJZ7PKcHeESx8mVWmhlrP840SxT283Hzzzeju7sZ3vvMdWCwWLFq0CJs2bRqZlNvS0gJRPPvmWb16NZ599ll8+9vfxre+9S3U1tbixRdfxLx58+LdVEoCZ58XR17rROO2Hvg9EiAA5ReZMfeGYhTM4OZqiXLuxMdQZCl4nHmaBuZp0YUPu8WPo8/a4OwMnJ10KQOZZWrM+awpZOAZT8Arw9HpB2TAWKSGWjd15jcMB5g/zZiJlTf0YvfvT4c5GJh5dWHMz5HuwcXnkND4mh2dBz2QhxbFiWqgeJkeNddmTqn3UzqIe52XRGOdF+UY7HRj08P18Dr8o8bxh4ckLr2vJi67Raf7SXoiBtt82PcLa8Tjlv97TtRzDFy9Aez9RT8CXnnsHkkioDEKWPr1HOhM4QNMwCvj1JsOtO92I+AJnq5EDVCyTI/q9RlT6pOz3HcGkGXM3bgFR1/rhCBi5G9HEIOrdC69rwYVMVadTqV5LvHgc0nY/5gVrt7A2PeiEOyBWfSVbG5vkWQpU+eFKJx3f9U8JrgAwZOxLAe/7x4Ms68RTZqsMg1yajWh66kLQMF8bUyTI0+/7Rg/uACABPicMs4MbaAnBWTI0tjPUQGfjA9+Z0XrO66R4AIAkg9o2+XGgcetwR67KULILQcEAUe+dQWu+e4sVF+cC3OpHjmVBsy5vhgf+8m8lA4uUkCGs9sPZ7cfkj9xn5tbtjrh6hknuACADAy2+dG+i5s5Kgln11FS9J5yorcpTOVWOXiia9zeg7nXj7/SjCbX3FtN+PBpGwaafCOf6If/m1OrwaxPh/8kdK6ATw5OAg6XKyTgzE4Xeuu9I8NK2TUalF9qQP5QXZn291ywtfhHlm2PIgMOSwCt21yovjojth9WwYaHkP511UX4/PTwlXgjSVRwkfwyWrY50bbDBZ8j+GKqjQLKVhpQeYUxrj0eUkBGx3vu8d9Dw2SgbacLFZdxtZxSMLxQUnSfsAc/5Uc4oXTX24HrE9WqqU1tELHoS2ZYm3zo3O+B1x6AzqRC0WI9zNPUMVU09julkXkF4ch+BIMLAMiAtckHa6MPVVcagxODd7oivkfad7lQdaVxStX4CFUHJhYJCy4BGR/+cQD9J32jXku/U8bpLU70N3qx8O6JD9lIfhm9x71w9wegMYrIm62Fxnh2UMFnlyIu8wcAd78EyS+z0KJCMLxQUkR9HeR5JKEEQUDOdC1ypl9YsTmVXowcTsczdPzpzU5klavh7os8JORzyvA5pIhzZ9LNhQSYRA4Vdex2o/+Eb/xvyoCtxY8z7zhRdUXsvWcde91ofM0Ov1Meeb8JquCmojXXZEBUCRCiDSPCeSUAKKXxpaKkKKjNjHxhE4DCOq44UiK1LriEesLhUwTadkW5ISQwpXpdzjWR8JHI4CLLMs7siDCXZGjIZrw5T+F0vO9C/XODweAy9DgAIAeCFaFP/CO495M2Q0RmmTr8e1EEcus0EMSp+T5SIoYXSorcaUbkTzeG/qQjDO+SO/ES55RcVVdcwPwBCRho9geL4oW7nghAZqkKmoypeyoTcsvDbuR4rkSvKpL8CE6UjcA7KI/MhYlGwCej4VVH2GMsez0j9VwqLzeE/7AkgfNdFGbq/sVT0l18Tw10WeoxASa45BO45P+rhi6LI5tKZa7SYO7nTBCHX0IBsfXEyEDFZREuOjIvOkB0ASYZy6Fj2fg9liGb3qMeBCLMYxFEwLI3uE1C4UL9SJge9TxD7cup1UBQCUizyiFpjVcGSpqsIh2u+/5sHP1nJxq29sDvliCIQPnibMz9SDHyp0+dFSTpqmCeDtn/Nw+Wfe7gqiEApio1Tr3pDH/xEYCsCg3yZulQvd6I5tedY2qayBJQcbkBhYtYzwkYCjDAuPNfklXHRVQLMFWpQ68YG2IsUkFtDCYJa7MPbTuc6D/pgywDWRVqlK82IG+OdmTSuLtfCn70DjMlSpYQ3LRxSPX6DOTO1KJtpwt9J73BTUbl4HvJ2uhD/0krMktVmHebGfqcqTV/SokYXiipjDlaLL21Aos/Uw6fKwC1ToRKzQ7BdKIxiqi4dHTviHdQQuu2MCuJZKBsdXAzvqorMpBdo8WZHU5YG4MXNPM0DcovNiBnBnexPt/5E3iTXYCu4lIjjvzJFvEYQRDQss2Jpn86RgUTa6MP1gYfipfqMPOTWRBEIVgROsJcbkEEVPrRXT/maRrIkoyuQ56zc2TOeRy7JYADj1ux9Bs50Bh4HkplDC+UEkRRgC6Db8epouqKDFgbfBhsG/8TefFSHQrmnw0mwS0JzAlsoTKdvwIp2cEFAPLnaVFxuSEYVs/tLRlaHVSyXI/ipTr0N3iDwQUYHUyG3h+WvR5klWtQtsqA/Dk6nHzRHnaHbVkCCheM7ZVr2hRmrowEeAYkWN53czgyxTFaElHcyJIMV28Azp7RFVXVOgGLvpyNqiuM0BjPfjo25Iuo+3hm8BN2LBMmaMT5O1Enu+S/IAiYfl0m5t9pQs50DUR1cDmzuVqDuZ83oe4TmRAEIVhtOcIVqXW7E7IkQ5slomS5PuQcKkEEMkpUyK0b3TPn6gvAdjr8EBZkoGOPO7YfkhKOH3WJaNJJgWDp/zM7XPDagh+P1UYBpSsNqFprhEorQKUVUH11BqquNMJrkyCoAG2WyNAyCYZ7YJIdXM6VN0uHvFmh5yf1nfRGHApy90lwWyUYclWYcUMmfE4Z3Yc8Z+dDDfXsGItUWHBX9pilz97B6LaS8ER5HCUPwwsRTSopIOPIMzb0HvOO+rrfKaNlixPWBi8WfulsRVVRJXCCZBxEG1x8Tgnu/gBEjQBjgSop4VGW5bBDQKOOHaoHI6oFzL3VBNtlPlj2uuHuD0BtFFG4QIe8Wdpxa7ZEu6Rem8EAneoYXohoUrXvco0JLiNkwNbqR+s2J6at42qyZHL3B9C0yYHuQ56R4KDPFVG51oiSZfqEhhhBEJBZooK9IxB2SEelE6Azjw66pgoNTBWaqJ7HmK9CZpka9vYwQ0cCULxUH2XLKVk454WIJs3AKV/E4mHDFVWlAGtqJIurN4B9j/aPCi5AcFjmxPN2NP0rwmsYB2WrI9T0EYKTey90E8fq9Rmhn0cENBkCSlYYLug5KP4YXohoUritAXzwO2tU+xn5HHLU8w9o8p18cRA+V+ihmtZtLthaQuxHFCdFi/XInRViSwkBMBaqMO3KC18BlDdTi9m3ZEEc6qwRxLOF6/TZIhZ9ORvaKVyxWSk4bEREk6J9lwtSDNc7boKXHK6+APpCbZQ4RBCBtl0umCqjG46ZDKJKwLzbTGjZ4sSZHa6RPYtEDVCyTI9pV2dAPUm1V4ou0iNvthadBzywt/shqoHcOi1yZ44/V4ZSD8MLEU2KzoOeqI815InQZjG9JINjaL+fcGQJGGyNfNxkE1UCpq3LQOUaI5zdAciSDGOBGirt5AcKtV5E2SoODykVwwsRTYqAJ/o5LOVDFVUp8YQoF3YJSbw6iGoBmSW8PFFo/OhDRJNCn6uKauPF/HlalK7gao5kMVVpIgcYITg3hChVMbwQ0aQoXaGPOFk3o0SFuZ8zcV5BEmkMIkqWha5OCwTnvJSu5JAKpS6GFyKaFEWL9cgsU4dcLSJqgNmfNiVkuMhh8aN9jwsde1xwdid+7kaqm359JsxVw8ttzn59eOXNnM+aWDiQUhoHFYloUqg0AhbebcaJ5wfRfdg7qhfGWKDCrE9nIbM0vqccV28Ax/9uw8Cp0YEle7oGsz6dBX02L8gAoNIGX6vO/W607XLB2R2ssFswT4eyiw3ILJ7818nnktCxx42O993wDkrQZggoXqpHyQoDlyZTzBheiGjSaAwi5n7ODLc1gP6TXkgBILNEDVOlOu49Lh5bAPt/1Q+fc+zYlbXZhwO/smLJ13KgzeSFEghOii1ZbkDJ8vgPD7n6Ajj4Gys8A9JIqHW5ZTS/EVwWfdFXsmEs4OWIose/YiKadPpsFUqWGVC20gBzlSYhQ0UtW5zB4DJe4TUJ8NgknHnXGfd20GiyLOPw0wPw2KSxc6LkYMHCQ08OjOxZRBQNhhciUjzJL6PjfXf4XYlloP09N2SZF8lEGjjlg6MjEPq1kQF3r4S2na6EtouUjf10RKR4PqcUVXVfv0tGwAuodfFvUypyWwOw7HPD3RuASicip1YNt1WCZ0CCWh/ckdmQN/68IFmWJ9SD1nfCB0FExF2jG15zwFyjRVac50VReuC7hIgUT6WL8qIqAOIkn/WszV6ceceFvpNeQAIyS9Uou9iAwgW6lFkSLssyTr3pxOm3Rw+bte0M/lcQAVkGmjc5ULhIh5mfyoJKI8A7KOHMzuCqLZ9dhlovoGiJHuWXGGDIjW7ysxztBpwS0PiaHYvuzo7hJ6OpiuGFiBRPrRORU6dB/0lf2B2DC+ZqIaomL1C0bnei8TXHqJ4FW6sftr8MouewB3M+mxo1bc6848LpzaHn+5zbK9L1gQcBj4zpH8nAwcet8Nrlkd+p3y2jbZcLlvddWHh3dlR7H2WWqCP2ugyzNvjg6gtEHYxo6uKcFyJKC1VrI+w4LAMVl0felTjgleHqC8DnDH/FtTb70PiaI/jQ5x46dKHv/tCL1u2R53EEfDJ6j3vQecANa7Nv0ufkBHwyToUJLmPIQO8xLw79YQBehzw2DEpAwAd8+NQAJH/ktubP00FtiD7AufsC0beVpiz2vBBRWsiu0WLWp7NQ/9wgZBlnL7pCcFhk9i0mmCpC9xQ4e/w4/ZYTXYc8kIeunzm1GlRdYUR2zdhS+WfedQY//oXJOGfedaL8UsO4vT2yLKN1uwstbzvhd58NAfpcETNuyET+nMmZmNN/wouAO8ZAJAQn0YY0tEqo+0MPii4Kv9WDSiNg1qezcPiPtqieOh6bMFL6YXghorRRvFiPnOkadOxxY+CUDxCA7BoNipcZoAuzi7W93Y8Dj1sR8I1eat3f4EN/wwDmfCYLhQtHX6T7T/rCr24C4B2U4eoJIKNo7Km26V8OtG4b2zPj7pNw+I82zP28CQXzggFGCsjoPe4NTrTVCsibo4XOFN3Qis8R5ZjNuaLJOmLw9xMpvABA/hwdii7SofNA+J3HtSYRWeW8LFFkfJcQkeL5HBIs+9wYbPdDFIGcOi0q1xohqiN/ipdlGUeetSHgHWeIZOjfx/4+CEOBCj2HvXANBYhohkyAYIE2Y4Fq1NwXZ49/3OByrhP/GETebC16Dntw8iU7fA45WMpfBvAiULxUh9qPBSfWhqM1TWB2wPDzRBDLEFfNtRnoOeJBwBv6mMo1xpSYI0Spj+GFiBStY68bJ/4xeHbeiQBY9nnQmOXA/DtNyCoLP6l0oNkHV3f4eRayH9j3c+vIPkCCEHnp77DDT9mgM4sov9SA8osNEEQBHXvcEZcP+xwyml8/r3dGPvtfy14PfIMy5t0Rfr+onFotNJkCfPYYho6iOVQCTOWRJ+wO05lVmP+FbHz45AACnrNPMPx7KL/UgLLV3G2cosMJu0SkWL3HPcE5LgEEL7gyRoZyvHYJH/x2AB5b+GBia/WH3WF5lKHniDa4DPMMSGh81YHjfx+ELAWHkiI+hgC073aHbUvvcS+sjeEL3IgqAdOvy4y+sQJgnqaG1iSE/b2IGqBocWzzcrKrNVj5QC6mfyQD2TUaZFWoUbxMjyVfy8aMj2QmpBIzpQf2vBCRYjW/4Qw9xCEPL+11o2Z9RsjHSOT1svOAB/lzdcFJqZGGZmREnmgrAh173MiZMXZC8bmKl+gh+WQ0vGaH5MWYicbC0MdYWQJypmsw93MmODoD+OC3VkjS6GOHA83sW0xQ62P//Ksxiqi41IiKSyOv/CIKheGFiBTJ1RuAvc0f/iAZ6NwXPrxk12iiGyaZDAJwZqcL5RcbIk5ejYoUnFMTjdKVBhRepEf3ITdcvRJUOgGmChUGTvnh7pegNggoXKgbWZFlniZi8X05OPWWAz1Hzu4SnjNDg6orjciuDh+YiOKJ4YWIFMnnim7sJtJxWeXB4YvBNn/E1UMXTAYGz/iQN9sMQ54Id780/vDR0NDNQHOEcAbAYfHD1uoLuwx8mFonoGTZ6F2kc2aEHvrJLFFj3ufNsJ7yYrDVD41RRN4cLTQGzjig5OI7kIgUSRflKppolhTP+awJ2kxx7ByPOAwpCaIAUSVgwV3ZZ1cCDT/P0H+zazSY+zkzVFF0bkg+4OBvrLB3RA46sRps82Hfo/04+OuBkTk7u37Qi5Mv26NebUUUDwwvRKRIOpMKObWa8AFDAEqWR17BYshVYenXc1C51giNMfiAogYoXqKDJmMSE4wI5NYGe0gMeSosvz8XMz+ViewaDTJKVMifo8X8O01Y+EUztJliVBWBAUAKAM1vOCavnRiqffNra7BH6tzn8gNtO104/PQAZIkBhpKDw0ZEpFjV6zNgbbIGh17Ou44KIqDLFlEaRXgBAG2miJr1GahZnwEpIEMQAUEQ0PWBG0efHZycBktA+SVnA4lKGxzGOX8oZ1jVFUb4HBLadoZZdTT0uL3HvPDapWAP0iRoeMUOyY+Qk6H76n3oOeodKaRHlEhx7Xnp6+vDrbfeCpPJhOzsbNx1112w2+1h77NmzRoIgjDq9pWvfCWezSQihTJVaLDgLjO0Q9VzBREjPTFZFWpc9JVsqCcwP0NUCSPLdgsX6jHr5iyo9MKY58ifq0XJct3Iap1Qhr8//foMmKdFXxtFEAXUfiwLQjTFdGXAMzA5+wK5egOwNoXZ5BIYWsodee8moniIa8/Lrbfeio6ODrz55pvw+Xy488478aUvfQnPPvts2Pvdfffd+N73vjfyb6ORS+qIaHw507VY9WAueuu9sLf7IaoE5NRqIhani0XxYj0K5uvQc9gDV08AKp2A/Lk6GPKCqaJ6vYSeox4E3DK0JhGSX0b7e27Y2/0QVEBurRbllxjG3SMpGmq9EKywG+m4SZpI6+qJIgTJgLOTmyhScsQtvBw7dgybNm3C+++/j6VLlwIAHn30UVx33XX4yU9+gtLS0pD3NRqNKC4ujlfTiCjNCKKA/Nk65M+O3xCGSiOE3sdHCNYvUetkZJaqYSxQo2Tp+ENBE1F0kR5ndrhC94QIQGapGobc6PY7ikSMcnNElY5F5Sg54jZstGvXLmRnZ48EFwBYt24dRFHE7t27w973z3/+M/Lz8zFv3jw8+OCDcDpj2M6diChB/B4Zx5+zYdcPenHkGRuOPjuIPT/px4HfWOHsnrzVP2UXGyBqEHpysgxMWzd5PdSmCvXIxOWQBKBgAee7UHLErefFYrGgsLBw9JOp1cjNzYXFYgl5v89+9rOoqqpCaWkpDh06hP/4j/9AfX09/vGPf4x7vMfjgcdzttiTzRbdtutERBdC8ss49DtrcHuB83pEBpp92P+YFUu+mjMytBSKLMnoO+mFtcGHgVM+eB0SVJrgsFTJcj302SoYclVYeFc2PnxqAH7XORs0DuWL2hszkT9n8oKEqBZQcbkRTf8KsYJJCK7GKl3BvYgoOWIOLw888AB+9KMfhT3m2LFjE27Ql770pZH/nz9/PkpKSnDllVeisbER06dPH3P8xo0b8fDDD0/4+YiIJsKyzw1bS4jeFTnYK9O0yY65t5pDPkbXB240vOKAd3BspTqHxYnWbU7Mu82M3JlamKdpsOpbeeg84Eb/SS+kAJBVpkbJcn1UtWxiVXGZAe7+ANrfO28TSQFQaYD5XzDH5XmJohFzeLn//vtxxx13hD2mpqYGxcXF6OrqGvV1v9+Pvr6+mOazrFixAgDQ0NAwbnh58MEHsWHDhpF/22w2VFRURP34REQT0bYrwkobCeg57IXPIUGTMXaEvvOgG8f+En4JtuQHDj89gGX358KQq4JKK6B0hQGlKyZvPk0ogiig7uNZKFqsR/suF+wW/0iPUPEyPbTj/ExEiRJzeCkoKEBBQUHE41atWgWr1Yp9+/ZhyZIlAIC3334bkiSNBJJoHDx4EABQUlIy7vd1Oh10Oo67ElFiuXojr7SRJcDdHxgTXiS/jJMvhS8bMXKsBLS/54ptZ+hJZK7SwFw1eSu3iCZD3KLz7Nmzcc011+Duu+/Gnj17sGPHDtx333245ZZbRlYatbW1YdasWdizZw8AoLGxEd///vexb98+nDp1Ci+//DJuu+02XHbZZViwYEG8mkpEFDOVJrqVNuOt3Ok97oXfGWV1Wgno/nASNnEkSiNx7ff785//jFmzZuHKK6/Eddddh0suuQRPPPHEyPd9Ph/q6+tHVhNptVq89dZbuPrqqzFr1izcf//9+OQnP4lXXnklns0kIopZwXxdxDOoIV+EsWDsvBBXbyCmfZMkL8vwE50rrkXqcnNzwxakmzZtGmT57B9lRUUFtm3bFs8mERFNirKLDeh43x22CG3l2oyRSr3nUuuF8NVrzyUAxiLu5EJ0Ls64IiKagIxCNeZ93gRBjdG9KENn1aorjCheMv58vLzZ2uh7XmSgdGX8J+gSKQnjPBHRBOXN1mHlf+SiY7cbvfVeyAEZWRUalK40IKs09OlVZ1KhZJkeHe+7I/bA5M7SomDexLYVIEpXDC9ERBdAZ1Jh2lUZmHZVRkz3m/HRTHgHJfQe854tOncOQQOUrzag+uoMCCLL8BOdi+GFiCgJVBoB8243YaDZh4733XD1BaBSC8gsVcNcrUH2dC3U3DuIaFwML0RESSIIArJrtBPebZpoquKEXSIiIlIUhhciIiJSFIYXIiIiUhSGFyIiIlIUhhciIiJSFIYXIiIiUhSGFyIiIlIUhhciIiJSFIYXIiIiUhSGFyIiIlIUhhciIiJSFIYXIiIiUhSGFyIiIlIUhhciIiJSFIYXIiIiUhSGFyIiIlIUhhciIiJSFIYXIiIiUhSGFyIiIlIUhhciIiJSFIYXIiIiUhSGFyIiIlIUhhciIiJSFIYXIiIiUhSGFyIiIlIUhhciIiJSFIYXIiIiUhSGFyIiIlIUhhciIiJSFIYXIiIiUhSGFyIiIlIUhhciIiJSFIYXIiIiUhSGFyIiIlIUhhciIiJSFIYXIiIiUhSGFyIiIlKUuIWXH/7wh1i9ejWMRiOys7Ojuo8sy/jOd76DkpISGAwGrFu3DidPnoxXE4mIiEiB4hZevF4vbrrpJtxzzz1R3+e///u/8Ytf/AKPP/44du/ejYyMDKxfvx5utztezSQiIiKFUcfrgR9++GEAwFNPPRXV8bIs45FHHsG3v/1tfOxjHwMAPP300ygqKsKLL76IW265JV5NJSIiIgVJmTkvzc3NsFgsWLdu3cjXzGYzVqxYgV27doW8n8fjgc1mG3UjIiKi9JUy4cVisQAAioqKRn29qKho5Hvj2bhxI8xm88itoqIiru0kIiKi5IopvDzwwAMQBCHs7fjx4/Fq67gefPBBDAwMjNxaW1sT+vxERESUWDHNebn//vtxxx13hD2mpqZmQg0pLi4GAHR2dqKkpGTk652dnVi0aFHI++l0Ouh0ugk9JxERESlPTOGloKAABQUFcWlIdXU1iouLsXnz5pGwYrPZsHv37phWLBEREVF6i9ucl5aWFhw8eBAtLS0IBAI4ePAgDh48CLvdPnLMrFmz8MILLwAABEHAN77xDfzgBz/Ayy+/jA8//BC33XYbSktLceONN8armURERKQwcVsq/Z3vfAd//OMfR/590UUXAQC2bNmCNWvWAADq6+sxMDAwcsw3v/lNOBwOfOlLX4LVasUll1yCTZs2Qa/Xx6uZREREpDCCLMtyshsxmWw2G8xmM3qe+BpMRs6FodGeqakDAAi55UluCRERnctld+DepZ/CwMAATCZT2GNTZqk0UbwxuBARpQeGF5oSGFyIiNIHwwulPQYXIqL0wvBCaY3BhYgo/TC8UNpicCEiSk8ML5SWGFyIiNIXwwulHQYXIqL0xvBCaYXBhYgo/TG8UNpgcCEimhoYXiitMLgQEaU/hhdKC8/U1DG4EBFNEQwvpHgMLkREUwvDCykagwsR0dTD8EKKNTxBl4iIphaGF1IkriwiIpq6GF5IcRhciIimNoYXUhQGFyIiYnghxWBwISIigOGFFILBhYiIhjG8UMpjcCEionMxvFBKY3AhIqLzMbxQymJwISKi8TC8UEpicCEiolAYXijlMLgQEVE4DC+UUhhciIgoEoYXSjkMLkREFA7DCxERESkKwwsREREpCsMLERERKQrDCxERESkKwwsREREpCsMLERERKQrDCxERESkKwwsREREpCsMLERERKQrDCxERESkKwwsREREpCsMLERERKQrDCxERESkKwwsREREpCsMLERERKQrDCxERESlK3MLLD3/4Q6xevRpGoxHZ2dlR3eeOO+6AIAijbtdcc028mkhEREQKpI7XA3u9Xtx0001YtWoVfv/730d9v2uuuQZPPvnkyL91Ol08mkdEREQKFbfw8vDDDwMAnnrqqZjup9PpUFxcHIcWERERUTpIuTkvW7duRWFhIWbOnIl77rkHvb29YY/3eDyw2WyjbkRERJS+Uiq8XHPNNXj66aexefNm/OhHP8K2bdtw7bXXIhAIhLzPxo0bYTabR24VFRUJbDERERElWkzh5YEHHhgzofb82/HjxyfcmFtuuQUf/ehHMX/+fNx444149dVX8f7772Pr1q0h7/Pggw9iYGBg5Nba2jrh5yciIqLUF9Ocl/vvvx933HFH2GNqamoupD1jHis/Px8NDQ248sorxz1Gp9NxUi8REdEUElN4KSgoQEFBQbzaMsaZM2fQ29uLkpKShD0nERERpba4rTZqaWlBX18fWlpaEAgEcPDgQQDAjBkzkJmZCQCYNWsWNm7ciI9//OOw2+14+OGH8clPfhLFxcVobGzEN7/5TcyYMQPr16+P+nllWQYADLo8k/4zUfy5HC4IWkeym0FERAnmsjsBnL2OhyXHye233y4DGHPbsmXLyDEA5CeffFKWZVl2Op3y1VdfLRcUFMgajUauqqqS7777btliscT0vK2treM+L2+88cYbb7zxlvq31tbWiNd6YShEpA1JktDe3o6srCwMDg6ioqICra2tMJlMyW7alGaz2fhapAi+FqmDr0Xq4GuRfLIsY3BwEKWlpRDF8OuJ4jZslCyiKKK8vBwAIAgCAMBkMvHNmCL4WqQOvhapg69F6uBrkVxmszmq41KqzgsRERFRJAwvREREpChpHV50Oh0eeugh1oFJAXwtUgdfi9TB1yJ18LVQlrSbsEtERETpLa17XoiIiCj9MLwQERGRojC8EBERkaIwvBAREZGiTInwcurUKdx1112orq6GwWDA9OnT8dBDD8Hr9Sa7aVPSD3/4Q6xevRpGoxHZ2dnJbs6U8thjj2HatGnQ6/VYsWIF9uzZk+wmTUnbt2/HDTfcgNLSUgiCgBdffDHZTZqyNm7ciGXLliErKwuFhYW48cYbUV9fn+xmUQRTIrwcP34ckiThN7/5DY4cOYKf/exnePzxx/Gtb30r2U2bkrxeL2666Sbcc889yW7KlPK3v/0NGzZswEMPPYT9+/dj4cKFWL9+Pbq6upLdtCnH4XBg4cKFeOyxx5LdlClv27ZtuPfee/Hee+/hzTffhM/nw9VXXw2HgxvEprIpu1T6xz/+MX7961+jqakp2U2Zsp566il84xvfgNVqTXZTpoQVK1Zg2bJl+OUvfwkguA9YRUUFvvrVr+KBBx5IcuumLkEQ8MILL+DGG29MdlMIQHd3NwoLC7Ft2zZcdtllyW4OhTAlel7GMzAwgNzc3GQ3gyghvF4v9u3bh3Xr1o18TRRFrFu3Drt27Upiy4hSy8DAAADw+pDipmR4aWhowKOPPoovf/nLyW4KUUL09PQgEAigqKho1NeLiopgsViS1Cqi1CJJEr7xjW/g4osvxrx585LdHApD0eHlgQcegCAIYW/Hjx8fdZ+2tjZcc801uOmmm3D33XcnqeXpZyKvBRFRKrn33ntx+PBh/PWvf012UygCdbIbcCHuv/9+3HHHHWGPqampGfn/9vZ2rF27FqtXr8YTTzwR59ZNLbG+FpRY+fn5UKlU6OzsHPX1zs5OFBcXJ6lVRKnjvvvuw6uvvort27ejvLw82c2hCBQdXgoKClBQUBDVsW1tbVi7di2WLFmCJ598EqKo6E6nlBPLa0GJp9VqsWTJEmzevHlkYqgkSdi8eTPuu+++5DaOKIlkWcZXv/pVvPDCC9i6dSuqq6uT3SSKgqLDS7Ta2tqwZs0aVFVV4Sc/+Qm6u7tHvsdPnYnX0tKCvr4+tLS0IBAI4ODBgwCAGTNmIDMzM7mNS2MbNmzA7bffjqVLl2L58uV45JFH4HA4cOeddya7aVOO3W5HQ0PDyL+bm5tx8OBB5ObmorKyMoktm3ruvfdePPvss3jppZeQlZU1MgfMbDbDYDAkuXUUkjwFPPnkkzKAcW+UeLfffvu4r8WWLVuS3bS09+ijj8qVlZWyVquVly9fLr/33nvJbtKUtGXLlnH/Bm6//fZkN23KCXVtePLJJ5PdNApjytZ5ISIiImXixA8iIiJSFIYXIiIiUhSGFyIiIlIUhhciIiJSFIYXIiIiUhSGFyIiIlIUhhciIiJSFIYXIiIiUhSGFyIiIlIUhhciIiJSFIYXIiIiUhSGFyIiIlKU/x+WudUokcUYzAAAAABJRU5ErkJggg==", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi8AAAGdCAYAAADaPpOnAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjYsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvq6yFwwAAAAlwSFlzAAAPYQAAD2EBqD+naQAAZpZJREFUeJzt3Xd4XOWdPvz7nOkjaUa9F0uy5F5wt2k2OBhDSEghkEILIeWFlDW7WZxfNoSU9WaTLCSEhFQIbEgISyiBhGZcwDY2bhgbW7aKLVm9j6aXc94/RsVCmiZpZs6R7s91zWV7dGbmsaace57yfQRZlmUQERERqYSY7AYQERERxYLhhYiIiFSF4YWIiIhUheGFiIiIVIXhhYiIiFSF4YWIiIhUheGFiIiIVIXhhYiIiFRFm+wGTDVJktDS0oK0tDQIgpDs5hAREVEUZFnGwMAACgsLIYrh+1amXXhpaWlBSUlJsptBREREE9DU1ITi4uKwx0y78JKWlgYAaPjZl5BmMiS5NURE6vGX8tkQMgqT3QyaoVx2J/51/S3D5/Fwpl14GRoqSjMZYDEzvBARRcuUYoKQmpLsZtAMF82UD07YJSIiIlVheCEiIiJVYXghIiIiVWF4ISIiIlVheCEiIiJVYXghIiIiVWF4ISIiIlVheCEiIiJVYXghIiIiVWF4ISIiIlVheCEiIiJVYXghIiIiVWF4ISIiIlVheCEiIiJVYXghIiIiVWF4ISIiIlVheCEiIiJVYXghIiIiVWF4ISIiIlVheCEiIiJVYXghIiIiVWF4ISIiIlVheCEiIiJVYXghIiIiVWF4ISIiAMDN9ach95xPdjOIImJ4ISKiYQwwpAYML0RENAYDDCkZwwsREY1yc/1pAAwwpFwML0RENAYDDCkZwwsREY2LAYaUiuGFiIhCYoAhJWJ4ISKisBhgSGkYXoiIKCIGGFIShhciIooKAwwpBcMLERFFjQGGlIDhhYiIYsIAQ8nG8EJERDFjgKFkYnghIqIJYYChZGF4ISKiCWOAoWRgeCEiokkZCjBEiRLX8LJ7925cd911KCwshCAIeO6558Iev3PnTgiCMObS1tYWz2YSEdEk3Vx/mr0vlDBxDS8OhwNLlizBww8/HNPtampq0NraOnzJzc2NUwuJiGiqMMBQomjjeeebN2/G5s2bY75dbm4u0tPTp75BREQUVzfXn8YTAITM4mQ3haYxRc55Wbp0KQoKCvChD30Ie/bsCXusx+OBzWYbdSEiouRiDwzFk6LCS0FBAR555BE888wzeOaZZ1BSUoL169fj8OHDIW+zbds2WK3W4UtJSUkCW0xERB/EFUgUb4Isy3JCHkgQ8Oyzz+L666+P6XaXX345SktL8cQTT4z7c4/HA4/HM/xvm82GkpISdP3ma7CYDZNpMhERTcITFdUAOIRE0XHZHbhrxSfR398Pi8US9lhF9byMZ9WqVaitrQ35c4PBAIvFMupCRETJxx4YihfFh5ejR4+ioKAg2c0gIqIJYICheIjraiO73T6q16ShoQFHjx5FZmYmSktLsXXrVjQ3N+Pxxx8HADz44IMoLy/HggUL4Ha78bvf/Q5vvPEGXn311Xg2k4iI4ujm+tN4oqIacs95DiHRlIhreDl48CA2bNgw/O8tW7YAAG699VY89thjaG1tRWNj4/DPvV4v7rnnHjQ3N8NsNmPx4sV4/fXXR90HERGpz1CAIZoKCZuwmyg2mw1Wq5UTdomIFOaJimr2vFBI02rCLhEREdGFGF6IiIhIVRheiIiISFUYXoiIiEhVGF6IiIhIVRheiIiISFUYXoiIiEhVGF6IiIhIVRheiIiISFUYXoiIiEhVGF6IiIhIVRheiIiISFUYXoiIiEhVGF6IiIhIVRheiIiISFUYXoiIiEhVGF6IiIhIVRheiIiISFUYXoiIiEhVGF6IiIhIVRheiIiISFUYXoiIiEhVGF6IiIhIVRheiIiISFUYXoiIiEhVGF6IiIhIVRheiIiISFUYXoiIiEhVtMluABHFTpZktJ4YQMepAUiSDI1OREqmDgaLDoULLdDo+b2EiKYvhhcilelrcmHXz+ow0O6BIACyPPrnOrMGiz6Sj3nX5EEQhOQ0kogojhheiFTE0e3Fqz+sgc8VADA2uACAzxnA4b80w+sMYOkNRQluIRFR/LFvmUhFTv6zHT5XALIU+djjf2+Do8sb/0YRESUYwwuRSsiyjLrdXVEFFwAQBKBud1d8G0VElAQcNiJSgYBXQsfpAfhcUSYXBIeUBjo8cWwVEVFyMLwQKZgUkPHe86049UoHfM5AbDeWgc7Tdvg9ErSG2DtZu+odOP16BzpqHBAEIH9BGuZszEV6iSnm+yIimkoML0QKJUsy3vplAxoP9E74PuxdXux5pAGXf70yptu991wr3n2mZdR1A+0enHmjC8s/U4x5m/Mm3CYioslieCFSqOZ3+ycVXAAAMtB0sA8955zILDNHdZPGd3rHBJcLHXryPFLzDChZlj65tgHwOPyof7MbnWccAIDc6hRUXJIFfQo/mogoNH5CECnU6e2dEEREPUE3FEEEGvb2RB1e3nuuNeIxBx49N+nwcv5wH958uAEB38h/sPGdXhz5awsuvascxVMQjohoeuJqIyKF6mt0TTq4DPHYfNEdN+BHb6Mr4nGuPj8c3ROfDNxd78Cun9ch4JUAGaMuAa+EXT+vQ3eDY8L3T0TTG3teiBQq6hL/AoIn/jBM6bqo7srviz4tvfGTWvgcAWgMIspWZaDqihykZOmjuu2JF9vCt1kOHnPZV2Obq0NEMwN7XogUqmR5OoQI71CdWUSkHQBkCSi/JCuqxzRZdBHvb0h/sxvOXh8G2jw48WIbXvjmcbSesEW8XcAnoelQX9heJVkKztUJxBCmiGjmYHghUqjqjTkQxPBJYt7mfMy/Nj/0AQJQfnEm0ouiW94sagWkl0Y3N+bCnhNZAgI+GTv/pxbOvtFDVG6bD6df78SxZ1twZkcnnD3eqIbDZAnwuxleiGisuIaX3bt347rrrkNhYSEEQcBzzz0X8TY7d+7EsmXLYDAYMHv2bDz22GPxbCKRYqXmGHDZ1yshaoVRPTBDfy9bk4GFH8nH0k8WYsGH84PXC4CoGTm+8rIsrPlCWUyPu+rWkok1WAYkn4zancGqvpIk49CTTXjma8dw4PFGvPd8K/b/oRF/v/d9iJrI3TtagwidWTOxthDRtBbXOS8OhwNLlizB5z//eXz84x+PeHxDQwOuvfZafPnLX8af/vQnbN++HV/4whdQUFCATZs2xbOpRIpUvNSKj/z3Apx5oxONB/sg+WSkl5hQvTEHhYstw7tGX3RjEeZuysXZfT1w9vpgSNNi1poMpOYYYn7MnKpULPxoPo4/3xbzbWUZaDrUh8XXF+DgE004/XrnyM8Ga+xJ/ggTdBAMaJWXZUUVcoho5olreNm8eTM2b94c9fGPPPIIysvL8dOf/hQAMG/ePLz11lt44IEHGF5oxkrNMeCiG4tx0Y3FYY8zpeumrHjc0k8WIaPEjGPPtqC/2R28MoqJwUBwtdBAh2dUcAlpnPsUREBv1oQfDiOiGU1Rq4327duHjRs3jrpu06ZN+MY3vhHyNh6PBx7PyJJNmy3yhEGiRHB0edFZawdkIHt2yoR6QZKpbHUGylZnwNHlhc8TgN8r4eXvnAp7G0EEMkpNqH+rO3KNGiEYuFy9vmCIAQAZSC824ZK7K0atXJJlGd0NTnTXOyAIAvLmp8FaYJz8f5KIVElR4aWtrQ15eaO/Oebl5cFms8HlcsFkGjvpcNu2bbj//vsT1USiiFx9Pux/9BzOH+kf1atQtNSK1Z8vhTkjuuXESpGSPdLe3Dmp6DxjDxlKZAmoviIH9Xu6I96vKAqYtTYDZSsz0XHGHrz/qlRkVZqHh8MAoK/JhT2PNIypP1OwMA2r7yhD/3k33DYfTOk65C+wcKiJaAZQVHiZiK1bt2LLli3D/7bZbCgpmeCEQ6JJ8tj9eOX7p+Do8o4ZDmk51o9XvleDzffPhdESXd0VpVl1ayle/t4pBLzSuAGm8vIs5M5NRfO7/RHvS5ZkGNN0yJ6dguzZKeMeM9Duxivfr4HfM3ZTytYTA3huy/FRv2ejRYulnyrC7Muzo/4/EZH6KCq85Ofno729fdR17e3tsFgs4/a6AIDBYIDBoK7ueJq+Tv6zHY6u8ZcCyxLg7PHi/ZfasezT4eevjCfglXDuQC/Ovt0DryOA1FwDZq/PRt7c1FE9FfGUXmLC1ffNxcE/NaHt+MDw9YbU4ByV+dfkQRAEzFqbifdfag9zT8HJvWWrM8Iec+xvrfB7AuP39Iwz/8Zt8+Pt351DwCthzodyo/kvEZEKKSq8rF27Fv/4xz9GXffaa69h7dq1SWoRUfRkScbp7Z0Ri6+d2dGFpZ8qiml4w97hwWv/dRqOTu/wJNfuegfO7u1ByfJ0XHJXOTS6xJRtSi82YeO/V8Pe6YGtzQOtQURWhRka7cjjZ5aZUbTUipZj/eP/PgSg4pKssPOAfK4Azu7vmdAWCYf/fB7lF2dBz6XWRNNSXD/t7HY7jh49iqNHjwIILoU+evQoGhsbAQSHfG655Zbh47/85S+jvr4e3/zmN3Hq1Cn88pe/xF//+lf8y7/8SzybSSogyzLaTw7gwGONeOuX9TjyVDNsre5kN2sUn1uC1zF2eGPMca4AvA5/1Pcb8Et4/Uen4ez2Bq8Y7HEYOqk3He7Dwf9tirW5k5aaY0DhIgtyq1NHBZchl9xVjoKFFgCAoAEE4YIaNasysPr20rD37+rzDS+vjlXAL+Psvp6J3ZiIFC+uPS8HDx7Ehg0bhv89NDfl1ltvxWOPPYbW1tbhIAMA5eXleOmll/Av//Iv+NnPfobi4mL87ne/4zLpGc4z4MeOB2rRdcYRXMEiB0+EJ15sQ/XGHKy4uQRihEq08ST5ZUAENDoh6uXE2mj3LQJw/lAf7B3e0AfIQO2uLiz5eCGMVuXMpdEZNdjwr7PRXedEw95uuG1+mDJ0qLgkK6odridToE4QBQy0T3zjSCJStriGl/Xr10OWQ3+Sj1c9d/369Thy5EgcW0VqIksy3vhpLXoGdxge6m0Yelmdfr0TOqMGF91YlNB2SQEZdbu6cOrVjuE6KNmzU2Cy6uDqC72DsyAC+fPToDVGf2JufKcPgjDyfx6PHACa3+1H5WXKmagqSzIkv4ysSnPICbnhmKw65FSloLPWEVUgHP3gMrRG7n5CNF0pas4L0Qe1Hrehu84R9piTL7dj/ofzYEhJzMtZ8svY9bM6NB/tH6lPAqCrNnw7gWD4mv/h2IqveV2BsMEFACAAPpcy9gHqPuvE+y+1ofGdPsgBGYY0LaquyMa8TXkwpMX2HC26vgBv/Lg25jbIElC6Ij3m2xGROvCrCSna2X29EXdWlvwyzh/qS0h7AOD9f7aPLAWOsUdg9edLUbDAEtNtLHmGiL8DyEBqXvJX3TUe7MXL951E44FeyIHgL8cz4MeJv7fhH985CWdvmOGvcRQutmL1HWUQRET+HQwSRKBgoQUZ0W4wSUSqw/BCiuax+yKuNhFEwGOf4MzOGEmSjJpXO2IfxgAwd1MuqjbkxHy72ZdnR/wdmNJ1KFwUWyiaau5+H956uAGyPLay7tAy8X2/ORvz/Vatz8bHHliEhR8tQOESC4qWWrH4EwWwFgcr7A6tEh8KN5nlKbjk7vJJ/E+ISOk4bESKZs7URywzL0uAOXP8iaqOLi9qd3Wh77wLGp2AoqXpKF2ZPuFlxY5OT9g5LeH0nHVO6HYZZWZUXZGNM290hTxm5S0lSa8sW7urC1JADhnsZAloPT4AW6sblhhL+5sz9Vjy8cJR1y38cAHOH+lD/ZvdcPX5YM7Uo+KyLBQtsSb9d0FE8cXwQopWeWn4kzYA6Ewiipelj7n+xIttOPLX5uBkVyn4zfzsvl4c/osOV/5bFdJLxi98GE7EuSfhbitN/Marbi2F0aLDyX+2w+8ZSXLmLB1W3lyKkuXpE2/YFGk7ORBVj1RHjT3m8DIeUSugdGUGSleGL3RHRNMPwwspWlalGaUr09F4sC/kiXHpDUVjlh7X7e7CkaeaAYwEjqHeG3e/D69tO42P/GgBRK0AnzsAQ6o2qt6YlGw99CmaqOq5XEgQgeyq1JhuM/r2ApZ8ohDzr81D63s2eJ0BpGbrkTcvDUISl4mPEmU2C7cCkYgoGgwvpGiCIODir5RD91gj6t4MbvYniALkgAyNXsRFnyoaUwZelmQce7Y15H3KUnAS6cvfO4WBtmAtEI1OQMWlWVh4XcGojQg/SKMVUb0xBydeaIupF0aWgxsWTpbOqFFsT0NOVSra3x+I+HvJmT1+iJMCMpqP9KP1uA1SQEbmLDPK12VCZ2KVXCIajeGFFE+jE7H2zllY/PFCNB3shdcRQEq2HqWrMqAbp15K91lncGPECIaCCwAEfDJqd3ah8UAvNn1nbthhjYUfKUDbiQF01UVRf2SwaN3Km0uQpoDVQPE0e0M2jr/QGvJ3IohAVmXKuMN1fU0u7PifWji6vBAGn9LancChJ89j3RdnRdwDiYhmFq42ItVIydJj7qY8LP54ISovyx43uACIeUhniCwBXmcAb/2qIexxWr2IjVursfhjBTBaRvJ/So4eOdUpEHUjwzjZlSlYv6VyRmwSmJKpx5o7ygCMXdYsiIA+RYuLvzR2FZCrz4fX/rMGzp5g4JQDGN4WIOCV8ObD9Wg7YYtr24lIXdjzQtNOSlboYZ9IZAnoaXCi+6wTWbNC1wnR6kUs/lghFn60AO4+HwRRgNGqhSAI8LsDcPX7oTOKiirXnwiVl2XDnKXHib+3oe1EcNdpjV5A5aXZWHBd/rjPzentnfA6Q+wcjWDn1bt/a0F+jPVxpspAuwfn9vfAYw8gJUuHWWszYbTMrOeVSGkYXmjasRYakVVhRneDc0L1WCAAXWfsYcPLEFEUYM4cfULWGjVIi6H8/3RTsMCCggUWeB1++DwSjKlaaMLs5VS3uyv8UngZ6DztgKPbO6lgGquAV8K+353D2X09wZ4kQYAsyTj05/NYeF0BFn+8AIKgkMnSRDMMwwtNS8s/U4zX/vN0MLvEvC8ORpX9v1DAK+HcgV407OmGe8CP1BwDKi/LQuESa1I3h4zGExXVAICb608n5PH0KVroo9jSyGOPbodt94A/oeHlrV81DFduDoar4AtJDgDvPdcKUSNg0fUFCWsPEY3gnBealnLnpOGKf62CKT3YvR/85hz97fPmpI25ztHtxYvfeh97f30WrScG0HvOhfOH+7Dzf+qw/b9Ow+dOTJXfiRgKLh/8uxJEO7Rmsibuu1Z3vQNNB/vCrpx674VWeB3RBS8imloMLzRtFSyy4GMPLsKGe2Zj6aeKsOJzJdh8/xxodKFTjCACudWpY1bESJKMN358BvbOwRVKH6gd01Fjx77fnY3D/2LyhsKKkFkMIbN41HVKMPvy7LDBUhCB/AVpMGckrtelfk9P5D21fDIa3+lLSHuIaDSGF5rWRFFA0VIrFlybj7lX5SKrIhWX3l0BQTP+ihijVYeLvzJ2RUzLMRv6m90h52bIEtC4vw/2Ds/4ByTJhcFlyIV/V4LqK3JgSteNHxaE4GXJJwrH+WH8uPoi76kFAPYuZT3fRDMFwwvNOMXL0rH5u/NQtjoDwuAeOPoUDeZfk49rvz9v3CJ1TQcj724NAWg63Df1DZ6g8YLLECGzWDG9L4Y0La769hxYi4K9XYKI4efFkKrFhi2zkTOJ6sQTYbRooxpm7Gt0xb8xRDQGJ+zSjJQ5y4xL/r8KXPxlGQGfDI1eCLtyxO+WIlaOFYTgcUoQLrgMETKL8QQSN4E3nLRcA6794Tx01NjRdmIAkl9G5iwTipenQ6NN/HesWesycPr1zojHddY6IEuycrZoIJohGF5oRhNEAVpD5BNPWr4huMFjmAAjS8Hjki2a4DJESQFGEATkzU1D3tyxk6UTLb0ouk07PQN+eB0BGNL4UUqUSBw2omnH1e/D6dc78d7zrah/q3tKVgHNviw74hwIvVmT9N2dYwku492OgmLp7RHDTAAnovjg1wWaFmRZhrPbh6P/14yGvT2AHJw7IUuA5lERS28oxNxNuRMuKpaaa8DCj+Tj+AttIY9ZeWtpVDtTx8tEg4uQWQy55zyeqKhWRA+MEmj0InLnpKLzjD105d/BvZpCbVNBRPHD8EKqd3ZfD0682IbeD0yeHDrpBLwSDv3pPABg3tV5E36cJZ8shD5Fi+PPt8LrHOnNMWfqsPyzJShblbzNAycaXIYwwIw1/5o87HzAHvLnshQ8hogSj+GFVO3Y31pw7NnWqFaGHH26BbPXh97QMRJBEDD/mjzM+VAO2k4MwGP3w5ylR96c1KRO2JxscBkykwKMLMtoe38A9W92w9njhdGqQ/m6zFGVkouXpWPJJwvx7v+1DPfiASM9eos+VoDSFdztmigZGF5ItbrrHcHgAkS1BUDAK6HxnT5UXpo1qcfV6EQULbVO6j6m2lTVbpkJAcbnDmDXg3VoOzEwHEQEETj3di8yy8244t+qYBycgLvoowXIn5+GU691oOOkHbIsI29uGuZ8KBe5cxK7fJuIRjC8kGrVvN456htxJIIGcPZ449uoBHuionrKi85N9wCz99dn0f5+cMfrodfO0J+955zY9UAtrvqPOcPzo3KqUhNeZ4aIwuNqI1KtcJMpxyNLgCFl+uT1eASXIUrcRmAq2FrdYfcskiWg84wDnWcciW0YEcWE4YVUK1LRuA8SBKBkZXpc2pJo8QwuQ6ZjgGk82BexUrKgARoP9CamQUQ0IdPnayjNGI4eLw4+0QR7ewz7yghA1RU5MEW5g7GSJTJMKGkIyePwo353N87t74HXJcGSb0DVhhwULrZEPWHa5wwEU2y4SVIy4HMpd4dwImJ4IZVx9vnw8ndPwd3vi+l25RdnYsVnS+LUqsSZqpVFsVBCgOk558T2/zoNjyMwnDsG2tw4f7gfRUstuOxrlVHV2EnN1UMORO6yS8kxwD3gh6PbC51BHKywzGJ0RErB8EKqcuyZZrj7o9vxV5+iQdnaDFRfkYuMkujKvStZMoLLkGQGGJ8rgO0/OhOsrXNB7hh6DTS/a8OhJ89j1a2lEe+rbE0mDj7RhIAvdICRZaCr1o73nm0ZfgxLgQELP1qAiosnt1KNiKYG57yQosmyDFurGz3nnHB0e1C/pydicBG0ApZ/phgf//lirL61jMFliiRrDkzD3h54Bvyhn3cZqN3ZBc+AP+J96U0aXHRT+N+hqBHQ+p5t1OPZWj3Y+8hZvPdcawwtJ6J4Yc8LKZIsy6jb1Y0TL7ZhYHBuS7TLogUZmLd5+lQ+VUJwGTLUA5NITYciT56V/DJaj9swa21mxGPnXpULUSvg6F+b4XWMzG3RGkToUzVw9vhCTol595kWlKxMj3rjRiKKD4YXUqQjTzXj/ZfaR10X7bLo6bRRnpKCy5BE70Ttd0f3xPs90a+br74iB5WXZqHlmA2uXh8MaVqkZOvx8ndPhb2dIAJntndi5S2Rh6iIKH4YXkhxOmvtY4JLtAQRKFmWPrUNShIlBpchiQww1mITuuocEcOrpdAY0/1qdOKoXcDr3+qOeBtZArobnDE9DhFNPc55IcU5s70zYi2OUGQZmHt17tQ2KAmUHFyGCJnFCZn/UrUhO3xwEYITanOqUib1OKImuh67aI8jovhheCHFieZbNoBRmzEKYvCy7kuzkFU+uZNYsqkhuFwo3gEmqzwFVVdkj/9DIfi8r769bNJLmfPmpUUVmg2pGrj6YluqT0RTi8NGpDiiNrpMnTs3Fc5uH0QNULjYiuorc2ApiG3oQGnUFlwStYR61a2lSMnS48RL7cFCc4MySkxYeUvplGySaErXoWxNJs69HX5FW9Ohfpw/cgwVl2Zh1S2l0Oj5HZAo0RheSHEKF1vQ3+wKewIRdQLWf70S+mm2VxGgnuAyJBEBRhAFLPxIAeZdnYf2Gjv87gBScw3ILDNP6eOsurUUthY3es6Gn9ciS0Dd7m64+nzYsGV21BV+iWhq8CsDKU71FTmDJdzHJwhA5WVZDC4KkqgaMBq9iMJFFpSuzJjy4AIAerMGV/3HHKy6rRTWIkP4g2Wg5V0b2gZ3qCaixGF4IcVJzTXg0rvKh+exDBvMMznVqVj+afWX+h+i9uAyZLps5KjVi6i+MgerbiuLeKwgAmd2diWgVUR0IYYXUqTSlRm49gfzB3tYNNDoRWSUmLD6jjJceW8VtIbp9dJVe3AZMl0CDAA4urwRj5ElwN4RwwahRDQlpk+/O0076SUmrLljFtbckeyWxM8TFdXTJrgMUcJGjlNBb9ZEPkgI7qFFRIk1vb6+EqnIdAwuQ6ZDD0z+Agu0xggfkTJQHsWWBDS9uXsDaNrtRP3LdjTvc8HniL7aM01MQsLLww8/jFmzZsFoNGL16tU4cOBAyGMfe+wxCIIw6mI0qnv5K9EHqSG4+N0S7K1+OLv8kOXQuzCHovYAozWIWHBtfsifCyKQkqVH2WqGl5lK8suo+T8b3v5RD+r+4UDTLhfOPG/H3h924+zrjgm9byg6cR82euqpp7BlyxY88sgjWL16NR588EFs2rQJNTU1yM0dvxKqxWJBTU3N8L8nW3yKSEmUfjL39AfQ8IoD7Uc9kAdLqpiyRJSuNyN/pTGm96Pah5AWfiQf7gEfal7tHN4YdOhPc5YeG/+9etrNv6LonXp6AB3veoY38hzKKnIAOPuaExCAWVequ2imUsU9vPzP//wP7rzzTtx+++0AgEceeQQvvfQS/vCHP+Dee+8d9zaCICA/P/Q3HqJoyLKM9lN2dNc5IIgC8uanIWvW1C+vjYXSVxa5ewM4/HAfvA4JuKDn29UtoeYZO5ydAVReG1tBODUHGEEUsPLmUlRtyEHtzi7Y2tzQGTUoXZmO4uXp0IxTUFGWZficAYgaAVoj58NMV/ZWPzqOhp+sfW67E0XrTNCZGHCnWlzDi9frxaFDh7B169bh60RRxMaNG7Fv376Qt7Pb7SgrK4MkSVi2bBn+8z//EwsWLBj3WI/HA49n5AVks9mm7j9AqtV91om3Hq7HQJsnuNxaDn4ryq4045K7K5CaHaGGRxwoPbgAwJnn7WOCy4WadruQvcAA6yxdTPerlgDjdwfQsLcH9Xu64bb5kZKlR+Xl2ShdmY4Vnwu/PN/vlVDzagdOvdoBV29w+4Ds2SmYf00eSldmJKL5FAPPgITmPS60vuOCzyFDaxKQv9yI4otNMGZEDp1th9zDvXChyAGg85gHhatNU9hyAuI856WrqwuBQAB5eXmjrs/Ly0NbW9u4t5kzZw7+8Ic/4Pnnn8f//u//QpIkrFu3DufPnx/3+G3btsFqtQ5fSkqmT/0Pmpj+Fjde+0HN8BJWWRrpzu1ucOKV79XAbUvs3jRqCC7uvgC6T3lDBhcAgAg073NN6P6VPgfG3uXB37/1PvY/2ojOMw4MtHnQ9v4A9vyyAa/cXwOP3R/ytn53AK//52kc+WvzcHABgO46B3b/vB5Hn25OxH+BouTs9OPggz1o3OmEzy4DMuB3yji/x4V3HuzFQHPkzwfvgIRIU1oEEfDYOHk3HhTXl7V27VrccsstWLp0KS6//HL87W9/Q05ODn7961+Pe/zWrVvR398/fGlqakpwi0lpjj3bgoBPGvcbkSwB7n4fal7rTFh71BBcAMDR6h8euw9JAmxNEw9+Sv0dyJKMHT+uhbN7sLbL0O9h8M/eRif2/Koh5O3f/VsruhscY35/Qye34y+0oe0Ee4WVQJZlHH/cBp9THvt6l4CAR8Z7j9kgBcK/GXRmMVwh8MHHAvQpijvNTgtx/a1mZ2dDo9Ggvb191PXt7e1Rz2nR6XS46KKLUFtbO+7PDQYDLBbLqAvNXF5XAI0HesN35UrAmR2JCS9KDy6yJMPnkiD5ZQia6CbiipPcx0fILFZc70vrcRv6W9whXzeyBLQcs6G/eWyvk98j4cyOzrCvOUFEQgMzhdZX74OzIxC6h1EGvDYJ3e+HL1KYd5Eh7HMOBLcyyVmU+CHqmSCu4UWv12P58uXYvn378HWSJGH79u1Yu3ZtVPcRCATw3nvvoaCgIF7NpGnE3e+L+IESPM4PWYrvMkYlBxevXULdP+x46/5u7PluN3Z/uwvn9zghRBjqF0Qgc45+0o+vtADTdKgvqv970+H+Mdf3t7jgd4d/0ckS0H6KeyApQV+tb/S2I+MQRKC3Lnx4SSvRIqNaN7xtyXgK1xqhT2PPSzzE/be6ZcsW/Pa3v8Uf//hHnDx5El/5ylfgcDiGVx/dcsstoyb0fu9738Orr76K+vp6HD58GJ/73Odw7tw5fOELX4h3U2kaiHazRq1RjOtOwEoOLu7eAA7+rBdNu10IuIfWdgK9p33DS6PDKVw7NZMPlRRgAl4p8pCZIASP+6BoMzBLfihCtLVXIr0XBEHAgs9ZkFkdnLwuiAieUQfPqgWrjTGvzKPoxX2p9I033ojOzk585zvfQVtbG5YuXYqXX355eBJvY2MjRHEkQ/X29uLOO+9EW1sbMjIysHz5cuzduxfz58+Pd1NpCsmyjO4GJ1y9PhjTtMienRLXsDDEmKZF3rw0dNQMhOyBEUSg4uKsuLdFicEFAE79dQBe+9iT9Zjfl4CRYwafupLLTDCmT+13HiWsQLIUGCP22MkBGZaCsQUzrUUmaI1i2N4XQQRy5/JEpgRpxTrIUvhJ57IU7FmJRGsQsfjz6bCd96HjqAc+pwSDVYP85QaYs7n7Tjwl5Ld799134+677x73Zzt37hz17wceeAAPPPBAAlpF8dJ0uA+H/3weA20jS9jNmTosvaEIFZfEPzQs+lgBXt8WooteAEStgLmbxi+QON052v3oq4884TZrvh7uvgCcbYHgpNPBENO404WWA27M2piConWxFawbj1KWUJetycTRp1vCHyQAJcusY67WGkRUbcjGqVc6ws6ZmXvVzHzNKU3WPD10qQJ8jnEm7A7S6IG8pdFXdrcU62Apjq18AE0OB+NoSjXs68GuB+pGBRcAcPb4sPfXZ3HqlfYQt5w6+fPScMlXyiFqhWCPgYDhMW6dSYMr/q1q3G/QM0H/uehWCkk+GaWXm4Mn4w98wPudMmpfsOPc684paZMSllB31NgjHyQDnWcc4/5oyccLkVmeMmb+w1C2W3BdPvIXcDGBEogaAfM/Ywl+Jnwwew9+Xsz7tAUaPSu7Kxn7tWjK+L0SDjx6Luwxh/58HrPWZsJoie+3lFlrM1Gw0IK6N7vRXe+AIAJ589JQvjaTVU+jIAdknHku/An97HYn8lcaYUyf/O8z2T0wnaftEQuOCSLQcdqOgkVjQ4jWqMGHvlWNUy+3o+a1Trj6giExqyIF867JQ9kqFqlTkoxKPZbdlY6zrzmDtY0GA3rGbB1mbUyJuQgjJR7DC02ZxgO98Lkir7qof7Mb86/Nh6vfhzNvdKJhbw+8jgBSsvSo2pCN8kuyoNVPvlPQkKbF/GvyIh84g1hLo/hQFgBtigi/K3RRtiGt77hR/qGp2bslmQEmqkmcEb6Ia/UiFn6kAAs+nA+vMwBRK0DHoKxYaUU6LLrNCp9DgtcuQZciQp/KwQi14DNFU6a/xR2xVoggBI/rOevE3//9BN57thUDbR54BvzoOefE/kcb8fJ3T8EzEPnESbFLydcGv1WGeecLAmDM0ERcTgoArq4olifFIFlDSDlVqVFM2AVyqiIHNUEUYEjVMriohC5FREqelsFFZfhs0ZTR6kVErpcdnDD7xo/PwOcKjD588O/9zS7s+XXoaqY0OXNvTAtW/fzgu39wDsDcT6XBkCZGfCohABrd1M8LSEaAmbUmEzqzJmTviiACKTl6FCy0wNnjxbvPtODVH9Tg1R/U4Mhfm2HvDL9BH8WfLMnoPunBuR1ONL3phKOdX4CmMw4b0ZQpusiKd58Jv2JDDgA6owZuW+gPFlkCWt61wdbmhiV/Zk6sjSdTpgbLv5aOpl0utB5wITBYiyuzWo/S9Wakl+vg7Aqg7qXxJ6cOk4DshZMvWDeeRA8haQ0iLvtqBXb8tBayJI/qhRHE4M+X3VSE7f99Bm0nRq9k6zxjx4kX27Dm82WYvT477m2lsXrOeIMlAGxSMITLQN2LDmRU6TDvJgt7VaYhhheaMpllZuTNT0PHqfFrrAgiYMrQo2Ffd+Q7E4Ll2Ble4sNg0WD2damouCYFfpcMjV4YtbrCnK1B9gI9ut73jrucVBABU7YGmdXxCS9A4gNMwUILNn93Lo6/2Da8xYSoFVB+cSbS8gx486HxewOHXutv//5csHeGq4oSqv+sD+/9oX+kp/CCz57eOh/e/W0flt2VwdVD0wzjKE2pS++ugLVosALr0GfF4J+iRoCz2wtXb+TuXEEAJB93Y403USNAnyqO+8E+91NpsJQOfr/5wHNpSBex+PPWuBceTPQQUkaZGZfeVYGbfnsRPvmLxbjxt0tRtSEncg0YBAPdib+3JaCVdKG6f9hH1SIaRQIcbQG0H3EnulkUZ+x5oSllTNNi8/1zcW5/L2p3dcHZ44PRqoXsD1bcjZYsAeklU1OGnsZydvnRut+NgRY/RA2QOdeA/IsM0JpGvs9ojSKWfikdPae8aH3HDXdvALoUEXnLjMhdYojLfJfxJGMVkkYvQjO44u3ky+0QhMjTuWQJaDsxAK8rAL2Jk3UTwdUdgO1cFKviDrhRuJqfJ9MJwwtNOY1ORMUlWcPVdO2dHjy35Xj0dyAEK/IWLGT3ezyc2+FEw8uOUeX/e2p8aHjFgcW3W0fVuBA1ArIXGJC9ILk74yZzGfX5w/1RbfY5xO+RGF4SxN0X3Wo3R7sf3gGJmyROI3wmKe7OH+6LWCNjiCAAoihg3ZfKE7IX0kzTdsgdDC7AmG72gEfGu7/vi/qEkGjJWkYt+aNPLlqDCEMqg0ui6MzRncIkH/DOAz2wt0XupZElGb11XrTsd6H9qBs+J4evlYg9LxR3PrcUrF4axTkxZ04qlt1YjOzZU1P4jEbIsoyz28OsIJKDH/It+1yo2KzMTQSHemASyVpkRN95dxS7TgOVl2dDo+V3wkRJydfAlC3C1RU5YPhcMt57tB+rv5kJMUQ9qp4aL04/OwB378j9CRqgcHCHaFHLL1RKwXcZxV1ariGK7eWB6o05uOr/zWFwiRNHewDu7kiV2ID2o8quWSJkFie096V6Y27k4ALAZNVhwXX58W8QDRMEAbOirfAsAZ4+Cd0nveP+uOe0F8ce7Ye7b/R7RA4AzfvceP9JW3SVmCkhGF4o7kqWpwcLgIUhy8C8zRMv5R/wS3AP+CH5+eESSsAd3e8m4FH+7zCRAabysizkzk0NO/SZU52CTffNgTmde+IkWt5SI2ZfF12AEcRg78oHybKMM88P1u8Z7+UvA10nvFHtyE6JwWEjijuNXsTKm0uw99dnQx4z/9o8pOXGPim0t8mFE39vxbkDfZAD8nBdjoXX5SMtjzViLmTMiOK7yuDWAGqRiAm8Gq2IK/6tCkefbkbtji74PYPfzAUgq9yMZZ8tRl51WlzbQOEVX2KGqyeA5r2Rh/fkwNgDbOf8kYeeRKB1vxsZlfGrbUTRY3ihhKi4JAuiRsChP5+Hq3fk24vOFNzMbv61sfe6tL0/gDd+fGZURVTJL6P+zW6c29+LD32rGlnlHIIaYrBqkFmtQ88ZX+gPeBkoXKOO0JfIFUhavYgVny3Bkk8UorvBCTkgI6PEBKOVPS1KkV6uR/Oe8PVcZBlILRx72nP1RDEhTwKcU7yXF00cwwslzKy1mShdnYH29wfg7PFCn6pFwULLhHaQDngl7P55HaSAPOZELEtDP6/H9T9dyFVLF6i4JhV9D/dC8mNsgBGCH+x5y9QRXoDEL6HWGTXIn8deFiXKmq+HLkWAzzn2M2GIoMG4r2+tIbrPCK2RnyVKwTkvlFCiKKBgoQWVl2WjZFn6hIILAJw70AuvIxDyQ0qWAEeXFy3HbJNo7fSTWqDF0i+nw5z7gaEhAchZqMeSO60JKz43VZK1hHqyZFlGX5MLHTV2OLrGn0QKAD53APVvdeP4C604vb0T7n7OuxiPqBEw70YLBAFj5ycN/nvOx9PGXV6dUaWHGMVoUO6S5NY7ohHseSFV6jhth6AJv/xa0AQ3zStaak1cw1TAUqzDyn/JgK3RD3trsMJuRpUexnT1zHX5oGQWsZuIhn09OPa3Fgy0jazsypuXhmU3FSGrYmSo89SrHTj612b4PYPlBiTgnccbUX1lDpZ/poRLdz8gc44eS7+UjvpXHOi/YHJtaqEW5VeZkTV3/PCh0QsoudSMc9tDVAEXAX2KiNyl6umVnO4YXkh1PA4/bK3umKqe0miCIMBapoO1bGJzNtx9AfidMvRpomKqlqolwJz8ZzsOPTm2Vk1HzQBe+X4NNt5bjdw5qTj1SgcO/m/T8M+HXu+yBNS83gmfW8K6L85KUKvVwzpLh4u+lA53bwAemwRdighzduRgPmujGR5bAG3veIaD4lAVan2qiCVfsEY9vETxx/BCqiFLMt59pgXv/6M9qiXRcgDInaPMYmtq1XPGi7OvOUbtJ5M5R4fyq1KQVpz8yatKDzCOLi8O/Xn8InuyFBxK2vubBlzz/Xk4+nRz6DuSgfo3uzF/cx73AAvBmKGJaeWcIAqY+0kLCtf40LrfDWdnABqDgJxFhoTu5UXRYXgh1Tj05HmceqUjqmMFETBn6bk/0hRqP+rGyb8MjLm+57QPvbV9WHyHVRHLSJUcYGp3dYXf5FEG7B1eHH+hbWRJdgiCCNS92YXlnymZ+obOYJZiHSxhgrjfI6HnlBc+lwyjVURGtT5kxV6KH4YXShrJL8PnCkBrEiOWVLd3eGIKLlqDiMu/VsmVRgC8Dgl9dT5IfhmpBVqkFsT+tve5JNQ8PRCygJcsASf/PIA1W0OXXk8kpQaY3kZnxOFOQQD6mlwjQxchyDLCTvSlqSVLMs5ud6JplxPSBXOmdSkCKj+cinwVrdKbDhheKOEG2t048WIb6vf0QPLJEDQCylZnYOGH80N2gde91R3xwxwIBpfZ67Mx/9r8CRW9m04CXhlnXhhA+yHPqN9bWrEWcz6ZFlOIaT/sCS6vDkUGvAPBb6TJ3oF6iBIDjKgVR+3mPR4ZgM6kCd07M0gQAH3q9P8Id3YF0HrABUebHxq9gKx5BuQsTvwwTu3f7cEieB/gc8g49dQAZAkoWMEAkyjKmGlHM0b3WSde+vZJ1O3uhuQLfjrLARnn3u7BP+47ibaTY4clgOi+YQoaAfM252H17WUzPrhIARnH/tCPtoOeMYFvoNmPI7/shSOKHXZHbuOL+GkhiMH7VhKlLaMuXGyJvE+SDMzdlBNc8hvuMClYO2m6kmUZDa85cODHPWh604WeGh86j3tx6q8D2P+jHthbE/dac3T4xw0uF6r9ux0Bn/K31pguGF4oYSRJxu6f1SHglcacUGUpOIw09PMP0qdEMfFOkmFQwDfRJyqqh0+aydLxrgf9DSEq6cpAwA/U/cMe9f2JohBua59hggI/UZQUYGatyYQhTRsymAgiULTUipyqNFRdkRNyPyVBBLKrUpA3Vz0T0mP9/bfud+Pc64NLl4c+EgZfz167hHd/2wefMzFLDtsOuiOeLQNuGd3vK3tT0+lEgR81NF21HrPB0eUNPfQjA15HAOcO9I750azVmRGHjGQZKFuVMfmGToISTpAA0LLPFXYjQUhAT40P7r7oyp1nVOki//6lYL0YJVJKgNEaRFzxr7OhNWlGB5jBv1uLTMPLn1d8tgQVFwd7VgQxeMxQOMyqTMGGLbMhROqeUYih33u0oX5ofknoAwCfU0brO+F7Q6aKqzswEqBCEETA1cP6DYmS/K+pNGN0nom+sFzFJVmjrs+qNCN/fhraTw2MfxIVgPK1mUhN4nBRrB/Q8eTqDl19+ELunkBUxemyFxigT3PAa5fGv18xWL3XUqrcj5ShOTDJllWRguv+az7OvNGFhj3d8DoCSMnWo+qKHFRckgWtIZhQRK2AdV8qx7zNeajb3R3cUiNFi1lrM5E3L3XaBhcAsDX64bVFSstA+xE3Si83T6Z5UdEaxcgTqKXotxmgyVPuJw3RBQRBwGVfq8DOB+rQUWMf/iAZ+rN4mRVr7ihLWvuUElxkWUbX+174PdGNvWui/LAVtQIW3GzBu7/tG7XSAgAgAIY0EQs+Z1H8CVXILMYTQNIn8Joz9FjyiUIs+URhxGMzSs1Y8bn4n6DjYaLvC787uh4Mvysxc0xyFumDQ0fhCED2AmX2PE5HDC+UMDnVqZBfCH+MHAByqsYfx9enaPGh/1eN9lN2nN3bA7fNB3OGHhWXZiG7Mnm7RyspuJx+1o7W/dF1pRusYtQrjhztfpz4X9vY4ALAUqLFwlss0KepY3sBpQSY6W4y74uoissJgCkzMa+5zGo9UvI1cHYEQvb85i83wGBVx3tgOmB4oYQpXGRBSrYezp4Q816E4MTccPNWBEFA/rw0xezsq5TgAgQnFUYbXACgdIM5qjo4fndwcqTXPv63XFujH60HPSjboJ7eAQaY+Jrs+yIlT4vUIi3sLf7Qw58yULA6MUuTBVHA4jusePe3/XB2BIaXuw/1/GbN1aPqemV8Js0UnLBLCSOIAi77eiW0BnHMqhRBDA5NXPa1SmgmuNN0oikpuMiyjKbdrqiPL7nchMI10X3wtx/2wDsgh51D07TTqcplosmewDsdTdX7YvZ1KePvEI3gdZZSLXIWJW6Om8GiwYpvZGDBzRZkL9TDWqFD3kUGLP2yFQtvtXD7gARjzwslVNYsM675/nyceKkN9W8Fa72Ig0XqFoQpUqc0SgouQLBQlrMjupVDy+5Oh6Uk+n2I2t+N3Jvjd8vor/chc456xvyVWMQulP4WN2pe7UDjO73weyVY8o2o3piD8oszodGKsHd4ULO9E40HeuH3SLAUGFB9ZS7KVmckpeLxVLwv0sv1WHyHFaeeHoCnTxop7icE56DM+WRawv9vokZAzkIDchbO7DpSSsDwQgmXlmfAms+XYdUtpfC5A9AaI28PoCRKCy5AsNBftGLdHiDaSZHRTrJUEjUEmPOH+7D75/WQZXl4uLXnnBNv/+4c6nZ3YcG1+dj9i3rIgZGfd9n96DzdgNpdXdhwz2xoE9SbOdU1jjJm67Hm3zPRW+uDs8MPUSsgc44+pg0XaXpSzxmDph1RK8CQqo0quPhcAZzZ2YXDfzmPY8+2oLcp+iGSqaTE4AIA+jQRupTI30JNORqI2ti+rZqyNFF9UhgTNHlyqimlBsx4HD1e7H6oHtIFwQTA8BBe52kHdv2sDpJ/9M+HthZoPzmAwyF2sZ5q8SrOKIgCMqv1KL7EjMI1JgYXAsDwQipwZmcX/u/uY9j/+3M49XIH3nuuFS99631s/9FpeOyJKxGu1OACBD/gi9aawhemA1B8cezDcoWrjRELdJlzNUgrVm9HrlIDTO2Oroi9arKEsJNaa3d2weuI7/tECVWlaWZheCFFa9jTjf2/Pze8ZcCF30Db3h/AG/99BlIMQyYTpeTgMqTkcnMwQISY4JhRrUPBqthXZ4h6hO7VGZxQWfVR9RRNC0WJAab5aH/EDRojkfwy2mui3woiVkr6fdHMod6vSjTtSZKMw081h/y5LAHdDU6cP9yH0pXx2xZADcEFADR6AUu/mI6z2x1oeduNgDt41tOlCChaZ0LpenPMExxb3nbh9LP2kD06BouIOZ9IQ8Zs9UzUDUdpc2AC/qmZRyTFaSVYIt4bsiSj/6wPPocMvUWEpSQY0Dvf86J5jwu2Jh8EAbBW6FB8sQlZczmZdiZgeCHF6jhlh6t3nKpoFxBEoG53V9zCi1qCyxCNXkDl5lSUfygFrq5gPQpTtmZCqzIcbX6cfm7wG3uIc1/GHJ2qVhhFQ0kBJrsiBbYWd8R9pSKJxyq+RLw3Wg+60fCKY9RWAcYMEcZMEX11/uEVSDKA3lofek/7ULrBhIqr1bNhJU0Mh41IsVx94YMLEOx9cfZEPm4y1BJcLiRqBaTka5GSp53wctLmSJs7Amg/5IHPpb5VRpEoZQip6sqc8MElwvMjiEBOdQqshVNbzC0RwaXpTSdqnh4Ys8eRu1cKBhdgdKgePKxxhwtd3N152mN4IcUypkXRMSgARmv0NUtiMdMnIfae8UWcqCsHgL66+IbHZFFCgMmuSMH8a/PG/ZkgAnqzBhWXZIb8udYgYvXtU7vnVyKCi9cuof4fjondWACa3kzOakRKHIYXUqy8eWkwWiIEGBmouDQr/DETMNODCxCs2huN95+04czzdlVW2I1ECQHmohuLsPrzpUjJGRmeE0SgZGUGNn9vHtZ+cRZW3lICU8YFIV4AChdbcPV35yG9eOqGjBI1jNp2yD3xicoy0F/vgyxNv9cjjUjInJeHH34YP/7xj9HW1oYlS5bgoYcewqpVq0Ie//TTT+M//uM/cPbsWVRVVeFHP/oRrrnmmkQ0lRRE1ApY8olC7H+0cdyfCyJgKTCidEX6lD4ug0uQtUwHT58n4nwLORAcYrK3+bHkDmvMdWSULtlzYARBQNWGHMy+PBv9rW74PRJScwyjeibnfCgXVVfmoPecE36PhLRcA8yZUzsXKZHzv5ydAQgCJrXSSpYjjqqRisW95+Wpp57Cli1bcN999+Hw4cNYsmQJNm3ahI6OjnGP37t3Lz796U/jjjvuwJEjR3D99dfj+uuvx/Hjx+PdVFKgqitycNFNRcG9kARA0AQvAJBRZsbGe6uh0U3dyzjZcxyUpOhiU/QTRQe/7bYdjn5jSDVRQpgVRAHpRSZkV6QMB5eBDg8aD/bi/JE++FwBZJWnIG9umqqDC4BJ7xNkzpvYJHVSD0GOtm94glavXo2VK1fiF7/4BQBAkiSUlJTgq1/9Ku69994xx994441wOBx48cUXh69bs2YNli5dikceeSTi49lsNlitVnT95muwmLlkbrpw9/tQ/1Y3bG0eaI0iSlekI6d6amuLqG1lUSI0vObAuded0R0sAKkFGqz4+vhzMKYDued80lcgAcBAuwcHHjuH1uMDw9eJWgGVl2Vh+WdKoDVMfaBP5Puit9aLd3/bP+HbV38sFYVr1LFPGo1w2R24a8Un0d/fD4vFEvbYuA4beb1eHDp0CFu3bh2+ThRFbNy4Efv27Rv3Nvv27cOWLVtGXbdp0yY899xz4x7v8Xjg8YzMLLfZbJNvOCmO0arD/Gvz43b/DC4j/G4JXe974bNLMOdoMPdTqTi3wwlXZ6Txo2B3/3QmZBbjCSCpAcbe6cHL3z0Jr3P071ryy6jd0YX+ZjeuvLdqSvYLS9b7Ir1Sh5QCDRztgYiTxkcRgMw5euSvnNrVVaQ8cR026urqQiAQQF7e6NnyeXl5aGtrG/c2bW1tMR2/bds2WK3W4UtJScnUNJ5mDAaXIFmWcfZ1B/b+oBunnhpA3T8cOPnnAZx+zo6sedH1Yk63+S7jETKLkzq8ePTpZnidgXGH9GQZ6Kixo+Gtnkk/TjLfF4IgYNFtVpgyB09RQy+rwT9TCjQoutgErWnk9aZPE1C+KQULb7FwyGgGUH2Ruq1bt47qqbHZbAwwFDUGlxENrzjQuOOCJaaDA8qSFzi/2wWtSQi7w7QgAtkLplfBunCSMYHXY/fj3P7eiLVfTm/vxOz12RN+HCW8L4zpGqz4RiY6jrrRdsgDn12C3iqiYIUROYsMwWGya1Lg7g0WYzRmcJ7LTBLX8JKdnQ2NRoP29vZR17e3tyM/f/whgPz8/JiONxgMMBg4t4Vip4QPaKVw9wXQuDN8bQy/O8IGgTJQfIl5KpulWMlagWTvjLz6CzJga534xGklvS80OgEFK00oWDn+/BVRK8Cco/rv4DQBcR020uv1WL58ObZv3z58nSRJ2L59O9auXTvubdauXTvqeAB47bXXQh5PNBFK+oBWgvYjUVQklYNzEYBgL8swMfjv+Z9OQ2rBzDmRJKMGjFYf3Ue2JsrjQuH7gpQu7p80W7Zswa233ooVK1Zg1apVePDBB+FwOHD77bcDAG655RYUFRVh27ZtAICvf/3ruPzyy/HTn/4U1157Lf7yl7/g4MGD+M1vfhPvplISSQEZXmcAOoM46Q/eSBhcxnL3RK6rIWiA1EItyq9KQfO+4IZ4okZA5hw9CteYYM7WjHs7WZbRf9YPW+PgBnrlOlhK4lMVOdES3QNjKTAiNVcPe4c3dJtEoHRl+oTufybUOPI5Jdhbg9sLpBVpoTWyVqsaxT283Hjjjejs7MR3vvMdtLW1YenSpXj55ZeHJ+U2NjZCFEdePOvWrcOTTz6Jb3/72/jWt76FqqoqPPfcc1i4cGG8m0pJ4Ozx4sRL7ajb1QW/RwIEoPgiKxZcl4+c2VO/uRqDy/gunPgYiiwFj7PO0sE6K7rwYW/z4/0nbXC2B0YmXcpAapEW8z9jCRl4xhPwynC0+wEZMOdpoTUoY35DIgOMIApYcF0B9v/+XJiDgDlX5cZ839M9uPgcEupesqP9qAfy4EItUQvkrzSiYnOqYl5PFJ2413lJNNZ5UY+Bdjdevr8GXod/1Dj+0JDEpXdXTOlu0QwuoQ00+3Do530Rj1v1rxlRzzFwdQdw8Oe9CHjlsctdRUBnFrDi6xkwWMIHmIBXxtnXHGjZ70bAE/y4EnVAwUojyjelKOabs9xzHkD8l1HLsowjTzXj/ZfaIYgYfu8IYnCVzqV3V6AkxqrT0/294XNJOPxwH1zd4yy9FoI9MEu/nD7p4ng0ObHUeVHGu55mpLd+2TAmuADBD2NZDv7cPeCfksea7h/Ok5VWpENGlS50PXUByFmkj2ly5Lk3HOMHFwCQAJ9TxvnBDfSkgDzuXjQBn4x3f9eHpjddw8EFACQf0LzPjSOP9AV77BQgUXNgBEHAspuKcfV356L84kxYC43IKDVh/rX5+OhPFio6uEgBGc5OP5ydfkj+xH1vbtzphKsrRM0YGRho9qNlHzdzVJOZM7uOFKX7rBPd9WEqt8rBD7q63V1YMMnidAwu0VnwWQvee9yG/nrf8Df6oT8zqnSY+6nw34QuFPDJwUnA4XKFBJzf60J3jXd4WCm9QofiS03IHqwr0/K2C7ZG//Cy7VFkwNEWQNMuF8qvSontPxsniRxCyq5MQXZl+aTuI1HvDckvo3GXE817XPA5gk+m1iygaI0JpVeY49rjIQVktL7tHv81NEQGmve6UHLZzFgtNx0wvFBSdJ62B7/lR/hA6ayxA9dO/vEYXCLTmkQs/aIVffU+tB/2wGsPwGDRIG+ZEdZZ2pi2YvA7peF5BeHIfgSDCwDIQF+9D311PpRdaQ5ODN7rivgaadnnQtmVZsXU+Ej2Ro7RSlhwCch474/96D3jG/Vc+p0yzu1worfOiyV3TnzIRvLL6D7lhbs3AJ1ZRNY8PXTmkUEFn12KuMwfANy9EiS/PCMKLU4HDC+UFFGfByf5OTLdJyFONUEQkFGpR0bl5IrNaYxi5HA6nsHjz213Iq1YC3dP5CEhn1OGzyFFnDuTSEoPMInsjWzd70bvad/4P5QBW6Mf5990ouyK2HvPWg+6UfeSHX6nPPx6EzTBTUUrrk6BqBEgRBtGhA+UACBF41NFSZFTlRr5xCYAudUTX3HE4JI8WkNwCfWEw6cINO+LckNIQDG9LhdKRh2YaCQyuMiyjPN7IswlGRyyGW/OUzit77hQ8/RAMLgM3g8AyIFgRejTfwtuWqlPEZFapA3/WhSBzGodBFF5ryMaH8MLJUXmLDOyK82hv+kIQ7vkTqzEOYNL8pVdMYn5AxLQ3+APFsULdz4RgNRCDXQpyvwoU1qASfT8L8mP4ETZCLwD8vBcmGgEfDJqX3SEPabtoGe4nkvp5abwX5YkcL6LyijzHU8zwsVfqYAhTTsmwASXfAKX/H/lMKTFPrKplBPFTGct02HB5ywQh55CAbH1xMhAyWURTjqy8k86SgkwyZi4HsM0qZiGbLrf9yAQYR6LIAJtB4PbJOQuMQ6H6VGPM9i+jCodBI2AaVY5ZFrjnBdKmrQ8A675/jy8/4921O7sgt8tQRCB4mXpWPDhfGRXxj4GzpVFypKz0ID0/5eFtkPu4KohAJYyLc6+5gx/8hGAtBIdsuYaUL7JjIZXnGNqmsgSUHK5CblLlV/PaWgOTLIk630hagVYyrShV4wNMudpoDUHk0Rfgw/Ne5zoPeODLANpJVoUrzMha75+eNK4u1cKfvUOMyVKlhDctHFQ+aYUZM7Ro3mvCz1nvMFNRuXga6mvzofeM31ILdRg4S1WGDOUM3+KxsfwQkllztBjxWdLsOzTxfC5AtAaRGi0E+sQZHBRJp1ZRMmlo3tHvAMSmnaFWUkkA0XrgpvxlV2RgvQKPc7vcaKvLnhCs87SofhiEzJmq2cXayGzGE8g/kXsPijZ74uSS8048b+2iMcIgoDGXU7U/8MxKpj01fnQV+tD/goD5nwiDYIoBCtCR5jLLYiAxji668c6SwdZktFxzDMyR+aC+7G3BXDkkT6s+EYGdCYOTCgZwwspgigKMKRM/OWY7A9oik3ZFSnoq/VhoHn8b+T5KwzIWTQSTIJbElgT2ML4SHSAUcL7InuhHiWXm4Jh9cLeksHVQQWrjMhfYUBvrTcYXIDRwWTw9dF20IO0Yh2K1pqQPd+AM8/Zw+6wLUtA7uKxvXL1L4eZKyMBnn4Jbe+4FT8cOdMxWpLqKeEDmsYnSzJc3QE4u0ZXVNUaBCz9UjrKrjBDZx75dmzKFlH9sdTgN+xYJkyoiJBZnJD5L0p5XwiCgMprUrHodgsyKnUQtcHlzNZyHRbcbEH1x1MhCEKw2nKEM1LTbidkSYY+TUTBKmPIOVSCCKQUaJBZPbpnztUTgO1c+CEsyEDrAXds/0lKOPa8kKop5QOaRpMCwdL/5/e44LUFvx5rzQIK15hQtsEMjV6ARi+g/KoUlF1phtcmQdAA+jRx2oaWD0pEgFHS+yJrrgFZc0PPT+o54404FOTukeDuk2DK1GD2danwOWV0HvOMzIca7Nkx52mw+I70MUufvQPRbSXhifI4Sh6GF1ItBhdlkgIyTjxhQ/dJ76jr/U4ZjTuc6Kv1YskXRyqqihphxk2QVNJr1ueU4O4NQNQJMOdokhIeZVkOOwQ06tjBejCiVsCCz1pgu8yHtoNuuHsD0JpF5C42IGuuftyaLdEuqdenzIwArWYML6RKDC7K1bLPNSa4DJMBW5MfTbucmLVRGfsRzVTu3gDqX3ag85hnODgYM0WUbjCjYKUxoSFGEASkFmhgbw2EHdLRGAQYrKODrqVEB0uJLqrHMWdrkFqkhb0lzNCRAOSvMEbZckoWznkh1WFwUa7+s76IxcOGKqpKAdbUSBZXdwCHHuodFVyA4LDM6WfsqP9nhOcwDorWRajpIwQn9052E8fyTSmhH0cEdCkCClabJvUYFH8ML6QqDC7K5e4L4N3f9UW1n5HPIUc9/4Cm3pnnBuBzhR6qadrlgq0xxH5EcZK3zIjMuSG2lBAAc64Gs66c/AqgrDl6zLspDeJgZ40gjhSuM6aLWPqldOgVWrGZRnDYiFSDwUXZWva5IMVwvuMmeMnh6gmgJ9RGiYMEEWje54KlNLrhmKkgagQsvMWCxh1OnN/jGt6zSNQBBSuNmHVVCrRTVHsl7yIjsubp0X7EA3uLH6IWyKzWI3PO+HNlSHkYXkgVGFyUr/2oJ+pjTVki9GlML8ngGNzvJxxZAgaaIh831USNgFkbU1C63gxnZwCyJMOco4VGP/WBQmsUUbSWw0NqxfBCisfgog4BT/RzWIoHK6pS4glRLuwSknh2ELUCUgt4eqLQ+NWHVIHBRfmMmZqoNl7MXqhH4Wqu5kgWS5kucoARgnNDiJSK4YUU7YmKagYXlShcbYw4WTelQIMFn7NwXkES6UwiClaGrk4LBOe8FK7hkAopF8MLKRaDi7rkLTMitUgbcrWIqAPmfcqSkOEiR5sfLQdcaD3ggrMz8XM3lK7y2lRYy4aW24xcP7TyZv5nLDOucCCpCwcVSZEYXNRHoxOw5E4rTj8zgM7j3lG9MOYcDeZ+Kg2phfH9yHF1B3Dqrzb0nx0dWNIrdZj7qTQY03lCBgCNPvhctR92o3mfC87OYIXdnIUGFF1sQmr+1D9PPpeE1gNutL7jhndAgj5FQP4KIwpWm7g0mWLG8EKKk4g9Xyg+dCYRCz5nhbsvgN4zXkgBILVAC0upNu49Lh5bAId/2Qufc+zYVV+DD0d+2YflX8uAPpUnSiA4KbZglQkFq+I/POTqCeDor/vg6ZeGQ63LLaPh1eCy6Iu+nA5zDk9HFD2+i0lRuLJoejCma1Cw0oSiNSZYy3QJGSpq3OEMBpfxCq9JgMcm4fxbzri3g0aTZRnHH++HxyaNnRMlBwsWHnu0f3jPIqJoMLyQ4jC4UKwkv4zWd9zhdyWWgZa33ZBlniQTqf+sD47WQOjnRgbc3RKa97oS2i5SN/bTEZHq+ZxSVNV9/S4ZAS+gNcS/TUrk7gug7ZAb7u4ANAYRGVVauPskePolaI3BHZlNWePPC5JleUI9aD2nfRBERNw1uvYlB6wVeqTFeV4UTQ98lRCR6mkMUZ5UBUCc4k+9vgYvzr/pQs8ZLyABqYVaFF1sQu5ig2KWhMuyjLOvOXHujdHDZs17g38KIiDLQMPLDuQuNWDOJ9Og0QnwDkg4vze4astnl6E1CshbbkTxJSaYMqOb/CxHuwGnBNS9ZMfSO9Nj+J/RTMXwQkSqpzWIyKjWofeML+yOwTkL9BA1UxcomnY7UfeSY1TPgq3JD9ufB9B13IP5n1FGTZvzb7pwbnvo+T4X9op0vOtBwCOj8sMpOPpIH7x2efh36nfLaN7nQts7Liy5Mz2qvY9SC7QRe12G9NX64OoJRB2MaObinBcimhbKNkTYcVgGSi6PvCtxwCvD1ROAzxn+jNvX4EPdS47gXV946OCJvvM9L5p2R57HEfDJ6D7lQfsRN/oafFM+Jyfgk3E2THAZQwa6T3px7A/98DrksWFQAgI+4L3H+iH5I7c1e6EBWlP0Ac7dE4i+rTRjseeFiKaF9Ao95n4qDTVPD0CWMXLSFYLDIvNussBSErqnwNnlx7nXneg45oE8eP7MqNKh7Aoz0ivGlso//5Yz+PUvTMY5/5YTxZeaxu3tkWUZTbtdaHzDCb97JAQYM0XMvi4V2fOnZmJO72kvAu4YA5EQnEQb0uAqoc73PMi7KPxWDxqdgLmfSsPxP9qieuh4bMJI0w/DCxFNG/nLjMio1KH1gBv9Z32AAKRX6JC/0gRDmF2s7S1+HHmkDwHf6KXWvbU+9Nb2Y/6n05C7ZPRJuveML/zqJgDeARmurgBS8sZ+1Nb/04GmXWN7Ztw9Eo7/0YYFN1uQszAYYKSAjO5T3uBEW72ArPl6GCzRDa34HFGO2VwomqwjBn8/kcILAGTPNyDvIgPaj4TfeVxvEZFWzNMSRcZXCRGpns8hoe2QGwMtfogikFGtR+kGM0Rt5G/xsizjxJM2BLzjDJEM/vvkXwdgytGg67gXrsEAEc2QCRAs0GbO0Yya++Ls8o8bXC50+m8DyJqnR9dxD848b4fPIQdL+csAngPyVxhQ9dHgxNpw9JYJzA4YepwIYhniqticgq4THgS8oY8pXW9WxBwhUj6GFyJStdaDbpz+28DIvBMBaDvkQV2aA4tutyCtKPyk0v4GH1yd4edZyH7g0M/6hvcBEoTIS3+HHH/MBoNVRPGlJhRfbIIgCmg94I64fNjnkNHwygd6Z+SRP9sOeuAbkLHwtvD7RWVU6aFLFeCzxzB0FM2hEmApjjxhd4jBqsGiz6fjvUf7EfCMPMDQ76H4UhOK1nG3cYoOJ+wSkWp1n/IE57gEEDzhyhgeyvHaJbz72354bOGDia3JH3aH5VEGHyPa4DLE0y+h7kUHTv11ALIUHEqKeB8C0LLfHbYt3ae86KsLX+BG1AiovCY1+sYKgHWWFnqLEPb3IuqAvGWxzctJL9dhzb2ZqPxwCtIrdEgr0SJ/pRHLv5aO2R9OTUglZpoe2PNCRKrV8Koz9BCHPLS0142KTSkh7yOR58v2Ix5kLzAEJ6VGGpqREXmirQi0HnAjY/bYCcUXyl9uhOSTUfuSHZIXYyYaC4NfY2UJyKjUYcHnLHC0B/Dub/sgSaOPHQo0826yQGuM/fuvziyi5FIzSi6NvPKLKBSGFyJSJVd3APZmf/iDZKD9UPjwkl6hi26YZCoIwPm9LhRfbIo4eTUqUnBOTTQK15iQe5ERncfccHVL0BgEWEo06D/rh7tXgtYkIHeJYXhFlnWWiGV3Z+Ds6w50nRjZJTxjtg5lV5qRXh4+MBHFE8MLEamSzxXd2E2k49KKg8MXA83+iKuHJk0GBs77kDXPClOWCHevNP7w0eDQTX9DhHAGwNHmh63JF3YZ+BCtQUDBytG7SGfMDj30k1qgxcKbreg768VAkx86s4is+XroTJxxQMnFVyARqZIhylU00Swpnv8ZC/Sp4tg5HnEYUhJEAaJGwOI70kdWAg09zuCf6RU6LPicFZooOjckH3D0132wt0YOOrEaaPbh0EO9OPqr/uE5O/t+0I0zL9ijXm1FFA8ML0SkSgaLBhlVuvABQwAKVkVewWLK1GDF1zNQusEMnTl4h6IOyF9ugC5lChOMCGRWBXtITFkarLonE3M+mYr0Ch1SCjTInq/HotstWPIFK/SpYlQVgQFACgANrzqmrp0YrH3zq75gj9SFj+UHmve6cPzxfsgSAwwlB4eNiEi1yjeloK++Lzj08oHzqCAChnQRhVGEFwDQp4qo2JSCik0pkAIyBBEQBAEd77rx/pMDU9NgCSi+ZCSQaPTBYZwPDuUMKbvCDJ9DQvPeMKuOBu+3+6QXXrsU7EGaArV/t0PyI+Rk6J4aH7re9w4X0iNKpLj2vPT09OCzn/0sLBYL0tPTcccdd8But4e9zfr16yEIwqjLl7/85Xg2k4hUylKiw+I7rNAPVs8VRAz3xKSVaHHRl9OhncD8DFEjDC/bzV1ixNwb06AxCmMeI3uBHgWrDMOrdUIZ+nnltSmwzoq+NoogCqj6aBqEaIrpyoCnf2r2BXJ1B9BXH2aTS2BwKXfkvZuI4iGuPS+f/exn0draitdeew0+nw+33347vvjFL+LJJ58Me7s777wT3/ve94b/bTZzSR0RjS+jUo+1WzPRXeOFvcUPUSMgo0oXsThdLPKXGZGzyICu4x64ugLQGARkLzDAlBVMFeWbJHS970HALUNvESH5ZbS87Ya9xQ9BA2RW6VF8iWncPZKioTUKwQq7kY6boom0rq4oQpAMONu5iSIlR9zCy8mTJ/Hyyy/jnXfewYoVKwAADz30EK655hr85Cc/QWFhYcjbms1m5Ofnx6tpRDTNCKKA7HkGZM+L3xCGRieE3sdHCNYv0RpkpBZqYc7RomDF+ENBE5F3kRHn97hC94QIQGqhFqbM6PY7ikSMcnNEjYFF5Sg54jZstG/fPqSnpw8HFwDYuHEjRFHE/v37w972T3/6E7Kzs7Fw4UJs3boVTmcM27kTESWI3yPj1NM27PtBN048YcP7Tw7gwE96ceTXfXB2Tt3qn6KLTRB1CD05WQZmbZy6HmpLiXZ44nJIApCzmPNdKDni1vPS1taG3Nzc0Q+m1SIzMxNtbW0hb/eZz3wGZWVlKCwsxLFjx/Dv//7vqKmpwd/+9rdxj/d4PPB4Roo92WzRbbtORDQZkl/Gsd/1BbcX+ECPSH+DD4cf7sPyr2YMDy2FIksyes540VfrQ/9ZH7wOCRpdcFiqYJURxnQNTJkaLLkjHe891g+/64INGgfzRdX1qcieP3VBQtQKKLncjPp/hljBJARXYxWu5l5ElBwxh5d7770XP/rRj8Iec/LkyQk36Itf/OLw3xctWoSCggJceeWVqKurQ2Vl5Zjjt23bhvvvv3/Cj0dENBFth9ywNYboXZGDvTL1L9ux4LPWkPfR8a4btX93wDswtlKdo82Jpl1OLLzFisw5elhn6bD2W1loP+JG7xkvpACQVqRFwSpjVLVsYlVymQnu3gBa3v7AJpICoNEBiz5vjcvjEkUj5vByzz334Lbbbgt7TEVFBfLz89HR0THqer/fj56enpjms6xevRoAUFtbO2542bp1K7Zs2TL8b5vNhpKSkqjvn4hoIpr3RVhpIwFdx73wOSToUsaO0LcfdePkn8MvwZb8wPHH+7HynkyYMjXQ6AUUrjahcPXUzacJRRAFVH8sDXnLjGjZ54K9zT/cI5S/0gj9OP8nokSJObzk5OQgJycn4nFr165FX18fDh06hOXLlwMA3njjDUiSNBxIonH06FEAQEFBwbg/NxgMMBg47kpEieXqjrzSRpYAd29gTHiR/DLOPB++bMTwsRLQ8rYrtp2hp5C1TAdr2dSt3CKaCnGLzvPmzcPVV1+NO++8EwcOHMCePXtw991346abbhpeadTc3Iy5c+fiwIEDAIC6ujp8//vfx6FDh3D27Fm88MILuOWWW3DZZZdh8eLF8WoqEVHMNLroVtqMt3Kn+5QXfmeU1WkloPO9KdjEkWgaiWu/35/+9CfMnTsXV155Ja655hpccskl+M1vfjP8c5/Ph5qamuHVRHq9Hq+//jquuuoqzJ07F/fccw8+8YlP4O9//3s8m0lEFLOcRYaIn6CmbBHmnLHzQlzdgZj2TZK8LMNPdKG4FqnLzMwMW5Bu1qxZkOWRN2VJSQl27doVzyaRgj1RUZ3sJhBFrehiE1rfcYctQlu6IWW4Uu+FtEYhfPXaCwmAOY87uRBdiDOuSBGGgouQWZzklhBFJyVXi4U3WyBoMboXZfBTtewKM/KXjz8fL2uePvqeFxkoXBP/CbpEasI4T0nH4EJqlTXPgDX/nonW/W5013ghB2SklehQuMaEtMLQH68GiwYFK41ofccdsQcmc64eOQsntq0A0XTF8EJJxeBCamewaDDrQymY9aGUmG43+yOp8A5I6D7pHSk6dwFBBxSvM6H8qhQIIsvwE12I4YWSjsGFZiKNTsDCWy3ob/Ch9R03XD0BaLQCUgu1sJbrkF6ph5Z7BxGNi+GFkuaJimoGF5rRBEFAeoV+wrtNE81UnLBLScHgQkREE8XwQgnH4EJERJPB8EIJxVouREQ0WQwvlDBcWURERFOB4YUSgsGFiIimCsMLxR2DCxERTSWGF4orBhciIppqDC8UNwwuREQUDwwvFBcMLkREFC8MLzTlGFyIiCieGF5oSjG4EBFRvDG80JRhcCEiokRgeKEpweBCRESJwvBCk8bgQkREicTwQlOCwYWIiBKF4YUmhTtEExFRojG80IQxuBARUTIwvNCEMLgQEVGyMLxQzIYm6BIRESUDwwvFhCuLiIgo2RheKGYMLkRElEwML0RERKQqDC9ERESkKgwvREREpCoML0RERKQqDC9ERESkKgwvREREpCoML0RERKQqDC9ERESkKgwvREREpCoML0RERKQqDC9ERESkKgwvREREpCoML0RERKQqDC9ERESkKgwvREREpCoML0RERKQqcQsvP/zhD7Fu3TqYzWakp6dHdRtZlvGd73wHBQUFMJlM2LhxI86cOROvJhIREZEKxS28eL1e3HDDDfjKV74S9W3++7//Gz//+c/xyCOPYP/+/UhJScGmTZvgdrvj1UwiIiJSGW287vj+++8HADz22GNRHS/LMh588EF8+9vfxkc/+lEAwOOPP468vDw899xzuOmmm+LVVCIiIlIRxcx5aWhoQFtbGzZu3Dh8ndVqxerVq7Fv376Qt/N4PLDZbKMuRERENH0pJry0tbUBAPLy8kZdn5eXN/yz8Wzbtg1Wq3X4UlJSEtd2EhERUXLFFF7uvfdeCIIQ9nLq1Kl4tXVcW7duRX9///ClqakpoY9PREREiRXTnJd77rkHt912W9hjKioqJtSQ/Px8AEB7ezsKCgqGr29vb8fSpUtD3s5gMMBgMEzoMYmIiEh9YgovOTk5yMnJiUtDysvLkZ+fj+3btw+HFZvNhv3798e0YomIiIimt7jNeWlsbMTRo0fR2NiIQCCAo0eP4ujRo7Db7cPHzJ07F88++ywAQBAEfOMb38APfvADvPDCC3jvvfdwyy23oLCwENdff328mklEREQqE7el0t/5znfwxz/+cfjfF110EQBgx44dWL9+PQCgpqYG/f39w8d885vfhMPhwBe/+EX09fXhkksuwcsvvwyj0RivZlIMnqioTnYTiIiIIMiyLCe7EVPJZrPBarWi6zdfg8XMuTBTZSi4CJnFSW4JERFNRy67A3et+CT6+/thsVjCHquYpdKkXAwuRESkJAwvFBaDCxERKQ3DC4XE4EJERErE8EJhMbgQEZHSMLzQuJ6oqGZwISIiRWJ4oTEYXIiISMkYXmgUBhciIlI6hhcaxiJ0RESkBgwvBIAri4iISD0YXojBhYiIVIXhZYZjcCEiIrVheJnBGFyIiEiNGF5mKAYXIiJSK4aXGYjBhYiI1IzhZYZhcCEiIrVjeJlBGFyIiGg6YHiZIRhciIhoumB4mQEYXIiIaDpheJnmGFyIiGi6YXiZARhciIhoOmF4mca4QzQREU1HDC/TFIMLERFNVwwv0xCDCxERTWcML9PM0ARdIiKi6YrhZRrhyiIiIpoJGF6mGQYXIiKa7hheiIiISFUYXoiIiEhVGF6IiIhIVRheiIiISFUYXoiIiEhVGF6IiIhIVRheiIiISFUYXoiIiEhVGF6IiIhIVRheiIiISFUYXoiIiEhVGF6IiIhIVRheiIiISFUYXoiIiEhVGF6IiIhIVRheiIiISFXiFl5++MMfYt26dTCbzUhPT4/qNrfddhsEQRh1ufrqq+PVRCIiIlIhbbzu2Ov14oYbbsDatWvx+9//PurbXX311Xj00UeH/20wGOLRPCIiIlKpuIWX+++/HwDw2GOPxXQ7g8GA/Pz8OLSIiIiIpgPFzXnZuXMncnNzMWfOHHzlK19Bd3d32OM9Hg9sNtuoCxEREU1figovV199NR5//HFs374dP/rRj7Br1y5s3rwZgUAg5G22bdsGq9U6fCkpKUlgi4mIiCjRYgov995775gJtR+8nDp1asKNuemmm/CRj3wEixYtwvXXX48XX3wR77zzDnbu3BnyNlu3bkV/f//wpampacKPT0RERMoX05yXe+65B7fddlvYYyoqKibTnjH3lZ2djdraWlx55ZXjHmMwGDipl4iIaAaJKbzk5OQgJycnXm0Z4/z58+ju7kZBQUHCHpOIiIiULW6rjRobG9HT04PGxkYEAgEcPXoUADB79mykpqYCAObOnYtt27bhYx/7GOx2O+6//3584hOfQH5+Purq6vDNb34Ts2fPxqZNm6J+XFmWAQADLs+U/5+UzuVwQdA7kt0MIiKimLnsTgAj5/Gw5Di59dZbZQBjLjt27Bg+BoD86KOPyrIsy06nU77qqqvknJwcWafTyWVlZfKdd94pt7W1xfS4TU1N4z4uL7zwwgsvvPCi/EtTU1PEc70wGCKmDUmS0NLSgrS0NAwMDKCkpARNTU2wWCzJbtqMZrPZ+FwoBJ8L5eBzoRx8LpJPlmUMDAygsLAQohh+PVHcho2SRRRFFBcXAwAEQQAAWCwWvhgVgs+FcvC5UA4+F8rB5yK5rFZrVMcpqs4LERERUSQML0RERKQq0zq8GAwG3HfffawDowB8LpSDz4Vy8LlQDj4X6jLtJuwSERHR9Date16IiIho+mF4ISIiIlVheCEiIiJVYXghIiIiVZkR4eXs2bO44447UF5eDpPJhMrKStx3333wer3JbtqM9MMf/hDr1q2D2WxGenp6spszozz88MOYNWsWjEYjVq9ejQMHDiS7STPS7t27cd1116GwsBCCIOC5555LdpNmrG3btmHlypVIS0tDbm4urr/+etTU1CS7WRTBjAgvp06dgiRJ+PWvf40TJ07ggQcewCOPPIJvfetbyW7ajOT1enHDDTfgK1/5SrKbMqM89dRT2LJlC+677z4cPnwYS5YswaZNm9DR0ZHsps04DocDS5YswcMPP5zspsx4u3btwl133YW3334br732Gnw+H6666io4HNzkVslm7FLpH//4x/jVr36F+vr6ZDdlxnrsscfwjW98A319fcluyoywevVqrFy5Er/4xS8ABPcBKykpwVe/+lXce++9SW7dzCUIAp599llcf/31yW4KAejs7ERubi527dqFyy67LNnNoRBmRM/LePr7+5GZmZnsZhAlhNfrxaFDh7Bx48bh60RRxMaNG7Fv374ktoxIWfr7+wGA5weFm5Hhpba2Fg899BC+9KUvJbspRAnR1dWFQCCAvLy8Udfn5eWhra0tSa0iUhZJkvCNb3wDF198MRYuXJjs5lAYqg4v9957LwRBCHs5derUqNs0Nzfj6quvxg033IA777wzSS2ffibyXBARKcldd92F48eP4y9/+Uuym0IRaJPdgMm45557cNttt4U9pqKiYvjvLS0t2LBhA9atW4ff/OY3cW7dzBLrc0GJlZ2dDY1Gg/b29lHXt7e3Iz8/P0mtIlKOu+++Gy+++CJ2796N4uLiZDeHIlB1eMnJyUFOTk5UxzY3N2PDhg1Yvnw5Hn30UYiiqjudFCeW54IST6/XY/ny5di+ffvwxFBJkrB9+3bcfffdyW0cURLJsoyvfvWrePbZZ7Fz506Ul5cnu0kUBVWHl2g1Nzdj/fr1KCsrw09+8hN0dnYO/4zfOhOvsbERPT09aGxsRCAQwNGjRwEAs2fPRmpqanIbN41t2bIFt956K1asWIFVq1bhwQcfhMPhwO23357sps04drsdtbW1w/9uaGjA0aNHkZmZidLS0iS2bOa566678OSTT+L5559HWlra8Bwwq9UKk8mU5NZRSPIM8Oijj8oAxr1Q4t16663jPhc7duxIdtOmvYceekguLS2V9Xq9vGrVKvntt99OdpNmpB07doz7Hrj11luT3bQZJ9S54dFHH0120yiMGVvnhYiIiNSJEz+IiIhIVRheiIiISFUYXoiIiEhVGF6IiIhIVRheiIiISFUYXoiIiEhVGF6IiIhIVRheiIiISFUYXoiIiEhVGF6IiIhIVRheiIiISFUYXoiIiEhV/n+LfnmB5Ho3HAAAAABJRU5ErkJggg==", "text/plain": [ "
" ] From d4c3562219db65e34dbd2a8d97876db892c170df Mon Sep 17 00:00:00 2001 From: manoflearning <77jwk0724@gmail.com> Date: Wed, 17 Sep 2025 19:27:58 +0900 Subject: [PATCH 6/7] build: make NumPy as optional --- pyproject.toml | 11 +++++++---- uv.lock | 22 +++++++++++++--------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 39651ef..e11dc1b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,9 +11,7 @@ classifiers = [ "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", ] -dependencies = [ - "numpy~=2.0", -] +dependencies = [] [project.optional-dependencies] viz = [ @@ -22,20 +20,25 @@ viz = [ datasets = [ "tqdm>=4.66.4,<5", ] +numpy = [ + "numpy>=2.0", +] all = [ "graphviz>=0.20.3,<0.21", "tqdm>=4.66.4,<5", + "numpy>=2.0", ] [dependency-groups] dev = [ - "pyright>=1.1.372,<2", + "pyright>=1.1.405,<2", "ruff>=0.5.4,<0.6", "pytest>=8.3.1,<9", "torch>=2.3.1,<3", "maturin>=1.7.1,<2", "matplotlib>=3.10.6", "scikit-learn>=1.7.2", + "numpy>=2.0", ] [build-system] diff --git a/uv.lock b/uv.lock index fb1135d..fd7772c 100644 --- a/uv.lock +++ b/uv.lock @@ -101,18 +101,19 @@ wheels = [ name = "cranberry" version = "0.1.3" source = { editable = "." } -dependencies = [ - { name = "numpy" }, -] [package.optional-dependencies] all = [ { name = "graphviz" }, + { name = "numpy" }, { name = "tqdm" }, ] datasets = [ { name = "tqdm" }, ] +numpy = [ + { name = "numpy" }, +] viz = [ { name = "graphviz" }, ] @@ -121,6 +122,7 @@ viz = [ dev = [ { name = "matplotlib" }, { name = "maturin" }, + { name = "numpy" }, { name = "pyright" }, { name = "pytest" }, { name = "ruff" }, @@ -132,17 +134,19 @@ dev = [ requires-dist = [ { name = "graphviz", marker = "extra == 'all'", specifier = ">=0.20.3,<0.21" }, { name = "graphviz", marker = "extra == 'viz'", specifier = ">=0.20.3,<0.21" }, - { name = "numpy", specifier = "~=2.0" }, + { name = "numpy", marker = "extra == 'all'", specifier = ">=2.0" }, + { name = "numpy", marker = "extra == 'numpy'", specifier = ">=2.0" }, { name = "tqdm", marker = "extra == 'all'", specifier = ">=4.66.4,<5" }, { name = "tqdm", marker = "extra == 'datasets'", specifier = ">=4.66.4,<5" }, ] -provides-extras = ["viz", "datasets", "all"] +provides-extras = ["viz", "datasets", "numpy", "all"] [package.metadata.requires-dev] dev = [ { name = "matplotlib", specifier = ">=3.10.6" }, { name = "maturin", specifier = ">=1.7.1,<2" }, - { name = "pyright", specifier = ">=1.1.372,<2" }, + { name = "numpy", specifier = ">=2.0" }, + { name = "pyright", specifier = ">=1.1.405,<2" }, { name = "pytest", specifier = ">=8.3.1,<9" }, { name = "ruff", specifier = ">=0.5.4,<0.6" }, { name = "scikit-learn", specifier = ">=1.7.2" }, @@ -784,15 +788,15 @@ wheels = [ [[package]] name = "pyright" -version = "1.1.389" +version = "1.1.405" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "nodeenv" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/72/4e/9a5ab8745e7606b88c2c7ca223449ac9d82a71fd5e31df47b453f2cb39a1/pyright-1.1.389.tar.gz", hash = "sha256:716bf8cc174ab8b4dcf6828c3298cac05c5ed775dda9910106a5dcfe4c7fe220", size = 21940, upload-time = "2024-11-13T16:35:41.84Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fb/6c/ba4bbee22e76af700ea593a1d8701e3225080956753bee9750dcc25e2649/pyright-1.1.405.tar.gz", hash = "sha256:5c2a30e1037af27eb463a1cc0b9f6d65fec48478ccf092c1ac28385a15c55763", size = 4068319, upload-time = "2025-09-04T03:37:06.776Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1b/26/c288cabf8cfc5a27e1aa9e5029b7682c0f920b8074f45d22bf844314d66a/pyright-1.1.389-py3-none-any.whl", hash = "sha256:41e9620bba9254406dc1f621a88ceab5a88af4c826feb4f614d95691ed243a60", size = 18581, upload-time = "2024-11-13T16:35:40.689Z" }, + { url = "https://files.pythonhosted.org/packages/d5/1a/524f832e1ff1962a22a1accc775ca7b143ba2e9f5924bb6749dce566784a/pyright-1.1.405-py3-none-any.whl", hash = "sha256:a2cb13700b5508ce8e5d4546034cb7ea4aedb60215c6c33f56cec7f53996035a", size = 5905038, upload-time = "2025-09-04T03:37:04.913Z" }, ] [[package]] From f3c35f968c0da0214b8c217da1c39e705d813168 Mon Sep 17 00:00:00 2001 From: manoflearning <77jwk0724@gmail.com> Date: Wed, 17 Sep 2025 19:41:30 +0900 Subject: [PATCH 7/7] remove NumPy deps --- cranberry/features/datasets.py | 23 +++++- cranberry/tensor.py | 141 ++++++++++++++++++++++++--------- 2 files changed, 122 insertions(+), 42 deletions(-) diff --git a/cranberry/features/datasets.py b/cranberry/features/datasets.py index 960522d..e75a009 100644 --- a/cranberry/features/datasets.py +++ b/cranberry/features/datasets.py @@ -8,9 +8,25 @@ import urllib.request import sys -import numpy as np +from typing import TYPE_CHECKING, Any + from cranberry import Tensor +if TYPE_CHECKING: # pragma: no cover - typing only + pass + +try: # pragma: no cover - optional dependency + import numpy as np # type: ignore[assignment] +except ImportError: # pragma: no cover + np = None # type: ignore[assignment] + + +def _require_numpy() -> Any: + if np is None: # pragma: no cover - optional path + raise RuntimeError("NumPy is required for dataset utilities. Install 'cranberry[numpy]' to enable them.") + return np + + # Platform detection without psutil OSX = sys.platform == "darwin" @@ -63,10 +79,11 @@ def fetch( def _fetch_mnist(file, offset): + np_mod = _require_numpy() return Tensor( - np.frombuffer( + np_mod.frombuffer( gzip.open(fetch("https://storage.googleapis.com/cvdf-datasets/mnist/" + file)).read()[offset:], - dtype=np.uint8, + dtype=np_mod.uint8, ) ) diff --git a/cranberry/tensor.py b/cranberry/tensor.py index ea1057f..1b035ed 100644 --- a/cranberry/tensor.py +++ b/cranberry/tensor.py @@ -1,8 +1,24 @@ from __future__ import annotations + import math -from typing import Iterable, List, Optional, Tuple, Union +from typing import Iterable, List, Optional, Tuple, TYPE_CHECKING, Union, Any + +if TYPE_CHECKING: # pragma: no cover - typing only + import numpy as _np + +try: # pragma: no cover - optional dependency + import numpy as np # type: ignore[assignment] +except ImportError: # pragma: no cover - handled gracefully at runtime + np = None # type: ignore[assignment] + +HAS_NUMPY = np is not None + + +def _require_numpy() -> Any: + if np is None: # pragma: no cover - optional path + raise RuntimeError("NumPy is required for this operation. Install 'cranberry[numpy]' to enable NumPy interoperability.") + return np -import numpy as np from .cranberry import StorageView from cranberry.ops import BinaryOps, MovementOps, Op, ReduceOps, UnaryOps @@ -34,6 +50,15 @@ def _recurse(value): return flattened +def _reshape_flat(values: List[float], shape: Tuple[int, ...]): + if not shape: + return values[0] if values else 0.0 + if len(shape) == 1: + return values[: shape[0]] + stride = _prod(shape[1:]) + return [_reshape_flat(values[i * stride : (i + 1) * stride], shape[1:]) for i in range(shape[0])] + + def _ensure_shape_tuple(shape: Optional[Union[Tuple[int, ...], Shape]]) -> Optional[Tuple[int, ...]]: if shape is None: return None @@ -63,7 +88,7 @@ def _contiguous(storage: StorageView) -> StorageView: def _coerce_to_storage( - data: Union["Tensor", StorageView, float, int, List, np.ndarray], + data: Union["Tensor", StorageView, float, int, List, "_np.ndarray"], shape: Optional[Union[Tuple[int, ...], Shape]], device: str, ) -> Tuple[StorageView, Tuple[int, ...]]: @@ -107,8 +132,9 @@ def _coerce_to_storage( storage = storage.reshape(_shape_to_list(target_shape)) return storage, target_shape - if isinstance(data, np.ndarray): - arr = np.asarray(data, dtype=np.float32) + if np is not None and isinstance(data, np.ndarray): + np_mod = _require_numpy() + arr = np_mod.asarray(data, dtype=np_mod.float32) inferred = tuple(int(dim) for dim in arr.shape) flat = arr.reshape(-1).tolist() storage = StorageView.from_vec(flat, device) @@ -125,8 +151,8 @@ def _coerce_to_storage( class Tensor: def __init__( self, - data: Union[float, int, List, np.ndarray, StorageView, "Tensor"], - grad: Optional[Union[List, np.ndarray, StorageView, "Tensor"]] = None, + data: Union[float, int, List, "_np.ndarray", StorageView, "Tensor"], + grad: Optional[Union[List, "_np.ndarray", StorageView, "Tensor"]] = None, shape: Optional[Union[Tuple[int, ...], Shape]] = None, requires_grad: bool = False, prev: Optional[Tuple[Tensor, ...]] = None, @@ -443,31 +469,55 @@ def backward(): if out_grad is None: return grad = _contiguous(out_grad) - input_shape = self.shape or (1,) - input_vals = np.array(_contiguous(self._storage).to_vec(), dtype=np.float32).reshape(input_shape) + dims = list(self.shape) - out_shape = out.shape or (1,) - max_vals = np.array(_contiguous(out._storage).to_vec(), dtype=np.float32).reshape(out_shape) - grad_vals = np.array(grad.to_vec(), dtype=np.float32).reshape(out_shape) + if not dims: + self._add_grad(grad) + return + grad_expanded = grad if axis is None: - if not keepdim and len(input_shape) > 0: - grad_vals = grad_vals.reshape((1,) * len(input_shape)) - max_vals = max_vals.reshape((1,) * len(input_shape)) - grad_broadcast = np.broadcast_to(grad_vals, input_shape) - max_broadcast = np.broadcast_to(max_vals, input_shape) + if not keepdim: + grad_expanded = grad_expanded.reshape([1] * len(dims)) + grad_expanded = grad_expanded.expand(dims) else: - axis_idx = axis if not keepdim: - grad_vals = np.expand_dims(grad_vals, axis_idx) - max_vals = np.expand_dims(max_vals, axis_idx) - grad_broadcast = np.broadcast_to(grad_vals, input_shape) - max_broadcast = np.broadcast_to(max_vals, input_shape) + reshape_shape = dims.copy() + reshape_shape[axis] = 1 + grad_expanded = grad_expanded.reshape(reshape_shape) + grad_expanded = grad_expanded.expand(dims) - mask = np.isclose(input_vals, max_broadcast, rtol=1e-6, atol=1e-6).astype(np.float32) - grad_result = (mask * grad_broadcast).astype(np.float32) - grad_storage = StorageView.from_vec(grad_result.reshape(-1).tolist(), self._device).reshape(self._shape_list()) - self._add_grad(grad_storage) + input_vals = _contiguous(self._storage).to_vec() + mask = [0.0] * len(input_vals) + + if axis is None: + if input_vals: + max_val = max(input_vals) + for idx, val in enumerate(input_vals): + if math.isclose(val, max_val, rel_tol=1e-6, abs_tol=1e-6): + mask[idx] = 1.0 + else: + axis_size = dims[axis] + outer = _prod(dims[:axis]) if axis > 0 else 1 + inner = _prod(dims[axis + 1 :]) if axis + 1 < len(dims) else 1 + for outer_idx in range(outer): + for inner_idx in range(inner): + best = -float("inf") + positions: List[int] = [] + for axis_idx in range(axis_size): + linear = ((outer_idx * axis_size) + axis_idx) * inner + inner_idx + val = input_vals[linear] + if val > best + 1e-6: + best = val + positions = [linear] + elif math.isclose(val, best, rel_tol=1e-6, abs_tol=1e-6): + positions.append(linear) + for pos in positions: + mask[pos] = 1.0 + + mask_storage = StorageView.from_vec(mask, self._device).reshape(self._shape_list()) + grad_contrib = _contiguous(grad_expanded).mul(_contiguous(mask_storage)) + self._add_grad(grad_contrib) out._backward = backward @@ -648,12 +698,17 @@ def sparse_categorical_crossentropy(self, Y: Tensor) -> Tensor: raise ValueError("shape mismatch between predictions and labels") y_pred = self.log_softmax() num_classes = self.shape[1] - onehot = [[0.0] * num_classes for _ in range(Y.shape[0])] - labels = Y.numpy().astype(np.int32) - for i, label in enumerate(labels): - onehot[i][int(label)] = 1.0 - y_onehot = Tensor(onehot) - return -(y_onehot * y_pred).sum() / Y.shape[0] + labels_vec = _contiguous(Y._storage).to_vec() + batch = len(labels_vec) + onehot_data = [0.0] * (batch * num_classes) + for i, label_val in enumerate(labels_vec): + idx = int(round(label_val)) + if idx < 0 or idx >= num_classes: + raise ValueError(f"label index {idx} is out of bounds for {num_classes} classes") + onehot_data[i * num_classes + idx] = 1.0 + onehot_storage = StorageView.from_vec(onehot_data, self._device).reshape([batch, num_classes]) + y_onehot = Tensor._from_storage(onehot_storage, requires_grad=False, prev=None, op=None, device=self._device) + return -(y_onehot * y_pred).sum() / batch # ******************************************************** # *************** random *************** @@ -711,20 +766,22 @@ def ones(shape: Tuple[int, ...], requires_grad: bool = False) -> Tensor: def detach(self) -> Tensor: return Tensor._from_storage(self._storage, requires_grad=False, prev=None, op=None, device=self._device) - def numpy(self) -> np.ndarray: + def numpy(self) -> "_np.ndarray": + np_mod = _require_numpy() contig = _contiguous(self._storage) - arr = np.array(contig.to_vec(), dtype=np.float32) + arr = np_mod.array(contig.to_vec(), dtype=np_mod.float32) return arr.reshape(self.shape) if self.shape else arr.reshape(()) @property - def data(self) -> np.ndarray: + def data(self) -> "_np.ndarray": return self.numpy() @property - def grad(self) -> np.ndarray: + def grad(self) -> "_np.ndarray": + np_mod = _require_numpy() grad_storage = self._ensure_grad() contig = _contiguous(grad_storage) - arr = np.array(contig.to_vec(), dtype=np.float32) + arr = np_mod.array(contig.to_vec(), dtype=np_mod.float32) return arr.reshape(self.shape) if self.shape else arr.reshape(()) @property @@ -753,6 +810,7 @@ def data_storage(self) -> StorageView: def set_data_storage(self, storage: StorageView): self._storage = storage + self._shape = _storage_shape(storage) def num_elements(self) -> int: return self._numel() @@ -767,13 +825,18 @@ def size(self, dim: Optional[int] = None): def item(self) -> float: if self.shape != (): raise ValueError("item() only supports tensors with a single element") - return float(self.numpy().item()) + values = _contiguous(self._storage).to_vec() + return float(values[0]) if values else 0.0 def __hash__(self): return id(self) def __repr__(self): - out = f"Tensor({self.numpy().round(4) if self.shape != () else self.item()}" + if HAS_NUMPY: + display = self.numpy().round(4) if self.shape else self.item() + else: + display = _reshape_flat(_contiguous(self._storage).to_vec(), self.shape) + out = f"Tensor({display}" if self._op is not None: out += f", op={self._op.__repr__()}" return out + ")"