From a6fbbfe63622cb841d286130b74b71a8c3aa2d49 Mon Sep 17 00:00:00 2001 From: Brian Przezdziecki Date: Mon, 9 Sep 2024 08:48:04 -0700 Subject: [PATCH] init --- .gitignore | 66 ++++++++++ .../UserInterfaceState.xcuserstate | Bin 37292 -> 37314 bytes server/.gitignore | 3 + server/package.json | 34 +++++ .../20240909154453_init/migration.sql | 63 +++++++++ server/prisma/migrations/migration_lock.toml | 3 + server/prisma/schema.prisma | 61 +++++++++ server/src/dao/emailDao.ts | 22 ++++ server/src/dao/embeddingDao.ts | 59 +++++++++ server/src/dao/fileDao.ts | 22 ++++ server/src/dao/linkDao.ts | 22 ++++ server/src/scripts/ingest.ts | 94 +++++++++++++ server/src/server.ts | 76 +++++++++++ server/src/services/clients/exa.ts | 37 ++++++ server/src/services/clients/gmail.ts | 98 ++++++++++++++ server/src/services/clients/openai.ts | 0 server/src/services/dataLoaders/email.ts | 22 ++++ server/src/services/dataLoaders/files.ts | 123 ++++++++++++++++++ server/src/services/embeddings/embed.ts | 48 +++++++ server/src/services/openai.ts | 21 +++ server/src/utils/container.ts | 33 +++++ server/tsconfig.json | 110 ++++++++++++++++ {server => server2}/apis/exa_client.py | 0 {server => server2}/apis/gmail_client.py | 0 {server => server2}/apis/openai_client.py | 0 {server => server2}/app.py | 0 {server => server2}/constants.py | 0 {server => server2}/data_loaders/files.py | 0 {server => server2}/data_loaders/gmail.py | 0 {server => server2}/data_loaders/history.py | 0 {server => server2}/database/db.py | 0 {server => server2}/database/queries.py | 0 {server => server2}/database/tables.py | 0 {server => server2}/embeddings/embed.py | 0 {server => server2}/types/email.py | 0 35 files changed, 1017 insertions(+) create mode 100644 server/.gitignore create mode 100644 server/package.json create mode 100644 server/prisma/migrations/20240909154453_init/migration.sql create mode 100644 server/prisma/migrations/migration_lock.toml create mode 100644 server/prisma/schema.prisma create mode 100644 server/src/dao/emailDao.ts create mode 100644 server/src/dao/embeddingDao.ts create mode 100644 server/src/dao/fileDao.ts create mode 100644 server/src/dao/linkDao.ts create mode 100644 server/src/scripts/ingest.ts create mode 100644 server/src/server.ts create mode 100644 server/src/services/clients/exa.ts create mode 100644 server/src/services/clients/gmail.ts create mode 100644 server/src/services/clients/openai.ts create mode 100644 server/src/services/dataLoaders/email.ts create mode 100644 server/src/services/dataLoaders/files.ts create mode 100644 server/src/services/embeddings/embed.ts create mode 100644 server/src/services/openai.ts create mode 100644 server/src/utils/container.ts create mode 100644 server/tsconfig.json rename {server => server2}/apis/exa_client.py (100%) rename {server => server2}/apis/gmail_client.py (100%) rename {server => server2}/apis/openai_client.py (100%) rename {server => server2}/app.py (100%) rename {server => server2}/constants.py (100%) rename {server => server2}/data_loaders/files.py (100%) rename {server => server2}/data_loaders/gmail.py (100%) rename {server => server2}/data_loaders/history.py (100%) rename {server => server2}/database/db.py (100%) rename {server => server2}/database/queries.py (100%) rename {server => server2}/database/tables.py (100%) rename {server => server2}/embeddings/embed.py (100%) rename {server => server2}/types/email.py (100%) diff --git a/.gitignore b/.gitignore index 2ea9e1a..eb6ca66 100644 --- a/.gitignore +++ b/.gitignore @@ -166,3 +166,69 @@ cython_debug/ #.idea/ links.csv *.xcuserstate +node_modules/ +.node_modules/ +built/* +tests/cases/rwc/* +tests/cases/perf/* +!tests/cases/webharness/compilerToString.js +test-args.txt +~*.docx +\#*\# +.\#* +tests/baselines/local/* +tests/baselines/local.old/* +tests/services/baselines/local/* +tests/baselines/prototyping/local/* +tests/baselines/rwc/* +tests/baselines/reference/projectOutput/* +tests/baselines/local/projectOutput/* +tests/baselines/reference/testresults.tap +tests/baselines/symlinks/* +tests/services/baselines/prototyping/local/* +tests/services/browser/typescriptServices.js +src/harness/*.js +src/compiler/diagnosticInformationMap.generated.ts +src/compiler/diagnosticMessages.generated.json +src/parser/diagnosticInformationMap.generated.ts +src/parser/diagnosticMessages.generated.json +rwc-report.html +*.swp +build.json +*.actual +tests/webTestServer.js +tests/webTestServer.js.map +tests/webhost/*.d.ts +tests/webhost/webtsc.js +tests/cases/**/*.js +tests/cases/**/*.js.map +*.config +scripts/eslint/built/ +scripts/debug.bat +scripts/run.bat +scripts/**/*.js +scripts/**/*.js.map +coverage/ +internal/ +**/.DS_Store +.settings +**/.vs +**/.vscode/* +!**/.vscode/tasks.json +!**/.vscode/settings.template.json +!**/.vscode/launch.template.json +!**/.vscode/extensions.json +!tests/cases/projects/projectOption/**/node_modules +!tests/cases/projects/NodeModulesSearch/**/* +!tests/baselines/reference/project/nodeModules*/**/* +.idea +yarn.lock +yarn-error.log +.parallelperf.* +tests/baselines/reference/dt +.failed-tests +TEST-results.xml +package-lock.json +.eslintcache +*v8.log +/lib/ \ No newline at end of file diff --git a/RealJeff/Jeff.xcodeproj/project.xcworkspace/xcuserdata/brianprzezdziecki.xcuserdatad/UserInterfaceState.xcuserstate b/RealJeff/Jeff.xcodeproj/project.xcworkspace/xcuserdata/brianprzezdziecki.xcuserdatad/UserInterfaceState.xcuserstate index 2c0162f4a8dae8da9a92557536acfe002988afb8..b81b8c2bb03d1472ea757ad817a07544f3e7e891 100644 GIT binary patch delta 16674 zcmaKT2S8Lu*Y=&cdmErEMNoRjg#|=FdhfmWvb3cxbr;Z}?j1Guz8a%3DqycMiN@Gt zVq%Xi#UvVI@7)+<)PL@-7<}LN`-80R^m)#6o^xjI&Bi`>pbyU7s-8+rBc>BGh?&G( z;sat4@iDQC=q37ywZsPE3t|WHC9#v(MeHUH5MLAD5r>Gw#1F)e#81S}#5v*@;v(?} zah146{6oAXUJRDTfE}<0&cFpo!3YonLO~b^2N4Az z4#a~5kO(kK^_zww5l4R7RK50N2kwZvJ(u%Yt?MQnvm<%C9 z$uKgUj36V)C^DLiA!ErnGKtJ2v&bT{m@FY{$Xc?FtS6huR#HWFkmJblq?+VOM9w4U zlNxdXxsd#TTtt3It{{8JUb2r|NvQOSkw25? z$n)eMX=j7ky3-YxHif9ol(ia(s#3EDCFp;&$M&vAV7kPq7YH2C|VRFN)%;_azwcznW$V;A*vMBh#E9fGF5#=bVGDg^iuRn^qL|lKtYP6 zL=;7_ls;uZ4Wf)Fb80AMNm)}ilmj(_@}>N!Kq`m|qr$0ZDuzm+5~*w|hsve$s3NMA zDx>PCda8kHq!d&urKHAD<0&=8QVjXF0R`0dJnyqK0qI)Ptm98 zGxT}-0{uIEk-kFTpdZi==|}Wq`UU+DW6BI+%ouZKC^L+)U@RFc#+tEV92g1X#*Acq z7+)rY31z~V1SXM5Vv?D3CR@!EGbKzX)5VNt#xU`?PdGemFz}#6T6w+!G6i^WOuRq z*ss~6>@oHVdzHP$UT1HxH`!b4ZT1d(m%Yb6VxO~rvoAQ36LAzbsE!-V8FIro3(k_W z;_Nsl&YAPzJh=oekxSx|xfCvyOXJeH3@($);RkYq8Y5#f$}^>< zR;g*zckn(#3?c>-hQv5x0wG;Vj3?9tPaxivAI?j7x0S?1ViNH_F_CxYJ$O(2>7ywd zRBJqom`#|iSiwtA6Z44qgoaq4nLoy15dO!)g0$@P$_9CrN+TX@sd-?{7!w~7O9-vi*3k}ytl@{F3Z{2%+kieX}H^IhUE-$S}Rm_h4szV z^3e(P&8j{{c}>lH`yryihGHX4ru}d`(?Pr#;N~Vndi6|;h3eeI9Jx}Z6vNZz zLx)-5MiR@VyTk-;V`bEsU#7fCF7 z_4H1=OFe`#cg+F&5MZQvZSNZ1OeirZQ*hW?PHe`3>PuoT@fC5L_?|dRTqpj-p$Omr z!~qi=eB410h{iD{3CEa9&<5TEYK`0>g3}?ll31lNF?6R^6KjZtnrOq>`MUCT#Cn<7 zBrQ9&Lse0!kjKX#)irCR;53x@$#$Gl+g0b^jo7xRPgPZ=-cePm+8#F*G@xdJm*Uac8qTI*?w3!+;Tii%dHyjmr1Qs%VF z31)Pi#Wj)WBZgO6svsbjV)ki2c<#+UQb{>~o2Q`fA& zz#ELh=z@{J2lxU%zLKxvt9kiK;12>oAPC}X_(Zaw{d~FYi;p?=} zT7GtT$UIKCVEE^hojm<*;Are%97Rqgf7wfxxp2JFZct*u&j0n@;A zY{<9nlC5pjhl^)u1xfc^*K`(`EfWvHCXT95Rn^5|fywO9+_U7!IoJWuTMpG2TUjtz z)WCe8(G*(QQ@vmTt|_$g^j-u$7RdD>SPYh6axLX2@Duq-m|X9#0?WX1!UgmYPW)tk z3UA0y#RT)w46?S?{9@&&*=c11#ZVxJ_XdDHb`{tNHi6Ax3;2Ye&d=ay^0WBatH7sP zhHeAf`8isK&gJI|rZ;fb&;~`Chx+E41ExdNeu3l0ofbqJ>;wB(@bh&0Ism>ddka8h zdwa#0K5zhhLoCcs%hrC?b@^7?rJsL?!3lwXN5D~V3>@bd@C*45_(j;e!AWpR>)jvn ziv{nNiB03`71%D>Doy>M9IPY`nr*g5I2?%wW19Ve`EvOUUzW}{%F*=NRA^Q?*yE7& zor622om2Bn`+b;W7%nYv91WIhPB{*-n8N?S|H%Ku|BOHe0tEQ2|b|~909%g9sEB2D1VIiDM*pmsPgLNU7ET6L*t54D#o;Qs8U<1<%L=GwRPH> z{xJh!s4!+A41&Qhg#VJ?$?xKKuf#gC97Yf>*h*9QJ=oknn)Ci$K3a%j0!&=N@6}o5 z8<^ZH7@>c>RG5w(2d44+dte6t748=nG%Nt-fc?EN7v}K?_(S}>{>Fu{lrZaqMX(r_ z@L%)a@CW-~8Q}uU`ET)*-(hE6GRjl^b~la@T^9-AzyAJpQ1M^=n{fXY{xI%;gqQxi zU&n~ayS07Apucwq98XBs!cN!)N5e7jJ=hJ$!g2g@{sjL$f094NpXSf-XV+?G1Yk3N zwkeCjmO@YX-%&;7g2k6s%s+Ys@A5y(EpNx+fVnlw?Vc)GHG5 zHJ#A%Px&V@agb(ibC8vGedHi=aB)mt4wevk25#0z8j@n%yTi>+JEk#dqFL%D6_KX6 z@R*y_g)}1!NpsLm4kInFetA|_DCNcY?{>MOLRH@>=L!Bf|CeUGB+Xb@lQn6Ro0g5& z+)G;H5yrcxs!0bD$Jz~~Bk4pslP;tyIh>S`ZlpUYB|Z3;{44%70t5m$>_7xa1Vji> z2+#;H8%Qt0MNneWhx8@=$Wi!T0fY+zEcP0lI3S>pfCU1U2uOu*9)kbN#1`2t9qm=} zXw1df>d(TY3Xn<0!^9pE(~1-H04p|`j9EvffaSt30($%u?FYYCy#0;ak=bM(Vb)9L zkhusLATX$x%qOub4o1KXJN(6bN6m#Sr{QENS%H0Gyte0yzyk zH#w1CZ~{75wJqQ8UY&wY!R?Szz!?cc>AaYM(reb;gE;31f9Sn)GM*fj7h}VU@y6wSonX` z{g-r*d&zy|e)21L3jgaH@*o1E5C}sc0s$YbcHple2!sz*5A`qPcjTdd`9L5D0pEV{ z(2?i_RuA%f@+5f*0Y3x+5D0uLAb6f1;9c!J{#wNl4m_TMny3_V!LyZO>VK-hTfwmC zSBy8xKwc7*0fAtx2!v{wDr1uY{&`dIMFc{zGX~>2BaNSr%;g^W2#YXzpL{?*L?9A@ zC<~d&CXg|HTe(us)u}uKpX<`+C7She^VO~DH4^uO>MH} zsyLqtHS9RcAb+)ADTqcw=9v86={^OMM4b|Y6M+&Q3lTmO6~1*G$B2nL28xI%PgL|@y^C@0 zQUqi|Z+u$P9m$+x&b)!FsDVE?P>lF$_+Un8V{C>co7m!5KV*!MDL3x z3xcmiV3Lp)z?WF4Ndl1ok@y}IFEas=wurP7l>f6C!POu!alPn6(PGgO@*B}oct*5L zv|O}8)PsNufldUlVq!OckAH^1SOmr+z~j>nw(Y-uBwDTWqYlmQeu?4%_k^oI>nEn7q746hjkHZsqX!eKeX*NX}5A42ASFe_d z!!+~4d^P*3^aoD)jdseA@R)&xhqZ-s!`~AFyw~p$1PF?RU^rZ}Il{^yQmK@yqAOGt zik4cNPX7N4)k-4s0*i4x?TD!2@ zEoMccTcXD}I~Cm)-4Wdt-4opxJrF$^ir|1OoZyRFmkgn3^3{rdEuuKx7mZ%q2xn?A zW%$k+o-(FP@#-lP1eW$tLlF2Fk0EO_H=~A87I=(hI&M;wRj(jgnwf3R)s!t|_wFE$ zlmwp&DJROAa-m!)9Qt|@Sc$-D1lA(3eih|Lxl>ZggTfJOBLbfxuoHoA5IFh{RFsbn zsy+cKB(c-1i#Glby`}LrAHP?# zr)HQ&uhz0sS2Klt)k{sKrXg?ufv?|AIP*;`!wDcY2S*}Z=l)|JrJ)uHD_B4+q%bLQ zj_@r4-wC@}Of3<1a|nSW1F=~l>)bY0iZht#qT>pSOslCF0yQtk1?tT)1(+HgHqxMnzsjm?D5rL}+Jbjmq z-|EP4N+yoPZ1mUcZ`Ly%;MmkrUHuu&!@BSRN=2R2mH(g#50CVGL!dL%Pr~$PsUIjj z*G~xijKH}*>SyX4^$P;{&t(LzV3bZa%^F2r6fE=y0>AQm&r(N<6! zP^rD^!?&pjTtMJF0>2?}>E9(J?%g{Qg|6yHgT5r0K+Aprrx^kF^-!DFSJ4dgQws^`ir^iRe+ZKOKO;p9nld;IBS9hz_Q4 z@WVOcYXk|Mi6nHCu%c)L{vNoZI69tA5LWaOfqxLdDt!raAOf#Qq}kbEU`?me8UL}W zOkq{8w5xj9V5S+=VjWKB(*^IYte7qp0t}Y<(LHn-f)Mjm$1A#m#!}KlS0YIEzzBSo zajd25=vG3yhOVa@=tf#WH_^>>3xe4983Z{5^${F|pdo@rYv?w*omSE+x`Xbd zyAU)%&FUAN5TijX&ULOStV`dMzPc zPcNgF(<|s6x|i;wSJJEK)$|$!O%WV|pc#VZ2o6PX7=jiES|VtLp!Irso$gFcZ=yHj zt5PR=D^9SW&6^W7v_tsDZVEp5LG5W;<0U7Q{DO8-FrNdJVO6N1hN zy7bZK=wI;R8M-1ky#Me#p810?q%Yx9@zFRt?4Ty6>8rTl8ZP)V&dwDV^v`^gzN0-I z)7XbG0B*f7g1(297UE;4COF>Sntnn*(;hr&>>8d3dcM=bJK3y`e#x)`y07Th48Z^f zG9)8nD28S*=|&*vjo?TGeGv3T&=0{;2>K%!fMDQyhSQ&bld;9a;Ao?$wS_rKu@%miZCi?Jz$e*xY}& zqrnx{%&*J^9SLw?$4=IYN$~F_l|X_k%maZ0SD9;?hlexv*X6`U|nS0EA1lti* zBB(;J1Hn!NF`T0j9E0F{2zDblb_4V94GErN5`=Tj){MpKf)Je0%MN2P z-!PM=VI%in46?SYBepba$J(u}-YBU}-$;)PF6_N(n>O0~>a8w$9S* z2wdQe3(jQ^bS>7G^%v&$6Zkb<;1?Spc>D}4zie`b#js&)Ean0m&PK42Y!n;K#vnKg z!Py96*3Lz69)k0=TwoK(S;Ry(`3)EF0c|*@16-)(LYwAvg@xuujzd2e*c?{t)Js_G z)R+qk{*wzs;!0T=rUHw@`Um|~D5n>D)D>$2+G+yEjQR&EjJcpZm;^ou$^odi&^jyf=dzn7{O%-ViT=Eum{0j1hGG@ zT+hCz;{`ijC&26kffcLzS+VA=0DmShqF;bh*|`D_FgC0Ed9dLB7v9*9*kytpma=#m zYY|-6%Pwc}Le?X=Nhbwm>?#)Ddg5DJc8ykiHwc?tFKlunFD>i8-cOM?*Q)A_^hfkV zvxWUkV9h7&R`yc_HzT+O!B6_w&)IDPYqlcz=>XR3#w6K;Npid34Quw}g0FCaS>ZdZ z`G);Yhc`aZU|;-v;8JvWA7_6S@IJwQ&z@vYv8UNH>{<2)_DA+71h*sj1%f*e{1U;P z2x4v7jUe8~UIh28XV1OC`#gI=NGI4o1ibh6seyRsviAkNA0YTuKi-c8 zzk9+y6@C#sfZ*40+0jv7T{%W2SEh@_ryFNVLVY93m+Y8LTpu$%3%7OL z_wMyM=fdH46!0xLhf^JFo6`fAE7(TOd2zv5D7g`wH#d^=;e0tiZWQOw1#p2}5Q1kB z#NzuSg4j|&BX|zMUl9Bi!QT)(kKlz3T!_v?6@ z7KHMJV5Wi3P(BNmwt!472iuFwMeyPqdvOKWUR)tqF8m_+hoF=A16ym_8(VSZTov9K zSHV>xcp1Sfy<9biqtaCbZx7H(u7PV7kZgAmceTH$j-27jzOIZ$q>#=V=_< zJ%Ln{)jqH634gDLyNE~@x8g2ips#ROg@=H`^GL_YZ{yb()|#6kQuhWGcb&U|Z~SBM z`vAS%^|ue&xm(;FJR{VObr+HP!i?J2NHmekRQv+M9Az+m7shbLiy4}e$_60)ELb%d zym?U}ts?-pn9}i~DcIkq7pNDc7mP>=BE1ml!_U+Bbb5*~S6wu-J8d)*J7t*ZZ zU^^jl1R}lhjj(MOewK+Z2{dWlL%~8#ZTBckI6|*MuMywj<0b+8QA7qJG6>)0;|n~^ z#cq4eZ{71<;@21%Bt6GFYY~fy9HNDjNA?)y{3V#L=L4*$xsQF;*ZFQ6061~lO zXY{V>o9GAX=j(UsFVNqv|F!-x{S*2p^-t@c)jzNQyZ#^gm-VmeU)O(SU~J%S5ND8W zkYkW*kZ(|K&}g7CKn9Zy<`}Fu_`+bP!ES?n245L`ZE(Th`XJFDdJt>aW4PY%jNvb0 zQd}o)6HgJ(7JnpODPApJD_$?&s1|P%e?N`}T*Y_~XNalzsbi^~?*EN)obvZO2rTe@3%SbAArv3zLx z#PUze=aw%lUs?^dvbPGeim-~ZdS)%MW~@1D1M9)oVr!{&pml_Gly!`CoOP;oy0tpf zI@>zeT4g=WdZG0u>%G>8tj}0qu)b)0$@+@*HS5RLf7t*V(uT5OY{WJeHdZ!HHXb(N zHgPuTHU&1VHXSyzY*yK9w)x)Xtj$xKzik;?V_Q>OGuxrI7PeNluC@|ecUuozFI#V0 zAKNtBI@^i1eYUG?_uC$~{n_>x+uv+2sBJIWUb4Mwd*AkJ7f2&-4(m5cGvBm*uAuSZ4c~8d&+*Wz1ZH^-qhaAeyF{d zeYAaveV6@w`{nk%_ABkz*srtSVE?WC_x5+}@7X_aSmm(UVVA>Rhy4y;I~;WQ&f&Jh z69-Ka-`-s!yCd9U++=L60conN}xxdggIxkS6fx}>>e zyOg_BxyW7WTpC;yF1*V$m)S0JUFN$iaOrVb>$2WuquOP&%QlxUT)uSK<+8`+sLRhT zw_TpNymZxbHF7m^9pY;4YU66>>fq|+>f$=WHPAKKHNmyewb-@PRpwgZTIDKtt#z$; zeb05Q>v&h*b%N_8*U7F^U8lRwblvHCYqlPD$QCG#a~CEFzjB$p&tBo8Hj zN&c4nBYEXUxiM~fZiCzm-JIO~-4flh+=|^w-OAmn+-ltF+@`s$ay#tytJ`z;LGHuc z{oMoIgWW^j!`-9YW8LH36WuG_JKQI^FLmGIepKy#-2Hp^Q|@Qne{lav%1Mo-!=#o{ zYpJc&Ug{|gl!i+qrP0z@X`!@IilmdJtE8VwKbLNo?vU=09+4iCo{*lDo|oQ~K9)X} zK9fH8i1kSGsP?G$PSBKYVuWqmLUdZc1uaCSw_FC@M z1Gz4wpa=e&RO{@weU_f79R z-uJy9j z-<`gDeE0hv^!?8Fi0?7q)4qTBzVxI0toiracEq?8OGyG=z z&GXavE%aODx7e@OZ=K&pzb$^B`hD*AmESjh-}xQ(JMZ_q+V76vJ--KjkNjSb0;9-L z)F|UoLq?g88a7Hks(qAdROhHMqYjRGHX1I7iY`G9EwJpro&_6D2`xEXLK;C{fvfX4w(1D*vEflQ!Y;GjTppmCsW zphKWjpi7`{U}Ru)U~FJ~U`}9OU_oF}U~OPSpdzq2aC_j_f!_um4m=k4ec-9UtAY0d z9|!&!_&o4M5F2C|WE5l)G$hC>$TrA6$SKGrXhcw8P+U-IP`WxOGblT#D5x|@7E~Ej z9n=}54nje*gL;Fu1?>p>D(HC7>7WZikAsO|%V3*e`(USF*I>6`kKhr(KEb1c1A{|? z!-J!OOM}aUtAcBS>w_DEn}geeRl!}s?*)$w=7T2$PYPZhygT@};F}@h5QmWPknE6# zkVzpQgd7gJ7;-a2{VEiMib9!Cz0g6ShM~rxLqdm!T83JOx`j$ZJwv@i$AnG}ofeqhk1r6!}zcXVUxn9gnb|OTiAuLi(!|;ox;7teZu|1 z1Hwm#zaKs&d|LR-@Z;gXgkKH65q>-TUiicCC*iLn@RO+sHbP$=F*rgT5gZ|pSRQdW z;&~(;$weAO8b%sNnnv13Iz~E24v%b(?2eonIVW;{uh+ zdn3P!{3i0d$Rm-*BfpQl8TmAdjdG0gj*5xOi>i)lifWE(i+V4rJ8FE?vZystpGR$r z+8(td>R{BNsH0IQqE1Dr&qQ5~x)*gn>QU5-Xd>Ds+C5qt?Gf!29TFWMofMrKoe`ZK zoflmZT^(H;-4NXr-5T8~6Qd_bAB#R0{cH63=!-GKWBg+LV*+DBV#dWxkC_=W zJ7!+Yk1>~HuEt!CxfMGqHaa#bHZ?XQHaoT`wj@>-TNzs&`+n@|*u(1B)3HCq{v7*j z?3LK7vDf23+~Bw&aYN%Q<80!b<6Pt1;ymKK;xgitaSP%$$9)mEGj31Z{K3);u65kd-J$_aE=J?Ozx5s}OzdQb5 z{I~Ik;!nk&jsGeBT>Q28yYUa=)sN%)q*uuxnM^iKwn?^6c1m_lc1vzf?n<7NJS}-<@|@%a z$%~Q~Cx4XOo4hW0U-F^k8_D-m3{#v_f>RPw5>t{=Qc?<2s#9uH8d924T2quM<5N(| zq?9Qs(^F=p%t`q&1)#0rEgHD zf0h1I`jzyn>DSV4q(4pnEB!_Ks|=7qW{5LPGR!iDWmsldX9Q-%W#ncQWE5wVWmIHT zXVhjiWQ@reo1xB_knw)T)QlM!voq#qe4Md7qbFl!#?_2R8Ba3)%y^y|oSBfBl$ny5 zp7~Mc+RXKt8#A|LKFbni(OGPkepY%`NtU`St30bJYeUwKtesiAv-V|E*~Zzf*>2e$ z*(0)jvPWeHW`|^lXGdknW*23bX3MfGva7Odvg@-OvzxM8v+=mIhk`V=TgphlzN zZF$PP&b-liQ}U+e&Cgq&w=!={-uk>Pd0X>7&)c4NFz?PJ?j;v_+DXO;SYs>7QQYrD6%bbC~_|f zC`wirr4ZjGu;ooyRdd??ef~*+Euk1 zYq!*XTDz_Gi`t8I#&te*A$5s$C3Uj8y1I_KsdY2zKCJt=Zbe;R-P*eKb(`w8)a|W1 zT6eMTvbyeC-Oai?b@%HY)jh5ItL{bJt9p}qv-+X+7WG#3w)GD6PW3MJl6v=gkNTwg z`ubV*Tk6l$KX0&Z2x%y87~OyxCN)fHnAR}A;iHCS4LuDj8`d!0 z2OEww9B(+;aJu1sqj95cqf?`6qg$g_qj#fk4!o zXzXfS(RjM?apOM<1 z)O@u0c=LmnK`j<7_AO2=t}SjY-Yvc@{w+Z*AuaNjMJ+p8zG^wq@<+>+mb)!~w+?C* zw_3M4v^uvAZ}n&$(K@o#w>7#ot~I^2q_w_P(c03gZg1^q9o^d9I<<91>#Wwft@B$K zwtm>Uq;*T{iPqa~Oq+9Cd|OUiVOvR?tgX7OwymK}(Ke=Sa@(}FnQe30=C>_q>uKw2 z`=o7K+m5zfZTs5}v>j~wuIEpcE+?C8so2 zS|}ZrE=n)uC}prRL7A*fRc0u&mAT3?Ww~;Wa;b8;vRAoExmLMec}{srd0Tl;`9d{V zWu!7ynX4>R)+#%dqsm1kQH7|&RFSG^Rh&wls7g_#tFlx%syzI^1C3VAQ>|3(Qk_)& zqI%S!-(lP_v?H`5x+ATlw4U8V$?)2#l>5T4->rCuS=}hY^?yT&Tch+?_cD8qRbdK)q?i|;-y7O!o(ZzM? z=XDM3vh8x{a_$=5<<&K^%dab-E2t~7E2%51yP~_gyQ#aodqVfL?hm^Ax>t9v>)zPC zrTf$FZQTdD4|adoeWd$%_xIhWy3cg~HP(7;{Mfd!?~R=}cIMbQnwj%#!KeQZE!u@m delta 16733 zcmaKT2V4}#|Mt%8ZKFs>Iw-v!9R%s+1UxwD)q{hh_hv!89lKFuSv9sq0bA6lvBku$ zF~$-#YK$eu-fN7;s44F(2ZsFq`Mlzt?Ck7!W}fF;XVydAaAP-|`?=-=VkR+*m`!|0 z%pv9zONiyf$3!Rb8L^4jOl%=OCw35D5W9$d#D3y1@ilRbI8Iz3ek3jumx#;672*bQ zlekOVBmN}*BK`&h0006KNB{+BzyKDQgMq*TSOP0x4F-Y1zy-JhZ{PfPy$q-5>$Z}&I~WBtU<#N4W`lWPK3D`k0;|CquokQX zlfkE8J=g%g0K34K;482j>;Z?sVQ^HK4&pRk&=>kae;5EmpbQR!NiZ3vz~N91Q(-R5 zgZZ!ks$esm05xz5oC#;axo`no2$#TRuoHH{HE=!L0Jp&H@GH0*?t}Z`0eBf+fj_~k z@Mm}pUWYf}O?V65hIilt_#FNM-@v!<9Z8Wi$&jpu)FTZ@6VjBlA#F)Jau7L~bR+}F zKr)C7CPTiDs-xNtn zD3;R0L|^@{q9dQJUK{XvtoF0DuF(`K|e?LZHr2h)zU zEA2^#(?jV9I+Bi}qv;qriB6^!^awhS&Zk@HHoBc2MUSS(&|~QidK^8ToWn$cm57)!>Av1S|@C&rnPGQNx-}89mZ;gvvPI>o6cslIcy25VoTXFR?Swi zwQM^(iXF|4VaKyPi`bd$EOs{gAv>QH*v0HJb_KhVUC(Y{H?p6xo7k=F4t76#fc=_1 z%6`xOz+PZ4vsc(_>~;1b`-pwaK4E`hU$einZ#cj~j^rd9$LVkbIGak&mb2sRIR|bK z=ghfq9-I#szy)$aTm%=%#d5EHZZWrnTgoltmUAn(F0Pwf#jWEuahtiV+;(n4<-(t?q>r?6D# zXJFscvPqW}=9yXM`{Xn<*R(X2l&a4W`osXjfEZ6qBD_`;69^5#6Ns3|yYgszbJtJKXcf}KOY@Xp@R>}-$N zxx~Dcy!Tl`AQlh{iA6%Oo@v0y)C_rBYl*5>9n;)gGNv2f_=s4XlbX?UnVXuCrm9w# zwrT||?GYf%cCckc8J`dYJ z^9j9^R0?)Rjz&KbKZ|p@$|rUb*LYdKx!l5YxzjtBVR$Z?!fBH#i~Gb2!s}Dw0r8M{ zL_8**5KoC`#B<^oL1Ah&_%|Yts325CA@PoP;2rtFdzgLn=YVC4WhKo{r%eK3H}=5zR5K9A2|4Gcx)04BhcFX&N@k-V};Ig;WN;`;QX zRbAI2Ofs(+Zh%`e)U}vhby;S!x=;In4Y0$j1Gao&C$Q&>_zVHzU(Q$Y)qH)AZd4e>f_PDG9AD8168K6jRy8jJyB2?Njp#)0u*g0Rol%Y_FBOazm_5P%7g^j~t7okMo%(6>IH#1pXUY;j@W#Wam7=p6{3=H*z>-ug+b`E>aGOV#lrw-=U z%zU5;Zgtn4?~EHQt-Ez}tOSj|nZ(x4zNfpPsXmYI3^M7`(K+kr1i}%JAutSKGw~H}J2?v$PQwgF%1cI9)m4pEmtp2x z)xspFbb)bpobO~IsGJ7rpM!4DoiKnNU^4U+J`8i!7y^EKk19iN=u?oIkyKx%9<3;8 zt?YtQVsZb5{Io`OeNug9iK@kr5X}aMR2WFi>x4o4m%MkcCc#h`MR=`&Ltq#TheKfm zjO4%Kck_Grz5G6Y{~8z#V_+=)jwhV>1HAr01ibkJ2>4<3_7z&($2g|J3~c650Y|`e z{t$nd|GFDy!Yr7@A3?wi|Hb_bJfg!2VaY031V=(8RPx^-V2XeT{>z`a2uooZRKs#S zzY16ht6(*(fwiy>*24za2%Cf%9-BQ*@K^Y&{80o<5HRL1@;_aGEwB}~;fLDcC^#C9 zfn#9@9EY)2c~7K$Jwr2iSkes$hr(1Zd#Vf0!5(6&*PwW< zw0U49{D41&P5)$%&i9SduU#9GKY~lU#2n)sT#hI4G5iFs0ON$KUIVGK{5hTw(nbu& zVO<=m8?F)zhMVJ|R^y@YZ`Rla!nJT6{4~E2Gq3e;{0068rD2#bFVaz%o31a+OwzH@ zKEDxuRuG?^nOIe;78Ru%Zi1V!py%EWqNjtO!>z(VA1?{qh8tx*UV3l`7QYjWgS+6D zg2rbcd6~Z?PRCR$ZV%j>m70;2KD-O=!B4#Lk(a@P@H@h513Ux|!>{2H_zgS?zlF!( zad-lr(go_xa;aPYNo`*l+-+m-q_y^c!iuv4Q z{ty07gu}W%0tUFvcN?hd0Lkz!Ea`;z_=o%%twKK(wdxTD9e?;o7fRZe3mrCos4iNi;)f*{-tAd z*-AqE(>oE;jI_WLAh9ys(N*g&Z%2QmNk^N)!~3JTwPz$ zTG_Ki(jJzqJj-~rW5IjZI^T=T}W5bjdUkH_`mqS5pX9Ez#$YOK&~OZ z2q)5qSV#Jjeh5fJJSjdH0UZQ%v1Kl8{LIHO>iNIrhLe%vT!)eo2+#;HU1Ss)jR1=P zhY`sPbTb)7CgWg2CLy5LNv0s6FNTxALDnP4O!3KdG6Mla1dRHvQ#ctIpuq}97Lr9c zQOCh9p{%O4p?O$U3;rIHgcGtZa-^7-{^z~_<0Z&qvV>HTrLZ3VR!&wRV1a-G0)r7S z>+ur!(hh+^{r!Yy6j?=9_c;m#Y!EQ-a}!#y8?m1to5*Ie1%ZJGSRr8j-c{gnM!^qy z#gw0eY-Os&a>3%lctP5v=Jawbpk zml3X%(ng5)jD#b>N|(9h0_<_gdE|UjK)?|JCj^|ku|_Q-KjPma;DUfFrdFC*A$%U< zp!tMc*-5TIzzr*gc9C5@QA?}GtI0LEX6Y?{GSscz5%4{a zeZ3l%;$&`|_yg<|P3 z{!Q$+BC*HFbPVyE45tyLwhge3D_lS92tDftY6C>UzBDT#fRO}tvwR#BmDj%ikxI7T;UU4rm)n5~AD=;Lft!bx-_x)MDE zu-PXgfX#mRD#-we0eMY1BIAfzZC<6 zaL9R7aNnbD5pltbl0;0DBmsfKP6zlcO1H30!0mt~^SyiX7$YefGf_(Tlv z=d>gnv`%Z3Fd`|zuwUmbJ)Lhx4C&u_hqiNvFfTr_fBRS3_VI#WgtuTAr6cT%H102K ze~+;1kqP}D`da%C5+c*02cGLU^fB#?NlL>=Ato(AxUA3?_!<} zq`tz6XnP?)x=ApOcCYW#iEEO3gx6Zhb;%9MP020EZOI+UT?A$#FdKn62+TtO`{9KM ze6&__U-CfmQ1VFfSn@>j6oDlO>_Fgq1YRTPfM7U+dAx7`iT$db*sOjNGw(O=-?g_t zR2oJJO+y@nCvqdfdxSy%HUFvYG8cykdB1iFYTM`c2eAKrMtgq&W=RY1r$Ywxzptmg zzX(5{)W6+G+rC)XpDwSYOeuT9YdvK~nNtHP3(AtRqO2(!%9g^RcPRqP5Lk}D#|V6a z01iPb5y0Bjg+TXu%0Zj|QBIUISV=fhZldn4(x!h1tU+Kaws4%SZ0j*{Py%``ChBH90Qu~9S(Egn1rH+cB@PmF=L5GeN2l zUl-wPzibC*{$<~w#Z+leu1Bf(-x1h?lOReh2IkN4kfXB)*-|xBeNUQ1)%B!F+yB+$ zbESDD)k5>kBJdRgyAjxfz+MFQA+Ucv z#rNVmxfj>zBCZGea6SAU*VFxR9YHM;as3E^gMGLz#kf+-z)JChz#;5Q#4ns%OZt}C zMXeUG?WR^C@HGNQx~Mf2#`qfqj^TwqueTMX6&{7uMryMbjZGpNN5!>%E~4=*AM!u> zu90qHpX{B~S0X82P`fC+%Hs&&uzj+d+D+{dN%;;0gjiZJ=)S0)Gddqwa|X4fRlj{%Rle*Z4mWc-ULh zyb{q@8d_yEv^AHi<8V~mHlVH^$f-9X4;Y=F`*`^CKP#MGoM;Kn;8oESO(Sp}fg4>k zOLGX^MBw&699Yu>Xd|(k0cnoFE&lf|(u%ai#m8IR2QS@~9!OhbxM>U8lD0zN4gz-( zxYteF(6%Dn_YrvT9&Xx+Frb|=ytoxC=@TN8siA!UW&2q0{NC9{OpV2VnYt!}Rx6U_I>91#|^w zpDv_}=#jLNE~ZOp6ri|C~yxgXJs=_Ls2AgGI=UN^mr zUXICycsc|AA-5ZoxeAl{R;49(ExulduU$&}IXHR)y-6H%BUUG9h}DVSOj;sn)T2*z zrIzvZ4tkHs{!aP}dKdjA{T00%K@$W`5i~;(KQs_Q3oZNmiFNcr5~pZlX$mcIX$q~e z3gBqGw6Q_RDRU64%k183zzMp?h=W?UIJ1}`ZP!5{>KyBSZ$OVpYW1Vi6z4HG~Z zFo9T0hF11y4HJT|L-BQGWj_zhgfl(c3h7Kl58h$@=Yef-U`0HWhT&!sm_$a#3}cd* zWG00f&d8Zm1cxFRfnX$pQ3ysO7=vIef^i7OBbczBQE1_2GMOwfnP757xD)%}PDU^r z8x7X4oc?fUG36rM6$r}u;I77SGc`=D_(5IRjRT9|S8zABev&B{TX!`;oaU5^|BbgkTkd)m_XL<|hPe5Nzu4fihR-26IQm z^(J$Rxs4zuw+_MjZssm?PsFtW!Nz}ZeTs2>hH*UDpvCnCzWx8CPB5@>a6AiG7CS)}vLq{EDVAojdu~T?6oOb;#vnKr!Hym$$m){5 ztbShzgyX~z2q*kk2xJGcRw9Q(S<4;=IR5`SK-PgBj2$40{gdXu9bn(NH0#EC_9z7F z(W4L(MTK}PYG{kr0V-HO7N1V8X8qX!HjoWsv2&h`AjW?>!uus~)@nAC9m0mO;Vd>! z9Df!fxE#SX2!8gjDu<2fsTtrDTr&jy&lVS(sJ%DseVu~0!1|RyY>M{&jJ^uVGD@#s zIl?Nmw`cxO{m5o%+h_OgoEdv-H)7aawh(Wgv3YDhi?w+Uf^!j^hd0mIk*rc&`g{cO z>k4@DY_g0k$2(^%b`J~scFx%9o}IH^wPWkpCQ(*B+rTy=xCp_I5L_(EYGGSNSxXRH z)+5VN@E9huWXFo~IuKm?Ufu-l`O&?EL2m3scB&|C5<8ilg5bvpeuChNZgv_wot=Ro z*8DC6SN|hUkDY@Jg~ikF_BZxC|?VcaIq<7B`p8~PV z@s6GLL2uy$nYB;fdOF$F;taajZgv%dYY|+B;HTnx*0Srw^{hv5L(c@njYWGIyIEY% z76doGA7$IW@Y%_J_22ULuzN*$n-JWLM_M|B57<-&O zf#6mIwfp;mxd#h?|({uk+-R#fW z`(KC~7(TsvbA!Djj(?NA#bW>QC4yfexVxLZ%id$}Be(~_BM5$j7h@pz6P6_x2Cz>> zc5t4smp^l!eSuGS@Hr3rEBlgt#r~Gp((}RrKHx?00D}7w+=t-Tc%BMd_AUD-;nl^y zWB)+#AcBXw*uNmo=MLjTC0}7#ii>z?!%-Y9JWFw4x;chp@n-0=lngni%jx~=-2%>l zGsD))8FEIPF=xV=BKR$W#}GV@Aja_IYR;S+$XRffoE3uKA$STw4CNUFFW`|P9Ha|3?UN3fF*7HM?n6yoG@82+jMI2P8!NL`_qYZQ`9JKfx^eB?Xt5m!lzW{V4k!=)rT^Rn zts39QUe{6B5~0(xji!t-&*u>L0p2F%CUTRw$=nofDmRUr&S59>5Wz~| z+WQ>AU)FImxmnz7?n7=4H)F$}K}e23s4cwVPtmYQ)NlPmF- zf!6VGE43HGgjnBnZZ)^&->d$V+aN9hdzRljxs3?E5vedxk8Ui%2is-hp-@tMi&l-c zaGzr*_*Q#7u$J4_C7x5vk8^hBc5+|*Tk2Qb9vlL>-3b2G$?Zk>Y!4sgHCg+YbP`59 z#9NrXz<$Ghi$@pRI=N$rguJhIsFT=1_Vn}?BAb0A_|+dfVP3PTFu8f`$a(y!PVPD) zS=^bsMHq0mxjXnpF`w3|I<;fWH;G@L+gaU-NbTV*caOV|&rIU+*?t#y51;n-4*ZaN zj0c82vYsFkpTzaP)*=+P$ni@A#Vu0dQi~pb;pW`6=t|*HOEn1Dmfbo4^uBOV)fS3- zb+s)vCFXVO4ABYG2}h(mBBh87;Jt;^b{~9ueEVmHUOE{%xZ;%{Tw;-a_!u>8l$XQ? zyAYj1ouZ=m2S0d%CEd6QpT7wcJAwz1r**1zYRJ>%X+(yKyZYo1M27Wj@8j3hl!k)^ zzTHlE*)iWO@#`Nhf8>DI+hAx!WD*TT8Bs!vAsp~`EB^K){E0C92^BVud!(t7LO&t9xGeg6>7#Te^32@993!eWd$D_n988H&}0& z-blSkdb9Lq>&?+ytkjH~@2wxEAE%$7FVj!b zPtniR&(_b?&(|;1AF1D_KSRGuf3HUWxc&+KllrIhf7Jg;|MdXL0A>I;KySc+0fqxS z2LuinF(6|=mVu6eiNR2VID=Y)?+soXN(}W44GfJ8O$^Np?F=0Z2OByWx){0{Mj2)p z)*4PT>@wVJxYcmG;TMKq8tyi{V0g{&55vEV2%{ZFhm3wO`qAi;(G{bsM%RpP7~L|` z+%dXm^w#JPqrZ#^V`wZfrj1!+9b-M?0mh-mg~l4=wZ_MdpP87O3^8dkSzvO|2uQ;rZ3Ha88jm`W|Wz+nYEd%nZ4N{Ge!{ zvtTp1S&7+LvxR0|W~$GcTUq;C$63p*bFE9Q)z%f(Rn|4uqpinVPqm(I{ekr?>xI@U ztUIkgwccv|we<<>^VV0bUs=DkF}Crr39!ku$+H=2qp_J~^O4OGn`Jg1+pMtZwApC0 z$!3eqR-5fMJ8gE^T(o&%t8eRO8)utfn{HcRTV$)Owk@$OwXLdrNz3ds};Z`$6^|_Fncr_P+N1 z_JQ^(_DcKF_JaK;``z~Y><`!Cxo&aY>bl)^ zr|XxldtCRq9&o+n`q=fIn~9r?TdZ5WTcX=Aw`8~BZmDj(+YGn4Zu8w1xGi#9?6%r% zi`!1OU2b2w?QuKfcFFB`w?Ex&+}+(h-M!tV?t$(x?s4u3?lO0UdzQOKpZ5OV`<(Z6?>pZ2ydQW!^?vUC!uzGqK%YRL7@uK2$v(q$+X}(k`Es>T=E2Y&^O|7(EI#xPE zI$tVC7fKgPJEfbXyQO=j`=y7ZN2K3MPe?CHuSl;-uS;)A??~@UA4>oBwet=4&GBvX z{m6Hv?`OVSeRufo^4;ru!1u845#N)(mwd1J-uAuY`@;8?@9)0veE;$z{Pg_>`dRw9 z_<8#I`1$z-`-S?2`3?1p^&6q_tMhC3o9#E>?^D0ye&6|>@jK^t!S9maPkz_@Zu-6R z`^z8rlm3)HO@o)2= z=s(4On*R*{AN+6n-}b-j{~#bJKou}6U`fErfbM`b0iOnJ3D_F2BVbp+R{@$U0e=PB z2Zjg62g(AI1Lc9)fjNPBfvUiU!1lm#fhcft;OxK!flC6H1+EHQ8@N7jW8jX!GlAy< zF9hBTd=U63@JZnFz&AlgLFPd|LD4~ZL4`reppu}{AazhhP-D=jps_*YgZQ9{L34uU z2Q3WxC}?%i)}ZY{JA-xw9SJ%bbW9U;BIsh!m7uFZ*MePx1A{|?!-6A%ql06EvxAkv zWx*A})xovFV}kkMNx@TtX9Uj;7J?TBFAiQBye@c0@VCKdgRcbN4}KQ>BKTGC@4j3ds#A2q_Dx2&oRK3uz2#4(SM)5;8YrUC53Q&AyO> zAzz0a4LKfiF64)hA44vMTnV`v@-EaOG(0pjv?R1Sv@LXWXh-OTP!u{jbXw>Kq02%) z4qXx28QLAXI&^L5r=c4{KMTDwM0ZH&kcuJmha4L6GR!f|H!L75IBZDR(6Gp`^ss`k z;;_=N^03OVwy<$wny`ssQ^ID2%?XyTiENc-^1R9GvR~7ox|P2J;Qy%{lWvoL&C$tBf_J@%tquo5S0}M~8QWPY6fhlf$Ql&j_Cxz9#%|_^t5gLkA9ZADTEce`wRtnL|GwdS>Xo zq0b{UR0JEL8!;flD8e+tJi;=T%Sws25SMqJEEh7fnZV(R$Ga(Z-r+v*>}*vgoGh4beZu5Hb2OHZk@w zgJYay++sXqd}91!qGRG>5@V8LhR3AEq{n2%m@eyF`mb=R~hWX`+8(P~yA}J%OFiD+MnN*!rm(-Bdl+=>MCrwG3o-{LQZjz9+C~0xhj--o8cak0^ zJx+R-^fKwUq&G?Ll68{xldX~mB|9a%CVM3NB?l&lBo9drPcBPdl>9~Vf#mO!uP5J3 zew6%sif)R2idBkz%HR~|6ps|I6lscIN_2`mrA(7jky4#fm(rNhlG2_sCS_cTCS_vE zl$7NuD^faBx>HuCtV>y+@>$B3l&vW{Qocy}GUYt`1#?#%8lhx zd8j;G9x0EOC&|<0neuFTt~_5}D6f*&%bVn_@-gypa*Z6xKan4lpOT-GUyxsvUzJ~z z-;ir=$zRC-PNh@1RJ~M#RP$8JRGU=0REN~0)Q;5UsjE}BrXEQ>ntCSnYU=aU7pZ@z z>7^N@8K;@0S)^H~*`;}=`K0-#1*8S14M_`6i%3gPYfAetZDZQ0w1;W0)BZ^NTLBe} zLPw#mFi_Yi+!bC5sls0oqzF;OD@G`?6?qy(p`t`ls!%H`6zz&hilvH=6)P3piZzN) z6&n?s6T5x_kPtbai@tdQ*C9dV9JieR}%L^x5fi)90rzNMDrRoxU!8L;9xlZRtDHzf9kq zuK79rT?U(BkYSu*mSK@$pD{SYIm0!>JtIA1a>nY6O&Pm0PGy|QxRh}><4wl9Og2+L z(=gLGb6}=brcI_@rgx@qW<+LkraV)TnUR^DnU`6ZS(RCvS)bXI*^=3wIVQ6sb8hD5 z%#)e-vvjiDvVyY0vLdpgvl6m~Wu;`vv+^}r6_@6SF2~&C6PpwIpkK zR%cds)|#w!Szly*opmYer>tvPH?!_!-OqZI^)&03te07@v-PqKvW>D$vdywBvaPdi zv+c77XFFxPWXENzvZrRR%RZU?BF8)@C?_+gDQ9fX_#8fGV$RH*1v!gy7UwL@S)TJr z&W0S#mYi)lJ9BpD?8`Zrb2#UEE}3hXYnD4O*DBXOcTlcVu1juEZfI^o?(p2S-1OY6 z+`?RCt}3@Iw>)=F?!nxfxsP&x&(qD*&oj?+%nQy7%}dBj&XeaU^0M-B^78Wv^J?-& z=grQWn%Ah;l^U|2y)L2AK>g3N;Kg4}}qf{KFLf`)?T zf>8xy3&s~{3YHh_FF0B7eZjec3k6pSt`=M`xLNQ^!OKEW$QJ4r4k$D#v?#PLv@3Kd z99)=Es2N}QN#WYU?S)4Rj}@LRyjJ+4@MYnjMLIw!ii(R$i^_|tifW4*ikgesie?qfDVkR#6fG=TT(qp{lcLU|?xHnC>x$MF9WA;$ zQfH*sNX5wZk)0!tjJ&NRl@3aGrI%8w^izf@j!=$NmMY7YRmys0 zld@IWu3VzrsXVMauKZ4UMtM$oS$S1?U3pV^yI8+Cs<^ngs<^#)M)Azz1;w4k+lzM= zA1?m3_(bt{#b=9uDE_hdQt|!bN5wBoK*@j-qY~4SfhATYwj~ZF-X*>z{v|;rp(WuZ z5hYP2n#_`xlDQ?DO3su#E_qY(mkOvPDo&-RGEf<*>{OmAAC;dfP!*yYqDoW^Q)Q|0 zRYj^|m0DGys#eviI#knDf@+a!iE6oOg{n*SnQDt_n`)=(OVu9LKGg};cdFB>v!xcL z&ZVxU?xkL(t)-}RQt6b^>7~a?FO*&^y{sv{S~j>$TIN?4P!?P^v20G+qOv7r%ga`j zttneqwxMiO*_N{J%U-I@)KYb*I$Ry8j!`G8Q`B;GzPdzRrmj#|tDDp<>UQ-Qb%%Pn zdawGr`hohf`kDHL`knd@^b)#y8KNAte`5`3Y`k)3jc~B6+lL>u?p8djcwX_M;_phjlB?9KG^jMMw5YVMw5xQe99o%BIjl0J zGPN?NGPg3nvZAuSvZ=DQa#ZDnN>n*nQ#q}2M&+W)#g$7dKd!u5`LObF<eSp8G=wdxzyx2hjj|5C%$7}ogKgw*8J)YVL?nOifjW`50r z8ck=-#+uDFTWfaI?5f#a^G(gMnv*rBYtGjEQ1fFgQ7f&@uANf5xpq(O-r9Y&2WrpN zUa!4Td$aa-?UUN)wZGQBs-x?;I=woBI^#M;U0z*zT}@qmT~l3q-I%(Ly76^$>prgg zq;5rBXWgc{9d*0vcGvBzJ5u*;-HEzWb!Y1SuJ_T@=i&qJvigeps`|0@=H!2!48nYYo8VefR8d2kd#?6g;8_zV}Y<%AMrtxj#pG|ZV)1=en(G<`W z-4xRl+Z5k4qA9Z}rzyXws7cw>)HJrKqiI6Zw5C~2>zlr4+ST-B)9$8kn@%*HYWlwE zeA9)d+f5IeUNpUGdfoJ0`uG)FhfnvR4d!6+d80iP^)9B zORHO}XRF4i)vqx|Zwtvg!xxBl4ry!CbKpKXKN+}Z-#V%t*M zhPS1*6||MLRkT&N)wMOYjcFU#rfHklHl=M^+sd}xZ5P_Pc7t~Fc8m5w?QZR!?LO^( z?E&r4?Xvddc6qy^J*Pdty{NsoUDZCb{mb?{?N8gEx3~W}N-~NWr87!@l>CV`<0Qj`ba%b!_R_*0HnW o%Z}Y0*T+%gq~r3&sm9feYaKUA`(iMJ+JALMYya*09=H7e0V}JsH~;_u diff --git a/server/.gitignore b/server/.gitignore new file mode 100644 index 0000000..11ddd8d --- /dev/null +++ b/server/.gitignore @@ -0,0 +1,3 @@ +node_modules +# Keep environment variables out of version control +.env diff --git a/server/package.json b/server/package.json new file mode 100644 index 0000000..0a17611 --- /dev/null +++ b/server/package.json @@ -0,0 +1,34 @@ +{ + "name": "server", + "version": "1.0.0", + "main": "index.js", + "license": "MIT", + "devDependencies": { + "@types/express": "^4.17.21", + "@types/marked": "^6.0.0", + "ts-node": "^10.9.2", + "typescript": "^5.5.4" + }, + "dependencies": { + "@prisma/client": "^5.19.1", + "axios": "^1.7.7", + "dotenv": "^16.4.5", + "express": "^4.19.2", + "google-auth-library": "^9.14.1", + "googleapis": "^144.0.0", + "gpt-3-encoder": "^1.1.4", + "inversify": "^6.0.2", + "marked": "^14.1.2", + "openai": "^4.58.1", + "pdf-lib": "^1.17.1", + "pdf2pic": "^3.1.3", + "pgvector": "^0.2.0", + "prisma": "^5.19.1", + "tesseract.js": "^5.1.1" + }, + "scripts": { + "start": "ts-node src/server.ts", + "generate": "yarn prisma generate", + "ingest": "ts-node src/scripts/ingest.ts" + } +} diff --git a/server/prisma/migrations/20240909154453_init/migration.sql b/server/prisma/migrations/20240909154453_init/migration.sql new file mode 100644 index 0000000..1128782 --- /dev/null +++ b/server/prisma/migrations/20240909154453_init/migration.sql @@ -0,0 +1,63 @@ +-- CreateExtension +CREATE EXTENSION IF NOT EXISTS "vector"; + +-- CreateEnum +CREATE TYPE "ContentType" AS ENUM ('EMAIL', 'FILE', 'LINK'); + +-- CreateTable +CREATE TABLE "Email" ( + "id" UUID NOT NULL, + "sender" TEXT NOT NULL, + "subject" TEXT NOT NULL, + "body" TEXT NOT NULL, + "messageId" TEXT NOT NULL, + + CONSTRAINT "Email_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "File" ( + "id" UUID NOT NULL, + "name" TEXT NOT NULL, + "path" TEXT NOT NULL, + "content" TEXT NOT NULL, + "contentHash" VARCHAR(64) NOT NULL, + + CONSTRAINT "File_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "Link" ( + "id" UUID NOT NULL, + "url" TEXT NOT NULL, + "title" TEXT NOT NULL, + "content" TEXT NOT NULL, + + CONSTRAINT "Link_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "Embedding" ( + "id" UUID NOT NULL, + "contentType" "ContentType" NOT NULL, + "emailId" UUID, + "fileId" UUID, + "linkId" UUID, + + CONSTRAINT "Embedding_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "Email_messageId_key" ON "Email"("messageId"); + +-- CreateIndex +CREATE INDEX "File_contentHash_idx" ON "File"("contentHash"); + +-- AddForeignKey +ALTER TABLE "Embedding" ADD CONSTRAINT "Embedding_fileId_fkey" FOREIGN KEY ("fileId") REFERENCES "File"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "Embedding" ADD CONSTRAINT "Embedding_emailId_fkey" FOREIGN KEY ("emailId") REFERENCES "Email"("id") ON DELETE SET NULL ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "Embedding" ADD CONSTRAINT "Embedding_linkId_fkey" FOREIGN KEY ("linkId") REFERENCES "Link"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/server/prisma/migrations/migration_lock.toml b/server/prisma/migrations/migration_lock.toml new file mode 100644 index 0000000..fbffa92 --- /dev/null +++ b/server/prisma/migrations/migration_lock.toml @@ -0,0 +1,3 @@ +# Please do not edit this file manually +# It should be added in your version-control system (i.e. Git) +provider = "postgresql" \ No newline at end of file diff --git a/server/prisma/schema.prisma b/server/prisma/schema.prisma new file mode 100644 index 0000000..dd8c331 --- /dev/null +++ b/server/prisma/schema.prisma @@ -0,0 +1,61 @@ +// This is your Prisma schema file, +// learn more about it in the docs: https://pris.ly/d/prisma-schema + +// Looking for ways to speed up your queries, or scale easily with your serverless or edge functions? +// Try Prisma Accelerate: https://pris.ly/cli/accelerate-init + +generator client { + provider = "prisma-client-js" + previewFeatures = ["postgresqlExtensions"] // Enable the postgresqlExtensions. Currently in preview +} + +datasource db { + provider = "postgresql" + url = env("DATABASE_URL") + extensions = [vector] +} + +enum ContentType { + EMAIL + FILE + LINK +} + +model Email { + id String @id @default(uuid()) @db.Uuid + sender String + subject String + body String + messageId String @unique + embeddings Embedding[] +} + +model File { + id String @id @default(uuid()) @db.Uuid + name String + path String + content String + contentHash String @db.VarChar(64) + embeddings Embedding[] + + @@index([contentHash]) +} + +model Link { + id String @id @default(uuid()) @db.Uuid + url String + title String + content String + embeddings Embedding[] +} + +model Embedding { + id String @id @default(uuid()) @db.Uuid + contentType ContentType + emailId String? @db.Uuid + fileId String? @db.Uuid + linkId String? @db.Uuid + file File? @relation(fields: [fileId], references: [id], onDelete: Cascade) + email Email? @relation(fields: [emailId], references: [id]) + link Link? @relation(fields: [linkId], references: [id], onDelete: Cascade) +} diff --git a/server/src/dao/emailDao.ts b/server/src/dao/emailDao.ts new file mode 100644 index 0000000..8baeacd --- /dev/null +++ b/server/src/dao/emailDao.ts @@ -0,0 +1,22 @@ +import { PrismaClient, Prisma } from '@prisma/client'; +import { container } from '../utils/container'; + +export async function createEmail(data: Prisma.EmailCreateInput) { + const prisma = container.get(PrismaClient); + return prisma.email.create({ data }); +} + +export async function getEmail(id: string) { + const prisma = container.get(PrismaClient); + return prisma.email.findUnique({ where: { id } }); +} + +export async function updateEmail(id: string, data: Prisma.EmailUpdateInput) { + const prisma = container.get(PrismaClient); + return prisma.email.update({ where: { id }, data }); +} + +export async function deleteEmail(id: string) { + const prisma = container.get(PrismaClient); + return prisma.email.delete({ where: { id } }); +} diff --git a/server/src/dao/embeddingDao.ts b/server/src/dao/embeddingDao.ts new file mode 100644 index 0000000..36b98cd --- /dev/null +++ b/server/src/dao/embeddingDao.ts @@ -0,0 +1,59 @@ +import { PrismaClient, ContentType } from '@prisma/client'; +import { container } from '../utils/container'; +import pgvector from 'pgvector'; + +// Prisma doesn't support pgvector types, so we need to use raw SQL and make our own create input types +type EmbeddingCreateInput = { + embedding: number[]; + contentType: ContentType; + emailId?: string; + fileId?: string; + linkId?: string; +}; + +export async function createEmbedding(data: EmbeddingCreateInput) { + const prisma = container.get(PrismaClient); + const embedding = pgvector.toSql(data.embedding); + return prisma.$executeRaw` + INSERT INTO "Embedding" (id, embedding, "contentType", "emailId", "fileId", "linkId") + VALUES (gen_random_uuid(), ${embedding}::vector, ${data.contentType}, ${data.emailId}, ${data.fileId}, ${data.linkId}) + `; +} + +export type SimilaritySearchResult = { + content_type: ContentType; + source: string; + title: string; + content: string; + distance: number; +}; + +export async function performSimilaritySearch(queryEmbedding: number[], limit: number): Promise { + const prisma = container.get(PrismaClient); + const embedding = pgvector.toSql(queryEmbedding); + const results = await prisma.$queryRaw` + SELECT + e."contentType" as content_type, + COALESCE(em."messageId", f.path, l.url) AS source, + COALESCE(em.subject, f.name, l.title) AS title, + COALESCE(em.body, f.content, l.content) AS content, + e.embedding <-> ${embedding}::vector AS distance + FROM + "Embedding" e + LEFT JOIN "Email" em ON e."emailId" = em.id + LEFT JOIN "File" f ON e."fileId" = f.id + LEFT JOIN "Link" l ON e."linkId" = l.id + ORDER BY distance + LIMIT ${limit} + `; + + return results; +} + +export async function clearAllTables() { + const prisma = container.get(PrismaClient); + await prisma.embedding.deleteMany(); + await prisma.email.deleteMany(); + await prisma.file.deleteMany(); + await prisma.link.deleteMany(); +} \ No newline at end of file diff --git a/server/src/dao/fileDao.ts b/server/src/dao/fileDao.ts new file mode 100644 index 0000000..7e63503 --- /dev/null +++ b/server/src/dao/fileDao.ts @@ -0,0 +1,22 @@ +import { PrismaClient, Prisma } from '@prisma/client'; +import { container } from '../utils/container'; + +export async function createFile(data: Prisma.FileCreateInput) { + const prisma = container.get(PrismaClient); + return prisma.file.create({ data }); +} + +export async function getFile(id: string) { + const prisma = container.get(PrismaClient); + return prisma.file.findUnique({ where: { id } }); +} + +export async function updateFile(id: string, data: Prisma.FileUpdateInput) { + const prisma = container.get(PrismaClient); + return prisma.file.update({ where: { id }, data }); +} + +export async function deleteFile(id: string) { + const prisma = container.get(PrismaClient); + return prisma.file.delete({ where: { id } }); +} diff --git a/server/src/dao/linkDao.ts b/server/src/dao/linkDao.ts new file mode 100644 index 0000000..3aa9a75 --- /dev/null +++ b/server/src/dao/linkDao.ts @@ -0,0 +1,22 @@ +import { PrismaClient, Prisma } from '@prisma/client'; +import { container } from '../utils/container'; + +export async function createLink(data: Prisma.LinkCreateInput) { + const prisma = container.get(PrismaClient); + return prisma.link.create({ data }); +} + +export async function getLink(id: string) { + const prisma = container.get(PrismaClient); + return prisma.link.findUnique({ where: { id } }); +} + +export async function updateLink(id: string, data: Prisma.LinkUpdateInput) { + const prisma = container.get(PrismaClient); + return prisma.link.update({ where: { id }, data }); +} + +export async function deleteLink(id: string) { + const prisma = container.get(PrismaClient); + return prisma.link.delete({ where: { id } }); +} diff --git a/server/src/scripts/ingest.ts b/server/src/scripts/ingest.ts new file mode 100644 index 0000000..0a9db5b --- /dev/null +++ b/server/src/scripts/ingest.ts @@ -0,0 +1,94 @@ +import { container } from '../utils/container'; +import { PrismaClient } from '@prisma/client'; +import { gmailClient } from '../services/clients/gmail'; +import { processAndStoreEmail } from '../services/dataLoaders/email'; +import { processFiles } from '../services/dataLoaders/files'; +import { getContentsForUrl } from '../services/clients/exa'; +import { createLink } from '../dao/linkDao'; +import dotenv from 'dotenv'; +import fs from 'fs'; +import path from 'path'; + +dotenv.config(); + +const FOLDER_PATH = process.env.FOLDER_PATH || 'jeff_storage'; + +async function ingestRecentEmails(days: number = 7) { + const prisma = container.get(PrismaClient); + + try { + const recentEmailThreads = await gmailClient.readEmails(days); + + for (const thread of Object.values(recentEmailThreads)) { + for (const email of thread) { + try { + const storedEmail = await processAndStoreEmail(email); + console.log(`Stored email: ${storedEmail.id} - ${storedEmail.subject}`); + } catch (error: any) { + if (error.code === 'P2002') { + console.log(`Email already exists: ${email.subject}`); + } else { + console.error(`Error processing email: ${error}`); + } + } + } + } + } catch (error) { + console.error('Error during email ingestion:', error); + } + + console.log("Email ingestion complete."); +} + +async function ingestFiles(folderPath: string) { + try { + await processFiles(folderPath); + } catch (error) { + console.error('Error during file ingestion:', error); + } +} + +async function ingestBrowserHistory() { + try { + console.log("Getting history"); + const historyFilePath = path.join(FOLDER_PATH, 'browser_history.json'); + const historyData = fs.readFileSync(historyFilePath, 'utf8'); + const historyEntries = JSON.parse(historyData); + + for (const entry of historyEntries) { + try { + const content = await getContentsForUrl(entry.url); + await createLink({ + url: entry.url, + title: entry.title, + content: content, + }); + console.log(`Stored link: ${entry.url}`); + } catch (error) { + console.error(`Error processing link ${entry.url}:`, error); + } + } + } catch (error) { + console.error('Error during history ingestion:', error); + } +} + +async function main() { + const prisma = container.get(PrismaClient); + + // Ingest emails + await ingestRecentEmails(); + + // Ingest files + await ingestFiles(FOLDER_PATH); + + // Ingest browser history + await ingestBrowserHistory(); + + await prisma.$disconnect(); +} + +main().catch((error) => { + console.error('Error in main ingest process:', error); + process.exit(1); +}); \ No newline at end of file diff --git a/server/src/server.ts b/server/src/server.ts new file mode 100644 index 0000000..a816a3b --- /dev/null +++ b/server/src/server.ts @@ -0,0 +1,76 @@ +import express, { Request, Response } from 'express'; +import { performSimilaritySearch } from './dao/embeddingDao'; +import { generateAnswerSummary } from './services/openai'; +import dotenv from 'dotenv'; +import { getEmbedding } from './services/embeddings/embed'; + +dotenv.config(); + +const app = express(); +app.use(express.json()); + + +enum ItemType { + FILE = "FILE", + URL = "URL" +} + +interface SearchResult { + type: ItemType; + source: string; + title: string; + distance: number; +} + +interface SearchResponse { + results: SearchResult[]; + answer_summary: string | null; + error?: string; +} + +interface SearchRequest { + query: string; + limit?: number; +} + +app.post('/search', async (req: Request<{}, {}, SearchRequest>, res: Response) => { + const { query, limit = 5 } = req.body; + console.log(`Search endpoint accessed with query: ${query}`); + + try { + const queryEmbedding = await getEmbedding(query); + const results = await performSimilaritySearch(queryEmbedding, limit); + + let summaryContextString = ""; + const searchResults: SearchResult[] = results.map(result => { + summaryContextString += "\n\n" + result.content; + let itemType = result.content_type === 'FILE' ? ItemType.FILE : ItemType.URL; + let source = result.source; + + if (result.content_type === 'EMAIL') { + source = `https://mail.google.com/mail/u/0/?tab=rm&ogbl#inbox/${source}`; + } + + return { + type: itemType, + source: source, + title: result.title, + distance: result.distance + }; + }); + + const answerSummary = await generateAnswerSummary(query, summaryContextString); + + console.log(`Search results: ${JSON.stringify(searchResults)}`); + res.json({ results: searchResults, answer_summary: answerSummary }); + } catch (error) { + console.error('Error during search:', error); + res.status(500).json({ results: [], answer_summary: null, error: 'An error occurred during the search' }); + } +}); + +const PORT = process.env.PORT || 8000; + +app.listen(PORT, () => { + console.log(`Server is running on http://localhost:${PORT}`); +}); diff --git a/server/src/services/clients/exa.ts b/server/src/services/clients/exa.ts new file mode 100644 index 0000000..c5104d7 --- /dev/null +++ b/server/src/services/clients/exa.ts @@ -0,0 +1,37 @@ +import axios from 'axios'; +import dotenv from 'dotenv'; + +dotenv.config(); + +const EXA_API_URL = 'https://api.exa.ai/contents'; +const EXA_API_KEY = process.env.EXA_KEY; + +interface ExaResponse { + results: Array<{ + text: string; + }>; +} + +export async function getContentsForUrl(webUrl: string): Promise { + try { + const response = await axios.post( + EXA_API_URL, + { + ids: [webUrl], + text: { includeHtmlTags: false }, + }, + { + headers: { + 'accept': 'application/json', + 'content-type': 'application/json', + 'x-api-key': EXA_API_KEY, + }, + } + ); + + return response.data.results[0].text; + } catch (error) { + console.error('Error fetching content from Exa:', error); + throw error; + } +} diff --git a/server/src/services/clients/gmail.ts b/server/src/services/clients/gmail.ts new file mode 100644 index 0000000..a2ced8d --- /dev/null +++ b/server/src/services/clients/gmail.ts @@ -0,0 +1,98 @@ +import { google } from 'googleapis'; +import { OAuth2Client } from 'google-auth-library'; +import dotenv from 'dotenv'; + +dotenv.config(); + +const SCOPES = ['https://www.googleapis.com/auth/gmail.readonly']; +const TOKEN_PATH = 'token.json'; + +interface EmailThread { + [threadId: string]: Array<{ + sender: string; + subject: string; + body: string; + messageId: string; + }>; +} + +class GmailClient { + private auth: OAuth2Client; + + constructor() { + this.auth = new google.auth.OAuth2( + process.env.GMAIL_CLIENT_ID, + process.env.GMAIL_CLIENT_SECRET, + 'urn:ietf:wg:oauth:2.0:oob' + ); + + this.auth.setCredentials({ + refresh_token: process.env.GMAIL_REFRESH_TOKEN, + }); + } + + async readEmails(days: number = 7): Promise { + const gmail = google.gmail({ version: 'v1', auth: this.auth }); + const date = new Date(); + date.setDate(date.getDate() - days); + const query = `after:${date.getFullYear()}/${date.getMonth() + 1}/${date.getDate()}`; + + try { + const res = await gmail.users.messages.list({ + userId: 'me', + q: query, + }); + + const messages = res.data.messages || []; + const emailThreads: EmailThread = {}; + + for (const message of messages) { + const fullMessage = await gmail.users.messages.get({ + userId: 'me', + id: message.id!, + }); + + const threadId = fullMessage.data.threadId!; + const headers = fullMessage.data.payload?.headers; + const sender = headers?.find(h => h.name === 'From')?.value || ''; + const subject = headers?.find(h => h.name === 'Subject')?.value || ''; + const messageId = headers?.find(h => h.name === 'Message-ID')?.value || ''; + const body = this.getEmailBody(fullMessage.data); + + if (!emailThreads[threadId]) { + emailThreads[threadId] = []; + } + + emailThreads[threadId].push({ + sender, + subject, + body, + messageId, + }); + } + + return Object.values(emailThreads); + } catch (error) { + console.error('The API returned an error:', error); + throw error; + } + } + + private getEmailBody(message: any): string { + if (message.payload?.body?.data) { + return Buffer.from(message.payload.body.data, 'base64').toString(); + } + + if (message.payload?.parts) { + for (const part of message.payload.parts) { + if (part.mimeType === 'text/plain' && part.body?.data) { + return Buffer.from(part.body.data, 'base64').toString(); + } + } + } + + return ''; + } +} + +export const gmailClient = new GmailClient(); diff --git a/server/src/services/clients/openai.ts b/server/src/services/clients/openai.ts new file mode 100644 index 0000000..e69de29 diff --git a/server/src/services/dataLoaders/email.ts b/server/src/services/dataLoaders/email.ts new file mode 100644 index 0000000..0303991 --- /dev/null +++ b/server/src/services/dataLoaders/email.ts @@ -0,0 +1,22 @@ +import { createEmail } from '../../dao/emailDao'; +import { createEmbedding } from '../../dao/embeddingDao'; +import { getEmbedding } from '../embeddings/embed'; + +export async function processAndStoreEmail(email: any) { + const storedEmail = await createEmail({ + sender: email.sender, + subject: email.subject, + body: email.body, + messageId: email.messageId, + }); + + const embedding = await getEmbedding(email.body); + + await createEmbedding({ + embedding, + contentType: 'EMAIL', + emailId: storedEmail.id, + }); + + return storedEmail; +} diff --git a/server/src/services/dataLoaders/files.ts b/server/src/services/dataLoaders/files.ts new file mode 100644 index 0000000..36055ac --- /dev/null +++ b/server/src/services/dataLoaders/files.ts @@ -0,0 +1,123 @@ +import fs from 'fs'; +import path from 'path'; +import crypto from 'crypto'; +import { marked } from 'marked'; +import { PDFDocument } from 'pdf-lib'; +import { createWorker } from 'tesseract.js'; +import { container } from '../../utils/container'; +import { PrismaClient } from '@prisma/client'; +import { getEmbedding, chunkContent } from '../embeddings/embed'; +import { createFile } from '../../dao/fileDao'; +import { createEmbedding } from '../../dao/embeddingDao'; +import { fromPath } from 'pdf2pic'; + +const FOLDER_PATH = process.env.FOLDER_PATH || 'jeff_storage'; + +async function getFileContents(filePath: string): Promise { + const ext = path.extname(filePath).toLowerCase(); + + if (ext === '.md' || ext === '.txt') { + const content = fs.readFileSync(filePath, 'utf-8'); + return ext === '.md' ? marked(content) : content; + } else if (ext === '.pdf') { + return ocrPdf(filePath); + } else { + throw new Error(`Unsupported file type: ${ext}`); + } +} + +async function ocrPdf(filePath: string): Promise { + const worker = await createWorker(); + let text = ''; + + const options = { + density: 300, + saveFilename: "temp_page", + savePath: "./temp", + format: "png", + width: 2480, + height: 3508 + }; + + const convert = fromPath(filePath, options); + const pdfDoc = await PDFDocument.load(fs.readFileSync(filePath)); + + for (let i = 0; i < pdfDoc.getPageCount(); i++) { + const result = await convert(i + 1); + const base64 = (result as any).base64; + const { data: { text: pageText } } = await worker.recognize(`data:image/png;base64,${base64}`); + text += pageText + '\n'; + } + + await worker.terminate(); + return text.replace(/\0/g, ''); +} + +function getContentHash(content: string): string { + return crypto.createHash('sha256').update(content).digest('hex'); +} + +export async function processFiles(folderPath: string): Promise { + const prisma = container.get(PrismaClient); + + const processFile = async (filePath: string) => { + const ext = path.extname(filePath).toLowerCase(); + + if (['.md', '.txt', '.pdf'].includes(ext)) { + try { + const content = await getFileContents(filePath); + const chunks = chunkContent(content); + + for (let i = 0; i < chunks.length; i++) { + const chunk = chunks[i]; + const contentHash = getContentHash(chunk); + + const existingFile = await prisma.file.findFirst({ + where: { contentHash } + }); + + if (existingFile) continue; + + const embedding = await getEmbedding(chunk); + + const fileData = { + name: `${path.basename(filePath)}_chunk_${i + 1}`, + path: filePath, + content: chunk, + contentHash: contentHash, + }; + + const storedFile = await createFile(fileData); + await createEmbedding({ + embedding, + contentType: 'FILE', + fileId: storedFile.id, + }); + + console.log(`Stored file chunk: ${storedFile.id} - ${storedFile.name}`); + } + } catch (error) { + console.error(`Error processing file ${filePath}:`, error); + } + } + }; + + const processDirectory = async (dir: string) => { + const entries = fs.readdirSync(dir, { withFileTypes: true }); + + for (const entry of entries) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + await processDirectory(fullPath); + } else { + await processFile(fullPath); + } + } + }; + + await processDirectory(folderPath); +} + +if (require.main === module) { + processFiles(FOLDER_PATH).catch(console.error); +} diff --git a/server/src/services/embeddings/embed.ts b/server/src/services/embeddings/embed.ts new file mode 100644 index 0000000..9386a01 --- /dev/null +++ b/server/src/services/embeddings/embed.ts @@ -0,0 +1,48 @@ +import OpenAI from 'openai'; +import dotenv from 'dotenv'; +import { encode } from 'gpt-3-encoder'; +import { container } from '../../utils/container'; + +dotenv.config(); + +export function numTokensFromString(string: string, modelName: string = "text-embedding-3-large"): number { + return encode(string).length; +} + +export function chunkContent(content: string, maxTokens: number = 5000): string[] { + const chunks: string[] = []; + let currentChunk = ""; + let currentTokens = 0; + + for (const line of content.split('\n')) { + const lineTokens = numTokensFromString(line); + if (currentTokens + lineTokens > maxTokens) { + chunks.push(currentChunk.trim()); + currentChunk = line; + currentTokens = lineTokens; + } else { + currentChunk += line + '\n'; + currentTokens += lineTokens; + } + } + + if (currentChunk) { + chunks.push(currentChunk.trim()); + } + + return chunks; +} + +export async function getEmbedding(text: string): Promise { + try { + const openai = container.get(OpenAI); + const response = await openai.embeddings.create({ + model: "text-embedding-3-large", + input: text, + }); + return response.data[0].embedding + } catch (error) { + console.error('Error getting embedding:', error); + throw error; + } +} diff --git a/server/src/services/openai.ts b/server/src/services/openai.ts new file mode 100644 index 0000000..7c81c7a --- /dev/null +++ b/server/src/services/openai.ts @@ -0,0 +1,21 @@ +import OpenAI from 'openai'; +import { container } from '../utils/container'; + +export async function generateAnswerSummary(question: string, content: string, model: string = 'gpt-4'): Promise { + const openai = container.get(OpenAI); + + const messages = [ + { + role: "user" as const, + content: `Answer the following question: ${question}. Only use information from the following text and provide as brief of an answer as possible: \n ${content}` + } + ]; + + const response = await openai.chat.completions.create({ + model: model, + messages: messages, + temperature: 0.0 + }); + + return response.choices[0].message.content || ''; +} \ No newline at end of file diff --git a/server/src/utils/container.ts b/server/src/utils/container.ts new file mode 100644 index 0000000..2ca5162 --- /dev/null +++ b/server/src/utils/container.ts @@ -0,0 +1,33 @@ +import { Container, ContainerModule, interfaces } from 'inversify' +import dotenv from 'dotenv' +import { PrismaClient } from '@prisma/client' +import OpenAI from 'openai' + +dotenv.config(); + +export const container = new Container({ + autoBindInjectable: true, + skipBaseClassChecks: true, + defaultScope: 'Singleton', +}) + +export const TYPES = { + Logger: Symbol.for('Logger'), + LoggerFactory: Symbol.for('LoggerFactory'), +} + +export const coreBindingsModule = new ContainerModule((bind: interfaces.Bind) => { + + const prismaClient = new PrismaClient({ + log: [ + { emit: 'event', level: 'info' }, + { emit: 'event', level: 'warn' }, + { emit: 'event', level: 'error' }, + ], + }) + bind(PrismaClient).toConstantValue(prismaClient) + + bind(OpenAI).toConstantValue(new OpenAI({ apiKey: process.env.OPENAI_API_KEY })) +}) + +container.load(coreBindingsModule) diff --git a/server/tsconfig.json b/server/tsconfig.json new file mode 100644 index 0000000..73d3514 --- /dev/null +++ b/server/tsconfig.json @@ -0,0 +1,110 @@ +{ + "exclude": ["node_modules"], + "compilerOptions": { + "baseUrl": "./src", + /* Visit https://aka.ms/tsconfig to read more about this file */ + + /* Projects */ + // "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */ + // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */ + // "tsBuildInfoFile": "./.tsbuildinfo", /* Specify the path to .tsbuildinfo incremental compilation file. */ + // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects. */ + // "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */ + // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ + + /* Language and Environment */ + "target": "es2016", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ + // "lib": [], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ + // "jsx": "preserve", /* Specify what JSX code is generated. */ + // "experimentalDecorators": true, /* Enable experimental support for legacy experimental decorators. */ + // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ + // "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */ + // "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */ + // "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */ + // "reactNamespace": "", /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */ + // "noLib": true, /* Disable including any library files, including the default lib.d.ts. */ + // "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */ + // "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */ + + /* Modules */ + "module": "commonjs", /* Specify what module code is generated. */ + // "rootDir": "./", /* Specify the root folder within your source files. */ + // "moduleResolution": "node10", /* Specify how TypeScript looks up a file from a given module specifier. */ + // "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ + // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ + // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ + // "typeRoots": [], /* Specify multiple folders that act like './node_modules/@types'. */ + // "types": [], /* Specify type package names to be included without being referenced in a source file. */ + // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ + // "moduleSuffixes": [], /* List of file name suffixes to search when resolving a module. */ + // "allowImportingTsExtensions": true, /* Allow imports to include TypeScript file extensions. Requires '--moduleResolution bundler' and either '--noEmit' or '--emitDeclarationOnly' to be set. */ + // "resolvePackageJsonExports": true, /* Use the package.json 'exports' field when resolving package imports. */ + // "resolvePackageJsonImports": true, /* Use the package.json 'imports' field when resolving imports. */ + // "customConditions": [], /* Conditions to set in addition to the resolver-specific defaults when resolving imports. */ + // "resolveJsonModule": true, /* Enable importing .json files. */ + // "allowArbitraryExtensions": true, /* Enable importing files with any extension, provided a declaration file is present. */ + // "noResolve": true, /* Disallow 'import's, 'require's or ''s from expanding the number of files TypeScript should add to a project. */ + + /* JavaScript Support */ + // "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */ + // "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */ + // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */ + + /* Emit */ + // "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ + // "declarationMap": true, /* Create sourcemaps for d.ts files. */ + // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ + // "sourceMap": true, /* Create source map files for emitted JavaScript files. */ + // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ + // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */ + // "outDir": "./", /* Specify an output folder for all emitted files. */ + // "removeComments": true, /* Disable emitting comments. */ + // "noEmit": true, /* Disable emitting files from a compilation. */ + // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ + // "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */ + // "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */ + // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ + // "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */ + // "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */ + // "newLine": "crlf", /* Set the newline character for emitting files. */ + // "stripInternal": true, /* Disable emitting declarations that have '@internal' in their JSDoc comments. */ + // "noEmitHelpers": true, /* Disable generating custom helper functions like '__extends' in compiled output. */ + // "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */ + // "preserveConstEnums": true, /* Disable erasing 'const enum' declarations in generated code. */ + // "declarationDir": "./", /* Specify the output directory for generated declaration files. */ + + /* Interop Constraints */ + // "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */ + // "verbatimModuleSyntax": true, /* Do not transform or elide any imports or exports not marked as type-only, ensuring they are written in the output file's format based on the 'module' setting. */ + // "isolatedDeclarations": true, /* Require sufficient annotation on exports so other tools can trivially generate declaration files. */ + // "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */ + "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ + // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */ + "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ + + /* Type Checking */ + "strict": true, /* Enable all strict type-checking options. */ + // "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */ + // "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */ + // "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ + // "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */ + // "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ + // "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */ + // "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */ + // "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ + // "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */ + // "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */ + // "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ + // "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ + // "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ + // "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */ + // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ + // "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */ + // "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ + // "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ + + /* Completeness */ + // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ + "skipLibCheck": true /* Skip type checking all .d.ts files. */ + } +} diff --git a/server/apis/exa_client.py b/server2/apis/exa_client.py similarity index 100% rename from server/apis/exa_client.py rename to server2/apis/exa_client.py diff --git a/server/apis/gmail_client.py b/server2/apis/gmail_client.py similarity index 100% rename from server/apis/gmail_client.py rename to server2/apis/gmail_client.py diff --git a/server/apis/openai_client.py b/server2/apis/openai_client.py similarity index 100% rename from server/apis/openai_client.py rename to server2/apis/openai_client.py diff --git a/server/app.py b/server2/app.py similarity index 100% rename from server/app.py rename to server2/app.py diff --git a/server/constants.py b/server2/constants.py similarity index 100% rename from server/constants.py rename to server2/constants.py diff --git a/server/data_loaders/files.py b/server2/data_loaders/files.py similarity index 100% rename from server/data_loaders/files.py rename to server2/data_loaders/files.py diff --git a/server/data_loaders/gmail.py b/server2/data_loaders/gmail.py similarity index 100% rename from server/data_loaders/gmail.py rename to server2/data_loaders/gmail.py diff --git a/server/data_loaders/history.py b/server2/data_loaders/history.py similarity index 100% rename from server/data_loaders/history.py rename to server2/data_loaders/history.py diff --git a/server/database/db.py b/server2/database/db.py similarity index 100% rename from server/database/db.py rename to server2/database/db.py diff --git a/server/database/queries.py b/server2/database/queries.py similarity index 100% rename from server/database/queries.py rename to server2/database/queries.py diff --git a/server/database/tables.py b/server2/database/tables.py similarity index 100% rename from server/database/tables.py rename to server2/database/tables.py diff --git a/server/embeddings/embed.py b/server2/embeddings/embed.py similarity index 100% rename from server/embeddings/embed.py rename to server2/embeddings/embed.py diff --git a/server/types/email.py b/server2/types/email.py similarity index 100% rename from server/types/email.py rename to server2/types/email.py