From fa4a85da64e5341028e9ac2a45a9256a7d4441c6 Mon Sep 17 00:00:00 2001 From: Serge Bornow Date: Wed, 4 Mar 2015 09:37:26 -0500 Subject: [PATCH 01/16] fixed broken apache storm distribution --- provision.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/provision.sh b/provision.sh index c8bdb24..2c0b88f 100755 --- a/provision.sh +++ b/provision.sh @@ -22,8 +22,8 @@ echo "Storm..." # TODO maybe make this use the best mirror always? sudo mkdir /opt/storm cd /opt/storm -sudo wget http://mirror.cogentco.com/pub/apache/incubator/storm/apache-storm-0.9.2-incubating/apache-storm-0.9.2-incubating.tar.gz -sudo tar xvzf apache-storm-0.9.2-incubating.tar.gz -sudo rm apache-storm-0.9.2-incubating.tar.gz -sudo chmod +x /opt/storm/apache-storm-0.9.2-incubating/bin/storm -sudo ln -s /opt/storm/apache-storm-0.9.2-incubating/bin/storm /usr/bin/storm +sudo wget http://apache.mirror.vexxhost.com/storm/apache-storm-0.9.3/apache-storm-0.9.3.tar.gz +sudo tar xvzf apache-storm-0.9.3.tar.gz +sudo rm apache-storm-0.9.3.tar.gz +sudo chmod +x /opt/storm/apache-storm-0.9.3/bin/storm +sudo ln -s /opt/storm/apache-storm-0.9.3/bin/storm /usr/bin/storm From b4f2b786ea40118dad3443c194f75cf812d9d370 Mon Sep 17 00:00:00 2001 From: Serge Bornow Date: Wed, 4 Mar 2015 11:38:03 -0500 Subject: [PATCH 02/16] Fixed the issue of re-running vagrant provision with updated storm and proper provision task in vagrant file --- Vagrantfile | 1 + provision.sh => bootstrap.sh | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) rename provision.sh => bootstrap.sh (75%) diff --git a/Vagrantfile b/Vagrantfile index bf4bca6..25b95e8 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -7,4 +7,5 @@ VAGRANTFILE_API_VERSION = "2" Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.box = "udacity/ud381" config.vm.network :forwarded_port, guest: 5000, host: 5000 + config.vm.provision :shell, path: "bootstrap.sh" end diff --git a/provision.sh b/bootstrap.sh similarity index 75% rename from provision.sh rename to bootstrap.sh index 2c0b88f..36f1694 100755 --- a/provision.sh +++ b/bootstrap.sh @@ -1,4 +1,4 @@ -#!/bin/bash -i +#!/usr/bin/env bash # The following are documented (and stolen from) here: # http://redsymbol.net/articles/unofficial-bash-strict-mode/ @@ -18,11 +18,19 @@ sudo apt-get -y install default-jdk maven vim zookeeper zookeeperd redis-server sudo pip install flask redis -echo "Storm..." -# TODO maybe make this use the best mirror always? +echo "Storm.." + + +sudo rm -rf /opt/storm 2> /dev/null | echo "Removed old" +sudo rm /usr/bin/storm 2> /dev/null | echo "Removed old" + + + sudo mkdir /opt/storm cd /opt/storm -sudo wget http://apache.mirror.vexxhost.com/storm/apache-storm-0.9.3/apache-storm-0.9.3.tar.gz + +# -q option will supress progress +sudo wget -q http://apache.mirror.vexxhost.com/storm/apache-storm-0.9.3/apache-storm-0.9.3.tar.gz sudo tar xvzf apache-storm-0.9.3.tar.gz sudo rm apache-storm-0.9.3.tar.gz sudo chmod +x /opt/storm/apache-storm-0.9.3/bin/storm From c6fab16db490125a6f1f0d3f37655939cf90fefa Mon Sep 17 00:00:00 2001 From: Serge Bornow Date: Wed, 4 Mar 2015 11:39:20 -0500 Subject: [PATCH 03/16] updated readme --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 51c6e3e..6779837 100644 --- a/README.md +++ b/README.md @@ -3,3 +3,10 @@ Udacity and Twitter bring you Real-Time Analytics with Apache Storm Join the course for free: www.udacity.com/course/ud381 + + +### Serge's changes + + - updated to point to fixed apache storm repo + - added provision task for vagrant + - run or re-run vagrant provision - will remove old, reinstall storm \ No newline at end of file From c9188e53f546fadce4d019310ab0a1ee4ad916b2 Mon Sep 17 00:00:00 2001 From: Serge Bornow Date: Fri, 6 Mar 2015 19:29:19 -0500 Subject: [PATCH 04/16] my pic --- viz/static/MyPicture.jpg | Bin 0 -> 7094 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 viz/static/MyPicture.jpg diff --git a/viz/static/MyPicture.jpg b/viz/static/MyPicture.jpg new file mode 100644 index 0000000000000000000000000000000000000000..405010774f84bdd7681c54caaedf62c51fa1ed81 GIT binary patch literal 7094 zcmbW5Wl$T?*X9!fgaXBi6WpaOQVPM{EjSb??iO5&OVbb}MT-~L7I)XSNb%yH0!0fg zR#^V;yR$p9pLX{-bI+H1?w#j1XXf0Qd%tkM4j@reQdI%~u>b(zLjdlV0P+AF5H>b8 z2fd3`H z{}d1lg#F+W?-4%1LqQ`601F5LVLkYN0C=bleb@(Jlj4vu3&`R=fmwlB;N*f4$pv_< za-VxCbibdl30ZqZKEkJ@qNbr`=iubx=6NCfQbZIgCa<8Vq^zQ z@8Ia!B^!;Qw#|vHTwvh!h)#Spb(z76!J0KVcD!z$2GSF8JL0h*e1Ugu>eEJ3b}biw*YE z|DgRB+5a6_~#25$9a?KQx+-(QsnS@C(QJ!KZM9)5hqcupc;!SIvY_lERRBd8Z@)o z7Obxa|6xLWON6}&4rEr@3gf?EO%7MKXxW}?s~0ai8nqTbl6A1hYmr1Gs?E;a!pcTB zick8?0~cM5l=ejvnd#r-vhI@`i8SqpW!kthWUT8am#ZR{Uv2UBV6J`}j4sFNZpwQ7 zA~fNY6vsb2pP-j_@W841lY)~VC&Y^tK<^Ts`h@PIE6JPVu)9tsQO$j=(g!aviI5`{zo!@_xPeD+sO*@@}d^Bl@F2f9nk>Y$^yKQz^gi<2#8 zoBcw)iv^o!s`m{N9W^(z@&c%(1i-VuX34eX;ucdxK zGm!jgZ<$d}(&cNVn*M6pU#?AN170Czwo?^c@!#66o=zWWMyqOk3SaiHJK~s6G zr#1Ompj_aP8C&{JA-zAnn}wD#fLdu&01`tl&0}@Mqjv{~ZRM4c1GpQ8SsQ`L?5#Ic_;SN zE0FgUs3fo=s49kOk(h1Ws2E(GNK`UpR;LN)H?bWB5Pn_hO^cVPdoODK%Zxerekb@)Z5jgmp7_&6>O~lK}VeYnRCOq!Z@&U$x zYX_!=*X-Nr%!W#CTPYvGCu2d){S&cNL6J1iHc$#%ICa{)VzlE1WJ)ui&OR)h1-tQV z>=MHIBO33_B7Q3Rz9GBx4laYH8JlrSW$wLJ3JBY9&$M$d++}=`B)cqz*wcz}adBp9 ziqYrVLZwDm($9QSl@7S_FuipBULY_8k6Ho?_wBDz&b;`8|5Kg;jwt`d_^pBBtI85SKTaVbgYKXXLJ6%*%<`kycHKoyV^%fTu53Y38QnnQw$X zJ>i5U>PWgc=$)LWH8%?=NYnv)E^34f8~IaHsRf(C^!nGddw=b#CV0k?HEoQ>PU&ga zG1(%1<7^0h!f1bN{lnv_4)JvdL`f9vuCKGzhVb1@_*&m;o`z9N_Ez%zl@Z!7uRfjo z*y@1i)2=)`C6pT<5wMx1Z2Dz)i%6TkJTC99QzDX>{;>#%LPl;K6V4r@^($G=0Te0d zgj8Keh2mh@KGlF;l$&@C&?dC`jbX7$YAP-Nb5_O_l#S3o!c3Lp_QPlH0cu>^5MZ*R zp!q}|n#sD{+1al3(MZpj#pnvszza3C%-_o)`R*|5-10&~?44|ug7*Ox%;~eGa?APo z7Uy~G>r3;KT}fAo!Ec5x%$jQhzr^maT54n4cnN|l=j2=87*EtvoOk!N<24-Ob4_ML z1qpAmH&dOK6nw0>ym{-wz_;R4=W7Dbo<{$o+wlishb8#Wy)KfK385n^`S| zy#zDB)!r7Ek6%818j@S7ps7BCQoz>;da&{XWw(xLZH)MjtASv|&V>tyf$MXM6)g~h za-)s&CnhD%Jm;~(&567&o>xyR75xzUv2-m5V*zmpvoP0X9>%058G4*h)l*h^b3LW^ zO5LKEi`tA3J*yW~DfJ~Q2@Ui1^py-sQjgVkEm8xYqKZ8N3ETp*`K1#=O*V2gh23c9 z4)Hh?up<`cM+0`d-d%DEc;G2jQ6JhwB?cjVLT854#-|;RMUOQn7#gaSLY;^NdTL+% zQ93WX0xdyH(M?%i$|XGEZ#w}cBr?0jt~P6p3O_`zCpm=U#|1sPiyZPylOiUn1!4tc z(u(+Arr-#~F`aW1C*g!#BWd5BW;sR4Q$wy{@}#%sTot72U>b2vlkXIadMNJCjaRo7 zRaQ+%RPeO_=;Moq5kb%yZEX!R;oQ`7|2PwO79^-2;DW!DoA$Uy$?s)~)VQqFMS;hz=!hPo*y?GKr80XT zuZ<`lul4FaJK)(I^?@3vJVCINJ2}Kt(}Xrip$I(z*KW>pbse4b9`wFr$-3U8jGWR< zQ4z>KuW)019;C3?c$DK9{S{6k^&!>;-7&}7=76GpOS2ItEPo>|^PTQcigjkU^8;HT zY-l{}I2C(Z`JIIDi-gCi#WO&II5=o@b1^ywmBK%fF+e=J~nQ z@at(}NfdVhb4`y3OHz`PmMRUSnvFw^Q$p0mX7cDcd3&t*ZZZOsy&1v6jjK%YgveQ$ zwaJv+H;qNA^3&ZY|bNgNevaiJiqZ2J; zHb48U;qIjrSEWC0=*uUGddwm4mEj++@&J5saqj@a9sBU+LfK-@X(V}}ri=8Ry1O$! zrYU(KL+p{taj0n3kx#HV zRaHa0>72R=#t&h?!o>x&xfpU@bZnDscw?SgsF2aS*jQTfiiXM1PghdWS-FH)VIT!S zAL?S<>_R=NC|;i?L|}JDen{Pe+cCXK&hy6zuRl#M;=m$vvr{r_1sQ5|loSY_>*yX# zCsW5~>j{ka`*hcsJ3jbBG=s5TInG*rY3U}-L)e~$8MSgfUmYUf)pW8Jdl6hGE9>}r zAH(zUl#e&&Qf3Ti@Rp=(i_CWj3M~x@>AbNGdKp<`a$Gan-2uNHzF=wC?a29N*~p)saA(I1SM#^hKx`QrqMr#4{$4mr`$MamSx10HS&zGX+#M6KV<;{9uma1n?ViBfE)b-1(@j zU+Dts*&#ji-x-O071A}D*tST1c^m zf7%5;BOR9mQEb-rA)(IJlXh*MZ!E}zvVNWeB9|RXc+wM?Q8(Hzi$7&zlOE_ciKh?R z<_ z$moLCTAZ5sWB2LVxM4v3-Zzrq~bj=cfYwCs#`OE8DLAeQL*}95OX9HiTYQYrn z+C#epLuFmn9t`J;!e5Gt{$6)Ay|aky@>;&r@x183RFw;<*?pM&u5k)K{J?DfT@|G! zgO;FQ7F~O5&iUuvZbD4Ers)OWta_mGx6#9_>MHD+JD#DTZI!mrjOA8r)>bH^nRRPU zkjs*QXOjrp?@<3iDt8^UV|fps!jRHYe@og znUl8qUQ&n{!8=Z zJ?^3tFwNV?(*EB94m6)eWdWHpMz$K2IW;22RD!U$>5``+NFZ=ULLet6xd8`j-!v2i zEpGwF?ZEP(7z%y}Xl<(PSKxP3{5Ztk9pfssFA%s+McCPP5?1WZ^kj%f#UX7~F^f$8 zBbI>t!W7Yz#`SccX*Bv3pIf`9Y1rEuT3*e=oR8IBdBlwi9xo2mTNbF8coD~qA4G*8 zGbn~^Dl*0FOa3-~dLc=$ruY8)v+efG?)v=TI17{X@a_b^xjUNK+I(rp6fT>kj*KDI zT9L_Tjok#t4bwJFnV=mupTnOr2ZJ6z_N92cJl%aq4`<5t=quu*Kr>`sVK7sr5if(+ z{RJKa40O|Oe7Uc36pjS8?M~AsB>dOsv}1kRInalseM<(tsz5xnNOEv~PVCT%lSI$Rcms3?(i1 zfcT)un%w^HTHLJU7ZUOVB%^#&BwpY#{Vs-pK6)WTbbnSkDcH-c#6A0W!*#Hc31#-# z-t>w}_NSg&!b?=i+Q2vR^XeFq3SGeKoY6^5kE#<_!3V#PlZH-?2cE@H-as z(3Gdr21s)}Gy%^kGi>2HiFelL-N9qES$95nGP@%%PH6l^joSNS&3zx8TH1=ev|jPR z7l-`?TQuF>&wP0xw%30#ZW;Fue8;J~_reoXA3ba8aOn@Mp z5ecG_rwI9Qp=^Y%2w|4Z^RFo74LOpbVbQ{j1+$HMMRB~-7wd^?N$5S*aAh+_ZzWH$ z?RlpjNg-5AITc`i#s-DATV#lBL!L2Sxse`Elv++l7ss-wcjoZ9Hoqa%j=hegBb$F6 z{hHI9^*v#9EH;Uve>d^1^K{RPgFSq0s%7h-8$++C6?*c@&`gGTVyb>-HG%UnpQc@G z*+$tvH3V{V%$j=cQ>o@$5>(zR9F& z3Rjuw9?)pE+FWM*W0)<0OJ9+bqegXMOqIhp?H{ZU9RnNll;Uo8b1S zRvN$dl?Bo;4N(G1_*HvgJEpL49Q}NAt)2O{*oBp2I*4R{?)1kV5yVwNa}2dmn{sPY z9_A7ddvr<2+C3sk?#a6*?W9fd3|;a(&UFVC=+|D_yDQo$Ref8zJI8eAyAl_r_UkwN z0_{xt;#P>++;pDi9$=DKqZovf{ri>0)!bQ!Zu!P!l49UQ+^D(r464hrR^3o{5K$&6 zC_aZa`ku|-HcO96-wN1h_MM&5{3&IWAjo)fQAiwrEk5$q8)vNT9zZ~`cC}UP<@O0l znhOOZ^(^Ze?d}2kMd-CGOn{?MqxJq%qB@UwF+Aw&CMogwuz_jI%cz3n#{+yTIKTgm6@EtAv zcaj409hs;WRK`O<$dIoa;7A3rlb>{NOyHSf6mKawa%nsBcxUyUIR$EWBcWY}lRg74FUKZyg#; zEF+FTJR!`yuf?5!oX>BNczJa6wTtl{kTh8P&xc}9%6OUD-E}_u(h*es3UjkI)%o(smMA37jpQk^8 z{6^t8RpH2N~*VrI- z3w3!Lmcw(h8ylOUwmR+~%xL+Ow!n^nL$)S&Rh9)KL3{Qb=kmCp_Mp76Tg3tNl(5u> zc%pVg>r{Ja0{;@u(bO#DEEcVy(@&eKv-&GIRLBfg`ts1`M~<_zNNd53?5*NGAX-_5 z)-aPL!AM80AkNrEg7P{y>h!!e$sN+3GLWLp(Rk~L@BJ1A-)+~DND8G%b+W)fS&nV3Qk7p)1Ats{Yo4}Z}_Xh@o#IboR0Fhu|t#NCk zOuRrin3H1nH8GW6N9j#v*2TMDPwS<;RJP~5FBA5D*=pkV-C2_?`?Lns>jPe6c+*1nAP_Y?LZ~C3bh~85hia(*NZ#~nY`??TU5kk z$7gi9)yNM*qDrf|sXcq(tJzl|=(;Cd(^;@1kV=<(;u5a-l5Ev=d<*It#>2(#?8$aI zsb{Xy-A*&)6<93WrTAJ8(PX_ioVYo>0n;?k+T^seH9|pvOSo0xxWP8HFD_(}sY3xJ zD&hcDjy5fIM?oW=Dp;HUIB(7=u4bhOn+IU-;s86kwp2K$yaya(pP9&E0D9reo*jvf z6pK=wL^&z#U~m(9P*)ShCOAL!B1UhyITUop#Skf=Gs4_ii^CBd$>5oo_>Z0{k4RSj zZF=i*Z+U;_JS$5oEE`@eEfk9CW@6~ecJtla?rbK} zJM_oP&J{6{VHuT-H7rpqstiFXFw*!lANGo^7VytABd=V82u}}8RBwhAC6RQM>h6)` zG&B;wjeJHzK2vMSV3zg$D>4|7w9*Ka zv)BUL^5q=%#TbX~g5fgMS}0eMAN+hm+9EQigL=X1m4q%UmbQ66GIuEH%OB(;Ux}o# z;VngF+V}05b@1E%$9tt}nkFf|;s)bEID&V{t_EzmOoqAYO$e_b2zC+T^F)wtXofu}A<3C15 zcs84Qin}HDjsN*dB!yY<-;d>wi%`b(tb}*^7=e}%Y+fpqjlt4{wXnk|Hk Date: Sat, 7 Mar 2015 22:02:54 -0500 Subject: [PATCH 05/16] updated storm and java target to java 1.8 --- lesson1/stage1/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lesson1/stage1/pom.xml b/lesson1/stage1/pom.xml index a4d7881..f7bffc8 100644 --- a/lesson1/stage1/pom.xml +++ b/lesson1/stage1/pom.xml @@ -59,7 +59,7 @@ org.apache.storm storm-core - 0.9.2-incubating + 0.9.3 provided @@ -174,7 +174,7 @@ 1.7 - 1.7 + 1.8 From 92567bd3151cc2cd9fd39920f44565d1741b6e0f Mon Sep 17 00:00:00 2001 From: Serge Bornow Date: Sun, 8 Mar 2015 08:29:27 -0400 Subject: [PATCH 06/16] redis integrated into report topology to push results from bolts --- lesson1/stage1/pom.xml | 136 +++++++++--------- .../storm/ReporterExclamationTopology.java | 23 +-- viz/templates/cloud.html | 4 +- 3 files changed, 89 insertions(+), 74 deletions(-) diff --git a/lesson1/stage1/pom.xml b/lesson1/stage1/pom.xml index f7bffc8..ac08328 100644 --- a/lesson1/stage1/pom.xml +++ b/lesson1/stage1/pom.xml @@ -27,75 +27,81 @@ - junit - junit - 3.8.1 - test - - - org.testng - testng - 6.8.5 - test - - - org.mockito - mockito-all - 1.9.0 - test - - - org.easytesting - fest-assert-core - 2.0M8 - test - - - org.jmock - jmock - 2.6.0 - test - - - org.apache.storm - storm-core - 0.9.3 - - provided - - - commons-collections - commons-collections - 3.2.1 - - - com.google.guava - guava - 15.0 - - - org.twitter4j - twitter4j-core - [3.0,) - - - org.twitter4j - twitter4j-stream - [3.0,) - + com.lambdaworks + lettuce + 2.3.3 + + + + junit + junit + 3.8.1 + test + + + org.testng + testng + 6.8.5 + test + + + org.mockito + mockito-all + 1.9.0 + test + + + org.easytesting + fest-assert-core + 2.0M8 + test + + + org.jmock + jmock + 2.6.0 + test + + + org.apache.storm + storm-core + 0.9.3 + + provided + + + commons-collections + commons-collections + 3.2.1 + + + com.google.guava + guava + 15.0 + + + org.twitter4j + twitter4j-core + [3.0,) + + + org.twitter4j + twitter4j-stream + [3.0,) + - + --> + - - src/jvm - + + src/jvm + + --> maven-assembly-plugin @@ -172,11 +178,11 @@ maven-compiler-plugin 3.1 - + 1.7 1.8 - - + + diff --git a/lesson1/stage1/src/jvm/udacity/storm/ReporterExclamationTopology.java b/lesson1/stage1/src/jvm/udacity/storm/ReporterExclamationTopology.java index 98f4c38..a6d4e95 100644 --- a/lesson1/stage1/src/jvm/udacity/storm/ReporterExclamationTopology.java +++ b/lesson1/stage1/src/jvm/udacity/storm/ReporterExclamationTopology.java @@ -16,16 +16,23 @@ import java.util.Map; + +import com.lambdaworks.redis.RedisClient; + +import com.lambdaworks.redis.RedisConnection; + + + //********* TO DO 1-of-4 imported http://mvnrepository.com/artifact/com.lambdaworks/lettuce/ // COPY AND PASE: following code into pom.xml file (located lesson1/stage1/pom.xml) -// +// // com.lambdaworks // lettuce // 2.3.3 -// -// +// + //********* END 1-of-4 /** @@ -54,7 +61,7 @@ public static class ExclamationBolt extends BaseRichBolt //********* TO DO 2-of-4 // place holder to keep the connection to redis - + RedisConnection redis; //********* END 2-of-4 @Override @@ -68,9 +75,9 @@ public void prepare( //********* TO DO 3-of-4 // instantiate a redis connection - + RedisClient client = new RedisClient("localhost",6379); // initiate the actual connection - + redis = client.connect(); //********* END 3-of-4 } @@ -88,8 +95,8 @@ public void execute(Tuple tuple) _collector.emit(tuple, new Values(exclamatedWord.toString())); //********* TO DO 4-of-4 Uncomment redis reporter - //long count = 30; - //redis.publish("WordCountTopology", exclamatedWord.toString() + "|" + Long.toString(count)); + long count = 30; + redis.publish("WordCountTopology", exclamatedWord.toString() + "|" + Long.toString(count)); //********* END 4-of-4 } diff --git a/viz/templates/cloud.html b/viz/templates/cloud.html index 3c69d58..91f49bc 100644 --- a/viz/templates/cloud.html +++ b/viz/templates/cloud.html @@ -40,7 +40,9 @@

Udacity and Twitter bring you Real-Time Analytics with

 Smiley face  Twitter logo -   Word Cloud!!!

+  Serge Profile +
+    Serge's Word Cloud!!!