From 0d2d278260d0ebd1f447aa832dfee09351a95229 Mon Sep 17 00:00:00 2001 From: RobViren Date: Mon, 1 Nov 2021 18:38:58 -0500 Subject: [PATCH] rebasing --- .gitignore | 9 ++ buildmac.sh | 3 + config.xlsx | Bin 0 -> 55503 bytes go.mod | 21 ++++ go.sum | 41 +++++++ main.go | 80 ++++++++++++++ runner/check.go | 9 ++ runner/facebook.go | 102 +++++++++++++++++ runner/instagram.go | 149 +++++++++++++++++++++++++ runner/js/facebook_data.js | 106 ++++++++++++++++++ runner/js/instagram_followers_urls.js | 24 ++++ runner/js/instagram_liked_by.js | 3 + runner/js/instagram_liked_by_url.js | 10 ++ runner/js/instagram_post_data.js | 37 +++++++ runner/js/scroll.js | 26 +++++ runner/logger.go | 81 ++++++++++++++ runner/result.go | 9 ++ runner/runner.go | 152 ++++++++++++++++++++++++++ runner/scroll.go | 13 +++ 19 files changed, 875 insertions(+) create mode 100644 .gitignore create mode 100755 buildmac.sh create mode 100755 config.xlsx create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 runner/check.go create mode 100644 runner/facebook.go create mode 100644 runner/instagram.go create mode 100644 runner/js/facebook_data.js create mode 100644 runner/js/instagram_followers_urls.js create mode 100644 runner/js/instagram_liked_by.js create mode 100644 runner/js/instagram_liked_by_url.js create mode 100644 runner/js/instagram_post_data.js create mode 100644 runner/js/scroll.js create mode 100644 runner/logger.go create mode 100644 runner/result.go create mode 100644 runner/runner.go create mode 100644 runner/scroll.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e38b63e --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +/browser +browser +data-* +log.tsv +datahop* +config.bak +data.xlsx +log.xlsx +log-* \ No newline at end of file diff --git a/buildmac.sh b/buildmac.sh new file mode 100755 index 0000000..55f889a --- /dev/null +++ b/buildmac.sh @@ -0,0 +1,3 @@ +#!/usr/bin/zsh + +GOOS=darwin GOARCH=arm64 go build -x \ No newline at end of file diff --git a/config.xlsx b/config.xlsx new file mode 100755 index 0000000000000000000000000000000000000000..b8b913771f935c3193a8afbe51183cf9b5ce4d2e GIT binary patch literal 55503 zcmeHw2|SeD|F$Sa$y$=K%bF1)>qxR@*G8JiWC{|n+U|WkT+S}$4wmM22nV<#&jq+M4^&BE#R36Mb5roonmJ+CP*9|;r=VCK+W{JU1Ke_{Vqy#s3F0jNMB1svzVzIj zY|2u`-9`@-jZGF$uobV_cv`ZvYHHjqG0(nACm=(CuO0VEcJSn(zA{W+5Os5a$n}T@ zTOFPVf8mnbOj9$(ixa`BwGziAQu+AaQv|%rDA*&dVi@oD>a}Z8bo|YL*y}MP%6(5y z4#yr}$`-Meqq{A&*V}$wg-Jfb`gt**L8^xhvr`4P0R)@R@i2c~bdQZ)>!CE)L-yN^ zjxLzKE1-1Bh`)e8d9zLHo~vum?e6%^*?uXkR2~#jp9+UWa17I8CIeLMT+pMRTn=Yl zC^Md@XxIx8xt3gPzU@ZszAL$SXxtqtypM*fsstUc?mW{|nA47zGJ8v<+NmVZh)wW! zI^0pxYBRgol2gMyc<3wxFKov!%ku*V601WEyHKG?6S+mAg3d*^wXd5HT{L%j+U;qn z^7F14*gBY|sFxsW%BALo@PT8zKPDtxFr(>iJJhjG{+cCS&#g69JqpGh=S*KMP+F_( zR4n&?B$C zTTP4m4o@Kt+MX4E_r3=hE)-B~QVxt57bGm&Np{f&xCm+01raj;1 z--ofot^bw~R_`4P&r?!R$O43*`%H*GI!9iNy;Lc;m%hM9c~y|&4jgn-uq#%DxY^Xp#~duJs-qTD(*R5a)f4c6J^-7XXhWa zNk0mwfBT+^cQi{lt2u;!p-Qw;>-6a6OL=Z#vTG^e%;nB^Oc*StcV2y!G#n!2ZfMJ! zP1R}_oVO{>e^%2lLU4mvQ2L49*t-$fi(FChu=9JLKDeFL&n{LhFC`jgSy|jAY(J#t z-R!k50~^`34KfimsAg(bDp|#77OP+dIo$a428GaN?YtWYOYX#IPuzA+FSr?%Dj>|{ zsk08V?TY61K@)4=9k;0rRmKa@ohc06=ccE1TX>RuV_!|}$jdHFd(kuY!pAQ>y`r#3 zKE1MZu;;L0S8Mzu%9xGkJRUGQ?6ld$@A@KZ)23GqoSU~aZ<(v497JM4_F z-x=@oG94_i*GCzcL(y+k;IbU#$TfPXXj5ts5zddYqJUzE6#X!y8X^IEc$mBV`5kg(1 z)8OAfV(eX6RV6y-WlxpDIPOh7*WTy8n`_JM^aqagnSnH)1~JDUon=eBiJTA^E6qY8 z2lX`Ed(YpkD7$xu&A{IM67iiVy5XqIajB9`v2j<2WfOEh+#9f!NH_^Onrp|Q`e+vO zXg5Asz7!j)oLW6xO&cW9bw+-2v^%cH=jF5c@!<=*icj&5q3-Q&LCvnn)aqsX~^gz_Eq;(WPi?T1ju4fEIyZwEAvlYhs zIqM7bZQ_=vWwnAXiCJE6hFYn8Wu@n0`P-#)gSj=&E@B|I54ONs_y5g%B_V9%Y zU%5x{baQJM$yXjmHK{EfIHaM`Cw?KTUV@iTyM?`P!?un@;=L*QZyO@;QTpp5paHmm z#$fomAwI8o6Pq5r9(9L~#*vq38J;Yw@aLl8 zY{$FGq|5dPy*7V@Q?O!7ajoquITh&Dm2g)yN+r!CQ2*|F)gv)ZtTzIcw_3zfgsm%M zz-BWTz*3ANGq)MF6gLZLS?$}{E_I})did;m4U34u(`(!)rA~Pes0xerRlG!Jox`<3 zIOge-%MBaL5qU?VrDHoAaQE{@+dPG8J?j@eqqQd&xt_S4z^?tal5LDzX+8jz(*fAF z5%?dryg7j^Z#10qq71aD_H6PdYCT=NQ$)j4=m0tJCZT?5O+iCfOqq1XWsk-At^3dQ z)pB^+#d~6z3cDN44H40ygVv|s-cJ!UaaEe5PNJs|mDfKgE_Y*Jln?KgHx94$q*7l+ zNh}x}!yj9mV$hs7VU^dfYH_@I3?r zDX+0_^d#DAM`zu3v1B&Z%dx}~j(Go@yl!u4jBti60#q<1e=2s0*wFGLqMyujfKaaI=G+2bM@zH zT?35Huqkj}YUmyH(+N+br^WRK3~!mC_$OQlBGamZ!!^ zXysmU)6^V35q&9QYEYj)^|F5H%Un}JP{lF*E67uF_qb*RF%=OqHryfpe*V`7vgnmR zY8|JeyYY-eWcNmq+2Ld2%|qwYMJF0cr3+4ov1+$-&0IPs*P>g-;5MK)&QZpY7FxsL z)aQDNVB6QIS7n0pcr#nY#tafGmW|zJ&e4qj6D@4kEnb+7F zi#?xwsp(wa`P}(*>ol4*XNEbxuzmC0ck<`QOsh=f8;3Km*3)na=(4`0`1qRe`qs6N zonvf-r0Fe!S$Hae5bhF)M%aO7(;mA9quXBDPYgqQVxLjnHp)}JD|fta##ZoElMh-H zK6s;*r99bLsg8&4rBRakYwgrDBSss~&M}s3KX?T;#7HZ7AStRzrC_V9PTH0YZK^&a zT3cu~>7?dtQ9SD_y+h`-`3r}&!cJ>CAIqI)#hE-yk6ouDDeO<{KY+fF9#`Rd&v1(Q zzVywpj*U_l5f{flz5diiD^ZVTzcJ6bX$DbbOA$!ZJEb{tMLm8K6Zb7X%HX##14;M8 zwH7uGu+Ury%FgU$5X-1zEa}#>a&eMP-A!pU{$XuE^!W?jO9gxA!t7`o-(KzOucFh(1E_%*Xx;mA-3!h3YMb8cZr<{Gb9aH|Gts(FTCjV# z($9+=m~_~tH6arnQs!mblqZu5o#E++71wX_ijuj5m&-fwF1c9aK5k;hle6+tm!1-f zdgBG^w4RVa3m^EUXON*tA$p-b4V(C6Dlk$uNhh+V#!Da%r-sa)_3Kl&54`-uvA=A# zJC)gaDf3_@VRPLA<4L=R#@ecDBo!B`UwW*UN-gTidqL;SklUZ6z6gnT-s?9QW&I%N z3}u$B@>}L$gY6Zw#;=#y9CIWEt!JxU9&Dj<*gnw~YwJF%a>CRQaw+rnOAg)bw~vl2 zUDtZkC44w$X#)D~@aZ5XQhI=zf?~Y@1qIXp7%!IhWMd{e2xj$d1d+4~RuudC-jp{5 zss75*+k~(5&$_$Se{8fpbEC&}U7c1;PD@T8&+O|1ZCh#5PY&g4YPT5%qnq7a?X9fP zc3B6!YXj@$C7Y&^FBbRQYa9@nR}9!391*=}pX-!r$KBWIsp^$!-^DQlwLPvO{4rd- zU&+SQh$l}ZA#Rd?fGM0yd2x!#!Ss&dqYOjM4)L&obsP+wA8JpyAA6MMsKx6O?%j4e z!?xs|=!I*zty2Z-(~D|P+cHZvN59^s;*8_)g;qw1`&Anx1bfM|>{xH|(zDpYMgKa~ zzsX6s>`=;q7ad!JI49HAl;n#(zH2Q0^5L`II^o^>w5VzH#UF%8sI}8WcbSE8y}E1L zLSt3jlRGoY&RTS}!aIMU-N|RtT8E{i3$1O1;ysjfa^KxX9KL5gA{4LYgOD@ra*zzY6?xO?~IrrBy)!PR+}QW0-UYpR*wY^Rk7O$1RTv_=9ClKQ!4qh`7Iw&bSlDSEX;J`?}JD(3(4__j1zL zs*WD1h+LRhcmz2(arTW{-$WeL6Wi3^)Qr(Xqp(Ax{K{~24JQu=1pnd%5z8e6PwytnZ-?O#=g=~wxFGj#{)wx)}Ls%ee5MOnaJ-E zthwWE;G}t~Ntfz|*BH0V4d|skOtR$tqI+K^%4gLaPpz_dvR2_T+atoFa;;$F(MeVF{5QfM z3--OIKH`wdy=Y|4&MvTJ?bQR@)GHVx3mH7M57Aw{b6W7xlgI}(w~}ZWLpWGe98NX% zN7y5q&lj+=bRP7Q*^E2Jp?vGq{pW`ZkndiY<2SK~;s+Sd2QtkDY#EV0|L)_dSCY); zbEU>!FKF=aW*Kwk~VS)s$9DnqH7vn}}f_O&#__k>W2^{Sk`ZiLn1 zZ$AhL6Lgn9@k$&vbj|bF*o49TJ4YH%Zrd&wS9U$#$?^`(-rXK%XI_aV#jwQ$ozrnO z-Y(J+c@L2fSC@*~AnE1ddyoBfzLZFVt>sS3U1u!QYor+8Pdg&#El!lWtDf0^@_0jT zOh;*2+APMG_e`DI*zJ$U)T8U@vh!^$z(mQHf zE(g^d)*P3B)*NnAGebNb#^tzl9C>9DGv5|6B*a_&)cufM4BJ3Kw%?HTuH5eFU>*z1 z&Yh{;2|}A{D)JcI;wGv zLH>lNX7%D&%YD6Rty5-WyK#3?jwCOBvh$M3-T7GHfx$CI>31P()ijOUi9Y+ zJTz=jcoWEPw~OyEZ)ty9SA3YU`nrqCJZiIE3qOYCqJRCk| z8pxf=aQ1ST{`30rHXfaD_-yEeha76OM3!9xA9qcOif-H|pwlu;;Ly_+yXbdEDcuH% zb(FWo3r@XRV*Pf?kts=izr12E2UhIcSEroSMf=yg>(2}L2HP5O20z%rj}@_r0s30e zCserGUwKh*=j~=TW$G9U$5d)j1yk7%JrdY+@8gTQo3nFghn}q8HFD{w{ucfXGrQh2 z7SVQ(#03g4Jnk^>vawW4Na%h*figO z+rnRc$TE!%Ej;m5s=>SS5ck}X{T-eBK^|cX$rH^|iI2?0$EiFB83&lcy*{u$(A`~l z>7JO==6WcO(`w93F4b!wI$!?DwGKC%tqv2m=BKT-+ba_nscM| zxoiHe4;MJb#jB-r`Fp9}d{Rm#q%g$>&l^1KlVUY)yE=a5-StVU@f-{#;S-vek0E|K zN+dMQLp>L}I0=MMJ=Amp1UcKlSxNX*gdy}M6A7VkU1Db^D;n2Zmrt1J%ty|5aaLj% zA%uAusOKbx*gH<>FB>Ng(i5i|WT1qR%sOIkAez_*LoQ9ih;!{|LhmRF`(Yl2pOC@e zM`ch8!@R_qu6ZKop*kkdIu%?ne{5@;Q;y32`tYAs0cMc#?c)q6mX2 zsdXdH=wXm}ou#gJ;@l#FFwvhZHief#F9JrA@W8Xj(1eNPql++f#rX;hlGuh?nu#F{ zPGg@Kg%M{Wp|J5*=JrK=5e!A_qQ~G{!%@?-ZfGJl8+*8th{I2vClE&E^*+wSyogZZ z7>wAd$4Y2Vt}I3^&ADL+3uq{`4yKTrU*3YxoM~%ERYqa5P2;dLGp$7g3>x3>NEkJw zC&smV5T+k+5JoM+iE*qKvC~gD&eSdJeq zn@WH}31dWepF!ZWB^bt+IBf+iN%FA+tjQ7!@tjZ$4(c^eoavZq3y0xrFkZwi)Y8ae z(@`9D>$x}-t{kN>2}R0H)eQPVhv}iiXkrb5@Tr|hs5?vW?9QRyx2GC z!87@|bygLZ$xNS&hNJvM{S^Fo{3QH@{p9?1`APeU`6>Bv`|a@)@{{%B_uK2Y+fUJt z*KeA2Vl-KLdazevX|`9v6FJlmad_h_egm55j(ZXucWPgfJ6*tZm1y^+UiUu5ZH))^ z-C&qv2w`}}ki#&-5YN!TP|AQ~2wf%E~>3+;i^^^QvZ z5ZUmLeMSrM)2vtgAadcAj{f;_s9ueQ!s%^y1V}69r0=W(es)%&Bx&OO;8Pnvh{Cdg zXVdScs^gN-j(nbD7iim=RX9`_RoGOvsO(f>R$*6RRpG2=sNPo1QoW;^sd{_$)@qJw z#%k=$Bab8^&Lathy%;@lc<{P`g5PR+a;A4Ebn*E~k%}GGSEqbwR;PR-WQw6&#}nIU zPv5=kCQur8vk+J{+%F;^!apKV-%me4-(Nql*{?aE*}pkZ+)q3}++RHKuHW5&yZ(0r z9sL|DJ@1CKqbtLX#xa&hdSXWaI~pr?c!OazVC*38h7|#JWB@zUC~U1gYE!r4ilLfd z45$Km-3^-u47tTlZd=;i``it2aUy^j@6Y>)U3ZB?jF)tb@V;Togtr5O zl5z5!gxA`Nk7YWh0t1(jyxrl*mVLXouDk<>Pkn};LUutm~bk6d>=lJ0FJl`KlA z{pGvid&xT*O!vj>ZSGUtaoH*f$$VwkM_-yK9=0gnb%s1!40-8Vcy_k%Qc3VIOYm+i z#l;Gl(;Fc`p5-H@Ccj6j^40tRe9yjWlFXEVjBerQqusn^}Ar!%s9kJaeV(V}O z2U`T=^$51(5nJ9y?39g|j&rKO%h(QNiaUD+t7?StX@pp6Tw&1&HrBW-r4j0)5x%KD z=umx_XnlxX{goZ{SnZN-0=xN?(VN0bRUSSyCcc9beCo96s>11N+tUxqrK?k0s_wE> z+iH1G+ERV<`=uEJ&n1^MB_yqHTbzd9V~tDc8UgP$e4;h{UupQ>*9e@c_YSM~dr^Ps zc74FddY>D~O2T2u4t%wG<}svIpjDl1vi0(n1))QFFdjxG-oHa1iObre^v>F!-uxhG z59m!BnRiUXDf%f6Y4fRA@a;F`JK&VAtd*`}o4(&9{eXw1vYMsJ8O!}gEDyN8S8fap z$92^tqKYJLYmihRB~N2Khl4SCeGVrn4P2w%Hr}f?docyaO+sx!8ed`0zoRJsi+UoA z(dwju*bjg>34oYR8i+`sg+Ll_M6v439w4EIcrIn(6uYyK;l@x>*i{#vgXzCKGOt0B z@uiFt2qMX;e`-X09wc?4OgL0QuD(A;{Kk-KX=gIw4NS73mKBuLO=)22by1m zwC0}$n*aAS{<(wxp7lfkEyPY*3ylFSU=%Bx+@h=_p_pB!pXd%*OPz=yP z$ARXTC$0JMK=c2e#y@w^-?N^{NN3(fFC~|q-Yl90(Hhy&nkB{>8OE9qUTNgL(yWlu zNR!gMcV8p-zGj(=#-RTt_&dQ%$@KouQ)skEOCIFEoSci5#N)}bUO-*9!sx(PO z!O>dRqqY5wwPK95FTc`Ce5LIxrFB(GJM_L*!hP*Pmy?3WY7?n(PpN4S8jA883d0S> zwipT>Hr&0}P~^NJgsw$Ur$uyki?Dr**v^&OjSgkqV5 zVkCs(XtSe*vtzes$H-;JQQJiCvWeYl6C-UCw|Tf+|8AH(-OM8}H1(xp2}wVum2Uhj z{e*D3VNSY#9pH_J!@w0$7i<@-M^ik>n1Kxcw;8}wK z?jVSdLoi+34h(p2!F2I^U|4Deri=d+pi9O22Y|_?01S8* zV8D9<2E0%(;IV=M4+4gzFfg-tC8$~aX8_%No-H3(K<0u0?felz<`$khNUuKSc(9{(w_pfi&^a= zuf)eg@gguR{V6~{m*7A;r}-e2T+9MU-H=(|~INvx(tOz&+;)o6u&9V$F<1O&Kc?Emqx zv{7lgsRvI;(&Qv2lo}@)lFSWwW~}@1slM)Tl*4ea;Dy@d)LO~=sbZXjG=9AO(28@> z)>C~dc~Yu5*#Ng`dlm_W(Mp=U=|c!>v1(S%I6?a#d^uY9yIPnHrove1W~`!>e5r*8 z3?qJgv>z=t9GesrWmRsS0!^D?lhPN8BL&2RRCuriPl9#EBq*8fQ-B4|r9PGMWNY65zE(%9I4m z0=ImSz)3R6;G78t=TXW-D8bew^+E|MpeWH5C8 zDKIaE@6Pxm_;Nxar&xixiU2TdNe|kCl`TzEvUhdocL>%`j?{tY%!MA^509E zcq&O&fc_BtfN|6lP};yu{l8wCm|V!6Mwx!{Pk!ou%Q6C8x3YtL{9AT<&+y3zkWR$lad7ByzJK zk4B%BC)y?j1Ev@LdTHW9Eub`mTFL(xD?meRFk;2Bqsao>@_i(vIAHvL;g<5-ia@{) zu^r$ysPipy;;#>(emxq<+HqjN1xlACC;s{n>er*O6c!FNA=x?Qs}(0bnElo-_W=UN z(P=(KywgS}dn ziD2CJ$cex94gU3L0ORO9aPkXWLPN6qAUW|jC%?ZOI^g6N4>&24om0L91^spk&`ql) z?D%qs&y%JI5?$DT(}1A=m&(g9O6?=NtQ{Z4&7qjsn|x7wUTi8anupdO5n?Z2JDj^& z9=AW_M;G)0>tY(kl?X6)#g*`~>9fT*i=LO;5N z7}z!mL@RxD)(=Fk{cTbH2txqEzqk(m;I>;p>GOc#PXvHdI7ITW{;Hs#8sDye-`Ezt0iCuf?7REC9P4&iH9Tp zr0B0lqs7Xz61cM)4Eyy~bPf{PzZ!!>BKueO-z2hsH3o-7_OHquB(i^XF;5zwKRddR zHpu)8nDudC%f7^`$4(?R3OVsgc;e>)`j?{tG~u8A8YDL1s)qtO@#-A=%h3Ru5DoM~ z7?`1vocQZr_{-4%n()u~L=xwq)%Zjb=b%+>lEgU(%$n-61Wei-1OqeGL%{GkZY7+B z#HoJu5b~deS?g;RfVAoRbBGFQ&|$zp2WnPXJ2<(!7%h^ zzzp0+)4m)Rg$C}JS$!$ZKPnjj`(EI>8^ny~%Ed0E0QgdwcsXh1))-ROO3NjGIlXcv z4JqvoC{QvUtM}cI(mvKHM75@s%XCQT>8x}!m?7-SO+BR03b?6GS zU;ebBHIQ|hFYm(oH%TumA|ol*v^D^BpyR0wws*vb&4{E+5`_rt- zVPu`=_t$Bf$T|&Krvc+J_FL;T{zUeIdtGvOsOZx2Efy+Y7M$5Dw`Tj^rfK)gV}}l@ zMy`z?=DL8p^Iey0of8hc;opYFM8xaw~9y{ND`s>IlH89oF{Dt}6$XdaJsb(uM zwv~YBS|1jzNEYqjXsuNr!x&@j%dfN&UupYFXv9WU zL(9r$e!bl8el04mGhpD8WEW)7NM+G1h}OskE=UKQ@4gp;x9Fq$ytqX<8Sr*irp?d@ z$xzkG0A9-i99C%MWT@F@=#FIU-=3ivpK;Ixc+)Api%D$^HSPs9?K(rjgNCB~hQh$B zkpOSb@5SIDQ(7f{8q~#Mg8)UGLDdoPmnFT9@4(b?5EPw?z|g4#6rF(6Bhu*vykL>^ z-uQE%K2aMAbQymzb%cQGGC+hT>2=%)hE6bGMj)L|)nMqv3WiRVKwK&5qtnlUTC9Hn z7#3Q<)G-cJ9f9aU((9N3icX-q3=r~2I-RsZ(FyRSBb`n^1?rt~lOw<^1BBF&UPlp7 zbp$obJOe|gqoC*n1Okyxrx-AF$_F(%{S>ILJa_lX@TmksNl35b15kAY)n$PA7t-l; z78IR8P3Rq9=+q91PCo_eI(T_Wi!uh(EK>`njxwO?2!z*=PN&OY=!5{(Wq_C<(&@ww zhE6{R>VAptlQt;`P+jIMm^zw*sw1db<~0~P6@jAD12A;T14XBw0<}3C6#vPMm1hW% zd8O0+@g;Rp&&M~_MUjuA)J44>7pjY*9*3yk_I#Y9ejE8XUj4S$<5Kn8sK){7cAk%K zsoNnR$Ew?TJ$|Te2kk(lVVhv6l9Cd7Oc#2w21T3-Z<2B2bbt~D+EI(Rz%pW~7jcnu zyl@_hU#RJ7Nak>znyurMo%je_e<^db0&&i7P;ZC(|+Tqy=OGB1vu%FPE$^iC4lX zo$p)pSGv%*cv)$wTW<#znW@K#Mds4GO+PNLgpDH93DcN)9QM;(zt=RDFxOfKLoEz^ z=vwRyM;e)yFh}?>`Y0Kp zE7luH=`DKoR?M zA;cEWq22+ESIJcjaiO=XZXQio`ZQg$xG+_jP?nEw$s|^ku%k+_+mN`$Ita11S0%6$ z)$Qr!AK1$2P)Q)Zuj3@ljp}$JkD|{l4)h^DRVP3z7CQqMG03vOz-|m-34s`R4jXFM z4nuci;?QONMJS@nd|ZCADry0$uT8aY2;K64NCS?D~U)AppiXm;R2PVFZ9v%3qF5pLKE&K zz|)P3m*+M$pi0)uta~?Zd)KQ$v`ozTL;mx(+|Oexe2E4{hcoz=9!_ObF+&$<5c*Hg z!cADVR0b>vv+KK6oYWKMPQCjuQKNIR?qt5-ara|stcY>UTP6d7g8GD-buBb4$-p4c zXKQ|SZef7M4*al8i-_-JPQ{S)LV-_;%9@@F!ZS261JCKEDX$wvW?CxAZ-M*b#v&pt zb5Z)oy?8x?YNGrkBC%n#SVF__xkO9dxX7U~qiEr|18V6yx-{1Zp08PAy%c`$#+}9E zc_(-ey?Z!jT=!J2NB+~RlpT?59MTt;G|IxC-en(kt!8zj{e#jL2*la% zJvV+WC8r6Hx)69a<2RY}H&ReoB2Mc&Asn3{rx8x@?`02sl>WL1$i@YjVE88cKA;wV zWH)zo{9f)h#;r6TfLuDj!-1!*e65yz=Vl6u)$X^_>Hdm$+*Gv!k&1#MA84&F4Yrz5 z6hHt#%f%XQ4`2Pc(slo8z(Gu;^Z+#l#d-k>3Z`$Kd;+ldt%2{AT&5)T{c_36fs$|k zCVvy#O8(WtFApYPkpFH)ZVqZ4@h+yI7?<8avF#f>V}Mzp9A6Dbe-1={t<7JDv1LQ6 faHO^V)yj7VHGS=M02mY$n}C0e0IpeW%fI~}hezq= literal 0 HcmV?d00001 diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..c6d5caf --- /dev/null +++ b/go.mod @@ -0,0 +1,21 @@ +module rodbiren.ddns.net/git/robviren/datahop + +go 1.17 + +require ( + github.com/go-rod/rod v0.101.8 + github.com/xuri/excelize/v2 v2.4.1 +) + +require ( + github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect + github.com/richardlehane/mscfb v1.0.3 // indirect + github.com/richardlehane/msoleps v1.0.1 // indirect + github.com/xuri/efp v0.0.0-20210322160811-ab561f5b45e3 // indirect + github.com/ysmood/goob v0.3.0 // indirect + github.com/ysmood/gson v0.6.4 // indirect + github.com/ysmood/leakless v0.7.0 // indirect + golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect + golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985 // indirect + golang.org/x/text v0.3.6 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..ab560b5 --- /dev/null +++ b/go.sum @@ -0,0 +1,41 @@ +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-rod/rod v0.101.8 h1:oV0O97uwjkCVyAP0hD6K6bBE8FUMIjs0dtF7l6kEBsU= +github.com/go-rod/rod v0.101.8/go.mod h1:N/zlT53CfSpq74nb6rOR0K8UF0SPUPBmzBnArrms+mY= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/richardlehane/mscfb v1.0.3 h1:rD8TBkYWkObWO0oLDFCbwMeZ4KoalxQy+QgniCj3nKI= +github.com/richardlehane/mscfb v1.0.3/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk= +github.com/richardlehane/msoleps v1.0.1 h1:RfrALnSNXzmXLbGct/P2b4xkFz4e8Gmj/0Vj9M9xC1o= +github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/xuri/efp v0.0.0-20210322160811-ab561f5b45e3 h1:EpI0bqf/eX9SdZDwlMmahKM+CDBgNbsXMhsN28XrM8o= +github.com/xuri/efp v0.0.0-20210322160811-ab561f5b45e3/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= +github.com/xuri/excelize/v2 v2.4.1 h1:veeeFLAJwsNEBPBlDepzPIYS1eLyBVcXNZUW79exZ1E= +github.com/xuri/excelize/v2 v2.4.1/go.mod h1:rSu0C3papjzxQA3sdK8cU544TebhrPUoTOaGPIh0Q1A= +github.com/ysmood/goob v0.3.0 h1:XZ51cZJ4W3WCoCiUktixzMIQF86W7G5VFL4QQ/Q2uS0= +github.com/ysmood/goob v0.3.0/go.mod h1:S3lq113Y91y1UBf1wj1pFOxeahvfKkCk6mTWTWbDdWs= +github.com/ysmood/got v0.15.1/go.mod h1:pE1l4LOwOBhQg6A/8IAatkGp7uZjnalzrZolnlhhMgY= +github.com/ysmood/gotrace v0.2.2/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM= +github.com/ysmood/gson v0.6.4 h1:Yb6tosv6bk59HqjZu2/7o4BFherpYEMkDkXmlhgryZ4= +github.com/ysmood/gson v0.6.4/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg= +github.com/ysmood/leakless v0.7.0 h1:XCGdaPExyoreoQd+H5qgxM3ReNbSPFsEXpSKwbXbwQw= +github.com/ysmood/leakless v0.7.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/image v0.0.0-20210220032944-ac19c3e999fb/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985 h1:4CSI6oo7cOjJKajidEljs9h+uP0rRZBPPPhcCbj5mw8= +golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go new file mode 100644 index 0000000..9d6d313 --- /dev/null +++ b/main.go @@ -0,0 +1,80 @@ +package main + +import ( + "log" + "os" + "path" + "path/filepath" + "strconv" + "time" + + "github.com/xuri/excelize/v2" + "rodbiren.ddns.net/git/robviren/datahop/runner" +) + +func main() { + + exe_path, err := os.Executable() + if err != nil { + log.Fatalln(err) + } + work_dir := filepath.Dir(exe_path) + + datahop := runner.NewRunner() + data := datahop.RunAll() + defer datahop.Close() + + var f *excelize.File + _, err = os.Stat(path.Join(work_dir, "data.xlsx")) + if err == nil { + f, err = excelize.OpenFile(path.Join(work_dir, "data.xlsx")) + if err != nil { + log.Println(err) + } + } else { + f = excelize.NewFile() + f.NewSheet("Facebook") + f.NewSheet("Instagram") + } + + f.DeleteSheet("Sheet1") + + f.SetCellValue("Facebook", "A1", "Brand") + f.SetCellValue("Facebook", "B1", "Followers") + f.SetCellValue("Facebook", "C1", "Engagement") + f.SetCellValue("Facebook", "D1", "Posts") + f.SetCellValue("Facebook", "E1", "Posts") + + f.SetCellValue("Instagram", "A1", "Brand") + f.SetCellValue("Instagram", "B1", "Followers") + f.SetCellValue("Instagram", "C1", "Engagement") + f.SetCellValue("Instagram", "D1", "Posts") + f.SetCellValue("Instagram", "E1", "Posts") + + for _, elm := range data { + rows, err := f.GetRows(elm.Site) + if err != nil { + log.Println(err) + } + f.SetCellValue(elm.Site, "A"+strconv.Itoa(1+len(rows)), elm.Name) + f.SetCellValue(elm.Site, "B"+strconv.Itoa(1+len(rows)), elm.Followers) + f.SetCellValue(elm.Site, "C"+strconv.Itoa(1+len(rows)), elm.Engagement) + f.SetCellValue(elm.Site, "D"+strconv.Itoa(1+len(rows)), elm.Posts) + f.SetCellValue(elm.Site, "E"+strconv.Itoa(1+len(rows)), time.Now().Format("01-02-2006")) + } + + current_day := time.Now() + + if err := f.SaveAs(path.Join(work_dir, "data-"+current_day.Format("01-02-2006")+".xlsx")); err != nil { + log.Println(err) + } + + if err := f.SaveAs(path.Join(work_dir, "data.xlsx")); err != nil { + log.Println(err) + } + + // signal_channel := make(chan os.Signal, 1) + // signal.Notify(signal_channel, os.Interrupt) + // <-signal_channel + +} diff --git a/runner/check.go b/runner/check.go new file mode 100644 index 0000000..0449ef6 --- /dev/null +++ b/runner/check.go @@ -0,0 +1,9 @@ +package runner + +import "log" + +func checkErr(err error) { + if err != nil { + log.Fatal(err) + } +} diff --git a/runner/facebook.go b/runner/facebook.go new file mode 100644 index 0000000..b3832f2 --- /dev/null +++ b/runner/facebook.go @@ -0,0 +1,102 @@ +package runner + +import ( + _ "embed" + "encoding/json" + "fmt" + "time" +) + +//go:embed js/facebook_data.js +var facebook_data string + +const ( + facebook_login_url = "https://m.facebook.com/login/?ref=dbl&fl" +) + +type FacebookRes struct { + Followers int `json:"followers"` + Posts []FacebookPost `json:"posts"` +} + +type FacebookPost struct { + Engagement int `json:"engagement"` + Body string `json:"body"` + Timestamp time.Time `json:"timestamp"` + URL string `json:"url"` +} + +func (r *Runner) CheckFacebook() { + r.page.MustNavigate(facebook_login_url) + r.page.WaitLoad() + time.Sleep(time.Millisecond * 113) + location := r.page.MustEval("window.location.href").Str() + if location == facebook_login_url { + r.page.MustElement(`input[name="email"]`).Input(r.FacebookEmail) + time.Sleep(time.Millisecond * 113) + r.page.MustElement(`input[name="pass"]`).Input(r.FacebookPass) + time.Sleep(time.Millisecond * 72) + r.page.MustElement(`button[name="login"]`).MustClick() + time.Sleep(time.Millisecond * 5000) + } +} + +func (r *Runner) GetFacebookData() []Result { + var results []Result + for i := range r.Targets { + fmt.Println("Processing " + r.Targets[i].Name + "'s Facebook") + results = append(results, r.GetFacebookDataByIndex(i)) + } + return results +} + +//Navigate with random stops +func (r *Runner) FacebookPageNavigate(index int) { + r.page.MustNavigate(r.Targets[index].Facebook) + r.page.MustWaitLoad() + r.Scroll(30, longTimeOut) +} + +func (r *Runner) FacebookPagePullData() FacebookRes { + data := r.page.MustEval(facebook_data).JSON("", "") + var res FacebookRes + err := json.Unmarshal([]byte(data), &res) + checkErr(err) + return res +} + +func (r *Runner) calcFacebookPostData(posts []FacebookPost) (int, int) { + total_engagement := 0 + total_posts := 0 + for _, elm := range posts { + if elm.Timestamp.After(r.weekago) && elm.Timestamp.Before(r.currentTime) { + total_engagement += elm.Engagement + total_posts += 1 + } + } + + return total_engagement, total_posts +} + +func (r *Runner) GetFacebookDataByIndex(index int) Result { + + r.FacebookPageNavigate(index) + + data := r.FacebookPagePullData() + + total_engagement, total_posts := r.calcFacebookPostData(data.Posts) + + r.logger.LogFacebookRes(data, r.Targets[index].Name) + + result := Result{ + r.Targets[index].Name, + "Facebook", + data.Followers, + total_engagement, + total_posts, + } + + r.logger.LogResult(result, "Facebook") + + return result +} diff --git a/runner/instagram.go b/runner/instagram.go new file mode 100644 index 0000000..c733dc0 --- /dev/null +++ b/runner/instagram.go @@ -0,0 +1,149 @@ +package runner + +import ( + _ "embed" + "encoding/json" + "fmt" + "time" +) + +type InstaRes struct { + Followers int `json:"followers"` + Urls []string `json:"urls"` + Engagement int `json:"engagement"` + Posts []InstaPost `json:"posts"` +} + +type InstaPost struct { + Engagement int `json:"engagement"` + Body string `json:"body"` + Timestamp time.Time `json:"timestamp"` + Comments int `json:"comments"` + Likes int `json:"likes"` + Views int `json:"views"` + URL string `json:"url"` +} + +//go:embed js/instagram_followers_urls.js +var instagram_followers_urls string + +//go:embed js/instagram_post_data.js +var instagram_post_data string + +//go:embed js/instagram_liked_by_url.js +var instagram_liked_by_url string + +//go:embed js/instagram_liked_by.js +var instagram_liked_by string + +const insta_login_url = "https://www.instagram.com/accounts/login/" + +func (r *Runner) CheckInstaGram() { + r.page.MustNavigate(insta_login_url) + r.page.WaitLoad() + location := r.page.MustEval("window.location.href").Str() + if location == insta_login_url { + r.page.MustElement(`input[name="username"]`).Input(r.InstaUser) + time.Sleep(time.Millisecond * 113) + r.page.MustElement(`input[name="password"]`).Input(r.InstaPass) + time.Sleep(time.Millisecond * 72) + r.page.MustElementsByJS(`document.querySelectorAll("button")`).Last().MustClick() + time.Sleep(time.Millisecond * 5000) + } +} + +func (r *Runner) InstaPageNavigate(index int) { + r.page.MustNavigate(r.Targets[index].Instagram) + r.page.MustWaitLoad() + r.Scroll(3, 250) +} + +func (r *Runner) InstaGetRes() InstaRes { + r.page.MustEval(instagram_followers_urls) + data := r.page.MustEval("data").JSON("", "") + var res InstaRes + err := json.Unmarshal([]byte(data), &res) + checkErr(err) + return res +} + +func (r *Runner) calcInstaPostData(posts []InstaPost) (int, int) { + total_engagement := 0 + total_posts := 0 + for _, elm := range posts { + if elm.Timestamp.After(r.weekago) { + total_engagement += elm.Engagement + total_posts += 1 + } + } + + return total_engagement, total_posts +} +func (r *Runner) GetInstaDataByIndex(index int) Result { + + r.InstaPageNavigate(index) + + res := r.InstaGetRes() + posts := r.GetIntsaEgagement(res.Urls) + total_engagement, total_posts := r.calcInstaPostData(posts) + + r.logger.LogInstaRes(posts, r.Targets[index].Name) + + result := Result{r.Targets[index].Name, "Instagram", res.Followers, total_engagement, total_posts} + r.logger.LogResult(result, "Insta") + return result +} + +func (r *Runner) GetInstaData() []Result { + var results []Result + for i := range r.Targets { + fmt.Println("Processing " + r.Targets[i].Name + "'s Instagram") + results = append(results, r.GetInstaDataByIndex(i)) + } + return results +} + +func (r *Runner) GetIntsaEgagement(urls []string) []InstaPost { + var posts []InstaPost + + for _, url := range urls { + fmt.Print("https://www.instagram.com" + url + " ---> ") + r.page.MustNavigate("https://www.instagram.com" + url) + r.page.MustWaitLoad() + r.Scroll(10, 250) + + data := r.page.MustEval(instagram_post_data).JSON("", "") + var res InstaPost + err := json.Unmarshal([]byte(data), &res) + checkErr(err) + if res.Timestamp.Before(r.weekago) { + fmt.Println("Reached post outside range ", res.Timestamp) + return posts + } + res.URL = "https://www.instagram.com" + url + if res.Timestamp.Before(r.currentTime) { + //Hard get the likes + if res.Likes == 0 { + r.page.MustElementByJS(instagram_liked_by_url).MustClick() + r.page.MustWaitLoad() + r.Scroll(99999, longTimeOut) + likes := r.page.MustEval(instagram_liked_by).Int() + + res.Likes = likes + } + + if res.Views != 0 { + res.Engagement = res.Views + res.Comments + } else { + res.Engagement = res.Likes + res.Comments + } + + fmt.Println("Likes :", res.Likes, ", Comments: ", res.Comments) + + posts = append(posts, res) + } else { + fmt.Println("Posted Today ", res.Timestamp) + } + } + return posts +} diff --git a/runner/js/facebook_data.js b/runner/js/facebook_data.js new file mode 100644 index 0000000..fe3623a --- /dev/null +++ b/runner/js/facebook_data.js @@ -0,0 +1,106 @@ +() => { + + function getTimeStamp(time_string) { + let d = new Date(); + if (time_string.includes("Yesterday")) { + return (new Date(d.setDate(d.getDate() - 1))) + } else if (time_string.includes("AM") || time_string.includes("PM")) { + var date_strings = time_string.split(" ") + if (date_strings.length === 5) { + var month_string = date_strings[0] + var month = 0 + switch (month_string) { + case "January": + month = 1 + break + case "February": + month = 2 + break + case "March": + month = 3 + break + case "April": + month = 4 + break + case "May": + month = 5 + break + case "June": + month = 6 + break + case "July": + month = 7 + break + case "August": + month = 8 + break + case "September": + month = 9 + break + case "October": + month = 10 + break + case "November": + month = 11 + break + case "December": + month = 12 + break + } + var day = parseInt(date_strings[1]) + var year = d.getFullYear() + if (month === 12 && d.getMonth() !== month) { + year += 1 + } + var hours = parseInt(date_strings[3].split(":")[0]) + if (hours < 12 && date_strings[4] === "PM") { + hours += 12 + } + var minutes = parseInt(date_strings[3].split(":")[1]) + return new Date(year, month - 1, day, hours, minutes) + } + } else { + return (new Date()) + } + } + + function getURL(elm) { + res = elm.getElementsByTagName("a") + for (let i = 0; i < res.length; i++) { + if (res[i].href.includes("story.php?") && res[i].getAttribute("aria-label") == "Open story") { + return res[i].href + } + } + } + + function getEngagement(elm) { + res = elm.getElementsByTagName("footer")[0] + engage_reg = /\d+/g + total = 0 + try { + res.innerText.match(engage_reg).forEach(match => { + total += parseInt(match) + }) + } catch (e) { return 0 } + + return total + } + + //Followers + followers = parseInt(document.getElementsByName("description")[0].getAttribute("content").replace(/,/g, '').match(/\d+/g)[0]) + + arts = document.getElementsByTagName("article") + posts = [] + for (var i = 0; i < arts.length; i++) { + var timestamp = getTimeStamp(arts[i].getElementsByTagName("abbr")[0].innerText) + //Not accurate + var body = arts[i].innerText + var url = getURL(arts[i]) + var engagement = getEngagement(arts[i]) + if(url){ + posts.push({ timestamp: timestamp.toISOString(), body: body, engagement: engagement, url: url }) + } + } + + return { followers: followers, posts: posts } +} \ No newline at end of file diff --git a/runner/js/instagram_followers_urls.js b/runner/js/instagram_followers_urls.js new file mode 100644 index 0000000..d7231c9 --- /dev/null +++ b/runner/js/instagram_followers_urls.js @@ -0,0 +1,24 @@ +() => { + + follower_data = document.getElementsByTagName("span") + followers = 0 + for (i = 0; i < follower_data.length; i++) { + if (!isNaN(parseInt(follower_data[i].title.replace(/,/g, "")))) { + followers = parseInt(follower_data[i].title.replace(/,/g, "")) + } + } + + //Urls + res = document.getElementsByTagName("a") + insta_urls = [] + for (let i = 0; i < res.length; i++) { + if (res[i].getAttribute("href").length == 15 && res[i].getAttribute("href").search("/p/") == 0) { + insta_urls.push(res[i].getAttribute("href")) + } + } + + window["data"] = { + urls: insta_urls, + followers: followers + } +} \ No newline at end of file diff --git a/runner/js/instagram_liked_by.js b/runner/js/instagram_liked_by.js new file mode 100644 index 0000000..87f52b7 --- /dev/null +++ b/runner/js/instagram_liked_by.js @@ -0,0 +1,3 @@ +() => { + return document.getElementsByTagName("main")[0].innerText.split("Follow\n").length +} \ No newline at end of file diff --git a/runner/js/instagram_liked_by_url.js b/runner/js/instagram_liked_by_url.js new file mode 100644 index 0000000..a0d1c6d --- /dev/null +++ b/runner/js/instagram_liked_by_url.js @@ -0,0 +1,10 @@ +() => { + res = document.getElementsByTagName("a") + for (i = 0; i < res.length; i++) { + if (res[i].href) { + if (res[i].href.includes("liked_by")) { + return res[i] + } + } + } +} \ No newline at end of file diff --git a/runner/js/instagram_post_data.js b/runner/js/instagram_post_data.js new file mode 100644 index 0000000..25bd933 --- /dev/null +++ b/runner/js/instagram_post_data.js @@ -0,0 +1,37 @@ +() => { + //TimeStamp + res = document.getElementsByTagName("time") + timestamp = res[0].getAttribute("datetime") + + //Engage + res = document.getElementsByTagName("article") + like_reg = /\d+ likes/ + comment_reg = /\d+ comments/ + view_reg = /\d+ views/ + comma_reg = /,/g + engagement = 0 + comments = 0 + likes = 0 + views = 0 + try { + engagement += parseInt(res[0].innerText.replace(comma_reg, "").match(like_reg)[0]) + likes += parseInt(res[0].innerText.replace(comma_reg, "").match(like_reg)[0]) + } catch (e) { } + try { + engagement += parseInt(res[0].innerText.replace(comma_reg, "").match(comment_reg)[0]) + comments += parseInt(res[0].innerText.replace(comma_reg, "").match(comment_reg)[0]) + } catch (e) { } + try { + engagement += parseInt(res[0].innerText.replace(comma_reg, "").match(view_reg)[0]) + views += parseInt(res[0].innerText.replace(comma_reg, "").match(view_reg)[0]) + } catch (e) { } + + //Text Content + start = res[0].innerText.search(like_reg) + finish = res[0].innerText.search(comment_reg) + body = res[0].innerText.substr(start, (finish - start)).split("\n")[1] + + return { + timestamp: timestamp, body: body, engagement: engagement, likes: likes, comments: comments, views: views + } +} \ No newline at end of file diff --git a/runner/js/scroll.js b/runner/js/scroll.js new file mode 100644 index 0000000..98e8d64 --- /dev/null +++ b/runner/js/scroll.js @@ -0,0 +1,26 @@ +async (limit,timeout) => { + const getScrollHeight = (element) => { + if (!element) return 0 + + const { scrollHeight, offsetHeight, clientHeight } = element + return Math.max(scrollHeight, offsetHeight, clientHeight) + } + + const position = await new Promise((resolve) => { + let count = 0 + let limit_counter = 0 + const intervalId = setInterval(() => { + const { body } = document + const availableScrollHeight = getScrollHeight(body) + + window.scrollBy(0, 250) + count += 250 + limit_counter += 1 + if (count >= availableScrollHeight || limit_counter > limit) { + clearInterval(intervalId) + resolve(count) + } + }, timeout) + }) + return position +} \ No newline at end of file diff --git a/runner/logger.go b/runner/logger.go new file mode 100644 index 0000000..bffdd9f --- /dev/null +++ b/runner/logger.go @@ -0,0 +1,81 @@ +package runner + +import ( + "path" + "strconv" + "time" + + "github.com/xuri/excelize/v2" +) + +type Logger struct { + log *excelize.File + work_dir string + weekago time.Time + current_time time.Time +} + +func NewLogger(work_dir string, weekago time.Time, current_time time.Time) *Logger { + + logfile := excelize.NewFile() + logfile.NewSheet("FacebookPosts") + logfile.NewSheet("InstaPosts") + logfile.NewSheet("Facebook") + logfile.NewSheet("Insta") + logfile.DeleteSheet("Sheet1") + var logger = new(Logger) + logger.log = logfile + logger.work_dir = work_dir + logger.weekago = weekago + logger.current_time = current_time + return logger +} + +const log_facebook_sheet = "FacebookPosts" +const log_insta_sheet = "InstaPosts" + +func (l *Logger) LogFacebookRes(data FacebookRes, target string) { + rows, err := l.log.GetRows(log_facebook_sheet) + checkErr(err) + current_index := len(rows) + for _, elm := range data.Posts { + if elm.Timestamp.After(l.weekago) && elm.Timestamp.Before(l.current_time) { + l.log.SetCellValue(log_facebook_sheet, "A"+strconv.Itoa(current_index+1), target) + l.log.SetCellValue(log_facebook_sheet, "D"+strconv.Itoa(current_index+1), elm.Timestamp.Format("01-02-2006")) + l.log.SetCellValue(log_facebook_sheet, "C"+strconv.Itoa(current_index+1), elm.Engagement) + l.log.SetCellValue(log_facebook_sheet, "B"+strconv.Itoa(current_index+1), elm.URL) + current_index += 1 + } + } +} + +func (l *Logger) LogInstaRes(posts []InstaPost, target string) { + rows, err := l.log.GetRows(log_insta_sheet) + checkErr(err) + current_index := len(rows) + for _, elm := range posts { + if elm.Timestamp.After(l.weekago) && elm.Timestamp.Before(l.current_time) { + l.log.SetCellValue(log_insta_sheet, "A"+strconv.Itoa(current_index+1), target) + l.log.SetCellValue(log_insta_sheet, "B"+strconv.Itoa(current_index+1), elm.URL) + l.log.SetCellValue(log_insta_sheet, "C"+strconv.Itoa(current_index+1), elm.Engagement) + l.log.SetCellValue(log_insta_sheet, "D"+strconv.Itoa(current_index+1), elm.Timestamp.Format("01-02-2006")) + current_index += 1 + } + } +} + +func (l *Logger) LogResult(data Result, sheet string) { + rows, err := l.log.GetRows(sheet) + checkErr(err) + current_index := len(rows) + l.log.SetCellValue(sheet, "A"+strconv.Itoa(current_index+1), data.Name) + l.log.SetCellValue(sheet, "B"+strconv.Itoa(current_index+1), data.Followers) + l.log.SetCellValue(sheet, "C"+strconv.Itoa(current_index+1), data.Engagement) + l.log.SetCellValue(sheet, "D"+strconv.Itoa(current_index+1), data.Posts) + l.log.SetCellValue(sheet, "E"+strconv.Itoa(current_index+1), time.Now().Format("01-02-2006")) +} + +func (l *Logger) Close() { + err := l.log.SaveAs(path.Join(l.work_dir, "log-"+time.Now().Format("01-02-2006")+".xlsx")) + checkErr(err) +} diff --git a/runner/result.go b/runner/result.go new file mode 100644 index 0000000..d5f0ce3 --- /dev/null +++ b/runner/result.go @@ -0,0 +1,9 @@ +package runner + +type Result struct { + Name string + Site string + Followers int + Engagement int + Posts int +} diff --git a/runner/runner.go b/runner/runner.go new file mode 100644 index 0000000..ace271b --- /dev/null +++ b/runner/runner.go @@ -0,0 +1,152 @@ +package runner + +import ( + "os" + "os/signal" + "path" + "path/filepath" + "time" + + "github.com/go-rod/rod" + "github.com/go-rod/rod/lib/devices" + "github.com/go-rod/rod/lib/launcher" + "github.com/xuri/excelize/v2" +) + +const ( + longTimeOut int = 2000 +) + +type Runner struct { + page *rod.Page + logger *Logger + Targets []Target + FacebookEmail string + FacebookPass string + InstaUser string + InstaPass string + work_dir string + weekago time.Time + currentTime time.Time +} + +type Target struct { + Name string `json:"name"` + Facebook string `json:"facebook"` + Instagram string `json:"instagram"` +} + +func getExecutablePath() string { + exe_path, err := os.Executable() + checkErr(err) + work_dir := filepath.Dir(exe_path) + return work_dir +} + +func parseConfig(r *Runner) { + f, err := excelize.OpenFile(path.Join(r.work_dir, "config.xlsx")) + checkErr(err) + + //Facebook + cell, err := f.GetCellValue("Sheet1", "E1") + checkErr(err) + r.FacebookEmail = cell + cell, err = f.GetCellValue("Sheet1", "E2") + checkErr(err) + r.FacebookPass = cell + + //Insta + cell, err = f.GetCellValue("Sheet1", "E3") + checkErr(err) + r.InstaUser = cell + cell, err = f.GetCellValue("Sheet1", "E4") + checkErr(err) + r.InstaPass = cell + + rows, err := f.GetRows("Sheet1") + checkErr(err) + var targets []Target + for i, row := range rows { + var target Target + if i != 0 { + for j, colCell := range row { + switch j { + case 0: + target.Name = colCell + case 1: + target.Facebook = colCell + case 2: + target.Instagram = colCell + } + } + } + if len(target.Name) > 0 { + targets = append(targets, target) + } + } + r.Targets = targets +} + +func configRod(r *Runner) { + u := launcher.New(). + Set("user-data-dir", path.Join(r.work_dir, "browser")). + Headless(false). + MustLaunch() + + r.page = rod.New().ControlURL(u).MustConnect().MustPage() + r.page.MustEmulate(devices.IPhone6or7or8Plus) +} + +func setTime(r *Runner) { + //Establish the date + weekago := time.Now().AddDate(0, 0, -7) + hours := weekago.Hour() + weekago = weekago.Add(time.Duration(-hours) * time.Hour) + r.weekago = weekago + + currentTime := time.Now() + hours = currentTime.Hour() + r.currentTime = currentTime.Add(time.Duration(-hours) * time.Hour) +} + +func NewRunner() *Runner { + work_dir := getExecutablePath() + var runner = new(Runner) + runner.work_dir = work_dir + parseConfig(runner) + configRod(runner) + setTime(runner) + runner.logger = NewLogger(work_dir, runner.weekago, runner.currentTime) + + return runner +} + +func (r *Runner) RunAll() []Result { + r.CheckInstaGram() + idata := r.GetInstaData() + r.CheckFacebook() + fdata := r.GetFacebookData() + return append(idata, fdata...) +} + +func (r *Runner) RunInsta() []Result { + r.CheckInstaGram() + idata := r.GetInstaData() + return idata +} + +func (r *Runner) RunFacebook() []Result { + r.CheckFacebook() + fdata := r.GetFacebookData() + return fdata +} + +func (r *Runner) Close() { + r.logger.Close() +} + +func (r *Runner) JustOpen() { + signal_channel := make(chan os.Signal, 1) + signal.Notify(signal_channel, os.Interrupt) + <-signal_channel +} diff --git a/runner/scroll.go b/runner/scroll.go new file mode 100644 index 0000000..86b2af7 --- /dev/null +++ b/runner/scroll.go @@ -0,0 +1,13 @@ +package runner + +import ( + _ "embed" +) + +//go:embed js/scroll.js +var scroll string + +func (r *Runner) Scroll(limit int, timeout int) { + r.page.MustEval(scroll, limit, timeout) + r.page.MustWaitLoad() +}