From 70ec5f52733f0a4563b096285157a553fce5de26 Mon Sep 17 00:00:00 2001 From: Orion Kindel Date: Tue, 3 Oct 2023 16:25:12 -0500 Subject: [PATCH] feat: add plugins --- .gitignore | 1 + bun.lockb | Bin 45650 -> 67459 bytes package.json | 7 +- packages.dhall | 105 ------- spago.dhall | 55 ---- spago.yaml | 46 +++ src/Puppeteer.Base.js | 3 + src/Puppeteer.Base.purs | 292 ++++++++---------- src/Puppeteer.Browser.purs | 96 ++++-- src/Puppeteer.Page.Navigate.js | 19 +- src/Puppeteer.Page.Navigate.purs | 25 +- src/Puppeteer.Page.WaitFor.purs | 4 +- src/Puppeteer.Page.purs | 8 +- src/Puppeteer.Plugin.AdBlock.js | 20 ++ src/Puppeteer.Plugin.AdBlock.purs | 92 ++++++ src/Puppeteer.Plugin.AnonymousUserAgent.js | 5 + src/Puppeteer.Plugin.AnonymousUserAgent.purs | 8 + src/Puppeteer.Plugin.Captcha.js | 23 ++ src/Puppeteer.Plugin.Captcha.purs | 308 +++++++++++++++++++ src/Puppeteer.Plugin.Stealth.js | 5 + src/Puppeteer.Plugin.Stealth.purs | 8 + src/Puppeteer.purs | 14 +- test/Puppeteer.Browser.Spec.purs | 4 +- test/Puppeteer.Handle.Spec.purs | 2 +- test/Puppeteer.Page.Event.Spec.purs | 2 +- test/Puppeteer.Page.Spec.purs | 2 +- test/Puppeteer.Plugin.Spec.purs | 99 ++++++ test/Puppeteer.Spec.purs | 6 +- test/Test.Main.purs | 2 + 29 files changed, 860 insertions(+), 401 deletions(-) delete mode 100644 packages.dhall delete mode 100644 spago.dhall create mode 100644 spago.yaml create mode 100644 src/Puppeteer.Plugin.AdBlock.js create mode 100644 src/Puppeteer.Plugin.AdBlock.purs create mode 100644 src/Puppeteer.Plugin.AnonymousUserAgent.js create mode 100644 src/Puppeteer.Plugin.AnonymousUserAgent.purs create mode 100644 src/Puppeteer.Plugin.Captcha.js create mode 100644 src/Puppeteer.Plugin.Captcha.purs create mode 100644 src/Puppeteer.Plugin.Stealth.js create mode 100644 src/Puppeteer.Plugin.Stealth.purs create mode 100644 test/Puppeteer.Plugin.Spec.purs diff --git a/.gitignore b/.gitignore index f887a09..ebf58ad 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ /.spago .log .purs-repl +.env diff --git a/bun.lockb b/bun.lockb index cded8fb5ce1b64c47f4ccca3047176786d31879a..d8c9f5d17dd205d98dbbec7d15ef117b75bba191 100755 GIT binary patch delta 24894 zcmeHv2UJs8*LD&?2?&CqBy;qE+pV|9eH*lNsG_PX5`$rRE#Y`Z` zZ3U?f=nc?@pwmSHfdi;YnWB;>2n31AX>nA1VKsrk82M?b@e1Mxs^sbEIRe2f@4F^dH?2R#qNpfmUI)T!p`c$B2>MuAe{6lG>6Sw9&ip&+*<@`>iiRf!qMe*+EV zt{e}4jFLc5spT(AHNCSzm#LQrz>~7Apw#{~lq2g)K}oqHBVL{?@1s`4Co8j*Xz016 z#@{|xnhL`}$)HkDYH%VdQZF)-RB2g>NrJengan03AW$k43O^rj!2$S+#w$~$uB5C3R6*l6b*+$vRq`Zu{UY_!uf`~sBf{R&Dwtf;3cm#WB2k!SQ1@a6q6=+v$vBVDCbr^p2r z$RLZ3I%pbFWMpM3lQRT+wES(YJToI%NsG1w3dohaKxyJ6Dl%QNR7%13X7nCM%>>*G zN_t9R82Xxf4fWNOk*-QhR4Fnt255V1;iR#=Hz-+}BhMO?48=FK`IkV&R8NuG2iXE+ z@Z{>|7kxDm>;u27F7HiujN1l=_z#LAOt1=W(&_hGm+oPf8 zrjv(e?2ENMI1EaP`5qNBC!L0BZ&mF!9cLTO8hyB0mn4tprrlRlNPr*oGDbv$N&;Kh3>zs^O3U?`%c4dPY8eoF239vF_X( zm5q+)4>2+qUCfS-yb#gj`SEg-!iVOPn->!6ENyb|(XBHVbnngZO`W~2=75JG_np00 ztF8Yq>}s=q?45Ov+2#he7B}`6+zP)n!}p|5OhTuq*wCyC4f1bIij_%iX6>uX#_HIq-F@D@9$a2g(zDfm z=gY-?9=TR)(7nM5=fTF~LwC3xHlNsEG`e2Pb+w-qD>4ehj*R^I>vFe}7d;HUYjbx_ zpIhf9*IIta@c@(SZgcMGwf%7U-TB-4tdT{!zn%9nzsb`vrbY*gx3~=3o^ed=;qyy? zEVt>APEyGtpK$k8v-Y&QaQpqb>%#~A9HW@+TH-%Bn03^#(;FUGfkY zlQx%l16FS4%sT4n8P)+iK%1+>Q$QLoWiJy44mJ_nG(U#0k&y1`40=Y73m6ePA#GW{E zJAYR<4>*k_|SKZE01fhK0B--H?4iS@88k#YWXEsW}*839V#IkPjCVxd0g+Z4z&zLnSU6E6qniIO_3NY77nMV}Rl zWg=Yz7V97rg&MFz2bp-Vfj|&|BKjC?KXD1TAWkK@%>Y{yIl4N@L@jHwSPPjr51FJv zhuNF?nd}Ca1dhB1M+)>bqh}_rX($k+QcK9xVF#kk`JpkK28mKDF{~;P=^8N$eVI56 z8yxNav@K$p=Vi zD*Q=e7>F(UM{Q#k%wAtJ**74Ux{=UGuevV2;@@FDb@OKj` zn*T|1##F2aK@SMhXwjI=(-zZI7YJVQ4T#@E5<&`Dxs{(ukg0~_%h~hgOlEu~vEe0# zS0JI0C6mS6Qil4PIJG)_k!ftmWrlr`+0lf{nFS`BkvS^^2{&gJ1~T167&3R7!{T$equ~;BW9%hiWd<^F zA&z}7WSX#Y13!_jl*QJRiCRlpA;tyoztnaI?dSs2Si(bg>1SSHTL^8_t_ zHY^rov<)i+DY0Q?CNgnNJOOFeJk*NX*s|E#GEtr_E37ROZ-9*URIW#&N@QXeb+cn- zAQ^Vd!b~PMwbyh^kK2TMfTKQOBZKxi_N)v=t|GIECKFw$QIEyek%?Q@!T^9$6)wl~sDP7LE5^!W3#s}`!ZNw}rWujh@E&YD2adX^8Ei)~0^>bpeAd8(?A;=>qR%Rs=H)*VC6Pq#YNB~EJhLvmP zC!PsT(^GJ|=fDNBd3u3j37-5k(^IbJr<(9-Qaol-BLY@#>?ewLVP&>5(Ha+KVTU>A%3|$g;t*F&pEQeN2)K@%ZE_<& z@kwx09WqqccVn^kSgLNU5M-=Gf#RjOqd}lu!B2b@91Viz z1T)3+D@`3Nsak%z$;5HpT7$b5$mgC1#E-$zAfOaxNFz^H<{;Be!NkSZ;t(iafjbgw z6g;MUa2QuBKarCcD{FvV#*0}vHdW&&p^r)e+S_Xb)d6fG9JK+Zy$@@M<1wwcM3nL| zdALNBcuWj|2yg-@eT$MD1H>gzHK!O35YYLDt&3|2%b!a`$uR7q0v!P70jJ;xP)gsT zWJnMRk%9o?g8@oJiElv+QVaUfsJYPyT54DUQ7Ras<%yDkp<13O^`xzquS&^aY|Na& zKZ4c(VgM@NO-pfpajAz^KOJaX1vw}wj^o4{N_@PQC#nbZ11LWkpj4HTqEs$dL#bSv zmM2Q`{#ss5H&j5S&G;6jIgt$z9R^SXxd5eHfa(ti$U`FmN>wR&U;;pN8bC=)S(=ls zn(EB}sGXSrrGJM~{Q`jKT!7k{r={~jDG?>R7Xp+P0>m%kUL0s{aOE<%hL+-<8e9QT z!BqgIZ&8x30Z8#$fD%y}sVxAtvmKyvI{`|cDdqrG+{I;UC{;WJP{kwK{Hm1lk8AUZ z>Hwu${#%sVKc$ruB}2|=={cH%)W9Wx8omNhB1#p@h(Y>SD7A9~pn5j}N<>LvIWb5? zNq&nMBsJYo#%+KMxC_uKdIV4+N)10D2B|8gdQZ7r4JAd-0cz(ZK=M}rrK*(je+NL- zg4ZNPB1#qC5rb5fQibt)ZOY($o;kU-K`FEuQ+`-we@vnW=MA2isfV4kJ-0<_n*x0zKu>0d1OAKtv-8(qi zrqrGh;{5JT&bb8>e{DZ%ikoZj5oyL%ch6HTMsMoYdfEz?#)eSsOcF=P@)xdq!$Fh8bmqF<-o?N~$X@hyKTQBp< zmL3*&)1NaT$NHxc*A3P(JD4LBXsn~z!6%)$oik7F?0hA+Fx0A%-kg#LVFzvFS{>`s zW}i;UhB2=$7>t=-;!v%tTpDpj<{Bm5{GjW$t8WLq^QzUmb?b#;ZyoesvPw5exG?JJ z^%esLUwZgx!a1c|(45fQgFBTypLt}ZcSXU#vU>HM=I?De-M^Q^qY0BQEckt9htBS1 zgEri(-JzNLo5_Q>h+4oL-ntr0+&r4Ca5rQr&BG$ZyrG~v4R;Q@VHPK9=Hu<*eyGc? z3B&%VQO`Lben$@-LFv4U0S8|+yEf>_W#01$I<3<@vdPH)!`5DJxz~B-D&1;h zZCQkeL_OAdvXw{gHCgX$`~6h^kN(S>2bS6&I3Mz0r+!hvgf6oW?Y!5^QU6Z(!n?XN z&I)I>@%e4ng2jHZHoD{c5AXIUcJ$aPrW#Z=?nC1zvK|ZEHw;Y5>~?ip?PaUj2UvGw z*$e$Pde<~5#_KS;-a>7Yc^5;#hHKsHgES1hW^FVW8}%}z;-+smb>XVHk0Z}?8&+p-$+MVH_3L^ye)SYh zXj*dGXk(rO%;AT*7H=A;j(??-t+L&Nwe|_=)_oth+2(P&{;JM1p9fAnoOn3IWSdc= z{XI6Y@x61m>^ILQ)?z)pB7?-Ll`#K}`3`ixq2hb1$x+RmW1;%cl5zVAJa%X?9N1 z6K9t1j;JpvF1x)<*~q8Sz0^C69cM0!S!%`%y(MbzU-ty-o-CTW@Q?A!JC_+oc{vS_^JG_1;jYb7>SJ_VYfROc3Pum%ZCg1yg!_fX-^ZV!yJ0)$Muv{8&Yj}I_ zE|Xes3yL*-pc?j~$AxywkFNBcevj>ebqAj0@6h+Hnf>FeUrs*^=sZ?WC35e$cHD!e z5jksa%)LK-a@%9K^tKPN-#q>Y%0K3sGizUo+B_`j@v8%EGTxi!49aZu%=^~kHXQ|e6I+#J*^KgOV7bd}M)tOf z{uM^yGtvin%hnqo+$dW*_)^<0zbl$|`duKU4>!DZ(oeR*cX_SeTs64x!M0lGLoQGD zn|6K{?zsHY(@wu#NXXh4d7z8%?yi^XM~)gPEkEERu_|{s8<*}kd-M>e2hxFe%msF( zmjhTStULFl-i1r*LPN_pDJLb#0rmIotnK(}fz7M17j8Y)+Q)e~wTc?AW+ zgFTacGkvD}itg>&6nJix)HkEcVcEd|`}0ROw}^=rbiQXfsl(d6wvUFhp?(te81`Uz zPTlc8`#m)dm#;DHeR!NzRNJWZzS|=Mo|q^n+B!LO9rdvKu#^rnk4=8QGJ8~L!93Sv z5}(C4p5I;ZAnZfaD%M$7wJz3p=t(EnieNVR*_mLgH_8y5$cM&#hR%sOo)XgT)N-eB zeIH%TaSnM=@Y*$F#f8;jwem7{NfSrE+sj=1CF(XSR~#&2jw60+QrYQ@ zaoc5~53aSeNUv|SeVF&iUoF?}{%J;x3(LJ#lokH_1&1b0#xHW7G^f!o{o|dYn{S-h zz9g}Vb+%Qls|a`OS-kkzzPvG}_OnwJ^U}g)*JJJPhU{O_@l+D&P zTQr*;V90dc!-VzOfzHvaO;bbGpi5XpgQ}Jn)!DynUABpLeEIEq<{rzf<(u9$4OqK+ z-lU3%v-@g1K7F=R-@0e}RUAFPS>O=yM@Zin9qR0Q(Q{Mq@Voi9CzNMcv&yCt^~mbU zjn`c+=@GN5<4NkUsT2LhEV|k_|@Y^F8lTPHW`(tU%#jDI~j;QhW(S;5% z8xreTJfB?r(*Wt=;d#oE1vh2mFXoLn-OG3A%?0rT8{S}zd$4(d5@r}0CUjy-c%W-D zL$xq}jAc}-8I86dIX$N2l);wTL+^On&f9+WTFtlR=UR73vs?c(5neb*^fw$!epCt2*?Kd3R-8;D+ScKe_r0SB z7FfM~8kN+2zOC8J9xbJFZ+Cd2DlqFYXP{o(mL-0rbu*9UtISGQm$DDd@Z9yT_q^6w z>kOwhe-gQL*y^dd9<|4&FV7y89p^b|ziY&zdWS}ZB`ltEVe{ zLD=QwF%NPZ%-G2ce~_qs*Y)js!)Wa8TY}q@YLA>~x=mVt_;1e|SH3ITsp{pJI8`vR z;oa=^ODonzPr9Y=_fXZYeEQzYA-^uZRjXHkJVHIQisAI`nNK=}kNTOksa|#>bMS9^ zqQM^`Hctp`Q7iRLZoR%4s~+S8K0AIT!{>c|vb3+CeaqjkP7XbtY5209^USO#IzO5} zJrUjYBHIJ&UZsz%U)+7%**QNixq18S#RtM}nI26Q4i86e-nM6Io5BZ&TGbz;wp%ni zV@>ZMvq+1U6LOXt-P+oCt!i7$-cEU&XrOtw`tYWK>NH#=8L(x{o@QG+y=kK>$ z{6q2TMg3j&1vp<{J#y-4X^M3JbGu8a%LcpKS=n`}7f}?|)3IT3P{9jk9V}5lv)q(; zVcXRFd}WiI(r5J|EjM|^)LU|6W6uRI^M0M_tT%1dtB;lE+coxkVt%^DHN&U9Vq=Fd zRty$xz0|IM!p%dnD%J&5wa&-&R_QCFd7>Yh&iY+`>Bq(`0s?KDw=1$8GWEn&za#Oz zyP9;Hx&72iX_(+_UgXm9l-gdW8b#K4;^tVReAvj~DdMZF6xMl}Y^*(Q^pS&cZr6H# z+?pO$(|W<-i%tor_gQpl(c@m%`;Wqc=08#;tDAMs7(M-3zq9>YUf%d_flXXig^9}9 zocrM+KeB;Ut@}ZctY`fDbgvN$CMxqQFJHX8ImuI5o*bxbc(PUZrYDX+tCbfR+jpyU zwzRg>!b-n=yP4VUD9OzSeX6seEhOs8%jbo>f4VDf$JLU<(?;E!mzZqX z(xp?i+uMyEgu1#pOZ!`e?p_&mW!%;p-KP~K?8%$;%pkMD?A4AR-PbhvvEo3tD!yx8 z)w=1nL$==Eyvsar(AL}Mj;>ZbS6D47br>R?dH!B%{FWGtuwR?Mx)VP}+S;^2xwMq12QLiU{;`oVz2NspukTOrAKy{>A$Di)?T$Jh zHvclQH;ZNYIjeR@Re$nJ?Z+PzH_mDu=r(BmnM;zWtA<@`9yQpadC#tSzNXJ9eA0<{ zeWG#U<-s|xACDQ*_kLQ#=MIyv7)NY4KX{L7?#lj~VxR13qpd-gzvqekC!J0X2FV#%x{otKpdk3VQX+j62w>fATu z&&-;=JcKoGC1DY+VZv6dz%`n^Ze_?mfD2`9@j$SmwIS<)2ZFXt?1|$r6ragG!-QdM z2ROYphRoV4Oc=qGUN|?vm4fTQq~16;+8VN<-eJN>b{w3fogs7a2@`f=IX=Rf6lr0{o-d*a$;5&p%AqgFOJ} z-`RzkHDoc=?~~1xS>CU3Dekda7mrfqo6Qh zf0h%3{&hi*z-2I}VDt}MesGvDi(LVi9fcmX2onxu<65A9(dZGl92U?rS~!SJ#dR=y zfa?$z91<-Y$_j8D#-8D-W^G$V3v*c!uEW_IT=Q6D>uBK!wi4HoEG9HsIErn=bu<&V zi58Awa$Lu<9k`BTCT*jIvVPn*BQ(sB3d|;jl*>oE5~&<3uqrLEMQY{Eo2XHox_4VL<{G#0$k^@ zXSmL1Z97H_7qB8+7qU0F7O}|4XyGEZ64%8{_+zwi35&sXDcgwaGA8a6EnLpzxUOJ3 za9znvI!6mvvHo46g{#?dT-PxBsA%C@mV@g$b{5z5%qhB4P5K^||M?$&{$s{AL^n~> z%XR)o_HoKo*S>V*{rB?J+^_rp2E|&v>i>F=zw9uE6{eP^dEO@TIw_ z_G{7qNuK(DQu!~atNzP+@}C)_7UaUKf4N)#WnpU0P}TJAr}Q&(BX|7%MAfew29_}}Z@KTu4Y8UBAu&Hcj` z{-^HKa{&KOUH_dt_1~pftC#%$-|g|29aiIe+1&q*tG`sN)%*GR?`tp61mXV=ZWg|k zGJ>JV`it}6|G!cR89fESSMj;GJT?ETM@D})4?nKb6l6lVY4Q8sQuAVW`51lu?dzx- zeyq2i*=9HAR;l0*?T0tYUjWcAQs_5Mlz5&DpdV~eS_lwNzp&tc__YZ4RFC!=?gnJB z_LhF@FaV%BOSF0P8?3(cluV5*)n<}X{#Sy_w0We|AE2}xphoBi33~xbD*;lh4{QS{ z(XUJ?4|72<8=$lrpuCy@e&EifH4z|`Nng`n1gPP)04c$|;C>fNjjRW#9KEZj-^h|| z13-C==mx1`6KkB)LQVEkGh`1{C4JH$K4{+(I}pdYMd0-b>_Ks$ic z^#K$Bp63OLKoXz?+yVL)0rN*dUl6AO z=|D1&0+8!J0PlcDKm||J{M99QYZy49o@Q0g_@|ko>n{w$@YA%BP`R z2hgy30BZnp-YS5c(FAY?8Urwc>x(g{2+-~vAE|#%0OeQvDxVxe`8rkdb#Y&dril@5 z3;{7v1E>z@0s25qzyP3p%Bv5MA`^gDY=?U@pf+F%(1=-Sd73(P0CT_spy^Z>kN}it zO4Gy|H#AE8=#aO`>og(+;(4Xi3tlnhRprTZl;`qQK9zSx9#MBtH^3L5`R+;6galro zKH5y8ErAw*44|Q$p9};!F?K#3Q+mc z058j>+I#~U5Ku*0VOc;XFa*d31_FbDLE8Hq(4hd|5Kl)TZzMp9h66NrMgV!*d&-{z zOa#UQ6G-_aARm|vQ17P!GsdB7ZCF0cSt z2rLFjz6e+ftN@k+o-1)7MccbqF`>^CN3p`=)|GX{|dboOe zxng=2qmUSqM>XxF26+uUMZ%umt{$%L0>N&rB%;pV$F1%AZh^$p!_~{xM{t;x4wI_S zLtYCt*ED`PASy574dh-BW3AkR#10bv|K_Qts|Shb@T?CB|DW}o#GC3gfQ0Js|9j81 z=1Uf~g2V-af{f#A6L2AS5OIf(5}}n~W95VL@NpzWLfFM6 zKAaC9fr8A|fg%q!Y?QegJ-~XYrWr0ic*SRhd`uudj0I-Bwpl(p5FgNjtK;G7hi539 zdPW#YKD5Q>*7!(5eDDjAP?wzU!G?~K*mIVAU*G8JOr-Wy^Rb2aa2V7ZYT`c_1ayQC z7^Deu5gE?4#x33Q>fC{x%ZJy11VinrhO7CoM0_L-tqJhAJ+^Av3HaC?kigGa?p%Z- zid4i$*WjAt274ju&?4gFa6saz>Cm_Is3C$PJ|Yqy#Di1hPIHHkt;B}|(JDe6KFSgw z7(^?9=f0sxYr>!Os5NeYB0hK%A8+Kd-F!4AK7xo=5$b&3gl~F|>nb1YNfV#M&TJK$ z(~kHz6nU_OTsx7>fIZEZh&CIrBjcQ#`)b_D2YY(|b30M1&Zov|4aW5Uz633pjrZ*N zok<6$et*|GkLmUAZH@_lMPu3x2FmHs{MLcRg2tKCmapccSWS7gbm30xYI&SP;H-bt ztHWrN*7hW6leZhLRiFKNr~7_?zdi-EChDT8U?a9DH(WHvh&|1fsQGAEeB2ipq&;K# zxLJIJn9mYEauy#W1`;1vUsq3$KOMftLKzk!B|aJ#A9seUUw& zt$o`Bl6=1oA9ah5FQaLVHY+|Z7axI!Bx>w05UEYLj|aJY^ct>G07>|0V0_#gNND@? z^!DZBh4GPbsE!An8J^$p54Rq?|Lg^E*o!%6U1MZw(YAh7)Ylf-V;K`r(D{!D#b zM|0W*Ev+M;eN6Y_=Oxp!E}O5GSn=`Dt~`n8_UnfxuS7z3S3ITQK`Op3lT8W{rPgJV zT)U3T@Vr8c;f#-2=a_3RYM2RS&3(8HdNU;YkR)ZS`}nrjYNatf=5VLfcNU$8C!($~ z{ox6s0TQMf?kO5BVaq@!B235d1S>vr+Ir>ciziL@)`d4{hsJ!qCSl$4B%(?Q8<6Me z%SUKyo_MCukTHE7s2rU@9s)iVTScGPF4vpw4}ruVEnu-9kg^+j5??-un{~bN`uoN8 zcA$WE`NUp{76d$)s&q$duUp#^dW zl=C6vJ|qh_q}K~6<0RYxd|ArYp=~}CTwH&L=0OIv{82!AHkDejTO-0n2dtUiVIcUR9a%9>li4`CHEvNKoMcu)XxoCox2JHXEmU)lzWVxd(L>ukc+)0ohBIBl0LnLE(%ZUgerchf(`!~By_|-{b^|W+@DXL=Oi99F@LpVHlrn0 zFHnHnA$J$%hGfTTt-WxtR;hn|c6m&=)opxoAnP|gzA zjX-oPXsoAbj|0mb8!kHPzz%}rOlZvP#_7>*<8hv%&5o?+IEiSFBg-Bat~p>mt-f)7 z!o4)Sn)6d+?8MrP_Y_$;u_5CnR=u6LcO1D>-D2|{w=@skn>e=thgJXM%g#MZjNBH#QF+nQQtX5H!GfSA@X~hScOY-Tp%&ke~40O#08!C)Q zc4nI=c#0l5v-=?Lotf1{i4`A@ZqQX*Bds?J+1hl zbF1yUE_yraBP~>#lPKQXgYC|jSQX=(qLHfdp^~0@P~hL&r2dH)>*`_;(X^;P_8fE~ z>78fqCxzf63ZfKJ#5}>t7P6i1Lc8sMiP>Wen~sg&^!`xC$Yr_?nqBU8W}ecxPur`t zz?(w8v|qOg38BGOUA6L@()I82vhVrCfj5DCaLT2SUlKhH_~3`zA2&G0p6MwwAgA_g z^@=oaWHh_=Ie&VnKzdYN$Vw*fwFttuEByz#$@|15BOs$f<&vJ2p03DLC{%3R6gTxJ zP7~eJB|SMSQJL!UsrdKgscES>Dat_#mn`{ril`Lv^7KrECj5>(BU2$y7G)-}XG^Wv zlnr|7FD>B2nMrOgZvUu$W|9jY0Pw`)nwdE0+XcU!>ym|^F}S3w6bOo`Krq*g^jgXC zoPjDOJ}GJp`Ho#**ZaPKUzdcX^o|3D_E9L(QxvL1#XqvGk33WE(nqOci)L7|$s5G2 zzw{5+I$!3W7{M1K3)tC0dlSvaa@5k?xOg$Q#r6H^(>3ylxcI@PUSw-RhARSsCa23Y zlTwta$`m<UdOJ!51Sq6QThl*e2xNxU7=ioTRO2we|apBC3N3>pfCOcJR zE7ja0J00~uGcN9Anf;g5NW&NHz<#*NoPTWqIe!bsy0c!>?4|t622O@hzcia@lk#Qd zbL=cWyAtv$j&<>vwxB+ZajP%J7=^wwLA?0U3Che#S7f+x!F#zG<$}4z&K6mDed4I6 zpBeZVpLuCusx;|9g@!s*Hu3zpJr?FCzrC`>l#>oF8XCXOc$C;4bY*g9@1DvDjGwi}}=8!UG0 zw%)$nXlafKir1czgD0yLZ5NX$_;;ELrz8y)wPN!*3oU(SXcVwKxMhZ79-o`yjN{wo zlzMUJ%O@`;EAUvANsc20)>M-+wB6zvO-;6Osa^ONBZKBKT!Ww=?{m1mR2juVSW78* zh|9?Mq7Du-KA-yK`aL1v_UGcHhW7eqUsmB;<16s|H>}E3DY5pf&3P-qbE^*orMbs1 zi>i$$Cu>*+IhdV_9{G2=QKW3dH#o& zZ@vHJ`@iq|zyF85_J7C5es6s&-Jd-2*3o~FWB+Bi)II8W+?uPSPk(vG%ZZ^=)~3!K zF!0f<^J~IidrnYGYiDtsH=;!&+V5~dXy1j$n;>^U21Bm!Xl-7IQ1A+8S#uSh>wNW_ zL_r8d{RfgD7$Fn=G`SRV0Ck`1^A{n)@ZNOr*KDR$VLhDTX=2cy1DXu`@Yo&(!uEO^%9pa=Tfx!Q+wM}; z2*PU^#SD{Fs&fcez}dhPn1%H?JHYGK?T|%WN$=TB6b__v&@WeHqhWbiZ ztxLm=waUIZm4oQJ)R*x&u@>^^c;lb@vV;Lwpti41Zq#~kiv}) zx5xPk4B|)~9HhzJkgR93&w*&o9T>UIK>CVn>XAcj5`zE z_4=bP?0@^k;~S^lp5=dHclyIKsM5JA0RHtA`RjInmeDiL}dO zu>WsCrGYEu)nEl+eiVuUhrz~z3Dk=Lm;9(J)-K zX!BDItO8>n0#x=GSU#Afs{YKMj0tu*4F^YtHXzg{*MV8p0qoHeVB^93Rdde=ka4hG z&cG5+RJEXsv`(gu!FFk{OkEKDjAR^Qm&YS*$DxaXJu<8SODB^hOZvT$y3qWoui1}! zgKcsuHVbD(K3owp2#vw~T45hA-c6ktvAx#+)OD+P_QZRI|O}c*ob&Rr0M+Q*W zD7$a6GW_NL{$n0LE=v=KnUFIwI`y zf1$Dfm73Mo2+dA|n(MU@WK6P4uSZaRl3f-fwc*%Cm^KrPy_MKMX?-LalkL)5k(3W% z97r9>c6k;KMoxKw3Zrc@fpG}5ARh%AtFqAHHu(ye&v&p)oJUz`Q@6D21>=d}k;NvT z1Y<3#4{{`q$nVB=f^n2ELx@d&53CFf=0?~I+0ihbOo>@?JDymdK%o|!v?rRnQtVP# z3>mF(wBgwJ7{1Ix`Kfkkr-eEoKC@6)Dv}xJ z?6)==9t5kRqp4YP&>%e}cx@(v@g!;hZKtJf1W0)n`TZNb%fca zjd9c!VV4faQGU2x3XG=?h+E^ym~J<$i%0Otl%6HOiYLtsp3ab<1y!KVWSf+eKt`Kg zx<7&PZR5Q-4g>~H96rxgiL?+T0L!cvmN;tppBS~UWRCSv`yr~TM0nKFSK~x6KfvYVy7cM~EO~+&Uk8?KNUcxSL$U%lz>1pzE-ZQ6awaI( zBcabLv|$h{SjnSNSn|ZHnV_)b_BBjUSTg6xu_0>#18^6>g(Vw$FB6o$lE>Yr)@oAX ze^WQ0f)#ZFBAw41=;dz24EP1Wg(dfTkO@j($^AB|wVGt}xA-_qZvR&w?<=|fAzwX9 z-h2})ndn-;tGr`f#v%Kq@~)iNcEtwUkS zq5o$$uQ%7H=m&Q%`f>PwLUQC-Nn<#gNBKlL_ zf?}GMYa-JuMU<&zRx$kz>=m#;l4lpw6H`p|`vr<otFEmXvt=*fk}6fh0r z7AayVl`JZzm%&~E3nO`PF;(Z8sBW<$M$j%WbH0gUOBHb-xk`)a5ZF<$D2iHAOf8Cu zRxeS+7&;6#qQFE+4n>TmR!1?N0DB)Sjz*Rh)7{few7yIc6X;#Aj2R}%E?2}MbWb__ z13M3vNSPJz?=}fkxKBzY=&`r-+m2U9dlb zW!EcWF5OcP|K`J6u&I>U0RI-i+Xh9OxAN^4QX za=N_*{(*f0=A@KX_~$gyFIyF{iq3#dt3;5n!_i9R!)o3DjMM(LAQ9ruLEXpnY&?y$LaPQ#c_tuPwlMkXRPzysEK{^ z;w9S8?D4e@dv9#`J>DC%?s^owH#Yok?+wj<&ryDR#@h};EiCmV;b$Fs9e4UM>-c}o zA64OX>hr3T|5V=nurctT!_WUV>o2|WHAHfahll#+Cd z3-z@V`{qUb|B>)>e&}BvT#ePfi$K*sU+2Yl5kIQJ>(n;`@4VW-&lC3een3Y1rFiYj z5AOGW*X`VW-j(XH4C|i_pxmZ@n&XZ)&X|{@jwDF7#IQ! z1rh;%(SV#%UoS>N@+%VdXEVUBS?&Vv2JQhm05%<_dR5!7f@61kONEr zCIb8h;ugRPaG?3UW-<^B!~pyf=~n>1a@q)RGOY&cfd+tAz7$vjH~{_!cPcO$7z;cA z@M|i53Fj%nFHV6npd6?G2yg-X!k`kU2DSn?<7-O$Sl{z1@mu4tOfS z{R;rSEf>%FcwUSYjz6cJsq>6HWA>U-w=o(SlQbg7^&AK-~NP4j?h0H+f_ zbNyUk7BCYi1d4#!z#PDPGk(njcujddYk?YIKESER>sSiO8)6}_09XuE0ZV`ipd4@j zWq=dllym`008Y)N0Ox)k&;Ymro-2q0{y&q4jq&(8j9r5Vl)r4Ci7yTF!xL4#R7V$H zij3I~BID4W9EJ8m7M&mf4FGR zPD>t(3C}{&eluD2L|gPrEBAl(*w)>#erwsQekp~Y@A#A(YpPW(hvik+i#VU4HSRCD&Y!A(Sb^-%|QQtz--ZPq$ zlw>6D76W~Lz-rO2x@1?qS+VM_nxWhQ?dVb7pX{%jE&3gory5dTsJ&4BoA0K-vp?;9 z-7FpHPxnEn`XUT4ORG=UFT2b+c5Y}}Lr4h*+Tc7o=vNzNjCpN)RM)<6w2aeQpd|R9 zSyDAb#GxzSkiogpUbbv#c23|;8EhW9!u~dy79K2+>W%cfgH|afkWN9245R^vtWrZD zO*v#u*RRLi@y^^~A2m-7#b9;~YV`{=tw#@@i(FaQuC}OZ*9XzlhphC}AEN1jKScP$ zF1`grm!Y-WL}L$|)3rM)Li?i?rK`6WG|a|W_6u30UpD!C|LVqvj&~nZTT;fs+OJHs z=CIkKU!&Qu+w|#{N4EAcC@h!;9WhIosrzuHv^AK34Nrc&TChDf6x!K6~MpZ_>OF$~aPB(eL6sd*ReQclj^KRcA{{O&*6^ zJhb(QRq6?)|2$G4od~7D-BvdB@}W`G-W|Y?9jG}FMti!=QcoD2=q`}V;WX^1)uLb6 zxqLid$}Q@-Thq!>MTgyctkQ}|dg7>6x;>Ik9yMF^`#7Cr=C2?2Vs@tV?H0@)NFhCD zY05wv)l(ofndw}wS=>RLJyE`GDZOH**L$oM{U%OzTIu6sMxXyBW=z8_$Bf6!6#j-) zniWOUAj+d?)f;AueqpEWy||F>@}yRD=On?Y(Hu?BzG0U9VyFwD0e?Nh%CM4^vyVa_|*GzP<&M2+C(clAOpCA6tA`D_h`P0)%BfIeS;Zk zfG65;W6nlC`UGL`MwVuq}$G(2TNN;~Io<9G?pGr?PXo}~bwY;>|AbuqNH~zTSl>h($ diff --git a/package.json b/package.json index 8a8eccd..976e491 100644 --- a/package.json +++ b/package.json @@ -15,9 +15,14 @@ "typescript": "^5.0.0" }, "dependencies": { + "@cliqz/adblocker-puppeteer": "1.23.8", "callsites": "^4.1.0", "puppeteer": "^21.3.5", "puppeteer-core": "^21.3.5", - "puppeteer-extra": "^3.3.6" + "puppeteer-extra": "^3.3.6", + "puppeteer-extra-plugin-adblocker": "^2.13.6", + "puppeteer-extra-plugin-anonymize-ua": "^2.4.6", + "puppeteer-extra-plugin-recaptcha": "^3.6.8", + "puppeteer-extra-plugin-stealth": "^2.11.2" } } diff --git a/packages.dhall b/packages.dhall deleted file mode 100644 index 2fafb1f..0000000 --- a/packages.dhall +++ /dev/null @@ -1,105 +0,0 @@ -{- -Welcome to your new Dhall package-set! - -Below are instructions for how to edit this file for most use -cases, so that you don't need to know Dhall to use it. - -## Use Cases - -Most will want to do one or both of these options: -1. Override/Patch a package's dependency -2. Add a package not already in the default package set - -This file will continue to work whether you use one or both options. -Instructions for each option are explained below. - -### Overriding/Patching a package - -Purpose: -- Change a package's dependency to a newer/older release than the - default package set's release -- Use your own modified version of some dependency that may - include new API, changed API, removed API by - using your custom git repo of the library rather than - the package set's repo - -Syntax: -where `entityName` is one of the following: -- dependencies -- repo -- version -------------------------------- -let upstream = -- -in upstream - with packageName.entityName = "new value" -------------------------------- - -Example: -------------------------------- -let upstream = -- -in upstream - with halogen.version = "master" - with halogen.repo = "https://example.com/path/to/git/repo.git" - - with halogen-vdom.version = "v4.0.0" - with halogen-vdom.dependencies = [ "extra-dependency" ] # halogen-vdom.dependencies -------------------------------- - -### Additions - -Purpose: -- Add packages that aren't already included in the default package set - -Syntax: -where `` is: -- a tag (i.e. "v4.0.0") -- a branch (i.e. "master") -- commit hash (i.e. "701f3e44aafb1a6459281714858fadf2c4c2a977") -------------------------------- -let upstream = -- -in upstream - with new-package-name = - { dependencies = - [ "dependency1" - , "dependency2" - ] - , repo = - "https://example.com/path/to/git/repo.git" - , version = - "" - } -------------------------------- - -Example: -------------------------------- -let upstream = -- -in upstream - with benchotron = - { dependencies = - [ "arrays" - , "exists" - , "profunctor" - , "strings" - , "quickcheck" - , "lcg" - , "transformers" - , "foldable-traversable" - , "exceptions" - , "node-fs" - , "node-buffer" - , "node-readline" - , "datetime" - , "now" - ] - , repo = - "https://github.com/hdgarrood/purescript-benchotron.git" - , version = - "v7.0.0" - } -------------------------------- --} -let upstream = - https://github.com/purescript/package-sets/releases/download/psc-0.15.10-20230921/packages.dhall - sha256:8c2123d78b41b74a5599f220cf526b48003804a490a85c324fd6a25215a94084 - -in upstream diff --git a/spago.dhall b/spago.dhall deleted file mode 100644 index fd1da6f..0000000 --- a/spago.dhall +++ /dev/null @@ -1,55 +0,0 @@ -{- -Welcome to a Spago project! -You can edit this file as you like. - -Need help? See the following resources: -- Spago documentation: https://github.com/purescript/spago -- Dhall language tour: https://docs.dhall-lang.org/tutorials/Language-Tour.html - -When creating a new Spago project, you can use -`spago init --no-comments` or `spago init -C` -to generate this file without the comments in this block. --} -{ name = "my-project" -, dependencies = - [ "aff" - , "aff-promise" - , "arrays" - , "bifunctors" - , "console" - , "control" - , "datetime" - , "effect" - , "either" - , "enums" - , "exceptions" - , "filterable" - , "foldable-traversable" - , "foreign" - , "identity" - , "integers" - , "maybe" - , "newtype" - , "node-buffer" - , "node-path" - , "node-process" - , "node-streams" - , "nullable" - , "ordered-collections" - , "parallel" - , "prelude" - , "simple-json" - , "spec" - , "st" - , "strings" - , "tailrec" - , "transformers" - , "tuples" - , "unsafe-coerce" - , "web-cssom" - , "web-dom" - , "web-html" - ] -, packages = ./packages.dhall -, sources = [ "src/**/*.purs", "test/**/*.purs" ] -} diff --git a/spago.yaml b/spago.yaml new file mode 100644 index 0000000..ee3ed61 --- /dev/null +++ b/spago.yaml @@ -0,0 +1,46 @@ +package: + dependencies: + - aff + - aff-promise + - arrays + - bifunctors + - console + - control + - datetime + - dotenv + - effect + - either + - enums + - exceptions + - filterable + - foldable-traversable + - foreign + - identity + - integers + - maybe + - newtype + - node-buffer + - node-path + - node-process + - node-streams + - nullable + - ordered-collections + - parallel + - prelude + - simple-json + - spec + - st + - strings + - tailrec + - transformers + - tuples + - unsafe-coerce + - web-cssom + - web-dom + - web-html + name: puppeteer +workspace: + extra_packages: {} + package_set: + url: https://raw.githubusercontent.com/purescript/package-sets/psc-0.15.10-20230921/packages.json + hash: sha256-pb4kxdVVOLZIDNaKgTY0oEdPEZlHuEvNj2xOz2nwMnM= diff --git a/src/Puppeteer.Base.js b/src/Puppeteer.Base.js index 9ea7f11..b2db30b 100644 --- a/src/Puppeteer.Base.js +++ b/src/Puppeteer.Base.js @@ -3,3 +3,6 @@ export const unsafeLog = a => { console.log(a) return a } + +/** @type {(a: A) => (b: B) => A & B} */ +export const unsafeUnion = a => b => ({ ...a, ...b }) diff --git a/src/Puppeteer.Base.purs b/src/Puppeteer.Base.purs index 75f23ab..4ffd1e6 100644 --- a/src/Puppeteer.Base.purs +++ b/src/Puppeteer.Base.purs @@ -4,19 +4,45 @@ import Prelude import Control.Alt ((<|>)) import Control.Monad.Error.Class (liftMaybe, try) +import Control.Monad.Except (runExcept) import Control.Parallel (parallel, sequential) -import Data.Either (hush) +import Data.Bifunctor (lmap) +import Data.Either (Either(..), hush) +import Data.Map (Map) import Data.Maybe (Maybe(..)) import Data.Time.Duration (Milliseconds) import Effect.Aff (Aff, delay) -import Effect.Exception (error) +import Effect.Exception (Error, error) import Foreign (Foreign, unsafeFromForeign) +import Foreign.Object (Object) +import Foreign.Object as Object import Prim.Row (class Union) import Puppeteer.FFI as FFI -import Simple.JSON (class ReadForeign, writeImpl) +import Simple.JSON (class ReadForeign, class WriteForeign, readImpl, writeImpl) import Web.HTML as HTML foreign import unsafeLog :: forall a. a -> a +foreign import unsafeUnion :: forall a b c. a -> b -> c + +data JsDuplex a ir = JsDuplex + { from :: ir -> Either String a + , into :: a -> ir + } + +duplex :: forall a ir. (a -> ir) -> (ir -> Either String a) -> JsDuplex a ir +duplex into from = JsDuplex { from, into } + +duplexRead :: forall a ir. ReadForeign ir => JsDuplex a ir -> Foreign -> Either Error a +duplexRead (JsDuplex { from }) = lmap error <<< flip bind from <<< lmap show <<< runExcept <<< readImpl + +duplexWrite :: forall a ir. WriteForeign ir => JsDuplex a ir -> a -> Foreign +duplexWrite (JsDuplex { into }) = writeImpl <<< into + +mapToObject :: forall v. WriteForeign v => Map String v -> Object Foreign +mapToObject = Object.fromFoldableWithIndex <<< map writeImpl + +merge :: forall a b c. Union a b c => Record a -> Record b -> Record c +merge a b = unsafeUnion a b timeout :: forall a. Milliseconds -> Aff a -> Aff (Maybe a) timeout t a = @@ -30,10 +56,10 @@ timeoutThrow t a = liftMaybe (error "timeout") =<< timeout t a newtype Context (a :: Symbol) = Context (Unit -> Aff Unit) -instance semicontext :: Semigroup (Context a) where +instance Semigroup (Context a) where append _ a = a -instance monoidcontext :: Monoid (Context a) where +instance Monoid (Context a) where mempty = Context $ const $ pure unit closeContext :: forall (a :: Symbol). Context a -> Aff Unit @@ -50,122 +76,51 @@ type Viewport = , isMobile :: Maybe Boolean } -prepareViewport :: Viewport -> Foreign -prepareViewport { deviceScaleFactor, hasTouch, height, width, isLandscape, isMobile } = - writeImpl - { deviceScaleFactor: FFI.maybeToUndefined deviceScaleFactor - , hasTouch: FFI.maybeToUndefined hasTouch - , isLandscape: FFI.maybeToUndefined isLandscape - , isMobile: FFI.maybeToUndefined isMobile - , height - , width - } +duplexViewport :: JsDuplex Viewport Viewport +duplexViewport = duplex identity pure --| [`PuppeteerNode`](https://pptr.dev/api/puppeteer.puppeteernode) foreign import data Puppeteer :: Row Type -> Type data LifecycleEvent = Load | DomContentLoaded | NetworkIdleZeroConnections | NetworkIdleAtMostTwoConnections -prepareLifecycleEvent :: LifecycleEvent -> Foreign -prepareLifecycleEvent Load = writeImpl "load" -prepareLifecycleEvent DomContentLoaded = writeImpl "domcontentloaded" -prepareLifecycleEvent NetworkIdleZeroConnections = writeImpl "networkidle0" -prepareLifecycleEvent NetworkIdleAtMostTwoConnections = writeImpl "networkidle2" - ---| A puppeteer plugin ---| ---| [`puppeteer-extra`](https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin) ---| ---| `src/DebugPlugin.js` ---| ```javascript ---| import { PuppeteerExtraPlugin } from 'puppeteer-extra-plugin' ---| import { PuppeteerExtra } from 'puppeteer-extra' ---| import { Page } from 'puppeteer' ---| ---| /** @typedef {Page & {sayHello: () => void}} DebugPluginPage */ ---| ---| class DebugPlugin extends PuppeteerExtraPlugin { ---| name = 'hello-world' ---| ---| constructor(opts = {}) { ---| super(opts) ---| } ---| ---| async onPageCreated(page) { ---| page.sayHello = () => console.log('hello') ---| } ---| } ---| ---| /** @type {() => DebugPlugin} */ ---| export const makeDebugPlugin = () => new DebugPlugin() ---| ---| /** @type {(_1: DebugPlugin) => (_2: PuppeteerExtra) => () => PuppeteerExtra} */ ---| export const registerDebugPlugin = dp => p => () => p.use(dp) ---| ---| /** @type {(_1: PuppeteerExtra) => (_2: DebugPluginPage) => () => void} */ ---| export const sayHello = () => page => () => page.sayHello() ---| ``` ---| ---| `src/DebugPlugin.purs` ---| ```purescript ---| module DebugPlugin where ---| ---| import Prelude ---| import Effect (Effect) ---| import Effect.Class (class MonadEffect) ---| import Puppeteer (class Plugin, Puppeteer) ---| import Puppeteer.Page (Page) ---| ---| foreign import data DebugPlugin :: Type ---| ---| foreign import makeDebugPlugin :: Effect DebugPlugin ---| ---| foreign import registerDebugPlugin :: forall (r :: Row Type) ---| . Puppeteer r ---| -> Effect (Puppeteer (debugPlugin :: DebugPlugin | r)) ---| ---| -- Note: ---| -- The puppeteer instance used here must have been ---| -- registered with `DebugPlugin`'s `use` in order to ---| -- invoke `sayHello` ---| foreign import sayHello :: forall (r :: Row Type) ---| . Puppeteer (debugPlugin :: DebugPlugin | r) ---| -> Page ---| -> Effect Unit ---| ---| instance debugPlugin :: Plugin DebugPlugin (debugPlugin :: DebugPlugin) where ---| use pptr _ = liftEffect $ registerDebugPlugin pptr ---| ``` -class Plugin p (r :: Row Type) | p -> r where - --| Register a given puppeteer instance with plugin `p` - --| - --| The row type `r` should be used in that plugin's purescript - --| API to ensure the puppeteer instance used has had that - --| plugin registered. - use :: forall b c. Union r b c => Puppeteer r -> p -> Aff (Puppeteer c) +duplexLifecycleEvent :: JsDuplex LifecycleEvent String +duplexLifecycleEvent = + let + toString Load = "load" + toString DomContentLoaded = "domcontentloaded" + toString NetworkIdleZeroConnections = "networkidle0" + toString NetworkIdleAtMostTwoConnections = "networkidle2" + fromString "load" = Right Load + fromString "domcontentloaded" = Right DomContentLoaded + fromString "networkidle0" = Right NetworkIdleZeroConnections + fromString "networkidle2" = Right NetworkIdleAtMostTwoConnections + fromString o = Left $ "unknown lifecycle event " <> o + in + duplex toString fromString --| [`Browser`](https://pptr.dev/api/puppeteer.browser) foreign import data Browser :: Type -instance browserForeign :: ReadForeign Browser where +instance ReadForeign Browser where readImpl = pure <<< unsafeFromForeign --| [`Page`](https://pptr.dev/api/puppeteer.page) foreign import data Page :: Type -instance pageForeign :: ReadForeign Page where +instance ReadForeign Page where readImpl = pure <<< unsafeFromForeign --| [`Frame`](https://pptr.dev/api/puppeteer.frame) foreign import data Frame :: Type -instance frameForeign :: ReadForeign Frame where +instance ReadForeign Frame where readImpl = pure <<< unsafeFromForeign --| [`BrowserContext`](https://pptr.dev/api/puppeteer.browsercontext) foreign import data BrowserContext :: Type -instance browserContextForeign :: ReadForeign BrowserContext where +instance ReadForeign BrowserContext where readImpl = pure <<< unsafeFromForeign --| Represents both [`JSHandle`](https://pptr.dev/api/puppeteer.jshandle) & [`ElementHandle`](https://pptr.dev/api/puppeteer.elementhandle) @@ -174,100 +129,103 @@ foreign import data Handle :: Type -> Type --| [`Keyboard`](https://pptr.dev/api/puppeteer.keyboard) foreign import data Keyboard :: Type +instance ReadForeign Keyboard where + readImpl = pure <<< unsafeFromForeign + foreign import data Request :: Type -instance foreignRequest :: ReadForeign Request where +instance ReadForeign Request where readImpl = pure <<< unsafeFromForeign foreign import data Response :: Type -instance foreignResponse :: ReadForeign Response where +instance ReadForeign Response where readImpl = pure <<< unsafeFromForeign --| `Browser` or `BrowserContext` class PageProducer :: Type -> Constraint class PageProducer a -instance bpp :: PageProducer Browser -instance bcpp :: PageProducer BrowserContext +instance PageProducer Browser +instance PageProducer BrowserContext --| `Page` or `Handle` class EvalTarget :: Type -> Constraint class EvalTarget a -instance pet :: EvalTarget Page -instance het :: EvalTarget (Handle a) +instance EvalTarget Page +instance EvalTarget (Handle a) --| `Page` or `BrowserContext` class BrowserAccess :: Type -> Constraint class BrowserAccess a -instance pba :: BrowserAccess Browser -instance bcba :: BrowserAccess BrowserContext +instance BrowserAccess Browser +instance BrowserAccess BrowserContext class IsElement :: Type -> Constraint class IsElement e -instance anchorIsElement :: IsElement HTML.HTMLAnchorElement -instance areaIsElement :: IsElement HTML.HTMLAreaElement -instance audioIsElement :: IsElement HTML.HTMLAudioElement -instance bRIsElement :: IsElement HTML.HTMLBRElement -instance baseIsElement :: IsElement HTML.HTMLBaseElement -instance bodyIsElement :: IsElement HTML.HTMLBodyElement -instance buttonIsElement :: IsElement HTML.HTMLButtonElement -instance canvasIsElement :: IsElement HTML.HTMLCanvasElement -instance dListIsElement :: IsElement HTML.HTMLDListElement -instance dataIsElement :: IsElement HTML.HTMLDataElement -instance dataListIsElement :: IsElement HTML.HTMLDataListElement -instance divIsElement :: IsElement HTML.HTMLDivElement -instance document :: IsElement HTML.HTMLDocument -instance element :: IsElement HTML.HTMLElement -instance embedIsElement :: IsElement HTML.HTMLEmbedElement -instance fieldSetIsElement :: IsElement HTML.HTMLFieldSetElement -instance formIsElement :: IsElement HTML.HTMLFormElement -instance hRIsElement :: IsElement HTML.HTMLHRElement -instance headIsElement :: IsElement HTML.HTMLHeadElement -instance headingIsElement :: IsElement HTML.HTMLHeadingElement -instance iFrameIsElement :: IsElement HTML.HTMLIFrameElement -instance imageIsElement :: IsElement HTML.HTMLImageElement -instance inputIsElement :: IsElement HTML.HTMLInputElement -instance keygenIsElement :: IsElement HTML.HTMLKeygenElement -instance lIIsElement :: IsElement HTML.HTMLLIElement -instance labelIsElement :: IsElement HTML.HTMLLabelElement -instance legendIsElement :: IsElement HTML.HTMLLegendElement -instance linkIsElement :: IsElement HTML.HTMLLinkElement -instance mapIsElement :: IsElement HTML.HTMLMapElement -instance mediaIsElement :: IsElement HTML.HTMLMediaElement -instance metaIsElement :: IsElement HTML.HTMLMetaElement -instance meterIsElement :: IsElement HTML.HTMLMeterElement -instance modIsElement :: IsElement HTML.HTMLModElement -instance oListIsElement :: IsElement HTML.HTMLOListElement -instance objectIsElement :: IsElement HTML.HTMLObjectElement -instance optGroupIsElement :: IsElement HTML.HTMLOptGroupElement -instance optionIsElement :: IsElement HTML.HTMLOptionElement -instance outputIsElement :: IsElement HTML.HTMLOutputElement -instance paragraphIsElement :: IsElement HTML.HTMLParagraphElement -instance paramIsElement :: IsElement HTML.HTMLParamElement -instance preIsElement :: IsElement HTML.HTMLPreElement -instance progressIsElement :: IsElement HTML.HTMLProgressElement -instance quoteIsElement :: IsElement HTML.HTMLQuoteElement -instance scriptIsElement :: IsElement HTML.HTMLScriptElement -instance selectIsElement :: IsElement HTML.HTMLSelectElement -instance sourceIsElement :: IsElement HTML.HTMLSourceElement -instance spanIsElement :: IsElement HTML.HTMLSpanElement -instance styleIsElement :: IsElement HTML.HTMLStyleElement -instance tableCaptionIsElement :: IsElement HTML.HTMLTableCaptionElement -instance tableCellIsElement :: IsElement HTML.HTMLTableCellElement -instance tableColIsElement :: IsElement HTML.HTMLTableColElement -instance tableDataCellIsElement :: IsElement HTML.HTMLTableDataCellElement -instance tableIsElement :: IsElement HTML.HTMLTableElement -instance tableHeaderCellIsElement :: IsElement HTML.HTMLTableHeaderCellElement -instance tableRowIsElement :: IsElement HTML.HTMLTableRowElement -instance tableSectionIsElement :: IsElement HTML.HTMLTableSectionElement -instance templateIsElement :: IsElement HTML.HTMLTemplateElement -instance textAreaIsElement :: IsElement HTML.HTMLTextAreaElement -instance timeIsElement :: IsElement HTML.HTMLTimeElement -instance titleIsElement :: IsElement HTML.HTMLTitleElement -instance trackIsElement :: IsElement HTML.HTMLTrackElement -instance uListIsElement :: IsElement HTML.HTMLUListElement -instance videoIsElement :: IsElement HTML.HTMLVideoElement +instance IsElement HTML.HTMLAnchorElement +instance IsElement HTML.HTMLAreaElement +instance IsElement HTML.HTMLAudioElement +instance IsElement HTML.HTMLBRElement +instance IsElement HTML.HTMLBaseElement +instance IsElement HTML.HTMLBodyElement +instance IsElement HTML.HTMLButtonElement +instance IsElement HTML.HTMLCanvasElement +instance IsElement HTML.HTMLDListElement +instance IsElement HTML.HTMLDataElement +instance IsElement HTML.HTMLDataListElement +instance IsElement HTML.HTMLDivElement +instance IsElement HTML.HTMLDocument +instance IsElement HTML.HTMLElement +instance IsElement HTML.HTMLEmbedElement +instance IsElement HTML.HTMLFieldSetElement +instance IsElement HTML.HTMLFormElement +instance IsElement HTML.HTMLHRElement +instance IsElement HTML.HTMLHeadElement +instance IsElement HTML.HTMLHeadingElement +instance IsElement HTML.HTMLIFrameElement +instance IsElement HTML.HTMLImageElement +instance IsElement HTML.HTMLInputElement +instance IsElement HTML.HTMLKeygenElement +instance IsElement HTML.HTMLLIElement +instance IsElement HTML.HTMLLabelElement +instance IsElement HTML.HTMLLegendElement +instance IsElement HTML.HTMLLinkElement +instance IsElement HTML.HTMLMapElement +instance IsElement HTML.HTMLMediaElement +instance IsElement HTML.HTMLMetaElement +instance IsElement HTML.HTMLMeterElement +instance IsElement HTML.HTMLModElement +instance IsElement HTML.HTMLOListElement +instance IsElement HTML.HTMLObjectElement +instance IsElement HTML.HTMLOptGroupElement +instance IsElement HTML.HTMLOptionElement +instance IsElement HTML.HTMLOutputElement +instance IsElement HTML.HTMLParagraphElement +instance IsElement HTML.HTMLParamElement +instance IsElement HTML.HTMLPreElement +instance IsElement HTML.HTMLProgressElement +instance IsElement HTML.HTMLQuoteElement +instance IsElement HTML.HTMLScriptElement +instance IsElement HTML.HTMLSelectElement +instance IsElement HTML.HTMLSourceElement +instance IsElement HTML.HTMLSpanElement +instance IsElement HTML.HTMLStyleElement +instance IsElement HTML.HTMLTableCaptionElement +instance IsElement HTML.HTMLTableCellElement +instance IsElement HTML.HTMLTableColElement +instance IsElement HTML.HTMLTableDataCellElement +instance IsElement HTML.HTMLTableElement +instance IsElement HTML.HTMLTableHeaderCellElement +instance IsElement HTML.HTMLTableRowElement +instance IsElement HTML.HTMLTableSectionElement +instance IsElement HTML.HTMLTemplateElement +instance IsElement HTML.HTMLTextAreaElement +instance IsElement HTML.HTMLTimeElement +instance IsElement HTML.HTMLTitleElement +instance IsElement HTML.HTMLTrackElement +instance IsElement HTML.HTMLUListElement +instance IsElement HTML.HTMLVideoElement diff --git a/src/Puppeteer.Browser.purs b/src/Puppeteer.Browser.purs index e0388dd..1599533 100644 --- a/src/Puppeteer.Browser.purs +++ b/src/Puppeteer.Browser.purs @@ -3,10 +3,12 @@ module Puppeteer.Browser , Product(..) , ChromeReleaseChannel(..) , Connect + , duplexConnect + , duplexProduct + , duplexChromeReleaseChannel , disconnect , websocketEndpoint , connected - , prepareConnectOptions , get , close ) where @@ -15,63 +17,93 @@ import Prelude import Control.Promise (Promise) import Control.Promise as Promise +import Data.Either (Either(..)) import Data.Enum (fromEnum) +import Data.Generic.Rep (class Generic) import Data.Maybe (Maybe) -import Data.Time (Millisecond) +import Data.Newtype (unwrap, wrap) +import Data.Show.Generic (genericShow) +import Data.Time.Duration (Milliseconds(..)) import Effect (Effect) import Effect.Aff (Aff) import Foreign (Foreign, unsafeToForeign) import Puppeteer.Base (Browser) as X -import Puppeteer.Base (class BrowserAccess, Browser, BrowserContext, Viewport) +import Puppeteer.Base (class BrowserAccess, Browser, BrowserContext, JsDuplex(..), Viewport, duplex) import Puppeteer.FFI as FFI +import Record (modify) import Simple.JSON (writeImpl) +import Type.Prelude (Proxy(..)) data Product = Chrome | Firefox +derive instance Generic Product _ +derive instance Eq Product +instance Show Product where + show = genericShow + +duplexProduct :: JsDuplex Product String +duplexProduct = + let + toString Chrome = "chrome" + toString Firefox = "firefox" + fromString "chrome" = pure Chrome + fromString "firefox" = pure Firefox + fromString o = Left $ "unknown browser product " <> o + in + duplex toString fromString + data ChromeReleaseChannel = ChromeStable | ChromeBeta | ChromeCanary | ChromeDev +derive instance Generic ChromeReleaseChannel _ +derive instance Eq ChromeReleaseChannel +instance Show ChromeReleaseChannel where + show = genericShow + +duplexChromeReleaseChannel :: JsDuplex ChromeReleaseChannel String +duplexChromeReleaseChannel = + let + toString ChromeStable = "chrome" + toString ChromeBeta = "chrome-beta" + toString ChromeCanary = "chrome-canary" + toString ChromeDev = "chrome-dev" + fromString "chrome" = pure ChromeStable + fromString "chrome-beta" = pure ChromeBeta + fromString "chrome-canary" = pure ChromeCanary + fromString "chrome-dev" = pure ChromeDev + fromString o = Left $ "unknown chrome release channel " <> o + in + duplex toString fromString + type Connect = { defaultViewport :: Maybe Viewport , ignoreHTTPSErrors :: Maybe Boolean - , protocolTimeout :: Maybe Millisecond - , slowMo :: Maybe Millisecond + , protocolTimeout :: Maybe Milliseconds + , slowMo :: Maybe Milliseconds } -prepareViewport :: Viewport -> Foreign -prepareViewport - { deviceScaleFactor - , hasTouch - , height - , width - , isLandscape - , isMobile - } = writeImpl - { deviceScaleFactor: FFI.maybeToUndefined deviceScaleFactor - , hasTouch: FFI.maybeToUndefined hasTouch - , height - , width - , isLandscape: FFI.maybeToUndefined isLandscape - , isMobile: FFI.maybeToUndefined isMobile +type ConnectRaw = + { defaultViewport :: Maybe Viewport + , ignoreHTTPSErrors :: Maybe Boolean + , protocolTimeout :: Maybe Number + , slowMo :: Maybe Number } -prepareConnectOptions :: Connect -> Foreign -prepareConnectOptions - { defaultViewport - , ignoreHTTPSErrors - , protocolTimeout - , slowMo - } = writeImpl - { defaultViewport: FFI.maybeToUndefined $ map prepareViewport defaultViewport - , ignoreHTTPSErrors: FFI.maybeToUndefined ignoreHTTPSErrors - , protocolTimeout: FFI.maybeToUndefined $ map fromEnum protocolTimeout - , slowMo: FFI.maybeToUndefined $ map fromEnum slowMo - } +duplexConnect :: JsDuplex Connect ConnectRaw +duplexConnect = + let + into r = modify (Proxy :: Proxy "protocolTimeout") (map unwrap) + $ modify (Proxy :: Proxy "slowMo") (map unwrap) r + from r = pure + $ modify (Proxy :: Proxy "protocolTimeout") (map wrap) + $ modify (Proxy :: Proxy "slowMo") (map wrap) r + in + duplex into from foreign import _close :: Browser -> Promise Unit foreign import _get :: Foreign -> Effect Browser diff --git a/src/Puppeteer.Page.Navigate.js b/src/Puppeteer.Page.Navigate.js index 05019e5..fb2c3fe 100644 --- a/src/Puppeteer.Page.Navigate.js +++ b/src/Puppeteer.Page.Navigate.js @@ -1,13 +1,16 @@ import { Page } from 'puppeteer' -/** @type {(ev: import('puppeteer').PuppeteerLifeCycleEvent) => (p: Page) => Promise} */ -export const _forward = ev => p => p.goForward({ timeout: 0, waitUntil: ev }) +/** @type {(ev: import('puppeteer').PuppeteerLifeCycleEvent) => (p: Page) => () => Promise} */ +export const _forward = ev => p => () => + p.goForward({ timeout: 0, waitUntil: ev }) -/** @type {(ev: import('puppeteer').PuppeteerLifeCycleEvent) => (p: Page) => Promise} */ -export const _back = ev => p => p.goBack({ timeout: 0, waitUntil: ev }) +/** @type {(ev: import('puppeteer').PuppeteerLifeCycleEvent) => (p: Page) => () => Promise} */ +export const _back = ev => p => () => p.goBack({ timeout: 0, waitUntil: ev }) -/** @type {(url: string) => (ev: import('puppeteer').PuppeteerLifeCycleEvent) => (p: Page) => Promise} */ -export const _to = url => ev => p => p.goto(url, { timeout: 0, waitUntil: ev }) +/** @type {(url: string) => (ev: import('puppeteer').PuppeteerLifeCycleEvent) => (p: Page) => () => Promise} */ +export const _to = url => ev => p => () => + p.goto(url, { timeout: 0, waitUntil: ev }) -/** @type {(ev: import('puppeteer').PuppeteerLifeCycleEvent) => (p: Page) => Promise} */ -export const _reload = ev => p => p.goForward({ timeout: 0, waitUntil: ev }) +/** @type {(ev: import('puppeteer').PuppeteerLifeCycleEvent) => (p: Page) => () => Promise} */ +export const _reload = ev => p => () => + p.goForward({ timeout: 0, waitUntil: ev }) diff --git a/src/Puppeteer.Page.Navigate.purs b/src/Puppeteer.Page.Navigate.purs index e649025..5ea41a7 100644 --- a/src/Puppeteer.Page.Navigate.purs +++ b/src/Puppeteer.Page.Navigate.purs @@ -7,27 +7,28 @@ import Control.Promise as Promise import Data.Maybe (Maybe) import Data.Newtype (unwrap) import Data.Time.Duration (Milliseconds(..)) +import Effect (Effect) import Effect.Aff (Aff) import Foreign (Foreign) -import Puppeteer.Base (LifecycleEvent(..), Page, URL, prepareLifecycleEvent) +import Puppeteer.Base (LifecycleEvent(..), Page, URL, duplexLifecycleEvent, duplexWrite) import Puppeteer.HTTP as HTTP -foreign import _forward :: Foreign -> Page -> Promise (Maybe HTTP.Response) -foreign import _back :: Foreign -> Page -> Promise (Maybe HTTP.Response) -foreign import _reload :: Foreign -> Page -> Promise (Maybe HTTP.Response) -foreign import _to :: String -> Foreign -> Page -> Promise (Maybe HTTP.Response) +foreign import _forward :: Foreign -> Page -> Effect (Promise (Maybe HTTP.Response)) +foreign import _back :: Foreign -> Page -> Effect (Promise (Maybe HTTP.Response)) +foreign import _reload :: Foreign -> Page -> Effect (Promise (Maybe HTTP.Response)) +foreign import _to :: String -> Foreign -> Page -> Effect (Promise (Maybe HTTP.Response)) forward :: LifecycleEvent -> Page -> Aff (Maybe HTTP.Response) -forward ev = Promise.toAff <<< _forward (prepareLifecycleEvent ev) +forward ev = Promise.toAffE <<< _forward (duplexWrite duplexLifecycleEvent ev) back :: LifecycleEvent -> Page -> Aff (Maybe HTTP.Response) -back ev = Promise.toAff <<< _back (prepareLifecycleEvent ev) +back ev = Promise.toAffE <<< _back (duplexWrite duplexLifecycleEvent ev) -to :: URL -> LifecycleEvent -> Page -> Aff (Maybe HTTP.Response) -to url ev = Promise.toAff <<< _to url (prepareLifecycleEvent ev) +to :: LifecycleEvent -> Page -> URL -> Aff (Maybe HTTP.Response) +to ev p u = Promise.toAffE $ _to u (duplexWrite duplexLifecycleEvent ev) p reload :: LifecycleEvent -> Page -> Aff (Maybe HTTP.Response) -reload ev = Promise.toAff <<< _reload (prepareLifecycleEvent ev) +reload ev = Promise.toAffE <<< _reload (duplexWrite duplexLifecycleEvent ev) forward_ :: Page -> Aff (Maybe HTTP.Response) forward_ = forward Load @@ -35,8 +36,8 @@ forward_ = forward Load back_ :: Page -> Aff (Maybe HTTP.Response) back_ = back Load -to_ :: URL -> Page -> Aff (Maybe HTTP.Response) -to_ url = to url Load +to_ :: Page -> URL -> Aff (Maybe HTTP.Response) +to_ = to Load reload_ :: Page -> Aff (Maybe HTTP.Response) reload_ = reload Load diff --git a/src/Puppeteer.Page.WaitFor.purs b/src/Puppeteer.Page.WaitFor.purs index b1d331c..6396f5b 100644 --- a/src/Puppeteer.Page.WaitFor.purs +++ b/src/Puppeteer.Page.WaitFor.purs @@ -17,7 +17,7 @@ import Data.Time.Duration (Milliseconds(..)) import Effect (Effect) import Effect.Aff (Aff) import Foreign (Foreign) -import Puppeteer.Base (Context(..), Handle, LifecycleEvent, Page, prepareLifecycleEvent) +import Puppeteer.Base (Context(..), Handle, LifecycleEvent, Page, duplexWrite, duplexLifecycleEvent) import Puppeteer.Selector (class Selector, toCSS) newtype NetworkIdleFor = NetworkIdleFor Milliseconds @@ -35,7 +35,7 @@ foreign import _selectorToBeHidden :: String -> Page -> Promise Unit navigation :: LifecycleEvent -> Page -> Effect (Context WaitingForNavigationHint) navigation ev p = do - promise <- _navigation (prepareLifecycleEvent ev) p + promise <- _navigation (duplexWrite duplexLifecycleEvent ev) p pure $ Context (\_ -> Promise.toAff $ promise) networkIdle :: NetworkIdleFor -> Page -> Aff Unit diff --git a/src/Puppeteer.Page.purs b/src/Puppeteer.Page.purs index d1cb092..8ca5425 100644 --- a/src/Puppeteer.Page.purs +++ b/src/Puppeteer.Page.purs @@ -34,14 +34,12 @@ import Control.Promise as Promise import Data.Array as Array import Data.Either (hush) import Data.Maybe (Maybe) -import Data.Nullable (Nullable) -import Data.Nullable as Nullable import Effect (Effect) import Effect.Aff (Aff) import Foreign (Foreign, unsafeToForeign) import Node.Path (FilePath) import Puppeteer.Base (Page) as X -import Puppeteer.Base (class PageProducer, Handle, Keyboard, LifecycleEvent, Page, URL, Viewport, prepareLifecycleEvent, prepareViewport) +import Puppeteer.Base (class PageProducer, Handle, Keyboard, LifecycleEvent, Page, URL, Viewport, duplexLifecycleEvent, duplexViewport, duplexWrite) import Puppeteer.Handle (unsafeCoerceHandle) import Puppeteer.Selector (class Selector, toCSS) import Simple.JSON (readImpl, undefined, writeImpl) @@ -147,10 +145,10 @@ content :: Page -> Aff String content = Promise.toAff <<< _content setContent :: String -> LifecycleEvent -> Page -> Aff Unit -setContent s ev = Promise.toAff <<< _setContent s (prepareLifecycleEvent ev) +setContent s ev = Promise.toAff <<< _setContent s (duplexWrite duplexLifecycleEvent ev) setViewport :: Viewport -> Page -> Aff Unit -setViewport vp = Promise.toAff <<< _setViewport (prepareViewport vp) +setViewport vp = Promise.toAff <<< _setViewport (duplexWrite duplexViewport vp) title :: Page -> Aff String title = Promise.toAff <<< _title diff --git a/src/Puppeteer.Plugin.AdBlock.js b/src/Puppeteer.Plugin.AdBlock.js new file mode 100644 index 0000000..234b3f6 --- /dev/null +++ b/src/Puppeteer.Plugin.AdBlock.js @@ -0,0 +1,20 @@ +import AdBlock, { + PuppeteerExtraPluginAdblocker, +} from 'puppeteer-extra-plugin-adblocker' +import { PuppeteerExtra } from 'puppeteer-extra' + +/** @type {(_: import('puppeteer-extra-plugin-adblocker').PluginOptions) => (_: PuppeteerExtra) => () => PuppeteerExtra} */ +export const _install = o => p => () => p.use(AdBlock(o)) + +/** @type {(_: PuppeteerExtra) => () => Promise} */ +export const _blocker = p => () => { + const adblock = p.plugins.find( + pl => pl instanceof PuppeteerExtraPluginAdblocker, + ) + + if (!adblock || !(adblock instanceof PuppeteerExtraPluginAdblocker)) { + throw new Error('Adblock plugin not registered') + } else { + return adblock.getBlocker() + } +} diff --git a/src/Puppeteer.Plugin.AdBlock.purs b/src/Puppeteer.Plugin.AdBlock.purs new file mode 100644 index 0000000..37c5d60 --- /dev/null +++ b/src/Puppeteer.Plugin.AdBlock.purs @@ -0,0 +1,92 @@ +module Puppeteer.Plugin.AdBlock + ( AdBlockMode(..) + , AdBlockOptions + , AdBlockPlugin + , AdBlocker + , install + , defaultOptions + , blocker + , cspInjectedH + , htmlFilteredH + , requestAllowedH + , requestBlockedH + , requestRedirectedH + , requestWhitelistedH + , scriptInjectedH + , styleInjectedH + ) where + +import Prelude + +import Control.Promise (Promise) +import Control.Promise as Promise +import Data.Maybe (Maybe(..)) +import Effect (Effect) +import Effect.Aff (Aff) +import Foreign (Foreign) +import Node.EventEmitter (EventEmitter) +import Node.EventEmitter as EventEmitter +import Node.EventEmitter.UtilTypes (EventHandle0) as EventEmitter +import Puppeteer.Base (Puppeteer) +import Puppeteer.FFI as FFI +import Simple.JSON (writeImpl) + +-- | https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-adblocker +foreign import data AdBlockPlugin :: Type +foreign import data AdBlocker :: Type + +-- | https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-adblocker#options +data AdBlockMode + -- | Block ads, but not trackers + = BlockAds + -- | Block ads & trackers + | BlockTrackers + -- | Block ads, trackers & annoyances + | BlockAnnoyances + +type AdBlockOptions = { mode :: AdBlockMode, useDiskCache :: Boolean, cacheDir :: Maybe String } + +defaultOptions :: AdBlockOptions +defaultOptions = { mode: BlockAds, useDiskCache: true, cacheDir: Nothing } + +prepareOptions :: AdBlockOptions -> Foreign +prepareOptions { mode, useDiskCache, cacheDir } = FFI.mergeRecords + [ writeImpl case mode of + BlockAds -> { blockTrackers: false, blockTrackersAndAnnoyances: false } + BlockTrackers -> { blockTrackers: true, blockTrackersAndAnnoyances: false } + BlockAnnoyances -> { blockTrackers: true, blockTrackersAndAnnoyances: true } + , writeImpl { useCache: useDiskCache, cacheDir: FFI.maybeToUndefined cacheDir } + ] + +foreign import _install :: forall (r :: Row Type). Foreign -> Puppeteer r -> Effect (Puppeteer (adblock :: AdBlockPlugin | r)) +foreign import _blocker :: forall (r :: Row Type). Puppeteer r -> Effect (Promise AdBlocker) + +install :: forall (r :: Row Type). AdBlockOptions -> Puppeteer r -> Effect (Puppeteer (adblock :: AdBlockPlugin | r)) +install o p = _install (prepareOptions o) p + +blocker :: forall (r :: Row Type). Puppeteer (adblock :: AdBlockPlugin | r) -> Aff AdBlocker +blocker = Promise.toAffE <<< _blocker + +cspInjectedH :: EventEmitter.EventHandle0 AdBlocker +cspInjectedH = EventEmitter.EventHandle "csp-injected" identity + +htmlFilteredH :: EventEmitter.EventHandle0 AdBlocker +htmlFilteredH = EventEmitter.EventHandle "html-filtered" identity + +requestAllowedH :: EventEmitter.EventHandle0 AdBlocker +requestAllowedH = EventEmitter.EventHandle "request-allowed" identity + +requestBlockedH :: EventEmitter.EventHandle0 AdBlocker +requestBlockedH = EventEmitter.EventHandle "request-blocked" identity + +requestRedirectedH :: EventEmitter.EventHandle0 AdBlocker +requestRedirectedH = EventEmitter.EventHandle "request-redirected" identity + +requestWhitelistedH :: EventEmitter.EventHandle0 AdBlocker +requestWhitelistedH = EventEmitter.EventHandle "request-whitelisted" identity + +scriptInjectedH :: EventEmitter.EventHandle0 AdBlocker +scriptInjectedH = EventEmitter.EventHandle "script-injected" identity + +styleInjectedH :: EventEmitter.EventHandle0 AdBlocker +styleInjectedH = EventEmitter.EventHandle "style-injected" identity diff --git a/src/Puppeteer.Plugin.AnonymousUserAgent.js b/src/Puppeteer.Plugin.AnonymousUserAgent.js new file mode 100644 index 0000000..f61c0fb --- /dev/null +++ b/src/Puppeteer.Plugin.AnonymousUserAgent.js @@ -0,0 +1,5 @@ +import AnonUA from 'puppeteer-extra-plugin-anonymize-ua' +import { PuppeteerExtra } from 'puppeteer-extra' + +/** @type {(_: PuppeteerExtra) => () => PuppeteerExtra} */ +export const install = p => () => p.use(AnonUA()) diff --git a/src/Puppeteer.Plugin.AnonymousUserAgent.purs b/src/Puppeteer.Plugin.AnonymousUserAgent.purs new file mode 100644 index 0000000..89a2ab4 --- /dev/null +++ b/src/Puppeteer.Plugin.AnonymousUserAgent.purs @@ -0,0 +1,8 @@ +module Puppeteer.Plugin.AnonymousUserAgent where + +import Effect (Effect) +import Puppeteer.Base (Puppeteer) + +-- | https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-anonymize-ua +foreign import data AnonymousUserAgentPlugin :: Type +foreign import install :: forall (r :: Row Type). Puppeteer r -> Effect (Puppeteer (userAgent :: AnonymousUserAgentPlugin | r)) diff --git a/src/Puppeteer.Plugin.Captcha.js b/src/Puppeteer.Plugin.Captcha.js new file mode 100644 index 0000000..fb81846 --- /dev/null +++ b/src/Puppeteer.Plugin.Captcha.js @@ -0,0 +1,23 @@ +import { PuppeteerExtra } from 'puppeteer-extra' +import Captcha from 'puppeteer-extra-plugin-recaptcha' + +/** @typedef {import('puppeteer-extra-plugin-recaptcha/dist/types').PluginOptions} PluginOptions */ +/** @typedef {import('puppeteer-extra-plugin-recaptcha/dist/types').CaptchaInfo} CaptchaInfo */ +/** @typedef {import('puppeteer-extra-plugin-recaptcha/dist/types').CaptchaSolution} CaptchaSolution */ +/** @typedef {import('puppeteer-extra-plugin-recaptcha/dist/types').CaptchaSolved} CaptchaSolved */ +/** @typedef {import('puppeteer').Page & import('puppeteer-extra-plugin-recaptcha/dist/types').RecaptchaPluginPageAdditions} Page */ + +/** @type {(_: PluginOptions) => (_: PuppeteerExtra) => () => PuppeteerExtra} */ +export const _captcha = o => p => () => p.use(Captcha(o)) + +/** @type {(_: Page) => Promise<{captchas: CaptchaInfo[], filtered: unknown[]}>} */ +export const _findCaptchas = p => p.findRecaptchas() + +/** @type {(_: Page) => (_: CaptchaInfo[]) => Promise<{solutions: CaptchaSolution[]}>} */ +export const _getSolutions = p => cs => p.getRecaptchaSolutions(cs) + +/** @type {(_: Page) => (_: CaptchaSolution[]) => Promise<{solved: CaptchaSolved[]}>} */ +export const _enterSolutions = p => cs => p.enterRecaptchaSolutions(cs) + +/** @type {(_: Page) => Promise<{captchas: CaptchaInfo[], filtered: unknown[], solutions: CaptchaSolution[], solved: CaptchaSolved[]}>} */ +export const _solveCaptchas = p => p.solveRecaptchas() diff --git a/src/Puppeteer.Plugin.Captcha.purs b/src/Puppeteer.Plugin.Captcha.purs new file mode 100644 index 0000000..adfae40 --- /dev/null +++ b/src/Puppeteer.Plugin.Captcha.purs @@ -0,0 +1,308 @@ +module Puppeteer.Plugin.Captcha + ( install + , findCaptchas + , solveCaptchas + , defaultOptions + , CaptchaCallback(..) + , Options + , CaptchaProvider(..) + , CaptchaVendor(..) + , CaptchaPlugin + , CaptchaKind(..) + , CaptchaFiltered(..) + , Token2Captcha(..) + , CaptchaInfo + , CaptchaInfoMaybeFiltered + , CaptchaSolution + , CaptchaSolved + , CaptchaInfoDisplay + , SolveResult + , getSolutions + , enterSolutions + ) where + +import Prelude + +import Control.Monad.Error.Class (liftEither) +import Control.Monad.Except (runExcept) +import Control.Promise (Promise) +import Control.Promise as Promise +import Data.Bifunctor (lmap) +import Data.Either (Either, hush) +import Data.Generic.Rep (class Generic) +import Data.JSDate (JSDate) +import Data.Maybe (Maybe(..)) +import Data.Newtype (class Newtype, unwrap, wrap) +import Data.Show.Generic (genericShow) +import Data.Traversable (for, sequence) +import Data.Tuple (Tuple) +import Data.Tuple.Nested ((/\)) +import Data.Variant (Variant) +import Effect (Effect) +import Effect.Aff (Aff) +import Effect.Exception (Error, error) +import Effect.Unsafe (unsafePerformEffect) +import Foreign (Foreign, unsafeFromForeign, unsafeReadTagged, unsafeToForeign) +import Puppeteer.Base (JsDuplex(..), Page, Puppeteer, duplex, duplexRead, duplexWrite) +import Puppeteer.FFI as FFI +import Record (modify, rename) +import Simple.JSON (class ReadForeign, class WriteForeign, readImpl, writeImpl) +import Type.Prelude (Proxy(..)) + +newtype CoerceDate = CoerceDate (Maybe JSDate) + +derive instance Newtype CoerceDate _ + +instance ReadForeign CoerceDate where + readImpl f = pure $ CoerceDate $ hush $ runExcept $ unsafeReadTagged "Date" f + +instance WriteForeign CoerceDate where + writeImpl (CoerceDate f) = unsafeToForeign f + +newtype Token2Captcha = Token2Captcha String + +derive instance Newtype Token2Captcha _ +derive instance Generic Token2Captcha _ +instance Show Token2Captcha where + show = genericShow + +data CaptchaKind = KindCheckbox | KindInvisible | KindScore | KindOther String + +derive instance Generic CaptchaKind _ +derive instance Eq CaptchaKind +instance Show CaptchaKind where + show = genericShow + +instance WriteForeign CaptchaKind where + writeImpl = writeImpl <<< case _ of + KindCheckbox -> "checkbox" + KindInvisible -> "invisible" + KindScore -> "score" + KindOther s -> s + +instance ReadForeign CaptchaKind where + readImpl = + let + fromStr = case _ of + "checkbox" -> KindCheckbox + "invisible" -> KindInvisible + "score" -> KindScore + s -> KindOther s + in + map fromStr <<< readImpl + +data CaptchaVendor = VendorReCaptcha | VendorHCaptcha | VendorOther String + +derive instance Generic CaptchaVendor _ +derive instance Eq CaptchaVendor +instance Show CaptchaVendor where + show = genericShow + +vendorFromString :: String -> CaptchaVendor +vendorFromString = case _ of + "recaptcha" -> VendorReCaptcha + "hcaptcha" -> VendorHCaptcha + s -> VendorOther s + +instance ReadForeign CaptchaVendor where + readImpl f = vendorFromString <$> readImpl f + +instance WriteForeign CaptchaVendor where + writeImpl VendorHCaptcha = writeImpl "hcaptcha" + writeImpl VendorReCaptcha = writeImpl "recaptcha" + writeImpl (VendorOther s) = writeImpl s + +data CaptchaFiltered = FilteredScoreBased | FilteredNotInViewport | FilteredInactive + +derive instance Generic CaptchaFiltered _ +derive instance Eq CaptchaFiltered +instance Show CaptchaFiltered where + show = genericShow + +newtype CaptchaCallback = CaptchaCallback Foreign + +derive instance Newtype CaptchaCallback _ +derive newtype instance WriteForeign CaptchaCallback +derive newtype instance ReadForeign CaptchaCallback +derive instance Generic CaptchaCallback _ +instance Show CaptchaCallback where + show _ = "CaptchaCallback" + +filteredFromString :: String -> Maybe CaptchaFiltered +filteredFromString = case _ of + "solveInViewportOnly" -> Just FilteredNotInViewport + "solveScoreBased" -> Just FilteredScoreBased + "solveInactiveChallenges" -> Just FilteredInactive + _ -> Nothing + +type CaptchaInfoDisplay = + { size :: Maybe Foreign + , theme :: Maybe String + , top :: Maybe Foreign + , left :: Maybe Foreign + , width :: Maybe Foreign + , height :: Maybe Foreign + } + +type CaptchaInfoMaybeFiltered = Tuple CaptchaInfo (Maybe CaptchaFiltered) + +type CaptchaSolution = + { vendor :: Maybe CaptchaVendor + , id :: Maybe String + , text :: Maybe String + , hasSolution :: Boolean + , requestAt :: Maybe JSDate + , responseAt :: Maybe JSDate + , duration :: Maybe Number + , provider :: Maybe String + , providerCaptchaId :: Maybe String + } + +type CaptchaSolved = + { vendor :: Maybe CaptchaVendor + , id :: Maybe String + , isSolved :: Maybe Boolean + , responseElement :: Maybe Boolean + , responseCallback :: Maybe Boolean + , solvedAt :: Maybe JSDate + } + +duplexSolved :: JsDuplex CaptchaSolved _ +duplexSolved = + let + toRaw r = modify (Proxy :: Proxy "solvedAt") CoerceDate + $ r + fromRaw r = pure + $ modify (Proxy :: Proxy "solvedAt") unwrap + $ r + in + duplex toRaw fromRaw + +type SolveResult = + { captchas :: Array CaptchaInfoMaybeFiltered + , solved :: Array CaptchaSolved + , solutions :: Array CaptchaSolution + } + +data CaptchaProvider + = Provider2Captcha Token2Captcha + | ProviderCustom (Array CaptchaInfo -> Aff (Array CaptchaSolution)) + +prepareCustomProvider :: (Array CaptchaInfo -> Aff (Array CaptchaSolution)) -> Array CaptchaInfo -> Promise { solutions :: Array CaptchaSolution } +prepareCustomProvider f = unsafePerformEffect <<< Promise.fromAff <<< map (\solutions -> { solutions }) <<< f + +type Options = + { visualize :: Maybe Boolean + , skipNotInViewport :: Maybe Boolean + , skipScoreBased :: Maybe Boolean + , skipInactive :: Maybe Boolean + , provider :: CaptchaProvider + } + +defaultOptions :: Token2Captcha -> Options +defaultOptions token = { visualize: Nothing, skipNotInViewport: Nothing, skipInactive: Nothing, skipScoreBased: Nothing, provider: Provider2Captcha token } + +prepareOptions :: Options -> Foreign +prepareOptions { provider, visualize, skipInactive, skipNotInViewport, skipScoreBased } = + writeImpl + { provider: case provider of + Provider2Captcha (Token2Captcha t) -> writeImpl { id: "2captcha", token: t } + ProviderCustom f -> writeImpl { fn: unsafeToForeign $ prepareCustomProvider f } + , visualFeedback: FFI.maybeToUndefined visualize + , solveInViewportOnly: FFI.maybeToUndefined $ skipNotInViewport + , solveScoreBased: FFI.maybeToUndefined $ not <$> skipScoreBased + , solveInactiveChallenges: FFI.maybeToUndefined $ not <$> skipInactive + , throwOnError: true + } + +type CaptchaInfo = + { kind :: Maybe CaptchaKind + , vendor :: Maybe CaptchaVendor + , id :: Maybe String + , sitekey :: Maybe String + , s :: Maybe String + , isInViewport :: Maybe Boolean + , isInvisible :: Maybe Boolean + , hasActiveChallengePopup :: Maybe Boolean + , hasChallengeFrame :: Maybe Boolean + , action :: Maybe String + , callback :: CaptchaCallback + , hasResponseElement :: Maybe Boolean + , url :: Maybe String + , display :: Maybe CaptchaInfoDisplay + } + +duplexSoln :: JsDuplex CaptchaSolution _ +duplexSoln = + let + toRaw r = modify (Proxy :: Proxy "requestAt") CoerceDate + $ modify (Proxy :: Proxy "responseAt") CoerceDate + $ r + fromRaw r = pure + $ modify (Proxy :: Proxy "requestAt") (unwrap) + $ modify (Proxy :: Proxy "responseAt") (unwrap) + $ r + in + duplex toRaw fromRaw + +duplexInfo :: JsDuplex CaptchaInfo _ +duplexInfo = + let + toRaw r = rename (Proxy :: Proxy "kind") (Proxy :: Proxy "_type") $ r + fromRaw r = pure $ rename (Proxy :: Proxy "_type") (Proxy :: Proxy "kind") r + in + duplex toRaw fromRaw + +foreign import data CaptchaPlugin :: Type + +foreign import _captcha :: forall (r :: Row Type). Foreign -> Puppeteer r -> Effect (Puppeteer (captcha :: CaptchaPlugin | r)) +foreign import _findCaptchas :: Page -> Promise Foreign +foreign import _getSolutions :: Page -> Foreign -> Promise Foreign +foreign import _enterSolutions :: Page -> Foreign -> Promise Foreign +foreign import _solveCaptchas :: Page -> Promise Foreign + +read :: forall @a. ReadForeign a => Foreign -> Either Error a +read = lmap (error <<< show) <<< runExcept <<< readImpl + +install :: forall (r :: Row Type). Options -> Puppeteer r -> Effect (Puppeteer (captcha :: CaptchaPlugin | r)) +install o p = _captcha (prepareOptions o) p + +infos :: Foreign -> Either Error (Array CaptchaInfoMaybeFiltered) +infos f = do + { captchas, filtered } <- read @({ captchas :: Array Foreign, filtered :: Array Foreign }) f + captchas' <- sequence $ duplexRead duplexInfo <$> captchas + let captchas'' = (_ /\ Nothing) <$> captchas' + filtered' <- for filtered \f' -> do + c <- duplexRead duplexInfo f' + { filtered: wasF, filteredReason } <- read @({ filtered :: Boolean, filteredReason :: String }) f' + pure $ case filteredFromString filteredReason of + Just r | wasF -> c /\ (Just r) + _ -> c /\ Nothing + pure $ captchas'' <> filtered' + +findCaptchas :: forall (r :: Row Type). Puppeteer (captcha :: CaptchaPlugin | r) -> Page -> Aff (Array CaptchaInfoMaybeFiltered) +findCaptchas _ p = do + f <- Promise.toAff $ _findCaptchas p + liftEither $ infos f + +getSolutions :: forall (r :: Row Type). Puppeteer (captcha :: CaptchaPlugin | r) -> Page -> Array CaptchaInfo -> Aff (Array CaptchaSolution) +getSolutions _ p is = do + f <- Promise.toAff $ _getSolutions p (writeImpl $ duplexWrite duplexInfo <$> is) + { solutions } <- liftEither $ read @({ solutions :: Array Foreign }) f + liftEither $ for solutions $ duplexRead duplexSoln + +enterSolutions :: forall (r :: Row Type). Puppeteer (captcha :: CaptchaPlugin | r) -> Page -> Array CaptchaSolution -> Aff (Array CaptchaSolved) +enterSolutions _ p sols = do + f <- Promise.toAff $ _enterSolutions p (writeImpl $ duplexWrite duplexSoln <$> sols) + { solved } <- liftEither $ read @({ solved :: Array Foreign }) f + liftEither $ for solved $ duplexRead duplexSolved + +solveCaptchas :: forall (r :: Row Type). Puppeteer (captcha :: CaptchaPlugin | r) -> Page -> Aff SolveResult +solveCaptchas _ p = do + f <- Promise.toAff $ _solveCaptchas p + { solved, solutions } <- liftEither $ read @({ solved :: Array Foreign, solutions :: Array Foreign }) f + captchas <- liftEither $ infos f + liftEither do + solved' <- for solved $ duplexRead duplexSolved + solutions' <- for solutions $ duplexRead duplexSoln + pure $ { captchas, solved: solved', solutions: solutions' } diff --git a/src/Puppeteer.Plugin.Stealth.js b/src/Puppeteer.Plugin.Stealth.js new file mode 100644 index 0000000..6056ed8 --- /dev/null +++ b/src/Puppeteer.Plugin.Stealth.js @@ -0,0 +1,5 @@ +import { PuppeteerExtra } from 'puppeteer-extra' +import Stealth from 'puppeteer-extra-plugin-stealth' + +/** @type {(_: PuppeteerExtra) => () => PuppeteerExtra} */ +export const install = p => () => p.use(Stealth()) diff --git a/src/Puppeteer.Plugin.Stealth.purs b/src/Puppeteer.Plugin.Stealth.purs new file mode 100644 index 0000000..0fcb3a7 --- /dev/null +++ b/src/Puppeteer.Plugin.Stealth.purs @@ -0,0 +1,8 @@ +module Puppeteer.Plugin.Stealth where + +import Effect (Effect) +import Puppeteer.Base (Puppeteer) + +-- | https://github.com/berstend/puppeteer-extra/tree/master/packages/puppeteer-extra-plugin-stealth +foreign import data StealthPlugin :: Type +foreign import install :: forall (r :: Row Type). Puppeteer r -> Effect (Puppeteer (stealth :: StealthPlugin | r)) diff --git a/src/Puppeteer.purs b/src/Puppeteer.purs index ee542d6..fde2767 100644 --- a/src/Puppeteer.purs +++ b/src/Puppeteer.purs @@ -1,6 +1,6 @@ module Puppeteer ( module X - , puppeteer + , new , connect , launch , connect_ @@ -23,12 +23,12 @@ import Effect (Effect) import Effect.Aff (Aff) import Effect.Unsafe (unsafePerformEffect) import Foreign (Foreign) -import Puppeteer.Base (Puppeteer) +import Puppeteer.Base (Puppeteer, duplexWrite) import Puppeteer.Base as X -import Puppeteer.Screenshot as X import Puppeteer.Browser (Browser) import Puppeteer.Browser as Browser import Puppeteer.FFI as FFI +import Puppeteer.Screenshot as X import Simple.JSON (writeImpl) --| [https://pptr.dev/api/puppeteer.puppeteerlaunchoptions] @@ -113,7 +113,7 @@ prepareConnectOptions , headers: FFI.maybeToUndefined $ map FFI.mapToRecord headers , transport: FFI.maybeToUndefined $ map transport' transport } - , writeImpl $ map Browser.prepareConnectOptions browser + , writeImpl $ map (duplexWrite Browser.duplexConnect) browser ] prepareLaunchOptions :: Launch -> Foreign @@ -157,7 +157,7 @@ prepareLaunchOptions , headless: if headless then writeImpl "new" else writeImpl false , userDataDir: FFI.maybeToUndefined userDataDir } - , writeImpl $ FFI.maybeToUndefined $ map Browser.prepareConnectOptions browser + , writeImpl $ FFI.maybeToUndefined $ map (duplexWrite Browser.duplexConnect) browser ] foreign import _puppeteer :: Effect (Promise (Puppeteer ())) @@ -168,8 +168,8 @@ foreign import _launch :: forall p. Foreign -> Puppeteer p -> Effect (Promise Br --| --| [`PuppeteerExtra`](https://github.com/berstend/puppeteer-extra/blob/master/packages/puppeteer-extra/src/index.ts) --| [`PuppeteerNode`](https://pptr.dev/api/puppeteer.puppeteernode) -puppeteer :: Unit -> Aff (Puppeteer ()) -puppeteer _ = Promise.toAffE _puppeteer +new :: Aff (Puppeteer ()) +new = Promise.toAffE _puppeteer --| Connect to an existing browser instance --| diff --git a/test/Puppeteer.Browser.Spec.purs b/test/Puppeteer.Browser.Spec.purs index 733ae6c..5f1ea91 100644 --- a/test/Puppeteer.Browser.Spec.purs +++ b/test/Puppeteer.Browser.Spec.purs @@ -12,7 +12,7 @@ import Test.Spec.Assertions (shouldEqual, shouldNotEqual) import Test.Util (test, testE) spec :: SpecT Aff Unit Effect Unit -spec = beforeAll (Pup.launch_ =<< Pup.puppeteer unit) +spec = beforeAll (Pup.launch_ =<< Pup.new) $ describe "Browser" do testE "websocketEndpoint" $ shouldNotEqual "" <=< Pup.Browser.websocketEndpoint testE "connected" $ shouldEqual true <=< Pup.Browser.connected @@ -22,6 +22,6 @@ spec = beforeAll (Pup.launch_ =<< Pup.puppeteer unit) connected <- liftEffect $ Pup.Browser.connected b connected `shouldEqual` false - pup <- Pup.puppeteer unit + pup <- Pup.new b' <- Pup.connect (Pup.connectDefault $ Pup.BrowserWebsocket ws) pup Pup.Browser.close b' diff --git a/test/Puppeteer.Handle.Spec.purs b/test/Puppeteer.Handle.Spec.purs index 1d871eb..d383f6f 100644 --- a/test/Puppeteer.Handle.Spec.purs +++ b/test/Puppeteer.Handle.Spec.purs @@ -100,7 +100,7 @@ withPage :: SpecT Aff Pup.Page Effect Unit -> SpecT Aff Unit Effect Unit withPage = let withPage' spec' _ = do - pup <- Pup.puppeteer unit + pup <- Pup.new b <- Pup.launch_ pup page <- Pup.Page.new b failOnPageError page do diff --git a/test/Puppeteer.Page.Event.Spec.purs b/test/Puppeteer.Page.Event.Spec.purs index 64501dd..9c9d1d0 100644 --- a/test/Puppeteer.Page.Event.Spec.purs +++ b/test/Puppeteer.Page.Event.Spec.purs @@ -61,7 +61,7 @@ withPage = spec :: SpecT Aff Unit Effect Unit spec = - beforeAll (Pup.launch_ =<< Pup.puppeteer unit) + beforeAll (Pup.launch_ =<< Pup.new) $ afterAll Pup.Browser.close $ do describe "Event" do diff --git a/test/Puppeteer.Page.Spec.purs b/test/Puppeteer.Page.Spec.purs index 5ee3eb4..e82d356 100644 --- a/test/Puppeteer.Page.Spec.purs +++ b/test/Puppeteer.Page.Spec.purs @@ -75,7 +75,7 @@ inputPage = """ spec :: SpecT Aff Unit Effect Unit -spec = beforeAll (Pup.launch_ =<< Pup.puppeteer unit) +spec = beforeAll (Pup.launch_ =<< Pup.new) $ afterAll Pup.Browser.close $ describe "Page" do test "new, close, isClosed" \b -> do diff --git a/test/Puppeteer.Plugin.Spec.purs b/test/Puppeteer.Plugin.Spec.purs new file mode 100644 index 0000000..3d22857 --- /dev/null +++ b/test/Puppeteer.Plugin.Spec.purs @@ -0,0 +1,99 @@ +module Puppeteer.Plugin.Spec where + +import Prelude + +import Control.Monad.Error.Class (liftMaybe, try) +import Control.Monad.ST as ST +import Control.Monad.ST.Global as ST +import Control.Monad.ST.Ref as ST +import Control.Parallel (parallel, sequential) +import Data.Array as Array +import Data.Foldable (for_) +import Data.Maybe (Maybe(..)) +import Data.Newtype (wrap) +import Data.Traversable (for) +import Effect (Effect) +import Effect.Aff (Aff, delay) +import Effect.Class (liftEffect) +import Effect.Console (warn) +import Effect.Exception (error) +import Node.EventEmitter as EventEmitter +import Node.Process as Process +import Puppeteer as Pup +import Puppeteer.Eval as Pup.Eval +import Puppeteer.Page as Pup.Page +import Puppeteer.Page.Navigate as Pup.Page.Nav +import Puppeteer.Page.WaitFor as Pup.Page.WaitFor +import Puppeteer.Plugin.AdBlock as Pup.AdBlock +import Puppeteer.Plugin.AnonymousUserAgent as Pup.AnonUA +import Puppeteer.Plugin.Captcha as Pup.Captcha +import Puppeteer.Plugin.Stealth as Pup.Stealth +import Test.Spec (SpecT(..), describe, focus, pending) +import Test.Spec.Assertions (shouldEqual, shouldSatisfy) +import Test.Util (test) + +spec :: SpecT Aff Unit Effect Unit +spec = describe "Plugin" do + args <- liftEffect Process.argv + + let + pendingUnlessArg a t b = + if not $ Array.any (_ == a) args then do + let msg = " (skipped unless `" <> a <> "`, ex. `spago test " <> a <> "`)" + pending (t <> msg) + else + test t b + + describe "Captcha" do + test "install" do + pup <- Pup.new + pup' <- liftEffect $ Pup.Captcha.install (Pup.Captcha.defaultOptions $ wrap "") pup + void $ Pup.launch_ pup' + + pendingUnlessArg "--test-captcha" "solves captchas" do + token <- liftMaybe (error "TWOCAPTCHA_API_KEY not present") <=< liftEffect <<< Process.lookupEnv $ "TWOCAPTCHA_API_KEY" + let + urls = + [ "https://www.google.com/recaptcha/api2/demo" + , "https://accounts.hcaptcha.com/demo" + , "https://democaptcha.com/demo-form-eng/hcaptcha.html" + ] + pup <- Pup.new + pup' <- liftEffect $ Pup.Captcha.install (Pup.Captcha.defaultOptions $ wrap token) pup + b <- Pup.launch_ pup' + sequential $ for_ urls \u -> parallel do + p <- Pup.Page.new b + _ <- Pup.Page.Nav.to_ p u + { solved } <- Pup.Captcha.solveCaptchas pup' p + Array.length solved `shouldSatisfy` (_ >= 1) + pure unit + describe "Adblock" do + test "install" do + pup <- Pup.new + pup' <- liftEffect $ Pup.AdBlock.install Pup.AdBlock.defaultOptions pup + void $ Pup.AdBlock.blocker pup' + pendingUnlessArg "--test-adblock" "blocks ads" do + pup <- Pup.new + pup' <- liftEffect $ Pup.AdBlock.install Pup.AdBlock.defaultOptions pup + blocker <- Pup.AdBlock.blocker pup' + requestsBlocked <- liftEffect $ ST.toEffect (ST.new 0) + stylesInjected <- liftEffect $ ST.toEffect (ST.new 0) + let add1On st h = liftEffect $ EventEmitter.on_ h (void $ ST.toEffect $ ST.modify (_ + 1) st) blocker + add1On requestsBlocked Pup.AdBlock.requestBlockedH + add1On stylesInjected Pup.AdBlock.styleInjectedH + b <- Pup.launch_ pup' + p <- Pup.Page.new b + _ <- Pup.Page.Nav.to_ p "https://www.google.com/search?q=rent%20a%20car" + Pup.Page.WaitFor.networkIdle (Pup.Page.WaitFor.NetworkIdleFor $ wrap 200.0) p + reqs <- liftEffect $ ST.toEffect $ ST.read requestsBlocked + stys <- liftEffect $ ST.toEffect $ ST.read stylesInjected + reqs `shouldSatisfy` (_ >= 1) + stys `shouldSatisfy` (_ >= 1) + describe "Stealth" do + test "install" do + pup <- Pup.new + void $ liftEffect $ Pup.Stealth.install pup + describe "AnonymousUserAgent" do + test "install" do + pup <- Pup.new + void $ liftEffect $ Pup.AnonUA.install pup diff --git a/test/Puppeteer.Spec.purs b/test/Puppeteer.Spec.purs index 40b3fc1..0727f98 100644 --- a/test/Puppeteer.Spec.purs +++ b/test/Puppeteer.Spec.purs @@ -11,6 +11,7 @@ import Puppeteer.Browser as Pup.Browser import Puppeteer.Browser.Spec as Spec.Browser import Puppeteer.Handle.Spec as Spec.Handle import Puppeteer.Page.Spec as Spec.Page +import Puppeteer.Plugin.Spec as Spec.Plugin import Puppeteer.Selector.Spec as Spec.Selector import Test.Spec (SpecT, describe, mapSpecTree) import Test.Spec.Assertions (shouldEqual) @@ -19,11 +20,11 @@ import Test.Util (test) spec :: SpecT Aff Unit Effect Unit spec = describe "Puppeteer" do test "launch" do - pup <- Pup.puppeteer unit + pup <- Pup.new map void Pup.launch_ pup test "connect" do - pup <- Pup.puppeteer unit + pup <- Pup.new b1 <- Pup.launch_ pup ws <- liftEffect $ Pup.Browser.websocketEndpoint b1 @@ -39,4 +40,5 @@ spec = describe "Puppeteer" do Spec.Browser.spec Spec.Page.spec Spec.Handle.spec + Spec.Plugin.spec mapSpecTree (pure <<< unwrap) identity Spec.Selector.spec diff --git a/test/Test.Main.purs b/test/Test.Main.purs index 2511673..0eb310e 100644 --- a/test/Test.Main.purs +++ b/test/Test.Main.purs @@ -19,11 +19,13 @@ import Test.Spec.Config (defaultConfig) import Test.Spec.Reporter (consoleReporter) import Test.Spec.Result (Result(..)) import Test.Spec.Runner (runSpecT) +import Dotenv as Dotenv foreign import errorString :: Error -> Effect String main :: Effect Unit main = launchAff_ do + Dotenv.loadFile let cfg = defaultConfig { timeout = Nothing, exit = false } run <- liftEffect $ runSpecT cfg [ consoleReporter ] Spec.spec res <- (map (join <<< map (foldl Array.snoc [])) run) :: Aff (Array Result)