
    a	h^\                        d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	m
Z
mZ d dlmZmZ d dlZd dlmZ 	 d dlZn+# e$ r#  ej        d            ej        d           Y nw xY w	 d dlmZ n# e$ r dZY nw xY w	 d dlZn# e$ r  ej        d           Y nw xY wd	 Zd
 Zd Zd Zd Zd Z	 ddZ 	 	 	 ddZ!dS )    N)urlparseunquote)WebDriverExceptionTimeoutException)
app_loggerz-Failed to import parser.py. Ensure it exists.   )	ConsoleUIzselenium_handlers.py not foundc                    | "t          | t          t          f          r| dk     rdS | dk    rdS dt          j        vr|  dS d}	 t          t          j        t          j        | d                              }|t          |          k    rt          |          d	z
  }t          j	        d|          }t          | |z  d
          }| d||          S # t          t          f$ r |  dcY S w xY w)z;Formats bytes into a human-readable string (B, KB, MB, GB).Nr   Unknown sizez0 Bmathz B)	BKBMBGBTBPBEBZBYBi   r       )
isinstanceintfloatsysmodulesr   floorloglenpowround
ValueErrorOverflowError)
size_bytes	size_nameipss        4c:\Users\brownies\Downloads\rentrytest\downloader.py_format_sizer*   '   s   Je!E!EVW~QuS[      EI	!
48J556677II"AHT1*q.!$$$$il$$$& ! ! !      !s   BC C,+C,c                     	 t          |           j        }t          j                            t	          |                    }|r|S n# t
          $ r Y nw xY wdS )z0Extracts a potential filename from the URL path.N)r   pathosbasenamer   	Exception)urlr,   filenames      r)   get_filename_from_urlr2   =   si    }}!7##GDMM22 	O	    4s   AA 
AAc                    | j                             d          }|sdS t          j        d|t          j                  }|r|d                                                             d          }|                                                    d          r+	 t          |dd         d	          }nb# t          $ r Y nVw xY w|                                                    d
          r+	 t          |dd         d	          }n# t          $ r Y nw xY w|                                S t          j        d|t          j                  }|r-|d                                                             d          S dS )z2Extracts filename from Content-Disposition header.zcontent-dispositionNzfilename\*=([^;
]*)r   z'"zutf-8''   zutf-8)encodingziso-8859-1''   z
iso-8859-1zfilename=([^;
]*))
headersgetrefindall
IGNORECASEstriplower
startswithr   r/   )responsecdfnamefns       r)   get_filename_from_headersrC   K   sz   				3	4	4B tJ-r2=AAE 1X^^##E**88::  ++ 		RVg666   XXZZ"">22 	RW|<<<    xxzz J+R??E -Qx~~%%e,,,4s$   B0 0
B=<B=(D 
DDc                 ,	   t          j                    j        }t          j                            |           st          j        d| d|             dS t          j                            |          se	 t          j        |           t          j	        d| d|            n5# t          $ r(}t          j        d| d| d|            Y d}~dS d}~ww xY wt          j        |          }t          j                            ||          }|}t          j                            |          rF	 t          j                            |          }t          j                            |           }||k    rt          j        d| d| dt!          |           d	           	 t          j        |            t          j	        d| d
|             n4# t          $ r'}t          j        d| d|  d|            Y d}~nd}~ww xY wdS t          j        d| d| dt!          |           dt!          |           d	           n4# t          $ r'}t          j        d| d| d|            Y d}~nd}~ww xY wt          j                            |          r5d}	t          j                            |          \  }
}t          j                            ||
 d|	 |           }t          j                            |          r|	dz  }	t          j                            ||
 d|	 |           }|	dk    rCt          j        d| d| d           	 t          j        |            n# t          $ r Y nw xY wdS t          j                            |          t          j        d| dt          j                            |                      	 t+          j        | |           t          j	        d| dt          j                            |            dt          j                            |                      |S # t.          $ rQ}t          j        d| d|  d| d|            	 t          j        |            n# t          $ r Y nw xY wY d}~dS d}~ww xY w)a"  
    Attempts to move source_path to target_dir with desired_filename.
    Checks for duplicates (exact name + size) before renaming/moving.
    Handles existing files (different size) by adding suffixes (_1, _2, ...).
    Returns the final path, "DUPLICATE_SKIPPED", or None on error.
    [z)] Source file for rename does not exist: Nz] Created target directory: z$] Failed to create target directory : z ] Skipping save: Existing file 'z' found with same size ().z] Deleted duplicate temp file: z'] Could not remove duplicate temp file DUPLICATE_SKIPPEDz] Target file z$ exists but size differs (Existing: z, New: z). Adding suffix.z0] Could not check size for potential duplicate 'z': r   _d   z1] Could not find available suffixed filename for z after 100 attempts.z] Renaming to suffixed name: z] Moved z -> z] Failed to move z to )	threadingcurrent_threadnamer-   r,   existsr   errormakedirsdebugOSErrorparsersanitize_filenamejoingetsizeinfor*   removewarningsplitextr.   shutilmover/   )source_path
target_dirdesired_filenamethread_nameeinitial_target_pathfinal_target_pathexisting_sizenew_sizecounterrM   exts               r)   attempt_renamerh   j   s!    *,,1K7>>+&& SSSkSS	
 	
 	
 t7>>*%% 	K
###VVV*VVWWWW 	 	 	VKVVZVVSTVV   44444		 /0@AA',,z3CDD+ 
w~~)** 	GOO,?@@Mw{33H(( O  O  OEU  O  Oo{  }J  pK  pK  O  O  O  Ik***$UKUUUU       &bKbbP[bb_`bb       
 +* " w  w  w3C  w  wiu  wD  jE  jE  w  w  NZ  [c  Nd  Nd  w  w  w     	 	 	iKiiQaiifgii       	 
w~~)** 
G$$%566	cGLL5M5Mw5M5M5MNNgnn.// 	qLG "ZD9Q9Q79QC9Q9Q R R}} |||Vf|||  Ik****   D t gnn.// 	 	___"':J:JK\:]:]__	
 	
 	

K!2333mmmRW%5%5k%B%BmmHXHXYjHkHkmm	
 	
 	
 !  
 
 
WWWkWW?PWWTUWW	
 	
 	
	Ik"""" 	 	 	D	 ttttt
s   4.B# #
C-CC/A/I  .G I  
G?G:5I  :G??I  ;I   
I1
I,,I1M! !
M.-M.
A-P8 8
R R#Q87R8
RRRRRc           
      4   t          j                    j        }d}d}d}||k     r*|dz  }t          j                            |           st          j        d| d|  d           dS 	 t          j                            |           }nH# t          $ r;}t          j        d| d|  d	|            t          j        |           Y d
}~d
}~ww xY w||k    r|dk    r|dz  }nd}|}||dz  k    r@|dk    r:t          j        d| dt          j                            |            d           dS ||k     rt          j        |           ||k     *t          j        d| dt          j                            |            dt          |           d           dS )z.Checks if a file size is stable over a period.r   r   rE   z] File z$ disappeared during stability check.Fz] Error getting size for rF   N   z' stability check exceeded max attempts.z confirmed stable (rG   T)rK   rL   rM   r-   r,   rN   r   rY   rV   rR   timesleepr.   rQ   r*   )	filepathintervalchecks_neededr`   stable_counter	last_sizecheck_countcurrent_sizera   s	            r)   check_file_stabilityru      s   *,,1KNIK
=
(
(qw~~h'' 	VKVVVVV   5	7??844LL 	 	 	IKII(IIaII   Jx   HHHH	 9$$)9)9aNNN 	***}q/@/@kKkk(8(8(B(Bkkk   5M))Jx   5 =
(
(6 jKjj 0 0 : :jj|\eOfOfjjj   4s   (B 
C1CCc           
      $    t          j                    j        }t          j                    }t	          j        d| d  d| d           t          j                    |z
  |k     r	  fdt          j                   D             }nn# t          $ r t	          j	        d| d             Y dS t          $ r;}t	          j	        d| d  d	|            t          j        |           Y d}~d}~ww xY w fd
|D             }|r=|d         }	t          |          dk    r:t	          j        d| d  dt          j                            |	                      t	          j        d| dt          j                            |	           d           t!          |	||          r	 t          j                            |	          }
t	          j        d| dt          j                            |	           dt'          |
           d           |	S # t          $ r(}t	          j	        d| d|	 d	|            Y d}~dS d}~ww xY wt          j        |           t          j                    |z
  |k     t	          j	        d| d| d             dS )z<Waits for a non-temporary file in download_dir to stabilize.rE   z] Starting download wait in 'z' (Timeout: zs)c                     g | ]V}|                     d           t          j                            t          j                            |                    T|WS ).)r>   r-   r,   isfilerU   .0fdownload_dirs     r)   
<listcomp>z.wait_for_download_complete.<locals>.<listcomp>  sc       ||C(( GNN27<<a#@#@AA	      z ] Download directory not found: Nz#] Error listing download directory rF   c                     g | ]I}|                                                     d           )t          j                            |          JS )).tmpz.crdownload.partz.partial)r=   endswithr-   r,   rU   rz   s     r)   r~   z.wait_for_download_complete.<locals>.<listcomp>  sV     
 
 
7799%%&RSS
GLLq))
 
 
r   r   r   z(] Multiple potential downloads found in z, checking first: z!] Potential download file found: z. Checking stability...z] Selenium download detected: z ()z$] Error getting size of stable file z] Timed out after z&s waiting for download to complete in )rK   rL   rM   rl   r   rQ   r-   listdirFileNotFoundErrorrO   rR   rm   r   rY   r,   r.   ru   rV   rW   r*   )r}   timeoutstability_intervalstability_checksr`   
start_timefilesra   potential_filesdl_filepath
final_sizes   `          r)   wait_for_download_completer      s    *,,1KJ[K[[l[[PW[[[   )++

"W
,
,	   L11  EE ! 	 	 	OKOOOO   44 	 	 	WKWWLWWTUWW   J)***HHHH	
 
 
 

 
 

  	 )!,K?##a''" M  M  M\  M  Mmomtm}m}  J  nK  nK  M  M   xKxx"'BRBRS^B_B_xxx   $K1CEUVV  
 !#!=!=JO DK  D  DrwGWGWXcGdGd  D  Dhtu  iA  iA  D  D  D   '&      $_K__[__\]__    44444	 
 	
%&&&[ )++

"W
,
,\ hKhh7hhZfhh   4s7   ' B $C3/	C381C..C3&A(H 
IH<<Ic                    t          j                    j        }d}d}d}	d}
d}	 |                    ddd          }|                    dd	d
          }d}d|                    dd          i}t          j        d| d|             t          j        | |d||f          5 }d|j         d|j	         d|  }
|
                                 |j                            d          }|rt          |          nd}|rt          |          nd}t          j        d| d|            t          |          }t!          |           }|p|p| d}t#          j        |          }t#          j        ||          }t(          j                            || d          }t(          j                            |          st)          j        |           t          j        d| dt(          j                            |                      t5          j                    }t7          |d          5 }|                    |          D ]P}|r#|                                rt=          d          |r'|                    |           |	tA          |          z  }	Q	 ddd           n# 1 swxY w Y   t5          j                    }||z
  }|dk    rt          |	|z             dnd }t          j        d| d!|d"d#| d$           tC          |||          }|d%k    rd}d&}
n~|rTt(          j                            |          r5d}d'}
	 t(          j        "                    |          }	n8# t<          $ r Y n,w xY wd}d(t(          j        #                    |pd)           }
d}ddd           n# 1 swxY w Y   n# t          j$        j%        $ r2}d| d*|  }
t          j&        d| d+|
            d}d}d}	Y d}~nd}~wt<          $ rr}d,tO          |          v r-tO          |          }
t          j(        d| d+|
 d*|             n$d-| }
t          j&        d| d+|
 d*|  d.           d}d}d}	Y d}~nd}~ww xY w|s|rt(          j                            |          rd	 t)          j)        |           t          j        d| d/|            n# tT          $ r'}t          j(        d| d0| d1|            Y d}~nd}~ww xY wn# |s|rt(          j                            |          rf	 t)          j)        |           t          j        d| d/|            w # tT          $ r'}t          j(        d| d0| d1|            Y d}~w d}~ww xY ww w w xY w|||	|
fS )2zZDownloads a file directly using requests. Returns (success, path_or_status, size, reason).FNr   zDownload initialization failedTimeoutsdirect_download_timeoutg      >@fallbackdirect_idle_timeoutg      N@i    z
User-AgentMisc
user_agentrE   z] Sending GET request to T)r7   streamr   zRequest error: r   z
 for url: zcontent-lengthr   z] File size: r   r   z] Saving temporarily to: wb)
chunk_sizez*Download cancelled due to shutdown requestz/sInfz] .part download finished in z.2fzs (r   rH   ,Duplicate file with same size already existsDownload successfulz Failed to rename/move temp file zN/A for ] shutdown requestzError during direct download: exc_infoz] Cleaned up partial file: z ] Could not remove partial file rF   )+rK   rL   rM   getfloatr8   r   rQ   requestsstatus_codereasonraise_for_statusr7   r   r*   rW   rC   r2   rS   rT   get_final_desired_filenamer-   r,   rU   rN   rP   relpathrl   openiter_contentis_setr/   writer   rh   rV   r.   
exceptionsRequestExceptionrO   strrY   rX   rR   ) r0   r^   desired_filename_baseconfiguishutdown_eventr`   successfinal_path_or_statusdownloaded_sizer   part_filepathtimeout_conntimeout_idler   r7   rcontent_length
total_sizesize_strfilename_headerfilename_urlactual_filenamefinal_desired_namedl_start_timer|   chunkdl_end_timeduration	speed_strreq_errra   s                                    r)   download_direct_filer   <  s    *,,1KGO-FMe1D ' 
 
 z3HSWXX
FL!A!ABH[HH3HHIII\l7S
 
 
 @	,Pq}PPqxPP3PPF   Y]]+;<<N0>H^,,,DJ3=Q|J///>HODDD(DDEEE7::O055LQ<Q6K3Q3Q3Q  %6GGO!'!B%" " GLL8J5Q5Q5QRRM7>>*-- (J'''OZKZZ"'//-:X:XZZ   !IKKMmT** 6a^^z^BB 6 6E% V.*?*?*A*A V'(TUUU 6'3u::566 6 6 6 6 6 6 6 6 6 6 6 6 6 6 )++K"]2H a<<  ( :;;???? 
 [K[[h[[[y[[[   $2z+=$ $  $':::G% 
,"'..9M*N*N 
,.&(goo6J&K&KOO    D  fBG<L<L]Mc^c<d<dff'+$A@	, @	, @	, @	, @	, @	, @	, @	, @	, @	, @	, @	, @	, @	, @	,D /   6766664[44F44555# 	 	 	Q''VVFD;DD&DDsDDEEEE9a99FBBBBBSBBTRRRR#	  		= 		RW^^M-J-J 			-((( OOOOO       "YYY]YYVWYY         		= 		RW^^M-J-J 			-((( OOOOO       "YYY]YYVWYY       		 		 		 (/6AAs   A>N, "FN &A(JN J	N "J	#BN <MN 
M)&N (M))+N N,  N$$N, 'N$(N, +S7 ,Q, (O-(S7 -Q,:A(Q'"S7 'Q,,S7 .S 
S3S..S37$V.U
V
U<U72V7U<<Vc
                    t          j                    j        }
d}d}d}d}d}	 |	r#|	                                rt	          d          t          j        d|
 d|            |                    dd	d
          }|                    ddd          }| 	                    |           | 
                    |           |	r#|	                                rt	          d          t          j        |           |	r#|	                                rt	          d           || ||          sd}t          j        d|
 d| d| d           ddd|f|s|rt          j                            |          rl|dk    rgt          j        d|
 d|            	 t          j        |           S # t&          $ r'}t          j        d|
 d| d|            Y d}~S d}~ww xY wS S S S |	r#|	                                rt	          d          |                    ddd          }|                    ddd
          }|                    ddd          }t          j        d|
 d| d            t)          ||||          }|	r#|	                                rt	          d!          |r!	 t          j                            |          }n6# t&          $ r)}t          j        d|
 d"| d|            d}Y d}~nd}~ww xY wt          j                            |          }t/          j        |          }t/          j        ||          }t5          |||          }|dk    rd#}d$}|}nl|rVt          j                            |          r7d#}d%}	 t          j                            |          }n&# t          $ r |}Y nw xY wd}d&| }|}d}nd}d'}d}d}n# t6          $ r4}d(| }t          j        d|
 d| d| d)           d}d}d}Y d}~nd}~wt8          $ r4}d*| }t          j        d|
 d| d| d)           d}d}d}Y d}~nd}~wt          $ rr}d+t;          |          v r-t;          |          }t          j        d|
 d| d|            n$d,| }t          j        d|
 d| d| d#)           d}d}d}Y d}~nd}~ww xY w|s|rt          j                            |          rj|dk    rdt          j        d|
 d|            	 t          j        |           n# t&          $ r'}t          j        d|
 d| d|            Y d}~nd}~ww xY wn# |s|rt          j                            |          rl|dk    rgt          j        d|
 d|            	 t          j        |           w # t&          $ r'}t          j        d|
 d| d|            Y d}~w d}~ww xY ww w w w xY w||||fS )-zJDownloads using Selenium. Returns (success, path_or_status, size, reason).FNr   z'Selenium download initialization failedzSkipped due to shutdown requestrE   z] Navigating to page: Seleniumnavigation_waitg      @r   page_load_timeoutZ   z2Cancelled after navigation due to shutdown requestz0Cancelled before handler due to shutdown requestz1Selenium site handler failed to initiate downloadr   r   rx   rH   z.] Cleaning up potentially orphaned temp file: z&] Could not remove orphaned temp file rF   z/Cancelled after handler due to shutdown requestr   download_wait_timeoutih  download_stability_intervaldownload_stability_checksrk   z"] Selenium handler successful for z. Waiting for download...z5Cancelled after download wait due to shutdown requestz"] Could not get size of temp file Tr   r   z+Failed to move/rename downloaded file from z5Download did not complete or stabilize within timeoutz%Selenium timeout during interaction: r   zSelenium WebDriver error: r   z.Unexpected error during Selenium interaction: )rK   rL   rM   r   r/   r   rW   r   getintset_page_load_timeoutr8   rl   rm   rO   r-   r,   rN   rY   rX   rR   r   rV   r.   rS   rT   r   rh   r   r   r   )driverr0   r^   r   handler_funcr   selenium_worker_dl_dirr   	item_infor   r`   r   r   r   r   downloaded_temp_filepathnav_waitr   ra   r   r   r   	temp_sizer   actual_filename_sanitizedr   selenium_timeoutselenium_errors                               r)   download_with_seleniumr     s    *,,1KGO6F#u 	?n3355 	?=>>>DKDDsDDEEE??:/@3?OO"MM*6ITVMWW$$%6777

3 	Rn3355 	RPQQQ
8 	Pn3355 	PNOOO|FFB// 	*HFCCCCCSCCCDDD$6)n 	(	 788	 %(;;;iKiiOgii  	23333   "jjjKcjjghjj       	 	 	 <q  	On3355 	OMNNN--
,Cc-RR#__5 - 
 
 "==3a ) 
 
 	]]]s]]]	
 	
 	
 $>"G-?AQ$
 $
   	Un3355 	USTTT# '	(GOO,DEE		   "fffG_ffcdff   							 !g../GHHO(.(@(Q(Q%!'!B%'@" " $2(*6H$ $  $':::G"+% ,"'..9M*N*N ,.0&(goo6J&K&KOO  0 0 0&/OOO0  _G]__"+'+$$GLFO#'    K9IKK>[>>F>>>>OOOO#   >n>>>[>>F>>>>OOOO# 	 	 	Q''VVFD;DD&DDsDDEEEEIaIIFBBBBBSBBTRRRR#	 	(	 788	 %(;;;iKiiOgii  	23333   "jjjKcjjghjj        	(	 788	 %(;;;iKiiOgii  	23333   "jjjKcjjghjj       	 	 	 < (/6AAs  D)O
 F&&
G0GGCO
 !K  O
 
K4K/*O
 /K44BO
 N# "O
 #N2/O
 1N22O
 	U 

R>*P>U R>*P?:U ?R>A(R94U 9R>>U T 
U$UUAW!V)(W!)
W3WW!WW!)NN)NNN)"r   r-   rl   r[   r9   r   rK   	tracebackr   urllib.parser   r   selenium.common.exceptionsr   r   loggingloggerr   rS   ImportErrorcriticalexit
console_uir	   selenium_handlersrY   r*   r2   rC   rh   ru   r   r   r    r   r)   <module>r      s;    				   				 



          * * * * * * * * K K K K K K K K       MMMM   JGHHHCHQKKKKK$$$$$$$   III9 9 9 9J7888889! ! !,    >h h hV$ $ $N: : :B MQrB rB rB rBz KB KB KB KB KB KBs6    A %A-,A-1A8 8BBB B#"B#