+
    Lߎii                         ^ RI t ^ RIt^ RIt^ RIHt ^ RItRt] P                  ! ]RR7       R tR t	R t
R	 tR
 t]R8X  d
   ]! 4        R# R# )    N)urlparseknowledge_baseT)exist_okc                     \         P                  ! V ^
R7      pVP                  4        VP                  #   \         P                  P
                   d-   p\        RT  RT 2\        P                  R7        Rp?R# Rp?ii ; i)
   timeoutzError fetching URL : fileN	requestsgetraise_for_statustext
exceptionsRequestExceptionprintsysstderrurlresponsees   &  scripts/kb_ingest.pyfetch_url_contentr      sj     <<R0!!#}}// #C51#.SZZ@   36 B!A<<Bc                     \         P                  ! V ^
R7      pVP                  4        VP                  #   \         P                  P
                   d-   p\        RT  RT 2\        P                  R7        Rp?R# Rp?ii ; i)r   r   zKB_INGEST: Error fetching URL r
   r   Nr   r   s   &  r   fetch_content_from_urlr       sh    <<R0!!#}}// .se2aS9

Kr   c                    V '       g!   \        R V 2\        P                  R7       R# \        V4      pVP                  VP
                  ,           P                  RR4      P                  RR4      P                  RR4      p\        P                  ! RRV4      R	,          R
,           p\        P
                  P                  \        V4      p \        VRRR7      ;_uu_ 4       pVP                  V 4       RRR4       \        RV RV 2\        P                  R7       R#   + '       g   i     L3; i  \         d-   p\        RT RT 2\        P                  R7        Rp?R# Rp?ii ; i)z'KB_INGEST: No content to save for URL: r   F/_.:z[^a-zA-Z0-9_-] :Nd   Nz.mdwzutf-8)encodingNzKB_INGEST: Content from z successfully saved to Tz#KB_INGEST: Error saving content to r
   )r   r   r   r   netlocpathreplaceresubosjoinKNOWLEDGE_BASE_DIRopenwriteIOError)contentr   
parsed_urlfilename	file_pathfr   s   &&     r   save_content_to_filer9   *   s   7u=CJJO #J!!JOO3<<S#FNNsTWX``adfijHvv'X6t<uDH/:I)S733qGGG 4(-DYKPWZWaWab 43  3I;bD3::Vs0   D2 "D4)D2 D/	*D2 2E)=!E$$E)c                 ~    \        R V  2\        P                  R7       \        V 4      pV'       d   \	        W4      # R# )z'KB_INGEST: Ingesting content from URL: r   F)r   r   r   r   r9   )r   r4   s   & r   
ingest_urlr;   B   s3    	3C5
9

K$S)G#G11    c            	      v   \        \        P                  4      ^8  d2   \        R\        P                  R7       \        P
                  ! ^4       \        P                  ^,          p V P                  R4      '       g   V P                  R4      '       d   \        V 4       R# \        P                  P                  V 4      '       d   \        RV  2\        P                  R7        \        P                  P                  V 4      p\        P                  P                  \        V4      p\        V R4      ;_uu_ 4       p\        VR4      ;_uu_ 4       pVP                  VP!                  4       4       RRR4       RRR4       \        R	V  R
V 2\        P                  R7       R# \        RV  2\        P                  R7       \        P
                  ! ^4       R#   + '       g   i     Lq; i  + '       g   i     L|; i  \"         d-   p\        RT  RT 2\        P                  R7        Rp?R# Rp?ii ; i)   z*Usage: python kb_ingest.py <URL|FILE_PATH>r   zhttp://zhttps://z!KB_INGEST: Ingesting local file: rbwbNzKB_INGEST: File z successfully copied to z'KB_INGEST: Error processing local file r
   z>KB_INGEST: Invalid input. Must be a URL or a local file path: )lenr   argvr   r   exit
startswithr;   r.   r*   existsbasenamer/   r0   r1   r2   read	Exception)	input_argr6   destination_pathsrcdstr   s         r   mainrM   J   s   
388}q:LII&&)*>*>z*J*J9			"	"1)=CJJO	_ww''	2H!ww||,>Ii&&#t4Dd/K/Ks		#((*% 0L&$YK/GHXGYZadakakl 	NykZadakakl 0L/K&&  	_;I;bLSVS]S]^^	_sI   &AH =G. G	2G.:)H G+&G..G>	9H H8!H33H8__main__)r.   r   r   urllib.parser   r,   r0   makedirsr   r   r9   r;   rM   __name__ r<   r   <module>rS      sZ    	 
  ! 	 &   .$04 zF r<   