
    mj)                        d Z ddlZddlZddlZddlZddlZddlZddlZddl	m
Z
 ddlmZmZ  e
e                                          j        j        Zedz  ZdZdZ ej        d          Z ej        d	          Z ej        d
ej                  ZddZd Zd Zi Zg Zd Zd Z d Z!d Z"efdZ#d Z$e%dk    r e$             dS dS )u  Processa o wiki-sac-export.json (gerado por coletar_no_navegador.js) e monta a base local.

Uso:
    python scripts/processar_wiki.py caminho/para/wiki-sac-export.json

Gera, na pasta base-sac/:
    livros/...            um .md por página (com frontmatter) + imagens baixadas ao lado
    indice.json           estrutura livros > capítulos > páginas
    base_sac_completa.md  tudo em um único arquivo
    chunks.jsonl          trechos com metadados para RAG
Pode ser executado de novo: imagens já baixadas não são baixadas outra vez.
    N)Path)urlparseunquotelivrosg333333?i  z,!\[([^\]]*)\]\(([^)\s]+)(?:\s+\"[^\"]*\")?\)z+\[!\[([^\]]*)\]\(([^)\s]+)\)\]\(([^)\s]+)\)z<img[^>]+src=\"([^\"]+)\"[^>]*><   c                 ,   t          j        d|                               dd                                          } t	          j        dd|                               d                                          } | d |                             d          pdS )NNFKDasciiignorez[^a-zA-Z0-9]+-z
sem-titulo)unicodedata	normalizeencodedecoderesubstriplower)smaxlens     processar_wiki.pyslugifyr   "   s    fa((//BBIIKKA
a((..s3399;;AWfW:C  0L0    c                 f    d| pd                     dd                               dd          z   dz   S )N" \z\\z\")replace)r   s    r   yaml_strr   (   s6    !'r""40088eDDDsJJr   c                 .    t          j        dd|           S )z6Troca a miniatura redimensionada pela imagem original.z/scaled-\d+-//)r   r   )us    r   original_urlr#   ,   s    6"C+++r   c           	         | t           v rt           |          S t          t          t          |           j                  j                  pd}t          j        dd|          }||z  }|                                r|	                                j
        dk    r|                    dd           t          d          D ]}	 t          j                            | dd	i
          }t          j                            |d          5 }|                    |                                           d d d            n# 1 swxY w Y   t'          j        t*                      n# t          j        j        $ r}|j        dk    rDt'          j        t3          |j                            d          pd          dz              Y d }~t8                              |  d|j                    Y d }~ d S d }~wt<          $ rK}|dk    r&t8                              |  d|            Y d }~ d S t'          j        d           Y d }~d }~ww xY w|t           | <   |S )Nimagemz[<>:\"/\\|?*]_r   Tparentsexist_ok   z
User-AgentzMozilla/5.0 (base-sac export))headersr   )timeouti  zRetry-After   z: HTTP    z:    )baixadasr   r   r   pathnamer   r   existsstatst_sizemkdirrangeurllibrequestRequesturlopenwrite_bytesreadtimesleep	PAUSA_IMGerror	HTTPErrorcodeintr+   getfalhasappend	Exception)urldestino_dirnomedestino	tentativareqres           r   baixarrQ   5   s   
h }4*++011=XD6"C..DD G>> w||~~5: $666q 	 	In,,S<Ih:i,jj^++C+<< 2''1112 2 2 2 2 2 2 2 2 2 2 2 2 2 2
9%%%<)   6S= Js19==#?#?#E2FFJKKKHHHH55QV55666tttttt   >  MMS--A--000444444
1	
 HSMNsV   AE!(D;/E!;D?	?E!D?	E!!I5A	G/$G//I<%I(IIc                     fddfd	fd}fd}fd}t                               ||           } t                              ||           } t                              ||           } | S )uI   Baixa as imagens da página e reescreve os links para caminhos relativos.c                     t          t          j                            |                                                     S N)r   osr1   relpathas_posix)ppasta_mds    r   relzprocessar_imagens.<locals>.relU   s-    BGOOAx0011::<<<r   Nc                     |r|gng }|t          |           | gz  }t                              d |D                       D ]}t          |          }|r|c S d S )Nc              3   H   K   | ]}||                     d           |V  dS )httpN)
startswith).0cs     r   	<genexpr>z8processar_imagens.<locals>.local_para.<locals>.<genexpr>[   s8      MMQ!MV8L8LMqMMMMMMr   )r#   dictfromkeysrQ   )	url_thumburl_origcandr"   rX   	pasta_imgs        r   
local_paraz%processar_imagens.<locals>.local_paraX   s~    %-zz2i(()44MM$MMMMM 	 	Aq)$$A tr   c                    |                                  \  }}}t          j        dt          |          j        t          j                  sd } ||          }|rd| d |           dn|                     d          S )Nz\.(png|jpe?g|gif|webp|bmp|svg)$![]()r   )groupsr   searchr   r1   Igroup)maltthumborigrX   rh   rZ   s        r   sub_linkz#processar_imagens.<locals>.sub_linka   s    88::UDy;Xd^^=PRTRVWW 	DJud##()9$C$$33q66$$$$qwwqzz9r   c                    |                      d          |                      d          }}|                    d          s|                      d          S  |          }|rd| d |           dn|                      d          S )N   r-   r]   r   rj   rk   rl   rp   r^   )rq   rr   rI   rX   rh   rZ   s       r   sub_imgz"processar_imagens.<locals>.sub_imgh   s    771::qwwqzzS~~f%% 	771::JsOO()9$C$$33q66$$$$qwwqzz9r   c                     |                      d          }|                    d          s|                      d          S  |          }|rd |           dn|                      d          S )Nrw   r]   r   z![](rl   rx   )rq   rI   rX   rh   rZ   s      r   sub_htmlz#processar_imagens.<locals>.sub_htmlo   sq    ggajj~~f%% 	771::JsOO#$4cc!ff!''!**4r   rT   )IMG_LINK_REr   IMG_REHTML_IMG_RE)mdrY   rg   ru   ry   r{   rh   rZ   s    ``   @@r   processar_imagensr   S   s    = = = = =     : : : : : :: : : : : :5 5 5 5 5 5 
2	&	&B	GR	 	 B	2	&	&BIr   c                 D    fd}t                               ||           S )uM   Troca imagens por uma marcação textual com o caminho a partir de base-sac/.c                 |   |                      d          |                      d          }}|                    d          rd|pd d| dS |z                                  }	 |                    t                                                    }n# t          $ r Y nw xY wd|pt          |          j         d| dS )Nrw   r-   r]   z	[imagem: zsem nome    — ])	rp   r^   resolverelative_toBASErW   
ValueErrorr   r2   )rq   rr   caminhoabs_rY   s       r   r   ztexto_para_ia.<locals>.sub~   s    wwqzz1771::Wf%% 	BAs0jAAwAAAA7"++--	&&t,,5577GG 	 	 	D	E34$w--"4EE7EEEEs   $,B 
BB)r}   r   )r   rY   r   s    ` r   texto_para_iar   |   s7    	F 	F 	F 	F 	F ::c2r   c                 `   |                                                                  }|rs|d                              d                                          |                                k    r0d                    |dd                                         d          S |                                 S )Nr   # 
rw   )lstrip
splitlinesr   join)r   titulolinhass      r   remover_titulo_duplicador      s    YY[[##%%F 2&)""4((..00FLLNNB 2yy$$++D11188::r   c                    t          j        d|           }dg }}|D ]}|                                }|st          |          |k    r|                    dd|          }||dz  k    r|n|}|r|                    |           d}|                    |d |                                                    ||d                                          }t          |          |k    t          |          t          |          z   dz   |k    r|r|                    |           |}	|r| d| n|}|r|                    |           |S )Nz\n\s*\nr    r   r-   

)r   splitr   lenrfindrG   )textomaximoblocosatualsaidabcortes          r   dividir_chunksr      sr   Xj%((Fr5E 6 6GGII 	!ffvo 	"GGCF++E"Vq[0<EEfE U###LL6E6**+++%&&	!!A !ffvo 	" u::A"V+ 	6 	6LLEE).5u%%!%%%AEE ULr   c                  ~   t          t          j                  dk     r(t          t                     t          j        d           t          j        t          t          j        d                   	                    d                    } | d         | d         g d}d| d          d	| d         d d
          dg}g }dx}}t          | d         d          D ]\  }}t          |dd|d          z  }|                    dd           |dz                      dt          |d                    d|d          d|d          d|                    d          pd d	d           |d         |                    d          |d         |                    t"                                                    g d}	|                    d|d          d|                    d          pd d           t          d|dd |d                     d}
|d!         D ]m}|
dz  }
|d"         d#k    r||
dd|d          z  }|                    dd           |d$         }|d         }||                    d          |d         |                    t"                                                    g d%}|	d!                             |           |                    d&| d           |d$         }d}n||gd }}}|	d!         }d'}t          |d          D ]t\  }}|dz  }|r|dn|
d}| dt)          |                    d          p|d(                    }|| d)z  }|                    d*          }|s|dz  }d+}t+          ||d(                   }t-          |||d,z  |z            }d-t          |d(                    d.t          |d                    d|rd/t          |           dndz   d0|d          d1z   }|                    | d2|d(          d| dd           |                    t"                                                    }|                    |d(         |d         |d3           t/          ||          }d4                    d5 |d         ||d(         fD                       }|                    d6|d(          d7| d8|d          d9| d	           t          t3          |          pdg          D ]n\  }}|                    |d          d:|                    d|           d;| |d         ||d(         |d         ||| d|                                 d<           ot          d=|d(                     vo|d                             |	           t"          d>z                      t          j        |d'd?          d           t"          d@z                      d                    |          d           t9          t"          dAz  dBd          5 }|D ].} |                    t          j        | d'C          dz              /	 d d d            n# 1 swxY w Y   t          dD           t          dEt          | d                    dF| dG|            t          dHt          t<                     dIt          t>                     dJt          |                      t>          D ]}!t          dK|!           t"          dLz                      d                    t>                    d           d S )MNr-   rw   zutf-8)encodingorigemcoletado_em)r   r   r   z## Base de Conhecimento SAC

Fonte: z (coletado em 
   z)
r   r   02dr   slugTr'   z	_livro.mdz---
livro: rK   z
url_original: rI   z
---

# r   	descricaor   r   )rK   r   rI   pastaitensz

# LIVRO: [z] r   tipocapitulopaginas)r   r   rI   r   r   u   

## CAPÍTULO: Fr   z.mdmarkdownu   _(conteúdo não coletado)_imagensz---
titulo: z
livro: z
capitulo: zurl_original: z
---

r   )r   rI   arquivoz > c              3      K   | ]}||V  	d S rT    )r_   xs     r   ra   zmain.<locals>.<genexpr>   s(      #]#]![\#]A#]#]#]#]#]#]r   z

### z

_Caminho: r   z_

r!   #)idlivror   paginarI   r   trechor   z     - zindice.json)ensure_asciiindentzbase_sac_completa.mdzchunks.jsonlw)r   u   
=== Relatório ===zLivros: u     Páginas: u     Sem conteúdo: u   Imagens (únicas): z
  Falhas: z
  Chunks: z  FALHA:zrelatorio_falhas.txt) r   sysargvprint__doc__exitjsonloadsr   	read_text	enumerate
LIVROS_DIRr6   
write_textr   rE   r   r   rW   rG   r   r   r   r   r   r   r   dumpsopenwriter0   rF   )"dadosindicecompletochunkstotal_paginassem_conteudolir   pasta_livro	ind_livro
ordem_itemitemr   r   r   ind_capdestino_indnumerarpipagprefixo	nome_basearqr   corpofmcaminho_relr   trilhacichfr`   fls"                                     r   mainr      s   
38}}q gJtCHQK((22G2DDEEEhm8LXZ[[FuhuuW\]jWklomoloWpuuuvHF#$$MLuX22 =+ =+	E b#>#>#>uV}#>#>>$666	{	"..E8E&M22 E EE%L E EvE E$)IIk$:$:$@bE E EOV 	/ 	X 	X 	X #6];9O9OX]^cXd)55d;;DDFFQSU U	[f[[599[;Q;Q;WUW[[[\\\+"+++E&M++,,, 
'N /	1 /	1D!OJF|z)  #&H&H&H$v,&H&HHD4888y/<'/dhh{>S>S\`af\g$)$5$5d$;$;$D$D$F$FSUW W'"))'222 AX A A ABBB%i0+6w'0$Wa00 1 1C")0IR+++6I6I&TT1QCM)R)RTT	////WWZ(( 7 A%L6B0S]CC)%	8II8UVVchs8}&=&=ccQVW]Q^H_H_ccc@HP<hx&8&8<<<<bR>U>>>? "DDHDD5DDDwWWW!ood33<<>>""c(mCJ[f#g#ghhh%eU33#]#]fxX/W#]#]#]]] o3x= o o o oUXY^U_ o ofk o o oppp'u(=(=(E"FF  FBMM!&vRR1K1KRRbRR!&vHPST\P]"5zkR$*!4!4!4!4!:!:!<!<	# #     /H//0000;1< 	x	****	M%%djeTU&V&V&Vah%iii	""..rwwx/@/@7.SSS	d^#S7	;	;	; >q 	> 	>AGGDJqu555<====	>> > > > > > > > > > > > > > > 

 !!!	
eSx))
e
e}
e
eWc
e
efff	
]H
]
]V
]
]PSTZP[P[
]
]^^^  j"	""..tyy/@/@7.SSSSSs   2YY"%Y"__main__)r   )&r   r   rU   r   r   r>   r   urllib.requestr8   pathlibr   urllib.parser   r   __file__r   parentr   r   r@   	CHUNK_MAXcompiler}   r|   ro   r~   r   r   r#   r0   rF   rQ   r   r   r   r   r   __name__r   r   r   <module>r      s     				 				 



                * * * * * * * *tH~~&-H_
			C	D	DbjGHHbj;RTBB1 1 1 1K K K, , ,
 	  <& & &R     "+    2TT TT TTn z DFFFFF r   