o

;��gDN�@
sR
dd
lm
Z

ddlmZ
ddlmZ
ddlmZmZm	Z	
ddl
mZmZm
Z
mZmZmZmZmZmZmZmZmZmZmZmZmZmZ
Gdd�d�ZGd	d
�d
e�ZGdd�de�ZGd
d�de�ZGdd�de�Z Gdd�de�Z!Gdd�de�Z"Gdd�de�Z#Gdd�de�Z$Gdd�de�Z%edd�
d/d"d#��
Z&ed$d�
	&d0d1d,d-��
Z'd.S)2�)
�annotations)
�	lru_cache)
�	getLogger�
)�COMMON_SAFE_ASCII_CHARACTERS�TRACE�UNICODE_SECONDARY_RANGE_KEYWORD)�is_accentuated�	is_arabic�is_arabic_isolated_form�is_case_variable�is_cjk�is_emoticon�	is_hangul�is_hiragana�is_katakana�is_latin�is_punctuation�is_separator�	is_symbol�is_thai�is_unprintable�
remove_accent�
unicode_rangec@
s<eZ
dZd
Zddd�Zdd	d
�Zddd�Zeddd��
ZdS)�MessDetectorPluginzy
    Base abstract class used for mess detection plugins.
    All detectors MUST extend and implement given methods.
    �	character�str�return�boolc
C
�t�
)
z@
        Determine if given character should be fed in.
        �
�NotImplementedError��selfr�r$�@/usr/local/lib/python3.10/dist-packages/charset_normalizer/md.py�eligible&�zMessDetectorPlugin.eligible�Nonec
C
r)
z�
        The main routine to be executed upon character.
        Insert the logic in witch the text would be considered chaotic.
        r r"r$r$r%�feed,szMessDetectorPlugin.feedc


C
r)
zB
        Permit to reset the plugin to the initial state.
        r �
r#r$r$r%�reset3r'zMessDetectorPlugin.reset�floatc


C
r)
z�
        Compute the chaos ratio based on what your feed() has seen.
        Must NOT be lower than 0.; No restriction gt 0.
        r r*r$r$r%�ratio9szMessDetectorPlugin.ratioN�rrrr�rrrr(�rr(�rr,)	�__name__�
__module__�__qualname__�__doc__r&r)r+�propertyr-r$r$r$r%r s




rc@
�BeZ
dZddd�Zddd	�Zdd
d�Zddd
�Zeddd��
ZdS)� TooManySymbolOrPunctuationPluginrr(c

C
s"d
|_d
|_
d
|_d|_d|_dS)Nr
F)�_punctuation_count�
_symbol_count�_character_count�_last_printable_char�_frenzy_symbol_in_wordr*r$r$r%�__init__Cs





z)TooManySymbolOrPunctuationPlugin.__init__rrrcC
�|
��S�
N�
�isprintabler"r$r$r%r&K�
z)TooManySymbolOrPunctuationPlugin.eligiblecC
sp|jd
7_|
|j
kr3|
tv
r3t|
�
r|jd
7_n|
��dur3t|
�
r3t|
�
dur3|jd7_|
|_
dS)NrF�)	r;r<rrr9�isdigitrrr:r"r$r$r%r)Ns




�
z%TooManySymbolOrPunctuationPlugin.feedc

C
sd
|_d
|_
d
|_dS�Nr
)r9r;r:r*r$r$r%r+`�



z&TooManySymbolOrPunctuationPlugin.resetr,c
C
s0|jd
krdS|j
|j|j}
|
dkr|
SdS)Nr
��333333�?)r;r9r:)r#�ratio_of_punctuationr$r$r%r-es



�z&TooManySymbolOrPunctuationPlugin.ratioNr0r.r/r1�	r2r3r4r>r&r)r+r6r-r$r$r$r%r8Bs





r8c@
r7)�TooManyAccentuatedPluginrr(c

C
�d
|_d
|_
dSrF�r;�_accentuated_countr*r$r$r%r>r�


z!TooManyAccentuatedPlugin.__init__rrrcC
r?r@)
�isalphar"r$r$r%r&vrCz!TooManyAccentuatedPlugin.eligiblecC
�,|jd
7_t
|
�
r|jd
7_dSdS�Nr)r;r	rOr"r$r$r%r)y�

�zTooManyAccentuatedPlugin.feedc

C
rMrFrNr*r$r$r%r+rPzTooManyAccentuatedPlugin.resetr,c
C
s*|jd
krdS|j
|j}
|
dkr|
SdS)N�rHgffffff�?rN)r#�ratio_of_accentuationr$r$r%r-�s


zTooManyAccentuatedPlugin.ratioNr0r.r/r1rKr$r$r$r%rLqs





rLc@
r7)�UnprintablePluginrr(c

C
rMrF)�_unprintable_countr;r*r$r$r%r>�rPzUnprintablePlugin.__init__rrrc
C
�d
S�NTr$r"r$r$r%r&��
zUnprintablePlugin.eligiblecC
s(t|
�
r|j
d
7_
|jd
7_dSrS)rrXr;r"r$r$r%r)�s


zUnprintablePlugin.feedc

C
s
d
|_dSrF)
rXr*r$r$r%r+�s

zUnprintablePlugin.resetr,c

C
�|jd
krdS|j
d|jS)Nr
rHrU)r;rXr*r$r$r%r-��

zUnprintablePlugin.ratioNr0r.r/r1rKr$r$r$r%rW�s





rWc@
r7)�SuspiciousDuplicateAccentPluginrr(c

C
�d
|_d
|_
d|_dSrF��_successive_countr;�_last_latin_characterr*r$r$r%r>�s


z(SuspiciousDuplicateAccentPlugin.__init__rrrcC
s|
��ot
|
�
Sr@)rQrr"r$r$r%r&�s
z(SuspiciousDuplicateAccentPlugin.eligiblecC
st|jd
7_|j
du
r5t|
�
r5t|j
�
r5|
��r%|j
��r%|jd
7_t|
�
t|j
�
kr5|jd
7_|
|_
dSrS)r;rbr	�isupperrarr"r$r$r%r)�s


��



z$SuspiciousDuplicateAccentPlugin.feedc

C
r_rFr`r*r$r$r%r+�rGz%SuspiciousDuplicateAccentPlugin.resetr,c

C
r\)Nr
rHrD)r;rar*r$r$r%r-�r]z%SuspiciousDuplicateAccentPlugin.ratioNr0r.r/r1rKr$r$r$r%r^�s





r^c@
r7)�SuspiciousRangerr(c

C
r_rF)�"_suspicious_successive_range_countr;�_last_printable_seenr*r$r$r%r>�rGzSuspiciousRange.__init__rrrcC
r?r@rAr"r$r$r%r&�rCzSuspiciousRange.eligiblecC
sx|jd
7_|
�
�st|
�
s|
tvrd|_dS|jdur"|
|_dSt|j�
}t|
�
}t||�r7|jd
7_|
|_dSrS)r;�isspacerrrfr� is_suspiciously_successive_rangere)r#r�unicode_range_a�unicode_range_br$r$r%r)�s 
��








zSuspiciousRange.feedc

C
r_rF)r;rerfr*r$r$r%r+�rGzSuspiciousRange.resetr,c
C
s"|jd
k
rdS|j
d|j}
|
S)N�
rHrD)r;re)r#�ratio_of_suspicious_range_usager$r$r%r-�s


�zSuspiciousRange.ratioNr0r.r/r1rKr$r$r$r%rd�s





rdc@
r7)�SuperWeirdWordPluginrr(c

C
s@d
|_d
|_
d
|_d|_d|_d
|_d
|_d|_d
|_d
|_	dS)Nr
F�)
�_word_count�_bad_word_count�_foreign_long_count�_is_current_word_bad�_foreign_long_watchr;�_bad_character_count�_buffer�_buffer_accent_count�_buffer_glyph_countr*r$r$r%r>�s







zSuperWeirdWordPlugin.__init__rrrc
C
rYrZr$r"r$r$r%r&

r[zSuperWeirdWordPlugin.eligiblecC
s�|
��rc|j
|
7_
t|
�
r|jd
7_|jdurFt|
�
dus%t|
�
rFt|
�
durFt|
�
durFt|
�
durFt	|
�
durFt
|
�
durFd|_t|
�
sZt|
�
sZt|
�
sZt	|
�
sZt
|
�
ra|jd
7_dS|j
shdS|
��sut
|
�
sut|
�
�
r$|j
�
r$|jd
7_t|j
�
}|j|7_|dkr�|j|dkr�d|_n4t|j
d�
r�|j
d��r�tdd�|j
D�
�
dur�|jd
7_d|_n|jd
kr�d|_|jd
7_|d	kr�|jr�d
d�t|j
td|��D�
}d}|r�t|�
|d
k
r�d}|s�|jd
7_d|_|j�
r|jd
7_|jt|j
�
7_d|_d|_d|_
d|_d|_dS|
dv
�
rA|
��du�
rCt|
�
�
rEd|_|j
|
7_
dSdSdSdS)NrFT�g�?���c
s
s�|]}
|
��V
qdSr@�
rc)�.0�
_r$r$r%�	<genexpr>7
s�z,SuperWeirdWordPlugin.feed.<locals>.<genexpr>�c
S
sg|]
\}
}|
��r|�qSr$rz)r{�
c�
ir$r$r%�
<listcomp>?
s
�
�z-SuperWeirdWordPlugin.feed.<locals>.<listcomp>r
rIrn>r|�
-�
<�
=�
>�
|�
~)rQrur	rvrsrr
rrrrrwrgrrro�lenr;rrrc�allrq�zip�rangerprtrEr)r#r�
buffer_length�camel_case_dst�probable_camel_casedr$r$r%r)

s�










�����


�
�
��



��






�












�
�
zSuperWeirdWordPlugin.feedc

C
s4d
|_d|_
d|_d|_d|_d|_d|_d|_dS)NrnFr
)rurrrsrpror;rtrqr*r$r$r%r+^
s








zSuperWeirdWordPlugin.resetr,c

C
s$|jd
k
r|j
dkrdS|j|jS)N�
r
rH)rorqrtr;r*r$r$r%r-h
s
zSuperWeirdWordPlugin.ratioNr0r.r/r1rKr$r$r$r%rm�s




Q

rmc@
sFeZ
dZd
Zddd�Zdd	d
�Zddd�Zdd
d�Zeddd��
Z	dS)�CjkInvalidStopPluginu�
    GB(Chinese) based encoding often render the stop incorrectly when the content does not fit and
    can be easily detected. Searching for the overuse of '丅' and '丄'.
    rr(c

C
rMrF��_wrong_stop_count�_cjk_character_countr*r$r$r%r>v
rPzCjkInvalidStopPlugin.__init__rrrc
C
rYrZr$r"r$r$r%r&z
r[zCjkInvalidStopPlugin.eligiblecC
s8|
d
vr
|jd7_dSt
|
�
r|jd7_dSdS)N>�丄�丅r)r�r
r�r"r$r$r%r)}
s




�zCjkInvalidStopPlugin.feedc

C
rMrFr�r*r$r$r%r+�
rPzCjkInvalidStopPlugin.resetr,c

C
s|jd
krdS|j
|jS)N�rH)r�r�r*r$r$r%r-�
s


zCjkInvalidStopPlugin.ratioNr0r.r/r1)
r2r3r4r5r>r&r)r+r6r-r$r$r$r%r�p
s





r�c@
r7)�ArchaicUpperLowerPluginrr(c

C
s.d
|_d|_
d|_d|_d|_d|_d|_dS)NFr
T)�_buf�_character_count_since_last_sep�_successive_upper_lower_count�#_successive_upper_lower_count_finalr;�_last_alpha_seen�_current_ascii_onlyr*r$r$r%r>�
s



z ArchaicUpperLowerPlugin.__init__rrrc
C
rYrZr$r"r$r$r%r&�
r[z ArchaicUpperLowerPlugin.eligiblecC
s$
|
��ot
|
�
}|d
u}|rC|jdkrC|jdk
r+|
��d
ur+|jd
ur+|j|j7_d|_d|_d|_d
|_|j	d7_	d|_dS|jdurQ|
�
�d
urQd
|_|jdu
r|
��r_|j��sh|
��r||j��r||jdurx|jd7_d
|_nd|_nd
|_|j	d7_	|jd7_|
|_dS)NFr
�@rTrD)
rQrr�rEr�r�r�r�r�r;�isasciirc�islower)r#r�is_concerned�	chunk_sepr$r$r%r)�
s@






�








�
�





zArchaicUpperLowerPlugin.feedc

C
s.d
|_d
|_
d
|_d
|_d|_d|_d|_dS)Nr
FT)r;r�r�r�r�r�r�r*r$r$r%r+�
s







zArchaicUpperLowerPlugin.resetr,c

C
s|jd
krdS|j
|jS)Nr
rH)r;r�r*r$r$r%r-�
s

zArchaicUpperLowerPlugin.ratioNr0r.r/r1rKr$r$r$r%r��
s




*	
r�c@
sBeZ
dZddd�Zddd�Zdd
d�Zddd
�Zeddd��
ZdS)�ArabicIsolatedFormPluginrr(c

C
rMrF�r;�_isolated_form_countr*r$r$r%r>�
rPz!ArabicIsolatedFormPlugin.__init__c

C
rMrFr�r*r$r$r%r+�
rPzArabicIsolatedFormPlugin.resetrrrcC
st|
�
Sr@)
r
r"r$r$r%r&�
rCz!ArabicIsolatedFormPlugin.eligiblecC
rRrS)r;rr�r"r$r$r%r)�
rTzArabicIsolatedFormPlugin.feedr,c
C
s|jd
krdS|j
|j}
|
S)NrUrHr�)r#�isolated_form_usager$r$r%r-�
s

zArabicIsolatedFormPlugin.ratioNr0r.r/r1)	r2r3r4r>r+r&r)r6r-r$r$r$r%r��
s





r��)
�maxsizeri�
str | NonerjrrcC
sv
|d
us|
d
ur
dS||
krdSd|vrd|
vrdSd|vs"d|
vr$dSd|vs,d|
vr6d|vs4d|
vr6dS|�d�
|
�d�
}}|D]}|t
vrJqC||vrQ
dSqC|dv|
dv}}|s_|rid	|vsgd	|
vridS|ro|rodSd
|vswd
|
vr�d	|vsd	|
vr�dS|dks�|
dkr�dSd	|vs�d	|
vs�|dvr�|
dvr�d
|vs�d
|
vr�dSd|vs�d|
vr�dS|dks�|
dkr�dSdS)za
    Determine if two Unicode range seen next to each other can be considered as suspicious.
    NTF�Latin�	Emoticons�	Combining�
 )�Hiragana�Katakana�CJK�HangulzBasic Latin)r�r��Punctuation�Forms)�splitr)rirj�keywords_range_a�keywords_range_b�el�range_a_jp_chars�range_b_jp_charsr$r$r%rh�
sZ





�



�
��













rhi皙�����?F�decoded_sequencer�maximum_thresholdr,�debugc	C
sR
d
d�t�
�D�
}t|�
d}d}|dkrd}n	|dk
rd}nd	}t|d
t|�
�D]2\}}|D]}	|	�|�
r<|	�|�

q0|dkrG||dksM||dkr\tdd
�|D�
�
}||
kr\
n
q*|r�td�
}
|
�	t
d|�d|�d|
���
t|�
dkr�|
�	t
d|dd����
|
�	t
d|dd����
|D]}|
�	t
|j�d|j���
q�t
|d�S)zw
    Compute a mess ratio given a decoded bytes sequence. The maximum threshold does stop the computation earlier.
    c
S
sg|]}
|
��qSr$r$)r{�md_classr$r$r%r�Is
�zmess_ratio.<locals>.<listcomp>rrHi� r�r���

r
c
s
s�|]}
|
jV
qdSr@)
r-)r{�dtr$r$r%r}`s�zmess_ratio.<locals>.<genexpr>�charset_normalizerzIMess-detector extended-analysis start. intermediary_mean_mess_ratio_calc=z mean_mess_ratio=z maximum_threshold=r�zStarting with: Nz
Ending with: i�z: �)r�__subclasses__r�r�r�r&r)�sumr�logr�	__class__r-�round)r�r�r��	detectors�length�mean_mess_ratio�!intermediary_mean_mess_ratio_calcr�index�detector�loggerr�r$r$r%�
mess_ratioAsN
�







�


�



�
���



r�N)rir�rjr�rr)r�F)r�rr�r,r�rrr,)(�
__future__r�	functoolsr�loggingr�constantrrr�utilsr	r
rrr
rrrrrrrrrrrrrr8rLrWr^rdrmr�r�r�rhr�r$r$r$r%�<module>
s(
L"/%1vL
I�