File: //proc/1233/root/usr/local/lib/python3.10/dist-packages/tiktoken/__pycache__/load.cpython-310.pyc
o
;��g� � @ sn d dl mZ d dlZd dlZd dlZd dd�Zd d d
�Zd!d"dd�Z d#d$dd�Zd%dd �Z d!d&d d �Z
dS )'� )� annotationsN�blobpath�str�return�bytesc
C s� | � d�s<| � d�s<zdd l}W n ty! } ztd�|�d }~ww |�| d�� }|�� W d � S 1 s7w Y dd l}|�| �}|�� |jS )Nzhttp://zhttps://r �Oblobfile is not installed. Please install it by running `pip install blobfile`.�rb) �
startswith�blobfile� ImportError�BlobFile�read�requests�get�raise_for_status�content)r r
�e�fr �resp� r �8/usr/local/lib/python3.10/dist-packages/tiktoken/load.py� read_file s$ ���� �
r �data�
expected_hash�boolc C s t �| ��� }||kS �N)�hashlib�sha256� hexdigest)r r � actual_hashr r r �
check_hash s r �
str | Nonec C s� d}dt jv r
t jd }ndt jv rt jd }ndd l}t j�|�� d�}d}|dkr/t| �S t�| � � ��
� }t j�||�}t j� |�rzt |d�� }|�
� }W d � n1 sYw Y |d u sgt||�ri|S zt �|� W n tyy Y nw t| �} |r�t| |�s�td | � d
|� d ���dd l}
z6t j|dd � |d
t|
�� � d } t | d��
}|�| � W d � n1 s�w Y t �| |� W | S ty� |rԂ Y | S w )NT�TIKTOKEN_CACHE_DIR�DATA_GYM_CACHE_DIRr zdata-gym-cacheF� r z'Hash mismatch for data downloaded from z (expected z<). This may indicate a corrupted download. Please try again.)�exist_ok�.z.tmp�wb)�os�environ�tempfile�path�join�
gettempdirr r �sha1�encoder �exists�openr
r �remove�OSError�
ValueError�uuid�makedirsr �uuid4�write�rename) r r �user_specified_cache� cache_dirr* � cache_key�
cache_pathr r �contentsr5 � tmp_filenamer r r �read_file_cached sV
� �� � ���r@ �vocab_bpe_file�encoder_json_file�vocab_bpe_hash�encoder_json_hash�dict[bytes, int]c s> dd� t d�D �}dd� |D �� d}t d�D ]}||vr-|�|� |� td| �< |d7 }qt|�dks6J �t| |��� }dd� |�d �dd
� D �}d� fdd� �dd� t|�D �} t| �}|D ]\}
} || �|
��| � < |d7 }qadd l} | � t||��}
�fdd�|
�
� D �}|� dd � |� dd � | |ks�J �| S )Nc S s( g | ]}t |��� rt |�d kr|�qS )� )�chr� isprintable��.0�br r r �
<listcomp>] s ( z3data_gym_to_mergeable_bpe_ranks.<locals>.<listcomp>� c S s i | ]}t |�|�qS r )rG rI r r r �
<dictcomp>_ s z3data_gym_to_mergeable_bpe_ranks.<locals>.<dictcomp>r � c S s g | ]}t |�� ��qS r )�tuple�split)rJ � merge_strr r r rL j s �
����valuer r r c s t � fdd�| D ��S )Nc 3 s � | ]}� | V qd S r r rI ��data_gym_byte_to_byter r � <genexpr>m s � zKdata_gym_to_mergeable_bpe_ranks.<locals>.decode_data_gym.<locals>.<genexpr>�r )rU rV r r �decode_data_gyml s z8data_gym_to_mergeable_bpe_ranks.<locals>.decode_data_gymc S s i | ]
\}}t |g�|�qS r rY )rJ �irK r r r rN p s c s i | ] \}}� |�|�qS r r )rJ �k�v)rZ r r rN } s s
<|endoftext|>s <|startoftext|>)rU r r r ) �range�appendrG �lenr@ �decoderQ � enumerate�json�loads�items�pop)rA rB rC rD �rank_to_intbyte�nrK �vocab_bpe_contents�
bpe_merges� bpe_ranks�first�secondrc � encoder_json�encoder_json_loadedr )rW rZ r � data_gym_to_mergeable_bpe_ranksV s2
�
rp rk �tiktoken_bpe_file�Nonec
C s� zdd l }W n ty } ztd�|�d }~ww |�|d��*}t| �� dd� d�D ]\}}|�t�|�d t|�� � d � q)W d � d S 1 sKw Y d S ) Nr r r' c S s | d S )NrO r )�xr r r �<lambda>� s z#dump_tiktoken_bpe.<locals>.<lambda>)�key� �
)
r
r r �sortedre r8 �base64� b64encoder r/ )rk rq r
r r �token�rankr r r �dump_tiktoken_bpe� s ���� &�"�r} c C sv t | |�}i }|�� D ]-}|sq z|�� \}}t|�|t�|�< W q ty8 } z td|�d| � ��|�d }~ww |S )NzError parsing line z in )r@ �
splitlinesrQ �intry � b64decode� Exceptionr4 )rq r r> �ret�liner{ r| r r r r �load_tiktoken_bpe� s
��r� )r r r r )r r r r r r r )r r r r! r r )NN)
rA r rB r rC r! rD r! r rE )rk rE rq r r rr )rq r r r! r rE ) �
__future__r ry r r( r r r@ rp r} r� r r r r �<module> s
9 �
0