o
    yqi70                     @   s   d dl Z d dlmZ d dlZddlmZ ddlmZm	Z	 ddl
mZmZ e	dr,d dlZedG d	d
 d
ZG dd dZedG dd dZdS )    N)Path   )logging)class_requires_depsis_dep_available   )
YAMLReader
YAMLWriterz	faiss-cpuc                   @   s   e Zd ZdZdZdZdZdd Zedd Z	ed	d
 Z
edd Zedd Zedd Zedd Zedd Zdd Zdd Zedd Zdd Zedd ZdS ) 	IndexDataZvectorz.indexid_mapz.yamlc                 C   s.   || _ || _|d | _|d | _|d | _d S )Nr   metric_type
index_type)_indexZ_index_info_id_map_metric_type_index_type)selfindex
index_info r   u/home/app/PaddleOCR-VL/.venv_paddleocr/lib/python3.10/site-packages/paddlex/inference/pipelines/components/faisser.py__init__#   s
   

zIndexData.__init__c                 C      | j S N)r   r   r   r   r   r   *      zIndexData.indexc                 C   s   t | jS r   )faissserialize_indexr   r   r   r   r   index_bytes.   s   zIndexData.index_bytesc                 C   r   r   )r   r   r   r   r   r   2   r   zIndexData.id_mapc                 C   r   r   )r   r   r   r   r   r   6   r   zIndexData.metric_typec                 C   r   r   )r   r   r   r   r   r   :   r   zIndexData.index_typec                 C   s   | j | j| | jdS )N)r   r   r   )r   r   _convert_intr   r   r   r   r   r   >   s   
zIndexData.index_infoc                 C   s&   t |}t|d }| ||d S )Nr      )pickleloadsr   Zdeserialize_index)clsbytestupr   r   r   r   
from_bytesF   s   
zIndexData.from_bytesc                 C   s   t | j| jf}t|S r   )r   r   r   r   r!   dumps)r   r%   r   r   r   to_bytesL   s   
zIndexData.to_bytesc                 C   s   dd |  D S )Nc                 S   s   i | ]\}}t |t|qS r   )intstr.0kvr   r   r   
<dictcomp>Q       z*IndexData._convert_int.<locals>.<dictcomp>items)r   r   r   r   r   r   P   s   zIndexData._convert_intc                 C   s   dd |   D S )Nc                 S   s    i | ]\}}t |t|qS r   )npint64r*   r+   r   r   r   r/   U   s     z,IndexData._convert_int64.<locals>.<dictcomp>r1   )r   r   r   r   _convert_int64S   s   zIndexData._convert_int64c                 C   s   t |}|jddd || j | j   }|| j | j   }| jtj	v r1t
| j| nt
| j| t }|j|| jddd d S )NT)parentsexist_okF)Zdefault_flow_styleZallow_unicode)r   mkdir	VECTOR_FNVECTOR_SUFFIXas_posixIDMAP_FNIDMAP_SUFFIXr   FaissBuilderBINARY_METRIC_TYPEr   Zwrite_index_binaryr   Zwrite_indexr	   writer   )r   save_dirvector_pathindex_info_pathZyaml_writerr   r   r   saveW   s   
zIndexData.savec                 C   sD  t |trt|}|| j | j  }|| j | j  }| s0J d| j | j d| d| sCJ d| j | j d| dt }|	|}d|v rWd|v rWd|v s_J d| dt
|d }|d tjv rut| }nt| }|jt|ksJ d	|||d |d fS t |t
sJ |j|j|j|jfS )
NzNot found the z	 file in !r   r   r   zThe index_info file(z]) may have been damaged, `id_map` or `metric_type` or `index_type` not found in `index_info`.z.data number in index is not equal in in id_map)
isinstancer*   r   r9   r:   r<   r=   existsr   readr
   r5   r>   r?   r   Zread_index_binaryr;   Z
read_indexZntotallenr   r   r   r   )r#   r   Z
index_rootrB   rC   Zyaml_readerr   r   r   r   r   loadj   s>   



zIndexData.loadN)__name__
__module____qualname__r9   r:   r<   r=   r   propertyr   r   r   r   r   r   classmethodr&   r(   r   staticmethodr5   rD   rJ   r   r   r   r   r
      s6    







r
   c                       s$   e Zd Z fddZdd Z  ZS )FaissIndexerc                    s&   t    t|\| _| _| _}d S r   )superr   r
   rJ   _indexerr   r   )r   r   r   	__class__r   r   r      s   
zFaissIndexer.__init__c                 C   s   | j t||\}}g }t||D ]/\}}	|g g d t||	D ]\}
}|dkrA|d d |
 |d d | j|  q$q| jtj	v rYt
|d d df |kd }nt
|d d df |k d }|D ]	}d d d||< qj|S )N)scorelabelr   rV   rW   )rS   searchr3   arrayzipappendr   r   r>   r?   where)r   featureZscore_thresZhamming_radiusZtopkZscores_listZids_listpredsZscoresidsrV   idZidxsidxr   r   r   __call__   s     zFaissIndexer.__call__)rK   rL   rM   r   rc   __classcell__r   r   rT   r   rQ      s    rQ   c                   @   sz   e Zd ZdZdZdZdZedddZedd	 Z	e	
	dddZ
edd Zedd Zedd ZedddZdS )r>   )hammingIPL2)FlatIVFHNSW32)re   )rh   ri   Z
BinaryHashNc                 C   s   |dkr|t tt|d d }|| jv r|d7 }n|d7 }|| jv r9|| jv s5J d| d| j dd	| }|d
krDtd d
}|dkrJd}|S )Nri      i   z,BFlatz,FlatzThe metric type(z) only support z index types!Brj   z5The HNSW32 method dose not support 'remove' operationrh   )r*   minr)   r?   BINARY_SUPPORT_INDEX_TYPEr   warning)r#   r   r   numr   r   r   _get_index_type   s    



zFaissBuilder._get_index_typec                 C   s<   |dkrt jS |dkrt jS |dkrt jS |dkrt jS d S )Nre   Zjaccardrf   rg   )r   ZMETRIC_HammingZMETRIC_JaccardZMETRIC_INNER_PRODUCTZ	METRIC_L2)r#   r   r   r   r   _get_metric_type   s   zFaissBuilder._get_metric_typerf   rj   c                 C   s"  || j v sJ d| j  d|| jv sJ d| j dt|tr*| ||\}}n||}}dd ||D }|| jv r@tjntj}	t	|
|	}|j\}
}|| jv ret|| |||
| |}nt|| |||
| |}t|}i }| j|||||dd\}}t||||dS )	NzSupported index types only: rE   Supported metric types only: c                 S      g | ]}|d  qS r^   r   r,   resr   r   r   
<listcomp>       z&FaissBuilder.build.<locals>.<listcomp>newmoder   r   r   )SUPPORT_INDEX_TYPESUPPORT_METRIC_TYPErF   r*   load_galleryr?   r3   uint8float32rZ   astypeshaper   Zindex_binary_factoryrq   rr   Zindex_factoryZIndexIDMap2_add_galleryr
   )r#   gallery_imgsgallery_labelpredict_funcr   r   gallery_docsgallery_listfeaturesdtypeZ
vector_numZ
vector_dimr   r`   r   r   r   build   s@   






zFaissBuilder.buildc           
         s   t |\}}}}|dkrtdt trMg }t }| }W d    n1 s+w   Y  g  |D ]}t| 	dd }	 
|	 q4t  nt  |   fdd| D }t ||||dS )Nrj   z:The index_type: HNSW32 dose not support 'remove' operation r   c                    s   i | ]\}}| vr||qS r   r   r+   
remove_idsr   r   r/   #  r0   z'FaissBuilder.remove.<locals>.<dictcomp>r}   )r
   rJ   RuntimeErrorrF   r*   open	readlinesr)   stripsplitr\   r3   Zasarrayr   r2   )
r#   r   r   r`   r   r   linesflineZid_r   r   r   remove
  s*   




zFaissBuilder.removec                 C   s   t |\}}}}|| jv sJ d| j dt|tr%| ||\}}	n||}}	dd ||	D }
|| jv r;tjntj	}t
|
|}
| j||||
|dd\}}t ||||dS )Nrs   rE   c                 S   rt   ru   r   rv   r   r   r   rx   4  ry   z'FaissBuilder.append.<locals>.<listcomp>r\   r{   r}   )r
   rJ   r   rF   r*   r   r?   r3   r   r   rZ   r   r   )r#   r   r   r   r   r`   r   r   r   r   r   r   r   r   r   r\   (  s    


zFaissBuilder.appendc                 C   s   |r
t | d nd}tdt|| tj}|dkr.|| jv r)|| n|	| || jvr9|
|| tt||D ]\}	}
|
||	< q@||fS )Nr    r   rz   )maxkeysr3   ZarangerI   r   r4   r?   addtrainZadd_with_idsr[   list)r#   r   r   r`   Zgallery_featuresr   r|   Zstart_idZids_nowidr   r   r   r   @  s   



zFaissBuilder._add_gallery r   c                 C   s   g }g }g }t |}t|ddd}| }W d    n1 s w   Y  |D ]}	|	 |\}
}||
 }||  || q'||fS )Nrzutf-8)encoding)r   r   r   r   r   r\   r;   )r#   Zgallery_label_pathZgallery_imgs_root	delimiterr   fileslabelsrootr   r   pathrW   	file_pathr   r   r   r   X  s   
zFaissBuilder.load_galleryr   )rf   rj   )r   r   )rK   rL   rM   r   r~   r?   rn   rO   rq   rr   r   r   r\   r   r   r   r   r   r   r>      s*    

1


r>   )r!   pathlibr   numpyr3   utilsr   Z
utils.depsr   r   Zutils.ior   r	   r   r
   rQ   r>   r   r   r   r   <module>   s   r