o
    i                      @  s   d dl mZ d dlmZ d dlmZ d dlmZ d dlm	Z	m
Z
 d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ d dlmZ e	rjd dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d dl m!Z! eddG dd dZ"dS )    )annotations)	dataclass)partial)perf_counter)TYPE_CHECKINGAny)eprint)+_get_credentials_from_provider_expiry_aware)0_extract_table_statistics_from_delta_add_actions)scan_parquet)ScanCastOptions)Schema)datetime)
DeltaTable)StorageOptionsDict)NoPickleOption)CredentialProviderBuilder	LazyFrameT)kw_onlyc                   @  s   e Zd ZU dZded< ded< ded< ded	< d
ed< ded< ded< ded< ded< d/ddZddddddd0d d!Zd1d#d$Zd2d&d'Zd3d)d*Z	d4d-d.Z
dS )5DeltaDatasetzDataset interface for Delta.zNoPickleOption[DeltaTable]table_
str | None
table_uri_zint | str | datetime | NoneversionzStorageOptionsDict | Nonestorage_optionsz CredentialProviderBuilder | Nonecredential_provider_builderzdict[str, Any] | Nonedelta_table_optionsbooluse_pyarrowpyarrow_optionsrechunkreturnr   c                 C  s   t |   S )zFetch the schema of the table.)r   tableschemaself r'   O/home/app/Keep/.python/lib/python3.10/site-packages/polars/io/delta/_dataset.pyr$   1   s   zDeltaDataset.schemaN)existing_resolved_version_keylimit
projectionfilter_columnspyarrow_predicater)   r*   
int | Noner+   list[str] | Noner,   r-   tuple[LazyFrame, str] | Nonec                  s  ddl }ddl}|jj }|r%td| j d| d| d| d| j 
 |  }	| jdur1| jn|	 }
t	|
}|durM||krM|rKtd|d	 dS | jr~ddl
}dd
lm} |	jdi | jpbi }t|jjjj||||d}|j|j|ddd|fS |	 }t|j |  }t fdd| D }t }|rtd |	 }|  drdd |D }|rt | }tdt| d|dd |durt |!|	" |||dnd}t#|t dkr|ndt dkt$% dd| j&| j'| j(|d
|fS )zConstruct a LazyFrame scan.r   Nz*DeltaDataset: to_dataset_scan(): version: z	, limit: z, projection: z, filter_columns: z, use_pyarrow: z=DeltaDataset: to_dataset_scan(): early return (version_key = )r   )Zn_rows	predicateZwith_columnsT)ZpyarrowZis_purec                   s   i | ]\}}| v r||qS r'   r'   ).0kvpartition_columnsr'   r(   
<dictcomp>t   s    z0DeltaDataset.to_dataset_scan.<locals>.<dictcomp>z5DeltaDataset: to_dataset_scan(): begin path expansion	lakefs://c                 S  s   g | ]}| d dqS )r9   zs3://)replace)r3   pathr'   r'   r(   
<listcomp>   s    z0DeltaDataset.to_dataset_scan.<locals>.<listcomp>zCDeltaDataset: to_dataset_scan(): native scan_parquet(): num_files: z, path expansion time: z.3fs)r,   r$   verboseinsertignore)	hive_schemaZhive_partitioningZcast_optionsZmissing_columnsZextra_columnsr   Zcredential_providerr!   Z_table_statisticsr'   ))polarspolars._utils.logging_utilsloggingr>   r   r   r   r#   strZ(polars.io.pyarrow_dataset.anonymous_scanpolars.lazyframe.framer   Zto_pyarrow_datasetr    r   ioZpyarrow_datasetZanonymous_scanZ_scan_pyarrow_dataset_implZ_scan_python_functionr$   metadatasetr7   r   itemsr   Z	file_uris	table_uri
startswithlenr
   Z	DataFrameZget_add_actionsr   r   Z_default_icebergr   r   r!   )r&   r)   r*   r+   r,   r-   plrB   r>   r#   r   Zversion_keyr   ZdatasetfuncZtable_mdr$   rA   
start_timepathselapsedZtable_statisticsr'   r6   r(   to_dataset_scan5   s   
	





zDeltaDataset.to_dataset_scanrF   c                 C  s.   | j du r| j dusJ |  j| _ | j S )zFetch the table URI.N)r   r   getr#   rL   r%   r'   r'   r(   rL      s   
zDeltaDataset.table_urir   c                 C  s>  | j  du rddlm} ddlm}m}m} ddlm	} | j
dus$J i }| jr6| j  }r6t|p5i }|| j
| j| jdusE| jdurNi | jpJi |nd| jd}| }	|	j|ksb|	j|krrd|	j d| d	| }
||
|	jd
kr|	jdurh |	j|}t|dkrd| d}
||
| j | | j  S )zFetch the DeltaTable object.Nr   )DeltaProtocolError)MAX_SUPPORTED_READER_VERSIONNOT_SUPPORTED_READER_VERSIONSUPPORTED_READER_FEATURES)_get_delta_lake_table)Z
table_pathr   r   r   z&The table's minimum reader version is z5 but polars delta scanner only supports version 1 or z with these reader features:    z)The table has set these reader features: z= but these are not yet supported by the polars delta scanner.)r   rU   Zdeltalake.exceptionsrV   Zdeltalake.tablerW   rX   rY   polars.io.delta._utilsrZ   r   r   Zbuild_credential_providerr	   r   r   r   protocolZmin_reader_versionZreader_features
differencerN   rJ   )r&   rV   rW   rX   rY   rZ   Zcredential_provider_credsproviderr#   Ztable_protocolmsgZmissing_featuresr'   r'   r(   r#      sP   









zDeltaDataset.tabledict[str, Any]c                 C  s   |    | jS N)rL   __dict__r%   r'   r'   r(   __getstate__   s   zDeltaDataset.__getstate__stateNonec                 C  s
   || _ d S rb   )rc   )r&   re   r'   r'   r(   __setstate__   s   
zDeltaDataset.__setstate__)r"   r   )r)   r   r*   r.   r+   r/   r,   r/   r-   r   r"   r0   )r"   rF   )r"   r   )r"   ra   )re   ra   r"   rf   )__name__
__module____qualname____doc____annotations__r$   rT   rL   r#   rd   rg   r'   r'   r'   r(   r      s,   
 

q

=r   N)#
__future__r   dataclassesr   	functoolsr   timer   typingr   r   rC   r   Z.polars.io.cloud.credential_provider._providersr	   r\   r
   Zpolars.io.parquet.functionsr   Z#polars.io.scan_options.cast_optionsr   Zpolars.schemar   r   Z	deltalaker   Zpolars._typingr   Zpolars.io.cloud._utilsr   Z,polars.io.cloud.credential_provider._builderr   rG   r   r   r'   r'   r'   r(   <module>   s(    