B
    lnd8                 @   s   d dl Z d dlZd dlZd dlZd dlZd dlmZ d dl	Z
d dlmZmZmZmZ edddZG dd dZG d	d
 d
eZG dd deZG dd deZdd Zdd ZdddZdS )    N)join)
implements_stringify_path_is_path_like	_DEPR_MSGzfilesystem.LocalFileSystemz2.0.0zfs.LocalFileSystemc               @   s   e Zd ZdZdd Zdd Zd(ddZd	d
 Zdd Zdd Z	d)ddZ
dd Zdd Zd*ddZdd Zdd Zdd Zdd Zd+d!d"Zd,d$d%Zed&d' Zd S )-
FileSystemz(
    Abstract filesystem interface.
    c          	   C   s    |  |d
}| S Q R X dS )z
        Return contents of file as a bytes object.

        Parameters
        ----------
        path : str
            File path to read content from.

        Returns
        -------
        contents : bytes
        rbN)openread)selfpathf r   6/tmp/pip-unpacked-wheel-i2z0tf32/pyarrow/filesystem.pycat)   s    zFileSystem.catc             C   s   t dS )z
        Return list of file paths.

        Parameters
        ----------
        path : str
            Directory to list contents from.
        N)NotImplementedError)r   r   r   r   r   ls9   s    	zFileSystem.lsFc             C   s   t dS )z
        Delete the indicated file or directory.

        Parameters
        ----------
        path : str
            Path to delete.
        recursive : bool, default False
            If True, also delete child paths for directories.
        N)r   )r   r   	recursiver   r   r   deleteD   s    zFileSystem.deletec       	      C   st   t |}| |}|d dkr&|d S d}xD| |D ]6\}}}x*|D ]"}| ||}|| |d 7 }qFW q6W |S )z
        Compute bytes used by all contents under indicated path in file tree.

        Parameters
        ----------
        path : str
            Can be a file path or directory.

        Returns
        -------
        usage : int
        kindfilesizer   )r   statwalk
_path_join)	r   r   Z	path_infototalrootdirectoriesfilesZ
child_pathabspathr   r   r   
disk_usageQ   s    

zFileSystem.disk_usagec             G   s   | j |S )N)pathsepr   )r   argsr   r   r   r   k   s    zFileSystem._path_joinc             C   s   t ddS )zl
        Information about a filesystem entry.

        Returns
        -------
        stat : dict
        zFileSystem.statN)r   )r   r   r   r   r   r   n   s    zFileSystem.statc             C   s   | j ||dS )z.
        Alias for FileSystem.delete.
        )r   )r   )r   r   r   r   r   r   rmx   s    zFileSystem.rmc             C   s   |  ||S )z.
        Alias for FileSystem.rename.
        )rename)r   r   new_pathr   r   r   mv~   s    zFileSystem.mvc             C   s   t ddS )z
        Rename file, like UNIX mv command.

        Parameters
        ----------
        path : str
            Path to alter.
        new_path : str
            Path to move to.
        zFileSystem.renameN)r   )r   r   r%   r   r   r   r$      s    zFileSystem.renameTc             C   s   t dS )z
        Create a directory.

        Parameters
        ----------
        path : str
            Path to the directory.
        create_parents : bool, default True
            If the parent directories don't exists create them as well.
        N)r   )r   r   create_parentsr   r   r   mkdir   s    zFileSystem.mkdirc             C   s   t dS )z
        Return True if path exists.

        Parameters
        ----------
        path : str
            Path to check.
        N)r   )r   r   r   r   r   exists   s    	zFileSystem.existsc             C   s   t dS )z
        Return True if path is a directory.

        Parameters
        ----------
        path : str
            Path to check.
        N)r   )r   r   r   r   r   isdir   s    	zFileSystem.isdirc             C   s   t dS )z
        Return True if path is a file.

        Parameters
        ----------
        path : str
            Path to check.
        N)r   )r   r   r   r   r   isfile   s    	zFileSystem.isfilec             C   s   t dS )zf
        Returns True if this FileSystem is a unix-style file store with
        directories.
        N)r   )r   r   r   r   _isfilestore   s    zFileSystem._isfilestoreNc       	      C   s,   ddl m} ||||| d}|j|||dS )a]  
        Read Parquet data from path in file system. Can read from a single file
        or a directory of files.

        Parameters
        ----------
        path : str
            Single file path or directory
        columns : List[str], optional
            Subset of columns to read.
        metadata : pyarrow.parquet.FileMetaData
            Known metadata to validate files against.
        schema : pyarrow.parquet.Schema
            Known schema to validate files against. Alternative to metadata
            argument.
        use_threads : bool, default True
            Perform multi-threaded column reads.
        use_pandas_metadata : bool, default False
            If True and file has custom pandas schema metadata, ensure that
            index columns are also loaded.

        Returns
        -------
        table : pyarrow.Table
        r   )ParquetDataset)schemametadata
filesystem)columnsuse_threadsuse_pandas_metadata)Zpyarrow.parquetr-   r
   )	r   r   r1   r/   r.   r2   r3   r-   Zdatasetr   r   r   read_parquet   s
    zFileSystem.read_parquetr   c             C   s   t dS )z3
        Open file for reading or writing.
        N)r   )r   r   moder   r   r   r	      s    zFileSystem.openc             C   s   dS )N/r   )r   r   r   r   r!      s    zFileSystem.pathsep)F)F)T)NNNTF)r   )__name__
__module____qualname____doc__r   r   r   r    r   r   r#   r&   r$   r(   r)   r*   r+   r,   r4   r	   propertyr!   r   r   r   r   r   $   s&   



 
 
r   c                   s   e Zd ZdZ fddZedd Zedd Zee	j
dd	 Z
ee	jdddZee	jdd Zee	jdd Zee	jdd Zee	jdd Zee	jdddZedd Zdd Z  ZS )LocalFileSystemNc                s   t jttdd t   d S )N   )
stacklevel)warningswarn_FS_DEPR_MSGFutureWarningsuper__init__)r   )	__class__r   r   rD      s    zLocalFileSystem.__init__c          	   C   s6   | j d kr0t  td t | _ W d Q R X | j S )Nignore)	_instancer?   catch_warningssimplefilterr<   )clsr   r   r   _get_instance   s
    


zLocalFileSystem._get_instancec             C   s   t jttdd |  S )Nr=   )r>   )r?   r@   rA   rB   rK   )rJ   r   r   r   get_instance  s    zLocalFileSystem.get_instancec                s$   t   t fddt D S )Nc             3   s   | ]}t  |V  qd S )N)pjoin).0x)r   r   r   	<genexpr>
  s    z%LocalFileSystem.ls.<locals>.<genexpr>)r   sortedoslistdir)r   r   r   )r   r   r     s    zLocalFileSystem.lsTc             C   s&   t |}|rt| n
t| d S )N)r   rR   makedirsr(   )r   r   r'   r   r   r   r(     s    zLocalFileSystem.mkdirc             C   s   t |}tj|S )N)r   rR   r   r*   )r   r   r   r   r   r*     s    zLocalFileSystem.isdirc             C   s   t |}tj|S )N)r   rR   r   r+   )r   r   r   r   r   r+     s    zLocalFileSystem.isfilec             C   s   dS )NTr   )r   r   r   r   r,     s    zLocalFileSystem._isfilestorec             C   s   t |}tj|S )N)r   rR   r   r)   )r   r   r   r   r   r)   "  s    zLocalFileSystem.existsr   c             C   s   t |}t||dS )z3
        Open file for reading or writing.
        )r5   )r   r	   )r   r   r5   r   r   r   r	   '  s    zLocalFileSystem.openc             C   s   t jjS )N)rR   r   sep)r   r   r   r   r!   /  s    zLocalFileSystem.pathsepc             C   s   t |}t|S )z8
        Directory tree generator, see os.walk.
        )r   rR   r   )r   r   r   r   r   r   3  s    zLocalFileSystem.walk)T)r   )r7   r8   r9   rG   rD   classmethodrK   rL   r   r   r   r(   r*   r+   r,   r)   r	   r;   r!   r   __classcell__r   r   )rE   r   r<      s   r<   c               @   s   e Zd ZdZdd Zeejdd Zeejdd Zeej	dd	 Z	eej
dddZ
eejdd ZeejdddZeejdddZdddZdd ZdS )DaskFileSystemzJ
    Wraps s3fs Dask filesystem implementation like s3fs, gcsfs, etc.
    c             C   s   t jdtdd || _d S )Nz~The pyarrow.filesystem.DaskFileSystem/S3FSWrapper are deprecated as of pyarrow 3.0.0, and will be removed in a future version.r=   )r>   )r?   r@   rB   fs)r   rY   r   r   r   rD   @  s    
zDaskFileSystem.__init__c             C   s   t dd S )NzUnsupported file system API)r   )r   r   r   r   r   r*   G  s    zDaskFileSystem.isdirc             C   s   t dd S )NzUnsupported file system API)r   )r   r   r   r   r   r+   K  s    zDaskFileSystem.isfilec             C   s   dS )zh
        Object Stores like S3 and GCSFS are based on key lookups, not true
        file-paths.
        Fr   )r   r   r   r   r,   O  s    zDaskFileSystem._isfilestoreFc             C   s   t |}| jj||dS )N)r   )r   rY   r#   )r   r   r   r   r   r   r   W  s    zDaskFileSystem.deletec             C   s   t |}| j|S )N)r   rY   r)   )r   r   r   r   r   r)   \  s    zDaskFileSystem.existsTc             C   s(   t |}|r| j|S | j|S d S )N)r   rY   Zmkdirsr(   )r   r   r'   r   r   r   r(   a  s    zDaskFileSystem.mkdirr   c             C   s   t |}| jj||dS )z3
        Open file for reading or writing.
        )r5   )r   rY   r	   )r   r   r5   r   r   r   r	   i  s    zDaskFileSystem.openc             C   s   t |}| jj||dS )N)detail)r   rY   r   )r   r   rZ   r   r   r   r   q  s    zDaskFileSystem.lsc             C   s   t |}| j|S )z9
        Directory tree generator, like os.walk.
        )r   rY   r   )r   r   r   r   r   r   u  s    zDaskFileSystem.walkN)F)T)r   )F)r7   r8   r9   r:   rD   r   r   r*   r+   r,   r   r)   r(   r	   r   r   r   r   r   r   rX   ;  s   
rX   c               @   s:   e Zd Zeejdd Zeejdd Zd	ddZdS )
S3FSWrapperc             C   sV   t t|}y0| j|}t|dkr6|d |kr6dS dS W n tk
rP   dS X d S )N   r   FT)_sanitize_s3r   rY   r   lenOSError)r   r   contentsr   r   r   r*     s    zS3FSWrapper.isdirc             C   sJ   t t|}y$| j|}t|dko0|d |kS  tk
rD   dS X d S )Nr\   r   F)r]   r   rY   r   r^   r_   )r   r   r`   r   r   r   r+     s    zS3FSWrapper.isfileFc             #   s   t t|}t  t }xTt| jj||dD ]<}|d }|d dkrR | q.|d dkr`q.|| q.W t fdd|D }tdd  D  | |fV  x  D ]}| j||dE d	H  qW d	S )
z
        Directory tree generator, like os.walk.

        Generator version of what is in s3fs, which yields a flattened list of
        files.
        )refreshZKeyZStorageClassZ	DIRECTORYZBUCKETc                s"   g | ]}| krt |d  qS )r\   )	posixpathsplit)rN   r   )r   r   r   
<listcomp>  s    z$S3FSWrapper.walk.<locals>.<listcomp>c             S   s   g | ]}t |d  qS )r\   )rb   rc   )rN   rO   r   r   r   rd     s   N)	r]   r   setlistrY   Z_lsaddrQ   r   )r   r   ra   r   key	directoryr   )r   r   r     s     

zS3FSWrapper.walkN)F)r7   r8   r9   r   r   r*   r+   r   r   r   r   r   r[   }  s   	r[   c             C   s   |  dr| ddS | S d S )Nzs3:// )
startswithreplace)r   r   r   r   r]     s    
r]   c             C   sN   t | }t|tsFdtjkr6tjd }t| |jr6| S td|n| S d S )NfsspeczUnrecognized filesystem: {})	type
issubclassr   sysmodules
isinstanceZAbstractFileSystemr_   format)rY   Zfs_typerm   r   r   r   _ensure_filesystem  s    


rt   c       	      C   s,  t | s |dk	rtd|| fS |dk	rdt|}t|trDt| }nt| tsXtdn| }||fS t| }tj	
|}|jdks|jdkr|jd}|d }|dkrd	}n|jd
 | }d}t|dkr|d  rt|d }tjj||d}|j}n(|jdkrt }|j}nt }|}||fS )zn
    Return filesystem from path which could be an HDFS URI, a local URI,
    or a plain filesystem path.
    NzVfilesystem passed but where is file-like, so there is nothing to open with filesystem.zPExpected string path; path-like objects are only allowed with a local filesystemhdfsZviewfs:r   rj   defaultz://r=   r\   )hostportr   )r   
ValueErrorrt   rr   r<   r   str	TypeErrorurllibparseurlparseschemenetlocrc   r^   	isnumericintparu   _connectr   rK   )	wherer0   r   Z
parsed_uriZnetloc_splitrx   ry   rY   Zfs_pathr   r   r   resolve_filesystem_and_path  s@    


r   )N)rR   rb   rp   urllib.parser}   r?   os.pathr   rM   Zpyarrowr   Zpyarrow.utilr   r   r   r   rs   rA   r   r<   rX   r[   r]   rt   r   r   r   r   r   <module>   s"   
 OIB7