o i@sddlZddlZddlZddlZddlZddlmZmZddlm Z m Z m Z m Z m Z mZmZmZmZmZmZmZmZmZmZddlmZmZmZddlmZddlmZddl m!Z!m"Z"dd l#m$Z$m%Z%dd l&m'Z'm(Z(m)Z)m*Z*m+Z+erdd lm,Z,dd l&m-Z-dd l.m/Z/ddl0m1Z1ddl2m3Z3ddl4m5Z5ede6e7Z8de9de9dee9de e7e fdeddeee9e9fee9e9fff ddZ:de de9de9de e7e fde6f dd Z;Gd!d"d"e<Z=Gd#d$d$e<Z>Gd%d&d&ee8Z?dS)'N)BytesIOStringIO)AnyCallablecastDict GeneratorGenericIOIteratorListOptionaloverloadTupleTypeVarUnion TYPE_CHECKING) DecodeErrorHttpResponseErrorIncompleteReadError)with_current_context)!validate_and_format_range_headers)parse_length_from_content_rangeprocess_storage_error)deserialize_blob_propertiesget_page_ranges_result)adjust_blob_size_for_encryption decrypt_blob&get_adjusted_download_range_and_offsetis_encryption_v2parse_encryption_data)IncrementalDecoder)_EncryptionData)AzureBlobStorage)BlobOperations)BlobProperties)StorageConfigurationT start_range end_rangelengthencryption_optionsencryption_datar#returncCsBd\}}|ddus|ddurt||||S||f||ffS)N)rrkeyresolver)getr)r)r*r+r,r- start_offset end_offsetr4S/home/app/Keep/.python/lib/python3.10/site-packages/azure/storage/blob/_download.pyprocess_range_and_offset,sr6datar2r3 encryptionc Cs|durtddt|}|r|ddus|ddurMzt|dp'd|d|d||||jjWStyL}z td|j|d|d}~ww|S) NzResponse cannot be None.r/r0requiredFzDecryption failed.)messageresponseerror) ValueErrorjoinlistr1rr<headers Exceptionr)r7r2r3r8contentr=r4r4r5process_content>s&   rDc@sLeZdZ     d+dddededededed ed eeefd ed d edeedee eeefdee eeegdfdeddfddZ dede eeffddZ deeddffddZdeddfddZdede eeffddZdeddfdd Zd!ededdfd"d#Zd$ed%edefd&d'Zded(ede eeffd)d*ZdS),_ChunkDownloaderNclientr% total_size chunk_sizecurrent_progressr)r*validate_contentr,r-r#streamparallelnon_empty_ranges progress_hookkwargsr.cKs||_| |_||_||_||_||_| |_| rtnd|_ | r$tnd|_ | |_ | r0| nd|_ ||_||_| |_||_||_dSNr)rFrMrHrG start_index end_indexrK threadingLock stream_lock progress_lockrNtell stream_startprogress_totalr,r-rJrequest_options)selfrFrGrHrIr)r*rJr,r-rKrLrMrNrOr4r4r5__init__Us  z_ChunkDownloader.__init__ chunk_startcCs0||j|jkr|j}||fS||j}||fSN)rHrRr[r] chunk_endr4r4r5_calculate_ranges  z!_ChunkDownloader._calculate_rangeccs4|j}||jkr|V||j7}||jks dSdSr^)rQrRrH)r[indexr4r4r5get_chunk_offsetss   z"_ChunkDownloader.get_chunk_offsetscCsP||\}}|||d\}}||}|dkr&|||||dSdS)Nrr)ra_download_chunk_write_to_stream_update_progress)r[r]r` chunk_data_r+r4r4r5 process_chunks z_ChunkDownloader.process_chunkcCs||\}}|||dS)Nr)rardr_r4r4r5 yield_chunksz_ChunkDownloader.yield_chunkr+cCsh|jr|j|j|7_Wdn1swYn|j|7_|jr2||j|jdSdSr^)rVrYrNrG)r[r+r4r4r5rfsz!_ChunkDownloader._update_progressrgcCsf|jr+|j|j|j||j|j|WddS1s$wYdS|j|dSr^)rUrKseekrXrQwrite)r[rgr]r4r4r5res "z!_ChunkDownloader._write_to_streamgiven_range_startgiven_range_endcCsB|jdurdS|jD]}||dkrdS|d|krq dSdS)NFstartTend)rM)r[rmrnZ source_ranger4r4r5 _do_optimizes    z_ChunkDownloader._do_optimizer`c Cs|jdur tdt||||j|j\}}||d|dr1|d|dd}d|}||fSt|d|d|jd\}}d} d} | rd} z|jjd |||j|j |j d|j \} } Wnt yt} z t | WYd} ~ nd} ~ wwzt| |d|d|j}d } Wn)tt tfy} z| d8} | dkrt | | d | tdWYd} ~ nd} ~ ww| sE| j}|j d r| jj|j d _||fS) Nz0Required argument is missing: encryption_optionsrr)check_content_md5TrangeZrange_get_content_md5rJdata_stream_totaldownload_stream_currentFr=modified_access_conditionsr4)r,r>r6r-rqrrJrFdownloadrGrYrZrrrDrrtimesleepcontent_lengthr1 propertiesetagif_match)r[r]r`Zdownload_rangeoffsetr~rg range_headerrange_validation retry_active retry_totalr<rhr=r4r4r5rds^ '  z _ChunkDownloader._download_chunk)NNNNN)__name__ __module__ __qualname__intboolrstrrr r rr\rrarrcribytesrjrfrerqrdr4r4r4r5rETsZ       / "rEc @sreZdZdZdededeededdf dd Zdefd d Z de efd d Z defddZ e Z defddZdS)_ChunkIteratorz,Iterator for chunks in blob download stream.sizerC downloaderrHr.NcCs,||_||_||_||_d|_|dk|_dSrP)r _chunk_size_current_content_iter_downloader _iter_chunks _complete)r[rrCrrHr4r4r5r\s z_ChunkIterator.__init__cC|jSr^rr[r4r4r5__len__z_ChunkIterator.__len__cCs|Sr^r4rr4r4r5__iter__ sz_ChunkIterator.__iter__c Cs|jrtd|jst|j|jkr|Sd|_|jS|js%|j|_t|j|jkr1|Szt |j}|j|j |d7_W|Stye}zd|_|jr_|jWYd}~S|d}~ww)NzDownload completeTr) r StopIterationrlenrr_get_chunk_datarrcnextrj)r[ next_chunker4r4r5__next__s,   z_ChunkIterator.__next__cCs&|jd|j}|j|jd|_|Sr^)rr)r[rgr4r4r5r/sz_ChunkIterator._get_chunk_data)rrr__doc__rrr rEr\rr rrrrr4r4r4r5rs"rc@seZdZUdZeed< eed< ded< eed<           d9d d d d d eedeedede ee fdedededeedee de ddfddZ ddZ d:ddZeddZddZdeefd d!Zed;dedefd#d$Zedd%d&eedefd'd$Zd;dd%ded&eedefd(d$Zdefd)d*Zd+eedefd,d-Zd.d/Zd0d1ZdStorageStreamDownloaderz< A streaming object to download from Azure Storage. name containerr&rrNrclientsr$configr'r)r*rJr,max_concurrencyencoding download_clsrOr.c Ks||_| |_d|_||_||_||_||_||_| |_||_ |p!i|_ | dd|_ | |_ d|_d|_d|_d|_d|_d|_d|_d|_d|_d|_d|_d|_d|_|jpXd|_| |j d<|j dduso|j ddurs||j sz|jjn|jj} |j}|jdur|j|| kr|j}n|| d}t |||j|j |j\|_!|_"|#|_t$d |jj%|_%|j|j%_|j|j%_|j|j%_d |jd |jdur|jn|jdd |j|j%_&d|j%_'dS) NrrNr9Tclsr/r0rr&zbytes -/)(rrr_clients_config _start_range _end_range_max_concurrency _encoding_validate_content_encryption_optionspop_progress_hook_request_options _response_location_moder _file_size_non_empty_ranges_encryption_data_download_offset_raw_download_offset _read_offset_current_content_offset _text_mode_decoder _first_chunk_download_startr1_get_encryption_data_requestZmax_single_get_sizemax_chunk_get_sizer6_initial_range_initial_offset_initial_requestrr content_rangeZ content_md5)r[rrr)r*rJr,rrrrrrOZfirst_get_sizeZinitial_request_startZinitial_request_endr4r4r5r\Esl           z StorageStreamDownloader.__init__cCrr^rrr4r4r5rrzStorageStreamDownloader.__len__cCsr|jdd}|jdd}t|jd<td|jjjdi|j}t|j|_ ||jd<|dur7||jd<dSdS)Nr decompressr&r4) rrrrrblobZget_propertiesr!metadatar)r[rrrr4r4r5rs   z4StorageStreamDownloader._get_encryption_data_requestcCs"t|jr |j|jkS|j|jkSr^)r rrrrrr4r4r5_download_completes   z*StorageStreamDownloader._download_completec Cst|jd|jddd|jd\}}d}d}|r zftttttf|jj j d|||jddd|j \}}||_ t |jj|_|jdurJtdt|j|j|_|jdurm|jdurmt|j|j|j|jd|_n|jdurz|j|j|_n|j|_WnVty}zJ|jdur|jr|jjd krz|jj j d|jddd |j \}}Wnty} z t| WYd} ~ nd} ~ wwd|_d|_nt|WYd}~nd}~wwz|jdkrd |_nt||jd|jd|j|_d}Wn+t tt!fy}z|d8}|dkrt||d |t"#dWYd}~nd}~ww|s|j$t%|j7_$|j&|j'7_&|jj(d krSz|jj)*} t+| d|_,Wn tyRYnw|j-sf|j .drf|jj/|j d_0|S)NrrF)Zstart_range_requiredZend_range_requiredrsTrtruz?Required Content-Range response header is missing or malformed.i)rJrwrxr9ryZPageBlobrzr4)1rrrrrr rrrrr{rrrrrrr>rrrrminrrr< status_coderrrDrrrrr|r}rrrr~Z blob_typeZ page_blobZget_page_rangesrrrr1rr) r[rrrrZ location_moder<r=rhrZ page_rangesr4r4r5rs    "    D z(StorageStreamDownloader._initial_requestcCs|jrtd|jrtdd}|jr|jsQ|jr(|jt|j }t|j }n|j}d}|j|j }t d|j j |j|j |jj||||j|j|j|jd |j}|jrW|j nd}t|j tt|||jjdS) a Iterate over chunks in the download stream. Note, the iterator returned will iterate over the entire download content, regardless of any data that was previously read. NOTE: If the stream has been partially read, some data may be re-downloaded by the iterator. :return: An iterator of the chunks in the download stream. :rtype: Iterator[bytes] .. admonition:: Example: .. literalinclude:: ../samples/blob_samples_hello_world.py :start-after: [START download_a_blob_in_chunk] :end-before: [END download_a_blob_in_chunk] :language: python :dedent: 12 :caption: Download a blob using chunks(). zRStream has been partially read in text mode. chunks is not supported in text mode.zrwarningswarnrrrrrrrErrrrrrrrrrrrr)r[Ziter_downloaderrorIrpZinitial_contentr4r4r5chunks&sD     zStorageStreamDownloader.chunkscCdSr^r4)r[rr4r4r5readazStorageStreamDownloader.read)charsrcCrr^r4)r[rr4r4r5rercCs*|dkr |jr td|dkr|durtd|js#|dur#td|jr.|dkr.td|jdur;|dur;td|d ksN|d ksN|jrU|jt|jkrU|jsSd Sd S|js||dur||jdur|d |_t |jd |_ |j j t t|j|jd|_n|jdurd|_|jrt}|dus|d krtjn|}n t}|d kr|ntj}|tjk}d }|j}tt|j|j||}||j|||}||7}|j|7_|j|7_|||} | d kr|js|j|j}|j|j} |jdk} td|jj|j|j|jj |j|| || |j!|j"|j#|j$|j%d|j&} d|_'| (} |rt|jst| rd|j|j|jj krdd dl)}|j*+|j}t,|-t.| j/| (Wdn 1s^wYn | D]}| /|qf|0nxt1| d}dur| d kr| 2|\}}|jt|7_|j3|7_3|jr|j dur|j j ||jd|_n||_| t|jkr||jd| }n||j}||_|j|7_| |8} |t1| d}dur| d ks|4}|js|jrz t t| |j}W|St5ytdw|S)a Read the specified bytes or chars from the stream. If `encoding` was specified on `download_blob`, it is recommended to use the chars parameter to read a specific number of chars to avoid decoding errors. If size/chars is unspecified or negative all bytes will be read. :param int size: The number of bytes to download from the stream. Leave unspecified or set negative to download all bytes. :keyword Optional[int] chars: The number of chars to download from the stream. Leave unspecified or set negative to download all chars. Note, this can only be used when encoding is specified on `download_blob`. :return: The requested data as bytes or a string if encoding was specified. If the return value is empty, there is no more data to read. :rtype: T rzSize parameter specified with text encoding enabled. It is recommended to use chars to read a specific number of characters instead.Nz#Cannot specify both size and chars.z$Must specify encoding to read chars.z>Stream has been partially read in text mode. Please use chars.Fz>Stream has been partially read in bytes mode. Please use size.rr9Tstrict)finalrrFrMrGrHrIr)r*rKrLrJr,r-rrNzEncountered a decoding error while decoding blob data from a partial read. Try using the `chars` keyword instead to read in text mode.r4)6rrrr>rrrrrcodecsgetincrementaldecoderrdecoderrrsysmaxsizerrrlr_check_and_report_progressrrrrrErrrrrrrrrrrrrcconcurrent.futuresfuturesThreadPoolExecutorr@maprri_complete_readrrjrgetvalueUnicodeDecodeError)r[rrZ output_streamreadallcountror+r remainingrprLrZ chunks_iter concurrentexecutorrchunkrgr~r7r4r4r5rjs       cCs|S)z Read the entire contents of this blob. This operation is blocking until all data is downloaded. :return: The requested data as bytes or a string if encoding was specified. :rtype: T )rrr4r4r5rszStorageStreamDownloader.readallrKcCs|jrtd|jrtd|jdk}|rBd}tjdkr%|s%t|z | | Wnt t fyA}zt||d}~ww|j |j}|dkrNdSt|j|j}|j}|tt|j|||}|j|7_|j|7_|jr||j|j |jr|S|j|j} |j|j } td |jj|j|j |jj|j| | |||j|j|j |j!|jd|j"} |rddl#} | j$%|j} t&| 't(| j)| *Wdn1swYn | *D]}| )|q|+|S) aeDownload the contents of this file to a stream. :param IO[bytes] stream: The stream to download to. This can be an open file-handle, or any writable stream. The stream must be seekable if the download uses more than one parallel connection. :return: The number of bytes read. :rtype: int zTStream has been partially read in text mode. readinto is not supported in text mode.zEEncoding is ignored with readinto as only byte streams are supported.rz&Target stream handle must be seekable.)rtNrrr4),rr>rrrrr version_infoseekablerkrWNotImplementedErrorAttributeErrorrrrrrrlrrrrrrErrrrrrrrrrrrrr@rrrircr)r[rKrL error_messageexcZremaining_sizeZcurrent_remainingrorZ data_startZdata_endrrrrr4r4r5readintosx         z StorageStreamDownloader.readintocCs(|j|_|j|_|j|_t|j|_dS)z/Adjusts all offsets to the end of the download.N)rrrrrrrrr4r4r5rVsz&StorageStreamDownloader._complete_readcCs2|jr|jt|jkr||j|jdSdSdS)zReports progress if necessary.N)rrrrrrrr4r4r5r]sz2StorageStreamDownloader._check_and_report_progresscCs(tdt|jr td||_|S)apDEPRECATED: Download the contents of this file. This operation is blocking until all data is downloaded. This method is deprecated, use func:`readall` instead. :param int max_concurrency: The number of parallel connections with which to download. :return: The contents of the file as bytes. :rtype: bytes z3content_as_bytes is deprecated, use readall insteadz\Stream has been partially read in text mode. content_as_bytes is not supported in text mode.)rrDeprecationWarningrr>rr)r[rr4r4r5content_as_bytesds z(StorageStreamDownloader.content_as_bytesUTF-8cCs.tdt|jr td||_||_|S)aDEPRECATED: Download the contents of this blob, and decode as text. This operation is blocking until all data is downloaded. This method is deprecated, use func:`readall` instead. :param int max_concurrency: The number of parallel connections with which to download. :param str encoding: Test encoding to decode the downloaded bytes. Default is UTF-8. :return: The content of the file as a str. :rtype: str z2content_as_text is deprecated, use readall insteadz[Stream has been partially read in text mode. content_as_text is not supported in text mode.)rrrrr>rrr)r[rrr4r4r5content_as_text{sz'StorageStreamDownloader.content_as_textcCs0tdt|jr td||_|||jS)a&DEPRECATED: Download the contents of this blob to a stream. This method is deprecated, use func:`readinto` instead. :param IO[T] stream: The stream to download to. This can be an open file-handle, or any writable stream. The stream must be seekable if the download uses more than one parallel connection. :param int max_concurrency: The number of parallel connections with which to download. :return: The properties of the downloaded blob. :rtype: Any z6download_to_stream is deprecated, use readinto insteadz^Stream has been partially read in text mode. download_to_stream is not supported in text mode.)rrrrr>rrr)r[rKrr4r4r5download_to_streams z*StorageStreamDownloader.download_to_stream) NNNNNNrNNNN)r.N)r)r)rr) rrrrr__annotations__rr rrrrr\rrpropertyrrr rrrr(rrr rrrrrrr4r4r4r5r5s        ^  c;" Q  r)@rrrSr|riorrtypingrrrrrr r r r r rrrrrZazure.core.exceptionsrrrZazure.core.tracing.commonrZ_shared.request_handlersrZ_shared.response_handlersrrZ _deserializerrZ _encryptionrrrr r!r"r#Z _generatedr$Z_generated.operationsr%Z_modelsr&Z_shared.modelsr'rrr(rr6rDobjectrErrr4r4r4r5sND           &*8