
    ciY                       d Z ddlmZ ddlmZmZ ddlmZmZm	Z	m
Z
mZmZ ddlmZ ddlmZmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ  ee          Z eddg          Ze                    d          d[d            Z e                    d           edd           eddd          fd\d             Z!e"                    d!d"#          d]d&            Z#e"                    d'd"#           ed(d)*          fd^d-            Z$e"                    d.d"#          d[d/            Z%e                    d0           edd           eddd           ed12           ed12           ed12           ed12           ed12           ed12           ed12           ed12           ed12           ed12           ed12          fd_d?            Z&e                    d@          d[dA            Z'e(                    dB           edCdDdE          fd`dG            Z)e                    dH          dadI            Z*e                    dJ           edd           edddK           ed12           ed12           ed12          fdbdN            Z+e                    dO          dcdQ            Z,e                    dR           edd           edSdd"           ed12           ed12          fdddU            Z-e                    dV           edW2          fdedZ            Z.d1S )fu   Admin / system-status endpoints.

管理后台接口模块。
提供系统健康状态统计、入库日志分页查询、文档删除（含 ES + Neo4j），
以及入库全链路追踪（Trace / Event / Artifact）的查询接口。
    )annotations)	AnnotatedAny)	APIRouterBodyDependsHTTPExceptionQueryRequest)JSONResponse)UserContextget_current_user)settings)ESClient)Neo4jClient)RedisClient)
get_loggerz/adminadmin)prefixtagsz/statsuser1Annotated[UserContext, Depends(get_current_user)]requestr   returndict[str, Any]c           
     z  	
K   |j         j        j        |j         j        j        
|j         j        j        ddlddl	dddddddddd	dfd}d
fd}dfd	}dfd
}d	fd}                     |             |             |             |             |                       d{V  S )u   Return system health and document statistics.

    聚合 ES、Neo4j、Redis 三大组件的连通状态和文档/分片/图谱统计。
    r   Nunknown)	total_documentstotal_chunksgraph_nodestoday_ingested	es_statusneo4j_statusredis_statuscelery_statusconverter_statusr   Nonec                   K   	                      j                            t          j                  j                            t          j                  j                            t          j        ddddddiii                     d {V \  } }}t          | t                    r| n| j        }t          |t                    r|n|j        }t          |t                    r|n|j        }|	                    d	d
          	d<   |	                    d	d
          	d<   |	                    d	d
          	d<   d	d<   d S # t          $ r:}t                              dt          |          |           d	d<   Y d }~d S d }~ww xY w)N)indexqueryrange
created_atznow/dznow+1d/d)gteltr)   bodycountr   r   r   r!   healthyr"   admin_es_errorerrorexc_infor5   )gatherrawr1   r   es_meta_indexes_chunk_index
isinstancedictr0   get	Exceptionloggerr5   str)

meta_countchunk_count
today_respraw_meta	raw_chunk	raw_todayeasyncio	es_clientresults
          *D:\work\zm-rag\backend\app\api\v1\admin.pycheck_eszsystem_stats.<locals>.check_es9   s     	*8?##(*@#AA##(*A#BB##"0!GlGS]<^<^-_#`a $  9 9 3 3 3 3 3 3/JZ &0
D%A%AVzzzH'1+t'D'DZ+JZI&0T&B&BW


I(0Wa(@(@F$%%.]]7A%>%>F>"'0}}Wa'@'@F#$"+F; 	* 	* 	*LL)Q!LDDD")F;	*s   D>E 
F	/FF	c                    K   	                                   d {V d<   dd<   d S # t          $ r9} t                              dt	          |                      dd<   Y d } ~ d S d } ~ ww xY w)Nr    r2   r#   admin_neo4j_errorr5   r5   )get_node_countr>   r?   warningr@   )rG   neo4j_clientrJ   s    rK   check_neo4jz!system_stats.<locals>.check_neo4jN   s      	-*6*E*E*G*G$G$G$G$G$G$GF=!%.F>""" 	- 	- 	-NN.c!ffN===%,F>"""""""	-s   ") 
A,.A''A,c                    K   	                                   d {V rdndd<   d S # t          $ r9} t                              dt	          |                      dd<   Y d } ~ d S d } ~ ww xY w)Nr2   r5   r$   admin_redis_errorrO   )pingr>   r?   rQ   r@   )rG   redis_clientrJ   s    rK   check_redisz!system_stats.<locals>.check_redisV   s      	-8D8I8I8K8K2K2K2K2K2K2K%XYYQXF>""" 	- 	- 	-NN.c!ffN===%,F>"""""""	-s   !( 
A+.A&&A+c                   K   	 ddl m                                                     d fd           d {V } | rdndd<   d S # t          $ r9}t
                              dt          |                     dd<   Y d }~d S d }~ww xY w)	Nr   
celery_appc                 ^     j                             d                                          S )N   timeout)controlinspectrV   rZ   s   rK   <lambda>z4system_stats.<locals>.check_celery.<locals>.<lambda>a   s'    j0888CCHHJJ     r2   r5   r%   admin_celery_errorrO   )app.tasks.celery_appr[   get_event_looprun_in_executorr>   r?   rQ   r@   )ping_resultrG   r[   rH   rJ   s     @rK   check_celeryz"system_stats.<locals>.check_celery]   s      	.777777 ' 6 6 8 8 H HJJJJ! !      K 4?&KiiGF?### 	. 	. 	.NN/s1vvN>>>&-F?#######	.s   A A 
B.BBc                   K   	                      d          4 d {V } |                     t          j         d           d {V }|j        dk    rdndd<   d d d           d {V  d S # 1 d {V swxY w Y   d S # t
          $ r9}t                              dt          |          	           dd<   Y d }~d S d }~ww xY w)
N   r^   z/actuator/health   r2   r5   r&   admin_converter_errorrO   )	AsyncClientr=   r   converter_base_urlstatus_coder>   r?   rQ   r@   )clientresprG   httpxrJ   s      rK   check_converterz%system_stats.<locals>.check_converterh   s     	1(((33 _ _ _ _ _ _ _v#ZZ8+F(X(X(XYYYYYYYY:>:Jc:Q:QYYW^)*_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _  	1 	1 	1NN2#a&&NAAA)0F%&&&&&&&	1s:   B ;A/B /
A99B <A9=B 
C.C  C)r   r'   )appstaterI   rR   rW   rH   rs   r7   )r   r   rL   rS   rX   ri   rt   rH   rI   rs   rR   rW   rJ   s          @@@@@@rK   system_statsrw      s      "++5I ' 1 >L ' 1 >LNNNLLL !!"%
 
F* * * * * * * **- - - - - - -- - - - - - -	. 	. 	. 	. 	. 	. 	.1 1 1 1 1 1 1 ..

KKMM;;==))        
 Mrc   z/ingest-logs   )defaultge   d   )ry   rz   lepageint	page_sizec           	     4  K   |j         j        j        }|dz
  |z  }	 |j                            t
          j        di idddiig||g dd           d	{V }t          |t                    r|n|j	        }|
                    d
i           }|
                    di           
                    dd          }	d |
                    d
g           D             }
|	|||
dS # t          $ r0}t                              d           t          dd          d	}~ww xY w)u   Return recent document ingest records from the meta index.

    从 ES meta 索引按创建时间倒序分页返回入库记录，用于管理后台入库日志页面。
    rx   	match_allr,   orderdescdoc_idtitleoriginal_filename
doc_numberissuing_orgdoc_typestatusrB   	file_path	file_typer,   r5   task_idacl_ids)r*   sortfromsize_sourcer/   Nhitstotalvaluer   c                   g | ]}|d                               d|d                   |d                               dd          |d                               dd          |d                               dd          |d                               dd          |d                               d	d          |d                               d
d          |d                               dd          |d                               dd          |d                               dd          |d                               dd          |d                               d          |d                               d          |d                               dg           dS )r   r   _idr   u   —r    r   r   r   r   	completedrB   r   r   r   r,   r5   r   r   r   )r=   ).0hs     rK   
<listcomp>zingest_logs.<locals>.<listcomp>   st    
 
 
"  I,**8QuX>>9))'599%&y\%5%56I2%N%N	l..|R@@ |//rBBiL,,Z<<I,**8[AA |//qAAy\--k2>>y\--k2>>	l..|R@@9))'22Y<++I66Y<++Ir:: 
 
 
rc   )r   r~   r   recordsadmin_ingest_logs_error  Internal server errorrp   detail)ru   rv   rI   r8   searchr   r9   r;   r<   r0   r=   r>   r?   	exceptionr	   )r   r   r~   r   rI   from_idxrr   r8   r   r   r   rG   s               rK   ingest_logsr   z   s      "++5IqI%H,M]))(%r*&&(9:; !    * 
 
 
 
 
 
 
 
" !t,,;dd$)wwvr"""%%))'155
 
" XXfb))#
 
 
& 9QXYYY M M M23334KLLLLMs   B?C 
D'+DDz/document/{doc_id}rl   )rp   r   r@   c                   K   |j         j        j        }|j         j        j        }	 |                    |            d{V }n@# t
          $ r3}t                              d|            t          dd          |d}~ww xY w|d         dk    r9|d	         s1|	                    d
d          dk    rt          ddd|  di          S ddi}	 |                    |            d{V }nA# t
          $ r4}t                              d| t          |                     Y d}~nd}~ww xY wt          j        	 d| |j        d|| | |d         t!          |d	                   |	                    d
d          |d         dS )u  Permanently delete a document from ES indices and Neo4j graph.

    Steps executed in order:
    1. Delete all chunks from ``gov_doc_chunks`` (ES delete_by_query).
    2. Delete the meta record from ``gov_doc_meta`` (ES delete by ID).
    3. Delete the Document node and all its relationships from Neo4j
       (DETACH DELETE — non-blocking: failure is logged but not raised).

    Returns a summary of what was deleted.
    Nadmin_delete_es_error)r   r   r   r   deleted_chunksr   deleted_metadeleted_guides  r   z
Document 'z' not found in search index.)rp   contentdeleted_nodesadmin_delete_neo4j_errorr   r5   admin_document_deleted)r   operator)r   r   r   r   deleted_graph_nodes)r   )ru   rv   rI   rR   delete_documentr>   r?   r   r	   r=   r   delete_document_graphrQ   r@   infouser_idbool)r   r   r   rI   rR   	es_resultexcneo4j_results           rK   r   r      s       "++5I ' 1 >LV#33F;;;;;;;;		 V V V0@@@4KLLLRUUV 	"#q((.) 	)MM*A..!33 PFPPPQ
 
 
 	
 %4Q#7LR)??GGGGGGGG R R R1&CQQQQQQQQR K   	
    #$45Y~677#--(8!<<+O<  s/   A 
A?.A::A?C( (
D&2*D!!D&z/documents/batch.T)embeddoc_ids	list[str]c           	       K   |st          dd          t          |          dk    rt          dd          |j        j        j        }|j        j        j        }d}d}d}d}g }	|D ]'}
	 |                    |
           d{V }||d         z  }|t          t          |d	                             z  }||	                    d
d          z  }nV# t          $ rI}t                              d|
t          |                     |	                    |
           Y d}~d}~ww xY w	 |                    |
           d{V }||d         z  }# t          $ r5}t                              d|
t          |                     Y d}~!d}~ww xY wt                              d| j        t          |          |t          |	                     t          |          |||||	dS )u.   批量删除选中的文档（ES + Neo4j）。i  u   doc_ids 不能为空r   r   u   单次最多删除 500 条r   Nr   r   r   batch_delete_es_errorr   r   batch_delete_neo4j_erroradmin_documents_batch_deleted)r   	requestedr   errors)r   r   r   r   r   r   )r	   lenru   rv   rI   rR   r   r   r   r=   r>   r?   rQ   r@   appendr   r   r   )r   r   r   rI   rR   r   
total_metatotal_guidestotal_graphr   r   r   r   r   s                 rK   delete_documents_batchr      sN       L4JKKKK
7||c4PQQQQ!++5I ' 1 >LLJLKF V V	'77????????II&677L#d9^#<==>>>JIMM*:A>>>LL 	 	 	NN26SNRRRMM&!!!HHHH	
	V!-!C!CF!K!KKKKKKKL<88KK 	V 	V 	VNN5fCPSHHNUUUUUUUU	V KK'g,,6{{     \\"&&*  s1   ,A$C
D$?DD$(&E
F*F		Fz/documents/allc                @  K   |j         j        j        }|j         j        j        }d}d}d}d}t          j        dft          j        dft          j        dffD ]\  }}		 |j        	                    |ddi iid           d	{V }
t          |
t                    r|
n|
j        }|                    d
d          }|	dk    r|}n|	dk    r|}n|}v# t          $ r7}t                              d|	 dt#          |                     Y d	}~d	}~ww xY w	 |                                 d	{V }|d         }n@# t          $ r3}t                              dt#          |                     Y d	}~nd	}~ww xY wt                              d| j        ||||           ||||dS )u?   删除全部文档数据（ES 三个索引 + Neo4j 全图）。r   metachunksguidesr*   r   T)r)   r0   refreshNdeleteddelete_all__errorrO   r   delete_all_neo4j_erroradmin_all_documents_deleted)r   r   r   r   r   )r   r   r   r   )ru   rv   rI   rR   r   r9   r:   es_service_guide_indexr8   delete_by_queryr;   r<   r0   r=   r>   r?   rQ   r@   delete_all_graphr   r   )r   r   rI   rR   r   r   r   deleted_graph
index_namekeyrr   r8   r1   r   r   s                  rK   delete_all_documentsr   ,  sD      "++5I ' 1 >LLNNM 
	(		 (+		((3 F F
C
	F"66 R01 7        D
 %T400?$$diCGGIq))Ef}}$!&!& 	F 	F 	FNN4444CHHNEEEEEEEE	FA)::<<<<<<<<$_5 A A A/s3xx@@@@@@@@A KK%!%%)     %((,	  s1   A/C
D-DD"D4 4
E1>)E,,E1z/ingest-tracesN)ry   
str | Nonetrace_idr   r   current_stager   source_type	has_errorbool | None
start_timeend_timekeywordc                   K   |j         j        j        }	 |                    ||||||||	|
||||           d{V S # t          $ r0}t
                              d           t          dd          d}~ww xY w)u   Query ingest traces with filters and pagination.

    多条件筛选入库追踪记录，支持按文档 ID、状态、阶段、文件类型等过滤。
    )r~   r   r   r   r   r   r   r   r   r   r   r   r   Nadmin_ingest_traces_errorr   r   r   )ru   rv   rI   query_ingest_tracesr>   r?   r   r	   )r   r   r~   r   r   r   r   r   r   r   r   r   r   r   r   rI   rG   s                    rK   list_ingest_tracesr   l  s      , "++5IM22'#! 3 
 
 
 
 
 
 
 
 	
  M M M45554KLLLLMs   '= 
A7+A22A7z/ingest-traces/statsc                   K   |j         j        j        }	 |                                 d{V S # t          $ r<}t
                              dt          |          |           ddddddcY d}~S d}~ww xY w)u   Get aggregate statistics for ingest traces.

    返回入库追踪的聚合统计：今日总数、运行中、已完成、失败数及平均耗时。
    Nadmin_trace_stats_errorr4   r   )today_totalrunningr   failedavg_duration_ms)ru   rv   rI   get_trace_statsr>   r?   r5   r@   )r   r   rI   rG   s       rK   ingest_trace_statsr     s       "++5I

..000000000 
 
 
.c!ffqIII 
 
 	
 	
 	
 	
 	
 	

s   / 
A51A0*A50A5z/ingest-traces/cleanup-stale      i  stale_minutesc                  K   ddl m }m}m} |j        j        j        }|                    |j                   ||          z
                                  }	 |j	        
                    t          j        dddddiid	d
d|iiigiidg dd           d{V }t          |t                    r|n|j        }	|	                    di                               dg           }
|                    |j                                                  }d}|
D ]}|d         }|d                             d
d          }	 |                    |          }t%          |                    |j                  |z
                                  dz            }n# t(          $ r d}Y nw xY w|j	                            t          j        |dddd| d|||di           d{V  |dz  }t,                              d||           |||d S # t(          $ r0}t,                              d!           t3          d"d#$          d}~ww xY w)%u	  将超时仍为 running 的 trace 标记为 failed，防止 worker 崩溃导致 trace 永远卡住。

    Mark traces stuck in 'running' beyond stale_minutes as 'failed'.
    This handles cases where the worker crashed/was killed without calling finish_trace.
    r   )datetimetimezone	timedelta)minutesr   filtertermr   r   r+   
started_atr.   rl   )r   r   r   )r*   r   r   r/   Nr   r   r   r   i  docr   INGEST_STALE_TIMEOUTu   Trace 超过 u;    分钟未完成，标记为失败 (worker 可能已崩溃))r   
error_codeerror_messagefinished_atduration_ms
updated_at)r)   idr0   rx   stale_trace_cleaned)r   r   )cleanedcutoffr   cleanup_stale_traces_errorr   r   r   )r   r   r   ru   rv   rI   nowutc	isoformatr8   r   r   es_trace_indexr;   r<   r0   r=   fromisoformatr   total_secondsr>   updater?   r   r   r	   )r   r   r   r   r   r   rI   r  rr   r8   r   now_isor  hitr   r   startedr  rG   s                      rK   cleanup_stale_tracesr    s
      7666666666!++5Ill8<((99]+K+K+KKVVXXF5M])))  #h	%:;$|dF^&DE# FFF  * 
 
 
 
 
 
 
 
 !t,,;dd$)wwvr""&&vr22,,x|,,6688 	_ 	_C5zHY++L"==J "00<<!\\(,//'9HHJJTQ          -&&-"*&< *D  *D  *D  *D'.'2&- 	 '          qLGKK-P]K^^^^"f}UUU M M M56664KLLLLMs?   C H 9AFH FH FA%H 
H=+H88H=z/ingest-traces/{trace_id}c                   K   |j         j        j        }|                    |            d{V }|st	          dd|  d          |S )zGet a single trace detail.Nr   zTrace '' not foundr   )ru   rv   rI   get_ingest_tracer	   )r   r   r   rI   traces        rK   r  r    g       "++5I,,X66666666E U4Sh4S4S4STTTTLrc   z /ingest-traces/{trace_id}/eventsr   stage
event_typec                   K   |j         j        j        }	 |                    | |||||           d{V S # t          $ r0}	t
                              d           t          dd          d}	~	ww xY w)z)Query events for a trace, ordered by seq.)r~   r   r  r   r  Nadmin_ingest_events_errorr   r   r   )ru   rv   rI   query_ingest_eventsr>   r?   r   r	   )
r   r   r   r~   r   r  r   r  rI   rG   s
             rK   list_ingest_eventsr    s       "++5IM22! 3 
 
 
 
 
 
 
 
 	
  M M M45554KLLLLMs    6 
A0 +A++A0z/ingest-events/{event_id}event_idc                   K   |j         j        j        }|                    |            d{V }|st	          dd|  d          |S )z1Get a single event detail with full details JSON.Nr   zEvent 'r  r   )ru   rv   rI   get_ingest_eventr	   )r   r   r   rI   events        rK   r"  r"  !  r  rc   z#/ingest-traces/{trace_id}/artifacts2   artifact_typec                   K   |j         j        j        }	 |                    | ||||           d{V S # t          $ r0}t
                              d           t          dd          d}~ww xY w)zQuery artifacts for a trace.)r~   r   r%  r  Nadmin_ingest_artifacts_errorr   r   r   )ru   rv   rI   query_ingest_artifactsr>   r?   r   r	   )	r   r   r   r~   r   r%  r  rI   rG   s	            rK   list_ingest_artifactsr)  3  s       "++5IM55' 6 
 
 
 
 
 
 
 
 	
  M M M78884KLLLLMs   5 
A/+A**A/z/ingest-artifacts/{artifact_id}previewartifact_idmodec                   K   |j         j        j        }|                    |            d{V }|st	          dd|  d          |dk    r(|                    d          r|d         dd         |d<   |S )	z0Get a single artifact detail. mode=preview|full.Nr   z
Artifact 'r  r   r*  payload_texti  )ru   rv   rI   get_ingest_artifactr	   r=   )r+  r   r   r,  rI   artifacts         rK   r/  r/  M  s       "++5I22;????????H [4Y4Y4Y4YZZZZyX\\.99#+N#;ETE#B Orc   )r   r   r   r   r   r   )
r   r   r   r   r~   r   r   r   r   r   )r   r@   r   r   r   r   r   r   )r   r   r   r   r   r   r   r   ) r   r   r   r   r~   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   )r   r   r   r   r   r   r   r   )r   r@   r   r   r   r   r   r   )r   r@   r   r   r   r   r~   r   r   r   r  r   r   r   r  r   r   r   )r   r@   r   r   r   r   r   r   )r   r@   r   r   r   r   r~   r   r   r   r%  r   r  r   r   r   )
r+  r@   r   r   r   r   r,  r@   r   r   )/__doc__
__future__r   typingr   r   fastapir   r   r   r	   r
   r   fastapi.responsesr   app.api.depsr   r   
app.configr   app.infrastructure.es_clientr   app.infrastructure.neo4j_clientr   app.infrastructure.redis_clientr   app.utils.loggerr   __name__r?   routerr=   rw   r   deleter   r   r   r   r   postr  r  r  r"  r)  r/   rc   rK   <module>rA     s    # " " " " " ! ! ! ! ! ! ! ! K K K K K K K K K K K K K K K K * * * * * * 6 6 6 6 6 6 6 6       1 1 1 1 1 1 7 7 7 7 7 7 7 7 7 7 7 7 ' ' ' ' ' '	H			('	3	3	3 H[ [ [ [| N aA&&&U2!444	9M 9M 9M 9M 9Mx #55: : : 65:z !s33 c...3 3 3 3 433l S118 8 8 218~  aA&&&U2!444t,,, 5...%---t,,, %d 3 3 3!E$///#eD111"U4000"U4000 5...%---)M )M )M )M )MX "##
 
 
 $#
, +,, raD999CM CM CM CM -,CML '((
 
 
 )(
 .//
 aA&&&U31555d+++t,,,"U4000M M M M 0/M6 '((
 
 
 )(
" 122
 aA&&&U2!444 %d 3 3 3d+++M M M M 32M2 -..
 i(((	    /.  rc   