NgjddlmZddlZddlZddlZddlZddlZddlm Z m Z ddl m Z ddl mZddlmZmZmZddlZddlZddlmZddlmZmZdd lmZdd lmZdd lm Z m!Z!dd l"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,ej-d Z.ej/Z0de0_1ej2dZ3e04e3dde.j5Dvre.6e0e.7ej8gdZ9ddddddZ:ddgZ;e Gdde Ze Gd d!e<Z? d>d?d-Z@ d@dAd7ZAe Gd8d9er$0s>>>af>>>)metricaverage sample_sd population_sdcountr&r'r(r)r*)indexrrrrjsontxtceZdZUdZded<ded<dZeedZeedZ ed&d Z d'd(dZ d)d*dZ e dZe d+dZd,d Zd-d$Zed-d%Zd S).BaseMetricsCalculatorzFoundation class for specialized metrics calculators. It provides a common interface for calculating metrics based on outputs and ground truths. Those can be provided as either directories or lists of files. z str | Path documents_dirground_truths_dirc`tj_tj_fdjdD_fdjdD_dS)z/Discover all files in the provided directories.clg|]0}||j1Sris_file relative_tor0r!pathselfs r#r$z7BaseMetricsCalculator.__post_init__..QH   ||~~   T/ 0 0   r%*clg|]0}||j1Sr)r5r6r1r7s r#r$z7BaseMetricsCalculator.__post_init__..VsH$ $ $ ||~~$   T3 4 4$ $ $ r%N)rr0resolver1glob_document_paths_ground_truth_pathsr9s`r# __post_init__z#BaseMetricsCalculator.__post_init__Ks!$"455==??!%d&.vs#D#D#DDGG#D#D#Dr%c,g|]}t|SrrrSs r#r$z2BaseMetricsCalculator.on_files..ys'L'L'LAQ'L'L'Lr%)r?r@)r9rNrPs r#on_fileszBaseMetricsCalculator.on_filesosH  E#D#D^#D#D#DD  M'L'L9K'L'L'LD $ r%Texecutor%Optional[concurrent.futures.Executor] export_dirOptional[str | Path]visualize_progressbooldisplay_agg_df pd.DataFramec||}|||}||\}}|,t||j|t||j||durt ||S)aCalculates metrics for each document using the provided executor. * Optionally, the results can be exported and displayed. * It loops through the list of structured output from all of `documents_dir` or selected files from `document_paths`, and compares them with gold-standard of the same file name under `ground_truths_dir` or selected files from `ground_truth_paths`. Args: executor: concurrent.futures.Executor instance export_dir: directory to export the results visualize_progress: whether to display progress bar display_agg_df: whether to display the aggregated results Returns: Metrics for each document as a pandas DataFrame NT)_default_executor_process_all_documentsrMrrDrGr)r9rWrYr[r]rHdfagg_dfs r# calculatezBaseMetricsCalculator.calculate}s.  --//H**85GHH..t44 F  ! :t'.s)    ?#??r%F)totalleavedisableN)r map_try_process_documentr?len)r9rWr[s r#raz,BaseMetricsCalculator._process_all_documentss    LL!;T=QRRd233 22                      sAAA"%A"docrOptional[list]ctd| ||S#t$r+}td|d|Yd}~dSd}~wwxYw)z3Safe wrapper around the document processing method.z Processing Failed to process document : N)rmrn_process_document Exceptionerror)r9res r#rz+BaseMetricsCalculator._try_process_documents '#''((( ))#.. .    LLAsAAaAA B B B44444 s4 A) A$$A)cdS)z=Should return all metadata and metrics for a single document.Nr)r9rs r#rz'BaseMetricsCalculator._process_documentrEr%)rHrIrJrKNN)rNrOrPrOrJr/NNTT rWrXrYrZr[r\r]r\rJr^)rJrr)rWrxr[r\rJrIrrrJr)__name__ __module__ __qualname____doc____annotations__rBpropertyrrDrGrMrVrd classmethodr`rorarrrr%r#r/r/@s !!!!   "BB^XB@@^X@   ^ 6:9=      ;?+/#'# """""HEE[E 666[6 &LLL^LLLr%r/ceZdZUdZdZded<dZded<dZded<fd Ze d Z e d Z e d Z ddZ dZxZS)TableStructureMetricsCalculatoraHCalculates the following metrics for tables: - tables found accuracy - table-level accuracy - element in column index accuracy - element in row index accuracy - element's column content accuracy - element's row content accuracy It also calculates the aggregated accuracy. NzOptional[float]cutoffTr\weighted_averageinclude_false_positivescHtdSrt)superrBr9 __class__s r#rBz-TableStructureMetricsCalculator.__post_init__s r%c gdS)N) total_tablestable_level_acctable_detection_recalltable_detection_precisiontable_detection_f1composite_structure_accelement_col_level_index_accelement_row_level_index_accelement_col_level_content_accelement_row_level_content_accrrAs r#supported_metric_namesz6TableStructureMetricsCalculator.supported_metric_namess    r%cdS)Nz%all-docs-table-structure-accuracy.tsvrrAs r#rDz0TableStructureMetricsCalculator.default_tsv_name66r%cdS)Nz&aggregate-table-structure-accuracy.tsvrrAs r#rGz4TableStructureMetricsCalculator.default_agg_tsv_name77r%rrrJrcn t|}|j}t|jdd}|dz}t|jdkr |jdnd}||jvrdS|j|z }|s t d|ddS|j |z }|s t d|ddStj |||j d} |  ||| jg fd |jDzS) N.jsonPrediction file  does not exist, skippingGround truth file html)prediction_fileground_truth_filer source_typec0g|]}t|Srgetattr)r!r&report_from_htmls r#r$zETableStructureMetricsCalculator._process_document..s$ Y Y Y6W%v . . Y Y Yr%)rstemsuffixrpartsr@r0existsrmwarningr1rfrom_json_filesr process_filetotal_predicted_tablesr) r9rdoc_path out_filenamedoctypesrc_gt_filename connectorrrprocessor_from_text_as_htmlrs @r#rz1TableStructureMetricsCalculator._process_documentsq99} |$$+ABB/&0*-hn*=*=*A*AHN2&&t d6 6 64,s2%%''  NNXoXXX Y Y Y4 2_D ''))  NN\0A\\\ ] ] ]4&8&H+/; ' ' ' # 7CCEE     3  Z Y Y YT=X Y Y Y Z Zr%c"gd|jz}tj||}|d|d<|jr>d|d|jd|jdz<||ddk}|js|d d|d<|j r7tjd|jD }ni}|jD]F}||| }|| ttt g} |dr||| d <n|d r9t)jt)j|||d d | d <n8t)jt)j|||d d | d <| j rtjgd gd||<A| ||<Htj| }|t2}||fS)N)filenamerrrcolumnsr_table_weightsrrc|dkrdndS)Nrrr) table_weights r#zFTableStructureMetricsCalculator._generate_dataframes../s,!*;*;QQr%cg|] }|ddddg S)Nrr)r!r&s r#r$zHTableStructureMetricsCalculator._generate_dataframes..4s#YYY6&$dA.YYYr%rr)weightsNNNrrrrrdatar+)rpd DataFramerreqrgtrapplyempty reset_indexnotnullaggrrr transpose startswithmeannproundr'SeriesrenameAGG_HEADERS_MAPPING) r9rHheadersrb has_tables_dfrcelement_metrics_resultsr& metric_df agg_metrics r#rMz4TableStructureMetricsCalculator._generate_dataframess    ' (\$ 0 0 0!.1   ' ^\]B !3!3A!6!69R9U9UVW9X9X!X Y2./!34 $ #%&6#7#=#=BB$$B    U\YYT=XYYYkmm F') #5 A A)-*?*G*G*I*IJ &v.22FGV3LMMWWYY $$^44 *3F*;*@*@*B*BJw''&&'899 *,( 9V#4iHX>YZZZ++Jw''+-( 9V#4i>WXXX++Jw'#A68i222:b:b:b777+F337A+F33\"9::DDFFRRTTF':;;6zr%r)rrrrrrrrrBrrrDrGrrM __classcell__rs@r#rrs#F""""!!!!!$(((((        X  77X788X8 Z Z Z ZD7777777r%rceZdZUdZdZded<dZded<dZd ed <fd Ze d!d Z e d!dZ d"d#fd Z dZ d$dZd%dZd ZxZS)&TextExtractionMetricsCalculatorzCalculates text accuracy and percent missing between document and ground truth texts. It also calculates the aggregated accuracy and percent missing. N Optional[str]group_by)rrrztuple[int, int, int]rr,str document_typecpt|dSrt)rrB_validate_inputsrs r#rBz-TextExtractionMetricsCalculator.__post_init___s1  r%rJcdS)Nzall-docs-cct.tsvrrAs r#rDz0TextExtractionMetricsCalculator.default_tsv_namecs!!r%cdS)Nzaggregate-scores-cct.tsvrrAs r#rGz4TextExtractionMetricsCalculator.default_agg_tsv_namegs))r%TrWrXrYrZr[r\r]r^ct||||}||jrt|j||d|S)0See the parent class for the method's docstring.rWrYr[r]Ntext_extractionrrdrget_mean_groupingr9rWrYr[r]rbrs r#rdz)TextExtractionMetricsCalculator.calculateksZWW  !1)     !dm ! dmR=N O O O r%c*js.tdtjdjt vrtdjdjD]r} |jdn.#t$r!t d|dY;wxYw|jddjkrt d stfd jDst d dSdS) Nz;No output files to calculate to edit distances for, exitingrztSpecified file type under `documents_dir` or `output_list` should be one of `json` or `txt`. The given file type is z , exiting.File z! does not have a suffix, skipping.zpThe directory contains file type inconsistent with the given input. Please note that some files will be skipped.c3JK|]}|jddjkVdS)rrN)suffixesrr7s r# zCTextExtractionMetricsCalculator._validate_inputs..s;bbT4=$(@D,>(@(@@bbbbbbr%) r?rmrnsysexitrOUTPUT_TYPE_OPTIONS ValueErrorr  IndexErrorrrall)r9r8s` r#rz0TextExtractionMetricsCalculator._validate_inputs~sj#  KKU V V V HQKKK  %8 8 8Z;?;MZZZ (  D  b!!!    LTLLLMMM }R $<(:$<$<<<CbbbbTMabbbbb  NN?       s' A55(B B rrrc|j}|jd}t|jdkr |jdnd}||\}}dt|t|z cxkrdkr(nn%t t|||jd}nd}t t||d}|||||gS)Nrrg?g@rg{Gz?) rr rr _get_cctsencoderrrr) r9rrrr output_cct source_cctaccuracypercent_missings r#rz1TextExtractionMetricsCalculator._process_documents8,q/$' NNQ$6$6CIaLLD !%!4!4 J Z&&(())C 0A0A0C0C,D,DD J J J Js J J J J J/ J UUWXYYHHH >z: V VXYZZ'9hHHr%tuple[str, str]ct|j|z |j}t|j|dz }||fS)N)docpath output_type.txt)rr0rrr1 with_suffix)r9rrrs r#rz)TextExtractionMetricsCalculator._get_cctssT(&,$:L   %T%;coof>U>U%UVV :%%r%c8gd}tj||}|dgttt t g}|dgttt t g}|jddkr,|jddkrtjt}n4tj ||f }t|_ ||fS)N)rrr cct-accuracy cct-%missingrr r!rr) rrrrrrrrshape AGG_HEADERSconcatrr)r9rHrrbaccmissrcs r#rMz4TextExtractionMetricsCalculator._generate_dataframessVVV \$ 0 0 0.!"&&vw'GHHRRTT>"#''(HIISSUU 9Q<1  A!!3!3\+666FFYT{++7799F(FN6zr%rJrrrr)rrrJr)rrrrrrrrrBrrDrGrdrrrrMrrs@r#rrTs4 #H""""$-G----M     """X"***X* ;?+/#'# &4IIII &&&&       r%rc|eZdZUdZdZded< ddfd ZeddZeddZ ddZ dZ xZ S)ElementTypeMetricsCalculatorz Calculates element type frequency accuracy, percent missing and aggregated accuracy between document and ground truth. NrrTFrWrXrYrZr[r\r]rJr^ct||||}||jrt|j||d|S)rrN element_typerrs r#rdz&ElementTypeMetricsCalculator.calculatesYWW  !1)     !dm ! dmR^ L L L r%rcdS)Nz#all-docs-element-type-frequency.tsvrrAs r#rDz-ElementTypeMetricsCalculator.default_tsv_names44r%cdS)Nz!aggregate-scores-element-type.tsvrrAs r#rGz1ElementTypeMetricsCalculator.default_agg_tsv_names22r%rrrcv|j}|jd}t|jdkr |jdnd}t t |j|z }t t |j|dz }tt||d}||||gS)Nrrrr) rr rrrrr0r1rrr )r9rrrroutputsourcers r#rz.ElementTypeMetricsCalculator._process_documents8,q/$' NNQ$6$6CIaLLD +OD|drtj |dd}ntd|}|j rtd|dkrH||j vs,|| rtd|dg}|rm|dkrg|D]d} |t!||| t&t(t*t,gie|dkrld|d <|D]d} |t!|d | t&t(t*t,giet/|}d |j dvr|d d!d"}|r-|ds|dz}t5|||dSt5|d#|d$|d|dS)%aAggregates accuracy and missing metrics by column name 'doctype' or 'connector', or 'all' for all rows. Export to TSV. If `all`, passing export_name is recommended. Args: group_by (str): Grouping category ('doctype' or 'connector' or 'all'). data_input (Union[pd.DataFrame, str]): DataFrame or path to a CSV/TSV file. export_dir (str): Directory for the exported TSV file. eval_name (str): Evaluated metric ('text_extraction' or 'element_type'). agg_name (str, optional): String to use with export filename. Default is `cct` for group_by `text_extraction` and `element-type` for `element_type` export_name (str, optional): Export filename. )rrrzQ>Q>S>S>W>W>Y>Y : DX D D D   JH%%  E   *JJx((,,eeVWf5U-VWW     5>  E   *JJ~..22EE67TZ;[3\]]     )*5J+<|d rtj|dd }ntd |}t|trtj|st d|d|drtj|d}nh|drtj|d}n<|d rtj|dd }ntd |j ddd f tj }n$t|tstd ||jvrtd||||}|jrt%d|dkr|S|dkr|rt'|||dS|dkr|stdtd)a2Reads the data_input file and filter only selected row available in filter_list. Args: data_input (str, dataframe): the source data, path to file or dataframe filter_list (str, list): the filter, path to file or list of string filter_by (str): data_input's column to filter the filter_list to export_filename (str, optional): export filename. required when return_type is "file" export_dir (str, optional): export directory. default to /metrics return_type (str): "file" or "dataframe" rr>r?Nr@rBrCrDrrFrGrz1Please provide a List of strings or path to file.z5`filter_by` key does not exists in the data provided.zANo common file names between data_input and filter_list. Exiting. dataframerYz!Please provide `export_filename`.z1Return type must be either `dataframe` or `file`.)rKrrir8rrLrMrrNrilocastypevaluestolistrIrisinrrOr) r3r[r]r7rYr^rb filter_dfress r#filter_metricsriVs$*c"" w~~j)) E#$CJ$C$C$CDD D   v & & DZ555BB   ( ( DZT222BB   ( ( DZT$???BBBCC C +s## Nw~~k** F#$DK$D$D$DEE E    ' ' D K===II  ! !& ) ) D KT:::II  ! !& ) ) D KT$GGGIIBCC CnQQQT*11#66=DDFF  T * *NLMMM ""PQQQ R ]   , , -C y^\]]]k!!   ? z?C88888    <===LMMMr%c|eZdZdZfdZedZedZedZdd Z ddZ ddZ xZ S)$ObjectDetectionMetricsCalculatorBasez Calculates object detection metrics for each document: - f1 score - precision - recall - average precision (mAP) It also calculates aggregated metrics. ctfdjdD_dS)Nclg|]0}||j1Srr4r7s r#r$zFObjectDetectionMetricsCalculatorBase.__post_init__..r:r%z,analysis/*/layout_dump/object_detection.json)rrBr0rglobr?rs`r#rBz2ObjectDetectionMetricsCalculatorBase.__post_init__s]     *001_``   r%c gdSNr< precisionrecallr=rrAs r#rz;ObjectDetectionMetricsCalculatorBase.supported_metric_names::::r%cdSNz%all-docs-object-detection-metrics.tsvrrAs r#rDz5ObjectDetectionMetricsCalculatorBase.default_tsv_namerr%cdSNz&aggregate-object-detection-metrics.tsvrrAs r#rGz9ObjectDetectionMetricsCalculatorBase.default_agg_tsv_namerr% file_stemrrJOptional[Path]c\|jD]#}t|jj|kr|cS$dS)azFind the file corresponding to OD model dump file among the set of ground truth files The files in ground truth paths keep the original extension and have .json suffix added, e.g.: some_document.pdf.json poster.jpg.json To compare to `file_stem` we need to take the prefix part of the file, thus double-stem is applied. N)r@rr)r9ryr8s r#_find_file_in_ground_truthz?ObjectDetectionMetricsCalculatorBase._find_file_in_ground_truths?,  DDI#y00 1tr%rrtuple(str, Path, Path)c$t|}|jd}||}||jvrt d|dt|jjdd}|j|z }|s1t d|dt d|d|j |z }|s1t d|dt d|d|||fS) aResolves ground doctype, prediction file path and ground truth path. As OD dump directory structure differes from other simple outputs, it needs a specific processing to match the output OD dump file with corresponding OD GT file. The outputs are placed in a dicrectory structure: analysis |- document_name |- layout_dump |- object_detection.json |- bboxes # not used in this evaluation and the GT file is pleced in od_gt directory for given dataset dataset_name |- od_gt |- document_name.pdf.json Args: doc (Path): path to the OD dump file Returns: tuple: doctype, prediction file path, ground truth path rz not found in list of GT filesrNrrz does not exist) rrr|r@rrrr0rrmrr1)r9r od_dump_pathryrrrrs r# _get_pathsz/ObjectDetectionMetricsCalculatorBase._get_pathss86Cyy  &r* 99)DD $": : :a/aaabb b+,,3ABB7,s2%%'' R NNXoXXX Y Y YPPPPQQ Q 2_D '')) V NN\0A\\\ ] ] ]T2CTTTUU U):::r%rKcPgd|jz}tj||}|jrtjt}ni}|jD]}|||}||ttttg }|jrtj gdgd||<|||<tj| }t|_||fS)N)rrrrrrr)rrrrr#rrrrrrrrrr) r9rHrrbrcrr&rrs r#rMz9ObjectDetectionMetricsCalculatorBase._generate_dataframess06669TT \$ 0 0 0 8 U\+666FF&( #5 A Ar&z11334 &v.22E67F3STT^^`` #A68i222:b:b:b777+F337A+F33\"9::DDFFRRTTF$6zr%)ryrrJrz)rrrJr})rJrK) rrrrrBrrrDrGr|rrMrrs@r#rkrks     ;;X;77X788X8 /;/;/;/;br%rkcneZdZfdZedZedZedZd d Zd Z xZ S) (ObjectDetectionPerClassMetricsCalculatorc~td|_|dSrt)rrBper_class_metric_names_set_supported_metricsrs r#rBz6ObjectDetectionPerClassMetricsCalculator.__post_init__s9 8<# ##%%%%%r%c<|jr|jStd)Nz:per_class_metrics not initialized - cannot get class names)rrrAs r#rz?ObjectDetectionPerClassMetricsCalculator.supported_metric_namess&  & [. .YZZ Zr%cdS)Nz/all-docs-object-detection-metrics-per-class.tsvrrAs r#rDz9ObjectDetectionPerClassMetricsCalculator.default_tsv_names@@r%cdS)Nz0aggregate-object-detection-metrics-per-class.tsvrrAs r#rGz=ObjectDetectionPerClassMetricsCalculator.default_agg_tsv_name sAAr%rrrJrc ||\}}}n8#t$r+}td|d|Yd}~dSd}~wwxYwt j||}|\}}|j|dg} |jD]x} d | ddd} | dd} t|| } | | | y| S)zCalculate both class-aggregated and per-class metrics for a single document. Args: doc (Path): path to the OD dump file Returns: tuple: a tuple of aggregated and per-class metrics for a single document rrNprediction_file_pathground_truth_file_path_r) rrrmrrr get_metricsrrjoinsplitrrQ)r9rrrrr processorrper_class_metricsper_class_metrics_rowcombined_metric_namer& class_name class_metricss r#rz:ObjectDetectionPerClassMetricsCalculator._process_document$sJ :>//#:N:N 7G_&7&7    LLAsAAaAA B B B44444 1@!0#4    )4466   "  !  %)$? D D XX288==crcBCCF-33C88>M ! ( (z)B C C C C$$s A A  Acvgd}t}|jD]d}|j|z }t|5}t j|}|d}||dddn #1swxYwYeg}|D]!} |D]} || d| "t||_ dS)aSets the supported metrics based on the classes found in the ground truth files. The difference between per class and aggregated calculator is that the list of classes (so the metrics) bases on the contents of the GT / prediction files. rqobject_detection_classesNr) setr@r1openr,loadupdaterQsortedr) r9rXclassesgt_file gt_file_pathfr gt_classesrr&rs r#rz?ObjectDetectionPerClassMetricsCalculator._set_supported_metricsFs. >==%%/ + +G1G;Ll## +qYq\\ :; z*** + + + + + + + + + + + + + + +"$ H HF% H H &--.F.F*.F.FGGGG H&,-C&D&D###s2A33A7 :A7 r) rrrrBrrrDrGrrrrs@r#rrs&&&&& [[X[ AAXABBXB % % % %DEEEEEEEr%rc\eZdZdZedZedZedZd d Zd S) *ObjectDetectionAggregatedMetricsCalculatorzSCalculates object detection metrics for each document and aggregates by all classesc gdSrprrAs r#rzAObjectDetectionAggregatedMetricsCalculator.supported_metric_names]rtr%cdSrvrrAs r#rDz;ObjectDetectionAggregatedMetricsCalculator.default_tsv_namearr%cdSrxrrAs r#rGz?ObjectDetectionAggregatedMetricsCalculator.default_agg_tsv_nameerr%rrrJrc@ ||\}}}n8#t$r+}td|d|Yd}~dSd}~wwxYwt j||}|\}|j|dgfd|jDzS)zCalculate both class-aggregated and per-class metrics for a single document. Args: doc (Path): path to the OD dump file Returns: list: a list of aggregated metrics for a single document rrNrc0g|]}t|Srr)r!r&rXs r#r$zPObjectDetectionAggregatedMetricsCalculator._process_document..s# P P P&WWf % % P P Pr%) rrrmrrrrrr) r9rrrrrrrrXs @r#rz//#:N:N 7G_&7&7    LLAsAAaAA B B B44444 1@!0#4   **,,   "    Q P P PD4O P P P Q Qs A A  ANr) rrrrrrrDrGrrr%r#rrZs]] ;;X;77X788X8QQQQQQr%rr)rrr3r4rYrr5rr6rr7rrJr8)rNrXrY)r3rZr[r\r]rr7rrYrr^rrJr_)E __future__rconcurrent.futuresrur,loggingrir abcrr dataclassesrpathlibrtypingr r r numpyrpandasrr !unstructured.metrics.element_typer r%unstructured.metrics.object_detectionr%unstructured.metrics.table.table_evalr$unstructured.metrics.text_extractionrrunstructured.metrics.utilsrrrrrrrrrr getLoggerrm StreamHandlerhandlerr Formatter formatter setFormatterhandlers addHandlersetLevelDEBUGr#rrr/rrr)rrirkrrrr%r#rs#"""""  ########!!!!!!((((((((((EDDDDDcccccccc                          . / / '  ! !! G Z [ [ Y>>fo>>>>> g JJJ   uo JLJLJLJLJLCJLJL JLZ BBBBB&;BB BJ ggggg&;gg gT :::::#8:: :D#%) VUVUVUVUVUx %) >N>N>N>N>NB sssss+@#ss slIEIEIEIEIE/SIEIEIEX(Q(Q(Q(Q(Q1U(Q(Q(Q(Q(Qr%