NgT ,dZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z m Z m Z mZmZmZmZmZmZmZmZmZddlmZmZddlmZddlmZmZddlm Z m!Z!m"Z"m#Z#m$Z$m%Z%e rddl&Z'ddl(m)cm*cm+Z,ej-e.Z/d Z0d Z1d Z2d'dZ3GddZ4e4Z5e4Z6Gdde7e Z8Gdde!Z9Gdde!Z:GddZ;dee7deddd e7fdZ?dedd ee7ddffd Z@eeAe4fZBe7ZCGd!d"ZDeDZEe7ZFGd#d$e9e:eeEeFfZGee7eefZHeAZIGd%d&e9e:eeHeIfZJeGZKeJZLeDZMdS)(z*A common module for NVIDIA Riva Runnables.N)Enum) TYPE_CHECKINGAnyAsyncGenerator AsyncIteratorDict GeneratorIteratorListOptionalTupleUnioncast) AnyMessage BaseMessage) PromptValue)RunnableConfigRunnableSerializable) AnyHttpUrl BaseModelField parse_obj_asroot_validator validator?i) .!?¡¿return riva.clientc` ddl}n"#t$r}td|d}~wwxYw|jS)z5Import the riva client and raise an error on failure.rNziCould not import the NVIDIA Riva client library. Please install it with `pip install nvidia-riva-client`.) riva.client ImportErrorclient)rivaerrs e/var/www/html/ai-engine/env/lib/python3.11/site-packages/langchain_community/utilities/nvidia_riva.py_import_riva_clientr+1s\  G    ;s &!&ceZdZdZdS) SentinelTzAn empty Sentinel type.N)__name__ __module__ __qualname____doc__r*r-r->s!!!!r3r-cbeZdZdZdZdZdZdZdZdZ e de d dfd Z e dd Zd S)RivaAudioEncodingaAn enum of the possible choices for Riva audio encoding. The list of types exposed by the Riva GRPC Protobuf files can be found with the following commands: ```python import riva.client print(riva.client.AudioEncoding.keys()) # noqa: T201 ``` ALAWENCODING_UNSPECIFIEDFLAC LINEAR_PCMMULAWOGGOPUS format_coder"c |j|j|jd|S#t$r}t d||d}~wwxYw)zReturn the audio encoding specified by the format code in the wave file. ref: https://mmsp.ece.mcgill.ca/Documents/AudioFormats/WAVE/WAVE.html )z>The following wave file format code is not supported by Riva: N)r9r6r:KeyErrorNotImplementedError)clsr<r)s r*from_wave_format_codez'RivaAudioEncoding.from_wave_format_codeXsj  ~#(syAA+N N   %8*588  s ?:?riva.client.AudioEncodingcHt}t|j|S)z-Returns the Riva API object for the encoding.)r+getattr AudioEncodingself riva_clients r*riva_pb2zRivaAudioEncoding.riva_pb2fs"*++ {0$777r3N)r"rE)r.r/r0r1r6r7r8r9r:r; classmethodintrDpropertyrLr2r3r*r5r5Fs D1 DJ EG  8K   [ 888X888r3r5ceZdZUdZeeddddgZeeefe d<edd Z e ee d <e dd Z edddeded efdZdS) RivaAuthMixinzBConfiguration for the authentication to a Riva service connection.zhttp://localhost:50051z1The full URL where the Riva service can be found.z"https://user@pass:riva.example.com) descriptionexamplesurlNz@A full path to the file where Riva's public ssl key can be read.rRssl_certr"riva.client.Authct}tt|j}|jdk}t |jdd}||j||S)z!Return a riva client auth object.https/)rVuse_ssluri) r+rrrTschemestrsplitAuthrV)rJrKrTr\ url_no_schemes r*authzRivaAuthMixin.authzsr*++ :tx((*'DH ++C003 ]G    r3T)pre allow_reusevalct|tr(ttt t|Stt|S)z:Do some initial conversations for the URL before checking.) isinstancer_rrr)rCrfs r* _validate_urlzRivaAuthMixin._validate_urlsA c3   C LS$A$ABB BJ$$$r3)r"rW)r.r/r0r1rrrTrr___annotations__rVr rOrcrrMrrir2r3r*rQrQmsLL"'% +,,G*,PQ###Cz3  $e VHhsm    X Yu$D111%% %%%[21%%%r3rQceZdZUdZeejdZeed<eddZ e ed<edd Z e ed <d S) RivaCommonConfigMixinz%A collection of common Riva settings.z!The encoding on the audio stream.)defaultrRencodingi@z*The sample rate frequency of audio stream.sample_rate_hertzzen-USzaThe [BCP-47 language code](https://www.rfc-editor.org/rfc/bcp/bcp47.txt) for the target language. language_codeN) r.r/r0r1rr5r9rnrjrorNrpr_r2r3r*rlrls//"'%!,7###H#U"Ns #M3r3rlcpeZdZUdZejed<ejed<d dZd dZ d dZ de fd Z d d Z d d ZdS) _Eventz3A combined event that is threadsafe and async safe._event_aeventr"Ncftj|_tj|_dS)zInitialize the event.N) threadingEventrsasynciortrJs r*__init__z_Event.__init__s"o'' } r3cj|j|jdSzSet the event.N)rssetrtrys r*r}z _Event.sets.  r3cj|j|jdSr|)rsclearrtrys r*rz _Event.clears0  r3c4|jS)zIndicate if the event is set.)rsis_setrys r*rz _Event.is_sets{!!###r3c8|jdS)zWait for the event to be set.N)rswaitrys r*rz _Event.waits r3cHK|jd{VdS)z#Async wait for the event to be set.N)rtrrys r* async_waitz_Event.async_waits2l!!!!!!!!!!!r3r"N)r.r/r0r1rvrwrjrxrzr}rboolrrrr2r3r*rrrrs== O ]''''   $$$$$""""""r3rroutput_directory sample_ratec(|rtjddd|5}|j}dddn #1swxYwYtj|d}|d|d||||fSd S) zECreate a new wave file and return the wave write object and filename.bxz.wavF)modesuffixdeletedirNwbr>r[)NN)tempfileNamedTemporaryFilenamewaveopen setnchannels setsampwidth setframerate)rrf wav_file_namewav_files r* _mk_wave_filers )  (fU8H    # FM # # # # # # # # # # # # # # #9]D11a   a   k***x(( rN)r_SENTENCE_TERMINATORSr`len_MAX_TEXT_LENGTHrange)rbufferchunk terminator last_sentenceidxs r*_process_chunksrs Fu%%0  J%%',{{:q'A'A$ u},z9999%% % v;;) ) )QF -=>> , ,S37]+++++F r3ceZdZUdZejed<ejed<ejed<e ed<e ed<e ed<e ej ed<dd e d d fd Z d eed d ffdZd eefdZed efdZed efdZed efdZed efdZddede e d d fdZddede e d d fdZdde e d d fdZdde e d d fdZdedd d fdZd S) AudioStreamz%A message containing streaming audio. _put_lock_queueoutputhangup user_talking user_quiet_workerrmaxsizer"Nctj|_tj||_tj|_t|_t|_ t|_ d|_ dS)zInitialize the queue.)rN) rvLockrqueueQueuerrrrrrrr)rJrs r*rzzAudioStream.__init__ s_"))k'222 kmm hh  (("HH r3c#K |jdt}n#tj$rY4wxYw|t krdS|V|jb)zReturn an error.TN)rget_QUEUE_GET_TIMEOUTrEmptyHANGUP task_donerJnext_vals r*__iter__zAudioStream.__iter__s $ ;??41CDD;    6!!NNN K ! ! # # # $s &88c K tjd|jjdt d{V}n#t j$rYRwxYw|tkrdS|WV|j )z4Iterate through all items in the queue until HANGUP.TN) rxget_event_looprun_in_executorrrrrrrrrs r* __aiter__zAudioStream.__aiter__&s $ !(!7!9!9!I!I$+/41C"";    6!!NNNN K ! ! # # # $s>AAAc4|jS)z(Indicate if the audio stream has hungup.)rrrys r*hungupzAudioStream.hungup9s{!!###r3c4|jS)z-Indicate in the input stream buffer is empty.)remptyrys r*rzAudioStream.empty>s{  """r3c|jo|j}|jduo2|j o|j}|o|S)z;Indicate if the audio stream has hungup and been processed.N)rrris_aliver)rJ input_done output_dones r*completezAudioStream.completeCs][/TZ L $ $L))+++ $ !!##  )k)r3cF|jr|jSdS)z&Indicate if the ASR stream is running.F)rrrys r*runningzAudioStream.runningNs' < +<((** *ur3itemtimeoutc|j5|jrtd|tur|j|j||ddddS#1swxYwYdS)zPut a new item into the queue.z?The audio stream has already been hungup. Cannot put more data.rN)rr RuntimeErrorrrr}rput)rJrrs r*rzAudioStream.putUs ^ 3 3{ "Uv~~ !!! KOOD'O 2 2 2 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3sAA**A.1A.cKtj}tj|d|j||d{VdS)z$Async put a new item into the queue.N)rxrwait_forrr)rJrrloops r*aputzAudioStream.aput`sS%''t33D$(DII7SSSSSSSSSSSr3c<|t|dS)zSend the hangup signal.N)rrrJrs r*closezAudioStream.closees !!!!!r3cLK|t|d{VdS)zAsync send the hangup signal.N)rrrs r*aclosezAudioStream.acloseis2ii(((((((((((r3 responseszrasr.StreamingRecognizeResponsecjrtdtjddd fd }tj|_d j_jdS) zIDrain the responses from the provided iterator and put them into a queue.z,An ASR instance has already been registered.r[rrr"NcD]}|js |jD]}|js |jrrjjtt|jdj }j |j s2jjόdS)zConsume the ASR Generator.rN)rresults alternativesis_finalrrrr}rr_ transcriptrrr)responseresultr has_startedrrJs r*workerz$AudioStream.register..workerts      % 0 0'&. 0 0F!.! 0)//111++---%)#v/B1/E/P%Q%Q   3333!.55770)--///--/// 0  0 0r3)targetTr) rrrvBarrierThreadrdaemonstartr)rJrrrs`` @r*registerzAudioStream.registerms < OMNN N'1555  0 0 0 0 0 0 0 0*!'v666 "  r3)rN) r.r/r0r1rvrrjrrrrr rrNrzr bytesrrStreamInputTyperrOrrrrrrrrrr rr2r3r*rrsq//~ K K NNN i& ''''D$)E4$56$$$$"$!?$$$$&$$$$X$#t###X#*$***X*X 3 3 3(3- 34 3 3 3 3TTT#TRVTTTT ""Xc]"d""""))HSM)T))))(+L"MRVr3rc HeZdZUdZdZeed<dZeed<eddZ e ed <ed d Z e ed <ed d Z e ed<ed edeeefdeeeffdZeddZddZ ddedeededefdZdS)RivaASRzNA runnable that performs Automatic Speech Recognition (ASR) using NVIDIA Riva.nvidia_riva_asrrzA Runnable for converting audio bytes to a string.This is useful for feeding an audio stream into a chain andpreprocessing that audio to create an LLM prompt.rRr>z7The number of audio channels in the input audio stream.rUaudio_channel_countTz\Controls whether or not Riva should attempt to filter profanity out of the transcribed text.profanity_filterz]Controls whether Riva should attempt to correct senetence puncuation in the transcribed text.enable_automatic_punctuationrdvaluesr"c"t}|Sz4Validate the Python environment and input arguments.r+rCr_s r*_validate_environmentzRivaASR._validate_environment ! ! r3&riva.client.StreamingRecognitionConfigc t}|d||j|j|jd|j|j|jS)z)Create and return the riva config object.Tr>)rnrormax_alternativesrrrp)interim_resultsconfig) r+StreamingRecognitionConfigRecognitionConfigrnrorrrrprIs r*r zRivaASR.configsm*++ 55 00"&"8$($d38n[    X ,0   (         r3rc eZdZUdZdZeed<dZeed<eddZ eed <ed d Z e eed <e d e deeefdeeeffdZed e dedefdZddZ ddede ededefdZ ddeede ede edeefdZ ddeede ede edeed ffdZd S)RivaTTSz?A runnable that performs Text-to-Speech (TTS) with NVIDIA Riva.nvidia_riva_ttsrz_A tool for converting text to speech.This is useful for converting LLM output into audio bytes.rRzEnglish-US.Female-1zThe voice model in Riva to use for speech. Pre-trained models are documented in [the Riva documentation](https://docs.nvidia.com/deeplearning/riva/user-guide/docs/tts/tts-overview.html).rU voice_nameNzThe directory where all audio files should be saved. A null value indicates that wave files should not be saved. This is useful for debugging purposes.rTrrr"c"t}|Srrrs r*rzRivaTTS._validate_environment rr3vc|rLtj|}|ddt|S|S)NT)parentsexist_ok)pathlibPathmkdirr_absolute)rCr1dirpaths r*_output_directory_validatorz#RivaTTS._output_directory_validator'sN  +l1ooG MM$M 6 6 6w''))** *r3"riva.client.SpeechSynthesisServicect} ||jS#t$r}t d|d}~wwxYw)rz5Error raised while connecting to the Riva TTS server.N)r+SpeechSynthesisServicercrrrs r*rzRivaTTS._get_service0s_)++  55di@@ @   G  rrr rc nd|t|gS)zDPerform TTS by taking a string and outputting the entire audio file.r3)r$ transformiter)rJrr rs r*r)zRivaTTS.invoke:s*xxtUG}}55666r3c+K|}t|j|j\}}t |D]}t d||||j|j |j j |j}|D]7} tt| j} |r|| | V8|r1|t d|dSdS)zHPerform TTS by taking a stream of characters and streaming output bytes.zRiva TTS chunk: %s)textr/rprnsample_rate_hzzRiva TTS wrote file: %sN)rrrrorr!r"synthesize_onliner/rprnrLrraudiowriteframesrawr) rJrr rr&rrrrresprEs r*r?zRivaTTS.transformCs0##%%#0  !4#9# #  x %U++  E MM. 6 6 6 11?"0/#5 2I"  UDJ//3++E222    D NN    MM3] C C C C C D Dr3c V Ktj tj tj d  fd }dtt f fd d fd d fd } |} |} tj dd{V}n#tj j $rYFwxYw |turn|WVm|d{V|d{VdS) zGIntercept async transforms and route them to the synchronous transform.r"NcK23d{V}|6tdS)z#Produce input into the input queue.N) put_nowait_TRANSFORM_END)rfr input_queues r* _producerz%RivaTTS.atransform.._produceruse" , , , , , , ,c&&s++++#  " "> 2 2 2 2 2s#c3K d}n#tj$rY*wxYw|tkrdS|V?)zIterate over the input_queue.TrrN)rrrrK)rfrLs r*_input_iteratorz+RivaTTS.atransform.._input_iterator{sf %//#/66CC{H.((E  s //cD]}|tdS)z!Consume the input with transform.N)r?rJrK)rfrO out_queuerJs r* _consumerz%RivaTTS.atransform.._consumersW~~oo&7&788 * *$$S))))   0 0 0 0 0r3cDKdd{VdS)z"Coroutine that wraps the consumer.N)r)rRrsr*_consumer_coroz*RivaTTS.atransform.._consumer_coros5&&tY77 7 7 7 7 7 7 7 7 7r3Trr) rxget_running_looprrr r create_taskrr exceptions TimeoutErrorrrK)rJrr rrMrTproducerconsumerrfrRrOrLrrQs`` @@@@@r* atransformzRivaTTS.atransformjs'))#(;== #*=??  3 3 3 3 3 3 3 ,!7       1 1 1 1 1 1 1 1  8 8 8 8 8 8 8##IIKK00##NN$4$455  #,Y]]__cBBBBBBBB%2        ! ! !n$$IIII s--CC21C2)r"r;r)r.r/r0r1rr_rjrRrr/rr rrMrrrrr:rrr TTSOutputTyper)r r?rrr[r2r3r*r-r-sg JI!D#!!! E e aJ',e  5'''hsm^4S>d38n[ Y!""CC[#",0777(7 7  7777,0%D%D %%D(%D3- %D - %D%D%D%DT,055\*5(53- 5  t+ , 555555r3r-)r"r#)Nr1rxloggingr5rrrvrenumrtypingrrrrrr r r r r rrlangchain_core.messagesrrlangchain_core.prompt_valuesrlangchain_core.runnablesrrpydanticrrrrrrr%r(riva.client.proto.riva_asr_pb2r'proto riva_asr_pb2rasr getLoggerr.r!rrrr+r-rrKr_r5rQrlrrfloat Wave_writerrrrrStreamOutputTyperr*r+rrr\r- NVIDIARivaASR NVIDIARivaTTSNVIDIARivaStreamr2r3r*ros00                              <;;;;;;;444444IIIIIIII2111111111111 ' H % %9    """""""" $8$8$8$8$8T$8$8$8N%%%%%I%%%BI(""""""""Dsm27 8C=(4?3 34"  3    H^43d?9S6y()LLLLLLLL`  ccccc}45cccPS*k12  _____}45___F  r3