
    Ngj                    8   d Z 	 ddlZddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlmZ ddlmZ 	 dZddlmZmZmZmZmZmZmZmZmZ ddlmZ ddlmZ dd	lm Z  n# e!$ r d
Ze"ZY nw xY wddl#m$Z$m%Z% ddl&m'Z' ddl(Z( G d d          Z) G d d          Z* G d d          Z+ G d de+          Z, G d de+          Z- G d de+          Z. G d de+          Z/ G d de+          Z0 G d de+          Z1 G d d e+          Z2 G d! d"e+          Z3 G d# d$e+          Z4 G d% d&e+          Z5 G d' d(e+          Z6 G d) d*e+          Z7 G d+ d,e+          Z8 G d- d.          Z9 G d/ d0          Z: G d1 d2          Z;d3 Z<d4 Z=d`d5Z>d`d6Z?d7 Z@dad9ZAd: ZBd; ZCd< ZDd= ZE e9            ZFeFjG        ZGd> ZHd? ZId@ ZJeKdAk    rKddBlLmMZM  eM            ZNeNO                    dCdDdEdFdGH           eNO                    dIdJdKdLd
dMN           eNO                    dOdPdQdLd
dRN           eNO                    dSdTdUdLd
dVN           eNO                    dWdXdYejP        Q                    dZ          d[\           eNR                                \  ZSZT e9eSjU        ]          ZVeTrEeTD ]@ZWeVG                    eWeSjX        eSjY        eSjZ        eSj[        ^          Z\e\d
k    r
eSj[        r dS AdS eVG                    eSjX        eSjY        eSjZ        eSj[        _           dS dS )baH	  
The NLTK corpus and module downloader.  This module defines several
interfaces which can be used to download corpora, models, and other
data packages that can be used with NLTK.

Downloading Packages
====================
If called with no arguments, ``download()`` will display an interactive
interface which can be used to download and install new packages.
If Tkinter is available, then a graphical interface will be shown,
otherwise a simple text interface will be provided.

Individual packages can be downloaded by calling the ``download()``
function with a single argument, giving the package identifier for the
package that should be downloaded:

    >>> download('treebank') # doctest: +SKIP
    [nltk_data] Downloading package 'treebank'...
    [nltk_data]   Unzipping corpora/treebank.zip.

NLTK also provides a number of "package collections", consisting of
a group of related packages.  To download all packages in a
colleciton, simply call ``download()`` with the collection's
identifier:

    >>> download('all-corpora') # doctest: +SKIP
    [nltk_data] Downloading package 'abc'...
    [nltk_data]   Unzipping corpora/abc.zip.
    [nltk_data] Downloading package 'alpino'...
    [nltk_data]   Unzipping corpora/alpino.zip.
      ...
    [nltk_data] Downloading package 'words'...
    [nltk_data]   Unzipping corpora/words.zip.

Download Directory
==================
By default, packages are installed in either a system-wide directory
(if Python has sufficient access to write to it); or in the current
user's home directory.  However, the ``download_dir`` argument may be
used to specify a different installation target, if desired.

See ``Downloader.default_download_dir()`` for more a detailed
description of how the default download directory is chosen.

NLTK Download Server
====================
Before downloading any packages, the corpus and module downloader
contacts the NLTK download server, to retrieve an index file
describing the available packages.  By default, this index file is
loaded from ``https://raw.githubusercontent.com/nltk/nltk_data/gh-pages/index.xml``.
If necessary, it is possible to create a new ``Downloader`` object,
specifying a different URL for the package index file.

Usage::

    python nltk/downloader.py [-d DATADIR] [-q] [-f] [-k] PACKAGE_IDS

or::

    python -m nltk.downloader [-d DATADIR] [-q] [-f] [-k] PACKAGE_IDS
    N)md5)ElementTreeT)	ButtonCanvasEntryFrameIntVarLabelMenuTclErrorTk)	showerror)Table)ShowTextF)	HTTPErrorURLError)urlopenc                   R    e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 d
dZed             Zd Zd	 ZdS )Packagea4  
    A directory entry for a downloadable package.  These entries are
    extracted from the XML index file that is downloaded by
    ``Downloader``.  Each package consists of a single file; but if
    that file is a zip file, then it can be automatically decompressed
    when the package is installed.
    N UnknownTc                 D   || _         	 |p|| _        	 || _        	 || _        	 t	          |          | _        	 t	          |          | _        	 || _        	 || _        	 |	| _	        	 |
| _
        	 || _        	 || _        	 t          j                            |                    d          d                   d         }t          j                            |||z             | _        	 t'          t	          |                    | _        	 | j                            |           d S )N/   )idnamesubdirurlintsizeunzipped_sizechecksumsvn_revision	copyrightcontactlicenseauthorospathsplitextsplitjoinfilenameboolunzip__dict__update)selfr   r   r   r   r!   r"   r#   r$   r%   r&   r'   r(   r0   kwexts                   K/var/www/html/ai-engine/env/lib/python3.11/site-packages/nltk/downloader.py__init__zPackage.__init__   s   " 3JB	-	4 EII	: //	 !4(<"0	, 3%gsyy~~b12215VR#X66	3 #e**%%
	 	R         c                     t          | t                    rt          j        |           } | j        D ]$}t          | j        |                   | j        |<   %t          di | j        S )N )
isinstancestrr   parseattribr   )xmlkeys     r6   fromxmlzPackage.fromxml  sf    c3 	)#C((C: 	3 	3C!#*S/22CJsOO$$$$$r8   c                 "    | j         |j         k     S Nr   r3   others     r6   __lt__zPackage.__lt__      w!!r8   c                     d| j         z  S )Nz<Package %s>rD   r3   s    r6   __repr__zPackage.__repr__   s    ''r8   )Nr   NNNNr   r   r   r   T	__name__
__module____qualname____doc__r7   staticmethodrA   rG   rK   r:   r8   r6   r   r      s          C! C! C! C!J % % \%" " "( ( ( ( (r8   r   c                   <    e Zd ZdZddZed             Zd Zd ZdS )
Collectionz
    A directory entry for a collection of downloadable packages.
    These entries are extracted from the XML index file that is
    downloaded by ``Downloader``.
    Nc                 ~    || _         	 |p|| _        	 || _        	 d | _        	 | j                            |           d S rC   )r   r   childrenpackagesr1   r2   )r3   r   rU   r   r4   s        r6   r7   zCollection.__init__+  sR    6JB	0 	, 	3 	R     r8   c                    t          | t                    rt          j        |           } | j        D ]$}t          | j        |                   | j        |<   %d |                     d          D             }t          dd|i| j        S )Nc                 8    g | ]}|                     d           S )refget).0childs     r6   
<listcomp>z&Collection.fromxml.<locals>.<listcomp>C  s$    FFFEIIe$$FFFr8   itemrU   r:   )r;   r<   r   r=   r>   findallrS   )r?   r@   rU   s      r6   rA   zCollection.fromxml=  s    c3 	)#C((C: 	3 	3C!#*S/22CJsOOFF#++f2E2EFFF::8:sz:::r8   c                 "    | j         |j         k     S rC   rD   rE   s     r6   rG   zCollection.__lt__F  rH   r8   c                     d| j         z  S )Nz<Collection %s>rD   rJ   s    r6   rK   zCollection.__repr__I  s     47**r8   rC   rL   r:   r8   r6   rS   rS   $  sk         ! ! ! !$ ; ; \;" " "+ + + + +r8   rS   c                       e Zd ZdZdS )DownloaderMessagezSA status message object, used by ``incr_download`` to
    communicate its progress.N)rM   rN   rO   rP   r:   r8   r6   rd   rd   R  s        ! ! ! !r8   rd   c                       e Zd ZdZd ZdS )StartCollectionMessagez<Data server has started working on a collection of packages.c                     || _         d S rC   
collectionr3   ri   s     r6   r7   zStartCollectionMessage.__init__Z      $r8   NrM   rN   rO   rP   r7   r:   r8   r6   rf   rf   W  s)        FF% % % % %r8   rf   c                       e Zd ZdZd ZdS )FinishCollectionMessagez=Data server has finished working on a collection of packages.c                     || _         d S rC   rh   rj   s     r6   r7   z FinishCollectionMessage.__init__a  rk   r8   Nrl   r:   r8   r6   rn   rn   ^  s)        GG% % % % %r8   rn   c                       e Zd ZdZd ZdS )StartPackageMessagez-Data server has started working on a package.c                     || _         d S rC   packager3   rt   s     r6   r7   zStartPackageMessage.__init__h      r8   Nrl   r:   r8   r6   rq   rq   e  )        77    r8   rq   c                       e Zd ZdZd ZdS )FinishPackageMessagez.Data server has finished working on a package.c                     || _         d S rC   rs   ru   s     r6   r7   zFinishPackageMessage.__init__o  rv   r8   Nrl   r:   r8   r6   ry   ry   l  )        88    r8   ry   c                       e Zd ZdZd ZdS )StartDownloadMessagez.Data server has started downloading a package.c                     || _         d S rC   rs   ru   s     r6   r7   zStartDownloadMessage.__init__v  rv   r8   Nrl   r:   r8   r6   r}   r}   s  r{   r8   r}   c                       e Zd ZdZd ZdS )FinishDownloadMessagez/Data server has finished downloading a package.c                     || _         d S rC   rs   ru   s     r6   r7   zFinishDownloadMessage.__init__}  rv   r8   Nrl   r:   r8   r6   r   r   z  )        99    r8   r   c                       e Zd ZdZd ZdS )StartUnzipMessagez,Data server has started unzipping a package.c                     || _         d S rC   rs   ru   s     r6   r7   zStartUnzipMessage.__init__  rv   r8   Nrl   r:   r8   r6   r   r     s)        66    r8   r   c                       e Zd ZdZd ZdS )FinishUnzipMessagez-Data server has finished unzipping a package.c                     || _         d S rC   rs   ru   s     r6   r7   zFinishUnzipMessage.__init__  rv   r8   Nrl   r:   r8   r6   r   r     rw   r8   r   c                       e Zd ZdZd ZdS )UpToDateMessagez/The package download file is already up-to-datec                     || _         d S rC   rs   ru   s     r6   r7   zUpToDateMessage.__init__  rv   r8   Nrl   r:   r8   r6   r   r     r   r8   r   c                       e Zd ZdZd ZdS )StaleMessagez3The package download file is out-of-date or corruptc                     || _         d S rC   rs   ru   s     r6   r7   zStaleMessage.__init__  rv   r8   Nrl   r:   r8   r6   r   r     s)        ==    r8   r   c                       e Zd ZdZd ZdS )ErrorMessagez Data server encountered an errorc                 x    || _         t          |t                    rt          |          | _        d S || _        d S rC   )rt   r;   	Exceptionr<   message)r3   rt   r   s      r6   r7   zErrorMessage.__init__  s7    gy)) 	#w<<DLLL"DLLLr8   Nrl   r:   r8   r6   r   r     s)        **# # # # #r8   r   c                       e Zd ZdZd ZdS )ProgressMessagez4Indicates how much progress the data server has madec                     || _         d S rC   )progress)r3   r   s     r6   r7   zProgressMessage.__init__  s     r8   Nrl   r:   r8   r6   r   r     s)        >>! ! ! ! !r8   r   c                       e Zd ZdZd ZdS )SelectDownloadDirMessagez:Indicates what download directory the data server is usingc                     || _         d S rC   )download_dirr3   r   s     r6   r7   z!SelectDownloadDirMessage.__init__  s    (r8   Nrl   r:   r8   r6   r   r     s)        DD) ) ) ) )r8   r   c                   H   e Zd ZdZdZ	 dZ	 dZ	 dZ	 dZ	 dZ		 d(d	Z
	 	 	 	 	 	 d)dZd Zd Zd Zd Zd Zd*dZd Zd Zd Zdddddd
dej        fdZd+dZd+dZd+dZd+dZd Zd,dZd+dZd Zd  Z d! Z!d" Z"d# Z# e$e"e#          Z%d$ Z&d% Z'd& Z( e$e'e(          Z)d' Z*dS )-
Downloaderzy
    A class used to access the NLTK data server, which can be used to
    download corpora and other data packages.
    i  zChttps://raw.githubusercontent.com/nltk/nltk_data/gh-pages/index.xml	installedznot installedzout of datepartialNc                     |p| j         | _        	 i | _        	 i | _        	 || _        	 d | _        	 d | _        	 i | _        	 d | _        	 | j        | 	                                | _        d S d S rC   )
DEFAULT_URL_url_collections	_packages_download_dir_index_index_timestamp_status_cache_errorsdefault_download_dir)r3   server_index_urlr   s      r6   r7   zDownloader.__init__  s    $8(8	7E?)I@ $	L  	
 R %!%!:!:!<!<D &%r8   TFc                    d}|| j         }t          d|z             |rjt          ddt          | j                  z   z             t          d| j        z             t          ddt          | j                  z   z             |dz  }dx}}	g }
|r|
                    d           |r|
                    d	           |
D ]}t          d
|                                z             |dz  }t           t          | |                      t                    D ](}| 	                    ||          }|| j
        k    r|r'|| j        k    rd}|| j        k    rd}	| j
        d| j        d| j        d| j        di|         }t          j        d|j        p|j        z   dd          dd          }t          d                    ||j                            dd          |                     |t          |                    d                    z  }|r1|dk    r+t+          d          }|                                dv r  d S d}*t                       d}|r|dz  }|	r|dz  }t          t          j        |d z   dd!"                     d S )#Nr   z!Using default data directory (%s)=   z Data server index for <%s>   FrV   collectionsz%s:r   r@   T*-P ---------------------------K                              subsequent_indent   z  [{}] {} {}   .
zHit Enter to continue: )xqz([*] marks installed packagesz+; [-] marks out-of-date or corrupt packagesz+; [P] marks partially installed collections)L   )r   width)r   printlenr   append
capitalizesortedgetattrr<   status	INSTALLEDSTALEPARTIALNOT_INSTALLEDtextwrapfillr   r   formatljustr,   inputlower)r3   r   show_packagesshow_collectionsheadermore_promptskip_installedlinesstaler   
categoriescategoryinfor   prefixr   
user_inputmsgs                     r6   listzDownloader.list	  s    -L5DEEE 	#c$)nn,-.../$);<<<#c$)nn,-...QJE
 	*j))) 	-m,,," 	 	H%(--///000QJE6wtX6688cBBB  T<88T^+++TZ'' ET\))"GNCJL#&	
   }	 4TW5rX  ## n++FDGMM"c4J4JDQQRRRTZZ--... 52::!&'@!A!AJ!''))Z77EGGGG- 	A@@C 	A@@ChmC#IBGGGHHHHHr8   c                 \    |                                   | j                                        S rC   )_update_indexr   valuesrJ   s    r6   rV   zDownloader.packagesE  s(    ~$$&&&r8   c                 p    |                                   d | j                                        D             S )Nc                 .    g | ]\  }}|j         d k    |S corporar   r\   r   pkgs      r6   r^   z&Downloader.corpora.<locals>.<listcomp>K  (    WWW	SszY?V?V?V?V?Vr8   r   r   itemsrJ   s    r6   r   zDownloader.corporaI  6    WWT^%9%9%;%;WWWWr8   c                 p    |                                   d | j                                        D             S )Nc                 .    g | ]\  }}|j         d k    |S r   r   r   s      r6   r^   z%Downloader.models.<locals>.<listcomp>O  r   r8   r   rJ   s    r6   modelszDownloader.modelsM  r   r8   c                 \    |                                   | j                                        S rC   )r   r   r   rJ   s    r6   r   zDownloader.collectionsQ  s)     '')))r8   c                 Z    t          |t                    r|                     |          S |S rC   )r;   r<   r   )r3   
info_or_ids     r6   _info_or_idzDownloader._info_or_idY  s,    j#&& 	99Z(((r8   c              #   @  K   || j         }t          |          V  t          |t          t          f          r|                     |||          E d {V  d S 	 |                     |          }n7# t          t          f$ r#}t          d d| d|           V  Y d }~d S d }~ww xY wt          |t                    rFt          |          V  |                     |j        ||          E d {V  t          |          V  d S |                     |||          E d {V  d S )NzError loading : )r   r   r;   r   tuple_download_listr   OSError
ValueErrorr   rS   rf   incr_downloadrU   rn   _download_package)r3   r   r   forcer   es         r6   r   zDownloader.incr_downloadh  s     -L*<88888 j4-00 	**:|UKKKKKKKKKF	##J//DD$ 	 	 	t%Gj%G%GA%G%GHHHHHFFFFF	
 dJ'' 	I(.....))$-uMMMMMMMMM)$/////// --dL%HHHHHHHHHHHs   A/ /B# BB#c                 X    t          |t                    rdS t          |j                  S )Nr   )r;   r   r   rV   )r3   r_   s     r6   _num_packageszDownloader._num_packages  s)    dG$$ 	&1t}%%%r8   c              #   p   K   t          t          |                    D ]Y}	                      ||                   ||<   "# t          t          f$ r$}t          ||         |          V  Y d }~ d S d }~ww xY wt           fd|D                       }d}t          |          D ]\  }}t          |t                    rd|z  }	nt          |j
                  |z  }	                     |||          D ]8}
t          |
t                    rt          ||
j        |	z  z             V  4|
V  9|d|	z  z  }d S )Nc              3   B   K   | ]}                     |          V  d S rC   )r  r\   r_   r3   s     r6   	<genexpr>z,Downloader._download_list.<locals>.<genexpr>  s1      FF4--d33FFFFFFr8   r   g      ?d   )ranger   r   r   r   r   sum	enumerater;   r   rV   r   r   r   )r3   r   r   r  ir  num_packagesr   r_   deltar   s   `          r6   r   zDownloader._download_list  s     s5zz"" 	 	A++E!H55aZ(   "58Q/////
 FFFFFFFFF '' 	$ 	$GAt$(( :l*DM**\9))$eDD  c?33 )(S\E5I*IJJJJJJIIIIe#HH	$ 	$s   AA7A22A7c              #   :  K   t          |          V  t          d          V  |                     ||          }|s@|| j        k    r5t	          |          V  t          d          V  t          |          V  d S | j                            |j        d            t          j
                            ||j                  }t          j
                            |          r0|| j        k    rt          |          V  t          j        |           t          j        |d           t          j        t          j
                            ||j                  d           t'          |          V  t          d          V  	 t)          |j                  }t-          |d          5 }t/          d|j        dz            }t3          j                    D ]a}	|                    d          }
|                    |
           |
s n2|	d	z  dk    r(t          t;          d
dd|	|z  z  z                       V  bd d d            n# 1 swxY w Y   |                                 n=# t>          $ r0}tA          |d|j        d|j        d|          V  Y d }~d S d }~ww xY wtC          |          V  t          d
          V  |j        "                    d          rt          j
                            ||j                  }|j#        sBt          j
                            t          j
                            ||j                            rBtI          |          V  tK          ||d          D ]}||_&        |V  tO          |          V  t          |          V  d S )Nr   r	  T)exist_ok   wbr    @     P   r   zError downloading z from <>:
  .zipF)verbose)(rq   r   r   r   r   ry   r   popr   r)   r*   r-   r.   existsr   r   removemakedirsr   r}   r   r   openmaxr!   	itertoolscountreadwritemincloser   r   r   endswithr0   r   _unzip_iterrt   r   )r3   r   r   r  r   filepathinfileoutfile
num_blocksblocksr  zipdirr   s                 r6   r   zDownloader._download_package  s     !$'''''a      T<00 	4>11!$'''''!#&&&&&&t,,,,,F 	tw--- 7<<dm<<7>>(## 	 ##"4(((((Ih 	L40000
BGLLt{;;dKKKK #4(((((a     	TX&&Fh%% V DI$;<<
&_.. V VEI..AMM!$$$ qyA~~-c"a"
@R:S6S.T.TUUUUUV V V V V V V V V V V V V V V LLNNNN 	 	 	>BgggtxxxQRQRS     FFFFF	 $D)))))b!!!!! =!!&)) 
	/W\\,<<F z /RW^^BGLL,I,IJJ /'-----&xGGG  C"&CKIIII(....."4(((((((s=   $I! 'BI5I! II! I	I! !
J+%JJ[nltk_data] c	           
         t          j        t          |          |||| _        |                                  dS dfd	}	|                     |||          D ] }
t          |
t                    r |	|
j                   |rt          |
j                  |r dS d| _
        |s_ d           t                                                      }|dv r)|                     |
j        j        |||||          s dS n|dv r dS |sRt          |
t                     r) |	d	|
j        j        z             d
z              t          |
t$                    rP            d d         | j
        r |	d|
j        j        z             7 |	d|
j        j        z             Qt          |
t&                    r |	d|
j        j        d|d           t          |
t(                    r |	d|
j        j        z  d           t          |
t*                    r |	d|
j        j        z  d           t          |
t.                    r|
j        }dS )N)fileTr   c                 Z     t          j        | |z   |z   dz                        d S )N    )initial_indentr   )r   r   )r-  prefix2r   print_tos     r6   showz!Downloader.download.<locals>.show  sM    M'-'7*07*:W*D      r8   Fz(Error installing package. Retry? [n/y/e])yY)r  EDownloading collection %rz   | z$Downloaded collection %r with errorszDone downloading collection %szDownloading package z to z...z!Package %s is already up-to-date!  zUnzipping %s.r   )	functoolsr   r   r   _interactive_downloadr   r;   r   r   r   r   r   stripdownloadrt   r   rf   ri   rn   rq   r   r   r.   r   r   )r3   r   r   quietr  r   halt_on_errorraise_on_errorprint_error_tor7  r   choicer6  s        `      @r6   rB  zDownloader.download  s    $U@@@ '%1"&&(((4       ))*lEJJ <8 <8c<00 )D%%%% 6(555$ %$uu#'DL  ) !KLLL!&!Z//#'== # , % % & - .$ $ 	- (-uu	- $z11#(55  !8!#'=>> 883>;LLMMM') ((((#C)@AA 8 (((!'< W D F"%."3!4   
 !D!ACNDU!UVVVV $C)<== 8"{~~~|||=    $C99 
8@3;>QSWXXXX $C):;; 8_s{/CCTJJJJ $C)ABB 8'*'7tr8   c                 @    |                      ||          | j        k    S rC   )r   r   r3   r   r   s      r6   is_stalezDownloader.is_staleE  s    {{:|44
BBr8   c                 @    |                      ||          | j        k    S rC   )r   r   rI  s      r6   is_installedzDownloader.is_installedH  s    {{:|44FFr8   c                 v    || j                                          d S | j                             |d            d S rC   )r   clearr  r3   r   s     r6   clear_status_cachezDownloader.clear_status_cacheK  sA    :$$&&&&&""2t,,,,,r8   c                 B    | j         }                     |          }t          |t                    rc fd|j        D             } j        |v r j        S  j        |v r j        S  j        |v r j        |v r j        S  j        |v r j        S  j        S t          j
                            ||j                  }| j         k    r                     ||          S |j         j        vr#                     ||           j        |j        <    j        |j                 S )z
        Return a constant describing the status of the given package
        or collection.  Status can be one of ``INSTALLED``,
        ``NOT_INSTALLED``, ``STALE``, or ``PARTIAL``.
        Nc                 D    g | ]}                     |j                  S r:   )r   r   )r\   r   r3   s     r6   r^   z%Downloader.status.<locals>.<listcomp>]  s'    GGG#$++cf--GGGr8   )r   r   r;   rS   rV   r   r   r   r   r)   r*   r-   r.   _pkg_statusr   r   )r3   r   r   r   
pkg_statusr(  s   `     r6   r   zDownloader.statusQ  s<    -L
++ dJ'' 	3GGGGGGGJzZ''z!++|#:--$2D
2R2R|##z11))~% w||L$-@@Ht111''h7777$"444262B2B42R2RD&tw/)$'22r8   c                    t           j                            |          s| j        S 	 t          j        |          }n# t
          $ r
 | j        cY S w xY w|j        t          |j                  k    r| j	        S t          |          |j        k    r| j	        S |                    d          r|d d         }t           j                            |          s| j        S t           j                            |          s| j	        S t          d t          j        |          D                       }||j        k    r| j	        S | j        S )Nr  r<  c              3      K   | ]D\  }}}|D ];}t          j        t           j                            ||                    j        V  <Ed S rC   )r)   statr*   r-   st_size)r\   d_filesfs        r6   r  z)Downloader._pkg_status.<locals>.<genexpr>  sp          Aq%     Q**++3             r8   )r)   r*   r  r   rW  r   rX  r    r!   r   md5_hexdigestr#   r&  r   isdirr  walkr"   )r3   r   r(  filestatunzipdirr"   s         r6   rS  zDownloader._pkg_statuss  sV   w~~h'' 	&%%	&wx((HH 	& 	& 	&%%%%	&s49~~--: ""dm33: V$$ 	"}H7>>(++ &~%7==** "z!    #%78#4#4       M
  222z! ~s   = AAc                     |                                   |                                 D ]8}|                     |          | j        k    r|                     |||           9dS )zA
        Re-download any packages whose status is STALE.
        )rC  r   N)rP  rV   r   r   rB  )r3   rC  r   r   s       r6   r2   zDownloader.update  si     	!!!==?? 	? 	?C{{34:--cv>>>	? 	?r8   c                    | j         (|&t          j                    | j        z
  | j        k    sdS |p| j        | _        t
          j                            t          j	        t          | j                                                                      | _         t          j                    | _        d | j                             d          D             }d |D             | _        d | j                             d          D             }d |D             | _        | j                                        D ]}t!          |j                  D ]m\  }}|| j        v r| j        |         |j        |<   $|| j        v r| j        |         |j        |<   Ct%          d                    |                     |j        |= n| j                                        D ]u}i }|g}|D ]R}t)          |t*                    r|                    |j                   2t)          |t.                    r|||j        <   RS|                                |_        v| j                                         dS )	zA helper function that ensures that self._index is
        up-to-date.  If the index is older than self.INDEX_TIMEOUT,
        then download it again.Nc                 B    g | ]}t                               |          S r:   )r   rA   r\   ps     r6   r^   z,Downloader._update_index.<locals>.<listcomp>  s$    XXX1GOOA&&XXXr8   packages/packagec                     i | ]
}|j         |S r:   rD   re  s     r6   
<dictcomp>z,Downloader._update_index.<locals>.<dictcomp>  s    444a!$444r8   c                 B    g | ]}t                               |          S r:   )rS   rA   r\   cs     r6   r^   z,Downloader._update_index.<locals>.<listcomp>  s3     
 
 
&'Jq!!
 
 
r8   collections/collectionc                     i | ]
}|j         |S r:   rD   rk  s     r6   ri  z,Downloader._update_index.<locals>.<dictcomp>  s    :::QT1:::r8   z.removing collection member with no package: {})r   timer   INDEX_TIMEOUTr   nltk	internalsElementWrapperr   r=   r   getrootr`   r   r   r   r  rU   r   r   r;   rS   extendr   r   rV   r   rN  )	r3   r   rV   r   ri   r  child_idqueuer]   s	            r6   r   zDownloader._update_index  s    Ky{{T22T5GGGF $49	 n33gdi001199;;
 
 !%	 YX0C0CDV0W0WXXX448444
 
+/;+>+>?W+X+X
 
 
 ;:k::: +2244 	/ 	/J()<== / /8t~---1^H-EJ'**!222-1->x-HJ'**HOO$   
 #+A../ +2244 
	4 
	4JHLE  eZ00 LL0000w// ).HUX&&"*//"3"3J 	  """""r8   c                 8    |                                   | j        S )z
        Return the XML index describing the packages available from
        the data server.  If necessary, this index will be downloaded
        from the data server.
        )r   r   rJ   s    r6   indexzDownloader.index  s     	{r8   c                     |                                   || j        v r| j        |         S || j        v r| j        |         S t          d|z            )zKReturn the ``Package`` or ``Collection`` record for the
        given item.Package %r not found in index)r   r   r   r   rO  s     r6   r   zDownloader.info  s`     	>"%%"""$R((82=>>>r8   c                 6   |                                   | j                            d          D ]}|                    d          |k    r|c S  | j                            d          D ]}|                    d          |k    r|c S  t	          d|z            )z-Return the XML info record for the given itemrg  r   rm  r{  )r   r   r`   r[   r   )r3   r   rt   ri   s       r6   xmlinfozDownloader.xmlinfo  s    {**+=>> 	 	G{{4  B&& '+--.FGG 	" 	"J~~d##r))!!!! *82=>>>r8   c                     | j         S )z)The URL for the data server's index file.)r   rJ   s    r6   _get_urlzDownloader._get_url  s
    yr8   c                 Z    | j         }	 |                     |           dS #  || _          xY w)z
        Set a new URL for the data server. If we're unable to contact
        the given url, then the original url is kept.
        N)r   r   )r3   r   original_urls      r6   _set_urlzDownloader._set_url  s=    
 y	s#####	$DIs     
*c                    dt           j        v rdS t          j        j        D ]D}t           j                            |          r#t          j                            |          r|c S Et          j	        dk    r!dt           j        v rt           j        d         }n4t           j        
                    d          }|dk    rt          d          t           j                            |d          S )a  
        Return the directory to which packages will be downloaded by
        default.  This value can be overridden using the constructor,
        or on a case-by-case basis using the ``download_dir`` argument when
        calling ``download()``.

        On Windows, the default download directory is
        ``PYTHONHOME/lib/nltk``, where *PYTHONHOME* is the
        directory containing Python, e.g. ``C:\Python25``.

        On all other platforms, the default directory is the first of
        the following which exists or which can be created with write
        permission: ``/usr/share/nltk_data``, ``/usr/local/share/nltk_data``,
        ``/usr/lib/nltk_data``, ``/usr/local/lib/nltk_data``, ``~/nltk_data``.
        APPENGINE_RUNTIMENwin32APPDATAz~/z+Could not find a default download directory	nltk_data)r)   environrq  datar*   r  rr  is_writablesysplatform
expanduserr   r-   )r3   nltkdirhomedirs      r6   r   zDownloader.default_download_dir  s    " "*,,F y~ 	 	Gw~~g&& 4>+E+Eg+N+N  <7""yBJ'>'>j+GG g((..G$ !NOOO w||G[111r8   c                     | j         S )a  
        The default directory to which packages will be downloaded.
        This defaults to the value returned by ``default_download_dir()``.
        To override this default on a case-by-case basis, use the
        ``download_dir`` argument when calling ``download()``.
        )r   rJ   s    r6   _get_download_dirzDownloader._get_download_dir=  s     !!r8   c                 F    || _         | j                                         d S rC   )r   r   rN  r   s     r6   _set_download_dirzDownloader._set_download_dirF  s%    )  """""r8   c                    t           rV	 t          |                                            d S # t          $ r% t	          |                                            Y d S w xY wt	          |                                            d S rC   )TKINTERDownloaderGUImainloopr   DownloaderShellrunrJ   s    r6   r@  z Downloader._interactive_downloadQ  s      	(,d##,,..... , , ,%%))++++++, D!!%%'''''s   !, +AA)NN)NTTTFF)NFrC   )Fr/  )+rM   rN   rO   rP   rp  r   r   r   r   r   r7   r   rV   r   r   r   r   r   r  r   r   r  stderrrB  rJ  rL  rP  r   rS  r2   r   ry  r   r}  r  r  propertyr   r   r  r  r   r@  r:   r8   r6   r   r     s         M XK! I$#ME"GE= = = =N :I :I :I :Ix' ' 'X X XX X X* * *  I I I I8& & &$ $ $2@) @) @)H z^ ^ ^ ^@C C C CG G G G- - - - 3  3  3  3D" " "H? ? ? ?<# <# <# <#|  ? ? ?	? 	? 	?  
 
 
 (8X
&
&C%2 %2 %2N" " "# # #
 8-/@AAL	( 	( 	( 	( 	(r8   r   c                   >    e Zd Zd Zd Zd Zd Zd Zd Zd Z	d Z
d	S )
r  c                     || _         d S rC   )_ds)r3   
dataservers     r6   r7   zDownloaderShell.__init__^  s    r8   c                     t          d           dt          d |D                       z
  t          |          dz
  z  dz  }t          d|                    |          z              t          d           d S )NzK---------------------------------------------------------------------------D   c              3   4   K   | ]}t          |          V  d S rC   )r   )r\   os     r6   r  z;DownloaderShell._simple_interactive_menu.<locals>.<genexpr>c  s(      001A000000r8   r   r   r3  )r   r  r   r-   )r3   optionsspcs      r6   _simple_interactive_menuz(DownloaderShell._simple_interactive_menua  sq    hC00000000c'llQ6FG#Mfsxx((()))hr8   c                    t          d           	 |                     dddddd           t          d	                                          }|st                       M|                                                                d
         }|                                dd          }	 |dk    r6t                       | j                            | j        j        dd           n|dk    r| 	                                 nj|dk    r| 
                                 nO|dv rd S |dk    r|                     |           n-|dk    r|                                  nt          d|z             nR# t          $ r}t          d|z             Y d }~n1d }~wt          $ r!}t          d|j        z             Y d }~nd }~ww xY wt                       )NNLTK DownloaderTzd) Downloadzl) Listz
 u) Updatez	c) Configzh) Helpzq) QuitzDownloader> r   r   lF)r   r   hrl  )r   r   rY  uzCommand %r unrecognizedzError reading from server: %szError connecting to server: %s)r   r  r   rA  r   r,   r  r   r   _simple_interactive_help_simple_interactive_config_simple_interactive_download_simple_interactive_updater   r   reason)r3   r   commandargsr  s        r6   r  zDownloaderShell.rung  s      %	))   ~..4466J  &&((..003G##%%abb)DCc>>GGGHMM$("7SWMXXXX^^113333^^335555
**F^^55d;;;;^^3355553j@AAA ; ; ;59:::::::: C C C6ABBBBBBBBC GGGK%	s,   &A6E( A	E( (
F72F		F7F22F7c                    |rQ|D ]L}	 | j                             |d            # t          t          f$ r}t	          |           Y d }~Ed }~ww xY wd S 	 t	                       t	          d           t          d          }|                                dk    r)| j                             | j         j        ddd           n|                                d	v rd S |rc|	                                D ]L}	 | j                             |d            # t          t          f$ r}t	          |           Y d }~Ed }~ww xY wd S )
Nr3  r   Tz*Download which package (l=list; x=cancel)?  Identifier> r  F)r   r   r   r   r   r   )
r  rB  r   r   r   r   r   r   r   r,   )r3   r  argr  r   r   s         r6   r  z,DownloaderShell._simple_interactive_download  s    	  H%%c&%9999,   !HHHHHHHH BCCC"#344
##%%,,HMM-$$('+	 "    %%''>99F (..00 % %% H--b-@@@@ '4 % % %!!HHHHHHHH%E)s,   %AA

A3DD:!D55D:c           	      v   	 g }dx}}t           t          | j        d                      t                    D ]K}| j                            |          | j        j        k    r!|                    |j        |j        f           Lt                       |rt          d           |D ]^\  }}t          j        d|z   dd	          d
d          }t          d                    |                    dd          |                     _t                       t          d          }|                                dk    rT|D ]O\  }}	 | j                            |d           ## t"          t$          f$ r}	t          |	           Y d }	~	Hd }	~	ww xY wd S |                                dv rd S nt          d           d S )NTFrV   r   z/Will update following packages (o=ok; x=cancel)r   r   r   r   r   z  [ ] {} {}r   r   r  r  r3  r  r  zNothing to update.)r   r   r  r<   r   r   r   r   r   r   r   r   r   r   r   r   rB  r   r   )
r3   stale_packagesr   r   r   pidpnamer   r   r  s
             r6   r  z*DownloaderShell._simple_interactive_update  s   	N##EG<wtx<<>>CHHH @ @8??4((DHN::"))47DI*>???GGG GHHH"0 J JJC#= E*B(  ccD -..syyS/A/A4HHIIII"#344
##%%,,&4 % %
U% H--c&-AAAA '4 % % %!!HHHHHHHH%E%%''>99F : *+++;	s   E  F
1FF
c                     t                       t          d           t          d           t          d           t          d           d S )Nz	Commands:zH  d) Download a package or collection     u) Update out of date packagesz1  l) List packages & collections          h) Helpz1  c) View & Modify Configuration          q) Quit)r   rJ   s    r6   r  z(DownloaderShell._simple_interactive_help  sT    kV	
 	
 	
 	ABBBABBBBBr8   c                    t                       t          d           t          d| j        j        z             t          dt          | j                                                  z             t          dt          | j                                                  z             t                       t          d           t          d| j        j        z             d S )NzData Server:z  - URL: <%s>z$  - %d Package Collections Availablez$  - %d Individual Packages AvailablezLocal Machine:z  - Data directory: %s)r   r  r   r   r   rV   r   rJ   s    r6   _show_configzDownloaderShell._show_config  s    no,---4s48;O;O;Q;Q7R7RRSSS4s48;L;L;N;N7O7OOPPP&)>>?????r8   c                 <   |                                   	 t                       |                     dddd           t          d                                                                          }|dk    r|                                   n|dk    rtt          d	                                          }|d
v rt          d           nt          j                            |          r|| j	        _
        nt          d|z             n|dk    rt          d                                          }|d
v rt          d           n\|                    d          sd|z   }	 || j	        _        n4# t          $ r}t          d|d|            Y d }~nd }~ww xY w|dk    rd S )NTzs) Show Configzu) Set Server URLzd) Set Data Dirzm) Main MenuzConfig> r-  rY  z  New Directory> )r   r   r   XQz  Cancelled!z)Directory %r not found!  Create it first.r  z  New URL> )zhttp://https://r  zError reading <r  m)r  r   r  r   rA  r   r)   r*   r^  r  r   
startswithr   r   )r3   r   
new_dl_dirnew_urlr  s        r6   r  z*DownloaderShell._simple_interactive_config  s   	GGG)) "57H.   z**002288::JS  !!####s"""#677==??
!999.))))W]]:.. T,6DH))E
RSSSSs""..4466666.))))"--.EFF 7",w"6F'.$ F F FDDDDDEEEEEEEEFs""9	s   E( (
F2FFN)rM   rN   rO   r7   r  r  r  r  r  r  r  r:   r8   r6   r  r  ]  s            ' ' 'R  <  @C C C@ @ @    r8   r  c                   "   e Zd ZdZg dZ	 dddddZ	 dddd	d	d
dZ	 dZ	 g dZ	 eD ]Z	e	ev sJ 	eD ]Z	e	ev sJ 	eD ]Z	e	ev sJ 	dZ
ej        dej        dej        dej        diZdZdZdZdZdZdCdZd Zd Zd Zd Zd Zd ZdDd Zd! Zd" Zd# Z d$ Z!d% Z"d& Z#d' Z$d(Z%dZ&d) Z'd* Z(d+ Z)d	Z*d, Z+d- Z,d. Z-d/ Z.d0 Z/d1 Z0d2 Z1d3 Z2d4 Z3d5 Z4d6 Z5 e6j7        d7          Z8d8 Z9d9 Z:dZ;d: Z<d; Z=d< Z>d= Z?d> Z@ G d? d@eAjB                  ZCdAZDdB ZEdS )Er  zU
    Graphical interface for downloading packages from the NLTK data
    server.
    )r   
IdentifierNameSizeStatusUnzipped Size	CopyrightContactLicenseAuthorSubdirChecksumr   r  )r   r  r  r  r   r   -   
      )r   r  r  r  r  r     )r   r  r  r  r  )z#000z#ccc)z#afaz#080)z#ffaz#880)z#faaz#800)#fffz#888)r  z#45c)#aaaz#67a)z#f00r  zhelvetica -16 boldTc                    || _         || _        t          j                    | _        g | _        g | _        d| _        i | _        g | _	        d| _
        |                     d           t                      x}| _        |                    d           |                    d           |                    | j        d                    |                    d| j                   |                    d	| j                   d| _        i | _        |                                  |                                  	 |                                  nN# t2          $ r}t5          d
|           Y d }~n/d }~wt6          $ r}t5          d|j                   Y d }~nd }~ww xY w|                                  |                                  | j                             d           | j                            d| j!                   d S )NFr   zNLTK Downloader Started!z+50+50r  r   )
backgroundz<Control-q>z<Control-x>Error reading from serverError connecting to serverz	<Destroy>)"r  _use_threads	threadingLock_download_lock_download_msg_queue_download_abort_queue_downloading_afterid_log_messages_log_indent_logr   topgeometrytitle	configure_BACKDROP_COLORbinddestroy
_destroyed_column_vars_init_widgets
_init_menu_fill_tabler   r   r   r  
_show_info_select_columns_tableselect_destroy)r3   r  use_threadsr  r  s        r6   r7   zDownloaderGUI.__init__\  s
   ' (n..#% %'"!   		,--- dhX		#$$$!5a!8999 	------ 		> 	6 	6 	61155555555 	> 	> 	>2AH========	> 	1 	dm44444s$   3E 
FE''F4FFc                     | j                             d                    t          j                    d| j        z  |                     d S )Nz{} {}{}z | )r  r   r   ro  ctimer  r3   r   s     r6   r  zDownloaderGUI._log  sJ    !!TZ\\543C+CSII	
 	
 	
 	
 	
r8   c                     t           j        dddd          }|                    ddd	           |                    dd
           |                    dd
           t          |d                              dd           t          |          }|                    dd
d           t          |          }|                    ddd           t          |          }|                    ddd           t          |d                              dd           t          |          }|                    ddd           t          |d                              dd           t           j        dd j        d
                   }|                    dd           d j        d<   d j        d<   g d _        i  _        t           j                  D ]s\  }}t          || j                  }	|	                    d|d
z   dz  dz             |	                    d j                   |	 j        |                                <   t fd  j        D             }
t!          | j        |
dd! j        "           _         j                            d j        d         #           t           j                  D ]A\  }} j                            | j                  } j                            ||$           B j                            dd%            j                                          j                            d& j                    j                            d' j                    j                            d( j                    j                            d) j                    j                            d* j                    j                            d+ j                   |                    d
d
           d,d- j        fd.d/ j         fg}i  _!        t          |          D ]\  }\  }}	}t          ||	0                              d|d1           tE          |d2d3d4d45          }||f j!        |<   |                    d( j#                   |                    d|f fd6	           |                    d
|d7            j                            d j#                   tI          |d8 j        d9           _%         j%                            d:           tI          |d; j&        d9           _'         j'                            d<:           t          |d= j        d          j        d
         >           _(        tS          |d?d! j*        d
         d@d
A           _+         ,                                  j+                            d<:            j(                            d:           d S )BNraisedr     r   )reliefborderpadxpadyr  Tboth)sidexpandr   r   )weight)height)columnrownews)r	  r
  stickyr      r     )r  r  r  bottomr   )sider   r  highlightthickness)CollectionsCorporaModelszAll Packages)textfontleftr  )r  r  z
<Button-1>c                 F    g | ]}j                             |d           S )r   )COLUMN_WEIGHTSr[   )r\   r	  r3   s     r6   r^   z/DownloaderGUI._init_widgets.<locals>.<listcomp>  s,    XXX$-11&!<<XXXr8      )column_weightsr  listbox_heightreprfunc)
foregroundr   )r  r   z<Double-Button-1>z<space>z<Return>z<Left>z<Right>z<Control-a>r   zServer Index:r   zDownload Directory:)r  r  couriergroovez#007aff)r  r   disabledforegroundr  c                 .                         |          S rC   
_info_edit)r  r@   r3   s     r6   <lambda>z-DownloaderGUI._init_widgets.<locals>.<lambda>  s    8L8L r8   ewDownload)r  r  r   )r  Refreshrightr   )r  r  r     sunken)r   r  r  r   r  )-r   r  packgrid_rowconfiguregrid_columnconfiguregridr  
_tab_names_tabsr  r
   	_TAB_FONTr  _select_tabr   COLUMNSr   _table_reprfuncr  columnconfig_MARK_COLORCOLUMN_WIDTHSr[   DEFAULT_COLUMN_WIDTHfocusbind_to_listboxes	_download_table_mark	_prev_tab	_next_tab	_mark_allr  r  _infor   
_info_saver   _download_button_refresh_refresh_button_progresslabelr   _PROGRESS_COLOR_progressbar_init_progressbar)r3   f1tabframe
tableframebuttonframe	infoframeprogressframer  tablabelr  r	  r   r   r@   callbackentrys   `                r6   r  zDownloaderGUI._init_widgets  sX   48HQQQGGG
E$V444
Qq)))
!,,,b  q 11199QAf5552YY
qa777Biiq888b  q 111"II	aQv666b  q 111H111Ea1H
 
 
 	s333)*%& ONN
00 	, 	,FAs(4>BBBEJJF1q5A+);J<<<JJ|T%5666&+DJsyy{{## YXXX4<XXXL) )
 
 
 	  t/?/B CCC"4<00 	5 	5IAv&**643LMMEK$$Qe$44446222%%&94>JJJD$4555T^4444>222DN333777 	&&q&333 OT]32D4JK
 
)24 	5 	5%A%UH)%(((--QAc-JJJ#,$  E  %h/DJsOJJz4?333JJ|3%L%L%L%L%LMMMJJaQtJ4444 	lDO444 !'j$.!
 !
 !
 	"""///%ia 
  
  
 	!!w!/// $+A.+A.	
 
 
 #+A.
 
 
 	   G,,,  f -----r8   c                     t           j                  }t          |d          }|                    dd j        d           |                                 |                    dd fd	           |                    d
d fd	           |                                 |                    dd j        	           |                                 |                    dd j        d           |                    dd|           t          |d          } j        j	        dd          D ]g}t           j                  }| j        vsJ | j        |<   | j        v r|                    d           |                    |d| j                   h|                    dd|           t          |d          } j        j	        dd          D ]!}|                    d|z  |f fd	           "|                                  j        j	        dd          D ]!}|                    d|z  |f fd	           "|                    dd|           t          |d          }|                    dd j        	           |                    dd j        d           |                    dd|            j                            d  j                    j                            |!           d S )"Nr   )tearoffr(  Return)rR  	underliner  acceleratorzChange Server Index   c                  .                          d          S )Nr   r$  rJ   s   r6   r&  z*DownloaderGUI._init_menu.<locals>.<lambda>  s    DOOE22 r8   )rR  rX  r  zChange Download Directoryc                  .                          d          S )Nr   r$  rJ   s   r6   r&  z*DownloaderGUI._init_menu.<locals>.<lambda>  s    DOON;; r8   zShow Logr  Exitr   zCtrl-xFile)rR  rX  menur  )rR  rX  variabler  Viewz
Sort by %sc                 :    j                             | d          S )N	ascendingr  sort_byrl  r3   s    r6   r&  z*DownloaderGUI._init_menu.<locals>.<lambda>5  s    $+*=*=a*M*M r8   )rR  r  zReverse sort by %sc                 :    j                             | d          S )N
descendingrd  rf  s    r6   r&  z*DownloaderGUI._init_menu.<locals>.<lambda><  s    $+*=*=a*N*N r8   SortAboutInstructionsF1Helpz<F1>)r_  )r   r  add_commandr=  add_separator	_show_logr  add_cascader  column_namesr	   r  INITIAL_COLUMNSsetadd_checkbuttonr  abouthelpr  config)r3   menubarfilemenuviewmenur	  varsortmenuhelpmenus   `       r6   r  zDownloaderGUI._init_menu  s   tx..+++4>x 	 	
 	
 	
 	   '2222 	 	
 	
 	

 	-;;;; 	 	
 	
 	

 	   :DNSSS   At| 	 	
 	
 	
 	&AHEEE
 +++k.qrr2 	 	F""C!22222(+Df%---


$$CAU %     	&AHEEE
 +++k.qrr2 	 	F  "V+"(MMMMM !     	   k.qrr2 	 	F  *V3"(NNNNN !     	&AHEEE+++7aLLL Atyd 	 	
 	
 	
 	&AHEEEfdi(((W%%%%%r8   c                     | j                                         D ]N\  }}|                                r| j                            |           4| j                            |           Od S rC   )r  r   r[   r  show_columnhide_column)r3   r	  r|  s      r6   r  zDownloaderGUI._select_columnsJ  sp    ,2244 	0 	0KFCwwyy 0''////''////		0 	0r8   c                 4   | j                                          	 |                                  nN# t          $ r}t	          d|           Y d }~n/d }~wt
          $ r}t	          d|j                   Y d }~nd }~ww xY w| j                            d           d S )Nr  r  r   )	r  rP  r  r   r   r   r  r  r  r3   r  s     r6   rE  zDownloaderGUI._refreshQ  s    ##%%%	> 	6 	6 	61155555555 	> 	> 	>2AH========	>1s    0 
A;AA;A66A;c                     |                                   | j        |         \  }}d|d<   d|d<   |                                 d S )Nnormalstater,  r   )rC  rB  r;  )r3   info_keyrT  rS  s       r6   r%  zDownloaderGUI._info_edit[  sF     Jx0!g"hr8   Nc                    | j         }| j                                        D ]R\  }}|d         dk    r||j        |u r|j        dk    r|}+d|d<   d|d<    ||                                           S|                                 d S )Nr  disabledrW  r!  r   )r  rB  r   widgetkeysymr[   r;  )r3   r  r;  rT  rS  s        r6   rC  zDownloaderGUI._info_saveb  s    #z0022 	& 	&OE8W~++}U!2!2qx87K7K!+g"*h%%%%r8   c                     | j         j        |                             d          r>t          |t                    rd|z  S |dk     rd|dz  z  S |dk     rd|dz  z  S d	|d
z  z  S |dv rt	          |          S d|z  S )Nr  z  %si   z	  %.1f KBg      @i   @z	  %.1f MBg      0Az	  %.1f GBg      A)r   r   )r  rr  r&  r;   r<   )r3   r
  colvals       r6   r6  zDownloaderGUI._table_reprfunco  s    ;#C(11&99 	7#s## 7|#w"cIo66w"cIo66"cIo66'>>s88OC<r8   c                     || j         j        k    rd S 	 || j         _        |                                  n4# t          $ r'}t	          dt          |                     Y d }~nd }~ww xY w|                                  d S )NzError Setting Server Index)r  r   r  r   r   r<   r  )r3   r   r  s      r6   r  zDownloaderGUI._set_url  s    $(,F	<DHL 	< 	< 	<2CFF;;;;;;;;	<s    5 
A&A!!A&c                 2   | j         j        |k    rd S || j         _        	 |                                  nN# t          $ r}t	          d|           Y d }~n/d }~wt
          $ r}t	          d|j                   Y d }~nd }~ww xY w|                                  d S )Nr  r  )r  r   r  r   r   r   r  r  )r3   r   r  s      r6   r  zDownloaderGUI._set_download_dir  s    8 L00F !-	> 	6 	6 	61155555555 	> 	> 	>2AH========	>s    5 
B AB !A;;B c                    t          d| j        j                   | j                                        D ] \  }}d|d<   |                    dd           !| j        d         d                             d| j        j                   | j        d         d                             d| j        j                   | j                                        D ]
\  }}d|d<   d S )	Nzshowing infor  r  r   endr   r   r  )r   r  r   rB  r   deleteinsertr   )r3   rT  cbs      r6   r  zDownloaderGUI._show_info  s    ndhl+++**,, 	# 	#IE2%E'NLLE""""
5!##Atx|444
>"1%,,Q0EFFF**,, 	( 	(IE2'E'NN	( 	(r8   c                    t          | j                  D ]\  }}|                                | j        k    r|dk    r| j        |dz
                                           | _        	 |                                 c S # t
          $ r}t          d|           Y d }~d }~wt          $ r}t          d|j                   Y d }~d }~ww xY wd S )Nr   r   r  r  )	r  r1  r   _tabr  r   r   r   r  r3   r  r  rQ  s       r6   r?  zDownloaderGUI._prev_tab  s   00 	F 	FFAsyy{{di''AEE OAE288::	F++-----  > > >91======== F F F:AHEEEEEEEEF	F 	Fs$   %A;;
CBC'CCc                    t          | j                  D ]\  }}|                                | j        k    r|t	          | j                  dz
  k     r| j        |dz                                            | _        	 |                                 c S # t          $ r}t          d|           Y d }~d }~wt          $ r}t          d|j
                   Y d }~d }~ww xY wd S )Nr   r  r  )r  r1  r   r  r   r2  r  r   r   r   r  r  s       r6   r@  zDownloaderGUI._next_tab  s   00 	F 	FFAsyy{{di''ATZ11D,E,E OAE288::	F++-----  > > >91======== F F F:AHEEEEEEEEF	F 	Fs$   :B
CB//C<CCc                    |j         d                                         | _        	 |                                  d S # t          $ r}t          d|           Y d }~d S d }~wt          $ r }t          d|j                   Y d }~d S d }~ww xY w)Nr  r  r  )r  r   r  r  r   r   r   r  )r3   eventr  s      r6   r4  zDownloaderGUI._select_tab  s    L(..00		> 	6 	6 	611555555555 	> 	> 	>2AH=========	>s!   < 
B	AB	)BB	r   c                      j                                         } j                                           j        dk    r j                                        }n j        dk    r j                                        }n\ j        dk    r j                                        }n7 j        dk    r j                                        }nJ d j        z               fd|D             } j         	                    |            j
                                        D ]k\  }}| j        k    r.|                     j        d          j        d         	           >|                     j        d          j        d         	           l j                             d
d                                              j                             |             j        j        d j         j        j        g j         j                                        R     j        j        d j         j        j        g j         j                                        R   d S )Nzall packagesr   r   r   r   zbad tab value %rc                 :    g | ]}                     |          S r:   )_package_to_columnsr  s     r6   r^   z-DownloaderGUI._fill_table.<locals>.<listcomp>  s'    AAA4((..AAAr8   r   r  r  r  rc  )order   i,  )r  selected_rowrN  r  r  rV   r   r   r   ru  r2  r   r  _FRONT_TAB_COLOR_BACK_TAB_COLORre  _color_tabler  r  after
_scrollbarrt  _mlbyview)r3   r  r   rowsrQ  rR  s   `     r6   r  zDownloaderGUI._fill_table  sN   {//119&&H%%''EEY)##H$$&&EEY(""HOO%%EEY-''H((**EE4(494444AAAA5AAA4    ***,, 
	 
	JCdi#4Q7#4Q7      
 #3A6#3A6      
 	L<<<<((( 	sDK26R9I9O9O9Q9QRRRRsDK26R9I9O9O9Q9QRRRRRRr8   c                     t          t          | j                            D ]5}| j                            | j        |df                   }|| j        |df<   6|                                  d S )Nr  r  )r
  r   r  r  r   r  )r3   row_numr   s      r6   _update_table_statusz"DownloaderGUI._update_table_status  sn    S--.. 	4 	4GX__T[,1F%GHHF-3DK)**r8   c                 b     j         r
  j        | S  fdt          t           j                            D             } j                                        }|s| j        |df         g} j                            | j        j                  }d _	         
                    ||           d S )Nc                 T    g | ]$}j         |d f         dk    j         |df         %S r   r   r  r  r\   r
  r3   s     r6   r^   z+DownloaderGUI._download.<locals>.<listcomp>  E     
 
 
{36"b(( K\)*(((r8   r  r   )r  _download_threadedr
  r   r  r  r  r   r   r  _download_cb)r3   r  marked	selectiondownload_iters   `    r6   r=  zDownloaderGUI._download  s      	/*4*A..
 
 
 
S--..
 
 

 K,,..	 	<)/k)\"9:;F..vtx7LMM-00000r8   c                 n    	 t          |          }nP# t          $ rC                                    j                            d j        d          }| j        d<   Y d S w xY w fd}t          |t                    r                     |j	                   nt          |t                    rM ||j                   |j                             |j        j                                        d            d S t          |t                    r* |d|j        j        z              xj        dz  c_        nJt          |t$                    r |d|j        j        z             nt          |t&                    r |d|j        j        z             nt          |t(                    r |d	|j        j        z             nt          |t*                    r |d
|j        j        z             nt          |t.                    rH xj        dz  c_         |d|j        j        z                                  |j        j                   n4t          |t2                    r                     |j        j                    j                             j         j        ||          }| j        d<   d S )Nr  r   r  c                 F    | j         d<                       |            d S Nr  rG  r  r-  r3   s    r6   r7  z(DownloaderGUI._download_cb.<locals>.show  #    *+D'IIaLLLLLr8   zDownloading collection %sr   zDownloading package %sPackage %s is up-to-date!Finished downloading %r.Unzipping %s#Finished downloading collection %r.)nextStopIterationr  r  r  _show_progressr  r;   r   r   r   r   rt   _selectr   rf   ri   r  rq   r   r   r   r.   rn   _clear_markry   	_DL_DELAYr  )r3   r  idsr   afteridr7  s   `     r6   r  zDownloaderGUI._download_cb	  s   	}%%CC 	 	 	%%'''hnnR)<a@@G,3DM.)FF		 	 	 	 	 c?++ 	-----\** 	-D{&S[^,,,%%%F344 	-D,s~/@@AAA!011 	-D)CKN:;;;;_-- 	-D,s{~=>>>> 233 		-D+ckn<====.// 	-D#+"667777455 	-!D69JJKKKS^.////122 	-S[^,,,(..1BMSVWW(/n%%%s    A	A A c                     t          t          | j                            D ]2}| j        |df         |k    r| j                            |            d S 3d S )Nr  )r
  r   r  r  r3   r   r
  s      r6   r  zDownloaderGUI._select5  sf    T[))** 	 	C{3,-33""3''' 4	 	r8   c                 8   t          t          | j                            D ]w}| j        | j        |df                  \  }}d\  }}| j                            |||||           | j                            |d| j        d         | j        d                    xd S )Nr  )blackwhite)r  selectforegroundr  selectbackgroundr   r   r  )r
  r   r  
_ROW_COLOR	rowconfigitemconfigurer8  )r3   r
  bgsbgfgsfgs         r6   r  zDownloaderGUI._color_table;  s    T[))** 	 	Codk#x-&@AGB(GBK!!!$!$ "    K%%Q4#3A#64CSTUCV &    	 	r8   c                     t          t          | j                            D ]!}| j        |df         |k    rd| j        |df<   "d S )Nr  r   r   r
  r   r  r  s      r6   r  zDownloaderGUI._clear_markL  sU    T[))** 	) 	)C{3,-33&(CF#	) 	)r8   c                 f    t          t          | j                            D ]}d| j        |df<   d S )Nr  r   r  )r3   r  r
  s      r6   rA  zDownloaderGUI._mark_allQ  s@    T[))** 	& 	&C"%DKQ	& 	&r8   c                     | j                                         }|dk    r0| j         |         d         dk    rd| j         |df<   nd| j         |df<   | j                             d           d S )Nr   r   r  r   )r  )r  r  r  )r3   r  r  s      r6   r>  zDownloaderGUI._table_markU  sw    K,,..	>>{9%a(B..,.IqL)),/IqL)#####r8   c                 f    d                     | j                  }t          | j        d|           d S )Nr   zNLTK Downloader Log)r-   r  r   r  )r3   r  s     r6   rp  zDownloaderGUI._show_log^  s1    yy+,,0$77777r8   c                    g }t          | j                  D ]\  }}|dk    r|                    d           !|dk    r|                    |j                   B|dk    r.|                    | j                            |                     v|                                                    dd          }|                    t          ||d                     |S )z
        Given a package, return a list of values describing that
        package, one for each column in ``self.COLUMNS``.
        r   r   r  r  r   rZ  zn/a)	r  r5  r   r   r  r   r   replacer   )r3   r   r
  column_indexcolumn_nameattrs         r6   r  z!DownloaderGUI._package_to_columnsb  s    
 )24<)@)@ 		6 		6%L+q  

2,,

36""""((

48??3//0000"((**223<<

73e445555
r8   c                 X    | j         rd S | j                                         d| _         d S )NT)r  r  r  r  s     r6   r  zDownloaderGUI.destroyx  s0    ? 	Fr8   c                     | j         6| j                                        D ]}| j                             |           | j        r| j        r|                                  | j                                         d S rC   )	r  r  r   after_cancelr  r  _abort_downloadr  rN  )r3   r  r  s      r6   r  zDownloaderGUI._destroy~  s    8=//11 / /%%g....  	#!2 	#  """
 	!!!!!r8   c                 *     | j         j        |i | d S rC   )r  r  )r3   r  kwargss      r6   r  zDownloaderGUI.mainloop  s"    4*6*****r8   an      This tool can be used to download a variety of corpora and models
    that can be used with NLTK.  Each corpus or model is distributed
    in a single zip file, known as a "package file."  You can
    download packages individually, or you can download pre-defined
    collections of packages.

    When you download a package, it will be saved to the "download
    directory."  A default download directory is chosen when you run

    the downloader; but you may also select a different download
    directory.  On Windows, the default download directory is


    "package."

    The NLTK downloader can be used to download a variety of corpora,
    models, and other data packages.

    Keyboard shortcuts::
      [return]	 Download
      [up]	 Select previous package
      [down]	 Select next package
      [left]	 Select previous tab
      [right]	 Select next tab
    c                     	 t          | j        d| j                                        dd           d S #  t          | j        d| j                                        d           Y d S xY w)NzHelp: NLTK Downloaderr   fixed)r   r  r  )r   r  HELPrA  r  s     r6   rw  zDownloaderGUI.help  s    		U'	!!     	UTX6	8I8IQSTTTTTTTs	   04 1A(c                     d}d}	 ddl m}  |||                                           d S # t          $ r t	          | j        ||           Y d S w xY w)Nz'NLTK Downloader
Written by Edward LoperzAbout: NLTK Downloaderr   )Message)r   r  )tkinter.messageboxr  r7  ImportErrorr   r  )r3   r  ABOUTTITLEr  s        r6   rv  zDownloaderGUI.about  s    ?(	-222222GE///4466666 	- 	- 	-TXue,,,,,,	-s   %-  AAc                 D   | j         }t          |d                   t          |d                   }}t          dt          |d                   dz  | j        z            D ]Z}|                    || j        z  dz   d|| j        z  |z
  dz
  |dz   | j        ddt          |d	z  d
z
            dz  z   z             [|                    d           |                    dd           |                    d|	                    dddd| j
        d                              d S )Nr   r  r   r  r   iz	#%02x0000r  r  r   r  )r   r   gradienthiddenr  redbox)r   )rI  r    r
  _gradient_widthcreate_lineabs
addtag_all
itemconfigaddtag_withtagcreate_rectanglerH  )r3   rl  r   r  r  s        r6   rJ  zDownloaderGUI._init_progressbar  sB   AgJQx[)9)9vq3qz??Q.43GGHH 	 	AMMD((2-D((61B6* BQUQY")<$<=      	
Z   	Zx000 	
a((Aq!$:Nq:Q(RR	
 	
 	
 	
 	
r8   c                 8   | j         }|2|                    ddddd           |                    dd           d S t          |d                   t          |d                   }}|t          |          z  dz  d	z   }|                    ddd||d	z              d S )
Nr  r   r  r  r  r   r  r	  r   )rI  coordsr  r    )r3   percentrl  r   r  r   s         r6   r  zDownloaderGUI._show_progress  s    ?HHXq!Q***LL8L44444'
OOS8-=-=6E#e**$+a/AHHXq!Q
33333r8   c                    | j         }| j        s|                    dd           d S |                    dd           |                    d          \  }}}}|dk    r#|                    d| j        dz  dz
  d           n|                    dd	d           | j                            d
| j                  }|| j	        d<   d S )Nr  r  r  r  ir  r  r   r<  r+  _progress_alive)
rI  r  r  bboxmover  r  r  r  r  )r3   rl  x1y1x2y2r  s          r6   r  zDownloaderGUI._progress_alive  s      
	7LL8L44444LL8L444VVJ//NBBTzzzD$81$<#A1EEEEz2q)))hnnS$*>??G/6DM+,,,r8   c                 f     j         r                                  d S d j        d<    fdt          t	           j                            D             } j                                        }|s| j        |df         g}t           j        j	         j        j
                  } j        g k    sJ  j        g k    sJ                      || j         j         j                                                   d _        d _                                                                             d S )NCancelr  c                 T    g | ]$}j         |d f         dk    j         |df         %S r  r  r  s     r6   r^   z4DownloaderGUI._download_threaded.<locals>.<listcomp>  r  r8   r  r   T)r  r  rD  r
  r   r  r  r   r  r   r   r  r  _DownloadThreadr  startr  _monitor_message_queuer  )r3   r  r  r  dss   `    r6   r  z DownloaderGUI._download_threaded  s`     	  """F )1f%
 
 
 
S--..
 
 

 K,,..	 	<)/k)\"9:;F
 dh&;<< '2----)R////$&	
 	
 %'''  ##%%% 	r8   c                     | j         rN| j                                         | j                            d           | j                                         d S d S )Nabort)r  r  acquirer  r   releaserJ   s    r6   r  zDownloaderGUI._abort_download)  s^     	*'')))&--g666'')))))	* 	*r8   c                       e Zd Zd Zd ZdS )DownloaderGUI._DownloadThreadc                     || _         || _        || _        || _        || _        t
          j                            |            d S rC   )data_serverr   lockmessage_queuer  r  Threadr7   )r3   r  r   r  r  r  s         r6   r7   z&DownloaderGUI._DownloadThread.__init__0  sD    *DDJDI!.DDJ%%d+++++r8   c                    | j                             | j                  D ]}| j                                         | j                            |           | j        r6| j                            d           | j                                          d S | j                                         | j                                         | j                            d           | j                                         d S )Nabortedfinished)	r  r   r   r  r  r  r   r  r  r  s     r6   r  z!DownloaderGUI._DownloadThread.run8  s    '55djAA $ $	!!###"))#...: &--i888I%%'''FF	!!####I%%j111Ir8   N)rM   rN   rO   r7   r  r:   r8   r6   r  r  /  s2        	, 	, 	,	  	  	  	  	 r8   r  r	  c                      fd} j                                         sd S  j        D ]D}|dk    s|dk    r                                  d _        d j        d<    j        d d =  j        d d =  j                                          |dk    r! |d                                d            n+ j	        
                    d j        d           }| j        d	<    d S t          |t                    r                     |j                   t          |t                    rU ||j                   |j                             |j        j                                        d            d _         d S t          |t(                    r* |d
|j        j        z              xj        dz  c_        t          |t.                    r> j                            |j        j                    |d|j        j        z             t          |t4                    r |d|j        j        z             t          |t6                    r |d|j        j        z             @t          |t8                    r |d|j        j        z             ot          |t<                    r |d|j        j        z             t          |t>                    rI xj        dz  c_         |d|j        j        z                                   |j        j                   t          |tB                    r3                                                        |j        j                   F j        r
d j"        d<    j        d d =  j                                           j	        
                     j#         j$                  }| j        d	<   d S )Nc                 F    | j         d<                       |            d S r  r  r  s    r6   r7  z2DownloaderGUI._monitor_message_queue.<locals>.showI  r  r8   r  r  Fr(  r  zDownload aborted!r	  r  r;  r   zDownloading package %rr  r  r  zFinished installing %sr  zAborting download...)%r  r  r  r  r  rD  r  r  r  r  r  r  r;   r   r   r   r   rt   r  r   rf   ri   r  rq   r  rP  r   r   r   r.   r   rn   r  ry   rG  _MONITOR_QUEUE_DELAYr  )r3   r7  r   r  s   `   r6   r  z$DownloaderGUI._monitor_message_queueH  s   	 	 	 	 	
 "**,, 	F+ 3	1 3	1Cj  C9$4$4))+++$)!0:%f-,QQQ/.qqq1#++---)##D,---''----"hnnS$2EtLLG>EDM":; C11  1##CL1111C.. 1S[!!!;*LL000##D)))$)!C!788 103>3DDEEE  A%   C!455 1++CKN;;;->????C11 103;>ABBBB C!677 1/#+.@AAAAC!233 
1^ck&::;;;;C!344 1->????C!899 1  A%  :S^=NNOOO  !23333C!566 1))+++  000
 % 	A*@D' $QQQ'##%%% (..!:D<WXX29.///r8   TrC   )FrM   rN   rO   rP   r5  r  r9  r:  rs  rl  r  r   r   r   r   r   r  r8  r  r  rH  r3  r7   r  r  r  r  rE  r%  rC  r6  r  r  r  r?  r@  r4  r  _rowsr  r  r=  r  r  r  r  r  rA  r>  rp  r  r  r  r  r   dedentr  rw  rv  r  rJ  r  r  r  r  r  r  r  r   r  r:   r8   r6   r  r    s          GD QQ??NC  M<   CBBOA   G|||||  G|||||  G||||| 'O 	.,* "2	J #K (&O&O$I/5 /5 /5 /5b
 
 
m. m. m.^C& C& C&J0 0 0                 ( ( (	F 	F 	F	F 	F 	F> > > DE'S 'S 'SR  1 1 1& I*0 *0 *0X    ") ) )
& & &$ $ $8 8 8  ,  " " "+ + + 8?	 D:U U U- - - O
 
 
(4 4 47 7 7$* * *X* * *         )*      . I: I: I: I: I:r8   r  c                     t          | t                    r7t          | d          5 }t          |          cddd           S # 1 swxY w Y   t          |           S )zz
    Calculate and return the MD5 checksum for a given file.
    ``file`` may either be a filename or an open stream.
    rbN)r;   r<   r  _md5_hexdigest)r1  r)  s     r6   r]  r]    s    
 $ *$ 	*!&))	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	* 	*$s   AA	Ac                     t                      }	 |                     d          }|sn|                    |           .|                                S )NTr  )r   r"  r2   	hexdigest)fp
md5_digestr,  s      r6   r&  r&    sY    J!	"" 	%   	!
 !!!r8   c                 v    t          | ||          D ]&}t          |t                    rt          |          'dS )z\
    Extract the contents of the zip file ``filename`` into the
    directory ``root``.
    N)r'  r;   r   r   )r.   rootr  r   s       r6   r0   r0     sN    
 xw77 % %g|,, 	%G$$$	%% %r8   c              #     K   |rct           j                            dt          j                            |           d         z             t           j                                         	 t          j        |           }nM# t          j	        $ r t          | d          V  Y d S t          $ r}t          | |          V  Y d }~d S d }~ww xY w|                    |           |rt                       d S d S )Nr  r   zError with downloaded zip file)r  stdoutr#  r)   r*   r,   flushzipfileZipFile
BadZipFiler   r   
extractallr   )r.   r,  r  zfr  s        r6   r'  r'    s      
"'--*A*A!*DDEEE
_X&&   8%EFFFFF   8Q''''' MM$  s   )A> >!C"	C+CCc                    g }t          t          j                            | d                    D ]5\  }}}t          j        |j                  }| d| dt          j                            |j                  d          }t          d |                                D                       }|	                    dd|z             |	                    dd|j
        z             |	                    ddt          |j                  z             |	                    d	|           |                    d
          s|	                    d
|           |                    |           7t          t          t          j                            | d                              }	t                      }
||	z   D ]f}|                    d          |
v r%t!          d|                    d          z            |
                    |                    d                     gt%          j        d          }|                    t%          j        d                     |d                             t+          |d                      |                    t%          j        d                     |d                             t+          |	d                      t-          |           |S )a}  
    Create a new data.xml index file, by combining the xml description
    files for various packages and collections.  ``root`` should be the
    path to a directory containing the package xml and zip files; and
    the collection xml files.  The ``root`` directory is expected to
    have the following subdirectories::

      root/
        packages/ .................. subdirectory for packages
          corpora/ ................. zip & xml files for corpora
          grammars/ ................ zip & xml files for grammars
          taggers/ ................. zip & xml files for taggers
          tokenizers/ .............. zip & xml files for tokenizers
          etc.
        collections/ ............... xml files for collections

    For each package, there should be two files: ``package.zip``
    (where *package* is the package name)
    which contains the package itself as a compressed zip file; and
    ``package.xml``, which is an xml description of the package.  The
    zipfile ``package.zip`` should expand to a single subdirectory
    named ``package/``.  The base filename ``package`` must match
    the identifier given in the package's xml file.

    For each collection, there should be a single file ``collection.zip``
    describing the collection, where *collection* is the name of the collection.

    All identifiers (for both packages and collections) must be unique.
    rV   r   r   c              3   $   K   | ]}|j         V  d S rC   )	file_size)r\   zf_infos     r6   r  zbuild_index.<locals>.<genexpr>  s%      KK'G-KKKKKKr8   r"   z%sr!   r#   r   r   r   r   zDuplicate UID: %sr  r   c                 ,    |                      d          S Nr   rZ   rs   s    r6   r&  zbuild_index.<locals>.<lambda>	  s    7;;t;L;L r8   r   c                 ,    |                      d          S r:  rZ   rh   s    r6   r&  zbuild_index.<locals>.<lambda>	  s    PTAUAU r8   )_find_packagesr)   r*   r-   rW  r.   r,   r  infolistrt  rX  r]  r[   r   r   _find_collectionsr   addr   Elementru  r   _indent_xml)r,  base_urlrV   pkg_xmlr4  r   zipstatr   r"   r   uidsr_   top_elts                r6   build_indexrG    s   > H-bgll4.L.LMM ! !V'"+&&DDFDDRW]]2;%?%?%BDDKKR[[]]KKKKK 	OTM%9:::FD7?2333J}R['A'A ABBBHf%%%{{5!! 	$KKs### 	     (dM)J)JKKLLK 55D;& ! !88D>>T!!0488D>>ABBB$     !+..GNN;&z22333AJfX+L+LMMMNNNNN;&}55666AJf[.U.UVVVWWWNr8   r   c                    t          |           dk    r| j        pd                                dz   |z   dz   | _        | D ]}t          ||dz              | dd         D ]+}|j        pd                                dz   |z   dz   |_        ,| d         j        pd                                dz   |z   | d         _        dS dS )z
    Helper for ``build_index()``: Given an XML ``ElementTree``, modify it
    (and its descendents) ``text`` and ``tail`` attributes to generate
    an indented tree, where each nested element is indented by 2
    spaces with respect to its parent.
    r   r   r   r=  Nr   )r   r  rA  rA  tail)r?   r   r]   s      r6   rA  rA  	  s     3xx!||HN))++d2V;dB 	. 	.Ev}----"X 	K 	KE**1133d:VCdJEJJB*1133d:VCB |r8   c                    t           j                            t           j                            |          d                   d         |                     d          k    r6t          d                    |                     d                              t          fd|                                D                       rt          ddd          d	S )
zr
    Helper for ``build_index()``: Perform some checks to make sure that
    the given package is consistent.
    r   r   r   z&package identifier mismatch ({} vs {})c              3   V   K   | ]#}|k    o|                     d z              V  $dS r   Nr  r\   r   uids     r6   r  z!_check_package.<locals>.<genexpr>9	  s?      
V
VDCK:c	 : ::
V
V
V
V
V
Vr8   Zipfile ..zip does not expand to a single subdirectory r   N)	r)   r*   r+   r,   r[   r   r   r  namelist)rC  zipfilenamer4  rO  s      @r6   _check_packagerT  ,	  s     '

27==55a8
9
9!
<C{{4C4;;GKK<M<MsSS
 
 	

 
V
V
V
V
V
V
VVV 
j"%##sss,
 
 	

 
r8   c                 t   t          j        ddd| gt           j        t           j                  }|                                \  }}|j        dk    s|s|sJt          dt          j                            |           d         dt          j
        |                    |                                d	         S )
z
    Helper for ``build_index()``: Calculate the subversion revision
    number for a given file (by using ``subprocess`` to run ``svn``).
    svnr   z-v)r.  r  r   z#Error determining svn_revision for r   r   r  )
subprocessPopenPIPEcommunicate
returncoder   r)   r*   r,   r   r   )r.   rf  r.  r  s       r6   _svn_revisionr\  A	  s    
 		$)	 	 	A
 }}VV|qF&jw}}X&&q)))8=+@+@+@B
 
 	
 <<>>!r8   c              #     K   t          j        |           D ]h\  }}}|D ]_}|                    d          rHt           j                            ||          }t          j        |                                          V  `idS )z
    Helper for ``build_index()``: Yield a list of ElementTree.Element
    objects, each holding the xml for a single package collection.
    .xmlN)r)   r_  r&  r*   r-   r   r=   rt  )r,  dirname_subdirsr[  r.   xmlfiles         r6   r>  r>  T	  s      
 %'GDMM ; ; 5 	; 	;H  (( ;',,w99!'0088:::::	;; ;r8   c           
   #   D  K   ddl m} g }t          j        |           D ]~\  }}}d                     || |                    }|D ]-}|                    d          rvt          j                            ||          }|dd         dz   }		 t          j        |	          }
n(# t          $ r}t          d|	d	|           |d}~ww xY w	 t          j        |                                          }n(# t          $ r}t          d|d	|           |d}~ww xY wt          j                            |dd                   d
         |                    d          k    r)t          d|                    d          dd          t!          fd|
                                D                       rt          ddd          ||
|fV  |                    d          rt          j                            |          d         }t          j                            ||dz             }t          j                            |          s"t)          j        | d|dz    d| dd           /	 |                    d           o# t          $ r Y |w xY wdS )a  
    Helper for ``build_index()``: Yield a list of tuples
    ``(pkg_xml, zf, subdir)``, where:
      - ``pkg_xml`` is an ``ElementTree.Element`` holding the xml for a
        package
      - ``zf`` is a ``zipfile.ZipFile`` for the package's contents.
      - ``subdir`` is the subdirectory (relative to ``root``) where
        the package was found (e.g. 'corpora' or 'grammars').
    r   )
_path_fromr   r^  Nr<  r  zError reading file z!
r   r   zpackage identifier mismatch (z vs r   c              3   V   K   | ]#}|k    o|                     d z              V  $dS rL  rM  rN  s     r6   r  z!_find_packages.<locals>.<genexpr>	  sS         S[Cs)C)C%C     r8   rP  rQ  z exists, but z' cannot be found! This could mean that z can not be downloaded.r  )
stacklevelz.svn)nltk.corpus.reader.utilrc  r)   r_  r-   r&  r*   r0  r1  r   r   r   r=   rt  r,   r[   r  rR  r+   r  warningswarnr  )r,  rc  rV   r_  subdirsr[  relpathr.   xmlfilenamerS  r4  r  rC  resourcenamerO  s                 @r6   r<  r<  `	  sX      322222 H#%74== 3 3%((::dG4455 +	 +	H  (( * gll7H==)#2#.7Y 55BB  Y Y Y$%P;%P%PQ%P%PQQWXXYY)/<<DDFFGG  Y Y Y$%P;%P%PQ%P%PQQWXXY gmmK$455a8;;t$$++$*$+KK$5$5$5$5sss<        "      %*58SS###?  
 r7*****""6** 	!w//99!< gll7L64IJJw~~k22 M# V V,2G V V0<V V V#$   	NN6"""" 	 	 	D	e3 3sB   B$$
C	.CC	&C44
D>DD8J
JJc                  R    t          t                                                     d S rC   )r  _downloaderr  r:   r8   r6   download_shellro  	  s"    K  $$&&&&&r8   c                  R    t          t                                                     d S rC   )r  rn  r  r:   r8   r6   download_guirq  	  s"    +'')))))r8   c                  8    t                                            d S rC   )rn  r2   r:   r8   r6   r2   r2   	  s    r8   __main__)OptionParserz-dz--dirdirz!download package to directory DIRDIR)destrw  metavarz-qz--quietrC  
store_truezwork quietly)rw  actiondefaultrw  z-fz--forcer  z"download even if already installedz-ez--exit-on-errorrD  zexit if an error occursz-uz--urlr   NLTK_DOWNLOAD_URLzdownload server index url)rw  r{  rw  )r   )r   r   rC  r  rD  )r   rC  r  rD  r!  r>  )]rP   r?  r   r)   shutilrW  r  r   r  ro  rg  r0  hashlibr   	xml.etreer   r  tkinterr   r   r   r   r	   r
   r   r   r   r  r   nltk.draw.tabler   nltk.draw.utilr   r  r   urllib.errorr   r   urllib.requestr   rq  r   rS   rd   rf   rn   rq   ry   r}   r   r   r   r   r   r   r   r   r   r  r  r]  r&  r0   r'  rG  rA  rT  r\  r>  r<  rn  rB  ro  rq  r2   rM   optparsert  parser
add_optionr  r[   
parse_argsr  r  r   
downloaderpkg_idru  rC  r  rD  rvr:   r8   r6   <module>r     s  < <~Yt         				      



               ! ! ! ! ! !	GWWWWWWWWWWWWWWWWWWWWWW,,,,,,%%%%%%'''''''   GHHH - , , , , , , , " " " " " " Z( Z( Z( Z( Z( Z( Z( Z(z&+ &+ &+ &+ &+ &+ &+ &+\! ! ! ! ! ! ! !
% % % % %. % % %% % % % %/ % % %    +       ,       ,       -       )       *       '       $   # # # # #$ # # #! ! ! ! !' ! ! !) ) ) ) )0 ) ) )_
( _
( _
( _
( _
( _
( _
( _
(Db b b b b b b bJO: O: O: O: O: O: O: O:p     " " "% % % %   4C C CLD D D D 
 
 
*  &	; 	; 	;A A AV jll' ' '* * *   z%%%%%%\^^F
0          1     &     
233(     ''))OWdW-EFFFJ 
 		 		F$$!$[mm%3 %  B U{{w4{		 		 	 --!/	 	 	
 	
 	
 	
 	
w s   *A( (	A43A4