o
    -g!                     @   s   d dl Z d dlZd dlmZ d dlZd dlZd dlmZ d dl	m
Z
 eejeje h dZdd Zdd	 Zd
d Zdd ZedkrLe  dS dS )    N)OrderedDict)Selector)get_raw_data>	   vai-Latnvai-Vaiicutkvokkjndsprgvaic                  C   s   d} t | }dd |D }|d i }|D ]}g ||< |D ]}t|d |r0|| | qqtD ]	}||v r=||= q4|S )Nz!../raw_data/cldr_dates_full/main/c                 S   s   g | ]
}t d |s|qS )-[A-Z0-9]+$)research.0	shortname r   c/var/www/html/django-vendor/venv/lib/python3.10/site-packages/dateparser_scripts/order_languages.py
<listcomp>   s    
z-_get_language_locale_dict.<locals>.<listcomp>rootr   )oslistdirremover   matchappendavoid_languages)cldr_dates_full_diravailable_locale_namesavailable_language_nameslanguage_locale_dictlanguage_namelocale_namelanguager   r   r   _get_language_locale_dict   s&   

r%   c              	      s  dd }d}t |}t|}W d    n1 sw   Y  |d d }i |D ];}t|| d }z(|| d }|D ]}	t||	 d | }
|	v rU|	  |
7  < q<|
|	< q<W q) tyd   Y q)w | }|t fd	d
dd ttfdd
d}t	dt
|D ]}tdd|| ||< q|  }d}dd t|D }t|t|  fdd|D }t t| }g }|D ]}	tdd|	}||v r|||d |	 q||	 q|t| }ttt|}|S )Nc               
   S   s   g d} t d}t|jd}|jrPz!|d }|s tddd |D }|d d	kr1td
W |S  tyO } zt	| t	d | W  Y d }~S d }~ww | S )N)&enruesdetrfrfajazhviitnlptarplidkoukthhecssvroeldahufisrskbgnbhrltnohislcaetz:https://w3techs.com/technologies/overview/content_language)textz//table[@class='bars']//a/@hrefzNo bars foundc                 S   s   g | ]}| d ddqS )z+https://w3techs.com/technologies/details/cl -)replacestrip)r   ir   r   r   r   ]   s    zH_get_language_order.<locals>.get_most_common_locales.<locals>.<listcomp>r   r&   z!English is not the first languagez6The website could have changed, please update the code)
requestsgetr   rL   okxpathgetall
ValueError	Exceptionprint)old_common_localesresponseselbarsnew_most_common_localeser   r   r   get_most_common_locales(   s0   )z4_get_language_order.<locals>.get_most_common_localesz5../raw_data/cldr_core/supplemental/territoryInfo.jsonsupplementalterritoryInfo_populationlanguagePopulation_populationPercentc                    s    |  | fS Nr   x)language_population_dictr   r   <lambda>   s    z%_get_language_order.<locals>.<lambda>T)keyreversec                    s
     | S rf   )indexrg   )language_order_with_duplicatesr   r   rj      s   
 )rk   r   _rN   zD../dateparser_data/supplementary_language_data/date_translation_datac                 S   s   g | ]}|d d qS )Nr   )r   rh   r   r   r   r      s    z'_get_language_order.<locals>.<listcomp>c                    s   g | ]}| v r|qS r   r   r   )available_languagesr   r   r      s    z-\w+rM      )openjsonloadintfloatrX   sortedkeyssetrangelenr   subr   r   unioninsertrm   r   listmapstr)r!   r`   territory_info_filefterritory_contentterritory_info_data	territory
population	lang_dictr$   language_populationmost_common_localeslanguage_orderrm   cldr_languagessupplementary_date_directorysupplementary_languagesabsent_languagesremaining_languagesparent_languager   )rq   rn   ri   r   _get_language_order'   sf   E




r   c                 C   sB   i }t | D ]}d|vr|g||< q||dd  | q|S )NrN   r   )rx   splitr   )r   datalangr   r   r   generate_language_map   s   r   c                  C   s   t   t } t| }d}d}tj|st| dtj|ddd }t	 }|D ]}|| 
 v r9t| | ||< q(g ||< q(dtj|ddd }t|}dtj|ddd }	|d	 |	 d	 | d
 }
t|d}||
 W d    d S 1 sxw   Y  d S )Nz../dateparser/data/z$../dateparser/data/languages_info.pyzlanguage_order = ),z:    )
separatorsindentzlanguage_locale_dict = zlanguage_map = z


w)r   r%   r   r   pathisdirmkdirrt   dumpsr   ry   rx   r   rs   write)r!   r   parent_directoryfilenamelanguage_order_stringcomplete_language_locale_dictrk   language_locale_dict_stringlanguage_map_datalanguage_map_data_stringlanguages_info_stringr   r   r   r   main   sJ   

"r   __main__)rt   r   collectionsr   regexr   rR   parselr   dateparser_scripts.utilsr   chdirr   dirnameabspath__file__r   r%   r   r   r   __name__r   r   r   r   <module>   s     
(
