o
    ùT·jâ ã                   @   sê   d dl mZ d dlmZ d dlZd dlmZ d dlZd dl	Z
d dlZd dlZd dlmZ d dlZd dlZd dlZd dlmZ d dlmZ d dlmZmZmZmZ d d	lmZm Z m!Z!m"Z"m#Z# d
d
d
dœZ$G dd„ deƒZ%G dd„ dƒZ&dS )é    )Únew_session)ÚiscloseN)ÚMapping)Úsliding_window_view)Úutils)ÚYfConfig)Ú
_BASE_URL_Ú_PRICE_COLNAMES_Úperiod_defaultÚ
_SENTINEL_)ÚYFDataExceptionÚYFInvalidPeriodErrorÚYFPricesMissingErrorÚYFRateLimitErrorÚYFTzMissingErrorç{®Gáz„?©ÚGBpÚZAcÚILAc                   @   s€   e Zd ZdZdZdZdd„ Zdd„ Zdd	„ Zd
d„ Z	dd„ Z
dd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ Zdd„ ZdS )ÚHistoryMetadataa#  
    Dict-like lazy wrapper for PriceHistory metadata.

    Yahoo only returns ``tradingPeriods`` for intraday chart requests.  Most
    metadata is available from the normal/base history fetch, so defer the
    intraday metadata fetch until callers actually request ``tradingPeriods``.
    )ÚtradingPeriodsz<lazy-loaded>c                 C   s
   || _ d S ©N)Ú_price_history)ÚselfÚprice_history© r   ú\/home/dinkstrade/pdmp-scanner/venv/lib/python3.10/site-packages/yfinance/scrapers/history.pyÚ__init__    s   
zHistoryMetadata.__init__c                 C   s   | j jpi S r   )r   Ú_history_metadata©r   r   r   r   Ú	_metadata#   ó   zHistoryMetadata._metadatac                 C   s2   t |  ¡  ¡ ƒ}| jD ]}||vr| |¡ q|S r   )Úlistr!   ÚkeysÚ
_LAZY_KEYSÚappend)r   r$   Úkr   r   r   Ú_keys&   s   

€zHistoryMetadata._keysc                 C   s.   t |  ¡ ƒ}| jD ]}||vr| j||< q	|S r   )Údictr!   r%   Ú_LAZY_VALUE)r   Údr'   r   r   r   Ú_display_dict-   s   

€zHistoryMetadata._display_dictc                 C   s,   || j v r||  ¡ vr| j ¡  |  ¡ | S r   )r%   r!   r   Ú&_load_history_metadata_trading_periods©r   Úkeyr   r   r   Ú__getitem__4   s   
zHistoryMetadata.__getitem__c                 C   ó   t |  ¡ ƒS r   )Úiterr(   r    r   r   r   Ú__iter__9   r"   zHistoryMetadata.__iter__c                 C   r1   r   )Úlenr(   r    r   r   r   Ú__len__<   r"   zHistoryMetadata.__len__c                 C   s   ||   ¡ v S r   ©r(   r.   r   r   r   Ú__contains__?   r"   zHistoryMetadata.__contains__c                 C   ó   |   ¡ S r   r6   r    r   r   r   r$   B   ó   zHistoryMetadata.keysc                    ó   ‡ fdd„ˆ   ¡ D ƒS )Nc                    s   g | ]}|ˆ | f‘qS r   r   ©Ú.0r'   r    r   r   Ú
<listcomp>F   ó    z)HistoryMetadata.items.<locals>.<listcomp>r6   r    r   r    r   ÚitemsE   ó   zHistoryMetadata.itemsc                    r:   )Nc                    s   g | ]}ˆ | ‘qS r   r   r;   r    r   r   r=   I   ó    z*HistoryMetadata.values.<locals>.<listcomp>r6   r    r   r    r   ÚvaluesH   r@   zHistoryMetadata.valuesc                 C   s   ddl m} ||  ¡ ƒS )Nr   )Úpformat)ÚpprintrC   r,   )r   rC   r   r   r   Ú__repr__K   s   zHistoryMetadata.__repr__c                 C   r8   r   )rE   r    r   r   r   Ú__str__O   r9   zHistoryMetadata.__str__N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r%   r*   r   r!   r(   r,   r0   r3   r5   r7   r$   r?   rB   rE   rF   r   r   r   r   r      s"    r   c                   @   sj  e Zd Zd5dd„Zejeddddddddddddfdejfd	d
„ƒZ	d6dejfdd„Z
dd„ Zdd„ Zefdefdd„Zd7dejfdd„Zd7dejfdd„Zd7dejfdd„Zd8dejfdd„Zd5dejfdd„Zejd9dd „ƒZd!d"„ Zd:d#d$„Zejd%d&„ ƒZejd'd(„ ƒZejd)d*„ ƒZejd+d,„ ƒZejd-d.„ ƒZejd/d0„ ƒZejd1d2„ ƒZejd;d3d4„ƒZ dS )<ÚPriceHistoryNc                 C   s\   || _ | ¡ | _|| _|ptƒ | _i | _d | _d| _d | _	d | _
d | _d | _d | _d | _d S )NF)Ú_dataÚupperÚtickerÚtzr   ÚsessionÚ_history_cacher   Ú_history_metadata_formattedÚ_history_metadata_lazyÚ
_dividendsÚ_splitsÚ_capital_gainsÚ_reconstruct_start_intervalÚ_last_error)r   ÚdatarN   rO   rP   r   r   r   r   T   s   

zPriceHistory.__init__Ú1dFTé
   Úreturnc           B   
      sª  t  ¡ }|rtjdtdd� |}|tkr d}|s|rd}nd}n|}|	rš|dv rš|dkr0tdƒ‚|du r˜|du r˜|dur˜| j}|du rft| j	ƒ}t
|ƒ}| d	d
¡d
 | _|s[tjjs]|‚| |¡ t  ¡ S |dkrzt tj d¡ |¡jd
d
¡}ntj d¡ |¡ ¡ }|t  |¡8 }|tjdd�8 }|}d}d}|}|}|sª|sª|rÔ| ¡ dkrÔ| j}|du rÔt| j	ƒ}t
|ƒ}| d	d
¡d
 | _|sÉtjjsË|‚| |¡ t  ¡ S |rât  ||¡}t| ¡ ƒ}|rðt  ||¡}t| ¡ ƒ}|du �r |sü|süd}n—|�s|t  d¡ }t| ¡ ƒ}n†|�stj d¡ |¡}t| ¡ ƒ}ns| ¡ dk�r^|du �r2tt  ¡ ƒ}|du �r]|dk�rA|d }n|dv �rK|d }n|dv �rU|d }n|d }|d7 }n5|�rh|�rhtdƒ‚|�sn|�r“t  |¡}|du �r‚|| }t| ¡ ƒ}|du �r‘|| }t| ¡ ƒ}d}|�s™|�rŸ||dœ}n| ¡ }d|i}| ¡ |d< ||d< |d dk�r¼d|d< d|d < t!|ƒ}| j}dD ]}||v �rãt
tj|| d!d"� "d¡ |¡ƒ||< �qÉ| | j	› d#t
|ƒ› �¡ t#› d$| j	› �}d}| j$j%}tj d¡}|du�r&tj|d!d"� "d¡}tjd%d&�} ||  |k�r&| j$j&}z||||d'�}d(|j'v �s9|du �r=t(d)ƒ‚| )¡ }W n t*�yK   ‚  t+�y]   |�sZtjj�s[‚ Y nw z1|�pci  %d*¡�pii }!|! %d+¡}"t,|"t-ƒ�rŒt.|"ƒd,k�rŒ|"d, �p‚i }#|# %d-¡�pŠi }$ni }$W n t+�y›   i }$Y nw |$| _/d.| _0d| _1|	| j/d/< |d d0 d1v }%d2}&|�sÃ|du �sÃ| ¡ dk�r8|&d3|› d4�7 }&|du�rÖ|&|› 7 }&n&|%�sì|&tj|d!d"� "d¡ |¡ ¡ › 7 }&n|&tj|d!d"� "d¡ |¡› 7 }&|&d57 }&|du�r|&|› d6�7 }&n3|%�s%|&tj|d!d"� "d¡ |¡ ¡ › d6�7 }&n|&tj|d!d"� "d¡ |¡› d6�7 }&n|&d7|› d6�7 }&d.}'|du �sMt,|t!ƒ�sVt2| j	|&ƒ}d8}'nªt,|t!ƒ�rtd9|v �rt|&d:|d9 › d6�7 }&t2| j	|&ƒ}d8}'nŒd*|v �r®|d* �r®|d* d; �r®|d* d; d< }(|d | ¡ k�r£|(d3|› d=|d › d6�7 }(t2| j	|&|(d>�}d8}'nRd*|v�s×|d* �r×|d* d+ du �s×|d* d+ �r×|d* d+ d, d? d@ d, �sàt2| j	|&ƒ}d8}'n |�r || j/dA v�r t  3|¡�s t4| j	|dB 5| j/dA ¡ƒ}d8}'|'�r2t
|ƒ}| d	d
¡d
 | _|�stjj�s|‚| |¡ | j6du�r.| j6|k�r.d| _6t  ¡ S | j/dC })|)dDv }*| j/dE }+| j/dF },|�rw|| j/ %dAg ¡v�rwtt  ¡ ƒ}tj|d!d"� "d¡}tj 7|¡}|t  |¡8 }|tjdd�8 }t  8|d* d+ d, ¡}-|�rŸ|-j9�sŸ|-j:d0 | d¡ "d¡k�rŸ|- ;|-j:d0 ¡}-|-j9�rª| j	› dG�}.n%t.|-ƒd
k�r¾| j	› dH|-j:d, › dI�}.n| j	› dH|-j:d, › d5|-j:d0 › �}.| |.¡ | ¡ dk�r| | j	› dJ�¡ |- <dK¡}/tj=|/ >¡ j:|/dL  ?¡ |/dM  @¡ |/dN  A¡ |/dO  >¡ |/dP  >¡ |/dQ  B¡ dRœdS�}-| j0�s t  C| j/¡| _/d8| _0t  D|-||+¡}-t  E|-|¡}-|d d0 d1v }%|�sG|%�rG| j/dT }0t  F|-||0¡}-|-j9�rR| j	› dU�}.n%t.|-ƒd
k�rf| j	› dV|-j:d, › dI�}.n| j	› dV|-j:d, › d5|-j:d0 › �}.| |.¡ t  G|d* d+ d, ¡\}1}2}3|*�s�d}3|2du�r¤t  D|2||+¡}2|2dW  HdX¡| _Int J¡ | _I|1du�røt  D|1||+¡}1dF|1jKv �rï|1 HdX¡| _L| j/dF }4|4du �rÍd2}4|1dF |4k}5|5 M¡ �rç|	�rç|4d2k�rç|  N|1|4|	¡}1|1j;dFd
dY�}1n|1dZ  HdX¡| _Lnt J¡ | _L|3du�rt  D|3||+¡}3|3d[  HdX¡| _Ont J¡ | _O|du�rL|-j9�sL|-j:d,  Pd\¡}6|1du�r4|1jQ|6d… }1|3du�r@|3jQ|6d… }3|2du�rL|2jQ|6d… }2|du�ry|t Rd
¡ }7|1du�rc|1d|7… }1|3du�rn|3d|7… }3|2du�ry|2d|7… }2|d d0 d1v }%|%�s¾t S|-j:j¡j"|+d8d]d^�|-_:|1jTd, d,k�r¨t S|1j:j¡j"|+d8d]d^�|1_:|2jTd, d,k�r¾t S|2j:j¡j"|+d8d]d^�|2_:|- U¡ ‰ |1jTd, d,k�rÑt  Vˆ |1|¡‰ dZˆ jKv �rãd,ˆ jQˆ dZ  W¡ dZf< nd_ˆ dZ< |2jTd, d,k�röt  Vˆ |2|¡‰ dWˆ jKv �rd,ˆ jQˆ dW  W¡ dWf< nd_ˆ dW< |*�r4|3jTd, d,k�rt  Vˆ |3|¡‰ d[ˆ jKv �r0d,ˆ jQˆ d[  W¡ d[f< nd_ˆ d[< ˆ j9�r?| j	› d`�}.n%t.ˆ ƒd
k�rS| j	› daˆ j:d, › dI�}.n| j	› daˆ j:d, › d5ˆ j:d0 › �}.| |.¡ t jXˆ |d |+||	|,db�\‰ }8|8du�rˆ|8dO |8jYdcœ| j/dd< ˆ ˆ j:jZdedf�  ‰ |	�rP| | j	› dg�¡ ˆ  U¡ ‰ |,}9|  [ˆ |,¡\‰ },}:|,| j/dF< ˆ dQ  W¡ };|; M¡ �rÄd,ˆ jQ|;dQf< |  \ˆ |||,¡‰ ˆ j9�sþ|  ]ˆ j^d0d… ||+|¡}<dhˆ jKv�rçd.ˆ dh< dh|<jKv�rñd.|<dh< t _ˆ  ;ˆ j:d0 ¡|<g¡‰ di| j	v�r|  `ˆ ||+|¡‰ |  aˆ ||+¡‰ |  ]ˆ ||+|¡‰ |  bˆ ¡‰ |:�rLtc|9 }=tdD ]}>ˆ |>  |=  < �q)ˆ dZ  |=  < |9| j/dF< dj| j/v �rL| j/dj= ˆ  U¡ ‰ z|�rZt  eˆ ¡‰ n|�rbt  fˆ ¡‰ W n5 t+�y˜ }? z(|�sttjj�su‚ |�r}dk|? }ndl|? }|| _| dm| j	|f ¡ W Y d}?~?nd}?~?ww |�r¬tg hˆ |d* d+ d, d- dn ¡‰ ˆ dQ  id,¡ jtgjk¡ˆ dQ< |%�rÁdoˆ j:_YndXˆ j:_Y|�sÑˆ j;g dp¢dqdr�‰ |
�sþtddQg g dp¢ }@‡ fdsdt„|@D ƒ}@ˆ |@  W¡ ˆ |@ d,kB jld
dY�}Aˆ  ;|Aj:|A ¡‰ ||k�	r|  mˆ |||¡‰ ˆ j9�	r| j	› du�}.n%t.ˆ ƒd
k�	r*| j	› dvˆ j:d, › dI�}.n| j	› dvˆ j:d, › d5ˆ j:d0 › �}.| |.¡ ˆ  n¡ ‰ | j6du�	rS| j6|k�	rSd| _6ˆ S )waµ  
        :Parameters:
            period : str
              | Valid periods: 1d,5d,1mo,3mo,6mo,1y,2y,5y,10y,ytd,max
              | Default: '1mo' if start & end None
              | Can combine with start/end e.g. end = start + period
            interval : str
              | Valid intervals: 1m,2m,5m,15m,30m,60m,90m,1h,1d,5d,1wk,1mo,3mo
              | Intraday data cannot extend last 60 days
              | Note: 30m data is fetched from Yahoo as 15m then resampled, to work around a Yahoo API bug
            start : str
              | Download start date string (YYYY-MM-DD) or _datetime, inclusive.
              | Default: 99 years ago
              | E.g. for start="2020-01-01", first data point = "2020-01-01"
            end : str
              | Download end date string (YYYY-MM-DD) or _datetime, exclusive.
              | Default: now
              | E.g. for end="2023-01-01", last data point = "2022-12-31"
            prepost : bool
              | Include Pre and Post market data in results?
              | Default: False
            auto_adjust : bool
              | Adjust all OHLC automatically?
              | Default: True
            back_adjust : bool
              | Back-adjusted data to mimic true historical prices
            repair : bool
              | Fixes price errors in Yahoo data: 100x, missing, bad dividend adjust.
              | Default: False
              | Full details at: :doc:`../advanced/price_repair`.
            keepna : bool
              | Keep NaN rows returned by Yahoo?
              | Default: False
            rounding : bool
              | Optional: Round values to 2 decimal places?
              | Default: False = use precision suggested by Yahoo!
            timeout : None or float
              | Optional: timeout fetches after N seconds
              | Default: 10 seconds
            raise_errors : bool
                If True, then raise errors as Exceptions instead of logging.
        zF'raise_errors' deprecated, do: yf.config.debug.hide_exceptions = Falseé   )Ú
stacklevelNÚ1mo)Ú5dÚ1wkr_   Ú3mor`   z<Yahoo's interval '5d' is nonsense, not supported with repairú: é   ÚytdÚUTCé   ©ÚdaysrZ   ÚmaxÚ1mi Œ
 )Ú2mÚ5mÚ15mÚ30mÚ90mi O )Ú1hÚ60mi gÂl   €j-t zASetting period, start and end is nonsense. Set maximum 2 of them.)Úperiod1Úperiod2ÚrangeÚintervalÚincludePrePostro   rn   zdiv,splits,capitalGainsÚeventsÚs)Úunitz: Yahoo GET parameters: ú/v8/finance/chart/é   )Úminutes)ÚurlÚparamsÚtimeoutzWill be right backz)*** YAHOO! FINANCE IS CURRENTLY DOWN! ***ÚchartÚresultr   ÚmetaFú
YF repair?éÿÿÿÿ©ÚmÚhÚ ú (ú ú -> ú)z	 (period=TÚstatus_codez(Yahoo status_code = ÚerrorÚdescriptionz resampled from )Úyahoo_reasonÚ
indicatorsÚquoteÚvalidRangesú, ÚinstrumentType)Ú
MUTUALFUNDÚETFÚexchangeTimezoneNameÚcurrencyz$: yfinance received OHLC data: EMPTYz: yfinance received OHLC data: z onlyz: resampling 30m OHLC from 15mÚ30minÚOpenÚHighÚLowÚCloseú	Adj CloseÚVolume)rœ   r�   rž   rŸ   r    r¡   )ÚindexrY   r   z: OHLC after cleaning: EMPTYz: OHLC after cleaning: úStock SplitsÚDate©ÚaxisÚ	DividendsúCapital GainsÚDÚshift_forward)Ú	ambiguousÚnonexistentç        z$: OHLC after combining events: EMPTYz: OHLC after combining events: )Úrepairrš   )ÚPriceÚTimeÚ	lastTradeÚfirst)Úkeepz: checking OHLC for repairs ...ú	Repaired?ú=ÚcurrencyRepairedzauto_adjust failed with %szback_adjust failed with %sz%s: %sÚ	priceHintÚDatetime©r§   r£   r¨   Úignore)ÚcolumnsÚerrorsc                    ó   g | ]	}|ˆ j v r|‘qS r   ©r»   ©r<   Úc©Údfr   r   r=   o  ó    z(PriceHistory.history.<locals>.<listcomp>z : yfinance returning OHLC: EMPTYz: yfinance returning OHLC: )or   Úget_yf_loggerÚwarningsÚwarnÚDeprecationWarningr
   Ú
ValueErrorrO   r   rN   ÚstrÚsplitrX   r   ÚdebugÚhide_exceptionsr�   Úempty_dfÚ	_datetimeÚdateÚpdÚ	TimestampÚnowÚ
tz_convertÚyearÚ_interval_to_timedeltaÚ	timedeltaÚlowerÚ_parse_user_dtÚintÚ	timestampÚ_timeÚtimer)   Útz_localizer   rL   ÚgetÚ	cache_getÚtextr   Újsonr   Ú	ExceptionÚ
isinstancer#   r4   r   rR   rS   r   Úis_valid_period_formatr   ÚjoinrW   ÚfromtimestampÚparse_quotesÚemptyr¢   ÚdropÚresampleÚ	DataFrameÚlastr²   rj   ÚminÚsumÚformat_history_metadataÚ	set_df_tzÚfix_Yahoo_dst_issueÚ'fix_Yahoo_returning_prepost_unrequestedÚparse_actionsÚrename_axisrU   ÚSeriesr»   rT   ÚanyÚ_dividends_convert_fxrV   ÚfloorÚlocÚ	TimedeltaÚto_datetimeÚshapeÚ
sort_indexÚsafe_merge_dfsÚisnaÚ!fix_Yahoo_returning_live_separateÚnameÚ
duplicatedÚ_standardise_currencyÚ_fix_bad_div_adjustÚ_fix_zeroesÚilocÚconcatÚ_fix_unit_mixupsÚ_fix_bad_stock_splitsÚ_repair_capital_gainsÚ_CURRENCY_CONVERSIONSr	   Úauto_adjustÚback_adjustÚnpÚroundÚfillnaÚastypeÚint64ÚallÚ	_resampleÚ_consolidate)Br   Úperiodrv   ÚstartÚendÚprepostÚactionsr  r  r®   ÚkeepnaÚroundingr€   Úraise_errorsÚloggerÚinterval_userÚperiod_userrO   Ú
_exceptionÚerr_msgÚ
start_userÚend_userÚstart_dtÚend_dtÚ	period_tdr   Úparams_prettyr'   r~   rY   Úget_fnÚdt_nowÚ
data_delayÚ
safe_chartÚresult_listÚ
first_itemrƒ   ÚintradayÚ_price_data_debugÚfailr‘   Ú
quote_typeÚexpect_capital_gainsÚtz_exchangerš   ÚquotesÚmsgÚquotes2ÚtpsÚ	dividendsÚsplitsÚcapital_gainsÚprice_currencyÚf_currency_mismatchÚstart_dÚend_dt_sub1Ú
last_tradeÚoriginal_currencyÚprices_scaledÚf_naÚdf_lastr‡   rÀ   ÚeÚdata_colnamesÚmask_nan_or_zeror   rÁ   r   Úhistoryh   sô  0

 


€







€



&€
ýÿÿ
€ÿ

& 
*&"R"



"








ú	
"

















"






€
 €ø
 
 
"
zPriceHistory.historyrj   c                 C   sN   |||f}|| j  ¡ vr"| j|||dd�}|| j| j| jdœ| j |< | j | S )NT)r  rv   r®   r  )Úpricesr9  r:  úcapital gains)rQ   r$   rH  rT   rU   rV   )r   r  rv   r®   Ú	cache_keyrÂ   r   r   r   Ú_get_history_cache…  s   
þ
zPriceHistory._get_history_cachec                 C   s,   |t kr| jd ur| j dd¡}|S d}|S )Nr„   F)r   r   rÞ   ©r   r®   r   r   r   Ú_resolve_metadata_repairŽ  s   
ÿz%PriceHistory._resolve_metadata_repairc                 C   sî   | j d urd| j v rd S | j d u r|  ¡  | j sd S t› d| j› �}ddddœ}d }z| jj||d� ¡ }|d d	 d
 d }|d }W n
 tyN   Y d S w | j  d¡}|d u r^| d¡}|d u rdd S ||dœ}t 	|¡}|d | j d< d S )Nr   r{   r`   rq   T)ru   rv   rw   )r~   r   r�   r‚   r   rƒ   r™   )r™   r   )
r   Úget_history_metadatar   rN   rL   rÞ   rá   râ   r   rï   )r   r~   r   rY   rƒ   Útrading_periodsrO   Úmdr   r   r   r-   —  s6   
ÿÿ
ÿ
z3PriceHistory._load_history_metadata_trading_periodsc                 C   sx   |   |¡}| jdu r z| jdd|d�d  W n	 ty   Y nw | jdu r/t | j¡| _d| _| jdu r9t| ƒ| _| jS )zÕ
        repair default value depends on whether user requested price repair
        with previous history() call. If user did not set repair here, then
        it is set to match previous history() call.
        Nr`   rZ   )r  rv   r®   rI  FT)	rN  r   rL  râ   rR   r   rï   rS   r   rM  r   r   r   rO  ¶  s   

ÿ


z!PriceHistory.get_history_metadatac                 C   ó   | j d||d�d S )NrZ   ©rv   r  r®   r9  ©rL  ©r   r  r®   r   r   r   Úget_dividendsÏ  ó   zPriceHistory.get_dividendsc                 C   rR  )NrZ   rS  rJ  rT  rU  r   r   r   Úget_capital_gainsÒ  rW  zPriceHistory.get_capital_gainsc                 C   rR  )NrZ   rS  r:  rT  rU  r   r   r   Ú
get_splitsÕ  rW  zPriceHistory.get_splitsc                    s  | j |d�}|d ‰|d }|d urCt|tjƒrCd|jv rCt ˆjddd�|d¡‰ˆd  d	¡ˆd< ˆd  d
¡ˆd< ˆj	ddid�‰g d¢}ˆ‡fdd„|D ƒ ‰ g d¢}‡ fdd„|D ƒ}ˆ ˆ | dkj
dd� ‰ |D ]}ˆ | d
k ¡ r~ˆ j|dd�‰ qmˆ S )N)r  rI  r9  rš   r§   rd   r¥   rZ   r‰   r­   úDividends FXr¾   )r§   rZ  r£   r¨   c                    r½   r   r¾   r¿   rÁ   r   r   r=   æ  rÃ   z,PriceHistory.get_actions.<locals>.<listcomp>r¹   c                    r½   r   r¾   r¿   )r  r   r   r=   é  rÃ   r   )rL  rã   rÐ   rë   r»   r   rþ   ré   r  Úrenamerö   r  )r   r  rY   ÚdivsÚcolsÚcols_numericrÀ   r   )r  rÂ   r   Úget_actionsØ  s$   €zPriceHistory.get_actionsc                 C   sÚ  ||kr|S d }d}|dkr+|dkr(d}t  tj ¡ j› d�¡}| |jj¡}nRd}nO|dkrId}|dkrHt  tj ¡ j› d�¡}| |jj¡}n1|d	krPd
}n*|dkrk|dkr[d}	n
tj ¡  	d¡ 
¡ }	d|	› �}n|dkrrd}ntd|› d�ƒ‚ddddddddœ}
d|jv r�d|
d< d|jv r˜|
d |
d< d|jv r¡d|
d< d|j|d  d!kd f< |dkr½|j|d"d"|d#� |
¡}n|j|d"d"|d$� |
¡}d!|j|d  dkd f< |d  d%¡ ¡ }d&D ]}||  |¡||< qß|S )'NÚepochra   re   Ú7Dz-01-01zW-MONr`   Ú5Dr_   ÚMSrb   ÚJANz%bzQS-rZ   Ú1Dz(Not implemented resampling to interval 'ú'r²   rí   rj   rì   rî   Úprod)rœ   rž   r�   rŸ   r¡   r§   r£   r´   rö   r    rŸ   r¨   ç      ð?r£   r­   Úleft)ÚlabelÚclosedÚorigin)rj  rk  Úoffsetrd   ©rœ   r�   rž   rŸ   )rÐ   rÑ   rÎ   ÚdatetimerÒ   rÔ   rÝ   r¢   rO   ÚstrftimerM   rÈ   r»   rù   rê   ÚaggÚshiftÚffillr  )r   rÂ   Údf_intervalÚtarget_intervalr  rm  rl  Úresample_periodÚ
year_startÚalign_monthÚresample_mapÚdf2Ú
prev_closerÀ   r   r   r   r  ð  sX   €þ


zPriceHistory._resampler…   c           X         s‚  t  ¡ }dˆ| jdœ}tˆ tjƒstdtˆ ƒƒ‚ˆdkrˆ S ddlm	} ˆdd … dv r1d	}d
}nd	}‡ fdd„t
D ƒ}	|	dg }
g d¢‰‡fdd„ˆD ƒ}‡fdd„ttˆƒd ƒD ƒ}d d tjdd�dœ}dD ]
}tjdd�||< qitjdd�|d< ˆ|v r‰|ˆ }|ˆ }n| dˆ› d�¡ dˆ jvr›d
ˆ d< ˆ S | jd u r¥ˆ| _ˆ| jkrÅˆ|| j krÅd | j|| j ˆ¡}|j||d� ˆ S ˆ  ¡ ‰ ˆ |
  ¡ |k}|jdd�}|| }|d u râd }n|tjdd�8 }tj d¡| }| ˆ jj¡ d¡}|jd |› d!ˆ› d"|› �|d� |d u�r?ˆ j|k}||@ }| ¡ �s?d#t |jdd�¡› d$�}|j||d� dˆ jv�r=d
ˆ d< ˆ S ˆ j| }t|ƒdk�r^|jd%|d� dˆ jv�r\d
ˆ d< ˆ S ˆ   ¡ }d|jv�rld
|d< ˆ |	  !¡ jdd� }|ˆ |	  ¡ |kj"dd�@ }ˆ | }|d gg}|d&k�rœt#j$j$d'd(�}n9|d)k�r©t#j$j$d'd(�}n,|d*k�r¶t#j$j$d'd(�}n|d+k�rÃt#j$j$dd(�}n|dk�rÏtjd,d�}ntjdd�}|jd-|› �|d� tdt|ƒƒD ]%}|| }| %¡ |d. d  %¡ | k �r|d.  &|¡ �qæ| &|g¡ �qæ|jd/|d� |D ]}| d0|d › d1|d. › �¡ �qtt|ƒƒD ]…}|| }|d }|jj'|gd2d3�d } | dk�rl|d u �sW|j| d  |k�rl|�rh|j| d   %¡ | %¡ k�rl| d8 } |d. }!|jj'|!gd2d3�d }"|"t|ƒd k �r™|�r•|j|"d   %¡ |! %¡ k�r™|"d7 }"|j| |"d … }#||  |# (¡ 7  < ||  )¡  �q.d}$tt|ƒd d.d.ƒD �]}}|| }ˆ ˆ j *|¡ }%| d4t+|%ƒ ¡ |d }&|& %¡ }'d
}(|d+k�røtj% ,¡ |' tjd5d�k�rød	}(n|d6v �rtj% ,¡ |' tjd7d�k�rd	}(|(�r$d8|�r|&n|'› d9�}|j||d� �qÀtjdd�})ˆd)v �r<|'| }*|d.  %¡ | }+nˆd*k�rL|'}*|d.  %¡ | }+n
|d }*|d. | }+|*|)8 }*|+|)7 }+|�rk|* %¡ }*|+ %¡ |) }+|d u�rwt-| %¡ |*ƒ}*|jd:|› d;|› d<|*› d=|+› �|d� t  ¡ }t.|d>ƒ�r�|j/},| 0t1j2¡ | j3|*|+|d
d	|d	d?�}-t.|d>ƒ�r´| 0|,¡ |-d u �s½|-j4�rÑd8|�rÃ|&n|'› d9�}|j||d� �qÀ|-j5|d |d. ||  tjdd@� …   ¡ }-|-j4�rdAˆ› dB|�rõ|&n|'› dC�}|j||d� �qÀd|-dD< ˆd)k�r0g dE¢}.|.|%jd  6¡ dF d dF  }/|-j 7d ¡ 8dG|/ ¡j9|-dH< dH}0n.ˆd*k�rAt :|-jj%¡|-dI< dI}0nd|-j5|-j *|%j¡dDf< |-dD  ;¡ |-dJ< |-j<dDdd�}-dJ}0|-|-|	dKg   !¡ j"dd�  }-|- =|0¡}1|1j>dLdMdNdOdPdQdRdSdT�j?dUdVidW�}2|0dXv �r•|2j 7|-jj¡|2_n|-dJ  @¡ |-dY< t &|-jd g|-j|-dJ  @¡ dk ¡}3|3|2_| dZt+|2ƒ ¡ t A|%j|2j¡}4t|4ƒdk�rÝd[ˆ› d\|'› d]�}|j||d� �qÀd^d_g}5|2|2j *|4¡ |5  ¡ }6|%|%j *|4¡ |5  ¡ }7|7|k}8|8t B|6¡ @ }8|8 ¡ �s|jd`|'› d]�|d� �qÀtt|5ƒƒD ]/}9|8d d …|9f  }:|: ¡ �rK|7jCjD�s6|7  ¡ }7|6jCjD�s?|6  ¡ }6d|7|:|9f< d|6|:|9f< �q|7|8 |6|8  };|1 E¡ }<|2j|<_|<|<j *|4¡  ¡  FtG¡}<|<jCjD�ss|<  ¡ }<|<d d …d f }<t H|<t|5ƒ¡}<|<|8 }<t|;ƒdk�rÝ|;  ¡ }=da}>db}?t I|=¡ Jd.d¡}@t-t K|>¡t K|> ¡ ƒ}A||A|?dc� L|@¡‰ddd„ t Mˆ¡D ƒ}B|B�rÛt-|B‡fdedf„dg�}Ct N|=ˆ|Ck ¡}D|D|=ˆ|Ck< |=};tjO|;|<dh�}EtP|Ed ƒdak �rðd}En=di}Fd|F }GtP|E|F d ƒdak �rt
D ]}H|2|H  |F9  < �qd}EntP|E|G d ƒdak �r-t
D ]}H|2|H  |G9  < �qd}E|jdj|Edk›�|d� tQdl|E dƒ}ItQ|Edƒ}E|Edk�rO|Idk�rOn-|Edk�re|2|	  |E9  < |2d  |E  < n|Idk�r||2|	  dl|I 9  < |2d  |I9  < |j R|%jd. ¡d }J|Jt|ƒk�rÀ|%dV |%d_  }K|%|	  ¡ |kjdd�}L|L ¡ �r²tjS|K|L< |K T¡  U¡ }K|KjVd. }Mt B|M¡�r¿dl}Mn|dV jV|J |d_ jV|J  }M|2dV jVd. |2d_ jVd.  }Nt B|N¡�rût B|2dK jVd. ¡�sû|2dV jVdm |2d_ jVdm  }N|Jt|ƒk �r$|dK jV|J dk�r$|dK jV|J }O|2d_ jVd. }Pd|O|P  }K|M|K9 }M|M|N }Q|Qdk�r5|2dV  |Q9  < |%j|%|	dg  |k ¡ jdd� }Rg }S|RD ]}||2jv�rX|S &|¡ �qKt|Sƒdk�rl|jdnt+|Sƒ |d� |RD ]Î}||2jv�rx�qn|2j5| }Tˆd)k�r”|2jV|2j R|¡d  }U|-j5|d … }-ˆ j5| }V|Vj|V|k  ¡ }Wdo|Wv �r°|Tdo |j5|dof< dp|Wv �r¾|Tdp |j5|dpf< d^|Wv �r÷ˆd)k�rî||-jd k�rî|Ud_ |j5|d^f< tW|j5|d^f |j5|dpf ƒ|j5|dpf< n	|Td^ |j5|d^f< d_|Wv �r|Td_ |j5|d_f< |TdV |j5|dVf< ndV|Wv �r|TdV |j5|dVf< d|Wv �r0|Td  Q¡  Fdq¡|j5|df< d	|j5|df< |$d7 }$�qn�qÀ|S )rNzprice-reconstruct©Úyf_catÚyf_intervalÚ	yf_symbolz#'df' must be a Pandas DataFrame notrk   r   )ÚDBSCANrd   )r+   ÚwkÚmoTFc                    s   g | ]}|ˆ v r|‘qS r   r   r¿   rÁ   r   r   r=   =  r>   z=PriceHistory._reconstruct_intervals_batch.<locals>.<listcomp>r¡   )ra   rZ   rq   ro   rn   rm   rl   rk   c                    s   i | ]}|t  ˆ ¡“qS r   )r   rÕ   ©r<   Úi)rv   r   r   Ú
<dictcomp>C  r>   z=PriceHistory._reconstruct_intervals_batch.<locals>.<dictcomp>c                    s   i | ]}ˆ | ˆ |d   “qS )rd   r   rƒ  )Ú	intervalsr   r   r…  D  s    iÚ  rh   )ra   rZ   rq   )ro   rn   rm   rl   é<   r|   z,Have not implemented price reconstruct for 'z' interval. Contact developersr´   z%Hit max depth of 2 ('{}'->'{}'->'{}')©Úextrar¥   rf   r©   zmin_dt=z
 interval=z sub_interval=z	Too old (z rows tagged)z/Nothing needs repairing (dts_to_repair[] empty)r_   é   ©Úyearsra   rZ   rq   r]   zgrp_max_size = r…   zRepair groups:z- rŒ   Únearest)Úmethodz
df_block:
iÙ  )ro   rn   é;   z"Cannot reconstruct block starting z9, too old, Yahoo will reject request for finer-grain dataz	Fetching z	 prepost=r‹   ú->Úlevel)r  r  rv   r  r  r  r®   )ÚmillisecondszCannot reconstruct z block range z3, Yahoo not returning finer-grain data within rangeÚctr)ÚMONÚTUEÚWEDÚTHUÚFRIÚSATÚSUNé   zW-ú
Week Startú	Day StartÚ
intervalIDr§   )rœ   r²   )rŸ   rì   )r    rì   )rž   rí   )r�   rj   )r§   rî   )r¡   rî   )r´   rö   )rœ   rŸ   ÚAdjCloserž   r�   r§   r¡   ÚRepairedrŸ  r    r¾   )rœ  r�  Údiffzdf_new:
zCan't calibrate z block starting z so aborting repairrœ   rŸ   zCan't calibrate block starting çš™™™™™¹?é   )ÚepsÚmin_samplesc                 S   ó   g | ]}|d kr|‘qS ©r…   r   )r<   rj  r   r   r   r=   >  r>   c                    s   t  ˆ | k¡S r   )r  Úcount_nonzero)rj  )Úlabelsr   r   Ú<lambda>B  s    z;PriceHistory._reconstruct_intervals_batch.<locals>.<lambda>©r/   )Úweightséd   z Price calibration ratio (raw) = Ú6frh  éþÿÿÿz:Yahoo didn't return finer-grain data for these intervals: r�   rž   rÙ   )Xr   rÄ   rN   rã   rÐ   rë   rÈ   ÚtypeÚsklearn.clusterr€  r	   ru   r4   rÎ   rÖ   Úwarningr»   rW   ÚformatÚinforý   Úto_numpyrö   rÑ   rÒ   rÓ   r¢   rO   ÚceilrË   r  rî   Úcopyrÿ   r  Ú	_dateutilÚrelativedeltarÏ   r&   Úget_indexerÚto_listÚsortÚisinrÉ   Útodayrj   Úhasattrr‘  ÚsetLevelÚloggingÚCRITICALrH  rè   rù   ÚweekdayrÝ   Ú	to_periodÚ
start_timerû   Úcumsumré   Úgroupbyrq  r[  r¡  Úintersect1dÚisnanÚflagsÚ	writeableÚsizer  ÚfloatÚtileÚlogÚreshapeÚlog1pÚfit_predictÚuniqueÚmedianÚaverageÚabsr  Úget_locÚnanrs  Úbfillr  rí   )Xr   rÂ   rv   r  Útagr  Ú
log_extrasr€  r/  Ú
price_colsÚ	data_colsÚitdsÚnextsÚmin_lookbacksr„  Úsub_intervalÚtd_ranger6  Úf_repairÚf_repair_rowsr‡   Úmin_dtÚf_recentÚdts_to_repairÚdf_v2Úf_goodÚdf_goodÚ
dts_groupsÚgrp_max_sizeÚdtÚgÚg0Úi0ÚglÚilÚgood_dtsÚn_fixedÚdf_blockr%  r>  ÚrejectÚtd_1dÚfetch_startÚ	fetch_endÚ	log_levelÚdf_fineÚweekdaysÚweek_end_dayÚgrp_colÚdf_fine_grpÚdf_newÚ	new_indexÚcommon_indexÚ
calib_colsÚdf_new_calibÚdf_block_calibÚcalib_filterÚjÚfÚratiosr¬  ÚxÚrelative_tolerancer¥  Úlogxr¤  Úcluster_labelsÚlargestÚreplacementÚratioÚrÚr_rcprÀ   Ú	ratio_rcpÚpost_idxÚadjÚf_tagÚpost_adjÚdf_new_adjLastÚdivÚclose_beforeÚadj_correctionÚbad_dtsÚno_fine_data_dtsÚ
df_new_rowÚdf_last_weekÚ
df_bad_rowÚ
bad_fieldsr   )rÂ   rv   r†  r©  r   Ú_reconstruct_intervals_batch'  st  



 








"
ÿ"&&


&
.

 
øø
	&



€





€"
$
€





*


ß%z)PriceHistory._reconstruct_intervals_batchc                 C   s   d}|t vr|||fS t | }ddddœ| }|d dk}| ¡ s&|||fS |jt |¡d d  }d	}|jtj d
¡t	j
dd� kriz| jd |d  }	t|	| d ƒdk rYd}W n tyh   tjjsf‚ Y nw |rztD ]
}
||
  |9  < qmd	}|| jd< d	| jd< |d dk}| ¡ rË|ddg  ¡ }|d  ¡ jd|d jd d�|d< || }|d |d   ¡ }t|ƒdkrËt |¡dkrË|d  |9  < |||fS )NFÚGBPÚZARÚILSr   r¡   r   r…   Trf   r|   rh   ÚregularMarketPricerŸ   rd   r¢  rš   r¶   r§   r­   )Ú
fill_value)r  rö   r  r  Úwherer  rÐ   rÑ   rÒ   rÎ   rÖ   r   rÖ  râ   r   rË   rÌ   r	   r·  rs  rr  rµ  r4   rÕ  )r   rÂ   rš   rB  r‡   Ú	currency2Úf_volumeÚlast_rowÚprices_in_subunitsr  rÀ   Úf_divr\  Údiv_pctsr   r   r   r  ¶  sH   

€ü

$
z"PriceHistory._standardise_currencyc                    sH  ‡ fdd„|d   ¡ D ƒ}g d¢}|D ]Š}d }|dkr#ˆ › d�}d}	n*ˆ dkr/ˆ › d�}d}	n||v rAˆ |v rA|› ˆ › d�}d}	n|› d�}d}	ˆ › d�}t| j|| jƒ}
|
jd	|d
�d jd }|	rgd| }|j|d |kdf  |9  < |d ur�t| j|| jƒ}|jd	|d
�d jd }|j|d |kdf  |9  < qˆ |d< |S )Nc                    s   g | ]}|ˆ kr|‘qS r   r   r¿   ©Úfxr   r   r=   è  r>   z6PriceHistory._dividends_convert_fx.<locals>.<listcomp>rš   )ÚUSDÚJPYÚEURÚCNYr#  ÚCADr1  ú=XFTr_   )r  r®   rŸ   r…   rd   r§   )rÓ  rK   rL   rP   rH  r  rù   )r   r9  r0  r®   Úbad_div_currenciesÚmajor_currenciesrÀ   Úfx2_tkrÚfx_tkrÚreverseÚfx_datÚfx_rateÚfx2_datÚfx2_rater   r/  r   r÷   ç  s8   



€z"PriceHistory._dividends_convert_fxc                 C   s,   |j r|S |  |||¡}|  ||||¡}|S r   )rè   Ú_fix_unit_switchÚ_fix_unit_random_mixups)r   rÂ   rv   r4  r  rz  Údf3r   r   r   r    s
   zPriceHistory._fix_unit_mixupsc           #         s~  |j r|S t ¡ }d|| jdœ}|jd dkr"d|jvr d|d< |S |jd dkr;|jd|d� d|jvr9d|d< |S | ¡ ‰ ˆ jj	d u rMˆ j 
|¡ˆ _nˆ jj	|krZˆ j |¡ˆ _dd	lm} g d
¢}‡ fdd„|D ƒ}ˆ | dkjdd� ¡ }	|	 ¡ rŒˆ |	 }
ˆ |	  ‰ ||	  }nd }
|}ˆ jd dkr©|jd|d� d|jvr§d|d< |S ˆ |  ¡ }|j|ddd�}|| }|d  ¡ d }|dk}d| }|d  ¡ d }|dk|dkB }||B }| ¡ sõ|jd|d� d|jvród|d< |S d}tt|ƒƒD ]}|d d …|f }|| }|ˆ j||f< qý||k ¡ }|  ˆ |||¡‰ ˆ |  ¡ |k}ˆ |  ¡ |k ¡ }|dk�rCˆ |  ¡ |k|@ }t|jd ƒD ]m}||d d …f }| ¡ �sZ�qIˆ j| }dD ]}| |¡}|| �r{|j||f d ˆ j||f< �qad}| |¡}|| �r™ˆ j|ddgf  ¡ ˆ j||f< d}| |¡}|| �rµˆ j|ddgf  ¡ ˆ j||f< �qIˆ |  ¡ |k|@ }t|jd ƒD ]m}||d d …f }| ¡ �sÙ�qÈˆ j| }dD ]}| |¡}|| �rú|j||f d ˆ j||f< �qàd}| |¡}|| �rˆ j|ddgf  ¡ ˆ j||f< d}| |¡}|| �r4ˆ j|ddgf  ¡ ˆ j||f< �qÈˆ |  ¡ |k}| ¡ }n|}|| }|| } |dk�rod|› d|› d �}!| dk�rh|!d!| › d"�7 }!|j|!|d� ˆ |  ¡ |k}tt|ƒƒD ] }|d d …|f }"|" ¡ �rœ|| }|j|"|f ˆ j|"|f< �q}|
d u�r½d|
jv�r­d|
d< t ˆ |
g¡ ¡ ‰ t ˆ j¡ˆ _ˆ S )#Núprice-repair-100xr|  r   r´   Frd   z3Cannot check single-row table for 100x price errorsrˆ  )Úndimage)r�   rœ   rž   rŸ   r    c                    r½   r   r¾   r¿   ©rz  r   r   r=   =  rÃ   z8PriceHistory._fix_unit_random_mixups.<locals>.<listcomp>r¥   z6Insufficient good data for detecting 100x price errors)r£  r£  Úwrap)rÌ  Úmodeé   r­  rh  zNo sporadic 100x errorsç      ð¿)rœ   rŸ   r   r�   rœ   rŸ   rž   ç      Y@zfixed ú/z currency unit mixups ú(z	 crudely))rè   r   rÄ   rN   rü   r»   rË   r·  r¢   rO   rÝ   rÓ   ÚscipyrD  rö   rµ  r´  Úmedian_filterr  ru   r4   rù   rî   r"  rj   rí   rÐ   r  rý   rû   )#r   rÂ   rv   r4  r  r  rÛ  Ú_ndimagerÝ  Úf_zeroesÚ
df2_zeroesÚdf_origÚdf2_datarÔ  r  Úratio_roundedr  r  Úratio_rcp_roundedÚf_rcpÚf_eitherrÚ  r„  ÚfirÀ   Ún_beforeÚ
df2_taggedÚn_afterÚidxr  Ún_after_cruderô  Ún_fixed_crudelyÚ
report_msgÚfjr   rE  r   rA    sð   	









€

 

 €



€

 

 €



€
z$PriceHistory._fix_unit_random_mixupsc           
      C   s¨   | j d }|dkrd}nd}d|vrd|d< |d  ¡ }| j||||ddd�}|d  ¡ }|| @ }| ¡ rR|d	v r=d
}	n|dv rDd}	n	|dv rKd}	n|S |	| j d< |S )Nrš   ÚKWFéè  r­  r´   FT)Úunit_switchÚcorrect_dividend)r   r#  r#  )r   r$  r$  )r   r%  r%  )r   rµ  Ú_fix_prices_sudden_changerö   )
r   rÂ   rv   r4  rš   ÚnÚf_repair_beforeÚf_repair_afterÚf_repair_unitr)  r   r   r   r@  ±  s(   



zPriceHistory._fix_unit_switchc           -         sn  |j r|S t ¡ }d|| jdœ}|d dv }| ¡ }| ¡ ‰ ˆ jjd u r,ˆ j |¡ˆ _nˆ jj|kr9ˆ j 	|¡ˆ _‡ fdd„t
D ƒ}ˆ | dkˆ |  ¡ B }	ˆ d ˆ d	 k }
d
|	j|
dd	gf< ˆ d ˆ d k}d
|	j|ddgf< ˆ d ˆ d	 k }d
|	j|dd	gf< ˆ d ˆ d k}d
|	j|ddgf< d }|rätj|	jdd�dd� |	jj¡}| ¡ | ¡  }|j|dk }t |	jj|¡}ˆ | }ˆ |  ‰ ˆ j rÔd|jvrÒd|d< |S ˆ  ¡ ‰ ˆ | dkˆ |  ¡ B }	ˆ d  ¡ ˆ d	  ¡ k}| j d¡rùd }nCˆ d  ¡  ¡  ˆ d	  ¡  ¡  @ }ˆ d dk ¡ }||@ |@ }|�s<ˆ d  ¡ }d|jd< t |ˆ d  ¡}|dk ¡ |@ }||B }dˆ jv �r]ˆ d dk ¡ }| ¡ �r]|| @ }| ¡ �r]d
|	|< |	 ¡ }	|	jdd�}|d u�rp||B }| ¡ �sˆ|jd|d� d|jv�r†d|d< |S |	 ¡ t|ƒtˆ ƒ k�r¨|jd|d� d|jv�r¦d|d< |S |dg }d}tt|ƒƒD ]}|| } |ˆ j|	d d …|f | f< �qµ|d u�rÖ|ˆ j|df< ˆ d  ¡ dkˆ d  ¡  ¡ B }!|ˆ j|	jdd�|!@ df< |ˆ j||!@ df< ˆ |  ¡ |k}"|" ¡ }#ˆ j|"jdd� }$|  ˆ |||¡‰ ˆ |  ¡ |k}"|" ¡ }%ˆ j|"jdd� }&|#|% }'|'dk�rd| j› d|'› d|#› d|› d �}(|'d!k �r]t t!t"|$ƒ #|&¡ƒƒ})|(d"|)› �7 }(|j|(|d� |d u�r†d|jv�rsd|d< dˆ jv�r}dˆ d< t $ˆ |g¡ ¡ ‰ ˆ |  ¡ |k}*tt|ƒƒD ] }+|*d d …|+f },|, ¡ �r³||+ } |j|,| f ˆ j|,| f< �q”ˆ S )#Nzprice-repair-zeroesr|  r…   r†   c                    r½   r   r¾   r¿   rE  r   r   r=   í  rÃ   z,PriceHistory._fix_zeroes.<locals>.<listcomp>r­   rŸ   rž   Tr�   rœ   rd   r¥   rØ  )r  ç      à?r´   Fr6  r¡   r   çš™™™™™©?r£   zNo price=0 errors to repairrˆ  z;No good data for calibration so cannot fix price=0 bad datarI  z: fixed rK  z value=0 errors in z price datarg   rc   )%rè   r   rÄ   rN   rý   r·  r¢   rO   rÝ   rÓ   r	   rÿ   rù   rÐ   rõ   rö   rÇ  rÏ   rî   Úcountr  r½  r»   rµ  Úendswithr¡  r  rÖ  rË   r4   ru   r"  Úsortedr#   ÚsetÚ
differencer  )-r   rÂ   rv   r4  r  r  rÛ  r/  rÜ  Úf_prices_badÚfclÚfchÚfolÚfohÚdf2_reserveÚgrpÚnan_pctÚdtsÚf_zero_or_nan_ignoreÚf_changeÚ	f_vol_badÚf_high_low_goodÚ
f_vol_zeroÚ
close_diffÚclose_chg_pct_absÚf_bad_price_chgÚf_splitÚf_change_expected_but_missingÚ
f_bad_rowsrÝ  rÚ  r„  rÀ   Úf_vol_zero_or_nanrZ  rY  Ú
dts_taggedr[  Údts_not_repairedrô  r6  Údts_repairedr  r  r`  r   rE  r   r  ×  sÒ    

$







 



€zPriceHistory._fix_zeroesc              	   C   sô  d|j vr|S |d dk ¡ r|S d}t ¡ }d| jdœ}| ¡ }| ¡ }|d jd d� ¡ |d< |d	 dk|d dk@ }|j	|df  
¡ }|jdd
d�}d|j vrVd|d< |d |d  |d< |r�tj|d< tj|d< |d |d   d¡|d< td|d›�ƒ ||d dk j}|d  ¡ }|d  ¡ }	i }
|D ] }|j |¡}|dk�r:|d	 j| }|d j| }||k rºqš|||d
   }|||d
   }||d
  ||  ||d
   }|| }t|| ƒ}t|||  ƒ}||k }||
|< |�r:td| ¡ › d|d d›d|d d›d�ƒ td|d d›d�ƒ td|d d›d�ƒ td|d›�ƒ td|d›�ƒ td|› �ƒ qšt|
 ¡ ƒt|
ƒ }|�rStd |d d›d�ƒ |d!k�rÙ|
 ¡ D ]|}|j| }|d	 j| }|d j| }|| }||j	|d	f< |	|d
  ||d
   |	| ||   }d"|| ||d
    }|| }|j	d |t d
¡ …df  |9  < d#|j	d |…df< d$| ¡ › d%|d›�}|j||d&� |�r×||j	|df< �q\|d |d  |d< |�rñ|d  d¡|d< |S |jdd
d�}|S )'Nr¨   r   Fzrepair-capital-gains)r}  r  rŸ   )Úfill_methodzPrice_Change%r§   rd   r¥   r´   r    ÚAdjÚScaleFactorÚ
correctionrg   ÚAdjYahooz# price_drop_pct_mean = ú.4fz# z: div = r­  ú.1fz%, cg = ú%z- price_drop_pct = z- price_drop_pct_excl_vol = z- diff_div = z- diff_total = z- cg_is_double_counted = z- pct_double_counted = gZd;ßOå?rh  Tz'Repaired capital-gains double-count at z. Adj correction = rˆ  )r»   r  r   rÄ   rN   r·  rý   Ú
pct_changerÖ  rù   Úmeanré   r  rØ  r  Úprintr¢   rµ  r×  r  rÏ   rî   rB   r4   r$   rÎ   rÖ   r´  )r   rÂ   rË   r  rÛ  Úno_distributionsÚprice_drop_pct_meanry  rÀ   ÚacÚdcsrí  r\  Údividendr;  Údiv_pctÚcg_pctÚprice_drop_pctÚprice_drop_pct_excl_volÚdiff_divÚ
diff_totalÚcg_is_double_countedÚpct_double_countedÚdividend_trueÚ
adj_beforeÚadj_correctrŒ  r6  r   r   r   r
  g  s”   




 ,€

($€þz"PriceHistory._repair_capital_gainsc           £   	      s 0  |d u s|j r	|S |dv r|S |d dv }d|jv r$|d dk ¡ r$|S t ¡ }d|| jdœ}|d d	k ¡ }| ¡ sD|jd
|d� |S | jd dkrNd}	nd}	d ‰d}
| 	¡ }| 
¡ }d|jvred|d< d}|d  ¡  ¡ }||  
¡ }||   
¡ }g d¢}||  ¡ jdd�}| ¡ rª|j||f  
¡ }|jd d …|f  ¡ |jd d …|f< nd }|r¿| 
¡ }|  ||d¡}|d dk}|d d	k ¡ }| ¡ sÔ|jd
|d� |S t |¡d }|d  tjtj g¡}| ¡ �r)| }|d  ¡ }|jjsý| 
¡ }|d  ¡ }|d }|||k@ }| ¡ �r%|| d ||< |||k@ }| ¡ �s||d< |d  tjtj gtj¡|d< |d |d  |d< |d  ¡ |d< |d  ¡ |d  ¡  @ }|d | |d |  |j|df< |jddd�}g }tt|ƒd ddƒD ]”}|| }|dk�r‰�q||j|d  }|d |d  } |d j| }!| dk�r| |! d dk �r|j|d  }"|d |! }#|#|d k�r|#|d k�r|#|j|"df< |d j| |d j|  }$|$d|!|d j|d     }%|d j|d  |% |j|"df< d |j|"df< d }| |j|  ¡ ¡ �q|t|ƒdk�r)d!d"d#„ |D ƒ› �}&|j|&|d� tt|ƒd ddƒD �]£}|| }|j| }'|d j| }!|dk�rM�q3|!|d j|d   }(d})d$}*t |d j| |d j|d  d d%d&��rè|d j|d  d |d j|  }+|!|d j|d  d  }(d|!|d j|d  d   },|d j|d  |d j|d   }-t |-|,d%d&��sç|,|d jd |'t!j"dd'� …  |jd |'t!j"dd'� …df< d |jd |'t!j"dd'� …df< nœt |d j| |d j|d  d d%d&��rt|d j|d  d |d j|  }+|!|d j|d  d  }(d|!|d j|d  d   },|d j|d  |d j|d   }-t |-|,d%d&��ss|,|d jd |'t!j"dd'� …  |jd |'t!j"dd'� …df< d |jd |'t!j"dd'� …df< n|d j|d  |d j|  }+|t|ƒd k �r¨t #|+|d j| |d j|d   g¡}.t $|.¡}/nt #|+g¡}.|+}/t|ƒ| d(k �rÊt%t|ƒ|d( ƒ}0t$d|0d) ƒ}1nt$d|d( ƒ}1t%t|ƒ|1d) ƒ}0|0|1 d(k �råtj}2n"|d j|1|0d …  ¡ |d j|1d |0…  ¡  }3t &t '|3¡¡}2g }4|.d	k (¡ �rQ|d* j| dk�rQt )|d* d	k¡t|ƒ }5|(d d+k �r8|4 d,d	d-œ¡ �n8|5d.k�rB|(d/k�sG|(dk�rO|4 d0d	d-œ¡ �n|d1 j|' }6|6d	k�r`d }7n0|!|6 }7|7|!k�rn|+|2 }8n|/}8|8dk�r�t'|!|8 ƒ} t'|7|8 ƒ}9|9|* | k�r�|4 d2|9d-œ¡ |(|
k�rò|/d	k�r£|4 d0d	d-œ¡ nOt'|!|/ ƒ} t'|!|	 |/ ƒ}:|7d u �rÆ|:|* | k�rÅ|4 d0|:d-œ¡ n,t'|7|	 |/ ƒ};|:|;k �rã|:|* | k�râ|4 d0|:d-œ¡ n|;|* | k�rò|4 d3|;d-œ¡ t *|2¡�sp|+|2 }<|<dk�r|�r|�r|d4 j| |d j|  d5|< k �rd}<|<dk�rpt'|!|< ƒ} t'|!|	 |< ƒ}:|7d u �rD|:|) | k�rC|4 d,|:d-œ¡ n,t'|7|	 |< ƒ};|:|;k �ra|:|* | k�r`|4 d,|:d-œ¡ n|;|* | k�rp|4 d6|;d-œ¡ |'||!|(d7œ}=|+|=d8< |/|=d9< |d* j| |=d:< |2|=d;< d|=d<< d|=d=< d|=d>< d|=d?< d|=d@< t|4ƒdk�r½t+|4dAdB„ dC�}4|4d }>d |=|>dD  dEdF¡< t, -|=g¡ .dG¡‰ˆd u �rÏˆ‰�q3t, /ˆˆg¡‰�q3ˆd u �râ|�sâ|S dHd#„ ˆjD ƒ}?ˆ 	¡ ‰dždKdL„}@ttˆƒƒD ]·}ˆdM j| }ˆj| }'ˆdI j| }!|dk�r�qù|!|d j|d   }(|d j|d  |d j|d   }A|d j| |d j|  }B|B|Ak}C|A|B }-d|- }D|Dd+|( k }E|Dd|( k}F|C�r]d}EdN|-i}=|C|=dO< |F|=dP< |E|=dQ< |= 0¡ D ]=\}G}H|Gˆv�r¦t1|Ht2tj3fƒ�rˆdˆ|G< nt1|Ht4ƒ�r“dˆ|G< nt1|Ht5ƒ�ržd	ˆ|G< nt6|G|Ht7|Hƒƒ‚|Hˆj|'|Gf< �qq�qù|?g dR¢7 }?dˆdS< t!j"dTdU�}Iˆd<dQg jdd�}| ¡ �rutˆƒdk�rut |¡d }Jˆj| }K|JD ]�}ˆj| }!|!j8}Ld }M|dk�rþˆj|d  }Nnˆj|d  }N|!dI |	 |NdI  }O|!dI |NdI  }Pt%t'|Od ƒt'|Pd ƒƒ}Qt'|L|Nj8 ƒ|Ik�rV|NdS �sV|Qdk �rV|Nj8|Kv �rT|!d8 }+|Nd8 }R|+dV|R k�rQ|Nj8}Mn|L}Mn|L}M|M�rsd ˆj|MdSf< |?D ]}S|Sˆjv �rqdˆj|M|Sf< �qb�qådW}Tˆ 	¡ ‰tdtˆƒƒD ]€}ˆj| }!|!j8}Ld}Ud}V|!d8 }+ˆj|d  }W|!dI |WdI  }Xt'|L|Wj8 ƒ|Ik�rÕ|WdS �sÕ|!dS �sÕt'|Xd ƒ|Tk �rÕˆd8 j|d  }Y|+dV|Y k�rÓd }Vnd }U|V�sÛ|U�r|U�rà|Lnˆj|d  }Zd ˆj|ZdSf< |?D ]}S|Sˆjv �rÿdˆj|Z|Sf< �qð�q‚|? dS¡ dSˆjv �r¬ˆdS }[|[ˆdXd#„ |?D ƒ jdd� @ }[|[ ¡ �r›ˆj|[ }\dYdZd#„ |\D ƒ› �}&|j|&|d� ˆj|[ }]|]D ]O}'|'t!j"dd'� }^|jd |^…df  ˆdN j|'   < d |jd |^…df< |jd |^…df  ˆdN j|'   < d |jd |^…df< d|j|'df< d }ˆ |'¡‰�qDdˆj|[dSf< ˆjdSdd�‰dS|?v �r¬|? 9dS¡ ˆ|?  ¡  ¡ �	s!tˆƒdk�	r!tdtˆƒƒD ]]}d[\}_}`|dk�rÞˆd\ j|d  ˆd\ j|  }_|tˆƒd k �r÷ˆd\ j|d  ˆd\ j|  }`|_�pû|`}_|`�	p |_}`t'|_|	 ƒd]k �	rt'|`|	 ƒd]k �	rˆj| }Ld ˆj|Ld=f< �qÃˆ|?  ¡  ¡ �	sï|d u�	r7||j|j|f< |�	rí|�	rÞ|d |d  |d< |d |d  |d< |d dk}t |¡d }t,j:ddU�}att|ƒd ddƒD ]f}|| }|j| }L|L ¡ }b|jt;|b|a ƒt;|bƒ… }c|cd jd }d|d|j|Ldf< |dd	k�	rŸ�	qn|cd jd |cd jd  }e|d j|d  |d j|  }f|e|f }g|jd |Lt, :d¡ …df  |g9  < �	qn|jddd�}|}|j �	sët, /||g¡ 	¡ }|S |S ttˆƒƒD �]š}ˆdM j| }ˆj| }'ˆdI j| }!|dk�
r�	qõ|!|d j|d   }(|'t!j"d^dU� }ht< =|j|h¡}it%|it|ƒd ƒ}it$d|d_ ƒ}j|d j||id …  >¡ }k|kd$k ¡ |kd`k  ¡ B }l|l ¡ �
rl|t |l¡d d  d }i|j|i }hd}md}nt,j?}o|i|jk�r¸|j|j|id …  
¡ }p|pd |pd  |pd< |pd |pd  |pda< |pd jdd …  ¡ |pd jd d…  ¡  }qt d	g|q¡}q|q|pdb< |pda jdd …  ¡ |pd jd d…  ¡  }rt d	g|r¡}r|r|pdc< |pdbdcg }q|(ddk�r¸|(dk �r¸|!|pd jd  }s|qdc |sde k}| ¡ �r¸t |¡d }J|JD ]¯}t|qdc j|t }u|udf|qdb j|t  k�r¶d|qdc  |u }v|vdgk}w|d1 j|' }6ˆd> j|' }x|6d	k�s?|x�s¢|w ¡ �r¢t |w¡d }yt|yƒdk�r•t @t|yƒ¡}ztt|yƒƒD ]+}|y| }t|vj|t }{|{|'k �r||z|  |'|{ jA7  < �q_|z|  d+|{|' jA 7  < �q_t B|z¡}|y| }|n|yd }|d }n|vj|| }o n|udh|s k�r¶|pd dk  ¡ �r´d }m n�qˆj|'dQf }E|m�rÇ|E�rÇd}mi }=|m|=di< |n|=dj< |o|=dk< |m�r(|d1 j|' }6|6d	k�r(|!|6 }7|7|!k�r ˆd; j|' }2ˆd8 j|' }+|+|2 }8n	ˆd9 j|' }/|/}8|8dk�r(t'|!|8 ƒ} t'|7|8 ƒ}9|9| dl k�r(d ˆj|'d>f< |= 0¡ D ]K\}G}H|Gˆv�rot1|Ht2tj3fƒ�rCdˆ|G< n,t1|Ht4ƒ�rNdˆ|G< n!t1|Ht5ƒ�rYd	ˆ|G< n|Gdkk�rgt,jCdmdn�ˆ|G< nt6|G|Ht7|Hƒƒ‚|Hˆj|'|Gf< �q,d<ˆjv �r�djˆjv �r�dˆjˆdj  ¡ d<f< �	qõ|?didjg7 }?|?D ]}Sˆ|S  ¡ �s©ˆj|Sdd�‰�q™dk}S|Sˆjv �rÃˆ|S  ¡  (¡ �rÃˆj|Sdd�‰‡fdod#„|?D ƒ}?ˆ Dd\¡‰|@ˆd\dp�ˆdq< ˆdq  E¡ }}|}D �]±}~ˆdq |~k}ˆ|  	¡ ‰ tˆ ƒ}€ˆ d\g  
¡ }�t|�ƒdk�r)|�d\ j F¡  G¡ jH I¡ dr }‚|‚jd |‚j|‚jd < |‚|�ds< |�d\ |�ds  |�dt< |?D ]}Sˆ |S  ¡  ¡ �s=ˆ j|Sdd�‰ �q+‡ fdud#„|?D ƒ}ƒ|ƒD �]F}Sˆ |S  ¡ }„t )|„¡}…|…d|€fv �ra�qJ|…|€ }†|Sd<k�r7d}‡dv}ˆdjˆ jv �r�ˆ |S ˆ dj k (¡ �r��qJdiˆ jv �rÁˆ |S ˆ |S ˆ di @ k (¡ �rÁdv}‡ˆ di  ¡ }‰tˆ ƒ}€t )|‰¡}…|…|€ }†|†|‡k�r¿|ˆdi  ¡ @ }d ˆj||Sf< �qJdQˆ jv �ráˆ dQ  (¡ �ráˆ d; | |„ dk (¡ �rÞdw}ˆn"d/}‡ndiˆ jv �rõˆ |S ˆ di k (¡ �rõdv}‡n| j Jdx¡�rþdy}‡dv}ˆ|†|‡k�r)d ˆj||Sf< djˆjv �r'dˆj|djf< t,j?ˆj|dkf< ˆ|  	¡ ‰ �qJ|†|ˆk�r7dˆj||Sf< �qJ|Sd=k�rfd}‡dz}ˆdPˆ jv�rJd{}‡dv}ˆ|†|‡k�rXd ˆj||Sf< �qJ|†|ˆk�rfdˆj||Sf< �qJ|SdOk�rm�qJ|SdQk�rt�qJ|Sdik�r{�qJ|SdSk�r‚�qJ|Sdjk�r‰�qJ|Sd|v �r��qJ�qJ�qád<|?v �rdQ|?v �rd}}Sdˆ|S< ttˆƒƒD ]E}ˆj| }'ˆj| ‰ˆd< �rîˆdQ �rîˆdI }Šˆd\ }‹|Š|‹ }ˆdN }Œd|Œ | }D|Š|D }Xt'|X|	|	  ƒ|	k }�|��rîd ˆj|'|Sf< �qªˆ|S  ¡ �sÿˆj|Sdd�‰n|? |S¡ d?ˆjv �r8d~D ]}S|Sˆv �rˆ|S ˆd? B ˆ|S< �qˆd? ˆ|S< |? |S¡ �qˆjd?dd�‰|? 9d?¡ ˆ 	¡ ‰ˆˆ|? jdd� ‰ˆj �r
|�rî|d |d  |d< |d |d  |d< |d dk}t |¡d }t,j:ddU�}att|ƒd ddƒD ]f}|| }|j| }L|L ¡ }b|jt;|b|a ƒt;|bƒ… }c|cd jd }d|d|j|Ldf< |dd	k�r¯�q~|cd jd |cd jd  }e|d j|d  |d j|  }f|e|f }g|jd |Lt, :d¡ …df  |g9  < �q~|jddd�}|}|d u�rû||j|j|f< |j �st, /||g¡ 	¡ }|S i }ŽtKˆdq  E¡ ƒD �]Ç}~ˆˆdq |~k ‰ ˆ j	dd�‰ dˆ d€< ttˆ ƒd ddƒD �] }ˆ j| ‰ˆj8}'|'t!j"dd'� }^dOˆv �oNˆdO }�dQˆv �oWˆdQ }�dPˆv �o`ˆdP }‘diˆv �oiˆdi }’d=ˆv �orˆd= }“d<ˆv �o{ˆd< }”d>ˆv �o„ˆd> }•djˆv �o�ˆdj }nd}ˆv �o–ˆd} }–t )‡fd�d#„|?D ƒ¡}—|”�r´|’�r´|—d$k�r´d}’|—d8 }—|n�r×|”�rÇd}”dˆ j|'d<f< |—d8 }—|��r×d}�dˆ j|'dQf< |—d8 }—|•�rê|’�rêd}’dˆ j|'dif< |—d8 }—|—dk�rÁ|��sõ|‘�rD|��rúd‚ndƒ}G|Ž L|Gg ¡ |'¡ dˆd\  ˆdN  }g|jd |^…df  |g9  < d |jd |^…df< |jd |^…df  |g9  < d |jd |^…df< d ˆ j|'d€f< �q3|“�rªd„}G|	}˜ˆdI |˜ }™|™|j|'df< |Gd…7 }GddˆdN  |˜  }š|šˆdN  }g|jd |^…df  |g9  < d |jd |^…df< |jd |^…df  |g9  < d |jd |^…df< d ˆ j|'d€f< |Ž L|Gg ¡ |'¡ �q3|”�rd†}Gd|	 }˜ˆdI |˜ }™|™|j|'df< ˆd\ |˜ }›d|› }šˆdN }-|Gd…7 }G|š|- }g|jd |^…df  |g9  < d |jd |^…df< |jd |^…df  |g9  < d |jd |^…df< d ˆ j|'d€f< |Ž L|Gg ¡ |'¡ �q3|��r]d‡}G|Ž L|Gg ¡ |'¡ dˆd\  }g|jd |^…df  |g9  < d |jd |^…df< |jd |^…df  |g9  < d |jd |^…df< d ˆ j|'d€f< �q3|n�rñdˆ}G|Ž L|Gg ¡ |'¡ dˆdN  }g|jd |^…df  |g9  < |jd |^…df  |g9  < ˆdk }o|d jˆdM  }œˆdI }!d|!|œ  }�|ot!j"dd'� }ž|jd |ž…df  |�9  < |jd |ž…df  |�9  < |j|odf  |!7  < d|j|'df< d |jd |^…df< d |jd |^…df< d ˆ j|'d€f< �q3|’�rRdˆd\  }šˆdN }-t'|š|- d ƒddk�rF|Gd…7 }G|š|- }g|jd |^…df  |g9  < d |jd |^…df< |jd |^…df  |g9  < d |jd |^…df< d ˆ j|'d€f< �q3ˆ |'¡‰ˆ  |'¡‰ �q3|•�r¿d‰}Gd|d1 j|'  }˜ˆdI |˜ }™|™|j|'df< ˆd\ |˜ }›d|› }šˆdN }-|Gd…7 }G|š|- }g|jd |^…df  |g9  < d |jd |^…df< |jd |^…df  |g9  < d |jd |^…df< d ˆ j|'d€f< |Ž L|Gg ¡ |'¡ �q3|—d$k�ré|”�r|��rdŠ}G|Ž L|Gg ¡ |'¡ dˆd\ |	  }g|j|'df  |	  < |jd |^…df  |g9  < d |jd |^…df< |jd |^…df  |g9  < d |jd |^…df< d ˆ j|'d€f< �q3|“�rx|��rxd‹}G|Ž L|Gg ¡ |'¡ dˆd\ |	  }g|j|'df  |	9  < |jd |^…df  |g9  < d |jd |^…df< |jd |^…df  |g9  < d |jd |^…df< d ˆ j|'d€f< �q3|”�r«|��r«ˆdI }!|!ˆd\  }ˆdN }Œ|j|'df  |	  < d0}G|Ž L|Gg ¡ |'¡ d ˆ j|'d€f< �q3|”�r|’�rdŒ}G|Ž L|Gg ¡ |'¡ ˆd\ |	 }›d|› }š|šˆdN  }g|j|'df  |	  < |jd |^…df  |g9  < d |jd |^…df< |jd |^…df  |g9  < d |jd |^…df< d ˆ j|'d€f< �q3|“�r{|‘�r{|j|'df  |	9  < d„}Gd�ˆv �riˆd� �rid }&dˆd\ |	  }Ÿ|ŸˆdN  }g|jd |^…df  |g9  < d |jd |^…df< |jd |^…df  |g9  < d |jd |^…df< |GdŽ7 }G|Ž L|Gg ¡ |'¡ d ˆ j|'d€f< �q3|•�rç|��rçd�}Gd|d1 j|'  }˜ˆdI |˜ }™|™|j|'df< ˆd\ |˜ }›d|› }šd|› ˆdN  }g|jd |^…df  |g9  < d |jd |^…df< |jd |^…df  |g9  < d |jd |^…df< d ˆ j|'d€f< |Ž L|Gg ¡ |'¡ �q3|—d�k�rÓ|”�rg|��rg|•�rgd‘}Gd|	 d|d1 j|'   }˜ˆdI |˜ }™|™|j|'df< ˆd\ |˜ }›d|› }šˆdN }-|Gd…7 }G|š|- }g|jd |^…df  |g9  < d |jd |^…df< |jd |^…df  |g9  < d |jd |^…df< d ˆ j|'d€f< |Ž L|Gg ¡ |'¡ �q3|”�rÓ|��rÓ|–�rÓˆdI }!|!ˆd\  }ˆdN }Œd’}G|!|	 } | | }¡| |j|'df< d|¡ }¢|¢|Œ }g|jd |^…df  |g9  < d |jd |^…df< |jd |^…df  |g9  < d |jd |^…df< |Ž L|Gg ¡ |'¡ d ˆ j|'d€f< �q3ˆ j �rÛ�q�q|ŽD ]}Gd“|G› d”d•d#„ t+|Ž|G ƒD ƒ› �}&|j|&|d� �qß|�rô|d |d  |d< |d |d  |d< |d dk}t |¡d }t,j:ddU�}att|ƒd ddƒD ]»}|| }|j| }L|L ¡ }b|b|jd  ¡ k�rX|d jd }d|d|j|Ldf< �q/|jt;|bdJ|a  ƒt;|bƒ… }ct|cƒdk�r tMd–ƒ tM|ƒ tMd—|› �ƒ tMd˜|L› �ƒ tMd™ƒ tM|cƒ tMdš|jd › d›|jd › �ƒ tNdœƒ‚|cd jd }d|d|j|Ldf< |dd	k�rµ�q/|cd jd� |cd jd  }e|d j|d  |d j|  }f|e|f }g|jd |Lt, :d¡ …df  |g9  < �q/|jddd�}|}|d u�r||j|j|f< |j �st, /||g¡ 	¡ }|S )ŸN)ra   r_   rb   Ú1yr…   )rˆ   r‡   r¨   r   zdiv-adjust-repair-badr|  r§   r­   zNo dividends to checkrˆ  rš   ra  rb  r­  gìQ¸…ë¡?r´   FrŸ   ©rœ   r�   rž   rŸ   r    rd   r¥   rZ   r    r[   çü©ñÒMbP?rŠ  rž   r   r�   rh  Tz'Repaired double-adjustment on div days c                 S   s   g | ]}t |ƒ‘qS r   )rÉ   )r<   r+   r   r   r   r=   S  rA   z4PriceHistory._fix_bad_div_adjust.<locals>.<listcomp>rŠ  gš™™™™™™?)Úrel_tol)Úsecondsrg   é   r¡   r¢  zdiv-too-small)Ústater¡  g      è?g      Ð?zdiv-too-bigr£   zdiv-pre-splitzdiv-too-big-and-pre-splitrœ   çš™™™™™É?zdiv-too-small-and-pre-split)rÏ   r\  r  r�  ré   Ú
drop_2DmaxÚvolumeÚvolÚdiv_too_bigÚdiv_too_smallÚdiv_pre_splitÚdiv_too_big_and_pre_splitÚdiv_too_small_and_pre_splitc                 S   s   | d S )Nr¡  r   )r'   r   r   r   rª  ñ  s    z2PriceHistory._fix_bad_div_adjust.<locals>.<lambda>r«  rª  ú-Ú_rÏ   c                 S   s   g | ]	}|  d ¡r|‘qS )Údiv_)Ú
startswithr¿   r   r   r   r=   ý  rÃ   r  r›  c                 S   sì   t | ƒ}|  |¡}g }|jd g}|| jd g}td|ƒD ]-}|j| }	|| j| }
|
t |¡ |k r@| |	¡ | |
¡ q| |¡ |	g}|
g}q| |¡ t dg| ¡}d}t	|ƒD ]\}}t |ƒ}||||| …< ||7 }q_|S )Nr   rd   r…   )
r4   Úsort_valuesr¢   r  ru   r  r’  r&   ÚarrayÚ	enumerate)rÂ   ÚcolumnÚ	thresholdrf  Ú	sorted_dfÚclustersÚcurrent_dtsÚcurrents_valsr„  rí  r  r  r“  ÚclusterÚncr   r   r   Úcluster_dividends   s,   





z;PriceHistory._fix_bad_div_adjust.<locals>.cluster_dividendsr\  zpresent adjÚadj_missingÚadj_exceeds_divÚdiv_exceeds_adj)rÄ  rÅ  rÆ  Úphantomé   rh   ç      ø?g{®Gáz´?c                 S   r¦  )rÇ  r   r¿   r   r   r   r=   ’  r>   zRemoving phantom div(s): c                 S   ó   g | ]}t | ¡ ƒ‘qS r   ©rÉ   rÏ   ©r<   rí  r   r   r   r=   •  r>   )NNr�  rH  é#   é   gÍÌÌÌÌÌì¿zAdj LowÚdeltaÚadjDeltark  ç333333ã?gj¼t“ð?çš™™™™™é?gö(\�ÂõØ?Úadj_exceeds_pricesÚdiv_date_wrongÚdiv_true_dategš™™™™™ñ?zdatetime64[ns, UTC]©Údtypec                    r½   r   r¾   r¿   )Údiv_status_dfr   r   r=   e	  rÃ   )r»  rÁ  g    ~~Ar  zavg yr yieldc                    r½   r   r¾   r¿   )rÁ  r   r   r=   }	  rÃ   rj  gUUUUUUå?z.TAg®Gáz®ç?g)\�Âõ(¼?gtÑE]tá?)r±  r²  Úadj_too_small)r¯  r±  ©Ú	ascendingzFixed?c                    s   g | ]
}|ˆ v rˆ | ‘qS r   r   r¿   )Úrowr   r   r=   5
  ó    ztoo-small div-adjustztoo-big div-adjustztoo-small divz & div-adjustztoo-big divzmissing div-adjustzwrong ex-div datezpre-split divz"too-big div and missing div-adjustz$too-small div and missing div-adjustztoo-big div & div-adjustzFX was repairedz and FX mixupz pre-split & too-small div-adjustr£  ztoo-big div & pre-splitzdiv-too-big and adj-too-smallz	Repaired rc   c                 S   rÊ  r   rË  rÌ  r   r   r   r=   T  r>   z# df2:z# div_idx = z# div dt = z# df2_div_d:z# df2 date range: rŒ   z,Only 1d of prices - how to apply div-adjust?r¯  )r  r›  )Orè   r»   rö   r   rÄ   rN   rµ  rË   r   rý   r·  rÿ   rù   rs  r  r  r(  r½  ÚinfrÊ  rË  ÚreplacerØ  rÙ  ré   ru   r4   r  r¢   r&   rÏ   r´  r   rÎ   rÖ   r¹  rj   rí   r’  rÖ  r  rî   rÉ  rn  rÐ   rë   Ú	set_indexr  r?   rã   ÚboolÚbool_rÙ   rÍ  rÈ   r°  r  Úremoverú   rÉ   ÚbisectÚbisect_leftr‘  ÚNaTÚzerosri   Úargminrõ   r¸  rÓ  Ú	to_seriesr¡  rí  Útotal_secondsrm  r#   Ú
setdefaultr“  râ   )£r   rÂ   rv   r  rš   r/  r  rÛ  r-  Úcurrency_divideÚtoo_big_check_thresholdrz  Údf_modifiedÚf_nanÚdf2_nanÚOHLCAÚf_nan_pricesÚdf2_nan_pricesÚdf2_intradayr  Údiv_indicesÚf_infÚf_ninfÚadjCloseÚcloseÚclose10xÚf_hugeÚf_adjClose_naÚfixed_datesr„  Údiv_idxÚprices_beforer¡  r  Ú	dt_beforeÚ	new_closeÚ	adj_afterr  r6  rí  r™  Ú#div_too_small_improvement_thresholdÚ!div_too_big_improvement_thresholdré   Útrue_adjustÚpresent_adjÚdropsr¬  r  r  Útypical_volatilityÚdiffsÚpossibilitiesÚpct_zero_volrÊ   Údiv_postSplitÚ_dropÚdiff_postSplitÚdiff_fxÚdiff_fxPostSplitÚdrop_wo_volÚ
div_statusÚpÚchecksrÃ  Úpre_adjr  Údiv_missing_from_adjcloseÚimplied_div_yieldÚdiv_adj_is_too_smallÚdiv_adj_exceeds_divr'   ÚvÚphantom_proximity_thresholdÚindicesÚdts_to_checkÚdiv_dtÚ
phantom_dtÚ	other_divÚratio1Úratio2Ú
divergenceÚ	drop_nextrÀ   Úratio_thresholdÚthis_is_phantomÚlast_is_phantomÚlast_divr  Ú	last_dropÚphantom_div_dtÚ	f_phantomÚdiv_dtsÚphantom_div_dtsÚenddtÚr_preÚr_postÚtd1Údiv_dÚ	df2_div_dÚdiv_correctÚdiv_adj_correctÚdiv_adj_currr  Úlookahead_dateÚlookahead_idxÚlookback_idxÚfuture_changesÚf_big_changeÚdiv_adj_exceeds_pricesrÔ  rÕ  r
  ÚdeltasÚ	adjDeltasÚadjDivr\  ÚadjDelta_dropr	  Úf_near1_or_aboveÚ	pre_splitÚnear_indicesÚ	penaltiesÚdtiÚreversal_idxÚcluster_idsÚcidÚfcrf  r.  Ú
time_diffsÚcluster_checksÚf_failÚn_failÚpct_failÚtrue_thresholdÚfals_thresholdÚf_adj_exceeds_pricesÚ	div_yieldÚpctÚadj_presentÚalso_correct_adjÚdiv_repairsrÄ  rÆ  rÅ  rÓ  r°  r¯  r±  rÙ  Ún_failed_checksrŒ  Úcorrect_divÚ
target_adjÚtarget_div_pctr  Útrue_adjÚenddt2Údiv_adjÚdiv_trueÚpct_truer£  r   )rÁ  rØ  rÜ  r   r  Ù  sæ  

&


þ "
 €

,$$<€,$$<€ *4"€






€
€&

€
€



$



ô


(€
:€
 


$$


  

$
€


 (




,,





€








€€

€


"*

"










þ©[

€




 (



€
€
€ÿ"
 
 (
z PriceHistory._fix_bad_div_adjustc              	   C   sÆ  |j r|S t ¡ }d|| jdœ}|dv }|s|S | ¡ }|d  ¡ dk}| ¡ s/| d¡ |S |jdt|d |  	¡ ƒ› �|d� d	|j
vrJd
|d	< t |¡d D ]�}|j| }	|j|	df }
|	|jd krgqQ|dv rp|d7 }n|d7 }t|jd |ƒ}|jd|d … }|jd|› d|	 ¡ › d|
d›�|d� |jd|jd  ¡ › d|jd  ¡ › �|d� | j||||
ddd�}||jd d krÆ|}qQ|j|d d … }|j r×| ¡ }qQt | ¡ |g¡}qQ|S )Nzsplit-repairr|  ©rZ   ra   r_   rb   r£   r   z%price-repair-split: No splits in datazSplits: rˆ  r´   F©ra   r_   rb   rd   r]   z
split_idx=z
 split_dt=z split=rŽ  zdf dt range: rŒ   r…   T)Úcorrect_volumerd  )rè   r   rÄ   rN   rý   rµ  rö   rË   rÉ   Úto_dictr»   r  r(  r¢   rù   rí   rü   r  rÏ   re  rÐ   r  )r   rÂ   rv   r4  r  rÛ  ÚinterdayÚsplit_fÚ	split_idxÚsplit_dtrÊ   Ú
cutoff_idxÚdf_pre_splitÚdf_pre_split_repairedÚdf_post_cutoffr   r   r   r	  ˆ  sF   
$


&.
z"PriceHistory._fix_bad_stock_splitsc           ”         sØ#  |j r|S t ¡ }d|| jdœ}	|‰dˆ }
tˆ|
ƒ}|dv }|dv }|r.d}d|	d< d }nd	}d
|	d< |d  ¡ dk}|j|  ¡ tj	j	dd�  
¡ }|jd|› d|d›dd| d›d�|	d� g d¢}d}|| jdd�|| jdd� d  ¡ d| k ¡ r…|r…d}|jd|› �|	d� dˆ  k r™dk r¤n n	|jd|	d� |S | ¡ jdd�}|jjd u rº|j |¡|_n|jj|krÇ|j |¡|_||  ¡ jdd�}||  ¡ }||   ¡ }|jd }|d  dk}|||  ¡ jdd�B }| ¡ �ot |¡d d dk}| }t |t |d¡@ ¡d }t|ƒdk�r#t |¡d }t|ƒdk�r-d }nt|d ƒ}d!|› d"|› �}|d u�rM|d#|j|  
¡ › d�7 }|j||	d� | ¡ }|jg d$¢dd%d&�}|jd d'id(�}|d'  ¡ }| ¡ �r¥d)|d*< d+|j|d*f< |d' |  d,  d-¡ d.¡d/ |j| d*f< |d* |d'< |jd*dd� n|d' d,  d-¡ d.¡d/ |d'< d0g‰|j‡fd1d2„|D ƒdd%d&�}|�rë|d3k�rëˆd4v�rët |d5fd¡}d6d0g}||  ¡ } | dk}!nt |d7fd¡}|}||  ¡ } | dk}!| j j!�s|  ¡ } |! ¡ �rd| |!< |d0 dk}!|! ¡ �r9t "t|ƒ¡}"|d8  ¡ |!  |d0  ¡ |!   |"|! < n|d8  ¡ |d0  ¡  }"| j#}#|#tj$k�rS|  d9¡} t%| jd ƒD ]!}$| d d …|$f  |"9  < ||$ |j&v �rz|||$   |"9  < �qZ|#tj$k�r‡|  d-¡} | dd …f | d d:…f  |dd …< |d   ¡ }%|%dk ¡ �r¯|jd;|	d� |S d<d=„ }&|!tj|!ddd�B }'|' ¡ �rÆd||'< |�rÖ|d3k�rÖtj'|dd�}(ntj(|dd�}(t )|(¡})|) ¡ �rëd|(|)< t |(¡|d d d k �rt |(¡d|d d d  k�r|jd>|› d?�|	d� |S t *|(d@dAg¡\}*}+|+|* },|*dB|,  }-|+dB|,  }.|(|-k|(|.k@ }t +|(| ¡}/t ,|(| ¡}0|0|/ }1|jdC|/dD›dE|0d›d#|1dF dG›dH�|	d� dI|1 }2|�ry|d3k�ry|2dJ9 }2|dKv �ry|2d59 }2tˆ|
ƒd|2 k �r›|jdL|	d� |jdM|1d›dN|2d›�|	d� |S |(|
 }3tˆ|
ƒ}|jdO|dP›dQ|2d›�|	d� d|d |2 dR  }4|jdS|4dP›dTd|4 dP›�|	d� t t|ƒd¡}5|�r t |d7fd¡}||  -dd¡ ¡ } |}| dd …f | d d:…f  |dd …< n|(}||
 }3|d|4 k }6||4k}7|6|7B }|�r?t%t|ƒƒD ] }$||$ }8|d d …|$f ||8dU < ||8dU   .dJ¡||8dU < �qn|(|dV< |dV  .dJ¡|dV< t |7|6B ¡d }9|�s„t|9ƒdk�r„|9d }:t |d¡};|&|%d |:… ƒ|;d |:…< t%t|9ƒƒD ])}<|<t|9ƒd k�r�|9|< }=|}:n
|9|< }=|9|<d  }:|&|%|=|:… ƒ|;|=|:…< �q{t |d¡}>|;d d:… dk}!|! ¡ �sÉ|;dd … |;d d:…  |>dd …< n!d|>dd … |!< |;dd … |!  |;d d:… |!   |>dd … |! < |�rû|>|dW< |dW  .dJ¡|dW< n|>|dW< |dW  .dJ¡|dW< t *|>d@dAg¡\}*}+|+|* },|*dB|,  }-|+dB|,  }.|>|-k|>|.k@ }t +|>| ¡}/t ,|>| ¡}0|0|/ }1|jdX|/dD›dE|0d›d#|1dF dG›dH�|	d� dI|1 }?|�rk|d3k�rk|?dJ9 }?|dKv �rk|?d59 }?d|d |? dY  }@|jdZ|?d›d[|@dD›�|	d� t|7jƒ}A|Adk�r�|7n|7jdd�}B|B ¡ �r¸|B }Ct |C¡d }Dt|6jƒ}E|j|Edk�r³|6n|6jdd� }Ft |B¡d D ]ö}G|Gd }<|j|< }H|d  j/|< }I|Idk�rÚdn
|d  j/|<d  |I }J|�r|<d t|ƒk �r|d  j/|<d  }K|Kdk�rt|J|d  j/|< |K ƒ}J|Cd |<…  0¡ }Ltd|Ld\ ƒ}Mtt|Dƒ|Md] d ƒ}N|j/|D|M|N…  }O|O ¡ }O|F|F|Hk }Pt|Pƒdk�r]t|Pƒ}Q|Q|Hk�rKd }Rn|Oj|Ht1 2d¡ |Qt1 2d¡ … }Rn|Oj|Ht1 2d¡ d … }R|Rd u�rt|Rj �rtd }Rd^d_„ }Sd`da„ }T|Rd u�r¶|T|I|Rƒ}U|T|Rd  j/d |Rƒ}Vt|U|Vƒd5k�r¶| db|H 
¡ › dc�¡ |Adk�r®d|7|G< �qÁd|7|Gd d …f< �qÁ|6|7B }|�sÜ|3|dd< |6|de< |7|df< |dd  .d5¡ d.¡|dd< ||dg< nNt%t|ƒƒD ]G}$||$ }8|3d d …|$f ||8dh < ||8dh   .d5¡ d.¡||8dh < |6d d …|$f ||8di < |7d d …|$f ||8dj < |d d …|$f ||8dk < �qât |¡d D ]ð}G|j|G }Htt|ƒd |Gd5 ƒ}W| 3dl¡�rLdm}Xn| 3d/¡�rUdn}XndJ}Xtd|G|X ƒ}Y|j/|Y|W… }Z|�rk|}[ndog}[|[D ]¯}8|8dok�rƒ|ZdV |Zdg    ¡ }\n|Z|8dU  |Z|8dk     ¡ }\t +|\¡}/t ,|\¡}0|0|/ }1dI|1 }2|�r¹|d3k�r¹|2dJ9 }2|dKv �r¹|2d59 }2d|d |2 d  }4|�rÑ||8dU  j/|G }]�qp|dV j/|G }]|]|4k �r|]d|4 k�r|�r| dp|8› dq|H 
¡ › dr|1dn dG›ds�¡ d|j|H|8dk f< �qp| dt|H 
¡ › dr|1dn dG›ds�¡ d|j|Hdgf< �qp�q1|�s6|6|dg  ¡ @ }6|7|dg  ¡ @ }7n;t%t|ƒƒD ]4}$||$ }8|8ˆv �ro|6d d …|$f ||8dk  @ |6d d …|$f< |7d d …|$f ||8dk  @ |7d d …|$f< �q<|6|7B }| ¡ �s‡|jd>|› d?�|	d� |S d]}^|d  ¡ dk}_|duv �	r(|_ ¡ �	r(t |_¡d }`t |¡d }at|`ƒ�r³t|aƒ�sµd S |ad d …d f |` }b|bd:9 }b|bdk}c|c ¡ �	r(|b|c  ¡ }dt 4|¡|d }et5|etj	j	ƒ�rétj	j	|^dv�}4nt1j2|^dv�}4t5|4tj	j	ƒ�	rt5|etj	j	ƒ�	rt |b|dk¡d d }G|j|G }H|H|e |H|4 k }fn|e|4k }f|f�	r(|j6dw|	d� |S | 7t8j9¡�	r®t:t%d|jd ƒƒ|dx< |jd d |dx  |dy< |�	rf|dzd2„ ˆD ƒ jdd�|d{d2„ ˆD ƒ jdd�B }gn|de |df B }g|gt |gd:¡B t |gd¡B t |gd|¡B t |gd5¡B }gt; <d}d d~dd€d�¡� | d‚t=||g ƒ ¡ W d   ƒ n	1 �	s©w   Y  ‡fdƒd„„}hd}i|d u�	rÑ|jd d | }j|jd…|› d†|j› �|	d� |�rxt |d¡}k|�	rét |d¡}lt |d¡}mg d‡¢‰ t%t|ƒƒD �]U}$||$ }8t |¡d d }n|�
rÕ|d u�
rÕ||nk�
rÕ|d d …|$f }o|7d d …|$f }p|6d d …|$f }q|h|o|d … |p|d … |q|d … ƒ}rt|rƒdk�
r`t%t|rƒƒD ]}<|r|< }3|3d | |3d | |3d5 f|r|<< �
qGt >t |pd:¡¡}st >t |qd:¡¡}t|t|sB }u|h|u|jd … |t|jd … |s|jd … ƒ}vt|vƒdk�
rÍt%t|vƒƒD ]}<|v|< }3|3d |j |3d |j |3d5 f|v|<< �
q•t%t|vƒƒD ]}<|v|< }3||3d  ||3d  |3d5 f|v|<< �
q´|r}w|w ?|v¡ n|h|d d …|$f |7d d …|$f |6d d …|$f ƒ}w|jdˆ|8› d‰|w› �|	d� |d u�r=t%t|wƒd d:d:ƒD ]3}<|w|< }3|j|3d   
¡ |k �r;|jdŠ|8› d‹|j|3d  › dŒ|j|3d d  › d��|	d� |w|<= �q	t|wƒdk�rH|wˆ |$< �	qót0dŽd2„ ˆ D ƒƒ}x|xdk�rZ�nÙ|xdk�rŽ‡ fd�d2„t%t|ƒƒD ƒ}yt t @|y¡d:k¡d d }G||G }z|jd�|› d‘|z› d’�|	d� �n¥g d“¢}{t%t|ƒƒD ]ã}$||$ }8ˆ |$ }w|wd u �r©g }w|wD ]Î}3|3d5 d”k�r¹ˆ}||
}}n|
}|ˆ}}|i�pÃ||d•k }i|�rêd–|› d—|8› d˜|j|3d d   
¡ › d™|j|3d   
¡ › dš||d›�
}~nd–|› d—|8› d˜|j|3d d  › d™|j|3d  › dš||d›�
}~|j|~|	d� |{|$  |3d |3d  7  < |j/|3d |3d …|j& A|8¡f  ||9  < |8d0k�rL|j/|3d |3d …|j& Ad8¡f  ||9  < |�rn|8d6k�r_d|l|3d |3d …< n|8d0k�rnd|m|3d |3d …< d|k|3d |3d …< �q«�q˜t0|{ƒdk�r¹d)}t%t|ƒƒD ]!}$|{|$ dk�r«|d)k�r�|d›7 }|||$ › dœ|{|$ › d��7 }�q‹dž|› �}~|j6|~|	d� t%t|ƒƒD ]s}$||$ }8ˆ |$ }w|wd u �rÐg }w|wD ]^}3|3d5 d”k�ràˆ}||
}}n|
}|ˆ}}||dk�r|d6d0g j/|3d |3d … jdd�|j/|3d |3d …|j& AdŸ¡f< �qÒ|d6d0g j/|3d |3d … jdd�|j/|3d |3d …|j& Ad ¡f< �qÒ�q¿|�rr|l|m@ }€t B|l|m¡}�|€ ¡ �rX|j|€d f |}  .¡  d-¡|j|€d f< |� ¡ �rr|j|�d f d |}  .¡  d-¡|j|�d f< d|5|k< �nÜd}{t |¡d d }n|�r=|d u�r=||nk�r=|h||d … |7|d … |6|d … ƒ}rt|rƒdk�rÈt%t|rƒƒD ]}<|r|< }3|3d | |3d | |3d5 f|r|<< �q¯t >t |7d:¡¡}‚t >t |6d:¡¡}ƒ|ƒ|‚B }„|h|„|jd … |ƒ|jd … |‚|jd … ƒ}vt|vƒdk�r5t%t|vƒƒD ]}<|v|< }3|3d |j |3d |j |3d5 f|v|<< �qýt%t|vƒƒD ]}<|v|< }3||3d  ||3d  |3d5 f|v|<< �q|r}w|w ?|v¡ n|h||7|6ƒ}w|d u�rƒt%t|wƒd d:d:ƒD ]0}<|w|< }3|j|3d   
¡ |k �r�|jd¡|j|3d  › dŒ|j|3d d  › d��|	d� |w|<= �qRt%t|wƒƒD �]Ž}<|w|< }3|3d5 d”k�rœˆ}||
}}n|
}|ˆ}}|%|3d |3d … }…t @g ¡}†|<dk�rË|3d dk�rË|%td|3d dm ƒ|3d … }†n#|<t|wƒd k�rî|3d t|%ƒk �rî|%|3d t|3d dm t|%ƒƒ… }†t%dt|wƒƒD ]±}‡t 0|†dk¡dmk�rt 0|…dk¡dmk�r nš|<|‡ }ˆ|ˆd }‰|<|‡ }Š|Šd }‹|ˆdk�ra|w|ˆ }Œt 0|…dk¡dmk �r>t C|%|Œd |Œd … |…¡}…t 0|†dk¡dmk �ra|‰t|wƒk �ra|w|‰ }�t C|%|Œd |�d … |†¡}†|Št|wƒk �r¥|w|Š }Žt 0|…dk¡dmk �r„t C|%|Žd |Žd … |…¡}…t 0|†dk¡dmk �r¥|‹dk�r¥|w|‹ }�t C|%|�d |Žd … |†¡}†�qõ|†|†dk }†|…|…dk }…|&|†ƒ}�|&|…ƒ}‘t|‘ƒdk�sÉt|�ƒdk�rË�q‰t +|‘¡t +|�¡ }’|�sø|’d|@ k �rè|7|3d  �rè�q‰|’|@k�rö|6|3d  �rö�q‰�q‰|’d|@ k �r|7|3d  �r�q‰|’|@k�r|6|3d  �r�q‰	 �q‰t%t|wƒƒD ]ý}<|w|< }3|3d5 d”k�r1ˆ}||
}}n|
}|ˆ}}|i�p;||d•k }i|jd¢|3› d£||› �|	d� d¤D ]}8|j/|3d |3d …|j& A|8¡f  ||9  < �qK|�r||j/|3d |3d …|j& Ad¥¡f  ||9  < |�r¤|j& Ad ¡}“|j/|3d |3d …|“f |}  .¡  d-¡|j/|3d |3d …|“f< d|5|3d |3d …< |3d |3d d k�rÚ|�rÌd–|› d¦|j|3d   
¡ › �}~n>d–|› d¦|j|3d  › �}~n0|j|3d d  }M|j|3d  }N|�rþd–|› d§|M 
¡ › d¨|N 
¡ › d©�}~nd–|› d§|M› d¨|N› d©�}~|j|~|	d� |{|3d |3d  7 }{�qt|wƒd5k�rGdª}~|wD ]}3|~d«|j|3d d   
¡ › d¨|j|3d   
¡ › �7 }~�q(ndž|{› d��}~|j6|~|	d� |�r˜|i�r˜d¬| jDv �r˜| jDd¬ �r˜|}|d| }}d¤D ]}8||8  ||9  < �qn|�r…|d¥  ||9  < |�r•|d  |}  .¡  d-¡|d < |5 }5d­|j&v�r¢d|d­< |d­  ¡ |5B |d­< |�rÚ|d   ¡ })|) ¡ �rÎ|d  |)   .d¡ d-¡|j|) d f< n|d   .d¡ d-¡|d < t|ƒdk�rèt; E||g¡}| ¡ S )®Nzprice-change-repairr|  rh  r`  ra  z
100x errorrC  r}  z	bad splitzprice-repair-splitr£   r­   rd   r‹  z
start_min=z change=rŽ  z (rcp=r�   rˆ  rn  Fr¥   rj  Tzcorrect_columns_individually=rÒ  g      ô?z(Split ratio too close to 1. Won't repairrÚ  r   r¡   zappears_suspended=z, idx_latest_active=rŠ   )r    r§   r£   r´   rº   )r¦   r¼   ÚVolr¾   r‰   ÚVolStrÚNaNg    €„.ArÙ   rÉ   r‡   rŸ   c                    s   g | ]}|ˆ vr|‘qS r   r   r¿   )Ú
debug_colsr   r   r=      r>   z:PriceHistory._fix_prices_sudden_change.<locals>.<listcomp>rZ   )rJ  r­  r¦  rŠ  rœ   rg   r    rÍ  r…   zNo Volume datac                 S   s  t dt| ƒƒ}|d@ dkr|d8 }|d }t | ¡}|dk}t |t t|ƒ¡t|ƒd ¡}tj |d d d… ¡d d d… }|| }|dk}t |t t|ƒ¡d¡}tj |¡}|| }t|ƒdkrgt g ¡S tj	|t
d�}tj|||fdtjd�}tjt||ƒdd	�}|S )
Né	   rd   r   rŠ  r…   rÖ  Úconstant)rG  Úconstant_valuesr¥   )rí   r4   r  r¹  r(  ÚarangeÚminimumÚ
accumulateÚmaximumÚasarrayrÍ  ÚpadrØ  Ú	nanmedianr   )r®  ÚWrx  Úvol_denoisedÚmaskr\  Úvol_denoised_paddedr   r   r   Údenoise_volumeT  s,   
  
þz>PriceHistory._fix_prices_sudden_change.<locals>.denoise_volumezNo z
s detectedé   éK   rÉ  z+Estimation of true 1D change stats: mean = z.2fz, StdDev = rJ  r�  z
% of mean)r]   r£  )r_   rb   z>Split ratio too close to normal price volatility. Won't repairz	sd_pct = z  largest_change_pct = z
split_max=z.3fz largest_change_pct=rÑ  z
threshold=z, threshold_rcp=z 1D %z1D %zvol 1D %z+Estimation of true 1D volChg stats: mean = r«  zlargest_volChg_pct = z, threshold_volUnitChg = é   r|   c           
      S   sŠ   t  |j|  ¡ ¡}|| ¡  }t  | ¡}t  |¡t  |¡ }|d  ¡ }t  || ¡}t  ||| d  ¡}t  	|¡}| | | }	|	S )Nr¡   rŠ  )
r  rÖ  r¢   rê  rj   Úexpr¹  rî   rµ  Úsqrt)
r­  rí  ÚblockÚ	distancesr¬  rB   Úweighted_meanÚweighted_varianceÚweighted_stdÚz_scorer   r   r   Ú_calc_volume_zscore_weighted%  s   
zLPriceHistory._fix_prices_sudden_change.<locals>._calc_volume_zscore_weightedc                 S   sl   |d   ¡ }t|ƒdks|dk ¡ rdS t|ƒdkrdS tj|dd�}|dkr)dS t |¡}| | | }|S )Nr¡   r   rd   )Úddofr­   )rµ  r4   r  r  Ústdr’  )r­  r„  rB   rŒ  r’  r‰  r   r   r   Ú_calc_volume_zscore4  s   
zCPriceHistory._fix_prices_sudden_change.<locals>._calc_volume_zscorez'Detected false-positive split error on z, ignoring price dropr  ÚdownÚupr  Ú_rÚ_downÚ_upÚ_fr+   r[   r­  zn/az	Unusual 'z' price action @ zM is actually similar to local price volatility, so ignoring (StdDev % mean = z%)zUnusual price action @ )rJ  r   rh   z<100x changes are too soon after stock split events, abortingr„  Úi_revc                 S   ó   g | ]}|d  ‘qS )r‘  r   r¿   r   r   r   r=   À  rA   c                 S   r•  )r’  r   r¿   r   r   r   r=   À  rA   r¯  zdisplay.max_rowszdisplay.max_columnsé   zdisplay.widthi°  z!price-repair-split: my workings:
c                    s*  | d rt  | ¡} d| d< t  |¡}d|d< t  |¡}d|d< |  ¡ s%g S t  | ¡d }g }tt|ƒd ƒD ]/}|d dkreˆ dkrM|||  rJdnd}n
|||  rUdnd}| || ||d  |f¡ q6t|ƒd dkr“ˆ dkr}||d  rzdnd}n
||d  r…dnd}| |d t| ƒ|f¡ |S )	Nr   Frd   rŠ  rh  rÊ   z	1.0/splitr…   )r  r·  rö   r(  ru   r4   r&   )r  Úf_upÚf_downÚtrue_indicesÚrangesr„  r  )rÊ   r   r   Úmap_signals_to_rangesÇ  s0   


€zEPriceHistory._fix_prices_sudden_change.<locals>.map_signals_to_rangeszidx_latest_active=z, idx_rev_latest_active=)NNNNzcolumn 'z
' ranges: zPruning z range r�  z because too old.c                 S   s   g | ]
}|d ur
dnd‘qS )Nrd   r   r   )r<   r
  r   r   r   r=   )  rÝ  c                    s   g | ]
}ˆ | r
|nd ‘qS r§  r   rƒ  )ÚOHLC_correct_rangesr   r   r=   .  rÝ  z
Potential z detected only in column z(, so treating as false positive (ignore))r   r   r   r   rÊ   g®Gáz®ï?z
Corrected z on col=z range=[ú:z] m=r•   rµ   r
  zCorrected: rž   r�   zPruning range zrange=z m=r¥  r§   z on interval z across intervals rŒ   z (inclusive)z
Corrected:r‹   r¶   r´   )Frè   r   rÄ   rN   rj   rµ  r¢   rí   r¸  r¹  rÏ   rË   rÖ  rö   r·  rý   rO   rÝ   rÓ   rÿ   rü   r  r  r(  Úrollr4   rÙ   ré   r[  rù   r  ÚfullrÊ  rË  Úonesr×  r  ru   r»   rÕ  rÔ  rÉ  Ú
percentiler’  rŒ  rß  r  r  rî   rÎ   rÖ   rm  rÕ   rã   r´  ÚisEnabledForrÁ  ÚDEBUGr#   rÐ   Úoption_contextrÉ   ÚflipÚextendr¹  r×  Úlogical_xorr&   r   r  )”r   rÂ   rv   r4  Úchangerc  rb  rd  r  rÛ  Ú	split_rcpÚ	split_maxrd  ÚmultidayÚfix_typeÚ	start_minr  ÚOHLCÚcorrect_columns_individuallyrz  rï  rð  rf  Úf_no_activityÚappears_suspendedÚf_activeÚidx_latest_activeÚlog_msgÚdf_workingsÚfnaÚ_1d_change_xÚprice_data_colsÚ
price_dataÚf_zeror  Údf_dtyper  r®  r~  Úf_zero_num_denomÚ_1d_change_denoisedrC  Úq1Úq3ÚiqrÚlower_boundÚupper_boundÚavgÚsdÚsd_pctÚlargest_change_pctr  r¼  Úsudden_change_repairedr˜  r—  rÀ   r  Úidx1r{  r„  Úidx0Ú
_1d_volChgÚlargest_volChg_pctÚthreshold_volUnitChgÚ
f_up_ndimsÚf_up_shiftsÚnf_up_shiftsÚflat_indicesÚf_down_ndimsÚdown_dtsr\  rí  r  Úvol_change_pctÚnext_vÚi_pos_in_flat_indicesr  r  r„  Údown_dts_fromÚnext_down_dtÚblock_afterrŠ  r�  Úz_score_afterÚz_score_after_d1Úidx_endÚlookbackÚ	idx_startÚchanges_localr]  Úclean_changesÚ
big_changeÚthreshold_daysÚf_splitsÚ	indices_AÚ	indices_BÚgapsÚf_posÚgap_minÚgap_tdÚwithin_thresholdr{  r›  Ú
any_m_lt_1Úidx_rev_latest_activeÚf_correctedÚf_open_fixedÚf_close_fixedÚidx_first_fr`  Úf_upjÚf_downjÚranges_beforeÚf_rev_downjÚ	f_rev_upjÚf_revjÚranges_afterrš  rl  ÚidxsÚcolÚn_correctedr‡   Úm_rcpr6  Úcounts_prettyÚf_open_and_closed_fixedÚf_open_xor_closed_fixedÚ
f_rev_downÚf_rev_upÚf_revÚ
vol_duringÚvol_outsideÚstepÚi2Úi2nÚi3Úi3bÚr2Úr2nÚr3Úr3bÚvolOutside_denoisedÚvolDuring_denoisedÚboundary_vol_changeÚcol_locr   )rœ  ro  rÊ   r   re  Å  s–  
 *4
 

. 



,
€
$


<,

"&ý
"2,



 
$

$


€
 




("€æ
((€

 
64ÿ!
$&$&&0
:€€

 
F<*
*

é
€


HHô
&
*$&$&&
4€
 $"(

€.*> "8ÿ
(z&PriceHistory._fix_prices_sudden_changer   )rj   rZ   F)rj   F)rj   r§  )F)FFF)!rG   rH   rI   r   r   Úlog_indent_decoratorr
   rÐ   rë   rH  rL  rN  r-   r   r)   rO  rõ   rV  rX  rY  r_  r  r"  r  r÷   r  rA  r@  r  r
  r  r	  re  r   r   r   r   rK   S   st    
üü     		7   
1'

 
%
 
q
         7
<rK   )'Úyfinance._httpr   Úmathr   rä  Úcollections.abcr   ro  rÎ   Údateutilr¸  rÁ  Únumpyr  Únumpy.lib.stride_tricksr   ÚpandasrÐ   rÜ   rÛ   rÅ   Úyfinancer   Úyfinance.configr   Úyfinance.constr   r	   r
   r   Úyfinance.exceptionsr   r   r   r   r   r  r   rK   r   r   r   r   Ú<module>   s&    >