o Tj.s@sddlZddlmZddlZddlZddlmZmZmZm Z ddl m Z m Z ddl mZddlZddlmZmZddlmZdd lmZddlZdd lmZmZmZd d Zd ZddZddZGdddeZ Gddde dZ!dZ"ddddZ#GdddZ$dS)N) lru_cache)requests new_sessionis_supported_session cookie_jar)urlspliturljoin) BeautifulSoup)utilscache) frozendict)YfConfig) YFExceptionYFDataExceptionYFRateLimitErrorcCs0t|ttjtfr dSt|j}hd}||vS)zDCheck if error is transient (network/timeout) and should be retried.T>Timeout ReadTimeout TimeoutErrorConnectTimeoutConnectionErrorRemoteDisconnectedChunkedEncodingError) isinstancersocketerrorOSErrortype__name__) exceptionerror_type_nametransient_error_typesr"P/home/dinkstrade/pdmp-scanner/venv/lib/python3.10/site-packages/yfinance/data.py_is_transient_errors  r$@cCst|tr ||dS|S)N)httphttps)rstr)proxyr"r"r#_normalize_proxy!s  r*cs*tfdd}j|_j|_|S)z Decorator transforms mutable dictionary and list arguments into immutable types Needed so lru_cache can cache method calls what has dict or list arguments. csVtdd|D}dd|D}tdd|D}dd|D}|i|S)NcS"g|] }t|tr t|n|qSr"rdictr .0argr"r"r# /"z9lru_cache_freezeargs..wrapped..cS(i|]\}}|t|trt|n|qSr"r,r/kvr"r"r# 0(z9lru_cache_freezeargs..wrapped..cSr+r"rlisttupler.r"r"r#r11r2cSr3r"r9r4r"r"r#r72r8)r;items)argskwargsfuncr"r#wrapped-s z%lru_cache_freezeargs..wrapped) functoolswraps cache_info cache_clear)r@rAr"r?r#lru_cache_freezeargs's  rFcs,eZdZdZiZeZfddZZ S) SingletonMetaz6 Metaclass that creates a Singleton instance. cs|jB||jvrtj|i|}||j|<n!d|vs$|r9t|dkr9d|vr-|dn|d}|j|||j|WdS1sHwYdS)Nsessionr)_lock _instancessuper__call__lenget _set_session)clsr=r>instancerH __class__r"r#rLCs  $zSingletonMeta.__call__) r __module__ __qualname____doc__rJ threadingLockrIrL __classcell__r"r"rRr#rG<s rGc@s2eZdZdZd2ddZddZddZd d Zd3d d Ze j ddZ e j ddZ e j d4ddZ e j d4ddZe j ddZe j ddZe j d4ddZe j d4ddZe j d5dd Ze j d6d!d"Ze j d6d#d$Zeeed%d5d&d'Zd5d(d)Zd*ed+efd,d-Zd.ejd/e d+ejfd0d1Z!dS)7YfDataz Have one place to retrieve data from Yahoo API in order to ease caching and speed up operations. Singleton means one session one cookie shared by all threads. NcCsBd|_d|_d|_d|_t|_d|_d|_| |pt dS)NFbasic) _crumb_cookie _logged_in_cookie_strategyrWrX _cookie_lockfundamentals_use_chunked_sessionrOrselfrHr"r"r#__init__Vs zYfData.__init__cCsR|j|jj||dd|_d|_d|_WddS1s"wYdS)N)TYT)r`rbcookiesupdater]r^r\)rdcookie_tcookie_yr"r"r#set_login_cookiesos"zYfData.set_login_cookiescCs2|j ||_WddS1swYdS)a!Thread-safe update of the login flag (also read under this lock in _set_cookie_strategy). Auth calls this after verifying the real login state, possibly from another thread, so the write must be serialized to avoid a stale value clobbering a concurrent fresh login.N)r`r^)rdvaluer"r"r#_set_logged_ins"zYfData._set_logged_incCs|durdSt|dddurtdt|s tdt|d|j"||_tjjdur=t tjj|j_ WddSWddS1sHwYdS)Nr znCaching sessions (e.g. requests_cache) are not supported. Solution: stop setting session, let yfinance handle.zUnsupported session type z^; expected curl_cffi or requests Session. Solution: stop setting session, let yfinance handle.) getattrrrrr`rbrnetworkr)r*proxiesrcr"r"r#rOs "zYfData._set_sessionFcCs||jkrdS|s|jz5|jdkr-td|jd|js)|jj d|_ntd|jdd|_d|_ d|_ Wn t yP|j w|sZ|j dSdS)Ncsrfztoggling cookie strategy z -> basicr[z -> csrf)r_r`acquirer get_yf_loggerdebugr^rbrhclearr]r\ Exceptionrelease)rdstrategy have_lockr"r"r#_set_cookie_strategys*       zYfData._set_cookie_strategycCs|jdurdSt|jj}t|dkrdSdd|D}t|dkr+dd|D}t|dkr=td|dSt|dkrEdS|d}|||i}t d|d S) NFrcSsg|]}d|vr|qS)yahoor"r/r5r"r"r#r1z0YfData._save_cookie_curlCffi..rcSsg|]}d|vr|qS)consentr"r}r"r"r#r1r~z1Multiple Yahoo cookies, not sure which to cache: curlCffiT) rbr_cookiesrMkeysr rtrur get_cookie_cachestore)rdrh yh_domains yh_domain yh_cookier"r"r#_save_cookie_curlCffis"       zYfData._save_cookie_curlCfficCs|jdurdStd}|dust|dkrdS|d}t|d}||dd}|j}|dkr9|d}tjj |tj j d }|tj tj j k}|rXt d dSt|jj|||_d S) NFrrcookie/A3geAg@@)tzzcached cookie expiredT)rbr rlookuprMr:rexpiresdatetime fromtimestamptimezoneutcnowr rtrurrrir])rd cookie_dictrhdomainr expiry_ts expiry_dtexpiredr"r"r#_load_cookie_curlCffis& zYfData._load_cookie_curlCffic Cs|jdurtddS|rtddSz |jjd|ddWnEtjj yF}ztdt |WYd}~dSd}~wt yk}zt |rft dt|jd WYd}~dSd}~ww|dS) Nreusing cookieTreusing persistent cookiezhttps://fc.yahoo.comurltimeoutallow_redirectsz$Handling DNS error on cookie fetch: Fz'Cookie fetch from fc.yahoo.com failed (z), continuing without it)r]r rtrurrbrNr exceptionsDNSErrorr(rwr$warningrrr)rdrer"r"r#_get_cookie_basics6   zYfData._get_cookie_basiccCs|jdurtd|jS|sdSd|dd}|jjd i|}|j|_|jdks2d|jvr@td|jt |jdusJd|jvrStd dStd |jd |jS) N reusing crumbz1https://query1.finance.yahoo.com/v1/test/getcrumbTrToo Many RequestsDidn't receive crumb Didn't receive crumb crumb = ''r") r\r rtrurrbrNtext status_coder)rdrget_argscrumb_responser"r"r#_get_crumb_basics& zYfData._get_crumb_basiccCs||sdS||SN)rr)rdrr"r"r#_get_cookie_and_crumb_basics  z"YfData._get_cookie_and_crumb_basiccCs|jdurtddS|rtdd|_dSd|i}i|ddi}z |jjdi|}Wntjj yHtdYdSwt |j d }|j d d d id }|duretddS|d}td||j d d did }|d} td| d} d} ddgd| || | d} i|d| | d} i|d| | d}z|jj di| |jjdi|Wntjj ytdYnwd|_|dS)NrTrrrzhttps://guce.yahoo.com/consentzR_get_cookie_csrf() encountering requests.exceptions.ChunkedEncodingError, abortingF html.parserinputname csrfToken)attrsz&Failed to find "csrfToken" in responsermz csrfToken = sessionIdz sessionId='zhttps://finance.yahoo.com/r|agreedefault)r consentUUIDrroriginalDoneUrl namespacez6https://consent.yahoo.com/v2/collectConsent?sessionId=)rdataz-https://guce.yahoo.com/copyConsent?sessionId=r")r]r rtrurrbrNrrrr contentfindpostr)rdr base_argsrresponsesoupcsrfTokenInputrsessionIdInputrrrr post_argsr"r"r#_get_cookie_csrf#sh  zYfData._get_cookie_csrfcCs|jdurtd|jS||sdSd|d}|jjd i|}|j|_|jdks2d|jvr@td|jt |jdusOd|jvsO|jdkrXtd dStd |jd |jS) Nrz1https://query2.finance.yahoo.com/v1/test/getcrumb)rrrrrrrrrr") r\r rtrurrbrNrrr)rdrrrr"r"r#_get_crumb_csrf^s$  zYfData._get_crumb_csrfcCsd\}}td|jd|j<|jdkr.|}|dur-|jddd||}n||}|durB|jddd|}|j}Wd||fS1sRwY||fS)N)NNzcookie_mode = 'rrrr[T)rz)r rtrur_r`rr{r)rdrcrumbryr"r"r#_get_cookie_and_crumb{s$    zYfData._get_cookie_and_crumbcCs8|j||jj||d}||js |S|||}|S)N)request_methodparamsr) _make_requestrbrN_is_this_consent_urlr_accept_consent_formrdrrrrr"r"r#rNs   z YfData.getcCs|j||jj||||dS)N)rbodyrrr)rrbr)rdrrrrrr"r"r#rsz YfData.postc Cst|dkrtd|dddn td|td|tjjdur8ttjj|j_ |dur>i}d|vrFt dz| \}}WnAt ydt dd|j}}Yn,ty} z t| rt dt| jd d|j}}nWYd} ~ nd} ~ ww|durd|i} ni} |i|| |d } |r|| d <|r|| d <d di| d<ttjjdD]1} z |di| } Wn%ty} zt| r| tjjkrtd| nWYd} ~ qd} ~ wwtd| j| jdkrtz|dkr|dn|d| |\}}Wn&t y'd}YntyA} zt| s5d}WYd} ~ nd} ~ wwd| dvrN| dd=|durY|| dd<|di| } td| j| jdkrtt | S)Nzurl=z...zparams=rz@Don't manually add 'crumb' to params dict, let data.py handle itz=Crumb fetch rate-limited (HTTP 429), continuing without crumbzCookie/crumb fetch failed (z), continuing without crumb)rrrjsonrz Content-Typezapplication/jsonheadersrzresponse code=ir[rrrrr")rMr rtrurrpr)r*rbrqrrrrr_rwr$rrrangeretries_timesleeprr{)rdrrrrrrrryrcrumbs request_argsattemptrr"r"r#rs                  zYfData._make_request)maxsizecCs||||SrrN)rdrrrr"r"r# cache_getszYfData.cache_getcCs4td||j|||d}||S)Nzget_raw_json(): )rr)r rtrurNraise_for_statusrrr"r"r# get_raw_jsonszYfData.get_raw_json response_urlreturncCs2zt|jo t|jdWStyYdSw)z Check if given response_url is consent page Args: response_url (str) : response.url Returns: True : This is cookie-consent page False : This is not cookie-consent page zconsent.yahoo.comF)rhostnameendswithrw)rdrr"r"r#r s  zYfData._is_this_consent_url consent_resprcCst|jd}|d}|s|S|dp|j}t|j|} i}|dD]>}|d}|s/q%|dp5d} |dp>d } | d vr_d |vsTd |vsT|d r^| d krZ| nd||<q%| ||<q%dd| D} t dd| Dszd|d <d|ji} |j j ||| |dd} | S)a Click 'Accept all' to cookie-consent form and return response object. Args: consent_resp (requests.Response) : Response instance of cookie-consent page timeout (int) : Raise TimeoutError if post doesn't respond Returns: response (requests.Response) : Response instance received from the server after accepting cookie-consent post. rformactionrrrrrmr)checkboxradioracceptchecked1cSsh|]}|qSr")lowerr}r"r"r# Nsz.YfData._accept_consent_form..css |] }d|vp d|vVqdS)rrNr"r}r"r"r# Osz.YfData._accept_consent_form..RefererT)rrrr) r rrrNrr find_allrhas_attrranyrbr)rdrrrrrrinprtypvalloweredrrr"r"r#rs<        zYfData._accept_consent_formr)F)r)Nr)NNrN)"rrTrUrVrerlrnrOr{r log_indent_decoratorrrrrrrrrrNrrrFr cache_maxsizerrr(boolrrResponseintrr"r"r"r#rZPsR        :     Y rZ) metaclasszDhttps://query1.finance.yahoo.com/ws/obi-integration/v1/subscriptionsgoldsilverbronze)c@steZdZdddZdededefddZdedBfd d Zdefd d Z dedBfd dZ e dedBfddZ dS)AuthNcCs||_t||_dSr)rbrZ_datarcr"r"r#redsz Auth.__init__rjrkrcCs,|j|||}|std|S)aW Set the login cookies and verify they are valid. How to Obtain the Cookies: 1. Open your browser (e.g., Chrome, Firefox). 2. Log in to Yahoo Finance (https://finance.yahoo.com). 3. Open the browser's Developer Tools: Press `F12` or `Ctrl + Shift + I` (Windows/Linux) or `Cmd + Option + I` (Mac). 4. Go to the "Application" tab (Chrome) or "Storage" tab (Firefox). 5. In the "Cookies" section, select `https://finance.yahoo.com`. 6. Look for the cookies named `T` and `Y`. 7. Copy the values of these cookies and pass them to this function. Args: cookie_t (str): The value for the 'T' cookie. cookie_y (str): The value for the 'Y' cookie. Returns: bool: ``True`` if the cookies are valid (the account is logged in), ``False`` otherwise (also emitted as a warning). The cookies are stored regardless. ``False`` can also mean Yahoo was transiently unreachable, so it is not treated as a hard error. zTset_login_cookies: the provided cookies are not logged in (or Yahoo is unreachable).)r rl check_loginr rtr)rdrjrk logged_inr"r"r#rlhszAuth.set_login_cookiesc Csz0|jt}|jdkr(|pid}t|tr(|dr(|jd|WS|jdWdStyR}zt j j s=t d|WYd}~dSd}~ww)aFetch the account's subscription entitlement (live, not cached). A single lightweight JSON call to the OBI subscriptions endpoint determines both login state and subscription tier, avoiding any consumer-web-page scraping. The result is intentionally not cached: the endpoint is cheap and Yahoo does not rate-limit it at any realistic volume, so a fresh call each time keeps the answer from going stale (e.g. if the login session expires part-way through a long-running process). Returns: dict | None: The entitlement ``result`` object when logged in, or ``None`` when not logged in (anonymous sessions return HTTP 401). rresultguidTFNzError confirming login: )r rN_SUBSCRIPTIONS_URLrrrr-rnrwrruhide_exceptionsr rtr)rdrrrr"r"r#_fetch_entitlements     zAuth._fetch_entitlementcCs |duS)uCheck whether the user is logged in to Yahoo Finance. Note: ``False`` during a transient error (e.g. Yahoo briefly unreachable) means "could not confirm" rather than "logged out" — the stored login cookies are kept and remain protected in that case. Nr)rdr"r"r#r s zAuth.check_logincCsJ|}|dur dSdd|dpgD}|sdSt|dddS) aMReturn the Yahoo Finance subscription tier of the logged-in account. Read directly from the account's tier id in ``subscriptionView`` (the value the account is billed against) rather than inferring it from the granted feature set, which Yahoo reshuffles between tiers. Returns: str | None: ``'gold'``, ``'silver'`` or ``'bronze'`` for a named subscription (``'premium'`` for a subscribed tier with no marketed name), ``'free'`` when logged in without a subscription, or ``None`` when not logged in. NcSsg|] }|ddkr|qS)rACTIVEr)r/sr"r"r#r1sz*Auth.subscription_tier..subscriptionViewfreertierpremium)rrN _TIER_NAMES)rd entitlementactiver"r"r#subscription_tiers zAuth.subscription_tiercCs|}|r d|diSdS)z Get the logged-in user's details. Returns: dict | None: ``{'guid': ...}`` if logged in, or ``None`` if not. rNr)rdrr"r"r#usersz Auth.userr) rrTrUrer(rrlr-rr rpropertyrr"r"r"r#r cs !# r )%rBrrtimer_httprrrr urllib.parserr bs4r rrr r r configrrWrrrrr$rr*rFrrGrZrrr r"r"r"r#s4