o Tje@sddlmZddlZddlmZmZddlZddlZddl m m Z ddl mZddlmZddlmZddlmZmZddlmZmZmZmZdd lmZdd lmZddlm m!Z"dd lm#Z#m$Z$dd l%m&Z&dd l'm(Z(ddl)m*Z*ddl+m,Z,m-Z-m.Z.ddl/m0Z0ddl1m2Z2ddl3m4Z4m5Z5m6Z6m7Z7m8Z8erddl9m:Z:m;Z;mZ>ddl?m@Z@GdddZA dCdDddZBdCdEd d!ZC dCdFd$d%ZDdGd(d)ZEdHdId-d.ZFdJdId/d0ZGdKd3d4ZH dHdLd7d8ZIdMd=d>ZJdNdAdBZKdS)O) annotationsN) TYPE_CHECKINGcast)PerformanceWarning)cache_readonly)find_stack_level)find_common_type maybe_promote)ensure_platform_intis_1d_only_ea_dtype is_integerneeds_i8_conversion)ExtensionDtype)notna) factorizeunique)factorize_from_iterable)ensure_wrapped_if_datetimelike) DataFrame)Index MultiIndex RangeIndex)concat)Series)compress_group_indexdecons_obs_group_idsget_compressed_idsget_group_indexget_group_index_sorter) ArrayLikeLevelnpt)ExtensionArray) FrozenListc@seZdZdZ d,d-d d Zed.ddZed/ddZd0ddZddZ ed1ddZ ed2ddZ d3dd Z d4d"d#Z d5d&d'Zed6d(d)Zed7d*d+Zd!S)8 _Unstackera  Helper class to unstack data / pivot with multi-level index Parameters ---------- index : MultiIndex level : int or str, default last level Level to "unstack". Accepts a name for the level. fill_value : scalar, optional Default value to fill in missing values if subgroups do not have the same set of labels. By default, missing values will be replaced with the default fill value for that data type, NaN for float, NaT for datetimelike, etc. For integer types, by default data will converted to float and missing values will be set to NaN. constructor : object Pandas ``DataFrame`` or subclass used to create unstacked response. If None, DataFrame will be used. Examples -------- >>> index = pd.MultiIndex.from_tuples([('one', 'a'), ('one', 'b'), ... ('two', 'a'), ('two', 'b')]) >>> s = pd.Series(np.arange(1, 5, dtype=np.int64), index=index) >>> s one a 1 b 2 two a 3 b 4 dtype: int64 >>> s.unstack(level=-1) a b one 1 2 two 3 4 >>> s.unstack(level=0) one two a 1 3 b 2 4 Returns ------- unstacked : DataFrame Tindexrlevelr sortboolreturnNonec Cs||_||_||_|j||_d|jj|jvrdnd|_t|jj |_ t|jj |_ |j |j|_|j |j|_|j |j|_|js_t|jj|j}|j||_|j||_tdd|j D}|jj}||}|ttjjkrtjd|dttd|dS) NrcSsg|]}|jqS)size).0 index_levelr-r-^/home/dinkstrade/pdmp-scanner/venv/lib/python3.10/site-packages/pandas/core/reshape/reshape.py sz'_Unstacker.__init__..z%The following operation may generate z& cells in the resulting pandas object.) stacklevel) constructorr'remove_unused_levelsr%_get_level_numberr&codesliftlistlevelsnew_index_levelsnamesnew_index_namespop removed_name removed_levelremoved_level_fullrtakenpmaxr.iinfoint32warningswarnrr_make_selectors) selfr%r&r4r' unique_codesnum_rows num_columns num_cellsr-r-r1__init__ns0   z_Unstacker.__init__-tuple[npt.NDArray[np.intp], list[np.ndarray]]c Cs|j}t|jj}t|jj}|d|||dd||g}tdd|d|||dd||gD}t||\}}t|}t||} | |fS)Nr,cs|]}t|VqdSNlenr/xr-r-r1 z2_Unstacker._indexer_and_to_sort..) r&r9r%r7r:tuplerrTr) rJvr7levsto_sortsizes comp_indexobs_idsngroupsindexerr-r-r1_indexer_and_to_sorts  &4 z_Unstacker._indexer_and_to_sortlist[np.ndarray]cs&|j\}|jrfdd|DS|S)Ncg|]}|qSr-rB)r/linerar-r1r2z,_Unstacker.sorted_labels..)rbr')rJr\r-rgr1 sorted_labelss z_Unstacker.sorted_labelsvalues np.ndarraycCs(|jr|j\}}tj||dd}|S|S)Nr)axis)r'rbalgostake_nd)rJrjra_ sorted_valuesr-r-r1_make_sorted_valuess  z_Unstacker._make_sorted_valuesc Cs|j}|jdd}tdd|D}t||\}}t|}t|}|jj|j|j }||f|_ |jd|||j }t j t |j td} | |d| t|jkr[td||_| |_|jro|t ||_dSt t j|ddd|_dS) Nr+csrQrRrSrUr-r-r1rWrXz-_Unstacker._make_selectors..dtypeTz0Index contains duplicate entries, cannot reshape) return_indexr,)r;rirYrrTr r%levshaper&r8 full_shaperCzerosprodr(putsum ValueError group_indexmaskr' searchsortedarange compressorr) rJ new_levelsremaining_labels level_sizesr^r_r`strideselectorr}r-r-r1rIs$  z_Unstacker._make_selectorscCst|jSrR)r(r}allrJr-r-r1mask_allsz_Unstacker.mask_all2tuple[npt.NDArray[np.intp], npt.NDArray[np.bool_]]cCs6tjt|jtjd}|j|dd\}}||dfS)Nrrr+ fill_valuer)rCrrTr%intpget_new_valuesany)rJ dummy_arr new_valuesr}r-r-r1 arange_resultsz_Unstacker.arange_resultrcCsn|jdkr|ddtjf}|dur|jddkrtd|||\}}||}|j}|j||||j dS)Nr,z-must pass column labels for multi-column data)r%columnsrs) ndimrCnewaxisshaper{rget_new_columns new_indexr4rs)rJrj value_columnsrrorr%r-r-r1 get_results   z_Unstacker.get_resultNc Cs|jdkr|ddtjf}||}|j\}}|jd}||}||f}|j} |j} | rJt|rJ| ||| dd |} tj |t d} | | fS|j } | rZ|j } tj|| d} n*t| trq| }|j|| d} || dd<nt| |\} }tj|| d} | || j}tj|t d} t|j r|d}| d} n|j|dd}t|| d|||| | dt|j r| d} t| } | |j } | | fS) Nr,rri8Fcopyu1zM8[ns])rrCrrqrvrr}rrTreshapeswapaxesonesr(rsempty isinstancerconstruct_array_type_emptyr fillnamerwr viewastype libreshapeunstackr)rJrjrrplengthwidthr result_width result_shaper}rrnew_maskrsclsrr-r-r1rs^             z_Unstacker.get_new_valuesr Index | Nonec s|dur!|jdkr|jj|jdS|jjd|jjd}||jSt|j|j}t|}t t ||t |t rT|j |jf}|j|jf}fdd|jD}n||jg}|j|jg}g}|j}|t||t |||ddS)Nrr)itemcrdr-rer/lab propagatorr-r1r2Zrhz._Unstacker.get_new_columns..Fr:r7r<verify_integrity)r8r@_renamer?insert _na_valuerenamerTrCrepeatrrrr:rAr<r7r _repeaterappendtile) rJrlevrrr new_names new_codesrepeaterr-rr1rEs0    z_Unstacker.get_new_columnscCs^t|jt|jkr|j|j}|jrt|dd}|St|j|j}t||j}|S)Nrr+)rTrAr@ get_indexerr8rCrr)rJrrr-r-r1rksz_Unstacker._repeatercsfddjddD}tjdkr9jd|d}}|dkr.|t||j}||jdSt j|jddS)Ncsg|]}|jqSr-)rBrrrr-r1r2}z(_Unstacker.new_index..r+r,rFr) rirTr;rrrrBrr=r)rJ result_codesr& level_codesr-rr1rzs z_Unstacker.new_index)T)r%rr&r r'r(r)r*)r)rP)r)rc)rjrkr)rk)r)r()r)r)r)rrR)rr)r)rk)r)r)__name__ __module__ __qualname____doc__rOrrbrirqrIrrrrrrrr-r-r-r1r$@s*. .        O& r$TdataSeries | DataFramer'r(csxtdkr|S|jttjvrgfddDfddtjD}fddD}fddD}fddD}fdd|D}fd d|D} fd d|D} td d |D} t|| d d d} t | d d\} }t | || |d d}|st |dd}nt||g| | g| dgd d}t |t r|}||_|jd||d}|}|}|}njt |jtr|}r܈d|j||d}fddDs|S|jd d}||_|jd||d}t |t r|jn|jt tsJjdg|}|jjg|}jdg}|fdd |Dt|||d d}t |t r7||_|S||_|S)Nrcsg|]}|qSr-)r6r/ir%r-r1r2rhz%_unstack_multiple..cg|]}|vr|qSr-r-r)clocsr-r1r2rcg|]}j|qSr-r:rrr-r1r2rhcrr-r7rrr-r1r2rhcrr-r<rrr-r1r2rhcrr-rrrr-r1r2rhcrr-rrrr-r1r2rhcrr-rrrr-r1r2rhcsrQrRrSrUr-r-r1rWrXz$_unstack_multiple..F)r'xnullr')r__placeholder__rrrr'cs g|] }|kr |n|dqSr,r-r/rZ)valr-r1r2 deepc3s |] }|jdVqdS)r+N)rBr7)r/rec)unstcolsr-r1rWs)rTr%rrr<rangenlevelsrYrrrrrrrrrr>r:rr7extend)rrrr'rlocsclevelsccodescnamesrlevelsrcodesrnamesrr|comp_idsr_ recons_codes dummy_indexdummy unstackedrrrresultdummy_df new_columnsr-)rr%rrr1_unstack_multiples|          robjcCst|ttfrt|dkrt||||dS|d}t|s'|dks'|j|t|trAt|jt r:t ||||dS|j j ddSt|jt sRt dt|jdt|jr_t||||d St|j||j|d }|j|jd|d S) Nr,rrrT) future_stackz'index must be a MultiIndex to unstack, z was passedrr&r4r'rr)rrYr9rTrr r%r6rr_unstack_frameTstackr{typer rs_unstack_extension_seriesr$_constructor_expanddimr_values)rr&rr' unstackerr-r-r1rs,       rrr)cCs^t|jtsJt|j||j|d}|js%|jj||d}|j||j dS|j |j |j |dS)Nrr)axesr) rr%rr$ _constructor_can_fast_transpose_mgrr_constructor_from_mgrrrrr)rr&rr'rmgrr-r-r1r s  rseriesrcCs,|}|j|||d}|jdg|_|S)an Unstack an ExtensionArray-backed Series. The ExtensionDtype is preserved. Parameters ---------- series : Series A Series with an ExtensionArray for values level : Any The level name or number. fill_value : Any The user-level (not physical storage) fill value to use for missing values introduced by the reshape. Passed to ``series.values.take``. sort : bool Whether to sort the resulting MuliIndex levels Returns ------- DataFrame Each column of the DataFrame will have the same dtype as the input Series. )r&rr'r)to_framerr_drop_level_numbers)r r&rr'dfrr-r-r1rsrr+framedropnacsdd}|j\}|j|}t|jtrt||||dSt|jtrat|jj}fdd|jj D}||j\} } | | | t | | t|jj} | |jjt||| dd} n*tt||j|jf\} \}} |t | | f}t| ||jj|jjgdd} |js|jrt|jj}|d}t|tr|}|d d|D}t||}n |j }n|j }|rt|}||}| |} |j|| d S) z Convert DataFrame to Series with multi-level Index. Columns become the second level of the resulting hierarchical index Returns ------- stacked : Series or DataFrame cSs,|jr |tt|fSt|\}}||fSrR) is_uniquerCrrTr)r%r7 categoriesr-r-r1stack_factorizeMs zstack..stack_factorize) level_numrr'crdr-rrKr-r1r2^rhzstack..FrrcSsg|]\}}|jqSr-)r)r/rocolr-r-r1r2}rhr)rrr6rr_stack_multi_columnsr%r9r:r7rrCrravelr<rzipmaprr_is_homogeneous_typedtypesrrr_concat_same_typeitems"_reorder_for_extension_array_stackr_constructor_sliced)rr&rr'rNrrrclevclabrrr:ilabr7rrsarrrr}r-rr1rCsT            rcstfdd|Dr}|D] t|||d}q|Stdd|DrL}fdd|D}|rJ|dt|||d}fdd|D}|s2|Std ) Nc3s|] }|jjvVqdSrR)rr<r/rrr-r1rWsz!stack_multiple..)rr'css|]}t|tVqdSrR)rintr(r-r-r1rWscsg|]}j|qSr-)rr6r(r)r-r1r2rz"stack_multiple..rcs g|] }|kr |n|dqSrr-rrr-r1r2rzTlevel should contain all level names or all level numbers, not a mixture of the two.)rrr>r{)rr&rr'rr-)rrr1stack_multiples"  r,rrcCst|jdkr|jdj|jddSddt|jdd|jddD}t|}dd t|D}t|}tj d dt||jD|jddd S) zBCreates a MultiIndex from the first N-1 levels of this MultiIndex.rrrcs"g|] \}fdd|DqS)cs g|] }|dkr |ndqS)rNr-)r/cr+r-r1r2rz8_stack_multi_column_index...r-r/r7r-r+r1r2sz-_stack_multi_column_index..Nr+css|]\}}|VqdSrRr-)r/keyror-r-r1rWrXz,_stack_multi_column_index..cSs*g|]\}}d|vrt||jdn|qS)Nrr)rrs)r/new_levrr-r-r1r2sr) rTr:rr<rr7 itertoolsgroupbyr from_arrays)rr[tuples unique_tuplesnew_levsr-r-r1_stack_multi_column_indexs  r7rr*c$ snddd}|jdd}|j}t|tsJ||jd kr@|}t||jd D]}|||} ||d |} || | }q&||_}|sU|rU|d |} |j| d d }|j}t t|}t |} i} |j d }t |j d }|rst|}|t|d}t||}t|g}| D]}z|j|}Wnty||Yqwt|tst|}n|j|j}|kr|jdd|j|f}||jj d |_|j|d j}nB|jdd|f}t|jtt r!"fdd|#D}|j$\}}t%||&||j'(}||}n|j}|j)d kr |(}|| |<qt|d kr1| *|} t|}t|j+trTt,|j+j }t,|j+j-}fdd|j+j D}nt.|j+\}} | g}|/g}|j+j0g}|||t1||||jj-|t|||dd}!|j2| |!| d}"|jjd kr|j3|g }#|"j4|#s|"|#}"|r|"j5d dd}"|"S)Nrr*rrcSs||jvr |j|S|S)z Logic for converting the level number to something we can safely pass to swaplevel. If `level_num` matches a column name return the name from position `level_num`, otherwise return `level_num`. r)rrr-r-r1_convert_level_numbers  z3_stack_multi_columns.._convert_level_numberFrr,r)r&rlr+)rcs g|] \}}|jjddqS)Fr)rr)r/rorVrrr-r1r2rz(_stack_multi_columns..crdr-rr)levsizer-r1r22rhr)r%rr)rlhow)rr*rr)6rrrrrr swaplevel _is_lexsorted sort_indexrr7r:rr7rCr'rrTrBget_locKeyErrorrslicestopstartlocreindexrjilocrrtolistrrrr rrrrrr differencer%r9r<rrrrrr equalsr)$rrrr'r8thismi_cols roll_columnsrlev1lev2 level_to_sortrnew_data level_valsrlevel_vals_nanlevel_vals_used drop_colsr/rC slice_lenchunk value_slicesubsetr#ridxrrr old_codes old_levelsrrdesired_columnsr-)rsr9r1rs                          rr'r"n_rows n_columnscCs&t||||j}||S)a Re-orders the values when stacking multiple extension-arrays. The indirect stacking method used for EAs requires a followup take to get the order correct. Parameters ---------- arr : ExtensionArray n_rows, n_columns : int The number of rows and columns in the original DataFrame. Returns ------- taken : ExtensionArray The original `arr` with elements re-ordered appropriately Examples -------- >>> arr = np.array(['a', 'b', 'c', 'd', 'e', 'f']) >>> _reorder_for_extension_array_stack(arr, 2, 3) array(['a', 'c', 'e', 'b', 'd', 'f'], dtype='>> _reorder_for_extension_array_stack(arr, 3, 2) array(['a', 'd', 'b', 'e', 'c', 'f'], dtype='.r+r,c3s(|]}|vr tntdVqdSrR)nextr@r`)genr&r-r1rWs  zstack_v3..r)rrsF)use_na_sentinelcsg|] }t|tqSr-)rCrrTr.r)r-r1r2srrr))rnuniquerTr{sortedr rrrCargsort_reorder_ilevelsrriterrYrCrrrrrrrrrsrHrr%rr:r9rr7rdrop_duplicatesr<rrrBrrE)rr& drop_levnums stack_colssorterordered_stack_colsstack_cols_uniqueordered_stack_cols_uniquebufrXrcolumn_indexerrratiorr[ index_levels index_codesr7uniques column_levels column_codeslen_df n_uniquesraidxsr-)rrcr&r1stack_v3us "              "  r|)NT)rrr'r()rrr'r()rrr'r(r)r)r rr'r(r)r)r+TT)rrrr(r'r()TT)rrr)r) rrrr*rr(r'r(r)r)r'r"r\r*r]r*r)r")rrr&r^r)r)L __future__rr1typingrrrGnumpyrCpandas._libs.reshape_libsrr pandas.errorsrpandas.util._decoratorsrpandas.util._exceptionsrpandas.core.dtypes.castrr pandas.core.dtypes.commonr r r r pandas.core.dtypes.dtypesrpandas.core.dtypes.missingrpandas.core.algorithmscore algorithmsrmrrpandas.core.arrays.categoricalrpandas.core.constructionrpandas.core.framerpandas.core.indexes.apirrrpandas.core.reshape.concatrpandas.core.seriesrpandas.core.sortingrrrrrpandas._typingrr r!pandas.core.arraysr"pandas.core.indexes.frozenr#r$rrrrrr,r7rr!r|r-r-r-r1sV             Q [%  % M !  %