o 5cړ@sddlmZddlmZmZmZddlZddlm Z ddl m Z m Z m Z mZddlmZmZmZmZmZmZmZddlmZmZddlmZmZmZmZmZdd l m!Z!dd l"m#Z#dd l$m%Z%dd l&m'Z'dd l(m)Z)m*Z*m+Z+esHddl,Z-ddl.m/Z0ddl1m2Z2ddl3m4Z4e0j5e0j6e0j7e0j8e0j9e0j:dZ;ereereF r1cCr(r))r* or_kleener,r/r/r0r1Jr2cCr(r))r*xorr,r/r/r0r1Lr2)andrandorrorr4rxor arrow_arraypa.ChunkedArray pa_objectpa.Array | pa.ScalarreturncCs.tj|jrtj|jr|tS|Sr))patypesrtypecastfloat64)r:r<r/r/r0cast_for_truedivOs rDleft&pa.ChunkedArray | pa.Array | pa.ScalarrightcCs>tt||}tj|jrtj|jr||j}|Sr))r*floordivide_checkedr?r@rrArB)rErGresultr/r/r0floordiv_compatZs rKaddraddcCr(r))r* add_checkedr,r/r/r0r1ir2subrsubcCr(r))r*subtract_checkedr,r/r/r0r1mr2mulrmulcCr(r))r*multiply_checkedr,r/r/r0r1qr2truedivcCstt|||Sr)r*rIrDr,r/r/r0r1trtruedivcCst|t||Sr)rVr,r/r/r0r1wrWfloordivcCs t||Sr)rKr,r/r/r0r1z rfloordivcCs t||Sr)rZr,r/r/r0r1}r[modrmoddivmodrdivmodpowrpowcCr(r))r* power_checkedr,r/r/r0r1r2)SeriesArrowExtensionArrayTArrowExtensionArray)bounddtype'ArrowDtype | pa.DataType | Dtype | Nonepa.DataType | NonecCsBt|tr |j}|St|tjr|}|S|rt|}|Sd}|S)z3 Convert dtype to a pyarrow type instance. N) isinstancer! pyarrow_dtyper?DataTypefrom_numpy_dtype)rhpa_dtyper/r/r0to_pyarrow_types   rpcs6eZdZUdZded<ded<dd d Zed d ddddZed d ddddZdddZ dddZ dddZ ddd Z dd!d"Z dd#d$Zd%d&Zd'd(Zd)d*Zd+d,Zdd.d/Zedd0d1Zedd3d4Zdd5d6Zedd7d8Zdd:d;Zed dgd< = > ?ddfdEdF ZdfdIdJ ZdddKdLZdddMdNZddOdPZdfdQdR Z dfdSdT Z!ddVdWZ"e#e$j%e&j'e&j'fdd]d^Z%d_d`Z( dddfdgZ)dfdhdi Z*dddldmZ+eddpdqZ,d=drdfdtduZ-ddydzZ.dd|d}Z/dddZ0ddddZ1ddZ2dddZ3edddZ4Z5S)rfaW Pandas ExtensionArray backed by a PyArrow ChunkedArray. .. warning:: ArrowExtensionArray is considered experimental. The implementation and parts of the API may change without warning. Parameters ---------- values : pyarrow.Array or pyarrow.ChunkedArray Attributes ---------- None Methods ------- None Returns ------- ArrowExtensionArray Notes ----- Most methods are implemented using `pyarrow compute functions. `__ Some methods may either raise an exception or raise a ``PerformanceWarning`` if an associated compute function is not available based on the installed version of PyArrow. Please install the latest version of PyArrow to enable the best functionality and avoid potential bugs in prior versions of PyArrow. Examples -------- Create an ArrowExtensionArray with :func:`pandas.array`: >>> pd.array([1, 1, None], dtype="int64[pyarrow]") [1, 1, ] Length: 3, dtype: int64[pyarrow] r;_datar!_dtypevaluespa.Array | pa.ChunkedArrayr>NonecCsftrd}t|t|tjrt|g|_nt|tjr ||_n tdt |dt |jj |_ dS)NzBpyarrow>=1.0.0 is required for PyArrow backed ArrowExtensionArray.zUnsupported type 'z' for ArrowExtensionArray) r ImportErrorrkr?Array chunked_arrayrq ChunkedArray ValueErrorrAr!rr)selfrsmsgr/r/r0__init__s  zArrowExtensionArray.__init__NFrhcopyrh Dtype | NonecCsbt|}t||}|st|tjtjfr$|r|j}|r ||}||S|ttj||ddS)zL Construct a new ExtensionArray from a sequence of scalars. TrA from_pandas) rprkr?rwryrqrBrxarray)clsscalarsrhrrois_clsr/r/r0_from_sequences  z"ArrowExtensionArray._from_sequencec Cs6t|}|dur |}ntj|rddlm}||dd}nutj|r2ddlm}||ddj}natj|rEddl m }||dd}nNtj |rXddl m }||dd}n;tj|rldd lm} | |}n'tj|s~tj|s~tj|rdd lm} | |dd}ntd |d |j|||d S)zL Construct a new ExtensionArray from a sequence of strings. Nr) to_datetimeraise)errors) to_timedelta)to_timecoerce BooleanArray) to_numericzConverting strings to z is not implemented.r~)rpr?r@ is_timestamppandas.core.tools.datetimesris_datedate is_durationpandas.core.tools.timedeltasris_timepandas.core.tools.timesr is_booleanpandas.core.arraysr_from_sequence_of_stringsto_numpyr is_floating is_decimalpandas.core.tools.numericrNotImplementedErrorr) rstringsrhrpa_typerrrrrrr/r/r0rs<               z-ArrowExtensionArray._from_sequence_of_stringsitemrcCst||}t|tjrLt|s/|jjdkr |jjdkr t }n|jj }t |tj g|dSt |jr9||St|jrHt ||j|Stdt|trUt|}|tur]td}t|rit|sitd|j|}t|tjrzt ||S|}|dur|jjS|S)a8Select a subset of self. Parameters ---------- item : int, slice, or ndarray * int: The position in 'self' to get. * slice: A slice object, where 'start', 'stop', and 'step' are integers or None * ndarray: A 1-d boolean NumPy ndarray the same length as 'self' Returns ------- item : scalar or ExtensionArray Notes ----- For scalar ``item``, return a scalar value suitable for the array's type. This should be an instance of ``self.dtype.type``. For slice ``key``, return an instance of ``ExtensionArray``, even if the slice is length 0 or 1. For a boolean mask, return an instance of ``ExtensionArray``, filtered to the values where ``item`` is True. stringpyarrowrAzFOnly integers, slices and integer or boolean arrays are valid indices.Nzuonly integers, slices (`:`), ellipsis (`...`), numpy.newaxis (`None`) and integer or boolean arrays are valid indices)rrknpndarraylenrrnamestorager?rrlrArxrrhtakerrqfilter IndexErrortuplerEllipsisslicerrryas_pyna_value)r{rrovaluescalarr/r/r0 __getitem__s:          zArrowExtensionArray.__getitem__cC|jS)z)Convert myself to a pyarrow ChunkedArray.)rq)r{rAr/r/r0__arrow_array__esz#ArrowExtensionArray.__arrow_array__r{recCs trtdt|t|jS)Nz*__invert__ not implement for pyarrow < 2.0)r rrAr*invertrqr{r/r/r0 __invert__iszArrowExtensionArray.__invert__cCt|t|jSr))rAr*negate_checkedrqrr/r/r0__neg__nzArrowExtensionArray.__neg__cCt||jSr)rArqrr/r/r0__pos__qzArrowExtensionArray.__pos__cCrr))rAr* abs_checkedrqrr/r/r0__abs__trzArrowExtensionArray.__abs__c Csddlm}t|j}t|tr||j|j}n_t|tjt fr'||j|}nPt |rkz ||jt |}Wn@t j jt j jfyjt|t|B}|}tjt|dd}|t|||||<|||YSwt|jdt|tr|j}n|}||S)Nrrboolrh not implemented for ) pandas.arraysrARROW_CMP_FUNCS__name__rkrfrqrrlistrr?rrArrowNotImplementedError ArrowInvalidrzerosrrrrAr to_pandasrsrr)r{otheroprpc_funcrJmaskvalidr/r/r0 _cmp_methodws.     zArrowExtensionArray._cmp_methodcCs||j}|turt|jdt|tr||j|j}n.t|tjtfr2||jt j |dd}nt |r@||jt |}n t|jdt |t ||S)Nz not implemented.Trr)rNotImplementedrrkrfrqrrrr?rrrrA)r{rr arrow_funcsrrJr/r/r0_evaluate_op_methods   z'ArrowExtensionArray._evaluate_op_methodcC|||tSr))rARROW_LOGICAL_FUNCSr{rrr/r/r0_logical_methodrz#ArrowExtensionArray._logical_methodcCrr))rARROW_ARITHMETIC_FUNCSrr/r/r0 _arith_methodrz!ArrowExtensionArray._arith_methodrcCst|tsdS|j|jkS)NF)rkrfrq)r{rr/r/r0equalss  zArrowExtensionArray.equalscCr)z2 An instance of 'ExtensionDtype'. )rrrr/r/r0rhszArrowExtensionArray.dtypeintcCs|jjS)zL The number of bytes needed to store this object in memory. )rqnbytesrr/r/r0rszArrowExtensionArray.nbytescCs t|jS)z] Length of this array. Returns ------- length : int )rrqrr/r/r0__len__s zArrowExtensionArray.__len__cCs |jjdkS)Nr)rq null_countrr/r/r0_hasnas zArrowExtensionArray._hasnanpt.NDArray[np.bool_]cCs"tr |jjS|jS)z Boolean NumPy array indicating if each value is missing. This should return a 1-D array the same length as 'self'. )r rqis_nullrrsrrr/r/r0rszArrowExtensionArray.isna)version allowed_argsT quicksortlast ascendingkindstr na_position np.ndarrayc s~|rdnd}ddd|d}|dustr"tdtj|||dStj|j||d}tr3| j } n| } | j t jd d S) Nr descendingat_endat_start)rfirst7)rrr)ordernull_placementFr)getrr superargsortr*array_sort_indicesrqr rrsrastyperintp) r{rrrargskwargsrrrJ np_result __class__r/r0rs   zArrowExtensionArray.argsortskipnamethodcsr|jd|jjfvs|jr|sttd||dStr&td|dtt||j|d}t |j| S)Nrargrz, only implemented for pyarrow version >= 6.0 skip_nulls) rqlengthrrgetattrrrrr*indexr)r{rrrrr/r0 _argmin_maxs zArrowExtensionArray._argmin_maxcC ||dS)Nminrr{rr/r/r0argmin zArrowExtensionArray.argmincCr)Nmaxrrr/r/r0argmaxrzArrowExtensionArray.argmaxcCr)z Return a shallow copy of the array. Underlying ChunkedArray is immutable, so a deep copy is unnecessary. Returns ------- type(self) rrr/r/r0r s zArrowExtensionArray.copycs,tr tddtSt|t|jS)z| Return ArrowExtensionArray without NA values. Returns ------- ArrowExtensionArray 6r)rr rdropnarAr* drop_nullrqrrr/r0r  zArrowExtensionArray.dropnacsvtr tddt|St|stjt|tdSi}tr"d|d<t j |j fdt j |ddi|}tj |tjdS)N2rrT skip_null value_setr)r r risinrrrrr r*is_inrqr?rbool_)r{rsrrJrr/r0r%s    zArrowExtensionArray.isintuple[np.ndarray, Any]cCs(tr |jj}n|j}||jjfS)a+ Return an array and missing value suitable for factorization. Returns ------- values : ndarray na_value : pd.NA Notes ----- The values returned by this method are also used in :func:`pandas.util.hash_pandas_object`. )r rqrrsrrhr)r{rsr/r/r0_values_for_factorize<s  z)ArrowExtensionArray._values_for_factorize na_sentinelint | lib.NoDefaultuse_na_sentinelbool | lib.NoDefault!tuple[np.ndarray, ExtensionArray]c CsFt||}tr |j}n|durdnd}|jj|d}tjdd|jD|jjd }|j j dkr@|dur9|nd|t |<|jt jd d }|jrt||d j}|durtr|jdk}|} || r| d krnd n |d| d } || |j j}||| kd 7<| ||dk<n t|tjg|jjd}|j|fS) Nrencode) null_encodingcSsg|]}|jqSr/)indices).0cr/r/r0 ]rWz1ArrowExtensionArray.factorize..rfFrr)rrrqdictionary_encoder?rxchunksrA index_typerrhrrisnanrint64 num_chunkschunk dictionaryrsrrinsertrr value_type) r{r$r&resolved_na_sentinelencodedr*r+uniquesna_maskna_indexna_coder/r/r0 factorizePs6         zArrowExtensionArray.factorizecOstt|d)NzA does not support reshape as backed by a 1D pyarrow.ChunkedArray.)rrA)r{rrr/r/r0reshapevs zArrowExtensionArray.reshaper+r allow_fill fill_valuercCs t|s t|}n|}t|jdkr|dkrtd|jdkr/|t|jkr/td|rn|dk}|rdt |t|jt j ||d}|j |}t |rXt||St||}|||<|St||j |S|dkrt|}||dkt|j7<t||j |S)a Take elements from an array. Parameters ---------- indices : sequence of int or one-dimensional np.ndarray of int Indices to be taken. allow_fill : bool, default False How to handle negative values in `indices`. * False: negative values in `indices` indicate positional indices from the right (the default). This is similar to :func:`numpy.take`. * True: negative values in `indices` indicate missing values. These values are set to `fill_value`. Any other other negative values raise a ``ValueError``. fill_value : any, optional Fill value to use for NA-indices when `allow_fill` is True. This may be ``None``, in which case the default NA value for the type, ``self.dtype.na_value``, is used. For many ExtensionArrays, there will be two representations of `fill_value`: a user-facing "boxed" scalar, and a low-level physical NA value. `fill_value` should be the user-facing version, and the implementation should handle translating that to the physical version for processing the take if necessary. Returns ------- ExtensionArray Raises ------ IndexError When the indices are out of bounds for the array. ValueError When `indices` contains negative values other than ``-1`` and `allow_fill` is True. See Also -------- numpy.take api.extensions.take Notes ----- ExtensionArray.take is called by ``Series.__getitem__``, ``.loc``, ``iloc``, when `indices` is a sequence of values. Additionally, it's called by :meth:`Series.reindex`, or any other method that causes realignment, with a `fill_value`. rzcannot do a non-empty takez!out of bounds value in 'indices'.)r)rr asanyarrayrrqanyrsizerrr?rrrrAr)r{r+rDrE indices_array fill_maskrJr/r/r0r|s.<      zArrowExtensionArray.takecs,tr tddtSt|t|jS)z Compute the ArrowExtensionArray of unique values. Returns ------- ArrowExtensionArray rr)r r runiquerAr*rqrrr/r0rKrzArrowExtensionArray.uniquerrdc Csddlm}m}|j}|d}|d}|r-|jjdkr-|}||}||}t |}|t ||}|||d dS)a@ Return a Series containing counts of each unique value. Parameters ---------- dropna : bool, default True Don't include counts of missing values. Returns ------- counts : Series See Also -------- Series.value_counts r)Indexrdr1)r Int64) pandasrLrdrq value_countsfieldris_validrrrrAr) r{rrLrdvcrscountsrr r/r/r0rOs      z ArrowExtensionArray.value_countsrtype[ArrowExtensionArrayT]cCs dd|D}t|}||S)z Concatenate multiple ArrowExtensionArrays. Parameters ---------- to_concat : sequence of ArrowExtensionArrays Returns ------- ArrowExtensionArray cSs g|] }|jD]}|q qSr/)rq iterchunks)r,earr/r/r0r.!s z9ArrowExtensionArray._concat_same_type..)r?rx)r to_concatr3arrr/r/r0_concat_same_types z%ArrowExtensionArray._concat_same_typerrc  s|dkr fdd}n!ddddd||}tt|d }|d ur,tj|fd |i|Sz |jfd |i|}Wn+tttfyd}zd t j d j d|dt j d|d }t||d }~wwt|rpj jS|S)a Return a scalar result of performing the reduction operation. Parameters ---------- name : str Name of the function, supported values are: { any, all, min, max, sum, mean, median, prod, std, var, sem, kurt, skew }. skipna : bool, default True If True, skip NaN values. **kwargs Additional keyword arguments passed to the reduction function. Currently, `ddof` is the only supported kwarg. Returns ------- scalar Raises ------ TypeError : subclass does not define reductions semcsDtj|fd|i|}tttjj|d|d}t||S)Nr r ddof)r*stddev sqrt_checkedrQcountrqrI)datarr numerator denominatorrr/r0 pyarrow_meth?s z1ArrowExtensionArray._reduce..pyarrow_methapproximate_medianproductr\variance)medianprodstdvarNrr 'z ' with dtype z does not support reduction 'z' with pyarrow version z. 'z(' may be supported by upgrading pyarrow.)rr r*r_reducerqAttributeErrorr TypeErrorrArrhr? __version__rrr) r{rrrrb pyarrow_namerJerrr|rrr0rk%s:   zArrowExtensionArray._reducekeyint | slice | np.ndarrayrcCst||}||}||}t|}||}t|r%t|t|}nt|t|kr1tdt ||}|j ||d|_ dS)aSet one or more values inplace. Parameters ---------- key : int, ndarray, or slice When called from, e.g. ``Series.__setitem__``, ``key`` will be one of * scalar int * ndarray of integers. * boolean ndarray * slice object value : ExtensionDtype.type, Sequence[ExtensionDtype.type], or object value or values to be set of ``key``. Returns ------- None %Length of indexer and values mismatch)r+rN) r_indexing_key_to_indices_maybe_convert_setitem_valuerrr broadcast_torrzasarray_set_via_chunk_iterationrq)r{rqrr+rr/r/r0 __setitem__ds    zArrowExtensionArray.__setitem__npt.NDArray[np.intp]cCst|}t|trt||}|St|r t||g}|St|r;t|}t||kr3td| d}|St|}t||}|S)z Convert indexing key for self into positional indices. Parameters ---------- key : int | slice | np.ndarray Returns ------- npt.NDArray[np.intp] rsr) rrkrrarangerrrwrznonzero)r{rqnr+r/r/r0rts        z,ArrowExtensionArray._indexing_key_to_indicesqsnpt.NDArray[np.float64] interpolationcCs*trtdtj|j||d}t||S)z Compute the quantiles of self for each quantile in `qs`. Parameters ---------- qs : np.ndarray[float64] interpolation: str Returns ------- same type as self z2quantile only supported for pyarrow version >= 4.0)qr)rrr*quantilerqrA)r{r~rrJr/r/r0 _quantiles  zArrowExtensionArray._quantilecCs`trtdt|jt|j}|d}|d}t||d}| |}t ||S)a| Returns the mode(s) of the ExtensionArray. Always returns `ExtensionArray` even if only one value. Parameters ---------- dropna : bool, default True Don't consider counts of NA values. Not implemented by pyarrow. Returns ------- same type as self Sorted, if possible. z.mode only supported for pyarrow version >= 6.0rr1) rrr*moderqcount_distinctrrPequalrrA)r{rmodesrsrSr most_commonr/r/r0_modes    zArrowExtensionArray._modecCs|S)z-Maybe convert value to be pyarrow compatible.r/)r{rr/r/r0rusz0ArrowExtensionArray._maybe_convert_setitem_valuenpt.NDArray[Any]c Csg}d}|jD]P}||t|}}t|dks ||dkr&||q ttj||dd}|d||}||d}t|}|d|||d} }||||| q t |S)a Loop through the array chunks and set the new values while leaving the chunking layout unchanged. Parameters ---------- indices : npt.NDArray[np.intp] Position indices for the underlying ChunkedArray. value : ExtensionDtype.type, Sequence[ExtensionDtype.type], or object value or values to be set of ``key``. Notes ----- Assumes that indices is sorted. Caller is responsible for sorting. rrE)sideN) rqrUrappendrr searchsorted_replace_with_indicesr?rx) r{r+rnew_datastopr8startr}c_indc_valuer/r/r0rxs   z,ArrowExtensionArray._set_via_chunk_iterationr8pa.Arrayc Cst|}|dkr |S|ddg\}}|||dkrF|d|tj||jdd||ddg}dd|D}t|dkrA|dSt|Stjt|tjd }d||<trh|j d d } || |<tj| |jd St | rut |d|St |||S) a Replace items selected with a set of positional indices. Analogous to pyarrow.compute.replace_with_mask, except that replacement positions are identified via indices rather than a mask. Parameters ---------- chunk : pa.Array indices : npt.NDArray[np.intp] value : npt.NDArray[Any] Replacement value(s). Returns ------- pa.Array rr0r1NTrcSsg|]}t|r|qSr/)r)r,rXr/r/r0r..sz=ArrowExtensionArray._replace_with_indices..rF)zero_copy_onlyr)rr?rrA concat_arraysrrr!rrrallr*if_elsereplace_with_mask) rr8r+rr}rrarraysrrXr/r/r0rs,     z)ArrowExtensionArray._replace_with_indices)rsrtr>ru)rhr)rrr))r{rer>re)r>r)r>r!)r>r)r>r)Trr)rrrrrrr>r)rrrrr>r)T)rrr>r)r>r")r$r%r&r'r>r()FN)r+r rDrrErr>rf)rrr>rd)rrTr>re)rrrr)rqrrrrr>ru)rqrrr>rz)r{rer~rrrr>re)r{rerrr>re)r+rzrrr>r;)r8rr+rzrrr>r)6r __module__ __qualname____doc____annotations__r} classmethodrrrrrrrrrrrrrpropertyrhrrrrrrrrrrrrr#rrrBr no_defaultrCrrKrOrYrkryrtrrrurxr __classcell__r/r/rr0rfs| +  - H                %  b &  ? % '  ")r:r;r<r=r>r;)rErFrGrFr>r;)rhrir>rj)M __future__rtypingrrrnumpyr pandas._libsrpandas._typingrrr r pandas.compatr r r rrrrpandas.util._decoratorsrrpandas.core.dtypes.commonrrrrrpandas.core.dtypes.missingrpandas.core.algorithmsrpandas.core.arraylikerpandas.core.arrays.baserpandas.core.indexersrrrrr?pyarrow.computecomputer*%pandas.core.arrays.arrow._arrow_utilsr pandas.core.arrays.arrow.dtyper!r not_equallessgreater less_equal greater_equalrrr+r3r4rrDrKrNrQrTrcrrNrdrerprfr/r/r/r0s  $                #