a S/b @s&ddlmZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl m Z ddl mZmZmZmZmZmZmZddlmZmZmZddlmZmZddlmZmZddlmZm Z dd l!m"Z"dd l#m$Z$ddl%Z&dd l'm(Z(dd l)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/dd l0m1Z1ddl2m3Z3m4Z4m5Z5m6Z6ddl7m8Z8m9Z9m:Z:m;Z;mm>Z?ddl>m@Z@ddlAmBZBddl5mCZCddlDmEZEmDZDddlFmGZGmHZHddlImJZJmKZKddlLmLZLddlMmNZNmOZOmPZPmQZQmRZRmSZSmTZTmUZUmVZVmWZWmXZXmYZYmZZZm[Z[m\Z\m]Z]m^Z^m_Z_m`Z`maZaddlbmcZcddl2mdZdddldmeZeddlfmgZgmhZhddlimjZjmkZkmlZldd lmmnZnmoZodd!lpmqZqmrZrmsZse4td"d#d$d%id&ZuGd'd(d(evZwdd*d+Zxdd,d-Zydd.d/Zzdd0l{m|Z|m}Z}iZ~d1d2Zd3d4d5d6Zd7d8Zd9d:Zexd;d)dd;fdZdd?d@ZgfdAdBZddDdEZdFdGZdHdIZddddgdddJdKdLZdMdNZddOdPdQd3d3dRdSdTZdUdVZdWdXZdYZGdZd[d[e8Zd\d]Zdd^d_Zd`d`dadbdcZddddeZdfdgZdhdiZddkdlZddmdnZddodpZdqdrZddsdtZddduifdvdwZdxdyZdzd{Zd|d}Zdd~dZdddZddZddZddZd)dd;d;ddddddd3d3ddddZdddddddZdddddZddd;dddZddZddZdd)dddddZddZddZddZddZddZdddZeTe&d;dddZddZddZddZddZdddZdddZddZdd„Zd)dÜddńZddDŽZddɄZdd˄Zdd̈́ZddτZddфZddӄZddՄZdddׄZdddڄZddd܄ZGddބdރZddl>m>Z>ddlMmZmZdS)) annotationsNbisect) CollectionHashableIterableIteratorMappingMutableMappingSequence)partialreducewraps)product zip_longest)IntegralNumber)addmul)Lock)Any) get_mapper) accumulateconcatfirst frequenciesgroupby partition)pluck)computeconfigcorethreaded)DaskMethodsMixincompute_as_if_collection dont_optimizeis_dask_collectionpersisttokenize) blockwise)broadcast_dimensions) globalmethod)quoteDelayeddelayed)HighLevelGraphMaterializedLayer) ArraySliceDep reshapelist)sizeof) IndexCallableMSerializableLock cached_cumsumcached_propertyconcrete derived_fromfactors format_bytesfuncname has_keyword is_arraylikeis_dataframe_like is_index_like is_integeris_series_likendeepmapndimlist parse_bytestypename) get_template)chunk)getitem)is_valid_array_chunkis_valid_chunk_type)concatenate_lookup einsum_lookuptensordot_lookup) _numpy_120 _Recurser)replace_ellipsis setitem_array slice_arrayarrayZ128MiB)z chunk-sizezrechunk-thresholdz A possible solution: https://docs.dask.org/en/latest/array-chunks.html#unknown-chunks Summary: to compute chunks sizes, use x.compute_chunk_sizes() # for Dask Array `x` ddf.to_dask_array(lengths=True) # for Dask DataFrame `ddf`c@seZdZdZdS)PerformanceWarningzezgetter..css|]}|dur|VqdSrar_rbr_r_r`refrfcss.|]&}t|ts|durdntddVqdSra) isinstancerslicerbr_r_r`regs asarraylock) rgtupleanygetteracquirerAnpZmatrixrjrelease)abrjrkb2Zb3cr_r_r`rnds"   rncCst||||dS)zA simple wrapper around ``getter``. Used to indicate to the optimization passes that the backend doesn't support fancy indexing. rirnrrrsrjrkr_r_r`getter_nofancy~srxcCst||||dS)aA getter function that optimizations feel comfortable inlining Slicing operations with this function may be inlined into a graph, such as in the following rewrite **Before** >>> a = x[:10] # doctest: +SKIP >>> b = a + 1 # doctest: +SKIP >>> c = a * 2 # doctest: +SKIP **After** >>> b = x[:10] + 1 # doctest: +SKIP >>> c = x[:10] * 2 # doctest: +SKIP This inlining can be relevant to operations when running off of disk. rirvrwr_r_r` getter_inlinesry) fuse_sliceoptimizecsfdd}|S)aRegister an __array_function__ implementation for dask.array.Array Register that a function implements the API of a NumPy function (or several NumPy functions in case of aliases) which is handled with ``__array_function__``. Parameters ---------- \*numpy_functions : callables One or more NumPy functions that are handled by ``__array_function__`` and will be mapped by `implements` to a `dask.array` function. csD] }|t|<q|Sra)_HANDLED_FUNCTIONS)Z dask_funcZnumpy_functionnumpy_functionsr_r` decorators zimplements..decoratorr_)r~rr_r}r` implementss rboolreturncCsBt|dr|jdurdSt|dr>t|s>t|jtjur>dSdS)zCheck whether Dask should delegate to the other. This implementation follows NEP-13: https://numpy.org/neps/nep-0013-ufunc-overrides.html#behavior-in-combination-with-python-s-binary-operations __array_ufunc__NTF)hasattrrrNtypeArray)otherr_r_r`_should_delegatesrcstfdd}|S)zCheck if method is handled by Dask given type of other Ensures proper deferral to upcast types in dunder operations without assuming unknown types are automatically downcast types. cst|r tS||SdSra)rNotImplementedselfrfr_r`wrappersz-check_if_handled_given_other..wrapper)r)rrr_rr`check_if_handled_given_othersrcCs.dd|D}ddt||D}tt|S)aTranslate chunks tuple to a set of slices in product order >>> slices_from_chunks(((2, 2), (3, 3, 3))) # doctest: +NORMALIZE_WHITESPACE [(slice(0, 2, None), slice(0, 3, None)), (slice(0, 2, None), slice(3, 6, None)), (slice(0, 2, None), slice(6, 9, None)), (slice(2, 4, None), slice(0, 3, None)), (slice(2, 4, None), slice(3, 6, None)), (slice(2, 4, None), slice(6, 9, None))] cSsg|]}t|ddqSTZ initial_zeror9rcbdsr_r_r` rfz&slices_from_chunks..cSs$g|]\}}ddt||DqS)cSsg|]\}}t|||qSr_rh)rcsdimr_r_r`rrfz1slices_from_chunks...zip)rcstartsshapesr_r_r`rs)rlistr)chunksZcumdimsslicesr_r_r`slices_from_chunkss rFr1c  Cst|||d}ttt|} t|drJt|drJ|r:|rJt|||d} n|} |rzt| || |dt|| id} t || Sd|} i} t | |i| | <t| || | dt|| id| |<| t || hi}t| |SdS)a HighLevelGraph for slicing chunks from an array-like according to a chunk pattern. If ``inline_array`` is True, this make a Blockwise layer of slicing tasks where the array-like is embedded into every task., If ``inline_array`` is False, this inserts the array-like as a standalone value in a MaterializedLayer, then generates a Blockwise layer of slicing tasks that refer to it. >>> dict(graph_from_arraylike(arr, chunks=(2, 3), shape=(4, 6), name="X", inline_array=True)) # doctest: +SKIP {(arr, 0, 0): (getter, arr, (slice(0, 2), slice(0, 3))), (arr, 1, 0): (getter, arr, (slice(2, 4), slice(0, 3))), (arr, 1, 1): (getter, arr, (slice(2, 4), slice(3, 6))), (arr, 0, 1): (getter, arr, (slice(0, 2), slice(3, 6)))} >>> dict( # doctest: +SKIP graph_from_arraylike(arr, chunks=((2, 2), (3, 3)), shape=(4,6), name="X", inline_array=False) ) {"original-X": arr, ('X', 0, 0): (getter, 'original-X', (slice(0, 2), slice(0, 3))), ('X', 1, 0): (getter, 'original-X', (slice(2, 4), slice(0, 3))), ('X', 1, 1): (getter, 'original-X', (slice(2, 4), slice(3, 6))), ('X', 0, 1): (getter, 'original-X', (slice(0, 2), slice(3, 6)))} dtyperjrkriN) numblocksz original-) normalize_chunksrlrangelenr@r core_blockwiser3r1from_collectionsr2set)arrrshapenamerMrkrjr inline_arrayout_indrnlayer original_namelayersZdepsr_r_r`graph_from_arraylikesR$  rcKs:|rt||}|rt||}ttttjfi|||S)aDot product of many aligned chunks >>> x = np.array([[1, 2], [1, 2]]) >>> y = np.array([[10, 20], [10, 20]]) >>> dotmany([x, x, x], [y, y, y]) array([[ 90, 180], [ 90, 180]]) Optionally pass in functions to apply to the left and right chunks >>> dotmany([x, x, x], [y, y, y], rightfunc=np.transpose) array([[150, 150], [150, 150]]) )mapsumr rpdot)ABZleftfuncZ rightfunckwargsr_r_r`dotmanyBs   rcsdkrt|tr|dS|St|tr0t|}t|ttfsB|Stdkr`fdd|D}ttt|ddd}t|dt rt|d t fd d |DsJt }D]*|tfd d |Ddd |<q|S||dd Sd S)aRecursively concatenate nested lists of arrays along axes Each entry in axes corresponds to each level of the nested list. The length of axes should correspond to the level of nesting of arrays. If axes is an empty list or tuple, return arrays, or arrays[0] if arrays is a list. >>> x = np.array([[1, 2], [3, 4]]) >>> _concatenate2([x, x], axes=[0]) array([[1, 2], [3, 4], [1, 2], [3, 4]]) >>> _concatenate2([x, x], axes=[1]) array([[1, 2, 1, 2], [3, 4, 3, 4]]) >>> _concatenate2([[x, x], [x, x]], axes=[0, 1]) array([[1, 2, 1, 2], [3, 4, 3, 4], [1, 2, 1, 2], [3, 4, 3, 4]]) Supports Iterators >>> _concatenate2(iter([x, x]), axes=[1]) array([[1, 2, 1, 2], [3, 4, 3, 4]]) Special Case >>> _concatenate2([x, x], axes=()) array([[1, 2], [3, 4]]) r_rrKcs g|]}t|dddqS)rKNaxes) _concatenate2rcrrrr_r`rrfz!_concatenate2..cSs t|ddSN__array_priority__rgetattrrdr_r_r`rfz_concatenate2..keyc3s|]}t|kVqdSra)rkeysr)rr_r`rerfz _concatenate2..c3s|]}|VqdSrar_rkr_r`rerfaxisN) rgrrrlrrPdispatchrmaxdictrall)arraysr concatenateretr_)rrrr`rXs*#   (rrc sddlmfdd|D}z>tjdd||i|}Wdn1sP0YWnrty}zZt\}} } dt | } |rd j |d nd} d |d | d |d| } WYd}~nd}~00d} | durt | |dur|j St dd|DS)a Tries to infer output dtype of ``func`` for a small set of input arguments. Parameters ---------- func: Callable Function for which output dtype is to be determined args: List of array like Arguments to the function, which would usually be used. Only attributes ``ndim`` and ``dtype`` are used. kwargs: dict Additional ``kwargs`` to the ``func`` funcname: String Name of calling function to improve potential error messages suggest_dtype: None/False or String If not ``None`` adds suggestion to potential error message to specify a dtype via the specified kwarg. Defaults to ``'dtype'``. nout: None or Int ``None`` if function returns single output, integer if many. Deafults to ``None``. Returns ------- : dtype or List of dtype One or many dtypes (depending on ``nout``) rKmeta_from_arraycs4g|],}t|r,tj|d|j|jdn|qS)rKrr)rArpZ ones_likendimrrbrr_r`rsz%apply_infer_dtype..ignorerNz@Please specify the dtype explicitly using the `{dtype}` kwarg. rz`dtype` inference failed in `z`. z2Original error is below: ------------------------ z Traceback: --------- css|] }|jVqdSrarrcer_r_r`rerfz$apply_infer_dtype..)utilsrrpZerrstate Exceptionsysexc_infojoin traceback format_tbformat ValueErrorrrl)funcargsrr? suggest_dtypenoutorexc_type exc_value exc_tracebacktbZsuggestmsgr_rr`apply_infer_dtypes<  0  rcCsdt|r |St|tr*td|r*t|St|trHt|dkrHt|St|dkr\t|S|SdS)a>Normalize user provided arguments to blockwise or map_blocks We do a few things: 1. If they are string literals that might collide with blockwise_token then we quote them 2. IF they are large (as defined by sizeof) then we put them into the graph on their own by using dask.delayed z_\d+ g.AN) r'rgstrrematchr0rrr5rr_r_r` normalize_args  rcOs*|t||||t|di|S)zHelper for :func:`dask.array.map_blocks` to pass `block_info` or `block_id`. For each element of `keys`, a corresponding element of args is changed to a keyword argument with that key, before all arguments re passed on to `func`. N)updaterr)rrrrr_r_r`_pass_extra_kwargssr)rtokenrr drop_axisnew_axismetac svt|sd} t| t|j|r4tjdtd|}|p>t|dt||||g|Ri| }i} t t rxgt |t r|g}dd|D} dd|D} | rt t t dd | Dd d d nd | }|d ur*|d ur*zt||g|Ri| }WntyYn0t|||d }rttfdd DrftdddfddDt fdd tD|d ur|d urtt|krt t|t}|rHtt|D]B}tt}|||d ur||| |<nd| |<qt t |t krHtd|d urt|tkrtdt|dtdtt|}nd }t|gt| R|| |dd||d| g}g}t|dr>> import dask.array as da >>> x = da.arange(6, chunks=3) >>> x.map_blocks(lambda x: x * 2).compute() array([ 0, 2, 4, 6, 8, 10]) The ``da.map_blocks`` function can also accept multiple arrays. >>> d = da.arange(5, chunks=2) >>> e = da.arange(5, chunks=2) >>> f = da.map_blocks(lambda a, b: a + b**2, d, e) >>> f.compute() array([ 0, 2, 6, 12, 20]) If the function changes shape of the blocks then you must provide chunks explicitly. >>> y = x.map_blocks(lambda x: x[::2], chunks=((2, 2),)) You have a bit of freedom in specifying chunks. If all of the output chunk sizes are the same, you can provide just that chunk size as a single tuple. >>> a = da.arange(18, chunks=(6,)) >>> b = a.map_blocks(lambda x: x[:3], chunks=(3,)) If the function changes the dimension of the blocks you must specify the created or destroyed dimensions. >>> b = a.map_blocks(lambda x: x[None, :, None], chunks=(1, 6, 1), ... new_axis=[0, 2]) If ``chunks`` is specified but ``new_axis`` is not, then it is inferred to add the necessary number of axes on the left. Map_blocks aligns blocks by block positions without regard to shape. In the following example we have two arrays with the same number of blocks but with different shape and chunk sizes. >>> x = da.arange(1000, chunks=(100,)) >>> y = da.arange(100, chunks=(10,)) The relevant attribute to match is numblocks. >>> x.numblocks (10,) >>> y.numblocks (10,) If these match (up to broadcasting rules) then we can map arbitrary functions across blocks >>> def func(a, b): ... return np.array([a.max(), b.max()]) >>> da.map_blocks(func, x, y, chunks=(2,), dtype='i8') dask.array >>> _.compute() array([ 99, 9, 199, 19, 299, 29, 399, 39, 499, 49, 599, 59, 699, 69, 799, 79, 899, 89, 999, 99]) Your block function can get information about where it is in the array by accepting a special ``block_info`` or ``block_id`` keyword argument. During computation, they will contain information about each of the input and output chunks (and dask arrays) relevant to each call of ``func``. >>> def func(block_info=None): ... pass This will receive the following information: >>> block_info # doctest: +SKIP {0: {'shape': (1000,), 'num-chunks': (10,), 'chunk-location': (4,), 'array-location': [(400, 500)]}, None: {'shape': (1000,), 'num-chunks': (10,), 'chunk-location': (4,), 'array-location': [(400, 500)], 'chunk-shape': (100,), 'dtype': dtype('float64')}} The keys to the ``block_info`` dictionary indicate which is the input and output Dask array: - **Input Dask array(s):** ``block_info[0]`` refers to the first input Dask array. The dictionary key is ``0`` because that is the argument index corresponding to the first input Dask array. In cases where multiple Dask arrays have been passed as input to the function, you can access them with the number corresponding to the input argument, eg: ``block_info[1]``, ``block_info[2]``, etc. (Note that if you pass multiple Dask arrays as input to map_blocks, the arrays must match each other by having matching numbers of chunks, along corresponding dimensions up to broadcasting rules.) - **Output Dask array:** ``block_info[None]`` refers to the output Dask array, and contains information about the output chunks. The output chunk shape and dtype may may be different than the input chunks. For each dask array, ``block_info`` describes: - ``shape``: the shape of the full Dask array, - ``num-chunks``: the number of chunks of the full array in each dimension, - ``chunk-location``: the chunk location (for example the fourth chunk over in the first dimension), and - ``array-location``: the array location within the full Dask array (for example the slice corresponding to ``40:50``). In addition to these, there are two extra parameters described by ``block_info`` for the output array (in ``block_info[None]``): - ``chunk-shape``: the output chunk shape, and - ``dtype``: the output dtype. These features can be combined to synthesize an array from scratch, for example: >>> def func(block_info=None): ... loc = block_info[None]['array-location'][0] ... return np.arange(loc[0], loc[1]) >>> da.map_blocks(func, chunks=((4, 4),), dtype=np.float_) dask.array >>> _.compute() array([0, 1, 2, 3, 4, 5, 6, 7]) ``block_id`` is similar to ``block_info`` but contains only the ``chunk_location``: >>> def func(block_id=None): ... pass This will receive the following information: >>> block_id # doctest: +SKIP (4, 3) You may specify the key name prefix of the resulting task in the graph with the optional ``token`` keyword argument. >>> x.map_blocks(lambda x: x + 1, name='increment') dask.array For functions that may not handle 0-d arrays, it's also possible to specify ``meta`` with an empty array matching the type of the expected result. In the example below, ``func`` will result in an ``IndexError`` when computing ``meta``: >>> da.map_blocks(lambda x: x[2], da.random.random(5), meta=np.array(())) dask.array Similarly, it's possible to specify a non-NumPy array to ``meta``, and provide a ``dtype``: >>> import cupy # doctest: +SKIP >>> rs = da.random.RandomState(RandomState=cupy.random.RandomState) # doctest: +SKIP >>> dt = np.float32 >>> da.map_blocks(lambda x: x[2], rs.random(5, dtype=dt), meta=cupy.array((), dtype=dt)) # doctest: +SKIP dask.array z~First argument must be callable function, not %s Usage: da.map_blocks(function, x) or: da.map_blocks(function, x, y, z)zThe `token=` keyword to `map_blocks` has been moved to `name=`. Please use `name=` instead as the `token=` keyword will be removed in a future release.)category-cSsg|]}t|tr|qSr_rgrrr_r_r`rrfzmap_blocks..cSs:g|]2}t|tr.|tt|jdddfn|dfqSN)rgrrlrrrr_r_r`rscss|] }|jVqdSrarrr_r_r`rerfzmap_blocks..Nrr_ map_blocksc3s |]}| kp|kVqdSrar_rcindim_outr_r`rerfz"drop_axis out of range (drop_axis=z, but output is zd).csg|] }|qSr_r_rrr_r`rrfc3s|]\}}|vr|VqdSrar_)rcrrd)rr_r`rerfrKz-New_axis values do not fill in all dimensionszProvided chunks have z dims; expected z dimsTF)rnew_axesrr align_arrays adjust_chunksrblock_idz block-id-csi|]}f||qSr_r_)rcr ) block_id_namer_r` Hszmap_blocks..css|]}tt|VqdSrarrrcrur_r_r`reJrfcss|]}dt|VqdSrNrrr_r_r`reOrfrr block_infocs8g|]0\}}|vr&tj|ddn dj|gqS)Trr)r9rrrcjind)argrr_r`rbscss|]}t|dVqdSrKNrrcrr_r_r`rejrfcSsg|]}t|ddqSrrrr_r_r`rlscSsg|]}t|ddqSrrrr_r_r`rprfz block-info-css|]}tt|VqdSrar rr_r_r`retrfcsi|]\}}||qSr_r_)rcrloc)rr_r`r vrfc3s2|]*\}}|dkr&|dndVqdSrKrNgetr)rlocation num_chunksr_r`re}scs4g|],\}}||||dfqSrr_rcZijr)rrr_r`rs)r num-chunksarray-locationchunk-locationcs,g|]$\}}||||dfqSrr_r) out_startsr_r`rsc3s |]\}}j||VqdSrarroutr_r`res)rr r!r"z chunk-shapercss|]}dt|VqdSrrrr_r_r`rerf)rrrrr r))callable TypeErrorrr[warningswarn FutureWarningr?r)rgrrlrr compute_metarrrrmr enumeratersortedinsertrrr*rr@rrrrrpZobject_appendrritemsrr)rrrrrrrrrrrrZarrsZargpairsZoriginal_kwargsaxnr Zextra_argpairsZ extra_namesZ block_id_dskZblock_id_arrayrZin_indZblock_info_nameZblock_info_dskr inforZarr_krr_) rr rrrrrr&rr#rr`rsPf0  &&                        rcs|sdSt|dkr|dSttt|fdd|D}g}tD]jfdd|D}tdd|Drv|}nd d|D}tt|dkrtd t|||dqHt |S) a\Construct a chunks tuple that broadcasts many chunks tuples >>> a = ((5, 5),) >>> b = ((5, 5),) >>> broadcast_chunks(a, b) ((5, 5),) >>> a = ((10, 10, 10), (5, 5),) >>> b = ((5, 5),) >>> broadcast_chunks(a, b) ((10, 10, 10), (5, 5)) >>> a = ((10, 10, 10), (5, 5),) >>> b = ((1,), (5, 5),) >>> broadcast_chunks(a, b) ((10, 10, 10), (5, 5)) >>> a = ((10, 10, 10), (5, 5),) >>> b = ((3, 3,), (5, 5),) >>> broadcast_chunks(a, b) Traceback (most recent call last): ... ValueError: Chunks do not align: [(10, 10, 10), (3, 3)] r_rKrcs g|]}dt||qS)rrr)r3r_r`rrfz$broadcast_chunks..csg|] }|qSr_r_rrr_r`rrfcss|]}|dkVqdSrr_rr_r_r`rerfz#broadcast_chunks..cSsg|]}|dkr|qSr5r_rr_r_r`rrfzChunks do not align: %s) rrrrrrrrr0rl)chunkssZchunkss2resultZstep1Zstep2r_)rr3r`broadcast_chunkss   r9zArray | Collection[Array]z bool | Lockz8tuple[slice, ...] | Collection[tuple[slice, ...]] | None)sourcesrkregionsr return_storedc s0t|tr|g}|g}tdd|Dr0tdt|t|krXtdt|t|ft|tsj|durp|g}t|dkrt|dkr|t|9}t|t|krtdt|t|t|ftjdd |D}t|t t d d |D}d t |} | |i} | t i} g} g} |D]@}t|trL| |j| |nt|r td q | rtj| }t|| }d t | }|| |<t | |<|o| }dd t|||D}g}t||||D]v\}}}}t||jt|tr|jn||||||d}|| |<t|tr.| |h| |<n | h| |<||7}q|rt| | |rfdd |D}t|i|}tjdd |D}t||dd |D}tfddt||DS|rt| | tt|fi|dSdt |}||i| |<t || |<t| | t|SdS)aStore dask arrays in array-like objects, overwrite data in target This stores dask arrays into object that supports numpy-style setitem indexing. It stores values chunk by chunk so that it does not have to fill up memory. For best performance you can align the block size of the storage target with the block size of your array. If your data fits in memory then you may prefer calling ``np.array(myarray)`` instead. Parameters ---------- sources: Array or collection of Arrays targets: array-like or Delayed or collection of array-likes and/or Delayeds These should support setitem syntax ``target[10:20] = ...`` lock: boolean or threading.Lock, optional Whether or not to lock the data stores while storing. Pass True (lock each file individually), False (don't lock) or a particular :class:`threading.Lock` object to be shared among all writes. regions: tuple of slices or collection of tuples of slices Each ``region`` tuple in ``regions`` should be such that ``target[region].shape = source.shape`` for the corresponding source and target in sources and targets, respectively. If this is a tuple, the contents will be assumed to be slices, so do not provide a tuple of tuples. compute: boolean, optional If true compute immediately; return :class:`dask.delayed.Delayed` otherwise. return_stored: boolean, optional Optionally return the stored result (default False). kwargs: Parameters passed to compute/persist (only used if compute=True) Returns ------- If return_stored=True tuple of Arrays If return_stored=False and compute=True None If return_stored=False and compute=False Delayed Examples -------- >>> import h5py # doctest: +SKIP >>> f = h5py.File('myfile.hdf5', mode='a') # doctest: +SKIP >>> dset = f.create_dataset('/data', shape=x.shape, ... chunks=x.chunks, ... dtype='f8') # doctest: +SKIP >>> store(x, dset) # doctest: +SKIP Alternatively store many arrays at the same time >>> store([x, y, z], [dset1, dset2, dset3]) # doctest: +SKIP css|]}t|t VqdSrarrr_r_r`re1rfzstore..z&All sources must be dask array objectsz1Different number of sources [%d] and targets [%d]NrKzCDifferent number of sources [%d] and targets [%d] than regions [%d]cSsg|] }|qSr_)__dask_graph__rr_r_r`rGrfzstore..cSsg|] }|qSr_) __dask_keys__rr_r_r`rIrfzstore-sources-z5Targets must be either Delayed objects or array-likeszstore-targets-cSs4g|],\}}}dt|t|tr"|nt||qS)z store-map-)r)rgr/id)rcrtrr_r_r`rbs)rrr&rrkregionr< load_storedcsg|]}t||ddqS)rrr/rcr) store_dskr_r`r~rfcSsg|] }|jqSr_daskrr_r_r`rrfcSsg|] }d|qS)load-r_)rcr3r_r_r`rrfc3s$|]\}}t||j|dVqdS)rN)rr)rcrr3)load_store_dskr_r`reszstore-)rgrrmrrrlr1merge__dask_optimize__rr"flattenr)rr/r0rr=r'r(r insert_to_oocr>rrr(retrieve_from_oocr%)r:targetsrkr;r r<rZ sources_hlgZ sources_layerZ sources_namer dependenciesZ targets_keysZ targets_dsksr@Z targets_hlgZ targets_layerZ targets_namerCZ map_namesZmap_keysrr3rAZ map_layerZ store_dlydsZ store_dsk_2rr_)rLrGr`storesD                         rTcCs|durtd|dur tdtt|s>> blockdims_from_blockshape((10, 10), (4, 3)) ((4, 4, 2), (3, 3, 3, 1)) >>> blockdims_from_blockshape((10, 0), (4, 0)) ((4, 4, 2), (0,)) Nz$Must supply chunks= keyword argumentz#Must supply shape= keyword argumentz6Array chunk sizes are unknown. shape: %s, chunks: %s%sz!chunks can only contain integers.z shape can only contain integers.css>|]6\}}|r2|f||||r,||fndndVqdS)r_rNr_)rcdbdr_r_r`resz,blockdims_from_blockshape..) r(rpisnanrrunknown_chunk_messagerrrDrlintrrrr_r_r`blockdims_from_blockshapes&r\cCsD|s t|S|}t|ttfrZ*ed?d@Z+e+j!dAd@Z+edBdCZ,e,j!dDdCZ,dEdFZ-dGZ.d!dHdIZ/dJdKZ0edLdMZ1e2e3dNdOZ3d"dQdRZ4dSdTZ5d#dUdVZ6dWdXZ7e7Z8dYdZZ9d[d\Z:e:Z;d]d^ZdcddZ?dedfZ@dgdhZAedidjZBedkdlZCedmdnZDeEeFjGdodpZHedqdrZIedsdtZJeEeFjGdudvZKeEeFjGdwdxZLeLZMeEeFjGdydzZNeEeFjGd{dd|d}d~ZOd$ddZPd%ddZQddZRddZSeTddZUeTddZVeTddZWeTddZXeTddZYeTddZZeTddZ[eTddZ\eTddZ]ddZ^eTddZ_eTddZ`eTddZaeTddZbeTddZceTddZdeTddZeeTddZfeTddZgddZheTddZiddZjeTddZkeTddZleTddZmeTddZneTddZoeTddZpeTddZqeTddÄZreTddńZseTddDŽZteTddɄZueTdd˄ZveTdd̈́ZweTddτZxeTddфZyeTddӄZzeTddՄZ{eEeFjGd&dd؄Z|eEeFjGd'ddڄZ}eEeFjGd(dd܄Z~eEeFjGd)ddބZeEeFjGd*ddZeEeFjGd+ddZeEeFjGd,ddZeEeFjGd-ddZeEeFjGd.ddZeEeFjGd/ddZeEeFjGd0ddZeEeFjGd1ddZd2ddZe2eddZd3ddZeEeFjGd4ddddZeEeFjGd5ddddZeEeFjGd6ddZd7ddZeddZeddZddZeEeFjGd8dd Zd9d d ZeEeFjGd dZeEeFjGd:ddZddZddZd;ddZeEeFjGd<ddZeEeFjGddZddZddZZS(=raParallel Dask Array A parallel nd-array comprised of many numpy arrays arranged in a grid. This constructor is for advanced uses only. For normal use see the :func:`dask.array.from_array` function. Parameters ---------- dask : dict Task dependency graph name : string Name of array in dask shape : tuple of ints Shape of the entire array chunks: iterable of tuples block sizes along each dimension dtype : str or dtype Typecode or data-type for the new Dask Array meta : empty ndarray empty ndarray created with same NumPy backend, ndim and dtype as the Dask Array being created (overrides dtype) See Also -------- dask.array.from_array )rIZ__name _cached_keysZ__chunks_meta__dict__Nc s~t|}t|tsJt|ts4tj||dd}||_t||_t ||d}t|tszt|t r|rt dd|Dr|j }nd}t |||d|_|jdurttt ||j|d|_tddD]} | |} | dur| }qz|jj|} WnttfyYnr0| jdurF|j|j |j|jtt|tt|jd| _n4| j|j|j |j|jtt|tt|jd|S) Nr_rSrcss|]}t|tVqdSrargrrr_r_r`rerfz Array.__new__..)rrZ array_plugins)rr chunksizerrZ chunk_type)super__new__rgr r1rrIr_namerrlrmrr_chunksrrCHUNKS_NONE_ERROR_MESSAGErrbr!rrAttributeErrorKeyErrorZcollection_annotationsrrfrIrr) clsrIrrrrrrdtZpluginr8r __class__r_r`rhs^           z Array.__new__cCst|j|j|j|jffSra)rrIrrrrr_r_r` __reduce__#szArray.__reduce__cCs|jSrarHrrr_r_r`r=&szArray.__dask_graph__cCs|jfSrarrrr_r_r`__dask_layers__)szArray.__dask_layers__csH|jdur|jS|j|j|jfdd|_}|S)Ncsbs fgSt}|dtkrBfddt|D}nfddt|D}|S)NrKcsg|]}f|fqSr_r_r)rrr_r`r7rfz5Array.__dask_keys__..keys..csg|]}|fqSr_r_r)rrr_r`r9rf)rr)rrr8rrrr)rr`r2sz!Array.__dask_keys__..keys)rarrr)rr8r_rvr`r>,s   zArray.__dask_keys__cCs|jSrartrrr_r_r`__dask_tokenize__?szArray.__dask_tokenize__Zarray_optimize)rZfalseycCstdfSNr_)r`rrr_r_r`__dask_postcompute__GszArray.__dask_postcompute__cCs |jdfSrx)_rebuildrrr_r_r`__dask_postpersist__JszArray.__dask_postpersist__)renamecCs,|j}|r|||}t|||j|j|jSra)rirrrrrb)rdskr|rr_r_r`rzMs zArray._rebuildcCs&|dur|jn|j|ddS)z Reset cached properties. Parameters ---------- key : str, optional Remove specified key. The default removes all items. N)rcclearpoprrr_r_r` _reset_cacheSs  zArray._reset_cachecCstj|tdS)Nr)rprXr>objectrrr_r_r` _key_arrayaszArray._key_arraycCsttt|jSra)rlrrrrrr_r_r`reszArray.numblockscCstt|jdSNrK)r rrrrr_r_r` npartitionsiszArray.npartitionscCs|}|jtttdd|jD|jff|jd}g}t|jD]<}|jdg|g}td||<t|}|t||q@tddt t|dD|_ |S)a Compute the chunk sizes for a Dask array. This is especially useful when the chunk sizes are unknown (e.g., when indexing one Dask array with another). Notes ----- This function modifies the Dask array in-place. Examples -------- >>> import dask.array as da >>> import numpy as np >>> x = da.from_array([-2, -1, 0, 1, 2], chunks=2) >>> x.chunks ((2, 2, 1),) >>> y = x[x <= 0] >>> y.chunks ((nan, nan, nan),) >>> y.compute_chunk_sizes() # in-place computation dask.array >>> y.chunks ((2, 1, 0),) css|]}t|dVqdSrrrr_r_r`rerfz,Array.compute_chunk_sizes..rrrrNcss |]}tdd|DVqdS)css|]}t|VqdSra)rZ)rcrLr_r_r`rerfz6Array.compute_chunk_sizes...Nrl)rcrr_r_r`res) r_get_chunk_shaperZrlrrrrhr0r rj)rrdZ chunk_shapesrurrr_r_r`compute_chunk_sizesms"  zArray.compute_chunk_sizescCstdd|jDS)Ncss|]}t|dddVqdS)TrrNrrr_r_r`rerfzArray.shape..rlrrrr_r_r`rsz Array.shapecCstdd|jDS)Ncss|]}t|VqdSrarrr_r_r`rerfz"Array.chunksize..rrrr_r_r`rfszArray.chunksizecCs&t|jtr|jdj}n|jj}|SNr)rgrbrlr)rrr_r_r`rs z Array.dtypecCs|jS)z9Non-public chunks property. Allows setting a chunk value._Array__chunksrrr_r_r`rjsz Array._chunkscCs||_dD]}||q dS)N)rrrrsizer)rr)rrrr_r_r`rjscCs|jS)zChunks property.rrrr_r_r`rsz Array.chunkscCstd|ddS)NzPCan not set chunks directly Please use the rechunk method instead: x.rechunk(zC) If trying to avoid unknown chunks, use x.compute_chunk_sizes()r()rrr_r_r`rs cCsF|jstdt|jdr4dt}t|tt|jdS)Nzlen() of unsized objectrz4Cannot call len() on object with unknown chunk size.) rr(rprXrmrYrrZr)rrr_r_r`__len__sz Array.__len__c OsB|dd}||D]}t|rtSq|dkr|tjurVddlm}||i|S|jdurddlm}|||jg|Ri|S|j dkrddl m } zt | |j } WntytYS0| |i|St|g|Ri|SnT|dkr:ddl m } zt | |j } Wnty(tYS0| j|i|StSdS) Nr&r___call__rKmatmul) apply_gufunc)ufuncouter)rrrrprroutinesZ signatureZgufuncrrrrrr[rlelemwiser) rZ numpy_ufuncmethodinputsrr&rdrrrZda_ufuncr_r_r`rsB              zArray.__array_ufunc__c CsNt|j}|jddd}d||j|j|t|jj ddt|jj S)z >>> import dask.array as da >>> da.ones((10, 10), chunks=(5, 5), dtype='i4') dask.array<..., shape=(10, 10), dtype=int32, chunksize=(5, 5), chunktype=numpy.ndarray> rrKrzAdask.array<{}, shape={}, dtype={}, chunksize={}, chunktype={}.{}>.) rrfrrsplitrrrrrbr\splitr[)rrfrr_r_r`__repr__s  zArray.__repr__cCsz|jtddd}Wnty.d}Yn0dtt|jvrLd}d}n8t|j s|t |j }t t |j |jj}nd}d}tdj||||dS) Nzarray.svg.sizexrrZsparseunknownz array.html.j2)rXgridnbytescbytes)to_svgr!rNotImplementedErrorrIrrbmathrXrr>rpprodrfritemsizerJZrender)rrrrr_r_r` _repr_html_s$    zArray._repr_html_cCs t|jSrarrrrr_r_r`r(sz Array.ndimcCstt|jdS)zNumber of elements in arrayrK)r rrrrr_r_r`r,sz Array.sizecCs|j|jjS)zNumber of bytes in array)rrrrrr_r_r`r1sz Array.nbytescCs|jjS)z$Length of one array element in bytes)rrrrr_r_r`r6szArray.itemsizecCs|jSra _Array__namerrr_r_r`ri;sz Array._namecCs||_d|_|ddS)Nr)rrarrvalr_r_r`ri?scCs|jSrarrrr_r_r`rFsz Array.namecCs tddS)NzCannot set name directly Name is used to relate the array to the task graph. It is uncommon to need to change it, but if you do please set ``._name``rrr_r_r`rJsccs tt|D]}||Vq dSrar )rrr_r_r`__iter__SszArray.__iter__ cKs:|}|r |j|kr ||}t|tjs6t|}|Sra)r rastypergrpndarrayrX)rrrrdr_r_r` __array__Ys    zArray.__array__c Csddlm}dd}tdd|Ds*tS|jdddD]6}zt||}Wq>tyr||||YS0q>t||j s||||St||j }||ur||||St |dr||d<||i|S) NrcSsT|tvrBtd|jd|jtt||\}}||i|St||i|S)NzThe `{}` function is not implemented by Dask array. You may want to use the da.map_blocks function or something similar to silence this warning. Your code may stop working in a future release.r)r|r)r*rr\r[r+r )rrrr_r_r`handle_nonmatching_namesds z:Array.__array_function__..handle_nonmatching_namescss|]}|tupt|VqdSra)rrO)rcrr_r_r`reyrfz+Array.__array_function__..rrKlike) Z dask.arrayrXrrr\rrrlrr[r@) rrtypesrrmodulerZ submoduleZda_funcr_r_r`__array_function__as"       zArray.__array_function__cCstSra)rrrr_r_r` _elemwiseszArray._elemwisecKs.t|g|gfi|}|ddr*|d}|S)Nr<Fr)rTr)rtargetrrAr_r_r`rTs z Array.storecCsddlm}||j|dS)aeConvert chunks from Dask Array into an SVG Image Parameters ---------- chunks: tuple size: int Rough size of the image Examples -------- >>> x.to_svg(size=500) # doctest: +SKIP Returns ------- text: An svg string depicting the array as a grid of chunks rK)svgr)rr)rrrr_r_r`rs z Array.to_svgcKst|||fi|S)acStore array in HDF5 file >>> x.to_hdf5('myfile.hdf5', '/x') # doctest: +SKIP Optionally provide arguments as though to ``h5py.File.create_dataset`` >>> x.to_hdf5('myfile.hdf5', '/x', compression='lzf', shuffle=True) # doctest: +SKIP See Also -------- da.store h5py.File.create_dataset )to_hdf5)rfilenamedatapathrr_r_r`rsz Array.to_hdf5cCsddlm}|||||dS)a^Convert dask Array to dask Dataframe Parameters ---------- columns: list or string list of column names if DataFrame, single string if Series index : dask.dataframe.Index, optional An optional *dask* Index to use for the output Series or DataFrame. The default output index depends on whether the array has any unknown chunks. If there are any unknown chunks, the output has ``None`` for all the divisions (one per chunk). If all the chunks are known, a default index with known divsions is created. Specifying ``index`` can be useful if you're conforming a Dask Array to an existing dask Series or DataFrame, and you would like the indices to match. meta : object, optional An optional `meta` parameter can be passed for dask to specify the concrete dataframe type to use for partitions of the Dask dataframe. By default, pandas DataFrame is used. See Also -------- dask.dataframe.from_dask_array r)from_dask_array)columnsindexr)Z dataframer)rrrrrr_r_r`to_dask_dataframes zArray.to_dask_dataframecCs0|jdkr td|jjdn t|SdS)NrKzThe truth value of a z& is ambiguous. Use a.any() or a.all().)rrrqr[rr rrr_r_r`__bool__s  zArray.__bool__cCs$|jdkrtdn ||SdS)NrKz7Only length-1 arrays can be converted to Python scalars)rr(r )rZ cast_typer_r_r` _scalarfuncs  zArray._scalarfunccCs |tSra)rrZrrr_r_r`__int__sz Array.__int__cCs |tSra)rfloatrrr_r_r` __float__szArray.__float__cCs |tSra)rcomplexrrr_r_r` __complex__szArray.__complex__cCs |tjSra)roperatorrrrr_r_r` __index__szArray.__index__c Cs^|tjjurtjd}t|trddlm}t|trJ|jdkrJt dz||||}Wn.t y}zt d|WYd}~n d}~00|j |_ |j |_ |j |_ |j|_dSt|jrt dtt|}dt|||}t||||}t|j }t|rt|}tj|||gd} t| ||j|j|d }|j |_ |j |_ |j |_ |j|_dS) Nr_rKwherez+boolean index array should have 1 dimensionzBoolean index assignment in Dask expects equally shaped arrays. Example: da1[da2] = da3 where da1.shape == (4,), da2.shape == (4,) and da3.shape == (4,).z Arrays chunk sizes are unknown. zsetitem-rd)rrr)rpZmaZmaskedZ masked_allrgrrrrrrbrIrrirrjrXrrmrY asanyarrayr)rVrisscalarrXr1rr) rrvalueryrr&r}rgraphr_r_r` __setitem__sB       zArray.__setitem__cst|ts*t|tr|rtdd|Drt|tr@j|}n4t|fdd|Dfdd|Djjd}|jrttj j t |j}j t dd|jD}j t||j||dSj t||d St|t s|f}d d lm}m}m}||j}jh} |D]} t| tr| | jqtd d|DrT||\}td d|Drv||\}tdd|DrSdt|} t| jj |j\} }tj| | gd} tjt |d}t|rt|}t| | ||dS)Ncss|]}t|tVqdSrarerr_r_r`re5rfz$Array.__getitem__..csg|]}jj|dqSrUrfieldsrcrrrr_r`r=rfz%Array.__getitem__..csg|]}jj|dqSrrrrrr_r`r>rf)namesformatsZoffsetsrcss|] }|fVqdSrar_rr_r_r`reErfrrrK)normalize_indexslice_with_bool_dask_arrayslice_with_int_dask_arraycss$|]}t|to|jjdvVqdS)iuN)rgrrkindrr_r_r`re[rfcss"|]}t|to|jtkVqdSra)rgrrrrr_r_r`re]rfcss$|]}t|to|tdkVqdSrargrhrr_r_r`re`rfzgetitem-rdrrK) rgrrrrrprrrrrrrlrrMbaseslicingrrrrrrrmr)rWr1rrrbrrX)rrrorrrrrindex2rSrr&r}rrr_rrr` __getitem__2sX          zArray.__getitem__cCs~t|ts|f}tdd|Dr0td|tdd|Drntddt||jDr`|Std|t|g|RS)Ncss|]}|duVqdSrar_rFr_r_r`reqrfz Array._vindex..z?vindex does not support indexing with None (np.newaxis), got {}css|]}t|tVqdSrarrFr_r_r`revrfcss,|]$\}}||td||kVqdSrN)indicesrh)rcrrVr_r_r`rewszvindex requires at least one non-slice to vectorize over when the slices are not over the entire array (i.e, x[:]). Use normal slicing instead when only using slices. Got: {}) rgrlrm IndexErrorrrrr_vindexrr_r_r`rns&  z Array._vindexcCs t|jS)alVectorized indexing with broadcasting. This is equivalent to numpy's advanced indexing, using arrays that are broadcast against each other. This allows for pointwise indexing: >>> import dask.array as da >>> x = np.array([[1, 2, 3], [4, 5, 6], [7, 8, 9]]) >>> x = da.from_array(x, chunks=2) >>> x.vindex[[0, 1, 2], [0, 1, 2]].compute() array([1, 5, 9]) Mixed basic/advanced indexing with slices/arrays is also supported. The order of dimensions in the result follows those proposed for `ndarray.vindex `_: the subspace spanned by arrays is followed by all slices. Note: ``vindex`` provides more general functionality than standard indexing, but it also has fewer optimizations and can be significantly slower. )r6rrrr_r_r`vindexsz Array.vindexcCst|S)a@An array-like interface to the blocks of an array. This returns a ``Blockview`` object that provides an array-like interface to the blocks of a dask array. Numpy-style indexing of a ``Blockview`` object returns a selection of blocks as a new dask array. You can index ``array.blocks`` like a numpy array of shape equal to the number of blocks in each dimension, (available as array.blocks.size). The dimensionality of the output array matches the dimension of this array, even if integer indices are passed. Slicing with ``np.newaxis`` or multiple lists is not supported. Examples -------- >>> import dask.array as da >>> x = da.arange(8, chunks=2) >>> x.blocks.shape # aliases x.numblocks (4,) >>> x.blocks[0].compute() array([0, 1]) >>> x.blocks[:3].compute() array([0, 1, 2, 3, 4, 5]) >>> x.blocks[::2].compute() array([0, 1, 4, 5]) >>> x.blocks[[-1, 0]].compute() array([6, 7, 0, 1]) >>> x.blocks.ravel() # doctest: +NORMALIZE_WHITESPACE [dask.array, dask.array, dask.array, dask.array] Returns ------- An instance of ``dask.array.Blockview`` ) BlockViewrrr_r_r`blockss&z Array.blockscCs|jS)aSlice an array by partitions. Alias of dask array .blocks attribute. This alias allows you to write agnostic code that works with both dask arrays and dask dataframes. This returns a ``Blockview`` object that provides an array-like interface to the blocks of a dask array. Numpy-style indexing of a ``Blockview`` object returns a selection of blocks as a new dask array. You can index ``array.blocks`` like a numpy array of shape equal to the number of blocks in each dimension, (available as array.blocks.size). The dimensionality of the output array matches the dimension of this array, even if integer indices are passed. Slicing with ``np.newaxis`` or multiple lists is not supported. Examples -------- >>> import dask.array as da >>> x = da.arange(8, chunks=2) >>> x.partitions.shape # aliases x.numblocks (4,) >>> x.partitions[0].compute() array([0, 1]) >>> x.partitions[:3].compute() array([0, 1, 2, 3, 4, 5]) >>> x.partitions[::2].compute() array([0, 1, 4, 5]) >>> x.partitions[[-1, 0]].compute() array([6, 7, 0, 1]) >>> x.partitions.ravel() # doctest: +NORMALIZE_WHITESPACE [dask.array, dask.array, dask.array, dask.array] Returns ------- An instance of ``da.array.Blockview`` )rrrr_r_r` partitionss)zArray.partitionscCs.ddlm}||||jdf|jdffdS)NrK) tensordotrr)rrr)rrrr_r_r`rs z Array.dotcCs|Srar_rrr_r_r`rszArray.AcCs|Sra transposerrr_r_r`TszArray.TcGstddlm}|sd}n"t|dkr8t|dtr8|d}|tt|jks`|tt|j dkrd|S|||dSdS)NrKrrr)rrrrgrrlrr)rrrr_r_r`rs (zArray.transposecCsddlm}||S)NrK)ravel)rr)rrr_r_r`r s z Array.ravelcCsddlm}|||S)NrK)choose)rr)rchoicesrr_r_r`rs z Array.chooseT merge_chunkslimitcGs>ddlm}t|dkr.t|dts.|d}|||||dS)z .. note:: See :meth:`dask.array.reshape` for an explanation of the ``merge_chunks`` and `limit` keywords. rK)reshaperr)rrrgr)rrrrrr_r_r`rs z Array.reshapercCsddlm}|||||dS)z]The top k elements of an array. See :func:`dask.array.topk` for docstring. rK)topkr split_every) reductionsr)rrrrrr_r_r`r%s z Array.topkcCsddlm}|||||dS)zoThe indices of the top k elements of an array. See :func:`dask.array.argtopk` for docstring. rK)argtopkr)rr)rrrrrr_r_r`r/s z Array.argtopkcKst|ddh}|r&tdt||dd}t|}|j|krJ|Stj|j||dsxtd|jd|d||jtj f||d |S) aCopy of the array, cast to a specified type. Parameters ---------- dtype : str or dtype Typecode or data-type to which the array is cast. casting : {'no', 'equiv', 'safe', 'same_kind', 'unsafe'}, optional Controls what kind of data casting may occur. Defaults to 'unsafe' for backwards compatibility. * 'no' means the data types should not be cast at all. * 'equiv' means only byte-order changes are allowed. * 'safe' means only casts which can preserve values are allowed. * 'same_kind' means only safe casts or casts within a kind, like float64 to float32, are allowed. * 'unsafe' means any data conversions may be done. copy : bool, optional By default, astype always returns a newly allocated array. If this is set to False and the `dtype` requirement is satisfied, the input array is returned instead of a copy. castingcopyz6astype does not take the following keyword arguments: ZunsaferzCannot cast array from z to z according to the rule )rZ astype_dtype) rr(rrrprcan_castrrLr)rrrZextrarr_r_r`r9s     z Array.astypecCs ttj|Sra)rrabsrrr_r_r`__abs__asz Array.__abs__cCsttj||Srarrrrr_r_r`__add__dsz Array.__add__cCsttj||Srarrr_r_r`__radd__hszArray.__radd__cCsttj||Srarrand_rr_r_r`__and__lsz Array.__and__cCsttj||Srarrr_r_r`__rand__pszArray.__rand__cCsttj||SrarrZdivrr_r_r`__div__tsz Array.__div__cCsttj||Srar rr_r_r`__rdiv__xszArray.__rdiv__cCsttj||Sra)rreqrr_r_r`__eq__|sz Array.__eq__cCsttj||Sra)rrgtrr_r_r`__gt__sz Array.__gt__cCsttj||Sra)rrgerr_r_r`__ge__sz Array.__ge__cCs ttj|Sra)rrinvertrrr_r_r` __invert__szArray.__invert__cCsttj||Srarrlshiftrr_r_r` __lshift__szArray.__lshift__cCsttj||Srarrr_r_r` __rlshift__szArray.__rlshift__cCsttj||Sra)rrltrr_r_r`__lt__sz Array.__lt__cCsttj||Sra)rrlerr_r_r`__le__sz Array.__le__cCsttj||Srarrmodrr_r_r`__mod__sz Array.__mod__cCsttj||Srarrr_r_r`__rmod__szArray.__rmod__cCsttj||Srarrrrr_r_r`__mul__sz Array.__mul__cCsttj||Srar"rr_r_r`__rmul__szArray.__rmul__cCsttj||Sra)rrnerr_r_r`__ne__sz Array.__ne__cCs ttj|Sra)rrnegrrr_r_r`__neg__sz Array.__neg__cCsttj||Srarror_rr_r_r`__or__sz Array.__or__cCs|Srar_rrr_r_r`__pos__sz Array.__pos__cCsttj||Srar)rr_r_r`__ror__sz Array.__ror__cCsttj||Srarrpowrr_r_r`__pow__sz Array.__pow__cCsttj||Srar.rr_r_r`__rpow__szArray.__rpow__cCsttj||Srarrrshiftrr_r_r` __rshift__szArray.__rshift__cCsttj||Srar2rr_r_r` __rrshift__szArray.__rrshift__cCsttj||Srarrsubrr_r_r`__sub__sz Array.__sub__cCsttj||Srar6rr_r_r`__rsub__szArray.__rsub__cCsttj||Srarrtruedivrr_r_r` __truediv__szArray.__truediv__cCsttj||Srar:rr_r_r` __rtruediv__szArray.__rtruediv__cCsttj||Srarrfloordivrr_r_r` __floordiv__szArray.__floordiv__cCsttj||Srar>rr_r_r` __rfloordiv__szArray.__rfloordiv__cCsttj||Srarrxorrr_r_r`__xor__sz Array.__xor__cCsttj||SrarBrr_r_r`__rxor__szArray.__rxor__cCsddlm}|||SNrKrrrrrrr_r_r` __matmul__s zArray.__matmul__cCsddlm}|||SrFrGrHr_r_r` __rmatmul__s zArray.__rmatmul__cCsddlm}|||SNrK)divmodrrLrrrLr_r_r` __divmod__s zArray.__divmod__cCsddlm}|||SrKrMrNr_r_r` __rdivmod__s zArray.__rdivmod__FcCsddlm}||||||dS)NrK)rmrkeepdimsrr&)rrm)rrrRrr&rmr_r_r`rm s z Array.anycCsddlm}||||||dS)NrKrrQ)rr)rrrRrr&rr_r_r`r s z Array.allcCsddlm}||||||dS)NrK)minrQ)rrS)rrrRrr&rSr_r_r`rS s z Array.mincCsddlm}||||||dS)NrKrrQ)rr)rrrRrr&rr_r_r`r s z Array.maxcCsddlm}|||||dS)NrK)argminrrr&)rrT)rrrr&rTr_r_r`rT s z Array.argmincCsddlm}|||||dS)NrK)argmaxrU)rrV)rrrr&rVr_r_r`rV# s z Array.argmaxcCs ddlm}|||||||dS)NrKrrrrRrr&)rr)rrrrRrr&rr_r_r`r) s z Array.sumrrKcCsddlm}||||||dS)NrK)trace)offsetaxis1axis2r)rrY)rrZr[r\rrYr_r_r`rY6 s z Array.tracecCs ddlm}|||||||dS)NrK)rrX)rr)rrrrRrr&rr_r_r`r< s z Array.prodcCs ddlm}|||||||dS)NrK)meanrX)rr])rrrrRrr&r]r_r_r`r]I s z Array.meanc Cs"ddlm}||||||||dS)NrK)stdrrrRddofrr&)rr^)rrrrRr`rr&r^r_r_r`r^V s z Array.stdc Cs"ddlm}||||||||dS)NrK)varr_)rra)rrrrRr`rr&rar_r_r`raf s z Array.varc Cs$ddlm}|||||||||dS)aCCalculate the nth centralized moment. Parameters ---------- order : int Order of the moment that is returned, must be >= 2. axis : int, optional Axis along which the central moment is computed. The default is to compute the moment of the flattened array. dtype : data-type, optional Type to use in computing the moment. For arrays of integer type the default is float64; for arrays of float types it is the same as the array type. keepdims : bool, optional If this is set to True, the axes which are reduced are left in the result as dimensions with size one. With this option, the result will broadcast correctly against the original array. ddof : int, optional "Delta Degrees of Freedom": the divisor used in the calculation is N - ddof, where N represents the number of elements. By default ddof is zero. Returns ------- moment : ndarray References ---------- .. [1] Pebay, Philippe (2008), "Formulas for Robust, One-Pass Parallel Computation of Covariances and Arbitrary-Order Statistical Moments", Technical Report SAND2008-6212, Sandia National Laboratories. rK)momentr_)rrb) rorderrrrRr`rr&rbr_r_r`rbv s, z Array.momentcOst||g|Ri|Sra)r)rrrrr_r_r`r szArray.map_blockscKs&ddlm}|||f|||d|S)a Map a function over blocks of the array with some overlap We share neighboring zones between blocks of the array, then map a function, then trim away the neighboring strips. Note that this function will attempt to automatically determine the output array type before computing it, please refer to the ``meta`` keyword argument in :func:`map_blocks ` if you expect that the function will not succeed when operating on 0-d arrays. Parameters ---------- func: function The function to apply to each extended block depth: int, tuple, or dict The number of elements that each block should share with its neighbors If a tuple or dict then this can be different per axis boundary: str, tuple, dict How to handle the boundaries. Values include 'reflect', 'periodic', 'nearest', 'none', or any constant value like 0 or np.nan trim: bool Whether or not to trim ``depth`` elements from each block after calling the map function. Set this to False if your mapping function already does this for you **kwargs: Other keyword arguments valid in :func:`map_blocks `. Examples -------- >>> import dask.array as da >>> x = np.array([1, 1, 2, 3, 3, 3, 2, 1, 1]) >>> x = da.from_array(x, chunks=5) >>> def derivative(x): ... return x - np.roll(x, 1) >>> y = x.map_overlap(derivative, depth=1, boundary=0) >>> y.compute() array([ 1, 0, 1, 1, 0, 0, -1, -1, 0]) >>> import dask.array as da >>> x = np.arange(16).reshape((4, 4)) >>> d = da.from_array(x, chunks=(2, 2)) >>> y = d.map_overlap(lambda x: x + x.size, depth=1, boundary='reflect') >>> y.compute() array([[16, 17, 18, 19], [20, 21, 22, 23], [24, 25, 26, 27], [28, 29, 30, 31]]) >>> func = lambda x: x + x.size >>> depth = {0: 1, 1: 1} >>> boundary = {0: 'reflect', 1: 'none'} >>> d.map_overlap(func, depth, boundary).compute() # doctest: +NORMALIZE_WHITESPACE array([[12, 13, 14, 15], [16, 17, 18, 19], [20, 21, 22, 23], [24, 25, 26, 27]]) >>> x = np.arange(16).reshape((4, 4)) >>> d = da.from_array(x, chunks=(2, 2)) >>> y = d.map_overlap(lambda x: x + x[2], depth=1, boundary='reflect', meta=np.array(())) >>> y dask.array<_trim, shape=(4, 4), dtype=float64, chunksize=(2, 2), chunktype=numpy.ndarray> >>> y.compute() array([[ 4, 6, 8, 10], [ 8, 10, 12, 14], [20, 22, 24, 26], [24, 26, 28, 30]]) >>> import cupy # doctest: +SKIP >>> x = cupy.arange(16).reshape((4, 4)) # doctest: +SKIP >>> d = da.from_array(x, chunks=(2, 2)) # doctest: +SKIP >>> y = d.map_overlap(lambda x: x + x[2], depth=1, boundary='reflect', meta=cupy.array(())) # doctest: +SKIP >>> y # doctest: +SKIP dask.array<_trim, shape=(4, 4), dtype=float64, chunksize=(2, 2), chunktype=cupy.ndarray> >>> y.compute() # doctest: +SKIP array([[ 4, 6, 8, 10], [ 8, 10, 12, 14], [20, 22, 24, 26], [24, 26, 28, 30]]) rK) map_overlap)depthboundarytrim)Zoverlaprd)rrrerfrgrrdr_r_r`rd sS zArray.map_overlapZ sequential)rcCsddlm}||||||dS)aDask added an additional keyword-only argument ``method``. method : {'sequential', 'blelloch'}, optional Choose which method to use to perform the cumsum. Default is 'sequential'. * 'sequential' performs the cumsum of each prior block before the current block. * 'blelloch' is a work-efficient parallel cumsum. It exposes parallelism by first taking the sum of each block and combines the sums via a binary tree. This method may be faster or more memory efficient depending on workload, scheduler, and hardware. More benchmarking is necessary. rK)cumsumr&r)rrh)rrrr&rrhr_r_r`rh s z Array.cumsumcCsddlm}||||||dS)aDask added an additional keyword-only argument ``method``. method : {'sequential', 'blelloch'}, optional Choose which method to use to perform the cumprod. Default is 'sequential'. * 'sequential' performs the cumprod of each prior block before the current block. * 'blelloch' is a work-efficient parallel cumprod. It exposes parallelism by first taking the product of each block and combines the products via a binary tree. This method may be faster or more memory efficient depending on workload, scheduler, and hardware. More benchmarking is necessary. rK)cumprodri)rrj)rrrr&rrjr_r_r`rj s z Array.cumprodcCsddlm}|||S)NrK)squeeze)rrk)rrrkr_r_r`rk. s z Array.squeezeautocCsddlm}||||||S)zSee da.rechunk for docstringrKrechunkrm)rrZ thresholdZblock_size_limitZbalancernr_r_r`rn4 s z Array.rechunkcCsddlm}||S)NrK)real)rro)rror_r_r`ro< s z Array.realcCsddlm}||S)NrK)imag)rrp)rrpr_r_r`rpB s z Array.imagcCsddlm}||S)NrK)conj)rrq)rrqr_r_r`rqH s z Array.conjcCsddlm}||||S)NrK)clip)rrr)rrSrrrr_r_r`rrM s z Array.clipCcs|dur|j}n t|}|jj|j|dkr^|jddtfdd|jdDf}n>|dkrtfdd|jdDf|jd d}ntd |jtj||||d S) a@Get a view of the array as a new data type Parameters ---------- dtype: The dtype by which to view the array. The default, None, results in the view having the same data-type as the original array. order: string 'C' or 'F' (Fortran) ordering This reinterprets the bytes of the array under a new dtype. If that dtype does not have the same size as the original array then the shape will change. Beware that both numpy and dask.array can behave oddly when taking shape-changing views of arrays under Fortran ordering. Under some versions of NumPy this function will fail when taking shape-changing views of Fortran ordered arrays if the first dimension has chunks of size one. Nrsrc3s|]}t|VqdSra ensure_intrZmultr_r`req rfzArray.view..Fc3s|]}t|VqdSrartrrvr_r`reu rfrrKzOrder must be one of 'C' or 'F')rcrr) rrprrrlrrrLview)rrrcrr_rvr`rxS s"    z Array.viewcCsddlm}||||S)NrK)swapaxes)rry)rr[r\ryr_r_r`ry~ s zArray.swapaxescCsddlm}|||dS)NrK)round)decimals)rrz)rr{rzr_r_r`rz s z Array.roundcCs0|jdkr|tjSt|j|j|j|dSdS)zS Copy array. This is a no-op for dask.arrays, which are immutable rKrKN)rrr7rrrIrrrrr_r_r`r s  z Array.copycCs|}||t|<|Sra)rr?)rmemorur_r_r` __deepcopy__ s zArray.__deepcopy__csl|}||d|rF||d|jtjddt|jfdd|}t j |t dS)aiConvert into an array of :class:`dask.delayed.Delayed` objects, one per chunk. Parameters ---------- optimize_graph : bool, optional If True [default], the graph is optimized before converting into :class:`dask.delayed.Delayed` objects. See Also -------- dask.array.from_delayed rzdelayed-r_rdcst|dS)NrDrErrrr_r`r rfz"Array.to_delayed..r) r>r=rurNrr1rrFrrprXr)rZoptimize_graphrLr_r~r` to_delayed s    zArray.to_delayedcCsddlm}||||dS)NrK)repeatr)Zcreationr)rZrepeatsrrr_r_r`r s z Array.repeatcCsddlm}||S)NrK)nonzero)rr)rrr_r_r`r s z Array.nonzerocOst|g|Ri|S)zSave array to the zarr storage format See https://zarr.readthedocs.io for details about the format. See function :func:`dask.array.to_zarr` for parameters. )to_zarr)rrrr_r_r`r sz Array.to_zarrcOs$ddlm}|||g|Ri|S)zSave array to the TileDB storage manager See function :func:`dask.array.to_tiledb` for argument documentation. See https://docs.tiledb.io for details about the format and engine. rK) to_tiledb)Z tiledb_ior)rZurirrrr_r_r`r s zArray.to_tiledb)NNN)N)N)r)NNN)rN)rN)NFNN)NFNN)NFNN)NFNN)NNN)NNN)NNFNN)rrrKN)NNFNN)NNFNN)NNFrNN)NNFrNN)NNFrNN)NT)NN)NN)N)rlNNF)NN)Nrs)r)T)N)r[r\r]r^ __slots__rhrsr=rur>rwr,r{r&rN staticmethodr#rZ__dask_scheduler__ryr{rzrr:rrrrrpropertyrfrrjsetterrrrrrrrrrrirrrrrrrrTrrrrZ __nonzero__rrZ__long__rrrrrrrrrr<rprrrrrrrOrrrrrrrrrr r r r rrrrrrrrr r!r#r$r&r(r+r,r-r0r1r4r5r8r9r<r=r@rArDrErIrJrOrPrmrrSrrTrVrrYrr]r^rarbrrdrhrjrkrnrorprqrrrxryrzrr}rrrrr __classcell__r_r_rpr`rs<     1        '         2     /<  ' *       (                                          9  Y +  rcCs t|}||krtd||S)NzCould not coerce %f to integer)rZr)rrr_r_r`ru s rucs|rt|tjst|}dur*ttttr Normalize chunks to tuple of tuples This takes in a variety of input types and information and produces a full tuple-of-tuples result for chunks, suitable to be passed to Array or rechunk or any other operation that creates a Dask array. Parameters ---------- chunks: tuple, int, dict, or string The chunks to be normalized. See examples below for more details shape: Tuple[int] The shape of the array limit: int (optional) The maximum block size to target in bytes, if freedom is given to choose dtype: np.dtype previous_chunks: Tuple[Tuple[int]] optional Chunks from a previous array that we should use for inspiration when rechunking auto dimensions. If not provided but auto-chunking exists then auto-dimensions will prefer square-like chunk shapes. Examples -------- Specify uniform chunk sizes >>> from dask.array.core import normalize_chunks >>> normalize_chunks((2, 2), shape=(5, 6)) ((2, 2, 1), (2, 2, 2)) Also passes through fully explicit tuple-of-tuples >>> normalize_chunks(((2, 2, 1), (2, 2, 2)), shape=(5, 6)) ((2, 2, 1), (2, 2, 2)) Cleans up lists to tuples >>> normalize_chunks([[2, 2], [3, 3]]) ((2, 2), (3, 3)) Expands integer inputs 10 -> (10, 10) >>> normalize_chunks(10, shape=(30, 5)) ((10, 10, 10), (5,)) Expands dict inputs >>> normalize_chunks({0: 2, 1: 3}, shape=(6, 6)) ((2, 2, 2), (3, 3)) The values -1 and None get mapped to full size >>> normalize_chunks((5, -1), shape=(10, 10)) ((5, 5), (10,)) Use the value "auto" to automatically determine chunk sizes along certain dimensions. This uses the ``limit=`` and ``dtype=`` keywords to determine how large to make the chunks. The term "auto" can be used anywhere an integer can be used. See array chunking documentation for more information. >>> normalize_chunks(("auto",), shape=(20,), limit=5, dtype='uint8') ((5, 5, 5, 5),) You can also use byte sizes (see :func:`dask.utils.parse_bytes`) in place of "auto" to ask for a particular size >>> normalize_chunks("1kiB", shape=(2000,), dtype='float32') ((250, 250, 250, 250, 250, 250, 250, 250),) Respects null dimensions >>> normalize_chunks((), shape=(0, 0)) ((0,), (0,)) Nc3s|]}|dVqdSrarrr$r_r`re+ rfz#normalize_chunks..css|]}|dkVqdSrr_rr_r_r`re. rfrUrKcss|]}t|ttfVqdSra)rgrrrr_r_r`re5 rfzNChunks and shape must be of the same length/dimension. Got chunks=%s, shape=%srcss*|]"\}}|dks|dur|n|VqdSrNr_rcrurr_r_r`re? rfrlzDOnly one consistent value of limit or chunk is allowed.Used %s != %scss(|] }t|tr|dkrdn|VqdSrlNrerr_r_r`reN rfcss|]}|dkVqdSrr_rr_r_r`reP rfcss"|]\}}|dvr|n|VqdS)>NrNr_rr_r_r`reT rfcss4|],\}}t|ttfs&t|f|fn|fVqdSra)rgrlrr\)rcrrur_r_r`reX s r_zZEmpty tuples are not allowed in chunks. Express zero length dimensions with 0(s) in chunkszLInput array has %d dimensions but the supplied chunks has only %d dimensionscss.|]&\}}||kp$t|p$t|VqdSra)rrXrr_r_r`rem sz6Chunks do not add up to shape. Got chunks=%s, shape=%scss |]}tdd|DVqdS)css$|]}t|st|n|VqdSra)rrXrZrbr_r_r`rev rfz-normalize_chunks...Nrrr_r_r`rev rf)rgrprrrkrrlrrrrrrtolistrrrHrm auto_chunksrr)rrrrprevious_chunksruparsedr_r$r`r sK             rrZ)r largest_blockcCs*||j|ttdd|DS)z Utility function for auto_chunk, to fin how much larger or smaller the ideal chunk size is relative to what we have now. css|]}|dkr|ndVqdSrrKNr_)rcrAr_r_r`re rfz&_compute_multiplier..)rrprrvalues)rrrr8r_r_r`_compute_multipliery srcsdurtddDt|}ddt|D}|s@t|S|durRtd}t|trdt|}|durttd|j rt dt|tD]<}t|t rt |st|trt |rtd tqtd |}t d d |D}rrfd d|D}g} tD]d\} } t| } t| ddd\} }| d krj|t| dkrj| | n | | qt||||}d}t}||ks||krN|}t|}t|D]}| |dkrd||<q|||d t|}||kr&||||9}|||<||=nt|| |||<qt||||}q|D]\}}|||<qVt|S||j|d t|fdd |D}|r|D]} | f|| <qt|||S|D]} t| || <qt|SdS)adDetermine automatic chunks This takes in a chunks value that contains ``"auto"`` values in certain dimensions and replaces those values with concrete dimension sizes that try to get chunks to be of a certain size in bytes, provided by the ``limit=`` keyword. If multiple dimensions are marked as ``"auto"`` then they will all respond to meet the desired byte limit, trying to respect the aspect ratio of their dimensions in ``previous_chunks=``, if given. Parameters ---------- chunks: Tuple A tuple of either dimensions or tuples of explicit chunk dimensions Some entries should be "auto" shape: Tuple[int] limit: int, str The maximum allowable size of a chunk in bytes previous_chunks: Tuple[Tuple[int]] See also -------- normalize_chunks: for full docstring and parameters Ncss"|]}t|tr|n|fVqdSra)rgrlrr_r_r`re szauto_chunks..cSsh|]\}}|dkr|qSrlr_rcrrur_r_r` rfzauto_chunks..zarray.chunk-sizez%dtype must be known for auto-chunkingziCan not use auto rechunking with object dtype. We are unable to estimate the size in bytes of object datazFCan not perform automatic rechunking with unknown (nan) chunk sizes.%srKcSs*g|]"}|dkrt|tr|nt|qSr)rgrr)rccsr_r_r`r rfzauto_chunks..csi|]}|t|qSr_)rpZmedianr)rr_r`r  rfzauto_chunks..cSs|dSrr_)Zkvr_r_r`r rfzauto_chunks..rrrcsg|]}|kr|qSr_r_r)rrr_r`r rf)rlrr-r!rrgrrHr(Z hasobjectrrrprXrmrrYrrrr1rr0rrr.removeround_torr)rrrrrZautosrdrr8Z ideal_shaperrZchunk_frequenciesmodecountZ multiplierZlast_multiplierZ last_autosrrZproposedrvZsmallr_)rrrr`r s             rcsX|krHztfddt|DWStyDtdtYS0n ||SdS)aReturn a chunk dimension that is close to an even multiple or factor We want values for c that are nicely aligned with s. If c is smaller than s then we want the largest factor of s that is less than the desired chunk size, but not less than half, which is too much. If no such factor exists then we just go with the original chunk size and accept an uneven chunk at the end. If c is larger than s then we want the largest multiple of s that is still smaller than c. c3s.|]&}d|krkrnq|VqdS)rNr_)rcrrur_r`re rfzround_to..rKN)rr=rrZ)rurr_rr`r s  rcCs*tj|jtd}|t|dtdfS)Nrra)rprjrrZrrh)rrrr_r_r`r srrlc  sttrtdntr&tdttttft j rFt |durZt d }t dd} t|jj| d}|dvrt||||||} |pd| }n|d urdtt}|d urt}tt ju} td d |D} | rB| sB|sBt|} t|ggd d |DR}fdd| D}tt||}nZ| rd| rd|fdji}n8|dur~|rzt}nt}t |j||||j|d }j!j"#dddkrt drt|||jdS|dur}t||||t dddS)aCreate dask array from something that looks like an array. Input must have a ``.shape``, ``.ndim``, ``.dtype`` and support numpy-style slicing. Parameters ---------- x : array_like chunks : int, tuple How to chunk the array. Must be one of the following forms: - A blocksize like 1000. - A blockshape like (1000, 1000). - Explicit sizes of all blocks along all dimensions like ((1000, 1000, 500), (400, 400)). - A size in bytes, like "100 MiB" which will choose a uniform block-like shape - The word "auto" which acts like the above, but uses a configuration value ``array.chunk-size`` for the chunk size -1 or None as a blocksize indicate the size of the corresponding dimension. name : str or bool, optional The key name to use for the array. Defaults to a hash of ``x``. Hashing is useful if the same value of ``x`` is used to create multiple arrays, as Dask can then recognise that they're the same and avoid duplicate computations. However, it can also be slow, and if the array is not contiguous it is copied for hashing. If the array uses stride tricks (such as :func:`numpy.broadcast_to` or :func:`skimage.util.view_as_windows`) to have a larger logical than physical size, this copy can cause excessive memory usage. If you don't need the deduplication provided by hashing, use ``name=False`` to generate a random name instead of hashing, which avoids the pitfalls described above. Using ``name=True`` is equivalent to the default. By default, hashing uses python's standard sha1. This behaviour can be changed by installing cityhash, xxhash or murmurhash. If installed, a large-factor speedup can be obtained in the tokenisation step. .. note:: Because this ``name`` is used as the key in task graphs, you should ensure that it uniquely identifies the data contained within. If you'd like to provide a descriptive name that is still unique, combine the descriptive name with :func:`dask.base.tokenize` of the ``array_like``. See :ref:`graphs` for more. lock : bool or Lock, optional If ``x`` doesn't support concurrent reads then provide a lock here, or pass in True to have dask.array create one for you. asarray : bool, optional If True then call np.asarray on chunks to convert them to numpy arrays. If False then chunks are passed through unchanged. If None (default) then we use True if the ``__array_function__`` method is undefined. fancy : bool, optional If ``x`` doesn't support fancy indexing (e.g. indexing with lists or arrays) then set to False. Default is True. meta : Array-like, optional The metadata for the resulting dask array. This is the kind of array that will result from slicing the input array. Defaults to the input array. inline_array : bool, default False How to include the array in the task graph. By default (``inline_array=False``) the array is included in a task by itself, and each chunk refers to that task by its key. .. code-block:: python >>> x = h5py.File("data.h5")["/x"] # doctest: +SKIP >>> a = da.from_array(x, chunks=500) # doctest: +SKIP >>> dict(a.dask) # doctest: +SKIP { 'array-original-': , ('array-', 0): (getitem, "array-original-", ...), ('array-', 1): (getitem, "array-original-", ...) } With ``inline_array=True``, Dask will instead inline the array directly in the values of the task graph. .. code-block:: python >>> a = da.from_array(x, chunks=500, inline_array=True) # doctest: +SKIP >>> dict(a.dask) # doctest: +SKIP { ('array-', 0): (getitem, , ...), ('array-', 1): (getitem, , ...) } Note that there's no key in the task graph with just the array `x` anymore. Instead it's placed directly in the values. The right choice for ``inline_array`` depends on several factors, including the size of ``x``, how expensive it is to create, which scheduler you're using, and the pattern of downstream computations. As a heuristic, ``inline_array=True`` may be the right choice when the array ``x`` is cheap to serialize and deserialize (since it's included in the graph many times) and if you're experiencing ordering issues (see :ref:`order` for more). This has no effect when ``x`` is a NumPy array. Examples -------- >>> x = h5py.File('...')['/data/path'] # doctest: +SKIP >>> a = da.from_array(x, chunks=(1000, 1000)) # doctest: +SKIP If your underlying datastore does not support concurrent reads then include the ``lock=True`` keyword argument or ``lock=mylock`` if you want multiple arrays to coordinate around the same lock. >>> a = da.from_array(x, chunks=(1000, 1000), lock=True) # doctest: +SKIP If your underlying datastore has a ``.chunks`` attribute (as h5py and zarr datasets do) then a multiple of that chunk shape will be used if you do not provide a chunk shape. >>> a = da.from_array(x, chunks='auto') # doctest: +SKIP >>> a = da.from_array(x, chunks='100 MiB') # doctest: +SKIP >>> a = da.from_array(x) # doctest: +SKIP If providing a name, ensure that it is unique >>> import dask.base >>> token = dask.base.tokenize(x) # doctest: +SKIP >>> a = da.from_array('myarray-' + token) # doctest: +SKIP NumPy ndarrays are eagerly sliced and then embedded in the graph. >>> import dask.array >>> a = dask.array.from_array(np.array([[1, 2], [3, 4]]), chunks=(1,1)) >>> a.dask[a.name, 0, 0][0] array([1]) Chunks with exactly-specified, different sizes can be created. >>> import numpy as np >>> import dask.array as da >>> x = np.random.random((100, 6)) >>> a = da.from_array(x, chunks=((67, 33), (6,))) zBArray is already a dask array. Use 'asarray' or 'rechunk' instead.ztPassing an object to dask.array.from_array which is already a Dask collection. This can lead to unexpected behavior.NrrrrNTzarray-FTcss|]}t|dkVqdSrrrr_r_r`re rfzfrom_array..css|]}tt|VqdSrar rr_r_r`re rfcsg|] }|qSr_r_rcslcrr_r`r rfzfrom_array..rU)rMrkrjrrrrZtiledbZ_ctx_rr)rr)$rgrrr'r)r*rrl memoryviewrpZ ScalarTyperXrrrrrr)ruuiduuid1r8rrrrrrrrrnrxrrqr\r)rdrrrkrjZfancyrMrrrrZ is_ndarrayZis_single_blockrrrr}r_rr` from_array sj       $ rc Ksddl}|pi}t||jr"|}nlt|ttjfrrt|tjrHt|}t|fi|} |j| fd|d|}n|} |j| fd|d|}|dur|n|j}|durdt ||||fi|}t ||||dS)a"Load array from the zarr storage format See https://zarr.readthedocs.io for details about the format. Parameters ---------- url: Zarr Array or str or MutableMapping Location of the data. A URL can include a protocol specifier like s3:// for remote data. Can also be any MutableMapping instance, which should be serializable if used in multiple processes. component: str or None If the location is a zarr group rather than an array, this is the subcomponent that should be loaded, something like ``'foo/bar'``. storage_options: dict Any additional parameters for the storage backend (ignored for local paths) chunks: tuple of ints or tuples of ints Passed to :func:`dask.array.from_array`, allows setting the chunks on initialisation, if the chunking scheme in the on-disc dataset is not optimal for the calculations to follow. name : str, optional An optional keyname for the array. Defaults to hashing the input kwargs: Passed to :class:`zarr.core.Array`. inline_array : bool, default False Whether to inline the zarr Array in the values of the task graph. See :meth:`dask.array.from_array` for an explanation. See Also -------- from_array rNT)Z read_onlypathz from-zarr-)rr) zarrrgrrosPathLikefspathrrr)r) url componentstorage_optionsrrrrrzmapperr_r_r` from_zarr s)   rc sxddl} t|jr$tdtt|| jr|} t| j t t fr\t dddvr\td|durv|| j}d} nZdd lmm} t| j| j} | || j}tfd d t| j| |D}||}|g} |j | d | ||d S|durtdt|jstd|pi}t|tr0t|fi|}n|}dd|jD}| jf|j||j|||d|} |j | d ||dS)a Save array to the zarr storage format See https://zarr.readthedocs.io for details about the format. Parameters ---------- arr: dask.array Data to store url: Zarr Array or str or MutableMapping Location of the data. A URL can include a protocol specifier like s3:// for remote data. Can also be any MutableMapping instance, which should be serializable if used in multiple processes. component: str or None If the location is a zarr group rather than an array, this is the subcomponent that should be created/over-written. storage_options: dict Any additional parameters for the storage backend (ignored for local paths) overwrite: bool If given array already exists, overwrite=False will cause an error, where overwrite=True will replace the existing data. region: tuple of slices or None The region of data that should be written if ``url`` is a zarr.Array. Not to be used with other types of ``url``. compute: bool See :func:`~dask.array.store` for more details. return_stored: bool See :func:`~dask.array.store` for more details. **kwargs: Passed to the :func:`zarr.creation.create` function, e.g., compression options. Raises ------ ValueError If ``arr`` has unknown chunk sizes, which is not supported by Zarr. If ``region`` is specified and ``url`` is not a zarr.Array See Also -------- dask.array.store dask.array.Array.compute_chunk_sizes rNzRSaving a dask array with unknown chunk sizes is not currently supported by Zarr.%sZ schedulerr)zdask.distributedZ distributedzGCannot store into in memory Zarr Array using the Distributed Scheduler.rK) new_blockdimrc3s$|]\}}}t|||VqdSrar)rcrrurArr_r`re szto_zarr..F)rkr;r r<z;Cannot use `region` keyword when url is not a `zarr.Array`.z\Attempt to save array to zarr with irregular chunking, please call `arr.rechunk(...)` first.cSsg|] }|dqSrUr_rr_r_r`r rfzto_zarr..)rrrrTr overwrite)rkr r<)rrprXrrmrrYrgrrTrr r!r RuntimeErrorrnrrrrrrlr_check_regular_chunksrrZcreater)rrrrrrBr r<rrrr;rZ old_chunksrrrr_rr`rB sj6           rcCsP|D]F}t|dkrqtt|dddkr4dS|d|dkrdSqdS)aCheck if the chunks are regular "Regular" in this context means that along every axis, the chunks all have the same size, except the last one, which may be smaller Parameters ---------- chunkset: tuple of tuples of ints From the ``.chunks`` attribute of an ``Array`` Returns ------- True if chunkset passes, else False Examples -------- >>> import dask.array as da >>> arr = da.zeros(10, chunks=(5, )) >>> _check_regular_chunks(arr.chunks) True >>> arr = da.zeros(10, chunks=((3, 3, 3, 1), )) >>> _check_regular_chunks(arr.chunks) True >>> arr = da.zeros(10, chunks=((3, 1, 3, 3), )) >>> _check_regular_chunks(arr.chunks) False rKNrFrT)rr)Zchunksetrr_r_r`r s rc Csddlm}m}t||s,t|dr,||}|p@dt||||}|fdt||ji}tdd|D}tj |||gd} t | ||||d S) auCreate a dask array from a dask delayed value This routine is useful for constructing dask arrays in an ad-hoc fashion using dask delayed, particularly when combined with stack and concatenate. The dask array will consist of a single chunk. Examples -------- >>> import dask >>> import dask.array as da >>> import numpy as np >>> value = dask.delayed(np.ones)(5) >>> array = da.from_delayed(value, (5,), dtype=float) >>> array dask.array >>> array.compute() array([1., 1., 1., 1., 1.]) rr.rz from-value-rUcss|] }|fVqdSrar_rcrVr_r_r`re rfzfrom_delayed..rdrr) r0r/rgrr)rrrlr1rr) rrrrrr/r0r}rrr_r_r` from_delayed srr_cCsn|pdt|||||}|s |r6t|g|Ri|}|fdt||fi}tdd|D}t||||S)a:Create dask array in a single block by calling a function Calling the provided function with func(*args, **kwargs) should return a NumPy array of the indicated shape and dtype. Examples -------- >>> a = from_func(np.arange, (3,), dtype='i8', args=(3,)) >>> a.compute() array([0, 1, 2]) This works particularly well when coupled with dask.array functions like concatenate and stack: >>> arrays = [from_func(np.array, (), dtype='i8', args=(n,)) for n in range(5)] >>> stack(arrays).compute() array([0, 1, 2, 3, 4]) z from_func-rUcss|] }|fVqdSrar_rr_r_r`rerfzfrom_func..)r)r rrlr)rrrrrrr}rr_r_r` from_funcs rcCst|s dSdd|D}t|dkr.t|St|dkrFt|tdStttt|rftd|tt tt|dkrtd|d d |D}tt|}d}g}||krt d d |D}| ||D](}|d |8<|d dkr| q||7}qt |S)aFind the common block dimensions from the list of block dimensions Currently only implements the simplest possible heuristic: the common block-dimension is the only one that does not span fully span a dimension. This is a conservative choice that allows us to avoid potentially very expensive rechunking. Assumes that each element of the input block dimensions has all the same sum (i.e., that they correspond to dimensions of the same size). Examples -------- >>> common_blockdim([(3,), (2, 1)]) (2, 1) >>> common_blockdim([(1, 2), (2, 1)]) (1, 1, 1) >>> common_blockdim([(2, 2), (3, 1)]) # doctest: +SKIP Traceback (most recent call last): ... ValueError: Chunks do not align r_cSsh|]}t|dkr|qSrrrr_r_r`r9rfz"common_blockdim..rKrrzUArrays' chunk sizes (%s) are unknown. A possible solution: x.compute_chunk_sizes()z"Chunks do not add up to same valuecSsg|]}t|dddqSrr)rcZntdr_r_r`rTrfz#common_blockdim..css|]}|dVqdSrr_rr_r_r`reZrfz"common_blockdim..r)rmrrrrprXrrrrrSr0rrl)Z blockdimsZnon_trivial_dimsZrchunkstotalrr&mrur_r_r`common_blockdim!s8          rc s|s igfSddtd|D}|dd}t|\tddDrVitfStfddDrtfd dDrttd d jfSg}t}d }|D]H\}|d ur|j|fj|j<t |j }q||fqt ||t d t ttt}|rR|rR||d krRtjd||tddg|D]p\} | d urxnPtfddt| D} | jkrtjr| n qZfS)aT Unify chunks across a sequence of arrays This utility function is used within other common operations like :func:`dask.array.core.map_blocks` and :func:`dask.array.core.blockwise`. It is not commonly used by end-users directly. Parameters ---------- *args: sequence of Array, index pairs Sequence like (x, 'ij', y, 'jk', z, 'i') Examples -------- >>> import dask.array as da >>> x = da.ones(10, chunks=((5, 2, 3),)) >>> y = da.ones(10, chunks=((2, 3, 5),)) >>> chunkss, arrays = unify_chunks(x, 'i', y, 'i') >>> chunkss {'i': (2, 3, 2, 3)} >>> x = da.ones((100, 10), chunks=(20, 5)) >>> y = da.ones((10, 100), chunks=(4, 50)) >>> chunkss, arrays = unify_chunks(x, 'ij', y, 'jk', 'constant', None) >>> chunkss # doctest: +SKIP {'k': (50, 50), 'i': (20, 20, 20, 20, 20), 'j': (4, 1, 3, 2)} >>> unify_chunks(0, None) ({}, [0]) Returns ------- chunkss : dict Map like {index: chunks}. arrays : list List of rechunked arrays. See Also -------- common_blockdim cSs(g|] \}}|durt|n||fqSra)r)rcrrrr_r_r`rsz unify_chunks..rr*Tcss|]}|duVqdSrar_rcrr_r_r`rerfzunify_chunks..c3s|]}|dkVqdSrr_r)indsr_r`rerfc3s|]}|jdjkVqdSrr$r)rr_r`resrN)Z consolidaterz+Increasing number of chunks by factor of %d) stacklevelc3sH|]@\}}j|dkr |ntt|s<j|ndVqdSr)rrprXr)rcr3r)rrr7r_r`res   )rrrrrrrr0rrrr+rrprrrrr)r*rZrlr-rn) rrZargindsr*ZnameindsZ blockdim_dictZ max_partsrZnpartsrrr_)rrrr7rr` unify_chunksesR*   "      rc Cs>t|ttfr:z |d}Wqtttfy6Yq:Yq0q|S)z >>> unpack_singleton([[[[1]]]]) 1 >>> unpack_singleton(np.array(np.datetime64('2000-01-01'))) array('2000-01-01', dtype='datetime64[D]') r)rgrrlrr(rmrr_r_r`r^s   r^c s,dddd}tddd}d}d }||D]\}}}t|turXtd |||sft|} n t|d kr0t|d } d }nq0|dur|| krtd|| ||| }q0|rtd|j|t t d}|j|ddt d} t || |} |j|fddt d}|j|fdddd| dS)a& Assemble an nd-array from nested lists of blocks. Blocks in the innermost lists are concatenated along the last dimension (-1), then these are concatenated along the second-last dimension (-2), and so on until the outermost list is reached Blocks can be of any dimension, but will not be broadcasted using the normal rules. Instead, leading axes of size 1 are inserted, to make ``block.ndim`` the same for all blocks. This is primarily useful for working with scalars, and means that code like ``block([v, 1])`` is valid, where ``v.ndim == 1``. When the nested list is two levels deep, this allows block matrices to be constructed from their components. Parameters ---------- arrays : nested list of array_like or scalars (but not tuples) If passed a single ndarray or scalar (a nested list of depth 0), this is returned unmodified (and not copied). Elements shapes must match along the appropriate axes (without broadcasting), but leading 1s will be prepended to the shape as necessary to make the dimensions match. allow_unknown_chunksizes: bool Allow unknown chunksizes, such as come from converting from dask dataframes. Dask.array is unable to verify that chunks line up. If data comes from differently aligned sources then this can cause unexpected results. Returns ------- block_array : ndarray The array assembled from the given blocks. The dimensionality of the output is equal to the greatest of: * the dimensionality of all the inputs * the depth to which the input list is nested Raises ------ ValueError * If list depths are mismatched - for instance, ``[[a, b], c]`` is illegal, and should be spelt ``[[a, b], [c]]`` * If lists are empty - for instance, ``[[a, b], []]`` See Also -------- concatenate : Join a sequence of arrays together. stack : Stack arrays in sequence along a new dimension. hstack : Stack arrays in sequence horizontally (column wise). vstack : Stack arrays in sequence vertically (row wise). dstack : Stack arrays in sequence depth wise (along third dimension). vsplit : Split array into a list of multiple sub-arrays vertically. Notes ----- When called with only scalars, ``block`` is equivalent to an ndarray call. So ``block([[1, 2], [3, 4]])`` is equivalent to ``array([[1, 2], [3, 4]])``. This function does not enforce that the blocks lie on a fixed grid. ``block([[a, b], [c, d]])`` is not restricted to arrays of the form:: AAAbb AAAbb cccDD But is also allowed to produce, for some ``a, b, c, d``:: AAAbb AAAbb cDDDD Since concatenation happens along the last axis first, `block` is _not_ capable of producing the following directly:: AAAbb cccbb cccDD Matlab's "square bracket stacking", ``[A, B, ...; p, q, ...]``, is equivalent to ``block([[A, B, ...], [p, q, ...]])``. cSs:t|}t||jd}|dkr$|S|d|tfSdS)Nrra)rrrEllipsis)rdrdiffr_r_r` atleast_nd5s zblock..atleast_ndcSsdddd|DS)Nrrcss|]}d|dVqdS)[]Nr_rr_r_r`re>rfz.block..format_index..)rrr_r_r` format_index=szblock..format_indexcSs t|tuSra)rrrr_r_r`r@rfzblock..)Z recurse_ifNFz{} is a tuple. Only lists can be used to arrange blocks, and np.block does not allow implicit conversion from tuple to ndarray.rrKTzcList depths are mismatched. First element was at depth {}, but there is an element at depth {} ({})zLists cannot be empty)Zf_mapf_reducecSs|jSrarZxir_r_r`rlrfcs |Srar_r)rrr_r`rtrfcstt||dS)N)rallow_unknown_chunksizes)rr)Zxsrrr_r`rzs cSst|ddS)NrKr)rrr_r_r`r}rf)rZf_kwargsr) rTwalkrrlr(rrrZ map_reducerrr) rrrZrecZ list_ndimZ any_emptyrrZenteringZ curr_depthZ elem_ndimZ first_axisr_)rrrr`blocksP]       rcsddlm}fddDs*tddurDddDdd dD}ttt|d d d }||d fddDtdjt fddt D}ddDsʈdkrڈkrd}t|ft}|dkrHz|j ||j dWSt yD|j||j dYS0n|dkrZdSstfddt DstttjdjrtdtdjtdddDfddt |D} t| D]\} } | d | <qttt| } t| ddi\} ddD}djdtfdd|Ddfdjdd}dgtttfddDd dDd!t}tt|ggd"d|DR}fd#d|D}t t||}t!j"||d$}t#|||d%S)&a Concatenate arrays along an existing axis Given a sequence of dask Arrays form a new dask Array by stacking them along an existing dimension (axis=0 by default) Parameters ---------- seq: list of dask.arrays axis: int Dimension along which to align all of the arrays. If axis is None, arrays are flattened before use. allow_unknown_chunksizes: bool Allow unknown chunksizes, such as come from converting from dask dataframes. Dask.array is unable to verify that chunks line up. If data comes from differently aligned sources then this can cause unexpected results. Examples -------- Create slices >>> import dask.array as da >>> import numpy as np >>> data = [da.from_array(np.ones((4, 4)), chunks=(2, 2)) ... for i in range(3)] >>> x = da.concatenate(data, axis=0) >>> x.shape (12, 4) >>> da.concatenate(data, axis=1).shape (4, 12) Result is a new dask Array See Also -------- stack rKwrapcsg|]}t|dqSrrjrrr_r`rrfzconcatenate..zNeed array(s) to concatenateNcSsg|] }|qSr_)rOrr_r_r`rrfrcSsg|] }t|qSr_rrr_r_r`rrfcSs t|ddSrrrr_r_r`rrfzconcatenate..rrcsg|]}|jqSr_rrrrKr_r`rrfc3s:|]2kr$tfddDn djVqdS)c3s|]}|jVqdSrarrr6r_r`rerf(concatenate...rN)rrrcrseqr6r`reszconcatenate..cSsg|]}|jr|qSr_rrr_r_r`rrfzXAxis must be less than than number of dimensions Data has %d dimensions, but got axis=%drrrrc3s.|]&kp$tfddDVqdS)c3s&|]}|jdjkVqdSrrrb)rseq2r_r`rerfrNrr)rrr6r`reszTried to concatenate arrays with unknown shape %s. Two solutions: 1. Force concatenation pass allow_unknown_chunksizes=True. 2. Compute shapes with [x.compute_chunk_sizes() for x in seq]zShapes do not align: %scSsg|] }|jqSr_rrbr_r_r`rrfcsg|]}ttqSr_)rrrrr_r`rrfr*FcSsg|] }|jqSr_r$rr_r_r`rrfc3s|]}|VqdSrar_rcrWrr_r`rerfr_csg|]}t|jqSr_)rrrrr_r`rrfcSsg|] }|jqSr_rtrr_r_r`rrfz concatenate-cSsg|]}tt|qSr_r rr_r_r`rrfc spg|]h}t|ddf|dd|dt|ddf|ddqSrKrNrrcr)rcum_dimsrr_r`rs&rdrK)$rrrrPrrrrrrlr empty_likerr(emptyrrmrrprXrr-rrrrrrrrr)rrr1rr)rrrrZ seq_metasZ _concatenaterrr3rrruc_args_rrrrrr}rr_)rrrrrrrrr`rs+      " rcCsxd}|r|s|}|r|zH|durFt|r8|||<nt|||<|rV|rV||}W|rt|n|rr|0|S)a A function inserted in a Dask graph for storing a chunk. Parameters ---------- x: array-like An array (potentially a NumPy one) out: array-like Where to store results too. index: slice-like Where to store result from ``x`` in ``out``. lock: Lock-like or False Lock to use before writing to ``out``. return_stored: bool Whether to return ``out``. load_stored: bool Whether to return the array stored in ``out``. Ignored if ``return_stored`` is not ``True``. Examples -------- >>> a = np.ones((5, 6)) >>> b = np.empty(a.shape) >>> load_store_chunk(a, b, (slice(None), slice(None)), False, False, False) N)rorArprrq)rdr&rrkr<rCr8r_r_r`load_store_chunks"    rcCst|||||dS)NFr)rdr&rrkr<r_r_r` store_chunkCsrcCstd|||ddSrr)r&rrkr_r_r` load_chunkGsr)rkrBr<rCrztuple[tuple[int, ...], ...]rz Lock | boolz slice | Noner)rrrrkrBr<rCrc stdurtt|}r,fdd|D}r@|r@t|fntdfddtt||D} | S)a Creates a Dask graph for storing chunks from ``arr`` in ``out``. Parameters ---------- keys: list Dask keys of the input array chunks: tuple Dask chunks of the input array out: array-like Where to store results to name: str First element of dask keys lock: Lock-like or bool, optional Whether to lock or with what (default is ``True``, which means a :class:`threading.Lock` instance). region: slice-like, optional Where in ``out`` to store ``arr``'s results (default is ``None``, meaning all of ``out``). return_stored: bool, optional Whether to return ``out`` (default is ``False``, meaning ``None`` is returned). load_stored: bool, optional Whether to handling loading from ``out`` at the same time. Ignored if ``return_stored`` is not ``True``. (default is ``False``, meaning defer to ``return_stored``). Returns ------- dask graph of store operation Examples -------- >>> import dask.array as da >>> d = da.ones((5, 6), chunks=(2, 3)) >>> a = np.empty(d.shape) >>> insert_to_ooc(d.__dask_keys__(), d.chunks, a, "store-123") # doctest: +SKIP Tcsg|]}t|qSr_)rzr)rBr_r`rrfz!insert_to_ooc..r_c s4i|],\}}f|dd||fqSrr_)rcr@r)rrrkrr&r<r_r`r sz!insert_to_ooc..)rrrrrr"rO) rrr&rrkrBr<rCrr}r_)rrrkrr&rBr<r`rPKs2rPzCollection[Hashable]r )rdsk_predsk_postrcsfdd|D}|S)a Creates a Dask graph for loading stored ``keys`` from ``dsk``. Parameters ---------- keys: Collection A sequence containing Dask graph keys to load dsk_pre: Mapping A Dask graph corresponding to a Dask Array before computation dsk_post: Mapping A Dask graph corresponding to a Dask Array after computation Examples -------- >>> import dask.array as da >>> d = da.ones((5, 6), chunks=(2, 3)) >>> a = np.empty(d.shape) >>> g = insert_to_ooc(d.__dask_keys__(), d.chunks, a, "store-123") >>> retrieve_from_ooc(g.keys(), g, {k: k for k in g.keys()}) # doctest: +SKIP cs@i|]8}d|df|ddt|f|ddqS)rJrrKNrr)rrFrrr_r`r sz%retrieve_from_ooc..r_)rrrZload_dskr_rr`rQs rQ)rcKs|durt|tr|St|dr(|St|jdddkrTt|drTt|jSt|t t frt dd|Drt ||d Stt |d dtstj|||d }nDtstd t|}t|tr|jtj|||d Stj||||d }t|fdti|S)uConvert the input to a dask array. Parameters ---------- a : array-like Input data, in any form that can be converted to a dask array. This includes lists, lists of tuples, tuples, tuples of tuples, tuples of lists and ndarrays. allow_unknown_chunksizes: bool Allow unknown chunksizes, such as come from converting from dask dataframes. Dask.array is unable to verify that chunks line up. If data comes from differently aligned sources then this can cause unexpected results. dtype : data-type, optional By default, the data-type is inferred from the input data. order : {‘C’, ‘F’, ‘A’, ‘K’}, optional Memory layout. ‘A’ and ‘K’ depend on the order of input array a. ‘C’ row-major (C-style), ‘F’ column-major (Fortran-style) memory representation. ‘A’ (any) means ‘F’ if a is Fortran contiguous, ‘C’ otherwise ‘K’ (keep) preserve input order. Defaults to ‘C’. like: array-like Reference object to allow the creation of Dask arrays with chunks that are not NumPy arrays. If an array-like passed in as ``like`` supports the ``__array_function__`` protocol, the chunk type of the resulting array will be definde by it. In this case, it ensures the creation of a Dask array compatible with that passed in via this argument. If ``like`` is a Dask array, the chunk type of the resulting array will be defined by the chunk type of ``like``. Requires NumPy 1.20.0 or higher. Returns ------- out : dask array Dask array interpretation of a. Examples -------- >>> import dask.array as da >>> import numpy as np >>> x = np.arange(3) >>> da.asarray(x) dask.array >>> y = [[1, 2, 3], [4, 5, 6]] >>> da.asarray(y) dask.array N to_dask_arrayrrxarraydatacss|]}t|tVqdSrarrr_r_r`rerfzasarray..rrrrc*The use of ``like`` required NumPy >= 1.20rrrcrM)rgrrrrr\rrjrrrlrmstackrrrprSrrrrry)rrrrrcrr like_metar_r_r`rjs$2  "    rj)rrcCs|durt|tr|St|dr(|St|jdddkrTt|drTt|jSt|t t fr|t dd|Dr|t |Stt |d dtstj|||d }nDtstd t|}t|tr|jtj|||d Stj||||d }t||jtd |dS)uConvert the input to a dask array. Subclasses of ``np.ndarray`` will be passed through as chunks unchanged. Parameters ---------- a : array-like Input data, in any form that can be converted to a dask array. This includes lists, lists of tuples, tuples, tuples of tuples, tuples of lists and ndarrays. dtype : data-type, optional By default, the data-type is inferred from the input data. order : {‘C’, ‘F’, ‘A’, ‘K’}, optional Memory layout. ‘A’ and ‘K’ depend on the order of input array a. ‘C’ row-major (C-style), ‘F’ column-major (Fortran-style) memory representation. ‘A’ (any) means ‘F’ if a is Fortran contiguous, ‘C’ otherwise ‘K’ (keep) preserve input order. Defaults to ‘C’. like: array-like Reference object to allow the creation of Dask arrays with chunks that are not NumPy arrays. If an array-like passed in as ``like`` supports the ``__array_function__`` protocol, the chunk type of the resulting array will be definde by it. In this case, it ensures the creation of a Dask array compatible with that passed in via this argument. If ``like`` is a Dask array, the chunk type of the resulting array will be defined by the chunk type of ``like``. Requires NumPy 1.20.0 or higher. inline_array: Whether to inline the array in the resulting dask graph. For more information, see the documentation for ``dask.array.from_array()``. Returns ------- out : dask array Dask array interpretation of a. Examples -------- >>> import dask.array as da >>> import numpy as np >>> x = np.arange(3) >>> da.asanyarray(x) dask.array >>> y = [[1, 2, 3], [4, 5, 6]] >>> da.asanyarray(y) dask.array Nrrrrrcss|]}t|tVqdSrarrr_r_r`re1rfzasanyarray..rrrrF)rrMrjr)rgrrrrr\rrrrrlrmrrrrprSrrrrrry)rrrrcrrrr_r_r`rs00  "   rcCsZt|dd}t|t p(tdd|D}t|pX|pXt|tjpXt|tjoX|jdkS)a >>> is_scalar_for_elemwise(42) True >>> is_scalar_for_elemwise('foo') True >>> is_scalar_for_elemwise(True) True >>> is_scalar_for_elemwise(np.array(42)) True >>> is_scalar_for_elemwise([1, 2, 3]) True >>> is_scalar_for_elemwise(np.array([1, 2, 3])) False >>> is_scalar_for_elemwise(from_array(np.array(0), chunks=())) False >>> is_scalar_for_elemwise(np.dtype('i4')) True rNcss|]}t|VqdSrar'rbr_r_r`re^sz)is_scalar_for_elemwise..r) rrgrrmrprrrr)rZ maybe_shapeZshape_conditionr_r_r`is_scalar_for_elemwiseGs   rc st|dkr|dSg}ttt|ddiD]j}t|rFtjnd|vrRdnt|tfdd|Drt d d tt || q,tt|S) a2 Determines output shape from broadcasting arrays. Parameters ---------- shapes : tuples The shapes of the arguments. Returns ------- output_shape : tuple Raises ------ ValueError If the input shapes cannot be successfully broadcast together. rKr fillvaluerc3s*|]"}|dddfvo t| VqdS)rrrKN)rprXrrr_r`rerfz#broadcast_shapes..z7operands could not be broadcast together with shapes {} )rrrreversedrprXrmnanrrrrrr0rl)rr&Zsizesr_rr`broadcast_shapesjs  r)r&rrrcOs|rt|jdt|t|}t|}dd|D}g}|D]0}t|dd} tdd| Drhd} || qBt|t r||j t|t r||j dd|D}t t |} t t| d d d } |d urd } nPd d|D} zt|| id dd}WntytYS0tdd|D} |sXt|dt||g||R}t||t|dd}|d ur||d<t}|||g| r||d<||d<t}t|| gtdd|DRi|}t||S)aApply an elementwise ufunc-like function blockwise across arguments. Like numpy ufuncs, broadcasting rules are respected. Parameters ---------- op : callable The function to apply. Should be numpy ufunc-like in the parameters that it accepts. *args : Any Arguments to pass to `op`. Non-dask array-like objects are first converted to dask arrays, then all arrays are broadcast together before applying the function blockwise across all arguments. Any scalar arguments are passed as-is following normal numpy ufunc behavior. out : dask array, optional If out is a dask.array then this overwrites the contents of that array with the result. where : array_like, optional An optional boolean mask marking locations where the ufunc should be applied. Can be a scalar, dask array, or any other array-like object. Mirrors the ``where`` argument to numpy ufuncs, see e.g. ``numpy.add`` for more information. dtype : dtype, optional If provided, overrides the output array dtype. name : str, optional A unique key name to use when building the backing dask graph. If not provided, one will be automatically generated based on the input arguments. Examples -------- >>> elemwise(add, x, y) # doctest: +SKIP >>> elemwise(sin, x) # doctest: +SKIP >>> elemwise(sin, x, out=dask_array) # doctest: +SKIP See Also -------- blockwise z/ does not take the following keyword arguments cSs(g|] }t|ttfr t|n|qSr_)rgrrlrprjrr_r_r`rrfzelemwise..rr_css|]}t|VqdSrarrbr_r_r`rerfzelemwise..cSsg|]}t|tr|ndqS)r_)rgrrr_r_r`rrfNrTcSs4g|],}t|s,tjdtd|j|jdn|qS)rrKr)rrprrrrrr_r_r`rsrF)rcss"|]}t| o|jdkVqdSr)rrrr_r_r`resrr)rrrelemwise_where_function enforce_dtypeenforce_dtype_functioncss6|].}|t|s(tt|jdddndfVqdSr)rrlrrrr_r_r`res)r(r[r._elemwise_normalize_out_elemwise_normalize_whererrmr0rgrrrrrlrrrrr?r)rstrip_elemwise_handle_whereextend_enforce_dtyper*r handle_out)opr&rrrrrrrrZout_ndimZ expr_indsZneed_enforce_dtypeZvalsZblockwise_kwargsr8r_r_r`rst(        $  rcCs(|dur dS|dus|dur dSt|S)NTFrrr_r_r`rs rcOs<|d}|^}}}t|dr(|}||||d|S)Nrr)rr&)rrr)rrfunctionrr&r_r_r`rs    rcCsft|tr:t|dkr |d}nt|dkr6tdnd}|dusbt|tsbtdt|jd|S)NrKrz(The out parameter is not fully supportedz8The out parameter is not fully supported. Received type z, expected Dask Array)rgrlrrrrr[r%r_r_r`rs     rcCsft|}t|tr^|j|jkr:tdt|jt|jf|j|_|j|_|j |_ |j |_ |S|SdS)zzHandle out parameters If out is a dask.array then this overwrites the contents of that array with the result zEMismatched shapes between result and out parameter. out=%s, result=%sN) rrgrrrrrrjrIrbrri)r&r8r_r_r`r&s  rcOs|d}|d}||i|}t|dr||jkr|tkrtj||ddsntdt|t|t|jft |r| |}n0z|j |dd}Wnt y| |}Yn0|S) a6Calls a function and converts its result to the given dtype. The parameters have deliberately been given unwieldy names to avoid clashes with keyword arguments consumed by blockwise A dtype of `object` is treated as a special case and not enforced, because it is used as a dummy value in some places when the result will not be a block in an Array. Parameters ---------- enforce_dtype : dtype Result dtype enforce_dtype_function : callable The wrapped function, which will be passed the remaining arguments rrrZ same_kindrz`Inferred dtype from function %r was %r but got %r, which can't be cast using casting='same_kind'Fr) rrrrrprrr?rrrr()rrrrr8r_r_r`r<s"     rc Cst|}t|}|dur t|}|j|kr@|dus<||jkr@|St||j}|dksxtddt||d|jDrt d|jd||durtdd|d|Dtddt|j|j||dD}nZt |||j |jd }t|j||dD]0\}}||kr|d krt d |j|fqd t |||}i}t d d|D} dd| DD]V\} } tddt|j| |dD} |jf| } |f| }tj| t| f||<qftj|||gd}t||||j |dS)aHBroadcast an array to a new shape. Parameters ---------- x : array_like The array to broadcast. shape : tuple The shape of the desired array. chunks : tuple, optional If provided, then the result will use these chunks instead of the same chunks as the source array. Setting chunks explicitly as part of broadcast_to is more efficient than rechunking afterwards. Chunks are only allowed to differ from the original shape along dimensions that are new on the result or have size 1 the input array. meta : empty ndarray empty ndarray created with same NumPy backend, ndim and dtype as the Dask Array being created (overrides dtype) Returns ------- broadcast : dask array See Also -------- :func:`numpy.broadcast_to` Nrcss"|]\}}|dkr||kVqdSrr_)rcnewoldr_r_r`reszbroadcast_to..zcannot broadcast shape z to shape css|] }|fVqdSrar_rr_r_r`rerfcss&|]\}}}|dkr|n|fVqdSrr_)rcrWr r r_r_r`resrrzqcannot broadcast chunks %s to chunks %s: new chunks must either be along a new dimension or a dimension of size 1z broadcast_to-css|]}t|VqdSra)r-rr_r_r`rerfcss|]}t|VqdSrar)rcZecr_r_r`rerfcss"|]\}}|dkrdn|VqdS)rrNr_)rcrWrr_r_r`resrdr)rjrlrrrrrrmrrrrr)rrrp broadcast_tor-r1rr)rdrrrZndim_newZold_bdZnew_bdrr}Zenumerated_chunksZ new_indexZ chunk_shapeZ old_indexZold_keyZnew_keyrr_r_r`r fsJ     r )subokcst|}|rtnttfdd|D}dd|D}tt||}t|ddi\}}tdd|Dtdd|Dfd d|D}|S) Nc3s|]}|VqdSrar_r)to_arrayr_r`rerfz#broadcast_arrays..cSsg|]}ttt|jqSr_)rrrrrbr_r_r`rrfz$broadcast_arrays..r*Fcss|] }|jVqdSrarrr_r_r`rerfcss|] }|jVqdSrar$rr_r_r`rerfcsg|]}t|dqS)r[)r r)rrr_r`rrf) rrrjrlrrrrr9)r rrrrr8r_)rrrr`broadcast_arrayss rcsHfdd}ttdj|_Wdn1s:0Y|S)zOffsets inputs by offset >>> double = lambda x: x * 2 >>> f = offset_func(double, (10,)) >>> f(1) 22 >>> f(300) 620 csttt|}|Sra)rrr)rZargs2rrZr_r`_offsetszoffset_func.._offsetZoffset_N) contextlibsuppressrr[)rrZrrr_rr` offset_funcs  *rcs^|sdSg}dddt|ttfrV|tfdd|D|d}d7qt|S)a_Chunks tuple from nested list of arrays >>> x = np.array([1, 2]) >>> chunks_from_arrays([x, x]) ((2, 2),) >>> x = np.array([[1, 2]]) >>> chunks_from_arrays([[x], [x]]) ((1, 1), (2,)) >>> x = np.array([[1, 2]]) >>> chunks_from_arrays([[x, x]]) ((1,), (2, 2)) >>> chunks_from_arrays([1, 1]) ((1, 1),) r_rcSs"z|jWStyYdS0dS)Nr)rrlrr_r_r`rs z!chunks_from_arrays..shapec3s|]}t|VqdSra) deepfirstrrrr_r`rerfz%chunks_from_arrays..rK)rgrrlr0)rr8r_rr`chunks_from_arrayss rcCs"t|ttfs|St|dSdS)z`First element in a nested list >>> deepfirst([[[1, 2], [3, 4]], [5, 6], [7, 8]]) 1 rN)rgrrlrrr_r_r`rsrcCs2t|tur*tt|gtt|dSdSdS)zGet the shape of nested listrr_N)rrrlr shapelist)rrr_r_r`r s rcstt|krtd|dkr(tdtttkrDtdtd}t|fddt||D}tt |}t ||S)zPermute axes of nested list >>> transposelist([[1,1,1],[1,1,1]], [2,1]) [[[1, 1], [1, 1], [1, 1]]] >>> transposelist([[1,1,1],[1,1,1]], [2,1], extradims=1) [[[[1], [1]], [[1], [1]], [[1], [1]]]] 2Length of axes should equal depth of nested arraysrz`newdims` should be positivez`axes` should be uniquerKcs&g|]}|vr|ndqSrrrrrr_r`r$sz!transposelist..) rrGrrrrrrr"rOr4)rr extradimsrZnewshaper8r_rr` transposelists    rcsjddlm}fddDs*tdstfddDstfddtD}td d jd |d dd |djd tjddDdfddDj dd krֈdt fddt j D}ddD}|s }t |}|d kr\z|j ||jdWStyZ|j||jdYS0tt ttfdd|D}t|\} }t dd|DdksJ|d jdd|f|d jd} dd|Ddt} tt| ggdd| DR} fdd| D} fdd| D}tt| |}tj| ||d}t|| | d S)!a Stack arrays along a new axis Given a sequence of dask arrays, form a new dask array by stacking them along a new dimension (axis=0 by default) Parameters ---------- seq: list of dask.arrays axis: int Dimension along which to align all of the arrays allow_unknown_chunksizes: bool Allow unknown chunksizes, such as come from converting from dask dataframes. Dask.array is unable to verify that chunks line up. If data comes from differently aligned sources then this can cause unexpected results. Examples -------- Create slices >>> import dask.array as da >>> import numpy as np >>> data = [da.from_array(np.ones((4, 4)), chunks=(2, 2)) ... for i in range(3)] >>> x = da.stack(data, axis=0) >>> x.shape (3, 4, 4) >>> da.stack(data, axis=1).shape (4, 3, 4) >>> da.stack(data, axis=-1).shape (4, 4, 3) Result is a new dask Array See Also -------- concatenate rKrcsg|]}t|dqSrrrrr_r`r[rfzstack..zNeed array(s) to stackc3s|]}|jdjkVqdSrrrbrr_r`re_rfzstack..c3s&|]}|djdjkr|VqdSrrrrr_r`re`rfzCStacked arrays must have the same shape. The first array had shape rz, while array z has shape rcSsg|] }t|qSr_rrr_r_r`rfrfrcsg|]}|jqSr_rrbrKr_r`rgrfc3sF|]>}|krtn&|kr,dj|ndj|dVqdSrrrrr_r`rels cSsg|]}|jr|qSr_rrr_r_r`rsrfrrc3s|]}|fVqdSrar_rb)rr_r`rerfcSsh|] }|jqSr_r$rr_r_r`rrfzstack..NrcSsg|] }|jqSr_rtrr_r_r`rrfzstack-cSsg|]}tt|qSr_r rr_r_r`rrfcs>g|]6}|df|dd|ddqSrr_r)rrr_r`rsc s>g|]6}t|tdddfdtdddffqS)Nra)rMrh)rcinp)rrr_r`rsrdrK)rrrrrr-rrprrrlrrrrr(rrrrrr)rrrr1rr)rrrridxrrr3rrrrrrrrrr_)rrrrrrrr`r,sb-       .    rc shttjddt|}|s$tdSttj|tt fdddd}t fdd tj|tt fdDsz t |}t |t t |jd WStyYn0tt|tjurt |}t |tt |jd St|}|s|St|}t tt|}d d }tj||t|d }tt|tj|tt fdD]6\}} t| drX| j|krX| d} q@| ||<q,|S)aRecursive np.concatenate Input should be a nested list of numpy arrays arranged in the order they should appear in the array itself. Each array should have the same number of dimensions as the desired output and the nesting of the lists. >>> x = np.array([[1, 2]]) >>> concatenate3([[x, x, x], [x, x, x]]) array([[1, 2, 1, 2, 1, 2], [1, 2, 1, 2, 1, 2]]) >>> concatenate3([[x, x], [x, x], [x, x]]) array([[1, 2, 1, 2], [1, 2, 1, 2], [1, 2, 1, 2]]) rNr) containercSs t|ddSrrrr_r_r`rrfzconcatenate3..rc3s"|]}tt|duVqdS)rN)rr)rcrZNDARRAY_ARRAY_FUNCTIONr_r`reszconcatenate3..rcSs(z|jWSty"t|YS0dSra)rrlrrr_r_r`rs zconcatenate3..dtyperr)N.)rrprr;rrr"rOrrlrr^rrrr(rPrrrrGrrrrrrr) rZadvancedrdrrrrr8rrr_r!r`r]sD        r]cCsFt|t|krtdtdt|jt|d}tt|||dS)z*Recursively call np.concatenate along axesrrrK)r)rrGrrrrr]r)rrrr_r_r`concatenate_axessr"r$cst|dkr$t|dtr$|d}nBt|dkr^t|dtr^t|dtr^|d|di}ntdddl}|j|dd<fdd |D}t t | |Wdn1s0YdS) aStore arrays in HDF5 file This saves several dask arrays into several datapaths in an HDF5 file. It creates the necessary datasets and handles clean file opening/closing. Parameters ---------- chunks: tuple or ``True`` Chunk shape, or ``True`` to pass the chunks from the dask array. Defaults to ``True``. Examples -------- >>> da.to_hdf5('myfile.hdf5', '/x', x) # doctest: +SKIP or >>> da.to_hdf5('myfile.hdf5', {'/x': x, '/y': y}) # doctest: +SKIP Optionally provide arguments as though to ``h5py.File.create_dataset`` >>> da.to_hdf5('myfile.hdf5', '/x', x, compression='lzf', shuffle=True) # doctest: +SKIP >>> da.to_hdf5('myfile.hdf5', '/x', x, chunks=(10,20,30)) # doctest: +SKIP This can also be used as a method on a single Array >>> x.to_hdf5('myfile.hdf5', '/x') # doctest: +SKIP See Also -------- da.store h5py.File.create_dataset rKrrz/Please provide {'/data/path': array} dictionaryNrr)rc sJg|]B\}}j|f|j|jdur8tdd|jDndqS)Tcss|]}|dVqdSrr_rr_r_r`rerfz%to_hdf5...)rrr)Zrequire_datasetrrrlr)rcZdprdrrrr_r`rszto_hdf5..) rrgrrrrh5pyZFiler1rTrr)rrrrrr$Zdsetsr_r#r`rs$ ( rcCstg}d}}t|t|}|||krl||durL||||d7}q||||d7}|d7}qt|S)zK >>> interleave_none([0, None, 2, None], [1, 3]) (0, 1, 2, 3) rNrK)rr0rl)rrrsr8rrr3r_r_r`interleave_none's    r%cCs||ftdd|DS)zE >>> keyname('x', 3, [None, None, 0, 2]) ('x', 3, 0, 2) css|]}|dur|VqdSrar_rFr_r_r`reArfzkeyname..r)rrokeyr_r_r`keyname;sr'cGs,t|j|}g}g}t|D]Z\}}t|tr:||qt|tr^|||tdq|td||qt|}t|}||}i}tt||j D]r\}\}}t|tst j |dd}|j j dkrtd||k|| kBrtd|||f||;}|||<q|r(t||}|S)a3Point wise indexing with broadcasting. >>> x = np.arange(56).reshape((7, 8)) >>> x array([[ 0, 1, 2, 3, 4, 5, 6, 7], [ 8, 9, 10, 11, 12, 13, 14, 15], [16, 17, 18, 19, 20, 21, 22, 23], [24, 25, 26, 27, 28, 29, 30, 31], [32, 33, 34, 35, 36, 37, 38, 39], [40, 41, 42, 43, 44, 45, 46, 47], [48, 49, 50, 51, 52, 53, 54, 55]]) >>> d = from_array(x, chunks=(3, 4)) >>> result = _vindex(d, [0, 1, 6, 0], [0, 1, 0, 7]) >>> result.compute() array([ 0, 9, 48, 7]) NTr rsz4vindex does not support indexing with boolean arrayszNvindex key has entries out of bounds for indexing along axis %s of size %s: %r)rUrr-rgrr0rhrlrrrprXrrrrm _vindex_array)rdindexesZnonfancy_indexesZreduced_indexesrrZ array_indexesrr_r_r`rDs>          rc sztj|}WnJty\}z2ddd|D}td||WYd}~n d}~00|dj}tt||fddt |j D}| dg|j t |d d|D}d d|j D}d dt||Dt|}t||} d | } t} ttd d|DD]R\} } ddt| D}fddtt| |D}| | t|t|fqddt||j D}|d| rt | fndt|}| rtd| ddDttddt||j D}dd|D}d| d | }i}|D]tD]L\} }tt|jft|t|tttd|f|f|t| <qtfddDfddt t Df|t|d<qtt j!| ||gd| ||j"|j#d}|$||jddSddl%m&}|tt't(|||j"| d}|$||jddS)z+Point wise indexing with only NumPy Arrays.rcss|]}t|jVqdSra)rrrr_r_r`rerfz _vindex_array..zLshape mismatch: indexing arrays could not be broadcast together with shapes Nrcs(g|] }|vr |ndqSra)rrr)lookupr_r`rsz!_vindex_array..cSs g|]}|durt|n|qSrar)rcrr_r_r`rscSsg|]}tttd|qSr)rrrrr_r_r`rrfcSsg|]\}}|dur|qSrar_)rcrrsr_r_r`rrfz vindex-merge-cSsg|]}|dur|qSrar_rr_r_r`rrfcSsg|]\}}t||dqSrr)rcrsrr_r_r`rrfcs$g|]\}\}}|||qSr_r_)rcrrr)bounds2r_r`rscSsg|]\}}|dur|qSrar_rr_r_r`rrfrUrKcSsi|]\}}|r||qSr_r_)rcrrr_r_r`r rfz!_vindex_array..cSs.g|]&\}}|dur$ttt|ndgqSra)rrrrr_r_r`rscSs"g|]}|durtddndqSrarrr_r_r`rrfz vindex-slice-rcsg|]}ttd|qSrU)rrr) per_blockr_r`rrfcsg|]}t|qSr_)r'r)rr&r_r`rrfrdrK)r)rrr))rprrrrrrrrrrrrr _get_axisr)rr-r0rlr/rr1r_vindex_transpose _vindex_slicerr%rr' _vindex_mergerr1rrrbrrrrr)rdZ dict_indexesZbroadcast_indexesrZ shapes_strZbroadcast_shapeZ flat_indexesZboundsrrZout_namepointsrrZ block_idxZ inblock_idxrZ other_blocksZ full_slicesZvindex_merge_namer}rZ result_1drr_)r+r*rr&r,r`r(}s          r(cCs<t|}dd|D}td|}|t|}|jdS)aOGet axis along which point-wise slicing results lie This is mostly a hack because I can't figure out NumPy's rule on this and can't be bothered to go reading. >>> _get_axis([[1, 2], None, [1, 2], None]) 0 >>> _get_axis([None, [1, 2], [1, 2], None]) 1 >>> _get_axis([None, None, [1, 2], [1, 2]]) 2 cSs$g|]}|durtddndgqSrrrr_r_r`rrfz_get_axis..)rrK)rrprrlrr)r)rrdZx2r_r_r`r-s  r-cCsdd|D}|t|S)z%Pull out point-wise slices from blockcSs"g|]}t|tr|nt|qSr_)rgrhr)rcpr_r_r`rrfz!_vindex_slice..r)rr1r_r_r`r/sr/cCs0|gtt|tt|d|j}||S)z6Rotate block so that points are on the first dimensionrK)rrrr)rrrr_r_r`r.s&r.c Csttt|}t|}ttt|}t|dj}||d<t|}|dj}tj|d||d}ddt |j D}t ||D]\}}||d<||t|<q~|S)z >>> locations = [0], [2, 1] >>> values = [np.array([[1, 2, 3]]), ... np.array([[10, 20, 30], [40, 50, 60]])] >>> _vindex_merge(locations, values) array([[ 1, 2, 3], [40, 50, 60], [10, 20, 30]]) r)rrcSsg|]}tddqSrarrr_r_r`rrfz!_vindex_merge..) rrrrrrlrrprrrr) Z locationsrr3rrrdrrrr_r_r`r0s  r0c stfddt|jD}||}tjs>> x = da.ones((5, 10, 10), chunks=(2, 4, 4)) # doctest: +SKIP >>> da.to_npy_stack('data/', x, axis=0) # doctest: +SKIP The ``.npy`` files store numpy arrays for ``x[0:2], x[2:4], and x[4:5]`` respectively, as is specified by the chunk size along the zeroth axis:: $ tree data/ data/ |-- 0.npy |-- 1.npy |-- 2.npy |-- info The ``info`` file stores the dtype, chunks, and axis information of the array. You can load these stacks with the :func:`dask.array.from_npy_stack` function. >>> y = da.from_npy_stack('data/') # doctest: +SKIP See Also -------- from_npy_stack c3s(|] \}}|kr|nt|fVqdSrarWrrr_r`re>rfzto_npy_stack..)rrrr4wbNz to-npy-stack-c s0i|](\}}|ftjtjd||fqSz%d.npy)rpZsaverrr)rcrr)dirnamerr_r`r Jsz to_npy_stack..rd)rlr-rrnrrexistsmkdirropenrpickledumprrrr"rOr>r1rr%rr) r5rdrrxxrrr}rr_)rr5rr` to_npy_stacks   * r<rAc sttjdd}t|}Wdn1s40Y|d}|d}|d}d}tt|ggdd |DR}fd d tt ||D} t t || } t | |||S) zLoad dask array from stack of npy files See :func:`dask.array.to_npy_stack` for docstring. Parameters ---------- dirname: string Directory of .npy files mmap_mode: (None or 'r') Read data in memory map mode r4rbNrrrzfrom-npy-stack-%scSsg|]}tt|qSr_r rr_r_r`rgrfz"from_npy_stack..cs&g|]}tjtjd|fqSr4)rploadrrrrr5 mmap_moder_r`rhs) r8rrrr9r>rrrrrrr) r5r@rr4rrrrrrr}r_r?r`from_npy_stackSs (  rAcCs|t|st|st|rfddlm}tdd|ddDsBJdgt|dd}|||||St|||||dSdS) zGeneric constructor for dask.array or dask.dataframe objects. Decides the appropriate output class based on the type of `meta` provided. r) new_dd_objectcss|]}t|dkVqdSrrrr_r_r`reyrfz new_da_object..rKNr)rrrr)rBrErCZdataframe.corerBrrr)r}rrrrrBZ divisionsr_r_r` new_da_objectqs  rCc@sreZdZdZdddddZddddd Zd d d d dZeddddZeddddZ ddddZ dS)raAn array-like interface to the blocks of an array. ``BlockView`` provides an array-like interface to the blocks of a dask array. Numpy-style indexing of a ``BlockView`` returns a selection of blocks as a new dask array. You can index ``BlockView`` like a numpy array of shape equal to the number of blocks in each dimension, (available as array.blocks.size). The dimensionality of the output array matches the dimension of this array, even if integer indices are passed. Slicing with ``np.newaxis`` or multiple lists is not supported. Examples -------- >>> import dask.array as da >>> from dask.array.core import BlockView >>> x = da.arange(8, chunks=2) >>> bv = BlockView(x) >>> bv.shape # aliases x.numblocks (4,) >>> bv.size 4 >>> bv[0].compute() array([0, 1]) >>> bv[:3].compute() array([0, 1, 2, 3, 4, 5]) >>> bv[::2].compute() array([0, 1, 4, 5]) >>> bv[[-1, 0]].compute() array([6, 7, 0, 1]) >>> bv.ravel() # doctest: +NORMALIZE_WHITESPACE [dask.array, dask.array, dask.array, dask.array] Returns ------- An instance of ``da.array.Blockview`` r)rXrcCs ||_dSra)_array)rrXr_r_r`__init__szBlockView.__init__z-int | Sequence[int] | slice | Sequence[slice])rrcsddlm}t|ts|f}tdd|Ddkr:tdtdd|DrTtd|||jj}tdd|D}d t |j||jj |td dt |jj |D}t d d|D}fd d |D}tj||jgd}t|||jdS)NrK)rcss|]}t|tjtfVqdSra)rgrprrrr_r_r`rerfz(BlockView.__getitem__..z!Can only slice with a single listcss|]}|duVqdSrar_rr_r_r`rerfz0Slicing with np.newaxis or None is not supportedcss*|]"}t|trt||dn|VqdSr)rgrrhrFr_r_r`rerfzblocks-css(|] \}}tt||VqdSra)rlrprXr)rcrurr_r_r`rescss|]}tt|VqdSrar rr_r_r`rerfcs$i|]}f|t|qSr_)rlrrrZnew_keysr_r`r rfz)BlockView.__getitem__..rdrK)rrrgrlrrrmrDrr)rrrrr1rr)rrrrrrrr_rFr`rs$    zBlockView.__getitem__rr)rrcCst|tr|j|juStSdSra)rgrrDrrr_r_r`rs  zBlockView.__eq__rZrcCs t|jS)z: The total number of blocks in the array. )rprrrrr_r_r`rszBlockView.sizeztuple[int, ...]cCs|jjS)zS The number of blocks per axis. Alias of ``dask.array.numblocks``. )rDrrrr_r_r`rszBlockView.shapez list[Array]csfddtjDS)zT Return a flattened list of all the blocks in the array in C order. csg|] }|qSr_r_)rcrrrr_r`rrfz#BlockView.ravel..)rpZndindexrrrr_rrr`rszBlockView.ravelN) r[r\r]r^rErrrrrrr_r_r_r`rs)r)r,r)TN)TN)TN)NN)rN)TNTF)NNNN)N)rlNFNTNNF)NNNNF)NNFNTF)NNN)F)rF)FNN)NN)NN)r)rF)r)rA)NN)Z __future__rrrrrr9rrrrr)rcollections.abcrrrrr r r functoolsr r r itertoolsrrZnumbersrrrr threadingrtypingrZnumpyrpZfsspecrZtlzrrrrrrZ tlz.curriedrrr r!r"r#rr$r%r&r'r(r)r*rr+contextr,r-r0r/Zhighlevelgraphr1r2rr3r4r5rr6r7r8r9r:r;r<r=r>r?r@rArBrCrDrErFrGrHrIZwidgetsrJrLrMZ chunk_typesrNrOrrPrQrRZ numpy_compatrSrTrrUrVrWZupdate_defaultsrYWarningrZrnrxry optimizationrzr{r|rrrrrrrrrrrr9rTr\r`rrkrrurrrrrrrrrrrrrr^rrrrrrPrQrjrrrrrrrrrr rrrrrrrr]r"rr%r'rr(r-r/r.r0r<rArCrr,rr_r_r_r`st  $            X      T  ? H @/.   # } k ? y( "Dc + 1  G  IM##y* K$   qF ; 9a! 4  e