Exceptions

Task: ('shuffle-p2p-86bd619d06d55badca637dddcb6987c5', 10)

Worker(s):

Exception: RuntimeError('shuffle_unpack failed during shuffle d4f2bcb88bd283c797b0c8a2e03a2799')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 86, in shuffle_unpack
    raise RuntimeError(msg)

Task: ('shuffle-p2p-86bd619d06d55badca637dddcb6987c5', 4)

Worker(s):

Exception: RuntimeError('shuffle_unpack failed during shuffle d4f2bcb88bd283c797b0c8a2e03a2799')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 86, in shuffle_unpack
    raise RuntimeError(msg)

Task: ('shuffle-p2p-86bd619d06d55badca637dddcb6987c5', 14)

Worker(s):

Exception: RuntimeError('shuffle_unpack failed during shuffle d4f2bcb88bd283c797b0c8a2e03a2799')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 86, in shuffle_unpack
    raise RuntimeError(msg)

Task: ('shuffle-p2p-86bd619d06d55badca637dddcb6987c5', 5)

Worker(s):

Exception: RuntimeError('shuffle_unpack failed during shuffle d4f2bcb88bd283c797b0c8a2e03a2799')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 86, in shuffle_unpack
    raise RuntimeError(msg)

Task: ('shuffle-p2p-48f700aa684aa26006c15258b8c5382c', 10)

Worker(s):

Exception: RuntimeError('shuffle_unpack failed during shuffle d104ddfcb6f659d4029e4c482e880637')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 86, in shuffle_unpack
    raise RuntimeError(msg)

Task: ('shuffle-p2p-9b0ee56ac071c0b9d77dc56cc7cb3c54', 14)

Worker(s):

Exception: RuntimeError('shuffle_unpack failed during shuffle 3d9e1529a4b4519387e33a6566423ca2')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 86, in shuffle_unpack
    raise RuntimeError(msg)

Task: ('shuffle-p2p-d1ae0c101f210d7c5f8ea8c7dc3cf426', 6)

Worker(s):

Exception: RuntimeError('shuffle_unpack failed during shuffle 8f4adf35dc92d235c8357c5e96d5cfa6')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 86, in shuffle_unpack
    raise RuntimeError(msg)

Task: ('shuffle-p2p-d1ae0c101f210d7c5f8ea8c7dc3cf426', 9)

Worker(s):

Exception: RuntimeError('shuffle_unpack failed during shuffle 8f4adf35dc92d235c8357c5e96d5cfa6')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 86, in shuffle_unpack
    raise RuntimeError(msg)

Task: ('shuffle-p2p-d1ae0c101f210d7c5f8ea8c7dc3cf426', 13)

Worker(s):

Exception: RuntimeError('shuffle_unpack failed during shuffle 8f4adf35dc92d235c8357c5e96d5cfa6')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 86, in shuffle_unpack
    raise RuntimeError(msg)

Task: ('shuffle-p2p-d1ae0c101f210d7c5f8ea8c7dc3cf426', 3)

Worker(s):

Exception: RuntimeError('shuffle_unpack failed during shuffle 8f4adf35dc92d235c8357c5e96d5cfa6')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 86, in shuffle_unpack
    raise RuntimeError(msg)

Task: ('shuffle-p2p-d1ae0c101f210d7c5f8ea8c7dc3cf426', 11)

Worker(s):

Exception: RuntimeError('shuffle_unpack failed during shuffle 8f4adf35dc92d235c8357c5e96d5cfa6')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 86, in shuffle_unpack
    raise RuntimeError(msg)

Task: ('shuffle-p2p-d1ae0c101f210d7c5f8ea8c7dc3cf426', 10)

Worker(s):

Exception: RuntimeError('shuffle_unpack failed during shuffle 8f4adf35dc92d235c8357c5e96d5cfa6')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 86, in shuffle_unpack
    raise RuntimeError(msg)

Task: ('shuffle-p2p-178fa069fdc4e10d687c9f8c1b4055d8', 10)

Worker(s):

Exception: RuntimeError('shuffle_unpack failed during shuffle b643354404bbe7799c8c94f4036bf1af')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 86, in shuffle_unpack
    raise RuntimeError(msg)

Task: ('shuffle-p2p-178fa069fdc4e10d687c9f8c1b4055d8', 4)

Worker(s):

Exception: RuntimeError('shuffle_unpack failed during shuffle b643354404bbe7799c8c94f4036bf1af')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 86, in shuffle_unpack
    raise RuntimeError(msg)

Task: ('shuffle-p2p-9270f973ad2e83734eab55b4347e70bb', 3)

Worker(s):

Exception: RuntimeError('shuffle_unpack failed during shuffle 96a1e233ed5b7bc19a76f4ed3d570b08')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 86, in shuffle_unpack
    raise RuntimeError(msg)

Task: ('assign-2ade601b15327aa8df2aa4173ec1022e', 12)

Worker(s):

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('assign-796376d92aac72905c42944a6382b6d8', 7)

Worker(s):

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('assign-85d164f98716a34ec38e1f7e22494803', 7)

Worker(s):

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('assign-7976d0369d518ba530f0de94fbdd515f', 6)

Worker(s):

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('lambda-fac61fb1513c9ecdae17b261810b86b4', 0)

Worker(s):

Exception: ValueError('Found array with 1 sample(s) (shape=(1, 1)) while a minimum of 2 is required by AgglomerativeClustering.')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/edge_driving_time_metric_worker.py", line 47, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/sklearn/cluster/_agglomerative.py", line 1099, in fit_predict
    return super().fit_predict(X, y)
  File "/opt/conda/lib/python3.10/site-packages/sklearn/base.py", line 753, in fit_predict
    self.fit(X)
  File "/opt/conda/lib/python3.10/site-packages/sklearn/cluster/_agglomerative.py", line 955, in fit
    X = self._validate_data(X, ensure_min_samples=2)
  File "/opt/conda/lib/python3.10/site-packages/sklearn/base.py", line 565, in _validate_data
    X = check_array(X, input_name="X", **check_params)
  File "/opt/conda/lib/python3.10/site-packages/sklearn/utils/validation.py", line 931, in check_array
    raise ValueError(

Task: ('assign-fbbc74300fd18669aa11b97fc58d7a55', 7)

Worker(s):

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('assign-96356a2d6db04b819cb16e4ccb058d2b', 2)

Worker(s):

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('hash-join-503a0712c875f18d8e1f083a31665de2', 14)

Worker(s):

Exception: RuntimeError('Worker tcp://172.21.25.6:38605 left during active shuffle 6a467b9020e603a14a42abd7e57c8fc7')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: ('hash-join-c9bcbabf5e1d277f0382eee9abe4bff6', 10)

Worker(s):

Exception: RuntimeError('Worker tcp://172.21.25.101:36249 left during active shuffle e1f614eaa1c52cb9d9911c29f12ec66e')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: ('hash-join-5c8f964fdea0da526c6019829c1f7484', 10)

Worker(s):

Exception: RuntimeError('Worker tcp://172.21.25.101:46203 left during active shuffle 5ee862de629847ea081687dc1b08a1f6')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: ('hash-join-ac76a017c1999ccc3fa2be6896153632', 10)

Worker(s):

Exception: RuntimeError('Worker tcp://172.21.25.101:32879 left during active shuffle 7e7c8f7fd24ded13a690a4aa92887301')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: ('hash-join-ccaa8528e53722d0f9b96e687eadf4c4', 4)

Worker(s):

Exception: ArrowInvalid('Schema at index 3 was different: \nschema_version: int64\ndetector_id: int64\nsource_id: int64\ntimestamp: int64\ngdt: int64\nsrt: int64\nheading: double\nspeed: double\nevent_type: string\nexterior_lights: string\nlongitude: double\nlatitude: double\naltitude: int64\ngeometry_type: string\nis_day: bool\nhashed_source_and_date: string\nindex: int64\nheading_diff_prev_and_cur: double\nheading_diff_cur_and_next: double\n__hash_partition: uint64\ndate: timestamp[ns]\nvs\nschema_version: int64\ndetector_id: int64\nsource_id: int64\ntimestamp: int64\ngdt: int64\nsrt: int64\nheading: double\nspeed: double\nevent_type: string\nexterior_lights: null\nlongitude: double\nlatitude: double\naltitude: int64\ngeometry_type: string\nis_day: bool\nhashed_source_and_date: string\nindex: int64\nheading_diff_prev_and_cur: double\nheading_diff_cur_and_next: double\n__hash_partition: uint64\ndate: timestamp[ns]')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 903, in get_output_partition
    return sync(self.worker.loop, shuffle.get_output_partition, output_partition)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 529, in get_output_partition
    out = await self.offload(_)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 132, in offload
    return await asyncio.get_running_loop().run_in_executor(
  File "/opt/conda/lib/python3.10/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 526, in _
    df = convert_partition(data)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_arrow.py", line 57, in convert_partition
    return pa.concat_tables(shards)
  File "pyarrow/table.pxi", line 5224, in pyarrow.lib.concat_tables
  File "pyarrow/error.pxi", line 144, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow/error.pxi", line 100, in pyarrow.lib.check_status

Task: ('shuffle-p2p-8c0c543644f0e53db6e051b942a01544', 8)

Worker(s):

Exception: RuntimeError('shuffle_unpack failed during shuffle 9ec91486a56c2c7ef771082ed2149971')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 86, in shuffle_unpack
    raise RuntimeError(msg)

Task: ('shuffle-p2p-8c0c543644f0e53db6e051b942a01544', 7)

Worker(s):

Exception: RuntimeError('shuffle_unpack failed during shuffle 9ec91486a56c2c7ef771082ed2149971')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 86, in shuffle_unpack
    raise RuntimeError(msg)

Task: ('hash-join-4e4149ba6e1885c825102688aec6b7f6', 1)

Worker(s):

Exception: RuntimeError('Worker tcp://172.21.12.165:39893 left during active shuffle 1653fd00984f93ed84f8e029c6ecefc6')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: ('hash-join-c459311cf8eefec35104c4cbdc890dbc', 3)

Worker(s):

Exception: RuntimeError('Worker tcp://172.21.12.165:44285 left during active shuffle 53c4d07b43fb6496174eea92e7316d44')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 176, in merge_unpack
    right = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: ('hash-join-fa0fb355181618f1e99a9941a6568f61', 1)

Worker(s):

Exception: RuntimeError('Worker tcp://172.21.12.165:41041 left during active shuffle c956f7e0429ca6ff4935fd66ff26d898')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: ('hash-join-7f4bc2eddbed6625d454dba445850922', 1)

Worker(s):

Exception: RuntimeError('Worker tcp://172.21.12.165:36527 left during active shuffle 3f2d5724fac4cbaac4184a29ec51985f')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: ('hash-join-transfer-a717b19e4029336aa0178251a7cc1958', 8)

Worker(s):

Exception: RuntimeError('shuffle_transfer failed during shuffle a717b19e4029336aa0178251a7cc1958')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 145, in merge_transfer
    return shuffle_transfer(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 71, in shuffle_transfer
    raise RuntimeError(msg)

Task: shuffle-barrier-2690b3fa24a763d2ad62478112e17d8c

Worker(s):

Exception: RuntimeError('shuffle_barrier failed during shuffle 2690b3fa24a763d2ad62478112e17d8c')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: ('hash-join-7da5c100c2b4ab75ce0a4c70e132b1b0', 2)

Worker(s):

Exception: RuntimeError('Worker tcp://172.21.12.191:43191 left during active shuffle ee85854fb99849da5f75fbca0487f836')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: ('assign-44edc5152ad0b3d2f0ae016b7fb99252', 3)

Worker(s):

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('assign-851b9b049df3dc54c535eb84f533fa0e', 13)

Worker(s):

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('assign-a1ca1211d630b21e5b87038005d3d25a', 3)

Worker(s):

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('assign-4f93a6c6a2cb051e7e62030ff9f8d150', 3)

Worker(s):

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('assign-5d394e4e49343a449512234e2973692d', 3)

Worker(s):

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('assign-7f9d3741f6b65ad9f6b5deabb3451288', 13)

Worker(s):

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('assign-d997eb77bc2a46d555c67f3b7f9dca26', 7)

Worker(s):

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('hash-join-b3ca13d96a793a016aac6927d2e27062', 11)

Worker(s):

Exception: RuntimeError('Worker tcp://172.21.25.102:36113 left during active shuffle 27bad0c1f67d6637d20c98e1d939cd30')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: shuffle-barrier-520f47582da650d63e845a0a5d37f79b

Worker(s):

Exception: RuntimeError('shuffle_barrier failed during shuffle 520f47582da650d63e845a0a5d37f79b')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: run-aa94d0b861f7376f08eaeb58cb14f222

Worker(s):

Exception:

Traceback

Task: shuffle-barrier-28f00e97d23da0fca4b21ae82b15e662

Worker(s):

Exception: RuntimeError('shuffle_barrier failed during shuffle 28f00e97d23da0fca4b21ae82b15e662')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: shuffle-barrier-7f4b41aa11157f657ffa138e7ba1f6c2

Worker(s):

Exception: RuntimeError('shuffle_barrier failed during shuffle 7f4b41aa11157f657ffa138e7ba1f6c2')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: shuffle-barrier-5ee7b335f5ad48bea8bdceb7d566fc19

Worker(s):

Exception: RuntimeError('shuffle_barrier failed during shuffle 5ee7b335f5ad48bea8bdceb7d566fc19')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: shuffle-barrier-a5e30cfa8bb2a924f876a8c4dbb06a59

Worker(s):

Exception: RuntimeError('shuffle_barrier failed during shuffle a5e30cfa8bb2a924f876a8c4dbb06a59')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: ('hash-join-08a42e91366e36177f1ab33259a5c846', 10)

Worker(s):

Exception: RuntimeError('Worker tcp://172.21.25.101:40477 left during active shuffle 98878ffca3ef011147192afec6684507')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: ('assign-a4457a4a9aab7e76cd3af87442776506', 0)

Worker(s):

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('assign-35c2123ae83a937a125cee7c5a38007e', 15)

Worker(s):

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('hash-join-542a811894f139680afbb3af04395fd4', 9)

Worker(s):

Exception: RuntimeError('Worker tcp://172.21.159.252:37885 left during active shuffle fa3ba46b3a37cf9aee29d0c126554108')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: shuffle-barrier-01009e915c7139fa0f31a3c0534664c7

Worker(s):

Exception: RuntimeError('shuffle_barrier failed during shuffle 01009e915c7139fa0f31a3c0534664c7')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: ('hash-join-5428f24842a7450f952c6b486b18f3b5', 9)

Worker(s):

Exception: RuntimeError('Worker tcp://172.21.159.252:41723 left during active shuffle 1e659d087c05ac679886c3e218b98523')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: shuffle-barrier-9c85337d9ebdbf4376cc35340c7ff7ef

Worker(s):

Exception: RuntimeError('shuffle_barrier failed during shuffle 9c85337d9ebdbf4376cc35340c7ff7ef')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: ('hash-join-b5a820aa927a12900a3e0a91d28f511f', 9)

Worker(s):

Exception: RuntimeError('Worker tcp://172.21.159.252:34943 left during active shuffle 4b3c44f6655057858492667cc9221e30')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: ('hash-join-2fc944b7af0b761e66d31aef0ed7f180', 3)

Worker(s):

Exception: RuntimeError('Worker tcp://172.21.159.233:46261 left during active shuffle 24fd7af501fb942d7c27dfa0ff196812')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: shuffle-barrier-accfd2c78b98ab833598e5f444758bf0

Worker(s):

Exception: RuntimeError('shuffle_barrier failed during shuffle accfd2c78b98ab833598e5f444758bf0')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: shuffle-barrier-04f61006fd61c699ae7ef329627796af

Worker(s):

Exception: RuntimeError('shuffle_barrier failed during shuffle 04f61006fd61c699ae7ef329627796af')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: ('hash-join-6f9200bd4e06cbe4d32127091e6d698d', 13)

Worker(s):

Exception: ArrowInvalid('Schema at index 1 was different: \nschema_version: int64\ndetector_id: int64\nsource_id: int64\ntimestamp: int64\ngdt: int64\nsrt: int64\nheading: double\nspeed: double\nevent_type: string\nexterior_lights: null\nlongitude: double\nlatitude: double\naltitude: int64\ngeometry_type: string\nis_day: bool\nhashed_source_and_date: string\nindex: int64\nheading_diff_prev_and_cur: double\nheading_diff_cur_and_next: double\n__hash_partition: uint64\ndate: timestamp[ns]\nvs\nschema_version: int64\ndetector_id: int64\nsource_id: int64\ntimestamp: int64\ngdt: int64\nsrt: int64\nheading: double\nspeed: double\nevent_type: string\nexterior_lights: string\nlongitude: double\nlatitude: double\naltitude: int64\ngeometry_type: string\nis_day: bool\nhashed_source_and_date: string\nindex: int64\nheading_diff_prev_and_cur: double\nheading_diff_cur_and_next: double\n__hash_partition: uint64\ndate: timestamp[ns]')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 903, in get_output_partition
    return sync(self.worker.loop, shuffle.get_output_partition, output_partition)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 529, in get_output_partition
    out = await self.offload(_)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 132, in offload
    return await asyncio.get_running_loop().run_in_executor(
  File "/opt/conda/lib/python3.10/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 526, in _
    df = convert_partition(data)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_arrow.py", line 57, in convert_partition
    return pa.concat_tables(shards)
  File "pyarrow/table.pxi", line 5224, in pyarrow.lib.concat_tables
  File "pyarrow/error.pxi", line 144, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow/error.pxi", line 100, in pyarrow.lib.check_status

Task: shuffle-barrier-5df853716780d1aedcef235c7fd3396a

Worker(s):

Exception: RuntimeError('shuffle_barrier failed during shuffle 5df853716780d1aedcef235c7fd3396a')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: shuffle-barrier-6c742d76f8795f7c6be2ec6d0c086c4c

Worker(s):

Exception: RuntimeError('shuffle_barrier failed during shuffle 6c742d76f8795f7c6be2ec6d0c086c4c')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: shuffle-barrier-8e6aa9ef5ae540effea7b163b4d6bf42

Worker(s):

Exception: RuntimeError('shuffle_barrier failed during shuffle 8e6aa9ef5ae540effea7b163b4d6bf42')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: shuffle-barrier-bdf5d09cede7dcf7cc5c1359073103a7

Worker(s):

Exception: RuntimeError('shuffle_barrier failed during shuffle bdf5d09cede7dcf7cc5c1359073103a7')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: ('hash-join-94374102b41c6d37ee09da60b10f93fd', 4)

Worker(s):

Exception: RuntimeError('Worker tcp://172.21.12.199:39093 left during active shuffle 4623a32eb3e199f35eab3c309ce2c655')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: ('hash-join-aae28ec3b952674bc0aebcf7e2e61033', 4)

Worker(s):

Exception: RuntimeError('Worker tcp://172.21.12.199:46413 left during active shuffle eaca08679c208a99b4015bf3c3a49e09')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: shuffle-barrier-673705ce530f61573981f79df58d282e

Worker(s):

Exception: RuntimeError('shuffle_barrier failed during shuffle 673705ce530f61573981f79df58d282e')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: ('hash-join-18744b1044f6a82f5747bc76419ce5c8', 4)

Worker(s):

Exception: RuntimeError('Worker tcp://172.21.12.199:41897 left during active shuffle aa92d10a14e9c3376a5ba6bb6f120ac5')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: ('hash-join-af9c7f1246728d52a8cbbbc47624d3d0', 4)

Worker(s):

Exception: RuntimeError('Worker tcp://172.21.12.199:44619 left during active shuffle 6a132d8b28e74d6585222c2a8b254b78')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: ('hash-join-cb216b993bac846786e4e6afcc3c4166', 13)

Worker(s):

Exception: RuntimeError('Worker tcp://172.21.25.24:37771 left during active shuffle d74e400756e7a11164e0a63f59a05928')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: shuffle-barrier-d26b6e3ecde3e0acc7af27bb1603df07

Worker(s):

Exception: RuntimeError('shuffle_barrier failed during shuffle d26b6e3ecde3e0acc7af27bb1603df07')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: ('shuffle-transfer-bed18f06c690d0ef051883c90797a2cd', 6)

Worker(s):

Exception: RuntimeError('shuffle_transfer failed during shuffle bed18f06c690d0ef051883c90797a2cd')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 71, in shuffle_transfer
    raise RuntimeError(msg)

Task: ('hash-join-transfer-849559c0082f84220cf3361bcb8dcb7b', 10)

Worker(s):

Exception: RuntimeError('shuffle_transfer failed during shuffle 849559c0082f84220cf3361bcb8dcb7b')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 145, in merge_transfer
    return shuffle_transfer(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 71, in shuffle_transfer
    raise RuntimeError(msg)

Task: shuffle-barrier-da7fec5ecb0608e2668fa35aad021412

Worker(s):

Exception: RuntimeError('shuffle_barrier failed during shuffle da7fec5ecb0608e2668fa35aad021412')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: ('hash-join-e48e996e143f23f365350e0c260cac0a', 9)

Worker(s):

Exception: RuntimeError('Worker tcp://172.21.159.252:38069 left during active shuffle d2bfd1b8cc73f1263b68a849f6eebdbe')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: ('hash-join-transfer-172949d1a63c679eeac0e91f99849060', 10)

Worker(s):

Exception: RuntimeError('shuffle_transfer failed during shuffle 172949d1a63c679eeac0e91f99849060')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 145, in merge_transfer
    return shuffle_transfer(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 71, in shuffle_transfer
    raise RuntimeError(msg)

Task: ('hash-join-transfer-d53c96a78ff735c24f4732c5cd2eb82f', 8)

Worker(s):

Exception: RuntimeError('shuffle_transfer failed during shuffle d53c96a78ff735c24f4732c5cd2eb82f')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 145, in merge_transfer
    return shuffle_transfer(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 71, in shuffle_transfer
    raise RuntimeError(msg)

Task: ('hash-join-330856b8cc6b0cf704dd953710d8345f', 14)

Worker(s):

Exception: RuntimeError('Worker tcp://172.21.25.6:45767 left during active shuffle 6cea3e1345b31246120a47cc45a8fc4b')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: ('hash-join-d29798391e160f012d89371411070a6b', 3)

Worker(s):

Exception: RuntimeError('Worker tcp://172.21.12.197:46281 left during active shuffle c2fb2e153c38d37558902f98f31ac671')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: ('hash-join-72ca15c1a8fef4719ebd4d2d63e06a53', 3)

Worker(s):

Exception: RuntimeError('Worker tcp://172.21.12.197:38741 left during active shuffle ff3cc01d0643e2e88537ebf16323fdb8')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: ('hash-join-50121078faf36b39015db5adb84841a2', 3)

Worker(s):

Exception: RuntimeError('Worker tcp://172.21.12.197:39491 left during active shuffle 3a1eaf2d37ba92359ba144166fd3d7cb')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: ('hash-join-93663b22a20b1fa2792d886b9b9d5ab5', 3)

Worker(s):

Exception: RuntimeError('Worker tcp://172.21.12.197:44031 left during active shuffle 2faa77d685b63a1e2fde75c33ef3a741')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: ('hash-join-7d9bebf36a6b1cf6fb16788549c9e06b', 2)

Worker(s):

Exception: RuntimeError('Worker tcp://172.21.12.191:43707 left during active shuffle 744c8c32cc1979948949f48d1073244f')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: ('hash-join-3b6e0477a0f9387b2df373d3b51dc089', 3)

Worker(s):

Exception: RuntimeError('Worker tcp://172.21.12.197:42943 left during active shuffle 99c3b74af73741f952a04a4b8730fa62')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: ('lambda-2d65dc7ce5849b87330df0d9465fe480', 13)

Worker(s):

Exception: ValueError('Found array with 1 sample(s) (shape=(1, 1)) while a minimum of 2 is required by AgglomerativeClustering.')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/special_vehicle_stop_metric_worker.py", line 48, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/sklearn/cluster/_agglomerative.py", line 1099, in fit_predict
    return super().fit_predict(X, y)
  File "/opt/conda/lib/python3.10/site-packages/sklearn/base.py", line 753, in fit_predict
    self.fit(X)
  File "/opt/conda/lib/python3.10/site-packages/sklearn/cluster/_agglomerative.py", line 955, in fit
    X = self._validate_data(X, ensure_min_samples=2)
  File "/opt/conda/lib/python3.10/site-packages/sklearn/base.py", line 565, in _validate_data
    X = check_array(X, input_name="X", **check_params)
  File "/opt/conda/lib/python3.10/site-packages/sklearn/utils/validation.py", line 931, in check_array
    raise ValueError(

Task: shuffle-barrier-711f47cc615828847e9ac68c161861fe

Worker(s):

Exception: RuntimeError('shuffle_barrier failed during shuffle 711f47cc615828847e9ac68c161861fe')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: shuffle-barrier-3e4516dc4c05066cb12c88993a579238

Worker(s):

Exception: RuntimeError('shuffle_barrier failed during shuffle 3e4516dc4c05066cb12c88993a579238')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: shuffle-barrier-176d1b411cdc1a86207751762410511c

Worker(s):

Exception: RuntimeError('shuffle_barrier failed during shuffle 176d1b411cdc1a86207751762410511c')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: shuffle-barrier-c1e1bcf833a1c27777403625338086a7

Worker(s):

Exception: RuntimeError('shuffle_barrier failed during shuffle c1e1bcf833a1c27777403625338086a7')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: ('hash-join-0bb79c8cee3def35eabc37a5aea85427', 6)

Worker(s):

Exception: RuntimeError('Worker tcp://172.21.159.210:43399 left during active shuffle 1f1c8d224c96d56d8aaeb1a72638da51')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: ('hash-join-cfef60d881a6e14546dff2f16a8b3342', 1)

Worker(s):

Exception: ArrowInvalid('Schema at index 1 was different: \nschema_version: int64\ndetector_id: int64\nsource_id: int64\ndate: timestamp[ns]\ntimestamp: int64\ngdt: int64\nsrt: int64\nheading: double\nspeed: double\nevent_type: string\nexterior_lights: null\nlongitude: double\nlatitude: double\naltitude: int64\ngeometry_type: string\nis_day: bool\nhashed_source_and_date: string\n__hash_partition: uint64\n__index_level_0__: int64\nvs\nschema_version: int64\ndetector_id: int64\nsource_id: int64\ndate: timestamp[ns]\ntimestamp: int64\ngdt: int64\nsrt: int64\nheading: double\nspeed: double\nevent_type: string\nexterior_lights: string\nlongitude: double\nlatitude: double\naltitude: int64\ngeometry_type: string\nis_day: bool\nhashed_source_and_date: string\n__hash_partition: uint64\n__index_level_0__: int64')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 903, in get_output_partition
    return sync(self.worker.loop, shuffle.get_output_partition, output_partition)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 529, in get_output_partition
    out = await self.offload(_)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 132, in offload
    return await asyncio.get_running_loop().run_in_executor(
  File "/opt/conda/lib/python3.10/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 526, in _
    df = convert_partition(data)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_arrow.py", line 57, in convert_partition
    return pa.concat_tables(shards)
  File "pyarrow/table.pxi", line 5224, in pyarrow.lib.concat_tables
  File "pyarrow/error.pxi", line 144, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow/error.pxi", line 100, in pyarrow.lib.check_status

Task: ('hash-join-cfef60d881a6e14546dff2f16a8b3342', 1)

Worker(s):

Exception: ArrowInvalid('Schema at index 1 was different: \nschema_version: int64\ndetector_id: int64\nsource_id: int64\ndate: timestamp[ns]\ntimestamp: int64\ngdt: int64\nsrt: int64\nheading: double\nspeed: double\nevent_type: string\nexterior_lights: string\nlongitude: double\nlatitude: double\naltitude: int64\ngeometry_type: string\nis_day: bool\nhashed_source_and_date: string\n__hash_partition: uint64\n__index_level_0__: int64\nvs\nschema_version: int64\ndetector_id: int64\nsource_id: int64\ndate: timestamp[ns]\ntimestamp: int64\ngdt: int64\nsrt: int64\nheading: double\nspeed: double\nevent_type: string\nexterior_lights: null\nlongitude: double\nlatitude: double\naltitude: int64\ngeometry_type: string\nis_day: bool\nhashed_source_and_date: string\n__hash_partition: uint64\n__index_level_0__: int64')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 903, in get_output_partition
    return sync(self.worker.loop, shuffle.get_output_partition, output_partition)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 529, in get_output_partition
    out = await self.offload(_)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 132, in offload
    return await asyncio.get_running_loop().run_in_executor(
  File "/opt/conda/lib/python3.10/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 526, in _
    df = convert_partition(data)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_arrow.py", line 57, in convert_partition
    return pa.concat_tables(shards)
  File "pyarrow/table.pxi", line 5224, in pyarrow.lib.concat_tables
  File "pyarrow/error.pxi", line 144, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow/error.pxi", line 100, in pyarrow.lib.check_status

Task: ('hash-join-cfef60d881a6e14546dff2f16a8b3342', 1)

Worker(s):

Exception: ArrowInvalid('Schema at index 1 was different: \nschema_version: int64\ndetector_id: int64\nsource_id: int64\ndate: timestamp[ns]\ntimestamp: int64\ngdt: int64\nsrt: int64\nheading: double\nspeed: double\nevent_type: string\nexterior_lights: null\nlongitude: double\nlatitude: double\naltitude: int64\ngeometry_type: string\nis_day: bool\nhashed_source_and_date: string\n__hash_partition: uint64\n__index_level_0__: int64\nvs\nschema_version: int64\ndetector_id: int64\nsource_id: int64\ndate: timestamp[ns]\ntimestamp: int64\ngdt: int64\nsrt: int64\nheading: double\nspeed: double\nevent_type: string\nexterior_lights: string\nlongitude: double\nlatitude: double\naltitude: int64\ngeometry_type: string\nis_day: bool\nhashed_source_and_date: string\n__hash_partition: uint64\n__index_level_0__: int64')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 903, in get_output_partition
    return sync(self.worker.loop, shuffle.get_output_partition, output_partition)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 529, in get_output_partition
    out = await self.offload(_)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 132, in offload
    return await asyncio.get_running_loop().run_in_executor(
  File "/opt/conda/lib/python3.10/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 526, in _
    df = convert_partition(data)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_arrow.py", line 57, in convert_partition
    return pa.concat_tables(shards)
  File "pyarrow/table.pxi", line 5224, in pyarrow.lib.concat_tables
  File "pyarrow/error.pxi", line 144, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow/error.pxi", line 100, in pyarrow.lib.check_status

Task: ('hash-join-cfef60d881a6e14546dff2f16a8b3342', 1)

Worker(s):

Exception: ArrowInvalid('Schema at index 1 was different: \nschema_version: int64\ndetector_id: int64\nsource_id: int64\ndate: timestamp[ns]\ntimestamp: int64\ngdt: int64\nsrt: int64\nheading: double\nspeed: double\nevent_type: string\nexterior_lights: null\nlongitude: double\nlatitude: double\naltitude: int64\ngeometry_type: string\nis_day: bool\nhashed_source_and_date: string\n__hash_partition: uint64\n__index_level_0__: int64\nvs\nschema_version: int64\ndetector_id: int64\nsource_id: int64\ndate: timestamp[ns]\ntimestamp: int64\ngdt: int64\nsrt: int64\nheading: double\nspeed: double\nevent_type: string\nexterior_lights: string\nlongitude: double\nlatitude: double\naltitude: int64\ngeometry_type: string\nis_day: bool\nhashed_source_and_date: string\n__hash_partition: uint64\n__index_level_0__: int64')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 903, in get_output_partition
    return sync(self.worker.loop, shuffle.get_output_partition, output_partition)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 529, in get_output_partition
    out = await self.offload(_)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 132, in offload
    return await asyncio.get_running_loop().run_in_executor(
  File "/opt/conda/lib/python3.10/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 526, in _
    df = convert_partition(data)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_arrow.py", line 57, in convert_partition
    return pa.concat_tables(shards)
  File "pyarrow/table.pxi", line 5224, in pyarrow.lib.concat_tables
  File "pyarrow/error.pxi", line 144, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow/error.pxi", line 100, in pyarrow.lib.check_status

Task: ('hash-join-cfef60d881a6e14546dff2f16a8b3342', 1)

Worker(s):

Exception: ArrowInvalid('Schema at index 1 was different: \nschema_version: int64\ndetector_id: int64\nsource_id: int64\ndate: timestamp[ns]\ntimestamp: int64\ngdt: int64\nsrt: int64\nheading: double\nspeed: double\nevent_type: string\nexterior_lights: string\nlongitude: double\nlatitude: double\naltitude: int64\ngeometry_type: string\nis_day: bool\nhashed_source_and_date: string\n__hash_partition: uint64\n__index_level_0__: int64\nvs\nschema_version: int64\ndetector_id: int64\nsource_id: int64\ndate: timestamp[ns]\ntimestamp: int64\ngdt: int64\nsrt: int64\nheading: double\nspeed: double\nevent_type: string\nexterior_lights: null\nlongitude: double\nlatitude: double\naltitude: int64\ngeometry_type: string\nis_day: bool\nhashed_source_and_date: string\n__hash_partition: uint64\n__index_level_0__: int64')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 903, in get_output_partition
    return sync(self.worker.loop, shuffle.get_output_partition, output_partition)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 529, in get_output_partition
    out = await self.offload(_)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 132, in offload
    return await asyncio.get_running_loop().run_in_executor(
  File "/opt/conda/lib/python3.10/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 526, in _
    df = convert_partition(data)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_arrow.py", line 57, in convert_partition
    return pa.concat_tables(shards)
  File "pyarrow/table.pxi", line 5224, in pyarrow.lib.concat_tables
  File "pyarrow/error.pxi", line 144, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow/error.pxi", line 100, in pyarrow.lib.check_status

Task: ('assign-5ba8908945c2e48dfa88ba16ebabb159', 7)

Worker(s):

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('assign-5ee13ad14ce9a575efff9b80d68193b2', 0)

Worker(s):

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('assign-ce819f05615dc55da714c34d3049328a', 0)

Worker(s):

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback 
  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)