Exceptions

Task: ('assign-614862eb877a3147fa3bbe0140397e40', 13)

Worker(s):

tcp://172.21.12.244:45931

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('assign-2b4fe7e1e8495ba40104b3a03d635a65', 1)

Worker(s):

tcp://172.21.151.165:40415

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('assign-1f48eafd0194490336db83dcf3900cc4', 1)

Worker(s):

tcp://172.21.151.165:40415

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('assign-40771b150f9d93cf6cd28d980990ead3', 13)

Worker(s):

tcp://172.21.159.254:33329

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('shuffle-transfer-ea0a90b691c3e303b96e36db26de1837', 12)

Worker(s):

tcp://172.21.25.68:33971

Exception: RuntimeError('shuffle_transfer failed during shuffle ea0a90b691c3e303b96e36db26de1837')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 71, in shuffle_transfer
    raise RuntimeError(msg)

Task: ('shuffle-transfer-dd0235e2b443afff522d4ae5c408aeda', 15)

Worker(s):

tcp://172.21.25.62:44951

Exception: RuntimeError('shuffle_transfer failed during shuffle dd0235e2b443afff522d4ae5c408aeda')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 71, in shuffle_transfer
    raise RuntimeError(msg)

Task: ('hash-join-transfer-f8e59ce3dd3848ae2f76070607d62973', 0)

Worker(s):

tcp://172.21.12.242:41381

Exception: RuntimeError('shuffle_transfer failed during shuffle f8e59ce3dd3848ae2f76070607d62973')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 145, in merge_transfer
    return shuffle_transfer(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 71, in shuffle_transfer
    raise RuntimeError(msg)

Task: ('shuffle-transfer-40c48032c2a8567c1cb703aa3460d1c2', 13)

Worker(s):

tcp://172.21.12.247:34539

Exception: RuntimeError('shuffle_transfer failed during shuffle 40c48032c2a8567c1cb703aa3460d1c2')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 71, in shuffle_transfer
    raise RuntimeError(msg)

Task: ('shuffle-transfer-19b050aaaf35e9e0332052c16d1adfa7', 10)

Worker(s):

tcp://172.21.12.247:34539

Exception: RuntimeError('shuffle_transfer failed during shuffle 19b050aaaf35e9e0332052c16d1adfa7')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 71, in shuffle_transfer
    raise RuntimeError(msg)

Task: ('lambda-6c0f5861c2467ebeef4630a613755e15', 13)

Worker(s):

tcp://172.21.159.211:41463

Exception: ValueError('Found array with 1 sample(s) (shape=(1, 1)) while a minimum of 2 is required by AgglomerativeClustering.')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/special_vehicle_stop_metric_worker.py", line 48, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/sklearn/cluster/_agglomerative.py", line 1099, in fit_predict
    return super().fit_predict(X, y)
  File "/opt/conda/lib/python3.10/site-packages/sklearn/base.py", line 753, in fit_predict
    self.fit(X)
  File "/opt/conda/lib/python3.10/site-packages/sklearn/cluster/_agglomerative.py", line 955, in fit
    X = self._validate_data(X, ensure_min_samples=2)
  File "/opt/conda/lib/python3.10/site-packages/sklearn/base.py", line 565, in _validate_data
    X = check_array(X, input_name="X", **check_params)
  File "/opt/conda/lib/python3.10/site-packages/sklearn/utils/validation.py", line 931, in check_array
    raise ValueError(

Task: ('hash-join-e286c972f21d9925673d10a728b4b70c', 12)

Worker(s):

tcp://172.21.151.165:40415

Exception: RuntimeError('Worker tcp://172.21.25.37:44341 left during active shuffle 007491f1c91dcc736549d14c5ba9c6fb')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: shuffle-barrier-54b9c2a97c6027c584d9cbbe910240a6

Worker(s):

tcp://172.21.12.242:41381

Exception: RuntimeError('shuffle_barrier failed during shuffle 54b9c2a97c6027c584d9cbbe910240a6')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: shuffle-barrier-75ac88329e3fce621aa4985cad5595f2

Worker(s):

tcp://172.21.12.242:41381

Exception: RuntimeError('shuffle_barrier failed during shuffle 75ac88329e3fce621aa4985cad5595f2')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: shuffle-barrier-5a497579a30c9658a4d67f6581a2f0a5

Worker(s):

tcp://172.21.25.68:33971

Exception: RuntimeError('shuffle_barrier failed during shuffle 5a497579a30c9658a4d67f6581a2f0a5')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: shuffle-barrier-532c3502a7cc6fe65633f4948f3996bb

Worker(s):

tcp://172.21.25.37:44341

Exception: RuntimeError('shuffle_barrier failed during shuffle 532c3502a7cc6fe65633f4948f3996bb')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: ('assign-c50bf8ad38cc8d0603b9f4d391bcd98c', 12)

Worker(s):

tcp://172.21.151.188:37283

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('assign-3a432aa46fcff6a8a79ca3e5b7fdefcd', 9)

Worker(s):

tcp://172.21.159.225:42711

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('assign-b6b82d33da70aed296cd127e77ccd7cd', 9)

Worker(s):

tcp://172.21.159.225:42711

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('assign-4d092cfcbcdc03d5275e9c9561b8f02b', 12)

Worker(s):

tcp://172.21.25.92:33649

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('assign-16d26042a00fc441423ad71accc306e5', 7)

Worker(s):

tcp://172.21.12.242:41381

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('assign-1a22242dd115d5ec668a08d102f51969', 3)

Worker(s):

tcp://172.21.25.37:44341

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('assign-fd97c2e46edf8891111972010e6a3be8', 7)

Worker(s):

tcp://172.21.25.37:44341

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('hash-join-b275930b26b3393e7034ee303146b7d4', 1)

Worker(s):

tcp://172.21.151.159:35235

Exception: RuntimeError('Worker tcp://172.21.12.244:45097 left during active shuffle 3ce9a0f1972060724bb499a59f41a827')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: ('shuffle-transfer-7ed48425efbc55e5015c886b2036fc5b', 1)

Worker(s):

tcp://172.21.151.159:35235

Exception: RuntimeError('shuffle_transfer failed during shuffle 7ed48425efbc55e5015c886b2036fc5b')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 71, in shuffle_transfer
    raise RuntimeError(msg)

Task: ('shuffle-transfer-ebc8095d3f53a53e1bc1d381943a9194', 1)

Worker(s):

tcp://172.21.151.165:40415

Exception: RuntimeError('shuffle_transfer failed during shuffle ebc8095d3f53a53e1bc1d381943a9194')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 71, in shuffle_transfer
    raise RuntimeError(msg)

Task: ('shuffle-transfer-7824cbfb7b72b8fc4e08526fd250e853', 1)

Worker(s):

tcp://172.21.159.225:42711

Exception: RuntimeError('shuffle_transfer failed during shuffle 7824cbfb7b72b8fc4e08526fd250e853')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 71, in shuffle_transfer
    raise RuntimeError(msg)

Task: ('hash-join-transfer-e0ab3fdc821ed28eee64f3629f35cd59', 1)

Worker(s):

tcp://172.21.12.244:45097

Exception: RuntimeError('shuffle_transfer failed during shuffle e0ab3fdc821ed28eee64f3629f35cd59')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 145, in merge_transfer
    return shuffle_transfer(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 71, in shuffle_transfer
    raise RuntimeError(msg)

Task: ('shuffle-p2p-9291c3c1493baaccec61f91e0e6dda53', 14)

Worker(s):

tcp://172.21.151.159:35235

Exception: RuntimeError('shuffle_unpack failed during shuffle 354fc7a0ba20a550dfa9a53f19c331e4')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 86, in shuffle_unpack
    raise RuntimeError(msg)

Task: ('assign-403859437dea4dca40afa9f359994048', 7)

Worker(s):

tcp://172.21.12.244:45097

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('lambda-b0b81a77da451c9730d5a8fbc98aa503', 0)

Worker(s):

tcp://172.21.12.247:33153

Exception: ValueError('Found array with 1 sample(s) (shape=(1, 1)) while a minimum of 2 is required by AgglomerativeClustering.')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/edge_driving_time_metric_worker.py", line 47, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/sklearn/cluster/_agglomerative.py", line 1099, in fit_predict
    return super().fit_predict(X, y)
  File "/opt/conda/lib/python3.10/site-packages/sklearn/base.py", line 753, in fit_predict
    self.fit(X)
  File "/opt/conda/lib/python3.10/site-packages/sklearn/cluster/_agglomerative.py", line 955, in fit
    X = self._validate_data(X, ensure_min_samples=2)
  File "/opt/conda/lib/python3.10/site-packages/sklearn/base.py", line 565, in _validate_data
    X = check_array(X, input_name="X", **check_params)
  File "/opt/conda/lib/python3.10/site-packages/sklearn/utils/validation.py", line 931, in check_array
    raise ValueError(

Task: shuffle-barrier-5ea19e8ed0835e1767f3d466556b87eb

Worker(s):

tcp://172.21.12.242:41381

Exception: RuntimeError('shuffle_barrier failed during shuffle 5ea19e8ed0835e1767f3d466556b87eb')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: shuffle-barrier-e4d9e92a361ec89a00d53a070b22c840

Worker(s):

tcp://172.21.12.242:41381

Exception: RuntimeError('shuffle_barrier failed during shuffle e4d9e92a361ec89a00d53a070b22c840')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: ('hash-join-faa12ab6bc70a2df5a66c1930f78c280', 4)

Worker(s):

tcp://172.21.25.68:32987

Exception: RuntimeError('Worker tcp://172.21.151.136:46017 left during active shuffle 2491b23d228317b970a6df3814e500cc')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: shuffle-barrier-23ec8e5079772a5917b0d27dac7535a5

Worker(s):

tcp://172.21.12.242:41381

Exception: RuntimeError('shuffle_barrier failed during shuffle 23ec8e5079772a5917b0d27dac7535a5')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: shuffle-barrier-ab8e8ea9bf2e4e0be0d66dd1f30e9997

Worker(s):

tcp://172.21.159.225:42711

Exception: RuntimeError('shuffle_barrier failed during shuffle ab8e8ea9bf2e4e0be0d66dd1f30e9997')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: shuffle-barrier-7c52d3d0d8538952953be808b5865164

Worker(s):

tcp://172.21.25.62:44951

Exception: RuntimeError('shuffle_barrier failed during shuffle 7c52d3d0d8538952953be808b5865164')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: ('assign-c12e221df6ec34ebb97539af6024fcd8', 0)

Worker(s):

tcp://172.21.159.228:36567

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('assign-5e0b6d80ad051794189fc21b7c95b2e9', 2)

Worker(s):

tcp://172.21.25.62:44951

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('assign-8acd5c3e7499411a28a7c179e4e2c5ae', 2)

Worker(s):

tcp://172.21.12.250:35843

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: shuffle-barrier-d7aeb8c66a738d8bbce53053563cebbb

Worker(s):

tcp://172.21.12.242:41381

Exception: RuntimeError('shuffle_barrier failed during shuffle d7aeb8c66a738d8bbce53053563cebbb')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: shuffle-barrier-55f01372aa03c81debc1170522181d2f

Worker(s):

tcp://172.21.12.242:41381

Exception: RuntimeError('shuffle_barrier failed during shuffle 55f01372aa03c81debc1170522181d2f')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: shuffle-barrier-fdfe932b91c54d95899e6ec92d671def

Worker(s):

tcp://172.21.12.242:41381

Exception: RuntimeError('shuffle_barrier failed during shuffle fdfe932b91c54d95899e6ec92d671def')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: shuffle-barrier-b0ecc915a1a8910e42b90cce4a505ac4

Worker(s):

tcp://172.21.12.242:41381

Exception: RuntimeError('shuffle_barrier failed during shuffle b0ecc915a1a8910e42b90cce4a505ac4')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: shuffle-barrier-53e266817263c3464d1e8b6236c652e9

Worker(s):

tcp://172.21.159.211:33779

Exception: RuntimeError('shuffle_barrier failed during shuffle 53e266817263c3464d1e8b6236c652e9')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: shuffle-barrier-fc9d8cf104f5e7272aca35981ee0f338

Worker(s):

tcp://172.21.12.250:35843

Exception: RuntimeError('shuffle_barrier failed during shuffle fc9d8cf104f5e7272aca35981ee0f338')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: ('assign-0c25682cf86e8375a42d3fc19d6a9d08', 0)

Worker(s):

tcp://172.21.151.136:38831

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('assign-628f434b9fd60adee0e9d8802f24e506', 8)

Worker(s):

tcp://172.21.151.159:35235

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('assign-f90cf42327d2c4a513b58d85bfc751c1', 9)

Worker(s):

tcp://172.21.25.37:44341

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('hash-join-6ecf3f6b6bbf2dcd44736aa9f15af56e', 3)

Worker(s):

tcp://172.21.12.250:35843

Exception: ArrowInvalid('Schema at index 1 was different: \nschema_version: int64\ndetector_id: int64\nsource_id: int64\ndate: timestamp[ns]\ntimestamp: int64\ngdt: int64\nsrt: int64\nheading: double\nspeed: double\nevent_type: string\nexterior_lights: string\nlongitude: double\nlatitude: double\naltitude: int64\ngeometry_type: string\nis_day: bool\nhashed_source_and_date: string\n__hash_partition: uint64\n__index_level_0__: int64\nvs\nschema_version: int64\ndetector_id: int64\nsource_id: int64\ndate: timestamp[ns]\ntimestamp: int64\ngdt: int64\nsrt: int64\nheading: double\nspeed: double\nevent_type: string\nexterior_lights: null\nlongitude: double\nlatitude: double\naltitude: int64\ngeometry_type: string\nis_day: bool\nhashed_source_and_date: string\n__hash_partition: uint64\n__index_level_0__: int64')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 903, in get_output_partition
    return sync(self.worker.loop, shuffle.get_output_partition, output_partition)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 529, in get_output_partition
    out = await self.offload(_)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 132, in offload
    return await asyncio.get_running_loop().run_in_executor(
  File "/opt/conda/lib/python3.10/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 526, in _
    df = convert_partition(data)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_arrow.py", line 57, in convert_partition
    return pa.concat_tables(shards)
  File "pyarrow/table.pxi", line 5224, in pyarrow.lib.concat_tables
  File "pyarrow/error.pxi", line 144, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow/error.pxi", line 100, in pyarrow.lib.check_status

Task: ('hash-join-6ecf3f6b6bbf2dcd44736aa9f15af56e', 3)

Worker(s):

tcp://172.21.12.250:35843

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 903, in get_output_partition
    return sync(self.worker.loop, shuffle.get_output_partition, output_partition)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 529, in get_output_partition
    out = await self.offload(_)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 132, in offload
    return await asyncio.get_running_loop().run_in_executor(
  File "/opt/conda/lib/python3.10/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 526, in _
    df = convert_partition(data)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_arrow.py", line 57, in convert_partition
    return pa.concat_tables(shards)
  File "pyarrow/table.pxi", line 5224, in pyarrow.lib.concat_tables
  File "pyarrow/error.pxi", line 144, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow/error.pxi", line 100, in pyarrow.lib.check_status

Task: ('hash-join-6ecf3f6b6bbf2dcd44736aa9f15af56e', 3)

Worker(s):

tcp://172.21.12.250:35843

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 903, in get_output_partition
    return sync(self.worker.loop, shuffle.get_output_partition, output_partition)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 529, in get_output_partition
    out = await self.offload(_)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 132, in offload
    return await asyncio.get_running_loop().run_in_executor(
  File "/opt/conda/lib/python3.10/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 526, in _
    df = convert_partition(data)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_arrow.py", line 57, in convert_partition
    return pa.concat_tables(shards)
  File "pyarrow/table.pxi", line 5224, in pyarrow.lib.concat_tables
  File "pyarrow/error.pxi", line 144, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow/error.pxi", line 100, in pyarrow.lib.check_status

Task: shuffle-barrier-fe06b586dfce930cd1b658299dc67b10

Worker(s):

tcp://172.21.12.242:41381

Exception: RuntimeError('shuffle_barrier failed during shuffle fe06b586dfce930cd1b658299dc67b10')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: shuffle-barrier-8f462bddb5ec7e824950eb2e1ac2b8c1

Worker(s):

tcp://172.21.12.242:41381

Exception: RuntimeError('shuffle_barrier failed during shuffle 8f462bddb5ec7e824950eb2e1ac2b8c1')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: ('hash-join-d3d3de609917719a421bb797b7c7005c', 6)

Worker(s):

tcp://172.21.151.136:38831

Exception: RuntimeError('Worker tcp://172.21.151.165:37763 left during active shuffle 319c238f91df853e5799c58392a3d7f5')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: shuffle-barrier-4d586856617ee5ce730c1546cd326eb5

Worker(s):

tcp://172.21.12.242:41381

Exception: RuntimeError('shuffle_barrier failed during shuffle 4d586856617ee5ce730c1546cd326eb5')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: shuffle-barrier-10e71ebdaa6717842c946bc0ad63e876

Worker(s):

tcp://172.21.151.136:38831

Exception: RuntimeError('shuffle_barrier failed during shuffle 10e71ebdaa6717842c946bc0ad63e876')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: shuffle-barrier-65addc0b168fbcef73dc352078766523

Worker(s):

tcp://172.21.151.165:39925

Exception: RuntimeError('shuffle_barrier failed during shuffle 65addc0b168fbcef73dc352078766523')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: shuffle-barrier-7518eea2542f0e2cb57d7675f1e12290

Worker(s):

tcp://172.21.12.244:45097

Exception: RuntimeError('shuffle_barrier failed during shuffle 7518eea2542f0e2cb57d7675f1e12290')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: shuffle-barrier-3d0a8f8f5a11912cba88bbfbf44751be

Worker(s):

tcp://172.21.12.242:33997

Exception: RuntimeError('shuffle_barrier failed during shuffle 3d0a8f8f5a11912cba88bbfbf44751be')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: shuffle-barrier-60c35d0ea4443c09c2d8d3d7bba99684

Worker(s):

tcp://172.21.12.242:33997

Exception: RuntimeError('shuffle_barrier failed during shuffle 60c35d0ea4443c09c2d8d3d7bba99684')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: shuffle-barrier-8de2f05ec5f3165cd47803d97814cda0

Worker(s):

tcp://172.21.12.242:33997

Exception: RuntimeError('shuffle_barrier failed during shuffle 8de2f05ec5f3165cd47803d97814cda0')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: ('hash-join-d707193e9f85b7ea1d92ed9f214b8c6e', 3)

Worker(s):

tcp://172.21.12.250:35843

Exception: ArrowInvalid('Schema at index 11 was different: \nschema_version: int64\ndetector_id: int64\nsource_id: int64\ntimestamp: int64\ngdt: int64\nsrt: int64\nheading: double\nspeed: double\nevent_type: string\nexterior_lights: string\nlongitude: double\nlatitude: double\naltitude: int64\ngeometry_type: string\nis_day: bool\nhashed_source_and_date: string\nindex: int64\nheading_diff_prev_and_cur: double\nheading_diff_cur_and_next: double\n__hash_partition: uint64\ndate: timestamp[ns]\nvs\nschema_version: int64\ndetector_id: int64\nsource_id: int64\ntimestamp: int64\ngdt: int64\nsrt: int64\nheading: double\nspeed: double\nevent_type: string\nexterior_lights: null\nlongitude: double\nlatitude: double\naltitude: int64\ngeometry_type: string\nis_day: bool\nhashed_source_and_date: string\nindex: int64\nheading_diff_prev_and_cur: double\nheading_diff_cur_and_next: double\n__hash_partition: uint64\ndate: timestamp[ns]')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 903, in get_output_partition
    return sync(self.worker.loop, shuffle.get_output_partition, output_partition)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 529, in get_output_partition
    out = await self.offload(_)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 132, in offload
    return await asyncio.get_running_loop().run_in_executor(
  File "/opt/conda/lib/python3.10/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 526, in _
    df = convert_partition(data)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_arrow.py", line 57, in convert_partition
    return pa.concat_tables(shards)
  File "pyarrow/table.pxi", line 5224, in pyarrow.lib.concat_tables
  File "pyarrow/error.pxi", line 144, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow/error.pxi", line 100, in pyarrow.lib.check_status

Task: shuffle-barrier-d5e03160a9793cb8edc1822fb4edcfcf

Worker(s):

tcp://172.21.159.254:33329

Exception: RuntimeError('shuffle_barrier failed during shuffle d5e03160a9793cb8edc1822fb4edcfcf')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: ('hash-join-2d0a7852f9f9f972c32d178b0cf71672', 13)

Worker(s):

tcp://172.21.12.244:45097

Exception: RuntimeError('Worker tcp://172.21.25.62:42429 left during active shuffle 8727f5e5328b17a198eaddff831498be')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: shuffle-barrier-059e40fcdaa61756cf3f0531d22e8e31

Worker(s):

tcp://172.21.12.242:33997

Exception: RuntimeError('shuffle_barrier failed during shuffle 059e40fcdaa61756cf3f0531d22e8e31')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: ('hash-join-7c445614942b6a8613ee0d9843697639', 13)

Worker(s):

tcp://172.21.12.244:45097

Exception: RuntimeError('Worker tcp://172.21.25.62:45499 left during active shuffle c98d9c3d9914db8c8838fce41542ce90')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: ('hash-join-258b4a0d5a7184c6e538a869303da290', 13)

Worker(s):

tcp://172.21.12.242:33997

Exception: RuntimeError('Worker tcp://172.21.25.62:36369 left during active shuffle f53d45c6ceeb185231dc7b60f9035eb2')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: ('hash-join-bd9b87c9a2dc9bdd0105b2ec75cdcf9c', 13)

Worker(s):

tcp://172.21.12.244:45097

Exception: RuntimeError('Worker tcp://172.21.25.62:32951 left during active shuffle 4e20bde84fcba5a004b5b31ca28c139f')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: ('shuffle-p2p-2b1ee98d9ccf4975d92436d3616a7a5e', 13)

Worker(s):

tcp://172.21.151.165:37861

Exception: RuntimeError('shuffle_unpack failed during shuffle e3c791b8ceaaa8a1c8cacf1cf56c3031')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 86, in shuffle_unpack
    raise RuntimeError(msg)

Task: ('lambda-6c27b5c7b65f128681bbca9824fde700', 13)

Worker(s):

tcp://172.21.12.250:35843

Exception: ValueError('Found array with 1 sample(s) (shape=(1, 1)) while a minimum of 2 is required by AgglomerativeClustering.')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/special_vehicle_stop_metric_worker.py", line 48, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/sklearn/cluster/_agglomerative.py", line 1099, in fit_predict
    return super().fit_predict(X, y)
  File "/opt/conda/lib/python3.10/site-packages/sklearn/base.py", line 753, in fit_predict
    self.fit(X)
  File "/opt/conda/lib/python3.10/site-packages/sklearn/cluster/_agglomerative.py", line 955, in fit
    X = self._validate_data(X, ensure_min_samples=2)
  File "/opt/conda/lib/python3.10/site-packages/sklearn/base.py", line 565, in _validate_data
    X = check_array(X, input_name="X", **check_params)
  File "/opt/conda/lib/python3.10/site-packages/sklearn/utils/validation.py", line 931, in check_array
    raise ValueError(

Task: ('assign-c42e5f8be81257ae645e5fb7b2338191', 8)

Worker(s):

tcp://172.21.159.225:32895

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: ('hash-join-021a772ec8fd0e83e9184b6ff4d012ab', 14)

Worker(s):

tcp://172.21.25.62:32951

Exception: RuntimeError('Worker tcp://172.21.25.92:45321 left during active shuffle 02dbb7b836a3da5d826cab5dea2e547f')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: ('hash-join-25fbdbe960ab751ba7e6c8bb5d750018', 14)

Worker(s):

tcp://172.21.25.62:32951

Exception: RuntimeError('Worker tcp://172.21.25.68:46011 left during active shuffle 52fec0dd53c6da3945d9046c5ff261d5')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: ('hash-join-a50b1b75b388b3764a21803852da3cde', 14)

Worker(s):

tcp://172.21.25.62:32951

Exception: RuntimeError('Worker tcp://172.21.151.159:40617 left during active shuffle a5b6d449ba3b53d8dfe186c0f2d62af5')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: ('hash-join-d700b7386a3a9ae6e5e69108e2798e2f', 14)

Worker(s):

tcp://172.21.25.62:32951

Exception: RuntimeError('Worker tcp://172.21.25.68:36885 left during active shuffle f92aa63088262c3be73cd4a4692f768b')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: shuffle-barrier-c666bc3f4123de0f986c642fb0fe1fe6

Worker(s):

tcp://172.21.25.68:36885

Exception: RuntimeError('shuffle_barrier failed during shuffle c666bc3f4123de0f986c642fb0fe1fe6')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: shuffle-barrier-e5ad0483476a2b47ad913819c05f632f

Worker(s):

tcp://172.21.12.244:45097

Exception: RuntimeError('shuffle_barrier failed during shuffle e5ad0483476a2b47ad913819c05f632f')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: shuffle-barrier-5b4e50195f12bd7da015c6c57c3bc554

Worker(s):

tcp://172.21.12.242:33997

Exception: RuntimeError('shuffle_barrier failed during shuffle 5b4e50195f12bd7da015c6c57c3bc554')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: shuffle-barrier-35c1c4cbfc75ca4311e3ef47bdd6e921

Worker(s):

tcp://172.21.12.242:33997

Exception: RuntimeError('shuffle_barrier failed during shuffle 35c1c4cbfc75ca4311e3ef47bdd6e921')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: shuffle-barrier-eb0539f1b040290c2d74a3d9dcf32dc8

Worker(s):

tcp://172.21.12.242:33997

Exception: RuntimeError('shuffle_barrier failed during shuffle eb0539f1b040290c2d74a3d9dcf32dc8')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: ('hash-join-fe8a8b5e0aedb9f2195eddc391aaa09e', 11)

Worker(s):

tcp://172.21.159.254:33329

Exception: ArrowInvalid('Schema at index 6 was different: \nschema_version: int64\ndetector_id: int64\nsource_id: int64\ntimestamp: int64\ngdt: int64\nsrt: int64\nheading: double\nspeed: double\nevent_type: string\nexterior_lights: string\nlongitude: double\nlatitude: double\naltitude: int64\ngeometry_type: string\nis_day: bool\nhashed_source_and_date: string\nindex: int64\nheading_diff_prev_and_cur: double\nheading_diff_cur_and_next: double\n__hash_partition: uint64\ndate: timestamp[ns]\nvs\nschema_version: int64\ndetector_id: int64\nsource_id: int64\ntimestamp: int64\ngdt: int64\nsrt: int64\nheading: double\nspeed: double\nevent_type: string\nexterior_lights: null\nlongitude: double\nlatitude: double\naltitude: int64\ngeometry_type: string\nis_day: bool\nhashed_source_and_date: string\nindex: int64\nheading_diff_prev_and_cur: double\nheading_diff_cur_and_next: double\n__hash_partition: uint64\ndate: timestamp[ns]')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 903, in get_output_partition
    return sync(self.worker.loop, shuffle.get_output_partition, output_partition)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 529, in get_output_partition
    out = await self.offload(_)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 132, in offload
    return await asyncio.get_running_loop().run_in_executor(
  File "/opt/conda/lib/python3.10/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 526, in _
    df = convert_partition(data)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_arrow.py", line 57, in convert_partition
    return pa.concat_tables(shards)
  File "pyarrow/table.pxi", line 5224, in pyarrow.lib.concat_tables
  File "pyarrow/error.pxi", line 144, in pyarrow.lib.pyarrow_internal_check_status
  File "pyarrow/error.pxi", line 100, in pyarrow.lib.check_status

Task: ('hash-join-926f1aa69ef3d02fdca73e9b42fe1e4c', 5)

Worker(s):

tcp://172.21.25.92:35319

Exception: RuntimeError('Worker tcp://172.21.151.159:40437 left during active shuffle 828b7c799727a708637c2ed1195150f9')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: ('lambda-25724e4d8dfc1f27c0b6457db5b67b60', 0)

Worker(s):

tcp://172.21.25.62:32951

Exception: ValueError('Found array with 1 sample(s) (shape=(1, 1)) while a minimum of 2 is required by AgglomerativeClustering.')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/edge_driving_time_metric_worker.py", line 47, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/sklearn/cluster/_agglomerative.py", line 1099, in fit_predict
    return super().fit_predict(X, y)
  File "/opt/conda/lib/python3.10/site-packages/sklearn/base.py", line 753, in fit_predict
    self.fit(X)
  File "/opt/conda/lib/python3.10/site-packages/sklearn/cluster/_agglomerative.py", line 955, in fit
    X = self._validate_data(X, ensure_min_samples=2)
  File "/opt/conda/lib/python3.10/site-packages/sklearn/base.py", line 565, in _validate_data
    X = check_array(X, input_name="X", **check_params)
  File "/opt/conda/lib/python3.10/site-packages/sklearn/utils/validation.py", line 931, in check_array
    raise ValueError(

Task: ('assign-d195a83320d467cd4e93783d813cfae6', 0)

Worker(s):

tcp://172.21.159.228:36567

Exception: GEOSException('IllegalArgumentException: point array must contain 0 or >1 elements\n')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/dask/optimization.py", line 990, in __call__
    return core.get(self.dsk, self.outkey, dict(zip(self.inkeys, args)))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 149, in get
    result = _execute_task(task, cache)
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in <genexpr>
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in _execute_task
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 113, in <listcomp>
    return [_execute_task(a, cache) for a in arg]
  File "/opt/conda/lib/python3.10/site-packages/dask/core.py", line 119, in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
  File "/opt/conda/lib/python3.10/site-packages/dask/utils.py", line 73, in apply
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/core.py", line 7006, in apply_and_enforce
    df = func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/dask/dataframe/groupby.py", line 230, in _groupby_slice_apply
    return g.apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/generic.py", line 254, in apply
    return super().apply(func, *args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1567, in apply
    result = self._python_apply_general(f, self._selected_obj)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/groupby.py", line 1629, in _python_apply_general
    values, mutated = self.grouper.apply(f, data, self.axis)
  File "/opt/conda/lib/python3.10/site-packages/pandas/core/groupby/ops.py", line 839, in apply
    res = f(group)
  File "/app/vehicle_distance_metric_worker.py", line 43, in <lambda>
  File "/opt/conda/lib/python3.10/site-packages/shapely/geometry/linestring.py", line 73, in __new__
    geom = shapely.linestrings(coordinates)
  File "/opt/conda/lib/python3.10/site-packages/shapely/decorators.py", line 77, in wrapped
    return func(*args, **kwargs)
  File "/opt/conda/lib/python3.10/site-packages/shapely/creation.py", line 120, in linestrings
    return lib.linestrings(coords, out=out, **kwargs)

Task: run-18545fd47974bfc641d16f91bfa274e0

Worker(s):

tcp://172.21.12.250:37171

Exception:

Traceback

Task: ('hash-join-a21a5b2952b103c6c94d9c80b7b85aee', 13)

Worker(s):

tcp://172.21.151.136:38831

Exception: RuntimeError('Worker tcp://172.21.12.242:34949 left during active shuffle e30936af7e96676cc0d3c7cd00d76eb6')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: shuffle-barrier-6c2bf87000cbfb3a717169b7c532b0e6

Worker(s):

tcp://172.21.12.242:34949

Exception: RuntimeError('shuffle_barrier failed during shuffle 6c2bf87000cbfb3a717169b7c532b0e6')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: shuffle-barrier-805ead32baa062c1f7211f9b49f87fdc

Worker(s):

tcp://172.21.12.242:34949

Exception: RuntimeError('shuffle_barrier failed during shuffle 805ead32baa062c1f7211f9b49f87fdc')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: ('hash-join-3f9d5ad47c24bcd38afc1f2bdc5bc159', 1)

Worker(s):

tcp://172.21.12.242:34949

Exception: RuntimeError('Worker tcp://172.21.12.244:41309 left during active shuffle 41ede99eb10352b0adf9d2e9467bab05')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: shuffle-barrier-544354b950710abae0e54acaf619d2b3

Worker(s):

tcp://172.21.12.242:34949

Exception: RuntimeError('shuffle_barrier failed during shuffle 544354b950710abae0e54acaf619d2b3')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: ('shuffle-p2p-b91bcc65eff283ac2264a83330a9aedf', 9)

Worker(s):

tcp://172.21.159.225:37655

Exception: RuntimeError('shuffle_unpack failed during shuffle 2a87cc2748f0e6b216f7058809febaec')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 86, in shuffle_unpack
    raise RuntimeError(msg)

Task: ('shuffle-p2p-b91bcc65eff283ac2264a83330a9aedf', 3)

Worker(s):

tcp://172.21.12.250:37171

Exception: RuntimeError('shuffle_unpack failed during shuffle 2a87cc2748f0e6b216f7058809febaec')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 86, in shuffle_unpack
    raise RuntimeError(msg)

Task: ('shuffle-p2p-b91bcc65eff283ac2264a83330a9aedf', 10)

Worker(s):

tcp://172.21.159.228:41827

Exception: RuntimeError('shuffle_unpack failed during shuffle 2a87cc2748f0e6b216f7058809febaec')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 86, in shuffle_unpack
    raise RuntimeError(msg)

Task: shuffle-barrier-212f056f1d49fa63d0d10a9c4e817a1d

Worker(s):

tcp://172.21.12.244:36079

Exception: RuntimeError('shuffle_barrier failed during shuffle 212f056f1d49fa63d0d10a9c4e817a1d')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: ('hash-join-6c96bba4467079249400d9f11f858a50', 14)

Worker(s):

tcp://172.21.12.247:45705

Exception: RuntimeError('Worker tcp://172.21.25.68:43811 left during active shuffle b85e8254b2cf9ad51bde4c4e7bb78bd1')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: shuffle-barrier-a31d5f4d89198a41d82e588538c3a6cc

Worker(s):

tcp://172.21.12.244:36079

Exception: RuntimeError('shuffle_barrier failed during shuffle a31d5f4d89198a41d82e588538c3a6cc')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_shuffle.py", line 97, in shuffle_barrier
    raise RuntimeError(msg)

Task: ('hash-join-1bf3e7f89603eb4294278efa6d6b5a84', 14)

Worker(s):

tcp://172.21.12.247:45705

Exception: RuntimeError('Worker tcp://172.21.25.68:44371 left during active shuffle a2c7feb2c3dbe3ed91f76bb15772073f')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: ('hash-join-40779926199f34ca1f87c44b42c6e60c', 15)

Worker(s):

tcp://172.21.151.159:40437

Exception: RuntimeError('Worker tcp://172.21.25.68:42109 left during active shuffle 13dc8037120052ae223a27a012443b10')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: ('hash-join-b92d947d47703295a852fb9c3c44f935', 15)

Worker(s):

tcp://172.21.159.228:41827

Exception: RuntimeError('Worker tcp://172.21.25.92:44641 left during active shuffle 63421263fa2a324624cf58c6ab14ad0a')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])

Task: ('hash-join-2f40876a1990955a7b102b75992854e6', 4)

Worker(s):

tcp://172.21.25.37:34813

Exception: RuntimeError('Worker tcp://172.21.151.136:37029 left during active shuffle d80ca885af5629fabf9e0ab97c6ef2a6')

Traceback

  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_merge.py", line 173, in merge_unpack
    left = ext.get_output_partition(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 902, in get_output_partition
    shuffle = self.get_shuffle_run(shuffle_id, run_id)
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 873, in get_shuffle_run
    return sync(
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 416, in sync
    raise exc.with_traceback(tb)
  File "/opt/conda/lib/python3.10/site-packages/distributed/utils.py", line 389, in f
    result = yield future
  File "/opt/conda/lib/python3.10/site-packages/tornado/gen.py", line 769, in run
    value = future.result()
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 678, in _get_shuffle_run
    shuffle = await self._refresh_shuffle(
  File "/opt/conda/lib/python3.10/site-packages/distributed/shuffle/_worker_extension.py", line 780, in _refresh_shuffle
    raise RuntimeError(result["message"])