from dask_yarn import YarnCluster
from dask.distributed import Client


cluster = YarnCluster(environment='venv:///opt/hadoop/pyenv/',
                      worker_vcores=2,
                      worker_memory="4GiB")

# Connect to the cluster
client = Client(cluster)
cluster.adapt(2, 10)
cluster.dashboard_link