start_cluster.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e5b3f5be-f09d-4761-b7e5-6177326750ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "import boto3\n",
    "import dask\n",
    "from dask_gateway import Gateway\n",
    "def get_dask_cluster(number_in_cluster):\n",
    "\n",
    "    gateway = Gateway()\n",
    "    running_clusters =  gateway.list_clusters()\n",
    "    if len(running_clusters) > 0:\n",
    "        cluster = gateway.connect(running_clusters[0].name)\n",
    "    else:\n",
    "        options = gateway.cluster_options()\n",
    "        session = boto3.Session(profile_name=\"default\")\n",
    "        credentials = session.get_credentials()\n",
    "        #options['conda_environment']= 'nebari-git/nebari-git-dask'\n",
    "\n",
    "        options['conda_environment']= 'nebari-git/nebari-git-dask-analysis'\n",
    "        options['profile'] = 'Small Worker'\n",
    "        #options['profile'] = 'Medium Worker'\n",
    "        options['environment_vars']['AWS_ACCESS_KEY_ID'] = credentials.access_key\n",
    "        options['environment_vars']['AWS_SECRET_ACCESS_KEY']= credentials.secret_key\n",
    "        cluster = gateway.new_cluster(options)\n",
    "\n",
    "        #cluster.adapt(minimum=1,maximum=10)\n",
    "\n",
    "    dask.config.set({\n",
    "        \"distributed.comm.timeouts.tcp\": \"1000 seconds\",\n",
    "        \"distributed.comm.timeouts.connect\": \"1000 seconds\",\n",
    "        \"distributed.scheduler.worker-ttl\": \"500 minutes\"\n",
    "    }) \n",
    "    cluster.scale(number_in_cluster)\n",
    "    #cluster.adapt(minimum=1,maximum=20)\n",
    "    return cluster.get_client()\n",
    "\n",
    "\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "972d57d3-c137-43dd-895b-65f0b4a2d73f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nebari-git-nebari-git-dask-analysis",
   "language": "python",
   "name": "conda-env-nebari-git-nebari-git-dask-analysis-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}