{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "uPvSdOSwMmnB"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import openai\n",
    "import sqlite3\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from google.colab import userdata\n",
    "import wandb\n",
    "\n",
    "oai_token = userdata.get('OPENAI_API_KEY')\n",
    "\n",
    "oai = openai.OpenAI(api_key=oai_token)\n",
    "onet = sqlite3.connect(\"onet.database\") # Run ./create_onet_database.sh to create it\n",
    "# This dataset comes from https://epoch.ai/gradient-updates/consequences-of-automating-remote-work\n",
    "# It contains labels for whethere a O*NET task can be done remotely or not (labeled by GPT-4o)\n",
    "# You can download it here: https://drive.google.com/file/d/1GrHhuYIgaCCgo99dZ_40BWraz-fzo76r/view?usp=sharing\n",
    "df_remote_status = pd.read_csv(\"epoch_task_data.csv\")\n",
    "\n",
    "# BLS OEWS: https://www.bls.gov/oes/special-requests/oesm23nat.zip\n",
    "df_oesm = pd.read_excel(\"oesm23national.xlsx\")\n",
    "\n",
    "# Run uv run enrich_task_ratings.py to get this file\n",
    "df = pd.read_json(\"task_ratings_enriched.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "collapsed": true,
    "id": "mO2expAK2b8F",
    "jupyter": {
     "outputs_hidden": true
    },
    "outputId": "f1f2c2d0-26e0-41a0-f22d-367294f49561"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "summary": "{\n  \"name\": \"df\",\n  \"rows\": 39467,\n  \"fields\": [\n    {\n      \"column\": \"onetsoc_code\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 879,\n        \"samples\": [\n          \"27-2012.05\",\n          \"21-1093.00\",\n          \"51-8013.00\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"task_id\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 5430,\n        \"min\": 1,\n        \"max\": 24001,\n        \"num_unique_values\": 17639,\n        \"samples\": [\n          10675,\n          22981,\n          798\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"task\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 16580,\n        \"samples\": [\n          \"File grooved, contoured, and irregular surfaces of metal objects, such as metalworking dies and machine parts, to conform to templates, other parts, layouts, or blueprint specifications.\",\n          \"Measure tires to determine mold size requirements.\",\n          \"Instruct customers in the installation, repair, or maintenance of products.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"occupation_title\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 879,\n        \"samples\": [\n          \"Media Technical Directors/Managers\",\n          \"Social and Human Service Assistants\",\n          \"Power Plant Operators\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"occupation_description\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 879,\n        \"samples\": [\n          \"Coordinate activities of technical departments, such as taping, editing, engineering, and maintenance, to produce radio or television programs.\",\n          \"Assist other social and human service providers in providing client services in a wide variety of fields, such as psychology, rehabilitation, or social work, including support for families. May assist clients in identifying and obtaining available benefits and social and community services. May assist social workers with developing, organizing, and conducting programs to prevent and resolve problems relevant to substance abuse, human relationships, rehabilitation, or dependent care.\",\n          \"Control, operate, or maintain machinery to generate electric power. Includes auxiliary equipment operators.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"frequency_category_1\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 13.496877695589955,\n        \"min\": 0.0,\n        \"max\": 98.43,\n        \"num_unique_values\": 2814,\n        \"samples\": [\n          8.24,\n          19.74,\n          38.65\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"frequency_category_2\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 16.678101090168983,\n        \"min\": 0.0,\n        \"max\": 88.89,\n        \"num_unique_values\": 3954,\n        \"samples\": [\n          10.98,\n          2.91,\n          26.81\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"frequency_category_3\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 14.06670148855046,\n        \"min\": 0.0,\n        \"max\": 73.08,\n        \"num_unique_values\": 3959,\n        \"samples\": [\n          28.72,\n          43.5,\n          4.48\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"frequency_category_4\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 13.650817839999881,\n        \"min\": 0.0,\n        \"max\": 80.77,\n        \"num_unique_values\": 3793,\n        \"samples\": [\n          25.44,\n          3.63,\n          17.33\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"frequency_category_5\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 15.456854044334518,\n        \"min\": 0.0,\n        \"max\": 93.02,\n        \"num_unique_values\": 5296,\n        \"samples\": [\n          2.41,\n          6.77,\n          80.96\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"frequency_category_6\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 8.831930521377236,\n        \"min\": 0.0,\n        \"max\": 84.0,\n        \"num_unique_values\": 3394,\n        \"samples\": [\n          4.46,\n          22.17,\n          18.61\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"frequency_category_7\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 13.023172761542176,\n        \"min\": 0.0,\n        \"max\": 97.54,\n        \"num_unique_values\": 4340,\n        \"samples\": [\n          12.67,\n          50.0,\n          58.93\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"importance_average\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.5405272645655124,\n        \"min\": 1.44,\n        \"max\": 5.0,\n        \"num_unique_values\": 293,\n        \"samples\": [\n          4.08,\n          4.96,\n          3.5700000000000003\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"relevance_average\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 17.754075691110696,\n        \"min\": 25.0,\n        \"max\": 100.0,\n        \"num_unique_values\": 5376,\n        \"samples\": [\n          49.07,\n          67.11,\n          53.23\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"dwas\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"remote_status\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          \"not remote\",\n          \"remote\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}",
       "type": "dataframe",
       "variable_name": "df"
      },
      "text/html": [
       "\n",
       "  <div id=\"df-bc7b085b-97a7-42d1-82cf-afa1e7f2f3c6\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>onetsoc_code</th>\n",
       "      <th>task_id</th>\n",
       "      <th>task</th>\n",
       "      <th>occupation_title</th>\n",
       "      <th>occupation_description</th>\n",
       "      <th>frequency_category_1</th>\n",
       "      <th>frequency_category_2</th>\n",
       "      <th>frequency_category_3</th>\n",
       "      <th>frequency_category_4</th>\n",
       "      <th>frequency_category_5</th>\n",
       "      <th>frequency_category_6</th>\n",
       "      <th>frequency_category_7</th>\n",
       "      <th>importance_average</th>\n",
       "      <th>relevance_average</th>\n",
       "      <th>dwas</th>\n",
       "      <th>remote_status</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>11-1011.00</td>\n",
       "      <td>8823</td>\n",
       "      <td>Direct or coordinate an organization's financi...</td>\n",
       "      <td>Chief Executives</td>\n",
       "      <td>Determine and formulate policies and provide o...</td>\n",
       "      <td>5.92</td>\n",
       "      <td>15.98</td>\n",
       "      <td>29.68</td>\n",
       "      <td>21.18</td>\n",
       "      <td>19.71</td>\n",
       "      <td>4.91</td>\n",
       "      <td>2.63</td>\n",
       "      <td>4.52</td>\n",
       "      <td>74.44</td>\n",
       "      <td>[Direct financial operations.]</td>\n",
       "      <td>remote</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>11-1011.00</td>\n",
       "      <td>8824</td>\n",
       "      <td>Confer with board members, organization offici...</td>\n",
       "      <td>Chief Executives</td>\n",
       "      <td>Determine and formulate policies and provide o...</td>\n",
       "      <td>1.42</td>\n",
       "      <td>14.44</td>\n",
       "      <td>27.31</td>\n",
       "      <td>25.52</td>\n",
       "      <td>26.88</td>\n",
       "      <td>2.52</td>\n",
       "      <td>1.90</td>\n",
       "      <td>4.32</td>\n",
       "      <td>81.71</td>\n",
       "      <td>[Confer with organizational members to accompl...</td>\n",
       "      <td>remote</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11-1011.00</td>\n",
       "      <td>8827</td>\n",
       "      <td>Prepare budgets for approval, including those ...</td>\n",
       "      <td>Chief Executives</td>\n",
       "      <td>Determine and formulate policies and provide o...</td>\n",
       "      <td>15.50</td>\n",
       "      <td>38.21</td>\n",
       "      <td>32.73</td>\n",
       "      <td>5.15</td>\n",
       "      <td>5.25</td>\n",
       "      <td>0.19</td>\n",
       "      <td>2.98</td>\n",
       "      <td>4.30</td>\n",
       "      <td>93.41</td>\n",
       "      <td>[Prepare operational budgets.]</td>\n",
       "      <td>remote</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11-1011.00</td>\n",
       "      <td>8826</td>\n",
       "      <td>Direct, plan, or implement policies, objective...</td>\n",
       "      <td>Chief Executives</td>\n",
       "      <td>Determine and formulate policies and provide o...</td>\n",
       "      <td>3.03</td>\n",
       "      <td>17.33</td>\n",
       "      <td>20.30</td>\n",
       "      <td>18.10</td>\n",
       "      <td>33.16</td>\n",
       "      <td>2.01</td>\n",
       "      <td>6.07</td>\n",
       "      <td>4.24</td>\n",
       "      <td>97.79</td>\n",
       "      <td>[Implement organizational process or policy ch...</td>\n",
       "      <td>remote</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11-1011.00</td>\n",
       "      <td>8834</td>\n",
       "      <td>Prepare or present reports concerning activiti...</td>\n",
       "      <td>Chief Executives</td>\n",
       "      <td>Determine and formulate policies and provide o...</td>\n",
       "      <td>1.98</td>\n",
       "      <td>14.06</td>\n",
       "      <td>42.60</td>\n",
       "      <td>21.24</td>\n",
       "      <td>13.18</td>\n",
       "      <td>6.24</td>\n",
       "      <td>0.70</td>\n",
       "      <td>4.17</td>\n",
       "      <td>92.92</td>\n",
       "      <td>[Prepare financial documents, reports, or budg...</td>\n",
       "      <td>remote</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39462</th>\n",
       "      <td>53-7121.00</td>\n",
       "      <td>12807</td>\n",
       "      <td>Unload cars containing liquids by connecting h...</td>\n",
       "      <td>Tank Car, Truck, and Ship Loaders</td>\n",
       "      <td>Load and unload chemicals and bulk solids, suc...</td>\n",
       "      <td>6.05</td>\n",
       "      <td>29.21</td>\n",
       "      <td>6.88</td>\n",
       "      <td>13.95</td>\n",
       "      <td>27.65</td>\n",
       "      <td>7.93</td>\n",
       "      <td>8.34</td>\n",
       "      <td>4.08</td>\n",
       "      <td>64.04</td>\n",
       "      <td>[Connect hoses to equipment or machinery.]</td>\n",
       "      <td>not remote</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39463</th>\n",
       "      <td>53-7121.00</td>\n",
       "      <td>12804</td>\n",
       "      <td>Clean interiors of tank cars or tank trucks, u...</td>\n",
       "      <td>Tank Car, Truck, and Ship Loaders</td>\n",
       "      <td>Load and unload chemicals and bulk solids, suc...</td>\n",
       "      <td>1.47</td>\n",
       "      <td>6.33</td>\n",
       "      <td>21.70</td>\n",
       "      <td>25.69</td>\n",
       "      <td>32.35</td>\n",
       "      <td>12.47</td>\n",
       "      <td>0.00</td>\n",
       "      <td>4.02</td>\n",
       "      <td>44.33</td>\n",
       "      <td>[Clean vessels or marine equipment.]</td>\n",
       "      <td>not remote</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39464</th>\n",
       "      <td>53-7121.00</td>\n",
       "      <td>12803</td>\n",
       "      <td>Lower gauge rods into tanks or read meters to ...</td>\n",
       "      <td>Tank Car, Truck, and Ship Loaders</td>\n",
       "      <td>Load and unload chemicals and bulk solids, suc...</td>\n",
       "      <td>4.52</td>\n",
       "      <td>1.76</td>\n",
       "      <td>4.65</td>\n",
       "      <td>17.81</td>\n",
       "      <td>37.42</td>\n",
       "      <td>23.31</td>\n",
       "      <td>10.55</td>\n",
       "      <td>3.88</td>\n",
       "      <td>65.00</td>\n",
       "      <td>[Measure the level or depth of water or other ...</td>\n",
       "      <td>not remote</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39465</th>\n",
       "      <td>53-7121.00</td>\n",
       "      <td>12805</td>\n",
       "      <td>Operate conveyors and equipment to transfer gr...</td>\n",
       "      <td>Tank Car, Truck, and Ship Loaders</td>\n",
       "      <td>Load and unload chemicals and bulk solids, suc...</td>\n",
       "      <td>6.97</td>\n",
       "      <td>12.00</td>\n",
       "      <td>2.52</td>\n",
       "      <td>5.90</td>\n",
       "      <td>35.48</td>\n",
       "      <td>22.08</td>\n",
       "      <td>15.05</td>\n",
       "      <td>3.87</td>\n",
       "      <td>47.90</td>\n",
       "      <td>[Operate conveyors or other industrial materia...</td>\n",
       "      <td>not remote</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39466</th>\n",
       "      <td>53-7121.00</td>\n",
       "      <td>12810</td>\n",
       "      <td>Perform general warehouse activities, such as ...</td>\n",
       "      <td>Tank Car, Truck, and Ship Loaders</td>\n",
       "      <td>Load and unload chemicals and bulk solids, suc...</td>\n",
       "      <td>5.91</td>\n",
       "      <td>10.85</td>\n",
       "      <td>6.46</td>\n",
       "      <td>14.46</td>\n",
       "      <td>34.14</td>\n",
       "      <td>16.39</td>\n",
       "      <td>11.78</td>\n",
       "      <td>3.53</td>\n",
       "      <td>47.84</td>\n",
       "      <td>[Weigh materials to ensure compliance with spe...</td>\n",
       "      <td>not remote</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>39467 rows × 16 columns</p>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-bc7b085b-97a7-42d1-82cf-afa1e7f2f3c6')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-bc7b085b-97a7-42d1-82cf-afa1e7f2f3c6 button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-bc7b085b-97a7-42d1-82cf-afa1e7f2f3c6');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "    <div id=\"df-6bfe183f-09ce-41f5-a906-79d8016964e7\">\n",
       "      <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-6bfe183f-09ce-41f5-a906-79d8016964e7')\"\n",
       "                title=\"Suggest charts\"\n",
       "                style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "      </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "      <script>\n",
       "        async function quickchart(key) {\n",
       "          const quickchartButtonEl =\n",
       "            document.querySelector('#' + key + ' button');\n",
       "          quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "          quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "          try {\n",
       "            const charts = await google.colab.kernel.invokeFunction(\n",
       "                'suggestCharts', [key], {});\n",
       "          } catch (error) {\n",
       "            console.error('Error during call to suggestCharts:', error);\n",
       "          }\n",
       "          quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "          quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "        }\n",
       "        (() => {\n",
       "          let quickchartButtonEl =\n",
       "            document.querySelector('#df-6bfe183f-09ce-41f5-a906-79d8016964e7 button');\n",
       "          quickchartButtonEl.style.display =\n",
       "            google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "        })();\n",
       "      </script>\n",
       "    </div>\n",
       "\n",
       "  <div id=\"id_7fc802be-1a27-4a30-b21a-2324fc2424c0\">\n",
       "    <style>\n",
       "      .colab-df-generate {\n",
       "        background-color: #E8F0FE;\n",
       "        border: none;\n",
       "        border-radius: 50%;\n",
       "        cursor: pointer;\n",
       "        display: none;\n",
       "        fill: #1967D2;\n",
       "        height: 32px;\n",
       "        padding: 0 0 0 0;\n",
       "        width: 32px;\n",
       "      }\n",
       "\n",
       "      .colab-df-generate:hover {\n",
       "        background-color: #E2EBFA;\n",
       "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "        fill: #174EA6;\n",
       "      }\n",
       "\n",
       "      [theme=dark] .colab-df-generate {\n",
       "        background-color: #3B4455;\n",
       "        fill: #D2E3FC;\n",
       "      }\n",
       "\n",
       "      [theme=dark] .colab-df-generate:hover {\n",
       "        background-color: #434B5C;\n",
       "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "        fill: #FFFFFF;\n",
       "      }\n",
       "    </style>\n",
       "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('df')\"\n",
       "            title=\"Generate code using this dataframe.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "    <script>\n",
       "      (() => {\n",
       "      const buttonEl =\n",
       "        document.querySelector('#id_7fc802be-1a27-4a30-b21a-2324fc2424c0 button.colab-df-generate');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      buttonEl.onclick = () => {\n",
       "        google.colab.notebook.generateWithVariable('df');\n",
       "      }\n",
       "      })();\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "      onetsoc_code  task_id  \\\n",
       "0       11-1011.00     8823   \n",
       "1       11-1011.00     8824   \n",
       "2       11-1011.00     8827   \n",
       "3       11-1011.00     8826   \n",
       "4       11-1011.00     8834   \n",
       "...            ...      ...   \n",
       "39462   53-7121.00    12807   \n",
       "39463   53-7121.00    12804   \n",
       "39464   53-7121.00    12803   \n",
       "39465   53-7121.00    12805   \n",
       "39466   53-7121.00    12810   \n",
       "\n",
       "                                                    task  \\\n",
       "0      Direct or coordinate an organization's financi...   \n",
       "1      Confer with board members, organization offici...   \n",
       "2      Prepare budgets for approval, including those ...   \n",
       "3      Direct, plan, or implement policies, objective...   \n",
       "4      Prepare or present reports concerning activiti...   \n",
       "...                                                  ...   \n",
       "39462  Unload cars containing liquids by connecting h...   \n",
       "39463  Clean interiors of tank cars or tank trucks, u...   \n",
       "39464  Lower gauge rods into tanks or read meters to ...   \n",
       "39465  Operate conveyors and equipment to transfer gr...   \n",
       "39466  Perform general warehouse activities, such as ...   \n",
       "\n",
       "                        occupation_title  \\\n",
       "0                       Chief Executives   \n",
       "1                       Chief Executives   \n",
       "2                       Chief Executives   \n",
       "3                       Chief Executives   \n",
       "4                       Chief Executives   \n",
       "...                                  ...   \n",
       "39462  Tank Car, Truck, and Ship Loaders   \n",
       "39463  Tank Car, Truck, and Ship Loaders   \n",
       "39464  Tank Car, Truck, and Ship Loaders   \n",
       "39465  Tank Car, Truck, and Ship Loaders   \n",
       "39466  Tank Car, Truck, and Ship Loaders   \n",
       "\n",
       "                                  occupation_description  \\\n",
       "0      Determine and formulate policies and provide o...   \n",
       "1      Determine and formulate policies and provide o...   \n",
       "2      Determine and formulate policies and provide o...   \n",
       "3      Determine and formulate policies and provide o...   \n",
       "4      Determine and formulate policies and provide o...   \n",
       "...                                                  ...   \n",
       "39462  Load and unload chemicals and bulk solids, suc...   \n",
       "39463  Load and unload chemicals and bulk solids, suc...   \n",
       "39464  Load and unload chemicals and bulk solids, suc...   \n",
       "39465  Load and unload chemicals and bulk solids, suc...   \n",
       "39466  Load and unload chemicals and bulk solids, suc...   \n",
       "\n",
       "       frequency_category_1  frequency_category_2  frequency_category_3  \\\n",
       "0                      5.92                 15.98                 29.68   \n",
       "1                      1.42                 14.44                 27.31   \n",
       "2                     15.50                 38.21                 32.73   \n",
       "3                      3.03                 17.33                 20.30   \n",
       "4                      1.98                 14.06                 42.60   \n",
       "...                     ...                   ...                   ...   \n",
       "39462                  6.05                 29.21                  6.88   \n",
       "39463                  1.47                  6.33                 21.70   \n",
       "39464                  4.52                  1.76                  4.65   \n",
       "39465                  6.97                 12.00                  2.52   \n",
       "39466                  5.91                 10.85                  6.46   \n",
       "\n",
       "       frequency_category_4  frequency_category_5  frequency_category_6  \\\n",
       "0                     21.18                 19.71                  4.91   \n",
       "1                     25.52                 26.88                  2.52   \n",
       "2                      5.15                  5.25                  0.19   \n",
       "3                     18.10                 33.16                  2.01   \n",
       "4                     21.24                 13.18                  6.24   \n",
       "...                     ...                   ...                   ...   \n",
       "39462                 13.95                 27.65                  7.93   \n",
       "39463                 25.69                 32.35                 12.47   \n",
       "39464                 17.81                 37.42                 23.31   \n",
       "39465                  5.90                 35.48                 22.08   \n",
       "39466                 14.46                 34.14                 16.39   \n",
       "\n",
       "       frequency_category_7  importance_average  relevance_average  \\\n",
       "0                      2.63                4.52              74.44   \n",
       "1                      1.90                4.32              81.71   \n",
       "2                      2.98                4.30              93.41   \n",
       "3                      6.07                4.24              97.79   \n",
       "4                      0.70                4.17              92.92   \n",
       "...                     ...                 ...                ...   \n",
       "39462                  8.34                4.08              64.04   \n",
       "39463                  0.00                4.02              44.33   \n",
       "39464                 10.55                3.88              65.00   \n",
       "39465                 15.05                3.87              47.90   \n",
       "39466                 11.78                3.53              47.84   \n",
       "\n",
       "                                                    dwas remote_status  \n",
       "0                         [Direct financial operations.]        remote  \n",
       "1      [Confer with organizational members to accompl...        remote  \n",
       "2                         [Prepare operational budgets.]        remote  \n",
       "3      [Implement organizational process or policy ch...        remote  \n",
       "4      [Prepare financial documents, reports, or budg...        remote  \n",
       "...                                                  ...           ...  \n",
       "39462         [Connect hoses to equipment or machinery.]    not remote  \n",
       "39463               [Clean vessels or marine equipment.]    not remote  \n",
       "39464  [Measure the level or depth of water or other ...    not remote  \n",
       "39465  [Operate conveyors or other industrial materia...    not remote  \n",
       "39466  [Weigh materials to ensure compliance with spe...    not remote  \n",
       "\n",
       "[39467 rows x 16 columns]"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.merge(df, df_remote_status[['Task', 'Remote']], left_on='task', right_on='Task', how='left')\n",
    "df = df.drop('Task', axis=1) \\\n",
    "       .rename(columns={'Remote': 'remote_status'})\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "WvFNPFjrNB-B"
   },
   "outputs": [],
   "source": [
    "FREQUENCY_MAP = {\n",
    "    'frequency_category_1': \"Yearly or less\",\n",
    "    'frequency_category_2': \"More than yearly\",\n",
    "    'frequency_category_3': \"More than monthly\",\n",
    "    'frequency_category_4': \"More than weekly\",\n",
    "    'frequency_category_5': \"Daily\",\n",
    "    'frequency_category_6': \"Several times daily\",\n",
    "    'frequency_category_7': \"Hourly or more\"\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "kZ-L4EFoNLnT"
   },
   "outputs": [],
   "source": [
    "# Cross-reference woth BLS OEWS\n",
    "# It doesn't really make sens to have it per-task, we only need it per-occupation...\n",
    "df_oesm_detailed = df_oesm[df_oesm['O_GROUP'] == 'detailed'][['OCC_CODE', 'TOT_EMP', 'H_MEAN', 'A_MEAN']].copy()\n",
    "df['occ_code_join'] = df['onetsoc_code'].str[:7]\n",
    "df = pd.merge(\n",
    "    df,\n",
    "    df_oesm_detailed,\n",
    "    left_on='occ_code_join',\n",
    "    right_on='OCC_CODE',\n",
    "    how='left'\n",
    ")\n",
    "df = df.drop(columns=['occ_code_join']).rename(columns={\"OCC_CODE\": \"occ_code\", \"TOT_EMP\": \"total_employment\", \"H_MEAN\": \"hourly_wage_average\", \"A_MEAN\": \"annual_wage_average\"})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "collapsed": true,
    "id": "3LkV9k591LW9",
    "jupyter": {
     "outputs_hidden": true
    },
    "outputId": "18d78a28-3f28-4d13-ca4f-dda263daedc5"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "summary": "{\n  \"name\": \"remote_df\",\n  \"rows\": 21698,\n  \"fields\": [\n    {\n      \"column\": \"onetsoc_code\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 697,\n        \"samples\": [\n          \"17-2199.10\",\n          \"41-2021.00\",\n          \"29-1299.02\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"task_id\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 4674,\n        \"min\": 1,\n        \"max\": 23997,\n        \"num_unique_values\": 6166,\n        \"samples\": [\n          5616,\n          15580,\n          173\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"task\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 5399,\n        \"samples\": [\n          \"Teach or supervise students and perform research at universities and colleges.\",\n          \"Identify methods for interfacing Web application technologies with enterprise resource planning or other system software.\",\n          \"Examine documentation such as mortgages, liens, judgments, easements, plat books, maps, contracts, and agreements to verify factors such as properties' legal descriptions, ownership, or restrictions.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"occupation_title\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 697,\n        \"samples\": [\n          \"Wind Energy Engineers\",\n          \"Counter and Rental Clerks\",\n          \"Orthoptists\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"occupation_description\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 697,\n        \"samples\": [\n          \"Design underground or overhead wind farm collector systems and prepare and develop site specifications.\",\n          \"Receive orders, generally in person, for repairs, rentals, and services. May describe available options, compute cost, and accept payment.\",\n          \"Diagnose and treat visual system disorders such as binocular vision and eye movement impairments.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"frequency_category_1\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 15.392290836442328,\n        \"min\": 0.0,\n        \"max\": 90.91,\n        \"num_unique_values\": 1778,\n        \"samples\": [\n          32.69,\n          5.4,\n          22.48\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"frequency_category_2\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 16.365064366617172,\n        \"min\": 0.0,\n        \"max\": 88.89,\n        \"num_unique_values\": 2552,\n        \"samples\": [\n          36.44,\n          60.47,\n          20.88\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"frequency_category_3\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 14.0698466229559,\n        \"min\": 0.0,\n        \"max\": 72.73,\n        \"num_unique_values\": 2559,\n        \"samples\": [\n          15.36,\n          37.23,\n          19.64\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"frequency_category_4\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 14.203638081510212,\n        \"min\": 0.0,\n        \"max\": 80.77,\n        \"num_unique_values\": 2451,\n        \"samples\": [\n          17.62,\n          28.41,\n          20.55\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"frequency_category_5\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 13.941091224152455,\n        \"min\": 0.0,\n        \"max\": 85.29,\n        \"num_unique_values\": 2788,\n        \"samples\": [\n          40.45,\n          23.45,\n          37.61\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"frequency_category_6\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 6.524498762360237,\n        \"min\": 0.0,\n        \"max\": 73.91,\n        \"num_unique_values\": 1828,\n        \"samples\": [\n          7.54,\n          27.76,\n          25.85\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"frequency_category_7\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 7.496218170101234,\n        \"min\": 0.0,\n        \"max\": 89.67,\n        \"num_unique_values\": 1776,\n        \"samples\": [\n          3.99,\n          44.89,\n          59.12\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"importance_average\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0.5343251977372085,\n        \"min\": 1.75,\n        \"max\": 5.0,\n        \"num_unique_values\": 270,\n        \"samples\": [\n          3.52,\n          2.81,\n          4.44\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"relevance_average\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 16.85307890304865,\n        \"min\": 25.0,\n        \"max\": 100.0,\n        \"num_unique_values\": 2662,\n        \"samples\": [\n          93.84,\n          88.0,\n          68.32\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"dwas\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"remote_status\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"remote\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"occ_code\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 579,\n        \"samples\": [\n          \"27-2032\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"total_employment\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 261963.7301986826,\n        \"min\": 260.0,\n        \"max\": 3684740.0,\n        \"num_unique_values\": 566,\n        \"samples\": [\n          584630.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"hourly_wage_average\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 507,\n        \"samples\": [\n          66.83\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"annual_wage_average\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 564,\n        \"samples\": [\n          41900\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}",
       "type": "dataframe",
       "variable_name": "remote_df"
      },
      "text/html": [
       "\n",
       "  <div id=\"df-ac6b9bba-3f44-4d0c-ad52-b83abce95a33\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>onetsoc_code</th>\n",
       "      <th>task_id</th>\n",
       "      <th>task</th>\n",
       "      <th>occupation_title</th>\n",
       "      <th>occupation_description</th>\n",
       "      <th>frequency_category_1</th>\n",
       "      <th>frequency_category_2</th>\n",
       "      <th>frequency_category_3</th>\n",
       "      <th>frequency_category_4</th>\n",
       "      <th>frequency_category_5</th>\n",
       "      <th>frequency_category_6</th>\n",
       "      <th>frequency_category_7</th>\n",
       "      <th>importance_average</th>\n",
       "      <th>relevance_average</th>\n",
       "      <th>dwas</th>\n",
       "      <th>remote_status</th>\n",
       "      <th>occ_code</th>\n",
       "      <th>total_employment</th>\n",
       "      <th>hourly_wage_average</th>\n",
       "      <th>annual_wage_average</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>11-1011.00</td>\n",
       "      <td>8823</td>\n",
       "      <td>Direct or coordinate an organization's financi...</td>\n",
       "      <td>Chief Executives</td>\n",
       "      <td>Determine and formulate policies and provide o...</td>\n",
       "      <td>5.92</td>\n",
       "      <td>15.98</td>\n",
       "      <td>29.68</td>\n",
       "      <td>21.18</td>\n",
       "      <td>19.71</td>\n",
       "      <td>4.91</td>\n",
       "      <td>2.63</td>\n",
       "      <td>4.52</td>\n",
       "      <td>74.44</td>\n",
       "      <td>[Direct financial operations.]</td>\n",
       "      <td>remote</td>\n",
       "      <td>11-1011</td>\n",
       "      <td>211230.0</td>\n",
       "      <td>124.47</td>\n",
       "      <td>258900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>11-1011.00</td>\n",
       "      <td>8824</td>\n",
       "      <td>Confer with board members, organization offici...</td>\n",
       "      <td>Chief Executives</td>\n",
       "      <td>Determine and formulate policies and provide o...</td>\n",
       "      <td>1.42</td>\n",
       "      <td>14.44</td>\n",
       "      <td>27.31</td>\n",
       "      <td>25.52</td>\n",
       "      <td>26.88</td>\n",
       "      <td>2.52</td>\n",
       "      <td>1.90</td>\n",
       "      <td>4.32</td>\n",
       "      <td>81.71</td>\n",
       "      <td>[Confer with organizational members to accompl...</td>\n",
       "      <td>remote</td>\n",
       "      <td>11-1011</td>\n",
       "      <td>211230.0</td>\n",
       "      <td>124.47</td>\n",
       "      <td>258900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11-1011.00</td>\n",
       "      <td>8827</td>\n",
       "      <td>Prepare budgets for approval, including those ...</td>\n",
       "      <td>Chief Executives</td>\n",
       "      <td>Determine and formulate policies and provide o...</td>\n",
       "      <td>15.50</td>\n",
       "      <td>38.21</td>\n",
       "      <td>32.73</td>\n",
       "      <td>5.15</td>\n",
       "      <td>5.25</td>\n",
       "      <td>0.19</td>\n",
       "      <td>2.98</td>\n",
       "      <td>4.30</td>\n",
       "      <td>93.41</td>\n",
       "      <td>[Prepare operational budgets.]</td>\n",
       "      <td>remote</td>\n",
       "      <td>11-1011</td>\n",
       "      <td>211230.0</td>\n",
       "      <td>124.47</td>\n",
       "      <td>258900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>11-1011.00</td>\n",
       "      <td>8826</td>\n",
       "      <td>Direct, plan, or implement policies, objective...</td>\n",
       "      <td>Chief Executives</td>\n",
       "      <td>Determine and formulate policies and provide o...</td>\n",
       "      <td>3.03</td>\n",
       "      <td>17.33</td>\n",
       "      <td>20.30</td>\n",
       "      <td>18.10</td>\n",
       "      <td>33.16</td>\n",
       "      <td>2.01</td>\n",
       "      <td>6.07</td>\n",
       "      <td>4.24</td>\n",
       "      <td>97.79</td>\n",
       "      <td>[Implement organizational process or policy ch...</td>\n",
       "      <td>remote</td>\n",
       "      <td>11-1011</td>\n",
       "      <td>211230.0</td>\n",
       "      <td>124.47</td>\n",
       "      <td>258900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11-1011.00</td>\n",
       "      <td>8834</td>\n",
       "      <td>Prepare or present reports concerning activiti...</td>\n",
       "      <td>Chief Executives</td>\n",
       "      <td>Determine and formulate policies and provide o...</td>\n",
       "      <td>1.98</td>\n",
       "      <td>14.06</td>\n",
       "      <td>42.60</td>\n",
       "      <td>21.24</td>\n",
       "      <td>13.18</td>\n",
       "      <td>6.24</td>\n",
       "      <td>0.70</td>\n",
       "      <td>4.17</td>\n",
       "      <td>92.92</td>\n",
       "      <td>[Prepare financial documents, reports, or budg...</td>\n",
       "      <td>remote</td>\n",
       "      <td>11-1011</td>\n",
       "      <td>211230.0</td>\n",
       "      <td>124.47</td>\n",
       "      <td>258900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39438</th>\n",
       "      <td>53-7081.00</td>\n",
       "      <td>7172</td>\n",
       "      <td>Fill out defective equipment reports.</td>\n",
       "      <td>Refuse and Recyclable Material Collectors</td>\n",
       "      <td>Collect and dump refuse or recyclable material...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.75</td>\n",
       "      <td>9.69</td>\n",
       "      <td>3.08</td>\n",
       "      <td>85.29</td>\n",
       "      <td>0.09</td>\n",
       "      <td>0.09</td>\n",
       "      <td>4.27</td>\n",
       "      <td>91.18</td>\n",
       "      <td>[Prepare accident or incident reports.]</td>\n",
       "      <td>remote</td>\n",
       "      <td>53-7081</td>\n",
       "      <td>135430.0</td>\n",
       "      <td>22.99</td>\n",
       "      <td>47810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39442</th>\n",
       "      <td>53-7081.00</td>\n",
       "      <td>7178</td>\n",
       "      <td>Communicate with dispatchers concerning delays...</td>\n",
       "      <td>Refuse and Recyclable Material Collectors</td>\n",
       "      <td>Collect and dump refuse or recyclable material...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>1.04</td>\n",
       "      <td>5.92</td>\n",
       "      <td>3.74</td>\n",
       "      <td>69.00</td>\n",
       "      <td>8.98</td>\n",
       "      <td>11.32</td>\n",
       "      <td>3.96</td>\n",
       "      <td>97.50</td>\n",
       "      <td>[Report vehicle or equipment malfunctions., No...</td>\n",
       "      <td>remote</td>\n",
       "      <td>53-7081</td>\n",
       "      <td>135430.0</td>\n",
       "      <td>22.99</td>\n",
       "      <td>47810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39443</th>\n",
       "      <td>53-7081.00</td>\n",
       "      <td>7179</td>\n",
       "      <td>Check road or weather conditions to determine ...</td>\n",
       "      <td>Refuse and Recyclable Material Collectors</td>\n",
       "      <td>Collect and dump refuse or recyclable material...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>8.98</td>\n",
       "      <td>4.23</td>\n",
       "      <td>8.60</td>\n",
       "      <td>61.70</td>\n",
       "      <td>11.87</td>\n",
       "      <td>4.63</td>\n",
       "      <td>3.81</td>\n",
       "      <td>89.52</td>\n",
       "      <td>[Gather information about work conditions or l...</td>\n",
       "      <td>remote</td>\n",
       "      <td>53-7081</td>\n",
       "      <td>135430.0</td>\n",
       "      <td>22.99</td>\n",
       "      <td>47810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39447</th>\n",
       "      <td>53-7081.00</td>\n",
       "      <td>7183</td>\n",
       "      <td>Organize schedules for refuse collection.</td>\n",
       "      <td>Refuse and Recyclable Material Collectors</td>\n",
       "      <td>Collect and dump refuse or recyclable material...</td>\n",
       "      <td>11.57</td>\n",
       "      <td>25.97</td>\n",
       "      <td>14.88</td>\n",
       "      <td>0.00</td>\n",
       "      <td>43.02</td>\n",
       "      <td>4.56</td>\n",
       "      <td>0.00</td>\n",
       "      <td>3.29</td>\n",
       "      <td>42.06</td>\n",
       "      <td>[Schedule operational activities.]</td>\n",
       "      <td>remote</td>\n",
       "      <td>53-7081</td>\n",
       "      <td>135430.0</td>\n",
       "      <td>22.99</td>\n",
       "      <td>47810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39455</th>\n",
       "      <td>53-7121.00</td>\n",
       "      <td>12796</td>\n",
       "      <td>Record operating data such as products and qua...</td>\n",
       "      <td>Tank Car, Truck, and Ship Loaders</td>\n",
       "      <td>Load and unload chemicals and bulk solids, suc...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>2.49</td>\n",
       "      <td>2.07</td>\n",
       "      <td>0.41</td>\n",
       "      <td>45.74</td>\n",
       "      <td>27.92</td>\n",
       "      <td>21.37</td>\n",
       "      <td>4.26</td>\n",
       "      <td>90.86</td>\n",
       "      <td>[Record operational or production data.]</td>\n",
       "      <td>remote</td>\n",
       "      <td>53-7121</td>\n",
       "      <td>11400.0</td>\n",
       "      <td>29.1</td>\n",
       "      <td>60530</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>21698 rows × 20 columns</p>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ac6b9bba-3f44-4d0c-ad52-b83abce95a33')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-ac6b9bba-3f44-4d0c-ad52-b83abce95a33 button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-ac6b9bba-3f44-4d0c-ad52-b83abce95a33');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "    <div id=\"df-7ae94a8c-a85f-4e45-a4fe-25523c2eac2d\">\n",
       "      <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-7ae94a8c-a85f-4e45-a4fe-25523c2eac2d')\"\n",
       "                title=\"Suggest charts\"\n",
       "                style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "      </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "      <script>\n",
       "        async function quickchart(key) {\n",
       "          const quickchartButtonEl =\n",
       "            document.querySelector('#' + key + ' button');\n",
       "          quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "          quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "          try {\n",
       "            const charts = await google.colab.kernel.invokeFunction(\n",
       "                'suggestCharts', [key], {});\n",
       "          } catch (error) {\n",
       "            console.error('Error during call to suggestCharts:', error);\n",
       "          }\n",
       "          quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "          quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "        }\n",
       "        (() => {\n",
       "          let quickchartButtonEl =\n",
       "            document.querySelector('#df-7ae94a8c-a85f-4e45-a4fe-25523c2eac2d button');\n",
       "          quickchartButtonEl.style.display =\n",
       "            google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "        })();\n",
       "      </script>\n",
       "    </div>\n",
       "\n",
       "  <div id=\"id_321a43c5-dad7-4e5d-8e78-03fa0ed47698\">\n",
       "    <style>\n",
       "      .colab-df-generate {\n",
       "        background-color: #E8F0FE;\n",
       "        border: none;\n",
       "        border-radius: 50%;\n",
       "        cursor: pointer;\n",
       "        display: none;\n",
       "        fill: #1967D2;\n",
       "        height: 32px;\n",
       "        padding: 0 0 0 0;\n",
       "        width: 32px;\n",
       "      }\n",
       "\n",
       "      .colab-df-generate:hover {\n",
       "        background-color: #E2EBFA;\n",
       "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "        fill: #174EA6;\n",
       "      }\n",
       "\n",
       "      [theme=dark] .colab-df-generate {\n",
       "        background-color: #3B4455;\n",
       "        fill: #D2E3FC;\n",
       "      }\n",
       "\n",
       "      [theme=dark] .colab-df-generate:hover {\n",
       "        background-color: #434B5C;\n",
       "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "        fill: #FFFFFF;\n",
       "      }\n",
       "    </style>\n",
       "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('remote_df')\"\n",
       "            title=\"Generate code using this dataframe.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "    <script>\n",
       "      (() => {\n",
       "      const buttonEl =\n",
       "        document.querySelector('#id_321a43c5-dad7-4e5d-8e78-03fa0ed47698 button.colab-df-generate');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      buttonEl.onclick = () => {\n",
       "        google.colab.notebook.generateWithVariable('remote_df');\n",
       "      }\n",
       "      })();\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "      onetsoc_code  task_id  \\\n",
       "0       11-1011.00     8823   \n",
       "1       11-1011.00     8824   \n",
       "2       11-1011.00     8827   \n",
       "3       11-1011.00     8826   \n",
       "4       11-1011.00     8834   \n",
       "...            ...      ...   \n",
       "39438   53-7081.00     7172   \n",
       "39442   53-7081.00     7178   \n",
       "39443   53-7081.00     7179   \n",
       "39447   53-7081.00     7183   \n",
       "39455   53-7121.00    12796   \n",
       "\n",
       "                                                    task  \\\n",
       "0      Direct or coordinate an organization's financi...   \n",
       "1      Confer with board members, organization offici...   \n",
       "2      Prepare budgets for approval, including those ...   \n",
       "3      Direct, plan, or implement policies, objective...   \n",
       "4      Prepare or present reports concerning activiti...   \n",
       "...                                                  ...   \n",
       "39438              Fill out defective equipment reports.   \n",
       "39442  Communicate with dispatchers concerning delays...   \n",
       "39443  Check road or weather conditions to determine ...   \n",
       "39447          Organize schedules for refuse collection.   \n",
       "39455  Record operating data such as products and qua...   \n",
       "\n",
       "                                occupation_title  \\\n",
       "0                               Chief Executives   \n",
       "1                               Chief Executives   \n",
       "2                               Chief Executives   \n",
       "3                               Chief Executives   \n",
       "4                               Chief Executives   \n",
       "...                                          ...   \n",
       "39438  Refuse and Recyclable Material Collectors   \n",
       "39442  Refuse and Recyclable Material Collectors   \n",
       "39443  Refuse and Recyclable Material Collectors   \n",
       "39447  Refuse and Recyclable Material Collectors   \n",
       "39455          Tank Car, Truck, and Ship Loaders   \n",
       "\n",
       "                                  occupation_description  \\\n",
       "0      Determine and formulate policies and provide o...   \n",
       "1      Determine and formulate policies and provide o...   \n",
       "2      Determine and formulate policies and provide o...   \n",
       "3      Determine and formulate policies and provide o...   \n",
       "4      Determine and formulate policies and provide o...   \n",
       "...                                                  ...   \n",
       "39438  Collect and dump refuse or recyclable material...   \n",
       "39442  Collect and dump refuse or recyclable material...   \n",
       "39443  Collect and dump refuse or recyclable material...   \n",
       "39447  Collect and dump refuse or recyclable material...   \n",
       "39455  Load and unload chemicals and bulk solids, suc...   \n",
       "\n",
       "       frequency_category_1  frequency_category_2  frequency_category_3  \\\n",
       "0                      5.92                 15.98                 29.68   \n",
       "1                      1.42                 14.44                 27.31   \n",
       "2                     15.50                 38.21                 32.73   \n",
       "3                      3.03                 17.33                 20.30   \n",
       "4                      1.98                 14.06                 42.60   \n",
       "...                     ...                   ...                   ...   \n",
       "39438                  0.00                  1.75                  9.69   \n",
       "39442                  0.00                  1.04                  5.92   \n",
       "39443                  0.00                  8.98                  4.23   \n",
       "39447                 11.57                 25.97                 14.88   \n",
       "39455                  0.00                  2.49                  2.07   \n",
       "\n",
       "       frequency_category_4  frequency_category_5  frequency_category_6  \\\n",
       "0                     21.18                 19.71                  4.91   \n",
       "1                     25.52                 26.88                  2.52   \n",
       "2                      5.15                  5.25                  0.19   \n",
       "3                     18.10                 33.16                  2.01   \n",
       "4                     21.24                 13.18                  6.24   \n",
       "...                     ...                   ...                   ...   \n",
       "39438                  3.08                 85.29                  0.09   \n",
       "39442                  3.74                 69.00                  8.98   \n",
       "39443                  8.60                 61.70                 11.87   \n",
       "39447                  0.00                 43.02                  4.56   \n",
       "39455                  0.41                 45.74                 27.92   \n",
       "\n",
       "       frequency_category_7  importance_average  relevance_average  \\\n",
       "0                      2.63                4.52              74.44   \n",
       "1                      1.90                4.32              81.71   \n",
       "2                      2.98                4.30              93.41   \n",
       "3                      6.07                4.24              97.79   \n",
       "4                      0.70                4.17              92.92   \n",
       "...                     ...                 ...                ...   \n",
       "39438                  0.09                4.27              91.18   \n",
       "39442                 11.32                3.96              97.50   \n",
       "39443                  4.63                3.81              89.52   \n",
       "39447                  0.00                3.29              42.06   \n",
       "39455                 21.37                4.26              90.86   \n",
       "\n",
       "                                                    dwas remote_status  \\\n",
       "0                         [Direct financial operations.]        remote   \n",
       "1      [Confer with organizational members to accompl...        remote   \n",
       "2                         [Prepare operational budgets.]        remote   \n",
       "3      [Implement organizational process or policy ch...        remote   \n",
       "4      [Prepare financial documents, reports, or budg...        remote   \n",
       "...                                                  ...           ...   \n",
       "39438            [Prepare accident or incident reports.]        remote   \n",
       "39442  [Report vehicle or equipment malfunctions., No...        remote   \n",
       "39443  [Gather information about work conditions or l...        remote   \n",
       "39447                 [Schedule operational activities.]        remote   \n",
       "39455           [Record operational or production data.]        remote   \n",
       "\n",
       "      occ_code  total_employment hourly_wage_average annual_wage_average  \n",
       "0      11-1011          211230.0              124.47              258900  \n",
       "1      11-1011          211230.0              124.47              258900  \n",
       "2      11-1011          211230.0              124.47              258900  \n",
       "3      11-1011          211230.0              124.47              258900  \n",
       "4      11-1011          211230.0              124.47              258900  \n",
       "...        ...               ...                 ...                 ...  \n",
       "39438  53-7081          135430.0               22.99               47810  \n",
       "39442  53-7081          135430.0               22.99               47810  \n",
       "39443  53-7081          135430.0               22.99               47810  \n",
       "39447  53-7081          135430.0               22.99               47810  \n",
       "39455  53-7121           11400.0                29.1               60530  \n",
       "\n",
       "[21698 rows x 20 columns]"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "remote_df = df[df['remote_status'] == 'remote'].copy()\n",
    "remote_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "-RBsW3TVOfsm",
    "outputId": "e47de86d-6184-4f10-8f3b-fb5b1a254f68"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sample size:  1599\n"
     ]
    }
   ],
   "source": [
    "# We sample a N unique occupations to have about a thousands associated tasks\n",
    "unique_occ_code = remote_df['occ_code'].unique()\n",
    "np.random.shuffle(unique_occ_code)\n",
    "unique_occ_code = unique_occ_code[:25]\n",
    "remote_sample_df = remote_df[remote_df['occ_code'].isin(unique_occ_code)]\n",
    "\n",
    "print(\"Sample size: \", len(remote_sample_df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 17
    },
    "id": "fisyTqQX2ZEv",
    "outputId": "3af15e93-6767-4a5f-b7be-3270ec2b8b45"
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "    async function download(id, filename, size) {\n",
       "      if (!google.colab.kernel.accessAllowed) {\n",
       "        return;\n",
       "      }\n",
       "      const div = document.createElement('div');\n",
       "      const label = document.createElement('label');\n",
       "      label.textContent = `Downloading \"${filename}\": `;\n",
       "      div.appendChild(label);\n",
       "      const progress = document.createElement('progress');\n",
       "      progress.max = size;\n",
       "      div.appendChild(progress);\n",
       "      document.body.appendChild(div);\n",
       "\n",
       "      const buffers = [];\n",
       "      let downloaded = 0;\n",
       "\n",
       "      const channel = await google.colab.kernel.comms.open(id);\n",
       "      // Send a message to notify the kernel that we're ready.\n",
       "      channel.send({})\n",
       "\n",
       "      for await (const message of channel.messages) {\n",
       "        // Send a message to notify the kernel that we're ready.\n",
       "        channel.send({})\n",
       "        if (message.buffers) {\n",
       "          for (const buffer of message.buffers) {\n",
       "            buffers.push(buffer);\n",
       "            downloaded += buffer.byteLength;\n",
       "            progress.value = downloaded;\n",
       "          }\n",
       "        }\n",
       "      }\n",
       "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
       "      const a = document.createElement('a');\n",
       "      a.href = window.URL.createObjectURL(blob);\n",
       "      a.download = filename;\n",
       "      div.appendChild(a);\n",
       "      a.click();\n",
       "      div.remove();\n",
       "    }\n",
       "  "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "download(\"download_38eda09c-f7fe-418f-9599-1fe1c8cfa841\", \"df_sample.csv\", 580368)"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from google.colab import files\n",
    "\n",
    "remote_sample_df.to_csv('df_sample.csv', encoding = 'utf-8-sig')\n",
    "files.download('df_sample.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6aLuq9DC2QXZ"
   },
   "source": [
    "Run `uv run add_task_estimates_to_samples.py` (this calls OpenAI and might be costly), then import the df_sample.csv again, it has two new columns lb_estimate and ub_estimate."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "collapsed": true,
    "id": "dZLyRpBp2Kqc",
    "jupyter": {
     "outputs_hidden": true
    },
    "outputId": "30c0fa91-0015-43de-ab7f-dfe0be7fbd76"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Unnamed: 0', 'onetsoc_code', 'task_id', 'task', 'occupation_title',\n",
       "       'occupation_description', 'frequency_category_1',\n",
       "       'frequency_category_2', 'frequency_category_3', 'frequency_category_4',\n",
       "       'frequency_category_5', 'frequency_category_6', 'frequency_category_7',\n",
       "       'importance_average', 'relevance_average', 'remote_status', 'occ_code',\n",
       "       'total_employment', 'hourly_wage_average', 'annual_wage_average',\n",
       "       'lb_estimate', 'ub_estimate'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "enriched_sample_df = pd.read_csv(\"df_sample_with_estimates.csv\")\n",
    "enriched_sample_df.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "-fe8ybLQP6Bx"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from google.colab import files\n",
    "\n",
    "sdf = df[df['remote_status'] == 'remote'].sample(frac=1, random_state=42)  # frac=1 shuffles all rows\n",
    "sample_tasks = sdf.iloc[:45]\n",
    "tasks_df = sample_tasks[['task', 'occupation_title', 'occupation_description']].copy()\n",
    "tasks_df.to_csv('sampled_tasks.csv', index=False)\n",
    "files.download('sampled_tasks.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "collapsed": true,
    "id": "bMliWSzU1_wl",
    "jupyter": {
     "outputs_hidden": true
    },
    "outputId": "aa1a9846-9cdb-4b90-84b4-1732d84007f4"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "dataframe",
       "variable_name": "enriched_sample_df"
      },
      "text/html": [
       "\n",
       "  <div id=\"df-c6f9f750-9827-4609-a964-39629cc23935\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>onetsoc_code</th>\n",
       "      <th>task_id</th>\n",
       "      <th>task</th>\n",
       "      <th>occupation_title</th>\n",
       "      <th>occupation_description</th>\n",
       "      <th>frequency_category_1</th>\n",
       "      <th>frequency_category_2</th>\n",
       "      <th>frequency_category_3</th>\n",
       "      <th>frequency_category_4</th>\n",
       "      <th>...</th>\n",
       "      <th>relevance_average</th>\n",
       "      <th>remote_status</th>\n",
       "      <th>occ_code</th>\n",
       "      <th>total_employment</th>\n",
       "      <th>hourly_wage_average</th>\n",
       "      <th>annual_wage_average</th>\n",
       "      <th>lb_estimate</th>\n",
       "      <th>ub_estimate</th>\n",
       "      <th>lb_estimate_in_hours</th>\n",
       "      <th>ub_estimate_in_hours</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>362</td>\n",
       "      <td>11-3071.00</td>\n",
       "      <td>21343</td>\n",
       "      <td>Plan, organize, or manage the work of subordin...</td>\n",
       "      <td>Transportation, Storage, and Distribution Mana...</td>\n",
       "      <td>Plan, direct, or coordinate transportation, st...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>3.10</td>\n",
       "      <td>7.15</td>\n",
       "      <td>...</td>\n",
       "      <td>97.17</td>\n",
       "      <td>remote</td>\n",
       "      <td>11-3071</td>\n",
       "      <td>198780.0</td>\n",
       "      <td>53.79</td>\n",
       "      <td>111870</td>\n",
       "      <td>4 hours</td>\n",
       "      <td>3 days</td>\n",
       "      <td>4.0</td>\n",
       "      <td>72.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>363</td>\n",
       "      <td>11-3071.00</td>\n",
       "      <td>21344</td>\n",
       "      <td>Collaborate with other departments to integrat...</td>\n",
       "      <td>Transportation, Storage, and Distribution Mana...</td>\n",
       "      <td>Plan, direct, or coordinate transportation, st...</td>\n",
       "      <td>3.33</td>\n",
       "      <td>6.67</td>\n",
       "      <td>33.33</td>\n",
       "      <td>10.00</td>\n",
       "      <td>...</td>\n",
       "      <td>100.00</td>\n",
       "      <td>remote</td>\n",
       "      <td>11-3071</td>\n",
       "      <td>198780.0</td>\n",
       "      <td>53.79</td>\n",
       "      <td>111870</td>\n",
       "      <td>1 week</td>\n",
       "      <td>3 weeks</td>\n",
       "      <td>168.0</td>\n",
       "      <td>504.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>364</td>\n",
       "      <td>11-3071.00</td>\n",
       "      <td>21345</td>\n",
       "      <td>Analyze all aspects of corporate logistics to ...</td>\n",
       "      <td>Transportation, Storage, and Distribution Mana...</td>\n",
       "      <td>Plan, direct, or coordinate transportation, st...</td>\n",
       "      <td>10.00</td>\n",
       "      <td>13.33</td>\n",
       "      <td>20.00</td>\n",
       "      <td>26.67</td>\n",
       "      <td>...</td>\n",
       "      <td>100.00</td>\n",
       "      <td>remote</td>\n",
       "      <td>11-3071</td>\n",
       "      <td>198780.0</td>\n",
       "      <td>53.79</td>\n",
       "      <td>111870</td>\n",
       "      <td>1 week</td>\n",
       "      <td>3 weeks</td>\n",
       "      <td>168.0</td>\n",
       "      <td>504.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>366</td>\n",
       "      <td>11-3071.00</td>\n",
       "      <td>21347</td>\n",
       "      <td>Develop and document standard and emergency op...</td>\n",
       "      <td>Transportation, Storage, and Distribution Mana...</td>\n",
       "      <td>Plan, direct, or coordinate transportation, st...</td>\n",
       "      <td>23.81</td>\n",
       "      <td>23.81</td>\n",
       "      <td>28.57</td>\n",
       "      <td>9.52</td>\n",
       "      <td>...</td>\n",
       "      <td>91.67</td>\n",
       "      <td>remote</td>\n",
       "      <td>11-3071</td>\n",
       "      <td>198780.0</td>\n",
       "      <td>53.79</td>\n",
       "      <td>111870</td>\n",
       "      <td>3 days</td>\n",
       "      <td>1 week</td>\n",
       "      <td>72.0</td>\n",
       "      <td>168.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>368</td>\n",
       "      <td>11-3071.00</td>\n",
       "      <td>21349</td>\n",
       "      <td>Analyze the financial impact of proposed logis...</td>\n",
       "      <td>Transportation, Storage, and Distribution Mana...</td>\n",
       "      <td>Plan, direct, or coordinate transportation, st...</td>\n",
       "      <td>3.45</td>\n",
       "      <td>27.59</td>\n",
       "      <td>34.48</td>\n",
       "      <td>31.03</td>\n",
       "      <td>...</td>\n",
       "      <td>96.67</td>\n",
       "      <td>remote</td>\n",
       "      <td>11-3071</td>\n",
       "      <td>198780.0</td>\n",
       "      <td>53.79</td>\n",
       "      <td>111870</td>\n",
       "      <td>4 hours</td>\n",
       "      <td>1 week</td>\n",
       "      <td>4.0</td>\n",
       "      <td>168.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>747</th>\n",
       "      <td>35501</td>\n",
       "      <td>49-1011.00</td>\n",
       "      <td>15264</td>\n",
       "      <td>Review, evaluate, accept, and coordinate compl...</td>\n",
       "      <td>First-Line Supervisors of Mechanics, Installer...</td>\n",
       "      <td>Directly supervise and coordinate the activiti...</td>\n",
       "      <td>4.43</td>\n",
       "      <td>14.98</td>\n",
       "      <td>44.06</td>\n",
       "      <td>9.94</td>\n",
       "      <td>...</td>\n",
       "      <td>65.94</td>\n",
       "      <td>remote</td>\n",
       "      <td>49-1011</td>\n",
       "      <td>589880.0</td>\n",
       "      <td>37.99</td>\n",
       "      <td>79020</td>\n",
       "      <td>1 hour</td>\n",
       "      <td>3 days</td>\n",
       "      <td>1.0</td>\n",
       "      <td>72.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>748</th>\n",
       "      <td>35502</td>\n",
       "      <td>49-1011.00</td>\n",
       "      <td>2926</td>\n",
       "      <td>Compile operational or personnel records, such...</td>\n",
       "      <td>First-Line Supervisors of Mechanics, Installer...</td>\n",
       "      <td>Directly supervise and coordinate the activiti...</td>\n",
       "      <td>4.25</td>\n",
       "      <td>6.22</td>\n",
       "      <td>27.27</td>\n",
       "      <td>16.93</td>\n",
       "      <td>...</td>\n",
       "      <td>64.30</td>\n",
       "      <td>remote</td>\n",
       "      <td>49-1011</td>\n",
       "      <td>589880.0</td>\n",
       "      <td>37.99</td>\n",
       "      <td>79020</td>\n",
       "      <td>30 minutes</td>\n",
       "      <td>1 hour</td>\n",
       "      <td>0.5</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>749</th>\n",
       "      <td>35503</td>\n",
       "      <td>49-1011.00</td>\n",
       "      <td>2931</td>\n",
       "      <td>Develop or implement electronic maintenance pr...</td>\n",
       "      <td>First-Line Supervisors of Mechanics, Installer...</td>\n",
       "      <td>Directly supervise and coordinate the activiti...</td>\n",
       "      <td>3.26</td>\n",
       "      <td>3.23</td>\n",
       "      <td>9.64</td>\n",
       "      <td>18.78</td>\n",
       "      <td>...</td>\n",
       "      <td>50.24</td>\n",
       "      <td>remote</td>\n",
       "      <td>49-1011</td>\n",
       "      <td>589880.0</td>\n",
       "      <td>37.99</td>\n",
       "      <td>79020</td>\n",
       "      <td>3 weeks</td>\n",
       "      <td>6 weeks</td>\n",
       "      <td>504.0</td>\n",
       "      <td>1008.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>750</th>\n",
       "      <td>35504</td>\n",
       "      <td>49-1011.00</td>\n",
       "      <td>2932</td>\n",
       "      <td>Design equipment configurations to meet person...</td>\n",
       "      <td>First-Line Supervisors of Mechanics, Installer...</td>\n",
       "      <td>Directly supervise and coordinate the activiti...</td>\n",
       "      <td>6.07</td>\n",
       "      <td>10.37</td>\n",
       "      <td>42.18</td>\n",
       "      <td>22.01</td>\n",
       "      <td>...</td>\n",
       "      <td>57.57</td>\n",
       "      <td>remote</td>\n",
       "      <td>49-1011</td>\n",
       "      <td>589880.0</td>\n",
       "      <td>37.99</td>\n",
       "      <td>79020</td>\n",
       "      <td>4 hours</td>\n",
       "      <td>8 hours</td>\n",
       "      <td>4.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>751</th>\n",
       "      <td>36002</td>\n",
       "      <td>49-3093.00</td>\n",
       "      <td>8383</td>\n",
       "      <td>Order replacements for tires or tubes.</td>\n",
       "      <td>Tire Repairers and Changers</td>\n",
       "      <td>Repair and replace tires.</td>\n",
       "      <td>5.26</td>\n",
       "      <td>4.94</td>\n",
       "      <td>9.61</td>\n",
       "      <td>7.48</td>\n",
       "      <td>...</td>\n",
       "      <td>69.89</td>\n",
       "      <td>remote</td>\n",
       "      <td>49-3093</td>\n",
       "      <td>101520.0</td>\n",
       "      <td>17.92</td>\n",
       "      <td>37280</td>\n",
       "      <td>30 minutes</td>\n",
       "      <td>1 hour</td>\n",
       "      <td>0.5</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>752 rows × 24 columns</p>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-c6f9f750-9827-4609-a964-39629cc23935')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-c6f9f750-9827-4609-a964-39629cc23935 button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-c6f9f750-9827-4609-a964-39629cc23935');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "    <div id=\"df-75d7d34f-050b-443c-bd53-cbf5c4069f42\">\n",
       "      <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-75d7d34f-050b-443c-bd53-cbf5c4069f42')\"\n",
       "                title=\"Suggest charts\"\n",
       "                style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "      </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "      <script>\n",
       "        async function quickchart(key) {\n",
       "          const quickchartButtonEl =\n",
       "            document.querySelector('#' + key + ' button');\n",
       "          quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "          quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "          try {\n",
       "            const charts = await google.colab.kernel.invokeFunction(\n",
       "                'suggestCharts', [key], {});\n",
       "          } catch (error) {\n",
       "            console.error('Error during call to suggestCharts:', error);\n",
       "          }\n",
       "          quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "          quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "        }\n",
       "        (() => {\n",
       "          let quickchartButtonEl =\n",
       "            document.querySelector('#df-75d7d34f-050b-443c-bd53-cbf5c4069f42 button');\n",
       "          quickchartButtonEl.style.display =\n",
       "            google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "        })();\n",
       "      </script>\n",
       "    </div>\n",
       "\n",
       "  <div id=\"id_12604fbd-be00-409a-b8ac-abaff19edee3\">\n",
       "    <style>\n",
       "      .colab-df-generate {\n",
       "        background-color: #E8F0FE;\n",
       "        border: none;\n",
       "        border-radius: 50%;\n",
       "        cursor: pointer;\n",
       "        display: none;\n",
       "        fill: #1967D2;\n",
       "        height: 32px;\n",
       "        padding: 0 0 0 0;\n",
       "        width: 32px;\n",
       "      }\n",
       "\n",
       "      .colab-df-generate:hover {\n",
       "        background-color: #E2EBFA;\n",
       "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "        fill: #174EA6;\n",
       "      }\n",
       "\n",
       "      [theme=dark] .colab-df-generate {\n",
       "        background-color: #3B4455;\n",
       "        fill: #D2E3FC;\n",
       "      }\n",
       "\n",
       "      [theme=dark] .colab-df-generate:hover {\n",
       "        background-color: #434B5C;\n",
       "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "        fill: #FFFFFF;\n",
       "      }\n",
       "    </style>\n",
       "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('enriched_sample_df')\"\n",
       "            title=\"Generate code using this dataframe.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "    <script>\n",
       "      (() => {\n",
       "      const buttonEl =\n",
       "        document.querySelector('#id_12604fbd-be00-409a-b8ac-abaff19edee3 button.colab-df-generate');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      buttonEl.onclick = () => {\n",
       "        google.colab.notebook.generateWithVariable('enriched_sample_df');\n",
       "      }\n",
       "      })();\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "     Unnamed: 0 onetsoc_code  task_id  \\\n",
       "0           362   11-3071.00    21343   \n",
       "1           363   11-3071.00    21344   \n",
       "2           364   11-3071.00    21345   \n",
       "3           366   11-3071.00    21347   \n",
       "4           368   11-3071.00    21349   \n",
       "..          ...          ...      ...   \n",
       "747       35501   49-1011.00    15264   \n",
       "748       35502   49-1011.00     2926   \n",
       "749       35503   49-1011.00     2931   \n",
       "750       35504   49-1011.00     2932   \n",
       "751       36002   49-3093.00     8383   \n",
       "\n",
       "                                                  task  \\\n",
       "0    Plan, organize, or manage the work of subordin...   \n",
       "1    Collaborate with other departments to integrat...   \n",
       "2    Analyze all aspects of corporate logistics to ...   \n",
       "3    Develop and document standard and emergency op...   \n",
       "4    Analyze the financial impact of proposed logis...   \n",
       "..                                                 ...   \n",
       "747  Review, evaluate, accept, and coordinate compl...   \n",
       "748  Compile operational or personnel records, such...   \n",
       "749  Develop or implement electronic maintenance pr...   \n",
       "750  Design equipment configurations to meet person...   \n",
       "751             Order replacements for tires or tubes.   \n",
       "\n",
       "                                      occupation_title  \\\n",
       "0    Transportation, Storage, and Distribution Mana...   \n",
       "1    Transportation, Storage, and Distribution Mana...   \n",
       "2    Transportation, Storage, and Distribution Mana...   \n",
       "3    Transportation, Storage, and Distribution Mana...   \n",
       "4    Transportation, Storage, and Distribution Mana...   \n",
       "..                                                 ...   \n",
       "747  First-Line Supervisors of Mechanics, Installer...   \n",
       "748  First-Line Supervisors of Mechanics, Installer...   \n",
       "749  First-Line Supervisors of Mechanics, Installer...   \n",
       "750  First-Line Supervisors of Mechanics, Installer...   \n",
       "751                        Tire Repairers and Changers   \n",
       "\n",
       "                                occupation_description  frequency_category_1  \\\n",
       "0    Plan, direct, or coordinate transportation, st...                  0.00   \n",
       "1    Plan, direct, or coordinate transportation, st...                  3.33   \n",
       "2    Plan, direct, or coordinate transportation, st...                 10.00   \n",
       "3    Plan, direct, or coordinate transportation, st...                 23.81   \n",
       "4    Plan, direct, or coordinate transportation, st...                  3.45   \n",
       "..                                                 ...                   ...   \n",
       "747  Directly supervise and coordinate the activiti...                  4.43   \n",
       "748  Directly supervise and coordinate the activiti...                  4.25   \n",
       "749  Directly supervise and coordinate the activiti...                  3.26   \n",
       "750  Directly supervise and coordinate the activiti...                  6.07   \n",
       "751                          Repair and replace tires.                  5.26   \n",
       "\n",
       "     frequency_category_2  frequency_category_3  frequency_category_4  ...  \\\n",
       "0                    0.00                  3.10                  7.15  ...   \n",
       "1                    6.67                 33.33                 10.00  ...   \n",
       "2                   13.33                 20.00                 26.67  ...   \n",
       "3                   23.81                 28.57                  9.52  ...   \n",
       "4                   27.59                 34.48                 31.03  ...   \n",
       "..                    ...                   ...                   ...  ...   \n",
       "747                 14.98                 44.06                  9.94  ...   \n",
       "748                  6.22                 27.27                 16.93  ...   \n",
       "749                  3.23                  9.64                 18.78  ...   \n",
       "750                 10.37                 42.18                 22.01  ...   \n",
       "751                  4.94                  9.61                  7.48  ...   \n",
       "\n",
       "     relevance_average  remote_status  occ_code  total_employment  \\\n",
       "0                97.17         remote   11-3071          198780.0   \n",
       "1               100.00         remote   11-3071          198780.0   \n",
       "2               100.00         remote   11-3071          198780.0   \n",
       "3                91.67         remote   11-3071          198780.0   \n",
       "4                96.67         remote   11-3071          198780.0   \n",
       "..                 ...            ...       ...               ...   \n",
       "747              65.94         remote   49-1011          589880.0   \n",
       "748              64.30         remote   49-1011          589880.0   \n",
       "749              50.24         remote   49-1011          589880.0   \n",
       "750              57.57         remote   49-1011          589880.0   \n",
       "751              69.89         remote   49-3093          101520.0   \n",
       "\n",
       "     hourly_wage_average annual_wage_average lb_estimate  ub_estimate  \\\n",
       "0                  53.79              111870     4 hours       3 days   \n",
       "1                  53.79              111870      1 week      3 weeks   \n",
       "2                  53.79              111870      1 week      3 weeks   \n",
       "3                  53.79              111870      3 days       1 week   \n",
       "4                  53.79              111870     4 hours       1 week   \n",
       "..                   ...                 ...         ...          ...   \n",
       "747                37.99               79020      1 hour       3 days   \n",
       "748                37.99               79020  30 minutes       1 hour   \n",
       "749                37.99               79020     3 weeks      6 weeks   \n",
       "750                37.99               79020     4 hours      8 hours   \n",
       "751                17.92               37280  30 minutes       1 hour   \n",
       "\n",
       "    lb_estimate_in_hours  ub_estimate_in_hours  \n",
       "0                    4.0                  72.0  \n",
       "1                  168.0                 504.0  \n",
       "2                  168.0                 504.0  \n",
       "3                   72.0                 168.0  \n",
       "4                    4.0                 168.0  \n",
       "..                   ...                   ...  \n",
       "747                  1.0                  72.0  \n",
       "748                  0.5                   1.0  \n",
       "749                504.0                1008.0  \n",
       "750                  4.0                   8.0  \n",
       "751                  0.5                   1.0  \n",
       "\n",
       "[752 rows x 24 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DURATION_TO_HOUR_ESTIMATE = {\n",
    "    '10 minutes': 0.1,\n",
    "    '30 minutes': .5,\n",
    "    '1 hour': 1,\n",
    "    '4 hours': 4,\n",
    "    '8 hours': 8,\n",
    "    '16 hours': 16,\n",
    "    '3 days': 72,\n",
    "    '1 week': 168,\n",
    "    '3 weeks': 504,\n",
    "    '6 weeks': 1008,\n",
    "    '3 months': 3 * (365.25 / 12) * 24,\n",
    "    '6 months': 6 * (365.25 / 12) * 24,\n",
    "    '1 year': 1 * 365.25 * 24,\n",
    "    '3 years': 3 * 365.25 * 24,\n",
    "    '10 years': 10 * 365.25 * 24,\n",
    "    '30 years': 10 * 365.25 * 24,\n",
    "    '60 years': 10 * 365.25 * 24,\n",
    "}\n",
    "\n",
    "enriched_sample_df['lb_estimate_in_hours'] = enriched_sample_df['lb_estimate'].map(DURATION_TO_HOUR_ESTIMATE)\n",
    "enriched_sample_df['ub_estimate_in_hours'] = enriched_sample_df['ub_estimate'].map(DURATION_TO_HOUR_ESTIMATE)\n",
    "enriched_sample_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "collapsed": true,
    "id": "QkEXAsALhyVi",
    "jupyter": {
     "outputs_hidden": true
    },
    "outputId": "2f747bb7-7a6d-41f4-e65b-6eea0b66cbd2"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "dataframe",
       "variable_name": "hardest_task_df"
      },
      "text/html": [
       "\n",
       "  <div id=\"df-cecc1b4f-478b-4f78-ab56-1f16bb70bb1e\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>onetsoc_code</th>\n",
       "      <th>task_id</th>\n",
       "      <th>task</th>\n",
       "      <th>occupation_title</th>\n",
       "      <th>occupation_description</th>\n",
       "      <th>frequency_category_1</th>\n",
       "      <th>frequency_category_2</th>\n",
       "      <th>frequency_category_3</th>\n",
       "      <th>frequency_category_4</th>\n",
       "      <th>...</th>\n",
       "      <th>relevance_average</th>\n",
       "      <th>remote_status</th>\n",
       "      <th>occ_code</th>\n",
       "      <th>total_employment</th>\n",
       "      <th>hourly_wage_average</th>\n",
       "      <th>annual_wage_average</th>\n",
       "      <th>lb_estimate</th>\n",
       "      <th>ub_estimate</th>\n",
       "      <th>lb_estimate_in_hours</th>\n",
       "      <th>ub_estimate_in_hours</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>142</th>\n",
       "      <td>5334</td>\n",
       "      <td>23-1021.00</td>\n",
       "      <td>7627</td>\n",
       "      <td>Determine existence and amount of liability ac...</td>\n",
       "      <td>Administrative Law Judges, Adjudicators, and H...</td>\n",
       "      <td>Conduct hearings to recommend or make decision...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>15.55</td>\n",
       "      <td>6.73</td>\n",
       "      <td>10.48</td>\n",
       "      <td>...</td>\n",
       "      <td>92.69</td>\n",
       "      <td>remote</td>\n",
       "      <td>23-1021</td>\n",
       "      <td>14670.0</td>\n",
       "      <td>57.67</td>\n",
       "      <td>119940</td>\n",
       "      <td>3 days</td>\n",
       "      <td>3 weeks</td>\n",
       "      <td>72.0</td>\n",
       "      <td>504.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>698</th>\n",
       "      <td>32910</td>\n",
       "      <td>41-3011.00</td>\n",
       "      <td>4593</td>\n",
       "      <td>Identify new advertising markets, and propose ...</td>\n",
       "      <td>Advertising Sales Agents</td>\n",
       "      <td>Sell or solicit advertising space, time, or me...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>2.33</td>\n",
       "      <td>25.23</td>\n",
       "      <td>18.22</td>\n",
       "      <td>...</td>\n",
       "      <td>78.05</td>\n",
       "      <td>remote</td>\n",
       "      <td>41-3011</td>\n",
       "      <td>108100.0</td>\n",
       "      <td>36.45</td>\n",
       "      <td>75820</td>\n",
       "      <td>1 week</td>\n",
       "      <td>3 weeks</td>\n",
       "      <td>168.0</td>\n",
       "      <td>504.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>676</th>\n",
       "      <td>31849</td>\n",
       "      <td>35-3011.00</td>\n",
       "      <td>2241</td>\n",
       "      <td>Create drink recipes.</td>\n",
       "      <td>Bartenders</td>\n",
       "      <td>Mix and serve drinks to patrons, directly or t...</td>\n",
       "      <td>0.61</td>\n",
       "      <td>25.31</td>\n",
       "      <td>16.89</td>\n",
       "      <td>20.69</td>\n",
       "      <td>...</td>\n",
       "      <td>74.93</td>\n",
       "      <td>remote</td>\n",
       "      <td>35-3011</td>\n",
       "      <td>711140.0</td>\n",
       "      <td>17.83</td>\n",
       "      <td>37090</td>\n",
       "      <td>30 minutes</td>\n",
       "      <td>1 hour</td>\n",
       "      <td>0.5</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>714</th>\n",
       "      <td>33264</td>\n",
       "      <td>43-3031.00</td>\n",
       "      <td>2488</td>\n",
       "      <td>Comply with federal, state, and company polici...</td>\n",
       "      <td>Bookkeeping, Accounting, and Auditing Clerks</td>\n",
       "      <td>Compute, classify, and record numerical data t...</td>\n",
       "      <td>0.35</td>\n",
       "      <td>6.24</td>\n",
       "      <td>3.95</td>\n",
       "      <td>6.89</td>\n",
       "      <td>...</td>\n",
       "      <td>80.15</td>\n",
       "      <td>remote</td>\n",
       "      <td>43-3031</td>\n",
       "      <td>1501910.0</td>\n",
       "      <td>23.84</td>\n",
       "      <td>49580</td>\n",
       "      <td>1 hour</td>\n",
       "      <td>3 years</td>\n",
       "      <td>1.0</td>\n",
       "      <td>26298.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>682</th>\n",
       "      <td>32364</td>\n",
       "      <td>39-3092.00</td>\n",
       "      <td>9631</td>\n",
       "      <td>Recommend vendors and monitor their work.</td>\n",
       "      <td>Costume Attendants</td>\n",
       "      <td>Select, fit, and take care of costumes for cas...</td>\n",
       "      <td>39.34</td>\n",
       "      <td>26.73</td>\n",
       "      <td>16.48</td>\n",
       "      <td>8.90</td>\n",
       "      <td>...</td>\n",
       "      <td>42.45</td>\n",
       "      <td>remote</td>\n",
       "      <td>39-3092</td>\n",
       "      <td>6300.0</td>\n",
       "      <td>28.96</td>\n",
       "      <td>60230</td>\n",
       "      <td>4 hours</td>\n",
       "      <td>1 week</td>\n",
       "      <td>4.0</td>\n",
       "      <td>168.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>738</th>\n",
       "      <td>34247</td>\n",
       "      <td>45-2093.00</td>\n",
       "      <td>13426</td>\n",
       "      <td>Maintain growth, feeding, production, and cost...</td>\n",
       "      <td>Farmworkers, Farm, Ranch, and Aquacultural Ani...</td>\n",
       "      <td>Attend to live farm, ranch, open range or aqua...</td>\n",
       "      <td>0.17</td>\n",
       "      <td>18.90</td>\n",
       "      <td>43.56</td>\n",
       "      <td>27.40</td>\n",
       "      <td>...</td>\n",
       "      <td>46.50</td>\n",
       "      <td>remote</td>\n",
       "      <td>45-2093</td>\n",
       "      <td>32590.0</td>\n",
       "      <td>17.82</td>\n",
       "      <td>37060</td>\n",
       "      <td>30 minutes</td>\n",
       "      <td>1 hour</td>\n",
       "      <td>0.5</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>2965</td>\n",
       "      <td>17-2111.02</td>\n",
       "      <td>8990</td>\n",
       "      <td>Study the relationships between ignition sourc...</td>\n",
       "      <td>Fire-Prevention and Protection Engineers</td>\n",
       "      <td>Research causes of fires, determine fire prote...</td>\n",
       "      <td>27.78</td>\n",
       "      <td>38.89</td>\n",
       "      <td>16.67</td>\n",
       "      <td>5.56</td>\n",
       "      <td>...</td>\n",
       "      <td>90.00</td>\n",
       "      <td>remote</td>\n",
       "      <td>17-2111</td>\n",
       "      <td>22510.0</td>\n",
       "      <td>52.28</td>\n",
       "      <td>108740</td>\n",
       "      <td>3 days</td>\n",
       "      <td>6 weeks</td>\n",
       "      <td>72.0</td>\n",
       "      <td>1008.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>749</th>\n",
       "      <td>35503</td>\n",
       "      <td>49-1011.00</td>\n",
       "      <td>2931</td>\n",
       "      <td>Develop or implement electronic maintenance pr...</td>\n",
       "      <td>First-Line Supervisors of Mechanics, Installer...</td>\n",
       "      <td>Directly supervise and coordinate the activiti...</td>\n",
       "      <td>3.26</td>\n",
       "      <td>3.23</td>\n",
       "      <td>9.64</td>\n",
       "      <td>18.78</td>\n",
       "      <td>...</td>\n",
       "      <td>50.24</td>\n",
       "      <td>remote</td>\n",
       "      <td>49-1011</td>\n",
       "      <td>589880.0</td>\n",
       "      <td>37.99</td>\n",
       "      <td>79020</td>\n",
       "      <td>3 weeks</td>\n",
       "      <td>6 weeks</td>\n",
       "      <td>504.0</td>\n",
       "      <td>1008.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125</th>\n",
       "      <td>4770</td>\n",
       "      <td>19-4043.00</td>\n",
       "      <td>22270</td>\n",
       "      <td>Interview individuals, and research public dat...</td>\n",
       "      <td>Geological Technicians, Except Hydrologic Tech...</td>\n",
       "      <td>Assist scientists or engineers in the use of e...</td>\n",
       "      <td>5.12</td>\n",
       "      <td>19.99</td>\n",
       "      <td>30.66</td>\n",
       "      <td>25.32</td>\n",
       "      <td>...</td>\n",
       "      <td>75.63</td>\n",
       "      <td>remote</td>\n",
       "      <td>19-4043</td>\n",
       "      <td>8860.0</td>\n",
       "      <td>31.05</td>\n",
       "      <td>64590</td>\n",
       "      <td>3 days</td>\n",
       "      <td>1 week</td>\n",
       "      <td>72.0</td>\n",
       "      <td>168.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>4274</td>\n",
       "      <td>19-2042.00</td>\n",
       "      <td>19769</td>\n",
       "      <td>Research geomechanical or geochemical processe...</td>\n",
       "      <td>Geoscientists, Except Hydrologists and Geograp...</td>\n",
       "      <td>Study the composition, structure, and other ph...</td>\n",
       "      <td>75.00</td>\n",
       "      <td>8.33</td>\n",
       "      <td>8.33</td>\n",
       "      <td>8.33</td>\n",
       "      <td>...</td>\n",
       "      <td>38.71</td>\n",
       "      <td>remote</td>\n",
       "      <td>19-2042</td>\n",
       "      <td>24620.0</td>\n",
       "      <td>50</td>\n",
       "      <td>104000</td>\n",
       "      <td>3 weeks</td>\n",
       "      <td>6 weeks</td>\n",
       "      <td>504.0</td>\n",
       "      <td>1008.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>2946</td>\n",
       "      <td>17-2111.00</td>\n",
       "      <td>21877</td>\n",
       "      <td>Develop industry standards of product safety.</td>\n",
       "      <td>Health and Safety Engineers, Except Mining Saf...</td>\n",
       "      <td>Promote worksite or product safety by applying...</td>\n",
       "      <td>23.53</td>\n",
       "      <td>29.41</td>\n",
       "      <td>35.29</td>\n",
       "      <td>11.76</td>\n",
       "      <td>...</td>\n",
       "      <td>78.26</td>\n",
       "      <td>remote</td>\n",
       "      <td>17-2111</td>\n",
       "      <td>22510.0</td>\n",
       "      <td>52.28</td>\n",
       "      <td>108740</td>\n",
       "      <td>3 weeks</td>\n",
       "      <td>6 weeks</td>\n",
       "      <td>504.0</td>\n",
       "      <td>1008.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>4390</td>\n",
       "      <td>19-3032.00</td>\n",
       "      <td>7567</td>\n",
       "      <td>Formulate and implement training programs, app...</td>\n",
       "      <td>Industrial-Organizational Psychologists</td>\n",
       "      <td>Apply principles of psychology to human resour...</td>\n",
       "      <td>15.38</td>\n",
       "      <td>42.31</td>\n",
       "      <td>19.23</td>\n",
       "      <td>15.38</td>\n",
       "      <td>...</td>\n",
       "      <td>100.00</td>\n",
       "      <td>remote</td>\n",
       "      <td>19-3032</td>\n",
       "      <td>1030.0</td>\n",
       "      <td>74.22</td>\n",
       "      <td>154380</td>\n",
       "      <td>3 weeks</td>\n",
       "      <td>6 weeks</td>\n",
       "      <td>504.0</td>\n",
       "      <td>1008.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>740</th>\n",
       "      <td>34749</td>\n",
       "      <td>47-2132.00</td>\n",
       "      <td>13560</td>\n",
       "      <td>Read blueprints and specifications to determin...</td>\n",
       "      <td>Insulation Workers, Mechanical</td>\n",
       "      <td>Apply insulating materials to pipes or ductwor...</td>\n",
       "      <td>4.81</td>\n",
       "      <td>3.22</td>\n",
       "      <td>16.08</td>\n",
       "      <td>29.99</td>\n",
       "      <td>...</td>\n",
       "      <td>95.42</td>\n",
       "      <td>remote</td>\n",
       "      <td>47-2132</td>\n",
       "      <td>22850.0</td>\n",
       "      <td>29.1</td>\n",
       "      <td>60530</td>\n",
       "      <td>30 minutes</td>\n",
       "      <td>1 hour</td>\n",
       "      <td>0.5</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>651</th>\n",
       "      <td>26473</td>\n",
       "      <td>25-2012.00</td>\n",
       "      <td>6519</td>\n",
       "      <td>Collaborate with other teachers and administra...</td>\n",
       "      <td>Kindergarten Teachers, Except Special Education</td>\n",
       "      <td>Teach academic and social skills to kindergart...</td>\n",
       "      <td>3.51</td>\n",
       "      <td>8.18</td>\n",
       "      <td>24.22</td>\n",
       "      <td>27.09</td>\n",
       "      <td>...</td>\n",
       "      <td>97.10</td>\n",
       "      <td>remote</td>\n",
       "      <td>25-2012</td>\n",
       "      <td>118580.0</td>\n",
       "      <td>*</td>\n",
       "      <td>67790</td>\n",
       "      <td>3 days</td>\n",
       "      <td>3 weeks</td>\n",
       "      <td>72.0</td>\n",
       "      <td>504.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>359</th>\n",
       "      <td>20317</td>\n",
       "      <td>25-1112.00</td>\n",
       "      <td>6222</td>\n",
       "      <td>Maintain regularly scheduled office hours to a...</td>\n",
       "      <td>Law Teachers, Postsecondary</td>\n",
       "      <td>Teach courses in law. Includes both teachers p...</td>\n",
       "      <td>1.02</td>\n",
       "      <td>4.08</td>\n",
       "      <td>17.48</td>\n",
       "      <td>50.25</td>\n",
       "      <td>...</td>\n",
       "      <td>91.64</td>\n",
       "      <td>remote</td>\n",
       "      <td>25-1112</td>\n",
       "      <td>14570.0</td>\n",
       "      <td>*</td>\n",
       "      <td>142440</td>\n",
       "      <td>1 hour</td>\n",
       "      <td>3 years</td>\n",
       "      <td>1.0</td>\n",
       "      <td>26298.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>140</th>\n",
       "      <td>5069</td>\n",
       "      <td>21-1013.00</td>\n",
       "      <td>9196</td>\n",
       "      <td>Provide public education and consultation to o...</td>\n",
       "      <td>Marriage and Family Therapists</td>\n",
       "      <td>Diagnose and treat mental and emotional disord...</td>\n",
       "      <td>17.32</td>\n",
       "      <td>31.77</td>\n",
       "      <td>33.40</td>\n",
       "      <td>13.85</td>\n",
       "      <td>...</td>\n",
       "      <td>72.63</td>\n",
       "      <td>remote</td>\n",
       "      <td>21-1013</td>\n",
       "      <td>63340.0</td>\n",
       "      <td>33.04</td>\n",
       "      <td>68730</td>\n",
       "      <td>3 days</td>\n",
       "      <td>1 week</td>\n",
       "      <td>72.0</td>\n",
       "      <td>168.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132</th>\n",
       "      <td>4965</td>\n",
       "      <td>19-5012.00</td>\n",
       "      <td>11097</td>\n",
       "      <td>Prepare documents to be used in legal proceedi...</td>\n",
       "      <td>Occupational Health and Safety Technicians</td>\n",
       "      <td>Collect data on work environments for analysis...</td>\n",
       "      <td>75.00</td>\n",
       "      <td>18.75</td>\n",
       "      <td>6.25</td>\n",
       "      <td>0.00</td>\n",
       "      <td>...</td>\n",
       "      <td>76.19</td>\n",
       "      <td>remote</td>\n",
       "      <td>19-5012</td>\n",
       "      <td>27270.0</td>\n",
       "      <td>30.89</td>\n",
       "      <td>64250</td>\n",
       "      <td>3 days</td>\n",
       "      <td>1 week</td>\n",
       "      <td>72.0</td>\n",
       "      <td>168.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>674</th>\n",
       "      <td>31127</td>\n",
       "      <td>31-9095.00</td>\n",
       "      <td>2057</td>\n",
       "      <td>Prepare, maintain, and record records of inven...</td>\n",
       "      <td>Pharmacy Aides</td>\n",
       "      <td>Record drugs delivered to the pharmacy, store ...</td>\n",
       "      <td>2.09</td>\n",
       "      <td>0.24</td>\n",
       "      <td>7.58</td>\n",
       "      <td>33.19</td>\n",
       "      <td>...</td>\n",
       "      <td>61.29</td>\n",
       "      <td>remote</td>\n",
       "      <td>31-9095</td>\n",
       "      <td>43830.0</td>\n",
       "      <td>18.74</td>\n",
       "      <td>38980</td>\n",
       "      <td>1 hour</td>\n",
       "      <td>3 days</td>\n",
       "      <td>1.0</td>\n",
       "      <td>72.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>736</th>\n",
       "      <td>33799</td>\n",
       "      <td>43-5051.00</td>\n",
       "      <td>20905</td>\n",
       "      <td>Order retail items and other supplies for offi...</td>\n",
       "      <td>Postal Service Clerks</td>\n",
       "      <td>Perform any combination of tasks in a United S...</td>\n",
       "      <td>5.22</td>\n",
       "      <td>9.12</td>\n",
       "      <td>28.52</td>\n",
       "      <td>37.90</td>\n",
       "      <td>...</td>\n",
       "      <td>41.14</td>\n",
       "      <td>remote</td>\n",
       "      <td>43-5051</td>\n",
       "      <td>78130.0</td>\n",
       "      <td>28.48</td>\n",
       "      <td>59240</td>\n",
       "      <td>30 minutes</td>\n",
       "      <td>3 days</td>\n",
       "      <td>0.5</td>\n",
       "      <td>72.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>670</th>\n",
       "      <td>29100</td>\n",
       "      <td>27-4014.00</td>\n",
       "      <td>18661</td>\n",
       "      <td>Convert video and audio recordings into digita...</td>\n",
       "      <td>Sound Engineering Technicians</td>\n",
       "      <td>Assemble and operate equipment to record, sync...</td>\n",
       "      <td>8.00</td>\n",
       "      <td>16.00</td>\n",
       "      <td>36.00</td>\n",
       "      <td>8.00</td>\n",
       "      <td>...</td>\n",
       "      <td>92.59</td>\n",
       "      <td>remote</td>\n",
       "      <td>27-4014</td>\n",
       "      <td>14600.0</td>\n",
       "      <td>35.62</td>\n",
       "      <td>74100</td>\n",
       "      <td>4 hours</td>\n",
       "      <td>8 hours</td>\n",
       "      <td>4.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>403</td>\n",
       "      <td>11-3071.04</td>\n",
       "      <td>15692</td>\n",
       "      <td>Develop or implement procedures or systems to ...</td>\n",
       "      <td>Supply Chain Managers</td>\n",
       "      <td>Direct or coordinate production, purchasing, w...</td>\n",
       "      <td>55.56</td>\n",
       "      <td>27.78</td>\n",
       "      <td>16.67</td>\n",
       "      <td>0.00</td>\n",
       "      <td>...</td>\n",
       "      <td>85.71</td>\n",
       "      <td>remote</td>\n",
       "      <td>11-3071</td>\n",
       "      <td>198780.0</td>\n",
       "      <td>53.79</td>\n",
       "      <td>111870</td>\n",
       "      <td>3 weeks</td>\n",
       "      <td>3 months</td>\n",
       "      <td>504.0</td>\n",
       "      <td>2191.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>706</th>\n",
       "      <td>33145</td>\n",
       "      <td>41-9041.00</td>\n",
       "      <td>4618</td>\n",
       "      <td>Deliver prepared sales talks, reading from scr...</td>\n",
       "      <td>Telemarketers</td>\n",
       "      <td>Solicit donations or orders for goods or servi...</td>\n",
       "      <td>7.95</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.94</td>\n",
       "      <td>...</td>\n",
       "      <td>85.23</td>\n",
       "      <td>remote</td>\n",
       "      <td>41-9041</td>\n",
       "      <td>81580.0</td>\n",
       "      <td>17.64</td>\n",
       "      <td>36680</td>\n",
       "      <td>1 hour</td>\n",
       "      <td>3 days</td>\n",
       "      <td>1.0</td>\n",
       "      <td>72.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>739</th>\n",
       "      <td>34499</td>\n",
       "      <td>47-2044.00</td>\n",
       "      <td>2854</td>\n",
       "      <td>Prepare cost and labor estimates, based on cal...</td>\n",
       "      <td>Tile and Stone Setters</td>\n",
       "      <td>Apply hard tile, stone, and comparable materia...</td>\n",
       "      <td>22.25</td>\n",
       "      <td>0.95</td>\n",
       "      <td>22.13</td>\n",
       "      <td>36.75</td>\n",
       "      <td>...</td>\n",
       "      <td>46.55</td>\n",
       "      <td>remote</td>\n",
       "      <td>47-2044</td>\n",
       "      <td>42420.0</td>\n",
       "      <td>25.92</td>\n",
       "      <td>53920</td>\n",
       "      <td>1 hour</td>\n",
       "      <td>3 days</td>\n",
       "      <td>1.0</td>\n",
       "      <td>72.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>751</th>\n",
       "      <td>36002</td>\n",
       "      <td>49-3093.00</td>\n",
       "      <td>8383</td>\n",
       "      <td>Order replacements for tires or tubes.</td>\n",
       "      <td>Tire Repairers and Changers</td>\n",
       "      <td>Repair and replace tires.</td>\n",
       "      <td>5.26</td>\n",
       "      <td>4.94</td>\n",
       "      <td>9.61</td>\n",
       "      <td>7.48</td>\n",
       "      <td>...</td>\n",
       "      <td>69.89</td>\n",
       "      <td>remote</td>\n",
       "      <td>49-3093</td>\n",
       "      <td>101520.0</td>\n",
       "      <td>17.92</td>\n",
       "      <td>37280</td>\n",
       "      <td>30 minutes</td>\n",
       "      <td>1 hour</td>\n",
       "      <td>0.5</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>364</td>\n",
       "      <td>11-3071.00</td>\n",
       "      <td>21345</td>\n",
       "      <td>Analyze all aspects of corporate logistics to ...</td>\n",
       "      <td>Transportation, Storage, and Distribution Mana...</td>\n",
       "      <td>Plan, direct, or coordinate transportation, st...</td>\n",
       "      <td>10.00</td>\n",
       "      <td>13.33</td>\n",
       "      <td>20.00</td>\n",
       "      <td>26.67</td>\n",
       "      <td>...</td>\n",
       "      <td>100.00</td>\n",
       "      <td>remote</td>\n",
       "      <td>11-3071</td>\n",
       "      <td>198780.0</td>\n",
       "      <td>53.79</td>\n",
       "      <td>111870</td>\n",
       "      <td>1 week</td>\n",
       "      <td>3 weeks</td>\n",
       "      <td>168.0</td>\n",
       "      <td>504.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118</th>\n",
       "      <td>4534</td>\n",
       "      <td>19-3051.00</td>\n",
       "      <td>233</td>\n",
       "      <td>Supervise or coordinate the work of urban plan...</td>\n",
       "      <td>Urban and Regional Planners</td>\n",
       "      <td>Develop comprehensive plans and programs for u...</td>\n",
       "      <td>4.17</td>\n",
       "      <td>4.17</td>\n",
       "      <td>8.33</td>\n",
       "      <td>20.83</td>\n",
       "      <td>...</td>\n",
       "      <td>96.00</td>\n",
       "      <td>remote</td>\n",
       "      <td>19-3051</td>\n",
       "      <td>42690.0</td>\n",
       "      <td>41.32</td>\n",
       "      <td>85940</td>\n",
       "      <td>3 weeks</td>\n",
       "      <td>6 weeks</td>\n",
       "      <td>504.0</td>\n",
       "      <td>1008.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>3846</td>\n",
       "      <td>19-1023.00</td>\n",
       "      <td>23958</td>\n",
       "      <td>Conduct literature reviews.</td>\n",
       "      <td>Zoologists and Wildlife Biologists</td>\n",
       "      <td>Study the origins, behavior, diseases, genetic...</td>\n",
       "      <td>16.50</td>\n",
       "      <td>36.10</td>\n",
       "      <td>21.62</td>\n",
       "      <td>17.76</td>\n",
       "      <td>...</td>\n",
       "      <td>82.88</td>\n",
       "      <td>remote</td>\n",
       "      <td>19-1023</td>\n",
       "      <td>17100.0</td>\n",
       "      <td>36.41</td>\n",
       "      <td>75740</td>\n",
       "      <td>1 week</td>\n",
       "      <td>3 weeks</td>\n",
       "      <td>168.0</td>\n",
       "      <td>504.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>27 rows × 24 columns</p>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-cecc1b4f-478b-4f78-ab56-1f16bb70bb1e')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-cecc1b4f-478b-4f78-ab56-1f16bb70bb1e button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-cecc1b4f-478b-4f78-ab56-1f16bb70bb1e');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "    <div id=\"df-3d0f60a8-2728-46d7-9a6c-db0f3922801f\">\n",
       "      <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-3d0f60a8-2728-46d7-9a6c-db0f3922801f')\"\n",
       "                title=\"Suggest charts\"\n",
       "                style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "      </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "      <script>\n",
       "        async function quickchart(key) {\n",
       "          const quickchartButtonEl =\n",
       "            document.querySelector('#' + key + ' button');\n",
       "          quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "          quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "          try {\n",
       "            const charts = await google.colab.kernel.invokeFunction(\n",
       "                'suggestCharts', [key], {});\n",
       "          } catch (error) {\n",
       "            console.error('Error during call to suggestCharts:', error);\n",
       "          }\n",
       "          quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "          quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "        }\n",
       "        (() => {\n",
       "          let quickchartButtonEl =\n",
       "            document.querySelector('#df-3d0f60a8-2728-46d7-9a6c-db0f3922801f button');\n",
       "          quickchartButtonEl.style.display =\n",
       "            google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "        })();\n",
       "      </script>\n",
       "    </div>\n",
       "\n",
       "  <div id=\"id_690b2ac3-8d5d-4f80-b9b4-04583e2f04fd\">\n",
       "    <style>\n",
       "      .colab-df-generate {\n",
       "        background-color: #E8F0FE;\n",
       "        border: none;\n",
       "        border-radius: 50%;\n",
       "        cursor: pointer;\n",
       "        display: none;\n",
       "        fill: #1967D2;\n",
       "        height: 32px;\n",
       "        padding: 0 0 0 0;\n",
       "        width: 32px;\n",
       "      }\n",
       "\n",
       "      .colab-df-generate:hover {\n",
       "        background-color: #E2EBFA;\n",
       "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "        fill: #174EA6;\n",
       "      }\n",
       "\n",
       "      [theme=dark] .colab-df-generate {\n",
       "        background-color: #3B4455;\n",
       "        fill: #D2E3FC;\n",
       "      }\n",
       "\n",
       "      [theme=dark] .colab-df-generate:hover {\n",
       "        background-color: #434B5C;\n",
       "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "        fill: #FFFFFF;\n",
       "      }\n",
       "    </style>\n",
       "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('hardest_task_df')\"\n",
       "            title=\"Generate code using this dataframe.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "    <script>\n",
       "      (() => {\n",
       "      const buttonEl =\n",
       "        document.querySelector('#id_690b2ac3-8d5d-4f80-b9b4-04583e2f04fd button.colab-df-generate');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      buttonEl.onclick = () => {\n",
       "        google.colab.notebook.generateWithVariable('hardest_task_df');\n",
       "      }\n",
       "      })();\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "     Unnamed: 0 onetsoc_code  task_id  \\\n",
       "142        5334   23-1021.00     7627   \n",
       "698       32910   41-3011.00     4593   \n",
       "676       31849   35-3011.00     2241   \n",
       "714       33264   43-3031.00     2488   \n",
       "682       32364   39-3092.00     9631   \n",
       "738       34247   45-2093.00    13426   \n",
       "65         2965   17-2111.02     8990   \n",
       "749       35503   49-1011.00     2931   \n",
       "125        4770   19-4043.00    22270   \n",
       "81         4274   19-2042.00    19769   \n",
       "56         2946   17-2111.00    21877   \n",
       "97         4390   19-3032.00     7567   \n",
       "740       34749   47-2132.00    13560   \n",
       "651       26473   25-2012.00     6519   \n",
       "359       20317   25-1112.00     6222   \n",
       "140        5069   21-1013.00     9196   \n",
       "132        4965   19-5012.00    11097   \n",
       "674       31127   31-9095.00     2057   \n",
       "736       33799   43-5051.00    20905   \n",
       "670       29100   27-4014.00    18661   \n",
       "31          403   11-3071.04    15692   \n",
       "706       33145   41-9041.00     4618   \n",
       "739       34499   47-2044.00     2854   \n",
       "751       36002   49-3093.00     8383   \n",
       "2           364   11-3071.00    21345   \n",
       "118        4534   19-3051.00      233   \n",
       "70         3846   19-1023.00    23958   \n",
       "\n",
       "                                                  task  \\\n",
       "142  Determine existence and amount of liability ac...   \n",
       "698  Identify new advertising markets, and propose ...   \n",
       "676                              Create drink recipes.   \n",
       "714  Comply with federal, state, and company polici...   \n",
       "682          Recommend vendors and monitor their work.   \n",
       "738  Maintain growth, feeding, production, and cost...   \n",
       "65   Study the relationships between ignition sourc...   \n",
       "749  Develop or implement electronic maintenance pr...   \n",
       "125  Interview individuals, and research public dat...   \n",
       "81   Research geomechanical or geochemical processe...   \n",
       "56       Develop industry standards of product safety.   \n",
       "97   Formulate and implement training programs, app...   \n",
       "740  Read blueprints and specifications to determin...   \n",
       "651  Collaborate with other teachers and administra...   \n",
       "359  Maintain regularly scheduled office hours to a...   \n",
       "140  Provide public education and consultation to o...   \n",
       "132  Prepare documents to be used in legal proceedi...   \n",
       "674  Prepare, maintain, and record records of inven...   \n",
       "736  Order retail items and other supplies for offi...   \n",
       "670  Convert video and audio recordings into digita...   \n",
       "31   Develop or implement procedures or systems to ...   \n",
       "706  Deliver prepared sales talks, reading from scr...   \n",
       "739  Prepare cost and labor estimates, based on cal...   \n",
       "751             Order replacements for tires or tubes.   \n",
       "2    Analyze all aspects of corporate logistics to ...   \n",
       "118  Supervise or coordinate the work of urban plan...   \n",
       "70                         Conduct literature reviews.   \n",
       "\n",
       "                                      occupation_title  \\\n",
       "142  Administrative Law Judges, Adjudicators, and H...   \n",
       "698                           Advertising Sales Agents   \n",
       "676                                         Bartenders   \n",
       "714       Bookkeeping, Accounting, and Auditing Clerks   \n",
       "682                                 Costume Attendants   \n",
       "738  Farmworkers, Farm, Ranch, and Aquacultural Ani...   \n",
       "65            Fire-Prevention and Protection Engineers   \n",
       "749  First-Line Supervisors of Mechanics, Installer...   \n",
       "125  Geological Technicians, Except Hydrologic Tech...   \n",
       "81   Geoscientists, Except Hydrologists and Geograp...   \n",
       "56   Health and Safety Engineers, Except Mining Saf...   \n",
       "97             Industrial-Organizational Psychologists   \n",
       "740                     Insulation Workers, Mechanical   \n",
       "651    Kindergarten Teachers, Except Special Education   \n",
       "359                        Law Teachers, Postsecondary   \n",
       "140                     Marriage and Family Therapists   \n",
       "132         Occupational Health and Safety Technicians   \n",
       "674                                     Pharmacy Aides   \n",
       "736                              Postal Service Clerks   \n",
       "670                      Sound Engineering Technicians   \n",
       "31                               Supply Chain Managers   \n",
       "706                                      Telemarketers   \n",
       "739                             Tile and Stone Setters   \n",
       "751                        Tire Repairers and Changers   \n",
       "2    Transportation, Storage, and Distribution Mana...   \n",
       "118                        Urban and Regional Planners   \n",
       "70                  Zoologists and Wildlife Biologists   \n",
       "\n",
       "                                occupation_description  frequency_category_1  \\\n",
       "142  Conduct hearings to recommend or make decision...                  0.00   \n",
       "698  Sell or solicit advertising space, time, or me...                  0.00   \n",
       "676  Mix and serve drinks to patrons, directly or t...                  0.61   \n",
       "714  Compute, classify, and record numerical data t...                  0.35   \n",
       "682  Select, fit, and take care of costumes for cas...                 39.34   \n",
       "738  Attend to live farm, ranch, open range or aqua...                  0.17   \n",
       "65   Research causes of fires, determine fire prote...                 27.78   \n",
       "749  Directly supervise and coordinate the activiti...                  3.26   \n",
       "125  Assist scientists or engineers in the use of e...                  5.12   \n",
       "81   Study the composition, structure, and other ph...                 75.00   \n",
       "56   Promote worksite or product safety by applying...                 23.53   \n",
       "97   Apply principles of psychology to human resour...                 15.38   \n",
       "740  Apply insulating materials to pipes or ductwor...                  4.81   \n",
       "651  Teach academic and social skills to kindergart...                  3.51   \n",
       "359  Teach courses in law. Includes both teachers p...                  1.02   \n",
       "140  Diagnose and treat mental and emotional disord...                 17.32   \n",
       "132  Collect data on work environments for analysis...                 75.00   \n",
       "674  Record drugs delivered to the pharmacy, store ...                  2.09   \n",
       "736  Perform any combination of tasks in a United S...                  5.22   \n",
       "670  Assemble and operate equipment to record, sync...                  8.00   \n",
       "31   Direct or coordinate production, purchasing, w...                 55.56   \n",
       "706  Solicit donations or orders for goods or servi...                  7.95   \n",
       "739  Apply hard tile, stone, and comparable materia...                 22.25   \n",
       "751                          Repair and replace tires.                  5.26   \n",
       "2    Plan, direct, or coordinate transportation, st...                 10.00   \n",
       "118  Develop comprehensive plans and programs for u...                  4.17   \n",
       "70   Study the origins, behavior, diseases, genetic...                 16.50   \n",
       "\n",
       "     frequency_category_2  frequency_category_3  frequency_category_4  ...  \\\n",
       "142                 15.55                  6.73                 10.48  ...   \n",
       "698                  2.33                 25.23                 18.22  ...   \n",
       "676                 25.31                 16.89                 20.69  ...   \n",
       "714                  6.24                  3.95                  6.89  ...   \n",
       "682                 26.73                 16.48                  8.90  ...   \n",
       "738                 18.90                 43.56                 27.40  ...   \n",
       "65                  38.89                 16.67                  5.56  ...   \n",
       "749                  3.23                  9.64                 18.78  ...   \n",
       "125                 19.99                 30.66                 25.32  ...   \n",
       "81                   8.33                  8.33                  8.33  ...   \n",
       "56                  29.41                 35.29                 11.76  ...   \n",
       "97                  42.31                 19.23                 15.38  ...   \n",
       "740                  3.22                 16.08                 29.99  ...   \n",
       "651                  8.18                 24.22                 27.09  ...   \n",
       "359                  4.08                 17.48                 50.25  ...   \n",
       "140                 31.77                 33.40                 13.85  ...   \n",
       "132                 18.75                  6.25                  0.00  ...   \n",
       "674                  0.24                  7.58                 33.19  ...   \n",
       "736                  9.12                 28.52                 37.90  ...   \n",
       "670                 16.00                 36.00                  8.00  ...   \n",
       "31                  27.78                 16.67                  0.00  ...   \n",
       "706                  0.00                  0.00                  0.94  ...   \n",
       "739                  0.95                 22.13                 36.75  ...   \n",
       "751                  4.94                  9.61                  7.48  ...   \n",
       "2                   13.33                 20.00                 26.67  ...   \n",
       "118                  4.17                  8.33                 20.83  ...   \n",
       "70                  36.10                 21.62                 17.76  ...   \n",
       "\n",
       "     relevance_average  remote_status  occ_code  total_employment  \\\n",
       "142              92.69         remote   23-1021           14670.0   \n",
       "698              78.05         remote   41-3011          108100.0   \n",
       "676              74.93         remote   35-3011          711140.0   \n",
       "714              80.15         remote   43-3031         1501910.0   \n",
       "682              42.45         remote   39-3092            6300.0   \n",
       "738              46.50         remote   45-2093           32590.0   \n",
       "65               90.00         remote   17-2111           22510.0   \n",
       "749              50.24         remote   49-1011          589880.0   \n",
       "125              75.63         remote   19-4043            8860.0   \n",
       "81               38.71         remote   19-2042           24620.0   \n",
       "56               78.26         remote   17-2111           22510.0   \n",
       "97              100.00         remote   19-3032            1030.0   \n",
       "740              95.42         remote   47-2132           22850.0   \n",
       "651              97.10         remote   25-2012          118580.0   \n",
       "359              91.64         remote   25-1112           14570.0   \n",
       "140              72.63         remote   21-1013           63340.0   \n",
       "132              76.19         remote   19-5012           27270.0   \n",
       "674              61.29         remote   31-9095           43830.0   \n",
       "736              41.14         remote   43-5051           78130.0   \n",
       "670              92.59         remote   27-4014           14600.0   \n",
       "31               85.71         remote   11-3071          198780.0   \n",
       "706              85.23         remote   41-9041           81580.0   \n",
       "739              46.55         remote   47-2044           42420.0   \n",
       "751              69.89         remote   49-3093          101520.0   \n",
       "2               100.00         remote   11-3071          198780.0   \n",
       "118              96.00         remote   19-3051           42690.0   \n",
       "70               82.88         remote   19-1023           17100.0   \n",
       "\n",
       "     hourly_wage_average annual_wage_average lb_estimate  ub_estimate  \\\n",
       "142                57.67              119940      3 days      3 weeks   \n",
       "698                36.45               75820      1 week      3 weeks   \n",
       "676                17.83               37090  30 minutes       1 hour   \n",
       "714                23.84               49580      1 hour      3 years   \n",
       "682                28.96               60230     4 hours       1 week   \n",
       "738                17.82               37060  30 minutes       1 hour   \n",
       "65                 52.28              108740      3 days      6 weeks   \n",
       "749                37.99               79020     3 weeks      6 weeks   \n",
       "125                31.05               64590      3 days       1 week   \n",
       "81                    50              104000     3 weeks      6 weeks   \n",
       "56                 52.28              108740     3 weeks      6 weeks   \n",
       "97                 74.22              154380     3 weeks      6 weeks   \n",
       "740                 29.1               60530  30 minutes       1 hour   \n",
       "651                    *               67790      3 days      3 weeks   \n",
       "359                    *              142440      1 hour      3 years   \n",
       "140                33.04               68730      3 days       1 week   \n",
       "132                30.89               64250      3 days       1 week   \n",
       "674                18.74               38980      1 hour       3 days   \n",
       "736                28.48               59240  30 minutes       3 days   \n",
       "670                35.62               74100     4 hours      8 hours   \n",
       "31                 53.79              111870     3 weeks     3 months   \n",
       "706                17.64               36680      1 hour       3 days   \n",
       "739                25.92               53920      1 hour       3 days   \n",
       "751                17.92               37280  30 minutes       1 hour   \n",
       "2                  53.79              111870      1 week      3 weeks   \n",
       "118                41.32               85940     3 weeks      6 weeks   \n",
       "70                 36.41               75740      1 week      3 weeks   \n",
       "\n",
       "    lb_estimate_in_hours  ub_estimate_in_hours  \n",
       "142                 72.0                 504.0  \n",
       "698                168.0                 504.0  \n",
       "676                  0.5                   1.0  \n",
       "714                  1.0               26298.0  \n",
       "682                  4.0                 168.0  \n",
       "738                  0.5                   1.0  \n",
       "65                  72.0                1008.0  \n",
       "749                504.0                1008.0  \n",
       "125                 72.0                 168.0  \n",
       "81                 504.0                1008.0  \n",
       "56                 504.0                1008.0  \n",
       "97                 504.0                1008.0  \n",
       "740                  0.5                   1.0  \n",
       "651                 72.0                 504.0  \n",
       "359                  1.0               26298.0  \n",
       "140                 72.0                 168.0  \n",
       "132                 72.0                 168.0  \n",
       "674                  1.0                  72.0  \n",
       "736                  0.5                  72.0  \n",
       "670                  4.0                   8.0  \n",
       "31                 504.0                2191.5  \n",
       "706                  1.0                  72.0  \n",
       "739                  1.0                  72.0  \n",
       "751                  0.5                   1.0  \n",
       "2                  168.0                 504.0  \n",
       "118                504.0                1008.0  \n",
       "70                 168.0                 504.0  \n",
       "\n",
       "[27 rows x 24 columns]"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hardest_task_df = enriched_sample_df.loc[enriched_sample_df.groupby('occupation_title')['ub_estimate_in_hours'].idxmax()]\n",
    "hardest_task_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "collapsed": true,
    "id": "BMsZWYvM4Bfh",
    "jupyter": {
     "outputs_hidden": true
    },
    "outputId": "81b7418f-534d-446d-97c0-0ef8cbfe6554"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.google.colaboratory.intrinsic+json": {
       "type": "dataframe",
       "variable_name": "enriched_sample_df"
      },
      "text/html": [
       "\n",
       "  <div id=\"df-d918d25e-643f-4586-9439-a5a4be2222d8\" class=\"colab-df-container\">\n",
       "    <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>onetsoc_code</th>\n",
       "      <th>task_id</th>\n",
       "      <th>task</th>\n",
       "      <th>occupation_title</th>\n",
       "      <th>occupation_description</th>\n",
       "      <th>frequency_category_1</th>\n",
       "      <th>frequency_category_2</th>\n",
       "      <th>frequency_category_3</th>\n",
       "      <th>frequency_category_4</th>\n",
       "      <th>...</th>\n",
       "      <th>relevance_average</th>\n",
       "      <th>remote_status</th>\n",
       "      <th>occ_code</th>\n",
       "      <th>total_employment</th>\n",
       "      <th>hourly_wage_average</th>\n",
       "      <th>annual_wage_average</th>\n",
       "      <th>lb_estimate</th>\n",
       "      <th>ub_estimate</th>\n",
       "      <th>lb_estimate_in_hours</th>\n",
       "      <th>ub_estimate_in_hours</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>362</td>\n",
       "      <td>11-3071.00</td>\n",
       "      <td>21343</td>\n",
       "      <td>Plan, organize, or manage the work of subordin...</td>\n",
       "      <td>Transportation, Storage, and Distribution Mana...</td>\n",
       "      <td>Plan, direct, or coordinate transportation, st...</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.00</td>\n",
       "      <td>3.10</td>\n",
       "      <td>7.15</td>\n",
       "      <td>...</td>\n",
       "      <td>97.17</td>\n",
       "      <td>remote</td>\n",
       "      <td>11-3071</td>\n",
       "      <td>198780.0</td>\n",
       "      <td>53.79</td>\n",
       "      <td>111870</td>\n",
       "      <td>4 hours</td>\n",
       "      <td>3 days</td>\n",
       "      <td>4.0</td>\n",
       "      <td>72.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>363</td>\n",
       "      <td>11-3071.00</td>\n",
       "      <td>21344</td>\n",
       "      <td>Collaborate with other departments to integrat...</td>\n",
       "      <td>Transportation, Storage, and Distribution Mana...</td>\n",
       "      <td>Plan, direct, or coordinate transportation, st...</td>\n",
       "      <td>3.33</td>\n",
       "      <td>6.67</td>\n",
       "      <td>33.33</td>\n",
       "      <td>10.00</td>\n",
       "      <td>...</td>\n",
       "      <td>100.00</td>\n",
       "      <td>remote</td>\n",
       "      <td>11-3071</td>\n",
       "      <td>198780.0</td>\n",
       "      <td>53.79</td>\n",
       "      <td>111870</td>\n",
       "      <td>1 week</td>\n",
       "      <td>3 weeks</td>\n",
       "      <td>168.0</td>\n",
       "      <td>504.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>364</td>\n",
       "      <td>11-3071.00</td>\n",
       "      <td>21345</td>\n",
       "      <td>Analyze all aspects of corporate logistics to ...</td>\n",
       "      <td>Transportation, Storage, and Distribution Mana...</td>\n",
       "      <td>Plan, direct, or coordinate transportation, st...</td>\n",
       "      <td>10.00</td>\n",
       "      <td>13.33</td>\n",
       "      <td>20.00</td>\n",
       "      <td>26.67</td>\n",
       "      <td>...</td>\n",
       "      <td>100.00</td>\n",
       "      <td>remote</td>\n",
       "      <td>11-3071</td>\n",
       "      <td>198780.0</td>\n",
       "      <td>53.79</td>\n",
       "      <td>111870</td>\n",
       "      <td>1 week</td>\n",
       "      <td>3 weeks</td>\n",
       "      <td>168.0</td>\n",
       "      <td>504.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>366</td>\n",
       "      <td>11-3071.00</td>\n",
       "      <td>21347</td>\n",
       "      <td>Develop and document standard and emergency op...</td>\n",
       "      <td>Transportation, Storage, and Distribution Mana...</td>\n",
       "      <td>Plan, direct, or coordinate transportation, st...</td>\n",
       "      <td>23.81</td>\n",
       "      <td>23.81</td>\n",
       "      <td>28.57</td>\n",
       "      <td>9.52</td>\n",
       "      <td>...</td>\n",
       "      <td>91.67</td>\n",
       "      <td>remote</td>\n",
       "      <td>11-3071</td>\n",
       "      <td>198780.0</td>\n",
       "      <td>53.79</td>\n",
       "      <td>111870</td>\n",
       "      <td>3 days</td>\n",
       "      <td>1 week</td>\n",
       "      <td>72.0</td>\n",
       "      <td>168.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>368</td>\n",
       "      <td>11-3071.00</td>\n",
       "      <td>21349</td>\n",
       "      <td>Analyze the financial impact of proposed logis...</td>\n",
       "      <td>Transportation, Storage, and Distribution Mana...</td>\n",
       "      <td>Plan, direct, or coordinate transportation, st...</td>\n",
       "      <td>3.45</td>\n",
       "      <td>27.59</td>\n",
       "      <td>34.48</td>\n",
       "      <td>31.03</td>\n",
       "      <td>...</td>\n",
       "      <td>96.67</td>\n",
       "      <td>remote</td>\n",
       "      <td>11-3071</td>\n",
       "      <td>198780.0</td>\n",
       "      <td>53.79</td>\n",
       "      <td>111870</td>\n",
       "      <td>4 hours</td>\n",
       "      <td>1 week</td>\n",
       "      <td>4.0</td>\n",
       "      <td>168.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>747</th>\n",
       "      <td>35501</td>\n",
       "      <td>49-1011.00</td>\n",
       "      <td>15264</td>\n",
       "      <td>Review, evaluate, accept, and coordinate compl...</td>\n",
       "      <td>First-Line Supervisors of Mechanics, Installer...</td>\n",
       "      <td>Directly supervise and coordinate the activiti...</td>\n",
       "      <td>4.43</td>\n",
       "      <td>14.98</td>\n",
       "      <td>44.06</td>\n",
       "      <td>9.94</td>\n",
       "      <td>...</td>\n",
       "      <td>65.94</td>\n",
       "      <td>remote</td>\n",
       "      <td>49-1011</td>\n",
       "      <td>589880.0</td>\n",
       "      <td>37.99</td>\n",
       "      <td>79020</td>\n",
       "      <td>1 hour</td>\n",
       "      <td>3 days</td>\n",
       "      <td>1.0</td>\n",
       "      <td>72.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>748</th>\n",
       "      <td>35502</td>\n",
       "      <td>49-1011.00</td>\n",
       "      <td>2926</td>\n",
       "      <td>Compile operational or personnel records, such...</td>\n",
       "      <td>First-Line Supervisors of Mechanics, Installer...</td>\n",
       "      <td>Directly supervise and coordinate the activiti...</td>\n",
       "      <td>4.25</td>\n",
       "      <td>6.22</td>\n",
       "      <td>27.27</td>\n",
       "      <td>16.93</td>\n",
       "      <td>...</td>\n",
       "      <td>64.30</td>\n",
       "      <td>remote</td>\n",
       "      <td>49-1011</td>\n",
       "      <td>589880.0</td>\n",
       "      <td>37.99</td>\n",
       "      <td>79020</td>\n",
       "      <td>30 minutes</td>\n",
       "      <td>1 hour</td>\n",
       "      <td>0.5</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>749</th>\n",
       "      <td>35503</td>\n",
       "      <td>49-1011.00</td>\n",
       "      <td>2931</td>\n",
       "      <td>Develop or implement electronic maintenance pr...</td>\n",
       "      <td>First-Line Supervisors of Mechanics, Installer...</td>\n",
       "      <td>Directly supervise and coordinate the activiti...</td>\n",
       "      <td>3.26</td>\n",
       "      <td>3.23</td>\n",
       "      <td>9.64</td>\n",
       "      <td>18.78</td>\n",
       "      <td>...</td>\n",
       "      <td>50.24</td>\n",
       "      <td>remote</td>\n",
       "      <td>49-1011</td>\n",
       "      <td>589880.0</td>\n",
       "      <td>37.99</td>\n",
       "      <td>79020</td>\n",
       "      <td>3 weeks</td>\n",
       "      <td>6 weeks</td>\n",
       "      <td>504.0</td>\n",
       "      <td>1008.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>750</th>\n",
       "      <td>35504</td>\n",
       "      <td>49-1011.00</td>\n",
       "      <td>2932</td>\n",
       "      <td>Design equipment configurations to meet person...</td>\n",
       "      <td>First-Line Supervisors of Mechanics, Installer...</td>\n",
       "      <td>Directly supervise and coordinate the activiti...</td>\n",
       "      <td>6.07</td>\n",
       "      <td>10.37</td>\n",
       "      <td>42.18</td>\n",
       "      <td>22.01</td>\n",
       "      <td>...</td>\n",
       "      <td>57.57</td>\n",
       "      <td>remote</td>\n",
       "      <td>49-1011</td>\n",
       "      <td>589880.0</td>\n",
       "      <td>37.99</td>\n",
       "      <td>79020</td>\n",
       "      <td>4 hours</td>\n",
       "      <td>8 hours</td>\n",
       "      <td>4.0</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>751</th>\n",
       "      <td>36002</td>\n",
       "      <td>49-3093.00</td>\n",
       "      <td>8383</td>\n",
       "      <td>Order replacements for tires or tubes.</td>\n",
       "      <td>Tire Repairers and Changers</td>\n",
       "      <td>Repair and replace tires.</td>\n",
       "      <td>5.26</td>\n",
       "      <td>4.94</td>\n",
       "      <td>9.61</td>\n",
       "      <td>7.48</td>\n",
       "      <td>...</td>\n",
       "      <td>69.89</td>\n",
       "      <td>remote</td>\n",
       "      <td>49-3093</td>\n",
       "      <td>101520.0</td>\n",
       "      <td>17.92</td>\n",
       "      <td>37280</td>\n",
       "      <td>30 minutes</td>\n",
       "      <td>1 hour</td>\n",
       "      <td>0.5</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>752 rows × 24 columns</p>\n",
       "</div>\n",
       "    <div class=\"colab-df-buttons\">\n",
       "\n",
       "  <div class=\"colab-df-container\">\n",
       "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d918d25e-643f-4586-9439-a5a4be2222d8')\"\n",
       "            title=\"Convert this dataframe to an interactive table.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
       "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "\n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    .colab-df-buttons div {\n",
       "      margin-bottom: 4px;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "    <script>\n",
       "      const buttonEl =\n",
       "        document.querySelector('#df-d918d25e-643f-4586-9439-a5a4be2222d8 button.colab-df-convert');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      async function convertToInteractive(key) {\n",
       "        const element = document.querySelector('#df-d918d25e-643f-4586-9439-a5a4be2222d8');\n",
       "        const dataTable =\n",
       "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                    [key], {});\n",
       "        if (!dataTable) return;\n",
       "\n",
       "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "          + ' to learn more about interactive tables.';\n",
       "        element.innerHTML = '';\n",
       "        dataTable['output_type'] = 'display_data';\n",
       "        await google.colab.output.renderOutput(dataTable, element);\n",
       "        const docLink = document.createElement('div');\n",
       "        docLink.innerHTML = docLinkHtml;\n",
       "        element.appendChild(docLink);\n",
       "      }\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "\n",
       "    <div id=\"df-562c4efd-3d0a-4564-afd9-113d3b641858\">\n",
       "      <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-562c4efd-3d0a-4564-afd9-113d3b641858')\"\n",
       "                title=\"Suggest charts\"\n",
       "                style=\"display:none;\">\n",
       "\n",
       "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "     width=\"24px\">\n",
       "    <g>\n",
       "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
       "    </g>\n",
       "</svg>\n",
       "      </button>\n",
       "\n",
       "<style>\n",
       "  .colab-df-quickchart {\n",
       "      --bg-color: #E8F0FE;\n",
       "      --fill-color: #1967D2;\n",
       "      --hover-bg-color: #E2EBFA;\n",
       "      --hover-fill-color: #174EA6;\n",
       "      --disabled-fill-color: #AAA;\n",
       "      --disabled-bg-color: #DDD;\n",
       "  }\n",
       "\n",
       "  [theme=dark] .colab-df-quickchart {\n",
       "      --bg-color: #3B4455;\n",
       "      --fill-color: #D2E3FC;\n",
       "      --hover-bg-color: #434B5C;\n",
       "      --hover-fill-color: #FFFFFF;\n",
       "      --disabled-bg-color: #3B4455;\n",
       "      --disabled-fill-color: #666;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart {\n",
       "    background-color: var(--bg-color);\n",
       "    border: none;\n",
       "    border-radius: 50%;\n",
       "    cursor: pointer;\n",
       "    display: none;\n",
       "    fill: var(--fill-color);\n",
       "    height: 32px;\n",
       "    padding: 0;\n",
       "    width: 32px;\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart:hover {\n",
       "    background-color: var(--hover-bg-color);\n",
       "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "    fill: var(--button-hover-fill-color);\n",
       "  }\n",
       "\n",
       "  .colab-df-quickchart-complete:disabled,\n",
       "  .colab-df-quickchart-complete:disabled:hover {\n",
       "    background-color: var(--disabled-bg-color);\n",
       "    fill: var(--disabled-fill-color);\n",
       "    box-shadow: none;\n",
       "  }\n",
       "\n",
       "  .colab-df-spinner {\n",
       "    border: 2px solid var(--fill-color);\n",
       "    border-color: transparent;\n",
       "    border-bottom-color: var(--fill-color);\n",
       "    animation:\n",
       "      spin 1s steps(1) infinite;\n",
       "  }\n",
       "\n",
       "  @keyframes spin {\n",
       "    0% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "      border-left-color: var(--fill-color);\n",
       "    }\n",
       "    20% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    30% {\n",
       "      border-color: transparent;\n",
       "      border-left-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    40% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-top-color: var(--fill-color);\n",
       "    }\n",
       "    60% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "    }\n",
       "    80% {\n",
       "      border-color: transparent;\n",
       "      border-right-color: var(--fill-color);\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "    90% {\n",
       "      border-color: transparent;\n",
       "      border-bottom-color: var(--fill-color);\n",
       "    }\n",
       "  }\n",
       "</style>\n",
       "\n",
       "      <script>\n",
       "        async function quickchart(key) {\n",
       "          const quickchartButtonEl =\n",
       "            document.querySelector('#' + key + ' button');\n",
       "          quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
       "          quickchartButtonEl.classList.add('colab-df-spinner');\n",
       "          try {\n",
       "            const charts = await google.colab.kernel.invokeFunction(\n",
       "                'suggestCharts', [key], {});\n",
       "          } catch (error) {\n",
       "            console.error('Error during call to suggestCharts:', error);\n",
       "          }\n",
       "          quickchartButtonEl.classList.remove('colab-df-spinner');\n",
       "          quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
       "        }\n",
       "        (() => {\n",
       "          let quickchartButtonEl =\n",
       "            document.querySelector('#df-562c4efd-3d0a-4564-afd9-113d3b641858 button');\n",
       "          quickchartButtonEl.style.display =\n",
       "            google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "        })();\n",
       "      </script>\n",
       "    </div>\n",
       "\n",
       "  <div id=\"id_c132fb94-dff1-4d25-b8e7-2646c5fb5c7a\">\n",
       "    <style>\n",
       "      .colab-df-generate {\n",
       "        background-color: #E8F0FE;\n",
       "        border: none;\n",
       "        border-radius: 50%;\n",
       "        cursor: pointer;\n",
       "        display: none;\n",
       "        fill: #1967D2;\n",
       "        height: 32px;\n",
       "        padding: 0 0 0 0;\n",
       "        width: 32px;\n",
       "      }\n",
       "\n",
       "      .colab-df-generate:hover {\n",
       "        background-color: #E2EBFA;\n",
       "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "        fill: #174EA6;\n",
       "      }\n",
       "\n",
       "      [theme=dark] .colab-df-generate {\n",
       "        background-color: #3B4455;\n",
       "        fill: #D2E3FC;\n",
       "      }\n",
       "\n",
       "      [theme=dark] .colab-df-generate:hover {\n",
       "        background-color: #434B5C;\n",
       "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "        fill: #FFFFFF;\n",
       "      }\n",
       "    </style>\n",
       "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('enriched_sample_df')\"\n",
       "            title=\"Generate code using this dataframe.\"\n",
       "            style=\"display:none;\">\n",
       "\n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
       "  </svg>\n",
       "    </button>\n",
       "    <script>\n",
       "      (() => {\n",
       "      const buttonEl =\n",
       "        document.querySelector('#id_c132fb94-dff1-4d25-b8e7-2646c5fb5c7a button.colab-df-generate');\n",
       "      buttonEl.style.display =\n",
       "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "      buttonEl.onclick = () => {\n",
       "        google.colab.notebook.generateWithVariable('enriched_sample_df');\n",
       "      }\n",
       "      })();\n",
       "    </script>\n",
       "  </div>\n",
       "\n",
       "    </div>\n",
       "  </div>\n"
      ],
      "text/plain": [
       "     Unnamed: 0 onetsoc_code  task_id  \\\n",
       "0           362   11-3071.00    21343   \n",
       "1           363   11-3071.00    21344   \n",
       "2           364   11-3071.00    21345   \n",
       "3           366   11-3071.00    21347   \n",
       "4           368   11-3071.00    21349   \n",
       "..          ...          ...      ...   \n",
       "747       35501   49-1011.00    15264   \n",
       "748       35502   49-1011.00     2926   \n",
       "749       35503   49-1011.00     2931   \n",
       "750       35504   49-1011.00     2932   \n",
       "751       36002   49-3093.00     8383   \n",
       "\n",
       "                                                  task  \\\n",
       "0    Plan, organize, or manage the work of subordin...   \n",
       "1    Collaborate with other departments to integrat...   \n",
       "2    Analyze all aspects of corporate logistics to ...   \n",
       "3    Develop and document standard and emergency op...   \n",
       "4    Analyze the financial impact of proposed logis...   \n",
       "..                                                 ...   \n",
       "747  Review, evaluate, accept, and coordinate compl...   \n",
       "748  Compile operational or personnel records, such...   \n",
       "749  Develop or implement electronic maintenance pr...   \n",
       "750  Design equipment configurations to meet person...   \n",
       "751             Order replacements for tires or tubes.   \n",
       "\n",
       "                                      occupation_title  \\\n",
       "0    Transportation, Storage, and Distribution Mana...   \n",
       "1    Transportation, Storage, and Distribution Mana...   \n",
       "2    Transportation, Storage, and Distribution Mana...   \n",
       "3    Transportation, Storage, and Distribution Mana...   \n",
       "4    Transportation, Storage, and Distribution Mana...   \n",
       "..                                                 ...   \n",
       "747  First-Line Supervisors of Mechanics, Installer...   \n",
       "748  First-Line Supervisors of Mechanics, Installer...   \n",
       "749  First-Line Supervisors of Mechanics, Installer...   \n",
       "750  First-Line Supervisors of Mechanics, Installer...   \n",
       "751                        Tire Repairers and Changers   \n",
       "\n",
       "                                occupation_description  frequency_category_1  \\\n",
       "0    Plan, direct, or coordinate transportation, st...                  0.00   \n",
       "1    Plan, direct, or coordinate transportation, st...                  3.33   \n",
       "2    Plan, direct, or coordinate transportation, st...                 10.00   \n",
       "3    Plan, direct, or coordinate transportation, st...                 23.81   \n",
       "4    Plan, direct, or coordinate transportation, st...                  3.45   \n",
       "..                                                 ...                   ...   \n",
       "747  Directly supervise and coordinate the activiti...                  4.43   \n",
       "748  Directly supervise and coordinate the activiti...                  4.25   \n",
       "749  Directly supervise and coordinate the activiti...                  3.26   \n",
       "750  Directly supervise and coordinate the activiti...                  6.07   \n",
       "751                          Repair and replace tires.                  5.26   \n",
       "\n",
       "     frequency_category_2  frequency_category_3  frequency_category_4  ...  \\\n",
       "0                    0.00                  3.10                  7.15  ...   \n",
       "1                    6.67                 33.33                 10.00  ...   \n",
       "2                   13.33                 20.00                 26.67  ...   \n",
       "3                   23.81                 28.57                  9.52  ...   \n",
       "4                   27.59                 34.48                 31.03  ...   \n",
       "..                    ...                   ...                   ...  ...   \n",
       "747                 14.98                 44.06                  9.94  ...   \n",
       "748                  6.22                 27.27                 16.93  ...   \n",
       "749                  3.23                  9.64                 18.78  ...   \n",
       "750                 10.37                 42.18                 22.01  ...   \n",
       "751                  4.94                  9.61                  7.48  ...   \n",
       "\n",
       "     relevance_average  remote_status  occ_code  total_employment  \\\n",
       "0                97.17         remote   11-3071          198780.0   \n",
       "1               100.00         remote   11-3071          198780.0   \n",
       "2               100.00         remote   11-3071          198780.0   \n",
       "3                91.67         remote   11-3071          198780.0   \n",
       "4                96.67         remote   11-3071          198780.0   \n",
       "..                 ...            ...       ...               ...   \n",
       "747              65.94         remote   49-1011          589880.0   \n",
       "748              64.30         remote   49-1011          589880.0   \n",
       "749              50.24         remote   49-1011          589880.0   \n",
       "750              57.57         remote   49-1011          589880.0   \n",
       "751              69.89         remote   49-3093          101520.0   \n",
       "\n",
       "     hourly_wage_average annual_wage_average lb_estimate  ub_estimate  \\\n",
       "0                  53.79              111870     4 hours       3 days   \n",
       "1                  53.79              111870      1 week      3 weeks   \n",
       "2                  53.79              111870      1 week      3 weeks   \n",
       "3                  53.79              111870      3 days       1 week   \n",
       "4                  53.79              111870     4 hours       1 week   \n",
       "..                   ...                 ...         ...          ...   \n",
       "747                37.99               79020      1 hour       3 days   \n",
       "748                37.99               79020  30 minutes       1 hour   \n",
       "749                37.99               79020     3 weeks      6 weeks   \n",
       "750                37.99               79020     4 hours      8 hours   \n",
       "751                17.92               37280  30 minutes       1 hour   \n",
       "\n",
       "    lb_estimate_in_hours  ub_estimate_in_hours  \n",
       "0                    4.0                  72.0  \n",
       "1                  168.0                 504.0  \n",
       "2                  168.0                 504.0  \n",
       "3                   72.0                 168.0  \n",
       "4                    4.0                 168.0  \n",
       "..                   ...                   ...  \n",
       "747                  1.0                  72.0  \n",
       "748                  0.5                   1.0  \n",
       "749                504.0                1008.0  \n",
       "750                  4.0                   8.0  \n",
       "751                  0.5                   1.0  \n",
       "\n",
       "[752 rows x 24 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "enriched_sample_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 615
    },
    "collapsed": true,
    "id": "PutUzHEG9E_M",
    "jupyter": {
     "outputs_hidden": true
    },
    "outputId": "13800b66-b5fb-4d32-d836-78baeeb3cd35"
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 108212 characters]\n",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "DURATION_TO_HOUR_ESTIMATE = {\n",
    "    '10 minutes': 0.1,\n",
    "    '30 minutes': .5,\n",
    "    '1 hour': 1,\n",
    "    '4 hours': 4,\n",
    "    '8 hours': 8,\n",
    "    '16 hours': 16,\n",
    "    '3 days': 72,\n",
    "    '1 week': 168,\n",
    "    '3 weeks': 504,\n",
    "    '6 weeks': 1008,\n",
    "    '3 months': 3 * (365.25 / 12) * 24,\n",
    "    '6 months': 6 * (365.25 / 12) * 24,\n",
    "    '1 year': 1 * 365.25 * 24,\n",
    "    '3 years': 3 * 365.25 * 24,\n",
    "    '10 years': 10 * 365.25 * 24,\n",
    "    '30 years': 10 * 365.25 * 24,\n",
    "    '60 years': 10 * 365.25 * 24,\n",
    "}\n",
    "\n",
    "# Calculate the count of occurrences for each (x, y) pair\n",
    "point_counts = enriched_sample_df.groupby(['lb_estimate_in_hours', 'ub_estimate_in_hours']).size().reset_index(name='count')\n",
    "\n",
    "# Create the scatter plot with size based on count\n",
    "plt.figure(figsize=(10, 6))\n",
    "sns.scatterplot(data=point_counts, x='lb_estimate_in_hours', y='ub_estimate_in_hours', size='count', sizes=(20, 200))  # Adjust sizes as needed\n",
    "\n",
    "# Add the diagonal line\n",
    "x = np.linspace(enriched_sample_df['lb_estimate_in_hours'].min(), enriched_sample_df['ub_estimate_in_hours'].max(), 100)\n",
    "plt.plot(x, x, color='red', linestyle='--', label='x=y')\n",
    "\n",
    "# Set log-log scale\n",
    "plt.xscale('log')\n",
    "plt.yscale('log')\n",
    "\n",
    "# Customize the plot\n",
    "plt.title('Lower Bound vs. Upper Bound Task Duration Estimates (Log-Log Scale)')\n",
    "plt.xlabel('Lower Bound Estimate (hours)')\n",
    "plt.ylabel('Upper Bound Estimate (hours)')\n",
    "plt.legend()\n",
    "\n",
    "# Set custom x and y ticks and labels\n",
    "x_ticks = list(DURATION_TO_HOUR_ESTIMATE.values())\n",
    "x_labels = list(DURATION_TO_HOUR_ESTIMATE.keys())\n",
    "plt.xticks(x_ticks, x_labels, rotation=45, ha='right')  # Rotate x labels for readability\n",
    "plt.yticks(x_ticks, x_labels)  # Use the same labels for y-axis\n",
    "\n",
    "# Show the plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "lYWiBiaSVLYe"
   },
   "outputs": [],
   "source": [
    "enriched_sample_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 476
    },
    "collapsed": true,
    "id": "9i3whbtOV3vz",
    "jupyter": {
     "outputs_hidden": true
    },
    "outputId": "00ba0ca0-2433-40c5-dd57-4f4da7b1f72a"
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 26400 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Placeholder #1\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "# Define the time range (0 to 3 years in hours)\n",
    "time_range = np.linspace(0, 3 * 365.25 * 24, 10000)  # 100 points for smoothness\n",
    "\n",
    "# Calculate the percentage of tasks for each time point\n",
    "percentages = []\n",
    "for time_point in time_range:\n",
    "    percentage = (enriched_sample_df['ub_estimate_in_hours'] <= time_point).mean() * 100\n",
    "    percentages.append(percentage)\n",
    "\n",
    "# Create the plot\n",
    "plt.plot(time_range, percentages)\n",
    "plt.xscale('log')\n",
    "plt.xlabel('Time horizon needed')\n",
    "plt.ylabel('% of tasks in the economy')\n",
    "plt.title('PLACEHOLDER #1')\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "3HFCARXtRIIh"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 476
    },
    "id": "DQ806XcnjoRC",
    "outputId": "01d9b861-744e-4b36-a453-25e51026383e"
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 35860 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "# Get unique upper bound estimates and sort them\n",
    "time_range = sorted(enriched_sample_df['ub_estimate_in_hours'].unique())\n",
    "\n",
    "# Calculate the percentage of tasks for each time point\n",
    "percentages = []\n",
    "for time_point in time_range:\n",
    "    percentage = (enriched_sample_df['ub_estimate_in_hours'] <= time_point).mean() * 100\n",
    "    percentages.append(percentage)\n",
    "\n",
    "# Create the plot\n",
    "plt.plot(time_range, percentages)\n",
    "plt.xscale('log')  # Keep the x-axis in log scale\n",
    "plt.xlabel('Time horizon needed')\n",
    "plt.ylabel('% of tasks in the economy')\n",
    "plt.title('Distribution of Task Completion Times')  # Update the title\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 476
    },
    "id": "XNp6VIDtkDuP",
    "outputId": "869aa51d-fb4c-44ef-fcfa-0262a79b8d85"
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 29560 characters]\n",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "# Get unique upper bound estimates and sort them\n",
    "time_range = sorted(enriched_sample_df['ub_estimate_in_hours'].unique())\n",
    "\n",
    "# Calculate the percentage of tasks for each time point\n",
    "percentages = []\n",
    "for time_point in time_range:\n",
    "    percentage = (enriched_sample_df['ub_estimate_in_hours'] <= time_point).mean() * 100\n",
    "    percentages.append(percentage)\n",
    "\n",
    "# Create the staircase plot\n",
    "plt.step(time_range, percentages, where='post')  # Use 'step' function with 'where='post''\n",
    "plt.xscale('log')  # Keep the x-axis in log scale\n",
    "plt.xlabel('Time horizon needed')\n",
    "plt.ylabel('% of tasks in the economy')\n",
    "plt.title('Distribution of Task Completion Times (Staircase)')  # Update the title\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 311
    },
    "collapsed": true,
    "id": "BSuO-HB1nPTS",
    "jupyter": {
     "outputs_hidden": true
    },
    "outputId": "f3fc3422-62e8-4054-f77b-8c514eae7fc7"
   },
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'Column not found: ub_estimate_in_hours'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-16-1b0d5f31b751>\u001b[0m in \u001b[0;36m<cell line: 0>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mhardest_task_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0menriched_sample_df\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloc\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0menriched_sample_df\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgroupby\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'occupation_title'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'ub_estimate_in_hours'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0midxmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mhardest_task_df\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.11/dist-packages/pandas/core/groupby/generic.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   1949\u001b[0m                 \u001b[0;34m\"Use a list instead.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1950\u001b[0m             )\n\u001b[0;32m-> 1951\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getitem__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1952\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1953\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_gotitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mndim\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msubset\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.11/dist-packages/pandas/core/base.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m    242\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    243\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mkey\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 244\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"Column not found: {key}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    245\u001b[0m             \u001b[0mndim\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    246\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_gotitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mndim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mndim\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'Column not found: ub_estimate_in_hours'"
     ]
    }
   ],
   "source": [
    "hardest_task_df = enriched_sample_df.loc[enriched_sample_df.groupby('occupation_title')['ub_estimate_in_hours'].idxmax()]\n",
    "hardest_task_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "h1ajw7Cpo5EG",
    "outputId": "62bc2c04-8d6a-40bd-ae2f-fc4154d28e7e"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "45"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_ground = pd.read_csv('groundtruth.csv')\n",
    "len(df_ground)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "TWXi34Z3H2zn",
    "outputId": "91b08575-a41b-4a9c-af77-e0c0cb7f01a8"
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "    async function download(id, filename, size) {\n",
       "      if (!google.colab.kernel.accessAllowed) {\n",
       "        return;\n",
       "      }\n",
       "      const div = document.createElement('div');\n",
       "      const label = document.createElement('label');\n",
       "      label.textContent = `Downloading \"${filename}\": `;\n",
       "      div.appendChild(label);\n",
       "      const progress = document.createElement('progress');\n",
       "      progress.max = size;\n",
       "      div.appendChild(progress);\n",
       "      document.body.appendChild(div);\n",
       "\n",
       "      const buffers = [];\n",
       "      let downloaded = 0;\n",
       "\n",
       "      const channel = await google.colab.kernel.comms.open(id);\n",
       "      // Send a message to notify the kernel that we're ready.\n",
       "      channel.send({})\n",
       "\n",
       "      for await (const message of channel.messages) {\n",
       "        // Send a message to notify the kernel that we're ready.\n",
       "        channel.send({})\n",
       "        if (message.buffers) {\n",
       "          for (const buffer of message.buffers) {\n",
       "            buffers.push(buffer);\n",
       "            downloaded += buffer.byteLength;\n",
       "            progress.value = downloaded;\n",
       "          }\n",
       "        }\n",
       "      }\n",
       "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
       "      const a = document.createElement('a');\n",
       "      a.href = window.URL.createObjectURL(blob);\n",
       "      a.download = filename;\n",
       "      div.appendChild(a);\n",
       "      a.click();\n",
       "      div.remove();\n",
       "    }\n",
       "  "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "download(\"download_e757f711-7909-4bab-b440-6b854050738b\", \"groundtruth_with_dwas.csv\", 20412)"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from google.colab import files\n",
    "\n",
    "# Rename columns in df_ground to match df\n",
    "df_ground = df_ground.rename(columns={\"Task\": \"task\", \"Occupation\": \"occupation_title\", \"Occupation Description\": \"occupation_description\"})\n",
    "\n",
    "# Drop duplicates from 'df' based on merge key columns, keeping only the first occurrence\n",
    "df_unique = df[['task', 'occupation_title', 'occupation_description', 'dwas']].drop_duplicates(subset=['task', 'occupation_title', 'occupation_description'], keep='first')\n",
    "\n",
    "# Merge the two DataFrames using the specified columns\n",
    "merged_df = pd.merge(df_ground, df_unique, on=['task', 'occupation_title', 'occupation_description'], how='left')\n",
    "\n",
    "# Restore original column names in merged_df if needed\n",
    "merged_df = merged_df.rename(columns={\"task\": \"Task\", \"occupation_title\": \"Occupation\", \"occupation_description\": \"Occupation Description\"})\n",
    "\n",
    "# Access the merged DataFrame with the added 'dwas' column\n",
    "merged_df.to_csv('groundtruth_with_dwas.csv', index=False)  # Save the DataFrame to a CSV file\n",
    "files.download('groundtruth_with_dwas.csv')  # Download the file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 17
    },
    "id": "5K-y59g0JdSu",
    "outputId": "e34e5e19-94b6-47bd-ff9f-9b9732eb7728"
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "    async function download(id, filename, size) {\n",
       "      if (!google.colab.kernel.accessAllowed) {\n",
       "        return;\n",
       "      }\n",
       "      const div = document.createElement('div');\n",
       "      const label = document.createElement('label');\n",
       "      label.textContent = `Downloading \"${filename}\": `;\n",
       "      div.appendChild(label);\n",
       "      const progress = document.createElement('progress');\n",
       "      progress.max = size;\n",
       "      div.appendChild(progress);\n",
       "      document.body.appendChild(div);\n",
       "\n",
       "      const buffers = [];\n",
       "      let downloaded = 0;\n",
       "\n",
       "      const channel = await google.colab.kernel.comms.open(id);\n",
       "      // Send a message to notify the kernel that we're ready.\n",
       "      channel.send({})\n",
       "\n",
       "      for await (const message of channel.messages) {\n",
       "        // Send a message to notify the kernel that we're ready.\n",
       "        channel.send({})\n",
       "        if (message.buffers) {\n",
       "          for (const buffer of message.buffers) {\n",
       "            buffers.push(buffer);\n",
       "            downloaded += buffer.byteLength;\n",
       "            progress.value = downloaded;\n",
       "          }\n",
       "        }\n",
       "      }\n",
       "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
       "      const a = document.createElement('a');\n",
       "      a.href = window.URL.createObjectURL(blob);\n",
       "      a.download = filename;\n",
       "      div.appendChild(a);\n",
       "      a.click();\n",
       "      div.remove();\n",
       "    }\n",
       "  "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "download(\"download_2c2b4f4a-16d1-4c18-960a-81b09b6daf2f\", \"groundtruth_with_dwas.csv\", 17625)"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from google.colab import files\n",
    "\n",
    "df_ground.to_csv('groundtruth_with_dwas.csv', index=False)  # Save the DataFrame to a CSV file\n",
    "files.download('groundtruth_with_dwas.csv')  # Download the file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 17
    },
    "id": "7pZgtEGtVGtH",
    "outputId": "5f0e938b-6393-4c7d-bdb3-0a4f9e6395e4"
   },
   "outputs": [
    {
     "data": {
      "application/javascript": [
       "\n",
       "    async function download(id, filename, size) {\n",
       "      if (!google.colab.kernel.accessAllowed) {\n",
       "        return;\n",
       "      }\n",
       "      const div = document.createElement('div');\n",
       "      const label = document.createElement('label');\n",
       "      label.textContent = `Downloading \"${filename}\": `;\n",
       "      div.appendChild(label);\n",
       "      const progress = document.createElement('progress');\n",
       "      progress.max = size;\n",
       "      div.appendChild(progress);\n",
       "      document.body.appendChild(div);\n",
       "\n",
       "      const buffers = [];\n",
       "      let downloaded = 0;\n",
       "\n",
       "      const channel = await google.colab.kernel.comms.open(id);\n",
       "      // Send a message to notify the kernel that we're ready.\n",
       "      channel.send({})\n",
       "\n",
       "      for await (const message of channel.messages) {\n",
       "        // Send a message to notify the kernel that we're ready.\n",
       "        channel.send({})\n",
       "        if (message.buffers) {\n",
       "          for (const buffer of message.buffers) {\n",
       "            buffers.push(buffer);\n",
       "            downloaded += buffer.byteLength;\n",
       "            progress.value = downloaded;\n",
       "          }\n",
       "        }\n",
       "      }\n",
       "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
       "      const a = document.createElement('a');\n",
       "      a.href = window.URL.createObjectURL(blob);\n",
       "      a.download = filename;\n",
       "      div.appendChild(a);\n",
       "      a.click();\n",
       "      div.remove();\n",
       "    }\n",
       "  "
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/javascript": [
       "download(\"download_cee274f5-c2ff-4298-bcb1-de5f2f8f04f0\", \"task_to_estimate.csv\", 11158577)"
      ],
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "remote_df.to_csv('task_to_estimate.csv', index=False)\n",
    "files.download('task_to_estimate.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "1hz5swsJwXrO"
   },
   "outputs": [],
   "source": [
    "METR = [\n",
    "    [2022, '1 min'],\n",
    "    [2024, '15 min'],\n",
    "    [2026, '4 hours'],\n",
    "    [2028, '68 hours']\n",
    "]\n",
    "\n",
    "# These are the values for the X axis"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
