{ "cells": [ { "cell_type": "code", "execution_count": 86, "id": "beace815-b5ae-44a4-a81c-a7f82cb66296", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\u001b[2K\u001b[2mResolved \u001b[1m118 packages\u001b[0m \u001b[2min 386ms\u001b[0m\u001b[0m \u001b[0m\n", "\u001b[2K\u001b[2mPrepared \u001b[1m2 packages\u001b[0m \u001b[2min 124ms\u001b[0m\u001b[0m \n", "\u001b[2K\u001b[2mInstalled \u001b[1m2 packages\u001b[0m \u001b[2min 5ms\u001b[0m\u001b[0m \u001b[0m\n", " \u001b[32m+\u001b[39m \u001b[1met-xmlfile\u001b[0m\u001b[2m==2.0.0\u001b[0m\n", " \u001b[32m+\u001b[39m \u001b[1mopenpyxl\u001b[0m\u001b[2m==3.1.5\u001b[0m\n" ] } ], "source": [ "!uv add pandas requests openai dotenv openpyxl" ] }, { "cell_type": "code", "execution_count": 9, "id": "941d511f-ad72-4306-bbab-52127583e513", "metadata": {}, "outputs": [], "source": [ "import os\n", "import dotenv\n", "import openai\n", "import sqlite3\n", "import pandas as pd\n", "\n", "dotenv.load_dotenv() # Copy .env.example to .env and fill in the blanks\n", "\n", "oai_token = os.getenv(\"OPENAI_API_KEY\")\n", "\n", "oai = openai.OpenAI(api_key=oai_token)\n", "onet = sqlite3.connect(\"onet.database\") # Run ./create_onet_database.sh to create it\n", "# This dataset comes from https://epoch.ai/gradient-updates/consequences-of-automating-remote-work\n", "# It contains labels for whethere a O*NET task can be done remotely or not (labeled by GPT-4o)\n", "# You can download it here: https://drive.google.com/file/d/1GrHhuYIgaCCgo99dZ_40BWraz-fzo76r/view?usp=sharing\n", "df_remote_status = pd.read_csv(\"epoch_task_data.csv\")\n", "\n", "# BLS OEWS: https://www.bls.gov/oes/special-requests/oesm23nat.zip\n", "df_oesm = pd.read_excel(\"oesm23national.xlsx\")\n", "\n", "# Run uv run enrich_task_ratings.py to get this file (trs = Task RatingS)\n", "df_enriched_trs = pd.read_json(\"task_ratings_enriched.json\")" ] }, { "cell_type": "code", "execution_count": 3, "id": "a5351f8b-c2ad-4d3e-af4a-992f539a6064", "metadata": {}, "outputs": [], "source": [ "FREQUENCY_MAP = {\n", " 'frequency_category_1': \"Yearly or less\",\n", " 'frequency_category_2': \"More than yearly\",\n", " 'frequency_category_3': \"More than monthly\",\n", " 'frequency_category_4': \"More than weekly\",\n", " 'frequency_category_5': \"Daily\",\n", " 'frequency_category_6': \"Several times daily\",\n", " 'frequency_category_7': \"Hourly or more\"\n", "}" ] }, { "cell_type": "code", "execution_count": 4, "id": "8b2ab22a-afab-41f9-81a3-48eab261b568", "metadata": {}, "outputs": [], "source": [ "background_prompt = '''\n", "Estimate the typical duration to complete *one instance* of the following job task from the moment a person starts to work on it to the last moment the person will need to keep it in mind\n", "\n", "Take into account that there might be delays between the steps to complete the task, which would lengthen the estimate.\n", "\n", "Output a range with the format [duration A] - [duration B] where [duration A] and [duration B] correspond to one of the durations below:\n", "- less than 30 minutes\n", "- 30 minutes\n", "- 1 hour\n", "- 4 hours\n", "- 8 hours\n", "- 16 hours\n", "- 3 days\n", "- 1 week\n", "- 3 weeks\n", "- 6 weeks\n", "- 3 months\n", "- 6 months\n", "- 1 year\n", "- 3 years\n", "- more than 3 year\n", "\n", "**Do not output anything besides the range**\n", "'''" ] }, { "cell_type": "code", "execution_count": 11, "id": "d2e4a855-f327-4b3d-ad0b-ed997e720639", "metadata": {}, "outputs": [], "source": [ "df_oesm_detailed = df_oesm[df_oesm['O_GROUP'] == 'detailed'][['OCC_CODE', 'TOT_EMP', 'H_MEAN', 'A_MEAN']].copy()\n", "df_enriched_trs['occ_code_join'] = df_enriched_trs['onetsoc_code'].str[:7]\n", "df_merged = pd.merge(\n", " df_enriched_trs,\n", " df_oesm_detailed,\n", " left_on='occ_code_join',\n", " right_on='OCC_CODE',\n", " how='left'\n", ")\n", "df_merged = df_merged.drop(columns=['occ_code_join'])" ] }, { "cell_type": "code", "execution_count": 12, "id": "9be7acb5-2374-4f61-bba3-13b0077c0bd2", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Task: Develop or recommend network security measures, such as firewalls, network security audits, or automated security probes.\n", "Occupation Description: Design and implement computer and information networks, such as local area networks (LAN), wide area networks (WAN), intranets, extranets, and other data communications networks. Perform network modeling, analysis, and planning, including analysis of capacity needs for network infrastructures. May also design network and computer security measures. May research and recommend network and data communications hardware and software.\n", "Occupation Title: Computer Network Architects\n" ] }, { "data": { "text/plain": [ "onetsoc_code 15-1241.00\n", "task_id 18971\n", "task Develop or recommend network security measures...\n", "occupation_title Computer Network Architects\n", "occupation_description Design and implement computer and information ...\n", "Yearly or less 0.0\n", "More than yearly 30.0\n", "More than monthly 15.0\n", "More than weekly 20.0\n", "Daily 15.0\n", "Several times daily 15.0\n", "Hourly or more 5.0\n", "importance_average 4.35\n", "relevance_average 100.0\n", "occ_code_join 15-1241\n", "remote remote\n", "Name: 45200, dtype: object" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "\n", "df_merged = pd \\\n", " .merge(left=df_enriched_trs, right=df_remote_status[['O*NET-SOC Code', 'Remote']], how='left', left_on='onetsoc_code', right_on='O*NET-SOC Code') \\\n", " .drop(columns=['O*NET-SOC Code']) \\\n", " .rename(columns={'Remote': 'remote'}) \\\n", " .rename(columns=FREQUENCY_MAP) \\\n", " .query('remote == \"remote\" and importance_average >= 3')\n", "\n", "row = df_merged.iloc[30000]\n", "print('Task: ', row['task'])\n", "print('Occupation Description: ', row['occupation_description'])\n", "print('Occupation Title: ', row['occupation_title'])\n", "\n", "row" ] }, { "cell_type": "code", "execution_count": 13, "id": "9e5ea89f-2c18-459d-851d-dacb379f4a2e", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
onetsoc_codetask_idtaskoccupation_titleoccupation_descriptionYearly or lessMore than yearlyMore than monthlyMore than weeklyDailySeveral times dailyHourly or moreimportance_averagerelevance_averageremoteOCC_CODETOT_EMPH_MEANA_MEAN
011-1011.008823Direct or coordinate an organization's financi...Chief ExecutivesDetermine and formulate policies and provide o...5.9215.9829.6821.1819.714.912.634.5274.44remote11-1011211230.0124.47258900
111-1011.008823Direct or coordinate an organization's financi...Chief ExecutivesDetermine and formulate policies and provide o...5.9215.9829.6821.1819.714.912.634.5274.44remote11-1011211230.0124.47258900
211-1011.008823Direct or coordinate an organization's financi...Chief ExecutivesDetermine and formulate policies and provide o...5.9215.9829.6821.1819.714.912.634.5274.44remote11-1011211230.0124.47258900
311-1011.008823Direct or coordinate an organization's financi...Chief ExecutivesDetermine and formulate policies and provide o...5.9215.9829.6821.1819.714.912.634.5274.44remote11-1011211230.0124.47258900
411-1011.008823Direct or coordinate an organization's financi...Chief ExecutivesDetermine and formulate policies and provide o...5.9215.9829.6821.1819.714.912.634.5274.44remote11-1011211230.0124.47258900
............................................................
12765353-7121.0012807Unload cars containing liquids by connecting h...Tank Car, Truck, and Ship LoadersLoad and unload chemicals and bulk solids, suc...6.0529.216.8813.9527.657.938.344.0864.04remote53-712111400.029.160530
12765453-7121.0012804Clean interiors of tank cars or tank trucks, u...Tank Car, Truck, and Ship LoadersLoad and unload chemicals and bulk solids, suc...1.476.3321.7025.6932.3512.470.004.0244.33remote53-712111400.029.160530
12765553-7121.0012803Lower gauge rods into tanks or read meters to ...Tank Car, Truck, and Ship LoadersLoad and unload chemicals and bulk solids, suc...4.521.764.6517.8137.4223.3110.553.8865.00remote53-712111400.029.160530
12765653-7121.0012805Operate conveyors and equipment to transfer gr...Tank Car, Truck, and Ship LoadersLoad and unload chemicals and bulk solids, suc...6.9712.002.525.9035.4822.0815.053.8747.90remote53-712111400.029.160530
12765753-7121.0012810Perform general warehouse activities, such as ...Tank Car, Truck, and Ship LoadersLoad and unload chemicals and bulk solids, suc...5.9110.856.4614.4634.1416.3911.783.5347.84remote53-712111400.029.160530
\n", "

127658 rows × 19 columns

\n", "
" ], "text/plain": [ " onetsoc_code task_id \\\n", "0 11-1011.00 8823 \n", "1 11-1011.00 8823 \n", "2 11-1011.00 8823 \n", "3 11-1011.00 8823 \n", "4 11-1011.00 8823 \n", "... ... ... \n", "127653 53-7121.00 12807 \n", "127654 53-7121.00 12804 \n", "127655 53-7121.00 12803 \n", "127656 53-7121.00 12805 \n", "127657 53-7121.00 12810 \n", "\n", " task \\\n", "0 Direct or coordinate an organization's financi... \n", "1 Direct or coordinate an organization's financi... \n", "2 Direct or coordinate an organization's financi... \n", "3 Direct or coordinate an organization's financi... \n", "4 Direct or coordinate an organization's financi... \n", "... ... \n", "127653 Unload cars containing liquids by connecting h... \n", "127654 Clean interiors of tank cars or tank trucks, u... \n", "127655 Lower gauge rods into tanks or read meters to ... \n", "127656 Operate conveyors and equipment to transfer gr... \n", "127657 Perform general warehouse activities, such as ... \n", "\n", " occupation_title \\\n", "0 Chief Executives \n", "1 Chief Executives \n", "2 Chief Executives \n", "3 Chief Executives \n", "4 Chief Executives \n", "... ... \n", "127653 Tank Car, Truck, and Ship Loaders \n", "127654 Tank Car, Truck, and Ship Loaders \n", "127655 Tank Car, Truck, and Ship Loaders \n", "127656 Tank Car, Truck, and Ship Loaders \n", "127657 Tank Car, Truck, and Ship Loaders \n", "\n", " occupation_description Yearly or less \\\n", "0 Determine and formulate policies and provide o... 5.92 \n", "1 Determine and formulate policies and provide o... 5.92 \n", "2 Determine and formulate policies and provide o... 5.92 \n", "3 Determine and formulate policies and provide o... 5.92 \n", "4 Determine and formulate policies and provide o... 5.92 \n", "... ... ... \n", "127653 Load and unload chemicals and bulk solids, suc... 6.05 \n", "127654 Load and unload chemicals and bulk solids, suc... 1.47 \n", "127655 Load and unload chemicals and bulk solids, suc... 4.52 \n", "127656 Load and unload chemicals and bulk solids, suc... 6.97 \n", "127657 Load and unload chemicals and bulk solids, suc... 5.91 \n", "\n", " More than yearly More than monthly More than weekly Daily \\\n", "0 15.98 29.68 21.18 19.71 \n", "1 15.98 29.68 21.18 19.71 \n", "2 15.98 29.68 21.18 19.71 \n", "3 15.98 29.68 21.18 19.71 \n", "4 15.98 29.68 21.18 19.71 \n", "... ... ... ... ... \n", "127653 29.21 6.88 13.95 27.65 \n", "127654 6.33 21.70 25.69 32.35 \n", "127655 1.76 4.65 17.81 37.42 \n", "127656 12.00 2.52 5.90 35.48 \n", "127657 10.85 6.46 14.46 34.14 \n", "\n", " Several times daily Hourly or more importance_average \\\n", "0 4.91 2.63 4.52 \n", "1 4.91 2.63 4.52 \n", "2 4.91 2.63 4.52 \n", "3 4.91 2.63 4.52 \n", "4 4.91 2.63 4.52 \n", "... ... ... ... \n", "127653 7.93 8.34 4.08 \n", "127654 12.47 0.00 4.02 \n", "127655 23.31 10.55 3.88 \n", "127656 22.08 15.05 3.87 \n", "127657 16.39 11.78 3.53 \n", "\n", " relevance_average remote OCC_CODE TOT_EMP H_MEAN A_MEAN \n", "0 74.44 remote 11-1011 211230.0 124.47 258900 \n", "1 74.44 remote 11-1011 211230.0 124.47 258900 \n", "2 74.44 remote 11-1011 211230.0 124.47 258900 \n", "3 74.44 remote 11-1011 211230.0 124.47 258900 \n", "4 74.44 remote 11-1011 211230.0 124.47 258900 \n", "... ... ... ... ... ... ... \n", "127653 64.04 remote 53-7121 11400.0 29.1 60530 \n", "127654 44.33 remote 53-7121 11400.0 29.1 60530 \n", "127655 65.00 remote 53-7121 11400.0 29.1 60530 \n", "127656 47.90 remote 53-7121 11400.0 29.1 60530 \n", "127657 47.84 remote 53-7121 11400.0 29.1 60530 \n", "\n", "[127658 rows x 19 columns]" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Cross-reference woth BLS OEWS\n", "# It doesn't really make sens to have it per-task, we only need it per-occupation...\n", "df_oesm_detailed = df_oesm[df_oesm['O_GROUP'] == 'detailed'][['OCC_CODE', 'TOT_EMP', 'H_MEAN', 'A_MEAN']].copy()\n", "df_merged['occ_code_join'] = df_merged['onetsoc_code'].str[:7]\n", "df_merged = pd.merge(\n", " df_merged,\n", " df_oesm_detailed,\n", " left_on='occ_code_join',\n", " right_on='OCC_CODE',\n", " how='left'\n", ")\n", "df_merged = df_merged.drop(columns=['occ_code_join'])\n", "df_merged" ] }, { "cell_type": "code", "execution_count": 76, "id": "08f45d91-039d-4ec0-94a2-f305a3312e6a", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Why did the scarecrow win an award?\n", "\n", "Because he was outstanding in his field!\n" ] } ], "source": [ "response = oai.chat.completions.create(messages=[{\"role\": \"user\", \"content\": \"Tell me a joke\"}], model=\"gpt-4.1-2025-04-14\", max_tokens=100, temperature=0.7, n=1, stop=None)\n", "joke = response.choices[0].message.content.strip()\n", "print(joke)" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.13.2" } }, "nbformat": 4, "nbformat_minor": 5 }