diff --git a/docs/source/forecaster.rst b/docs/source/forecaster.rst index 6072d0f4..ba5b71ef 100644 --- a/docs/source/forecaster.rst +++ b/docs/source/forecaster.rst @@ -73,9 +73,9 @@ Unless otherwise specified, the performance is reported using the ThresholdDecis +=====================+===================================+=======+======+=======+=======+======+==========+=========================+ | Gemma2 9B | ThresholdDecisionPolicy | 71.0 | 69.1 | 76.1 | 72.3 | 34.2 | 3.9 | +1.8 (8.4 - 6.6) | +---------------------+-----------------------------------+-------+------+-------+-------+------+----------+-------------------------+ -| Gemma2 9B | DeferralDecisionPolicy | 70.9 | 72.0 | 68.4 | 70.1 | 26.7 | 3.8 | -0.1 (7.0 - 7.1) | +| Gemma2 9B | DeferralDecisionPolicy | 70.9 | 72.1 | 68.4 | 70.2 | 26.7 | 3.8 | -0.1 (7.0 - 7.1) | +---------------------+-----------------------------------+-------+------+-------+-------+------+----------+-------------------------+ -| Gemma2 9B | SimulationAverageDecisionPolicy | 70.2 | 68.1 | 76.6 | 72.0 | 36.1 | 4.0 | -1.2 (9.3 - 10.5) | +| Gemma2 9B | SimulationAverageDecisionPolicy | 70.2 | 68.1 | 76.6 | 72.0 | 36.2 | 4.0 | -1.2 (9.3 - 10.5) | +---------------------+-----------------------------------+-------+------+-------+-------+------+----------+-------------------------+ | Mistral 7B | ThresholdDecisionPolicy | 70.7 | 68.8 | 76.0 | 72.1 | 34.6 | 4.0 | +2.9 (8.1 - 5.2) | +---------------------+-----------------------------------+-------+------+-------+-------+------+----------+-------------------------+ diff --git a/examples/forecaster/Run Transformer Fine-tuned Models.ipynb b/examples/forecaster/Run Transformer Fine-tuned Models.ipynb index 4f4e9c41..4cdb10ff 100644 --- a/examples/forecaster/Run Transformer Fine-tuned Models.ipynb +++ b/examples/forecaster/Run Transformer Fine-tuned Models.ipynb @@ -12,7 +12,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 2, "metadata": {}, "outputs": [ { @@ -26,26 +26,29 @@ "name": "stderr", "output_type": "stream", "text": [ - "2026-06-27 20:18:39.864582: I tensorflow/core/util/port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", - "2026-06-27 20:18:39.887405: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", + "/reef/lyk25/ConvoKit/dist/.venv/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n", "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n", - "E0000 00:00:1782591519.912052 859600 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", - "E0000 00:00:1782591519.919822 859600 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", - "W0000 00:00:1782591519.938735 859600 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.\n", - "W0000 00:00:1782591519.938755 859600 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.\n", - "W0000 00:00:1782591519.938757 859600 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.\n", - "W0000 00:00:1782591519.938760 859600 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.\n", - "2026-06-27 20:18:39.944875: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", - "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n" + "I0000 00:00:1782619476.409075 1013993 port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n", + "I0000 00:00:1782619476.481779 1013993 cpu_feature_guard.cc:227] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n", + "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n", + "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n", + "I0000 00:00:1782619477.990444 1013993 port.cc:153] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "INFO 06-27 20:18:48 [__init__.py:216] Automatically detected platform cuda.\n", "🦥 Unsloth Zoo will now patch everything to make training faster!\n" ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Unable to import `torchao` Tensor objects. This may affect loading checkpoints serialized with `torchao`\n" + ] } ], "source": [ @@ -78,16 +81,16 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "# CPU mode (noting that it will be slower)\n", "DEVICE = \"cuda\"\n", "\n", - "corpus_name = \"cga-wikiconv\"\n", + "# corpus_name = \"cga-wikiconv\"\n", "# corpus_name = \"cga-cmv-legacy\"\n", - "# corpus_name = \"cga-cmv-large\"\n", + "corpus_name = \"cga-cmv-large\"\n", "label_metadata = \"has_removed_comment\" if 'cmv' in corpus_name else 'conversation_has_personal_attack'\n", "\n", "YOUR_MODEL_DIRECTORY = \"YOUR_MODEL_DIRECTORY\"\n", @@ -96,7 +99,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -125,7 +128,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "metadata": {}, "outputs": [], "source": [ @@ -223,11 +226,11 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ - "def transform_selector(context_tuple, corpus_name):\n", + "def transform_selector(context_tuple):\n", " \"\"\"\n", " For transform we only need to check that the conversation is in the test split\n", " \"\"\"\n", @@ -238,7 +241,7 @@ " matches_split = (context_tuple.current_utterance.get_conversation().meta[\"split\"] == \"test\")\n", " is_end = (len(context_tuple.context) == convo_length)\n", "\n", - " if corpus_name.contains(\"cmv\"):\n", + " if \"cmv\" in corpus_name:\n", " return (matches_split)\n", " else:\n", " return (matches_split and not is_end)\n", @@ -505,7 +508,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 8, "metadata": {}, "outputs": [ { @@ -813,7 +816,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": ".venv", "language": "python", "name": "python3" }, @@ -827,7 +830,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.11" + "version": "3.12.4" } }, "nbformat": 4,