diff --git a/.gitignore b/.gitignore index e6458330..051e5a71 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *~ -*.pyc \ No newline at end of file +*.pyc +.DS_Store \ No newline at end of file diff --git a/docs/examples/dcn.ipynb b/docs/examples/dcn.ipynb index 12da7d01..c73d92a4 100644 --- a/docs/examples/dcn.ipynb +++ b/docs/examples/dcn.ipynb @@ -14,7 +14,10 @@ "execution_count": null, "metadata": { "cellView": "form", - "id": "uWqCArLO_kez" + "id": "uWqCArLO_kez", + "vscode": { + "languageId": "python" + } }, "outputs": [], "source": [ @@ -37,22 +40,22 @@ "id": "ikhIvrku-i-L" }, "source": [ - "# Deep \u0026 Cross Network (DCN)\n", - "\n", - "\u003ctable class=\"tfo-notebook-buttons\" align=\"left\"\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://www.tensorflow.org/recommenders/examples/dcn\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/tf_logo_32px.png\" /\u003eView on TensorFlow.org\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://colab.research.google.com/github/tensorflow/recommenders/blob/main/docs/examples/dcn.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/colab_logo_32px.png\" /\u003eRun in Google Colab\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca target=\"_blank\" href=\"https://github.com/tensorflow/recommenders/blob/main/docs/examples/dcn.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/GitHub-Mark-32px.png\" /\u003eView source on GitHub\u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd\u003e\n", - " \u003ca href=\"https://storage.googleapis.com/tensorflow_docs/recommenders/docs/examples/dcn.ipynb\"\u003e\u003cimg src=\"https://www.tensorflow.org/images/download_logo_32px.png\" /\u003eDownload notebook\u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/table\u003e" + "# Deep & Cross Network (DCN)\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + "
\n", + " View on TensorFlow.org\n", + " \n", + " Run in Google Colab\n", + " \n", + " View source on GitHub\n", + " \n", + " Download notebook\n", + "
" ] }, { @@ -61,16 +64,16 @@ "id": "Q-rOX95bAye4" }, "source": [ - "This tutorial demonstrates how to use Deep \u0026 Cross Network (DCN) to effectively learn feature crosses.\n", + "This tutorial demonstrates how to use Deep & Cross Network (DCN) to effectively learn feature crosses.\n", "\n", - "##Background\n", + "## Background\n", "\n", "**What are feature crosses and why are they important?** Imagine that we are building a recommender system to sell a blender to customers. Then, a customer's past purchase history such as `purchased_bananas` and `purchased_cooking_books`, or geographic features, are single features. If one has purchased both bananas **and** cooking books, then this customer will more likely click on the recommended blender. The combination of `purchased_bananas` and `purchased_cooking_books` is referred to as a **feature cross**, which provides additional interaction information beyond the individual features.\n", - "\u003cdiv\u003e\n", - "\u003ccenter\u003e\n", - "\u003cimg src=\"http://drive.google.com/uc?export=view\u0026id=1e8pYZHM1ZSwqBLYVkKDoGg0_2t2UPc2y\" width=\"600\"/\u003e\n", - "\u003c/center\u003e\n", - "\u003c/div\u003e\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", "\n", "\n", "\n", @@ -78,28 +81,29 @@ "**What are the challenges in learning feature crosses?** In Web-scale applications, data are mostly categorical, leading to large and sparse feature space. Identifying effective feature crosses in this setting often requires\n", "manual feature engineering or exhaustive search. Traditional feed-forward multilayer perceptron (MLP) models are universal function approximators; however, they cannot efficiently approximate even 2nd or 3rd-order feature crosses [[1](https://arxiv.org/pdf/2008.13535.pdf), [2](https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/18fa88ad519f25dc4860567e19ab00beff3f01cb.pdf)].\n", "\n", - "**What is Deep \u0026 Cross Network (DCN)?** DCN was designed to learn explicit and bounded-degree cross features more effectively. It starts with an input layer (typically an embedding layer), followed by a *cross network* containing multiple cross layers that models explicit feature interactions, and then combines\n", + "**What is Deep & Cross Network (DCN)?** DCN was designed to learn explicit and bounded-degree cross features more effectively. It starts with an input layer (typically an embedding layer), followed by a *cross network* containing multiple cross layers that models explicit feature interactions, and then combines\n", "with a *deep network* that models implicit feature interactions.\n", "\n", "\n", "* Cross Network. This is the core of DCN. It explicitly applies feature crossing at each layer, and the highest\n", "polynomial degree increases with layer depth. The following figure shows the $(i+1)$-th cross layer.\n", - "\u003cdiv class=\"fig figcenter fighighlight\"\u003e\n", - "\u003ccenter\u003e\n", - " \u003cimg src=\"http://drive.google.com/uc?export=view\u0026id=1QvIDptMxixFNp6P4bBqMN4AYAhAIAYQZ\" width=\"50%\" style=\"display:block\"\u003e\n", - " \u003c/center\u003e\n", - "\u003c/div\u003e\n", + "
\n", + "
\n", + " \n", + "
\n", + "
\n", + "\n", "* Deep Network. It is a traditional feedforward multilayer perceptron (MLP).\n", "\n", "The deep network and cross network are then combined to form DCN [[1](https://arxiv.org/pdf/2008.13535.pdf)]. Commonly, we could stack a deep network on top of the cross network (stacked structure); we could also place them in parallel (parallel structure). \n", "\n", "\n", - "\u003cdiv class=\"fig figcenter fighighlight\"\u003e\n", - "\u003ccenter\u003e\n", - " \u003cimg src=\"http://drive.google.com/uc?export=view\u0026id=1WtDUCV6b-eetUnWVCAmcPh8mJFut5EUd\" hspace=\"40\" width=\"30%\" style=\"margin: 0px 100px 0px 0px;\"\u003e\n", - " \u003cimg src=\"http://drive.google.com/uc?export=view\u0026id=1xo_twKb847hasfss7JxF0UtFX_rEb4nt\" width=\"20%\"\u003e\n", - " \u003c/center\u003e\n", - "\u003c/div\u003e" + "
\n", + "
\n", + " \n", + " \n", + "
\n", + "
" ] }, { @@ -124,7 +128,10 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "PjfZWVEWAmxS" + "id": "PjfZWVEWAmxS", + "vscode": { + "languageId": "python" + } }, "outputs": [], "source": [ @@ -136,7 +143,10 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "DqsyLA0UHeCl" + "id": "DqsyLA0UHeCl", + "vscode": { + "languageId": "python" + } }, "outputs": [], "source": [ @@ -199,7 +209,10 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "9rT3f6C3GX0u" + "id": "9rT3f6C3GX0u", + "vscode": { + "languageId": "python" + } }, "outputs": [], "source": [ @@ -236,7 +249,10 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "vrQWVYajgmNV" + "id": "vrQWVYajgmNV", + "vscode": { + "languageId": "python" + } }, "outputs": [], "source": [ @@ -267,7 +283,10 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "bwgAH2FTR4Fe" + "id": "bwgAH2FTR4Fe", + "vscode": { + "languageId": "python" + } }, "outputs": [], "source": [ @@ -312,7 +331,10 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "EwBwSHz_N3pW" + "id": "EwBwSHz_N3pW", + "vscode": { + "languageId": "python" + } }, "outputs": [], "source": [ @@ -340,7 +362,10 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "X6gD-NTF4eoj" + "id": "X6gD-NTF4eoj", + "vscode": { + "languageId": "python" + } }, "outputs": [], "source": [ @@ -361,7 +386,10 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "nFhrC7fV6szW" + "id": "nFhrC7fV6szW", + "vscode": { + "languageId": "python" + } }, "outputs": [], "source": [ @@ -382,7 +410,10 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "F8ZXXbmKuB8p" + "id": "F8ZXXbmKuB8p", + "vscode": { + "languageId": "python" + } }, "outputs": [], "source": [ @@ -394,7 +425,10 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "Tzg3KLKW2sdA" + "id": "Tzg3KLKW2sdA", + "vscode": { + "languageId": "python" + } }, "outputs": [], "source": [ @@ -416,7 +450,10 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "l4PM-goX6FoD" + "id": "l4PM-goX6FoD", + "vscode": { + "languageId": "python" + } }, "outputs": [], "source": [ @@ -452,7 +489,10 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "N8dga2Qck5IV" + "id": "N8dga2Qck5IV", + "vscode": { + "languageId": "python" + } }, "outputs": [], "source": [ @@ -506,7 +546,10 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "7Y_n3EPosR4A" + "id": "7Y_n3EPosR4A", + "vscode": { + "languageId": "python" + } }, "outputs": [], "source": [ @@ -535,7 +578,10 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "a5-l91jR_zEo" + "id": "a5-l91jR_zEo", + "vscode": { + "languageId": "python" + } }, "outputs": [], "source": [ @@ -559,7 +605,10 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "l9qhEcHq_VfI" + "id": "l9qhEcHq_VfI", + "vscode": { + "languageId": "python" + } }, "outputs": [], "source": [ @@ -588,7 +637,10 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "6lrDcBjiwnHU" + "id": "6lrDcBjiwnHU", + "vscode": { + "languageId": "python" + } }, "outputs": [], "source": [ @@ -685,7 +737,10 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "qeFjmfUbgzcS" + "id": "qeFjmfUbgzcS", + "vscode": { + "languageId": "python" + } }, "outputs": [], "source": [ @@ -706,7 +761,10 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "gTDk3GloquHO" + "id": "gTDk3GloquHO", + "vscode": { + "languageId": "python" + } }, "outputs": [], "source": [ @@ -743,7 +801,10 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "Zy3kWb5Dh0E7" + "id": "Zy3kWb5Dh0E7", + "vscode": { + "languageId": "python" + } }, "outputs": [], "source": [ @@ -758,18 +819,21 @@ }, "source": [ "**DCN (stacked).** We first train a DCN model with a stacked structure, that is, the inputs are fed to a cross network followed by a deep network.\n", - "\u003cdiv\u003e\n", - "\u003ccenter\u003e\n", - "\u003cimg src=\"http://drive.google.com/uc?export=view\u0026id=1X8qoMtIYKJz4yBYifvfw4QpAwrjr70e_\" width=\"140\"/\u003e\n", - "\u003c/center\u003e\n", - "\u003c/div\u003e\n" + "
\n", + "
\n", + "\n", + "
\n", + "
\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { - "id": "hiuYPJWhgw3J" + "id": "hiuYPJWhgw3J", + "vscode": { + "languageId": "python" + } }, "outputs": [], "source": [ @@ -785,18 +849,21 @@ "source": [ "**Low-rank DCN.** To reduce the training and serving cost, we leverage low-rank techniques to approximate the DCN weight matrices. The rank is passed in through argument `projection_dim`; a smaller `projection_dim` results in a lower cost. Note that `projection_dim` needs to be smaller than (input size)/2 to reduce the cost. In practice, we've observed using low-rank DCN with rank (input size)/4 consistently preserved the accuracy of a full-rank DCN.\n", "\n", - "\u003cdiv\u003e\n", - "\u003ccenter\u003e\n", - "\u003cimg src=\"http://drive.google.com/uc?export=view\u0026id=1ZZfUTNdxjGAaAuwNrweKkLJ1PGxMmiCm\" width=\"400\"/\u003e\n", - "\u003c/center\u003e\n", - "\u003c/div\u003e\n" + "
\n", + "
\n", + "\n", + "
\n", + "
\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": { - "id": "NYxbHI7ZNJX7" + "id": "NYxbHI7ZNJX7", + "vscode": { + "languageId": "python" + } }, "outputs": [], "source": [ @@ -818,7 +885,10 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "iBPpwD4cGtXF" + "id": "iBPpwD4cGtXF", + "vscode": { + "languageId": "python" + } }, "outputs": [], "source": [ @@ -839,7 +909,10 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "a1yj3pp0glEL" + "id": "a1yj3pp0glEL", + "vscode": { + "languageId": "python" + } }, "outputs": [], "source": [ @@ -872,14 +945,14 @@ "\n", "* *Concatenating cross layers.* The inputs are fed in parallel to multiple cross layers to capture complementary feature crosses.\n", "\n", - "\u003cdiv class=\"fig figcenter fighighlight\"\u003e\n", - "\u003ccenter\u003e\n", - " \u003cimg src=\"http://drive.google.com/uc?export=view\u0026id=11RpNuj9s0OgSav9TUuGA7v7PuFLL6nVR\" hspace=40 width=\"600\" style=\"display:block;\"\u003e\n", - " \u003cdiv class=\"figcaption\"\u003e\n", - " \u003cb\u003eLeft\u003c/b\u003e: DCN with a parallel structure; \u003cb\u003eRight\u003c/b\u003e: Concatenating cross layers. \n", - " \u003c/div\u003e\n", - " \u003c/center\u003e\n", - "\u003c/div\u003e" + "
\n", + "
\n", + " \n", + "
\n", + " Left: DCN with a parallel structure; Right: Concatenating cross layers. \n", + "
\n", + "
\n", + "
" ] }, { @@ -903,7 +976,10 @@ "cell_type": "code", "execution_count": null, "metadata": { - "id": "47ibaEBJxOoe" + "id": "47ibaEBJxOoe", + "vscode": { + "languageId": "python" + } }, "outputs": [], "source": [ @@ -951,12 +1027,12 @@ "---\n", "\n", "\n", - "##References\n", - "[DCN V2: Improved Deep \u0026 Cross Network and Practical Lessons for Web-scale Learning to Rank Systems](https://arxiv.org/pdf/2008.13535.pdf). \\\n", + "## References\n", + "[DCN V2: Improved Deep & Cross Network and Practical Lessons for Web-scale Learning to Rank Systems](https://arxiv.org/pdf/2008.13535.pdf). \\\n", "*Ruoxi Wang, Rakesh Shivanna, Derek Zhiyuan Cheng, Sagar Jain, Dong Lin, Lichan Hong, Ed Chi. (2020)*\n", "\n", "\n", - "[Deep \u0026 Cross Network for Ad Click Predictions](https://arxiv.org/pdf/1708.05123.pdf). \\\n", + "[Deep & Cross Network for Ad Click Predictions](https://arxiv.org/pdf/1708.05123.pdf). \\\n", "*Ruoxi Wang, Bin Fu, Gang Fu, Mingliang Wang. (AdKDD 2017)*" ] } diff --git a/docs/examples/images/blender-draw.gif b/docs/examples/images/blender-draw.gif new file mode 100644 index 00000000..a2f8da97 Binary files /dev/null and b/docs/examples/images/blender-draw.gif differ diff --git a/docs/examples/images/dcn-formula.png b/docs/examples/images/dcn-formula.png new file mode 100644 index 00000000..75d072a0 Binary files /dev/null and b/docs/examples/images/dcn-formula.png differ diff --git a/docs/examples/images/dcn-more-simple.png b/docs/examples/images/dcn-more-simple.png new file mode 100644 index 00000000..f95d9c85 Binary files /dev/null and b/docs/examples/images/dcn-more-simple.png differ diff --git a/docs/examples/images/dcn-parallel.png b/docs/examples/images/dcn-parallel.png new file mode 100644 index 00000000..6a370d55 Binary files /dev/null and b/docs/examples/images/dcn-parallel.png differ diff --git a/docs/examples/images/dcn-stack.png b/docs/examples/images/dcn-stack.png new file mode 100644 index 00000000..df56c649 Binary files /dev/null and b/docs/examples/images/dcn-stack.png differ diff --git a/docs/examples/images/dcn-stacked-simple.png b/docs/examples/images/dcn-stacked-simple.png new file mode 100644 index 00000000..4181a42d Binary files /dev/null and b/docs/examples/images/dcn-stacked-simple.png differ diff --git a/docs/examples/images/dcn_lowrank_simple.png b/docs/examples/images/dcn_lowrank_simple.png new file mode 100644 index 00000000..b445bade Binary files /dev/null and b/docs/examples/images/dcn_lowrank_simple.png differ