{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "provenance": [], "gpuType": "T4" }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "language_info": { "name": "python" }, "accelerator": "GPU", "widgets": { "application/vnd.jupyter.widget-state+json": { "cb93b0c0872446708f4f040336e4f261": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_df9772ff539145f1a4571b07a3961276", "IPY_MODEL_c97a2f8b3983426589da7b26468e4ccd", "IPY_MODEL_a16dccbeaebd4831a2a50421d6b0e260" ], "layout": "IPY_MODEL_fe703d3109f14ad48442fc27186b0c2f" } }, "df9772ff539145f1a4571b07a3961276": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_a0b1dd8bc303493dbec3eb13a2e4dea7", "placeholder": "​", "style": "IPY_MODEL_afd3ca7d3c5441ce93865e4819519940", "value": "tokenizer_config.json: 100%" } }, "c97a2f8b3983426589da7b26468e4ccd": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_6d8ee31edea849f19feb352de0293f19", "max": 48, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_8e2723a00a4344899763fc8cdb970182", "value": 48 } }, "a16dccbeaebd4831a2a50421d6b0e260": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_f4f35fb2dba845d185a417f0e2094b15", "placeholder": "​", "style": "IPY_MODEL_840edd2e3f1e421db7e6fd93597027ea", "value": " 48.0/48.0 [00:00<00:00, 2.90kB/s]" } }, "fe703d3109f14ad48442fc27186b0c2f": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "a0b1dd8bc303493dbec3eb13a2e4dea7": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "afd3ca7d3c5441ce93865e4819519940": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "6d8ee31edea849f19feb352de0293f19": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "8e2723a00a4344899763fc8cdb970182": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "f4f35fb2dba845d185a417f0e2094b15": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "840edd2e3f1e421db7e6fd93597027ea": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "cd1c1bf54c8a4474b7751feffe822e86": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_f4a32383d73f4bff823a6075ad56b268", "IPY_MODEL_b65ecdb616954c3f866e6ddbb3b9ee9e", "IPY_MODEL_cf58b2b0c09b443e9882cd4f71a85bc0" ], "layout": "IPY_MODEL_34826a684a5641e599a12c826b996c62" } }, "f4a32383d73f4bff823a6075ad56b268": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_acda006173dc420887738f8921893d2e", "placeholder": "​", "style": "IPY_MODEL_aa6c6b261c3e4b888a2e9fdbd2a7cd22", "value": "vocab.txt: 100%" } }, "b65ecdb616954c3f866e6ddbb3b9ee9e": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_e5beea48108c46ed88eb557c99ece4e2", "max": 231508, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_67c5fa90d29c45148ad385324be53ae1", "value": 231508 } }, "cf58b2b0c09b443e9882cd4f71a85bc0": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_0c1bd3fc37b743d1b3a3f13b3b0a7b69", "placeholder": "​", "style": "IPY_MODEL_039094aa47574c60a16edfbb7d326c42", "value": " 232k/232k [00:00<00:00, 3.72MB/s]" } }, "34826a684a5641e599a12c826b996c62": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "acda006173dc420887738f8921893d2e": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "aa6c6b261c3e4b888a2e9fdbd2a7cd22": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "e5beea48108c46ed88eb557c99ece4e2": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "67c5fa90d29c45148ad385324be53ae1": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "0c1bd3fc37b743d1b3a3f13b3b0a7b69": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "039094aa47574c60a16edfbb7d326c42": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "5260387ac9914fbea8f0d88a9ec0f058": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_50155b0e5e804e5097730bc9f450b9b4", "IPY_MODEL_b1654203849247f39884696ff1eda2f4", "IPY_MODEL_02b51525d9834a43bd9b555c077609f2" ], "layout": "IPY_MODEL_2299d5b1a59348618e275df847803cfa" } }, "50155b0e5e804e5097730bc9f450b9b4": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_b0edd8c2bb314d9ebb83a8b9fce170e7", "placeholder": "​", "style": "IPY_MODEL_42a439592989402abb6286db9f9b9fa3", "value": "tokenizer.json: 100%" } }, "b1654203849247f39884696ff1eda2f4": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_f768b9a5a1c84921ade0c1237af3fe80", "max": 466062, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_349a3525b93b452e89a9e334de00a3cd", "value": 466062 } }, "02b51525d9834a43bd9b555c077609f2": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_29085777a67d4bb39f319e204db5c05f", "placeholder": "​", "style": "IPY_MODEL_473d92a0a9f24a30b3d10c84a83a5103", "value": " 466k/466k [00:00<00:00, 5.69MB/s]" } }, "2299d5b1a59348618e275df847803cfa": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "b0edd8c2bb314d9ebb83a8b9fce170e7": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "42a439592989402abb6286db9f9b9fa3": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "f768b9a5a1c84921ade0c1237af3fe80": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "349a3525b93b452e89a9e334de00a3cd": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "29085777a67d4bb39f319e204db5c05f": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "473d92a0a9f24a30b3d10c84a83a5103": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "303f3f26dc1e4a78bf8cd3cb2e6f2d4a": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_f330f3d679b642f08014c1faa8b623cf", "IPY_MODEL_aa813a951ceb44209385914a1a7787f5", "IPY_MODEL_30aefa828c3e411f9490ae7e4a49aa3e" ], "layout": "IPY_MODEL_4e2392c88ae04ae18046b978d5c54058" } }, "f330f3d679b642f08014c1faa8b623cf": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_ae907e8f91a74d2199d245c08b8ba1ca", "placeholder": "​", "style": "IPY_MODEL_a45455994a1a42afa96f7d245b4536cc", "value": "config.json: 100%" } }, "aa813a951ceb44209385914a1a7787f5": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_903c4a8813db417b9db5762fa1cb6c4b", "max": 570, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_d6ecc30be6c44926b498f9988036a548", "value": 570 } }, "30aefa828c3e411f9490ae7e4a49aa3e": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_6257eb5097d149ddbad82bad21a6d278", "placeholder": "​", "style": "IPY_MODEL_7fdbf148a31d41eb80e3a40962e08e44", "value": " 570/570 [00:00<00:00, 41.7kB/s]" } }, "4e2392c88ae04ae18046b978d5c54058": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "ae907e8f91a74d2199d245c08b8ba1ca": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "a45455994a1a42afa96f7d245b4536cc": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "903c4a8813db417b9db5762fa1cb6c4b": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "d6ecc30be6c44926b498f9988036a548": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "6257eb5097d149ddbad82bad21a6d278": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "7fdbf148a31d41eb80e3a40962e08e44": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "5988ce482ad84ab5a9894ac83f87c6d2": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_e5385adcb0bd45309154fbd018fb4339", "IPY_MODEL_88ee9da252974899958be98a66726481", "IPY_MODEL_72a74a7b54b64870a1467a97a3e3202a" ], "layout": "IPY_MODEL_0424a95ba585487baf306be4c128a093" } }, "e5385adcb0bd45309154fbd018fb4339": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_3e8571b9bdb64bf09530553519ec4c9c", "placeholder": "​", "style": "IPY_MODEL_d20c707cfbe44454b00dad139afc2b09", "value": "model.safetensors: 100%" } }, "88ee9da252974899958be98a66726481": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_98b2f667c38e491db2510a9fe242ca02", "max": 440449768, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_407447ce58c04f8c9a8c65b2f39c044d", "value": 440449768 } }, "72a74a7b54b64870a1467a97a3e3202a": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_fc9e186ebc3f4fc8bdd79d13a58caa1b", "placeholder": "​", "style": "IPY_MODEL_8818f7d745b741878c194308a073f3e2", "value": " 440M/440M [00:03<00:00, 130MB/s]" } }, "0424a95ba585487baf306be4c128a093": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "3e8571b9bdb64bf09530553519ec4c9c": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "d20c707cfbe44454b00dad139afc2b09": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "98b2f667c38e491db2510a9fe242ca02": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "407447ce58c04f8c9a8c65b2f39c044d": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "fc9e186ebc3f4fc8bdd79d13a58caa1b": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "8818f7d745b741878c194308a073f3e2": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "b00a5318bf614a3fb9d5c05704eea9fe": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_d61397a2057541e19b38fc53bb1e6286", "IPY_MODEL_7247f5f59eeb4391a1a618d29924529c", "IPY_MODEL_84c7855f0298466295a0f07927156c58" ], "layout": "IPY_MODEL_db35422f35d74e0da37c39f24ad3a7c3" } }, "d61397a2057541e19b38fc53bb1e6286": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_7988e33c9e394e8a831d8ef7cb434973", "placeholder": "​", "style": "IPY_MODEL_ddf856a8bed641f8a15bf6d3a2a7aeb4", "value": "Downloading builder script: 100%" } }, "7247f5f59eeb4391a1a618d29924529c": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_634c152c9bef4b5ea7672ea87abe52cb", "max": 5937, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_c8ff7e1de7204ba2b2f60502aa8de390", "value": 5937 } }, "84c7855f0298466295a0f07927156c58": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_255b656e99ca4aec9dbb78065c948036", "placeholder": "​", "style": "IPY_MODEL_ae3d320ebb4d4a07a0187e6a474d736c", "value": " 5.94k/5.94k [00:00<00:00, 327kB/s]" } }, "db35422f35d74e0da37c39f24ad3a7c3": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "7988e33c9e394e8a831d8ef7cb434973": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "ddf856a8bed641f8a15bf6d3a2a7aeb4": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "634c152c9bef4b5ea7672ea87abe52cb": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "c8ff7e1de7204ba2b2f60502aa8de390": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "255b656e99ca4aec9dbb78065c948036": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "ae3d320ebb4d4a07a0187e6a474d736c": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "7f2b94cb9c1a4c74bdd4a6a01517a5e9": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_5b76a3cbec9b40cd88f72b871181d194", "IPY_MODEL_cf612ec51bc24437abea77ba697a6b8b", "IPY_MODEL_37c3d5260b464a29ae015de768475579" ], "layout": "IPY_MODEL_1badbd572b8a4b2494ba4ec24cfa4201" } }, "5b76a3cbec9b40cd88f72b871181d194": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_b6c28827a0cf4f098f9210822f3b6990", "placeholder": "​", "style": "IPY_MODEL_0d6eb59f405046e4baca4fd859f5632d", "value": "Downloading extra modules: " } }, "cf612ec51bc24437abea77ba697a6b8b": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_2c9b89a323b8494691f9edb1278beaea", "max": 1554, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_0c62e2a754634e608ca4cb64c99542be", "value": 1554 } }, "37c3d5260b464a29ae015de768475579": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_1ba081a449cd4663ba2e6db5d2a6151b", "placeholder": "​", "style": "IPY_MODEL_8d5bcdddfc3e4f1cb5e5e12b5ea9b3cc", "value": " 4.07k/? [00:00<00:00, 208kB/s]" } }, "1badbd572b8a4b2494ba4ec24cfa4201": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "b6c28827a0cf4f098f9210822f3b6990": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "0d6eb59f405046e4baca4fd859f5632d": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "2c9b89a323b8494691f9edb1278beaea": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "0c62e2a754634e608ca4cb64c99542be": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "1ba081a449cd4663ba2e6db5d2a6151b": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "8d5bcdddfc3e4f1cb5e5e12b5ea9b3cc": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "d9e4e9d0a4c34d32b152cddc533061b2": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HBoxModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HBoxView", "box_style": "", "children": [ "IPY_MODEL_b906934edda5424580fb5a676a42b482", "IPY_MODEL_67013f4714f9491c9a741399dfc38a85", "IPY_MODEL_c9a6c6b57a5c49ceabaf65de006a5a25" ], "layout": "IPY_MODEL_76ef22dfea0d48248e68c4e697f70c56" } }, "b906934edda5424580fb5a676a42b482": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_d9c26846632241499ae87589e011671c", "placeholder": "​", "style": "IPY_MODEL_ccc6082d5b284b38a72e88cd25fdc80c", "value": "Downloading extra modules: 100%" } }, "67013f4714f9491c9a741399dfc38a85": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "FloatProgressModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "ProgressView", "bar_style": "success", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_0a91f9152d004fc297ace0cd97cefe62", "max": 3344, "min": 0, "orientation": "horizontal", "style": "IPY_MODEL_592a6bb343504ebc99d4a66e39629685", "value": 3344 } }, "c9a6c6b57a5c49ceabaf65de006a5a25": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "model_module_version": "1.5.0", "state": { "_dom_classes": [], "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "HTMLModel", "_view_count": null, "_view_module": "@jupyter-widgets/controls", "_view_module_version": "1.5.0", "_view_name": "HTMLView", "description": "", "description_tooltip": null, "layout": "IPY_MODEL_b22679e161ce444abad828adc5f30c9d", "placeholder": "​", "style": "IPY_MODEL_ad88a23472eb461e96a3d8d934566de3", "value": " 3.34k/3.34k [00:00<00:00, 213kB/s]" } }, "76ef22dfea0d48248e68c4e697f70c56": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "d9c26846632241499ae87589e011671c": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "ccc6082d5b284b38a72e88cd25fdc80c": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } }, "0a91f9152d004fc297ace0cd97cefe62": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "592a6bb343504ebc99d4a66e39629685": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "ProgressStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "bar_color": null, "description_width": "" } }, "b22679e161ce444abad828adc5f30c9d": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "model_module_version": "1.2.0", "state": { "_model_module": "@jupyter-widgets/base", "_model_module_version": "1.2.0", "_model_name": "LayoutModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "LayoutView", "align_content": null, "align_items": null, "align_self": null, "border": null, "bottom": null, "display": null, "flex": null, "flex_flow": null, "grid_area": null, "grid_auto_columns": null, "grid_auto_flow": null, "grid_auto_rows": null, "grid_column": null, "grid_gap": null, "grid_row": null, "grid_template_areas": null, "grid_template_columns": null, "grid_template_rows": null, "height": null, "justify_content": null, "justify_items": null, "left": null, "margin": null, "max_height": null, "max_width": null, "min_height": null, "min_width": null, "object_fit": null, "object_position": null, "order": null, "overflow": null, "overflow_x": null, "overflow_y": null, "padding": null, "right": null, "top": null, "visibility": null, "width": null } }, "ad88a23472eb461e96a3d8d934566de3": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "model_module_version": "1.5.0", "state": { "_model_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_model_name": "DescriptionStyleModel", "_view_count": null, "_view_module": "@jupyter-widgets/base", "_view_module_version": "1.2.0", "_view_name": "StyleView", "description_width": "" } } } } }, "cells": [ { "cell_type": "code", "source": [ "from collections import Counter\n", "import random\n", "\n", "import torch\n", "import torch.nn.functional as F\n", "import torch.utils.data as data\n", "from torch.nn.utils.rnn import pad_sequence\n", "from torchtext.datasets import Multi30k\n", "from torchtext.data.utils import get_tokenizer\n", "from torchtext.vocab import vocab\n", "from tqdm import tqdm\n", "\n", "import torch.nn as nn\n", "import torch.optim as optim\n", "import math\n", "import copy\n", "import numpy as np\n", "\n", "\n", "random.seed(0)\n", "torch.manual_seed(seed=0)\n", "torch.backends.cudnn.deterministic = True\n", "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", "print(\"using device:\", device)" ], "metadata": { "id": "VCdQucUo7b5F", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "83ff22af-22b8-41df-9cfd-c3df92db7e98" }, "execution_count": 1, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "using device: cuda\n" ] } ] }, { "cell_type": "code", "source": [], "metadata": { "id": "1NXzmOrWtf2n" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "class MultiHeadAttention(nn.Module):\n", " def __init__(self, d_model, num_heads):\n", " super(MultiHeadAttention, self).__init__()\n", " # Ensure that the model dimension (d_model) is divisible by the number of heads\n", " assert d_model % num_heads == 0, \"d_model must be divisible by num_heads\"\n", "\n", " # Initialize dimensions\n", " self.d_model = d_model # Model's dimension\n", " self.num_heads = num_heads # Number of attention heads\n", " self.d_k = d_model // num_heads # Dimension of each head's key, query, and value\n", "\n", " # Linear layers for transforming inputs\n", " self.W_q = nn.Linear(d_model, d_model) # Query transformation\n", " self.W_k = nn.Linear(d_model, d_model) # Key transformation\n", " self.W_v = nn.Linear(d_model, d_model) # Value transformation\n", " self.W_o = nn.Linear(d_model, d_model) # Output transformation\n", "\n", " def scaled_dot_product_attention(self, Q, K, V, mask=None):\n", " # Calculate attention scores\n", " attn_scores = torch.matmul(Q, K.transpose(-2, -1)) / math.sqrt(self.d_k)\n", "\n", " # Apply mask if provided (useful for preventing attention to certain parts like padding)\n", " if mask is not None:\n", " attn_scores = attn_scores.masked_fill(mask == 0, -1e9)\n", "\n", " # Softmax is applied to obtain attention probabilities\n", " attn_probs = torch.softmax(attn_scores, dim=-1)\n", "\n", " # Multiply by values to obtain the final output\n", " output = torch.matmul(attn_probs, V)\n", " return output\n", "\n", " def split_heads(self, x):\n", " # Reshape the input to have num_heads for multi-head attention\n", " batch_size, seq_length, d_model = x.size()\n", " return x.view(batch_size, seq_length, self.num_heads, self.d_k).transpose(1, 2)\n", "\n", " def combine_heads(self, x):\n", " # Combine the multiple heads back to original shape\n", " batch_size, _, seq_length, d_k = x.size()\n", " return x.transpose(1, 2).contiguous().view(batch_size, seq_length, self.d_model)\n", "\n", " def forward(self, Q, K, V, mask=None):\n", " # Apply linear transformations and split heads\n", " Q = self.split_heads(self.W_q(Q))\n", " K = self.split_heads(self.W_k(K))\n", " V = self.split_heads(self.W_v(V))\n", "\n", " # Perform scaled dot-product attention\n", " attn_output = self.scaled_dot_product_attention(Q, K, V, mask)\n", "\n", " # Combine heads and apply output transformation\n", " output = self.W_o(self.combine_heads(attn_output))\n", " return output\n", "\n", "class PositionWiseFeedForward(nn.Module):\n", " def __init__(self, d_model, d_ff):\n", " super(PositionWiseFeedForward, self).__init__()\n", " self.fc1 = nn.Linear(d_model, d_ff)\n", " self.fc2 = nn.Linear(d_ff, d_model)\n", " self.relu = nn.ReLU()\n", "\n", " def forward(self, x):\n", " return self.fc2(self.relu(self.fc1(x)))\n", "\n", "class PositionalEncoding(nn.Module):\n", " def __init__(self, d_model, max_seq_length):\n", " super(PositionalEncoding, self).__init__()\n", "\n", " pe = torch.zeros(max_seq_length, d_model)\n", " position = torch.arange(0, max_seq_length, dtype=torch.float).unsqueeze(1)\n", " div_term = torch.exp(torch.arange(0, d_model, 2).float() * -(math.log(10000.0) / d_model))\n", "\n", " pe[:, 0::2] = torch.sin(position * div_term)\n", " pe[:, 1::2] = torch.cos(position * div_term)\n", "\n", " self.register_buffer('pe', pe.unsqueeze(0))\n", "\n", " def forward(self, x):\n", " return x + self.pe[:, :x.size(1)]\n", "\n", "\n", "class EncoderLayer(nn.Module):\n", " def __init__(self, d_model, num_heads, d_ff, dropout):\n", " super(EncoderLayer, self).__init__()\n", " self.self_attn = MultiHeadAttention(d_model, num_heads)\n", " self.feed_forward = PositionWiseFeedForward(d_model, d_ff)\n", " self.norm1 = nn.LayerNorm(d_model)\n", " self.norm2 = nn.LayerNorm(d_model)\n", " self.dropout = nn.Dropout(dropout)\n", "\n", " def forward(self, x, mask):\n", " attn_output = self.self_attn(x, x, x, mask)\n", " x = self.norm1(x + self.dropout(attn_output))\n", " ff_output = self.feed_forward(x)\n", " x = self.norm2(x + self.dropout(ff_output))\n", " return x\n", "\n", "\n", "class DecoderLayer(nn.Module):\n", " def __init__(self, d_model, num_heads, d_ff, dropout):\n", " super(DecoderLayer, self).__init__()\n", " self.self_attn = MultiHeadAttention(d_model, num_heads)\n", " self.cross_attn = MultiHeadAttention(d_model, num_heads)\n", " self.feed_forward = PositionWiseFeedForward(d_model, d_ff)\n", " self.norm1 = nn.LayerNorm(d_model)\n", " self.norm2 = nn.LayerNorm(d_model)\n", " self.norm3 = nn.LayerNorm(d_model)\n", " self.dropout = nn.Dropout(dropout)\n", "\n", " def forward(self, x, enc_output, src_mask, tgt_mask):\n", " attn_output = self.self_attn(x, x, x, tgt_mask)\n", " x = self.norm1(x + self.dropout(attn_output))\n", " attn_output = self.cross_attn(x, enc_output, enc_output, src_mask)\n", " x = self.norm2(x + self.dropout(attn_output))\n", " ff_output = self.feed_forward(x)\n", " x = self.norm3(x + self.dropout(ff_output))\n", " return x\n", "\n", "\n", "\n", "class Transformer(nn.Module):\n", " def __init__(self, src_vocab_size, tgt_vocab_size, d_model, num_heads, num_layers, d_ff, max_seq_length, dropout, graph_type):\n", " super(Transformer, self).__init__()\n", " self.encoder_embedding = nn.Embedding(src_vocab_size, d_model)\n", " self.decoder_embedding = nn.Embedding(tgt_vocab_size, d_model)\n", " self.positional_encoding = PositionalEncoding(d_model, max_seq_length)\n", "\n", " self.encoder_layers = nn.ModuleList([EncoderLayer(d_model, num_heads, d_ff, dropout) for _ in range(num_layers)])\n", " self.decoder_layers = nn.ModuleList([DecoderLayer(d_model, num_heads, d_ff, dropout) for _ in range(num_layers)])\n", "\n", " self.fc = nn.Linear(d_model, tgt_vocab_size)\n", " self.dropout = nn.Dropout(dropout)\n", " self.graph_type= graph_type\n", "\n", " def generate_mask(self, src, tgt):\n", " src_mask = (src != 0).unsqueeze(1).unsqueeze(2)\n", " tgt_mask = (tgt != 0).unsqueeze(1).unsqueeze(3)\n", " seq_length = tgt.size(1)\n", " nopeak_mask = (1 - torch.triu(torch.ones(1, seq_length, seq_length), diagonal=1)).bool()\n", "\n", " # tgt_mask = tgt_mask & nopeak_mask\n", " return src_mask, tgt_mask\n", "\n", " def forward(self, src, tgt, grf1, grf2, grf3, grf4):\n", " src_mask, tgt_mask = self.generate_mask(src, tgt)\n", " src_embedded = self.dropout(self.positional_encoding(self.encoder_embedding(src)))\n", "\n", " # print('src_embedded.shape', src_embedded.shape)\n", "\n", " grf_embedded1 = self.dropout(grf1)\n", " grf_embedded2 = self.dropout(grf2)\n", " grf_embedded3 = self.dropout(grf3)\n", " grf_embedded4 = self.dropout(grf4)\n", "\n", " tgt_embedded = self.dropout(self.positional_encoding(self.decoder_embedding(tgt)))\n", "\n", " if self.graph_type == \"basic\":\n", " enc_output = src_embedded # basic transformer\n", " elif self.graph_type == \"egcr\":\n", " enc_output = src_embedded + grf_embedded1 # basic paper\n", " elif self.graph_type == \"pos\":\n", " enc_output = src_embedded + grf_embedded2 # basicpaper transformer + pos graph\n", " elif self.graph_type == \"dp\":\n", " enc_output = src_embedded + grf_embedded3 # basicpaper transformer + DP graph\n", " elif self.graph_type == \"semantic\":\n", " enc_output = src_embedded + grf_embedded4 # basicpaper transformer + semantic graph\n", " else:\n", " enc_output = src_embedded + grf_embedded1 + grf_embedded2 + grf_embedded3 # basicpaper graph + pos graph + parsing-tree graph\n", " # print(grf_embedded.shape)\n", "\n", " for enc_layer in self.encoder_layers:\n", " enc_output = enc_layer(enc_output, src_mask)\n", "\n", " dec_output = tgt_embedded\n", " for dec_layer in self.decoder_layers:\n", " dec_output = dec_layer(dec_output, enc_output, src_mask, tgt_mask)\n", "\n", " output = self.fc(dec_output)\n", " return output" ], "metadata": { "id": "gtAcS2Ux2bQ9" }, "execution_count": 2, "outputs": [] }, { "cell_type": "code", "source": [ "def preprocess_english(sentence):\n", " \"\"\"\n", " preprocess english senttence\n", "\n", " Param sentence: input sentence\n", " \"\"\"\n", "\n", " sentence = sentence.strip()\n", " sentence = sentence.replace(';', '')\n", " return sentence\n", "\n", "def preprocess_persian(sentence):\n", " \"\"\"\n", " preprocess persian senttence\n", "\n", " Param sentence: input sentence\n", " \"\"\"\n", "\n", " sentence = sentence.strip().split('///')[0]\n", " sentence = sentence.replace(';', '')\n", " return sentence\n", "\n", "def process_input_line(l):\n", " \"\"\"\n", " preprocess input en --> fa sentendes\n", " each sentence is contain a persian sentcence and some of persian sentncess seperated by \"///\"\n", "\n", " Param: l: input row (line)\n", " \"\"\"\n", " l = l.strip().split('\\t')\n", " return [preprocess_english(l[0]), preprocess_persian(l[1])]\n", "\n", "def preprocess_tep_dataset(en_path, fa_path):\n", " \"\"\"\n", " making text pairs for tep dataset\n", "\n", " \"\"\"\n", "\n", " en_lines = []\n", " with open(en_path) as f:\n", " en_lines = f.readlines()\n", " en_lines = [line.strip() for line in en_lines]\n", "\n", " fa_lines = []\n", " with open(fa_path) as f:\n", " fa_lines = f.readlines()\n", " fa_lines = [line.strip() for line in fa_lines]\n", "\n", " text_pairs = []\n", " for en, fa in zip(en_lines, fa_lines):\n", " text_pairs.append([en, fa])\n", "\n", " return text_pairs\n", "\n", "\n" ], "metadata": { "id": "x0R-YNVTfC8l" }, "execution_count": 3, "outputs": [] }, { "cell_type": "code", "source": [ "\n", "# download file\n", "# download tep datste\n", "import os\n", "if not os.path.exists('tep-data.zip'):\n", " !wget -O tep-data.zip https://object.pouta.csc.fi/OPUS-TEP/v1/moses/en-fa.txt.zip\n", "\n", "\n", "if not os.path.exists('TEP.en-fa.en'):\n", " !unzip tep-data.zip\n" ], "metadata": { "id": "5pyiLVLb2gyE", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "e7fd05af-f404-4abd-bb0e-1122356cba26" }, "execution_count": 4, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "--2024-04-16 11:53:53-- https://object.pouta.csc.fi/OPUS-TEP/v1/moses/en-fa.txt.zip\n", "Resolving object.pouta.csc.fi (object.pouta.csc.fi)... 86.50.254.18, 86.50.254.19\n", "Connecting to object.pouta.csc.fi (object.pouta.csc.fi)|86.50.254.18|:443... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 16353318 (16M) [application/zip]\n", "Saving to: ‘tep-data.zip’\n", "\n", "tep-data.zip 100%[===================>] 15.60M 10.2MB/s in 1.5s \n", "\n", "2024-04-16 11:53:55 (10.2 MB/s) - ‘tep-data.zip’ saved [16353318/16353318]\n", "\n", "Archive: tep-data.zip\n", " inflating: TEP.en-fa.en \n", " inflating: TEP.en-fa.fa \n", " inflating: README \n" ] } ] }, { "cell_type": "code", "source": [ "# nltk modules\n", "from nltk.tokenize import sent_tokenize\n", "import nltk\n", "nltk.download('punkt')\n", "nltk.download('averaged_perceptron_tagger')" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "IKQyAmbDfoLm", "outputId": "ca2f570d-e9da-434c-dff4-588e7962ebb0" }, "execution_count": 5, "outputs": [ { "output_type": "stream", "name": "stderr", "text": [ "[nltk_data] Downloading package punkt to /root/nltk_data...\n", "[nltk_data] Unzipping tokenizers/punkt.zip.\n", "[nltk_data] Downloading package averaged_perceptron_tagger to\n", "[nltk_data] /root/nltk_data...\n", "[nltk_data] Unzipping taggers/averaged_perceptron_tagger.zip.\n" ] }, { "output_type": "execute_result", "data": { "text/plain": [ "True" ] }, "metadata": {}, "execution_count": 5 } ] }, { "cell_type": "code", "source": [ "# data pr3eprocessing and limit\n", "text_pairs = preprocess_tep_dataset('TEP.en-fa.en', 'TEP.en-fa.fa')\n", "text_pairs = text_pairs[:10000] # get just 10000 first rows\n", "\n", "\n", "lengths = []\n", "for src, tgt in text_pairs:\n", " l = sent_tokenize(src)\n", " for item in l:\n", " lengths.append(len(item))\n", "\n", "print(\"max sentecne length is: \", max(lengths))\n", "\n", "\n", "all_data = [(src.lower(), tgt) for src, tgt in text_pairs if len(src) > 0 and len(src) < 800 and len(tgt) < 800]\n", "\n" ], "metadata": { "id": "m8TfMZvpS-MD", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "8743955e-3482-4a72-89dd-1e0b7a11b993" }, "execution_count": 6, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "max sentecne length is: 156\n" ] } ] }, { "cell_type": "code", "source": [ "# import pandas as pd\n", "# data_df = {'persian':[], 'english':[]}\n", "\n", "\n", "# for li in all_data:\n", "# data_df['english'].append(li[0])\n", "# data_df['persian'].append(li[1])\n", "\n", "# data_df = pd.DataFrame(data_df)\n", "# data_df.to_csv('tep_slice.csv', index=False, encoding='utf-8-sig')" ], "metadata": { "id": "Gtv2Mfr6xIMz" }, "execution_count": 9, "outputs": [] }, { "cell_type": "code", "source": [ "# train test split\n", "\n", "# random.shuffle(all_data)\n", "n_val = int(0.15*len(all_data))\n", "n_train = len(all_data) - 2*n_val\n", "train_pairs = all_data[:n_train]\n", "val_pairs = all_data[n_train:n_train+n_val]\n", "test_pairs = all_data[n_train+n_val:]\n", "\n", "train_data = [(src, tgt) for src, tgt in train_pairs if len(src) > 0]\n", "val_data = [(src, tgt) for src, tgt in val_pairs if len(src) > 0]\n", "test_data = [(src, tgt) for src, tgt in test_pairs if len(src) > 0]\n", "\n", "print(\"train data samples count is: \", len(train_data))\n", "print(\"test data samples count is: \", len(test_data))\n", "print(\"validation data samples count is: \", len(val_data))\n", "\n", "\n" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "tPkW0nx3fdHy", "outputId": "a069f25d-710a-4bf6-9951-6d1ccf7a09f7" }, "execution_count": 22, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "train data samples count is: 7000\n", "test data samples count is: 1500\n", "validation data samples count is: 1500\n" ] } ] }, { "cell_type": "code", "source": [ "# build vocab\n", "# Build src and tgt vocabs from the training set.\n", "PAD, BOS, EOS = (\"\", \"\", \"\")\n", "tokenizer = get_tokenizer(\"basic_english\")\n", "en_counter, de_counter = Counter(), Counter()\n", "for src, tgt in all_data:\n", " en_counter.update(tokenizer(src))\n", " de_counter.update(tokenizer(tgt))\n", "de_vocab = vocab(de_counter, specials=[PAD, BOS, EOS])\n", "en_vocab = vocab(en_counter, specials=[PAD, BOS, EOS])\n", "pad_idx = de_vocab[PAD] # pad_idx is 0\n", "assert en_vocab[PAD] == de_vocab[PAD]\n", "\n", "\n" ], "metadata": { "id": "Kz5OZivL2g11" }, "execution_count": 23, "outputs": [] }, { "cell_type": "code", "source": [ "# graph modules\n", "# We assume that PyTorch is already installed\n", "import torch\n", "torchversion = torch.__version__\n", "\n", "# Install PyTorch Scatter, PyTorch Sparse, and PyTorch Geometric\n", "!pip install -q torch-scatter -f https://data.pyg.org/whl/torch-{torchversion}.html\n", "!pip install -q torch-sparse -f https://data.pyg.org/whl/torch-{torchversion}.html\n", "!pip install -q git+https://github.com/pyg-team/pytorch_geometric.git\n", "\n", "\n", "\n", "# Visualization\n", "import networkx as nx\n", "import matplotlib.pyplot as plt\n" ], "metadata": { "id": "hlEvSZ7V2g4t", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "ded0f172-9b18-4e65-9e1e-f0910f89f400" }, "execution_count": 24, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m10.9/10.9 MB\u001b[0m \u001b[31m43.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.0/5.0 MB\u001b[0m \u001b[31m29.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", " Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", " Building wheel for torch-geometric (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n" ] } ] }, { "cell_type": "code", "source": [], "metadata": { "id": "t57UsIi6-6L3" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "import torch\n", "from transformers import BertTokenizer, BertModel\n", "import transformers\n", "from sklearn.metrics.pairwise import cosine_similarity\n" ], "metadata": { "id": "AD_fyuZQ8-FL" }, "execution_count": 25, "outputs": [] }, { "cell_type": "code", "source": [ "def get_token_vector(bert_tokenizer, bert_embedding_model, token):\n", " token_bert = bert_tokenizer([token], return_tensors='pt')\n", " with torch.no_grad():\n", " outputs = bert_embedding_model(token_bert['input_ids'], attention_mask=token_bert['attention_mask'])\n", " word_embeddings = outputs.last_hidden_state\n", " if len(word_embeddings[0]) == 3:\n", " return word_embeddings[0][1]\n", " else:\n", " return torch.zeros(word_embeddings[0].shape[1])" ], "metadata": { "id": "mP4imrx3DXB7" }, "execution_count": 26, "outputs": [] }, { "cell_type": "code", "source": [ "train_pairs[20][0]" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 35 }, "id": "zV2gckqbFCYd", "outputId": "c5e2c506-9462-43b9-b92c-3dce9c2e69c8" }, "execution_count": 27, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "'alice , we cant make it rain together turn the dust into grass .'" ], "application/vnd.google.colaboratory.intrinsic+json": { "type": "string" } }, "metadata": {}, "execution_count": 27 } ] }, { "cell_type": "code", "source": [], "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 35 }, "id": "ShEQXsuPFanR", "outputId": "ca6b5e6b-7897-42f5-e365-30f0850c4610" }, "execution_count": null, "outputs": [ { "output_type": "execute_result", "data": { "text/plain": [ "'c'" ], "application/vnd.google.colaboratory.intrinsic+json": { "type": "string" } }, "metadata": {}, "execution_count": 176 } ] }, { "cell_type": "code", "source": [ "\n", "\n", "bert_tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n", "bert_embedding_model = BertModel.from_pretrained('bert-base-uncased')" ], "metadata": { "id": "wqfQqccS_y6G", "colab": { "base_uri": "https://localhost:8080/", "height": 177, "referenced_widgets": [ "cb93b0c0872446708f4f040336e4f261", "df9772ff539145f1a4571b07a3961276", "c97a2f8b3983426589da7b26468e4ccd", "a16dccbeaebd4831a2a50421d6b0e260", "fe703d3109f14ad48442fc27186b0c2f", "a0b1dd8bc303493dbec3eb13a2e4dea7", "afd3ca7d3c5441ce93865e4819519940", "6d8ee31edea849f19feb352de0293f19", "8e2723a00a4344899763fc8cdb970182", "f4f35fb2dba845d185a417f0e2094b15", "840edd2e3f1e421db7e6fd93597027ea", "cd1c1bf54c8a4474b7751feffe822e86", "f4a32383d73f4bff823a6075ad56b268", "b65ecdb616954c3f866e6ddbb3b9ee9e", "cf58b2b0c09b443e9882cd4f71a85bc0", "34826a684a5641e599a12c826b996c62", "acda006173dc420887738f8921893d2e", "aa6c6b261c3e4b888a2e9fdbd2a7cd22", "e5beea48108c46ed88eb557c99ece4e2", "67c5fa90d29c45148ad385324be53ae1", "0c1bd3fc37b743d1b3a3f13b3b0a7b69", "039094aa47574c60a16edfbb7d326c42", "5260387ac9914fbea8f0d88a9ec0f058", "50155b0e5e804e5097730bc9f450b9b4", "b1654203849247f39884696ff1eda2f4", "02b51525d9834a43bd9b555c077609f2", "2299d5b1a59348618e275df847803cfa", "b0edd8c2bb314d9ebb83a8b9fce170e7", "42a439592989402abb6286db9f9b9fa3", "f768b9a5a1c84921ade0c1237af3fe80", "349a3525b93b452e89a9e334de00a3cd", "29085777a67d4bb39f319e204db5c05f", "473d92a0a9f24a30b3d10c84a83a5103", "303f3f26dc1e4a78bf8cd3cb2e6f2d4a", "f330f3d679b642f08014c1faa8b623cf", "aa813a951ceb44209385914a1a7787f5", "30aefa828c3e411f9490ae7e4a49aa3e", "4e2392c88ae04ae18046b978d5c54058", "ae907e8f91a74d2199d245c08b8ba1ca", "a45455994a1a42afa96f7d245b4536cc", "903c4a8813db417b9db5762fa1cb6c4b", "d6ecc30be6c44926b498f9988036a548", "6257eb5097d149ddbad82bad21a6d278", "7fdbf148a31d41eb80e3a40962e08e44", "5988ce482ad84ab5a9894ac83f87c6d2", "e5385adcb0bd45309154fbd018fb4339", "88ee9da252974899958be98a66726481", "72a74a7b54b64870a1467a97a3e3202a", "0424a95ba585487baf306be4c128a093", "3e8571b9bdb64bf09530553519ec4c9c", "d20c707cfbe44454b00dad139afc2b09", "98b2f667c38e491db2510a9fe242ca02", "407447ce58c04f8c9a8c65b2f39c044d", "fc9e186ebc3f4fc8bdd79d13a58caa1b", "8818f7d745b741878c194308a073f3e2" ] }, "outputId": "07f805a9-962e-4023-f61c-d5f1b596db57" }, "execution_count": 28, "outputs": [ { "output_type": "display_data", "data": { "text/plain": [ "tokenizer_config.json: 0%| | 0.00/48.0 [00:00 x'),\n", " ReLU(inplace=True),\n", " (GCNConv(64, 64), 'x, edge_index -> x'),\n", " ReLU(inplace=True),\n", " Linear(64, out_channels),\n", " ])\n", "\n", " word_vectors = model(train_graph_X, train_edges)\n", "\n", "\n", "\n", " vectros = extract_sentence_vectors_v2(sentences_tokens, word_vectors, (max_senetnce_length, d_model))\n", "\n", " return vectros\n" ], "metadata": { "id": "1sZSqG8nkAAp" }, "execution_count": 40, "outputs": [] }, { "cell_type": "code", "source": [], "metadata": { "id": "QoaVjNn_08iI" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "# model parametes\n", "src_vocab_size = len(en_vocab)\n", "tgt_vocab_size = len(de_vocab)\n", "d_model = 512\n", "num_heads = 8\n", "num_layers = 6\n", "d_ff = 2048\n", "max_seq_length = max([ max([len(src), len(tgt)]) for src,tgt in all_data]) + 1\n", "dropout = 0.1" ], "metadata": { "id": "eCBFZ7OFjz_g" }, "execution_count": 41, "outputs": [] }, { "cell_type": "code", "source": [ "# train , test, val loader\n", "\n", "batch_size = 32\n", "train_loader = data.DataLoader(\n", " dataset=train_data,\n", " # batch_size=batch_size, shuffle=True, drop_last=True,\n", " batch_size=batch_size,\n", " collate_fn=lambda batch: (\n", " pad_sequence(\n", " [torch.LongTensor(en_vocab(tokenizer(x))) for x, _ in batch],\n", " batch_first=True, padding_value=pad_idx),\n", " pad_sequence(\n", " [torch.LongTensor(de_vocab([BOS] + tokenizer(y) + [EOS])) for _, y in batch],\n", " batch_first=True, padding_value=pad_idx\n", " ),\n", "\n", " pad_sequence(\n", " get_graph_vectors(batch, d_model, 'EGCR'),\n", " batch_first=True, padding_value=pad_idx\n", " ),\n", "\n", " pad_sequence(\n", " get_graph_vectors(batch, d_model, 'POS'),\n", " batch_first=True, padding_value=pad_idx\n", " ),\n", "\n", " pad_sequence(\n", " get_graph_vectors(batch, d_model, 'DP'),\n", " batch_first=True, padding_value=pad_idx\n", " ),\n", "\n", " pad_sequence(\n", " get_graph_vectors(batch, d_model, 'semantic'),\n", " batch_first=True, padding_value=pad_idx\n", " ),\n", "\n", " ),\n", "\n", " num_workers=0,\n", ")\n", "\n", "\n", "val_loader = data.DataLoader(\n", " dataset=val_data,\n", " # batch_size=batch_size, shuffle=True, drop_last=True,\n", " batch_size=batch_size,\n", " collate_fn=lambda batch: (\n", " pad_sequence(\n", " [torch.LongTensor(en_vocab(tokenizer(x))) for x, _ in batch],\n", " batch_first=True, padding_value=pad_idx),\n", " pad_sequence(\n", " [torch.LongTensor(de_vocab([BOS] + tokenizer(y) + [EOS])) for _, y in batch],\n", " batch_first=True, padding_value=pad_idx\n", " ),\n", "\n", " pad_sequence(\n", " get_graph_vectors(batch, d_model, 'EGCR'),\n", " batch_first=True, padding_value=pad_idx\n", " ),\n", "\n", " pad_sequence(\n", " get_graph_vectors(batch, d_model, 'POS'),\n", " batch_first=True, padding_value=pad_idx\n", " ),\n", "\n", " pad_sequence(\n", " get_graph_vectors(batch, d_model, 'DP'),\n", " batch_first=True, padding_value=pad_idx\n", " ),\n", " pad_sequence(\n", " get_graph_vectors(batch, d_model, 'semantic'),\n", " batch_first=True, padding_value=pad_idx\n", " ),\n", "\n", " ),\n", " num_workers=0,\n", ")\n", "\n", "\n", "\n", "test_loader = data.DataLoader(\n", " dataset=test_data,\n", " # batch_size=batch_size, shuffle=True, drop_last=True,\n", " batch_size=batch_size,\n", " collate_fn=lambda batch: (\n", " pad_sequence(\n", " [torch.LongTensor(en_vocab(tokenizer(x))) for x, _ in batch],\n", " batch_first=True, padding_value=pad_idx),\n", " pad_sequence(\n", " [torch.LongTensor(de_vocab([BOS] + tokenizer(y) + [EOS])) for _, y in batch],\n", " batch_first=True, padding_value=pad_idx\n", " ),\n", "\n", " pad_sequence(\n", " get_graph_vectors(batch, d_model, 'EGCR'),\n", " batch_first=True, padding_value=pad_idx\n", " ),\n", "\n", " pad_sequence(\n", " get_graph_vectors(batch, d_model, 'POS'),\n", " batch_first=True, padding_value=pad_idx\n", " ),\n", "\n", " pad_sequence(\n", " get_graph_vectors(batch, d_model, 'DP'),\n", " batch_first=True, padding_value=pad_idx\n", " ),\n", "\n", " pad_sequence(\n", " get_graph_vectors(batch, d_model, 'semantic'),\n", " batch_first=True, padding_value=pad_idx\n", " ),\n", "\n", "\n", " ),\n", " num_workers=0,\n", ")" ], "metadata": { "id": "4JE2XwBBEknZ" }, "execution_count": 42, "outputs": [] }, { "cell_type": "code", "source": [ "import re\n", "\n", "def remove_dot_from_text(text):\n", " \"\"\"\n", " most of token in pred is ., so we will remove them\n", " \"\"\"\n", " text = text.replace('.', '')\n", " text = re.sub(' +', ' ', text)\n", " return text" ], "metadata": { "id": "HFye9VhCr4Yw" }, "execution_count": 43, "outputs": [] }, { "cell_type": "code", "source": [ "# evaluation modules\n", "import numpy as np\n", "from tqdm import tqdm\n", "import pandas as pd\n", "import plotly.express as px\n", "\n", "from nltk.translate.bleu_score import sentence_bleu\n", "\n", "\n", "!pip install -q evaluate\n", "\n", "import evaluate\n", "import json\n", "\n", "!pip install -q jiwer\n", "\n", "from jiwer import wer\n", "\n", "# ! pip install --force-reinstall kaleido==0.1.0post1" ], "metadata": { "id": "zVxLoI0Qz04a", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "b61ae4df-23d4-407a-9e50-9ff6d29108a1" }, "execution_count": 44, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.1/84.1 kB\u001b[0m \u001b[31m2.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.4/3.4 MB\u001b[0m \u001b[31m12.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h" ] } ] }, { "cell_type": "code", "source": [ "epochs = 1\n" ], "metadata": { "id": "x4B5RJ7vE-_f" }, "execution_count": 47, "outputs": [] }, { "cell_type": "code", "source": [ "for g_tyep in [\"basic\", \"egcr\", \"pos\", \"dp\", \"semantic\", \"all\"]:\n", " try:\n", " del transformer\n", " except:\n", " pass\n", " transformer = Transformer(src_vocab_size, tgt_vocab_size, d_model, num_heads, num_layers, d_ff, max_seq_length, dropout, g_tyep)\n", " transformer = transformer.to(device)\n", "\n", "\n", " # train val\n", " criterion = nn.CrossEntropyLoss(ignore_index=0)\n", " optimizer = optim.Adam(transformer.parameters(), lr=0.0001, betas=(0.9, 0.98), eps=1e-9)\n", "\n", " # pbar = tqdm(total=epochs*64, desc=\"Iteration\")\n", " trainingEpoch_loss = []\n", " validationEpoch_loss = []\n", " for epoch in range(epochs):\n", " step_loss = []\n", " transformer.train()\n", " for src, tgt, grf1, grf2, grf3, grf4 in train_loader:\n", "\n", " src = src.to(device)\n", " tgt = tgt.to(device)\n", " grf1 = grf1.to(device)\n", " grf2 = grf2.to(device)\n", " grf3 = grf3.to(device)\n", " grf4 = grf4.to(device)\n", "\n", " # print(src.shape, grf.shape)\n", " # print(src.shape)\n", " # print(grf.shape)\n", " optimizer.zero_grad()\n", " output = transformer(src, tgt[:, :-1], grf1, grf2, grf3, grf4)\n", " loss = criterion(output.contiguous().view(-1, tgt_vocab_size), tgt[:, 1:].contiguous().view(-1))\n", " loss.backward()\n", " optimizer.step()\n", " step_loss.append(loss.item())\n", "\n", " trainingEpoch_loss.append(np.array(step_loss).mean())\n", "\n", " # model.eval() # Optional when not using Model Specific layer\n", " transformer.eval()\n", " validationStep_loss = []\n", " for src, tgt, grf1, grf2, grf3, grf4 in val_loader:\n", " src = src.to(device)\n", " tgt = tgt.to(device)\n", " grf1 = grf1.to(device)\n", " grf2 = grf2.to(device)\n", " grf3 = grf3.to(device)\n", " grf4 = grf4.to(device)\n", "\n", " # Forward Pass\n", " output = transformer(src, tgt[:, :-1], grf1, grf2, grf3, grf4)\n", " # Find the Loss\n", " validation_loss = criterion(output.contiguous().view(-1, tgt_vocab_size), tgt[:, 1:].contiguous().view(-1))\n", " # Calculate Loss\n", " validationStep_loss.append(validation_loss.item())\n", "\n", " validationEpoch_loss.append(np.array(validationStep_loss).mean())\n", "\n", "\n", " print(f\"Epoch: {epoch+1}, Loss: {loss.item()}\")\n", "\n", " # call back\n", " if len(validationEpoch_loss) > 3 and validationEpoch_loss[-1] >= validationEpoch_loss[-2] and validationEpoch_loss[-2] >= validationEpoch_loss[-3]:\n", " break\n", "\n", "\n", " # save model plots in a csv file\n", " pd_data = {'Train Loss': trainingEpoch_loss,\n", " 'Validation Loss': validationEpoch_loss,\n", " 'Epoch': list(range(1, len(trainingEpoch_loss)+1))}\n", "\n", " # Create DataFrame\n", " plot_df = pd.DataFrame(pd_data)\n", "\n", " plot_df.head()\n", " plot_df.to_csv(f'{g_tyep}.csv')\n", "\n", "\n", " # Creating the Figure instance\n", " fig = px.line(plot_df, x='Epoch', y=['Train Loss', 'Validation Loss'])\n", "\n", "\n", " # adding different style parameters to the legend\n", " fig.update_layout(\n", " legend=dict(\n", " x=0.7,\n", " y=0.9,\n", " title=\"Loss\",\n", " title_font_family=\"Times New Roman\",\n", " font=dict(\n", " family=\"Courier\",\n", " size=14,\n", " color=\"black\"\n", " ),\n", " # borderwidth=1\n", " ),\n", " autosize=False,\n", " width=1200,\n", " height=600,\n", " )\n", "\n", " fig.show()\n", " # fig.write_image(f\"{g_tyep}.png\", format='png',engine='kaleido')\n", "\n", " # evaluate\n", "\n", " transformer.eval()\n", "\n", " # # Generate random sample validation data\n", " # val_src_data = torch.randint(1, src_vocab_size, (64, max_seq_length)) # (batch_size, seq_length)\n", " # val_tgt_data = torch.randint(1, tgt_vocab_size, (64, max_seq_length)) # (batch_size, seq_length)\n", "\n", " pred_sentences = []\n", " real_sentences = []\n", "\n", "\n", " with torch.no_grad():\n", " for src, tgt, grf1, grf2, grf3, grf4 in test_loader:\n", " src = src.to(device)\n", " tgt = tgt.to(device)\n", " grf1 = grf1.to(device)\n", " grf2 = grf2.to(device)\n", " grf3 = grf3.to(device)\n", " grf4 = grf4.to(device)\n", "\n", " val_output = transformer(src, tgt[:, :-1], grf1, grf2, grf3, grf4)\n", "\n", " val_loss = criterion(val_output.contiguous().view(-1, tgt_vocab_size), tgt[:, 1:].contiguous().view(-1))\n", "\n", " pred_index_out = [ [ torch.argmax(token_preds).item() for token_preds in item] for item in val_output]\n", " pred_token_out = [ de_vocab.lookup_tokens(indices) for indices in pred_index_out]\n", " pred_sentences += [ [token for token in tokens if token not in ['', '', ''] ] for tokens in pred_token_out ]\n", "\n", " real_token_out = [ de_vocab.lookup_tokens(indices.cpu().numpy()) for indices in tgt]\n", " real_sentences += [ [token for token in tokens if token not in ['', '', ''] ] for tokens in real_token_out ]\n", "\n", "\n", " real_sentences = [' '.join(item) for item in real_sentences]\n", " pred_sentences = [' '.join(item) for item in pred_sentences]\n", "\n", "\n", " real_sentences = [remove_dot_from_text(item) for item in real_sentences]\n", " pred_sentences = [remove_dot_from_text(item) for item in pred_sentences]\n", " real_sentences1 = [ [item] for item in real_sentences]\n", "\n", " # evluate\n", "\n", "\n", " bleu = evaluate.load(\"bleu\")\n", " results = bleu.compute(predictions=pred_sentences, references=real_sentences1)\n", " json.dump( results, open( f\"{g_tyep}_4.json\", 'w' ) )\n", "\n", " bleu = evaluate.load(\"bleu\")\n", " results = bleu.compute(predictions=pred_sentences, references=real_sentences1, max_order=1)\n", " json.dump( results, open( f\"{g_tyep}_1.json\", 'w' ) )\n", "\n", " wer_scores = [] # [bleu_score, pred, ref]\n", " for r_sample, p_sample in zip(real_sentences1, pred_sentences):\n", " error = wer(r_sample, p_sample)\n", " wer_scores.append([error, p_sample, r_sample[0]])\n", "\n", " print(\"wer error is: \", sum([item[0] for item in wer_scores])/len(wer_scores))\n", "\n", " temp_wer_dict = {\"wer\": sum([item[0] for item in wer_scores])/len(wer_scores)}\n", " json.dump( temp_wer_dict, open( f\"{g_tyep}_wer.json\", 'w' ) )\n", "\n", " wer_scores = sorted(wer_scores, reverse=True)\n", "\n", " with open(f\"{g_tyep}_samples.txt\", \"w\") as f:\n", " for item in wer_scores:\n", " item = [str(i) for i in item]\n", " f.write(' ;; '.join(item))\n", " f.write(\"\\n\")\n", "\n" ], "metadata": { "id": "VqDSWCOd252m", "colab": { "base_uri": "https://localhost:8080/", "height": 1000, "referenced_widgets": [ "b00a5318bf614a3fb9d5c05704eea9fe", "d61397a2057541e19b38fc53bb1e6286", "7247f5f59eeb4391a1a618d29924529c", "84c7855f0298466295a0f07927156c58", "db35422f35d74e0da37c39f24ad3a7c3", "7988e33c9e394e8a831d8ef7cb434973", "ddf856a8bed641f8a15bf6d3a2a7aeb4", "634c152c9bef4b5ea7672ea87abe52cb", "c8ff7e1de7204ba2b2f60502aa8de390", "255b656e99ca4aec9dbb78065c948036", "ae3d320ebb4d4a07a0187e6a474d736c", "7f2b94cb9c1a4c74bdd4a6a01517a5e9", "5b76a3cbec9b40cd88f72b871181d194", "cf612ec51bc24437abea77ba697a6b8b", "37c3d5260b464a29ae015de768475579", "1badbd572b8a4b2494ba4ec24cfa4201", "b6c28827a0cf4f098f9210822f3b6990", "0d6eb59f405046e4baca4fd859f5632d", "2c9b89a323b8494691f9edb1278beaea", "0c62e2a754634e608ca4cb64c99542be", "1ba081a449cd4663ba2e6db5d2a6151b", "8d5bcdddfc3e4f1cb5e5e12b5ea9b3cc", "d9e4e9d0a4c34d32b152cddc533061b2", "b906934edda5424580fb5a676a42b482", "67013f4714f9491c9a741399dfc38a85", "c9a6c6b57a5c49ceabaf65de006a5a25", "76ef22dfea0d48248e68c4e697f70c56", "d9c26846632241499ae87589e011671c", "ccc6082d5b284b38a72e88cd25fdc80c", "0a91f9152d004fc297ace0cd97cefe62", "592a6bb343504ebc99d4a66e39629685", "b22679e161ce444abad828adc5f30c9d", "ad88a23472eb461e96a3d8d934566de3" ] }, "outputId": "8862d14a-21bb-4738-be30-692142d45c79" }, "execution_count": 48, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Epoch: 1, Loss: 5.21380090713501\n" ] }, { "output_type": "display_data", "data": { "text/html": [ "\n", "\n", "\n", "
\n", "
\n", "\n", "" ] }, "metadata": {} }, { "output_type": "display_data", "data": { "text/plain": [ "Downloading builder script: 0%| | 0.00/5.94k [00:00\n", "\n", "\n", "
\n", "
\n", "\n", "" ] }, "metadata": {} }, { "output_type": "stream", "name": "stdout", "text": [ "wer error is: 0.8561898101717572\n", "Epoch: 1, Loss: 5.208500385284424\n" ] }, { "output_type": "display_data", "data": { "text/html": [ "\n", "\n", "\n", "
\n", "
\n", "\n", "" ] }, "metadata": {} }, { "output_type": "stream", "name": "stdout", "text": [ "wer error is: 0.8837455819341059\n", "Epoch: 1, Loss: 5.316232681274414\n" ] }, { "output_type": "display_data", "data": { "text/html": [ "\n", "\n", "\n", "
\n", "
\n", "\n", "" ] }, "metadata": {} }, { "output_type": "stream", "name": "stdout", "text": [ "wer error is: 0.8948310318728848\n", "Epoch: 1, Loss: 5.2148966789245605\n" ] }, { "output_type": "display_data", "data": { "text/html": [ "\n", "\n", "\n", "
\n", "
\n", "\n", "" ] }, "metadata": {} }, { "output_type": "stream", "name": "stdout", "text": [ "wer error is: 0.8507563824692773\n", "Epoch: 1, Loss: 5.2134809494018555\n" ] }, { "output_type": "display_data", "data": { "text/html": [ "\n", "\n", "\n", "
\n", "
\n", "\n", "" ] }, "metadata": {} }, { "output_type": "stream", "name": "stdout", "text": [ "wer error is: 0.8490530758133504\n" ] } ] }, { "cell_type": "code", "source": [ "pip install -q transformers\n" ], "metadata": { "id": "L6ogPYEXZUi9" }, "execution_count": 52, "outputs": [] }, { "cell_type": "code", "source": [ "pip install -q huggingface_hub" ], "metadata": { "id": "ATOMbULh9Fin" }, "execution_count": 53, "outputs": [] }, { "cell_type": "code", "source": [ "! huggingface-cli login" ], "metadata": { "id": "zqnTnR9i9Fkm", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "f7ef7ac6-773a-4c95-8f13-aefe16966c41" }, "execution_count": 55, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "\n", " _| _| _| _| _|_|_| _|_|_| _|_|_| _| _| _|_|_| _|_|_|_| _|_| _|_|_| _|_|_|_|\n", " _| _| _| _| _| _| _| _|_| _| _| _| _| _| _| _|\n", " _|_|_|_| _| _| _| _|_| _| _|_| _| _| _| _| _| _|_| _|_|_| _|_|_|_| _| _|_|_|\n", " _| _| _| _| _| _| _| _| _| _| _|_| _| _| _| _| _| _| _|\n", " _| _| _|_| _|_|_| _|_|_| _|_|_| _| _| _|_|_| _| _| _| _|_|_| _|_|_|_|\n", "\n", " To login, `huggingface_hub` requires a token generated from https://huggingface.co/settings/tokens .\n", "Token: \n", "Add token as git credential? (Y/n) y\n", "Token is valid (permission: write).\n", "\u001b[1m\u001b[31mCannot authenticate through git-credential as no helper is defined on your machine.\n", "You might have to re-authenticate when pushing to the Hugging Face Hub.\n", "Run the following command in your terminal in case you want to set the 'store' credential helper as default.\n", "\n", "git config --global credential.helper store\n", "\n", "Read https://git-scm.com/book/en/v2/Git-Tools-Credential-Storage for more details.\u001b[0m\n", "Token has not been saved to git credential helper.\n", "Your token has been saved to /root/.cache/huggingface/token\n", "Login successful\n" ] } ] }, { "cell_type": "code", "source": [ "! transformers-cli -h\n" ], "metadata": { "id": "B9-GbCk9_9dM", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "17f79b1a-7f04-4a75-ab63-be65dc83f971" }, "execution_count": 56, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "2024-04-16 12:41:22.002906: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", "2024-04-16 12:41:22.002979: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", "2024-04-16 12:41:22.004290: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", "2024-04-16 12:41:23.117576: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n", "usage: transformers-cli []\n", "\n", "positional arguments:\n", " {convert,download,env,run,serve,login,whoami,logout,repo,add-new-model,add-new-model-like,lfs-enable-largefiles,lfs-multipart-upload,pt-to-tf}\n", " transformers-cli command helpers\n", " convert CLI tool to run convert model from original author checkpoints to\n", " Transformers PyTorch checkpoints.\n", " run Run a pipeline through the CLI\n", " serve CLI tool to run inference requests through REST and GraphQL endpoints.\n", " login Log in using the same credentials as on huggingface.co\n", " whoami Find out which huggingface.co account you are logged in as.\n", " logout Log out\n", " repo Deprecated: use `huggingface-cli` instead. Commands to interact with your\n", " huggingface.co repos.\n", " lfs-enable-largefiles\n", " Deprecated: use `huggingface-cli` instead. Configure your repository to\n", " enable upload of files > 5GB.\n", " lfs-multipart-upload\n", " Deprecated: use `huggingface-cli` instead. Command will get called by git-\n", " lfs, do not call it directly.\n", " pt-to-tf CLI tool to run convert a transformers model from a PyTorch checkpoint to\n", " a TensorFlow checkpoint. Can also be used to validate existing weights\n", " without opening PRs, with --no-pr.\n", "\n", "options:\n", " -h, --help show this help message and exit\n" ] } ] }, { "cell_type": "code", "source": [ "! huggingface-cli repo create dummy-model" ], "metadata": { "id": "gUp_4VFugaUi", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "3bfa2c16-fac2-4a46-bf6c-b1f631392ff0" }, "execution_count": 58, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "\u001b[90mgit version 2.34.1\u001b[0m\n", "\u001b[90mgit-lfs/3.0.2 (GitHub; linux amd64; go 1.18.1)\u001b[0m\n", "\n", "You are about to create \u001b[1mmohammad2928git/dummy-model\u001b[0m\n", "Proceed? [Y/n] y\n", "\n", "Your repo now lives at:\n", " \u001b[1mhttps://huggingface.co/mohammad2928git/dummy-model\u001b[0m\n", "\n", "You can clone it locally with the command below, and commit/push as usual.\n", "\n", " git clone https://huggingface.co/mohammad2928git/dummy-model\n", "\n" ] } ] }, { "cell_type": "code", "source": [ "\n" ], "metadata": { "id": "PWlFyKvQg5IZ" }, "execution_count": null, "outputs": [] } ] }