{ "cells": [ { "cell_type": "markdown", "id": "791b1383-2a89-4aa0-9f40-89c82cfb0d15", "metadata": {}, "source": [ "# Mosaic Model\n", "\n", "## This notebook clones the mosaicml/mosaic-bert-base-seqlen-2048 repository locally and modifies the config.json file so as to conform to our specifications" ] }, { "cell_type": "code", "execution_count": 4, "id": "3b173596-c9d3-4326-a0e9-e00e676b1c9e", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "/home/ec2-user/SageMaker\n" ] } ], "source": [ "import os\n", "print(os.getcwd())\n", "os.environ[\"PATH\"]=\"/usr/local/cuda-11.7/bin:\"+os.getenv(\"PATH\")" ] }, { "cell_type": "code", "execution_count": 5, "id": "508ac40d-b16c-45be-bc6e-d67c966ad0ee", "metadata": { "collapsed": true, "jupyter": { "outputs_hidden": true }, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Loaded plugins: dkms-build-requires, extras_suggestions, langpacks, priorities,\n", " : update-motd, versionlock\n", "Cleaning repos: amzn2-core amzn2extra-docker amzn2extra-epel\n", " : amzn2extra-kernel-5.10 amzn2extra-python3.8 centos-extras\n", " : copr:copr.fedorainfracloud.org:vbatts:shadow-utils-newxidmap\n", " : docker-ce-stable libnvidia-container neuron\n", "21 metadata files removed\n", "15 sqlite files removed\n", "0 metadata files removed\n", "Loaded plugins: dkms-build-requires, extras_suggestions, langpacks, priorities,\n", " : update-motd, versionlock\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "https://download.docker.com/linux/centos/2/x86_64/stable/repodata/repomd.xml: [Errno 14] HTTPS Error 404 - Not Found\n", "Trying other mirror.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "62 packages excluded due to repository priority protections\n", "Resolving Dependencies\n", "--> Running transaction check\n", "---> Package epel-release.noarch 0:7-11 will be installed\n", "--> Finished Dependency Resolution\n", "\n", "Dependencies Resolved\n", "\n", "================================================================================\n", " Package Arch Version Repository Size\n", "================================================================================\n", "Installing:\n", " epel-release noarch 7-11 amzn2extra-epel 15 k\n", "\n", "Transaction Summary\n", "================================================================================\n", "Install 1 Package\n", "\n", "Total download size: 15 k\n", "Installed size: 24 k\n", "Downloading packages:\n", "Running transaction check\n", "Running transaction test\n", "Transaction test succeeded\n", "Running transaction\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "Warning: RPMDB altered outside of yum.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ " Installing : epel-release-7-11.noarch 1/1 \n", " Verifying : epel-release-7-11.noarch 1/1 \n", "\n", "Installed:\n", " epel-release.noarch 0:7-11 \n", "\n", "Complete!\n", "Installing epel-release\n", " 0 ansible2 available \\\n", " [ =2.4.2 =2.4.6 =2.8 =stable ]\n", " 2 httpd_modules available [ =1.0 =stable ]\n", " 3 memcached1.5 available \\\n", " [ =1.5.1 =1.5.16 =1.5.17 ]\n", " 6 postgresql10 available [ =10 =stable ]\n", " 9 R3.4 available [ =3.4.3 =stable ]\n", " 10 rust1 available \\\n", " [ =1.22.1 =1.26.0 =1.26.1 =1.27.2 =1.31.0 =1.38.0\n", " =stable ]\n", " 18 libreoffice available \\\n", " [ =5.0.6.2_15 =5.3.6.1 =stable ]\n", " 19 gimp available [ =2.8.22 ]\n", " 20 docker=latest enabled \\\n", " [ =17.12.1 =18.03.1 =18.06.1 =18.09.9 =stable ]\n", " 21 mate-desktop1.x available \\\n", " [ =1.19.0 =1.20.0 =stable ]\n", " 22 GraphicsMagick1.3 available \\\n", " [ =1.3.29 =1.3.32 =1.3.34 =stable ]\n", " 23 tomcat8.5 available \\\n", " [ =8.5.31 =8.5.32 =8.5.38 =8.5.40 =8.5.42 =8.5.50\n", " =stable ]\n", " 24 epel=latest enabled [ =7.11 =stable ]\n", " 25 testing available [ =1.0 =stable ]\n", " 26 ecs available [ =stable ]\n", " 27 corretto8 available \\\n", " [ =1.8.0_192 =1.8.0_202 =1.8.0_212 =1.8.0_222 =1.8.0_232\n", " =1.8.0_242 =stable ]\n", " 29 golang1.11 available \\\n", " [ =1.11.3 =1.11.11 =1.11.13 =stable ]\n", " 30 squid4 available [ =4 =stable ]\n", " 32 lustre2.10 available \\\n", " [ =2.10.5 =2.10.8 =stable ]\n", " 33 java-openjdk11 available [ =11 =stable ]\n", " 34 lynis available [ =stable ]\n", " 36 BCC available [ =0.x =stable ]\n", " 37 mono available [ =5.x =stable ]\n", " 38 nginx1 available [ =stable ]\n", " 40 mock available [ =stable ]\n", " 41 postgresql11 available [ =11 =stable ]\n", " 43 livepatch available [ =stable ]\n", " 44 python3.8=latest enabled [ =stable ]\n", " 45 haproxy2 available [ =stable ]\n", " 46 collectd available [ =stable ]\n", " 47 aws-nitro-enclaves-cli available [ =stable ]\n", " 48 R4 available [ =stable ]\n", " _ kernel-5.4 available [ =stable ]\n", " 50 selinux-ng available [ =stable ]\n", " 51 php8.0 available [ =stable ]\n", " 52 tomcat9 available [ =stable ]\n", " 53 unbound1.13 available [ =stable ]\n", " 54 mariadb10.5 available [ =stable ]\n", " 55 kernel-5.10=latest enabled [ =stable ]\n", " 56 redis6 available [ =stable ]\n", " 57 ruby3.0 available [ =stable ]\n", " 58 postgresql12 available [ =stable ]\n", " 59 postgresql13 available [ =stable ]\n", " 60 mock2 available [ =stable ]\n", " 61 dnsmasq2.85 available [ =stable ]\n", " 62 kernel-5.15 available [ =stable ]\n", " 63 postgresql14 available [ =stable ]\n", " 64 firefox available [ =stable ]\n", " 65 lustre available [ =stable ]\n", " 66 php8.1 available [ =stable ]\n", " 67 awscli1 available [ =stable ]\n", " 68 php8.2 available [ =stable ]\n", " 69 dnsmasq available [ =stable ]\n", " 70 unbound1.17 available [ =stable ]\n", " 71 golang1.19 available [ =stable ]\n", " 72 collectd-python3 available [ =stable ]\n", "Loaded plugins: dkms-build-requires, extras_suggestions, langpacks, priorities,\n", " : update-motd, versionlock\n", "================================== repo: epel ==================================\n", "[epel]\n", "async = True\n", "bandwidth = 0\n", "base_persistdir = /var/lib/yum/repos/x86_64/2\n", "baseurl = \n", "cache = 0\n", "cachedir = /var/cache/yum/x86_64/2/epel\n", "check_config_file_age = True\n", "compare_providers_priority = 80\n", "cost = 1000\n", "deltarpm_metadata_percentage = 100\n", "deltarpm_percentage = \n", "enabled = True\n", "enablegroups = True\n", "exclude = \n", "failovermethod = priority\n", "ftp_disable_epsv = False\n", "gpgcadir = /var/lib/yum/repos/x86_64/2/epel/gpgcadir\n", "gpgcakey = \n", "gpgcheck = True\n", "gpgdir = /var/lib/yum/repos/x86_64/2/epel/gpgdir\n", "gpgkey = file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-7\n", "hdrdir = /var/cache/yum/x86_64/2/epel/headers\n", "http_caching = all\n", "includepkgs = \n", "ip_resolve = \n", "keepalive = True\n", "keepcache = False\n", "mddownloadpolicy = sqlite\n", "mdpolicy = group:small\n", "mediaid = \n", "metadata_expire = 21600\n", "metadata_expire_filter = read-only:present\n", "metalink = https://mirrors.fedoraproject.org/metalink?repo=epel-7&arch=x86_64\n", "minrate = 0\n", "mirrorlist = \n", "mirrorlist_expire = 86400\n", "name = Extra Packages for Enterprise Linux 7 - x86_64\n", "old_base_cache_dir = \n", "password = \n", "persistdir = /var/lib/yum/repos/x86_64/2/epel\n", "pkgdir = /var/cache/yum/x86_64/2/epel/packages\n", "priority = 99\n", "proxy = False\n", "proxy_dict = \n", "proxy_password = \n", "proxy_username = \n", "repo_gpgcheck = False\n", "report_instanceid = False\n", "retries = 7\n", "skip_if_unavailable = False\n", "ssl_check_cert_permissions = True\n", "sslcacert = \n", "sslclientcert = \n", "sslclientkey = \n", "sslverify = True\n", "throttle = 0\n", "timeout = 5.0\n", "ui_id = epel/x86_64\n", "ui_repoid_vars = releasever,\n", " basearch\n", "username = \n", "\n", "Loaded plugins: dkms-build-requires, extras_suggestions, langpacks, priorities,\n", " : update-motd, versionlock\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "https://download.docker.com/linux/centos/2/x86_64/stable/repodata/repomd.xml: [Errno 14] HTTPS Error 404 - Not Found\n", "Trying other mirror.\n", "http://mirror.es.its.nyu.edu/epel/7/x86_64/repodata/repomd.xml: [Errno 12] Timeout on http://mirror.es.its.nyu.edu/epel/7/x86_64/repodata/repomd.xml: (28, 'Failed to connect to mirror.es.its.nyu.edu port 80 after 5001 ms: Timeout was reached')\n", "Trying other mirror.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "286 packages excluded due to repository priority protections\n", "Resolving Dependencies\n", "--> Running transaction check\n", "---> Package git-lfs.x86_64 0:2.10.0-2.el7 will be installed\n", "--> Finished Dependency Resolution\n", "\n", "Dependencies Resolved\n", "\n", "================================================================================\n", " Package Arch Version Repository Size\n", "================================================================================\n", "Installing:\n", " git-lfs x86_64 2.10.0-2.el7 epel 3.7 M\n", "\n", "Transaction Summary\n", "================================================================================\n", "Install 1 Package\n", "\n", "Total download size: 3.7 M\n", "Installed size: 13 M\n", "Downloading packages:\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "warning: /var/cache/yum/x86_64/2/epel/packages/git-lfs-2.10.0-2.el7.x86_64.rpm: Header V4 RSA/SHA256 Signature, key ID 352c64e5: NOKEY\n", "Importing GPG key 0x352C64E5:\n", " Userid : \"Fedora EPEL (7) \"\n", " Fingerprint: 91e9 7d7c 4a5e 96f1 7f3e 888f 6a2f aea2 352c 64e5\n", " Package : epel-release-7-11.noarch (@amzn2extra-epel)\n", " From : /etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-7\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Public key for git-lfs-2.10.0-2.el7.x86_64.rpm is not installed\n", "Retrieving key from file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-7\n", "Running transaction check\n", "Running transaction test\n", "Transaction test succeeded\n", "Running transaction\n", " Installing : git-lfs-2.10.0-2.el7.x86_64 1/1 \n", " Verifying : git-lfs-2.10.0-2.el7.x86_64 1/1 \n", "\n", "Installed:\n", " git-lfs.x86_64 0:2.10.0-2.el7 \n", "\n", "Complete!\n", "Git LFS initialized.\n" ] }, { "data": { "text/plain": [ "0" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# install git-lfs\n", "os.system('sudo amazon-linux-extras install epel -y')\n", "os.system('sudo yum-config-manager --enable epel')\n", "os.system('sudo yum install git-lfs -y')\n", "os.system('git lfs install')" ] }, { "cell_type": "code", "execution_count": 6, "id": "7c916531-4e83-426d-9328-f48c67e6ccf0", "metadata": { "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Cloning into 'mosaic-bert-base-seqlen-2048'...\n" ] }, { "data": { "text/plain": [ "0" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# clone the mosaic model repository from HuggingFace\n", "os.system('git clone https://huggingface.co/mosaicml/mosaic-bert-base-seqlen-2048')" ] }, { "cell_type": "code", "execution_count": 7, "id": "c322ca06-ee24-4124-a39a-a01de29bb86a", "metadata": { "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "{'_name_or_path': 'mosaic-bert-base-seqlen-2048', 'alibi_starting_size': 2048, 'architectures': ['BertModel'], 'attention_probs_dropout_prob': 0.0, 'auto_map': {'AutoConfig': 'configuration_bert.BertConfig', 'AutoModelForMaskedLM': 'bert_layers.BertForMaskedLM', 'AutoModel': 'bert_layers.BertModel'}, 'classifier_dropout': None, 'gradient_checkpointing': False, 'hidden_act': 'gelu', 'hidden_dropout_prob': 0.1, 'hidden_size': 768, 'initializer_range': 0.02, 'intermediate_size': 3072, 'layer_norm_eps': 1e-12, 'max_position_embeddings': 512, 'model_type': 'bert', 'num_attention_heads': 12, 'num_hidden_layers': 12, 'pad_token_id': 0, 'position_embedding_type': 'absolute', 'torch_dtype': 'bfloat16', 'transformers_version': '4.28.1', 'type_vocab_size': 2, 'use_cache': True, 'vocab_size': 30528}\n" ] } ], "source": [ "import json\n", "import torch\n", "\n", "# modify the model's config.json file to satisfy our requirements\n", "contents = json.load(open('mosaic-bert-base-seqlen-2048/config.json'))\n", "contents['architectures'] = ['BertModel']\n", "contents['auto_map']['AutoModel'] = 'bert_layers.BertModel'\n", "contents['torch_dtype'] = 'bfloat16'\n", "contents['transformers_version'] = '4.28.1'\n", "contents['_name_or_path'] = 'mosaic-bert-base-seqlen-2048'\n", "print(contents)\n", "json.dump(contents, open('mosaic-bert-base-seqlen-2048/config.json', 'w'), ensure_ascii=True)" ] } ], "metadata": { "kernelspec": { "display_name": "conda_pytorch_p310", "language": "python", "name": "conda_pytorch_p310" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.10" } }, "nbformat": 4, "nbformat_minor": 5 }