{ "cells": [ { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "# middleschool-cardlist\n", "\n", "Note: this notebook is only for introductory purposes, and is not the way our list is compiled anymore. See the README for more information.\n", "\n", "## Prepare the data\n", "\n", "Download raw data from [MTGJSON](https://mtgjson.com/) (uncomment and run only once)\n" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "# !cd data\n", "# !wget \"https://mtgjson.com/api/v5/AllPrintings.json.bz2\"\n", "# !bunzip2 AllPrintings.json.bz2\n", "# !cd -" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "The Raw data is very large, so let's make JSON files for all relevant sets\n", "\n", "Note: this cell can take a couple minutes to run\n" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "setlist = ['4ED', 'ICE', 'CHR', 'HML', 'ALL', 'MIR', 'VIS', '5ED',\n", " 'WTH', 'POR', 'TMP', 'STH', 'EXO', 'P02', 'USG', 'ULG',\n", " '6ED', 'UDS', 'PTK', 'S99', 'MMQ', 'NEM', 'PCY', 'S00',\n", " 'INV', 'PLS', '7ED', 'APC', 'ODY', 'TOR', 'JUD', 'ONS',\n", " 'LGN', 'SCG', 'PDRC', 'PHPR', 'ATH', 'BRB', 'BTD', 'DKM']\n", "for set in setlist:\n", " # Write a separate JSON document for each Middle School legal set\n", " command = 'cat data/AllPrintings.json | jq \\'.data.\\\"' + \\\n", " set + '\\\".cards\\' > data/set_' + set + '.json'\n", " !{command}\n" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "Concatenate all set files into `middleschool.json`\n" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "command = \"jq -s add data/set_* > data/middleschool.json\"\n", "!{command}\n" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "Create a list with each card's oracle ID, English name, and Japanese name\n" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "5800 cards found\n", "These are the first and last 5 cards\n", " oracle_id name name_ja\n", "0 8adbba6e-03ef-4278-aec5-8a4496b377a8 Abandon Hope 断念\n", "0 5a70ccfa-d12d-4e62-a1a4-f05cda2fd442 Abandoned Outpost 見捨てられた前哨地\n", "0 c208b959-d0e4-4a9a-8255-2c7cc7596767 Abbey Gargoyles 修道院のガーゴイル\n", "0 62e3f285-886c-414e-b4ff-403a7c01c23a Abbey Matron None\n", "0 d0e1904e-1a37-41f6-8582-b9ea794bb886 Abduction 誘拐\n", " oracle_id name name_ja\n", "0 ae8773a3-05f2-4074-9a53-033b0c127235 Zuo Ci, the Mocking Sage 嘲笑する仙人 左慈\n", "0 c6eaa147-3566-43a9-999a-d58b877496f5 Zur's Weirding ズアーの運命支配\n", "0 ee0f883f-d7c9-4acf-a78f-f733b6f268d3 Zuran Enchanter None\n", "0 08cb8a30-9cb4-4517-bee5-8848aa60d1a2 Zuran Orb None\n", "0 bc7b90b1-3517-4e5d-9bd8-68b4d8a259fd Zuran Spellcaster None\n" ] } ], "source": [ "import json\n", "import pandas as pd\n", "\n", "with open(\"data/middleschool.json\") as json_data:\n", " cards = json.loads(json_data.read())\n", "\n", "# Create a pandas DataFrame with all cards from all legal sets\n", "column_names = [\"oracle_id\", \"name\", \"name_ja\"]\n", "middleschool_df = pd.DataFrame(columns=column_names)\n", "for card in cards:\n", " oracle_id = card[\"identifiers\"][\"scryfallOracleId\"]\n", " name = card[\"name\"]\n", " lang_ja = [lang for lang in card[\"foreignData\"] if lang[\"language\"] == \"Japanese\"]\n", " # Some cards do not have a Japanese name\n", " if len(lang_ja) > 0:\n", " name_ja = lang_ja[0][\"name\"]\n", " else:\n", " name_ja = None\n", " temporary_df = pd.DataFrame(\n", " {\"oracle_id\": [oracle_id], \"name\": [name], \"name_ja\": [name_ja]}\n", " )\n", " middleschool_df = pd.concat([middleschool_df, temporary_df])\n", "\n", "# For cards with multiple occurrences, put the rows that have the Japanese name on top\n", "middleschool_df = middleschool_df.sort_values(by=[\"name\", \"name_ja\"])\n", "# For cards with multiple occurrences, delete all rows except for the top one\n", "middleschool_df = middleschool_df.drop_duplicates(subset=[\"oracle_id\"])\n", "print(middleschool_df.shape[0], \"cards found\")\n", "print(\"These are the first and last 5 cards\")\n", "print(middleschool_df.head())\n", "print(middleschool_df.tail())" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "Remove Japanese card names that are wrong on MTGJSON\n" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Before:\n", " oracle_id name name_ja\n", "0 0fe602b7-9f88-4d3d-af24-7790df867ed5 Aether Barrier Æther Barrier\n", "0 1e33f39b-a61a-4a09-a541-16cc1bd53d02 Aether Burst Æther Burst\n", "0 15e83068-6253-4c65-8679-7295f3dc2075 Aether Charge Æther Charge\n", "0 a3c35742-e306-49b6-b042-db4f685c6f86 Aether Flash Æther Flash\n", "0 6697fe5b-90ac-4321-aa2f-cdc6ec283cb4 Aether Mutation Aether Mutation\n", "0 61105cb5-d7a1-4021-a006-dd1b947dfa68 Aether Sting Æther Sting\n", "0 ff4297d3-3d96-4bd6-a606-1bdc20a6df2b Aether Storm Æther Storm\n", "0 2fbf95b4-bcf4-4b5e-b5dc-0294f2b48d3e Aether Tide Æther Tide\n", "0 a61ceda1-5993-479e-945f-15753eeb7049 Tainted Aether Tainted Æther\n", "0 05a7ca83-e820-433f-b9e9-151e817d3708 Tar Pit Warrior Tar Pit Warrior\n", "After:\n", " oracle_id name name_ja\n", "0 0fe602b7-9f88-4d3d-af24-7790df867ed5 Aether Barrier None\n", "0 1e33f39b-a61a-4a09-a541-16cc1bd53d02 Aether Burst None\n", "0 15e83068-6253-4c65-8679-7295f3dc2075 Aether Charge None\n", "0 a3c35742-e306-49b6-b042-db4f685c6f86 Aether Flash None\n", "0 6697fe5b-90ac-4321-aa2f-cdc6ec283cb4 Aether Mutation None\n", "0 61105cb5-d7a1-4021-a006-dd1b947dfa68 Aether Sting None\n", "0 ff4297d3-3d96-4bd6-a606-1bdc20a6df2b Aether Storm None\n", "0 2fbf95b4-bcf4-4b5e-b5dc-0294f2b48d3e Aether Tide None\n", "0 a61ceda1-5993-479e-945f-15753eeb7049 Tainted Aether None\n", "0 05a7ca83-e820-433f-b9e9-151e817d3708 Tar Pit Warrior None\n" ] } ], "source": [ "wrongnames = [\n", " \"Aether Barrier\",\n", " \"Aether Burst\",\n", " \"Aether Charge\",\n", " \"Aether Flash\",\n", " \"Aether Mutation\",\n", " \"Aether Sting\",\n", " \"Aether Storm\",\n", " \"Aether Tide\",\n", " \"Tainted Aether\",\n", " \"Tar Pit Warrior\",\n", "]\n", "print(\"Before:\")\n", "print(middleschool_df.loc[middleschool_df[\"name\"].isin(wrongnames)])\n", "middleschool_df.loc[middleschool_df[\"name\"].isin(wrongnames), \"name_ja\"] = None\n", "print(\"After:\")\n", "print(middleschool_df.loc[middleschool_df[\"name\"].isin(wrongnames)])" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "Find Japanese names for cards that were not released in Japanese in Middle School legal sets\n" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "................................................................................\n", "................................................................................\n", "................................................................................\n", "................................................................................\n", "................................................................................\n", "................................................................................\n", "......................" ] } ], "source": [ "import time\n", "from requests_html import HTMLSession\n", "\n", "session = HTMLSession()\n", "\n", "\n", "def find_japanese_name(name):\n", " url = \"http://whisper.wisdom-guild.net/card/\" + name + \"/\"\n", " r = session.get(url)\n", " # Find the text on the element in the HTML document\n", " title = r.html.find(\"title\")[0].text\n", " # Find the position of the English card name within the title\n", " idx = title.find(name)\n", " # The Japanese name should be before the English name,\n", " # so if idx is 0, there is no Japanese name\n", " if idx == 0:\n", " return None\n", " # If the exact English card name can't be found, we look for a '/'\n", " if idx == -1:\n", " idx = title.find(\"/\")\n", " # No '/' means no Japanese name\n", " if idx == -1:\n", " return None\n", " # Take only the Japanese name from the title\n", " name_ja = title[0:idx]\n", " else:\n", " # Take only the Japanese name from the title\n", " name_ja = title[0 : idx - 1]\n", " return name_ja\n", "\n", "\n", "english_only_cards = middleschool_df[middleschool_df[\"name_ja\"].isnull()]\n", "name_list = english_only_cards[\"name\"].to_list()\n", "for idx, name in enumerate(name_list):\n", " middleschool_df.loc[middleschool_df[\"name\"] == name, \"name_ja\"] = (\n", " find_japanese_name(name)\n", " )\n", " # print(middleschool_df.loc[middleschool_df['name'] == name])\n", " print(\".\", end=\"\")\n", " if idx % 80 == 79:\n", " print()\n", " time.sleep(1)" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "Exclude all cards banned in Middle School\n" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Cards legal by set: 5800\n", "Banned cards: 25\n", "Cards legal by set and not banned: 5775\n" ] } ], "source": [ "banlist = [\n", " \"Amulet of Quoz\",\n", " \"Balance\",\n", " \"Brainstorm\",\n", " \"Bronze Tablet\",\n", " \"Channel\",\n", " \"Dark Ritual\",\n", " \"Demonic Consultation\",\n", " \"Flash\",\n", " \"Goblin Recruiter\",\n", " \"Imperial Seal\",\n", " \"Jeweled Bird\",\n", " \"Mana Crypt\",\n", " \"Mana Vault\",\n", " \"Memory Jar\",\n", " \"Mind's Desire\",\n", " \"Mind Twist\",\n", " \"Rebirth\",\n", " \"Strip Mine\",\n", " \"Tempest Efreet\",\n", " \"Timmerian Fiends\",\n", " \"Tolarian Academy\",\n", " \"Vampiric Tutor\",\n", " \"Windfall\",\n", " \"Yawgmoth's Bargain\",\n", " \"Yawgmoth's Will\",\n", "]\n", "print(\"Cards legal by set:\", middleschool_df.shape[0])\n", "# Find the rows with the banned cards\n", "banned_df = middleschool_df[\n", " pd.DataFrame(middleschool_df.name.tolist()).isin(banlist).any(axis=1).values\n", "]\n", "print(\"Banned cards:\", banned_df.shape[0])\n", "# Append the banned cards to the main Middle School DataFrame,\n", "# then remove any rows that appear twice,\n", "# effectively leaving only the legal cards\n", "middleschool_df = pd.concat([middleschool_df, banned_df]).drop_duplicates(keep=False)\n", "print(\"Cards legal by set and not banned:\", middleschool_df.shape[0])\n", "middleschool_df = middleschool_df.reset_index(drop=True)\n", "middleschool_df = middleschool_df[[\"oracle_id\", \"name\", \"name_ja\"]]" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "Save the list to a CSV file and a JSON file\n" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [], "source": [ "middleschool_df.to_csv(\"static/middleschool.csv\")\n", "middleschool_df.to_json(\"static/middleschool.json\")" ] }, { "attachments": {}, "cell_type": "markdown", "metadata": {}, "source": [ "Feel free to delete everything in the `data` directory after you are done\n" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.3" }, "orig_nbformat": 4, "vscode": { "interpreter": { "hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e" } } }, "nbformat": 4, "nbformat_minor": 2 }