{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "kernelspec": { "display_name": "Python [conda env:ml]", "language": "python", "name": "conda-env-ml-py" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.3" }, "colab": { "name": "run_all.ipynb", "provenance": [] }, "accelerator": "GPU", "widgets": { "application/vnd.jupyter.widget-state+json": { "134fb07f5e7b4f648166062a6f13405f": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "state": { "_view_name": "HBoxView", "_dom_classes": [], "_model_name": "HBoxModel", "_view_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_view_count": null, "_view_module_version": "1.5.0", "box_style": "", "layout": "IPY_MODEL_1313cbd66bf74792abf8f41f941e6b2d", "_model_module": "@jupyter-widgets/controls", "children": [ "IPY_MODEL_fedb6546740d46179de3367ae3b979c9", "IPY_MODEL_4a3f50ce50ed418db586f03f4b27d314" ] } }, "1313cbd66bf74792abf8f41f941e6b2d": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "state": { "_view_name": "LayoutView", "grid_template_rows": null, "right": null, "justify_content": null, "_view_module": "@jupyter-widgets/base", "overflow": null, "_model_module_version": "1.2.0", "_view_count": null, "flex_flow": "row wrap", "width": "100%", "min_width": null, "border": null, "align_items": null, "bottom": null, "_model_module": "@jupyter-widgets/base", "top": null, "grid_column": null, "overflow_y": null, "overflow_x": null, "grid_auto_flow": null, "grid_area": null, "grid_template_columns": null, "flex": null, "_model_name": "LayoutModel", "justify_items": null, "grid_row": null, "max_height": null, "align_content": null, "visibility": null, "align_self": null, "height": null, "min_height": null, "padding": null, "grid_auto_rows": null, "grid_gap": null, "max_width": null, "order": null, "_view_module_version": "1.2.0", "grid_template_areas": null, "object_position": null, "object_fit": null, "grid_auto_columns": null, "margin": null, "display": "inline-flex", "left": null } }, "fedb6546740d46179de3367ae3b979c9": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "state": { "_view_name": "ProgressView", "style": "IPY_MODEL_88fec4e3312a4e38a9f20b8a2ea97df6", "_dom_classes": [], "description": "Testing: 100%", "_model_name": "FloatProgressModel", "bar_style": "success", "max": 1, "_view_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "value": 1, "_view_count": null, "_view_module_version": "1.5.0", "orientation": "horizontal", "min": 0, "description_tooltip": null, "_model_module": "@jupyter-widgets/controls", "layout": "IPY_MODEL_5b668c94f7d44e08a94fd985e3fe31ee" } }, "4a3f50ce50ed418db586f03f4b27d314": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "state": { "_view_name": "HTMLView", "style": "IPY_MODEL_299e5d19924942509ad50453039df40a", "_dom_classes": [], "description": "", "_model_name": "HTMLModel", "placeholder": "​", "_view_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "value": " 63/63 [00:19<00:00, 3.26it/s]", "_view_count": null, "_view_module_version": "1.5.0", "description_tooltip": null, "_model_module": "@jupyter-widgets/controls", "layout": "IPY_MODEL_051d20e802b24f009a10f468425e6c1c" } }, "88fec4e3312a4e38a9f20b8a2ea97df6": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "state": { "_view_name": "StyleView", "_model_name": "ProgressStyleModel", "description_width": "initial", "_view_module": "@jupyter-widgets/base", "_model_module_version": "1.5.0", "_view_count": null, "_view_module_version": "1.2.0", "bar_color": null, "_model_module": "@jupyter-widgets/controls" } }, "5b668c94f7d44e08a94fd985e3fe31ee": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "state": { "_view_name": "LayoutView", "grid_template_rows": null, "right": null, "justify_content": null, "_view_module": "@jupyter-widgets/base", "overflow": null, "_model_module_version": "1.2.0", "_view_count": null, "flex_flow": null, "width": null, "min_width": null, "border": null, "align_items": null, "bottom": null, "_model_module": "@jupyter-widgets/base", "top": null, "grid_column": null, "overflow_y": null, "overflow_x": null, "grid_auto_flow": null, "grid_area": null, "grid_template_columns": null, "flex": "2", "_model_name": "LayoutModel", "justify_items": null, "grid_row": null, "max_height": null, "align_content": null, "visibility": null, "align_self": null, "height": null, "min_height": null, "padding": null, "grid_auto_rows": null, "grid_gap": null, "max_width": null, "order": null, "_view_module_version": "1.2.0", "grid_template_areas": null, "object_position": null, "object_fit": null, "grid_auto_columns": null, "margin": null, "display": null, "left": null } }, "299e5d19924942509ad50453039df40a": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "state": { "_view_name": "StyleView", "_model_name": "DescriptionStyleModel", "description_width": "", "_view_module": "@jupyter-widgets/base", "_model_module_version": "1.5.0", "_view_count": null, "_view_module_version": "1.2.0", "_model_module": "@jupyter-widgets/controls" } }, "051d20e802b24f009a10f468425e6c1c": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "state": { "_view_name": "LayoutView", "grid_template_rows": null, "right": null, "justify_content": null, "_view_module": "@jupyter-widgets/base", "overflow": null, "_model_module_version": "1.2.0", "_view_count": null, "flex_flow": null, "width": null, "min_width": null, "border": null, "align_items": null, "bottom": null, "_model_module": "@jupyter-widgets/base", "top": null, "grid_column": null, "overflow_y": null, "overflow_x": null, "grid_auto_flow": null, "grid_area": null, "grid_template_columns": null, "flex": null, "_model_name": "LayoutModel", "justify_items": null, "grid_row": null, "max_height": null, "align_content": null, "visibility": null, "align_self": null, "height": null, "min_height": null, "padding": null, "grid_auto_rows": null, "grid_gap": null, "max_width": null, "order": null, "_view_module_version": "1.2.0", "grid_template_areas": null, "object_position": null, "object_fit": null, "grid_auto_columns": null, "margin": null, "display": null, "left": null } }, "123c9d1f222d47e3b374f580ab7da346": { "model_module": "@jupyter-widgets/controls", "model_name": "HBoxModel", "state": { "_view_name": "HBoxView", "_dom_classes": [], "_model_name": "HBoxModel", "_view_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "_view_count": null, "_view_module_version": "1.5.0", "box_style": "", "layout": "IPY_MODEL_aa4c20448b2b4191a359366d20cfcd20", "_model_module": "@jupyter-widgets/controls", "children": [ "IPY_MODEL_6c15c5b234e2440093523921672b8348", "IPY_MODEL_a1acab3bf6854a1bbc9edf08a55cc90d" ] } }, "aa4c20448b2b4191a359366d20cfcd20": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "state": { "_view_name": "LayoutView", "grid_template_rows": null, "right": null, "justify_content": null, "_view_module": "@jupyter-widgets/base", "overflow": null, "_model_module_version": "1.2.0", "_view_count": null, "flex_flow": "row wrap", "width": "100%", "min_width": null, "border": null, "align_items": null, "bottom": null, "_model_module": "@jupyter-widgets/base", "top": null, "grid_column": null, "overflow_y": null, "overflow_x": null, "grid_auto_flow": null, "grid_area": null, "grid_template_columns": null, "flex": null, "_model_name": "LayoutModel", "justify_items": null, "grid_row": null, "max_height": null, "align_content": null, "visibility": null, "align_self": null, "height": null, "min_height": null, "padding": null, "grid_auto_rows": null, "grid_gap": null, "max_width": null, "order": null, "_view_module_version": "1.2.0", "grid_template_areas": null, "object_position": null, "object_fit": null, "grid_auto_columns": null, "margin": null, "display": "inline-flex", "left": null } }, "6c15c5b234e2440093523921672b8348": { "model_module": "@jupyter-widgets/controls", "model_name": "FloatProgressModel", "state": { "_view_name": "ProgressView", "style": "IPY_MODEL_91edbbe592df4958842778f18481eb38", "_dom_classes": [], "description": "Predicting: 100%", "_model_name": "FloatProgressModel", "bar_style": "success", "max": 1, "_view_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "value": 1, "_view_count": null, "_view_module_version": "1.5.0", "orientation": "horizontal", "min": 0, "description_tooltip": null, "_model_module": "@jupyter-widgets/controls", "layout": "IPY_MODEL_7db446afede0468292b3850b43251071" } }, "a1acab3bf6854a1bbc9edf08a55cc90d": { "model_module": "@jupyter-widgets/controls", "model_name": "HTMLModel", "state": { "_view_name": "HTMLView", "style": "IPY_MODEL_3a3e3368d12940eaa7f169e43cc9b522", "_dom_classes": [], "description": "", "_model_name": "HTMLModel", "placeholder": "​", "_view_module": "@jupyter-widgets/controls", "_model_module_version": "1.5.0", "value": " 157/157 [00:46<00:00, 3.35it/s]", "_view_count": null, "_view_module_version": "1.5.0", "description_tooltip": null, "_model_module": "@jupyter-widgets/controls", "layout": "IPY_MODEL_a1f3d0dc1c3b4f309aec430c8dc4cfe4" } }, "91edbbe592df4958842778f18481eb38": { "model_module": "@jupyter-widgets/controls", "model_name": "ProgressStyleModel", "state": { "_view_name": "StyleView", "_model_name": "ProgressStyleModel", "description_width": "initial", "_view_module": "@jupyter-widgets/base", "_model_module_version": "1.5.0", "_view_count": null, "_view_module_version": "1.2.0", "bar_color": null, "_model_module": "@jupyter-widgets/controls" } }, "7db446afede0468292b3850b43251071": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "state": { "_view_name": "LayoutView", "grid_template_rows": null, "right": null, "justify_content": null, "_view_module": "@jupyter-widgets/base", "overflow": null, "_model_module_version": "1.2.0", "_view_count": null, "flex_flow": null, "width": null, "min_width": null, "border": null, "align_items": null, "bottom": null, "_model_module": "@jupyter-widgets/base", "top": null, "grid_column": null, "overflow_y": null, "overflow_x": null, "grid_auto_flow": null, "grid_area": null, "grid_template_columns": null, "flex": "2", "_model_name": "LayoutModel", "justify_items": null, "grid_row": null, "max_height": null, "align_content": null, "visibility": null, "align_self": null, "height": null, "min_height": null, "padding": null, "grid_auto_rows": null, "grid_gap": null, "max_width": null, "order": null, "_view_module_version": "1.2.0", "grid_template_areas": null, "object_position": null, "object_fit": null, "grid_auto_columns": null, "margin": null, "display": null, "left": null } }, "3a3e3368d12940eaa7f169e43cc9b522": { "model_module": "@jupyter-widgets/controls", "model_name": "DescriptionStyleModel", "state": { "_view_name": "StyleView", "_model_name": "DescriptionStyleModel", "description_width": "", "_view_module": "@jupyter-widgets/base", "_model_module_version": "1.5.0", "_view_count": null, "_view_module_version": "1.2.0", "_model_module": "@jupyter-widgets/controls" } }, "a1f3d0dc1c3b4f309aec430c8dc4cfe4": { "model_module": "@jupyter-widgets/base", "model_name": "LayoutModel", "state": { "_view_name": "LayoutView", "grid_template_rows": null, "right": null, "justify_content": null, "_view_module": "@jupyter-widgets/base", "overflow": null, "_model_module_version": "1.2.0", "_view_count": null, "flex_flow": null, "width": null, "min_width": null, "border": null, "align_items": null, "bottom": null, "_model_module": "@jupyter-widgets/base", "top": null, "grid_column": null, "overflow_y": null, "overflow_x": null, "grid_auto_flow": null, "grid_area": null, "grid_template_columns": null, "flex": null, "_model_name": "LayoutModel", "justify_items": null, "grid_row": null, "max_height": null, "align_content": null, "visibility": null, "align_self": null, "height": null, "min_height": null, "padding": null, "grid_auto_rows": null, "grid_gap": null, "max_width": null, "order": null, "_view_module_version": "1.2.0", "grid_template_areas": null, "object_position": null, "object_fit": null, "grid_auto_columns": null, "margin": null, "display": null, "left": null } } } } }, "cells": [ { "cell_type": "code", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "4-rtCARZEx7R", "outputId": "dd43f538-ccad-43e5-bfa8-c1810495f51a" }, "source": [ "!git clone https://github.com/derInformatiker/AIcrowd-AIBlitz7-Solution.git\n", "!pip install -r AIcrowd-AIBlitz7-Solution/challenge1/requirements.txt\n", "!pip install aicrowd-cli==0.1" ], "execution_count": 2, "outputs": [ { "output_type": "stream", "text": [ "fatal: destination path 'AIcrowd-AIBlitz7-Solution' already exists and is not an empty directory.\n", "\u001b[K |████████████████████████████████| 10.1MB 5.2MB/s \n", "\u001b[K |████████████████████████████████| 28.2MB 108kB/s \n", "\u001b[K |████████████████████████████████| 829kB 54.0MB/s \n", "\u001b[K |████████████████████████████████| 122kB 58.1MB/s \n", "\u001b[K |████████████████████████████████| 112kB 59.1MB/s \n", "\u001b[K |████████████████████████████████| 276kB 59.0MB/s \n", "\u001b[K |████████████████████████████████| 829kB 48.3MB/s \n", "\u001b[K |████████████████████████████████| 952kB 52.8MB/s \n", "\u001b[K |████████████████████████████████| 1.3MB 54.7MB/s \n", "\u001b[K |████████████████████████████████| 296kB 56.1MB/s \n", "\u001b[K |████████████████████████████████| 143kB 57.6MB/s \n", "\u001b[?25h Building wheel for efficientnet-pytorch (setup.py) ... \u001b[?25l\u001b[?25hdone\n", " Building wheel for albumentations (setup.py) ... \u001b[?25l\u001b[?25hdone\n", " Building wheel for PyYAML (setup.py) ... \u001b[?25l\u001b[?25hdone\n", " Building wheel for future (setup.py) ... \u001b[?25l\u001b[?25hdone\n", "\u001b[31mERROR: google-colab 1.0.0 has requirement pandas~=1.1.0; python_version >= \"3.0\", but you'll have pandas 1.0.5 which is incompatible.\u001b[0m\n" ], "name": "stdout" } ] }, { "cell_type": "code", "metadata": { "id": "gTH1WwDQNgQD" }, "source": [ "#RESTART RUNTIME TO USE NEW PACKAGES" ], "execution_count": null, "outputs": [] }, { "cell_type": "code", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "fU7_L2JhJEHa", "outputId": "b50123fd-3b75-47c0-f0f4-49789293b37d" }, "source": [ "API_KEY = \"\" # Please enter your API Key from [https://www.aicrowd.com/participants/me]\n", "!aicrowd login --api-key $API_KEY" ], "execution_count": 3, "outputs": [ { "output_type": "stream", "text": [ "\u001b[32mAPI Key valid\u001b[0m\n", "\u001b[32mSaved API Key successfully!\u001b[0m\n" ], "name": "stdout" } ] }, { "cell_type": "code", "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "0tFvGMeqJfki", "outputId": "803a84f1-4f79-450a-8a16-a9c8c5c93c19" }, "source": [ "!aicrowd dataset download --challenge rover-classification\n", "\n", "!rm -rf data\n", "!mkdir data\n", "\n", "!unzip -q train.zip -d data/train\n", "!unzip -q val.zip -d data/val\n", "!unzip -q test.zip -d data/test\n", "\n", "!mv train.csv data/train.csv\n", "!mv val.csv data/val.csv\n", "!mv sample_submission.csv data/sample_submission.csv" ], "execution_count": 4, "outputs": [ { "output_type": "stream", "text": [ "sample_submission.csv: 100% 164k/164k [00:00<00:00, 1.30MB/s]\n", "test.zip: 100% 66.5M/66.5M [00:03<00:00, 20.8MB/s]\n", "train.csv: 100% 689k/689k [00:00<00:00, 3.15MB/s]\n", "train.zip: 100% 266M/266M [00:10<00:00, 25.9MB/s]\n", "val.csv: 100% 65.0k/65.0k [00:00<00:00, 855kB/s]\n", "val.zip: 100% 26.5M/26.5M [00:01<00:00, 23.0MB/s]\n" ], "name": "stdout" } ] }, { "cell_type": "code", "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 36 }, "id": "1Z7z8CmlF2mf", "outputId": "7c8018ac-711a-4bd5-e055-540978e34103" }, "source": [ "import shutil\n", "\n", "shutil.copy('AIcrowd-AIBlitz7-Solution/challenge1/model.py','model.py')\n", "shutil.copy('AIcrowd-AIBlitz7-Solution/challenge1/dataset.py','dataset.py')" ], "execution_count": 5, "outputs": [ { "output_type": "execute_result", "data": { "application/vnd.google.colaboratory.intrinsic+json": { "type": "string" }, "text/plain": [ "'dataset.py'" ] }, "metadata": { "tags": [] }, "execution_count": 5 } ] }, { "cell_type": "code", "metadata": { "id": "AzOSRQp9AwgC" }, "source": [ "import model\n", "import dataset\n", "\n", "import albumentations as A\n", "from albumentations.augmentations.transforms import Flip\n", "\n", "import torch\n", "import pytorch_lightning as pl\n", "from pytorch_lightning import Trainer" ], "execution_count": 1, "outputs": [] }, { "cell_type": "code", "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 358, "referenced_widgets": [ "134fb07f5e7b4f648166062a6f13405f", "1313cbd66bf74792abf8f41f941e6b2d", "fedb6546740d46179de3367ae3b979c9", "4a3f50ce50ed418db586f03f4b27d314", "88fec4e3312a4e38a9f20b8a2ea97df6", "5b668c94f7d44e08a94fd985e3fe31ee", "299e5d19924942509ad50453039df40a", "051d20e802b24f009a10f468425e6c1c", "123c9d1f222d47e3b374f580ab7da346", "aa4c20448b2b4191a359366d20cfcd20", "6c15c5b234e2440093523921672b8348", "a1acab3bf6854a1bbc9edf08a55cc90d", "91edbbe592df4958842778f18481eb38", "7db446afede0468292b3850b43251071", "3a3e3368d12940eaa7f169e43cc9b522", "a1f3d0dc1c3b4f309aec430c8dc4cfe4" ] }, "id": "HgKD5TpHAwgH", "outputId": "77f77947-2b09-4693-8bf4-c8d1a709f6b0" }, "source": [ "if __name__ == '__main__':\n", " trainer = Trainer(max_epochs = 6, gpus = 1, precision=16, amp_level='O1',deterministic=True)\n", " \n", " train_tr = A.Compose([\n", " A.CenterCrop(200,200,always_apply=True),\n", " Flip()\n", " ])\n", " \n", " val_tr = A.Compose([\n", " A.CenterCrop(200,200,always_apply=True)\n", " ])\n", " \n", " model = model.Classifier({'lr':3e-4,'batch_size':64,'train_tr':train_tr,'val_tr':val_tr})\n", " \n", " trainer.fit(model)\n", " trainer.test(model)\n", " out = trainer.predict(model)" ], "execution_count": 2, "outputs": [ { "output_type": "stream", "text": [ "GPU available: True, used: True\n", "TPU available: None, using: 0 TPU cores\n", "Using native 16bit precision.\n" ], "name": "stderr" }, { "output_type": "stream", "text": [ "Loaded pretrained weights for efficientnet-b3\n" ], "name": "stdout" }, { "output_type": "stream", "text": [ "/usr/local/lib/python3.7/dist-packages/torch/utils/data/dataloader.py:477: UserWarning: This DataLoader will create 6 worker processes in total. Our suggested max number of worker in current system is 2, which is smaller than what this DataLoader is going to create. Please be aware that excessive worker creation might get DataLoader running slow or even freeze, lower the worker number to avoid potential slowness/freeze if necessary.\n", " cpuset_checked))\n" ], "name": "stderr" }, { "output_type": "display_data", "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "134fb07f5e7b4f648166062a6f13405f", "version_minor": 0, "version_major": 2 }, "text/plain": [ "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Testing', layout=Layout(flex='2'), max=…" ] }, "metadata": { "tags": [] } }, { "output_type": "stream", "text": [ "/content/model.py:23: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n", " prob = F.softmax(x)\n" ], "name": "stderr" }, { "output_type": "stream", "text": [ "\n", "--------------------------------------------------------------------------------\n", "DATALOADER:0 TEST RESULTS\n", "{'test_f1': 0.48106613755226135, 'test_loss': 0.7106836438179016}\n", "--------------------------------------------------------------------------------\n" ], "name": "stdout" }, { "output_type": "display_data", "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "123c9d1f222d47e3b374f580ab7da346", "version_minor": 0, "version_major": 2 }, "text/plain": [ "HBox(children=(FloatProgress(value=1.0, bar_style='info', description='Predicting', layout=Layout(flex='2'), m…" ] }, "metadata": { "tags": [] } }, { "output_type": "stream", "text": [ "\n" ], "name": "stdout" } ] }, { "cell_type": "code", "metadata": { "id": "jrziiDvLAwgI" }, "source": [ "import pandas as pd\n", "def writeSub(p):\n", " labelmap = {0:'perseverance',1:'curiosity'}\n", " test_df = pd.read_csv('data/sample_submission.csv')\n", " output_list = p.int().tolist()\n", " output_list = [labelmap[i] for i in output_list]\n", " test_df['label'] = output_list\n", " test_df.to_csv(path_or_buf='data/submission.csv',index = False)" ], "execution_count": 3, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "kjmxQMJLAwgJ" }, "source": [ "output = torch.tensor([])\n", "\n", "for i in range(len(out)):\n", " output = torch.cat((output,torch.tensor(out[i][1]).argmax(1)))" ], "execution_count": 4, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "RFbEO7bRAwgJ" }, "source": [ "writeSub(output)" ], "execution_count": 6, "outputs": [] }, { "cell_type": "code", "metadata": { "id": "Y3nuhLUrM-Dk" }, "source": [ "" ], "execution_count": null, "outputs": [] } ] }