demo_OpenSetSegmentation_modelSelection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "OpenGAN: Open-Set Recognition via Open Data Generation\n",
    "================\n",
    "**Supplemental Material for ICCV2021 Submission**\n",
    "\n",
    "\n",
    "In this notebook is for demonstrating open-set semantic segmentation, especially for model selection in this task."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "import packages\n",
    "------------------\n",
    "\n",
    "Some packages are installed automatically through Anaconda. PyTorch should be also installed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3.7.4 (default, Aug 13 2019, 20:35:49) \n",
      "[GCC 7.3.0]\n",
      "1.4.0+cu92\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function, division\n",
    "import os, random, time, copy, scipy, pickle, sys, math, json, pickle\n",
    "\n",
    "import argparse, pprint, shutil, logging, time, timeit\n",
    "from pathlib import Path\n",
    "\n",
    "from skimage import io, transform\n",
    "import numpy as np\n",
    "import os.path as path\n",
    "import scipy.io as sio\n",
    "from scipy import misc\n",
    "from scipy import ndimage, signal\n",
    "import matplotlib.pyplot as plt\n",
    "# import PIL.Image\n",
    "from PIL import Image\n",
    "from io import BytesIO\n",
    "from skimage import data, img_as_float\n",
    "from skimage.measure import compare_ssim as ssim\n",
    "from skimage.measure import compare_psnr as psnr\n",
    "\n",
    "import torch, torchvision\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.optim import lr_scheduler \n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "from torchvision import datasets, models, transforms\n",
    "import torchvision.utils as vutils\n",
    "from collections import namedtuple\n",
    "\n",
    "from config_HRNet import models\n",
    "from config_HRNet import seg_hrnet\n",
    "from config_HRNet import config\n",
    "from config_HRNet import update_config\n",
    "from config_HRNet.modelsummary  import *\n",
    "from config_HRNet.utils import *\n",
    "\n",
    "\n",
    "from utils.dataset_tinyimagenet import *\n",
    "from utils.dataset_cityscapes4OpenGAN import *\n",
    "from utils.eval_funcs import *\n",
    "\n",
    "\n",
    "import warnings # ignore warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "print(sys.version)\n",
    "print(torch.__version__)\n",
    "\n",
    "# %load_ext autoreload\n",
    "# %autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Setup config parameters\n",
    " -----------------\n",
    " \n",
    " There are several things to setup, like which GPU to use, where to read images and save files, etc. Please read and understand this. By default, you should be able to run this script smoothly by changing nothing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./exp/step030_OpenGAN_num1000_w0.20\n"
     ]
    }
   ],
   "source": [
    "# set the random seed\n",
    "torch.manual_seed(10)\n",
    "\n",
    "\n",
    "################## set attributes for this project/experiment ##################\n",
    "# config result folder\n",
    "exp_dir = './exp' # experiment directory, used for reading the init model\n",
    "\n",
    "num_valimages = 1000\n",
    "weight_adversarialLoss = 0.20\n",
    "project_name = 'step030_OpenGAN_num{}_w{:.2f}'.format(num_valimages,weight_adversarialLoss)\n",
    "\n",
    "\n",
    "device ='cpu'\n",
    "if torch.cuda.is_available(): \n",
    "    device='cuda:0'\n",
    "    \n",
    "\n",
    "totalEpoch = 29 # the total number of models cached (we adopt early termination)\n",
    "\n",
    "\n",
    "ganBatchSize = 1000\n",
    "batch_size = 1\n",
    "newsize = (-1,-1)\n",
    "\n",
    "total_epoch_num = 150 # total number of epoch in training\n",
    "insertConv = False    \n",
    "embDimension = 64\n",
    "#isPretrained = False\n",
    "\n",
    "\n",
    "# Number of channels in the training images. For color images this is 3\n",
    "nc = 720\n",
    "# Size of z latent vector (i.e. size of generator input)\n",
    "nz = 64\n",
    "# Size of feature maps in generator\n",
    "ngf = 64\n",
    "# Size of feature maps in discriminator\n",
    "ndf = 64\n",
    "# Beta1 hyperparam for Adam optimizers\n",
    "beta1 = 0.5\n",
    "# Number of GPUs available. Use 0 for CPU mode.\n",
    "ngpu = 1\n",
    "\n",
    "\n",
    "\n",
    "nClassTotal = 10\n",
    "nClassCloseset = 6\n",
    "runIdx = 0\n",
    "\n",
    "\n",
    "save_dir = os.path.join(exp_dir, project_name)\n",
    "if not os.path.exists(exp_dir): os.makedirs(exp_dir)\n",
    "\n",
    "lr = 0.0001 # base learning rate\n",
    "\n",
    "num_epochs = total_epoch_num\n",
    "torch.cuda.device_count()\n",
    "torch.cuda.empty_cache()\n",
    "\n",
    "save_dir = os.path.join(exp_dir, project_name)\n",
    "print(save_dir)    \n",
    "if not os.path.exists(save_dir): os.makedirs(save_dir)\n",
    "\n",
    "log_filename = os.path.join(save_dir, 'train.log')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define model architecture\n",
    "---------\n",
    "\n",
    "Here is the definition of the model architecture. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "parser = argparse.ArgumentParser(description='Train segmentation network') \n",
    "parser.add_argument('--cfg',\n",
    "                    help='experiment configure file name',\n",
    "                    default='./config_HRNet/seg_hrnet_w48_train_512x1024_sgd_lr1e-2_wd5e-4_bs_12_epoch484.yaml',\n",
    "                    type=str)\n",
    "parser.add_argument('opts',\n",
    "                    help=\"Modify config options using the command-line\",\n",
    "                    default=None,\n",
    "                    nargs=argparse.REMAINDER)\n",
    "\n",
    "\n",
    "args = parser.parse_args(r'--cfg  ./config_HRNet/seg_hrnet_w48_train_512x1024_sgd_lr1e-2_wd5e-4_bs_12_epoch484.yaml ')\n",
    "args.opts = []\n",
    "update_config(config, args)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "model = eval(config.MODEL.NAME + '.get_seg_model_myModel')(config)\n",
    "model_dict = model.state_dict()\n",
    "\n",
    "model_state_file = '../openset/models/hrnet_w48_cityscapes_cls19_1024x2048_ohem_trainset.pth'\n",
    "pretrained_dict = torch.load(model_state_file, map_location=lambda storage, loc: storage)\n",
    "\n",
    "suppl_dict = {}\n",
    "suppl_dict['last_1_conv.weight'] = pretrained_dict['model.last_layer.0.weight'].clone()\n",
    "suppl_dict['last_1_conv.bias'] = pretrained_dict['model.last_layer.0.bias'].clone()\n",
    "\n",
    "suppl_dict['last_2_BN.running_mean'] = pretrained_dict['model.last_layer.1.running_mean'].clone()\n",
    "suppl_dict['last_2_BN.running_var'] = pretrained_dict['model.last_layer.1.running_var'].clone()\n",
    "# suppl_dict['last_2_BN.num_batches_tracked'] = pretrained_dict['model.last_layer.1.num_batches_tracked']\n",
    "suppl_dict['last_2_BN.weight'] = pretrained_dict['model.last_layer.1.weight'].clone()\n",
    "suppl_dict['last_2_BN.bias'] = pretrained_dict['model.last_layer.1.bias'].clone()\n",
    "\n",
    "suppl_dict['last_4_conv.weight'] = pretrained_dict['model.last_layer.3.weight'].clone()\n",
    "suppl_dict['last_4_conv.bias'] = pretrained_dict['model.last_layer.3.bias'].clone()\n",
    "\n",
    "\n",
    "pretrained_dict = {k[6:]: v for k, v in pretrained_dict.items()\n",
    "                   if k[6:] in model_dict.keys()}\n",
    "\n",
    "model_dict.update(pretrained_dict)\n",
    "model_dict.update(suppl_dict)\n",
    "model.load_state_dict(model_dict)\n",
    "\n",
    "model.eval();\n",
    "model.to(device);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def weights_init(m):\n",
    "    classname = m.__class__.__name__\n",
    "    if classname.find('Conv') != -1:\n",
    "        nn.init.normal_(m.weight.data, 0.0, 0.02)\n",
    "    elif classname.find('BatchNorm') != -1:\n",
    "        nn.init.normal_(m.weight.data, 1.0, 0.02)\n",
    "        nn.init.constant_(m.bias.data, 0)     \n",
    "        \n",
    "\n",
    "class Generator(nn.Module):\n",
    "    def __init__(self, ngpu=1, nz=100, ngf=64, nc=512):\n",
    "        super(Generator, self).__init__()\n",
    "        self.ngpu = ngpu\n",
    "        self.nz = nz\n",
    "        self.ngf = ngf\n",
    "        self.nc = nc\n",
    "        \n",
    "        self.main = nn.Sequential(\n",
    "            # input is Z, going into a convolution\n",
    "            nn.Conv2d( self.nz, self.ngf * 8, 1, 1, 0, bias=True),\n",
    "            nn.BatchNorm2d(self.ngf * 8),\n",
    "            nn.ReLU(True),\n",
    "            # state size. (self.ngf*8) x 4 x 4\n",
    "            nn.Conv2d(self.ngf * 8, self.ngf * 4, 1, 1, 0, bias=True),\n",
    "            nn.BatchNorm2d(self.ngf * 4),\n",
    "            nn.ReLU(True),\n",
    "            # state size. (self.ngf*4) x 8 x 8\n",
    "            nn.Conv2d( self.ngf * 4, self.ngf * 2, 1, 1, 0, bias=True),\n",
    "            nn.BatchNorm2d(self.ngf * 2),\n",
    "            nn.ReLU(True),\n",
    "            # state size. (self.ngf*2) x 16 x 16\n",
    "            nn.Conv2d( self.ngf * 2, self.ngf*4, 1, 1, 0, bias=True),\n",
    "            nn.BatchNorm2d(self.ngf*4),\n",
    "            nn.ReLU(True),\n",
    "            # state size. (self.ngf) x 32 x 32\n",
    "            nn.Conv2d( self.ngf*4, self.nc, 1, 1, 0, bias=True),\n",
    "            #nn.Tanh()\n",
    "            # state size. (self.nc) x 64 x 64\n",
    "        )\n",
    "\n",
    "    def forward(self, input):\n",
    "        return self.main(input)\n",
    "\n",
    "    \n",
    "class Discriminator(nn.Module):\n",
    "    def __init__(self, ngpu=1, nc=512, ndf=64):\n",
    "        super(Discriminator, self).__init__()\n",
    "        self.ngpu = ngpu\n",
    "        self.nc = nc\n",
    "        self.ndf = ndf\n",
    "        self.main = nn.Sequential(\n",
    "            nn.Conv2d(self.nc, self.ndf*8, 1, 1, 0, bias=True),\n",
    "            nn.LeakyReLU(0.2, inplace=True),\n",
    "            nn.Conv2d(self.ndf*8, self.ndf*4, 1, 1, 0, bias=True),\n",
    "            nn.BatchNorm2d(self.ndf*4),\n",
    "            nn.LeakyReLU(0.2, inplace=True),\n",
    "            nn.Conv2d(self.ndf*4, self.ndf*2, 1, 1, 0, bias=True),\n",
    "            nn.BatchNorm2d(self.ndf*2),\n",
    "            nn.LeakyReLU(0.2, inplace=True),\n",
    "            nn.Conv2d(self.ndf*2, self.ndf, 1, 1, 0, bias=True),\n",
    "            nn.BatchNorm2d(self.ndf),\n",
    "            nn.LeakyReLU(0.2, inplace=True),\n",
    "            nn.Conv2d(self.ndf, 1, 1, 1, 0, bias=True),\n",
    "            nn.Sigmoid()\n",
    "        )\n",
    "\n",
    "    def forward(self, input):\n",
    "        return self.main(input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "bestEpoch_GAN = 1 # random checkpoint for sanity check \n",
    "\n",
    "# netG = Generator(ngpu=ngpu, nz=nz, ngf=ngf, nc=nc).to(device)\n",
    "netD = Discriminator(ngpu=ngpu, nc=nc, ndf=ndf).to(device)\n",
    "\n",
    "\n",
    "# path_to_G = os.path.join(save_dir, 'epoch-{}.GNet'.format(bestEpoch_GAN))\n",
    "path_to_D = os.path.join(save_dir, 'epoch-{}.classifier'.format(bestEpoch_GAN))\n",
    "# netG.load_state_dict(torch.load(path_to_G))\n",
    "netD.load_state_dict(torch.load(path_to_D, map_location=lambda storage, loc: storage))\n",
    "\n",
    "# netG.eval()\n",
    "netD.eval();"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "setup dataset\n",
    "-----------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2900 10 500\n"
     ]
    }
   ],
   "source": [
    "# torchvision.transforms.Normalize(mean, std, inplace=False)\n",
    "imgTransformList = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),\n",
    "])\n",
    "\n",
    "targetTransformList = transforms.Compose([\n",
    "    transforms.ToTensor(),    \n",
    "])\n",
    "\n",
    "\n",
    "trainset = Cityscapes(root='/scratch/dataset/Cityscapes',\n",
    "                             newsize=newsize, split='train', trainnum=2900, mode='fine', target_type='semantic',\n",
    "                             transform=imgTransformList, target_transform=targetTransformList,\n",
    "                             transforms=None)\n",
    "\n",
    "# -10 means we fetch the last 10 images as our validation set\n",
    "valset = Cityscapes(root='/scratch/dataset/Cityscapes',\n",
    "                             newsize=newsize, split='train', trainnum=-10, mode='fine', target_type='semantic',\n",
    "                             transform=imgTransformList, target_transform=targetTransformList,\n",
    "                             transforms=None)\n",
    "\n",
    "testset = Cityscapes(root='/scratch/dataset/Cityscapes',\n",
    "                             newsize=newsize, split='val', mode='fine', target_type='semantic',\n",
    "                             transform=imgTransformList, target_transform=targetTransformList,\n",
    "                             transforms=None)\n",
    "\n",
    "\n",
    "\n",
    "dataloader_test = DataLoader(testset, batch_size=batch_size, shuffle=False, num_workers=4)\n",
    "dataloader_val = DataLoader(valset, batch_size=batch_size, shuffle=False, num_workers=4)\n",
    "dataloader_train = DataLoader(trainset, batch_size=batch_size, shuffle=True, num_workers=4)\n",
    "\n",
    "\n",
    "print(len(trainset), len(valset), len(testset))\n",
    "classDictionary = trainset.classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 unlabeled\n",
      "1 ego vehicle\n",
      "2 rectification border\n",
      "3 out of roi\n",
      "4 static\n",
      "5 dynamic\n",
      "6 ground\n",
      "9 parking\n",
      "10 rail track\n",
      "14 guard rail\n",
      "15 bridge\n",
      "16 tunnel\n",
      "18 polegroup\n",
      "29 caravan\n",
      "30 trailer\n",
      "34 license plate\n",
      "total# 16\n"
     ]
    }
   ],
   "source": [
    "id2trainID = {}\n",
    "id2color = {}\n",
    "trainID2color = {}\n",
    "id2name = {}\n",
    "opensetIDlist = []\n",
    "for i in range(len(classDictionary)):\n",
    "    id2trainID[i] = classDictionary[i][2]\n",
    "    id2color[i] = classDictionary[i][-1]\n",
    "    trainID2color[classDictionary[i][2]] = classDictionary[i][-1]\n",
    "    id2name[i] = classDictionary[i][0]\n",
    "    if classDictionary[i][-2]:\n",
    "        opensetIDlist += [i]\n",
    "\n",
    "id2trainID_list = []\n",
    "for i in range(len(id2trainID)):\n",
    "    id2trainID_list.append(id2trainID[i])\n",
    "id2trainID_np = np.asarray(id2trainID_list)        \n",
    "        \n",
    "for elm in opensetIDlist:\n",
    "    print(elm, id2name[elm])\n",
    "print('total# {}'.format(len(opensetIDlist)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "testing a single image\n",
    "-----------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 3, 1024, 2048]) torch.Size([256, 512]) torch.Size([1, 256, 512])\n"
     ]
    }
   ],
   "source": [
    "data_sampler = iter(dataloader_train)\n",
    "data = next(data_sampler)\n",
    "imageList, labelList = data[0], data[1]\n",
    "imageList = imageList.to(device)\n",
    "labelList = labelList.to(device)\n",
    "\n",
    "labelList = labelList.unsqueeze(1)\n",
    "labelList = F.interpolate(labelList, scale_factor=0.25, mode='nearest')\n",
    "labelList = labelList.squeeze()\n",
    "labelList.shape\n",
    "\n",
    "H, W = labelList.squeeze().shape\n",
    "trainlabelList = id2trainID_np[labelList.cpu().numpy().reshape(-1,).astype(np.int32)]\n",
    "trainlabelList = trainlabelList.reshape((1,H,W))\n",
    "trainlabelList = torch.from_numpy(trainlabelList)\n",
    "\n",
    "print(imageList.shape, labelList.shape, trainlabelList.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Initialize BCELoss function\n",
    "criterion = nn.BCELoss()\n",
    "\n",
    "# Establish convention for real and fake labels during training\n",
    "real_label = 1\n",
    "fake_label = 0\n",
    "\n",
    "upsampleFunc = nn.UpsamplingBilinear2d(scale_factor=4)\n",
    "with torch.no_grad():\n",
    "    imageList = imageList.to(device)\n",
    "    logitsTensor = model(imageList).detach().cpu()\n",
    "    #logitsTensor = upsampleFunc(logitsTensor)\n",
    "    softmaxTensor = F.softmax(logitsTensor, dim=1)    \n",
    "    feat1Tensor = model.feat1.detach()\n",
    "    feat2Tensor = model.feat2.detach()\n",
    "    feat3Tensor = model.feat3.detach()\n",
    "    feat4Tensor = model.feat4.detach()\n",
    "    feat5Tensor = model.feat5.detach()\n",
    "    torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([131072])\n"
     ]
    }
   ],
   "source": [
    "allPixList = feat4Tensor.squeeze()\n",
    "allPixList = allPixList.reshape(allPixList.shape[0], -1).permute(1,0).unsqueeze(-1).unsqueeze(-1).to(device)\n",
    "with torch.no_grad():\n",
    "    allPixConf = netD(allPixList).view(-1)\n",
    "    \n",
    "print(allPixConf.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "123188\n"
     ]
    }
   ],
   "source": [
    "closesetList = trainlabelList.reshape(-1,1)\n",
    "closesetList = ((closesetList>=0) & (closesetList<=18)).nonzero()\n",
    "closesetList = closesetList[:,0]\n",
    "#closesetList = closesetList[torch.randperm(closesetList.size()[0])]\n",
    "#closesetList = closesetList[:ganBatchSize]\n",
    "print(len(closesetList))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([123188, 1])\n"
     ]
    }
   ],
   "source": [
    "predConfidence_closeset = torch.from_numpy(np.array([0])).type(torch.float).view(-1,1)\n",
    "\n",
    "predConf = allPixConf[closesetList]\n",
    "predConf = predConf.view(-1,1).detach()\n",
    "\n",
    "\n",
    "predConfidence_closeset = torch.cat((predConfidence_closeset, predConf.detach().cpu()), 0)\n",
    "print(predConf.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "not existing --  0 tensor(0., device='cuda:0')\n",
      "1 ego vehicle\n",
      "not existing --  2 tensor(0., device='cuda:0')\n",
      "3 out of roi\n",
      "4 static\n",
      "not existing --  5 tensor(0., device='cuda:0')\n",
      "6 ground\n",
      "not existing --  9 tensor(0., device='cuda:0')\n",
      "not existing --  10 tensor(0., device='cuda:0')\n",
      "not existing --  14 tensor(0., device='cuda:0')\n",
      "not existing --  15 tensor(0., device='cuda:0')\n",
      "not existing --  16 tensor(0., device='cuda:0')\n",
      "not existing --  18 tensor(0., device='cuda:0')\n",
      "not existing --  29 tensor(0., device='cuda:0')\n",
      "not existing --  30 tensor(0., device='cuda:0')\n",
      "not existing --  34 tensor(0., device='cuda:0')\n"
     ]
    }
   ],
   "source": [
    "predConfidence_opensetlist = [torch.from_numpy(np.array([0])).type(torch.float).view(-1,1)]*len(opensetIDlist)\n",
    "\n",
    "for i in range(len(opensetIDlist)):\n",
    "    clsID = opensetIDlist[i]\n",
    "    tmpMap = (labelList==clsID).type(torch.float32)\n",
    "    tmpCount = tmpMap.sum()    \n",
    "    if tmpCount==0:   \n",
    "        print('not existing -- ', clsID, tmpCount)\n",
    "        continue\n",
    "                    \n",
    "    curClsList = labelList.reshape(-1,1)\n",
    "    curClsList = (curClsList==clsID).nonzero()\n",
    "    curClsList = curClsList[:,0]\n",
    "    \n",
    "    predConf = allPixConf[curClsList]\n",
    "    predConf = predConf.view(-1,1).detach()\n",
    "        \n",
    "    predConfidence_opensetlist[i] = torch.cat((predConfidence_opensetlist[i], predConf.detach().cpu()), 0)      \n",
    "    print(clsID, id2name[clsID])\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "for i in range(len(opensetIDlist)):\n",
    "    predConfidence_opensetlist[i] = predConfidence_opensetlist[i][1:]\n",
    "    predConfidence_opensetlist[i] = predConfidence_opensetlist[i].detach().cpu().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'trainlabel_to_color')"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV0AAADFCAYAAAD3yiEtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAOsklEQVR4nO3df6xkZ13H8c8XLOKy7ZYVTX9wKfHauPxINPUPMBjdmGapJgXCLaiwNCIYmm2widQSCMQbBNo01VjUzTYxlWy7jYHexhWom7HbbAhBUNPEpAWS7ZqWFVikXbfttoQf8vWPc2Y7d+45M2dmzjznec7zfiVNd++c+XnvvPc7z5w519xdAIAwXtD1DQCAnBBdAAiI6AJAQEQXAAIiugAQENEFgICILgAERHQxkZkdMLOPNtz202b28YbbHjOz9855m+Y+byizPBbIC9HtOTN7zMyunPf87n6du/95m7cpBDP7AzP7Ute3AxhHdDNmZj/V9W3AZnxP+o/o9piZ3SXpFZI+Z2ZnzewmM3Mze4+ZfVPSg+V2nzWzU2b2lJl90cxeM3IZ514mm9luM/tvM/uAmf2PmX3HzN5dc90vNbPPm9n3zOx/yz+/fGyzVTP7t/J6D5vZzpHzv97MvmxmZ8zsP81s9wz3+1WSDkj6tfJ+nym/vsPMDpa36XEz+4iZTX0OmNkfmdnXzewZM/uamV0xvJ5yqeOMmT1iZm+achmPmtlpM/snM7tk5DQ3s+vN7Lik403vJ9JEdHvM3d8l6ZuSrnb37ZI+U570m5JeJemN5d//WdLlkn5e0kOSDk242Isk7ZB0qaT3SPpbM3tpxXYvkPT3ki5TEf7vS/qbsW2ulfSHki6R9GNJn5IkM7tU0hckfVzSTkk3Stows59reL+/Luk6Sf/q7tvd/cLypL8ub/svqHgMrpVU+Y/GkJm9TdJ6ue0Fkt4k6UkzO0/S5yQNVDxu75d0yMx+qeIyfkvSzZLeLuliSY9L+oexzd4i6XWSXt3kPiJdRDdP6+7+rLt/X5Lc/U53f8bdf6AiML9sZjtqzvsjSR9z9x+5+/2SzkraEhp3f9LdN9z9OXd/RtInVIRu1F3u/rC7Pyvpo5LebmYvlLRX0v3ufr+7/8Td/0XSf0j6nXnvcHm5vyvpQ+V9fUzSX0h615SzvlfSre7+71541N0fl/R6Sdsl3eLuP3T3ByV9XtLvV1zGOyXd6e4PlY/xh1RM4a8c2eZmdz89/J6gv4hunk4O/2BmLzSzW8zshJk9Lemx8qSX1Zz3SXf/8cjfn1MRn03MbJuZ3VG+jH9a0hclXVjGb8vtUDH9nVde72WS3la+bD9TLg/8uoopcV4vk/Si8npGr/PSKedbkXSi4uuXSDrp7j9pcHmXjF6vu5+V9OTYtifHz4R+Irr9V3XsztGvvUPSmyVdqeKl9yvLr9uC1/sBFRPw69z9Akm/UXG5KyN/foWKKfoJFQG6y90vHPnvJe5+ywzXP36/nygv/7Kx6/zWlMs5KWm14uvflrQytiZcd3nfHr1eM3uJpJ8d25ZjrGaC6Pbfd1WsYdY5X9IPVExe2yR9sqXrPV/FOu6Z8g2yP6vYZq+ZvdrMtkn6mKR73f3/JN0t6Woze2M5ib+4fBNv/I24Sb4r6eVm9iJJKi/3M5I+YWbnm9llkv6kvK5J/k7SjWb2q1b4xfK8X5X0rKSbzOy88o2+q7V1rVaS7pH0bjP7FTP7aRWP8VfLJQ5khuj2382SPlK+RL+m4vSDKl76fkvS1yR9paXr/StJP6NiwvyKpCMV29wl6dOSTkl6saQ/liR3P6li+v6wpO+pmDb/VLP9vD4o6RFJp8zsifJr71cRyv+S9CUVMbxz0oW4+2dVrEffI+kZSf8oaae7/1DFm2q/Xd7H/ZKudfdvVFzGURVr1huSvqNicv69Ge4LesT4zREAEA6TLgAERHSRLCuOC3G24r8DXVwO0ATLCwAQ0MTPeZvdQJEBYEbut9fucjn14BrX/+UX2r01LTp+pGr3ya3W1tY2/X1jY2MZNwfAiP37B4233bdvzxJvSViDwa6Jpye7pts0uADi1qfgNpFsdAHErUlMcwuulGl0x5cbACCULKMLIIwcJ9lpko/u5Ved0OVXVR0ECkAM6sKba5CTji6xBdKUa3ClhKNLcIF05BzZcclGF0C6ZtmHt2/4zaMAlmJaWEdPz2kS7sWkywclgLjkPMlO04voAojHPMHdv3+QTaiJLoDW5BLORSQfXZYWgDgQ3GaSjy6A/sgh3ElHlykXQGqSjS7BBZCiJKNLcAGkKrnoElwAKUvmE2nEFkAfJDHpElwAfRH9pEtwgXzs27dn4d/sEvsvno02usQWyE8Ov0orqugSWiBdwyOFzfMBh6NH79j093mn1bW1tXPhjnXijSK6xBbIW1uB3NjYiH5a7jy6BBfol6YTb07H0B3VeXQB9NNoVHM9YHkVojsm9vUgIHaz/MaIoZxCnMR+ugDQF51HN8Rv9a2aWmNfbAfQTywvjBgMdqmuxVUviY4evYNlCAAz6XzSBYCcRBHdEEsMi6pb6B8MdgW+JQBSFsXyQir76u7bt+dcZB999FPnvj782p493+jkdgFIR6eT7vEjq9EF933ve6Ty64PBrk1T7fjHFofbAMAkUSwv9AnhBTDJ1OWF4SQ6y7prbNNraIPBLpYagFJOH3xoovGa7vEjq0m84QWgW0R2spmWF5pMsLlPuQAwSatrugQXACabObpVexzEuBfCstXt5QAAk7D3AgAEFMWHI1Iw3Bth2i5h7LUAYBIm3RYRXADTEN0W8cEIANMQXQAIiOgCQEC8kTbi+TXZ13R6OwD0F5MuAAREdAEgIJYXZrR5tzCWIQDMhuguiH1zAcyC5QUACIjoAkBARBcAAmoluin8RomNjY2ubwIAMOkCQEhEFwACIroAEBDRBYCAiC4ABER0F7S2thbsuvY/9Ny5/9A9vheYB9FNxPiTmyd7d4gtFkF0gQURYMyC6AJAQEQXaAHTLpoiugmoekLvu2JbB7cEUvHYVz3+hBdNcDzdyPFEjhPfl60Gg6tqT9uz50jAWxK3uaN7/MhqEge6ATC/SSHFfJh0F7CxsRF0P91Rw0mLZYbw+jzlEtnlY00XmFFf/6EjuGEw6UaszxNVX+1/6Lnkokxsw5p70mU9d/lSe/KikMo/loPBVQS3AywvADNKJaqTENvu9C66a2trnb25BaSA4HYrmegSUwB90MkbacN48ssi0Texr8Mz5XavtUmXN9YAYLqlLi+wJLC4us/5D/XhTR0gJ8ms6aJa7C9nAWzWeE2XdVigHv/4oamsJ92Ulj54UgP9wFHGErLvim2s4bbgvhMHFzr/KV3T+mWGsl33dH0TssexF9CaVMIDdKlxdLs8jGFu6uJ16qnNE9ZFO+7VfbzYCGr8e4B4ra2tRfkeVBaT7jBih3S49rRRh3S48dQ2y7ZtumjHvcGvE8DipkZ3NCjjgfmg3lC77XD7Wb4+SePz3CoduqmbEKL/Ltpxb9LT7tnVd5z78/YTrO92IYtJF1i2U09d08tXH6ORlgh1G+aO7gevf8P0jRJ228qBRttdrIsbb7uIvQ8/sOnvIa4TNVau3PL9kNL7nqyvlP8/dsHW03Y/LUm6TZvv0/pK9fZ15x/68py3sY+YdGvcePK6Rtsd0uFG26b2hEQ+RgPZJKhYTG+je+imw3rnrW+e67yznq/JdTWNeJ1T2jxZ7X34gUYvZ4k9ZjE+odZtMynOTS4jZ72NbpV5Ixyrvq4jpuDu125dYtj78AO6+7VXdnSLwiKs8+tldPsWV6n5LmJMtmjDoq/Mxr11lVXdoV5GN6SuA09kw6t6Ey0FbYd0FvedOKi3rl7b2fXHJJnodh232BBbNNFlaMeN7jufc4Bnju4ib1BhccS2ezGv28YU2UnGP7yUU4SDTLqEenHEFtOkEtwqOU3BnSwvEOBmCC1yVPUR/j6FOJk13ZwQW2CzYYj7EF9z9/oT7Yb6E5fk7OqHQ19lNIgtFpHy8sIixkM8egjaLg7tOBjskvvtVnd6VL+uJ9fg3rZygOBiYbn+DN134mBSRxWce3kh10C2IdcnB5Zv+LOV49Q76bjZMZkaXeLaLoKLEHKO76gY14KnrukS3cURWnQt9/iOW2aEp63psvfCEhFbxILJd7MuJ2Ci2zJCi5gR38262CeY6LaE2CIlxLfesqdgorsAQovUEd96y5qCie4ciC36hvg208YhKoluQ4QWOSC+0y16hDSiOwWxRY5uWzlAeBua9Qhp7KdbgdACzyO+s9l+4pPsp9sUsQW2YsmhXURXxBZogvi2I8voEllgfsR3MVEd2nHZOIQi0B6eS/Pp/aTLDwawPKPPLybfZno96RJcIByeb830atLlmw50i/Xe6Xoz6RJcIB48H+slPenyjQXixdRbLclJl70QgHTwXN0smUmXbxyQLqbe50UfXWIL9AfxjTS6hBbot5zjG92aLsEF8pHj8z2KSTfHBx5AIbdPtXU66bIXAoBROfQg+KSbw4MKYH59X+8NOukSXABN9bUXS510+/qgAQijj1PvUiZd1moBtKlPPWl10u3TAwMgLn2ZeluJLrEFEErq8Z07uoQWQJdSje/M0SW2AGKSWnwbR5fYAohZKvGdGl1ii75bP3a6/rTdOwPeErQh9vhGd8AbIKRJwW1yOuIV68AYxQFvgNBmien6sdNMvImK8WA6TLrIzjzTKxNv+mKZfIkusrJIPNePnSa+iYvh07JEF0B2ugwva7oAstTVei+TLrLB0gDqhJx8mXQBQOEmXyZdZIEpF7Fg0kUW1nfvJLxobJlTL5MussEHHDCPttd7mXSBGRDuPLU5+TLpAg0RXEiLT75EF1khnGjDIp9sI7rIzjzhJdaoMk94WdNFloYR5Vi6WNSs671MushaVVjXd+8kuJhLk8mXSRe90OY+uMvan5eQQyK6CCznDygQc0hEFy3JOaZda/LYE+Z4EF2cQzj7q43vLeFuB9HNDGHFvJr+7BDnyYhuwggoYjTrz2VukSa6kSOs6Lu6n/G+xpjoBkRAgeb6OjET3SUgrkB4qUzMRHcC4gmkL7Y3AIluicACeatqwDJCbO5ef6LdUH8iAKCS++1Wd9rE6AIA2sVRxgAgIKILAAERXQAIiOgCQEBEFwACIroAEND/A1gV4voArzGcAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "A = trainlabel_to_color(trainlabelList.squeeze().detach().cpu().numpy(), trainID2color)\n",
    "plt.imshow(A.astype(np.uint8))\n",
    "plt.axis('off')\n",
    "plt.clim(0, labelList.max())\n",
    "plt.title('trainlabel_to_color')    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 1.,  3.,  4.,  6.,  7.,  8., 11., 17., 20., 21., 24., 26.],\n",
       "       device='cuda:0')"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAV0AAADFCAYAAAD3yiEtAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAANc0lEQVR4nO3df4wcZR3H8c8XQSypVE0xBXOSeAVFSSRoCMgfNoE0VVOQLsZIK3/wB8UjBo1YohI9DVHBSjCaUjSoQU7QcNUGMHWlhkSDGoUYrQHT1ICVWoUgYEVA4PGPmy17e7uzszuzzzzPPO9X0rS9ndud+7Hv+96zs7PmnBMAwI8j6t4BAEgJ0QUAj4guAHhEdAHAI6ILAB4RXQDwiOgCgEdEF32Z2cNmdm6B7ZyZrR7zNsZ+X1+Kfh6AoogugmZm3zWza+reD6AqRBeYADM7su59QJiILnKZ2Rlm9isze9LM/m5m3zCzV/Zs9l4z+4uZPW5mXzGzI7re/xIze9DM/mVmPzWzE0e47UslbZS0xcwOmdmd2dtPMbN7s336k5mdV+C6lpnZV83sETN7ysx+aWbLssvOy67nyex6TxlwHUeb2Q1mdiD7c4OZHZ1dtsbM/mZmV5nZQUnfKfpxIi1EF8O8KOnjklZKOkvSOZJmera5QNI7JZ0u6XxJl0iSmb1f0qclbZB0nKRfSLqt6A07574paU7Sdc655c659WZ2lKQ7JbUlvV7SRyXNmdmbh1zdVknvkPQuSa+TtEXSS2Z2crZPH8v28SeS7uzzg0WSPiPpTEmnSXq7pDMkXd11+arsuk+UdGnRjxNpIbrI5Zy73zn3a+fcC865hyXdJOndPZtd65x7wjn3V0k3SPpQ9vbNkr7knHvQOfeCpC9KOm2UabePMyUtl/Rl59zzzrmfS7qr6zaXyCbvSyRd4Zx71Dn3onPuPufcc5I+KOlu59zPnHP/00Kcl2khzr02SvqCc+6fzrnHJH1e0oe7Ln9J0uecc8855/5b4mNEgxFd5DKzk83sLjM7aGZPayGcK3s229/170cknZD9+0RJX8t+bX9S0hOSTNIbSuzSCZL2O+de6rnNvOtcKelVkvYNuL5HOv/Jrnf/gOtbtK0Wf6yS9Jhz7tncvUfyiC6GuVHSQ5JOcs4dq4XlAuvZZqrr32+UdCD7935Jm51zr+n6s8w5d98It9977tEDkqa6142z23w05zoel/SspOk+lx3Qwg8HSZKZmRY+nn7Xt2hbLf5Y++0rsATRxTCvlvS0pENm9hZJH+mzzSfN7LVmNiXpCkk/yN6+XdKnzOxtkmRmK8zsAyPe/j8kvanr/7+R9B8tPLh2lJmtkbRe0u2DriCbXr8t6XozO8HMXmFmZ2UPgv1Q0vvM7JxsvfgTkp6T1O8Hw22Srjaz48xspaTPSrp1xI8HiSO6GOZKSRdJ+rekb+nloHbbKel+Sb+XdLekmyXJOfcjSddKuj1bmtgj6T0j3v7Nkt6aLVH82Dn3vKTzsut5XNI2SRc75x4q8HH8UdJvtbDMca2kI5xzf5a0SdLXs+tbL2l9dju9rpH0O0l/yK7rgextQGHGK0cAgD9MugDgEdFF7bInJhzq82djHdcDTBLLCwDgUe7zw82MIgPAiJxzvYdVHjb0pByXX9/v0MYw7N1VbN9ardai/8/Pz09idwB02batXXjbmZm1E9wTv9rt/I872jXdosEFELYmBbeIaKMLIGxFYppacKVEo9u73AAAviQZXQB+pDjJDhN9dE9at08nret38igAIRgU3lSDHHV0iS0Qp1SDK0UcXYILxCPlyPaKNroA4jXKMbxNwyuWApiIYWHtvjylSbgRky5PlADCkvIkO0wjogsgHOMEd9u2djKhJroAKpNKOMuIProsLQBhILjFRB9dAM2RQrijji5TLoDYRBtdggsgRlFGl+ACiFV00SW4AGIWzTPSiC2AJohi0iW4AJoi+EmX4ALpmJlZW/qVXUJ/4dlgo0tsgfSk8FJaQUWX0ALx6pwpbJwnOOzefdOi/487rbZarcPhDnXiDSK6xBZIW1WBnJ+fD35arj26BBdolqITb0rn0O1We3QBNFN3VFM9YXk/RLdH6OtBQOhGecWIjpRCHMVxugDQFLVH18er+vabWkNfbAfQTCwvdGm32wNj3O9Xot27b2IZAsBIap90ASAlQUTXxxJDWYMW+tvt5p/pHkB1gohuLMfqzsys1erV0urVi9/ebreJL4BCao3u3l3TwQV38+bNfd/eG9bepy12tgGAPEFMuk1CeAHkGXr0QmcSHWXdNbTp1bd2u621a9M52BvIk9ITH4oofMjY3l3TUTzgBaBeRDbfSMsLRSbY1KdcAMhT6ZouwQWAfCNHt98RByEehTBpg45yAIA8HL0AAB5x7oWCOkcjDDskjKMWAORh0q0QwQUwDNGtEE+MADAM0QUAj4guAHjEA2ldWJMFMGlMugDgEdEFAI9YXhgRSxAAyiC6JRFhAKNgeQEAPCK6AOAR0QUAjyqJbgyvKDE/P1/3LgAAky4A+ER0AcAjogsAHhFdAPCI6AKAR0S3pFar5e22tj3wzOE/qB9fC4yD6Eai987Nnb0+xBZlEF2gJAKMURBdAPCI6AIVYNpFUUQ3Av3u0DOnH1PDnkBa+Nz3+/wTXhTB+XQDxx05THxdlmq31w28bO3aXR73JGxjR3fvrukoTnQDYHx5IcV4mHRLmJ+f93qcbrfOpMUyg39NnnKJ7OSxpguMqKk/6AiuH0y6AWvyRNVU2x54JrooE1u/xp50Wc+dvNjuvFgQyw/Ldnsdwa0BywvAiGKJah5iW5/GRbfVatX24BYQA4Jbr2iiS0wBNEEtD6R14smLRaJpQl+HZ8qtX2WTLg+sAcBwE11eYEmgvEHP8+9owoM6QEqiWdNFf6H/OgtgscJruqzDAoPxww9FJT3pxrT0wZ0aaAbOMhaRmdOPYQ23Ajv23VLq/Q/qwsqv05fl+n7du5A8zr2AysQSHqBOhaNb52kMUzMoXgefWjxhrVpxh3bwy4ZXvV8DhKvVagX5GFQSk24nYnPaOfCybnPaWXhqG2XbKq1acYf32wRQ3tDodgelNzBX6eyB23a2H+XteQq/z3XS3JZ6QojmW7Xijqin3UPTFx3+9/J9rO/WIYlJF5i0g09d2MjfProjLRHqKowd3asuP3v4RhHbOrW90HbH6/jC25axac89i/7v4zYxwNS5S74eUnxfk9mp7O97j1162ZqnJUlbtfhjmp3qv/2g9++4b8x9bCIm3QGu3H9Zoe3mtLPQtrHdIZGO7kAWCSrKaWx057bs1Mbrzh/rfUd9vyK3VTTigxzU4slq0557Cv06S+wxit4JddA2eXEuch0pa2x0+xk3wqFq6jpiDG49dekSw6Y99+jWU8+taY/8Iqzja2R0mxZXqfghYky2qELZ38x6bZhmVbejkdH1qe7AE1n/+j2IFoOqQzqKHftu0Ybpi2u7/ZBEE9264xYaYosi6gxtr+5j51MO8MjRLfMAFcojtvULed02pMjm6X3yUkoR9jLpEuryiC2GiSW4/aQ0BdeyvECAiyG0SFG/p/A3KcTRrOmmhNgCi3VC3IT4mnNu8IVm7gJd4HF3pO9N3+j19kJCbFFGzMsLZfSGuPsUtHWc2rHdbss5Z4MuD+rlelIN7tap7QQXpaX6PbRj3y1RnVVw7OWFVANZhVTvHJi8zvdWilNv3nmzQzI0usS1WgQXPqQc324hrgUPXdM9NH3Q4+40E6FF3VKPb69JRnjYmi7RnSBii9AQ38UmEV+i6xmhRQyI72BlQ0x0PSG2iBHxHWzc+BLdCSK0aAriW0yREBPdCSC2aCriO9yw8BLdihBapIT4FtcbYaJbErFFqgjv6DZMX0x0x0FogZcR39Es37eK6BZFbIHBiG8xRLcAYgsUR3zzEd0+iCxQHvHtj+h2IbZAtQjvUslHl9ACfhDgBcOiG9RJzKtGcAF/uL8V06hJly86EIaUp95kJl2CC4SD++NgUU+6fGGB8KU29TbygTRiC8QlpfA2JrqEFohfCvGNPrrEFmieJsc3yugSWiANTYxvdEcvEFwgHSne34OYdFP8xANYqgmTb9DLC8QWQK/YwxtcdAktgCJijW9Qa7oEF0BRTe3FRCfdpn7SAPgV09Rby/ICsQVQtVjC6zW6xBbApIUeXy/RJbYAfAs1vhOLLqEFEILQ4lt5dIktgBCFEt/KoktsAcSg7viWju7smtlJ7BcQjNl7Zwdfxvd/tOqKb1BPjgBCkxfcIpcjXKH+ds6kiySNGlPuB/HzNfky6QI9xplemXjjF8rky6SLpFQRT+4T8Zvk1MukCwA96px6j6ztlgGgRt3h9XmkA5MuksG6LAbxOfky6QKA/E2+TLpIAlMuQkF0kQSOOMAotk5tP/ynakQXySC8GEfV4WVNFxgB4U5Tleu9TLpAQQQXUvnJl+giKYQTVSiz3kt0kZxxwkus0c844eXcC0ga59JFla7cfxnnXgDy9Avr7JpZgouxFJl8OXoBjVDlkx8m9UQKQg6J6MKzlJ8ZRswhEV1UJOWY1q3I554wh4Po4jDC2VycvD0cRDcxhBXjKvq9Q5zzEd2IEVCEiBf9zEd0A0dY0XSDvsebGmOi6xEBBYpr6sRMdCeAuAL+xTIxE90cxBOIX2gPABLdDIEF0tavAZMI8dAT3lR+iwDQcHknvMmNLgCgWpxlDAA8IroA4BHRBQCPiC4AeER0AcAjogsAHv0fE2uFzEyQ6ecAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "A = label_to_color(labelList.cpu().numpy().squeeze(), id2color)\n",
    "plt.imshow(A.astype(np.uint8))\n",
    "plt.axis('off')\n",
    "plt.clim(0, labelList.max())\n",
    "plt.title('label_to_color')    \n",
    "\n",
    "labelList.unique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "open-set recognition\n",
    "---------\n",
    "\n",
    "GAN for open-set recognition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "selectClassifier_step030_OpenGAN_num1000_w0.20_val.pkl\n",
      "0/29 roc:0.8045542657168512\n",
      "1/29 roc:0.8934519080962876\n",
      "2/29 roc:0.895303637323307\n",
      "3/29 roc:0.8884856603117062\n",
      "4/29 roc:0.8752582770801431\n",
      "5/29 roc:0.8645951547660959\n",
      "6/29 roc:0.9016062609698201\n",
      "7/29 roc:0.8743030888818354\n",
      "8/29 roc:0.9201573332277752\n",
      "9/29 roc:0.9175803657991842\n",
      "10/29 roc:0.9001231791977117\n",
      "11/29 roc:0.9022931083726095\n",
      "12/29 roc:0.9151644697136403\n",
      "13/29 roc:0.8983293563838922\n",
      "14/29 roc:0.9079718413008606\n",
      "15/29 roc:0.9014192120088449\n",
      "16/29 roc:0.8972301902583407\n",
      "17/29 roc:0.9093332890198818\n",
      "18/29 roc:0.8842788285838401\n",
      "19/29 roc:0.8634011060141903\n",
      "20/29 roc:0.9181098164822645\n",
      "21/29 roc:0.9031136943121291\n",
      "22/29 roc:0.9103406352765441\n",
      "23/29 roc:0.9004244134214663\n",
      "24/29 roc:0.9012821125791486\n",
      "25/29 roc:0.905546044493099\n",
      "26/29 roc:0.8820948616764375\n",
      "27/29 roc:0.9095462020058986\n",
      "28/29 roc:0.9149950117974504\n"
     ]
    }
   ],
   "source": [
    "rocList = []\n",
    "filename_to_save_roc = 'selectClassifier_' + project_name + '_val.pkl'\n",
    "print(filename_to_save_roc)\n",
    "num_valimages = 10\n",
    "for bestEpoch_GAN in range(totalEpoch):  \n",
    "    netD = Discriminator(ngpu=ngpu, nc=nc, ndf=ndf).to(device)\n",
    "    path_to_D = os.path.join(save_dir, 'epoch-{}.classifier'.format(bestEpoch_GAN+1))\n",
    "    netD.load_state_dict(torch.load(path_to_D, map_location=lambda storage, loc: storage))\n",
    "    netD.eval();\n",
    "    \n",
    "\n",
    "    predConfidence_closeset = torch.from_numpy(np.array([0])).type(torch.float).view(-1,1)\n",
    "    predConfidence_opensetlist = [torch.from_numpy(np.array([0])).type(torch.float).view(-1,1)]*len(opensetIDlist)\n",
    "    for sample in dataloader_val:    \n",
    "        if count%100==0 and count>0:\n",
    "            print('{}/{}'.format(count, len(dataloader_val)))\n",
    "        count += 1\n",
    "        imageList, labelList = sample\n",
    "        imageList = imageList.to(device)\n",
    "        labelList = labelList.to(device)\n",
    "\n",
    "        labelList = labelList.unsqueeze(1)\n",
    "        labelList = F.interpolate(labelList, scale_factor=0.25, mode='nearest')\n",
    "        labelList = labelList.squeeze()\n",
    "        H, W = labelList.squeeze().shape\n",
    "        trainlabelList = id2trainID_np[labelList.cpu().numpy().reshape(-1,).astype(np.int32)]\n",
    "        trainlabelList = trainlabelList.reshape((1,H,W))\n",
    "        trainlabelList = torch.from_numpy(trainlabelList)\n",
    "\n",
    "        with torch.no_grad():\n",
    "            imageList = imageList.to(device)\n",
    "            logitsTensor = model(imageList).detach().cpu()\n",
    "            featTensor = model.feat4.detach()\n",
    "\n",
    "        allPixList = featTensor.squeeze()\n",
    "        allPixList = allPixList.reshape(allPixList.shape[0], -1).permute(1,0).unsqueeze(-1).unsqueeze(-1).to(device)\n",
    "        with torch.no_grad():\n",
    "            allPixConf = netD(allPixList).view(-1)\n",
    "\n",
    "\n",
    "        ################    close-set pixels    ################\n",
    "        closesetList = trainlabelList.reshape(-1,1)\n",
    "        closesetList = ((closesetList>=0) & (closesetList<=18)).nonzero()\n",
    "        closesetList = closesetList[:,0]    \n",
    "        predConf = allPixConf[closesetList]\n",
    "        predConf = predConf.view(-1,1).detach()    \n",
    "        predConfidence_closeset = torch.cat((predConfidence_closeset, predConf.detach().cpu()), 0)\n",
    "\n",
    "        ################    open-set pixels    ################\n",
    "        for i in range(len(opensetIDlist)):        \n",
    "            clsID = opensetIDlist[i]\n",
    "            tmpMap = (labelList==clsID).type(torch.float32)\n",
    "            tmpCount = tmpMap.sum()    \n",
    "            if tmpCount==0:   \n",
    "                #print('not existing -- ', clsID, tmpCount)\n",
    "                continue\n",
    "\n",
    "            curClsList = labelList.reshape(-1,1)\n",
    "            curClsList = (curClsList==clsID).nonzero()\n",
    "            curClsList = curClsList[:,0]\n",
    "            predConf = allPixConf[curClsList]\n",
    "            predConf = predConf.view(-1,1).detach()\n",
    "            predConfidence_opensetlist[i] = torch.cat((predConfidence_opensetlist[i], predConf.detach().cpu()), 0)              \n",
    "\n",
    "    predConfidence_openset = torch.tensor([])        \n",
    "    for i in range(len(opensetIDlist)):\n",
    "        predConfidence_opensetlist[i] = predConfidence_opensetlist[i][1:].detach().cpu()\n",
    "        if i>=3: # remove \"ego vehicle\", \"out of roi\" and \"rectification border\"\n",
    "            predConfidence_openset = torch.cat((predConfidence_openset, predConfidence_opensetlist[i]),0)\n",
    "        predConfidence_opensetlist[i] = predConfidence_opensetlist[i].numpy()\n",
    "        \n",
    "    predConfidence_closeset = predConfidence_closeset[1:]\n",
    "    predConfidence_closeset = predConfidence_closeset.detach().cpu().numpy()\n",
    "    predConfidence_openset = predConfidence_openset.numpy()\n",
    "    roc_score, roc_to_plot = evaluate_openset(-predConfidence_closeset, -predConfidence_openset)\n",
    "    rocList += [roc_score]\n",
    "    print('{}/{} roc:{}'.format(bestEpoch_GAN, totalEpoch, roc_score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9201573332277752 8\n"
     ]
    }
   ],
   "source": [
    "bestROC = max(rocList)\n",
    "bestModelIdx = np.argmax(np.asarray(rocList))\n",
    "print(bestROC, bestModelIdx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'roc on valset')"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEGCAYAAAB/+QKOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXzcZbX48c/JvjRbszVLk7TpQkvpTkuBAoWCyGW3KKCIyuJVUC9X7xW9qFyuXjfUnwuguLEpiCDYq8gqW7XQpjvdF5I0TZe0yaRJZrKf3x8zk07TSTJJ5ptMkvN+vfJi5jvf+c4zpJkzz3me5zyiqhhjjDHdRQ13A4wxxkQmCxDGGGOCsgBhjDEmKAsQxhhjgrIAYYwxJqiY4W5AuGRlZWlJSclwN8MYY0aUdevWHVXV7GCPjZoAUVJSQllZ2XA3wxhjRhQRqejpMUsxGWOMCcoChDHGmKAsQBhjjAnKAoQxxpigLEAYY4wJytEAISKXishOEdkjIncHebxYRF4Tkc0i8oaIFPqOzxWR1SKy1ffYR5xspzHGmFM5FiBEJBp4APggMBO4QURmdjvtfuAxVZ0N3Ad823fcDXxcVU8HLgX+n4ikO9VWY4wxp3KyB7EI2KOq+1S1FXgKuKrbOTOB13y3X/c/rqq7VHW373Y1cAQIupDDRC5V5emy/WyorMPKyhsz8ji5UK4A2B9wvwpY3O2cTcCHgB8D1wApIpKpqsf8J4jIIiAO2OtgW40D9tY08p/PbAZgZl4qNy4u4up5BYyLHzXrM40Z1ZzsQUiQY92/Rn4JOF9ENgDnAweA9q4LiOQBjwOfVNXOU15A5HYRKRORspqamvC13ITF+0fdANx67iQUuOf591j8rVf56nNbeO9A/fA2zhjTJye/ylUBEwPuFwLVgSf40kfXAojIOOBDqlrvu58K/BW4R1XfCfYCqvow8DDAwoULLYcRYSqONQFw54VT+K9/mcGG/S5+/24lz66r4vfvVjJnYjofXVzEFbPzSYyLHubWGmO6c7IHsRaYKiKTRCQOuB5YGXiCiGSJiL8NXwF+4zseBzyHdwD7jw620Tio4pib1IQY0pPiEBHmF2Vw/3VzWPPV5Xz98pk0tbTzn89sZtH/vsq9K7ey+3DDcDfZOOR7L+7glkfWDnczTD85FiBUtR24E3gJ2A48rapbReQ+EbnSd9oFwE4R2QXkAt/yHf8wcB7wCRHZ6PuZ61RbjTPKjzVRkpV8yvG0pFg+de4kXrnrPP5w+1ksm57D79+t5NIfv82GyrphaKlx2up9x3htxxE27XcNd1NMPzg6WqiqLwAvdDv29YDbzwDPBHneE8ATTrbNOK+y1s0ZBWk9Pi4iLJ6cyeLJmRxpaGbZ99/gD2v3M68oYwhbaYbCQVczAI+uLueHE+273khhK6mNI9o6OjlQ56Ek89QeRDA5KQlcOiuPv245SHNbh8OtM0OpraOTww3NxMVE8ZdNBznW2DLcTTIhsgBhHFHt8tDeqRRlJoX8nGvnF9DQ3M5r24842DIz1A7VN6MKNy8pprWjk6fW7u/7SSYiWIAwjqg45p3iGmoPAuCsyZlMSE3gT+urnGpWr77/kg2kOqHa5QFg6dRszp2SxRPvVNDeccqsdTMAqsqDb+zhuy/ucOT6FiCMI/xTXIv70YOIjhKumpfPm7tqhjwNUe3y8PBb+3htxxG2VR8f0tce7Q7We8cf8tMT+fiSYg7WN/PKtsPD3KqRr62jk7uf3cL3XtxJVZ2Hjs7wz/S3AGEcUXHMTUJsFDkp8f163rXzCmnvVP5vU3XfJ4fRz9/ciyrERUfxx3WWAgmnA74eRH56AhfNyKUgPZFHV5cPa5tGunpPG5/47Rr+ULafz104hR9/ZC7RUcHWJg+OBQjjiPJjborHJyPSv3+00yekcHp+Ks9tOOBQy051qL6Zp9bsZ8WCQpbPzOHPG6tpbbcUSLhUuzxkJMWSFBdDdJRw05Ji3tlXy45D1lMbiP21bj700D9Z834t318xmy9eMp0oB4IDWIAwDqmsbepXeinQNfMK2FRVz54jjWFuVXA/f3MvHarcsWwK1y2YSG1TK3/fYQPl4VLt8pCXlth1/yMLJxIfE8VjqyuGsVUj04bKOq558B8cOd7MY59azHULJ/b9pEGwAGHCrrNTqTjmHnCAuHJuPlECz21wfrD6yPFmnlxTybXzCpg4PomlU7PISYnnGUszhc3B+mby008EiIzkOK6am89z6w9Q724bxpaNLC9sOcj1D79DUlwMz91xDktKMx1/TQsQJuyONLTQ0t5JcT9mMAXKSUlg6dRsnt9QTacDA2+BfvHWPto7lTsvnAJATHQU18wv4PWdNdQ02Hz9cDjg8lCQnnDSsY8vKcHT1mHjPSFQVR56Yy+f/d16ZhWk8dxnz6Y0e9yQvLYFCBN25QOYwdTdtfMLOODysKa8NlzNOkVNQwu/e7eCq+bmnxTMrlswkY5O5fkhHAcZrY43t9HQ3H5SDwJgVkEaC4szePydCse/BIxk/plK331xB1fMyed3ty4mc1z/Jn4MhgUIE3aVA1gD0d0lMyeQHBft6JqIX769j9b2Tu5cNuWk41NyxjGvKJ0/rttvGx0Nkr/ERl63AAFw89klVBxz8+augZXqb+/o5NsvbOft3aOz1H+wmUoJsUNb9dgChAm78mNNxEQJeWkJfZ/cg8S4aD54Rh5/23LIkdIbxxpbeHx1BVfOyWdykO76igWF7DrcyBbbt2JQquu9U1y7p5gALp01gZyUeB75Z3m/r6uq3PP8e/zirX18/skNo658xwGXp2um0v3XzXF0plJvLECYsKuodVOYkUhM9OD+eV07r4CGlnZHFlX98u33aW7v4M4LpwZ9/PLZ+cTHRPHHsuFZ1R1Oa8trWfy/r7K/1j3kr13dtQbi1B5EbHQUH11czJu7anj/aFO/rvuzv+/hqbX7+dD8Qhpb2vmfv2wLS3sjxQ9e3km1y8Ojn1rEigWFw9YOCxAm7CqONQ14gDrQWZMzyUtLCPuaiNqmVh5bXc7ls/OZkhN8sC8tMZYPnD6BP288MKKLB6oq3/rrdg4fb+G17UO/erna5SE6SshJCd6bvGHxRGKjhcdWl4d8zWfWVfGDV3ZxzbwC7r9uNncsm8LzG6t5fefomZq85v1azp+WzdmlWcPaDgsQJqxUBzfFNVBUlHDV3ALe3BXeGUW/XrUPT1sHn7twSq/nXbewkOPN7bw6yA/WX7y5d1g+nAFe2XaYjftdiHj3ZBhq1a5mJqQm9LjKNyclgcvOyOOZsiqaWtqDnhPo7d013P3sZs6Zksl3PzQbEeEzF5QyNWcc9zz3Ho0hXCPSHT7eTFWdhwXFw1/23gKECas6t3fWSjh6EOCdzdQRxtIbLncrj/6zgstm5TEtN6XXc88uzSI/LWFQaaYXthzk23/bwe2Pr+PF9w4O+DoD0dGp/ODlXUzOSubquQW8+37tkM8YqnZ5yA8y/hDo5rNLaGhp50999BS3VR/nM0+sZ0rOOB762ALiYrwfX/Ex0XznQ7Oprvdw/0s7w9b24VJW7t00a2HJ+GFuiQUIE2ZdRfrGD74HATAtN4VZBeErvfGbVe/T2NLO5y7qvfcA3uKB184v5O3dNRzyFZzrj9qmVr72/HvMKkhlTmEan3tyw5D2JFZuOsDOww38+yXTOHdKFi53G9uHuLxFdb0n6PhDoHkT0zmjII3H/lne46yxapeHTz6yhnHxMfz2k2eSmhB70uMLijP4+FnFPLq6nPUjfFfCteW1JMRGcXp+6nA3xQKECa+uMt9Z4QkQANfMK2TLgfpB71ld72njt/8o59LTJ3DahND++FYsKKRT4U8DWNV978qtHG9u4/7r5vDIpxYxIy+VzzyxnrcGOK2zP1rbO/nRK7uZmZfKZbPyulbdvrPPuXUl3XV0Koe6raIORkS4+ewSdh9pZPXeU9Ng/ume7pYOfvvJM08q2xHoPy49jQmpCdz97OYRXUtrXUUdcyemEzvISR7hMPwtMKNKxTE3IlCYEb4AceWcfKKjpM8URF8e+Uc5DSH2HvxKspJZVDKeZ8qq+rUm4uWth1i5qZo7l03ltAmppCbE8tinFlGaM47bHisL+kEYTk+X7aey1s1/fMA7PTI/PZHizCTHXzfQ0cYW2jqU/BCmO18+O4/xyXGnVHltae/gXx9fx76aJn5+0wJm5PUc2MfFx/DNq2ex63AjP39z7yBbH5oDLg9PrqkMW+quqaWdbQePs7B4+NNLYAHChFnFsSbyUhPCuqAnOyWepVOz+POGAwP+Qzze3MavV+3j4pm5nJ7f8z7ZwaxYUMi+o00hpy7q3W3c8/x7zMhL5bPLSruOpyfF8cQtiygan8Qtj66lzKFV4p7WDn7y2m7OLMnggunZXceXTM7k3fePObJvQDC9TXHtLiE2muvPnMgr2w5TVefthaoqX35mM6v3HeN7K2ZzzpS+Z/RcNCOXK+bk87O/72HPkcH1OHujqjy1ppIP/OgtvvKnLazaczQs192030VHp7KgZPgHqMEChAmzilp3v7YZDdU18wqorm/mnfcH9g34sX+Wc7y5nS9cFHzdQ28um51HYmw0z6wLLc1031+2cayple+vmH1KmiBzXDy/u20xE1IT+MRv17Jxv6vf7enLY6vLOdLQwn984LSTyq0vKc2kobl9yDZEqnad2CgoFB89qxiA371bCcD3X9rJ8xur+dIl07h2fuhrAb5xxUyS4qO5+9ktjgzKH6z3cPNv13L3n7YwqyCVuOiosAWIteV1iMD8IgsQZhSqONY0qBIbPblk5gTGxcfw3Pr+p5kaW9r51ar3uei0HGYV9K/3AN7UxWVn5PF/mw7iae19TcTrO4/w7PoqPnN+aY+vlZOSwO9vO4vxyXF8/Nfv8l4YV2sfb27joTf3csH0bBZNOjlNsWSydxxi9b7wfJj1pT89CICC9EQumTmBp9ZU8utV7/PgG3u5YdFE7lgWekoQIGtcPF/7l5mUVdTxuzWV/W53T1SVp8v2c8mP3mLt+7Xcd9Xp/P7Ws1hQnMHbu8Pz/7SsopbpuSmkJcb2ffIQsABhwqaxpZ2jja2O9CAS46L54KwJvLCl7w/p7h5bXY7L3cbnB9B78FuxwLti98WtPU9VPd7cxlf/tIVpueP6HOeYkJbA729bTEpCLDf9+t2wbZ7zq7f24XK38aVLpp/yWE5qApOzk4dsHOKAy0NyXDSpCTEhP+fjZxdT527jf/6yjWXTs/mfq2b1e9Mp8E6PXjo1i+/+bQcHfeU+BuNQfTOfemQt//nMZmbkpfLivy3l40tKiIoSlk7LYvvB44Neq9PRqWyodEXE+gc/CxAmbPxTXJ3oQQBcM7+AptYOXt52KOTnNLW088u39nHB9GzmTEwf8GsvnjSeieMTe00zffuF7Rw+3sz3V8whPqbvMZjCjCR+f9ti4mKi+Niv3h30BklHG1v41ar3+ZfZeT32XpZMzmRteR3tHc7P8jnom+Lanw/4JZMzmTsxnTkT0/nZjfMHXK5FRPjfa86go1O557n3Blx0UVX50/oqLvnRm6zed4xvXDGTp24766R1PkuneMd5/jHINNOOQ8dpbGnnzAhY/+BnAcKEjb+Ka1GY1kB0d9akTPJDLL1xtLGFx1eXc+Ov3qVukL0H8K7qXjF/Iv/ce6xrEDXQqt1HeXLNfm47b3K/AlFxZjK/v+0sQLjxl+9Q3s+aRIEefH0vLe2d/PvF03o8Z0lpJo0t7UNShLDa1fcU1+5EhKduP4vnPnM2yfGh9zyCmTg+iS9eMo3Xdhzhr1v6v0jxyPFmbnusjH9/ehPTclP42xfO45PnTDqlaN7p+alkJMUOOs20rsI7CcJ6ECbi/OiVXawc5Grlcl+ACEeZjWCiooSr5hXw9u6jHGk4deGay93KH9ZW8rFfvcuib73K1/68laaWdu676vSwDPpdO78AVXh23ckBqqmlnS8/u5nJWcnctbznD+eelGaP43e3Lqato5Mbf/kO+2r635M44PLwxDsVrJhf2OtmMmd1jUM4n2byrqLuX4AA74ymcFUu/cTZJcwuTOPelVtxuVv7PF9VOXK8mWfWVXHxj97i7d1HuedfZvCHTy9hUlbwnnFUlHDOlCze3l0zqPLwZeV15KbGU5jR//9nThlciO6DiFwK/BiIBn6lqt/p9ngx8BsgG6gFPqaqVb7Hbgbu8Z36TVV91Mm2jmUdncov3trL9AmpXDknf8DXqaxtIjM5jpQE5wbYrp1XwENv7GXlxmpuXTqZhuY2Xtl2mP/bVM3bu4/S3qkUZybx2QumcPmcPKbnpgwohx3MxPFJnF2ayTPrvfX5/R9i331xB9X1Hp751yUDnt47fUIKT9y6mI/+6l0u+8nbfOmS6XzynEk91jDq7iev7gbg88t77ylljYtnWu44Vu89xmcv6N/gb380t3VwrKk1pDUQToqJjuI7187mip+t4lt/3c73r5tDS3sHVXUeKo+5qax1U+H7b2VtE5W1bprbvOm3+UXpfP+6OSHt3rZ0ahZ/2XyQ3Uca+yzh0pN1FXUsLB4ftn+v4eBYgBCRaOAB4GKgClgrIitVNbAu7/3AY6r6qIhcCHwbuElExgPfABYCCqzzPXdkr6GPUPt9fxRbD9Tjae0gMW5gH3LlR8NTpK83U3NTOKMgjSfeqWBteS2v76yhtb2TgvREbjl3EpfPzmdWQapjf2TXLSzkrj9sYk15LWdNzuSdfcd4bHUFnzpnEgsGubjp9Pw0XvzCefzXc1v45l+388KWg3xvxZweK8767a1p5Jn1Vdy8pISCEL6xL5mcydNlVbS2d3bVMwq3g/X9m+LqpJn5qXz6vMk8+MZeVu05yqHjzQR+0U+MjaY4M4nizGTOm5pNUWYSk7KSObs0K+QAfe5U7zjEW7tqBhQgql0eDrg83Lp0Ur+f6yQnexCLgD2qug9ARJ4CrgICA8RM4C7f7deB5323PwC8oqq1vue+AlwKPOlge8esnb4SFu2dysb9rgFvhl5Z6z5laqUTrltYyNf/vBV3awc3Liriijn5zJuYPiQbqlx6eh5fi9/KH8uqmFOYzpef3UxxZhL/8YFTZw0NxIS0BH5180L+vLGae/9vK5f95G3uWj6N25ZO6nHA9oev7CI+JuqkRXm9WVKayaOrK9hc5XKsIFx/p7g67fMXTe2qp1WUmUTR+CSKM5OYOD6J7HHxg/5CUZCeyOTsZFbtOcqtSyf3+/llvvGHSFlB7edkgCgAAnckrwIWdztnE/AhvGmoa4AUEcns4bkF3V9ARG4HbgcoKioKW8PHml2HTqw4LSuvHVCAaGnvoLre43gPAuBji4tZNGk8U3NSQv6GFy6JcdFcPjuPP2+sJi5GqDjm5snbzhpwrysYEeHqeQWcPSWTrz+/le++uIO/vXeQ76+Yw/QJJ387fe9APX/dfJDPXziFrBD3Kl48KdNb/nvvMccCxAGXfye5yAgQCbHR/PAjcx19jaVTsni6rIqW9o6QZrEFWldeS1JcNDPyBpaecoqTg9TB/nK7j+B8CThfRDYA5wMHgPYQn4uqPqyqC1V1YXZ2dpCnmFDsPNxAYUYi03LHdX2T6a/9tR5UnRugDhQVJZw2IXXIg4PfigWFeNo6eHLNfm46q3jAPa6+5KQk8NDH5vPAjfM5UOfh8p++zU9e201bwBTV+1/eSVpiLLeeF/q31ozkOE6bkOroQLV/L+rctNCC1miwdGo2nraOrtlI/bG23Fugb7C7MIabk62pAiYG3C8ETpomo6rVqnqtqs4D/st3rD6U55rw2XW4gem5KSwsGc/6iroB1erpKvPt0BqISLKgOIPS7GQK0hP58gdPc/S1RIR/mZ3Hy3edx6Wz8vjhK7u48mf/4L0D9ax5v5Y3dtbwmQtKTyl/3ZclkzNZV1FHS7szu+VVuzxkp8T3+5v0SHZWaSYxUcKqfk53bWxpZ8eh4xGx/0N3TgaItcBUEZkkInHA9cDKwBNEJEtE/G34Ct4ZTQAvAZeISIaIZACX+I6ZMGtt72RfTRPTJqSwsDiDhpZ2dg2grLa/zHe49oGIZCLCY7cs5pnPLGHcIOfqhypzXDw/vWEev7hpATUNLVz9wD/4t6c2kJMSz81LSvp9vSWlmbS0d7KhMvy1oCC0fSBGm3HxMcwrSu/3eogNlXV0KiyMoPUPfo4FCFVtB+7E+8G+HXhaVbeKyH0icqXvtAuAnSKyC8gFvuV7bi3wP3iDzFrgPv+AtQmv8mNNtHcq03NTulZwDqTKaMWxJlLiYxifHBfuJkakgvTEHvclcNIHTp/Aq/9+HlfOzae6vpm7Lp42oPGPRZPGE+Ubh3DCAZeHgj52khuNlk7N5r3qeuqa+l5z4VdWXkeUwLyiga/0d4qjCS9VfUFVp6lqqar6P/y/rqorfbefUdWpvnNuVdWWgOf+RlWn+H5+62Q7x7KdvgHqabkpFGYkkpMSz9ry/udQ/VVcI2kO92iVnhTHDz88lzVfvYgbFg1sckZaYiyn56c5Mg6hqhx0NQ9LAB1u507NQhX+sTf0XkRZRS2nTUh1dP3QQEXWiIgZcrsONxAdJUzOTkZEOLNk/IAG2SqOOb8GwpwsJ3Vw39CXlGaysdJFc1t4xyFc7jY8bR1jLsUEMLsgjZSEGN7eFVqAaO/wpvkWRsj+D91ZgAgjVeW9A/X89LXdXPPgP5jz3y/zzhCUNBiMnYcaKMlM6loBvLAkgwMuT9c89lC0d3RSVeceEwPUo8mSyZm0dnQO6AtBb05McR17KaaY6CjOKc1i1Z6jIZXd2HGoAXdrR0TVXwpkAWKQGprbePG9g/znM5tY/L+vcflPV/HDV3fRqZCRFMutj5axpcr5wmgDtetww0lz6/0Ldfoz3fVgfTNtHTomBqhHkzMnjSc6SsI+DhFpi+SG2rlTszjg8rAvhMKL/vG+SJzBBA7XYhqNVJW9NU28vuMIr+88wtryWto6lJSEGM6bls2F03M4f3o2WePiOVjvYcVDq7n5t2t4+tNL+iyZMNQ8rR1U1Lq5et6JNYgz8lJIioumrLw25LpMXTOYrAcxooyLj+GMgvCPQ/jLbIzFMQiA83xlN1btPtpnHae1FXXkpyVEzILC7ixA9ENNQwsfeXg1+2q83wym56Zwy7mTWTY9mwXFGacscslLS+R3ty5mxc9Xc9Ov3+WP/7qEwozI+Za950gjqt734RcTHcW8onTK+jFQXd61BiJy3psJzZLSTH751j6aWtoHXV7br9rlIS4miswxMqOtO38pj7d3H+Xms0t6PE9VWVdex5lDUJ5moCzF1A+7Dzewr6aJO5aVsurLy3jprvO4+4OnsXhyZo8rIEuyknn8lkU0tbRz06/XDHrXqXDy12Ca1q18w8Li8ew4dJzjzW0hXaey1k1cTBQTBjloaobeksmZtHfqgFfQB3PA5SE/LWFIamNFqnOnZvHOvmMnrXrv7oDLw6HjzRG5/sHPAkQ/1Lm9H5hXzMnvV09gRl4qv/3kmRyqb+bjv1lDvSe0D16n7TrcQFxM1CljB2eWjKdTCXkRVfnRJorGJ43pD4SRakFxBjFhHocY6D4Qo8l5U7NobGln4/6e/4b8vfRIHaAGCxD94vJ4F7+kJ/a/67ygeDw/v2kBe440cMsja/u9r7ITdh5qYEr2uFN6P3OL0okSbwGxUFTWuimx9NKIlBwfw5yJ6WEdhzhYPzbXQARaUppFlMDbu2p6PKesopZx8TGcNiGyCvQFsgDRDy5fDyI9aWALWs6fls2Pr5/H+so6/vWJdbS2O78vcG+6z2DyGxcfw8z81JAWzKkqFcfcFI23AeqRasnkTN47UE9DiCnF3rR1dHL4ePOYnOIaKC0xljkT03m7l32qy8rrmFcUeQX6AkVuyyKQy91KQmzUgHcNA7jsjDz+95ozeHNXDXc9vXFAhfHCod7TxsH65h43N1lYPJ6N+1295lDBO3DvaeugJMt6ECPVktJMOjq1XxMTenL4eDOdOnanuAZaOiWLTftdQVPKx5vb2Hm4IaLTS2ABol9c7rYBpZe6u35REV+97DT+uvkg9zy/ZVD72A7Ubt8A9fQJwafhLSzJwNPWwbbq471ex78PdZGtgRixFhRnEBcdFZY0U7UrcnaSG27nTs2mU2F1kLIb6yvqUKWr/lmksgDRDy5P24DTS93dfl4pdywr5ck1+/nOizvCcs3+6JrB1EsPAmBtH+MQ/jLfJbYGYsRKiI1mblF6WAaqD9b7F8mN7RQTeIvvJcdFB63uuq6ijugoYe7EyCvQF8gCRD+43K1hCxAAX7pkOh87q4hfvLmPB9/YE7brhmLXoQaS46J7XKAzIS2BwozEPsswVBxzEx0lFGTYN8aRbMnkTLZW1w96hp2/zMZYH6QGiI2OYklpZtAAUVZex4y8lLCtPXGKBYh+CFeKyU9EuO/KWVw+O4/vvbizK+0zFHYebmDahJReq6+eWTKeteV1vabAKmrdFKQnEhvBA22mb0tKM+lUWPP+4KrqV7s8pCfFRvwH31A5d0oWlbVuKn2pWPAO5G/c74q4/aeDsb/qfghniskvKkr4n6tmkRQXzYNv7A3rtXuiquw81HDSCupgFhRncLSxhcpad4/nVBxrshXUo8C8onTiY6IGnWaqdjWTb72HLkunectuvL3nxHTXbdXH8bR1RGwF10AWIEKkqtS720hPCn/5gIzkOD66uIiVm6pP+qbhlKONrdS523ocf/DzD6D1Nt3VynyPDvEx0Swozhj0QLV3kZyNP/hNzkomPy3hpPLf/lXr1oMYRdytHbR2dIa9B+F329LJREcJD73pfC9iV9cMpt4DxNSccaQmxLCuInjaweVupd7TRrGtgRgVlkzOZPvB4/3aDa07W0V9MhFh6dRs/rn3KO2+KePrKmopSE9kQlrkB1ILECFy+Qbv0hOdCRA5qQl8eGEhz66r4pCvGqZTAneR601UlLCgOKPHHsSJKq7WgxgNlpRmAvDu+wPrRTS2tHO8ud0CRDfnTs3ieHM7mw/Uo6qsLa8bEeklsAARMpfbV2bDgRST36fPK6VDlYff2ufYa4C3BzE+OY6scX2/l3ZMeCAAAB2lSURBVIUl49lzpDHot8oTVVytBzEazC5MJzE2esDjEAfH+D4QPTlnShYi3vLf+2s91DS0ROz+D91ZgAjRYMtshGLi+CSunlvA79dUcKzRuaqvOw83MC13XEj7R/srTQab7lppi+RGlbiYKBaWDHwcwj/FNX8EpE6G0vjkOGblp7Fq91HKfOnaSK7gGsgCRIiGIkAAfHZZKS3tnfx61fuOXF9V2X24sc8ZTH5zJqYTGy2sDTIOUX7MTW5qPIlxAy89YiLL2aVZ7DrcyJHj/U9z2irqnp07NYv1lXW8uauGlPiYPtO7kcICRIj8lVwzHEwxAZRmj+OyWXk8vrrCkbLg1fXNNLa0n7IHRE8SYqM5oyCNdUHGISprmyy9NMosO807LfPvO470+7nVLg/RUUJOSny4mzXiLZ2SRXun8pfNB5lfnEH0CCmNbwEiRP4eRJpDg9SBPruslIaWdh77Z3nYr73LN0Adag8CvOMQm6vqaW47uUR5+TG37UM9ykzPTaEwI5FXtx/u93Or6z1MSE2I6Oqkw2VBSQYJsVF0dOqISS+BBYiQhaOSa6hOz0/jwtNy+M0/3qeppT2s1/bXYJranwBRnEFrRydbDtR3HXO3tlPT0EJJlvUgRhMRYfmMXN7efbTfe5ZUuzzk2fhDUPEx0Sye5J0ltmCEzGACCxAhc7nbHE8vBbpj2RTq3G08uaYyrNfddaiBvLSEfvWE/CWJA8tBV9gA9ai1fEYuLe2drOplL4Ngql3NNv7Qi8tn55GZHBfxBfoCORogRORSEdkpIntE5O4gjxeJyOsiskFENovIZb7jsSLyqIhsEZHtIvIVJ9sZijp325Ckl/wWFGewZHImD7+175TUzmB4ZzD1b4Asc1w8k7OTKQuo7GprIEavRZPGkxIfw6vbQk8zdXYqB+ttkVxvViwopOye5STFjZw6VY4FCBGJBh4APgjMBG4QkZndTrsHeFpV5wHXAw/6jl8HxKvqGcAC4NMiUuJUW0NR7wlvJddQ3HnhFI40tPDMuqqwXK+jU9l9pLHPFdTBnFk8nnWVdXT6Njjyl/m2VdSjT1xMFOdPz+a1HYe7ft99OdrUQluHjvmd5HojIiFNLY8kfQYIEZkUyrEgFgF7VHWfqrYCTwFXdTtHgVTf7TSgOuB4sojEAIlAK9D7zjUOC3cl11CcXZrJ3Inp/PzNvX3u7BaKimNNtLZ3DmiK3YKSDFzuNvbWNHqvVesmPSmWtCEOmmZoXDwzl6ONrWyscoV0vn+Kq5X5Hl1C6UE8G+TYMyE8rwDYH3C/yncs0L3Ax0SkCngB+FzA9ZuAg0AlcL+qnjIRX0RuF5EyESmrqel5c/BwcHnayEge2g9DEeHOZVOoqvOwcmN130/oQ1cNpgEECH/hPn+hMW8VV+s9jFYXTMshOkpCTjNV2yrqUanHACEip4nIh4A0Ebk24OcTQCj9yGB9qe791RuAR1S1ELgMeFxEovD2PjqAfGAS8EURmXzKxVQfVtWFqrowOzs7hCYNjKricreSNsQ9CICLZuRw2oQUHnxjT8jd/Z7sPNSICEzJCb7NaG9KMpPITI7r2mGuwqa4jmppSbEsKhkf8nRXf4DoaQMqMzL11oOYDlwOpANXBPzMB24L4dpVwMSA+4WcSCH53QI8DaCqq/EGnizgRuBFVW1T1SPAP4CFIbymI9ytHbR16JCPQYC3F3HHsinsrWnixa2HBnWtXYcbKBqfNKCVzyLCwpIMysrraG3vpNrlocQGqEe15TNz2XW4MaQS9NWuZpLjoklNHDkDsKZvPQYIVf2zqn4SuFxVPxnw83lV/WcI114LTBWRSSISh3cQemW3cyqBiwBEZAbeAFHjO36heCUDZwFDv3Gzj7+Sa8Yw5dsvOyOPyVnJPPD6nl53d+vLQGYwBTqzZDyVtW7WV9bRqVBkKaZRbfmMHICQehHVLg956YkjbhDW9C6UMYhjIvKaiLwHICKzReSevp6kqu3AncBLwHa8s5W2ish9InKl77QvAreJyCbgSeAT6v0EfAAYB7yHN9D8VlU39/fNhYu/kulwpJgAoqOEf72glK3Vx3lj58DGWlraO3j/aNOAxh/8/Osh/rTeO6vKehCjW3FmMtNyx4UWIGyK66gUSoD4JfAVoA3A90F9fSgXV9UXVHWaqpaq6rd8x76uqit9t7ep6jmqOkdV56rqy77jjap6naqerqozVfX7A3lz4eKviTQcKSa/a+YVUJCeyM8G2IvYV9NER6eGXIMpmNPz00iIjeKFLd5UV5EFiFFv+Yxc3n2/lnp373XBql0em+I6CoUSIJJUdU23Y+Gt/xDh/HWYhnIldXex0VF8+vzJrKuo4519/d9YfjAzmPziYqKYU5hOY0s7SXHRZI+zomyj3fKZuXR0Km/s6rl4X3NbB0cbW20v6lEolABxVERK8c1AEpEVeKefjhl1XZsFDe+c/w8vnEjWuHh++vfd/e5F7DzUQEyUMGmQtZP8012LxidZvnkMmFuYTta4OF7d3nOA8O+AmGcpplEnlABxB/AL4DQROQD8G/AZR1sVYfwppqEstRFMQmw0n72glH/uPdbrH2wwuw43MDk7mbiYwS2e92+VWGID1GNCVJRw0Wm5vLHzCK3twRdrnlgDYSmm0abPTwvfSujlQDZwmqqeq6rljrcsgrjcrSTGRg9JJde+3LSkmGm54/jv/9varxpNg53B5De/OIOYKGFytgWIsWL5zFwamtu71sB0d8DWQIxaoZTa+IKIpAJu4Ecisl5ELnG+aZHD5W4b9vSSX2x0FP995Syq6jw89MbekJ7T1NLO/lrPoMYf/FITYvnDp5dw+3mnrFs0o9S5U7KIj4nilR5WVR/0pZgmWKnvUSeUfMOnVPU4cAmQA3wS+I6jrYowQ13JtS9LSjO5Yk4+D725N6RFTLuPeOsnDWYGU6AFxRmkD+OAvRlaiXHRLJ2axavbDwcd+6p2ecgaF098zPD3sE14hRIg/CORl+Fdj7CJ4GU0Rq3hqOTal/+6bAYxUcJ9f9nW57kD2UXOmEAXzcilqs7TteFUoAM2xXXUCiVArBORl/EGiJdEJAUYfGnREWSoNwsKxYS0BD5/0VRe3X6Y1/vYP3jn4QYSYqOYaLWTzABddJpvVXWQNFO1yxbJjVahBIhbgLuBM1XVDcThTTONGXURNAYR6FPnTGJydjL39jFgvetwA1NzUkbMRukm8uSkJjBnYjqvdJs9p6ocrG+2Mt+jVG/VXOeLyHxgru/QZN/9YmDMVORSVeo9w1PJtS9xMVH895WnU3HMza/e3tfjeTsPhWcGkxnbLp6Rw6b9Lo4cb+46Vu9pw93aYVNcR6nePuh/0MtjClwY5rZEJH8l1+Eq1NeXpVOz+eCsCfzs9T1cPa+AwoyT00gudytHGlqYPqH/Jb6NCbR8Zi73v7yL13Yc4YZFRYBNcR3teqvmuqyXnzERHCByVlH35p7LvTu5fvMv2095bNdh3wwm60GYQZqem0JhRuJJ4xAHfTvJ2RjE6BTSsloRmSUiHxaRj/t/nG5YpPDXYYrEFJNfQXoidy6bwotbD/HWrpOrvfpnnQxkH2pjAokIy2fksmrPUTyt3jGv6npvDyLPUkyjUigL5b4B/NT3swz4HnBlr08aReqHeS+IUN123mRKMpO4d+XWk0oi7DrUQEpCDBNS7Q/YDN7FM3Npae9k1Z6jgDfFFBcdRVayFW4cjULpQazAu6nPId8GQnOAMfOv4USKKXJ7EADxMdF848rT2Xe0iV+ver/r+M7DDUzPTbHCeiYsFk0aT0pCTFeaqdrVTF56AlE2Q25UCiVAeFS1E2j3ldw4AoyZOgv+FFMkj0H4LZuew/IZufz077s5WO9BVdl1uCFsK6iNiY2O4oLpOby24zCdncpBl8fKfI9ioQSIMhFJx7tx0DpgPdB9f4hRK1IquYbqG1fMpL1T+dZft1PT0ILL3WYrqE1YLZ+Rw9HGVjZWuXxbjVr6crTqcz2Dqn7Wd/PnIvIikDqc238OtUiq5BqKieOT+Mz5pfz4td0U+VZO2wwmE04XTMshJkp46b1DHDrebFNcR7FQBqn/LCI3ikiyqpaPpeAAkbuKujefuaCUieMTedBX7XVarq2BMOGTlhTLoknj+eO6KjrVpriOZqGkmH4InAtsE5E/isgKERkzfUpXhFVyDUVCbDRfv/x0ALLGxZNpW4OaMFs+I5faJu8EDgsQo1coGwa96UszTQYeBj6Md6B6TKj3tEZcob5QLJ+RwxVz8lk2PXu4m2JGoeUzcrtu59s+EKNWSDWVRCQRuAL4CDAfeNTJRkWSOncbU3NGXopGRPjpDfOGuxlmlCrKTGJ6bgo7DzfYXtSjWChjEH8AtuOtvfQAUKqqn3O6YZEiknaTMyaSXLewkJl5qYyLHzO1O8ecUH6zvwVuVNXQN0AeJfyVXCN9kZwxw+HWpZO5demYWRI1JoUyzfXFoWhIJGryVXJNH2GD1MYYEw4hFesbKBG5VER2isgeEbk7yONFIvK6iGwQkc0iclnAY7NFZLWIbBWRLcMxc8o1Aiq5GmOMUxxLHopINN4xi4uBKmCtiKxU1cBNlO8BnlbVh0RkJvACUCIiMcATwE2quklEMoE2p9rakxNlNizFZIwZe0KdxVRAt53kVPWtPp62CNijqvt813gKuAoIDBAKpPpupwHVvtuXAJtVdZPvtY6F0s5w85fZsBSTMWYs6jNAiMh38U5v3Qb4B6oV6CtAFAD7A+5XAYu7nXMv8LKIfA5IBpb7jk8DVEReArKBp1T1e0HadjtwO0BRUVFfb6XfRkolV2OMcUIoPYirgemq2tLPawer/6vd7t8APKKqPxCRJcDjIjLL165zgTMBN/CaiKxT1ddOupjqw3gX77Fw4cLu1x40f4op0veCMMYYJ4QySL0PGMgnZBUwMeB+ISdSSH63AE8DqOpqIAHI8j33TVU9qqpuvGMT8wfQhkHxp5hSLcVkjBmDQgkQbmCjiPxCRH7i/wnheWuBqSIySUTigOuBld3OqcS7GREiMgNvgKgBXgJmi0iSb8D6fE4euxgSdU0jq5KrMcaEUygpppWc+sHeJ1VtF5E78X7YRwO/UdWtInIfUKaqK4EvAr8Ukbvwpp8+oaoK1InID/EGGQVeUNW/9rcNg+Xy2CpqY8zYFcpCuUd9PYBpvkM7VTWkKaeq+gLe9FDgsa8H3N4GnNPDc5/AO9V12HjLbNgAtTFmbAplFtMFeIvzleMdeJ4oIjeHMM11xHO5W22KqzFmzAolxfQD4BJV3QkgItOAJ4EFTjYsErg8I7OSqzHGhEMog9Sx/uAAoKq7GNisphHHUkzGmLEslB5EmYj8Gnjcd/+jwDrnmhQZVNWbYrJBamPMGBVKgPgMcAfwebxjEG8BDzrZqEjQ1NpBe6dVcjXGjF2hzGJqwbsv9Q+db07k8FdyHYnbjRpjTDg4Wu57JPOX2UizFJMxZoyyANGDrlLflmIyxoxRFiB64PJYJVdjzNjWZ4AQkVdEJD3gfoavDPeoZpVcjTFjXSg9iCxVdfnvqGodkONckyKDf5DaKrkaY8aqUAJEp4h07cYjIsWcuq/DqONyt1klV2PMmBbKOoj/AlaJyJu+++fh28VtNHN52iy9ZIwZ00JZB/GiiMwHzvIduktVjzrbrOHncreSZgPUxpgxLJQeBMDZeHsOfn9xoC0RxeVusymuxpgxLZRZTN8BvoB3R7dtwBdE5NtON2y4uTxtZCRbgDDGjF2h9CAuA+aqaieAiDwKbAC+4mTDhpvL3UZaoqWYjDFjV6gL5dIDbqc50ZBIYpVcjTEmtB7Et4ENIvI63mqu5zHKew/+Sq42i8kYM5b1GiBERIBVeGcwnYk3QHxZVQ8NQduGjX+RXLqlmIwxY1ivAUJVVUSeV9UFwMohatOws0quxhgT2hjEOyJypuMtiSBWydUYY0Ibg1gGfFpEKoAmvGkmVdXZjrZsGPkruWYkW4rJGDN2hRIgPuh4KyJMnfUgjDEmpFIbFUPRkEhS7xuktjEIY8xY5uiGQSJyqYjsFJE9InJ3kMeLROR1EdkgIptF5LIgjzeKyJecbGd3LncbSXHRxMdYJVdjzNjlWIAQkWjgAbwpqpnADSIys9tp9wBPq+o84HrgwW6P/wj4m1Nt7Emd1WEyxhhHexCLgD2quk9VW4GngKu6naNAqu92GlDtf0BErgb2AVsdbGNQ9R6r5GqMMU4GiAJgf8D9Kt+xQPcCHxORKuAF4HMAIpIMfBn4795eQERuF5EyESmrqakJV7txuW0vCGOMcTJASJBj3XeiuwF4RFUL8RYFfFxEovAGhh+pamNvL6CqD6vqQlVdmJ2dHZZGg7eSq9VhMsaMdaHuBzEQVcDEgPuFBKSQfG4BLgVQ1dUikgBkAYuBFSLyPbyFAjtFpFlVf+Zge7u43K1WydUYM+Y5GSDWAlNFZBJwAO8g9I3dzqkELgIeEZEZQAJQo6pL/SeIyL1A41AFB28lV0sxGWOMYykmVW0H7gReArbjna20VUTuE5Erfad9EbhNRDYBTwKfUNXuaagh5a/kaikmY8xY52QPAlV9Ae/gc+Cxrwfc3gac08c17nWkcT2oa7JKrsYYAw4vlBuJ6j1WydUYY8ACxCn8lVwzbB2EMWaMswDRTZ1/syDrQRhjxjgLEN24PFbJ1RhjwALEKaySqzHGeFmA6MYquRpjjJcFiG6skqsxxnhZgOim3tNKus1gMsYYCxDdudxWqM8YY8ACxCnq3K0WIIwxBgsQp6j3tFmKyRhjsABxEn8lVxukNsYYCxAnaWxpt0quxhjjYwEigL8Ok1VyNcYYCxAn8VdytR6EMcZYgDhJVw/CBqmNMcYCRCCr5GqMMSdYgAjgshSTMcZ0sQARoKuSq01zNcYYCxCB6qySqzHGdLEAEcDlbrOtRo0xxscCRIB6T6ull4wxxscCRIA6q+RqjDFdLEAEcLlbLcVkjDE+FiAC1HvabC9qY4zxcTRAiMilIrJTRPaIyN1BHi8SkddFZIOIbBaRy3zHLxaRdSKyxfffC51sJ1glV2OM6S7GqQuLSDTwAHAxUAWsFZGVqrot4LR7gKdV9SERmQm8AJQAR4ErVLVaRGYBLwEFTrUVrJKrMcZ052QPYhGwR1X3qWor8BRwVbdzFEj13U4DqgFUdYOqVvuObwUSRCTewbZaHSZjjOnGsR4E3m/8+wPuVwGLu51zL/CyiHwOSAaWB7nOh4ANqtriRCP9uiq5WorJGGMAZ3sQEuSYdrt/A/CIqhYClwGPi0hXm0TkdOC7wKeDvoDI7SJSJiJlNTU1g2rsiUJ91oMwxhhwNkBUARMD7hfiSyEFuAV4GkBVVwMJQBaAiBQCzwEfV9W9wV5AVR9W1YWqujA7O3tQjfWnmDJsDMIYYwBnA8RaYKqITBKROOB6YGW3cyqBiwBEZAbeAFEjIunAX4GvqOo/HGxjF38lV5vmaowxXo4FCFVtB+7EOwNpO97ZSltF5D4RudJ32heB20RkE/Ak8AlVVd/zpgBfE5GNvp8cp9oK4GrypZhsu1FjjAGcHaRGVV/AO3U18NjXA25vA84J8rxvAt90sm3duTxtJMdFExdjaweNMQZsJXUXl7vNBqiNMSaABQgfl9squRpjTCALED4uTxsZyRYgjDHGzwKEj8vdagPUxhgTwAKEj1VyNcaYk1mAwCq5GmNMMBYgOFHJ1TYLMsaYEyxAcKLMhqWYjDHmBAsQBJT6thSTMcZ0sQABuDzeMhsZyZZiMsYYPwsQWA/CGGOCsQCBdw0E2BiEMcYEsgBBYA/CUkzGGONnAQKr5GqMMcHYJyLe7UatkqsxxpzMAgRQ724j3cYfjDHmJBYg8KaYLEAYY8zJLEBglVyNMSYYCxB4ZzHZFFdjjDnZmA8QqurdLMgChDHGnGTMB4jGlnY6OtVSTMYY082YDxAdncrls/OYNiFluJtijDERJWa4GzDc0pPi+NmN84e7GcYYE3HGfA/CGGNMcBYgjDHGBGUBwhhjTFCOBggRuVREdorIHhG5O8jjRSLyuohsEJHNInJZwGNf8T1vp4h8wMl2GmOMOZVjg9QiEg08AFwMVAFrRWSlqm4LOO0e4GlVfUhEZgIvACW+29cDpwP5wKsiMk1VO5xqrzHGmJM52YNYBOxR1X2q2go8BVzV7RwFUn2304Bq3+2rgKdUtUVV3wf2+K5njDFmiDgZIAqA/QH3q3zHAt0LfExEqvD2Hj7Xj+ciIreLSJmIlNXU1ISr3cYYY3A2QEiQY9rt/g3AI6paCFwGPC4iUSE+F1V9WFUXqurC7OzsQTfYGGPMCU4ulKsCJgbcL+RECsnvFuBSAFVdLSIJQFaIzz3JunXrjopIxSDamwUcHcTzI5W9r5FntL43e1+RqbinB5wMEGuBqSIyCTiAd9D5xm7nVAIXAY+IyAwgAagBVgK/F5Ef4h2kngqs6e3FVHVQXQgRKVPVhYO5RiSy9zXyjNb3Zu9r5HEsQKhqu4jcCbwERAO/UdWtInIfUKaqK4EvAr8UkbvwppA+oaoKbBWRp4FtQDtwh81gMsaYoeVoLSZVfQHv4HPgsa8H3N4GnNPDc78FfMvJ9hljjOmZraQ+4eHhboBD7H2NPKP1vdn7GmHEm9ExxhhjTmY9CGOMMUFZgDDGGBPUmA8QfRUUHMlEpFxEtojIRhEpG+72DJSI/EZEjojIewHHxovIKyKy2/ffjOFs40D08L7uFZEDvt/ZxsACliOJiEz0FeLcLiJbReQLvuMj+vfWy/saFb+37sb0GISvoOAuAgoKAjd0Kyg4YolIObBQVUfyIh5E5DygEXhMVWf5jn0PqFXV7/gCe4aqfnk429lfPbyve4FGVb1/ONs2WCKSB+Sp6noRSQHWAVcDn2AE/956eV8fZhT83rob6z2IUAoKmmGmqm8Btd0OXwU86rv9KN4/0hGlh/c1KqjqQVVd77vdAGzHW09tRP/eenlfo9JYDxAhFQUcwRR4WUTWicjtw92YMMtV1YPg/aMFcoa5PeF0p29/lN+MtBRMMCJSAswD3mUU/d66vS8YZb83sAARUlHAEewcVZ0PfBC4w5fSMJHtIaAUmAscBH4wvM0ZHBEZBzwL/JuqHh/u9oRLkPc1qn5vfmM9QPS7KOBIoqrVvv8eAZ5jdO2pcdiXD/bnhY8Mc3vCQlUPq2qHqnYCv2QE/85EJBbvh+jvVPVPvsMj/vcW7H2Npt9boLEeILoKCopIHN6CgiuHuU1hISLJvkE0RCQZuAR4r/dnjSgrgZt9t28G/jyMbQkb/4enzzWM0N+ZiAjwa2C7qv4w4KER/Xvr6X2Nlt9bd2N6FhOAbzra/+NEQcFRUf9JRCbj7TWAt+bW70fqexORJ4EL8JZVPgx8A3geeBoowlsV+DpVHVEDvj28rwvwpikUKAc+7c/ZjyQici7wNrAF6PQd/irefP2I/b318r5uYBT83rob8wHCGGNMcGM9xWSMMaYHFiCMMcYEZQHCGGNMUBYgjDHGBGUBwhhjTFAWIIyJACJygYj8ZbjbYUwgCxDGGGOCsgBhTD+IyMdEZI2v5v8vRCRaRBpF5Acisl5EXhORbN+5c0XkHV8Bt+f8BdxEZIqIvCoim3zPKfVdfpyIPCMiO0Tkd75Vu8YMGwsQxoRIRGYAH8FbBHEu0AF8FEgG1vsKI76Jd0U0wGPAl1V1Nt6Vt/7jvwMeUNU5wNl4i7uBtzLovwEzgcnAOY6/KWN6ETPcDTBmBLkIWACs9X25T8RbbK4T+IPvnCeAP4lIGpCuqm/6jj8K/NFXH6tAVZ8DUNVmAN/11qhqle/+RqAEWOX82zImOAsQxoROgEdV9SsnHRT5Wrfzeqtf01vaqCXgdgf292mGmaWYjAnda8AKEcmBrv2Vi/H+Ha3wnXMjsEpV64E6EVnqO34T8KZv74AqEbnad414EUka0ndhTIjsG4oxIVLVbSJyD95d+qKANuAOoAk4XUTWAfV4xynAW876574AsA/4pO/4TcAvROQ+3zWuG8K3YUzIrJqrMYMkIo2qOm6422FMuFmKyRhjTFDWgzDGGBOU9SCMMcYEZQHCGGNMUBYgjDHGBGUBwhhjTFAWIIwxxgT1/wEml8L+sRJ/OAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(list(range(len(rocList))), rocList)\n",
    "plt.xlabel('epoch')\n",
    "plt.ylabel('roc on valset')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\t100/500\n",
      "\t200/500\n",
      "\t300/500\n",
      "\t400/500\n",
      "\t500/500\n",
      "0.9503467670459994\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3de5wcVZn/8c8z90lmEpJMEhISkgAJEJBwCQmIK4OgArqgiAIiygpm0UXW66qLqyy6v5/ioi6KYlZZLqtcdFGiG0VEBhAJJNwCCQSSEEhCyP0y18x097N/VHXo6fTM9EymutNT3/fr1a+uW1c9pzOpp+ucqnPM3RERkfgqK3YAIiJSXEoEIiIxp0QgIhJzSgQiIjGnRCAiEnNKBCIiMadEICISc0oEUjBmtsbM2s2sxczeMLNbzKwua5u3mtmfzazZzHaa2W/NbGbWNiPM7Ptm9lq4r5XhfENhSzQwFvi2mW0NX9eZmfWy7dVhWXeZ2Z1mNiJj/S1m1hl+D+lXecb6083sRTNrM7MHzWxKxrrrzGxtuN9XzezqrGO7mbVm7PenUXwfUnxKBFJof+vudcCxwHHAV9IrzOxk4I/AvcBEYBrwLPComR0SblMFPAAcBZwJjADeCmwF5hSuGHtirhjAx+YB7wNmAccA7wX+vodtPwpcApxC8J3UAj/I2uY6d6/LeCXD2BqAe4B/AUYDS4C7Mj73M+AId09/hx82s/Oy9j0rY7+XD6CsUgKUCKQo3P0N4D6ChJB2HXCbu/+Huze7+zZ3/yqwCLgm3OajwMHA+919ubun3H2Tu3/D3RdmHyf8Rf09M9sUXmEsNbOjw3W1ZnZ9+Gt4p5n9xcxqw3XnmNkyM9thZk1mdmTGPteY2ZfMbCnQamYVZjbRzP7HzDab2StmdlUvxf8YcL27r3P39cD1wKU9bPu3wM/cfa27twDfBi4ws2G9f8MAnAcsc/dfuntH+B3OMrMjANx9hbu3ZmyfAg7LY78yxCgRSFGY2STgLGBlOD+M4FfpL3NsfjfwznD6DOAP4UkxH+8C3g7MAA4ALiC4egD4d+CE8LijgX8CUmY2A7gD+AwwFlgI/Da8Gkm7CHhPuM8U8FuCq5eDgNOBz5jZu3uI6ahw27Rnw2W5WPjKnK8Gpmcs+5SZbTOzJ83sAz0dJzzpr8o8lpl92cxagHXAcOAXWcd/OKzGu8fMpvYQo5Q4JQIptN+YWTOwFtgEfD1cPprg73FDjs9sANL1/2N62KYnXUA9cARg7v6Cu28wszLg48A/uvt6d0+6+1/dfTdBsvhfd7/f3bsIEkYtQcJIuyH8ld4OnAiMdfdr3b3T3VcD/wlc2ENMdcDOjPmdQF0P7QS/By43s6lmNhL4Urg8fUVwA0FSGEdQBXSLmZ3Sw3HSx6pPz7j7t8L544Hbs7Y/FZgafnevA78bYFWY7OeUCKTQ3ufu9UAjwQkmfYLfTvDLekKOz0wAtoTTW3vYJid3/zPwQ+BGYKOZzQ8bWxuAGoJfyNkmAq9m7CNFkLgOythmbcb0FGBiWI20w8x2AP8MjO8hrBaCto20EUCL5+4B8maCq5MmYBnwYLh8XRjbU+6+1d0TYdXYzwmqhHIdJ32s5swFHngaaAf+NWP5w2Fi2wH8I0GbzZHIkKNEIEXh7g8BtxD82k5XWzwGfDDH5h8iaCAG+BPwbjMb3o9j3eDuJxBUicwAvkiQWDqAQ3N85HWCkzsQtDMAk4H1mbvNmF4LvOLuB2S86t397B5CWkbQUJw2K1yWK/aUu3/d3ae6+6Rwu/VZsXT7CG9WJXU7TvidHdrTsYAKcn8fufYtQ4m766VXQV7AGuCMjPmxQCtwbDj/tnD+KoLqilHAN4EdwPRwm2pgMfAHgiuKMoLqon8Gzs5xzBOBuUAlQR34H4BrwnU3EiSYiUA5cHK4/8PDOE4PP/cFYDVQ1UM5yoEnCaptasP5o4ETe/gergBeILjCmEhwYr6ih21HE5ycDZgJPA/My1h/PkEVUBlBe0gz0Jjx/e4EPkBw9fNtYFG4rozgTqVR4b7nEFS5XRWuP4qgIb883P/3gRVAZbH/jvQa/FfRA9ArPq/sE2i47MfA/2TMv42gGqQF2AX8L3B01mdGhiemteF2q4DvAmNyHPN0YGm43RaCqpO6cF1tuJ/14QnzYaA2XPd+YHm4/CHgqD7KMZGgCucNgmquRdnbZGxrBHdIbQtf1xG0X6TXtwB/E07PCE/AbQTVVZ/L2tcjYYy7CBqGL8xafwbwIkG1TxMwNVxeRpAUt4XHe4kgmVq4/h3hcVsJ2nJ+Q5iM9Rp6r/Q/uoiIxJTaCEREYk6JQEQk5pQIRERiTolARCTmSu4pwYaGBp86deqAPtva2srw4Xnffj4kqMzxoDLHw76U+cknn9zi7mNzrSu5RDB16lSWLFkyoM82NTXR2Ng4uAHt51TmeFCZ42Ffymxmr/a0TlVDIiIxp0QgIhJzSgQiIjGnRCAiEnNKBCIiMRdZIjCzm8PhAZ/vYb2Z2Q3hwONLzez4qGIREZGeRXlFcAvB4OI9OYtgZKXpBIN5/zjCWEREpAeRPUfg7g/3McbpuQQDlTuwyMwOMLMJ7t6fYQhFRCLn7iRTTiJ8JZNOIpXaaz69zZvvKRLJYD4Z7sOdPfOpjOUpd1KpYF1Xel/hfruSzu5EigPakjRGUL5iPlB2EN2H+1sXLtsrEZjZPIKrBsaPH09TU9OADtjS0jLgz5YqlTkehkqZU+4kUtCVgq6kB+8p6EqF08lgOpGC5rYO/rL+frqShJ95c/tEivAkHcwn3fdannRIpiDhWcv2nKjf3Ca5n/TWf8GhHsm/czETQa4h73J+3e4+H5gPMHv2bB/ok3V6EjEeVOZ94+7hL9AkHV0pOrqSe6a7L+v7fXf4mfSyzkSKjkSK3eH07kS4viuY7kym+hGpAZ17LS0zqK4op6qiLHiVB++V5UZVVTA/vDycLy+jcs/LqEi/l5VRUW5UlpdRXmZUlgXrKsqNijKjvCzYrrys5/ngvfurzIwyY8905vLgneA44X7ScZSXBbE+/PBDkfxtFzMRrCMYBzZtEsFYsSKSIZFM0d6VpL0rSUdnio5EkvbOJB1dyT0n1ac2JNi0ZC27M07EPZ2w05/Z8571md2JJKl9+AVcVV5GdWUZ1RVlVFeUB++VwYm5uqKMkbWVVNdX71lfU1m258RdXVFGTWXwmcz59HRVer68jKVPP8nbTj6J6sqyPcesKi+jolw3Q/ZXMRPBAuBKM7uTYEzZnWofkFLVmUjRujtBy+4ErZ0JWncnaQvf27sStHUGJ++2Pa9EMN8VLE9Pd3QmM076wXsi37Pys0u7zZYZe06qNZXle6bTJ+b0CbmmspyajBNwsO2bJ+DM5Zn7qqkso6YiY7/hCb28rDDj2295uYyDxwwryLGGusgSgZndATQCDWa2Dvg6wUDguPtNwELgbGAlwXisfxdVLCJ96ehKsqOtix3tnWxv7WJnexe72sP3jmC6uSPBro4EzR1dtOxO0NwRnPhbOhL9qtKoKi+jtqqcYVXlb75XljOipoIDR1RTu+dE++a6mspyavZMByfg2qo3f00vffpJ3n7KScEv8HB9ZblhVpiTspS2KO8auqiP9Q78Q1THl/hKubOttZOtLbvZ2trJ1pZOtrUG09tbO9nW1hW8t3ayva2THW1dtHcle9yfGdRXV1BfU8mI2krqqys4cEQN08dVMDxcXlddzvDqYH54VQXDw/na8GQ+vLoiOOlXlkdSdbHl5TImjdKvYxmYkuuGWuKpM5FiW2snW1p2s7llN1uag/etLcEJf0tLsG5rmABS992fcz8jaioYU1fNqGGVTBhZw5ETRjBqWCWjhldxwLBKDqgN3kfWhq9hldRVVVBWoOoOkWJQIpCi6Uqm2BqewDeHJ/bNzcFrS0v61cnm5t3sbO/KuY/aynIa6qsYM7yaSaOGcezkA2jZ+gbHz5zOmLpgeUN9FaOHVzFqWBWVakgU2YsSgQwqd2dXe4KNzR1s2rWbzS3B+6bmvU/y29tyn9zrqitoqKtibH0108fVcfIhY2ioC07oDXXVNNRVMzacH1a1959wU9M2Gt82LeqiigwZSgSSt7bOBK/v6GDjrvRrNxt3dbCpOXN6N52JvRtOayvLGVtfzdj6ag4dW8fcQ0YHJ/T66j3vY8OTfG1VeRFKJxJfSgSyRyKZYv2OdlZvaeXVLa28tq2dddvbWLe9nfU72nNWz9RVVzBuRDUHjqhh9pRRjB9Rw9j6asaNqGFcfXXwGlFDXbX+1ET2V/rfGUPuztpt7SzfsJMXNjTz8qZmXt7YwpqtrXRlPEtfW1nOpFG1TB49jOOnHMDEA2qZMLKGA0fUcuDI4EQ/XCd4kZKn/8UxsGFnO8+8toNn1u7g2XU7WLZ+F827E0Bwa+SU0cM4bFw97zhyHIc21DFt7HCmjBnG2Lpq3YcuEgNKBENMKuW8+EYzi9ds44k123jspTa2/eHPQPAg05ET6jn3uIkcNXEkMyeMYMb4etXJi8ScEkGJc3dWb2nlkZc289jqrTz+yjZ2hHfjTBxZw/QDyjjzxMM5dvIBzJw4guoKnfRFpDslghLU3pnkr6u28OCKTTSt2My67e0ATB5dy7tmjmfutDHMPWQ0k0YNC3qlPEW3UopIz5QISkRzRxcPvLCJ3z+/gYde2kxHV4phVeW89dAG/v7UQzl1+lh1wCUiA6JEsB/r6Ery22df508vbOTBFZvpTKQYP6KaD54wmXfOHM/cQ0arqkdE9pkSwX5oxRvN3PHEa/z66fXsbO+iuqKMD885mPceM4HjDx6lfm9EZFApEewnupIp/vD8G9z61zUseXU7VeVlvPvoA/ngCZM45bCGgvXxLiLxo0RQZC27E9zx+Gvc/OgrbNjZwZQxw7j67CP5wAmTGD28qtjhiUgMKBEUSXNHF//16Bp+9pdX2NnexUmHjOYb5x7NO44Yp6ofESkoJYIC6+hKcttja/hR0yp2tHVxxpHj+IfTDuO4g0cVOzQRiSklggJxdxY+9wb/b+ELrN/RzttnjOWL7zqct0waWezQRCTmlAgKYOWmFr527/P8ddVWjpwwguvOP4ZTDmsodlgiIoASQaQ6Eyl+3LSKGx9cSU1lGd849yg+PHeK7gASkf2KEkFEXtrYzGfufIblG3ZxzqyJ/Mt7ZzK2vrrYYYmI7EWJYJC5O//9+Gt883fLqa+pYP4lJ/Cuow4sdlgiIj1SIhhEbZ0JvnLPc9z7zOucOmMs139oFg11ugoQkf2bEsEgeX1HO5ffuoQX3tjFF941g081HqbnAUSkJCgRDIKl63Zw2a1L6OhMcvPHTuS0I8YVOyQRkbwpEeyjR1du4RO3LWHUsCp+8am5TB9fX+yQRET6RYlgH/z5xY1c8d9PMW3McG6/bA7jRtQUOyQRkX5TIhigB1ds4orbn+LwA+u57eNzGKUO4kSkRCkRDMATr2zjitufZPr4Ov77srmMHFZZ7JBERAasrNgBlJpVm1v4xG1LOGhULbd9fI6SgIiUPCWCftjZ1sXlty6hosy49e/mMEbPCIjIEBBpIjCzM81shZmtNLMv51h/sJk9aGZPm9lSMzs7ynj2RSrlfPbuZ1i3vY2bLjmByaM1ULyIDA2RJQIzKwduBM4CZgIXmdnMrM2+Ctzt7scBFwI/iiqefXXTw6v484ub+Jf3zuTEqaOLHY6IyKCJ8opgDrDS3Ve7eydwJ3Bu1jYOjAinRwKvRxjPgC1dt4Pv/vEl3nPMBC45aUqxwxERGVTm7tHs2Ox84Ex3vzycvwSY6+5XZmwzAfgjMAoYDpzh7k/m2Nc8YB7A+PHjT7jzzjsHFFNLSwt1dXX9+kxn0rnmr+20J+Cbb6tleGVpdRsxkDKXOpU5HlTm/jnttNOedPfZudZFeftorjNmdta5CLjF3a83s5OB283saHdPdfuQ+3xgPsDs2bO9sbFxQAE1NTXR389+574Xeb11Fbd9fA5vnzF2QMctpoGUudSpzPGgMg+eKKuG1gGTM+YnsXfVz2XA3QDu/hhQA+w3Q3e9tLGZnzy0mg8cP6kkk4CISD6iTASLgelmNs3MqggagxdkbfMacDqAmR1JkAg2RxhT3tydr9+7jLqaCq5+z5HFDkdEJDKRJQJ3TwBXAvcBLxDcHbTMzK41s3PCzT4PfMLMngXuAC71qBot+ulPL2zisdVb+fw7ZzBa3UeIyBAWaRcT7r4QWJi17GsZ08uBU6KMYSCSKeeaBcs4ZOxwLppzcLHDERGJlJ4szuHeZ9azfkc7n3/n4VSU6ysSkaFNZ7ksyZRzwwMvc8SB9Zx1tMYaFpGhT4kgy4Jn17NmaxtXvkNDTYpIPCgRZPn+n16moa6aM4/S1YCIxIMSQYbHVm3l1a1tfHjuwWobEJHY0Nkuwy+fXAvAvLcfUuRIREQKR4kgtLO9i3ueWs/Fcw+mrloDt4lIfCgRhH63NOj94pxZE4sciYhIYSkRhH791Hqmj6tjzjSNNSAi8ZJXIjCzKjM7LOpgimX9jnaWvLqdc4+diJluGRWReOkzEZjZe4DngPvD+WPN7NdRB1ZIv39uAwDvPUbVQiISP/lcEVwLzAV2ALj7M8CQujq4c/FajjiwnqkNw4sdiohIweWTCLrcfUfWsv2ih9DBsL21k5WbWtQ2ICKxlc99ki+Y2YeAMjObBvwjsCjasArnvmVvAHDW0ROKHImISHHkc0VwJXACkALuAToIksGQ8OSr2wF0RSAisZXPFcG73f1LwJfSC8zsPIKkUPIWr9nGO2eOp1wdzIlITOVzRfDVHMuuHuxAimHjrg7WbG1jzlRdDYhIfPV4RWBm7wbOBA4ys+9mrBpBUE1U8p54ZRugaiERibfeqoY2Ac8TtAksy1jeDHw5yqAKZfGabQyrKueoiSOKHYqISNH0mAjc/WngaTP7ubt3FDCmgnnilW2cMGWUupwWkVjLp7H4IDP7N2AmUJNe6O4zIouqAHZ1dLFiYzNnv0W3jYpIvOXzU/gW4L8AA84C7gbujDCmgnh+/U7cYdbkA4odiohIUeWTCIa5+30A7r7K3b8KnBZtWNFbtn4XAEerfUBEYi6fqqHdFnTJucrMrgDWA+OiDSt6yzfs4sARNYypqy52KCIiRZVPIvgsUAdcBfwbMBL4eJRBFcKLbzRzxIT6YochIlJ0fSYCd388nGwGLgEws0lRBhW1RDLFqk0tvH16Q7FDEREpul7bCMzsRDN7n5k1hPNHmdltlHinc69ta6MzmeKwcXXFDkVEpOh6TARm9v+BnwMXA38ws6uBB4FngZK+dXTV5lYAJQIREXqvGjoXmOXu7WY2Gng9nF9RmNCis3pzCwCHNCgRiIj0VjXU4e7tAO6+DXhxKCQBgDVbWxk9vIqRwyqLHYqISNH1dkVwiJmlu5o2YGrGPO5+Xl87N7Mzgf8AyoGfuvu3cmzzIeAaglHPnnX3D+cf/sC8urWNKWOGRX0YEZGS0Fsi+EDW/A/7s2MzKwduBN4JrAMWm9kCd1+esc104CvAKe6+3cwK8nzC2u1tHH/wqEIcSkRkv9dbp3MP7OO+5wAr3X01gJndSdDusDxjm08AN7r79vCYm/bxmH1KJFO8vqODc2bVRn0oEZGSkM8DZQN1ELA2Y34dMDdrmxkAZvYoQfXRNe7+h+wdmdk8YB7A+PHjaWpqGlBALS0t3PvHJpIpp3XTWpqa3hjQfkpJS0vLgL+vUqUyx4PKPHiiTAS5xn70HMefDjQCk4BHzOxod9/R7UPu84H5ALNnz/bGxsYBBdTU1MSBU4+Bhx6jcc4sGg8v+Z4y+tTU1MRAv69SpTLHg8o8ePLuiN/M+tspzzpgcsb8JIJbULO3udfdu9z9FWAFQWKIzOs7g6EVJh6gqiEREcgjEZjZHDN7Dng5nJ9lZj/IY9+LgelmNs3MqoALgQVZ2/yGsCfT8OnlGcDqfsTfbxvDRHDgyJo+thQRiYd8rghuAN4LbAVw92fJoxtqd08AVwL3AS8Ad7v7MjO71szOCTe7D9hqZssJnlr+ortv7X8x8rdxVwe1leXUV0dZKyYiUjryORuWufurQU/UeyTz2bm7LwQWZi37Wsa0A58LXwWxsXk340ZUk1UeEZHYyicRrDWzOYCHzwZ8Gngp2rCis6V5N2M1BoGIyB75VA19kuAX+8HARuCkcFlJ2tyymwYlAhGRPfK5Iki4+4WRR1IgW1t2M3fa6GKHISKy38jnimCxmS00s4+ZWUkP6ZVMOdvbujQ8pYhIhj4TgbsfCnwTOAF4zsx+Y2YleYXQmggnPPu5NhGR+MrrgTJ3/6u7XwUcD+wiGLCm5LR0BgngUA1IIyKyRz4PlNWZ2cVm9lvgCWAz8NbII4tAa1eQCEYNqypyJCIi+498GoufB34LXOfuj0QcT6TSiWBkrQakERFJyycRHOLuqcgjKQAlAhGRvfWYCMzsenf/PPA/ZrZX62o+I5Ttb9rCxuL6GnUvISKS1tsZ8a7wvV8jk+3P2hNBPquv0RWBiEhabyOUPRFOHunu3ZKBmV0J7OsIZgXXnoCqijKqKvLufVtEZMjL54z48RzLLhvsQAqhI+HqdVREJEtvbQQXEIwhMM3M7slYVQ/syP2p/VtHwhmuRCAi0k1vZ8UnCMYgmATcmLG8GXg6yqCi0pFEiUBEJEtvbQSvAK8AfypcONHqSDjDasuLHYaIyH6lt6qhh9z9VDPbTvdB541gTJmS68KzMwljKpUIREQy9VZPkh6OsqEQgRRCZwpqq5QIREQy9XjXUMbTxJOBcndPAicDfw8ML0Bsg64z6dToikBEpJt8bh/9DcEwlYcCtwFHAr+INKqIdCahtlLPEIiIZMrnrJhy9y7gPOD77v5p4KBow4pGV0pXBCIi2fJJBAkz+yBwCfC7cFlJ9tHQlYJqPVUsItJNvk8Wn0bQDfVqM5sG3BFtWNEIEoGuCEREMvX5dJW7P29mVwGHmdkRwEp3/7foQxtciWSKlKN+hkREsvSZCMzsb4DbgfUEzxAcaGaXuPujUQc3mLqSwaMQSgQiIt3l09/C94Cz3X05gJkdSZAYZkcZ2GDrTAR3w1aWKxGIiGTK56xYlU4CAO7+AlByg/52JoNEoCsCEZHu8rkieMrMfkJwFQBwMSXY6VxXmAgqy6zIkYiI7F/ySQRXAFcB/0TQRvAw8IMog4pCImwjqFDVkIhIN70mAjN7C3Ao8Gt3v64wIUWjK5VuI9AVgYhIph5/HpvZPxN0L3ExcL+Z5RqprGQkU8EVQbmqhkREuumtnuRi4Bh3/yBwIvDJ/u7czM40sxVmttLMvtzLduebmZtZZHcipdsIKspUNSQikqm3s+Jud28FcPfNfWy7FzMrJxjZ7CxgJnCRmc3MsV09QRvE4/3Zf3+lrwhUNSQi0l1vbQSHZIxVbMChmWMXu/t5fex7DsFTyKsBzOxO4FxgedZ23wCuA77Qn8D7K/1AmaqGRES66y0RfCBr/of93PdBwNqM+XXA3MwNzOw4YLK7/87MekwEZjYPmAcwfvx4mpqa+hkKrNiWBGDZc8/Bhvj0N9TS0jKg76uUqczxoDIPnt7GLH5gH/ed66f3niEvzayM4KnlS/vakbvPB+YDzJ492xsbG/sdTM3qrfDEIo47bhZvPXTIDLrWp6amJgbyfZUylTkeVObBE2XL6TqC0c3SJgGvZ8zXA0cDTWa2BjgJWBBVg3EqbCMoM1UNiYhkijIRLAamm9k0M6sCLgQWpFe6+053b3D3qe4+FVgEnOPuS6IIJulqIxARySXvRGBm1f3ZsbsngCuB+4AXgLvdfZmZXWtm5/QvzH0XXhDoikBEJEs+3VDPAX4GjAQONrNZwOXhkJW9cveFwMKsZV/rYdvGfAIeqDerhqI8iohI6cnniuAG4L3AVgB3f5ZgxLKSoieLRURyyycRlLn7q1nLklEEE6WUq7FYRCSXfHofXRtWD3n4tPCngZeiDWvwKRGIiOSWzxXBJ4HPAQcDGwlu8+x3v0PFFnY1pKohEZEs+Qxev4ng1s+Sltpz+2iRAxER2c/kc9fQf5LxRHCau8+LJKKIpBOBqWpIRKSbfNoI/pQxXQO8n+59CJUUpQERke7yqRq6K3PezG4H7o8sIhERKaiB1JhPA6YMdiCFoqohEZHu8mkj2M6bbQRlwDagx9HG9le+VyuHiIhA34PXGzALWB8uSrnrlCoiMpT0WjUUnvR/7e7J8FXySUAVQyIi3eXTRvCEmR0feSQR873vgBUREXqpGjKzirAr6bcBnzCzVUArwY9qd/eSTA5qKxYR6a63NoIngOOB9xUolkiVfqWWiEg0eksEBuDuqwoUS0GYWglERLrpLRGMNbPP9bTS3b8bQTyR0RWBiEhuvSWCcqAO3WgjIjKk9ZYINrj7tQWLpEDUWCwi0l1vt48OqVOmaoZERHLrLRGcXrAoRESkaHpMBO6+rZCBRG0IPBQtIhIJjdclIhJzsUsEaiwWEekuNolAFUMiIrnFJhGkaWAaEZHu4pMIdEkgIpJTfBKBiIjkFJtEkB6PQBVDIiLdxSYRiIhIbpEmAjM708xWmNlKM9trwHsz+5yZLTezpWb2gJlNiTKe4JhRH0FEpLRElgjMrBy4ETgLmAlcZGYzszZ7Gpjt7scAvwKuiyoePVgsIpJblFcEc4CV7r7a3TuBO4FzMzdw9wfdvS2cXQRMijAeERHJobduqPfVQcDajPl1wNxetr8M+H2uFWY2D5gHMH78eJqamvodzItruwBY9NhjjKqJT9NIS0vLgL6vUqYyx4PKPHiiTAS5auNzVtCY2UeA2cCpuda7+3xgPsDs2bO9sbGx38FseOI1WPYcJ5/8Vg4cWdPvz5eqpqYmBvJ9lTKVOR5U5sETZSJYB0zOmJ8EvJ69kZmdAVwNnOruu6MKJt1GoMZiEZHuoqwjWQxMN7NpZlYFXAgsyNzAzI4DfgKc4+6bIoxFRER6EFkicPcEcCVwH/ACcLe7LzOza83snHCz7xCMi/xLM3vGzBb0sLtBowsCEZHuoqwawt0XAguzln0tY/qMKI/f7bjqbEhEJKf43D4jIiI5xSYR7HmgTA76pegAAAoSSURBVHVDIiLdxCYRiIhIbrFJBG9eEOiSQEQkU2wSgYiI5KZEICISc/FJBGFrsZ4sFhHpLj6JQEREcopNItDdoyIiucUmEYiISG5KBCIiMRebRPBmN9SqHBIRyRSbRCAiIrnFJhF4+vbRIschIrK/iU0iEBGR3GKTCDQagYhIbrFJBGlqKxYR6S52iUBERLqLTSLYc/uomotFRLqJTSIQEZHcYpMI9jQW64JARKSb2CQCERHJLTaJwF03kIqI5BKbRJCm20dFRLqLXSIQEZHuYpcIdEEgItJd7BKBiIh0F5tEoLZiEZHcYpMI0jQwjYhId7FJBK7+R0VEcopNIkjT9YCISHeRJgIzO9PMVpjZSjP7co711WZ2V7j+cTObGmU8IiKyt8gSgZmVAzcCZwEzgYvMbGbWZpcB2939MOB7wLejikeNxSIiuUV5RTAHWOnuq929E7gTODdrm3OBW8PpXwGnW8StuWorFhHpriLCfR8ErM2YXwfM7Wkbd0+Y2U5gDLAlcyMzmwfMAxg/fjxNTU39DqZtU4LjGpy/PPIIVeXxyQYtLS0D+r5KmcocDyrz4IkyEeQ622ZX0OSzDe4+H5gPMHv2bG9sbOx3MI3AcU1NDOSzpaxJZY4FlTkeoipzlFVD64DJGfOTgNd72sbMKoCRwLYIYxIRkSxRJoLFwHQzm2ZmVcCFwIKsbRYAHwunzwf+7OovWkSkoCKrGgrr/K8E7gPKgZvdfZmZXQsscfcFwM+A281sJcGVwIVRxSMiIrlF2UaAuy8EFmYt+1rGdAfwwShjEBGR3sXuyWIREelOiUBEJOaUCEREYk6JQEQk5qzU7tY0s83AqwP8eANZTy3HgMocDypzPOxLmae4+9hcK0ouEewLM1vi7rOLHUchqczxoDLHQ1RlVtWQiEjMKRGIiMRc3BLB/GIHUAQqczyozPEQSZlj1UYgIiJ7i9sVgYiIZFEiEBGJuSGZCMzsTDNbYWYrzezLOdZXm9ld4frHzWxq4aMcXHmU+XNmttzMlprZA2Y2pRhxDqa+ypyx3flm5mZW8rca5lNmM/tQ+G+9zMx+UegYB1sef9sHm9mDZvZ0+Pd9djHiHCxmdrOZbTKz53tYb2Z2Q/h9LDWz4/f5oO4+pF4EXV6vAg4BqoBngZlZ23wKuCmcvhC4q9hxF6DMpwHDwulPxqHM4Xb1wMPAImB2seMuwL/zdOBpYFQ4P67YcRegzPOBT4bTM4E1xY57H8v8duB44Pke1p8N/J5ghMeTgMf39ZhD8YpgDrDS3Ve7eydwJ3Bu1jbnAreG078CTjcr6WHt+yyzuz/o7m3h7CKCEeNKWT7/zgDfAK4DOgoZXETyKfMngBvdfTuAu28qcIyDLZ8yOzAinB7J3iMhlhR3f5jeR2o8F7jNA4uAA8xswr4ccygmgoOAtRnz68JlObdx9wSwExhTkOiikU+ZM11G8IuilPVZZjM7Dpjs7r8rZGARyuffeQYww8weNbNFZnZmwaKLRj5lvgb4iJmtIxj/5NOFCa1o+vv/vU+RDkxTJLl+2WffI5vPNqUk7/KY2UeA2cCpkUYUvV7LbGZlwPeASwsVUAHk8+9cQVA91Ehw1feImR3t7jsiji0q+ZT5IuAWd7/ezE4mGPXwaHdPRR9eUQz6+WsoXhGsAyZnzE9i70vFPduYWQXB5WRvl2L7u3zKjJmdAVwNnOPuuwsUW1T6KnM9cDTQZGZrCOpSF5R4g3G+f9v3unuXu78CrCBIDKUqnzJfBtwN4O6PATUEnbMNVXn9f++PoZgIFgPTzWyamVURNAYvyNpmAfCxcPp84M8etsKUqD7LHFaT/IQgCZR6vTH0UWZ33+nuDe4+1d2nErSLnOPuS4oT7qDI52/7NwQ3BmBmDQRVRasLGuXgyqfMrwGnA5jZkQSJYHNBoyysBcBHw7uHTgJ2uvuGfdnhkKsacveEmV0J3Edwx8HN7r7MzK4Flrj7AuBnBJePKwmuBC4sXsT7Ls8yfweoA34Ztou/5u7nFC3ofZRnmYeUPMt8H/AuM1sOJIEvuvvW4kW9b/Is8+eB/zSzzxJUkVxayj/szOwOgqq9hrDd4+tAJYC730TQDnI2sBJoA/5un49Zwt+XiIgMgqFYNSQiIv2gRCAiEnNKBCIiMadEICISc0oEIiIxp0Qg+x0zS5rZMxmvqb1sO7WnXhr7ecymsIfLZ8PuGQ4fwD6uMLOPhtOXmtnEjHU/NbOZgxznYjM7No/PfMbMhu3rsWXoUiKQ/VG7ux+b8VpToONe7O6zCDok/E5/P+zuN7n7beHspcDEjHWXu/vyQYnyzTh/RH5xfgZQIpAeKRFISQh/+T9iZk+Fr7fm2OYoM3sivIpYambTw+UfyVj+EzMr7+NwDwOHhZ89Pezn/rmwn/jqcPm37M3xHf49XHaNmX3BzM4n6M/p5+Exa8Nf8rPN7JNmdl1GzJea2Q8GGOdjZHQ2ZmY/NrMlFoxD8K/hsqsIEtKDZvZguOxdZvZY+D3+0szq+jiODHFKBLI/qs2oFvp1uGwT8E53Px64ALghx+euAP7D3Y8lOBGvC7scuAA4JVyeBC7u4/h/CzxnZjXALcAF7v4WgifxP2lmo4H3A0e5+zHANzM/7O6/ApYQ/HI/1t3bM1b/CjgvY/4C4K4BxnkmQZcSaVe7+2zgGOBUMzvG3W8g6IfmNHc/Lex24qvAGeF3uQT4XB/HkSFuyHUxIUNCe3gyzFQJ/DCsE08S9KGT7THgajObBNzj7i+b2enACcDisGuNWoKkksvPzawdWEPQlfHhwCvu/lK4/lbgH4AfEoxv8FMz+18g726u3X2zma0O+4h5OTzGo+F++xPncIIuFzJHp/qQmc0j+H89gWCQlqVZnz0pXP5oeJwqgu9NYkyJQErFZ4GNwCyCK9m9Bppx91+Y2ePAe4D7zOxygi57b3X3r+RxjIszO6Uzs5xjVIT938wh6OjsQuBK4B39KMtdwIeAF4Ffu7tbcFbOO06Ckbq+BdwInGdm04AvACe6+3Yzu4Wg87VsBtzv7hf1I14Z4lQ1JKViJLAh7GP+EoJfw92Y2SHA6rA6ZAFBFckDwPlmNi7cZrTlP17zi8BUMzssnL8EeCisUx/p7gsJGmJz3bnTTNAVdi73AO8j6Ef/rnBZv+J09y6CKp6TwmqlEUArsNPMxgNn9RDLIuCUdJnMbJiZ5bq6khhRIpBS8SPgY2a2iKBaqDXHNhcAz5vZM8ARBMP5LSc4Yf7RzJYC9xNUm/TJ3TsIenb8pZk9B6SAmwhOqr8L9/cQwdVKtluAm9KNxVn73Q4sB6a4+xPhsn7HGbY9XA98wd2fJRireBlwM0F1U9p84Pdm9qC7bya4o+mO8DiLCL4riTH1PioiEnO6IhARiTklAhGRmFMiEBGJOSUCEZGYUyIQEYk5JQIRkZhTIhARibn/A0AqXDqM48gwAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "netD = Discriminator(ngpu=ngpu, nc=nc, ndf=ndf).to(device)\n",
    "path_to_D = os.path.join(save_dir, 'epoch-{}.classifier'.format(bestModelIdx+1))\n",
    "netD.load_state_dict(torch.load(path_to_D, map_location=lambda storage, loc: storage))\n",
    "netD.eval();\n",
    "\n",
    "num_valimages = 10\n",
    "\n",
    "\n",
    "predConfidence_closeset = torch.from_numpy(np.array([0])).type(torch.float).view(-1,1)\n",
    "predConfidence_opensetlist = [torch.from_numpy(np.array([0])).type(torch.float).view(-1,1)]*len(opensetIDlist)\n",
    "count = 0\n",
    "for sample in dataloader_val:   \n",
    "    count += 1 \n",
    "    if count<=num_valimages: \n",
    "        continue\n",
    "    if count%100==0 and count>0:\n",
    "        print('\\t{}/{}'.format(count, len(dataloader_val)))\n",
    "    imageList, labelList = sample\n",
    "    imageList = imageList.to(device)\n",
    "    labelList = labelList.to(device)\n",
    "\n",
    "    labelList = labelList.unsqueeze(1)\n",
    "    labelList = F.interpolate(labelList, scale_factor=0.25, mode='nearest')\n",
    "    labelList = labelList.squeeze()\n",
    "    H, W = labelList.squeeze().shape\n",
    "    trainlabelList = id2trainID_np[labelList.cpu().numpy().reshape(-1,).astype(np.int32)]\n",
    "    trainlabelList = trainlabelList.reshape((1,H,W))\n",
    "    trainlabelList = torch.from_numpy(trainlabelList)\n",
    "\n",
    "    with torch.no_grad():\n",
    "        imageList = imageList.to(device)\n",
    "        logitsTensor = model(imageList).detach().cpu()        \n",
    "        featTensor = model.feat4.detach()\n",
    "\n",
    "\n",
    "    allPixList = featTensor.squeeze()\n",
    "    allPixList = allPixList.reshape(allPixList.shape[0], -1).permute(1,0).unsqueeze(-1).unsqueeze(-1).to(device)\n",
    "    with torch.no_grad():\n",
    "        allPixConf = netD(allPixList).view(-1)\n",
    "\n",
    "\n",
    "    ################    close-set pixels    ################\n",
    "    closesetList = trainlabelList.reshape(-1,1)\n",
    "    closesetList = ((closesetList>=0) & (closesetList<=18)).nonzero()\n",
    "    closesetList = closesetList[:,0]    \n",
    "    predConf = allPixConf[closesetList]\n",
    "    predConf = predConf.view(-1,1).detach()    \n",
    "    predConfidence_closeset = torch.cat((predConfidence_closeset, predConf.detach().cpu()), 0)\n",
    "\n",
    "    ################    open-set pixels    ################\n",
    "    for i in range(len(opensetIDlist)):        \n",
    "        clsID = opensetIDlist[i]\n",
    "        tmpMap = (labelList==clsID).type(torch.float32)\n",
    "        tmpCount = tmpMap.sum()    \n",
    "        if tmpCount==0:   \n",
    "            #print('not existing -- ', clsID, tmpCount)\n",
    "            continue\n",
    "\n",
    "        curClsList = labelList.reshape(-1,1)\n",
    "        curClsList = (curClsList==clsID).nonzero()\n",
    "        curClsList = curClsList[:,0]\n",
    "        predConf = allPixConf[curClsList]\n",
    "        predConf = predConf.view(-1,1).detach()\n",
    "        predConfidence_opensetlist[i] = torch.cat((predConfidence_opensetlist[i], predConf.detach().cpu()), 0)              \n",
    "\n",
    "for i in range(len(opensetIDlist)):\n",
    "    predConfidence_opensetlist[i] = predConfidence_opensetlist[i][1:]\n",
    "    predConfidence_opensetlist[i] = predConfidence_opensetlist[i].detach().cpu().numpy()                    \n",
    "\n",
    "predConfidence_closeset = predConfidence_closeset[1:]\n",
    "predConfidence_closeset = predConfidence_closeset.detach().cpu().numpy()\n",
    "    \n",
    "predConfidence_openset = np.concatenate(predConfidence_opensetlist, 0)\n",
    "roc_score, roc_to_plot = evaluate_openset(-predConfidence_closeset, -predConfidence_openset)\n",
    "\n",
    "\n",
    "plt.plot(roc_to_plot['fp'], roc_to_plot['tp'])\n",
    "plt.grid('on')\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('ROC score {:.5f}'.format(roc_score))\n",
    "print(roc_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8756813035477743\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deZwdVZn/8c/T+5rupDsJ2TshiSQEEkKAsCgdAUVQcBhEEBHGJSMOOoo6OuJLHcb5jYM6KgMuqAzqoIALGDEYUWlAMCRhCZCEJSvpLCS9pvf1+f1R1eGm09256fTt233r+3697iu3qs6t+5zuTj1V51SdY+6OiIhEV1qyAxARkeRSIhARiTglAhGRiFMiEBGJOCUCEZGIUyIQEYk4JQIRkYhTIpBhY2bbzazFzBrNbK+Z3WVmBb3KnGVmfzGzBjOrN7Pfmdn8XmXGmNm3zey1cF+bw+XS4a3R4Fjgv8ysOnzdYmY2QPmPm9k2MztgZuvM7JyYbQ+FP4OeV7uZvRCzPfZn3mhmf+y171lm9mD4864ys1titpWZ2Uozqw1/X7eZWcZQ/zwk+ZQIZLi9y90LgEXAKcC/9mwwszOBPwK/BSYDM4H1wBNmNisskwX8GTgRuBAYA5wFVAOnD181DsY8mAPjcuDdwELgZOCdwD/2s/8zgK8BlwNFwI+B+80sHcDd3+HuBT0v4Engl712866YMm+L2XcW8DDwF+A4YCrwfzGf+y6wD5hE8Ps6F/jYIOorI52766XXsLyA7cD5Mcu3AL+PWX4c+G4fn3sI+Gn4/sPA60BBnN9pwLcIDmj1wPPAgnBbLvBNYEe47a9AbrjtEmADUAdUAPN61eNz4b7agAyCxPVrYD+wDfjEADE9CSyPWf4QsLqfsu8F1sQs5wMOTOqjbBnQBczs72feq/xy4PEB4twEXBSz/HXgB8n+O9Jr6F+6IpCkMLOpwDuAzeFyHsGZfe+zWYD7gAvC9+cDf3D3xji/6m3AW4C5QDHBgbU63PYN4NTwe8cB/wJ0m9lc4BfAJ4HxwErgd+EZdI+rgIvDfXYDvyO4epkCnAd80sze3k9MJ4Zle6wP1/XlISDdzM4IrwI+CDwH7O2j7AcIDuzbeq2/28z2m9kfzWxhzPqlwPaweanKzCrM7KSY7d8BrjSzPDObQvD7+kM/ccoopkQgw+0BM2sAdhKcpX85XD+O4O9xTx+f2QP0tP+X9FOmPx1AIXACYO6+yd33mFkawUH1n919l7t3ufuT7t5GkCx+7+4Pu3sHQcLIJUgYPW51953u3gKcBox395vdvd3dtwI/BK7sJ6YCgiuQHvVAQT/9BA0EVxp/Jbj6+DLB1URfg4R9ALir17qrCa4UZgCPAKvMrDjcNjWM8VaCK5rfA7+NSXiPEiSoA0AlsA54oJ86ySimRCDD7d3uXgiUExycew7wtQRn1pP6+MwkoCp8X91PmT65+1+A24DbgdfN7A4zGxN+bw6wpY+PTSZoLurZRzdB4poSU2ZnzPsZwGQzq+t5AV8AJvYTViNB30aPMUBjPwf3DxMkrBOBLOD9wINmNjm2UNiBfBzwq9j17v6Eu7e4e7O7/ydBU9ebw80twF/d/SF3bydIeCXAvDBRrgJ+Q9AcVQqMBf6rnzrJKKZEIEnh7o8SnL1+I1xuAv4GvKeP4lcQdBAD/Al4u5nlH8V33erupxIcTOcCnyVILK3A8X18ZDfBwR0I7vIBpgG7Yncb834nsM3di2Nehe5+UT8hbSDoKO6xMFzXl4XA79z9FXfvdvc/EFwRndWr3LXAb+JoMnOCfhMI+jj6G354HEGdb3P3NnevBv4X6K9OMoopEUgyfRu4wMwWhcufB641s0+YWaGZjTWzrwJnAv8WlvkZwYH312Z2gpmlmVmJmX3BzA47SJnZaWH7eibQRHDw7wrP8u8E/tvMJptZupmdaWbZBH0SF5vZeeHnPk3QLPNkP/VYAxwws8+ZWW64rwVmdlo/5X8K3GhmU8Iz+09zeJNOj7VhLLPC204vIEhmL8bUMZcggR6yDzObbmZnm1mWmeWY2WcJzuyfCIv8H7DUzM4P+x8+SZAgN7l7FUGn9/VmlhE2J13LoX0bkiqS3VutV3Re9HEHC/A94Ncxy+cQ3KXTSNA2/XvCu3xiyhQRJJGdYbktwH8DJX1853kEZ76NBAe5uwnvOCJo9/82wZl+PfAYb9w19HfAxnD9o8CJR6jHZIIO5r0EzVyre5eJKWsEd0zVhK9bCPoverY3Am+OKXsz8BpBf8Em4Jpe+7uKoCnLeq0/Max7E0GT2p+BJb3KXEbQYX8g/LnH1nNRuK42/Nn9EpiQ7L8jvYb+ZeEvXEREIkpNQyIiEadEICIScUoEIiIRp0QgIhJxo24kwdLSUi8rKxvUZ5uamsjPj/v285SgOkeD6hwNx1Lnp59+usrdx/e1bdQlgrKyMtatWzeoz1ZUVFBeXj60AY1wqnM0qM7RcCx1NrMd/W1T05CISMQpEYiIRJwSgYhIxCkRiIhEnBKBiEjEJSwRmNmdZrbPzF7sZ7uZ2a0WTDz+vJktTlQsIiLSv0ReEdxFMLl4f94BzAlfywlGoRQRkWGWsOcI3P0xMysboMilBBOSO7DazIrNbJK7H800hCIiI4q7097VTUt7Fy0dXbS0d9Hc3kVrRxetHd20dnTR0dVNe1c3nV1OZ3c37V1OZ1c33Q7d3U63O13u4Xvo6g6Giy5u6aI8ATEn84GyKRw63V9luO6wRGBmywmuGpg4cSIVFRWD+sLGxsZBf3a0Up2jQXUevODADc2dTksntIT/tnY6LZ1Oa1fwvrUTWructp7lLmjrDJbbug79tzsBo/sbcMXxnpDfczITQV8Tdff543P3O4A7AJYsWeKDfbJOTyJGg+ocDb3r7O40tHVS29ROTVM7dc0d1DYH7+tbOqhr7qCupYP6lg7qm4N1B1o7OdDSQWccR+6MNCM/O4OC7AzystLJy0lnfM/7rODfnMx08rPTyc1MJzerZ1uwPjcz+Dc7I42sjDQy09PITDcy09PISDcy0tJINyMtDdLTjDQLXsF7MLOE/Z6TmQgqCeZE7TGVYK5YERHcnab2Lqob26hqDA7owfs2qpva2bS1lR9tforqpnZqmtqobeqgvau7z32lGYzJzaQ4N5Oi3EyK8rKYXpJPUW4GhTnBusKc4H1hdgYFORnkZ2VQmJNBfnYG+dnpZGekD/NPYPgkMxGsAG4ws3uAM4B69Q+IpLbubqeupYOqxjaqGtrY39jG/obgwF7V8MZBvrqxnarGNto6+z6wF2RnkJvWzdTMTiYX5XDSlDGMy8+mJD+LcflZjM3PpDgvi3F5WRTnZTImJ5O0tL4aIQQSmAjM7BdAOVBqZpXAl4FMAHf/PrASuIhgvtRm4B8SFYuIJE5Xt1PTFBy4qxrb2Heg7eD7qsb2Q/6taWqnq49mmIw0o7Qgm9LCLErys5k9oYDSgmzG5WdRkp9FaUE2JQVZlBQEB/uczPSwmeTsJNQ49STyrqGrjrDdgX9K1PeLyLFpae9if0Mb+xtb2d/Qxr6G4Ox934E29jW0Hjybr2rs++Cek5nG+MJsxuVnM7koh4VTiygpyGJcfjYTCoMD+4TCbEoLsinKzcRMZ+zJMuqGoRaRwevudmqb29kfnrkfcoBvaGVfQxv7DrRS1dhOY1vnYZ9PMygtyGZ8YXAwn3fcGCaMyWZCYc7B9eMLsyktyKIgO0MH91FCiUBklHMPbnPcur8xPIM/9ABfFS73tMX3dfaen5UeHNzH5LBgSlF4MH/jwD6+IJsJY7Ipyc8mXW3tKUeJQGQEa+3oYt+BNl5vaGVvffg60MrrB1oPNtHsa2ijub0L/vToIZ/NSLODB/SJY3JYMLmI0sKsmDP6nINn9vnZOhREmX77IknS3tnN3vpWdtW1sKe+hT3hgX5PfQu764IDfk1T+2Gfy81MZ+KY4Oz9pKnFTCjMpmF/JUsXzg+aaAqzGF+Qzdi8LN0pI3FRIhBJkNaOLiprm6msbWFXXUvwb20LlbXN7K5r5fWGVrxXK01RbiaTinKYXJzLwmnFTCrK4biiHCaOyeG48DUm9/C294qKfZQvnjqMtZNUokQgcgwa2zrZtr+JrVWNbK9qZkdNE69VN/NaTTP7GtoOKZuZbkwqymVKcS5nzy5lythcphbnMmVsLpOKcphUlEtuVuo+tCQjlxKBSBxqmtp55fUGXt3XyJZ9jWwOX3sPtB5SblJRDtPH5XHu3PFMH5fHtHF5wQF/bC4TCnPU0SojkhKBSIzOrm62VjWxYXc9G3cf4KW9DWza00BV4xtn9/lZ6cyeUMBZs0s4fnwBs0rzmTW+gBkleeRk6oxeRh8lAoms7m5na1UTL+yqY/3OetZX1rFx94GDwxpkZaQxd2IB584dzwnHFTJnYgFzJhYyuShH98dLSlEikMioaWrnmR21PLuzlud21vH8znoawoemcjPTWTBlDO9fOoMFU8Zw4uQiZpXmk5Gu2Vwl9SkRSEpyd7ZXN7N2Ww1rt9ewbkct26qagGCI33mTCrlk0WQWTivm5KlFzB5foIO+RJYSgaSErm5n054DrN1ew++fbeUzf/3zwXb94rxMlswYyxVLprF4ejEnTy3W3TkiMZQIZFRq7+zm+co61myvYc22Gp7eXnuwmackx3jLvFJOKxvHaWVjmT2hQG36IgNQIpBRobWji+d21rFmW3DgX7ejhtaOoFN39oQC3rVoMqeXjeO0meN49bmnKC9flOSIRUYPJQIZkVo7uli3vZantlWzZlsNz+6so72zGzN408RCrjxtOktnlXBa2VhKCrIP+eyrSYpZZLRSIpARobm9k2d21LF6azWrt1azvrKOji4nzeDEyUV8YOmM8MA/jqK8zGSHK5JSlAgkKRpaO3h6Ry1PhU09z4cH/vQ0Y8GUIj549szgwD9zHAUaGVMkofQ/TIbFgdYO1m6rYfXWap7aVsOG3Qfo6nYy0oyTphbxwXNmHjzj14FfZHjpf5wkREt7F0/vqOWJLVU8ubmKF3bV0+2QlZ7GounFfKz8eM6YWcLiGcXkZenPUCSZ9D9QhkRnVzcv7Krnic1VPLG5mqd31NLe1U1GmnHK9GJuWDabpbNKWDxjrMbjERlhlAhkUNyDcXr++moVT2yu4m9bq2loDe7jnzdpDNeeNYOzZpdyetk4zX4lMsLpf6jErb65g8c37+fRl/fzxOYqdtcHQzBPHZvLxSdN4uzZpZx1fMlht3OKyMimRCADeq26mVUb9vLwptd5ekctXd1OUW4mZx1fwj+9tZRzZpcyoyQ/2WGKyDFQIpDD7KxpZsX63fz++T1s3HMAgBOOK+T6c49n2QnjWTi1WAO0iaQQJQIBgid5//DiXu5bt5Mnt1QDsHh6MV+8eB5vm38c00vykhyhiCSKEkHE7axp5idPbue+dTs50NrJ9HF5fOr8uVy2eArTxungLxIFSgQR9eKuer5XsYWHXtxDmhlvX3AcV58ejN+Tpnl1RSJFiSBinttZx7f/9AoVL++nMDuD5W85nuvOKuO4opxkhyYiSaJEEBGb9zVwyx9e5o8bX2dsXiafedtcrjmzjKJcDeAmEnVKBCmuvs35wv0vcM+a18jLyuDGC+bywXNmajwfETlIR4MU1dbZxZ1/3c53Hmum03fygTPL+PhbZ+thLxE5TEITgZldCHwHSAd+5O5f67V9OvAToDgs83l3X5nImKLgr69W8aXfvsjWqiZOmZDON645h+PHFyQ7LBEZoRKWCMwsHbgduACoBNaa2Qp33xhT7IvAfe7+PTObD6wEyhIVU6qramzjqw9u5IHndlNWksdd/3Aa7NmoJCAiA0rkFcHpwGZ33wpgZvcAlwKxicCBMeH7ImB3AuNJWe7Ob57Zxc0PbqS5vZNPvHU2H1s2m5zMdCr2bDzyDkQk0szdE7Njs8uBC939w+HyNcAZ7n5DTJlJwB+BsUA+cL67P93HvpYDywEmTpx46j333DOomBobGykoSK2z4wPtzv++2Maz+7qYU5zGPyzIZnLBG8M/pGKdj0R1jgbV+egsW7bsaXdf0te2RF4R9PVUUu+scxVwl7t/08zOBH5mZgvcvfuQD7nfAdwBsGTJEi8vLx9UQBUVFQz2syPRo6/s599/uZ76Zuemi+bxoXNmHvYwWKrVOR6qczSozkMnkYmgEpgWszyVw5t+PgRcCODufzOzHKAU2JfAuEa99s5uvr7qJX74+DbmTCjgpx88nXmTxhz5gyIifUhkIlgLzDGzmcAu4Ergfb3KvAacB9xlZvOAHGB/AmMa9fbUt/Cxu5/h2dfqeP/S6Xzx4vma8UtEjknCEoG7d5rZDcAqgltD73T3DWZ2M7DO3VcAnwZ+aGafImg2us4T1WmRAtZsq+Fjdz9NS3sX3716MRedNCnZIYlICkjocwThMwEre637Usz7jcDZiYwhVdy79jW++MCLTB2bxz3LlzJ7QmGyQxKRFKEni0e47m7nPx/axA8f38ab55Ry2/sWa3wgERlSSgQjWGtHF5++bz2/f2EPHzhzBl9653zNDCYiQ06JYIRqaO3gwz9Zx1Pbarjponl85C2zkh2SiKQoJYIRqL6lg2vvXMMLu+r5zpWLuHTRlGSHJCIpTIlghKlrbueaH6/hpb0H+O7Vi3n7icclOyQRSXFKBCPIgdYOrvnxGl5+vYEfXHMqbz1hYrJDEpEIUM/jCNHU1sl1dwZXAt9//2IlAREZNkoEI0B7ZzfX3/0Mz+2s43+uOkVJQESGlZqGkqy72/nsr9bz2Cv7ueXvT+bCBXpaWESGV1xXBGaWZWazEx1MFH39jy/z2+d289m3v4krTpt25A+IiAyxIyYCM7sYeAF4OFxeZGb3JzqwKLhv7U6+V7GF950xnY+VH5/scEQkouK5IrgZOAOoA3D35wBdHRyjddtruOmBF3jznFJuvuREzPqavkFEJPHiSQQd7l7Xa51GCD0Ge+tb+ej/Pc3UsXncdtViDRshIkkVT2fxJjO7AkgL5xb4Z2B1YsNKXe2d3Xz0/4KhpO9ZvpSiPA0gJyLJFc+p6A3AqUA38BuglSAZyCD850ObeG5nHV9/z0INJS0iI0I8VwRvd/fPAZ/rWWFmlxEkBTkKqzbs5X+f2M51Z5VpUhkRGTHiuSL4Yh/rbhrqQFLd3vpW/uVXz3PSlCL+9aITkh2OiMhB/V4RmNnbCSaWn2Jm/x2zaQxBM5HEqbvbufG+5+jo6uY7Vy4iO0NzDIvIyDFQ09A+4EWCPoENMesbgM8nMqhUc+cT23hySzVfu+wkZo0vSHY4IiKH6DcRuPuzwLNmdre7tw5jTClly/5Gbln1MufPm8B79eSwiIxA8XQWTzGz/wDmAzk9K919bsKiShFd3c5nfrmevKx0/t9lJ+mhMREZkeLpLL4L+F/AgHcA9wH3JDCmlPGzv23n2dfq+Mq7TmRCYc4Ry4uIJEM8iSDP3VcBuPsWd/8isCyxYY1+e+pb+Pqql3nL3PFcumhyssMREelXPE1DbRa0aWwxs48Cu4AJiQ1r9Pv3BzfS2e189dIFahISkREtnkTwKaAA+ATwH0AR8MFEBjXaPfbKfla+sJdPXzCX6SV5yQ5HRGRAR0wE7v5U+LYBuAbAzKYmMqjRrKOrm5sf3EhZSR4fecusZIcjInJEA/YRmNlpZvZuMysNl080s5+iQef69fOnXmPzvkZuung+OZl6cExERr5+E4GZ/SdwN3A18Aczuwl4BFgP6NbRPjS0dvDtP73CmbNKOH+eulFEZHQYqGnoUmChu7eY2Thgd7j88vCENvp850+vUtvcwRcumqcOYhEZNQZqGmp19xYAd68BXlIS6N+uuhZ+9NdtLHvTeE6aWpTscERE4jbQFcEsM+sZatqAsphl3P2yI+3czC4EvgOkAz9y96/1UeYK4CsEs56td/f3xR/+yHH7I5sBuOni+UmORETk6AyUCP6+1/JtR7NjM0sHbgcuACqBtWa2wt03xpSZA/wrcLa715rZqGxY31nTzM+feo1LF01m9gQNKicio8tAg879+Rj3fTqw2d23ApjZPQT9DhtjynwEuN3da8Pv3HeM35kUX/vDSwDcsGx2kiMRETl65p6YeejN7HLgQnf/cLh8DXCGu98QU+YB4BXgbILmo6+4+x/62NdyYDnAxIkTT73nnsENddTY2EhBwdCesde1dvPJihZOmZDOPy8eeeMJJaLOI53qHA2q89FZtmzZ0+6+pK9t8TxZPFh93TbTO+tkAHOAcmAq8LiZLXD3ukM+5H4HcAfAkiVLvLy8fFABVVRUMNjP9ufT960HKvmvq89m7sSRNwdxIuo80qnO0aA6D514Bp0DwMyyj3LflUDsAPxTCW5B7V3mt+7e4e7bgJcJEsOocKC1g1Ub9nLW8SUjMgmIiMTjiInAzE43sxeAV8PlhWb2P3Hsey0wx8xmmlkWcCWwoleZBwhHMg2fXp4LbD2K+JPqG6teprGtk89dqDmIRWT0iueK4FbgnUA1gLuvJ45hqN29E7gBWAVsAu5z9w1mdrOZXRIWWwVUm9lGgqeWP+vu1UdfjeHX0dXNQy/upawkj4XTipMdjojIoMXTR5Dm7jt6PSnbFc/O3X0lsLLXui/FvHfgxvA1qtz/zC72N7Tx5fedkuxQRESOSTyJYKeZnQ54+GzAxwnu9Im0O5/YRmlBNhctmJTsUEREjkk8TUPXE5yxTwdeB5aG6yLrxV31vLS3gQ+eU0ZamsYUEpHRLZ4rgk53vzLhkYwi33r4FTLTjatPn5HsUEREjlk8VwRrzWylmV1rZpG/R7KprZPVW6tZOquEorzMZIcjInLMjpgI3P144KvAqcALZvaAmUX2CuHXz1TS1N7FP2k4CRFJEXE9UObuT7r7J4DFwAGCCWsi6cHn91BakMUZM8clOxQRkSERzwNlBWZ2tZn9DlgD7AfOSnhkI9CO6ibWbKvhurPKNPGMiKSMeDqLXwR+B9zi7o8nOJ4R7b51OwF49ylTkhyJiMjQiScRzHL37oRHMsK5Ow+9sJcTJ49h6ti8ZIcjIjJk+k0EZvZNd/808GszO2ys6nhmKEslz+6sY2tVE19994JkhyIiMqQGuiK4N/z3qGYmS1UPrt8DwNtOnJjkSEREhtZAM5StCd/Oc/dDkoGZ3QAc6wxmo4a78/CmvZxeNo4JhSNv8hkRkWMRz+2jH+xj3YeGOpCRbH1lPTtrWrj81KnJDkVEZMgN1EfwXoI5BGaa2W9iNhUCdX1/KjXd/0wlAMtOmJDkSEREht5AfQRrCOYgmArcHrO+AXg2kUGNNE9uqWb+pDGMLzzaSdpEREa+gfoItgHbgD8NXzgjz7aqJl7d18iX3jk/2aGIiCTEQE1Dj7r7uWZWy6GTzhvBnDKRGGPhgWd3AXDBfN0tJCKpaaCmoZ7pKEuHI5CRau32GiaOyWbaOD1EJiKpqd+7hmKeJp4GpLt7F3Am8I9A/jDElnQNrR08uaVaQ0qISEqL5/bRBwimqTwe+CkwD/h5QqMaIf7y0j4Azp07PsmRiIgkTjyJoNvdO4DLgG+7+8eBSJwi/279bszgtLJIdIeISETFkwg6zew9wDXAg+G6lJ+ay93506Z9nDZjHJnpcU3bICIyKsX7ZPEygmGot5rZTOAXiQ0r+bbsbwJg6SxdDYhIajviMNTu/qKZfQKYbWYnAJvd/T8SH1pyPbG5CoC/17ASIpLijpgIzOzNwM+AXQTPEBxnZte4+xOJDi6ZntxSxdSxucwoicQNUiISYfFMTPMt4CJ33whgZvMIEsOSRAaWTF3dzuqtNbxND5GJSATE00eQ1ZMEANx9E5CVuJCSb31lHfUtHZw1uyTZoYiIJFw8VwTPmNkPCK4CAK4mxQedW/l8MAnNGTOVCEQk9cWTCD4KfAL4F4I+gseA/0lkUMm2els1aQaTi3OTHYqISMINmAjM7CTgeOB+d79leEJKLndnV22LrgZEJDL67SMwsy8QDC9xNfCwmfU1U1nK2byvkdrmDv5O4wuJSEQM1Fl8NXCyu78HOA24/mh3bmYXmtnLZrbZzD4/QLnLzczNLOl3Ij32avD8wKllY5MciYjI8BgoEbS5exOAu+8/QtnDmFk6wcxm7wDmA1eZ2WGzu5hZIUEfxFNHs/9E2bj7AFnpacwq1fMDIhINA/URzIqZq9iA42PnLnb3y46w79MJnkLeCmBm9wCXAht7lft34BbgM0cTeKI8+1otb5k7HjNLdigiIsNioETw972WbzvKfU8BdsYsVwJnxBYws1OAae7+oJn1mwjMbDmwHGDixIlUVFQcZSiBxsbGAT97oN3ZWtXM4rHtg/6OkeZIdU5FqnM0qM5DZ6A5i/98jPvu65T64JSXZpZG8NTydUfakbvfAdwBsGTJEi8vLx9UQBUVFQz02Ude2ges5aKzF1J+Qmo8VXykOqci1TkaVOehk8jxlSsJZjfrMRXYHbNcCCwAKsxsO7AUWJHMDuOKl4OJaHTrqIhESSITwVpgjpnNNLMs4EpgRc9Gd69391J3L3P3MmA1cIm7r0tgTAN6bmcdEwqzyc+O5zk7EZHUEHciMLPso9mxu3cCNwCrgE3Afe6+wcxuNrNLji7MxHN31lfWs2hacbJDEREZVvEMQ3068GOgCJhuZguBD4dTVg7I3VcCK3ut+1I/ZcvjCThRKmtbAFioRCAiERPPFcGtwDuBagB3X08wY1lKeWFXPQDnzC5NciQiIsMrnkSQ5u47eq3rSkQwybR+Zx0ZacabjitMdigiIsMqnl7RnWHzkIdPC38ceCWxYQ2/1dtqmFmaT05merJDEREZVvFcEVwP3AhMB14nuM3zqMcdGul21bZQpmElRCSC4pm8fh/BrZ8pa19DK1WNbZw5S88PiEj0xHPX0A+JeSK4h7svT0hESbBpTwMA8yaNSXIkIiLDL54+gj/FvM8B/o5DxxAa9V7acwCAeZPUUSwi0RNP09C9sctm9jPg4YRFlAQv7W3guDE5FOdlJTsUEZFhN5ghJmYCM4Y6kGRavbVat42KSGTF00dQyxt9BGlADdDvbGOjTWdXN3vqWzlbD5KJSEQdaRNmRz8AAA46SURBVPJ6AxYCu8JV3e5+WMfxaPZaTTMAJ+iKQEQiasCmofCgf7+7d4WvlEoCAK/uawTgtLJxSY5ERCQ54ukjWGNmixMeSZJsDhPB8RMKkhyJiEhy9Ns0ZGYZ4VDS5wAfMbMtQBPBzGPu7imRHJ7eUcukohwKNAeBiETUQEe/NcBi4N3DFEtS7KlvZVy+bhsVkegaKBEYgLtvGaZYhp27U1nTzKWnTE52KCIiSTNQIhhvZjf2t9Hd/zsB8Qyr/Q1tNLR1Mnu8+gdEJLoGSgTpQAHhlUEq2lbVBMBMJQIRibCBEsEed7952CJJgh3VwTMEM0s0/LSIRNdAt4+m7JVAj23VTWSkGZOLc5IdiohI0gyUCM4btiiSZEd1E9PG5ZGRPpghl0REUkO/R0B3rxnOQJLh2dfqmFKcm+wwRESSKrKnwu5OY1snxXmZyQ5FRCSpIpsI6ls6aGjtZNG04mSHIiKSVJFNBDtrWgCYOjYvyZGIiCRXZBNBZW1w6+i0ceojEJFoi3AiCK8IinVFICLRFtlEsKuuhcLsDIrUWSwiERfZRLBx9wGmjFWzkIhIZBNBTXM72ZnpyQ5DRCTpEpoIzOxCM3vZzDab2WET3pvZjWa20cyeN7M/m9mMRMYTq7K2mfmTNE+xiEjCEoGZpQO3A+8A5gNXmdn8XsWeBZa4+8nAr4BbEhVPrOb2Tlo7uinJzx6OrxMRGdESeUVwOrDZ3be6eztwD3BpbAF3f8Tdm8PF1cDUBMZzUM8dQxPHKBGIiCRyot4pwM6Y5UrgjAHKfwh4qK8NZrYcWA4wceJEKioqBhVQY2MjFRUVbKjqAqB+12YqKrYPal+jRU+do0R1jgbVeegkMhH0NYy191nQ7P3AEuDcvra7+x3AHQBLlizx8vLyQQVUUVFBeXk5VU9Xwrr1vLP8TMpKU3sugp46R4nqHA2q89BJZCKoBKbFLE8FdvcuZGbnAzcB57p7WwLjOej1A60AHFekeQhERBLZR7AWmGNmM80sC7gSWBFbwMxOAX4AXOLu+xIYyyE27K6nKDeTHN0+KiKSuETg7p3ADcAqYBNwn7tvMLObzeySsNjXCeZF/qWZPWdmK/rZ3ZBq7+wmLeXnXxMRiU8im4Zw95XAyl7rvhTz/vxEfn9/9je0sWBKUTK+WkRkxInkk8X7GtqYOEb9AyIiEMFE0N3t7G9oY0KhniEQEYEIJoLa5nY6u53xSgQiIkAEE8H+xuAO1QmFahoSEYEoJoKGIBGUFmQlORIRkZEhcomgpqkdgJICNQ2JiEAEE0FtmAjG5euKQEQEIpgIapraSTMoytUUlSIiEMVE0NxOcV4W6Xq0WEQEiGAiqG3qYKwmrBcROShyiaC6qU0zk4mIxIhcIqht6mBsvq4IRER6RC4R1DS3644hEZEYkUoE7k5tUztj85QIRER6RCoRNHdCZ7frikBEJEakEkFjezBlshKBiMgbIpUIGsJEMFaJQETkoGglgo4gEZQoEYiIHBSpRNDTNKTOYhGRN0QqETSoj0BE5DDRSgQdkJ2RRl5WerJDEREZMSKVCBrbg1tHzTTgnIhIj0glggPtrv4BEZFeIpUIeq4IRETkDdFKBB1KBCIivUUqERzQFYGIyGEikwg6urpp6dQzBCIivUUmEfRMWl+Um5HkSERERpbIJIKGtk4A0tMjU2URkbhE5qjY0BokgslFOUmORERkZIlMImgKrwgKstU0JCISK6GJwMwuNLOXzWyzmX2+j+3ZZnZvuP0pMytLVCw9VwT5SgQiIodIWCIws3TgduAdwHzgKjOb36vYh4Bad58NfAv4r0TF0xheERTmKBGIiMRK5BXB6cBmd9/q7u3APcClvcpcCvwkfP8r4DxL0EBAB1o6ACjKzUzE7kVERq1Enh5PAXbGLFcCZ/RXxt07zaweKAGqYguZ2XJgOcDEiROpqKg46mDqXu9kYYnzzFNPkBahQecaGxsH9fMazVTnaFCdh04iE0FfR1sfRBnc/Q7gDoAlS5Z4eXn5UQdTDiyuqGAwnx3NKlTnSFCdoyFRdU5k01AlMC1meSqwu78yZpYBFAE1CYxJRER6SWQiWAvMMbOZZpYFXAms6FVmBXBt+P5y4C/uftgVgYiIJE7CmobCNv8bgFVAOnCnu28ws5uBde6+Avgx8DMz20xwJXBlouIREZG+JfReSndfCazste5LMe9bgfckMgYRERlYZJ4sFhGRvikRiIhEnBKBiEjEKRGIiEScjba7Nc1sP7BjkB8vpddTyxGgOkeD6hwNx1LnGe4+vq8Noy4RHAszW+fuS5Idx3BSnaNBdY6GRNVZTUMiIhGnRCAiEnFRSwR3JDuAJFCdo0F1joaE1DlSfQQiInK4qF0RiIhIL0oEIiIRl5KJwMwuNLOXzWyzmX2+j+3ZZnZvuP0pMysb/iiHVhx1vtHMNprZ82b2ZzObkYw4h9KR6hxT7nIzczMb9bcaxlNnM7si/F1vMLOfD3eMQy2Ov+3pZvaImT0b/n1flIw4h4qZ3Wlm+8zsxX62m5ndGv48njezxcf8pe6eUi+CIa+3ALOALGA9ML9XmY8B3w/fXwncm+y4h6HOy4C88P31UahzWK4QeAxYDSxJdtzD8HueAzwLjA2XJyQ77mGo8x3A9eH7+cD2ZMd9jHV+C7AYeLGf7RcBDxHM8LgUeOpYvzMVrwhOBza7+1Z3bwfuAS7tVeZS4Cfh+18B55mN6omMj1hnd3/E3ZvDxdUEM8aNZvH8ngH+HbgFaB3O4BIknjp/BLjd3WsB3H3fMMc41OKpswNjwvdFHD4T4qji7o8x8EyNlwI/9cBqoNjMJh3Ld6ZiIpgC7IxZrgzX9VnG3TuBeqBkWKJLjHjqHOtDBGcUo9kR62xmpwDT3P3B4QwsgeL5Pc8F5prZE2a22swuHLboEiOeOn8FeL+ZVRLMf/Lx4QktaY72//sRJXRimiTp68y+9z2y8ZQZTeKuj5m9H1gCnJvQiBJvwDqbWRrwLeC64QpoGMTze84gaB4qJ7jqe9zMFrh7XYJjS5R46nwVcJe7f9PMziSY9XCBu3cnPrykGPLjVypeEVQC02KWp3L4peLBMmaWQXA5OdCl2EgXT50xs/OBm4BL3L1tmGJLlCPVuRBYAFSY2XaCttQVo7zDON6/7d+6e4e7bwNeJkgMo1U8df4QcB+Au/8NyCEYnC1VxfX//WikYiJYC8wxs5lmlkXQGbyiV5kVwLXh+8uBv3jYCzNKHbHOYTPJDwiSwGhvN4Yj1Nnd69291N3L3L2MoF/kEndfl5xwh0Q8f9sPENwYgJmVEjQVbR3WKIdWPHV+DTgPwMzmESSC/cMa5fBaAXwgvHtoKVDv7nuOZYcp1zTk7p1mdgOwiuCOgzvdfYOZ3Qysc/cVwI8JLh83E1wJXJm8iI9dnHX+OlAA/DLsF3/N3S9JWtDHKM46p5Q467wKeJuZbQS6gM+6e3Xyoj42cdb508APzexTBE0k143mEzsz+wVB015p2O/xZSATwN2/T9APchGwGWgG/uGYv3MU/7xERGQIpGLTkIiIHAUlAhGRiFMiEBGJOCUCEZGIUyIQEYk4JQIZccysy8yei3mVDVC2rL9RGo/yOyvCES7Xh8MzvGkQ+/iomX0gfH+dmU2O2fYjM5s/xHGuNbNFcXzmk2aWd6zfLalLiUBGohZ3XxTz2j5M33u1uy8kGJDw60f7YXf/vrv/NFy8Dpgcs+3D7r5xSKJ8I87vEl+cnwSUCKRfSgQyKoRn/o+b2TPh66w+ypxoZmvCq4jnzWxOuP79Met/YGbpR/i6x4DZ4WfPC8e5fyEcJz47XP81e2N+h2+E675iZp8xs8sJxnO6O/zO3PBMfomZXW9mt8TEfJ2Z/c8g4/wbMYONmdn3zGydBfMQ/Fu47hMECekRM3skXPc2M/tb+HP8pZkVHOF7JMUpEchIlBvTLHR/uG4fcIG7LwbeC9zax+c+CnzH3RcRHIgrwyEH3gucHa7vAq4+wve/C3jBzHKAu4D3uvtJBE/iX29m44C/A05095OBr8Z+2N1/BawjOHNf5O4tMZt/BVwWs/xe4N5BxnkhwZASPW5y9yXAycC5Znayu99KMA7NMndfFg478UXg/PBnuQ648QjfIyku5YaYkJTQEh4MY2UCt4Vt4l0EY+j09jfgJjObCvzG3V81s/OAU4G14dAauQRJpS93m1kLsJ1gKOM3Advc/ZVw+0+AfwJuI5jf4Edm9nsg7mGu3X2/mW0Nx4h5NfyOJ8L9Hk2c+QRDLsTOTnWFmS0n+H89iWCSlud7fXZpuP6J8HuyCH5uEmFKBDJafAp4HVhIcCV72EQz7v5zM3sKuBhYZWYfJhiy9yfu/q9xfMfVsYPSmVmfc1SE49+cTjDQ2ZXADcBbj6Iu9wJXAC8B97u7W3BUjjtOgpm6vgbcDlxmZjOBzwCnuXutmd1FMPhabwY87O5XHUW8kuLUNCSjRRGwJxxj/hqCs+FDmNksYGvYHLKCoInkz8DlZjYhLDPO4p+v+SWgzMxmh8vXAI+GbepF7r6SoCO2rzt3GgiGwu7Lb4B3E4yjf2+47qjidPcOgiaepWGz0higCag3s4nAO/qJZTVwdk+dzCzPzPq6upIIUSKQ0eK7wLVmtpqgWaipjzLvBV40s+eAEwim89tIcMD8o5k9DzxM0GxyRO7eSjCy4y/N7AWgG/g+wUH1wXB/jxJcrfR2F/D9ns7iXvutBTYCM9x9TbjuqOMM+x6+CXzG3dcTzFW8AbiToLmpxx3AQ2b2iLvvJ7ij6Rfh96wm+FlJhGn0URGRiNMVgYhIxCkRiIhEnBKBiEjEKRGIiEScEoGISMQpEYiIRJwSgYhIxP1//SS4YBuHqykAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "useless_void = ['unlabeled', 'ego vehicle', 'rectification border', 'out of roi', 'license plate']\n",
    "\n",
    "meaningful_openset = np.asarray([0.]).reshape(1,1)\n",
    "for i in range(len(predConfidence_opensetlist)):    \n",
    "    clsID = opensetIDlist[i]\n",
    "    \n",
    "    if id2name[clsID] not in useless_void:\n",
    "        meaningful_openset = np.concatenate((meaningful_openset, predConfidence_opensetlist[i]), 0)\n",
    "        \n",
    "        \n",
    "        \n",
    "roc_score, roc_to_plot = evaluate_openset(-predConfidence_closeset, -meaningful_openset)\n",
    "plt.plot(roc_to_plot['fp'], roc_to_plot['tp'])\n",
    "plt.grid('on')\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.title('ROC score {:.5f}'.format(roc_score))\n",
    "print(roc_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}