classification_aldipp.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "affd7b99-6df9-4163-b460-7fb668fe6dd4",
   "metadata": {},
   "source": [
    "# Thesis Test: aldi_gmm_dyn_none_daily"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8efcc8be-a9a2-4255-9324-9f23d51f14bd",
   "metadata": {},
   "source": [
    "Discord Detector ALDI with following features:\n",
    "- Use of GMM\n",
    "- Dynamic selecting left components\n",
    "- No PSU consideration "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ae549c95-73e8-4dd4-8f23-eccdacb82a19",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import yout ALDI version here\n",
    "from aldi_gmm_dyn_none_both import ALDI\n",
    "\n",
    "aldi_id = '07'\n",
    "aldi_name = 'aldi_gmm_dyn_none_daily'\n",
    "\n",
    "# Set experiment's parameter\n",
    "#dict_param_exp = {'exp_id' : 0, 'gmm_max_comp' : 2, 'gmm_data': 'D'}\n",
    "#dict_param_exp = {'exp_id' : 1, 'gmm_max_comp' : 3, 'gmm_data': 'D'}\n",
    "#dict_param_exp = {'exp_id' : 2, 'gmm_max_comp' : 4, 'gmm_data': 'D'}\n",
    "#dict_param_exp = {'exp_id' : 3, 'gmm_max_comp' : 5, 'gmm_data': 'D'}\n",
    "#dict_param_exp = {'exp_id' : 4, 'gmm_max_comp' : 6, 'gmm_data': 'D'}\n",
    "dict_param_exp = {'exp_id' : 5, 'gmm_max_comp' : 7, 'gmm_data': 'D'}\n",
    "#dict_param_exp = {'exp_id' : 6, 'gmm_max_comp' : 8, 'gmm_data': 'D'}\n",
    "#dict_param_exp = {'exp_id' : 7, 'gmm_max_comp' : 9, 'gmm_data': 'D'}\n",
    "#dict_param_exp = {'exp_id' : 8, 'gmm_max_comp' : 10, 'gmm_data': 'D'}\n",
    "#dict_param_exp = {'exp_id' : 9, 'gmm_max_comp' : 15, 'gmm_data': 'D'}\n",
    "#dict_param_exp = {'exp_id' : 10, 'gmm_max_comp' : 30, 'gmm_data': 'D'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7720043f-3a5f-4da2-b27e-3c877bf214b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# LOAD LIBRARIES\n",
    "\n",
    "import time\n",
    "import logging\n",
    "\n",
    "from datetime import datetime\n",
    "from utils import *\n",
    "from data_import_ashrae import DataImportAshrae\n",
    "from aldi_evaluation_metrics import AldiEvaluationMetrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "831151e2-4b7b-4969-b6b8-5df64922be49",
   "metadata": {},
   "outputs": [],
   "source": [
    "# AUXILLIARY VARIABLES\n",
    "\n",
    "myDataImport = DataImportAshrae()\n",
    "myEvalMetrics = AldiEvaluationMetrics()\n",
    "\n",
    "list_site_id = list(range(0,16)) \n",
    "list_site_name = [f'Site {i}' for i in list_site_id]\n",
    "meter_type = 0\n",
    "\n",
    "agg_method = 'majority'\n",
    "\n",
    "curr_timestamp = datetime.today().strftime('%Y%m%d-%H%M')\n",
    "\n",
    "dict_all_pred_labels = {}\n",
    "dict_all_pred_labels_pred = {}\n",
    "dict_all_true_labels_or = {}\n",
    "dict_all_true_labels_and = {}\n",
    "dict_all_true_labels_majo = {}\n",
    "dict_all_true_labels_majoplus = {}\n",
    "dict_all_roc_auc_or = {}\n",
    "dict_all_roc_auc_and = {}\n",
    "dict_all_roc_auc_majo = {}\n",
    "dict_all_roc_auc_majoplus = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "74f8ccb8-d105-441d-bb47-b6d705ecb824",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "10/20/2021 10:56:51 AM, aldi_gmm_dyn_none_daily, INFO: aldi_gmm_dyn_none_daily starts its experiments now\n"
     ]
    }
   ],
   "source": [
    "# PREPARE LOGGING TECHNIQUE\n",
    "\n",
    "# create logger\n",
    "logger = logging.getLogger(aldi_name)\n",
    "logger.setLevel(logging.DEBUG)\n",
    "\n",
    "# create console handler and set level to info\n",
    "ch = logging.StreamHandler()\n",
    "ch.setLevel(logging.DEBUG)\n",
    "\n",
    "# create file handler and set level to info\n",
    "fh = logging.FileHandler(filename=f'10_thesis_results/{aldi_id}_{aldi_name}/{aldi_id}_{aldi_name}.log')\n",
    "fh.setLevel(logging.INFO)\n",
    "\n",
    "# create formatter\n",
    "formatter = logging.Formatter('%(asctime)s, %(name)s, %(levelname)s: %(message)s',\n",
    "                              datefmt='%m/%d/%Y %I:%M:%S %p')\n",
    "\n",
    "# add formatter to ch & fh\n",
    "ch.setFormatter(formatter)\n",
    "fh.setFormatter(formatter)\n",
    "\n",
    "# add ch to logger\n",
    "logger.addHandler(ch)\n",
    "logger.addHandler(fh)\n",
    "\n",
    "logger.info(aldi_name + ' starts its experiments now')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5acafa0e-ab50-4281-a47a-71af9afcd8dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# LOAD ENERGY CONSUMPTION DATA AND META DATA\n",
    "\n",
    "df_metadata = myDataImport.get_meta_data()\n",
    "df_timestamps = myDataImport.get_timestamps()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "85c1fd80-328f-4d9b-a812-4ddd35d59d7b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "10/20/2021 10:57:10 AM, aldi_gmm_dyn_none_daily, DEBUG: Discord calculation: Run Site 0\n",
      "10/20/2021 10:57:30 AM, aldi_gmm_dyn_none_daily, DEBUG: Discord calculation: Run Site 1\n",
      "10/20/2021 10:57:40 AM, aldi_gmm_dyn_none_daily, DEBUG: Discord calculation: Run Site 2\n",
      "10/20/2021 10:58:05 AM, aldi_gmm_dyn_none_daily, DEBUG: Discord calculation: Run Site 3\n",
      "10/20/2021 10:58:29 AM, aldi_gmm_dyn_none_daily, DEBUG: Discord calculation: Run Site 4\n",
      "10/20/2021 10:58:46 AM, aldi_gmm_dyn_none_daily, DEBUG: Discord calculation: Run Site 5\n",
      "10/20/2021 10:59:03 AM, aldi_gmm_dyn_none_daily, DEBUG: Discord calculation: Run Site 6\n",
      "10/20/2021 10:59:10 AM, aldi_gmm_dyn_none_daily, DEBUG: Discord calculation: Run Site 7\n",
      "10/20/2021 10:59:14 AM, aldi_gmm_dyn_none_daily, DEBUG: Discord calculation: Run Site 8\n",
      "10/20/2021 10:59:27 AM, aldi_gmm_dyn_none_daily, DEBUG: Discord calculation: Run Site 9\n",
      "10/20/2021 10:59:49 AM, aldi_gmm_dyn_none_daily, DEBUG: Discord calculation: Run Site 10\n",
      "10/20/2021 10:59:56 AM, aldi_gmm_dyn_none_daily, DEBUG: Discord calculation: Run Site 11\n",
      "10/20/2021 10:59:58 AM, aldi_gmm_dyn_none_daily, DEBUG: Discord calculation: Run Site 12\n",
      "10/20/2021 11:00:06 AM, aldi_gmm_dyn_none_daily, DEBUG: Discord calculation: Run Site 13\n",
      "10/20/2021 11:00:32 AM, aldi_gmm_dyn_none_daily, DEBUG: Discord calculation: Run Site 14\n",
      "10/20/2021 11:00:50 AM, aldi_gmm_dyn_none_daily, DEBUG: Discord calculation: Run Site 15\n",
      "10/20/2021 11:01:11 AM, aldi_gmm_dyn_none_daily, INFO: Runtime: 241.4149 sec\n",
      "10/20/2021 11:01:11 AM, aldi_gmm_dyn_none_daily, INFO: Runtime: 4.0236 min\n",
      "10/20/2021 11:01:11 AM, aldi_gmm_dyn_none_daily, INFO: aldi_gmm_dyn_none_daily ends discord label caluclation\n"
     ]
    }
   ],
   "source": [
    "# RUN DISCORD DETECTOR ALDI\n",
    "\n",
    "glb_start_time = time.time()\n",
    "for site_id in list_site_id:\n",
    "    logger.debug(f'Discord calculation: Run {list_site_name[site_id]}')\n",
    "    \n",
    "    # Select relevant energy consumption data + meta data\n",
    "    df_site_meter = myDataImport.get_meter_data([meter_type], [site_id])\n",
    "        \n",
    "    # execute aldi\n",
    "    aldi = ALDI(df_meters = df_site_meter, \n",
    "                df_metadata = df_metadata, \n",
    "                m = 24, \n",
    "                col_id = 'building_id', \n",
    "                site_id=site_id, \n",
    "                meter_id=meter_type,\n",
    "                verbose=False, \n",
    "                gpu=False,\n",
    "                hourly_processing=False,\n",
    "                aldi_name=aldi_name)\n",
    "    aldi.set_gmm_model(gmm_data = dict_param_exp['gmm_data'],\n",
    "                       gmm_max_comp = dict_param_exp['gmm_max_comp'])\n",
    "    \n",
    "    # request predicted discord label from aldi\n",
    "    df_pred_labels = aldi.get_result_df(forecast_out = False)\n",
    "    \n",
    "    # request predict discord label for predictor\n",
    "    df_pred_labels_pred = aldi.get_result_df(forecast_out = True)\n",
    "    \n",
    "    # keep track of all sites' predicted discord labels\n",
    "    dict_all_pred_labels[list_site_name[site_id]] = df_pred_labels\n",
    "    dict_all_pred_labels_pred[list_site_name[site_id]] = df_pred_labels_pred\n",
    "\n",
    "# adjustment to ensure that the labels have the correct format for timekeeping\n",
    "df_all_pred_labels = pd.concat(dict_all_pred_labels, axis=1)\n",
    "df_all_pred_labels.columns = df_all_pred_labels.columns.get_level_values(1)\n",
    "df_ref_timestamps = pd.DataFrame(df_timestamps.timestamp)\n",
    "df_ref_timestamps['timestamp'] = df_ref_timestamps['timestamp'].dt.date\n",
    "df_all_pred_labels['timestamp'] = df_all_pred_labels.index.date\n",
    "df_all_pred_label_hourly = pd.merge(df_ref_timestamps, df_all_pred_labels, on='timestamp')\n",
    "    \n",
    "# calculate runtime and store the value in a log file\n",
    "runtime_sec = time.time() - glb_start_time\n",
    "logger.info(f'Runtime: {round(runtime_sec, 4)} sec')\n",
    "logger.info(f'Runtime: {round(runtime_sec / 60 , 4) } min')\n",
    "logger.info(f'{aldi_name} ends discord label caluclation')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a3ecfcf5-effd-4221-9b41-8ad1037a1070",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "10/20/2021 11:01:11 AM, aldi_gmm_dyn_none_daily, DEBUG: Evaluation: Run Site 0\n",
      "10/20/2021 11:01:15 AM, aldi_gmm_dyn_none_daily, DEBUG: Evaluation: Run Site 1\n",
      "10/20/2021 11:01:18 AM, aldi_gmm_dyn_none_daily, DEBUG: Evaluation: Run Site 2\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "10/20/2021 11:01:22 AM, aldi_gmm_dyn_none_daily, DEBUG: Evaluation: Run Site 3\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "10/20/2021 11:01:30 AM, aldi_gmm_dyn_none_daily, DEBUG: Evaluation: Run Site 4\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "10/20/2021 11:01:34 AM, aldi_gmm_dyn_none_daily, DEBUG: Evaluation: Run Site 5\n",
      "/home/matias/Documents/matrix-profile-bdg/aldi_evaluation_metrics.py:223: RuntimeWarning: More than 20 figures have been opened. Figures created through the pyplot interface (`matplotlib.pyplot.figure`) are retained until explicitly closed and may consume too much memory. (To control this warning, see the rcParam `figure.max_open_warning`).\n",
      "  fig, ax = plt.subplots(figsize=figsize)\n",
      "10/20/2021 11:01:37 AM, aldi_gmm_dyn_none_daily, DEBUG: Evaluation: Run Site 6\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "10/20/2021 11:01:40 AM, aldi_gmm_dyn_none_daily, DEBUG: Evaluation: Run Site 7\n",
      "10/20/2021 11:01:42 AM, aldi_gmm_dyn_none_daily, DEBUG: Evaluation: Run Site 8\n",
      "10/20/2021 11:01:45 AM, aldi_gmm_dyn_none_daily, DEBUG: Evaluation: Run Site 9\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "10/20/2021 11:01:49 AM, aldi_gmm_dyn_none_daily, DEBUG: Evaluation: Run Site 10\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "10/20/2021 11:01:52 AM, aldi_gmm_dyn_none_daily, DEBUG: Evaluation: Run Site 11\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/home/matias/anaconda3/envs/matrix-profile-bdg/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1245: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "10/20/2021 11:01:54 AM, aldi_gmm_dyn_none_daily, DEBUG: Evaluation: Run Site 12\n",
      "10/20/2021 11:01:56 AM, aldi_gmm_dyn_none_daily, DEBUG: Evaluation: Run Site 13\n",
      "10/20/2021 11:02:01 AM, aldi_gmm_dyn_none_daily, DEBUG: Evaluation: Run Site 14\n",
      "10/20/2021 11:02:04 AM, aldi_gmm_dyn_none_daily, DEBUG: Evaluation: Run Site 15\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# CALCULATE DIFFERENT METRICS\n",
    "\n",
    "# 1. SITE WISE METRICS\n",
    "for site_id in list_site_id:\n",
    "    logger.debug(f'Evaluation: Run {list_site_name[site_id]}')\n",
    "    \n",
    "    # load predicted sites' discord label (daily)\n",
    "    df_curr_pred_label_daily = dict_all_pred_labels[list_site_name[site_id]]\n",
    "    \n",
    "    # load true sites' discord label + aggregate\n",
    "    df_curr_true_labels_hourly = myDataImport.get_label_data([meter_type], [site_id])\n",
    "    \n",
    "    df_curr_true_labels_or_daily = get_daily_resolution(df_hourly_data=df_curr_true_labels_hourly, \n",
    "                                                        agg_method='logic_or')\n",
    "    df_curr_true_labels_and_daily = get_daily_resolution(df_hourly_data=df_curr_true_labels_hourly, \n",
    "                                                         agg_method='logic_and')\n",
    "    df_curr_true_labels_majo_daily = get_daily_resolution(df_hourly_data=df_curr_true_labels_hourly, \n",
    "                                                          agg_method='majority')\n",
    "    df_curr_true_labels_majoplus_daily = get_daily_resolution(df_hourly_data=df_curr_true_labels_hourly, \n",
    "                                                              agg_method='majority_plus')\n",
    "    \n",
    "    # keep track of all aggregate true labels\n",
    "    dict_all_true_labels_or[list_site_name[site_id]] = df_curr_true_labels_or_daily\n",
    "    dict_all_true_labels_and[list_site_name[site_id]] = df_curr_true_labels_and_daily\n",
    "    dict_all_true_labels_majo[list_site_name[site_id]] = df_curr_true_labels_majo_daily\n",
    "    dict_all_true_labels_majoplus[list_site_name[site_id]] = df_curr_true_labels_majoplus_daily\n",
    "    \n",
    "        \n",
    "    \n",
    "    # Calculate ROC AUC metric per site & aggregation method\n",
    "    df_roc_auc_or = myEvalMetrics.get_roc_auc(df_true=df_curr_true_labels_or_daily,\n",
    "                                              df_pred=df_curr_pred_label_daily)\n",
    "    df_roc_auc_and = myEvalMetrics.get_roc_auc(df_true=df_curr_true_labels_and_daily,\n",
    "                                               df_pred=df_curr_pred_label_daily)\n",
    "    df_roc_auc_majo = myEvalMetrics.get_roc_auc(df_true=df_curr_true_labels_majo_daily,\n",
    "                                                df_pred=df_curr_pred_label_daily)\n",
    "    df_roc_auc_majoplus = myEvalMetrics.get_roc_auc(df_true=df_curr_true_labels_majoplus_daily,\n",
    "                                                    df_pred=df_curr_pred_label_daily)\n",
    "    \n",
    "    # Keep track of all ROC AUC metric values\n",
    "    dict_all_roc_auc_or[list_site_name[site_id]] = df_roc_auc_or\n",
    "    dict_all_roc_auc_and[list_site_name[site_id]] = df_roc_auc_and\n",
    "    dict_all_roc_auc_majo[list_site_name[site_id]] = df_roc_auc_majo\n",
    "    dict_all_roc_auc_majoplus[list_site_name[site_id]] = df_roc_auc_majoplus\n",
    "    \n",
    "    # Prepare Confusion matrix/class report per site & aggregation method\n",
    "    myEvalMetrics.get_class_report(df_true = df_curr_true_labels_or_daily,\n",
    "                                   df_pred = df_curr_pred_label_daily, \n",
    "                                   aldi_impl = aldi_name, \n",
    "                                   level_name = list_site_name[site_id], \n",
    "                                   meter_type = meter_type, \n",
    "                                   path = f'10_thesis_results/{aldi_id}_{aldi_name}/or-exp_id{dict_param_exp[\"exp_id\"]}')\n",
    "    myEvalMetrics.get_class_report(df_true = df_curr_true_labels_and_daily,\n",
    "                                   df_pred = df_curr_pred_label_daily, \n",
    "                                   aldi_impl = aldi_name, \n",
    "                                   level_name = list_site_name[site_id], \n",
    "                                   meter_type = meter_type, \n",
    "                                   path = f'10_thesis_results/{aldi_id}_{aldi_name}/and-exp_id{dict_param_exp[\"exp_id\"]}')\n",
    "    myEvalMetrics.get_class_report(df_true = df_curr_true_labels_majo_daily,\n",
    "                                   df_pred = df_curr_pred_label_daily, \n",
    "                                   aldi_impl = aldi_name, \n",
    "                                   level_name = list_site_name[site_id], \n",
    "                                   meter_type = meter_type, \n",
    "                                   path = f'10_thesis_results/{aldi_id}_{aldi_name}/majo-exp_id{dict_param_exp[\"exp_id\"]}')\n",
    "    myEvalMetrics.get_class_report(df_true = df_curr_true_labels_majoplus_daily,\n",
    "                                   df_pred = df_curr_pred_label_daily, \n",
    "                                   aldi_impl = aldi_name, \n",
    "                                   level_name = list_site_name[site_id], \n",
    "                                   meter_type = meter_type, \n",
    "                                   path = f'10_thesis_results/{aldi_id}_{aldi_name}/majoplus-exp_id{dict_param_exp[\"exp_id\"]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "faf64d87-3da2-485c-b89c-846c2dc5c7a2",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "10/20/2021 11:02:08 AM, aldi_gmm_dyn_none_daily, DEBUG: Evaluation: dataset wide metrics/results\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'              precision    recall  f1-score   support\\n\\n           0       0.94      0.31      0.46    469166\\n           1       0.11      0.82      0.19     47992\\n\\n    accuracy                           0.36    517158\\n   macro avg       0.53      0.57      0.33    517158\\nweighted avg       0.87      0.36      0.44    517158\\n'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 720x720 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 2. DATASET WISE METRICS\n",
    "logger.debug(f'Evaluation: dataset wide metrics/results')\n",
    "\n",
    "# store all ROC AUC metric values\n",
    "df_all_roc_auc_or = pd.DataFrame.from_dict(dict_all_roc_auc_or, orient='index', columns=['roc_auc'])\n",
    "df_all_roc_auc_or.to_csv(f'10_thesis_results/{aldi_id}_{aldi_name}/or-exp_id{dict_param_exp[\"exp_id\"]}/{curr_timestamp}-roc_auc.csv')\n",
    "\n",
    "df_all_roc_auc_and = pd.DataFrame.from_dict(dict_all_roc_auc_and, orient='index', columns=['roc_auc'])\n",
    "df_all_roc_auc_and.to_csv(f'10_thesis_results/{aldi_id}_{aldi_name}/and-exp_id{dict_param_exp[\"exp_id\"]}/{curr_timestamp}-roc_auc.csv')\n",
    "\n",
    "df_all_roc_auc_majo = pd.DataFrame.from_dict(dict_all_roc_auc_majo, orient='index', columns=['roc_auc'])\n",
    "df_all_roc_auc_majo.to_csv(f'10_thesis_results/{aldi_id}_{aldi_name}/majo-exp_id{dict_param_exp[\"exp_id\"]}/{curr_timestamp}-roc_auc.csv')\n",
    "\n",
    "df_all_roc_auc_majoplus = pd.DataFrame.from_dict(dict_all_roc_auc_majoplus, orient='index', columns=['roc_auc'])\n",
    "df_all_roc_auc_majoplus.to_csv(f'10_thesis_results/{aldi_id}_{aldi_name}/majoplus-exp_id{dict_param_exp[\"exp_id\"]}/{curr_timestamp}-roc_auc.csv')\n",
    "\n",
    "\n",
    "# create a single confusion matrix for each agg method\n",
    "myEvalMetrics.get_class_report(df_true = pd.concat(dict_all_true_labels_or, axis=1), \n",
    "                               df_pred = pd.concat(dict_all_pred_labels, axis=1), \n",
    "                               aldi_impl = aldi_name, \n",
    "                               level_name = 'all', \n",
    "                               meter_type = meter_type,\n",
    "                               path = f'10_thesis_results/{aldi_id}_{aldi_name}/or-exp_id{dict_param_exp[\"exp_id\"]}')\n",
    "myEvalMetrics.get_class_report(df_true = pd.concat(dict_all_true_labels_and, axis=1), \n",
    "                               df_pred = pd.concat(dict_all_pred_labels, axis=1), \n",
    "                               aldi_impl = aldi_name, \n",
    "                               level_name = 'all', \n",
    "                               meter_type = meter_type,\n",
    "                               path = f'10_thesis_results/{aldi_id}_{aldi_name}/and-exp_id{dict_param_exp[\"exp_id\"]}')\n",
    "myEvalMetrics.get_class_report(df_true = pd.concat(dict_all_true_labels_majo, axis=1), \n",
    "                               df_pred = pd.concat(dict_all_pred_labels, axis=1), \n",
    "                               aldi_impl = aldi_name, \n",
    "                               level_name = 'all', \n",
    "                               meter_type = meter_type,\n",
    "                               path = f'10_thesis_results/{aldi_id}_{aldi_name}/majo-exp_id{dict_param_exp[\"exp_id\"]}')\n",
    "myEvalMetrics.get_class_report(df_true = pd.concat(dict_all_true_labels_majoplus, axis=1), \n",
    "                               df_pred = pd.concat(dict_all_pred_labels, axis=1), \n",
    "                               aldi_impl = aldi_name, \n",
    "                               level_name = 'all', \n",
    "                               meter_type = meter_type,\n",
    "                               path = f'10_thesis_results/{aldi_id}_{aldi_name}/majoplus-exp_id{dict_param_exp[\"exp_id\"]}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "52fcca82-f7a9-4b68-bd12-b52e06a925ab",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20216100, 4)\n"
     ]
    }
   ],
   "source": [
    "# 3. BUILD EXPORTABLE FILE FOR PREDICTOR\n",
    "\n",
    "df_export_labels = pd.concat(dict_all_pred_labels_pred, axis=0)\n",
    "df_export_labels['meter'] = [0] * df_export_labels.shape[0]\n",
    "df_export_labels.index = df_export_labels.index.get_level_values(1)\n",
    "\n",
    "df_left_keys = myDataImport.get_timestamps_buildings(resolution='H')\n",
    "df_exportable = pd.merge(df_left_keys, df_export_labels, how=\"left\", on=[\"timestamp\", \"building_id\", \"meter\"])\n",
    "\n",
    "# Attention: NaNs are padded with 0\n",
    "df_exportable = df_exportable.fillna(0)\n",
    "df_exportable['is_discord'] = df_exportable['is_discord'].astype('int8')\n",
    "\n",
    "print(df_exportable.shape)\n",
    "\n",
    "# Export\n",
    "df_exportable['is_discord'].to_csv(f'10_thesis_results/{aldi_id}_{aldi_name}/{curr_timestamp}-discords-exp_id{dict_param_exp[\"exp_id\"]}.csv', index=False) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e54ddd96-b64f-4e6d-b19f-c9944345176b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}