.. DO NOT EDIT.
.. THIS FILE WAS AUTOMATICALLY GENERATED BY SPHINX-GALLERY.
.. TO MAKE CHANGES, EDIT THE SOURCE PYTHON FILE:
.. "auto_examples/further_examples/plot_4_upperLower.py"
.. LINE NUMBERS ARE GIVEN BELOW.

.. only:: html

    .. note::
        :class: sphx-glr-download-link-note

        Click :ref:`here <sphx_glr_download_auto_examples_further_examples_plot_4_upperLower.py>`
        to download the full example code

.. rst-class:: sphx-glr-example-title

.. _sphx_glr_auto_examples_further_examples_plot_4_upperLower.py:


.. _pruebas:

Example: Use of Upper and Lower bound as error estimation
==================================================================

This example is an extension to `ex2` where we will prove how the upper and
lower bound of the loss are an unbiased estimator of the error. The models are
trained with different number of cases ranging from 10% to 80% of the data and
then are tested with 20% of the samples. The graphs show how in most of the
cases the error is between those bounds which proves the potential of this
feature of the MRCs. The results are for a
:mod:`MRC(phi = 'fourier', loss = '0-1', s = 1)`


.. note::    Note that there is an additional dataset related to COVID-19
             patients that is available upon requesting to HM Hospitales
             `here
             <www.hmhospitales.com/coronavirus/covid-data-save-lives/>`_.
             More information about this dataset can be found in the
             `COVID example<ex_covid>`

.. GENERATED FROM PYTHON SOURCE LINES 24-182

.. code-block:: default


    # Import needed modules
    import time
    import warnings

    import matplotlib.pyplot as plt
    import numpy as np
    import pandas as pd
    import seaborn as sns
    from imblearn.over_sampling import SMOTE
    from sklearn import preprocessing
    from sklearn.model_selection import RepeatedStratifiedKFold

    from MRCpy import MRC
    from MRCpy.datasets import *


    sns.set_style("whitegrid")
    sns.set_context("paper")
    warnings.filterwarnings("ignore")


    def load_covid(norm=False, array=True):
        data_consensus = pd.read_csv("data/data_consensus.csv", sep=";")

        variable_dict = {
            "CD0000AGE": "Age",
            "CORE": "PATIENT_ID",
            "CT000000U": "Urea",
            "CT00000BT": "Bilirubin",
            "CT00000NA": "Sodium",
            "CT00000TP": "Proth_time",
            "CT0000COM": "Com",
            "CT0000LDH": "LDH",
            "CT0000NEU": "Neutrophils",
            "CT0000PCR": "Pro_C_Rea",
            "CT0000VCM": "Med_corp_vol",
            "CT000APTT": "Ceph_time",
            "CT000CHCM": "Mean_corp_Hgb",
            "CT000EOSP": "Eosinophils%",
            "CT000LEUC": "Leukocytes",
            "CT000LINP": "Lymphocytes%",
            "CT000NEUP": "Neutrophils%",
            "CT000PLAQ": "Platelet_count",
            "CTHSDXXRATE": "Rate",
            "CTHSDXXSAT": "Sat",
            "ED0DISWHY": "Status",
            "F_INGRESO/ADMISSION_D_ING/INPAT": "Fecha_admision",
            "SEXO/SEX": "Sexo",
        }
        data_consensus = data_consensus.rename(columns=variable_dict)
        if norm:
            x_consensus = data_consensus[
                data_consensus.columns.difference(["Status", "PATIENT_ID"])
            ][:]
            std_scale = preprocessing.StandardScaler().fit(x_consensus)
            x_consensus_std = std_scale.transform(x_consensus)
            dataframex_consensus = pd.DataFrame(
                x_consensus_std, columns=x_consensus.columns
            )
            data_consensus.reset_index(drop=True, inplace=True)
            data_consensus = pd.concat(
                [dataframex_consensus, data_consensus[["Status"]]], axis=1
            )
        data_consensus = data_consensus[data_consensus.columns.difference(
            ["PATIENT_ID"])]
        X = data_consensus[data_consensus.columns.difference(
            ["Status", "PATIENT_ID"])]
        y = data_consensus["Status"]
        if array:
            X = X.to_numpy()
            y = y.to_numpy()
        return X, y


    def getUpperLowerdf(train_size, X, y, cv, paramsMRC, smote=True):
        """
        Parameters
        ----------
        train_size : array
            Array of different training sizes to train the model.
        cv : CrossValidator
            Cross validator.
        paramsMRC : TYPE
            Parameters for the MRCs.
        smote : Bool, optional
            Class imbalance corrector, set to false to disable. The default is
            True.
        Returns
        -------
        table : dataFrame
            Dataframe with the results of the training for each training size.

        """
        if smote:
            smotefit = SMOTE(sampling_strategy="auto")
            X, y = smotefit.fit_resample(X, y)
        table = pd.DataFrame()
        for train_set in train_size:
            for j, (train_index, test_index) in enumerate(cv.split(X, y)):
                X_train, X_test = X[train_index], X[test_index]
                y_train, y_test = y[train_index], y[test_index]

                random_indices = np.random.choice(
                    X_train.shape[0], size=int(X.shape[0] * train_set),
                    replace=False,
                )
                X_train = X_train[random_indices, :]
                y_train = y_train[random_indices]
                std_scale = preprocessing.StandardScaler().fit(X_train, y_train)
                X_train = std_scale.transform(X_train)
                X_test = std_scale.transform(X_test)
                start_time = time.time()
                MRC_model = MRC(phi="fourier", s=1, **
                                paramsMRC).fit(X_train, y_train)
                train_time = time.time() - start_time
                auxtable = pd.DataFrame(
                    columns=["Error", "Upper", "Lower", "iteration", "train_size",
                             "Time", ],
                    index=range(0, 1),
                )
                auxtable["train_size"] = train_set
                auxtable["iteration"] = j
                auxtable["Error"] = 1 - MRC_model.score(X_test, y_test)
                auxtable["Time"] = train_time
                auxtable["Upper"] = MRC_model.get_upper_bound()
                auxtable["Lower"] = MRC_model.get_lower_bound()

                table = table.append(auxtable, ignore_index=True)
        return table


    # Data sets
    loaders = [
        load_mammographic,
        load_haberman,
        load_indian_liver,
        load_diabetes,
        load_credit,
        load_covid,
    ]

    dataName = [
        "mammographic",
        "haberman",
        "indian_liver",
        "diabetes",
        "credit",
        "COVID",
    ]
    paramsMRC = {
        "deterministic": False,
        "fit_intercept": False,
        "use_cvx": True,
        "loss": "0-1",
    }
    train = np.arange(0.1, 0.81, 0.1)








.. GENERATED FROM PYTHON SOURCE LINES 183-187

Cross test validation
----------------------------------------
5 fold repeated Stratified Cross validation is performed where each of the
fold is trained with 80% of the data and then tested with the remaining 20%

.. GENERATED FROM PYTHON SOURCE LINES 187-193

.. code-block:: default


    n_splits = 5
    n_repeats = 10
    cv = RepeatedStratifiedKFold(n_splits=n_splits, n_repeats=n_repeats,
                                 random_state=1)








.. GENERATED FROM PYTHON SOURCE LINES 194-202

Results
------------------------------
We will present the results for the 6 datasets. For more information
about the dataset refer to the
`MRCpy documentation <https://machinelearningbcam.github.io/MRCpy>`_ of the
loaders. In the results we can see how the upper and lower bounds get closer
when the training size is increased. Furthermore, the standard deviation of
both bounds is reduced significantly.

.. GENERATED FROM PYTHON SOURCE LINES 204-206

Mammographic
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

.. GENERATED FROM PYTHON SOURCE LINES 206-221

.. code-block:: default

    X, y = load_mammographic()
    table = getUpperLowerdf(train, X, y, cv, paramsMRC)
    # dataframes.append(table)
    # plotUpperLower(table)
    means = table[table.columns.difference(["iteration"])].groupby(
        "train_size").mean()
    std = table[table.columns.difference(["iteration"])].groupby(
        "train_size").std()
    for column in means.columns:
        means[column] = (
            means[column].round(3).astype(str) + " ± " + std[column].round(
                3).astype(str)
        )
    means[["Error", "Upper", "Lower", "Time"]]






.. raw:: html

    <div class="output_subarea output_html rendered_html output_result">
    <div>
    <style scoped>
        .dataframe tbody tr th:only-of-type {
            vertical-align: middle;
        }

        .dataframe tbody tr th {
            vertical-align: top;
        }

        .dataframe thead th {
            text-align: right;
        }
    </style>
    <table border="1" class="dataframe">
      <thead>
        <tr style="text-align: right;">
          <th></th>
          <th>Error</th>
          <th>Upper</th>
          <th>Lower</th>
          <th>Time</th>
        </tr>
        <tr>
          <th>train_size</th>
          <th></th>
          <th></th>
          <th></th>
          <th></th>
        </tr>
      </thead>
      <tbody>
        <tr>
          <th>0.1</th>
          <td>0.206 ± 0.028</td>
          <td>0.214 ± 0.031</td>
          <td>0.104 ± 0.03</td>
          <td>0.189 ± 0.012</td>
        </tr>
        <tr>
          <th>0.2</th>
          <td>0.195 ± 0.027</td>
          <td>0.216 ± 0.025</td>
          <td>0.134 ± 0.027</td>
          <td>0.336 ± 0.016</td>
        </tr>
        <tr>
          <th>0.3</th>
          <td>0.2 ± 0.032</td>
          <td>0.215 ± 0.017</td>
          <td>0.145 ± 0.02</td>
          <td>0.482 ± 0.029</td>
        </tr>
        <tr>
          <th>0.4</th>
          <td>0.202 ± 0.031</td>
          <td>0.215 ± 0.018</td>
          <td>0.154 ± 0.022</td>
          <td>0.579 ± 0.027</td>
        </tr>
        <tr>
          <th>0.5</th>
          <td>0.191 ± 0.032</td>
          <td>0.214 ± 0.015</td>
          <td>0.158 ± 0.019</td>
          <td>0.672 ± 0.029</td>
        </tr>
        <tr>
          <th>0.6</th>
          <td>0.201 ± 0.038</td>
          <td>0.22 ± 0.014</td>
          <td>0.168 ± 0.017</td>
          <td>0.782 ± 0.033</td>
        </tr>
        <tr>
          <th>0.7</th>
          <td>0.201 ± 0.03</td>
          <td>0.217 ± 0.013</td>
          <td>0.171 ± 0.016</td>
          <td>0.877 ± 0.028</td>
        </tr>
        <tr>
          <th>0.8</th>
          <td>0.206 ± 0.03</td>
          <td>0.219 ± 0.012</td>
          <td>0.175 ± 0.013</td>
          <td>0.965 ± 0.043</td>
        </tr>
      </tbody>
    </table>
    </div>
    </div>
    <br />
    <br />

.. GENERATED FROM PYTHON SOURCE LINES 222-245

.. code-block:: default

    fig, ax = plt.subplots()
    sns.lineplot(data=table, x="train_size", y="Error", label="Test Error", ax=ax)
    sns.lineplot(
        data=table,
        x="train_size",
        y="Upper",
        color="red",
        label="Upper bound",
        linestyle="dotted",
        ax=ax,
    )
    sns.lineplot(
        data=table,
        x="train_size",
        y="Lower",
        color="green",
        label="Lower bound",
        linestyle="dotted",
        ax=ax,
    )
    plt.suptitle("Mammographic")
    plt.show()




.. image-sg:: /auto_examples/further_examples/images/sphx_glr_plot_4_upperLower_001.png
   :alt: Mammographic
   :srcset: /auto_examples/further_examples/images/sphx_glr_plot_4_upperLower_001.png
   :class: sphx-glr-single-img





.. GENERATED FROM PYTHON SOURCE LINES 246-248

Haberman
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

.. GENERATED FROM PYTHON SOURCE LINES 248-262

.. code-block:: default


    X, y = load_haberman()
    table = getUpperLowerdf(train, X, y, cv, paramsMRC)
    means = table[table.columns.difference(
        ["iteration"])].groupby("train_size").mean()
    std = table[table.columns.difference(
        ["iteration"])].groupby("train_size").std()
    for column in means.columns:
        means[column] = (
            means[column].round(3).astype(
                str) + " ± " + std[column].round(3).astype(str)
        )
    means[["Error", "Upper", "Lower", "Time"]]






.. raw:: html

    <div class="output_subarea output_html rendered_html output_result">
    <div>
    <style scoped>
        .dataframe tbody tr th:only-of-type {
            vertical-align: middle;
        }

        .dataframe tbody tr th {
            vertical-align: top;
        }

        .dataframe thead th {
            text-align: right;
        }
    </style>
    <table border="1" class="dataframe">
      <thead>
        <tr style="text-align: right;">
          <th></th>
          <th>Error</th>
          <th>Upper</th>
          <th>Lower</th>
          <th>Time</th>
        </tr>
        <tr>
          <th>train_size</th>
          <th></th>
          <th></th>
          <th></th>
          <th></th>
        </tr>
      </thead>
      <tbody>
        <tr>
          <th>0.1</th>
          <td>0.414 ± 0.061</td>
          <td>0.36 ± 0.037</td>
          <td>0.139 ± 0.053</td>
          <td>0.101 ± 0.005</td>
        </tr>
        <tr>
          <th>0.2</th>
          <td>0.384 ± 0.057</td>
          <td>0.376 ± 0.026</td>
          <td>0.206 ± 0.042</td>
          <td>0.171 ± 0.007</td>
        </tr>
        <tr>
          <th>0.3</th>
          <td>0.385 ± 0.045</td>
          <td>0.376 ± 0.019</td>
          <td>0.241 ± 0.03</td>
          <td>0.251 ± 0.008</td>
        </tr>
        <tr>
          <th>0.4</th>
          <td>0.369 ± 0.044</td>
          <td>0.381 ± 0.017</td>
          <td>0.254 ± 0.026</td>
          <td>0.319 ± 0.01</td>
        </tr>
        <tr>
          <th>0.5</th>
          <td>0.36 ± 0.045</td>
          <td>0.375 ± 0.013</td>
          <td>0.255 ± 0.022</td>
          <td>0.388 ± 0.017</td>
        </tr>
        <tr>
          <th>0.6</th>
          <td>0.351 ± 0.041</td>
          <td>0.379 ± 0.011</td>
          <td>0.266 ± 0.019</td>
          <td>0.467 ± 0.017</td>
        </tr>
        <tr>
          <th>0.7</th>
          <td>0.35 ± 0.048</td>
          <td>0.377 ± 0.009</td>
          <td>0.272 ± 0.016</td>
          <td>0.514 ± 0.02</td>
        </tr>
        <tr>
          <th>0.8</th>
          <td>0.357 ± 0.042</td>
          <td>0.378 ± 0.007</td>
          <td>0.278 ± 0.011</td>
          <td>0.57 ± 0.023</td>
        </tr>
      </tbody>
    </table>
    </div>
    </div>
    <br />
    <br />

.. GENERATED FROM PYTHON SOURCE LINES 263-286

.. code-block:: default

    fig, ax = plt.subplots()
    sns.lineplot(data=table, x="train_size", y="Error", label="Test Error", ax=ax)
    sns.lineplot(
        data=table,
        x="train_size",
        y="Upper",
        color="red",
        label="Upper bound",
        linestyle="dotted",
        ax=ax,
    )
    sns.lineplot(
        data=table,
        x="train_size",
        y="Lower",
        color="green",
        label="Lower bound",
        linestyle="dotted",
        ax=ax,
    )
    plt.suptitle("Haberman")
    plt.show()




.. image-sg:: /auto_examples/further_examples/images/sphx_glr_plot_4_upperLower_002.png
   :alt: Haberman
   :srcset: /auto_examples/further_examples/images/sphx_glr_plot_4_upperLower_002.png
   :class: sphx-glr-single-img





.. GENERATED FROM PYTHON SOURCE LINES 287-289

Indian liver
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

.. GENERATED FROM PYTHON SOURCE LINES 289-302

.. code-block:: default

    X, y = load_indian_liver()

    table = getUpperLowerdf(train, X, y, cv, paramsMRC)
    means = table[table.columns.difference(
        ["iteration"])].groupby("train_size").mean()
    std = table[table.columns.difference(
        ["iteration"])].groupby("train_size").std()
    for column in means.columns:
        means[column] = (
            means[column].round(3).astype(str) + " ± " +
            std[column].round(3).astype(str)
        )
    means[["Error", "Upper", "Lower", "Time"]]





.. raw:: html

    <div class="output_subarea output_html rendered_html output_result">
    <div>
    <style scoped>
        .dataframe tbody tr th:only-of-type {
            vertical-align: middle;
        }

        .dataframe tbody tr th {
            vertical-align: top;
        }

        .dataframe thead th {
            text-align: right;
        }
    </style>
    <table border="1" class="dataframe">
      <thead>
        <tr style="text-align: right;">
          <th></th>
          <th>Error</th>
          <th>Upper</th>
          <th>Lower</th>
          <th>Time</th>
        </tr>
        <tr>
          <th>train_size</th>
          <th></th>
          <th></th>
          <th></th>
          <th></th>
        </tr>
      </thead>
      <tbody>
        <tr>
          <th>0.1</th>
          <td>0.383 ± 0.036</td>
          <td>0.344 ± 0.03</td>
          <td>0.163 ± 0.031</td>
          <td>0.159 ± 0.005</td>
        </tr>
        <tr>
          <th>0.2</th>
          <td>0.367 ± 0.035</td>
          <td>0.358 ± 0.024</td>
          <td>0.223 ± 0.029</td>
          <td>0.299 ± 0.011</td>
        </tr>
        <tr>
          <th>0.3</th>
          <td>0.356 ± 0.038</td>
          <td>0.364 ± 0.015</td>
          <td>0.252 ± 0.02</td>
          <td>0.438 ± 0.025</td>
        </tr>
        <tr>
          <th>0.4</th>
          <td>0.353 ± 0.034</td>
          <td>0.361 ± 0.012</td>
          <td>0.266 ± 0.015</td>
          <td>0.539 ± 0.018</td>
        </tr>
        <tr>
          <th>0.5</th>
          <td>0.341 ± 0.035</td>
          <td>0.36 ± 0.012</td>
          <td>0.273 ± 0.015</td>
          <td>0.668 ± 0.02</td>
        </tr>
        <tr>
          <th>0.6</th>
          <td>0.348 ± 0.031</td>
          <td>0.361 ± 0.009</td>
          <td>0.279 ± 0.01</td>
          <td>0.805 ± 0.023</td>
        </tr>
        <tr>
          <th>0.7</th>
          <td>0.344 ± 0.035</td>
          <td>0.361 ± 0.007</td>
          <td>0.284 ± 0.008</td>
          <td>0.941 ± 0.028</td>
        </tr>
        <tr>
          <th>0.8</th>
          <td>0.346 ± 0.033</td>
          <td>0.36 ± 0.006</td>
          <td>0.288 ± 0.007</td>
          <td>1.062 ± 0.028</td>
        </tr>
      </tbody>
    </table>
    </div>
    </div>
    <br />
    <br />

.. GENERATED FROM PYTHON SOURCE LINES 303-325

.. code-block:: default

    fig, ax = plt.subplots()
    sns.lineplot(data=table, x="train_size", y="Error", label="Test Error", ax=ax)
    sns.lineplot(
        data=table,
        x="train_size",
        y="Upper",
        color="red",
        label="Upper bound",
        linestyle="dotted",
        ax=ax,
    )
    sns.lineplot(
        data=table,
        x="train_size",
        y="Lower",
        color="green",
        label="Lower bound",
        linestyle="dotted",
        ax=ax,
    )
    plt.suptitle("Indian Liver")
    plt.show()



.. image-sg:: /auto_examples/further_examples/images/sphx_glr_plot_4_upperLower_003.png
   :alt: Indian Liver
   :srcset: /auto_examples/further_examples/images/sphx_glr_plot_4_upperLower_003.png
   :class: sphx-glr-single-img





.. GENERATED FROM PYTHON SOURCE LINES 326-328

diabetes
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

.. GENERATED FROM PYTHON SOURCE LINES 328-342

.. code-block:: default

    X, y = load_diabetes()

    table = getUpperLowerdf(train, X, y, cv, paramsMRC)
    means = table[table.columns.difference(
        ["iteration"])].groupby("train_size").mean()
    std = table[table.columns.difference(
        ["iteration"])].groupby("train_size").std()
    for column in means.columns:
        means[column] = (
            means[column].round(3).astype(str) + " ± " +
            std[column].round(3).astype(str)
        )
    means[["Error", "Upper", "Lower", "Time"]]






.. raw:: html

    <div class="output_subarea output_html rendered_html output_result">
    <div>
    <style scoped>
        .dataframe tbody tr th:only-of-type {
            vertical-align: middle;
        }

        .dataframe tbody tr th {
            vertical-align: top;
        }

        .dataframe thead th {
            text-align: right;
        }
    </style>
    <table border="1" class="dataframe">
      <thead>
        <tr style="text-align: right;">
          <th></th>
          <th>Error</th>
          <th>Upper</th>
          <th>Lower</th>
          <th>Time</th>
        </tr>
        <tr>
          <th>train_size</th>
          <th></th>
          <th></th>
          <th></th>
          <th></th>
        </tr>
      </thead>
      <tbody>
        <tr>
          <th>0.1</th>
          <td>0.332 ± 0.034</td>
          <td>0.292 ± 0.027</td>
          <td>0.093 ± 0.032</td>
          <td>0.187 ± 0.006</td>
        </tr>
        <tr>
          <th>0.2</th>
          <td>0.31 ± 0.037</td>
          <td>0.302 ± 0.021</td>
          <td>0.173 ± 0.029</td>
          <td>0.356 ± 0.012</td>
        </tr>
        <tr>
          <th>0.3</th>
          <td>0.3 ± 0.038</td>
          <td>0.309 ± 0.013</td>
          <td>0.201 ± 0.016</td>
          <td>0.537 ± 0.02</td>
        </tr>
        <tr>
          <th>0.4</th>
          <td>0.298 ± 0.032</td>
          <td>0.31 ± 0.013</td>
          <td>0.217 ± 0.012</td>
          <td>0.654 ± 0.021</td>
        </tr>
        <tr>
          <th>0.5</th>
          <td>0.299 ± 0.033</td>
          <td>0.311 ± 0.01</td>
          <td>0.23 ± 0.013</td>
          <td>0.816 ± 0.023</td>
        </tr>
        <tr>
          <th>0.6</th>
          <td>0.296 ± 0.029</td>
          <td>0.311 ± 0.008</td>
          <td>0.236 ± 0.011</td>
          <td>0.981 ± 0.03</td>
        </tr>
        <tr>
          <th>0.7</th>
          <td>0.295 ± 0.031</td>
          <td>0.31 ± 0.006</td>
          <td>0.238 ± 0.008</td>
          <td>1.152 ± 0.048</td>
        </tr>
        <tr>
          <th>0.8</th>
          <td>0.301 ± 0.033</td>
          <td>0.31 ± 0.005</td>
          <td>0.24 ± 0.008</td>
          <td>1.299 ± 0.031</td>
        </tr>
      </tbody>
    </table>
    </div>
    </div>
    <br />
    <br />

.. GENERATED FROM PYTHON SOURCE LINES 343-365

.. code-block:: default

    fig, ax = plt.subplots()
    sns.lineplot(data=table, x="train_size", y="Error", label="Test Error", ax=ax)
    sns.lineplot(
        data=table,
        x="train_size",
        y="Upper",
        color="red",
        label="Upper bound",
        linestyle="dotted",
        ax=ax,
    )
    sns.lineplot(
        data=table,
        x="train_size",
        y="Lower",
        color="green",
        label="Lower bound",
        linestyle="dotted",
        ax=ax,
    )
    plt.suptitle("Diabetes")
    plt.show()



.. image-sg:: /auto_examples/further_examples/images/sphx_glr_plot_4_upperLower_004.png
   :alt: Diabetes
   :srcset: /auto_examples/further_examples/images/sphx_glr_plot_4_upperLower_004.png
   :class: sphx-glr-single-img





.. GENERATED FROM PYTHON SOURCE LINES 366-368

credit
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

.. GENERATED FROM PYTHON SOURCE LINES 368-382

.. code-block:: default

    X, y = load_credit()

    table = getUpperLowerdf(train, X, y, cv, paramsMRC)
    means = table[table.columns.difference(
        ["iteration"])].groupby("train_size").mean()
    std = table[table.columns.difference(
        ["iteration"])].groupby("train_size").std()
    for column in means.columns:
        means[column] = (
            means[column].round(3).astype(str) + " ± " +
            std[column].round(3).astype(str)
        )
    means[["Error", "Upper", "Lower", "Time"]]






.. raw:: html

    <div class="output_subarea output_html rendered_html output_result">
    <div>
    <style scoped>
        .dataframe tbody tr th:only-of-type {
            vertical-align: middle;
        }

        .dataframe tbody tr th {
            vertical-align: top;
        }

        .dataframe thead th {
            text-align: right;
        }
    </style>
    <table border="1" class="dataframe">
      <thead>
        <tr style="text-align: right;">
          <th></th>
          <th>Error</th>
          <th>Upper</th>
          <th>Lower</th>
          <th>Time</th>
        </tr>
        <tr>
          <th>train_size</th>
          <th></th>
          <th></th>
          <th></th>
          <th></th>
        </tr>
      </thead>
      <tbody>
        <tr>
          <th>0.1</th>
          <td>0.225 ± 0.032</td>
          <td>0.203 ± 0.024</td>
          <td>0.016 ± 0.014</td>
          <td>0.147 ± 0.006</td>
        </tr>
        <tr>
          <th>0.2</th>
          <td>0.195 ± 0.03</td>
          <td>0.207 ± 0.017</td>
          <td>0.049 ± 0.017</td>
          <td>0.288 ± 0.008</td>
        </tr>
        <tr>
          <th>0.3</th>
          <td>0.182 ± 0.024</td>
          <td>0.211 ± 0.016</td>
          <td>0.073 ± 0.018</td>
          <td>0.414 ± 0.014</td>
        </tr>
        <tr>
          <th>0.4</th>
          <td>0.176 ± 0.026</td>
          <td>0.205 ± 0.014</td>
          <td>0.082 ± 0.016</td>
          <td>0.525 ± 0.02</td>
        </tr>
        <tr>
          <th>0.5</th>
          <td>0.18 ± 0.023</td>
          <td>0.205 ± 0.011</td>
          <td>0.095 ± 0.013</td>
          <td>0.653 ± 0.022</td>
        </tr>
        <tr>
          <th>0.6</th>
          <td>0.174 ± 0.022</td>
          <td>0.202 ± 0.008</td>
          <td>0.099 ± 0.008</td>
          <td>0.766 ± 0.02</td>
        </tr>
        <tr>
          <th>0.7</th>
          <td>0.173 ± 0.029</td>
          <td>0.2 ± 0.008</td>
          <td>0.105 ± 0.008</td>
          <td>0.891 ± 0.028</td>
        </tr>
        <tr>
          <th>0.8</th>
          <td>0.174 ± 0.026</td>
          <td>0.198 ± 0.008</td>
          <td>0.11 ± 0.007</td>
          <td>1.01 ± 0.026</td>
        </tr>
      </tbody>
    </table>
    </div>
    </div>
    <br />
    <br />

.. GENERATED FROM PYTHON SOURCE LINES 383-405

.. code-block:: default

    fig, ax = plt.subplots()
    sns.lineplot(data=table, x="train_size", y="Error", label="Test Error", ax=ax)
    sns.lineplot(
        data=table,
        x="train_size",
        y="Upper",
        color="red",
        label="Upper bound",
        linestyle="dotted",
        ax=ax,
    )
    sns.lineplot(
        data=table,
        x="train_size",
        y="Lower",
        color="green",
        label="Lower bound",
        linestyle="dotted",
        ax=ax,
    )
    plt.suptitle("Credit")
    plt.show()



.. image-sg:: /auto_examples/further_examples/images/sphx_glr_plot_4_upperLower_005.png
   :alt: Credit
   :srcset: /auto_examples/further_examples/images/sphx_glr_plot_4_upperLower_005.png
   :class: sphx-glr-single-img





.. GENERATED FROM PYTHON SOURCE LINES 406-408

COVID
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

.. GENERATED FROM PYTHON SOURCE LINES 408-420

.. code-block:: default

    table = pd.read_csv('data/table.csv')
    means = table[table.columns.difference(
        ["iteration"])].groupby("train_size").mean()
    std = table[table.columns.difference(
        ["iteration"])].groupby("train_size").std()
    for column in means.columns:
        means[column] = (
            means[column].round(3).astype(str) + " ± " +
            std[column].round(3).astype(str)
        )
    means[["Error", "Upper", "Lower", "Time"]]






.. raw:: html

    <div class="output_subarea output_html rendered_html output_result">
    <div>
    <style scoped>
        .dataframe tbody tr th:only-of-type {
            vertical-align: middle;
        }

        .dataframe tbody tr th {
            vertical-align: top;
        }

        .dataframe thead th {
            text-align: right;
        }
    </style>
    <table border="1" class="dataframe">
      <thead>
        <tr style="text-align: right;">
          <th></th>
          <th>Error</th>
          <th>Upper</th>
          <th>Lower</th>
          <th>Time</th>
        </tr>
        <tr>
          <th>train_size</th>
          <th></th>
          <th></th>
          <th></th>
          <th></th>
        </tr>
      </thead>
      <tbody>
        <tr>
          <th>0.1</th>
          <td>0.259 ± 0.019</td>
          <td>0.273 ± 0.012</td>
          <td>0.113 ± 0.019</td>
          <td>0.531 ± 0.033</td>
        </tr>
        <tr>
          <th>0.2</th>
          <td>0.256 ± 0.017</td>
          <td>0.275 ± 0.009</td>
          <td>0.154 ± 0.013</td>
          <td>1.028 ± 0.03</td>
        </tr>
        <tr>
          <th>0.3</th>
          <td>0.258 ± 0.018</td>
          <td>0.278 ± 0.006</td>
          <td>0.174 ± 0.009</td>
          <td>1.518 ± 0.047</td>
        </tr>
        <tr>
          <th>0.4</th>
          <td>0.253 ± 0.016</td>
          <td>0.277 ± 0.007</td>
          <td>0.182 ± 0.01</td>
          <td>2.034 ± 0.053</td>
        </tr>
        <tr>
          <th>0.5</th>
          <td>0.253 ± 0.016</td>
          <td>0.278 ± 0.005</td>
          <td>0.19 ± 0.007</td>
          <td>2.659 ± 0.078</td>
        </tr>
        <tr>
          <th>0.6</th>
          <td>0.256 ± 0.017</td>
          <td>0.277 ± 0.005</td>
          <td>0.192 ± 0.006</td>
          <td>3.121 ± 0.118</td>
        </tr>
        <tr>
          <th>0.7</th>
          <td>0.256 ± 0.019</td>
          <td>0.277 ± 0.005</td>
          <td>0.196 ± 0.005</td>
          <td>3.836 ± 0.153</td>
        </tr>
        <tr>
          <th>0.8</th>
          <td>0.252 ± 0.015</td>
          <td>0.276 ± 0.005</td>
          <td>0.198 ± 0.006</td>
          <td>4.144 ± 0.11</td>
        </tr>
      </tbody>
    </table>
    </div>
    </div>
    <br />
    <br />

.. GENERATED FROM PYTHON SOURCE LINES 421-443

.. code-block:: default

    fig, ax = plt.subplots()
    sns.lineplot(data=table, x="train_size", y="Error", label="Test Error", ax=ax)
    sns.lineplot(
        data=table,
        x="train_size",
        y="Upper",
        color="red",
        label="Upper bound",
        linestyle="dotted",
        ax=ax,
    )
    sns.lineplot(
        data=table,
        x="train_size",
        y="Lower",
        color="green",
        label="Lower bound",
        linestyle="dotted",
        ax=ax,
    )
    plt.suptitle("COVID")
    plt.show()



.. image-sg:: /auto_examples/further_examples/images/sphx_glr_plot_4_upperLower_006.png
   :alt: COVID
   :srcset: /auto_examples/further_examples/images/sphx_glr_plot_4_upperLower_006.png
   :class: sphx-glr-single-img






.. rst-class:: sphx-glr-timing

   **Total running time of the script:** ( 38 minutes  39.470 seconds)


.. _sphx_glr_download_auto_examples_further_examples_plot_4_upperLower.py:


.. only :: html

 .. container:: sphx-glr-footer
    :class: sphx-glr-footer-example



  .. container:: sphx-glr-download sphx-glr-download-python

     :download:`Download Python source code: plot_4_upperLower.py <plot_4_upperLower.py>`



  .. container:: sphx-glr-download sphx-glr-download-jupyter

     :download:`Download Jupyter notebook: plot_4_upperLower.ipynb <plot_4_upperLower.ipynb>`


.. only:: html

 .. rst-class:: sphx-glr-signature

    `Gallery generated by Sphinx-Gallery <https://sphinx-gallery.github.io>`_