{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "455a3208",
   "metadata": {},
   "source": [
    "# Detecting and filtering outliers\n",
    "\n",
    "Filtering or transforming outliers is largely a matter of applying array operations. Consider a DataFrame with some normally distributed data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "35bb569f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-05-22T13:31:04.568333Z",
     "iopub.status.busy": "2026-05-22T13:31:04.568087Z",
     "iopub.status.idle": "2026-05-22T13:31:04.799772Z",
     "shell.execute_reply": "2026-05-22T13:31:04.799402Z",
     "shell.execute_reply.started": "2026-05-22T13:31:04.568315Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1000.000000</td>\n",
       "      <td>1000.000000</td>\n",
       "      <td>1000.000000</td>\n",
       "      <td>1000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.023623</td>\n",
       "      <td>0.029803</td>\n",
       "      <td>-0.028434</td>\n",
       "      <td>0.005751</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.012689</td>\n",
       "      <td>0.999117</td>\n",
       "      <td>0.980014</td>\n",
       "      <td>0.969842</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-3.240759</td>\n",
       "      <td>-2.757613</td>\n",
       "      <td>-3.372727</td>\n",
       "      <td>-2.697726</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>-0.633097</td>\n",
       "      <td>-0.650075</td>\n",
       "      <td>-0.685945</td>\n",
       "      <td>-0.684345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.001117</td>\n",
       "      <td>0.022371</td>\n",
       "      <td>-0.009798</td>\n",
       "      <td>0.002211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.705048</td>\n",
       "      <td>0.710901</td>\n",
       "      <td>0.657235</td>\n",
       "      <td>0.654270</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>3.262954</td>\n",
       "      <td>3.302385</td>\n",
       "      <td>2.956046</td>\n",
       "      <td>2.634073</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 0            1            2            3\n",
       "count  1000.000000  1000.000000  1000.000000  1000.000000\n",
       "mean      0.023623     0.029803    -0.028434     0.005751\n",
       "std       1.012689     0.999117     0.980014     0.969842\n",
       "min      -3.240759    -2.757613    -3.372727    -2.697726\n",
       "25%      -0.633097    -0.650075    -0.685945    -0.684345\n",
       "50%       0.001117     0.022371    -0.009798     0.002211\n",
       "75%       0.705048     0.710901     0.657235     0.654270\n",
       "max       3.262954     3.302385     2.956046     2.634073"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "rng = np.random.default_rng()\n",
    "df = pd.DataFrame(rng.normal(size=(1000, 4)))\n",
    "\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c159cc3",
   "metadata": {},
   "source": [
    "Suppose you want to find values in one of the columns whose absolute value is greater than 3:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "89dfec83",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-05-22T13:31:04.800495Z",
     "iopub.status.busy": "2026-05-22T13:31:04.800294Z",
     "iopub.status.idle": "2026-05-22T13:31:04.802979Z",
     "shell.execute_reply": "2026-05-22T13:31:04.802751Z",
     "shell.execute_reply.started": "2026-05-22T13:31:04.800485Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "220    3.011150\n",
       "640    3.201065\n",
       "674    3.302385\n",
       "Name: 1, dtype: float64"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "col = df[1]\n",
    "\n",
    "col[col.abs() > 3]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0a411bdb",
   "metadata": {},
   "source": [
    "To select all rows where value is greater than `3` or less than `-3` in one of the columns, you can apply [pandas.DataFrame.any](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.any.html) to a Boolean DataFrame, using `any(axis=1)` to check if a value is in a row:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ca08a1c8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-05-22T13:31:04.803480Z",
     "iopub.status.busy": "2026-05-22T13:31:04.803354Z",
     "iopub.status.idle": "2026-05-22T13:31:04.807356Z",
     "shell.execute_reply": "2026-05-22T13:31:04.807062Z",
     "shell.execute_reply.started": "2026-05-22T13:31:04.803468Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3.073224</td>\n",
       "      <td>0.392376</td>\n",
       "      <td>0.464029</td>\n",
       "      <td>-1.086741</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>0.456746</td>\n",
       "      <td>0.313551</td>\n",
       "      <td>-3.372727</td>\n",
       "      <td>0.232789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>3.262954</td>\n",
       "      <td>-1.511093</td>\n",
       "      <td>-0.243049</td>\n",
       "      <td>-0.424410</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>220</th>\n",
       "      <td>0.657494</td>\n",
       "      <td>3.011150</td>\n",
       "      <td>-0.733968</td>\n",
       "      <td>0.549828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>504</th>\n",
       "      <td>-3.240759</td>\n",
       "      <td>0.202480</td>\n",
       "      <td>-0.181495</td>\n",
       "      <td>0.088678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>640</th>\n",
       "      <td>0.913886</td>\n",
       "      <td>3.201065</td>\n",
       "      <td>-0.896181</td>\n",
       "      <td>1.048140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>674</th>\n",
       "      <td>-0.283886</td>\n",
       "      <td>3.302385</td>\n",
       "      <td>-0.541091</td>\n",
       "      <td>0.524652</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>833</th>\n",
       "      <td>3.010898</td>\n",
       "      <td>-0.341878</td>\n",
       "      <td>-0.409523</td>\n",
       "      <td>0.264089</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            0         1         2         3\n",
       "0    3.073224  0.392376  0.464029 -1.086741\n",
       "50   0.456746  0.313551 -3.372727  0.232789\n",
       "78   3.262954 -1.511093 -0.243049 -0.424410\n",
       "220  0.657494  3.011150 -0.733968  0.549828\n",
       "504 -3.240759  0.202480 -0.181495  0.088678\n",
       "640  0.913886  3.201065 -0.896181  1.048140\n",
       "674 -0.283886  3.302385 -0.541091  0.524652\n",
       "833  3.010898 -0.341878 -0.409523  0.264089"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[(df.abs() > 3).any(axis=1)]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "74382233",
   "metadata": {},
   "source": [
    "On this basis, the values can be limited to an interval between -3 and 3. For this we use the instruction `np.sign(df)`, which generates values 1 and -1, depending on whether the values in `df` are positive or negative:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6817f226",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-05-22T13:31:04.807718Z",
     "iopub.status.busy": "2026-05-22T13:31:04.807640Z",
     "iopub.status.idle": "2026-05-22T13:31:04.815314Z",
     "shell.execute_reply": "2026-05-22T13:31:04.815016Z",
     "shell.execute_reply.started": "2026-05-22T13:31:04.807710Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1000.000000</td>\n",
       "      <td>1000.000000</td>\n",
       "      <td>1000.000000</td>\n",
       "      <td>1000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.023276</td>\n",
       "      <td>0.029288</td>\n",
       "      <td>-0.028434</td>\n",
       "      <td>0.005751</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.011630</td>\n",
       "      <td>0.997518</td>\n",
       "      <td>0.980014</td>\n",
       "      <td>0.969842</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-3.240759</td>\n",
       "      <td>-2.757613</td>\n",
       "      <td>-3.372727</td>\n",
       "      <td>-2.697726</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>-0.633097</td>\n",
       "      <td>-0.650075</td>\n",
       "      <td>-0.685945</td>\n",
       "      <td>-0.684345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.001117</td>\n",
       "      <td>0.022371</td>\n",
       "      <td>-0.009798</td>\n",
       "      <td>0.002211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.705048</td>\n",
       "      <td>0.710901</td>\n",
       "      <td>0.657235</td>\n",
       "      <td>0.654270</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.956046</td>\n",
       "      <td>2.634073</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 0            1            2            3\n",
       "count  1000.000000  1000.000000  1000.000000  1000.000000\n",
       "mean      0.023276     0.029288    -0.028434     0.005751\n",
       "std       1.011630     0.997518     0.980014     0.969842\n",
       "min      -3.240759    -2.757613    -3.372727    -2.697726\n",
       "25%      -0.633097    -0.650075    -0.685945    -0.684345\n",
       "50%       0.001117     0.022371    -0.009798     0.002211\n",
       "75%       0.705048     0.710901     0.657235     0.654270\n",
       "max       3.000000     3.000000     2.956046     2.634073"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df > 3] = np.sign(df) * 3\n",
    "\n",
    "df.describe()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.13 Kernel",
   "language": "python",
   "name": "python313"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.0"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}