{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "455a3208",
   "metadata": {},
   "source": [
    "# Detecting and filtering outliers\n",
    "\n",
    "Filtering or transforming outliers is largely a matter of applying array operations. Consider a DataFrame with some normally distributed data:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "35bb569f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1000.000000</td>\n",
       "      <td>1000.000000</td>\n",
       "      <td>1000.000000</td>\n",
       "      <td>1000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>-0.034508</td>\n",
       "      <td>0.011824</td>\n",
       "      <td>-0.024031</td>\n",
       "      <td>-0.048423</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.023096</td>\n",
       "      <td>1.069939</td>\n",
       "      <td>1.037148</td>\n",
       "      <td>0.972926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-2.998919</td>\n",
       "      <td>-2.939683</td>\n",
       "      <td>-3.980539</td>\n",
       "      <td>-3.180228</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>-0.735324</td>\n",
       "      <td>-0.739318</td>\n",
       "      <td>-0.690162</td>\n",
       "      <td>-0.699223</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>-0.020213</td>\n",
       "      <td>0.009185</td>\n",
       "      <td>-0.041272</td>\n",
       "      <td>-0.046438</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.661472</td>\n",
       "      <td>0.728629</td>\n",
       "      <td>0.675814</td>\n",
       "      <td>0.588834</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>3.187850</td>\n",
       "      <td>3.693235</td>\n",
       "      <td>3.950033</td>\n",
       "      <td>3.089895</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 0            1            2            3\n",
       "count  1000.000000  1000.000000  1000.000000  1000.000000\n",
       "mean     -0.034508     0.011824    -0.024031    -0.048423\n",
       "std       1.023096     1.069939     1.037148     0.972926\n",
       "min      -2.998919    -2.939683    -3.980539    -3.180228\n",
       "25%      -0.735324    -0.739318    -0.690162    -0.699223\n",
       "50%      -0.020213     0.009185    -0.041272    -0.046438\n",
       "75%       0.661472     0.728629     0.675814     0.588834\n",
       "max       3.187850     3.693235     3.950033     3.089895"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "df = pd.DataFrame(np.random.randn(1000, 4))\n",
    "\n",
    "df.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c159cc3",
   "metadata": {},
   "source": [
    "Suppose you want to find values in one of the columns whose absolute value is greater than 3:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "89dfec83",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "435    3.693235\n",
       "Name: 1, dtype: float64"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "col = df[1]\n",
    "\n",
    "col[col.abs() > 3]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0a411bdb",
   "metadata": {},
   "source": [
    "To select all rows where value is greater than `3` or less than `-3` in one of the columns, you can apply [pandas.DataFrame.any](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.any.html) to a Boolean DataFrame, using `any(axis=1)` to check if a value is in a row:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ca08a1c8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>-0.477368</td>\n",
       "      <td>-0.100079</td>\n",
       "      <td>-1.466754</td>\n",
       "      <td>-3.180228</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>188</th>\n",
       "      <td>1.962728</td>\n",
       "      <td>-0.072791</td>\n",
       "      <td>3.950033</td>\n",
       "      <td>-0.012231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>210</th>\n",
       "      <td>1.498744</td>\n",
       "      <td>-0.057742</td>\n",
       "      <td>3.412662</td>\n",
       "      <td>0.586651</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>245</th>\n",
       "      <td>3.016760</td>\n",
       "      <td>1.527263</td>\n",
       "      <td>1.790951</td>\n",
       "      <td>-0.015122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>282</th>\n",
       "      <td>1.006073</td>\n",
       "      <td>-0.480924</td>\n",
       "      <td>0.259646</td>\n",
       "      <td>3.089895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>385</th>\n",
       "      <td>3.187850</td>\n",
       "      <td>-1.069850</td>\n",
       "      <td>-0.641928</td>\n",
       "      <td>1.733524</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>435</th>\n",
       "      <td>-0.303929</td>\n",
       "      <td>3.693235</td>\n",
       "      <td>-0.590390</td>\n",
       "      <td>0.052511</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>606</th>\n",
       "      <td>-0.220844</td>\n",
       "      <td>-0.479557</td>\n",
       "      <td>-3.012150</td>\n",
       "      <td>-1.476384</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>613</th>\n",
       "      <td>0.715983</td>\n",
       "      <td>0.134178</td>\n",
       "      <td>-3.835888</td>\n",
       "      <td>-1.358231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>666</th>\n",
       "      <td>-0.351409</td>\n",
       "      <td>1.919364</td>\n",
       "      <td>-3.014478</td>\n",
       "      <td>-0.340513</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>743</th>\n",
       "      <td>0.227552</td>\n",
       "      <td>-0.831102</td>\n",
       "      <td>-0.905155</td>\n",
       "      <td>-3.046226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>824</th>\n",
       "      <td>0.109159</td>\n",
       "      <td>0.501608</td>\n",
       "      <td>-3.980539</td>\n",
       "      <td>-0.783160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>829</th>\n",
       "      <td>3.075201</td>\n",
       "      <td>1.517391</td>\n",
       "      <td>1.191999</td>\n",
       "      <td>-0.690774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>882</th>\n",
       "      <td>-0.445649</td>\n",
       "      <td>0.455558</td>\n",
       "      <td>-3.241675</td>\n",
       "      <td>2.569407</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            0         1         2         3\n",
       "103 -0.477368 -0.100079 -1.466754 -3.180228\n",
       "188  1.962728 -0.072791  3.950033 -0.012231\n",
       "210  1.498744 -0.057742  3.412662  0.586651\n",
       "245  3.016760  1.527263  1.790951 -0.015122\n",
       "282  1.006073 -0.480924  0.259646  3.089895\n",
       "385  3.187850 -1.069850 -0.641928  1.733524\n",
       "435 -0.303929  3.693235 -0.590390  0.052511\n",
       "606 -0.220844 -0.479557 -3.012150 -1.476384\n",
       "613  0.715983  0.134178 -3.835888 -1.358231\n",
       "666 -0.351409  1.919364 -3.014478 -0.340513\n",
       "743  0.227552 -0.831102 -0.905155 -3.046226\n",
       "824  0.109159  0.501608 -3.980539 -0.783160\n",
       "829  3.075201  1.517391  1.191999 -0.690774\n",
       "882 -0.445649  0.455558 -3.241675  2.569407"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[(df.abs() > 3).any(axis=1)]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "74382233",
   "metadata": {},
   "source": [
    "On this basis, the values can be limited to an interval between -3 and 3. For this we use the instruction `np.sign(df)`, which generates values 1 and -1, depending on whether the values in `df` are positive or negative:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6817f226",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1000.000000</td>\n",
       "      <td>1000.000000</td>\n",
       "      <td>1000.000000</td>\n",
       "      <td>1000.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>-0.034787</td>\n",
       "      <td>0.011131</td>\n",
       "      <td>-0.023309</td>\n",
       "      <td>-0.048286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.022245</td>\n",
       "      <td>1.067774</td>\n",
       "      <td>1.025773</td>\n",
       "      <td>0.971934</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-2.998919</td>\n",
       "      <td>-2.939683</td>\n",
       "      <td>-3.000000</td>\n",
       "      <td>-3.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>-0.735324</td>\n",
       "      <td>-0.739318</td>\n",
       "      <td>-0.690162</td>\n",
       "      <td>-0.699223</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>-0.020213</td>\n",
       "      <td>0.009185</td>\n",
       "      <td>-0.041272</td>\n",
       "      <td>-0.046438</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.661472</td>\n",
       "      <td>0.728629</td>\n",
       "      <td>0.675814</td>\n",
       "      <td>0.588834</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 0            1            2            3\n",
       "count  1000.000000  1000.000000  1000.000000  1000.000000\n",
       "mean     -0.034787     0.011131    -0.023309    -0.048286\n",
       "std       1.022245     1.067774     1.025773     0.971934\n",
       "min      -2.998919    -2.939683    -3.000000    -3.000000\n",
       "25%      -0.735324    -0.739318    -0.690162    -0.699223\n",
       "50%      -0.020213     0.009185    -0.041272    -0.046438\n",
       "75%       0.661472     0.728629     0.675814     0.588834\n",
       "max       3.000000     3.000000     3.000000     3.000000"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.abs() > 3] = np.sign(df) * 3\n",
    "\n",
    "df.describe()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.13 Kernel",
   "language": "python",
   "name": "python313"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.0"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}