feat: add mean over time + NA for latest period

Author: vviers

notebooks/02-rapport_post_integration.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "markdown",
-   "id": "mental-deployment",
+   "id": "directed-vessel",
    "metadata": {},
    "source": [
     "# Automated Post-integration Report - Signaux Faibles\n",
@@ -12,7 +12,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "stock-shame",
+   "id": "electoral-joining",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -51,12 +51,15 @@
     "    \"paydex_nb_jours_past_12\",\n",
     "]\n",
     "# ces variables sont toujours requêtées\n",
-    "VARIABLES += [\"outcome\", \"periode\", \"siret\", \"siren\", \"time_til_outcome\", \"code_naf\"]"
+    "VARIABLES += [\"outcome\", \"periode\", \"siret\", \"siren\", \"time_til_outcome\", \"code_naf\"]\n",
+    "\n",
+    "# période actuelle\n",
+    "LATEST_PERIODE = \"2021-02-01\""
    ]
   },
   {
    "cell_type": "markdown",
-   "id": "adjustable-arkansas",
+   "id": "athletic-adams",
    "metadata": {},
    "source": [
     "## Fetch a random sample of the data"
@@ -65,7 +68,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "false-shield",
+   "id": "tutorial-congress",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -76,7 +79,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "formed-salvation",
+   "id": "mighty-feelings",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -86,20 +89,20 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "suspended-london",
+   "id": "extra-panama",
    "metadata": {},
    "outputs": [],
    "source": [
     "dataset = SFDataset(\n",
     "    fields = VARIABLES,\n",
-    "    sample_size=10_000\n",
+    "    sample_size=100_000\n",
     ")\n",
     "dataset.fetch_data();"
    ]
   },
   {
    "cell_type": "markdown",
-   "id": "pediatric-drama",
+   "id": "headed-aurora",
    "metadata": {},
    "source": [
     "## Temporal Coverage and NA values"
@@ -108,7 +111,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "theoretical-density",
+   "id": "comic-shift",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -118,7 +121,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "promotional-heritage",
+   "id": "optional-corner",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -129,7 +132,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "included-industry",
+   "id": "proof-horse",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -140,7 +143,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "logical-bailey",
+   "id": "numerous-senate",
    "metadata": {},
    "source": [
     "## Coverage over time for selected variables"
@@ -149,7 +152,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "fiscal-samoa",
+   "id": "pretty-memorabilia",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -160,14 +163,10 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "handled-tuning",
+   "id": "constitutional-audience",
    "metadata": {},
    "outputs": [],
    "source": [
-    "def count_na_prop(series):\n",
-    "    return (1 - series.isna().sum() / len(series)) * 100\n",
-    "\n",
-    "\n",
     "fig, axs = plt.subplots(len(VARIABLES), figsize=(10, 100))\n",
     "fig.tight_layout()\n",
     "for i, variable in enumerate(VARIABLES):\n",
@@ -178,26 +177,121 @@
     "    axs[i].set(adjustable='box')"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "aboriginal-dominican",
+   "metadata": {},
+   "source": [
+    "## Average over time"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "critical-category",
+   "id": "local-beijing",
    "metadata": {},
    "outputs": [],
-   "source": []
+   "source": [
+    "from pandas.api.types import is_numeric_dtype"
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "super-arabic",
+   "id": "purple-helicopter",
    "metadata": {},
    "outputs": [],
-   "source": []
+   "source": [
+    "VARIABLES_TO_AVERAGE = [var for var in VARIABLES if is_numeric_dtype(dataset.data[var])]\n",
+    "fig, axs = plt.subplots(len(VARIABLES_TO_AVERAGE), figsize=(10, 100))\n",
+    "fig.tight_layout()\n",
+    "for i, variable in enumerate(VARIABLES_TO_AVERAGE):\n",
+    "    grouped = dataset.data.groupby(pd.Grouper(key=\"periode\", freq=\"M\")).agg({f\"{variable}\": \"mean\"})\n",
+    "    axs[i].set_title(f\"{variable}\")\n",
+    "    #axs[i].set_ylim([0, 100])\n",
+    "    axs[i].plot_date(grouped.index, grouped[f\"{variable}\"], \"-\");\n",
+    "    axs[i].set(adjustable='box')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "guided-launch",
+   "metadata": {},
+   "source": [
+    "## Codes NAF"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "crude-wesley",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import seaborn as sns\n",
+    "grouped = dataset.data.groupby(\"code_naf\", as_index=False).agg({\"outcome\": \"count\"})\n",
+    "sns.barplot(x = grouped.code_naf, y = grouped.outcome);"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "distinguished-router",
+   "metadata": {},
+   "source": [
+    "## Codes NAF over time"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "satisfactory-selling",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "grouped = dataset.data.groupby([pd.Grouper(key = \"periode\", freq = \"2Q\"), \"code_naf\"]).agg({\"outcome\": \"count\"}).reset_index()\n",
+    "plt.figure(figsize = (15, 10))\n",
+    "sns.lineplot(x = grouped.periode, y = grouped.outcome, hue = grouped.code_naf);"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "wanted-retrieval",
+   "metadata": {},
+   "source": [
+    "## Analysis for latest period only"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "structural-bridal",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dataset = SFDataset(\n",
+    "    fields = VARIABLES,\n",
+    "    date_min = LATEST_PERIODE,\n",
+    "    sample_size=100_000\n",
+    ")\n",
+    "dataset.fetch_data();"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "continuing-minnesota",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "na_rates_df = (dataset.data.isna().sum() / len(dataset) * 100).sort_values(ascending = False).to_frame()\n",
+    "na_rates_df.columns = [\"NA rate\"]\n",
+    "na_rates_df"
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "desperate-button",
+   "id": "friendly-appearance",
    "metadata": {},
    "outputs": [],
    "source": []