feat: v0 rapport post integration

Author: vviers

notebooks/02-rapport_post_integration.ipynb

@@ -0,0 +1,225 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "relevant-fighter",
+   "metadata": {},
+   "source": [
+    "# Automated Post-integration Report - Signaux Faibles\n",
+    "This notebook can be run after each new data integration by the [opensignauxfaibles](https://github.com/signaux-faibles/opensignauxfaibles) codebase."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "straight-detroit",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "VARIABLES = [\n",
+    "    \"financier_court_terme\",\n",
+    "    \"interets\",\n",
+    "    \"ca\",\n",
+    "    \"equilibre_financier\",\n",
+    "    \"endettement\",\n",
+    "    \"degre_immo_corporelle\",\n",
+    "    \"liquidite_reduite\",\n",
+    "    \"poids_bfr_exploitation\",\n",
+    "    \"productivite_capital_investi\",\n",
+    "    \"rentabilite_economique\",\n",
+    "    \"rentabilite_nette\",\n",
+    "    \"cotisation\",\n",
+    "    \"cotisation_moy12m\",\n",
+    "    \"montant_part_ouvriere\",\n",
+    "    \"montant_part_ouvriere_past_1\",\n",
+    "    \"montant_part_ouvriere_past_12\",\n",
+    "    \"montant_part_ouvriere_past_2\",\n",
+    "    \"montant_part_ouvriere_past_3\",\n",
+    "    \"montant_part_ouvriere_past_6\",\n",
+    "    \"montant_part_patronale\",\n",
+    "    \"montant_part_patronale_past_1\",\n",
+    "    \"montant_part_patronale_past_12\",\n",
+    "    \"montant_part_patronale_past_2\",\n",
+    "    \"montant_part_patronale_past_3\",\n",
+    "    \"montant_part_patronale_past_6\",\n",
+    "    \"ratio_dette\",\n",
+    "    \"ratio_dette_moy12m\",\n",
+    "    \"effectif\",\n",
+    "    \"apart_heures_consommees_cumulees\",\n",
+    "    \"apart_heures_consommees\",\n",
+    "    \"paydex_nb_jours\",\n",
+    "    \"paydex_nb_jours_past_12\",\n",
+    "]\n",
+    "# ces variables sont toujours requêtées\n",
+    "VARIABLES += [\"outcome\", \"periode\", \"siret\", \"siren\", \"time_til_outcome\", \"code_naf\"]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "turkish-newport",
+   "metadata": {},
+   "source": [
+    "## Fetch a random sample of the data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "reported-peoples",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "%config Completer.use_jedi = False\n",
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "married-drinking",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from predictsignauxfaibles.data import SFDataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "authentic-rendering",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dataset = SFDataset(\n",
+    "    fields = VARIABLES,\n",
+    "    sample_size=10_000\n",
+    ")\n",
+    "dataset.fetch_data();"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "presidential-acrobat",
+   "metadata": {},
+   "source": [
+    "## Temporal Coverage and NA values"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "monthly-secretary",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dataset.data.periode = pd.to_datetime(dataset.data.periode)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "otherwise-culture",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "date_range = dataset.data.periode.min().date(), dataset.data.periode.max().date()\n",
+    "print(f\"Data goes from {date_range[0]} to {date_range[1]}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "sexual-chester",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "(dataset.data.isna().sum() / len(dataset) * 100).sort_values(ascending = False).to_frame()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "oriental-flush",
+   "metadata": {},
+   "source": [
+    "## Coverage over time for selected variables"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "historical-brick",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import matplotlib.pyplot as plt\n",
+    "%matplotlib inline"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "awful-nurse",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def count_na_prop(series):\n",
+    "    return (1 - series.isna().sum() / len(series)) * 100\n",
+    "\n",
+    "\n",
+    "fig, axs = plt.subplots(len(VARIABLES), figsize=(10, 100))\n",
+    "fig.tight_layout()\n",
+    "for i, variable in enumerate(VARIABLES):\n",
+    "    grouped = dataset.data.groupby(pd.Grouper(key=\"periode\", freq=\"M\")).agg({f\"{variable}\": count_na_prop})\n",
+    "    axs[i].set_title(f\"{variable}\")\n",
+    "    axs[i].set_ylim([0, 100])\n",
+    "    axs[i].plot_date(grouped.index, grouped[f\"{variable}\"], \"-\");\n",
+    "    axs[i].set(adjustable='box')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "lucky-clerk",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "proud-volunteer",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "sensitive-pipeline",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "sf",
+   "language": "python",
+   "name": "sf"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.8"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}