FP_Growth_Algorithm(Association).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "FP Growth Algorithm(Association).ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "mount_file_id": "1fkBzPfC76maXIy8ctChnNvwpUgxaZPgM",
      "authorship_tag": "ABX9TyO0Ai+pqCvHEGj9m6OIzXDa",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Dushanthimadhushika3/FP-Growth-Algorithm/blob/main/FP_Growth_Algorithm(Association).ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IVvyWlQ-pIKL",
        "outputId": "66329443-1c2d-4c3e-fab1-4d214ae4519a"
      },
      "source": [
        "!pip install pyspark"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting pyspark\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/89/db/e18cfd78e408de957821ec5ca56de1250645b05f8523d169803d8df35a64/pyspark-3.1.2.tar.gz (212.4MB)\n",
            "\u001b[K     |████████████████████████████████| 212.4MB 62kB/s \n",
            "\u001b[?25hCollecting py4j==0.10.9\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/9e/b6/6a4fb90cd235dc8e265a6a2067f2a2c99f0d91787f06aca4bcf7c23f3f80/py4j-0.10.9-py2.py3-none-any.whl (198kB)\n",
            "\u001b[K     |████████████████████████████████| 204kB 21.3MB/s \n",
            "\u001b[?25hBuilding wheels for collected packages: pyspark\n",
            "  Building wheel for pyspark (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for pyspark: filename=pyspark-3.1.2-py2.py3-none-any.whl size=212880768 sha256=7404de37e3494a2c880b931301ae3333dea7b3b329c18686596334e5669bae60\n",
            "  Stored in directory: /root/.cache/pip/wheels/40/1b/2c/30f43be2627857ab80062bef1527c0128f7b4070b6b2d02139\n",
            "Successfully built pyspark\n",
            "Installing collected packages: py4j, pyspark\n",
            "Successfully installed py4j-0.10.9 pyspark-3.1.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9iqk65qfuIoN"
      },
      "source": [
        "import pandas as pd\n",
        "import glob\n",
        "\n",
        "from pyspark.sql import SparkSession\n",
        "\n",
        "# Import the FPGrowth API from PySpark\n",
        "from pyspark.ml.fpm import FPGrowth\n",
        "from pyspark.sql import SQLContext\n",
        "\n",
        "spark = SparkSession.builder.master(\"local\").appName(\"Colab\")\\\n",
        "        .config('spark.ui.port', '4050').getOrCreate()"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "di9lx1Lzvgdw"
      },
      "source": [
        "csv = glob.glob('/content/drive/MyDrive/L4S1/Big Data/Data/Datasets'+'/*.csv')\n",
        "\n",
        "df = pd.DataFrame()\n",
        "# Read the data from the dataset\n",
        "for f in csv:\n",
        "  file = pd.read_csv(f)\n",
        "  df = df.append(file)"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 289
        },
        "id": "MckCsj1tDGve",
        "outputId": "20df1ba9-b9e0-47aa-f9cc-1038824686df"
      },
      "source": [
        "df.head()"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>event_time</th>\n",
              "      <th>event_type</th>\n",
              "      <th>product_id</th>\n",
              "      <th>category_id</th>\n",
              "      <th>category_code</th>\n",
              "      <th>brand</th>\n",
              "      <th>price</th>\n",
              "      <th>user_id</th>\n",
              "      <th>user_session</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2020-01-01 00:00:00 UTC</td>\n",
              "      <td>view</td>\n",
              "      <td>5809910</td>\n",
              "      <td>1602943681873052386</td>\n",
              "      <td>NaN</td>\n",
              "      <td>grattol</td>\n",
              "      <td>5.24</td>\n",
              "      <td>595414620</td>\n",
              "      <td>4adb70bb-edbd-4981-b60f-a05bfd32683a</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2020-01-01 00:00:09 UTC</td>\n",
              "      <td>view</td>\n",
              "      <td>5812943</td>\n",
              "      <td>1487580012121948301</td>\n",
              "      <td>NaN</td>\n",
              "      <td>kinetics</td>\n",
              "      <td>3.97</td>\n",
              "      <td>595414640</td>\n",
              "      <td>c8c5205d-be43-4f1d-aa56-4828b8151c8a</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2020-01-01 00:00:19 UTC</td>\n",
              "      <td>view</td>\n",
              "      <td>5798924</td>\n",
              "      <td>1783999068867920626</td>\n",
              "      <td>NaN</td>\n",
              "      <td>zinger</td>\n",
              "      <td>3.97</td>\n",
              "      <td>595412617</td>\n",
              "      <td>46a5010f-bd69-4fbe-a00d-bb17aa7b46f3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>2020-01-01 00:00:24 UTC</td>\n",
              "      <td>view</td>\n",
              "      <td>5793052</td>\n",
              "      <td>1487580005754995573</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>4.92</td>\n",
              "      <td>420652863</td>\n",
              "      <td>546f6af3-a517-4752-a98b-80c4c5860711</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>2020-01-01 00:00:25 UTC</td>\n",
              "      <td>view</td>\n",
              "      <td>5899926</td>\n",
              "      <td>2115334439910245200</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>3.92</td>\n",
              "      <td>484071203</td>\n",
              "      <td>cff70ddf-529e-4b0c-a4fc-f43a749c0acb</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                event_time  ...                          user_session\n",
              "0  2020-01-01 00:00:00 UTC  ...  4adb70bb-edbd-4981-b60f-a05bfd32683a\n",
              "1  2020-01-01 00:00:09 UTC  ...  c8c5205d-be43-4f1d-aa56-4828b8151c8a\n",
              "2  2020-01-01 00:00:19 UTC  ...  46a5010f-bd69-4fbe-a00d-bb17aa7b46f3\n",
              "3  2020-01-01 00:00:24 UTC  ...  546f6af3-a517-4752-a98b-80c4c5860711\n",
              "4  2020-01-01 00:00:25 UTC  ...  cff70ddf-529e-4b0c-a4fc-f43a749c0acb\n",
              "\n",
              "[5 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qTb4ah_aNsmw",
        "outputId": "af5b0465-4f97-47a0-b684-ba92c616e50b"
      },
      "source": [
        "sc = df[['user_session','brand','user_id','event_type']]\n",
        "sc = sc.loc[sc['event_type']=='purchase']\n",
        "sc = sc[['user_session','brand']]\n",
        "sc = sc.dropna()\n",
        "print(sc.shape)\n",
        "sc.head()\n",
        "\n",
        "sc.to_csv('/content/purchase.csv')"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(737314, 2)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zpjCqDI5cdcy",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2cca288d-3c00-4ab6-a08e-1d0b558f7f9d"
      },
      "source": [
        "session_id_col_name = 'user_session'\n",
        "product_id_col_name = 'brand'\n",
        "session_id_col_index = -1\n",
        "product_id_col_index = -1\n",
        "data = dict()\n",
        "\n",
        "with open('/content/purchase.csv') as f:\n",
        "    lis = [line.rstrip().split(',') for line in f]        \n",
        "    for i, x in enumerate(lis):              \n",
        "        if i == 0:\n",
        "            if type(x) is list and session_id_col_name in x and product_id_col_name in x: # column names\n",
        "                session_id_col_index = x.index(session_id_col_name)\n",
        "                product_id_col_index = x.index(product_id_col_name)\n",
        "            else:\n",
        "                print('Required columns does not exists in given csv file')\n",
        "                break\n",
        "        else:\n",
        "            if x[session_id_col_index] in data:\n",
        "                data[x[session_id_col_index]] = data[x[session_id_col_index]] + ',' + x[product_id_col_index]\n",
        "            else:\n",
        "                data[x[session_id_col_index]] = x[product_id_col_index]\n",
        "\n",
        "\n",
        "# Create the pandas DataFrame\n",
        "dataframe = pd.DataFrame(list(data.items()),columns=[session_id_col_name,product_id_col_name])\n",
        "dataframe.head()\n",
        "print(dataframe.shape)\n"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(137908, 2)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        },
        "id": "Mm-lR95RX8ht",
        "outputId": "544ba601-491c-4ff4-b43c-1cdb2b3fd2d3"
      },
      "source": [
        "# Preprocess the data\n",
        "records = []\n",
        "for i in range(0,137908):\n",
        "  record = []\n",
        "  for j in range(1,2):\n",
        "    val = dataframe.values[i, j].split(\",\")\n",
        "    for k in val:\n",
        "      if k not in record:\n",
        "        record.append(k)\n",
        "  records.append([i, record])\n",
        "\n",
        "print(record)\n",
        "# Create the pandas DataFrame\n",
        "out = pd.DataFrame(records, columns = ['id', 'items'])\n",
        "out.head()"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['ingarden']\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>items</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>[kiss]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>[cnd]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>[refectocil, cosmoprofi, zinger, runail, irisk]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>[roubloff, depilflax, irisk, domix, grattol, m...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>[concept]</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   id                                              items\n",
              "0   0                                             [kiss]\n",
              "1   1                                              [cnd]\n",
              "2   2    [refectocil, cosmoprofi, zinger, runail, irisk]\n",
              "3   3  [roubloff, depilflax, irisk, domix, grattol, m...\n",
              "4   4                                          [concept]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "si8JsJLUvrec",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "627d49fd-e2db-4466-884a-ca1b98b2fdef"
      },
      "source": [
        "# Convert the pandas dataframe into spark dataframe\n",
        "sqlContext = SQLContext(spark)\n",
        "sf = spark.createDataFrame(out,[\"id\", \"items\"])\n",
        "sf.show()"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+---+--------------------+\n",
            "| id|               items|\n",
            "+---+--------------------+\n",
            "|  0|              [kiss]|\n",
            "|  1|               [cnd]|\n",
            "|  2|[refectocil, cosm...|\n",
            "|  3|[roubloff, depilf...|\n",
            "|  4|           [concept]|\n",
            "|  5|             [estel]|\n",
            "|  6|[runail, irisk, l...|\n",
            "|  7|            [runail]|\n",
            "|  8|[ingarden, irisk,...|\n",
            "|  9|[irisk, runail, b...|\n",
            "| 10|[runail, concept,...|\n",
            "| 11|             [irisk]|\n",
            "| 12|[dartnails, ylloz...|\n",
            "| 13|[bluesky, grattol...|\n",
            "| 14|[konad, irisk, it...|\n",
            "| 15|[grattol, bpw.sty...|\n",
            "| 16|[irisk, grattol, ...|\n",
            "| 17|[irisk, runail, n...|\n",
            "| 18|             [estel]|\n",
            "| 19|[severina, cosmop...|\n",
            "+---+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YPyYzHn-vwPc",
        "outputId": "013d34d3-a7b1-417a-920a-fa1ee3b61de7"
      },
      "source": [
        "# Create fpGrowth object by instantiating the FPGrowth constructor with the necessary parameters\n",
        "fpGrowth = FPGrowth(itemsCol=\"items\", minSupport=0.01, minConfidence=0.01)\n",
        "\n",
        "# Fit the dataframe into the fpGrowth object to get the model prepared\n",
        "model = fpGrowth.fit(sf)\n",
        "# Display the frequently occuring items\n",
        "model.freqItemsets.show()"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+--------------------+-----+\n",
            "|               items| freq|\n",
            "+--------------------+-----+\n",
            "|           [staleks]| 4155|\n",
            "|    [staleks, irisk]| 1552|\n",
            "|   [staleks, runail]| 1704|\n",
            "|             [f.o.x]| 2763|\n",
            "|          [solomeya]| 1798|\n",
            "|           [bluesky]| 5559|\n",
            "|    [bluesky, irisk]| 2160|\n",
            "|[bluesky, irisk, ...| 1445|\n",
            "|   [bluesky, runail]| 2841|\n",
            "|             [domix]|10006|\n",
            "|    [domix, grattol]| 1785|\n",
            "|   [domix, ingarden]| 1612|\n",
            "|      [domix, irisk]| 4336|\n",
            "|[domix, irisk, ru...| 2737|\n",
            "|     [domix, runail]| 4868|\n",
            "|           [levrana]| 1580|\n",
            "|            [uskusi]| 2351|\n",
            "|           [lianail]| 3591|\n",
            "|   [lianail, runail]| 1607|\n",
            "|          [severina]| 7807|\n",
            "+--------------------+-----+\n",
            "only showing top 20 rows\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Z202vDb6v1ml",
        "outputId": "aa6a49c8-9787-41d5-e9d3-d4b6fd727a65"
      },
      "source": [
        "# Display the generated association rules\n",
        "model.associationRules.show()"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+-------------------+-----------+-------------------+------------------+--------------------+\n",
            "|         antecedent| consequent|         confidence|              lift|             support|\n",
            "+-------------------+-----------+-------------------+------------------+--------------------+\n",
            "| [severina, runail]|    [irisk]| 0.5723214285714285| 2.102608119010831|0.018592104881515213|\n",
            "|             [pole]|    [irisk]| 0.4037460978147763| 1.483292046924188|0.014067349247324303|\n",
            "|             [pole]|   [runail]| 0.5023933402705515|1.5267868566965164| 0.01750442323868086|\n",
            "|         [nagaraku]|    [irisk]|  0.498050139275766|1.8297484843956082|0.012965165182585493|\n",
            "|         [nagaraku]|   [runail]| 0.3896935933147632| 1.184289298284501|0.010144444122168403|\n",
            "|      [milv, irisk]|   [runail]| 0.6210562414266118|1.8874066009092572|0.013131943034486759|\n",
            "|           [zinger]|    [irisk]|0.41102542130819764|1.5100349992479867|0.020868985120515125|\n",
            "|           [zinger]|   [runail]|0.46329620108540415| 1.407969600460255|0.023522928329030947|\n",
            "|           [lovely]|    [irisk]| 0.4197686645636172|1.5421561349203294|0.014473417060649129|\n",
            "|           [lovely]|   [runail]| 0.4241850683491062|1.2891098174461433|0.014625692490645938|\n",
            "|[ingarden, grattol]|    [irisk]|0.45550957061562336|1.6734619282982415|0.012769382486875308|\n",
            "|[ingarden, grattol]|   [runail]| 0.5607863424728401| 1.704244758979802| 0.01572062534443252|\n",
            "|           [de.lux]|[bpw.style]| 0.4721669980119284| 4.989319313602714|0.013777300809235142|\n",
            "|           [de.lux]|    [irisk]| 0.4955268389662028|1.8204783235161994| 0.01445891463874467|\n",
            "|           [de.lux]|   [runail]| 0.5897117296222664|1.7921497875393357| 0.01720712358963947|\n",
            "|              [uno]|  [grattol]| 0.2722522208674447|1.7858523247425595|0.022667285436667923|\n",
            "|              [uno]|[bpw.style]|0.15859606340358823| 1.675861306555976| 0.01320445514400905|\n",
            "|              [uno]| [ingarden]|0.17305347500435464| 1.644646036172596|0.014408156162079067|\n",
            "|              [uno]|    [irisk]|0.33957498693607385|1.2475386887521998|0.028272471502740957|\n",
            "|              [uno]|   [masura]| 0.1268942692910643|1.1695338427716429|0.010565014357397685|\n",
            "+-------------------+-----------+-------------------+------------------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FzmRgl25v4pw",
        "outputId": "9c45ad5c-cac4-43fc-b310-ebcbe63d6ab4"
      },
      "source": [
        "# Predicts what the customer is most likely to buy with the items bought in the dataset\n",
        "model.transform(sf).show(20)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+---+--------------------+--------------------+\n",
            "| id|               items|          prediction|\n",
            "+---+--------------------+--------------------+\n",
            "|  0|              [kiss]|                  []|\n",
            "|  1|               [cnd]|                  []|\n",
            "|  2|[refectocil, cosm...|[staleks, bluesky...|\n",
            "|  3|[roubloff, depilf...|[runail, severina...|\n",
            "|  4|           [concept]|                  []|\n",
            "|  5|             [estel]|[irisk, runail, k...|\n",
            "|  6|[runail, irisk, l...|[staleks, bluesky...|\n",
            "|  7|            [runail]|[staleks, bluesky...|\n",
            "|  8|[ingarden, irisk,...|[grattol, bpw.sty...|\n",
            "|  9|[irisk, runail, b...|[staleks, domix, ...|\n",
            "| 10|[runail, concept,...|[estel, irisk, st...|\n",
            "| 11|             [irisk]|[staleks, bluesky...|\n",
            "| 12|[dartnails, ylloz...|[staleks, bluesky...|\n",
            "| 13|[bluesky, grattol...|[irisk, runail, d...|\n",
            "| 14|[konad, irisk, it...|[staleks, bluesky...|\n",
            "| 15|[grattol, bpw.sty...|[runail, irisk, d...|\n",
            "| 16|[irisk, grattol, ...|[domix, severina,...|\n",
            "| 17|[irisk, runail, n...|[staleks, bluesky...|\n",
            "| 18|             [estel]|[irisk, runail, k...|\n",
            "| 19|[severina, cosmop...|[irisk, runail, g...|\n",
            "+---+--------------------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}