classification/bertopic.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Topic Extraction with BERTopic"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "numpy.ndarray size changed, may indicate binary incompatibility. Expected 96 from C header, got 88 from PyObject",
     "output_type": "error",
     "traceback": [
      "\u001B[0;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[0;31mValueError\u001B[0m                                Traceback (most recent call last)",
      "\u001B[0;32m/tmp/ipykernel_46304/1159876270.py\u001B[0m in \u001B[0;36m<module>\u001B[0;34m\u001B[0m\n\u001B[0;32m----> 1\u001B[0;31m \u001B[0;32mfrom\u001B[0m \u001B[0mbertopic\u001B[0m \u001B[0;32mimport\u001B[0m \u001B[0mBERTopic\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m      2\u001B[0m \u001B[0;32mimport\u001B[0m \u001B[0mre\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m      3\u001B[0m \u001B[0;32mfrom\u001B[0m \u001B[0menum\u001B[0m \u001B[0;32mimport\u001B[0m \u001B[0mEnum\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m      4\u001B[0m \u001B[0;32mfrom\u001B[0m \u001B[0mnltk\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mcorpus\u001B[0m \u001B[0;32mimport\u001B[0m \u001B[0mstopwords\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m      5\u001B[0m \u001B[0;32mfrom\u001B[0m \u001B[0mspacy\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mlang\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mde\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mstop_words\u001B[0m \u001B[0;32mimport\u001B[0m \u001B[0mSTOP_WORDS\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/.local/lib/python3.9/site-packages/bertopic/__init__.py\u001B[0m in \u001B[0;36m<module>\u001B[0;34m\u001B[0m\n\u001B[0;32m----> 1\u001B[0;31m \u001B[0;32mfrom\u001B[0m \u001B[0mbertopic\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0m_bertopic\u001B[0m \u001B[0;32mimport\u001B[0m \u001B[0mBERTopic\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m      2\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m      3\u001B[0m \u001B[0m__version__\u001B[0m \u001B[0;34m=\u001B[0m \u001B[0;34m\"0.9.4\"\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m      4\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m      5\u001B[0m __all__ = [\n",
      "\u001B[0;32m~/.local/lib/python3.9/site-packages/bertopic/_bertopic.py\u001B[0m in \u001B[0;36m<module>\u001B[0;34m\u001B[0m\n\u001B[1;32m     19\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m     20\u001B[0m \u001B[0;31m# Models\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 21\u001B[0;31m \u001B[0;32mimport\u001B[0m \u001B[0mhdbscan\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m     22\u001B[0m \u001B[0;32mfrom\u001B[0m \u001B[0mumap\u001B[0m \u001B[0;32mimport\u001B[0m \u001B[0mUMAP\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m     23\u001B[0m \u001B[0;32mfrom\u001B[0m \u001B[0msklearn\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mfeature_extraction\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0mtext\u001B[0m \u001B[0;32mimport\u001B[0m \u001B[0mCountVectorizer\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/.local/lib/python3.9/site-packages/hdbscan/__init__.py\u001B[0m in \u001B[0;36m<module>\u001B[0;34m\u001B[0m\n\u001B[0;32m----> 1\u001B[0;31m \u001B[0;32mfrom\u001B[0m \u001B[0;34m.\u001B[0m\u001B[0mhdbscan_\u001B[0m \u001B[0;32mimport\u001B[0m \u001B[0mHDBSCAN\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mhdbscan\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0m\u001B[1;32m      2\u001B[0m \u001B[0;32mfrom\u001B[0m \u001B[0;34m.\u001B[0m\u001B[0mrobust_single_linkage_\u001B[0m \u001B[0;32mimport\u001B[0m \u001B[0mRobustSingleLinkage\u001B[0m\u001B[0;34m,\u001B[0m \u001B[0mrobust_single_linkage\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m      3\u001B[0m \u001B[0;32mfrom\u001B[0m \u001B[0;34m.\u001B[0m\u001B[0mvalidity\u001B[0m \u001B[0;32mimport\u001B[0m \u001B[0mvalidity_index\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m      4\u001B[0m from .prediction import (approximate_predict,\n\u001B[1;32m      5\u001B[0m                          \u001B[0mmembership_vector\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32m~/.local/lib/python3.9/site-packages/hdbscan/hdbscan_.py\u001B[0m in \u001B[0;36m<module>\u001B[0;34m\u001B[0m\n\u001B[1;32m     19\u001B[0m \u001B[0;32mfrom\u001B[0m \u001B[0mscipy\u001B[0m\u001B[0;34m.\u001B[0m\u001B[0msparse\u001B[0m \u001B[0;32mimport\u001B[0m \u001B[0mcsgraph\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m     20\u001B[0m \u001B[0;34m\u001B[0m\u001B[0m\n\u001B[0;32m---> 21\u001B[0;31m from ._hdbscan_linkage import (single_linkage,\n\u001B[0m\u001B[1;32m     22\u001B[0m                                \u001B[0mmst_linkage_core\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n\u001B[1;32m     23\u001B[0m                                \u001B[0mmst_linkage_core_vector\u001B[0m\u001B[0;34m,\u001B[0m\u001B[0;34m\u001B[0m\u001B[0;34m\u001B[0m\u001B[0m\n",
      "\u001B[0;32mhdbscan/_hdbscan_linkage.pyx\u001B[0m in \u001B[0;36minit hdbscan._hdbscan_linkage\u001B[0;34m()\u001B[0m\n",
      "\u001B[0;31mValueError\u001B[0m: numpy.ndarray size changed, may indicate binary incompatibility. Expected 96 from C header, got 88 from PyObject"
     ]
    }
   ],
   "source": [
    "from bertopic import BERTopic\n",
    "import re\n",
    "from enum import Enum\n",
    "from nltk.corpus import stopwords\n",
    "from spacy.lang.de.stop_words import STOP_WORDS"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "class Party(Enum):\n",
    "    AFD = 0\n",
    "    CDU = 1\n",
    "    FDP = 2\n",
    "    GRUENE = 3\n",
    "    LINKE = 4\n",
    "    SPD = 5"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "Load party texts"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%% md\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# stopwords\n",
    "nltk_stopwords = stopwords.words('german')\n",
    "\n",
    "# build stopwords list\n",
    "all_stopwords = list(set(STOP_WORDS) | set(nltk_stopwords))\n",
    "with open('custom_stopwords.txt', 'r', encoding='utf-8') as f:\n",
    "    all_stopwords += [line.strip() for line in f.readlines()]\n",
    "\n",
    "party_text = {}\n",
    "for party in Party:\n",
    "    all_stopwords.extend(['{}'.format(party.name.lower())])\n",
    "    with open('resources/' + party.name + '.txt', encoding='utf-8', errors='ignore') as txt:\n",
    "        file = \" \".join(l for l in txt)\n",
    "        # remove gender *\n",
    "        file = re.sub(r'\\*innen(\\w*)\\s', r'\\1 ', file)\n",
    "    party_text[party] = file"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "def split_in_256_chunks(text):\n",
    "    chunks = []\n",
    "    tokens = text.split(' ')\n",
    "    for i in range(0, len(tokens), 256):\n",
    "        chunks.append(\" \".join(tokens[i: min(i+255, len(tokens) - 1)]))\n",
    "    return chunks\n",
    "\n",
    "party_text = {p: split_in_256_chunks(t) for (p,t) in party_text.items()}"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "model = BERTopic(language='german')\n",
    "topics, probs = model.fit_transform(party_text[Party.FDP])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "model.visualize_topics()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}