learn_pytorch_53_pytorch自然语言处理实战前的准备之gensim主题模型.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1 TF-IDF模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim\n",
    "import pprint\n",
    "from gensim import corpora\n",
    "from gensim.utils import simple_preprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_list = [\n",
    "   \"Hello, how are you?\", \"How do you do?\", \n",
    "   \"Hey what are you doing? yes you What are you doing?\"\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_tokenized = [simple_preprocess(doc) for doc in doc_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "dictionary = corpora.Dictionary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "BoW_corpus = [dictionary.doc2bow(doc, allow_update=True) for doc in doc_tokenized]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['are', 1], ['hello', 1], ['how', 1], ['you', 1]]\n",
      "[['how', 1], ['you', 1], ['do', 2]]\n",
      "[['are', 2], ['you', 3], ['doing', 2], ['hey', 1], ['what', 2], ['yes', 1]]\n"
     ]
    }
   ],
   "source": [
    "for doc in BoW_corpus:\n",
    "   print([[dictionary[id], freq] for id, freq in doc])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from gensim import models\n",
    "tfidf = models.TfidfModel(BoW_corpus, smartirs='ntc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['are', 0.4], ['hello', 0.81], ['how', 0.4], ['you', 0.17]]\n",
      "[['how', 0.24], ['you', 0.1], ['do', 0.97]]\n",
      "[['are', 0.3], ['you', 0.18], ['doing', 0.59], ['hey', 0.3], ['what', 0.59], ['yes', 0.3]]\n"
     ]
    }
   ],
   "source": [
    "for doc in tfidf[BoW_corpus]:\n",
    "   print([[dictionary[id], np.around(freq,decimals=2)] for id, freq in doc])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2 LDA模型"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3 LSI 模型"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4 HDP模型"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5 Word2Vec 模型"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "lda  = gensim.models.ldamodel.LdaModel()    \n",
    "lsi  = gensim.models.lsimodel.LsiModel()    \n",
    "hdp = gensim.models.hdpmodel.HdpModel()   \n",
    "w2c = gensim.models.word2vec.Word2Vec()  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "详细内容参考官网：https://radimrehurek.com/gensim/auto_examples/index.html  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}