{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 机器学习与社会科学应用"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第四章 自然语言处理入门"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第三节 TF-IDF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<font face=\"宋体\" >郭峰    \n",
    "    教授、博士生导师  \n",
    "上海财经大学公共经济与管理学院  \n",
    "上海财经大学数实融合与智能治理实验室    \n",
    "    邮箱：guofengsfi@163.com</font> "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 目标：用语料库表征两个文本，然后计算他们的相似度\n",
    "# 这个例子没有考虑到自定义词典和停用词"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 安装gensim: pip install gensim\n",
    "import gensim \n",
    "print(gensim.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import jieba\n",
    "from gensim import corpora,models,similarities\n",
    "\n",
    "# 语料文档\n",
    "doc0 = \"我不喜欢上海\"\n",
    "doc1 = \"上海是一个好地方\"\n",
    "doc2 = \"北京是一个好地方\"\n",
    "doc3 = \"上海好吃的在哪里\"\n",
    "doc4 = \"上海好玩的在哪里\"\n",
    "doc5 = \"上海是好地方\"\n",
    "doc6 = \"上海路和上海人\"\n",
    "doc7 = \"喜欢小吃\"\n",
    "\n",
    "# 测试文档\n",
    "doc_test1 = \"我喜欢上海的小吃\"\n",
    "doc_test2 = \"我不喜欢上海\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 分词\n",
    "# 为了简化操作，把目标文档放到一个列表all_doc\n",
    "all_doc = []\n",
    "all_doc.append(doc0)\n",
    "all_doc.append(doc1)\n",
    "all_doc.append(doc2)\n",
    "all_doc.append(doc3)\n",
    "all_doc.append(doc4)\n",
    "all_doc.append(doc5)\n",
    "all_doc.append(doc6)\n",
    "all_doc.append(doc7)\n",
    "print(all_doc)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# 对目标文档进行分词，并且保存在列表all_doc_list中\n",
    "all_doc_list = []\n",
    "for doc in all_doc:\n",
    "    doc_list = [word for word in jieba.cut(doc)]\n",
    "    all_doc_list.append(doc_list)\n",
    "\n",
    "all_doc_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 制作语料库，首先用dictionary方法获取词袋（bag-of-words)\n",
    "dictionary = corpora.Dictionary(all_doc_list)\n",
    "print(len(dictionary))\n",
    "print(dictionary.keys())\n",
    "print(list(dictionary.values()))\n",
    "print(dictionary[0])\n",
    "print(dictionary[1])\n",
    "print(dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 编号与词之间的对应关系\n",
    "dictionary.token2id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 使用doc2bow制作语料库\n",
    "corpus = [dictionary.doc2bow(doc) for doc in all_doc_list]\n",
    "corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 使用TF-IDF模型对语料库建模\n",
    "tfidf = models.TfidfModel(corpus, dictionary=dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 测试文档也进行分词，并保存在列表doc_test_list中\n",
    "doc_test1_cut = [word for word in jieba.cut(doc_test1)]\n",
    "doc_test2_cut = [word for word in jieba.cut(doc_test2)]\n",
    "print(doc_test1_cut)\n",
    "print(doc_test2_cut)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 用同样的方法，把测试文档也转换为二元组的向量\n",
    "doc_test1_vec = dictionary.doc2bow(doc_test1_cut)\n",
    "doc_test2_vec = dictionary.doc2bow(doc_test2_cut)\n",
    "print(doc_test1_vec)\n",
    "print(doc_test2_vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 计算测试文档的tfidf向量表示\n",
    "doc_test1_tfidf = tfidf[doc_test1_vec]\n",
    "print(doc_test1_tfidf)\n",
    "doc_test2_tfidf = tfidf[doc_test2_vec]\n",
    "print(doc_test2_tfidf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 两个测试文档相似度的计算\n",
    "# gensim自带的similarities算法使用方法为计算某一文档（目标文档）与另外一组文档（对比文档集）的相似度，输出也是一个列表\n",
    "# 先将目标文档集列表化，这里列表只有一个元素\n",
    "\n",
    "doc_test1_list = [doc_test1_vec]\n",
    "doc_test1_tfidf = tfidf[doc_test1_list]\n",
    "print(doc_test1_tfidf[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 对文档集相似度计算进行建模\n",
    "tfidf_sim = similarities.SparseMatrixSimilarity(doc_test1_tfidf,num_features=len(dictionary.keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 计算目标文档与对比文档集（这里只有一个文件）的相似度列表\n",
    "sim = tfidf_sim[doc_test2_tfidf]\n",
    "print(sim[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 余弦相似度的具体计算（解析上面的相似度0.54680777到底是怎么得来的）\n",
    "vec1 = [0.08112725037593049,0,0.3909393754390612,0.5864090631585919,0,0,0,0,0,0,0,0,0.3909393754390612,0,0,0,0,0.58640906315859,0]\n",
    "vec2 = [0.08814189721744814,0.6371127720068723,0.4247418480045816,0.6371127720068723,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]\n",
    "def similarity(a_vect, b_vect):\n",
    "    dot_val = 0.0\n",
    "    a_norm = 0.0\n",
    "    b_norm = 0.0\n",
    "    cos = None\n",
    "    for a, b in zip(a_vect, b_vect):\n",
    "        dot_val += a*b\n",
    "        a_norm += a**2\n",
    "        b_norm += b**2\n",
    "    if a_norm == 0.0 or b_norm == 0.0:\n",
    "        cos = -1\n",
    "    else:\n",
    "        cos = dot_val / ((a_norm*b_norm)**0.5)\n",
    "    return cos\n",
    "print(similarity(vec1,vec2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 计算目标文档与测试集所有文档的相似度\n",
    "print(corpus)\n",
    "doc_train_tfidf=tfidf[corpus]\n",
    "print(doc_train_tfidf)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_sim = similarities.SparseMatrixSimilarity(doc_train_tfidf, num_features=len(dictionary.keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_test1_tfidf = tfidf[doc_test1_vec]\n",
    "sim1 = tfidf_sim[doc_test1_tfidf]\n",
    "print(sim1)\n",
    "print(sim1.max())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_test2_tfidf = tfidf[doc_test2_vec]\n",
    "sim2 = tfidf_sim[doc_test2_tfidf]\n",
    "print(sim2)\n",
    "print(sim2.max())  # 注：目标文档2就是测试集中的一个文档，所以相似度为1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 输出与目标文档最最相似文档的编号（文档）及相似度数值\n",
    "# 根据相似度排序\n",
    "sim1_new = sorted(enumerate(sim1), key=lambda item: -item[1])\n",
    "# 从分析结果来看，测试文档与doc7相似度最高，其次是doc0，与doc2的相似度为零\n",
    "print(sim1_new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(sim1_new[0][1])\n",
    "print(all_doc)\n",
    "print(all_doc[sim1_new[0][0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 本节结束"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}