learn_pytorch_55_pytorch自然语言处理实战之中文分词基于词典.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 基于词典的分词3种方法\n",
    "(1)正向最大匹配算法  \n",
    "(2)逆向最大匹配算法  \n",
    "(3)双向最大匹配算法  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy\n",
    "\n",
    "\n",
    "def maximum_forward_matching_method(word_dict, waited_word_string):\n",
    "    # 1 统计字典最常词语的长度\n",
    "    # 数组中每个元素都是一个字符串\n",
    "    max_length = max([len(word) for word in word_dict])\n",
    "\n",
    "    # 2 列表接收分词\n",
    "    result_word_list = []\n",
    "\n",
    "    # 3 按正向(从左到右)的方向，首先以最大长度进行匹配\n",
    "    word_index = 0\n",
    "    new_string = copy.deepcopy(waited_word_string)\n",
    "\n",
    "    while len(new_string) != 0:\n",
    "        length = word_index + max_length\n",
    "        # print(length)\n",
    "        if waited_word_string[word_index:length] in word_dict:\n",
    "            new_string = waited_word_string.replace(waited_word_string[word_index:length], \"\")\n",
    "\n",
    "            # print(new_string)\n",
    "            result_word_list.append(waited_word_string[word_index:length])\n",
    "            # print(result_word_list)\n",
    "            waited_word_string = copy.deepcopy(new_string)\n",
    "            # print(waited_word_string)\n",
    "            # 重置最大值\n",
    "            max_length = max([len(word) for word in word_dict])\n",
    "        else:\n",
    "            max_length -= 1\n",
    "\n",
    "    return result_word_list\n",
    "\n",
    "\n",
    "def maximum_reverse_matching_method(word_dict, waited_word_string):\n",
    "    # 1 统计字典最常词语的长度\n",
    "    # 词典中每个元素都是一个字符串\n",
    "    max_length = max([len(word) for word in word_dict])\n",
    "    # 2 初始化列表接收分词\n",
    "    result_word_list = []\n",
    "\n",
    "    # 3 从逆向进行统计\n",
    "    new_string = copy.deepcopy(waited_word_string)\n",
    "    word_index = len(new_string)\n",
    "\n",
    "    while len(new_string) != 0:\n",
    "        # print(len(new_string))\n",
    "\n",
    "        length = word_index - max_length\n",
    "        # print(length)\n",
    "\n",
    "        if length < 0:\n",
    "            length = 0\n",
    "        if waited_word_string[length:word_index] in word_dict:\n",
    "            new_string = waited_word_string.replace(waited_word_string[length:word_index], \"\")\n",
    "            # print(new_string)\n",
    "\n",
    "            result_word_list.append(waited_word_string[length:word_index])\n",
    "            # print(result_word_list)\n",
    "\n",
    "            waited_word_string = copy.deepcopy(new_string)\n",
    "            # print(waited_word_string)\n",
    "\n",
    "            # 重置最大值\n",
    "            max_length = max([len(word) for word in word_dict])\n",
    "\n",
    "            # 剩余词的长度在变化\n",
    "            word_index = len(new_string)\n",
    "        else:\n",
    "            max_length -= 1\n",
    "\n",
    "    result_word_list.reverse()\n",
    "    return result_word_list\n",
    "\n",
    "\n",
    "def calculate_list_single_char_length(object_list):\n",
    "    char_count = 0\n",
    "    for string in object_list:\n",
    "        if len(string) == 1:\n",
    "            char_count += 1\n",
    "\n",
    "    return char_count\n",
    "\n",
    "\n",
    "def bidirectional_maximum_matching_algorithm(word_dict, waited_word_string):\n",
    "    # 1 正向最大匹配算法\n",
    "    maximum_forward_matching_result = maximum_forward_matching_method(word_dict, waited_word_string)\n",
    "    print(maximum_forward_matching_result)\n",
    "    # 2 逆向最大匹配算法\n",
    "    maximum_reverse_matching_result = maximum_reverse_matching_method(word_dict, waited_word_string)\n",
    "    print(maximum_reverse_matching_result)\n",
    "    # 3 比较二者的分词数，分词数目不一样，取分词数量少的那一个。\n",
    "    if len(maximum_forward_matching_result) < len(maximum_reverse_matching_result):\n",
    "        return maximum_forward_matching_result\n",
    "    # 选分词数目少的那个\n",
    "    elif len(maximum_reverse_matching_result) < len(maximum_forward_matching_result):\n",
    "        return maximum_reverse_matching_result\n",
    "    # 数目相等时，比较结果，结果相同时都可以\n",
    "    elif maximum_forward_matching_result == maximum_reverse_matching_result:\n",
    "        return maximum_forward_matching_result\n",
    "    # 结果不同时，选包含单字少的\n",
    "    elif calculate_list_single_char_length(maximum_forward_matching_result) < calculate_list_single_char_length(\n",
    "            maximum_reverse_matching_result):\n",
    "        return maximum_forward_matching_result\n",
    "    # 分词数相同，单字也相同，不在考虑。思路是包含2个字的最少的，递归。\n",
    "    else:\n",
    "        return maximum_reverse_matching_result\n",
    "\n",
    "\n",
    "def main():\n",
    "    # word_dict = [\"\", \"习\", \"近平\", \"孙\", \"dh\", \"张三\", \"例子子子子\"]\n",
    "    # waited_word_string = \"孙dh习近平\"\n",
    "\n",
    "    word_dict = [\"研究\", \"研究生\", \"生命\", \"命\", \"的\", \"起源\"]\n",
    "    waited_word_string = \"研究生命的起源\"\n",
    "\n",
    "    result_forward = maximum_forward_matching_method(word_dict, waited_word_string)\n",
    "    print(\"正向匹配算法结果：\", result_forward)\n",
    "\n",
    "    result_reverse = maximum_reverse_matching_method(word_dict, waited_word_string)\n",
    "    print(\"逆向匹配算法结果：\", result_reverse)\n",
    "\n",
    "    bidirectional_result = bidirectional_maximum_matching_algorithm(word_dict, waited_word_string)\n",
    "    print(\"双向匹配算法结果：\", bidirectional_result)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "正向匹配算法结果： ['研究生', '命', '的', '起源']\n",
      "逆向匹配算法结果： ['研究', '生命', '的', '起源']\n",
      "['研究生', '命', '的', '起源']\n",
      "['研究', '生命', '的', '起源']\n",
      "双向匹配算法结果： ['研究', '生命', '的', '起源']\n"
     ]
    }
   ],
   "source": [
    "main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}