正向最大匹配算法：从左到右将待分词文本中的几个连续字符与词表匹配，如果匹配上，则切分出一个词，并且要做到最大匹配。
反向最大匹配算法：从右到左将待分词文本中的几个连续字符与词表匹配，如果匹配上，则切分出一个词，并且要做到最大匹配。
这份代码对正向最大匹配算法和反向最大匹配算法进行封装，需要在代码的目录下存放一份词典，词典取名为：chineseDic.txt。
样例如下：
冮,nr
劼人,nr
勍,nr
喆,nr
揳入,v
即可以是无headers的csv格式文件，词,词性这样的格式存放。
代码如下：
#!/usr/bin/python3
# -*- coding:utf-8 -*-
# Author:ChenYuan
import time
import os
class CSegment(object):
def __init__(self):
self.question = None
self.true_index = []
self.with_user_dict = False
self.reverse = False
self.result_reverse = False
self.MM_result_index = []
self.RMM_result_index = []
self.MM_result_list = []
self.RMM_result_list = []
self.word_pos_dict = {}
def read_user_dict(self, dict_path):
"""
:param dict_path: 用户定义的词典文件
:return:
"""
tic = time.clock()
word_pos = {}
if not os.path.exists(dict_path):
print('该文件不存在')
assert os.path.exists(dict_path) is True
with open(dict_path, 'r')as fp:
for line in fp:
line = line.strip()
word = line.split(',')[0]
pos = line.split(',')[1]
word_pos[word] = pos
self.word_pos_dict = word_pos
self.with_user_dict = True
toc = time.clock()
time_clock = toc - tic
print('\033[1;31;47m')
print('*' * 50)
print('*Load user dict:\t', dict_path)
print('*Load time:\t', time_clock)
print('*' * 50)
print('\033[0m')
def read_true_sentence(self, true_result):
"""
:param true_result: 正确的分词结果
:return: 分词结果的下表元组列表
"""
if len(true_result) == 0:
return []
else:
true_list = [t.strip() for t in true_result.split('/')]
true_index = []
index = 0
for t in true_list:
lth = len(t)
if index + lth == len(self.question):
break
if self.question[index:index + lth] == t:
true_index.append(str((index, index + lth)))
index += lth
return true_index
def get_true_index(self, result_list):
"""
:param result_list: 结果列表
:return: 结果对应的下表列表
"""
if self.reverse:
self.reverse = False
return self.RMM_result_index
else:
return self.MM_result_index
def evaluate(self, true_list, result_list):
"""
:param true_list: 正确的分词列表
:param result_list: 算法得到的分词列表
:return: 三种评价指标：{正确率，召回率，F1-score}
"""
true_index = self.read_true_sentence(true_list)
result_index = self.get_true_index(result_list)
if len(true_index) == 0:
print('未导入正确结果，不能进行评估')
assert len(true_index) > 0
tp = 0
fp = 0
fn = 0
for r, t in zip(result_index, true_index):
if r in true_index:
tp += 1
if r not in true_index:
fp += 1
if t not in result_index:
fn += 1
precision = tp / (tp + fp)
recall = tp / (tp + fn)
F1 = 2 * precision * recall / (precision + recall)
evaluate_result = {'Precision': precision, 'Recall': recall, 'F1': F1}
return evaluate_result
@staticmethod
def read_own_dict():
dict_path = './chineseDic.txt'
word_pos_dict = {}
if not os.path.exists(dict_path):
print('该文件不存在')
assert os.path.exists(dict_path) is True
with open(dict_path, 'r')as fp:
for line in fp:
line = line.strip()
word = line.split(',')[0]
w_pos = line.split(',')[1]
word_pos_dict[word] = w_pos
return word_pos_dict
def MM(self, sentence, lth, pos=False):
"""
:param sentence: 待分词句子
:param lth: 正向匹配的最大长度
:param pos: 结果是否显示词性标注
"""
self.reverse = False
self.result_reverse = False
if lth <= 1:
print('max_len 不能小于2')
assert lth > 1
if len(sentence) == 0:
print('原句子不能为空')
assert len(sentence) > 0
self.question = sentence
if self.with_user_dict:
word_pos_dict = self.word_pos_dict
else:
word_pos_dict = self.read_own_dict()
result_list = []
result_index = []
max_lth = lth
index = 0
index_last = index + max_lth
while index <= len(sentence):
if sentence[index:index_last] in word_pos_dict.keys():
if pos:
result_list.append(sentence[index:index_last] + '/' + word_pos_dict[sentence[index:index_last]])
else:
result_list.append(sentence[index:index_last])
result_index.append(str((index, index_last)))
index = index_last
index_last = index + max_lth
else:
index_last -= 1
self.MM_result_index = result_index
self.MM_result_list = result_list
def RMM(self, sentence, lth, pos=False):
"""
:param sentence: 待分词句子
:param lth: 反向匹配的最大长度
:param pos: 结果是否显示词性标注
:return:
"""
self.reverse = True
self.result_reverse = True
if lth <= 1:
print('max_len 不能小于2')
assert lth > 1
if len(sentence) == 0:
print('原句子不能为空')
assert len(sentence) > 0
self.question = sentence
if self.with_user_dict:
word_pos_dict = self.word_pos_dict
else:
word_pos_dict = self.read_own_dict()
result_list = []
result_index = []
max_lth = lth
index_last = len(sentence)
index = index_last - max_lth
while index_last != 0:
if sentence[index:index_last] in word_pos_dict.keys():
if pos:
result_list.append(sentence[index:index_last] + '/' + word_pos_dict[sentence[index:index_last]])
else:
result_list.append(sentence[index:index_last])
result_index.append(str((index, index_last)))
index_last = index
index = index_last - max_lth
else:
index += 1
result_list.reverse()
result_index.reverse()
self.RMM_result_index = result_index
self.RMM_result_list = result_list
def get_result(self):
"""
:return: 返回结果
"""
if self.result_reverse:
return self.RMM_result_list
else:
return self.MM_result_list
由于是课程作业，对方法的调用和定义的变量有要求，所以显得比较笨拙，但是代码简单，可修改性强，没有很复杂的函数调用和第三方模块调用。
下面是测试方法：
question = '命名时应考虑的因素：直观、时髦用语、暗示创业模型、有说服力、能吸引顾客的注意力。'
true = '命名/ 时/ 应/ 考虑/ 的/ 因素/ ：/ 直观/ 、/ 时髦/ 用语/ 、/ 暗示/ 创业/ 模型/ 、/ 有/ 说服力/ 、/ 能/ 吸引/ 顾客/ 的/ 注意力/ 。/ '
# 这里是指用户词典
chinese_dict = 'chineseDic.txt'
max_len = 3
# 实例化一个对象cut
cut = CSegment()
# 加载用户词典
cut.read_user_dict(chinese_dict)
# 正向最大匹配
cut.MM(sentence=question, lth=max_len, pos=True)
# 获取结果
MM_result = cut.get_result()
print(MM_result)
# 评价分词结果，需要有true（正确答案）
evaluate = cut.evaluate(true, MM_result)
print(evaluate)
# 反向最大匹配
cut.RMM(sentence=question, lth=max_len, pos=True)
RMM_result = cut.get_result()
print(RMM_result)
evaluate = cut.evaluate(true, RMM_result)
print(evaluate)
