论文链接：
https://arxiv.org/abs/1805.08092
论文解决的问题：
作者从对SQuAD等数据的观察看到大部分Question对应的答案只需要Context少数的几句话既可以给出直接的判断，并不需要全文信息完成QA推断。具体需要的Context的句数及占比见下图：
SQuAD 90%的问题仅需要单句信息。
现在对于答案从Context选择使用全文进行检索的精度都是比较高的，有很多常用的方法，感兴趣可以看之前
BI-DIRECTIONALATTENSION FLOW FOR MACHINE COMPREHENSION 论文阅读及实现
等文章。但是这些全文“检索”的模型在算法性能上有待提高，比如上面提到的模型要在Context及Question之间构建两个句子“直积”的attension表示，对于Context很长的情况显然是不利的，故本文给出了进行句子Selector的结构，希望通过对于Context的预挑选缩小Context输入信息来减少性能消耗，下面仅就 Sentence Selector部分给出其构造及简要介绍。
Sentence Selector基本构成：
给定Context:
后面不加说明地用Context指代Context中的一个句子。
Question
首先要找到Question的Context aware表示
方法就是如下attension
之后fuse信息进bilstm
再实现attension进行Context的重要度打分：
整体结构见下图：
在这里有一个疑问，在这种选择结构中好像没有用到，不同Context子句的关系信息，或上下文信息，比如一般地对于这种句子或者篇章分割结构常采取的方式是，对每个子句进行dense表示后按序送入rnn再次进行顺序信息提取，这样构建上下文信息。
在进行打分后有取threshold及topk的方法来进行选出句子数目限定。
下面仅就训练Sentence Selector的情况给出二分类场景的实现：
数据处理：（下载最新数据进行数据处理及编码过程）
import json
from nltk.tokenize import word_tokenize
from collections import Counter, defaultdict
import pickle
class DataItem():
def __init__(self, context_sequence_list, qest_answer_nest_tuple_list):
self.context_sequence_list = context_sequence_list
self.qest_answer_nest_tuple_list = qest_answer_nest_tuple_list
def sentense_process():
with open(r"E:\Coding\python\squad_model\data\train-v1.1.json", "r" ,encoding="utf-8") as f:
train_json_obj = json.load(f)
with open(r"E:\Coding\python\squad_model\data\dev-v1.1.json", "r", encoding="utf-8") as f:
dev_json_obj = json.load(f)
train_data = train_json_obj["data"]
dev_data = dev_json_obj["data"]
def data_split_func(input_data_obj):
req_list = []
for d in input_data_obj:
for k, v in d.items():
if k == "paragraphs":
for l in v:
for kk, vv in l.items():
data_item = DataItem([], [])
if kk == "qas":
temp_list = []
for aq_dict in vv:
answer_text = aq_dict["answers"][0]["text"]
question_text = aq_dict["question"]
temp_list.append((question_text, answer_text))
data_item.qest_answer_nest_tuple_list.append(temp_list)
elif kk == "context":
context = vv
data_item.context_sequence_list.append(context.split("."))
req_list.append(data_item)
return req_list
train_list_data, test_list_data = map(data_split_func, [train_data, dev_data])
print("construct end")
def tokenlize_and_encode_obj_data(input_list):
cnt = Counter()
context_sequence_pos_list = []
qest_answer_nest_tuple_pos_list = []
for data_item in input_list:
context_sequence_list = data_item.context_sequence_list
qest_answer_nest_tuple_list = data_item.qest_answer_nest_tuple_list
for inner_list in context_sequence_list:
temp_list = []
for context_str in inner_list:
context_pos = word_tokenize(context_str.lower())
temp_list.append(context_pos)
cnt.update(context_pos)
context_sequence_pos_list.append(temp_list)
for inner_list in qest_answer_nest_tuple_list:
temp_list = []
for t2 in inner_list:
t2_pos = list(map(word_tokenize, map(lambda x: x.lower(),t2)))
cnt.update(t2_pos[0] + t2_pos[1])
temp_list.append(tuple(t2_pos))
qest_answer_nest_tuple_pos_list.append(temp_list)
return (context_sequence_pos_list, qest_answer_nest_tuple_pos_list, cnt)
train_t3, dev_t3 = map(tokenlize_and_encode_obj_data, [train_list_data, test_list_data])
print("tokenize end")
train_c, train_q, train_cnt = train_t3
dev_c, dev_q, dev_cnt = dev_t3
total_cnt_dict = defaultdict(int)
for k, v in train_cnt.items():
total_cnt_dict[k] += v
for k, v in dev_cnt.items():
total_cnt_dict[k] += v
word2idx = dict((word, idx) for idx, word in enumerate(list(total_cnt_dict.keys())))
print("word2idx len: {}".format(len(word2idx)))
def index_func(t2):
context_input, qa_input = t2
req_c = []
for temp_list in context_input:
tl = []
for pos_list in temp_list:
tl.append(list(map(lambda w: word2idx[w], pos_list)))
req_c.append(tl)
req_qa = []
for temp_list in qa_input:
tl = []
for pos_t2_list in temp_list:
tl.append(tuple(map(lambda pos_list: list(map(lambda w: word2idx[w], pos_list)), pos_t2_list)))
req_qa.append(tl)
return (req_c, req_qa)
train_req, dev_req = map(index_func, [(train_c, train_q), (dev_c, dev_q)])
print("index end :")
with open("serlize.pkl", "wb") as f:
pickle.dump({
"train_req": train_req,
"dev_req": dev_req,
"word2idx": word2idx
}, f)
print("dump end")
if __name__ == "__main__":
sentense_process()
数据导出及网络构建：
import pickle
import numpy as np
import tensorflow as tf
from sklearn.utils import shuffle
from sklearn.metrics import f1_score
with open("data_preprocess/serlize.pkl", "rb") as f:
total_construct = pickle.load(f)
train_req = total_construct["train_req"]
dev_req = total_construct["dev_req"]
word2idx = total_construct["word2idx"]
def data_gen(data_req = train_req, padding_idx = len(word2idx), question_max_len = 50, context_max_len = 50,
batch_size = 64 * 8):
req_c, req_qa = data_req
assert len(req_c) == len(req_qa)
start_idx = 0
quest_data = np.full(shape=[batch_size, question_max_len], fill_value=padding_idx).astype(np.int32)
context_data = np.full(shape=[batch_size, context_max_len], fill_value=padding_idx).astype(np.int32)
quest_mask = np.zeros(shape=[batch_size]).astype(np.int32)
context_mask = np.zeros(shape=[batch_size]).astype(np.int32)
label_data = np.zeros(shape=[batch_size, 2])
for idx, req_c_ele in enumerate(req_c):
req_qa_ele = req_qa[idx]
req_c_ele_str = list(map(lambda pos_list: " ".join(map(str ,pos_list)), req_c_ele))
req_a_ele_str = list(map(lambda t2: " ".join(map(str ,t2[1])), req_qa_ele))
answer_boolean_nest_list = []
for answer_str in req_a_ele_str:
temp_list = []
for c_ele in req_c_ele_str:
temp_list.append(answer_str in c_ele)
answer_boolean_nest_list.append(temp_list)
answer_boolean_list = list(map(lambda inner_list: sum(map(int ,inner_list)), answer_boolean_nest_list))
for answer_idx ,answer_boolean in enumerate(answer_boolean_list):
if answer_boolean:
answer_boolean_inner_list = answer_boolean_nest_list[answer_idx]
have_answer_qa_ele = req_qa_ele[answer_idx]
# only use have answer ele for training and valid, and assign label by answer_boolean_nest_list
# boolean value.
for req_c_idx ,req_c in enumerate(req_c_ele):
for c_idx, c in enumerate(req_c):
if c_idx == context_max_len:
break
context_data[start_idx][c_idx] = c
for q_idx, q in enumerate(have_answer_qa_ele[0]):
if q_idx == question_max_len:
break
quest_data[start_idx][q_idx] = q
quest_mask[start_idx] = int(min(len(req_c_ele), context_max_len))
context_mask[start_idx] = int(min(len(have_answer_qa_ele[0]), context_max_len))
label_data[start_idx][int(answer_boolean_inner_list[req_c_idx])] = 1
start_idx += 1
if start_idx == batch_size:
# shuffle data before yield
quest_data, context_data, quest_mask, context_mask, label_data = shuffle(quest_data, context_data, quest_mask, context_mask, label_data)
true_indexes = label_data[:, 1] == 1
false_indexes = label_data[:, 1] == 0
true_num = np.sum(true_indexes).astype(np.int32)
quest_data_t, context_data_t, quest_mask_t, context_mask_t, label_data_t = quest_data[true_indexes], context_data[true_indexes], quest_mask[true_indexes], context_mask[true_indexes], label_data[true_indexes]
quest_data_f, context_data_f, quest_mask_f, context_mask_f, label_data_f = quest_data[false_indexes], context_data[false_indexes], quest_mask[false_indexes], context_mask[false_indexes], label_data[false_indexes]
quest_data_f, context_data_f, quest_mask_f, context_mask_f, label_data_f = quest_data_f[:true_num], context_data_f[:true_num], quest_mask_f[:true_num], context_mask_f[:true_num], label_data_f[:true_num]
quest_data, context_data, quest_mask, context_mask, label_data = np.append(quest_data_t, quest_data_f, axis=0), \
np.append(context_data_t, context_data_f, axis=0),\
np.append(quest_mask_t, quest_mask_f, axis=0), np.append(context_mask_t, context_mask_f, axis=0), np.append(label_data_t, label_data_f, axis=0)
quest_data, context_data, quest_mask, context_mask, label_data = shuffle(quest_data, context_data, quest_mask, context_mask, label_data)
yield (quest_data, context_data, quest_mask, context_mask, label_data)
start_idx = 0
quest_data = np.full(shape=[batch_size, question_max_len], fill_value=padding_idx).astype(np.int32)
context_data = np.full(shape=[batch_size, context_max_len], fill_value=padding_idx).astype(np.int32)
quest_mask = np.zeros(shape=[batch_size]).astype(np.int32)
context_mask = np.zeros(shape=[batch_size]).astype(np.int32)
label_data = np.zeros(shape=[batch_size, 2])
class MCOD(object):
def __init__(self, vocab_size = len(word2idx) + 1, question_max_len = 50, context_max_len = 50,
word_embedding_size = 100, first_hidden_size = 50, second_hidden_size = 50):
# for calculate convenient first and second hidden size must have follows relation
assert first_hidden_size == second_hidden_size
self.vocab_size = vocab_size
self.question_max_len = question_max_len
self.context_max_len = context_max_len
self.word_embedding_size = word_embedding_size
self.first_hidden_size = first_hidden_size
self.second_hidden_size = second_hidden_size
self.input_question = tf.placeholder(dtype=tf.int32, shape=[None, question_max_len])
self.input_context = tf.placeholder(dtype=tf.int32, shape=[None, context_max_len])
self.question_mask = tf.placeholder(dtype=tf.int32, shape=[None])
self.context_mask = tf.placeholder(dtype=tf.int32, shape=[None])
self.input_label = tf.placeholder(dtype=tf.int32, shape=[None, 2])
self.keep_prob = tf.placeholder(dtype = tf.float32, shape=[])
self.l2_param = tf.placeholder(dtype=tf.float32, shape=[])
self.is_training = tf.placeholder(dtype=tf.bool, shape=[])
with tf.name_scope("Word_Embed"):
self.Word_Embed = tf.Variable(
tf.random_normal(shape=[self.vocab_size, self.word_embedding_size])
, name="Word_Embed")
# [batch, question_max_len ,hd]
self.input_question_lookup = tf.nn.embedding_lookup(self.Word_Embed, self.input_question, name="question_lookup")
# [batch, context_max_len ,hd]
self.input_context_lookup = tf.nn.embedding_lookup(self.Word_Embed, self.input_context, name="context_lookup")
self.W1 = tf.Variable(tf.random_normal(
shape=[self.word_embedding_size, self.word_embedding_size]
), name="W1")
self.W2 = tf.Variable(tf.random_normal(
shape=[self.first_hidden_size * 2, self.first_hidden_size * 2, self.first_hidden_size * 2]
), name="W2")
self.model_construct()
self.opt_construct()
# fuse_input have the shape [question_max_len + context_max_len, hd]
def first_attension(self, fuse_input):
# [question_max_len, hd]
question_part = tf.slice(fuse_input ,[0, 0], [self.question_max_len, -1])
# [context_max_len, hd]
context_part = tf.slice(fuse_input, [self.question_max_len, 0], [-1, -1])
# [hd, question_max_len]
Q = tf.transpose(question_part, [1, 0])
# [hd, context_max_len]
D = tf.transpose(context_part, [1, 0])
# [context_max_len, question_max_len] softmax by row
alpha_matrix = tf.nn.softmax(tf.matmul(tf.matmul(context_part, self.W1), Q), axis=-1)
# [hd, context_max_len]
Dq = tf.transpose(tf.matmul(alpha_matrix, question_part), [1, 0], name="Dq")
# [hd, question_max_len + context_max_len + context_max_len]
QDDq = tf.concat([Q, D, Dq], axis=-1, name="QDDq")
return QDDq
# fuse_input have the shape [question_max_len, hd + 1]
def simple_matmul(self, fuse_input):
# [question_max_len, hd]
hd = int(fuse_input.get_shape()[-1]) - 1
question_part = tf.slice(fuse_input ,[0, 0], [-1, hd])
# [question_max_len, 1]
beta_part = tf.slice(fuse_input, [0, hd], [-1, -1])
# [hd]
return tf.squeeze(tf.matmul(tf.transpose(beta_part, [1, 0]), question_part))
# fuse_input have the shape [context_max_len + 1, hd]
def second_attension(self, fuse_input):
# [context_max_len, hd]
context_part = tf.slice(fuse_input, [0, 0], [self.context_max_len, -1])
# [1, hd]
qenc_part = tf.slice(fuse_input, [self.context_max_len, 0], [-1, -1])
# [context_max_len ,hd]
before_reduce_max = tf.matmul(context_part, tf.reshape(tf.matmul(tf.reshape(self.W2, shape=[-1, self.first_hidden_size * 2]), tf.transpose(qenc_part, [1, 0])),
[self.first_hidden_size * 2, self.first_hidden_size * 2]))
# [hd]
reduce_max_part = tf.reduce_max(before_reduce_max, axis=0)
return reduce_max_part
def model_construct(self):
#self.input_question_lookup
hd = self.word_embedding_size
first_fuse_input = tf.concat([self.input_question_lookup, self.input_context_lookup], axis=1)
first_fuse_input = tf.layers.batch_normalization(first_fuse_input, training=self.is_training)
QDDq = tf.map_fn(self.first_attension, first_fuse_input)
QDDq = tf.transpose(QDDq, [0, 2, 1])
# [batch, question_max_len, hd]
Q = tf.slice(QDDq, [0, 0, 0], [-1, self.question_max_len, -1])
# [batch, context_max_len, hd]
D = tf.slice(QDDq, [0, self.question_max_len, 0], [-1, self.context_max_len, -1])
# [batch, context_max_len, hd]
Dq = tf.slice(QDDq, [0, self.question_max_len + self.context_max_len, 0], [-1, -1, -1])
ff_lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(num_units=self.first_hidden_size, name="ff_lstm_cell")
fb_lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(num_units=self.first_hidden_size, name="fb_lstm_cell")
ff_lstm_cell = tf.nn.rnn_cell.DropoutWrapper(ff_lstm_cell, input_keep_prob=self.keep_prob,
output_keep_prob=self.keep_prob)
fb_lstm_cell = tf.nn.rnn_cell.DropoutWrapper(fb_lstm_cell, input_keep_prob=self.keep_prob,
output_keep_prob=self.keep_prob)
outputs, output_states = tf.nn.bidirectional_dynamic_rnn(
cell_fw=ff_lstm_cell,
cell_bw=fb_lstm_cell,
inputs=tf.concat([D, Dq],axis=-1),
sequence_length=self.context_mask, dtype=tf.float32)
# each shape [batch, context_max_len, first_hidden_size]
output_fw, output_bw = outputs
# [batch, context_max_len, hd]
Denc = tf.concat([output_fw, output_bw], axis=-1)
sf_lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(num_units=self.second_hidden_size, name="sf_lstm_cell")
sb_lstm_cell = tf.nn.rnn_cell.BasicLSTMCell(num_units=self.second_hidden_size, name = "sb_lstm_cell")
sf_lstm_cell = tf.nn.rnn_cell.DropoutWrapper(sf_lstm_cell, input_keep_prob=self.keep_prob,
output_keep_prob=self.keep_prob)
sb_lstm_cell = tf.nn.rnn_cell.DropoutWrapper(sb_lstm_cell, input_keep_prob=self.keep_prob,
output_keep_prob=self.keep_prob)
outputs, output_states = tf.nn.bidirectional_dynamic_rnn(
cell_fw=sf_lstm_cell,
cell_bw=sb_lstm_cell,
inputs=Q,
sequence_length=self.question_mask,
dtype=tf.float32)
# each shape [batch, question_max_len, second_hidden_size]
output_fw, output_bw = outputs
Qenc = tf.concat([output_fw, output_bw], axis=-1)
#Qenc_part = tf.reshape(tf.transpose(Qenc, [0, 2, 1]), [-1, self.question_max_len])
Qenc_fuse_last_dim = int(Qenc.get_shape()[-1])
Qenc_part = tf.reshape(Qenc, [-1, Qenc_fuse_last_dim])
w = tf.Variable(tf.random_normal(shape=[Qenc_fuse_last_dim, 1]))
# [batch, question_max_len] softmax by row
beta_matrix = tf.nn.softmax(tf.reshape(tf.matmul(Qenc_part, w), [-1, self.question_max_len]), axis=-1)
beta_matrix_expand = tf.expand_dims(beta_matrix, axis=-1)
Qenc_beta_fuse = tf.concat([Qenc, beta_matrix_expand], axis=-1)
# [batch, hd]
qenc = tf.map_fn(self.simple_matmul, Qenc_beta_fuse)
# [batch, 1, hd]
qenc_expand = tf.transpose(tf.expand_dims(qenc, axis=-1), [0, 2, 1])
# [batch, context_max_len + 1, hd]
Denc_qenc_fuse = tf.concat([Denc, qenc_expand], axis=1)
# [batch, hd]
h = tf.map_fn(self.second_attension, Denc_qenc_fuse)
self.score_before_softmax = tf.layers.dense(inputs=h, units=2, name="score_before_softmax")
# [batch, 2]
self.score = tf.nn.softmax(self.score_before_softmax, axis=-1)
def opt_construct(self):
logits = self.score_before_softmax
labels = tf.cast(self.input_label, tf.float32)
self.loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=labels, logits = logits))
self.l2_loss = None
for train_able_var in tf.trainable_variables():
if self.l2_loss is None:
self.l2_loss = tf.nn.l2_loss(train_able_var)
else:
self.l2_loss += tf.nn.l2_loss(train_able_var)
self.loss = self.loss + self.l2_param * self.l2_loss
self.train_op = tf.train.AdamOptimizer(0.001).minimize(self.loss)
self.pred = tf.cast(tf.argmax(self.score, axis=-1), tf.int32)
self.acc = tf.reduce_mean(tf.cast(tf.equal(self.pred, tf.cast(tf.argmax(self.input_label, axis=-1), tf.int32)), tf.float32))
@staticmethod
def train():
train_gen = data_gen(data_req=train_req)
test_gen = data_gen(data_req=dev_req)
epoch = 0
times = 0
mcod_ext = MCOD()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
print("init model end")
while True:
try:
quest_data, context_data, quest_mask, context_mask, label_data = train_gen.__next__()
except:
print("epoch {} end".format(epoch))
epoch += 1
train_gen = data_gen(data_req=train_req)
quest_data, context_data, quest_mask, context_mask, label_data = train_gen.__next__()
_, loss, acc, pred = sess.run([mcod_ext.train_op, mcod_ext.loss, mcod_ext.acc, mcod_ext.pred], feed_dict={
mcod_ext.input_question: quest_data,
mcod_ext.input_context: context_data,
mcod_ext.question_mask: quest_mask,
mcod_ext.context_mask: context_mask,
mcod_ext.input_label: label_data,
mcod_ext.l2_param: 0.00001,
mcod_ext.keep_prob: 0.7,
mcod_ext.is_training: True
})
times += 1
if times % 10 == 0:
f1_val = f1_score(y_true=np.argmax(label_data, -1), y_pred=pred)
print("train loss :{} acc :{}, f1 :{}".format(loss, acc, f1_val))
try:
quest_data, context_data, quest_mask, context_mask, label_data = test_gen.__next__()
except:
print("test epoch end")
test_gen = data_gen(data_req=dev_req)
quest_data, context_data, quest_mask, context_mask, label_data = test_gen.__next__()
loss, acc, pred = sess.run([mcod_ext.loss, mcod_ext.acc, mcod_ext.pred], feed_dict={
mcod_ext.input_question: quest_data,
mcod_ext.input_context: context_data,
mcod_ext.question_mask: quest_mask,
mcod_ext.context_mask: context_mask,
mcod_ext.input_label: label_data,
mcod_ext.l2_param: 0.0,
mcod_ext.keep_prob: 1.0,
mcod_ext.is_training: False
})
f1_val = f1_score(y_true=np.argmax(label_data, -1), y_pred=pred)
print("test loss :{} acc :{}, f1 :{}".format(loss, acc, f1_val))
if __name__ == "__main__":
MCOD.train()
若干步结果：
train loss :59.26350021362305 acc :0.5062500238418579, f1 :0.2882882882882883 test loss :0.7154014706611633 acc :0.46315789222717285, f1 :0.4950495049504951 train loss :58.38571548461914 acc :0.47340425848960876, f1 :0.019801980198019802 test loss :0.7559463381767273 acc :0.48181816935539246, f1 :0.3372093023255814 train loss :57.567787170410156 acc :0.49390244483947754, f1 :0.023529411764705882 test loss :1.0900768041610718 acc :0.47727271914482117, f1 :0.3551401869158879 train loss :56.78171920776367 acc :0.4545454680919647, f1 :0.0 test loss :0.6984333395957947 acc :0.4527558982372284, f1 :0.31527093596059114 train loss :55.96480941772461 acc :0.5, f1 :0.01694915254237288 test loss :0.6920817494392395 acc :0.5074626803398132, f1 :0.2903225806451613 train loss :55.181243896484375 acc :0.5, f1 :0.02247191011235955 test loss :0.7044786214828491 acc :0.4776119291782379, f1 :0.34579439252336447 train loss :54.416412353515625 acc :0.48924732208251953, f1 :0.02061855670103093 test loss :0.7037022113800049 acc :0.48235294222831726, f1 :0.3432835820895522 train loss :53.650962829589844 acc :0.5106382966041565, f1 :0.04166666666666667 test loss :0.7914647459983826 acc :0.5350877046585083, f1 :0.4421052631578947 train loss :52.9079475402832 acc :0.4819819927215576, f1 :0.0 test loss :0.6979129314422607 acc :0.43617022037506104, f1 :0.3026315789473684 train loss :52.16638946533203 acc :0.4933333396911621, f1 :0.11627906976744184 test loss :0.6840958595275879 acc :0.5131579041481018, f1 :0.41269841269841273 train loss :51.4479866027832 acc :0.4938271641731262, f1 :0.0 test loss :0.6917073130607605 acc :0.5428571701049805, f1 :0.3846153846153846 train loss :50.73869323730469 acc :0.4836065471172333, f1 :0.05970149253731344 test loss :0.6761440634727478 acc :0.5324675440788269, f1 :0.43750000000000006 train loss :50.040889739990234 acc :0.4675324559211731, f1 :0.163265306122449 test loss :0.6930208802223206 acc :0.5277777910232544, f1 :0.42696629213483145 train loss :49.34195327758789 acc :0.44736841320991516, f1 :0.467005076142132 test loss :0.6656744480133057 acc :0.5890411138534546, f1 :0.5652173913043479 train loss :48.66841506958008 acc :0.47777777910232544, f1 :0.5936599423631124 test loss :0.6944690942764282 acc :0.49568966031074524, f1 :0.4608294930875576 train loss :47.99543380737305 acc :0.5491803288459778, f1 :0.6646341463414633 test loss :0.706595242023468 acc :0.4651162922382355, f1 :0.4320987654320987 train loss :47.33315658569336 acc :0.508474588394165, f1 :0.6184210526315789 test loss :0.6912959814071655 acc :0.4852941036224365, f1 :0.467005076142132 train loss :46.690399169921875 acc :0.4652777910232544, f1 :0.5549132947976878 test loss :0.6757375001907349 acc :0.5663265585899353, f1 :0.5303867403314917 train loss :46.047576904296875 acc :0.5054945349693298, f1 :0.6311475409836065 test loss :0.6971669793128967 acc :0.49180328845977783, f1 :0.4789915966386554 train loss :45.41992950439453 acc :0.4864864945411682, f1 :0.6274509803921569 test loss :0.7327954769134521 acc :0.5138888955116272, f1 :0.4198895027624309 train loss :44.801719665527344 acc :0.5120481848716736, f1 :0.5888324873096447 test loss :0.6880180835723877 acc :0.5320000052452087, f1 :0.48000000000000004 train loss :44.208595275878906 acc :0.5979381203651428, f1 :0.6829268292682926 test loss :0.6992944478988647 acc :0.5, f1 :0.5294117647058824 train loss :43.58946990966797 acc :0.5148515105247498, f1 :0.65 test loss :0.6971734762191772 acc :0.5240384340286255, f1 :0.5025125628140704 train loss :42.99113464355469 acc :0.5210084319114685, f1 :0.6415094339622642 test loss :0.6851209402084351 acc :0.5235849022865295, f1 :0.5388127853881279 train loss :42.38849639892578 acc :0.5526315569877625, f1 :0.6046511627906976 test loss :0.6946607232093811 acc :0.5285714268684387, f1 :0.5352112676056338 train loss :41.83910369873047 acc :0.5, f1 :0.6495726495726496 test loss :0.6996318697929382 acc :0.5535714030265808, f1 :0.5562130177514792 train loss :41.275474548339844 acc :0.5090090036392212, f1 :0.6517571884984026 test loss :0.6812279224395752 acc :0.5333333611488342, f1 :0.5701754385964912 train loss :40.71938705444336 acc :0.5384615659713745, f1 :0.6307692307692307 test loss :0.6849081516265869 acc :0.5511363744735718, f1 :0.56353591160221 train loss :40.166107177734375 acc :0.5822784900665283, f1 :0.6944444444444443 test loss :0.7007064819335938 acc :0.49082568287849426, f1 :0.47887323943661975 train loss :39.615413665771484 acc :0.5898876190185547, f1 :0.6866952789699571 test loss :0.6977179646492004 acc :0.5144927501678467, f1 :0.54421768707483 train loss :39.09635925292969 acc :0.5, f1 :0.6536964980544746 test loss :0.6860984563827515 acc :0.6134020686149597, f1 :0.6478873239436621 train loss :38.57054138183594 acc :0.5159574747085571, f1 :0.6714801444043321 test loss :0.7047839164733887 acc :0.48148149251937866, f1 :0.44000000000000006 train loss :38.05504608154297 acc :0.5058139562606812, f1 :0.6443514644351465 test loss :0.6801943778991699 acc :0.5730336904525757, f1 :0.5681818181818181 train loss :37.54621505737305 acc :0.5, f1 :0.6535433070866141 test loss :0.6948246955871582 acc :0.532608687877655, f1 :0.532608695652174 train loss :37.044578552246094 acc :0.510869562625885, f1 :0.6666666666666667 test loss :0.7127021551132202 acc :0.5347222089767456, f1 :0.524822695035461 train loss :36.5418815612793 acc :0.5548780560493469, f1 :0.6666666666666666 test loss :0.7066271305084229 acc :0.4684210419654846, f1 :0.46560846560846564 train loss :36.06358337402344 acc :0.5185185074806213, f1 :0.6517857142857144 test loss :0.6894969344139099 acc :0.5645161271095276, f1 :0.5803108808290157 train loss :35.58399200439453 acc :0.5, f1 :0.6554621848739496 test loss :0.7011650800704956 acc :0.5204081535339355, f1 :0.53 train loss :35.080108642578125 acc :0.6038960814476013, f1 :0.702439024390244 test loss :0.6900454759597778 acc :0.5565217137336731, f1 :0.5446428571428571 train loss :34.64231872558594 acc :0.5397727489471436, f1 :0.6746987951807228 test loss :0.7002308368682861 acc :0.5148515105247498, f1 :0.5420560747663552 train loss :34.175662994384766 acc :0.5178571343421936, f1 :0.5970149253731343 test loss :0.6853282451629639 acc :0.5487805008888245, f1 :0.5487804878048781 train loss :33.72896957397461 acc :0.550632894039154, f1 :0.6635071090047393 test loss :0.6870244741439819 acc :0.5, f1 :0.5208333333333334 train loss :33.28495788574219 acc :0.5223880410194397, f1 :0.6595744680851062 test loss :0.6963727474212646 acc :0.5058139562606812, f1 :0.5251396648044693 train loss :32.85262680053711 acc :0.5055555701255798, f1 :0.6536964980544747 test loss :0.7045859098434448 acc :0.5117647051811218, f1 :0.5608465608465608 train loss :32.41871643066406 acc :0.5285714268684387, f1 :0.648936170212766 test loss :0.6848579049110413 acc :0.5465116500854492, f1 :0.5568181818181818 train loss :31.986242294311523 acc :0.5799999833106995, f1 :0.6735751295336788 test loss :0.7000165581703186 acc :0.4765625, f1 :0.45528455284552843 train loss :31.57253074645996 acc :0.5, f1 :0.6599326599326599 test loss :0.7299734950065613 acc :0.4803921580314636, f1 :0.5309734513274337 train loss :31.147296905517578 acc :0.5564516186714172, f1 :0.6706586826347305 test loss :0.6881290078163147 acc :0.5633803009986877, f1 :0.5441176470588235 train loss :30.750545501708984 acc :0.5098039507865906, f1 :0.6575342465753425 test loss :0.6888918876647949 acc :0.6458333134651184, f1 :0.6382978723404256 train loss :30.348840713500977 acc :0.49514561891555786, f1 :0.6533333333333333 test loss :0.7213895916938782 acc :0.5111111402511597, f1 :0.5 train loss :29.942609786987305 acc :0.5581395626068115, f1 :0.680672268907563 test loss :0.6902480125427246 acc :0.5163043737411499, f1 :0.5435897435897437 train loss :29.551250457763672 acc :0.5980392098426819, f1 :0.6611570247933883 test loss :0.7119031548500061 acc :0.4555555582046509, f1 :0.4431818181818182 train loss :29.173669815063477 acc :0.5368421077728271, f1 :0.6666666666666667 test loss :0.6843246221542358 acc :0.5299999713897705, f1 :0.53 train loss :28.794591903686523 acc :0.5443037748336792, f1 :0.660377358490566 test loss :0.694769024848938 acc :0.570652186870575, f1 :0.572972972972973 train loss :28.40526580810547 acc :0.5857142806053162, f1 :0.6627906976744186 test loss :0.6948185563087463 acc :0.5388888716697693, f1 :0.5464480874316939 train loss :28.05225372314453 acc :0.49494948983192444, f1 :0.6598639455782314 test loss :0.6353764533996582 acc :0.7058823704719543, f1 :0.7142857142857144 train loss :27.685585021972656 acc :0.5105262994766235, f1 :0.664259927797834 test loss :0.7199012637138367 acc :0.6052631735801697, f1 :0.6762589928057554 train loss :27.318653106689453 acc :0.6065573692321777, f1 :0.6962025316455697 test loss :0.7195006012916565 acc :0.45512819290161133, f1 :0.4910179640718563 train loss :26.971885681152344 acc :0.5393258333206177, f1 :0.6693548387096775 test loss :0.6822502017021179 acc :0.5592105388641357, f1 :0.5786163522012578 train loss :26.620277404785156 acc :0.54666668176651, f1 :0.6633663366336633 test loss :0.6909845471382141 acc :0.5174418687820435, f1 :0.5088757396449705 train loss :26.304800033569336 acc :0.5249999761581421, f1 :0.6637168141592921 test loss :0.6871962547302246 acc :0.5232558250427246, f1 :0.5638297872340425 train loss :25.914714813232422 acc :0.6209677457809448, f1 :0.6845637583892618 test loss :0.7010907530784607 acc :0.5368421077728271, f1 :0.5686274509803921 train loss :25.611587524414062 acc :0.49425286054611206, f1 :0.6507936507936507 test loss :0.7009632587432861 acc :0.5393258333206177, f1 :0.6057692307692307 train loss :25.272262573242188 acc :0.518750011920929, f1 :0.6607929515418502 test loss :0.6948155760765076 acc :0.5416666865348816, f1 :0.6071428571428571 train loss :24.955718994140625 acc :0.5045454502105713, f1 :0.6686930091185409 test loss :0.7053803205490112 acc :0.577464759349823, f1 :0.5774647887323944 train loss :24.63296890258789 acc :0.5461538434028625, f1 :0.6467065868263473 test loss :0.7043526768684387 acc :0.5324675440788269, f1 :0.5499999999999999 train loss :24.316829681396484 acc :0.5277777910232544, f1 :0.6558704453441295 test loss :0.6856452822685242 acc :0.49390244483947754, f1 :0.5257142857142858 train loss :24.008567810058594 acc :0.5116279125213623, f1 :0.6692913385826772 test loss :0.702958881855011 acc :0.5104166865348816, f1 :0.5436893203883495 train loss :23.702726364135742 acc :0.5092592835426331, f1 :0.6666666666666667 test loss :0.6802870035171509 acc :0.5376344323158264, f1 :0.5784313725490197 train loss :23.389034271240234 acc :0.5661764740943909, f1 :0.6628571428571428 test loss :0.6958106160163879 acc :0.48969072103500366, f1 :0.5352112676056339 train loss :23.10261344909668 acc :0.4939759075641632, f1 :0.6528925619834711 test loss :0.6864136457443237 acc :0.5430107712745667, f1 :0.5853658536585367 train loss :22.80520248413086 acc :0.5353982448577881, f1 :0.6788990825688073 test loss :0.6893994212150574 acc :0.5232558250427246, f1 :0.5444444444444444 train loss :22.52151870727539 acc :0.5220588445663452, f1 :0.5859872611464968 test loss :0.6879146099090576 acc :0.5, f1 :0.5614035087719298 train loss :22.231653213500977 acc :0.4883720874786377, f1 :0.6507936507936507 test loss :0.7094622850418091 acc :0.4195402264595032, f1 :0.482051282051282 train loss :21.94789695739746 acc :0.5170454382896423, f1 :0.669260700389105 test loss :0.6866780519485474 acc :0.5097087621688843, f1 :0.5258215962441315 train loss :21.651575088500977 acc :0.5423728823661804, f1 :0.6301369863013698 test loss :0.6798667311668396 acc :0.649350643157959, f1 :0.6707317073170731 train loss :21.406585693359375 acc :0.5, f1 :0.6334841628959277 test loss :0.6889396905899048 acc :0.5240963697433472, f1 :0.5683060109289617 train loss :21.128219604492188 acc :0.5526315569877625, f1 :0.6458333333333334 test loss :0.7063003182411194 acc :0.578125, f1 :0.6124401913875599 train loss :20.867286682128906 acc :0.5270270109176636, f1 :0.6416382252559727 test loss :0.7015010118484497 acc :0.5148515105247498, f1 :0.5196078431372549 train loss :20.597827911376953 acc :0.512499988079071, f1 :0.6548672566371682 test loss :0.6831575036048889 acc :0.5411764979362488, f1 :0.5760869565217391 train loss :20.33578109741211 acc :0.5132743120193481, f1 :0.6646341463414634 test loss :0.6876962780952454 acc :0.5364583134651184, f1 :0.5781990521327015 train loss :20.079830169677734 acc :0.5303030014038086, f1 :0.6782006920415226 test loss :0.6614402532577515 acc :0.5461538434028625, f1 :0.593103448275862 train loss :19.801908493041992 acc :0.5890411138534546, f1 :0.6590909090909092 test loss :0.6881267428398132 acc :0.474683552980423, f1 :0.5464480874316939 train loss :19.58233070373535 acc :0.5, f1 :0.6621160409556315 test loss :0.7162292003631592 acc :0.5049999952316284, f1 :0.5638766519823788 train loss :19.330663681030273 acc :0.5284090638160706, f1 :0.6666666666666667 test loss :0.6706855893135071 acc :0.5740740895271301, f1 :0.5964912280701755 train loss :19.093273162841797 acc :0.5142857432365417, f1 :0.6666666666666667 test loss :0.7046011686325073 acc :0.5384615659713745, f1 :0.5955056179775281 train loss :18.839134216308594 acc :0.5340909361839294, f1 :0.6611570247933884 test loss :0.6854276657104492 acc :0.550000011920929, f1 :0.5655172413793104 train loss :18.6164493560791 acc :0.52173912525177, f1 :0.6632653061224489 test loss :0.6994476914405823 acc :0.5703125, f1 :0.5925925925925926 train loss :18.396728515625 acc :0.5357142686843872, f1 :0.6355140186915887 test loss :0.6905453205108643 acc :0.5571428537368774, f1 :0.581081081081081 train loss :18.15584373474121 acc :0.5320512652397156, f1 :0.6755555555555556 test loss :0.6796745657920837 acc :0.5795454382896423, f1 :0.6407766990291263 train loss :17.929290771484375 acc :0.5408163070678711, f1 :0.6830985915492958 test loss :0.6621325016021729 acc :0.5714285969734192, f1 :0.5666666666666665 train loss :17.71531867980957 acc :0.5063291192054749, f1 :0.6666666666666665 test loss :0.6994466185569763 acc :0.4791666567325592, f1 :0.5238095238095237 train loss :17.40052604675293 acc :0.7272727489471436, f1 :0.7500000000000001 test loss :0.6809377670288086 acc :0.570652186870575, f1 :0.6425339366515838 train loss :17.282140731811523 acc :0.5, f1 :0.6646884272997032 test loss :0.695618748664856 acc :0.5411764979362488, f1 :0.6285714285714287 train loss :17.06243324279785 acc :0.5307692289352417, f1 :0.6390532544378699 test loss :0.6963691115379333 acc :0.5170454382896423, f1 :0.5893719806763283 train loss :16.836963653564453 acc :0.5661764740943909, f1 :0.6775956284153006 test loss :0.6902043223381042 acc :0.5101010203361511, f1 :0.6040816326530613 train loss :16.64530372619629 acc :0.5297619104385376, f1 :0.6550218340611355 test loss :0.7063679099082947 acc :0.5174418687820435, f1 :0.5951219512195123 train loss :16.442424774169922 acc :0.5421052575111389, f1 :0.6640926640926641 test loss :0.7098447680473328 acc :0.554347813129425, f1 :0.6132075471698113 train loss :16.23899269104004 acc :0.5531914830207825, f1 :0.6865671641791046 test loss :0.6823780536651611 acc :0.5671641826629639, f1 :0.6506024096385541 train loss :16.06553840637207 acc :0.5328947305679321, f1 :0.6243386243386243 test loss :0.7080391049385071 acc :0.46666666865348816, f1 :0.550561797752809 train loss :15.851235389709473 acc :0.49504950642585754, f1 :0.6577181208053691 test loss :0.6794351935386658 acc :0.5479452013969421, f1 :0.6071428571428571 train loss :15.652654647827148 acc :0.5101010203361511, f1 :0.6643598615916955 test loss :0.7032460570335388 acc :0.5393258333206177, f1 :0.6019417475728156 train loss :15.458317756652832 acc :0.5416666865348816, f1 :0.6764705882352942 test loss :0.6932806968688965 acc :0.5572916865348816, f1 :0.6118721461187214 train loss :15.265140533447266 acc :0.5439560413360596, f1 :0.6745098039215687 test loss :0.6794919371604919 acc :0.5416666865348816, f1 :0.5925925925925926 train loss :15.087337493896484 acc :0.49494948983192444, f1 :0.6428571428571429 test loss :0.7070552110671997 acc :0.5185185074806213, f1 :0.5979381443298969 train loss :14.888094902038574 acc :0.5625, f1 :0.6666666666666667 test loss :0.6823591589927673 acc :0.5572916865348816, f1 :0.6351931330472103 epoch 0 end train loss :14.721129417419434 acc :0.5416666865348816, f1 :0.6751054852320675 test loss :0.6462240219116211 acc :0.6306818127632141, f1 :0.6733668341708543 train loss :14.497365951538086 acc :0.6574074029922485, f1 :0.7086614173228347 test loss :0.6746686100959778 acc :0.5517241358757019, f1 :0.6060606060606061 train loss :14.37336254119873 acc :0.5053191781044006, f1 :0.6666666666666667 test loss :0.7225291132926941 acc :0.49494948983192444, f1 :0.5726495726495726 train loss :14.193147659301758 acc :0.5267857313156128, f1 :0.670807453416149 test loss :0.68392014503479 acc :0.5979381203651428, f1 :0.6608695652173913 train loss :14.005753517150879 acc :0.5833333134651184, f1 :0.6994535519125683 test loss :0.6741202473640442 acc :0.5833333134651184, f1 :0.5833333333333334 train loss :13.854649543762207 acc :0.5263158082962036, f1 :0.6666666666666666 test loss :0.6707935929298401 acc :0.5930232405662537, f1 :0.669811320754717 train loss :13.68435287475586 acc :0.5537634491920471, f1 :0.6891385767790262 test loss :0.6865785717964172 acc :0.5867347121238708, f1 :0.643171806167401 train loss :13.520797729492188 acc :0.5365853905677795, f1 :0.6695652173913044 test loss :0.6890022158622742 acc :0.5909090638160706, f1 :0.6582278481012658 train loss :13.36252212524414 acc :0.52734375, f1 :0.6666666666666666 test loss :0.691808819770813 acc :0.5494505763053894, f1 :0.5684210526315788 train loss :13.176122665405273 acc :0.5803571343421936, f1 :0.6758620689655173 test loss :0.7034547328948975 acc :0.5206185579299927, f1 :0.5866666666666667 train loss :13.06440258026123 acc :0.5301204919815063, f1 :0.6666666666666666 test loss :0.6771824955940247 acc :0.5882353186607361, f1 :0.6181818181818182 train loss :12.854766845703125 acc :0.5772727131843567, f1 :0.6804123711340206 test loss :0.6760649681091309 acc :0.573913037776947, f1 :0.6474820143884893 train loss :12.732070922851562 acc :0.5420560836791992, f1 :0.6620689655172414 test loss :0.6681626439094543 acc :0.5721153616905212, f1 :0.6276150627615064 train loss :12.600958824157715 acc :0.5656565427780151, f1 :0.6791044776119403 test loss :0.675197184085846 acc :0.5957446694374084, f1 :0.6380952380952382 train loss :12.433053970336914 acc :0.5555555820465088, f1 :0.6811594202898551 test loss :0.7061024308204651 acc :0.5297619104385376, f1 :0.5989847715736041 train loss :12.28217601776123 acc :0.5170454382896423, f1 :0.6382978723404255 test loss :0.6756644248962402 acc :0.5789473652839661, f1 :0.6097560975609756 train loss :12.140645980834961 acc :0.5441176295280457, f1 :0.6787564766839378 test loss :0.6740903854370117 acc :0.5654205679893494, f1 :0.6008583690987125 train loss :11.972416877746582 acc :0.6020408272743225, f1 :0.7067669172932332 test loss :0.6915537714958191 acc :0.5368421077728271, f1 :0.5555555555555556 train loss :11.843419075012207 acc :0.5688073635101318, f1 :0.6824324324324325 test loss :0.6900310516357422 acc :0.6038960814476013, f1 :0.6114649681528662 train loss :11.695077896118164 acc :0.6340206265449524, f1 :0.7193675889328064 test loss :0.7246748208999634 acc :0.5287356376647949, f1 :0.5729166666666667 train loss :11.603494644165039 acc :0.6342592835426331, f1 :0.6877470355731226 test loss :0.6662987470626831 acc :0.5423728823661804, f1 :0.5537190082644629 train loss :11.438796043395996 acc :0.5766128897666931, f1 :0.6920821114369502 test loss :0.6720055937767029 acc :0.5885416865348816, f1 :0.6325581395348838 train loss :11.310236930847168 acc :0.546875, f1 :0.6789667896678967 test epoch end test loss :0.6622927784919739 acc :0.6105263233184814, f1 :0.672566371681416 train loss :11.171144485473633 acc :0.5544554591178894, f1 :0.6875 test loss :0.6833247542381287 acc :0.6272727251052856, f1 :0.6962962962962964 train loss :11.045841217041016 acc :0.5877862572669983, f1 :0.7016574585635358 test loss :0.6786400079727173 acc :0.5530303120613098, f1 :0.6193548387096774 train loss :10.882343292236328 acc :0.6194030046463013, f1 :0.6909090909090908 test loss :0.6796627044677734 acc :0.5354330539703369, f1 :0.6402439024390244 train loss :10.768730163574219 acc :0.6625000238418579, f1 :0.7290969899665553 test loss :0.6585690975189209 acc :0.6268656849861145, f1 :0.6835443037974683 train loss :10.6437349319458 acc :0.6741071343421936, f1 :0.743859649122807 test loss :0.6748766899108887 acc :0.5597015023231506, f1 :0.6380368098159509 train loss :10.535074234008789 acc :0.6513158082962036, f1 :0.7225130890052356 test loss :0.668499767780304 acc :0.6117647290229797, f1 :0.65625 train loss :10.412888526916504 acc :0.6132075190544128, f1 :0.7028985507246377 test loss :0.6056484580039978 acc :0.7017543911933899, f1 :0.673076923076923 train loss :10.28891658782959 acc :0.6323529481887817, f1 :0.7058823529411764 test loss :0.6618160605430603 acc :0.5904255509376526, f1 :0.6751054852320676 train loss :10.141242980957031 acc :0.6302083134651184, f1 :0.7171314741035858 test loss :0.6681300401687622 acc :0.5855262875556946, f1 :0.676923076923077 train loss :9.993220329284668 acc :0.6875, f1 :0.7428571428571429 test loss :0.6843588352203369 acc :0.5857142806053162, f1 :0.6506024096385543 train loss :9.91484260559082 acc :0.6428571343421936, f1 :0.7290640394088671 test loss :0.6722180843353271 acc :0.6168830990791321, f1 :0.700507614213198 train loss :9.809072494506836 acc :0.5919540524482727, f1 :0.6635071090047393 test loss :0.656697154045105 acc :0.5833333134651184, f1 :0.6739130434782609 train loss :9.65066146850586 acc :0.6559139490127563, f1 :0.7117117117117117 test loss :0.6233893632888794 acc :0.6917808055877686, f1 :0.7513812154696133 train loss :9.587944984436035 acc :0.65625, f1 :0.6986301369863013 test loss :0.6533226370811462 acc :0.6206896305084229, f1 :0.7046979865771812 train loss :9.49934196472168 acc :0.6363636255264282, f1 :0.7113402061855669 test loss :0.643667995929718 acc :0.6395348906517029, f1 :0.7181818181818181 train loss :9.310962677001953 acc :0.686170220375061, f1 :0.7467811158798283 test loss :0.653975248336792 acc :0.6372548937797546, f1 :0.7153846153846154 train loss :9.255167007446289 acc :0.5914633870124817, f1 :0.6940639269406392 test loss :0.6433354020118713 acc :0.6632652878761292, f1 :0.7317073170731707 train loss :9.149104118347168 acc :0.6650000214576721, f1 :0.7242798353909465 test loss :0.6559503674507141 acc :0.6352459192276001, f1 :0.6983050847457627 train loss :9.052477836608887 acc :0.648809552192688, f1 :0.7121951219512196 test loss :0.6881930232048035 acc :0.5740740895271301, f1 :0.6515151515151515 train loss :8.939974784851074 acc :0.6333333253860474, f1 :0.7203389830508473 test loss :0.6763322949409485 acc :0.6079999804496765, f1 :0.7065868263473053 train loss :8.772387504577637 acc :0.7452830076217651, f1 :0.7731092436974791 test loss :0.6663002967834473 acc :0.6071428656578064, f1 :0.7006802721088435 train loss :8.752303123474121 acc :0.6136363744735718, f1 :0.6996466431095407 test loss :0.6524512767791748 acc :0.6346153616905212, f1 :0.7226277372262775 train loss :8.687384605407715 acc :0.6139240264892578, f1 :0.6934673366834171 test loss :0.6762941479682922 acc :0.5801886916160583, f1 :0.6962457337883958 train loss :8.59071159362793 acc :0.5961538553237915, f1 :0.7014218009478673 test loss :0.6860203146934509 acc :0.5666666626930237, f1 :0.6872852233676976 train loss :8.514771461486816 acc :0.5472972989082336, f1 :0.6763285024154589 test loss :0.6474296450614929 acc :0.6547619104385376, f1 :0.7339449541284403 train loss :8.321556091308594 acc :0.6875, f1 :0.7393364928909953 test loss :0.6748971343040466 acc :0.5714285969734192, f1 :0.6808510638297872 train loss :8.267824172973633 acc :0.630630612373352, f1 :0.7210884353741497 test loss :0.620971143245697 acc :0.6704545617103577, f1 :0.7363636363636363 train loss :8.21446704864502 acc :0.6162790656089783, f1 :0.6796116504854369 test loss :0.6525086164474487 acc :0.6697247624397278, f1 :0.7428571428571429 train loss :8.100052833557129 acc :0.6736842393875122, f1 :0.7075471698113207 test loss :0.6671063303947449 acc :0.6086956262588501, f1 :0.6949152542372882 train loss :8.03050708770752 acc :0.6180555820465088, f1 :0.708994708994709 test loss :0.6479334235191345 acc :0.6030927896499634, f1 :0.6980392156862745 train loss :7.930685997009277 acc :0.6066666841506958, f1 :0.7064676616915423 test loss :0.6826183795928955 acc :0.5679012537002563, f1 :0.6666666666666666 train loss :7.8363494873046875 acc :0.6089109182357788, f1 :0.7084870848708487 test loss :0.666271448135376 acc :0.6123595237731934, f1 :0.7012987012987013 train loss :7.749617099761963 acc :0.625, f1 :0.6823529411764706 test loss :0.6604633927345276 acc :0.60326087474823, f1 :0.6995884773662553 train loss :7.627895832061768 acc :0.6666666865348816, f1 :0.7457627118644068 test loss :0.6640687584877014 acc :0.5625, f1 :0.6666666666666667 train loss :7.537315368652344 acc :0.7333333492279053, f1 :0.7746478873239437 test loss :0.6792986392974854 acc :0.5631579160690308, f1 :0.6719367588932806 train loss :7.493045806884766 acc :0.6329787373542786, f1 :0.6986899563318777 test loss :0.6644682884216309 acc :0.5806451439857483, f1 :0.6929133858267715 train loss :7.387424468994141 acc :0.6585366129875183, f1 :0.7333333333333333 test loss :0.6573323011398315 acc :0.5867347121238708, f1 :0.6872586872586873 train loss :7.3155317306518555 acc :0.6386138796806335, f1 :0.7159533073929961 test loss :0.6648008227348328 acc :0.582608699798584, f1 :0.7 train loss :7.255756378173828 acc :0.6232876777648926, f1 :0.6927374301675977 test loss :0.6331043839454651 acc :0.698019802570343, f1 :0.7662835249042146 train loss :7.146876811981201 acc :0.6463414430618286, f1 :0.7211538461538461 test loss :0.6493303775787354 acc :0.6158536672592163, f1 :0.7014218009478673 train loss :7.088024616241455 acc :0.6646341681480408, f1 :0.7342995169082126 test loss :0.6654958128929138 acc :0.592391312122345, f1 :0.6963562753036437 train loss :6.9927873611450195 acc :0.6630434989929199, f1 :0.7372881355932204 test loss :0.6373470425605774 acc :0.6337209343910217, f1 :0.717488789237668 train loss :6.971644401550293 acc :0.6408450603485107, f1 :0.6946107784431137 test loss :0.6600098609924316 acc :0.6058823466300964, f1 :0.7099567099567099 train loss :6.872450828552246 acc :0.6494252681732178, f1 :0.7404255319148937 test loss :0.6265233755111694 acc :0.680232584476471, f1 :0.7488584474885844 train loss :6.817215919494629 acc :0.604651153087616, f1 :0.6936936936936937 test loss :0.7012857794761658 acc :0.5234375, f1 :0.6629834254143646 train loss :6.728199481964111 acc :0.613095223903656, f1 :0.6766169154228857 test loss :0.6986444592475891 acc :0.5686274766921997, f1 :0.6811594202898551 train loss :6.606841087341309 acc :0.7142857313156128, f1 :0.7647058823529412 test loss :0.6872422099113464 acc :0.5422534942626953, f1 :0.6666666666666667 train loss :6.570825099945068 acc :0.6755319237709045, f1 :0.7336244541484717 test loss :0.654042661190033 acc :0.6458333134651184, f1 :0.7301587301587301 train loss :6.4930033683776855 acc :0.6703296899795532, f1 :0.7321428571428572 test loss :0.6424410343170166 acc :0.6000000238418579, f1 :0.7000000000000001 train loss :6.397623062133789 acc :0.6949999928474426, f1 :0.7468879668049793 test loss :0.6706475615501404 acc :0.614130437374115, f1 :0.7193675889328063 train loss :6.410904407501221 acc :0.5904255509376526, f1 :0.6804979253112032 test loss :0.6759946346282959 acc :0.5833333134651184, f1 :0.6913580246913581 train loss :6.306626796722412 acc :0.6477272510528564, f1 :0.7047619047619049 test loss :0.6978817582130432 acc :0.5400000214576721, f1 :0.684931506849315 train loss :6.281235218048096 acc :0.6173469424247742, f1 :0.7035573122529644 test loss :0.6810269355773926 acc :0.5489130616188049, f1 :0.6795366795366796 train loss :6.19986629486084 acc :0.6187499761581421, f1 :0.7024390243902439 test loss :0.677791953086853 acc :0.5666666626930237, f1 :0.6929133858267716 train loss :6.140160083770752 acc :0.611940324306488, f1 :0.6867469879518073 test loss :0.678559422492981 acc :0.5735294222831726, f1 :0.6979166666666666 train loss :6.040308952331543 acc :0.6634615659713745, f1 :0.7348484848484849 test loss :0.6807653307914734 acc :0.5350877046585083, f1 :0.674846625766871 train loss :6.011411190032959 acc :0.7076923251152039, f1 :0.7397260273972603 test loss :0.7074670791625977 acc :0.5448718070983887, f1 :0.672811059907834 train loss :5.966691017150879 acc :0.6000000238418579, f1 :0.6818181818181818 test loss :0.6777371764183044 acc :0.5460526347160339, f1 :0.6666666666666667 train loss :5.892942905426025 acc :0.6268656849861145, f1 :0.7126436781609196 test loss :0.6706344485282898 acc :0.5755813717842102, f1 :0.6945606694560669 train loss :5.8286356925964355 acc :0.6447368264198303, f1 :0.7216494845360825 test loss :0.68108731508255 acc :0.5523256063461304, f1 :0.6804979253112032 train loss :5.759369850158691 acc :0.6521739363670349, f1 :0.7142857142857143 test loss :0.6698668003082275 acc :0.5842105150222778, f1 :0.6996197718631179 train loss :5.690567493438721 acc :0.65887850522995, f1 :0.7438596491228071 test loss :0.6664812564849854 acc :0.5786516666412354, f1 :0.6987951807228915 train loss :5.616798400878906 acc :0.6888889074325562, f1 :0.75 test loss :0.6811220645904541 acc :0.5763888955116272, f1 :0.6995073891625616 train loss :5.583926677703857 acc :0.6525423526763916, f1 :0.728476821192053 test loss :0.6716065406799316 acc :0.5845070481300354, f1 :0.7035175879396985 train loss :5.580383777618408 acc :0.5722222328186035, f1 :0.6695278969957081 test loss :0.670954704284668 acc :0.5844155550003052, f1 :0.7009345794392522 train loss :5.513225555419922 acc :0.6293103694915771, f1 :0.6386554621848739 test loss :0.67231684923172 acc :0.5792682766914368, f1 :0.7038626609442059 train loss :5.4365434646606445 acc :0.6428571343421936, f1 :0.7302904564315353 test loss :0.6749115586280823 acc :0.59375, f1 :0.7022900763358779 train loss :5.359493255615234 acc :0.6650485396385193, f1 :0.749090909090909 test loss :0.6518592238426208 acc :0.6397849321365356, f1 :0.7309236947791165 train loss :5.348880290985107 acc :0.6024096608161926, f1 :0.7053571428571429 test loss :0.6638138890266418 acc :0.623711347579956, f1 :0.7265917602996255 train loss :5.304172515869141 acc :0.6150000095367432, f1 :0.6956521739130435 test loss :0.6071746349334717 acc :0.6935483813285828, f1 :0.759493670886076 train loss :5.259949684143066 acc :0.5909090638160706, f1 :0.6985645933014353 test loss :0.6476115584373474 acc :0.6220930218696594, f1 :0.721030042918455 train loss :5.178959369659424 acc :0.6637930870056152, f1 :0.7310344827586207 test loss :0.6934933662414551 acc :0.5649999976158142, f1 :0.6925795053003534 train loss :5.079901695251465 acc :0.6971830725669861, f1 :0.7425149700598803 test loss :0.6562538743019104 acc :0.6091954112052917, f1 :0.7068965517241379 train loss :5.084912300109863 acc :0.6329787373542786, f1 :0.7294117647058824 test loss :0.6653070449829102 acc :0.5922330021858215, f1 :0.7042253521126761 train loss :5.031538009643555 acc :0.6546391844749451, f1 :0.7413127413127414 test loss :0.6465588808059692 acc :0.6103895902633667, f1 :0.7115384615384616 train loss :4.988809585571289 acc :0.6518987417221069, f1 :0.7058823529411765 test loss :0.6606163382530212 acc :0.6265060305595398, f1 :0.7232142857142858 train loss :4.9199538230896 acc :0.6782178282737732, f1 :0.7509578544061303 test loss :0.6434434056282043 acc :0.6458333134651184, f1 :0.7322834645669292 train loss :4.902554988861084 acc :0.5978260636329651, f1 :0.6890756302521008 test loss :0.690047562122345 acc :0.5742574334144592, f1 :0.6838235294117647 train loss :4.837471961975098 acc :0.649350643157959, f1 :0.7065217391304348 test loss :0.6855130195617676 acc :0.570588231086731, f1 :0.6755555555555556 train loss :4.7825493812561035 acc :0.6470588445663452, f1 :0.7087378640776698 test loss :0.6916613578796387 acc :0.5364583134651184, f1 :0.6615969581749049 train loss :4.747241973876953 acc :0.6811594367027283, f1 :0.6562500000000001 test loss :0.6609394550323486 acc :0.6153846383094788, f1 :0.7023809523809523 train loss :4.695973873138428 acc :0.6623376607894897, f1 :0.7346938775510204 test loss :0.6629752516746521 acc :0.6139240264892578, f1 :0.7162790697674417 train loss :4.661523342132568 acc :0.6372548937797546, f1 :0.7299270072992701 test loss :0.6841887831687927 acc :0.5849999785423279, f1 :0.6891385767790262 train loss :4.615486145019531 acc :0.5859375, f1 :0.6015037593984963 test loss :0.6373440623283386 acc :0.6419752836227417, f1 :0.721153846153846 epoch 1 end train loss :4.550625801086426 acc :0.5957446694374084, f1 :0.6122448979591838 test loss :0.6791767477989197 acc :0.6089743375778198, f1 :0.7024390243902439 train loss :4.497131824493408 acc :0.6760563254356384, f1 :0.7195121951219512 test loss :0.6448349356651306 acc :0.6428571343421936, f1 :0.7311827956989247 train loss :4.4931769371032715 acc :0.6212121248245239, f1 :0.7148288973384029 test loss :0.6952346563339233 acc :0.5078125, f1 :0.6480446927374302 train loss :4.47650146484375 acc :0.6415929198265076, f1 :0.7254237288135593 test loss :0.6751621961593628 acc :0.5857142806053162, f1 :0.6979166666666667 train loss :4.404613971710205 acc :0.6538461446762085, f1 :0.7410071942446044 test loss :0.6757574677467346 acc :0.5909090638160706, f1 :0.7096774193548387 train loss :4.322770595550537 acc :0.6479591727256775, f1 :0.7160493827160493 test loss :0.6493523716926575 acc :0.6208791136741638, f1 :0.7183673469387755 train loss :4.325937747955322 acc :0.6041666865348816, f1 :0.6964856230031948 test loss :0.6613491177558899 acc :0.6145833134651184, f1 :0.7196969696969697 train loss :4.261044025421143 acc :0.7105262875556946, f1 :0.744186046511628 test loss :0.701045572757721 acc :0.54347825050354, f1 :0.6793893129770994 train loss :4.2002716064453125 acc :0.6850393414497375, f1 :0.7419354838709677 test loss :0.6856261491775513 acc :0.5588235259056091, f1 :0.6938775510204082 train loss :4.213860511779785 acc :0.6428571343421936, f1 :0.719626168224299 test loss :0.6948525309562683 acc :0.5454545617103577, f1 :0.68 train loss :4.166141510009766 acc :0.7338709831237793, f1 :0.7401574803149606 test loss :0.6943566799163818 acc :0.560606062412262, f1 :0.6903914590747331 train loss :4.098399639129639 acc :0.6800000071525574, f1 :0.7515527950310559 test loss :0.7012938857078552 acc :0.5348837375640869, f1 :0.6799999999999999 train loss :4.020195007324219 acc :0.703125, f1 :0.7625000000000001 test loss :0.6639971733093262 acc :0.60326087474823, f1 :0.7044534412955465 train loss :4.0242085456848145 acc :0.703125, f1 :0.7564102564102563 test loss :0.7032093405723572 acc :0.5671641826629639, f1 :0.6881720430107526 train loss :3.980978488922119 acc :0.6796875, f1 :0.7302631578947368 test loss :0.6999689936637878 acc :0.5400000214576721, f1 :0.6729857819905214 train loss :3.936495780944824 acc :0.7193877696990967, f1 :0.7679324894514769 test loss :0.7138698101043701 acc :0.5136986374855042, f1 :0.6697674418604651 train loss :3.946786642074585 acc :0.6274510025978088, f1 :0.7054263565891473 test loss :0.6695820093154907 acc :0.584269642829895, f1 :0.6942148760330579 train loss :3.8878090381622314 acc :0.6648351550102234, f1 :0.7109004739336492 test loss :0.6872296929359436 acc :0.5625, f1 :0.6934306569343065 train loss :3.784642219543457 acc :0.7723214030265808, f1 :0.7901234567901235 test loss :0.6870884895324707 acc :0.5694444179534912, f1 :0.6990291262135923 train loss :3.8199596405029297 acc :0.6652542352676392, f1 :0.7392739273927393 test loss :0.7228165864944458 acc :0.5185185074806213, f1 :0.6666666666666666 train loss :3.7693943977355957 acc :0.6854838728904724, f1 :0.7328767123287672 test loss :0.7206511497497559 acc :0.546875, f1 :0.6765799256505576 train loss :3.734422206878662 acc :0.6926605701446533, f1 :0.7563636363636365 test loss :0.7106273174285889 acc :0.5170454382896423, f1 :0.669260700389105 train loss :3.724407911300659 acc :0.6829268336296082, f1 :0.7531645569620253 test loss :0.713482141494751 acc :0.5517241358757019, f1 :0.6904761904761904 train loss :3.684948444366455 acc :0.6746031641960144, f1 :0.7388535031847134 test loss :0.748045802116394 acc :0.5, f1 :0.6425992779783394 train loss :3.8993349075317383 acc :0.5740740895271301, f1 :0.34285714285714286 test loss :0.7123057842254639 acc :0.561855673789978, f1 :0.6953405017921147 train loss :3.6385371685028076 acc :0.6681416034698486, f1 :0.7508305647840532 test loss :0.6804325580596924 acc :0.6388888955116272, f1 :0.7234042553191489 train loss :3.654191493988037 acc :0.601123571395874, f1 :0.6978723404255319 test loss :0.685422956943512 acc :0.5988371968269348, f1 :0.7136929460580913 train loss :3.5665011405944824 acc :0.6644737124443054, f1 :0.7357512953367875 test loss :0.6870809197425842 acc :0.5867347121238708, f1 :0.7032967032967032 train loss :3.541166305541992 acc :0.6445783376693726, f1 :0.7230046948356806 test loss :0.6858209371566772 acc :0.5858585834503174, f1 :0.7007299270072993 train loss :3.5217833518981934 acc :0.635869562625885, f1 :0.7074235807860262 test loss :0.7039735317230225 acc :0.5439560413360596, f1 :0.6795366795366795 train loss :3.536576747894287 acc :0.5846154093742371, f1 :0.6896551724137931 test loss :0.6857467889785767 acc :0.5876288414001465, f1 :0.7080291970802919 train loss :3.483822822570801 acc :0.6325300931930542, f1 :0.6772486772486772 test loss :0.70633465051651 acc :0.5686274766921997, f1 :0.6944444444444443 train loss :3.3626863956451416 acc :0.7314814925193787, f1 :0.7128712871287127 test loss :0.6746574640274048 acc :0.591304361820221, f1 :0.7080745341614907 train loss :3.410275936126709 acc :0.6461538672447205, f1 :0.7195121951219511 test loss :0.6746877431869507 acc :0.6105769276618958, f1 :0.7177700348432056 train loss :3.3845481872558594 acc :0.6363636255264282, f1 :0.7272727272727272 test loss :0.6902097463607788 acc :0.5744680762290955, f1 :0.696969696969697 train loss :3.406694173812866 acc :0.6176470518112183, f1 :0.6941176470588235 test loss :0.688338041305542 acc :0.5714285969734192, f1 :0.6923076923076923 train loss :3.322779655456543 acc :0.6566265225410461, f1 :0.7219512195121952 test loss :0.6803468465805054 acc :0.5789473652839661, f1 :0.7037037037037037 train loss :3.338637113571167 acc :0.6043956279754639, f1 :0.7096774193548387 test loss :0.6961828470230103 acc :0.5607476830482483, f1 :0.6845637583892616 train loss :3.287754535675049 acc :0.593406617641449, f1 :0.672566371681416 test loss :0.7224342226982117 acc :0.5263158082962036, f1 :0.6762589928057554 train loss :3.258493185043335 acc :0.686170220375061, f1 :0.7203791469194313 test loss :0.6861772537231445 acc :0.5844155550003052, f1 :0.7009345794392522 train loss :3.2322731018066406 acc :0.6583333611488342, f1 :0.6870229007633588 test loss :0.6943454146385193 acc :0.5747126340866089, f1 :0.6991869918699186 train loss :3.224104642868042 acc :0.6118420958518982, f1 :0.70935960591133 test loss :0.7131195068359375 acc :0.5423728823661804, f1 :0.6804733727810651 train loss :3.201681613922119 acc :0.5945945978164673, f1 :0.6385542168674698 test loss :0.7186388373374939 acc :0.5416666865348816, f1 :0.6834532374100719 train loss :3.1286587715148926 acc :0.649350643157959, f1 :0.7127659574468086 test epoch end test loss :0.6885609030723572 acc :0.5894736647605896, f1 :0.7067669172932329 train loss :3.1642489433288574 acc :0.5961538553237915, f1 :0.6692913385826771 test loss :0.6517295837402344 acc :0.6499999761581421, f1 :0.7407407407407407 train loss :3.0909507274627686 acc :0.6513158082962036, f1 :0.7135135135135134 test loss :0.7354846596717834 acc :0.5378788113594055, f1 :0.6839378238341969 train loss :3.089146137237549 acc :0.6744186282157898, f1 :0.7407407407407407 test loss :0.6853955984115601 acc :0.5944882035255432, f1 :0.711484593837535 train loss :3.0479559898376465 acc :0.6534090638160706, f1 :0.7109004739336493 test loss :0.6776175498962402 acc :0.60447758436203, f1 :0.7135135135135133 train loss :2.97822904586792 acc :0.6692307591438293, f1 :0.7074829931972789 test loss :0.6797167658805847 acc :0.608208954334259, f1 :0.7154471544715446 train loss :3.0179190635681152 acc :0.6438356041908264, f1 :0.6904761904761905 test loss :0.6781443953514099 acc :0.6000000238418579, f1 :0.7142857142857143 train loss :2.9458584785461426 acc :0.7115384340286255, f1 :0.7513812154696132 test loss :0.6490747332572937 acc :0.6140350699424744, f1 :0.7179487179487178 train loss :2.876633644104004 acc :0.747474730014801, f1 :0.788135593220339 test loss :0.6892386674880981 acc :0.5744680762290955, f1 :0.6992481203007519 train loss :2.9322071075439453 acc :0.6557376980781555, f1 :0.7407407407407408 test loss :0.7005272507667542 acc :0.5526315569877625, f1 :0.6909090909090909
原文中用了三种方法来进行优化，且对于优化后选取子句后answer选择给出了结论，整体实现过程还是要费一些功夫的。