Архитектура сети
max_features = len(lexicon)
num_classes = len(labels_lexicon)
early_stop = EarlyStopping(monitor='val_acc', min_delta=0.001,
patience=3, verbose=1, mode='auto')
chkpt = ModelCheckpoint('architecture.hdf5',
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='auto')
callbacks = [early_stop, chkpt]
model = Sequential()
model.add(Embedding(max_features, 32))
model.add(SpatialDropout1D(0.2))
model.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(num_classes, activation="softmax"))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.fit(x_train, y_train, batch_size=128, epochs=50,
validation_data=(x_test, y_test), verbose=1, callbacks=callbacks)
scores = model.evaluate(x_test, y_test, batch_size=128)
Данные по текущей архитектуре сети
val_acc - 86%
val_loss - 0.3
acc - 89%
кол-во эпох обучения: 10 (после 10ой эпохи начинается переобучение)
Подключение модулей и создание словоря категорий
import sklearn
import numpy as np
from collections import Counter
from sklearn.model_selection import train_test_split as tts
from keras.preprocessing import sequence
from keras import utils as ku
from nltk.stem.snowball import SnowballStemmer
from keras.models import Sequential
from keras.layers import Dense, Activation, Embedding
from keras.layers import LSTM, SpatialDropout1D
import re
from nltk.stem.snowball import SnowballStemmer
from keras.models import load_model
from keras.callbacks import EarlyStopping, ModelCheckpoint
np.random.seed(42)
labels_lexicon = []
for i in range(0, 33):
labels_lexicon.append("_label_"+str(i))
Загрузка данных из файла
def get_data_from_the_file():
labels, descriptions, lexicon, lexicon_base = [], [], [], []
for i, line in enumerate(open('testtext.txt', 'r', encoding='utf8', errors='ignore')):
content = line.split()
labels.append(content[0].replace(u'\ufeff', ''))
descriptions.append(content[1:])
lexicon_base += content[1:]
count_lexicon = Counter(lexicon_base).most_common()
for count_item in count_lexicon:
if count_item[1] >= 10:
lexicon.append(count_item[0])
return labels, descriptions, lexicon
labels, descriptions, lexicon = get_data_from_the_file()
Токенизация обучающих текстов(функция)
def get_descriptions_to_index(lexicon): #функция для преоброзования слов в индесы(для описаний)
cache = {}
word2index = {}
for i,word in enumerate(lexicon):
if cache.get(word) == None:
cache[word] = i
word2index[word] = i
return word2index
word2index = get_descriptions_to_index(lexicon)
Токенизация label(функция)
def get_labels_to_index(labels_lexicon):
cache = {}
labels2index = {}
for i,word in enumerate(labels_lexicon):
if cache.get(word) == None:
cache[word] = i
labels2index[word] = i
return labels2index
labels2index = get_labels_to_index(labels_lexicon)
Токенизация обучающих текстов и label к ним
list_of_tokenize_descriptions = []
list_of_tokenize_labels = []
for description_item in descriptions: # преоброзование описаний тендеров из текста в индексы
prepare_list_of_tokenize_descriptions = []
for descriptions_piece in description_item:
if word2index.get(descriptions_piece) != None:
prepare_list_of_tokenize_descriptions.append(word2index[descriptions_piece])
list_of_tokenize_descriptions.append(prepare_list_of_tokenize_descriptions)
for label in labels: # преоброзование категорий тендеров из текста в индексы
if labels2index.get(label) != None:
list_of_tokenize_labels.append(labels2index[label])
Разделение на тестовую и обучающую части
x_train, x_test, y_train, y_test = tts(list_of_tokenize_descriptions, list_of_tokenize_labels, test_size=0.1, random_state=42) # разделения на тренироввые и тренировачны части
Привидение label к категориальному виду:
y_test = ku.to_categorical(y_test, num_classes=len(labels_lexicon))
y_train = ku.to_categorical(y_train, num_classes=len(labels_lexicon))
Привидение всех описаний к одной длине
MaxDescriptionLen = 0
for count_description_item in descriptions:
if len(count_description_item) > MaxDescriptionLen:
MaxDescriptionLen = len(count_description_item)
#преведение всех описаний к одинаковой длине
x_train = sequence.pad_sequences(x_train, maxlen=MaxDescriptionLen)
x_test = sequence.pad_sequences(x_test, maxlen=MaxDescriptionLen)
Ссылки: dataset, полный код
Задача
Максимально улучшить архитектуру данной сети.