Я написал скрипт нейронной сети, а точнее часть с подготовкой данных на вход. Но не уверен, правильно ли всё сделал для того, чтобы модель смогла корректно обучаться. Очень нужна оценка знающих людей.
Код:
import sklearn
import numpy as np
from collections import Counter
from keras.models import model_from_json
from keras.preprocessing import sequence
from sklearn.model_selection import train_test_split as tts
labels_lexicon = ['_label_0', '_label_1', '_label_2'] # список категорий
def get_data_from_the_file():
labels, descriptions, lexicon, lexicon_base = [], [], [], []
for i, line in enumerate(open('testtext.txt', 'r', encoding='utf8', errors='ignore')):
content = line.split()
labels.append([content[0]])
descriptions.append(content[1:])
lexicon_base += content[1:]
count_lexicon = Counter(lexicon_base).most_common(5000)
for count_item in count_lexicon:
lexicon.append(count_item[0])
return labels, descriptions, lexicon
labels, descriptions, lexicon = get_data_from_the_file()
def get_descriptions_to_index(lexicon):
cache = {}
word2index = {}
for i,word in enumerate(lexicon):
if cache.get(word) == None:
cache[word] = i
word2index[word] = i
return word2index
word2index = get_descriptions_to_index(lexicon)
def get_labels_to_index(labels_lexicon):
cache = {}
labels2index = {}
for i,word in enumerate(labels_lexicon):
if cache.get(word) == None:
cache[word] = i
labels2index[word] = i
return labels2index
labels2index = get_labels_to_index(labels_lexicon)
list_of_tokenize_descriptions = []
list_of_tokenize_labels = []
for descriptions_arrays in descriptions:
prepare_list_of_tokenize_descriptions = []
for descriptions_piece in descriptions_arrays:
if word2index.get(descriptions_piece) != None:
prepare_list_of_tokenize_descriptions.append(word2index[descriptions_piece])
list_of_tokenize_descriptions.append(prepare_list_of_tokenize_descriptions)
for labels_arrays in labels:
prepare_list_of_tokenize_labels = []
for labels_piece in labels_arrays:
if labels2index.get(labels_piece) != None:
prepare_list_of_tokenize_labels.append(labels2index[labels_piece])
list_of_tokenize_labels.append(prepare_list_of_tokenize_labels)
x_matrix_list = []
y_matrix_list = []
for i in range(len(list_of_tokenize_descriptions)):
matrix_i = np.zeros((len(lexicon)),dtype=int)
line = list_of_tokenize_descriptions[i]
for index in line:
matrix_i[index] = 1
x_matrix_list.append(matrix_i)
for i in range(len(list_of_tokenize_labels)):
matrix_i = np.zeros((len(labels_lexicon)),dtype=int)
line = list_of_tokenize_labels[i]
for index in line:
matrix_i[index] = 1
y_matrix_list.append(matrix_i)
x_train, x_test, y_train, y_test = tts(np.array(x_matrix_list), np.array(y_matrix_list), test_size=0.3)
Здесь ссылкуа на dataset.