3

Архитектура сети

max_features = len(lexicon)
num_classes = len(labels_lexicon)

    early_stop = EarlyStopping(monitor='val_acc', min_delta=0.001,
                               patience=3, verbose=1, mode='auto')
    chkpt = ModelCheckpoint('architecture.hdf5', 
                            monitor='val_loss', 
                            verbose=1, 
                            save_best_only=True, 
                            mode='auto')
    callbacks = [early_stop, chkpt]
    
    model = Sequential()
    model.add(Embedding(max_features, 32))
    model.add(SpatialDropout1D(0.2))
    model.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2)) 
    model.add(Dense(num_classes, activation="softmax"))
    model.compile(loss='categorical_crossentropy',
                  optimizer='adam',
                  metrics=['accuracy'])
    model.fit(x_train, y_train, batch_size=128, epochs=50,
              validation_data=(x_test, y_test), verbose=1, callbacks=callbacks)
    scores = model.evaluate(x_test, y_test, batch_size=128)

Данные по текущей архитектуре сети

val_acc - 86%

val_loss - 0.3

acc - 89%

кол-во эпох обучения: 10 (после 10ой эпохи начинается переобучение)

Подключение модулей и создание словоря категорий

import sklearn
import numpy as np
from collections import Counter
from sklearn.model_selection import train_test_split as tts
from keras.preprocessing import sequence
from keras import utils  as ku
from nltk.stem.snowball import SnowballStemmer 
from keras.models import Sequential
from keras.layers import Dense, Activation, Embedding
from keras.layers import LSTM, SpatialDropout1D
import re
from nltk.stem.snowball import SnowballStemmer 
from keras.models import load_model
from keras.callbacks import EarlyStopping, ModelCheckpoint

np.random.seed(42)

labels_lexicon =  []
for i in range(0, 33):
  labels_lexicon.append("_label_"+str(i))

Загрузка данных из файла

def get_data_from_the_file():
  labels, descriptions, lexicon, lexicon_base = [], [], [], [] 

  for i,  line in enumerate(open('testtext.txt', 'r', encoding='utf8', errors='ignore')):
    content = line.split()
    labels.append(content[0].replace(u'\ufeff', ''))
    descriptions.append(content[1:])
    lexicon_base += content[1:]

  count_lexicon = Counter(lexicon_base).most_common()
  for count_item in count_lexicon:
    if count_item[1] >= 10:
      lexicon.append(count_item[0])

  return labels, descriptions, lexicon

labels, descriptions, lexicon = get_data_from_the_file()

Токенизация обучающих текстов(функция)

def get_descriptions_to_index(lexicon): #функция для преоброзования слов в индесы(для описаний)
    cache = {}
    word2index = {}
    for i,word in enumerate(lexicon):
        if cache.get(word) == None:
            cache[word] = i
            word2index[word] = i
    return word2index
word2index = get_descriptions_to_index(lexicon)

Токенизация label(функция)

def get_labels_to_index(labels_lexicon):
    cache = {}
    labels2index = {}
    for i,word in enumerate(labels_lexicon):
        if cache.get(word) == None:
            cache[word] = i
            labels2index[word] = i
    return labels2index
labels2index = get_labels_to_index(labels_lexicon)

Токенизация обучающих текстов и label к ним

list_of_tokenize_descriptions = []
list_of_tokenize_labels = []

for description_item in descriptions: # преоброзование описаний тендеров из текста в индексы
    prepare_list_of_tokenize_descriptions = []
    for descriptions_piece in description_item:
        if word2index.get(descriptions_piece) != None:
            prepare_list_of_tokenize_descriptions.append(word2index[descriptions_piece])
    list_of_tokenize_descriptions.append(prepare_list_of_tokenize_descriptions)

for label in labels: # преоброзование категорий тендеров из текста в индексы
    if labels2index.get(label) != None:
        list_of_tokenize_labels.append(labels2index[label])

Разделение на тестовую и обучающую части

x_train, x_test, y_train, y_test = tts(list_of_tokenize_descriptions, list_of_tokenize_labels,  test_size=0.1, random_state=42) # разделения на тренироввые и тренировачны части

Привидение label к категориальному виду:

y_test = ku.to_categorical(y_test, num_classes=len(labels_lexicon))
y_train = ku.to_categorical(y_train, num_classes=len(labels_lexicon))

Привидение всех описаний к одной длине

MaxDescriptionLen = 0
for count_description_item in descriptions:
  if len(count_description_item) > MaxDescriptionLen:
    MaxDescriptionLen = len(count_description_item)

#преведение всех  описаний к одинаковой длине
x_train = sequence.pad_sequences(x_train, maxlen=MaxDescriptionLen)
x_test = sequence.pad_sequences(x_test, maxlen=MaxDescriptionLen)

Ссылки: dataset, полный код

Задача

Максимально улучшить архитектуру данной сети.

3 ответа 3

3

Вот CNN архитектура (без использования LSTM), которая даёт точность: 90.25% на тестовой выборке:

In [75]: model.summary()
_________________________________________________________________
Layer (type)                 Output Shape              Param #
=================================================================
embedding_7 (Embedding)      (None, 100, 50)           300000
_________________________________________________________________
dropout_10 (Dropout)         (None, 100, 50)           0
_________________________________________________________________
conv1d_4 (Conv1D)            (None, 98, 250)           37750
_________________________________________________________________
global_max_pooling1d_4 (Glob (None, 250)               0
_________________________________________________________________
dense_8 (Dense)              (None, 250)               62750
_________________________________________________________________
dropout_11 (Dropout)         (None, 250)               0
_________________________________________________________________
activation_4 (Activation)    (None, 250)               0
_________________________________________________________________
dense_9 (Dense)              (None, 8)                 2008
=================================================================
Total params: 402,508
Trainable params: 402,508
Non-trainable params: 0

In [76]: model.evaluate(X_test, Y_test)
6123/6123 [==============================] - 0s 71us/step
Out[76]: [0.3342062237993596, 0.9024987751005056]

Полный код:

from pathlib import Path
import pandas as pd
from keras.models import Sequential
from keras.layers import Embedding, Conv1D, MaxPool1D, GlobalMaxPool1D, Dense, Dropout, Activation
from keras.preprocessing.text import Tokenizer, text_to_word_sequence
from keras.preprocessing.sequence import pad_sequences
from keras import optimizers
from keras.callbacks import EarlyStopping, ModelCheckpoint, TensorBoard
from keras.models import save_model, load_model
from sklearn.model_selection import train_test_split

path = Path(r'D:\work\SO_ru\939264-Keras_CNN_LSTM')
filename = path / 'testtext.txt'
model_fn = path / 'model_CNN.h5'
max_features = 6000
maxlen = 100
embedding_dims = 50
filters = 250
kernel_size = 3
hidden_dims = 250
batch_size = 32
epochs = 15

os.chdir(str(path))

##################################################################

def get_data(filename, max_features=5000, maxlen=100, frac=1.0):
    data = (pd.read_csv(filename, header=None, names=['text'], sep='~')
              .sample(frac=frac))
    data[['label','text']] = data.pop('text').str.split(n=1, expand=True)
    data = data.dropna()
    data = data.loc[data['label'].str.contains(r'^_label')]

    # build vocabulary
    tok = Tokenizer(num_words=max_features)
    tok.fit_on_texts(data['text'])
    # convert texts to sequences
    X = tok.texts_to_sequences(data['text'])
    lb = LabelBinarizer()
    Y = pd.DataFrame(lb.fit_transform(data['label']), 
                     columns=lb.classes_, index=data.index)
    return (pad_sequences(X, maxlen=maxlen), Y, tok)


def build_model(max_features=1000, num_classes=1,
                embedding_dims=128, maxlen=100, fc_dropout=0.2,
                last_activation='sigmoid', optimizer='adam',
                loss='binary_crossentropy', metrics=['accuracy']):
    if num_classes == 1:
        last_activation = 'sigmoid'
        loss = 'binary_crossentropy'
    else:
        last_activation = 'softmax'
        loss = 'categorical_crossentropy'
    model = Sequential()
    model.add(Embedding(max_features, embedding_dims, input_length=maxlen))
    model.add(Dropout(0.1))
    model.add(Conv1D(filters,
              kernel_size,
              padding='valid',
              activation='relu',
              strides=1))
    model.add(GlobalMaxPool1D())
    model.add(Dense(hidden_dims))
    model.add(Dropout(fc_dropout))
    model.add(Activation('relu'))
    model.add(Dense(num_classes, activation=last_activation))
    model.compile(loss=loss,
                  optimizer=optimizer,
                  metrics=metrics)
    return model



X, Y, tok = get_data(filename, max_features=max_features)
# split data set to train / dev
X_train, X_test, Y_train, Y_test = \
    train_test_split(X, Y, test_size=0.05, random_state=123, stratify=Y)
print('X_train.shape:\t{}\t\tY_train.shape:\t{}'.format(X_train.shape, Y_train.shape))
print('X_test.shape:\t{}\t\tY_test.shape:\t{}'.format(X_test.shape, Y_test.shape))
#############################

early_stop = EarlyStopping(monitor='val_acc', min_delta=0.001,
                           patience=5, verbose=1, mode='auto')
chkpt = ModelCheckpoint(str(model_fn), 
                        monitor='val_acc', 
                        verbose=1, 
                        save_best_only=True, 
                        mode='auto')
callbacks = [early_stop, chkpt]

if Y_train.ndim == 1:
    num_classes = 1
elif Y_train.ndim == 2:
    num_classes = Y_train.shape[1]
else:
    raise Exception('[Y_train] must be either 1D or 2D array')

model = build_model(max_features=max_features, num_classes=num_classes,
                    embedding_dims=embedding_dims, maxlen=maxlen,
                    fc_dropout=0.4, last_activation='softmax',
                    optimizer='adam', loss='categorical_crossentropy',
                    metrics=['accuracy'])
model.fit(X_train, Y_train, batch_size=batch_size, epochs=epochs,
          validation_split=0.2, verbose=1,
          callbacks=callbacks)


print(model.evaluate(x_test, y_test, batch_size=batch_size))
4
1

Вариант решения использующий алгоритм TF-IDF + Logistic Regression (без использования нейронных сетей). Точность модели на тестовой выборке: 91.07%.

In [35]: grid.score(X_test, Y_test)
Out[35]: 0.9106944535179983

Полный код:

from datetime import datetime as DT
from pprint import pprint
import pandas as pd
import numpy as np
from sklearn.compose import ColumnTransformer
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.preprocessing import LabelEncoder, StandardScaler
from sklearn.model_selection import RandomizedSearchCV, GridSearchCV
from sklearn.linear_model import SGDClassifier, LogisticRegression
from sklearn.naive_bayes import MultinomialNB, GaussianNB,  BernoulliNB, ComplementNB
from sklearn.neural_network import MLPClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.gaussian_process import GaussianProcessClassifier
from sklearn.gaussian_process.kernels import RBF
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.pipeline import Pipeline, make_pipeline
from sklearn.externals import joblib
from pathlib import Path


def get_data(filename):
    df = pd.read_csv(filename, sep='~', header=None, names=['text'])
    df[['label', 'text']] = df.pop('text').str.split(n=1, expand=True)
    df['label'] = df['label'].astype('category')
    return df

def build_grid(**grid_search_parms):
    pipe = Pipeline([
        ('vect', TfidfVectorizer()),
        ('clf', MultinomialNB())
    ])

    param_grid = [
        #{      #   NOTE: poor accuracy !!!
        #    'vect': [TfidfVectorizer(), CountVectorizer()],
        #    'clf': [MultinomialNB(), BernoulliNB(), ComplementNB()],
        #    'vect__ngram_range': [(1, 3), (1,4)],
        #    'clf__alpha': [0.01, 0.1, 1],
        #},
        #{      #  NOTE: too slow !!!
        #    'vect': [TfidfVectorizer()],
        #    'clf': [SVC(gamma='scale')],
        #    'vect__ngram_range': [(1, 3)],
        #    'clf__kernel': ['rbf','linear'],
        #    #'clf__degree': [2,3],
        #},
        {
            'vect': [TfidfVectorizer()],
            'clf': [LogisticRegression(multi_class='auto', solver='lbfgs')],
            'vect__ngram_range': [(1, 3)],
            'clf__C': [1, 10, 15],
            'clf__max_iter': [1000],
        },
    ]

    grid = GridSearchCV(pipe, param_grid=param_grid, **grid_search_parms)
    return grid


path = Path(r'D:\work\SO_ru\939264-Keras_CNN_LSTM')
test_size = 0.25

os.chdir(str(path))

df = get_data(path / 'testtext.txt')
X_train, X_test, Y_train, Y_test = \
    train_test_split(df['text'], df['label'].cat.codes, test_size=test_size,
                     stratify=df['label'].cat.codes, random_state=123)

grid_search_parms = dict(cv=3, n_jobs=-1, verbose=2)

grid = build_grid(**grid_search_parms)
grid.fit(X_train, Y_train)
score = grid.score(X_test, Y_test)
print('Test Score: {:0.4f}'.format(score))
joblib.dump(grid, 'grid_{:%Y%m%d_%H%M%S}_val_acc_{:0.4f}.pkl'.format(DT.now(), score))
1

Модель с использованием 1D Convolution и LSTM - показывает точность 0.9000 на тестовой выборке после 5 эпох обучения, ~287 секунд/эпоха на GeForce GTX 1070 GPU:

Model Summary
_________________________________________________________________
Layer (type)                 Output Shape              Param #
=================================================================
embedding_1 (Embedding)      (None, 100, 64)           384000
_________________________________________________________________
dropout_1 (Dropout)          (None, 100, 64)           0
_________________________________________________________________
conv1d_1 (Conv1D)            (None, 96, 64)            20544
_________________________________________________________________
max_pooling1d_1 (MaxPooling1 (None, 24, 64)            0
_________________________________________________________________
lstm_1 (LSTM)                (None, 64)                33024
_________________________________________________________________
dense_1 (Dense)              (None, 250)               16250
_________________________________________________________________
activation_1 (Activation)    (None, 250)               0
_________________________________________________________________
dense_2 (Dense)              (None, 8)                 2008
=================================================================
Total params: 455,826
Trainable params: 455,826
Non-trainable params: 0

In [11]: score, acc = model.evaluate(X_test, Y_test, batch_size=batch_size)
    ...: print('Test score:', score)
    ...: print('Test accuracy:', acc)
    ...:
6123/6123 [==============================] - 6s 929us/step
Test score: 0.3491380948189445
Test accuracy: 0.9000489956390696

Полный код:

'''

Gets to 0.9000 test accuracy after 5 epochs. 287s/epoch on GTX 1070 GPU.

'''

from pathlib import Path
import pandas as pd
from keras.models import Sequential
from keras.layers import Embedding, Conv1D, MaxPool1D, GlobalMaxPool1D, Dense, Dropout, Activation, LSTM
from keras.preprocessing.text import Tokenizer, text_to_word_sequence
from keras.preprocessing.sequence import pad_sequences
from keras import optimizers
from keras.callbacks import EarlyStopping, ModelCheckpoint, TensorBoard
from keras.models import save_model, load_model
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelBinarizer

# Directories and file names
path = Path(r'D:\work\SO_ru\939264-Keras_CNN_LSTM')
filename = path / 'testtext.txt'
model_fn = path / 'model_CNN_LSTM.h5'

#####################
### Layers ...
#####################
# Embedding
max_features = 6000
maxlen = 100
embedding_dims = 64

# Dropout
dropout_1 = 0.2
dropout_2 = 0.2

# Convolution
filters = 64
kernel_size = 5
pool_size = 4

# LSTM
lstm_out = 64

# Dense
dense_1 = 250

# Training
batch_size = 32
epochs = 5

# changing working directory
os.chdir(str(path))

##################################################################

def get_data(filename, max_features=5000, maxlen=100, frac=1.0):
    data = (pd.read_csv(filename, header=None, names=['text'], sep='~')
              .sample(frac=frac))
    data[['label','text']] = data.pop('text').str.split(n=1, expand=True)
    data = data.dropna()
    data = data.loc[data['label'].str.contains(r'^_label')]

    # build vocabulary
    tok = Tokenizer(num_words=max_features)
    tok.fit_on_texts(data['text'])
    # convert texts to sequences
    X = tok.texts_to_sequences(data['text'])
    lb = LabelBinarizer()
    Y = pd.DataFrame(lb.fit_transform(data['label']), 
                     columns=lb.classes_, index=data.index)
    return (pad_sequences(X, maxlen=maxlen), Y, tok)


def build_model(num_classes=1,
                max_features=1000, embedding_dims=128, maxlen=100,
                dropout_1=0.1,
                filters=128, kernel_size=5, pool_size=4,
                lstm_out=64,
                dense_1=256, dropout_2=0.2,
                optimizer='adam', metrics=['accuracy']):
    if num_classes == 1:
        last_activation = 'sigmoid'
        loss = 'binary_crossentropy'
    else:
        last_activation = 'softmax'
        loss = 'categorical_crossentropy'
    model = Sequential()
    model.add(Embedding(max_features, embedding_dims, input_length=maxlen,
                        name='embedding_1'))
    model.add(Dropout(dropout_1, name='dropout_1'))
    model.add(Conv1D(filters,
              kernel_size,
              padding='valid',
              activation='relu',
              strides=1,
              name='conv1d_1'))
    model.add(MaxPool1D(pool_size=pool_size, name='max_pooling_1d_1'))
    model.add(LSTM(lstm_out, name='lstm_1'))
    model.add(Dense(dense_1, name='dense_1'))
    #model.add(Dropout(dropout_2, name='dropout_2'))
    model.add(Activation('relu', name='activation_1'))
    model.add(Dense(num_classes, activation=last_activation,
                    name='dense_output'))
    model.compile(loss=loss,
                  optimizer=optimizer,
                  metrics=metrics)
    return model
##################################################################

print('Loading data ...')
X, Y, tok = get_data(filename, max_features=max_features)
# split data set to train / test
X_train, X_test, Y_train, Y_test = \
    train_test_split(X, Y, test_size=0.05, random_state=123, stratify=Y)
print('X_train.shape:\t{}\t\tY_train.shape:\t{}'.format(X_train.shape, Y_train.shape))
print('X_test.shape:\t{}\t\tY_test.shape:\t{}'.format(X_test.shape, Y_test.shape))

#############################

# Keras callbacks
early_stop = EarlyStopping(monitor='val_acc', min_delta=0.001,
                           patience=5, verbose=1, mode='auto')
chkpt = ModelCheckpoint(str(model_fn), 
                        monitor='val_acc', 
                        verbose=1, 
                        save_best_only=True, 
                        mode='auto')
callbacks = [early_stop, chkpt]

if Y_train.ndim == 1:
    num_classes = 1
elif Y_train.ndim == 2:
    num_classes = Y_train.shape[1]
else:
    raise Exception('[Y_train] must be either 1D or 2D array')

print('Building model ...')
model_parms = dict(
  num_classes=num_classes,
  max_features=max_features, embedding_dims=embedding_dims, maxlen=maxlen,
  dropout_1=dropout_1,
  filters=filters, kernel_size=kernel_size, pool_size=pool_size,
  lstm_out=lstm_out,
  dense_1=dense_1, dropout_2=dropout_2,
  optimizer='adam', metrics=['accuracy']
)
model = build_model(**model_parms)

print('Model Summary')
print(model.summary())

print('Train...')
model.fit(X_train, Y_train, batch_size=batch_size, epochs=epochs,
          validation_split=0.2, verbose=1,
          callbacks=callbacks)


score, acc = model.evaluate(X_test, Y_test, batch_size=batch_size)
print('Test score:', score)
print('Test accuracy:', acc)
2
  • Как работает Conv1D слой? 2 фев 2019 в 10:09
  • @СергейАндреев, если хотите более менее подробный ответ, то задайте новый вопрос. Или вам ссылка подойдёт? 2 фев 2019 в 10:23

Ваш ответ

Нажимая на кнопку «Отправить ответ», вы соглашаетесь с нашими пользовательским соглашением, политикой конфиденциальности и политикой о куки

Всё ещё ищете ответ? Посмотрите другие вопросы с метками или задайте свой вопрос.