Econ 425T / Biostat 203B
Source: https://keras.io/examples/nlp/text_classification_with_transformer/
Display system information for reproducibility.
import IPython
print(IPython.sys_info()){'commit_hash': 'add5877a4',
'commit_source': 'installation',
'default_encoding': 'utf-8',
'ipython_path': '/Users/huazhou/opt/anaconda3/lib/python3.9/site-packages/IPython',
'ipython_version': '8.8.0',
'os_name': 'posix',
'platform': 'macOS-10.16-x86_64-i386-64bit',
'sys_executable': '/Users/huazhou/opt/anaconda3/bin/python3',
'sys_platform': 'darwin',
'sys_version': '3.9.12 (main, Apr 5 2022, 01:56:13) \n[Clang 12.0.0 ]'}sessionInfo()R version 4.2.2 (2022-10-31)
Platform: x86_64-apple-darwin17.0 (64-bit)
Running under: macOS Big Sur ... 10.16
Matrix products: default
BLAS: /Library/Frameworks/R.framework/Versions/4.2/Resources/lib/libRblas.0.dylib
LAPACK: /Library/Frameworks/R.framework/Versions/4.2/Resources/lib/libRlapack.dylib
locale:
[1] en_US.UTF-8/en_US.UTF-8/en_US.UTF-8/C/en_US.UTF-8/en_US.UTF-8
attached base packages:
[1] stats graphics grDevices utils datasets methods base
loaded via a namespace (and not attached):
[1] Rcpp_1.0.9 here_1.0.1 lattice_0.20-45 png_0.1-8
[5] withr_2.5.0 rprojroot_2.0.3 digest_0.6.29 grid_4.2.2
[9] jsonlite_1.8.0 magrittr_2.0.3 evaluate_0.15 rlang_1.0.6
[13] stringi_1.7.8 cli_3.4.1 rstudioapi_0.13 Matrix_1.5-1
[17] reticulate_1.27 rmarkdown_2.14 tools_4.2.2 stringr_1.4.0
[21] htmlwidgets_1.6.1 xfun_0.31 yaml_2.3.5 fastmap_1.1.0
[25] compiler_4.2.2 htmltools_0.5.4 knitr_1.39 Load libraries.
# Plotting tool
import matplotlib.pyplot as plt
# Load Tensorflow and Keras
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layerslibrary(keras)class TransformerBlock(layers.Layer):
def __init__(self, embed_dim, num_heads, ff_dim, rate = 0.1):
super().__init__()
self.att = layers.MultiHeadAttention(num_heads = num_heads, key_dim = embed_dim)
self.ffn = keras.Sequential(
[layers.Dense(ff_dim, activation = "relu"), layers.Dense(embed_dim),]
)
self.layernorm1 = layers.LayerNormalization(epsilon = 1e-6)
self.layernorm2 = layers.LayerNormalization(epsilon = 1e-6)
self.dropout1 = layers.Dropout(rate)
self.dropout2 = layers.Dropout(rate)
def call(self, inputs, training):
attn_output = self.att(inputs, inputs)
attn_output = self.dropout1(attn_output, training = training)
out1 = self.layernorm1(inputs + attn_output)
ffn_output = self.ffn(out1)
ffn_output = self.dropout2(ffn_output, training=training)
return self.layernorm2(out1 + ffn_output)Two seperate embedding layers, one for tokens, one for token index (positions).
class TokenAndPositionEmbedding(layers.Layer):
def __init__(self, maxlen, vocab_size, embed_dim):
super().__init__()
self.token_emb = layers.Embedding(input_dim=vocab_size, output_dim=embed_dim)
self.pos_emb = layers.Embedding(input_dim=maxlen, output_dim=embed_dim)
def call(self, x):
maxlen = tf.shape(x)[-1]
positions = tf.range(start=0, limit=maxlen, delta=1)
positions = self.pos_emb(positions)
x = self.token_emb(x)
return x + positionsvocab_size = 20000 # Only consider the top 20k words
maxlen = 200 # Only consider the first 200 words of each movie review
(x_train, y_train), (x_val, y_val) = keras.datasets.imdb.load_data(num_words = vocab_size)
print(len(x_train), "Training sequences")25000 Training sequencesprint(len(x_val), "Validation sequences")25000 Validation sequencesx_train = keras.preprocessing.sequence.pad_sequences(x_train, maxlen = maxlen)
x_val = keras.preprocessing.sequence.pad_sequences(x_val, maxlen = maxlen)Transformer layer outputs one vector for each time step of our input sequence. Here, we take the mean across all time steps and use a feed forward network on top of it to classify text.
embed_dim = 32 # Embedding size for each token
num_heads = 2 # Number of attention heads
ff_dim = 32 # Hidden layer size in feed forward network inside transformer
inputs = layers.Input(shape = (maxlen,))
embedding_layer = TokenAndPositionEmbedding(maxlen, vocab_size, embed_dim)
x = embedding_layer(inputs)
transformer_block = TransformerBlock(embed_dim, num_heads, ff_dim)
x = transformer_block(x)
x = layers.GlobalAveragePooling1D()(x)
x = layers.Dropout(0.1)(x)
x = layers.Dense(20, activation = "relu")(x)
x = layers.Dropout(0.1)(x)
outputs = layers.Dense(2, activation = "softmax")(x)
model = keras.Model(inputs=inputs, outputs=outputs)model.compile(
optimizer = "adam",
loss = "sparse_categorical_crossentropy",
metrics = ["accuracy"]
)
history = model.fit(
x_train,
y_train,
batch_size = 32,
epochs = 2,
validation_data = (x_val, y_val),
verbose = 2
)Epoch 1/2
782/782 - 67s - loss: 0.3909 - accuracy: 0.8138 - val_loss: 0.3012 - val_accuracy: 0.8747 - 67s/epoch - 86ms/step
Epoch 2/2
782/782 - 63s - loss: 0.2008 - accuracy: 0.9240 - val_loss: 0.3129 - val_accuracy: 0.8598 - 63s/epoch - 81ms/step