高周波エンジニアのためのAI・機械学習入門（GPU編19）PythonとKeras3.0とscikit-rfを使ってTransformerで同軸コネクタがついたマイクロストリップライン（誘電率、L、C、線幅、厚みを振った）のSパラメータからTDRでは出せない複素特性インピーダンスを推定する。

DNN、CNNと来て今回はTransformer。

コードはこんな感じ。

import os

os.environ["KERAS_BACKEND"] = "jax"

import keras

import jax

import numpy as np

import matplotlib.pyplot as plt

from sklearn.model_selection import train_test_split

from sklearn.preprocessing import StandardScaler

plt.rcParams['font.family'] = 'Noto Sans CJK JP'

# JAXの高速化

jax.config.update("jax_enable_x64", False)

jax.config.update("jax_disable_jit", False)

# mixed precision (JAX + Keras3対応)

keras.mixed_precision.set_global_policy("mixed_float16")

# ==========================

# データ読み込み

# ==========================

data_label = np.load("complex_tl_data.npz")

data = data_label["data"]     # (N,200,5)

label = data_label["label"]   # (N,2)

x_train, x_test, y_train, y_test = train_test_split(

    data, label, test_size=0.3, random_state=0

)

# ==========================

# ラベルのスケーリング

# ==========================

scaler_y = StandardScaler()

y_train_f = scaler_y.fit_transform(y_train)

y_test_f  = scaler_y.transform(y_test)

# ==========================

# Positional Encoding

# ==========================

def positional_encoding(seq_len, d_model):

    pos = np.arange(seq_len)[:, np.newaxis]

    i = np.arange(d_model)[np.newaxis, :]

    angle_rates = 1 / np.power(10000, (2 * (i // 2)) / np.float32(d_model))

    angle_rads = pos * angle_rates

    pe = np.zeros((seq_len, d_model))

    pe[:, 0::2] = np.sin(angle_rads[:, 0::2])

    pe[:, 1::2] = np.cos(angle_rads[:, 1::2])

    return keras.ops.convert_to_tensor(pe, dtype="float32")

# ==========================

# Transformer Encoder Block

# ==========================

def transformer_encoder(x, head_size, num_heads, ff_dim, dropout=0.1):

    # Self Attention

    attn_out = keras.layers.MultiHeadAttention(

        num_heads=num_heads,

        key_dim=head_size

    )(x, x)

    #attn_out = keras.layers.Dropout(dropout)(attn_out)

    x = keras.layers.LayerNormalization(epsilon=1e-6)(x + attn_out)

    # Feed Forwardネットワーク

    ffn = keras.layers.Dense(ff_dim, activation="relu")(x)

    ffn = keras.layers.Dense(x.shape[-1])(ffn)

    #ffn = keras.layers.Dropout(dropout)(ffn)

    x = keras.layers.LayerNormalization(epsilon=1e-6)(x + ffn)

    return x

# ==========================

# モデル定義

# ==========================

seq_len = 200

feat_dim = 5

embed_dim = 64

inputs = keras.Input(shape=(seq_len, feat_dim))

# 入力を高次元に射影

x = keras.layers.Dense(embed_dim)(inputs)

# Positional Encodingを加算

pe = positional_encoding(seq_len, embed_dim)

x = x + pe

# Transformer ブロック

x = transformer_encoder(x, head_size=64, num_heads=4, ff_dim=128)

#x = transformer_encoder(x, head_size=64, num_heads=4, ff_dim=128)

# 時系列次元を集約

x = keras.layers.GlobalAveragePooling1D()(x)

# 全結合

x = keras.layers.Dense(128, activation="relu")(x)

outputs = keras.layers.Dense(2)(x)

model = keras.Model(inputs, outputs)

model.compile(

    optimizer=keras.optimizers.Adam(learning_rate=1e-3),

    loss="mean_squared_error"

)

model.summary()

# ==========================

# 学習

# ==========================

batch_size = 64

epochs = 500

keras.utils.set_random_seed(1)

history = model.fit(

    x_train,

    y_train_f,

    batch_size=batch_size,

    epochs=epochs,

    validation_split=0.15,

)

# ==========================

# 予測

# ==========================

y_pred_f = model.predict(x_test)

y_pred = scaler_y.inverse_transform(y_pred_f)

metric = keras.metrics.R2Score()

metric.update_state(y_test, y_pred)

result = metric.result()

print(result)

error = np.abs((y_test - y_pred)/y_test*100)

print(error.mean(axis=0))

legend = ["Z0"]

fig, ax = plt.subplots(2, 2, figsize=(12,12))

maxrvalue = y_pred[:,0].max()

minrvalue = y_pred[:,0].min()

maxivalue = y_pred[:,1].max()

minivalue = y_pred[:,1].min()

ax[0, 0].scatter(y_pred[:,0], y_test[:,0], c="r", s=5)

ax[0, 0].plot([0,maxrvalue], [0,maxrvalue], "--", c="black")

ax[0, 0].set_xlabel("推定した値（実部）")

ax[0, 0].set_ylabel("実際の値（実部）")

ax[0, 0].set_xlim(minrvalue, maxrvalue)

ax[0, 0].set_ylim(minrvalue, maxrvalue)

ax[0, 0].grid()

ax[0, 0].legend([legend[0] + f" 平均誤差（実部）{error.mean(axis=0)[0]:.2f}%"])

ax[1, 0].hist(error[:, 0], bins = 100)

ax[1, 0].set_xlabel("誤差[%]（実部）")

ax[1, 0].set_ylabel("頻度")

ax[1, 0].grid()

ax[0, 1].scatter(y_pred[:,1], y_test[:,1], c="r", s=5)

ax[0, 1].plot([0,maxivalue], [0,maxivalue], "--", c="black")

ax[0, 1].set_xlabel("推定した値（虚部）")

ax[0, 1].set_ylabel("実際の値（虚部）")

ax[0, 1].set_xlim(minivalue, maxivalue)

ax[0, 1].set_ylim(minivalue, maxivalue)

ax[0, 1].grid()

ax[0, 1].legend([legend[0] + f" 平均誤差（虚部）{error.mean(axis=0)[1]:.2f}%"])

ax[1, 1].hist(error[:, 1], bins = 100)

ax[1, 1].set_xlabel("誤差[%]（虚部）")

ax[1, 1].set_ylabel("頻度")

ax[1, 1].grid()

fig.tight_layout()

plt.show()

結果はこちら。CNNと大体同じくらいになっている。時間は1.5倍くらいはかかる。

まあ今回もCNNを使うのが一番よさそうだ。