{"id":1190,"date":"2025-10-24T07:51:00","date_gmt":"2025-10-23T23:51:00","guid":{"rendered":"https:\/\/www.laoyulaoyu.com\/?p=1190"},"modified":"2025-10-23T19:45:40","modified_gmt":"2025-10-23T11:45:40","slug":"%e5%80%9f%e5%8a%a9-transformer-%e5%ae%9e%e7%8e%b0%e7%be%8e%e8%82%a1%e4%bb%b7%e6%a0%bc%e7%9a%84%e9%a2%84%e6%b5%8b%ef%bc%88python%e5%b9%b2%e8%b4%a7%ef%bc%89","status":"publish","type":"post","link":"https:\/\/laoyulaoyu.com\/index.php\/2025\/10\/24\/%e5%80%9f%e5%8a%a9-transformer-%e5%ae%9e%e7%8e%b0%e7%be%8e%e8%82%a1%e4%bb%b7%e6%a0%bc%e7%9a%84%e9%a2%84%e6%b5%8b%ef%bc%88python%e5%b9%b2%e8%b4%a7%ef%bc%89\/","title":{"rendered":"\u501f\u52a9 Transformer \u5b9e\u73b0\u7f8e\u80a1\u4ef7\u683c\u7684\u9884\u6d4b\uff08Python\u5e72\u8d27\uff09"},"content":{"rendered":"\n

<\/p>\n\n\n\n

\u4f5c\u8005\uff1a\u8001\u4f59\u635e\u9c7c<\/a><\/p>\n\n\n\n

\u539f\u521b\u4e0d\u6613\uff0c\u8f6c\u8f7d\u8bf7\u6807\u660e\u51fa\u5904\u53ca\u539f\u4f5c\u8005\u3002<\/mark><\/strong><\/p>\n\n\n\n

\"\"<\/figure>\n\n\n\n

\u5199\u5728\u524d\u9762\u7684\u8bdd\uff1a<\/strong>Transformer \u662f\u4e00\u79cd\u5728\u81ea\u7136\u8bed\u8a00\u5904\u7406\u7b49\u9886\u57df\u5e7f\u6cdb\u5e94\u7528\u7684\u6df1\u5ea6\u5b66\u4e60\u67b6\u6784\uff0c\u4e0e\u4f20\u7edf\u7684\u5faa\u73af\u795e\u7ecf\u7f51\u7edc\uff08RNN\uff09\u76f8\u6bd4\uff0cTransformer \u53ef\u4ee5\u5e76\u884c\u5904\u7406\u8f93\u5165\u5e8f\u5217\u7684\u5404\u4e2a\u4f4d\u7f6e\uff0c\u5927\u5927\u63d0\u9ad8\u4e86\u8ba1\u7b97\u6548\u7387\u3002\u800c\u4e14\u901a\u8fc7\u591a\u5c42\u7684\u6df1\u5ea6\u5806\u53e0\uff0c\u80fd\u591f\u5b66\u4e60\u5230\u66f4\u590d\u6742\u548c\u62bd\u8c61\u7684\u7279\u5f81\u8868\u793a\u3002\u672c\u6587\u5c06\u5229\u7528Python\u4ee3\u7801\u6765\u5b9e\u73b0\u7f8e\u80a1\u4ef7\u683c\u7684\u9884\u6d4b\u6a21\u62df\u3002<\/p>\n\n\n\n

\u8bdd\u4e0d\u591a\u8bf4\uff0c\u4ee3\u7801\u5982\u4e0b\uff1a<\/strong>\n
  • <\/li>
  • <\/li>
  • <\/li>
  • <\/li>
  • <\/li>
  • <\/li>
  • <\/li>
  • <\/li>
  • <\/li>
  • <\/li>
  • <\/li>
  • <\/li>
  • <\/li>
  • <\/li>
  • <\/li>
  • <\/li>
  • <\/li>
  • <\/li>
  • <\/li>
  • <\/li>
  • <\/li>
  • <\/li>
  • <\/li>
  • <\/li>\nimport numpy as np\nimport pandas as pd\nimport os, datetime\nimport tensorflow as tf\nfrom tensorflow.keras.models import *\nfrom tensorflow.keras.layers import *\nprint('Tensorflow version: {}'.format(tf.__version__))\n\nimport matplotlib.pyplot as plt\nplt.style.use('seaborn')\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nphysical_devices = tf.config.list_physical_devices()\nprint('Physical devices: {}'.format(physical_devices))\n\n# Filter out the CPUs and keep only the GPUs\ngpus = [device for device in physical_devices if 'GPU' in device.device_type]\n\n# If GPUs are available, set memory growth to True\nif len(gpus) > 0:\n    tf.config.experimental.set_visible_devices(gpus[0], 'GPU')\n    tf.config.experimental.set_memory_growth(gpus[0], True)\n    print('GPU memory growth: True')<\/code><\/pre>\n\n\n\n
    Tensorflow version: 2.9.1
    Physical devices: [PhysicalDevice(name=’\/physical_device:CPU:0′, device_type=’CPU’)]<\/p>\n\n\n\n
    Hyperparameters<\/strong><\/h3>\n\n\n\n
    batch_size = 32\nseq_len = 128\n\nd_k = 256\nd_v = 256\nn_heads = 12\nff_dim = 256<\/code><\/pre>\n\n\n\n
    Load IBM data<\/strong><\/h3>\n\n\n\n
    IBM_path = 'IBM.csv'<\/code>\n<\/code>df = pd.read_csv(IBM_path, delimiter=',', usecols=['Date', 'Open', 'High', 'Low', 'Close', 'Volume'])<\/code>\n<\/code># Replace 0 to avoid dividing by 0 later on<\/em><\/code>df['Volume'].replace(to_replace=0, method='ffill', inplace=True) <\/code>df.sort_values('Date', inplace=True)<\/code>df.tail()<\/code> df.head()<\/code><\/pre>\n\n\n\n
    \"\"<\/figure>\n\n\n\n
    \"\"<\/figure>\n\n\n\n
      <\/ul>\n\n\n\n
      # print the shape of the dataset<\/em><\/code>print('Shape of the dataframe: {}'.format(df.shape))<\/code>Shape of the dataframe: (14588, 6)<\/code><\/pre>\n\n\n\n
      Plot daily IBM closing prices and volume<\/strong><\/h3>\n\n\n\n
      fig = plt.figure(figsize=(15,10))<\/code>st = fig.suptitle(\"IBM Close Price and Volume\", fontsize=20)<\/code>st.set_y(0.92)<\/code>
      <\/code>ax1 = fig.add_subplot(211)<\/code>ax1.plot(df['Close'], label='IBM Close Price')<\/code>ax1.set_xticks(range(0, df.shape[0], 1464))<\/code>ax1.set_xticklabels(df['Date'].loc[::1464])<\/code>ax1.set_ylabel('Close Price', fontsize=18)<\/code>ax1.legend(loc=\"upper left\", fontsize=12)<\/code>
      <\/code>ax2 = fig.add_subplot(212)<\/code>ax2.plot(df['Volume'], label='IBM Volume')<\/code>ax2.set_xticks(range(0, df.shape[0], 1464))<\/code>ax2.set_xticklabels(df['Date'].loc[::1464])<\/code>ax2.set_ylabel('Volume', fontsize=18)<\/code>ax2.legend(loc=\"upper left\", fontsize=12)<\/code><\/code><\/pre>\n\n\n\n
      \"\"<\/figure>\n\n\n\n
      <\/p>\n\n\n\n
      Calculate normalized percentage change of all columns<\/strong><\/h3>\n\n\n\n
      '''Calculate percentage change'''<\/code>
      <\/code>df['Open'] = df['Open'].pct_change() # Create arithmetic returns column<\/em><\/code>df['High'] = df['High'].pct_change() # Create arithmetic returns column<\/em><\/code>df['Low'] = df['Low'].pct_change() # Create arithmetic returns column<\/em><\/code>df['Close'] = df['Close'].pct_change() # Create arithmetic returns column<\/em><\/code>df['Volume'] = df['Volume'].pct_change()<\/code>
      <\/code>df.dropna(how='any', axis=0, inplace=True) # Drop all rows with NaN values<\/em><\/code>
      <\/code>###############################################################################<\/em><\/code>'''Create indexes to split dataset'''<\/code>
      <\/code>times = sorted(df.index.values)<\/code>last_10pct = sorted(df.index.values)[-int(0.1*len(times))] # Last 10% of series<\/em><\/code>last_20pct = sorted(df.index.values)[-int(0.2*len(times))] # Last 20% of series<\/em><\/code>
      <\/code>###############################################################################<\/em><\/code>'''Normalize price columns'''<\/code>#<\/em><\/code>min_return = min(df[(df.index < last_20pct)][['Open', 'High', 'Low', 'Close']].min(axis=0))<\/code>max_return = max(df[(df.index < last_20pct)][['Open', 'High', 'Low', 'Close']].max(axis=0))<\/code>
      <\/code># Min-max normalize price columns (0-1 range)<\/em><\/code>df['Open'] = (df['Open'] - min_return) \/ (max_return - min_return)<\/code>df['High'] = (df['High'] - min_return) \/ (max_return - min_return)<\/code>df['Low'] = (df['Low'] - min_return) \/ (max_return - min_return)<\/code>df['Close'] = (df['Close'] - min_return) \/ (max_return - min_return)<\/code>
      <\/code>###############################################################################<\/em><\/code>'''Normalize volume column'''<\/code>
      <\/code>min_volume = df[(df.index < last_20pct)]['Volume'].min(axis=0)<\/code>max_volume = df[(df.index < last_20pct)]['Volume'].max(axis=0)<\/code>
      <\/code># Min-max normalize volume columns (0-1 range)<\/em><\/code>df['Volume'] = (df['Volume'] - min_volume) \/ (max_volume - min_volume)<\/code>
      <\/code>###############################################################################<\/em><\/code>'''Create training, validation and test split'''<\/code>
      <\/code>df_train = df[(df.index < last_20pct)]  # Training data are 80% of total data<\/em><\/code>df_val = df[(df.index >= last_20pct) & (df.index < last_10pct)]<\/code>df_test = df[(df.index >= last_10pct)]<\/code>
      <\/code># Remove date column<\/em><\/code>df_train.drop(columns=['Date'], inplace=True)<\/code>df_val.drop(columns=['Date'], inplace=True)<\/code>df_test.drop(columns=['Date'], inplace=True)<\/code>
      <\/code># Convert pandas columns into arrays<\/em><\/code>train_data = df_train.values<\/code>val_data = df_val.values<\/code>test_data = df_test.values<\/code>print('Training data shape: {}'.format(train_data.shape))<\/code>print('Validation data shape: {}'.format(val_data.shape))<\/code>print('Test data shape: {}'.format(test_data.shape))<\/code>
      <\/code>df_train.head()<\/code><\/code><\/pre>\n\n\n\n
      Training data shape: (11678, 5)
      Validation data shape: (1460, 5)
      Test data shape: (1459, 5)<\/pre>\n\n\n\n
      \"\"<\/figure>\n\n\n\n
      <\/p>\n\n\n\n
      Plot daily changes of close prices and volume<\/strong><\/h3>\n\n\n\n
        <\/ul>\n\n\n\n
        fig = plt.figure(figsize=(15,12))<\/code>st = fig.suptitle(\"Data Separation\", fontsize=20)<\/code>st.set_y(0.95)<\/code>
        <\/code>###############################################################################<\/em><\/code>
        <\/code>ax1 = fig.add_subplot(211)<\/code>ax1.plot(np.arange(train_data.shape[0]), df_train['Close'], label='Training data')<\/code>
        <\/code>ax1.plot(np.arange(train_data.shape[0], <\/code>                   train_data.shape[0]+val_data.shape[0]), df_val['Close'], label='Validation data')<\/code>
        <\/code>ax1.plot(np.arange(train_data.shape[0]+val_data.shape[0], <\/code>                   train_data.shape[0]+val_data.shape[0]+test_data.shape[0]), df_test['Close'], label='Test data')<\/code>ax1.set_xlabel('Date')<\/code>ax1.set_ylabel('Normalized Closing Returns')<\/code>ax1.set_title(\"Close Price\", fontsize=18)<\/code>ax1.legend(loc=\"best\", fontsize=12)<\/code>
        <\/code>###############################################################################<\/em><\/code>
        <\/code>ax2 = fig.add_subplot(212)<\/code>ax2.plot(np.arange(train_data.shape[0]), df_train['Volume'], label='Training data')<\/code>
        <\/code>ax2.plot(np.arange(train_data.shape[0], <\/code>                   train_data.shape[0]+val_data.shape[0]), df_val['Volume'], label='Validation data')<\/code>
        <\/code>ax2.plot(np.arange(train_data.shape[0]+val_data.shape[0], <\/code>                   train_data.shape[0]+val_data.shape[0]+test_data.shape[0]), df_test['Volume'], label='Test data')<\/code>ax2.set_xlabel('Date')<\/code>ax2.set_ylabel('Normalized Volume Changes')<\/code>ax2.set_title(\"Volume\", fontsize=18)<\/code>ax2.legend(loc=\"best\", fontsize=12)<\/code><\/code><\/pre>\n\n\n\n
        \"\"<\/figure>\n\n\n\n
        <\/p>\n\n\n\n
        Create chunks of training, validation and test data<\/strong><\/h3>\n\n\n\n
        # Training data<\/em><\/code>X_train, y_train = [], []<\/code>for i in range(seq_len, len(train_data)):<\/code>  X_train.append(train_data[i-seq_len:i]) # Chunks of training data with a length of 128 df-rows<\/em><\/code>  y_train.append(train_data[:, 3][i]) #Value of 4th column (Close Price) of df-row 128+1<\/em><\/code>X_train, y_train = np.array(X_train), np.array(y_train)<\/code>
        <\/code>###############################################################################<\/em><\/code>
        <\/code># Validation data<\/em><\/code>X_val, y_val = [], []<\/code>for i in range(seq_len, len(val_data)):<\/code>    X_val.append(val_data[i-seq_len:i])<\/code>    y_val.append(val_data[:, 3][i])<\/code>X_val, y_val = np.array(X_val), np.array(y_val)<\/code>
        <\/code>###############################################################################<\/em><\/code>
        <\/code># Test data<\/em><\/code>X_test, y_test = [], []<\/code>for i in range(seq_len, len(test_data)):<\/code>    X_test.append(test_data[i-seq_len:i])<\/code>    y_test.append(test_data[:, 3][i])    <\/code>X_test, y_test = np.array(X_test), np.array(y_test)<\/code>
        <\/code>print('Training set shape', X_train.shape, y_train.shape)<\/code>print('Validation set shape', X_val.shape, y_val.shape)<\/code>print('Testing set shape' ,X_test.shape, y_test.shape)<\/code><\/code><\/pre>\n\n\n\n
        Training set shape (11550, 128, 5) (11550,)
        Validation set shape (1332, 128, 5) (1332,)
        Testing set shape (1331, 128, 5) (1331,)<\/pre>\n\n\n\n
        TimeVector<\/strong><\/h3>\n\n\n\n
          <\/ul>\n\n\n\n
          class Time2Vector(Layer):<\/code>  def __init__(self, seq_len, **kwargs):<\/code>    super(Time2Vector, self).__init__()<\/code>    self.seq_len = seq_len<\/code>
          <\/code>  def build(self, input_shape):<\/code>    '''Initialize weights and biases with shape (batch, seq_len)'''<\/code>    self.weights_linear = self.add_weight(name='weight_linear',<\/code>                                shape=(int(self.seq_len),),<\/code>                                initializer='uniform',<\/code>                                trainable=True)<\/code>    <\/code>    self.bias_linear = self.add_weight(name='bias_linear',<\/code>                                shape=(int(self.seq_len),),<\/code>                                initializer='uniform',<\/code>                                trainable=True)<\/code>    <\/code>    self.weights_periodic = self.add_weight(name='weight_periodic',<\/code>                                shape=(int(self.seq_len),),<\/code>                                initializer='uniform',<\/code>                                trainable=True)<\/code>
          <\/code>    self.bias_periodic = self.add_weight(name='bias_periodic',<\/code>                                shape=(int(self.seq_len),),<\/code>                                initializer='uniform',<\/code>                                trainable=True)<\/code>
          <\/code>  def call(self, x):<\/code>    '''Calculate linear and periodic time features'''<\/code>    x = tf.math.reduce_mean(x[:,:,:4], axis=-1) <\/code>    time_linear = self.weights_linear * x + self.bias_linear # Linear time feature<\/em><\/code>    time_linear = tf.expand_dims(time_linear, axis=-1) # Add dimension (batch, seq_len, 1)<\/em><\/code>    <\/code>    time_periodic = tf.math.sin(tf.multiply(x, self.weights_periodic) + self.bias_periodic)<\/code>    time_periodic = tf.expand_dims(time_periodic, axis=-1) # Add dimension (batch, seq_len, 1)<\/em><\/code>    return tf.concat([time_linear, time_periodic], axis=-1) # shape = (batch, seq_len, 2)<\/em><\/code>   <\/code>  def get_config(self): # Needed for saving and loading model with custom layer<\/em><\/code>    config = super().get_config().copy()<\/code>    config.update({'seq_len': self.seq_len})<\/code>    return config<\/code><\/code><\/pre>\n\n\n\n
          Transformer<\/strong><\/h3>\n\n\n\n
          class SingleAttention(Layer):<\/code>  def __init__(self, d_k, d_v):<\/code>    super(SingleAttention, self).__init__()<\/code>    self.d_k = d_k<\/code>    self.d_v = d_v<\/code>
          <\/code>  def build(self, input_shape):<\/code>    self.query = Dense(self.d_k, <\/code>                       input_shape=input_shape, <\/code>                       kernel_initializer='glorot_uniform', <\/code>                       bias_initializer='glorot_uniform')<\/code>    <\/code>    self.key = Dense(self.d_k, <\/code>                     input_shape=input_shape, <\/code>                     kernel_initializer='glorot_uniform', <\/code>                     bias_initializer='glorot_uniform')<\/code>    <\/code>    self.value = Dense(self.d_v, <\/code>                       input_shape=input_shape, <\/code>                       kernel_initializer='glorot_uniform', <\/code>                       bias_initializer='glorot_uniform')<\/code>
          <\/code>  def call(self, inputs): # inputs = (in_seq, in_seq, in_seq)<\/em><\/code>    q = self.query(inputs[0])<\/code>    k = self.key(inputs[1])<\/code>
          <\/code>    attn_weights = tf.matmul(q, k, transpose_b=True)<\/code>    attn_weights = tf.map_fn(lambda x: x\/np.sqrt(self.d_k), attn_weights)<\/code>    attn_weights = tf.nn.softmax(attn_weights, axis=-1)<\/code>    <\/code>    v = self.value(inputs[2])<\/code>    attn_out = tf.matmul(attn_weights, v)<\/code>    return attn_out    <\/code>
          <\/code>#############################################################################<\/em><\/code>
          <\/code>class MultiAttention(Layer):<\/code>  def __init__(self, d_k, d_v, n_heads):<\/code>    super(MultiAttention, self).__init__()<\/code>    self.d_k = d_k<\/code>    self.d_v = d_v<\/code>    self.n_heads = n_heads<\/code>    self.attn_heads = list()<\/code>
          <\/code>  def build(self, input_shape):<\/code>    for n in range(self.n_heads):<\/code>      self.attn_heads.append(SingleAttention(self.d_k, self.d_v))  <\/code>    <\/code>    # input_shape[0]=(batch, seq_len, 7), input_shape[0][-1]=7 <\/em><\/code>    self.linear = Dense(input_shape[0][-1], <\/code>                        input_shape=input_shape, <\/code>                        kernel_initializer='glorot_uniform', <\/code>                        bias_initializer='glorot_uniform')<\/code>
          <\/code>  def call(self, inputs):<\/code>    attn = [self.attn_heads[i](inputs) for i in range(self.n_heads)]<\/code>    concat_attn = tf.concat(attn, axis=-1)<\/code>    multi_linear = self.linear(concat_attn)<\/code>    return multi_linear   <\/code>
          <\/code>#############################################################################<\/em><\/code>
          <\/code>class TransformerEncoder(Layer):<\/code>  def __init__(self, d_k, d_v, n_heads, ff_dim, dropout=0.1, **kwargs):<\/code>    super(TransformerEncoder, self).__init__()<\/code>    self.d_k = d_k<\/code>    self.d_v = d_v<\/code>    self.n_heads = n_heads<\/code>    self.ff_dim = ff_dim<\/code>    self.attn_heads = list()<\/code>    self.dropout_rate = dropout<\/code>
          <\/code>  def build(self, input_shape):<\/code>    self.attn_multi = MultiAttention(self.d_k, self.d_v, self.n_heads)<\/code>    self.attn_dropout = Dropout(self.dropout_rate)<\/code>    self.attn_normalize = LayerNormalization(input_shape=input_shape, epsilon=1e-6)<\/code>
          <\/code>    self.ff_conv1D_1 = Conv1D(filters=self.ff_dim, kernel_size=1, activation='relu')<\/code>    # input_shape[0]=(batch, seq_len, 7), input_shape[0][-1] = 7 <\/em><\/code>    self.ff_conv1D_2 = Conv1D(filters=input_shape[0][-1], kernel_size=1) <\/code>    self.ff_dropout = Dropout(self.dropout_rate)<\/code>    self.ff_normalize = LayerNormalization(input_shape=input_shape, epsilon=1e-6)    <\/code>  <\/code>  def call(self, inputs): # inputs = (in_seq, in_seq, in_seq)<\/em><\/code>    attn_layer = self.attn_multi(inputs)<\/code>    attn_layer = self.attn_dropout(attn_layer)<\/code>    attn_layer = self.attn_normalize(inputs[0] + attn_layer)<\/code>
          <\/code>    ff_layer = self.ff_conv1D_1(attn_layer)<\/code>    ff_layer = self.ff_conv1D_2(ff_layer)<\/code>    ff_layer = self.ff_dropout(ff_layer)<\/code>    ff_layer = self.ff_normalize(inputs[0] + ff_layer)<\/code>    return ff_layer <\/code>
          <\/code>  def get_config(self): # Needed for saving and loading model with custom layer<\/em><\/code>    config = super().get_config().copy()<\/code>    config.update({'d_k': self.d_k,<\/code>                   'd_v': self.d_v,<\/code>                   'n_heads': self.n_heads,<\/code>                   'ff_dim': self.ff_dim,<\/code>                   'attn_heads': self.attn_heads,<\/code>                   'dropout_rate': self.dropout_rate})<\/code>    return config<\/code><\/code><\/pre>\n\n\n\n
          Model<\/strong><\/h3>\n\n\n\n
          def create_model():<\/code>  '''Initialize time and transformer layers'''<\/code>  time_embedding = Time2Vector(seq_len)<\/code>  attn_layer1 = TransformerEncoder(d_k, d_v, n_heads, ff_dim)<\/code>  attn_layer2 = TransformerEncoder(d_k, d_v, n_heads, ff_dim)<\/code>  attn_layer3 = TransformerEncoder(d_k, d_v, n_heads, ff_dim)<\/code>\n<\/code>  '''Construct model'''<\/code>  in_seq = Input(shape=(seq_len, 5))<\/code>  x = time_embedding(in_seq)<\/code>  x = Concatenate(axis=-1)([in_seq, x])<\/code>  x = attn_layer1((x, x, x))<\/code>  x = attn_layer2((x, x, x))<\/code>  x = attn_layer3((x, x, x))<\/code>  x = GlobalAveragePooling1D(data_format='channels_first')(x)<\/code>  x = Dropout(0.1)(x)<\/code>  x = Dense(64, activation='relu')(x)<\/code>  x = Dropout(0.1)(x)<\/code>  out = Dense(1, activation='linear')(x)<\/code>\n<\/code>  model = Model(inputs=in_seq, outputs=out)<\/code>  model.compile(loss='mse', optimizer='adam', metrics=['mae', 'mape'])<\/code>  return model<\/code>\n<\/code>\n<\/code>model = create_model()<\/code>model.summary()<\/code>\n<\/code>callback = tf.keras.callbacks.ModelCheckpoint('Transformer+TimeEmbedding.hdf5', <\/code>                                              monitor='val_loss', <\/code>                                              save_best_only=True, verbose=1)<\/code>\n<\/code>history = model.fit(X_train, y_train, <\/code>                    batch_size=batch_size, <\/code>                    epochs=35, <\/code>                    callbacks=[callback],<\/code>                    validation_data=(X_val, y_val))  <\/code>\n<\/code>model = tf.keras.models.load_model('Transformer+TimeEmbedding.hdf5',<\/code>                                   custom_objects={'Time2Vector': Time2Vector, <\/code>                                                   'SingleAttention': SingleAttention,<\/code>                                                   'MultiAttention': MultiAttention,<\/code>                                                   'TransformerEncoder': TransformerEncoder})<\/code>\n<\/code>\n<\/code>###############################################################################<\/em><\/code>'''Calculate predictions and metrics'''<\/code>\n<\/code>#Calculate predication for training, validation and test data<\/em><\/code>train_pred = model.predict(X_train)<\/code>val_pred = model.predict(X_val)<\/code>test_pred = model.predict(X_test)<\/code>\n<\/code>#Print evaluation metrics for all datasets<\/em><\/code>train_eval = model.evaluate(X_train, y_train, verbose=0)<\/code>val_eval = model.evaluate(X_val, y_val, verbose=0)<\/code>test_eval = model.evaluate(X_test, y_test, verbose=0)<\/code>print(' ')<\/code>print('Evaluation metrics')<\/code>print('Training Data - Loss: {:.4f}, MAE: {:.4f}, MAPE: {:.4f}'.format(train_eval[0], train_eval[1], train_eval[2]))<\/code>print('Validation Data - Loss: {:.4f}, MAE: {:.4f}, MAPE: {:.4f}'.format(val_eval[0], val_eval[1], val_eval[2]))<\/code>print('Test Data - Loss: {:.4f}, MAE: {:.4f}, MAPE: {:.4f}'.format(test_eval[0], test_eval[1], test_eval[2]))<\/code>\n<\/code>###############################################################################<\/em><\/code>'''Display results'''<\/code>\n<\/code>fig = plt.figure(figsize=(15,20))<\/code>st = fig.suptitle(\"Transformer + TimeEmbedding Model\", fontsize=22)<\/code>st.set_y(0.92)<\/code>\n<\/code>#Plot training data results<\/em><\/code>ax11 = fig.add_subplot(311)<\/code>ax11.plot(train_data[:, 3], label='IBM Closing Returns')<\/code>ax11.plot(np.arange(seq_len, train_pred.shape[0]+seq_len), train_pred, linewidth=3, label='Predicted IBM Closing Returns')<\/code>ax11.set_title(\"Training Data\", fontsize=18)<\/code>ax11.set_xlabel('Date')<\/code>ax11.set_ylabel('IBM Closing Returns')<\/code>ax11.legend(loc=\"best\", fontsize=12)<\/code>\n<\/code>#Plot validation data results<\/em><\/code>ax21 = fig.add_subplot(312)<\/code>ax21.plot(val_data[:, 3], label='IBM Closing Returns')<\/code>ax21.plot(np.arange(seq_len, val_pred.shape[0]+seq_len), val_pred, linewidth=3, label='Predicted IBM Closing Returns')<\/code>ax21.set_title(\"Validation Data\", fontsize=18)<\/code>ax21.set_xlabel('Date')<\/code>ax21.set_ylabel('IBM Closing Returns')<\/code>ax21.legend(loc=\"best\", fontsize=12)<\/code>\n<\/code>#Plot test data results<\/em><\/code>ax31 = fig.add_subplot(313)<\/code>ax31.plot(test_data[:, 3], label='IBM Closing Returns')<\/code>ax31.plot(np.arange(seq_len, test_pred.shape[0]+seq_len), test_pred, linewidth=3, label='Predicted IBM Closing Returns')<\/code>ax31.set_title(\"Test Data\", fontsize=18)<\/code>ax31.set_xlabel('Date')<\/code>ax31.set_ylabel('IBM Closing Returns')<\/code>ax31.legend(loc=\"best\", fontsize=12)<\/code><\/code><\/pre>\n\n\n\n
          \"\"<\/figure>\n\n\n\n
          <\/p>\n\n\n\n
          ###############################################################################<\/em><\/code>'''Calculate predictions and metrics'''<\/code>
          <\/code>#Calculate predication for training, validation and test data<\/em><\/code>train_pred = model.predict(X_train)<\/code>val_pred = model.predict(X_val)<\/code>test_pred = model.predict(X_test)<\/code>
          <\/code>#Print evaluation metrics for all datasets<\/em><\/code>train_eval = model.evaluate(X_train, y_train, verbose=0)<\/code>val_eval = model.evaluate(X_val, y_val, verbose=0)<\/code>test_eval = model.evaluate(X_test, y_test, verbose=0)<\/code>print(' ')<\/code>print('Evaluation metrics')<\/code>print('Training Data - Loss: {:.4f}, MAE: {:.4f}, MAPE: {:.4f}'.format(train_eval[0], train_eval[1], train_eval[2]))<\/code>print('Validation Data - Loss: {:.4f}, MAE: {:.4f}, MAPE: {:.4f}'.format(val_eval[0], val_eval[1], val_eval[2]))<\/code>print('Test Data - Loss: {:.4f}, MAE: {:.4f}, MAPE: {:.4f}'.format(test_eval[0], test_eval[1], test_eval[2]))<\/code>
          <\/code>###############################################################################<\/em><\/code>'''Display results'''<\/code>
          <\/code>fig = plt.figure(figsize=(15,20))<\/code>st = fig.suptitle(\"Transformer + TimeEmbedding Model\", fontsize=22)<\/code>st.set_y(0.92)<\/code>
          <\/code>#Plot training data results<\/em><\/code>ax11 = fig.add_subplot(311)<\/code>ax11.plot(train_data[:, 3], label='IBM Closing Returns')<\/code>ax11.plot(np.arange(seq_len, train_pred.shape[0]+seq_len), train_pred, linewidth=3, label='Predicted IBM Closing Returns')<\/code>ax11.set_title(\"Training Data\", fontsize=18)<\/code>ax11.set_xlabel('Date')<\/code>ax11.set_ylabel('IBM Closing Returns')<\/code>ax11.legend(loc=\"best\", fontsize=12)<\/code>
          <\/code>#Plot validation data results<\/em><\/code>ax21 = fig.add_subplot(312)<\/code>ax21.plot(val_data[:, 3], label='IBM Closing Returns')<\/code>ax21.plot(np.arange(seq_len, val_pred.shape[0]+seq_len), val_pred, linewidth=3, label='Predicted IBM Closing Returns')<\/code>ax21.set_title(\"Validation Data\", fontsize=18)<\/code>ax21.set_xlabel('Date')<\/code>ax21.set_ylabel('IBM Closing Returns')<\/code>ax21.legend(loc=\"best\", fontsize=12)<\/code>
          <\/code>#Plot test data results<\/em><\/code>ax31 = fig.add_subplot(313)<\/code>ax31.plot(test_data[:, 3], label='IBM Closing Returns')<\/code>ax31.plot(np.arange(seq_len, test_pred.shape[0]+seq_len), test_pred, linewidth=3, label='Predicted IBM Closing Returns')<\/code>ax31.set_title(\"Test Data\", fontsize=18)<\/code>ax31.set_xlabel('Date')<\/code>ax31.set_ylabel('IBM Closing Returns')<\/code>ax31.legend(loc=\"best\", fontsize=12)<\/code><\/code><\/pre>\n\n\n\n
          Model metrics<\/strong><\/h3>\n\n\n\n
          '''Display model metrics'''<\/code>
          <\/code>fig = plt.figure(figsize=(15,20))<\/code>st = fig.suptitle(\"Transformer + TimeEmbedding Model Metrics\", fontsize=22)<\/code>st.set_y(0.92)<\/code>
          <\/code>#Plot model loss<\/em><\/code>ax1 = fig.add_subplot(311)<\/code>ax1.plot(history.history['loss'], label='Training loss (MSE)')<\/code>ax1.plot(history.history['val_loss'], label='Validation loss (MSE)')<\/code>ax1.set_title(\"Model loss\", fontsize=18)<\/code>ax1.set_xlabel('Epoch')<\/code>ax1.set_ylabel('Loss (MSE)')<\/code>ax1.legend(loc=\"best\", fontsize=12)<\/code>
          <\/code>#Plot MAE<\/em><\/code>ax2 = fig.add_subplot(312)<\/code>ax2.plot(history.history['mae'], label='Training MAE')<\/code>ax2.plot(history.history['val_mae'], label='Validation MAE')<\/code>ax2.set_title(\"Model metric - Mean average error (MAE)\", fontsize=18)<\/code>ax2.set_xlabel('Epoch')<\/code>ax2.set_ylabel('Mean average error (MAE)')<\/code>ax2.legend(loc=\"best\", fontsize=12)<\/code>
          <\/code>#Plot MAPE<\/em><\/code>ax3 = fig.add_subplot(313)<\/code>ax3.plot(history.history['mape'], label='Training MAPE')<\/code>ax3.plot(history.history['val_mape'], label='Validation MAPE')<\/code>ax3.set_title(\"Model metric - Mean average percentage error (MAPE)\", fontsize=18)<\/code>ax3.set_xlabel('Epoch')<\/code>ax3.set_ylabel('Mean average percentage error (MAPE)')<\/code>ax3.legend(loc=\"best\", fontsize=12)<\/code><\/code><\/pre>\n\n\n\n
          \"\"<\/figure>\n\n\n\n
          <\/p>\n\n\n\n
          Model architecture overview<\/strong><\/h3>\n\n\n\n
          tf.keras.utils.plot_model(
          model,
          to_file=\"IBM_Transformer+TimeEmbedding.png\",
          show_shapes=True,
          show_layer_names=True,
          expand_nested=True,
          dpi=96,)
          tf.keras.utils.plot_model(
          model,
          to_file=\"IBM_Transformer+TimeEmbedding.png\",
          show_shapes=True,
          show_layer_names=True,
          expand_nested=True,
          dpi=96,)<\/code><\/pre>\n\n\n\n
          Moving Average<\/strong><\/h2>\n\n\n\n
          Moving Average – Load IBM data again, to apply rolling window<\/strong><\/h3>\n\n\n\n
          IBM_path = 'IBM.csv'<\/code>
          <\/code>df = pd.read_csv(IBM_path, delimiter=',', usecols=['Date', 'Open', 'High', 'Low', 'Close', 'Volume'])<\/code>
          <\/code># Replace 0 to avoid dividing by 0 later on<\/em><\/code>df['Volume'].replace(to_replace=0, method='ffill', inplace=True) <\/code>df.sort_values('Date', inplace=True)<\/code>
          <\/code># Apply moving average with a window of 10 days to all columns<\/em><\/code>df[['Open', 'High', 'Low', 'Close', 'Volume']] = df[['Open', 'High', 'Low', 'Close', 'Volume']].rolling(10).mean() <\/code>
          <\/code># Drop all rows with NaN values<\/em><\/code>df.dropna(how='any', axis=0, inplace=True) <\/code>df.head()<\/code><\/code><\/pre>\n\n\n\n
          \"\"<\/figure>\n\n\n\n
          <\/p>\n\n\n\n
          Moving Average – Plot daily IBM closing prices and volume<\/strong><\/h3>\n\n\n\n