【AReLU: Attention-based-Rectified-Linear-Unit】tensorflow 实现arelu激活函数

import tensorflow as tf class TfModel(object): def __init__(self, max_len=10, vocab_size=500, embedding_size=100, alpha=0.90, beta=2.0): self.max_len = max_len self.vocab_size = vocab_size self.embedding_size = embedding_size self.alpha = alpha self.beta = beta self.build_model() def build_model(self): self.x = tf.placeholder(tf.float32, [None, self.max_len], name="x") self.y = tf.placeholder(tf.float32, [None, self.max_len], name="y") hidden_output = tf.layers.dense(self.x, units=10) # 进入arelu激活函数 self.alpha_tensor = tf.Variable([self.alpha], dtype=tf.float32) self.beta_tensor = tf.Variable([self.beta], dtype=tf.float32) self.alpha_tensor = tf.clip_by_value(self.alpha_tensor, clip_value_min=0.01, clip_value_max=0.99) self.beta_tensor = 1 + tf.nn.sigmoid(self.beta_tensor) hidden_action = tf.nn.relu(hidden_output) * self.beta_tensor - tf.nn.relu(-hidden_output) * self.alpha_tensor # arelu激活函数结束 logits = tf.layers.dense(hidden_action, units=1) self.loss = tf.reduce_mean(tf.losses.mean_squared_error(self.y, logits)) optimizer = tf.train.AdamOptimizer() self.train_op = optimizer.minimize(self.loss) model = TfModel(max_len=1) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) x = [[1], [2], [3]] y = [[14], [25], [36]] for i in range(1000): _, loss = sess.run([model.train_op, model.loss], {model.x: x, model.y: y}) print("index: {}, loss: {}".format(i, loss)) print(sess.run([model.alpha_tensor, model.beta_tensor]))