-
Notifications
You must be signed in to change notification settings - Fork 16
/
Copy pathff_cplx.py
39 lines (34 loc) · 1.04 KB
/
ff_cplx.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
#!/usr/bin/env python
# coding=utf-8
import torch
import torch.nn as nn
import torch.nn.functional as F
from linear_cplx import Complex_Linear
EPSILON = torch.finfo(torch.float32).eps
class FF_Cplx(nn.Module):
def __init__(self,
in_dim,
hidden_dim):
super(FF_Cplx, self).__init__()
self.layernorm_linear = nn.LayerNorm(in_dim)
self.linear1 = Complex_Linear(in_dim, hidden_dim)
self.linear2 = Complex_Linear(hidden_dim, in_dim)
self.prelu = nn.PReLU()
self.dropout = nn.Dropout(p=0.1)
def forward(self, x):
# N C F T 2
y = self.layernorm_linear(x.transpose(1, 4)).transpose(1, 4)
y = y.transpose(1, 3)
y = self.linear1(y)
y = self.prelu(y)
y = self.dropout(y)
y = self.linear2(y)
y = self.dropout(y)
y = y.transpose(1, 3)
y = y + x
return y
if __name__ == '__main__':
net = FF_Cplx(128, 64)
inputs = torch.ones([10, 128, 4, 397, 2])
y = net(inputs)
print(y.shape)