Update activator_only_GEGLU.py

activator_only_GEGLU.py

@@ -2,23 +2,6 @@ import torch
 from torch import nn
 
 
-
-
-
-"""class FeedForward(nn.Module):
-    def __init__(self, dim, hidden_dim, dropout):
-        super().__init__()
-        self.net = nn.Sequential(
-            nn.Linear(dim, hidden_dim),
-            nn.GELU(),
-            nn.Dropout(dropout),
-            nn.Linear(hidden_dim, dim),
-            nn.Dropout(dropout)
-        )
-    def forward(self, x):
-        return self.net(x)"""
-
-
    
 
 
@@ -54,16 +37,13 @@ class ActivatorBlock(nn.Module):
        
         self.norm = nn.LayerNorm(d_model)       
         self.actgu = ActivatorGatingUnit(d_model, d_ffn)
-        #self.ffn = FeedForward(d_model,d_ffn,dropout)
+        
     def forward(self, x):
         residual = x
         x = self.norm(x)
         x = self.actgu(x)           
         x = x + residual      
-        #residual = x
-        #x = self.norm(x)
-        #x = self.ffn(x)        
-        #out = x + residual
+        
         out = x
         return out