diff --git a/setup.py b/setup.py index 2b08364..ca864bc 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ from setuptools import setup, find_packages setup( name = 'vit-pytorch', packages = find_packages(exclude=['examples']), - version = '0.26.4', + version = '0.26.5', license='MIT', description = 'Vision Transformer (ViT) - Pytorch', author = 'Phil Wang', diff --git a/vit_pytorch/crossformer.py b/vit_pytorch/crossformer.py index 401aa4b..bc7c78a 100644 --- a/vit_pytorch/crossformer.py +++ b/vit_pytorch/crossformer.py @@ -62,9 +62,9 @@ class LayerNorm(nn.Module): self.b = nn.Parameter(torch.zeros(1, dim, 1, 1)) def forward(self, x): - std = torch.var(x, dim = 1, unbiased = False, keepdim = True).sqrt() + var = torch.var(x, dim = 1, unbiased = False, keepdim = True) mean = torch.mean(x, dim = 1, keepdim = True) - return (x - mean) / (std + self.eps) * self.g + self.b + return (x - mean) / (var + self.eps).sqrt() * self.g + self.b def FeedForward(dim, mult = 4, dropout = 0.): return nn.Sequential( diff --git a/vit_pytorch/cvt.py b/vit_pytorch/cvt.py index 6ac0827..62406ec 100644 --- a/vit_pytorch/cvt.py +++ b/vit_pytorch/cvt.py @@ -30,9 +30,9 @@ class LayerNorm(nn.Module): # layernorm, but done in the channel dimension #1 self.b = nn.Parameter(torch.zeros(1, dim, 1, 1)) def forward(self, x): - std = torch.var(x, dim = 1, unbiased = False, keepdim = True).sqrt() + var = torch.var(x, dim = 1, unbiased = False, keepdim = True) mean = torch.mean(x, dim = 1, keepdim = True) - return (x - mean) / (std + self.eps) * self.g + self.b + return (x - mean) / (var + self.eps).sqrt() * self.g + self.b class PreNorm(nn.Module): def __init__(self, dim, fn): diff --git a/vit_pytorch/nest.py b/vit_pytorch/nest.py index 47ee6ae..77edbec 100644 --- a/vit_pytorch/nest.py +++ b/vit_pytorch/nest.py @@ -20,9 +20,9 @@ class LayerNorm(nn.Module): self.b = nn.Parameter(torch.zeros(1, dim, 1, 1)) def forward(self, x): - std = torch.var(x, dim = 1, unbiased = False, keepdim = True).sqrt() + var = torch.var(x, dim = 1, unbiased = False, keepdim = True) mean = torch.mean(x, dim = 1, keepdim = True) - return (x - mean) / (std + self.eps) * self.g + self.b + return (x - mean) / (var + self.eps).sqrt() * self.g + self.b class PreNorm(nn.Module): def __init__(self, dim, fn): diff --git a/vit_pytorch/twins_svt.py b/vit_pytorch/twins_svt.py index 76eafe5..ec27cc2 100644 --- a/vit_pytorch/twins_svt.py +++ b/vit_pytorch/twins_svt.py @@ -38,9 +38,9 @@ class LayerNorm(nn.Module): self.b = nn.Parameter(torch.zeros(1, dim, 1, 1)) def forward(self, x): - std = torch.var(x, dim = 1, unbiased = False, keepdim = True).sqrt() + var = torch.var(x, dim = 1, unbiased = False, keepdim = True) mean = torch.mean(x, dim = 1, keepdim = True) - return (x - mean) / (std + self.eps) * self.g + self.b + return (x - mean) / (var + self.eps).sqrt() * self.g + self.b class PreNorm(nn.Module): def __init__(self, dim, fn):