allow for overriding alpha as well on forward in distillation wrapper

README.md

@@ -83,8 +83,7 @@ v = DistillableViT(
     heads = 8,
     mlp_dim = 2048,
     dropout = 0.1,
-    emb_dropout = 0.1,
-    pool = 'mean'
+    emb_dropout = 0.1
 )
 
 distiller = DistillWrapper(

setup.py

@@ -3,7 +3,7 @@ from setuptools import setup, find_packages
 setup(
   name = 'vit-pytorch',
   packages = find_packages(exclude=['examples']),
-  version = '0.6.0',
+  version = '0.6.2',
   license='MIT',
   description = 'Vision Transformer (ViT) - Pytorch',
   author = 'Phil Wang',

vit_pytorch/distill.py

@@ -68,11 +68,14 @@ class DistillWrapper(nn.Module):
             nn.Linear(dim, num_classes)
         )
 
-    def forward(self, img, labels, temperature = None, **kwargs):
-        b, *_, alpha = *img.shape, self.alpha
+    def forward(self, img, labels, temperature = None, alpha = None, **kwargs):
+        b, *_ = img.shape
+        alpha = alpha if exists(alpha) else self.alpha
         T = temperature if exists(temperature) else self.temperature
 
-        teacher_logits = self.teacher(img)
+        with torch.no_grad():
+            teacher_logits = self.teacher(img)
+
         student_logits, distill_tokens = self.student(img, distill_token = self.distillation_token, **kwargs)
         distill_logits = self.distill_mlp(distill_tokens)