91 lines
2.8 KiB
Python
91 lines
2.8 KiB
Python
import torch
|
|
from contextlib import contextmanager
|
|
|
|
class Linear(torch.nn.Linear):
|
|
def reset_parameters(self):
|
|
return None
|
|
|
|
class Conv2d(torch.nn.Conv2d):
|
|
def reset_parameters(self):
|
|
return None
|
|
|
|
class Conv3d(torch.nn.Conv3d):
|
|
def reset_parameters(self):
|
|
return None
|
|
|
|
class GroupNorm(torch.nn.GroupNorm):
|
|
def reset_parameters(self):
|
|
return None
|
|
|
|
class LayerNorm(torch.nn.LayerNorm):
|
|
def reset_parameters(self):
|
|
return None
|
|
|
|
def conv_nd(dims, *args, **kwargs):
|
|
if dims == 2:
|
|
return Conv2d(*args, **kwargs)
|
|
elif dims == 3:
|
|
return Conv3d(*args, **kwargs)
|
|
else:
|
|
raise ValueError(f"unsupported dimensions: {dims}")
|
|
|
|
def cast_bias_weight(s, input):
|
|
bias = None
|
|
if s.bias is not None:
|
|
bias = s.bias.to(device=input.device, dtype=input.dtype)
|
|
weight = s.weight.to(device=input.device, dtype=input.dtype)
|
|
return weight, bias
|
|
|
|
class manual_cast:
|
|
class Linear(Linear):
|
|
def forward(self, input):
|
|
weight, bias = cast_bias_weight(self, input)
|
|
return torch.nn.functional.linear(input, weight, bias)
|
|
|
|
class Conv2d(Conv2d):
|
|
def forward(self, input):
|
|
weight, bias = cast_bias_weight(self, input)
|
|
return self._conv_forward(input, weight, bias)
|
|
|
|
class Conv3d(Conv3d):
|
|
def forward(self, input):
|
|
weight, bias = cast_bias_weight(self, input)
|
|
return self._conv_forward(input, weight, bias)
|
|
|
|
class GroupNorm(GroupNorm):
|
|
def forward(self, input):
|
|
weight, bias = cast_bias_weight(self, input)
|
|
return torch.nn.functional.group_norm(input, self.num_groups, weight, bias, self.eps)
|
|
|
|
class LayerNorm(LayerNorm):
|
|
def forward(self, input):
|
|
weight, bias = cast_bias_weight(self, input)
|
|
return torch.nn.functional.layer_norm(input, self.normalized_shape, weight, bias, self.eps)
|
|
|
|
@classmethod
|
|
def conv_nd(s, dims, *args, **kwargs):
|
|
if dims == 2:
|
|
return s.Conv2d(*args, **kwargs)
|
|
elif dims == 3:
|
|
return s.Conv3d(*args, **kwargs)
|
|
else:
|
|
raise ValueError(f"unsupported dimensions: {dims}")
|
|
|
|
@contextmanager
|
|
def use_comfy_ops(device=None, dtype=None): # Kind of an ugly hack but I can't think of a better way
|
|
old_torch_nn_linear = torch.nn.Linear
|
|
force_device = device
|
|
force_dtype = dtype
|
|
def linear_with_dtype(in_features: int, out_features: int, bias: bool = True, device=None, dtype=None):
|
|
if force_device is not None:
|
|
device = force_device
|
|
if force_dtype is not None:
|
|
dtype = force_dtype
|
|
return Linear(in_features, out_features, bias=bias, device=device, dtype=dtype)
|
|
|
|
torch.nn.Linear = linear_with_dtype
|
|
try:
|
|
yield
|
|
finally:
|
|
torch.nn.Linear = old_torch_nn_linear
|