You move your model to the GPU, start training, and immediately hit:
RuntimeError: Expected all tensors to be on the same device, but found at least two devices, cuda:0 and cpu!
Or in some PyTorch versions:
RuntimeError: Tensors must be on the same device to be operated on, got devices: cuda:0 and cpu
model.cuda() looks like it should cover everything, but it only moves the model's own parameters. Your input tensors, label tensors, and anything you create inside a forward pass stay wherever they started, usually CPU, until you move them too. Five causes account for nearly every occurrence of this error, roughly in order of how often you'll actually hit them.
import torch
import torch.nn as nn
class SimpleNet(nn.Module):
def __init__(self):
super().__init__()
self.fc = nn.Linear(10, 2)
def forward(self, x):
return self.fc(x)
model = SimpleNet()
model.cuda() # model weights are now on cuda:0
# Input tensor is still on CPU (default)
x = torch.randn(32, 10) # cpu
# RuntimeError: Expected all tensors to be on the same device
output = model(x)
model.cuda() moves the model's registered parameters and buffers to the GPU. Every tensor you create with torch.randn(), torch.zeros(), or from a DataLoader defaults to CPU, and PyTorch has no way to multiply a CPU tensor by a CUDA weight matrix since they live in different memory spaces entirely.
Define a device variable once and move every tensor explicitly with .to(device). Avoid hardcoding .cuda() directly, since it crashes outright on any machine without a GPU:
import torch
import torch.nn as nn
from torch.utils.data import DataLoader, TensorDataset
# The canonical device idiom, use this everywhere
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print(f"Using device: {device}")
class SimpleNet(nn.Module):
def __init__(self):
super().__init__()
self.fc = nn.Linear(10, 2)
def forward(self, x):
return self.fc(x)
model = SimpleNet().to(device) # move model
# Move each batch in the training loop
X = torch.randn(200, 10)
y = torch.randint(0, 2, (200,))
dataset = TensorDataset(X, y)
loader = DataLoader(dataset, batch_size=32)
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
for inputs, labels in loader:
inputs = inputs.to(device) # move inputs
labels = labels.to(device) # move labels
optimizer.zero_grad()
outputs = model(inputs)
loss = criterion(outputs, labels)
loss.backward()
optimizer.step()
The .to(device) call is a no-op when the tensor is already on the right device, so it is safe to call unconditionally every iteration.
import torch
import torch.nn as nn
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = nn.Linear(10, 3).to(device)
criterion = nn.CrossEntropyLoss()
inputs = torch.randn(16, 10).to(device) # correctly moved
labels = torch.randint(0, 3, (16,)) # forgotten, still on CPU
outputs = model(inputs) # outputs is on cuda:0
loss = criterion(outputs, labels)
# RuntimeError: Expected all tensors to be on the same device
# found at least two devices, cuda:0 and cpu
nn.CrossEntropyLoss, nn.MSELoss, and every other loss function compute a scalar from two tensors: predictions and targets. If the model output landed on GPU but the label tensor was never moved, PyTorch raises the device mismatch error inside the loss function, not inside the forward pass. This surprises people who correctly moved their inputs but forgot that labels are a separate tensor.
import torch
import torch.nn as nn
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = nn.Linear(10, 3).to(device)
criterion = nn.CrossEntropyLoss()
# Move BOTH inputs and labels in one unpacking line
inputs = torch.randn(16, 10)
labels = torch.randint(0, 3, (16,))
inputs, labels = inputs.to(device), labels.to(device)
outputs = model(inputs)
loss = criterion(outputs, labels) # both on same device, works
print(f"Loss: {loss.item():.4f}")
In a real training loop with nested batches (dicts, lists of tensors), it's easy to move the top-level tensor and miss one buried inside. This recursive helper catches all of them:
def to_device(batch, device):
"""Recursively move a batch (tensor, list, dict, tuple) to device."""
if isinstance(batch, torch.Tensor):
return batch.to(device)
if isinstance(batch, (list, tuple)):
return type(batch)(to_device(b, device) for b in batch)
if isinstance(batch, dict):
return {k: to_device(v, device) for k, v in batch.items()}
return batch
for batch in loader:
batch = to_device(batch, device)
inputs, labels = batch
# both guaranteed on correct device
import torch
import torch.nn as nn
class AttentionModel(nn.Module):
def __init__(self, hidden_size):
super().__init__()
self.hidden_size = hidden_size
self.linear = nn.Linear(hidden_size, hidden_size)
def forward(self, x):
batch_size = x.size(0)
# BUG: torch.zeros always creates a CPU tensor
# even when x is on cuda:0
mask = torch.zeros(batch_size, self.hidden_size)
# RuntimeError: x is cuda:0, mask is cpu
return self.linear(x) + mask
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = AttentionModel(64).to(device)
x = torch.randn(8, 64).to(device)
output = model(x) # crashes
Tensor factory functions like torch.zeros(), torch.ones(), torch.randn(), and torch.arange() all default to CPU regardless of which device the surrounding model is on. Call one of them inside forward() and you get a CPU tensor sitting next to a GPU model, even though the model itself was moved correctly.
Pass device=x.device to the factory call so the new tensor inherits whatever device the input is actually on. That's the pattern to default to, it stays correct during CPU-only runs and across multiple GPUs without any extra logic:
import torch
import torch.nn as nn
class AttentionModel(nn.Module):
def __init__(self, hidden_size):
super().__init__()
self.hidden_size = hidden_size
self.linear = nn.Linear(hidden_size, hidden_size)
def forward(self, x):
batch_size = x.size(0)
# Fix 1: pass device= explicitly, inferred from input tensor
mask = torch.zeros(batch_size, self.hidden_size, device=x.device)
# Fix 2: create on CPU then move (less efficient, but also works)
# mask = torch.zeros(batch_size, self.hidden_size).to(x.device)
# Fix 3: use torch.zeros_like to match device AND dtype of x
# (only use when shape matches)
# mask = torch.zeros_like(x)
return self.linear(x) + mask
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = AttentionModel(64).to(device)
x = torch.randn(8, 64).to(device)
output = model(x) # works
print(output.device) # cuda:0
Anything else you compute inside forward() and expect to combine with the input, position encodings, attention masks, a padding tensor, needs the same treatment.
import torch
import torch.nn as nn
# Only relevant when you have 2+ GPUs
if torch.cuda.device_count() < 2:
print("This example requires at least 2 GPUs")
else:
model = nn.Linear(10, 2)
# DataParallel splits the model across all visible GPUs
# The "main" copy lives on cuda:0
model = nn.DataParallel(model)
model = model.cuda() # places on cuda:0
# BUG: tensor explicitly placed on cuda:1
x = torch.randn(16, 10).to("cuda:1")
# RuntimeError: module must have its parameters and buffers
# on device cuda:0 (device_ids[0]) but found one of them on device: cuda:1
output = model(x)
With nn.DataParallel, PyTorch distributes each batch across all GPUs but expects inputs to arrive on cuda:0 (the first device in device_ids). It then scatters the batch to the other GPUs internally. If you manually place your input on a different GPU, the scatter step fails. A related issue: if you save a tensor on cuda:1 (e.g., from a previous step) and pass it to a model on cuda:0, you get the same error.
import torch
import torch.nn as nn
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model = nn.Linear(10, 2)
if torch.cuda.device_count() > 1:
print(f"Using {torch.cuda.device_count()} GPUs")
# Specify device_ids explicitly so you know which is the primary
model = nn.DataParallel(model, device_ids=[0, 1])
model = model.to(device)
# Always send inputs to the PRIMARY device (device_ids[0])
x = torch.randn(16, 10).to(device) # to cuda:0, DataParallel scatters from here
output = model(x)
print(output.shape)
If you're setting up multi-GPU training today, skip DataParallel and go straight to torch.nn.parallel.DistributedDataParallel (DDP). It's more efficient, and device placement is explicit rather than something the framework infers: each process owns exactly one GPU, so there's no scatter step to get wrong in the first place.
import torch
import torch.nn as nn
import torch.distributed as dist
from torch.nn.parallel import DistributedDataParallel as DDP
# In a DDP training script each worker sets its own rank
# torchrun sets LOCAL_RANK automatically
import os
local_rank = int(os.environ.get("LOCAL_RANK", 0))
torch.cuda.set_device(local_rank)
device = torch.device(f"cuda:{local_rank}")
model = nn.Linear(10, 2).to(device)
# dist.init_process_group would be called before this in real usage
# model = DDP(model, device_ids=[local_rank])
x = torch.randn(16, 10).to(device) # matches this process's GPU
output = model(x)
Sometimes none of the four causes above match, or the error happens deep inside a large model where the traceback doesn't say which tensor is misplaced. These two snippets narrow it down fast:
import torch
import torch.nn as nn
# Inspect all model parameters and buffers
def audit_model_devices(model):
print("=== Model parameter devices ===")
for name, param in model.named_parameters():
print(f" {name}: {param.device} (dtype={param.dtype})")
print("=== Model buffer devices ===")
for name, buf in model.named_buffers():
print(f" {name}: {buf.device} (dtype={buf.dtype})")
model = nn.Sequential(
nn.Linear(10, 20),
nn.ReLU(),
nn.Linear(20, 2)
)
# Move only part of the model accidentally
model[0] = model[0].cuda()
# model[2] is still on CPU
audit_model_devices(model)
# === Model parameter devices ===
# 0.weight: cuda:0 (dtype=torch.float32)
# 0.bias: cuda:0 (dtype=torch.float32)
# 2.weight: cpu (dtype=torch.float32) ← mismatch
# 2.bias: cpu (dtype=torch.float32) ← mismatch
import torch
# Inspect a batch from your DataLoader
def audit_batch_devices(batch):
if isinstance(batch, torch.Tensor):
print(f" tensor: shape={tuple(batch.shape)}, device={batch.device}, dtype={batch.dtype}")
elif isinstance(batch, (list, tuple)):
for i, item in enumerate(batch):
print(f" [{i}]", end=" ")
audit_batch_devices(item)
elif isinstance(batch, dict):
for k, v in batch.items():
print(f" '{k}':", end=" ")
audit_batch_devices(v)
# Example: NLP batch with mixed devices
batch = {
"input_ids": torch.randint(0, 1000, (8, 128)).cuda(),
"attention_mask": torch.ones(8, 128), # still on CPU
"labels": torch.randint(0, 2, (8,)), # still on CPU
}
audit_batch_devices(batch)
# 'input_ids': tensor: shape=(8, 128), device=cuda:0, dtype=torch.int64
# 'attention_mask': tensor: shape=(8, 128), device=cpu, dtype=torch.float32 ← wrong
# 'labels': tensor: shape=(8,), device=cpu, dtype=torch.int64 ← wrong
One idiom covers the model and the training loop:
device = torch.device("cuda" if torch.cuda.is_available() else "cpu"),
then model.to(device) once and tensor.to(device) on
every input and label, every iteration. Inside forward(),
factory calls need device=x.device explicitly, since they don't
inherit it on their own. If you're on DataParallel, inputs go
to cuda:0 specifically, not just "a GPU". When none of that
points at the bug, model.named_parameters() plus the recursive
batch-auditing function above will usually surface the mismatched tensor
within a minute.