mirrors/tinygrad
2
0
Fork 0
mirror of https://github.com/tinygrad/tinygrad.git synced 2026-06-24 02:14:17 +00:00
Code Issues Projects Releases Packages Wiki Activity

hotfix: fix dataset reading for new llm.c

Browse source
This commit is contained in:
George Hotz 2024-06-03 14:03:16 +02:00
commit eecfdd2f6e
1 changed files with 3 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

5
examples/llm.c/train_gpt2.py
View file

@ -142,7 +142,8 @@ if __name__ == "__main__":
assert os.path.isfile(tokens_bin)
print(f"loading cached tokens in {tokens_bin}")
with open(tokens_bin, "rb") as f:
tokens = np.frombuffer(f.read(), dtype=np.int32)
f.seek(0x400)
tokens = np.frombuffer(f.read(), dtype=np.uint16).astype(np.int32)
tokens = Tensor(tokens)
# lightweight dataloader
@ -161,7 +162,7 @@ if __name__ == "__main__":
# forward backward for a few iterations
data_iter = iter(get_batch())
x, y = next(data_iter) # we'll overfit this batch below
optimizer = nn.optim.Adam(nn.state.get_parameters(model), lr=1e-4)
optimizer = nn.optim.AdamW(nn.state.get_parameters(model), lr=1e-4, weight_decay=0)
@TinyJit
def step(x, y):