File tree Expand file tree Collapse file tree 1 file changed +44
-0
lines changed
Expand file tree Collapse file tree 1 file changed +44
-0
lines changed Original file line number Diff line number Diff line change @@ -14,6 +14,50 @@ $ pip install transfusion-pytorch
1414
1515## Usage
1616
17+ One modality, say images
18+
19+ ``` python
20+ import torch
21+ from transfusion_pytorch import Transfusion
22+
23+ model = Transfusion(
24+ num_text_tokens = 256 ,
25+ dim_latent = 192 ,
26+ transformer = dict (
27+ dim = 512 ,
28+ depth = 8
29+ )
30+ )
31+
32+ text_ids = torch.randint(0 , 256 , (2 , 1024 ))
33+
34+ modality_tokens = [[
35+ torch.randn(6 , 192 ),
36+ torch.randn(4 , 192 )
37+ ], [
38+ torch.randn(5 , 192 ),
39+ torch.randn(3 , 192 )
40+ ]]
41+
42+ modality_positions = [[
43+ (2 , 6 ),
44+ (10 , 4 )
45+ ], [
46+ (2 , 5 ),
47+ (10 , 3 )
48+ ]] # (offset, length)
49+
50+ loss, breakdown = model(
51+ text_ids,
52+ modality_tokens = modality_tokens,
53+ modality_positions = modality_positions
54+ )
55+
56+ loss.backward()
57+ ```
58+
59+ Multiple modalities
60+
1761``` python
1862import torch
1963from transfusion_pytorch import Transfusion
You can’t perform that action at this time.
0 commit comments