-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathmodel.py
More file actions
199 lines (152 loc) · 10.1 KB
/
Copy pathmodel.py
File metadata and controls
199 lines (152 loc) · 10.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
import torch
import torch.nn.functional as F
import torch.optim as optim
import torch.nn as nn
from torch.autograd import Variable
from torch.utils.data import Dataset,DataLoader
import math
from typing import Optional, Any, Union, Callable,Tuple
from torch import Tensor
import copy
from collections import OrderedDict
from transformer import *
class MM4Rec(nn.Module):
def __init__(self,pad_id,d_model, nhead,n_section, num_encoder_layers,ff_dim,dropout,num_expert,llm_size,add_PE):
super(MM4Rec, self).__init__()
self.pad_id =pad_id
self.d_model = d_model
self.nhead = nhead
self.n_section = n_section
self.num_encoder_layers = num_encoder_layers
self.num_expert = num_expert
self.ff_dim = ff_dim
self.llm_size=llm_size
self.dropout = dropout
self.add_PE = add_PE
self.auxiliary_layer1 = nn.Linear(d_model, 2)
self.linear=nn.Linear(self.llm_size,d_model)
self.cos = nn.CosineSimilarity(dim=2, eps=1e-6)
self.sigmoid = nn.Sigmoid()
self.softmax = nn.Softmax(dim=1)
self.scenario_embbeddings = [nn.Embedding(2, d_model)]
self.source_embbeddings = [nn.Embedding(2, d_model)]
self.embbeddings = [
nn.Embedding(26, d_model),# hour
nn.Embedding(9, d_model),# weekday
nn.Embedding(500, d_model),# click_time delta
nn.Embedding(5, d_model),# charge mode
nn.Embedding(23, d_model),# price unit
nn.Embedding(self.n_section, d_model)]# section
self.TIL= SATE(src_pad_idx=pad_id,d_model=d_model, n_head=nhead,ffn_hidden=ff_dim, num_encoder_layers=num_encoder_layers, drop_prob=dropout)
self.CIL= SATE(src_pad_idx=pad_id,d_model=d_model, n_head=nhead,ffn_hidden=ff_dim, num_encoder_layers=num_encoder_layers, drop_prob=dropout)
self.expert_module = nn.ModuleList()
expert = nn.Sequential(
nn.Linear(d_model*2,d_model*2),
nn.ReLU(),
nn.Dropout(p=dropout),
nn.Linear(d_model*2,d_model))
for i in range(num_expert):
self.expert_module.add_module('Expert'+str(i+1), expert)
self.trend_attention = MultiHeadAttention(d_model=d_model, n_head=nhead)
self.project_layer = nn.Sequential(
nn.Linear(d_model,d_model),
)
self.gate_sc1 = nn.Sequential(
nn.Linear(d_model*3,d_model),
nn.ReLU(),
nn.Linear(d_model, num_expert))
self.gate_sc2 = nn.Sequential(
nn.Linear(d_model*3,d_model),
nn.ReLU(),
nn.Linear(d_model, num_expert))
self.tower = nn.Sequential(
nn.Linear(d_model*2,d_model),
)
self.gate_trend = nn.Linear(d_model*2, 1)
if self.add_PE == True:
self.pos_encoder = PositionalEmbedding(max_len=500,embedding_dim=d_model,mode='MODE_ADD')
def make_pad_mask(self, q, k, q_pad_idx, k_pad_idx):
len_q, len_k = q.size(1), k.size(1)
k = k.ne(k_pad_idx).unsqueeze(1).unsqueeze(2)
k = k.repeat(1, 1, len_q, 1)
q = q.ne(q_pad_idx).unsqueeze(1).unsqueeze(3)
q = q.repeat(1, 1, 1, len_k)
mask = k & q
return mask
def aggregation(self, input_seq):
return torch.mean(input_seq, dim=1, keepdim=True)
def fusion(self, input_seq1,input_seq2):
return input_seq1 + input_seq2
def forward(self,click_seq,click_seq_one,click_time_hour_seq,click_time_weekday_seq,click_time_delta_seq,\
scenario_push,scenario_browse,source_ads,source_news,pop_click_seq,pop_click_seq_one,target,target_one,ns_seq,ns_seq_one):
# shared embedding layer
source_emb = self.source_embbeddings[0](source_ads.type(torch.LongTensor)).to(device)
scenario_emb = self.scenario_embbeddings[0](scenario_push.type(torch.LongTensor)).to(device)
# temporal
click_time_h_emb = self.embbeddings[0](click_time_hour_seq.type(torch.LongTensor)).to(device) #bs,seq_len,128
click_time_w_emb = self.embbeddings[1](click_time_weekday_seq.type(torch.LongTensor)).to(device)
click_time_delta_emb = self.embbeddings[2](click_time_delta_seq.type(torch.LongTensor)).to(device)
time_emb = click_time_h_emb + click_time_w_emb + click_time_delta_emb
src_time_emb_ = time_emb.permute(1, 0, 2).type(torch.float)
# text
text_emb = self.linear(click_seq_one[:,:,0:self.llm_size]) #print(x_one.size()) bs,20,816
charge_mode_emb = self.embbeddings[3](click_seq_one[:,:,-3].type(torch.LongTensor)).to(device)
unit_price_emb = self.embbeddings[4](click_seq_one[:,:,-2].type(torch.LongTensor)).to(device)
section_emb = self.embbeddings[5](click_seq_one[:,:,-1].type(torch.LongTensor)).to(device)
context_emb = text_emb + unit_price_emb + charge_mode_emb + section_emb
src_context_emb_ = context_emb.permute(1, 0, 2).type(torch.float)
# add positional encoding
if self.add_PE == True:
src_context_emb = self.pos_encoder(src_context_emb_).permute(1, 0, 2)
src_time_emb = self.pos_encoder(src_time_emb_).permute(1, 0, 2)
else:
src_context_emb = src_context_emb_.permute(1, 0, 2)
src_time_emb = src_time_emb_.permute(1, 0, 2)
# learner
tran_out_context = self.CIL(src_context_emb,source_emb.unsqueeze(1) ,source_ads,click_seq) #torch.Size([bs, 20, 256])
tran_out_time = self.TIL(src_time_emb ,source_emb.unsqueeze(1) ,source_ads,click_seq) #torch.Size([bs, 20, 256])
seq_emb_ = torch.cat([self.aggregation(tran_out_context),self.aggregation(tran_out_time)],2)
#self.fusion(self.aggregation(tran_out_context),self.aggregation(tran_out_time))
w_push = self.softmax(self.gate_sc1(torch.cat([scenario_emb,seq_emb_.squeeze(1)],1)))
w_browse = self.softmax(self.gate_sc2(torch.cat([scenario_emb,seq_emb_.squeeze(1)],1)))
# Expert
expert_output=[]
for i, expert in enumerate(self.expert_module):
expert_output.append(expert(seq_emb_))
seq_emb_push = torch.stack([w_push[:, i].unsqueeze(1).unsqueeze(2) * expert_output[i] for i in range(w_push.size()[1])]).sum(dim=0)
seq_emb_browse = torch.stack([w_browse[:, i].unsqueeze(1).unsqueeze(2) * expert_output[i] for i in range(w_browse.size()[1])]).sum(dim=0)
seq_emb = self.tower(torch.cat([seq_emb_push.squeeze(1),seq_emb_browse.squeeze(1)],1)).unsqueeze(1)
seq_emb_score = self.sigmoid(self.auxiliary_layer1(seq_emb.squeeze(1)))
# pop trend
p_text_emb = self.linear(pop_click_seq_one[:,:,0:self.llm_size]) #print(x_one.size()) bs,20,816
p_charge_mode_emb = self.embbeddings[3](pop_click_seq_one[:,:,-3].type(torch.LongTensor)).to(device)
p_unit_price_emb = self.embbeddings[4](pop_click_seq_one[:,:,-2].type(torch.LongTensor)).to(device)
p_section_emb = self.embbeddings[5](pop_click_seq_one[:,:,-1].type(torch.LongTensor)).to(device)
p_context_emb = p_text_emb + p_unit_price_emb + p_charge_mode_emb + p_section_emb
p_mask = self.make_pad_mask(pop_click_seq, pop_click_seq, self.pad_id, self.pad_id)
p_mask = p_mask[:,:,-1,:].unsqueeze(2)
trend_out = self.trend_attention(q=p_context_emb, k=p_context_emb, v=p_context_emb, mask=p_mask)
trend_out = torch.mean(trend_out,dim=1,keepdim=True)
# gating control the weight
gate_weights = self.sigmoid(self.gate_trend(torch.cat([seq_emb,trend_out],2)))
seq_emb = (gate_weights * seq_emb) + ((1-gate_weights) * trend_out)
# target
target_text_emb = self.linear(target_one[:,:,0: self.llm_size])
t_charge_mode_emb = self.embbeddings[3](target_one[:,:,-3].type(torch.LongTensor)).to(device)
t_unit_price_emb = self.embbeddings[4](target_one[:,:,-2].type(torch.LongTensor)).to(device)
t_section_emb = self.embbeddings[5](target_one[:,:,-1].type(torch.LongTensor)).to(device)
target_emb = target_text_emb + t_unit_price_emb + t_charge_mode_emb + t_section_emb
# neg_sample
ns_text_emb = self.linear(ns_seq_one[:,:,0: self.llm_size]) #print(x_one.size()) bs,99,816
ns_charge_mode_emb = self.embbeddings[3](ns_seq_one[:,:,-3].type(torch.LongTensor)).to(device)
ns_unit_price_emb = self.embbeddings[4](ns_seq_one[:,:,-2].type(torch.LongTensor)).to(device)
ns_section_emb = self.embbeddings[5](ns_seq_one[:,:,-1].type(torch.LongTensor)).to(device)
ns_emb = ns_text_emb + ns_unit_price_emb + ns_charge_mode_emb + ns_section_emb
# target + neg_sample
emb = torch.cat([target_emb,ns_emb],1) #torch.Size([64, 100, 128])
# shared space
emb=self.project_layer(emb)
# similarity
similarity = self.cos(seq_emb, emb) # (bs,100)
similarity = (similarity + 1) / 2
return similarity