Coverage for hopwise/model/context_aware_recommender/dcnv2.py: 63%
100 statements
« prev ^ index » next coverage.py v7.16.2, created at 2026-09-30 13:25 +0000
1# @Time : 2022/8/29
2# @Author : Yifan Li
3# @Email : 295435096@qq.com
5r"""DCN V2
6################################################
7Reference:
8 Ruoxi Wang at al. "Dcn v2: Improved deep & cross network and practical lessons for web-scale
9 learning to rank systems." in WWW 2021.
11Reference code:
12 https://github.com/shenweichen/DeepCTR-Torch
14"""
16import torch
17from torch import nn
19from hopwise.model.abstract_recommender import ContextRecommender
20from hopwise.model.init import xavier_normal_initialization
21from hopwise.model.layers import MLPLayers
22from hopwise.model.loss import RegLoss
25class DCNV2(ContextRecommender):
26 r"""DCNV2 improves the cross network by extending the original weight vector to a matrix,
27 significantly improves the expressiveness of DCN. It also introduces the MoE and
28 low rank techniques to reduce time cost.
29 """
31 def __init__(self, config, dataset):
32 super().__init__(config, dataset)
34 # load and compute parameters info
35 self.mixed = config["mixed"]
36 self.structure = config["structure"]
37 self.cross_layer_num = config["cross_layer_num"]
38 self.embedding_size = config["embedding_size"]
39 self.mlp_hidden_size = config["mlp_hidden_size"]
40 self.reg_weight = config["reg_weight"]
41 self.dropout_prob = config["dropout_prob"]
43 if self.mixed:
44 self.expert_num = config["expert_num"]
45 self.low_rank = config["low_rank"]
47 self.in_feature_num = self.num_feature_field * self.embedding_size
49 # define cross layers and bias
50 if self.mixed:
51 # U: (in_feature_num, low_rank)
52 self.cross_layer_u = nn.ParameterList(
53 nn.Parameter(torch.randn(self.expert_num, self.in_feature_num, self.low_rank))
54 for _ in range(self.cross_layer_num)
55 )
56 # V: (in_feature_num, low_rank)
57 self.cross_layer_v = nn.ParameterList(
58 nn.Parameter(torch.randn(self.expert_num, self.in_feature_num, self.low_rank))
59 for _ in range(self.cross_layer_num)
60 )
61 # C: (low_rank, low_rank)
62 self.cross_layer_c = nn.ParameterList(
63 nn.Parameter(torch.randn(self.expert_num, self.low_rank, self.low_rank))
64 for _ in range(self.cross_layer_num)
65 )
66 self.gating = nn.ModuleList(nn.Linear(self.in_feature_num, 1) for _ in range(self.expert_num))
67 else:
68 # W: (in_feature_num, in_feature_num)
69 self.cross_layer_w = nn.ParameterList(
70 nn.Parameter(torch.randn(self.in_feature_num, self.in_feature_num))
71 for _ in range(self.cross_layer_num)
72 )
73 # bias: (in_feature_num, 1)
74 self.bias = nn.ParameterList(
75 nn.Parameter(torch.zeros(self.in_feature_num, 1)) for _ in range(self.cross_layer_num)
76 )
78 # define deep and predict layers
79 mlp_size_list = [self.in_feature_num] + self.mlp_hidden_size
80 self.mlp_layers = MLPLayers(mlp_size_list, dropout=self.dropout_prob, bn=True)
81 if self.structure == "parallel":
82 self.predict_layer = nn.Linear(self.in_feature_num + self.mlp_hidden_size[-1], 1)
83 elif self.structure == "stacked":
84 self.predict_layer = nn.Linear(self.mlp_hidden_size[-1], 1)
86 # define loss and activation functions
87 self.reg_loss = RegLoss()
88 self.sigmoid = nn.Sigmoid()
89 self.tanh = nn.Tanh()
90 self.softmax = nn.Softmax(dim=1)
91 self.loss = nn.BCELoss()
93 # parameters initialization
94 self.apply(xavier_normal_initialization)
96 def cross_network(self, x_0):
97 r"""Cross network is composed of cross layers, with each layer having the following formula.
99 .. math:: x_{l+1} = x_0 \odot (W_l x_l + b_l) + x_l
101 :math:`x_l`, :math:`x_{l+1}` are column vectors denoting the outputs from the l -th and
102 (l + 1)-th cross layers, respectively.
103 :math:`W_l`, :math:`b_l` are the weight and bias parameters of the l -th layer.
105 Args:
106 x_0(torch.Tensor): Embedding vectors of all features, input of cross network.
108 Returns:
109 torch.Tensor:output of cross network, [batch_size, num_feature_field * embedding_size]
110 """
111 x_0 = x_0.unsqueeze(dim=2)
112 x_l = x_0 # (batch_size, in_feature_num, 1)
113 for i in range(self.cross_layer_num):
114 xl_w = torch.matmul(self.cross_layer_w[i], x_l)
115 xl_w = xl_w + self.bias[i]
116 xl_dot = torch.mul(x_0, xl_w)
117 x_l = xl_dot + x_l
119 x_l = x_l.squeeze(dim=2)
120 return x_l
122 def cross_network_mix(self, x_0):
123 r"""Cross network part of DCN-mix, which add MoE and nonlinear transformation in low-rank space.
125 .. math::
126 x_{l+1} = \sum_{i=1}^K G_i(x_l)E_i(x_l)+x_l
127 .. math::
128 E_i(x_l) = x_0 \odot (U_l^i \dot g(C_l^i \dot g(V_L^{iT} x_l)) + b_l)
130 :math:`E_i` and :math:`G_i` represents the expert and gatings respectively,
131 :math:`U_l`, :math:`C_l`, :math:`V_l` stand for low-rank decomposition of weight matrix,
132 :math:`g` is the nonlinear activation function.
134 Args:
135 x_0(torch.Tensor): Embedding vectors of all features, input of cross network.
137 Returns:
138 torch.Tensor:output of mixed cross network, [batch_size, num_feature_field * embedding_size]
139 """
140 x_0 = x_0.unsqueeze(dim=2)
141 x_l = x_0 # (batch_size, in_feature_num, 1)
142 for i in range(self.cross_layer_num):
143 expert_output_list = []
144 gating_output_list = []
145 for expert in range(self.expert_num):
146 # compute gating output
147 gating_output_list.append(self.gating[expert](x_l.squeeze(dim=2))) # (batch_size, 1)
149 # project to low-rank subspace
150 xl_v = torch.matmul(self.cross_layer_v[i][expert].T, x_l) # (batch_size, low_rank, 1)
152 # nonlinear activation in subspace
153 xl_c = self.tanh(xl_v)
154 xl_c = torch.matmul(self.cross_layer_c[i][expert], xl_c) # (batch_size, low_rank, 1)
155 xl_c = self.tanh(xl_c)
157 # project back feature space
158 xl_u = torch.matmul(self.cross_layer_u[i][expert], xl_c) # (batch_size, in_feature_num, 1)
160 # dot with x_0
161 xl_dot = xl_u + self.bias[i]
162 xl_dot = torch.mul(x_0, xl_dot)
164 expert_output_list.append(xl_dot.squeeze(dim=2)) # (batch_size, in_feature_num)
166 expert_output = torch.stack(expert_output_list, dim=2) # (batch_size, in_feature_num, expert_num)
167 gating_output = torch.stack(gating_output_list, dim=1) # (batch_size, expert_num, 1)
168 moe_output = torch.matmul(expert_output, self.softmax(gating_output)) # (batch_size, in_feature_num, 1)
169 x_l = x_l + moe_output
171 x_l = x_l.squeeze(dim=2) # (batch_size, in_feature_num)
172 return x_l
174 def forward(self, interaction):
175 dcn_all_embeddings = self.concat_embed_input_fields(interaction) # (batch_size, num_field, embed_dim)
176 batch_size = dcn_all_embeddings.shape[0]
177 dcn_all_embeddings = dcn_all_embeddings.view(batch_size, -1) # (batch_size, in_feature_num)
179 if self.structure == "parallel":
180 deep_output = self.mlp_layers(dcn_all_embeddings) # (batch_size, mlp_hidden_size)
181 if self.mixed:
182 cross_output = self.cross_network_mix(dcn_all_embeddings) # (batch_size, in_feature_num)
183 else:
184 cross_output = self.cross_network(dcn_all_embeddings)
185 concat_output = torch.cat([cross_output, deep_output], dim=-1) # (batch_size, in_num + mlp_size)
186 output = self.sigmoid(self.predict_layer(concat_output))
188 return output.squeeze(dim=1)
190 elif self.structure == "stacked":
191 if self.mixed:
192 cross_output = self.cross_network_mix(dcn_all_embeddings) # (batch_size, in_feature_num)
193 else:
194 cross_output = self.cross_network(dcn_all_embeddings)
195 deep_output = self.mlp_layers(cross_output) # (batch_size, mlp_hidden_size)
196 output = self.sigmoid(self.predict_layer(deep_output))
198 return output.squeeze(dim=1)
200 def calculate_loss(self, interaction):
201 label = interaction[self.LABEL]
202 output = self.forward(interaction)
203 if self.mixed:
204 reg_loss = (
205 self.reg_loss(self.cross_layer_c)
206 + self.reg_loss(self.cross_layer_v)
207 + self.reg_loss(self.cross_layer_u)
208 )
209 else:
210 reg_loss = self.reg_loss(self.cross_layer_w)
211 l2_loss = self.reg_weight * reg_loss
212 return self.loss(output, label) + l2_loss
214 def predict(self, interaction):
215 return self.forward(interaction)