<filename>mmdet/models/anchor_heads/usd_head.py
import torch
import torch.nn as nn
from mmcv.cnn import normal_init
from mmdet.core.post_processing.bbox_nms import multiclass_nms_with_mask
from mmdet.core import distance2bbox, force_fp32, multi_apply
from ..builder import build_loss
from ..registry import HEADS
from ..utils import ConvModule, Scale, bias_init_with_prob

INF = 1e8


@HEADS.register_module
class USDHead(nn.Module):

    def __init__(self,
                 num_classes,
                 in_channels,
                 feat_channels=256,
                 stacked_convs=4,
                 strides=(4, 8, 16, 32, 64),
                 regress_ranges=((-1, 64), (64, 128), (128, 256), (256, 512),
                                 (512, INF)),
                 # use_dcn=False,
                 masknms=False,
                 loss_cls=dict(
                     type='FocalLoss',
                     use_sigmoid=True,
                     gamma=2.0,
                     alpha=0.25,
                     loss_weight=1.0),
                 loss_bbox=dict(type='IoULoss', loss_weight=1.0),
                 loss_coef=dict(
                     type='SmoothL1Loss',
                     beta=1.0,
                     reduction='mean',
                     loss_weight=1.0),
                 loss_centerness=dict(
                     type='CrossEntropyLoss',
                     use_sigmoid=True,
                     loss_weight=1.0),
                 conv_cfg=None,
                 norm_cfg=dict(type='GN', num_groups=32, requires_grad=True),
                 num_bases=32,
                 method='None',
                 use_center_sample=False,
                 loss_mask=None,
                 ):
        super(USDHead, self).__init__()

        self.num_classes = num_classes
        self.cls_out_channels = num_classes - 1
        self.in_channels = in_channels
        self.feat_channels = feat_channels
        self.stacked_convs = stacked_convs
        self.strides = strides
        self.regress_ranges = regress_ranges
        self.loss_cls = build_loss(loss_cls)
        self.loss_bbox = build_loss(loss_bbox)
        self.loss_coef = build_loss(loss_coef)
        self.loss_centerness = build_loss(loss_centerness)
        self.conv_cfg = conv_cfg
        self.norm_cfg = norm_cfg
        self.fp16_enabled = False

        # USD-Seg
        self.num_bases = num_bases
        # self.use_dcn=use_dcn  # TODO: Add DCN support
        if method not in ['var', 'cosine', 'cosine_r']:
            raise NotImplementedError('%s not supported.' % method)
        self.method = method
        self.use_center_sample = use_center_sample

        # Mask Loss
        if loss_mask is None:
            self.loss_mask = None
        else:
            self.loss_mask = build_loss(loss_mask)

        self._init_layers()

    def _init_layers(self):
        self.cls_convs = nn.ModuleList()
        self.reg_convs = nn.ModuleList()
        self.coef_convs = nn.ModuleList()
        for i in range(self.stacked_convs):
            chn = self.in_channels if i == 0 else self.feat_channels
            self.cls_convs.append(
                ConvModule(
                    chn,
                    self.feat_channels,
                    3,
                    stride=1,
                    padding=1,
                    conv_cfg=self.conv_cfg,
                    norm_cfg=self.norm_cfg,
                    bias=self.norm_cfg is None))
            self.reg_convs.append(
                ConvModule(
                    chn,
                    self.feat_channels,
                    3,
                    stride=1,
                    padding=1,
                    conv_cfg=self.conv_cfg,
                    norm_cfg=self.norm_cfg,
                    bias=self.norm_cfg is None))
            self.coef_convs.append(
                ConvModule(
                    chn,
                    self.feat_channels,
                    3,
                    stride=1,
                    padding=1,
                    conv_cfg=self.conv_cfg,
                    norm_cfg=self.norm_cfg,
                    bias=self.norm_cfg is None))
        self.usd_cls = nn.Conv2d(
            self.feat_channels, self.cls_out_channels, 3, padding=1)
        self.usd_reg = nn.Conv2d(self.feat_channels, 4, 3, padding=1)
        self.usd_coef = nn.Conv2d(
            self.feat_channels, self.num_bases, 3, padding=1)
        self.usd_centerness = nn.Conv2d(self.feat_channels, 1, 3, padding=1)

        self.scales_bbox = nn.ModuleList([Scale(1.0) for _ in self.strides])
        self.scales_coef = nn.ModuleList([Scale(1.0) for _ in self.strides])

    def init_weights(self):
        for m in self.cls_convs:
            normal_init(m.conv, std=0.01)
        for m in self.reg_convs:
            normal_init(m.conv, std=0.01)
        for m in self.coef_convs:
            normal_init(m.conv, std=0.01)
        bias_cls = bias_init_with_prob(0.01)
        normal_init(self.usd_cls, std=0.01, bias=bias_cls)
        normal_init(self.usd_reg, std=0.01)
        normal_init(self.usd_coef, std=0.01)
        normal_init(self.usd_centerness, std=0.01)

    def forward(self, feats):
        return multi_apply(self.forward_single, feats, self.scales_bbox, self.scales_coef)

    def forward_single(self, x, scale_boxx, scale_coef):
        cls_feat = x
        reg_feat = x
        coef_feat = x

        for cls_layer in self.cls_convs:
            cls_feat = cls_layer(cls_feat)
        cls_score = self.usd_cls(cls_feat)
        centerness = self.usd_centerness(cls_feat)

        for reg_layer in self.reg_convs:
            reg_feat = reg_layer(reg_feat)
        # scale the bbox_pred of different level
        # float to avoid overflow when enabling FP16
        bbox_pred = scale_boxx(self.usd_reg(reg_feat)).float().exp()

        for coef_layer in self.coef_convs:
            coef_feat = coef_layer(coef_feat)
        coef_pred = scale_coef(self.usd_coef(coef_feat)).float()
        return cls_score, bbox_pred, centerness, coef_pred

    @force_fp32(apply_to=('cls_scores', 'bbox_preds', 'coef_preds', 'centernesses'))
    def loss(self,
             cls_scores,
             bbox_preds,
             centernesses,
             coef_preds,
             gt_bboxes,
             gt_labels,
             gt_coefs,
             img_metas,
             cfg,
             gt_bboxes_ignore=None,
             extra_data=None,
             bases=None,
             gt_resized_masks=None):
        assert len(cls_scores) == len(bbox_preds) == len(
            centernesses) == len(coef_preds)
        featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores]
        all_level_points = self.get_points(featmap_sizes, bbox_preds[0].dtype,
                                           bbox_preds[0].device)

        labels, bbox_targets, coef_targets = self.usd_target(
            all_level_points, gt_bboxes, gt_labels, gt_coefs)

        num_imgs = cls_scores[0].size(0)
        # flatten cls_scores, bbox_preds and centerness
        flatten_cls_scores = [
            cls_score.permute(0, 2, 3, 1).reshape(-1, self.cls_out_channels)
            for cls_score in cls_scores
        ]
        flatten_bbox_preds = [
            bbox_pred.permute(0, 2, 3, 1).reshape(-1, 4)
            for bbox_pred in bbox_preds
        ]
        flatten_centerness = [
            centerness.permute(0, 2, 3, 1).reshape(-1)
            for centerness in centernesses
        ]
        flatten_coef_preds = [
            coef_pred.permute(0, 2, 3, 1).reshape(-1, self.num_bases)
            for coef_pred in coef_preds
        ]
        flatten_cls_scores = torch.cat(flatten_cls_scores)  # [num_pixel, 80]
        flatten_bbox_preds = torch.cat(flatten_bbox_preds)  # [num_pixel, 4]
        flatten_coef_preds = torch.cat(
            flatten_coef_preds)  # [num_pixel, num_bases]
        flatten_centerness = torch.cat(flatten_centerness)  # [num_pixel]

        flatten_labels = torch.cat(labels)                  # [num_pixel]
        flatten_bbox_targets = torch.cat(bbox_targets)      # [num_pixel, 4]
        flatten_coef_targets = torch.cat(coef_targets)      # [num_pixel, num_bases]
        # repeat points to align with bbox_preds
        flatten_points = torch.cat(
            [points.repeat(num_imgs, 1) for points in all_level_points])    # [num_pixel,2]

        pos_idx = flatten_labels.nonzero().reshape(-1)
        num_pos = len(pos_idx)

        loss_cls = self.loss_cls(
            flatten_cls_scores, flatten_labels,
            avg_factor=num_pos + num_imgs)  # avoid num_pos is 0

        pos_bbox_preds = flatten_bbox_preds[pos_idx]
        pos_coef_preds = flatten_coef_preds[pos_idx]
        pos_centerness = flatten_centerness[pos_idx]

        if num_pos > 0:
            pos_bbox_targets = flatten_bbox_targets[pos_idx]
            pos_coef_targets = flatten_coef_targets[pos_idx]
            pos_centerness_targets = self.centerness_target(pos_bbox_targets)

            pos_points = flatten_points[pos_idx]
            pos_decoded_bbox_preds = distance2bbox(pos_points, pos_bbox_preds)
            pos_decoded_target_preds = distance2bbox(pos_points,
                                                     pos_bbox_targets)
            # centerness weighted iou loss
            loss_bbox = self.loss_bbox(
                pos_decoded_bbox_preds,
                pos_decoded_target_preds,
                weight=pos_centerness_targets,
                avg_factor=pos_centerness_targets.sum())

            if self.method == 'var':
                # Change the weights of different coefs
                coef_weights = pos_centerness_targets.expand((self.num_bases,
                                                              pos_centerness_targets.size(0))).transpose(1, 0)
                loss_coef = self.loss_coef(pos_coef_preds,
                                           pos_coef_targets,
                                           weight=coef_weights,
                                           avg_factor=pos_centerness_targets.sum())
            elif self.method == 'cosine' or self.method == 'cosine_r':
                loss_coef = self.loss_coef(pos_coef_preds,
                                           pos_coef_targets,
                                           weight=pos_centerness_targets,
                                           avg_factor=pos_centerness_targets.sum())

            # Mask Loss
            if bases is not None:
                mask_targets = torch.cat(gt_resized_masks)
                coef_targets = torch.cat(gt_coefs).float()

                # Find the targets
                num_targets = pos_coef_targets.size(0)
                pos_mask_targets = torch.zeros(num_targets, 64 * 64, device=mask_targets.device)
                for target_idx in range(num_targets):
                    current_target_coef = pos_coef_targets[target_idx]
                    found = False
                    for gt_coef_idx, gt_coef in enumerate(coef_targets):
                        if (gt_coef == current_target_coef).sum().item() == self.num_bases:
                            pos_mask_targets[target_idx] = mask_targets[gt_coef_idx]
                            found = True
                            break
                    assert found

                mask_weights = pos_centerness_targets.expand((64 * 64,
                                                              pos_centerness_targets.size(0))).transpose(1, 0)

                loss_mask = self.loss_mask(torch.mm(pos_coef_preds, bases).gt(0).float(),
                                           pos_mask_targets.gt(0).float(),
                                           weight=mask_weights,
                                           avg_factor=pos_centerness_targets.sum() * 1024)

            loss_centerness = self.loss_centerness(pos_centerness,
                                                   pos_centerness_targets)
        else:
            loss_bbox = pos_bbox_preds.sum()
            loss_coef = pos_coef_preds.sum()
            loss_centerness = pos_centerness.sum()

        if bases is None:
            return dict(
                loss_cls=loss_cls,
                loss_bbox=loss_bbox,
                loss_coef=loss_coef,
                loss_centerness=loss_centerness)
        else:
            return dict(
                loss_cls=loss_cls,
                loss_bbox=loss_bbox,
                loss_coef=loss_coef,
                loss_centerness=loss_centerness,
                loss_mask=loss_mask)

    @force_fp32(apply_to=('cls_scores', 'bbox_preds', 'centernesses'))
    def get_bboxes(self,
                   cls_scores,
                   bbox_preds,
                   centernesses,
                   coef_preds,
                   img_metas,
                   cfg,
                   rescale=None):
        assert len(cls_scores) == len(bbox_preds)
        num_levels = len(cls_scores)

        featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores]
        mlvl_points = self.get_points(featmap_sizes, bbox_preds[0].dtype,
                                      bbox_preds[0].device)
        result_list = []
        for img_id in range(len(img_metas)):
            cls_score_list = [
                cls_scores[i][img_id].detach() for i in range(num_levels)
            ]
            bbox_pred_list = [
                bbox_preds[i][img_id].detach() for i in range(num_levels)
            ]
            centerness_pred_list = [
                centernesses[i][img_id].detach() for i in range(num_levels)
            ]
            coef_pred_list = [
                coef_preds[i][img_id].detach() for i in range(num_levels)
            ]
            img_shape = img_metas[img_id]['img_shape']
            scale_factor = img_metas[img_id]['scale_factor']
            det_bboxes = self.get_bboxes_single(cls_score_list,
                                                bbox_pred_list,
                                                coef_pred_list,
                                                centerness_pred_list,
                                                mlvl_points, img_shape,
                                                scale_factor, cfg, rescale)
            result_list.append(det_bboxes)
        return result_list

    def get_bboxes_single(self,
                          cls_scores,
                          bbox_preds,
                          coef_preds,
                          centernesses,
                          mlvl_points,
                          img_shape,
                          scale_factor,
                          cfg,
                          rescale=False):
        assert len(cls_scores) == len(bbox_preds) == len(mlvl_points)
        mlvl_bboxes = []
        mlvl_scores = []
        mlvl_coefs = []
        mlvl_centerness = []
        for cls_score, bbox_pred, coef_pred, centerness, points in zip(
                cls_scores, bbox_preds, coef_preds, centernesses, mlvl_points):
            assert cls_score.size()[-2:] == bbox_pred.size()[-2:]
            scores = cls_score.permute(1, 2, 0).reshape(
                -1, self.cls_out_channels).sigmoid()

            centerness = centerness.permute(1, 2, 0).reshape(-1).sigmoid()
            coef_pred = coef_pred.permute(1, 2, 0).reshape(-1, self.num_bases)
            bbox_pred = bbox_pred.permute(1, 2, 0).reshape(-1, 4)
            nms_pre = cfg.get('nms_pre', -1)
            if nms_pre > 0 and scores.shape[0] > nms_pre:
                max_scores, _ = (scores * centerness[:, None]).max(dim=1)
                _, topk_inds = max_scores.topk(nms_pre)
                points = points[topk_inds, :]
                bbox_pred = bbox_pred[topk_inds, :]
                coef_pred = coef_pred[topk_inds, :]
                scores = scores[topk_inds, :]
                centerness = centerness[topk_inds]
            bboxes = distance2bbox(points, bbox_pred, max_shape=img_shape)
            coefs = coef_pred
            mlvl_bboxes.append(bboxes)
            mlvl_scores.append(scores)
            mlvl_centerness.append(centerness)
            mlvl_coefs.append(coefs)

        mlvl_bboxes = torch.cat(mlvl_bboxes)
        mlvl_coefs = torch.cat(mlvl_coefs)
        if rescale:
            mlvl_bboxes /= mlvl_bboxes.new_tensor(scale_factor)
        mlvl_scores = torch.cat(mlvl_scores)
        padding = mlvl_scores.new_zeros(mlvl_scores.shape[0], 1)
        mlvl_scores = torch.cat([padding, mlvl_scores], dim=1)
        mlvl_centerness = torch.cat(mlvl_centerness)
        det_bboxes, det_labels, det_coefs = multiclass_nms_with_mask(
            mlvl_bboxes,
            mlvl_scores,
            mlvl_coefs,
            cfg.score_thr,
            cfg.nms,
            cfg.max_per_img,
            score_factors=mlvl_centerness,
            num_bases=self.num_bases)
        return det_bboxes, det_labels, det_coefs

    def get_points(self, featmap_sizes, dtype, device):
        """Get points according to feature map sizes.

        Args:
            featmap_sizes (list[tuple]): Multi-level feature map sizes.
            dtype (torch.dtype): Type of points.
            device (torch.device): Device of points.

        Returns:
            tuple: points of each image.
        """
        mlvl_points = []
        for i in range(len(featmap_sizes)):
            mlvl_points.append(
                self.get_points_single(featmap_sizes[i], self.strides[i],
                                       dtype, device))
        return mlvl_points

    def get_points_single(self, featmap_size, stride, dtype, device):
        h, w = featmap_size
        x_range = torch.arange(
            0, w * stride, stride, dtype=dtype, device=device)
        y_range = torch.arange(
            0, h * stride, stride, dtype=dtype, device=device)
        y, x = torch.meshgrid(y_range, x_range)
        points = torch.stack(
            (x.reshape(-1), y.reshape(-1)), dim=-1) + stride // 2
        return points

    def usd_target(self, points, gt_bboxes_list, gt_labels_list, gt_coefs_list):
        assert len(points) == len(self.regress_ranges)
        num_levels = len(points)
        # expand regress ranges to align with points
        expanded_regress_ranges = [
            points[i].new_tensor(self.regress_ranges[i])[None].expand_as(
                points[i]) for i in range(num_levels)
        ]
        # concat all levels points and regress ranges
        concat_regress_ranges = torch.cat(expanded_regress_ranges, dim=0)
        concat_points = torch.cat(points, dim=0)

        num_points = [center.size(0) for center in points]

        # get labels and bbox_targets of each image
        labels_list, bbox_targets_list, coef_targets_list = multi_apply(
            self.usd_target_single,
            gt_bboxes_list,
            gt_labels_list,
            gt_coefs_list,
            points=concat_points,
            regress_ranges=concat_regress_ranges,
            num_points_per_lvl=num_points)

        # split to per img, per level
        labels_list = [labels.split(num_points, 0) for labels in labels_list]
        bbox_targets_list = [
            bbox_targets.split(num_points, 0)
            for bbox_targets in bbox_targets_list
        ]
        coef_targets_list = [
            coef_targets.split(num_points, 0)
            for coef_targets in coef_targets_list
        ]

        # concat per level image
        concat_lvl_labels = []
        concat_lvl_bbox_targets = []
        concat_lvl_coef_targets = []
        for i in range(num_levels):
            concat_lvl_labels.append(
                torch.cat([labels[i] for labels in labels_list]))
            concat_lvl_bbox_targets.append(
                torch.cat(
                    [bbox_targets[i] for bbox_targets in bbox_targets_list]))
            concat_lvl_coef_targets.append(
                torch.cat(
                    [coef_targets[i] for coef_targets in coef_targets_list]))
        return concat_lvl_labels, concat_lvl_bbox_targets, concat_lvl_coef_targets

    def usd_target_single(self, gt_bboxes, gt_labels, gt_coefs, points, regress_ranges, num_points_per_lvl):
        num_points = points.size(0)
        num_gts = gt_labels.size(0)
        if num_gts == 0:
            return gt_labels.new_zeros(num_points), \
                gt_bboxes.new_zeros((num_points, 4)), \
                gt_labels.new_zeros((num_points, self.num_bases))

        areas = (gt_bboxes[:, 2] - gt_bboxes[:, 0] + 1) * (
            gt_bboxes[:, 3] - gt_bboxes[:, 1] + 1)  # gt_bboxes: [..., x1 y1 x2 y2]
        # TODO: figure out why these two are different
        # areas = areas[None].expand(num_points, num_gts)
        areas = areas[None].repeat(num_points, 1)

        regress_ranges = regress_ranges[:, None, :].expand(
            num_points, num_gts, 2)
        gt_bboxes = gt_bboxes[None].expand(num_points, num_gts, 4)
        # xs ys is the coord of x, y of points
        xs, ys = points[:, 0], points[:, 1]
        xs = xs[:, None].expand(num_points, num_gts)
        ys = ys[:, None].expand(num_points, num_gts)

        left = xs - gt_bboxes[..., 0]
        right = gt_bboxes[..., 2] - xs
        top = ys - gt_bboxes[..., 1]
        bottom = gt_bboxes[..., 3] - ys
        bbox_targets = torch.stack((left, top, right, bottom), -1)

        # condition1: inside a gt bbox
        if self.use_center_sample:
            inside_gt_bbox_mask = self.get_center_sample(gt_bboxes, self.strides, num_points_per_lvl,
                                                         xs.long(), ys.long(), radius=1.5)
        else:
            inside_gt_bbox_mask = bbox_targets.min(-1)[0] > 0

        # condition2: limit the regression range for each location
        max_regress_distance = bbox_targets.max(-1)[0]
        inside_regress_range = (
            max_regress_distance >= regress_ranges[..., 0]) & (
                max_regress_distance <= regress_ranges[..., 1])

        # if there are still more than one objects for a location,
        # we choose the one with minimal area
        areas[inside_gt_bbox_mask == 0] = INF
        areas[inside_regress_range == 0] = INF
        min_area, min_area_inds = areas.min(dim=1)

        labels = gt_labels[min_area_inds]
        labels[min_area == INF] = 0
        bbox_targets = bbox_targets[range(num_points), min_area_inds]

        # USD-Seg
        pos_inds = labels.nonzero().reshape(-1)
        coef_targets = torch.zeros((num_points, self.num_bases), device=gt_coefs.device)
        for p in pos_inds:
            pos_coef_id = min_area_inds[p]
            pos_coef = gt_coefs[pos_coef_id]
            coef_targets[p] = pos_coef

        return labels, bbox_targets, coef_targets

    def get_center_sample(self, gt, strides, num_points_per, gt_xs, gt_ys, radius=1):
        center_x = (gt[..., 0] + gt[..., 2]) / 2
        center_y = (gt[..., 1] + gt[..., 3]) / 2
        center_gt = gt.new_zeros(gt.shape)
        # no gt
        if center_x[..., 0].sum() == 0:
            return gt_xs.new_zeros(gt_xs.shape, dtype=torch.uint8)
        beg = 0
        for level, n_p in enumerate(num_points_per):
            end = beg + n_p
            stride = int(strides[level] * radius)
            xmin = center_x[beg:end] - stride
            ymin = center_y[beg:end] - stride
            xmax = center_x[beg:end] + stride
            ymax = center_y[beg:end] + stride
            # limit sample region in gt
            center_gt[beg:end, :, 0] = torch.where(xmin > gt[beg:end, :, 0], xmin, gt[beg:end, :, 0])
            center_gt[beg:end, :, 1] = torch.where(ymin > gt[beg:end, :, 1], ymin, gt[beg:end, :, 1])
            center_gt[beg:end, :, 2] = torch.where(xmax > gt[beg:end, :, 2], gt[beg:end, :, 2], xmax)
            center_gt[beg:end, :, 3] = torch.where(ymax > gt[beg:end, :, 3], gt[beg:end, :, 3], ymax)
            beg = end
        left = gt_xs - center_gt[..., 0]
        right = center_gt[..., 2] - gt_xs
        top = gt_ys - center_gt[..., 1]
        bottom = center_gt[..., 3] - gt_ys
        center_bbox = torch.stack((left, top, right, bottom), -1)
        inside_gt_bbox_mask = center_bbox.min(-1)[0] > 0
        return inside_gt_bbox_mask

    def centerness_target(self, pos_bbox_targets):
        # only calculate pos centerness targets, otherwise there may be nan
        left_right = pos_bbox_targets[:, [0, 2]]
        top_bottom = pos_bbox_targets[:, [1, 3]]
        centerness_targets = (
            left_right.min(dim=-1)[0] / left_right.max(dim=-1)[0]) * (
                top_bottom.min(dim=-1)[0] / top_bottom.max(dim=-1)[0])
        return torch.sqrt(centerness_targets)

<filename>mmdet/models/anchor_heads/usd_head.py
import torch
import torch.nn as nn
from mmcv.cnn import normal_init
from mmdet.core.post_processing.bbox_nms import multiclass_nms_with_mask
from mmdet.core import distance2bbox, force_fp32, multi_apply
from ..builder import build_loss
from ..registry import HEADS
from ..utils import ConvModule, Scale, bias_init_with_prob

INF = 1e8


@HEADS.register_module
class USDHead(nn.Module):

    def __init__(self,
                 num_classes,
                 in_channels,
                 feat_channels=256,
                 stacked_convs=4,
                 strides=(4, 8, 16, 32, 64),
                 regress_ranges=((-1, 64), (64, 128), (128, 256), (256, 512),
                                 (512, INF)),
                 # use_dcn=False,
                 masknms=False,
                 loss_cls=dict(
                     type='FocalLoss',
                     use_sigmoid=True,
                     gamma=2.0,
                     alpha=0.25,
                     loss_weight=1.0),
                 loss_bbox=dict(type='IoULoss', loss_weight=1.0),
                 loss_coef=dict(
                     type='SmoothL1Loss',
                     beta=1.0,
                     reduction='mean',
                     loss_weight=1.0),
                 loss_centerness=dict(
                     type='CrossEntropyLoss',
                     use_sigmoid=True,
                     loss_weight=1.0),
                 conv_cfg=None,
                 norm_cfg=dict(type='GN', num_groups=32, requires_grad=True),
                 num_bases=32,
                 method='None',
                 use_center_sample=False,
                 loss_mask=None,
                 ):
        super(USDHead, self).__init__()

        self.num_classes = num_classes
        self.cls_out_channels = num_classes - 1
        self.in_channels = in_channels
        self.feat_channels = feat_channels
        self.stacked_convs = stacked_convs
        self.strides = strides
        self.regress_ranges = regress_ranges
        self.loss_cls = build_loss(loss_cls)
        self.loss_bbox = build_loss(loss_bbox)
        self.loss_coef = build_loss(loss_coef)
        self.loss_centerness = build_loss(loss_centerness)
        self.conv_cfg = conv_cfg
        self.norm_cfg = norm_cfg
        self.fp16_enabled = False

        # USD-Seg
        self.num_bases = num_bases
        # self.use_dcn=use_dcn  # TODO: Add DCN support
        if method not in ['var', 'cosine', 'cosine_r']:
            raise NotImplementedError('%s not supported.' % method)
        self.method = method
        self.use_center_sample = use_center_sample

        # Mask Loss
        if loss_mask is None:
            self.loss_mask = None
        else:
            self.loss_mask = build_loss(loss_mask)

        self._init_layers()

    def _init_layers(self):
        self.cls_convs = nn.ModuleList()
        self.reg_convs = nn.ModuleList()
        self.coef_convs = nn.ModuleList()
        for i in range(self.stacked_convs):
            chn = self.in_channels if i == 0 else self.feat_channels
            self.cls_convs.append(
                ConvModule(
                    chn,
                    self.feat_channels,
                    3,
                    stride=1,
                    padding=1,
                    conv_cfg=self.conv_cfg,
                    norm_cfg=self.norm_cfg,
                    bias=self.norm_cfg is None))
            self.reg_convs.append(
                ConvModule(
                    chn,
                    self.feat_channels,
                    3,
                    stride=1,
                    padding=1,
                    conv_cfg=self.conv_cfg,
                    norm_cfg=self.norm_cfg,
                    bias=self.norm_cfg is None))
            self.coef_convs.append(
                ConvModule(
                    chn,
                    self.feat_channels,
                    3,
                    stride=1,
                    padding=1,
                    conv_cfg=self.conv_cfg,
                    norm_cfg=self.norm_cfg,
                    bias=self.norm_cfg is None))
        self.usd_cls = nn.Conv2d(
            self.feat_channels, self.cls_out_channels, 3, padding=1)
        self.usd_reg = nn.Conv2d(self.feat_channels, 4, 3, padding=1)
        self.usd_coef = nn.Conv2d(
            self.feat_channels, self.num_bases, 3, padding=1)
        self.usd_centerness = nn.Conv2d(self.feat_channels, 1, 3, padding=1)

        self.scales_bbox = nn.ModuleList([Scale(1.0) for _ in self.strides])
        self.scales_coef = nn.ModuleList([Scale(1.0) for _ in self.strides])

    def init_weights(self):
        for m in self.cls_convs:
            normal_init(m.conv, std=0.01)
        for m in self.reg_convs:
            normal_init(m.conv, std=0.01)
        for m in self.coef_convs:
            normal_init(m.conv, std=0.01)
        bias_cls = bias_init_with_prob(0.01)
        normal_init(self.usd_cls, std=0.01, bias=bias_cls)
        normal_init(self.usd_reg, std=0.01)
        normal_init(self.usd_coef, std=0.01)
        normal_init(self.usd_centerness, std=0.01)

    def forward(self, feats):
        return multi_apply(self.forward_single, feats, self.scales_bbox, self.scales_coef)

    def forward_single(self, x, scale_boxx, scale_coef):
        cls_feat = x
        reg_feat = x
        coef_feat = x

        for cls_layer in self.cls_convs:
            cls_feat = cls_layer(cls_feat)
        cls_score = self.usd_cls(cls_feat)
        centerness = self.usd_centerness(cls_feat)

        for reg_layer in self.reg_convs:
            reg_feat = reg_layer(reg_feat)
        # scale the bbox_pred of different level
        # float to avoid overflow when enabling FP16
        bbox_pred = scale_boxx(self.usd_reg(reg_feat)).float().exp()

        for coef_layer in self.coef_convs:
            coef_feat = coef_layer(coef_feat)
        coef_pred = scale_coef(self.usd_coef(coef_feat)).float()
        return cls_score, bbox_pred, centerness, coef_pred

    @force_fp32(apply_to=('cls_scores', 'bbox_preds', 'coef_preds', 'centernesses'))
    def loss(self,
             cls_scores,
             bbox_preds,
             centernesses,
             coef_preds,
             gt_bboxes,
             gt_labels,
             gt_coefs,
             img_metas,
             cfg,
             gt_bboxes_ignore=None,
             extra_data=None,
             bases=None,
             gt_resized_masks=None):
        assert len(cls_scores) == len(bbox_preds) == len(
            centernesses) == len(coef_preds)
        featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores]
        all_level_points = self.get_points(featmap_sizes, bbox_preds[0].dtype,
                                           bbox_preds[0].device)

        labels, bbox_targets, coef_targets = self.usd_target(
            all_level_points, gt_bboxes, gt_labels, gt_coefs)

        num_imgs = cls_scores[0].size(0)
        # flatten cls_scores, bbox_preds and centerness
        flatten_cls_scores = [
            cls_score.permute(0, 2, 3, 1).reshape(-1, self.cls_out_channels)
            for cls_score in cls_scores
        ]
        flatten_bbox_preds = [
            bbox_pred.permute(0, 2, 3, 1).reshape(-1, 4)
            for bbox_pred in bbox_preds
        ]
        flatten_centerness = [
            centerness.permute(0, 2, 3, 1).reshape(-1)
            for centerness in centernesses
        ]
        flatten_coef_preds = [
            coef_pred.permute(0, 2, 3, 1).reshape(-1, self.num_bases)
            for coef_pred in coef_preds
        ]
        flatten_cls_scores = torch.cat(flatten_cls_scores)  # [num_pixel, 80]
        flatten_bbox_preds = torch.cat(flatten_bbox_preds)  # [num_pixel, 4]
        flatten_coef_preds = torch.cat(
            flatten_coef_preds)  # [num_pixel, num_bases]
        flatten_centerness = torch.cat(flatten_centerness)  # [num_pixel]

        flatten_labels = torch.cat(labels)                  # [num_pixel]
        flatten_bbox_targets = torch.cat(bbox_targets)      # [num_pixel, 4]
        flatten_coef_targets = torch.cat(coef_targets)      # [num_pixel, num_bases]
        # repeat points to align with bbox_preds
        flatten_points = torch.cat(
            [points.repeat(num_imgs, 1) for points in all_level_points])    # [num_pixel,2]

        pos_idx = flatten_labels.nonzero().reshape(-1)
        num_pos = len(pos_idx)

        loss_cls = self.loss_cls(
            flatten_cls_scores, flatten_labels,
            avg_factor=num_pos + num_imgs)  # avoid num_pos is 0

        pos_bbox_preds = flatten_bbox_preds[pos_idx]
        pos_coef_preds = flatten_coef_preds[pos_idx]
        pos_centerness = flatten_centerness[pos_idx]

        if num_pos > 0:
            pos_bbox_targets = flatten_bbox_targets[pos_idx]
            pos_coef_targets = flatten_coef_targets[pos_idx]
            pos_centerness_targets = self.centerness_target(pos_bbox_targets)

            pos_points = flatten_points[pos_idx]
            pos_decoded_bbox_preds = distance2bbox(pos_points, pos_bbox_preds)
            pos_decoded_target_preds = distance2bbox(pos_points,
                                                     pos_bbox_targets)
            # centerness weighted iou loss
            loss_bbox = self.loss_bbox(
                pos_decoded_bbox_preds,
                pos_decoded_target_preds,
                weight=pos_centerness_targets,
                avg_factor=pos_centerness_targets.sum())

            if self.method == 'var':
                # Change the weights of different coefs
                coef_weights = pos_centerness_targets.expand((self.num_bases,
                                                              pos_centerness_targets.size(0))).transpose(1, 0)
                loss_coef = self.loss_coef(pos_coef_preds,
                                           pos_coef_targets,
                                           weight=coef_weights,
                                           avg_factor=pos_centerness_targets.sum())
            elif self.method == 'cosine' or self.method == 'cosine_r':
                loss_coef = self.loss_coef(pos_coef_preds,
                                           pos_coef_targets,
                                           weight=pos_centerness_targets,
                                           avg_factor=pos_centerness_targets.sum())

            # Mask Loss
            if bases is not None:
                mask_targets = torch.cat(gt_resized_masks)
                coef_targets = torch.cat(gt_coefs).float()

                # Find the targets
                num_targets = pos_coef_targets.size(0)
                pos_mask_targets = torch.zeros(num_targets, 64 * 64, device=mask_targets.device)
                for target_idx in range(num_targets):
                    current_target_coef = pos_coef_targets[target_idx]
                    found = False
                    for gt_coef_idx, gt_coef in enumerate(coef_targets):
                        if (gt_coef == current_target_coef).sum().item() == self.num_bases:
                            pos_mask_targets[target_idx] = mask_targets[gt_coef_idx]
                            found = True
                            break
                    assert found

                mask_weights = pos_centerness_targets.expand((64 * 64,
                                                              pos_centerness_targets.size(0))).transpose(1, 0)

                loss_mask = self.loss_mask(torch.mm(pos_coef_preds, bases).gt(0).float(),
                                           pos_mask_targets.gt(0).float(),
                                           weight=mask_weights,
                                           avg_factor=pos_centerness_targets.sum() * 1024)

            loss_centerness = self.loss_centerness(pos_centerness,
                                                   pos_centerness_targets)
        else:
            loss_bbox = pos_bbox_preds.sum()
            loss_coef = pos_coef_preds.sum()
            loss_centerness = pos_centerness.sum()

        if bases is None:
            return dict(
                loss_cls=loss_cls,
                loss_bbox=loss_bbox,
                loss_coef=loss_coef,
                loss_centerness=loss_centerness)
        else:
            return dict(
                loss_cls=loss_cls,
                loss_bbox=loss_bbox,
                loss_coef=loss_coef,
                loss_centerness=loss_centerness,
                loss_mask=loss_mask)

    @force_fp32(apply_to=('cls_scores', 'bbox_preds', 'centernesses'))
    def get_bboxes(self,
                   cls_scores,
                   bbox_preds,
                   centernesses,
                   coef_preds,
                   img_metas,
                   cfg,
                   rescale=None):
        assert len(cls_scores) == len(bbox_preds)
        num_levels = len(cls_scores)

        featmap_sizes = [featmap.size()[-2:] for featmap in cls_scores]
        mlvl_points = self.get_points(featmap_sizes, bbox_preds[0].dtype,
                                      bbox_preds[0].device)
        result_list = []
        for img_id in range(len(img_metas)):
            cls_score_list = [
                cls_scores[i][img_id].detach() for i in range(num_levels)
            ]
            bbox_pred_list = [
                bbox_preds[i][img_id].detach() for i in range(num_levels)
            ]
            centerness_pred_list = [
                centernesses[i][img_id].detach() for i in range(num_levels)
            ]
            coef_pred_list = [
                coef_preds[i][img_id].detach() for i in range(num_levels)
            ]
            img_shape = img_metas[img_id]['img_shape']
            scale_factor = img_metas[img_id]['scale_factor']
            det_bboxes = self.get_bboxes_single(cls_score_list,
                                                bbox_pred_list,
                                                coef_pred_list,
                                                centerness_pred_list,
                                                mlvl_points, img_shape,
                                                scale_factor, cfg, rescale)
            result_list.append(det_bboxes)
        return result_list

    def get_bboxes_single(self,
                          cls_scores,
                          bbox_preds,
                          coef_preds,
                          centernesses,
                          mlvl_points,
                          img_shape,
                          scale_factor,
                          cfg,
                          rescale=False):
        assert len(cls_scores) == len(bbox_preds) == len(mlvl_points)
        mlvl_bboxes = []
        mlvl_scores = []
        mlvl_coefs = []
        mlvl_centerness = []
        for cls_score, bbox_pred, coef_pred, centerness, points in zip(
                cls_scores, bbox_preds, coef_preds, centernesses, mlvl_points):
            assert cls_score.size()[-2:] == bbox_pred.size()[-2:]
            scores = cls_score.permute(1, 2, 0).reshape(
                -1, self.cls_out_channels).sigmoid()

            centerness = centerness.permute(1, 2, 0).reshape(-1).sigmoid()
            coef_pred = coef_pred.permute(1, 2, 0).reshape(-1, self.num_bases)
            bbox_pred = bbox_pred.permute(1, 2, 0).reshape(-1, 4)
            nms_pre = cfg.get('nms_pre', -1)
            if nms_pre > 0 and scores.shape[0] > nms_pre:
                max_scores, _ = (scores * centerness[:, None]).max(dim=1)
                _, topk_inds = max_scores.topk(nms_pre)
                points = points[topk_inds, :]
                bbox_pred = bbox_pred[topk_inds, :]
                coef_pred = coef_pred[topk_inds, :]
                scores = scores[topk_inds, :]
                centerness = centerness[topk_inds]
            bboxes = distance2bbox(points, bbox_pred, max_shape=img_shape)
            coefs = coef_pred
            mlvl_bboxes.append(bboxes)
            mlvl_scores.append(scores)
            mlvl_centerness.append(centerness)
            mlvl_coefs.append(coefs)

        mlvl_bboxes = torch.cat(mlvl_bboxes)
        mlvl_coefs = torch.cat(mlvl_coefs)
        if rescale:
            mlvl_bboxes /= mlvl_bboxes.new_tensor(scale_factor)
        mlvl_scores = torch.cat(mlvl_scores)
        padding = mlvl_scores.new_zeros(mlvl_scores.shape[0], 1)
        mlvl_scores = torch.cat([padding, mlvl_scores], dim=1)
        mlvl_centerness = torch.cat(mlvl_centerness)
        det_bboxes, det_labels, det_coefs = multiclass_nms_with_mask(
            mlvl_bboxes,
            mlvl_scores,
            mlvl_coefs,
            cfg.score_thr,
            cfg.nms,
            cfg.max_per_img,
            score_factors=mlvl_centerness,
            num_bases=self.num_bases)
        return det_bboxes, det_labels, det_coefs

    def get_points(self, featmap_sizes, dtype, device):
        """Get points according to feature map sizes.

        Args:
            featmap_sizes (list[tuple]): Multi-level feature map sizes.
            dtype (torch.dtype): Type of points.
            device (torch.device): Device of points.

        Returns:
            tuple: points of each image.
        """
        mlvl_points = []
        for i in range(len(featmap_sizes)):
            mlvl_points.append(
                self.get_points_single(featmap_sizes[i], self.strides[i],
                                       dtype, device))
        return mlvl_points

    def get_points_single(self, featmap_size, stride, dtype, device):
        h, w = featmap_size
        x_range = torch.arange(
            0, w * stride, stride, dtype=dtype, device=device)
        y_range = torch.arange(
            0, h * stride, stride, dtype=dtype, device=device)
        y, x = torch.meshgrid(y_range, x_range)
        points = torch.stack(
            (x.reshape(-1), y.reshape(-1)), dim=-1) + stride // 2
        return points

    def usd_target(self, points, gt_bboxes_list, gt_labels_list, gt_coefs_list):
        assert len(points) == len(self.regress_ranges)
        num_levels = len(points)
        # expand regress ranges to align with points
        expanded_regress_ranges = [
            points[i].new_tensor(self.regress_ranges[i])[None].expand_as(
                points[i]) for i in range(num_levels)
        ]
        # concat all levels points and regress ranges
        concat_regress_ranges = torch.cat(expanded_regress_ranges, dim=0)
        concat_points = torch.cat(points, dim=0)

        num_points = [center.size(0) for center in points]

        # get labels and bbox_targets of each image
        labels_list, bbox_targets_list, coef_targets_list = multi_apply(
            self.usd_target_single,
            gt_bboxes_list,
            gt_labels_list,
            gt_coefs_list,
            points=concat_points,
            regress_ranges=concat_regress_ranges,
            num_points_per_lvl=num_points)

        # split to per img, per level
        labels_list = [labels.split(num_points, 0) for labels in labels_list]
        bbox_targets_list = [
            bbox_targets.split(num_points, 0)
            for bbox_targets in bbox_targets_list
        ]
        coef_targets_list = [
            coef_targets.split(num_points, 0)
            for coef_targets in coef_targets_list
        ]

        # concat per level image
        concat_lvl_labels = []
        concat_lvl_bbox_targets = []
        concat_lvl_coef_targets = []
        for i in range(num_levels):
            concat_lvl_labels.append(
                torch.cat([labels[i] for labels in labels_list]))
            concat_lvl_bbox_targets.append(
                torch.cat(
                    [bbox_targets[i] for bbox_targets in bbox_targets_list]))
            concat_lvl_coef_targets.append(
                torch.cat(
                    [coef_targets[i] for coef_targets in coef_targets_list]))
        return concat_lvl_labels, concat_lvl_bbox_targets, concat_lvl_coef_targets

    def usd_target_single(self, gt_bboxes, gt_labels, gt_coefs, points, regress_ranges, num_points_per_lvl):
        num_points = points.size(0)
        num_gts = gt_labels.size(0)
        if num_gts == 0:
            return gt_labels.new_zeros(num_points), \
                gt_bboxes.new_zeros((num_points, 4)), \
                gt_labels.new_zeros((num_points, self.num_bases))

        areas = (gt_bboxes[:, 2] - gt_bboxes[:, 0] + 1) * (
            gt_bboxes[:, 3] - gt_bboxes[:, 1] + 1)  # gt_bboxes: [..., x1 y1 x2 y2]
        # TODO: figure out why these two are different
        # areas = areas[None].expand(num_points, num_gts)
        areas = areas[None].repeat(num_points, 1)

        regress_ranges = regress_ranges[:, None, :].expand(
            num_points, num_gts, 2)
        gt_bboxes = gt_bboxes[None].expand(num_points, num_gts, 4)
        # xs ys is the coord of x, y of points
        xs, ys = points[:, 0], points[:, 1]
        xs = xs[:, None].expand(num_points, num_gts)
        ys = ys[:, None].expand(num_points, num_gts)

        left = xs - gt_bboxes[..., 0]
        right = gt_bboxes[..., 2] - xs
        top = ys - gt_bboxes[..., 1]
        bottom = gt_bboxes[..., 3] - ys
        bbox_targets = torch.stack((left, top, right, bottom), -1)

        # condition1: inside a gt bbox
        if self.use_center_sample:
            inside_gt_bbox_mask = self.get_center_sample(gt_bboxes, self.strides, num_points_per_lvl,
                                                         xs.long(), ys.long(), radius=1.5)
        else:
            inside_gt_bbox_mask = bbox_targets.min(-1)[0] > 0

        # condition2: limit the regression range for each location
        max_regress_distance = bbox_targets.max(-1)[0]
        inside_regress_range = (
            max_regress_distance >= regress_ranges[..., 0]) & (
                max_regress_distance <= regress_ranges[..., 1])

        # if there are still more than one objects for a location,
        # we choose the one with minimal area
        areas[inside_gt_bbox_mask == 0] = INF
        areas[inside_regress_range == 0] = INF
        min_area, min_area_inds = areas.min(dim=1)

        labels = gt_labels[min_area_inds]
        labels[min_area == INF] = 0
        bbox_targets = bbox_targets[range(num_points), min_area_inds]

        # USD-Seg
        pos_inds = labels.nonzero().reshape(-1)
        coef_targets = torch.zeros((num_points, self.num_bases), device=gt_coefs.device)
        for p in pos_inds:
            pos_coef_id = min_area_inds[p]
            pos_coef = gt_coefs[pos_coef_id]
            coef_targets[p] = pos_coef

        return labels, bbox_targets, coef_targets

    def get_center_sample(self, gt, strides, num_points_per, gt_xs, gt_ys, radius=1):
        center_x = (gt[..., 0] + gt[..., 2]) / 2
        center_y = (gt[..., 1] + gt[..., 3]) / 2
        center_gt = gt.new_zeros(gt.shape)
        # no gt
        if center_x[..., 0].sum() == 0:
            return gt_xs.new_zeros(gt_xs.shape, dtype=torch.uint8)
        beg = 0
        for level, n_p in enumerate(num_points_per):
            end = beg + n_p
            stride = int(strides[level] * radius)
            xmin = center_x[beg:end] - stride
            ymin = center_y[beg:end] - stride
            xmax = center_x[beg:end] + stride
            ymax = center_y[beg:end] + stride
            # limit sample region in gt
            center_gt[beg:end, :, 0] = torch.where(xmin > gt[beg:end, :, 0], xmin, gt[beg:end, :, 0])
            center_gt[beg:end, :, 1] = torch.where(ymin > gt[beg:end, :, 1], ymin, gt[beg:end, :, 1])
            center_gt[beg:end, :, 2] = torch.where(xmax > gt[beg:end, :, 2], gt[beg:end, :, 2], xmax)
            center_gt[beg:end, :, 3] = torch.where(ymax > gt[beg:end, :, 3], gt[beg:end, :, 3], ymax)
            beg = end
        left = gt_xs - center_gt[..., 0]
        right = center_gt[..., 2] - gt_xs
        top = gt_ys - center_gt[..., 1]
        bottom = center_gt[..., 3] - gt_ys
        center_bbox = torch.stack((left, top, right, bottom), -1)
        inside_gt_bbox_mask = center_bbox.min(-1)[0] > 0
        return inside_gt_bbox_mask

    def centerness_target(self, pos_bbox_targets):
        # only calculate pos centerness targets, otherwise there may be nan
        left_right = pos_bbox_targets[:, [0, 2]]
        top_bottom = pos_bbox_targets[:, [1, 3]]
        centerness_targets = (
            left_right.min(dim=-1)[0] / left_right.max(dim=-1)[0]) * (
                top_bottom.min(dim=-1)[0] / top_bottom.max(dim=-1)[0])
        return torch.sqrt(centerness_targets)

# use_dcn=False, # USD-Seg # self.use_dcn=use_dcn  # TODO: Add DCN support # Mask Loss # scale the bbox_pred of different level # float to avoid overflow when enabling FP16 # flatten cls_scores, bbox_preds and centerness # [num_pixel, 80] # [num_pixel, 4] # [num_pixel, num_bases] # [num_pixel] # [num_pixel] # [num_pixel, 4] # [num_pixel, num_bases] # repeat points to align with bbox_preds # [num_pixel,2] # avoid num_pos is 0 # centerness weighted iou loss # Change the weights of different coefs # Mask Loss # Find the targets Get points according to feature map sizes.          Args:             featmap_sizes (list[tuple]): Multi-level feature map sizes.             dtype (torch.dtype): Type of points.             device (torch.device): Device of points.          Returns:             tuple: points of each image. # expand regress ranges to align with points # concat all levels points and regress ranges # get labels and bbox_targets of each image # split to per img, per level # concat per level image # gt_bboxes: [..., x1 y1 x2 y2] # TODO: figure out why these two are different # areas = areas[None].expand(num_points, num_gts) # xs ys is the coord of x, y of points # condition1: inside a gt bbox # condition2: limit the regression range for each location # if there are still more than one objects for a location, # we choose the one with minimal area # USD-Seg # no gt # limit sample region in gt # only calculate pos centerness targets, otherwise there may be nan